From 5fd206b4c3e69685e2466532363b8adfd8adfa48 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 03:48:18 -0700 Subject: [PATCH 01/14] chore(quest): claim quest/m2/evicted-refill.md Quest-Claim: 87138da3f642f8f2 From b8cbf65d7b16f91055460d75ef8d1874cb2eb1aa Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 05:10:45 -0700 Subject: [PATCH 02/14] fix(net): refill groups evicted before a reader took them A reader judged each sequence once, so a group the cache evicted after the reader judged it cached, but before it took it, was skipped silently. The track now records evictions until its slowest reader has seen them, and each reader judges such a group again as a miss, once: one handed out is never asked again, and a refill evicted again before it is taken is given up to the drop cursor, so refills never exceed what the reader reads. An ordered reader's drop poll probed from the window floor rather than its next sequence, asking fills again for evicted groups it had already taken. Co-Authored-By: Claude Opus 5.5 --- doc/concept/moq-lite.md | 3 +- quest/m1/unbounded-groups/flights.md | 4 - quest/m2/README.md | 1 - quest/m2/evicted-refill.md | 36 --- rs/moq-net/src/ledger.rs | 12 + rs/moq-net/src/model/track.rs | 365 +++++++++++++++++++++++++-- rs/moq-net/src/model/track/miss.rs | 104 +++++++- 7 files changed, 451 insertions(+), 74 deletions(-) delete mode 100644 quest/m2/evicted-refill.md diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index 26055fc1856..3b6fbd56ea9 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -301,7 +301,8 @@ starts) is asked of the track's dynamic handler as a fill, which a relay serves by fetching upstream one group at a time, skipping a group once every subscriber that asked for it narrows past it; with no handler it is reported as dropped once the feed stops waiting for it. One -evicted after the subscriber saw it cached, but before it read it, is skipped. A relay merges its subscribers' ranges into one upstream request, which +evicted after the subscriber saw it cached, but before it read it, is asked the same +way, once: evicted again before it is read, it is reported as dropped. A relay merges its subscribers' ranges into one upstream request, which moq-lite-07 carries range by range and older wire versions as the single span covering them. A relative or time anchor resolves on the hop that publishes the track, and over moq-lite-07, which carries a relative or time start upstream and diff --git a/quest/m1/unbounded-groups/flights.md b/quest/m1/unbounded-groups/flights.md index b62dccdea2e..54c2f4adf9c 100644 --- a/quest/m1/unbounded-groups/flights.md +++ b/quest/m1/unbounded-groups/flights.md @@ -75,7 +75,3 @@ Public API: none. Wire: none, beyond the existing frame-offset fills. ## Required - [Fastest-reader pacing](/quest/m1/unbounded-groups/fastest-reader.md) - flights pace on their fastest rider the same way the live edge does - -## Related - -- [Refill evicted groups](/quest/m2/evicted-refill.md) - refills whole groups evicted before a reader took them diff --git a/quest/m2/README.md b/quest/m2/README.md index 4f4f2452b0a..0626823ed2c 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -94,7 +94,6 @@ lite-07; bitrate enforcement is not part of this release. - [Read-only lookup](/quest/m2/read-only-lookup.md) - a live track lookup stops waking the front and every demand watcher - [Ordered owed wait](/quest/m2/ordered-owed-wait.md) - an ordered reader waits within max_delay for a missing group only while the track still owes it - [lite-05/06 gap judgement](/quest/m2/lite-legacy-gap.md) - a resumed copy on lite-05/06 never reads a descending track's newest group as its start -- [Refill evicted groups](/quest/m2/evicted-refill.md) - a group evicted after a reader judged it but before it took it is refilled, never delivered twice, and guarded against thrash - [Receive pool](/quest/m2/receive-pool.md) - one byte budget per worker covers every connection's receive credit and the cache, billing each frame once - [Rust-backed @moq/net](/quest/m2/rs2ts/README.md) - measure browser baselines, complete WASM bindings, and adopt Rust behind @moq/net before the later JS-only stage - [Front parking](/quest/m2/origin-front-parks.md) - an unroutable request waits on a front instead of re-asking on every route-table move diff --git a/quest/m2/evicted-refill.md b/quest/m2/evicted-refill.md deleted file mode 100644 index bdd740325ee..00000000000 --- a/quest/m2/evicted-refill.md +++ /dev/null @@ -1,36 +0,0 @@ -# [M] Refill groups evicted before a reader took them - -## Goal - -A reader behind the live edge still gets a group the cache evicted after the -reader judged it but before it took it: the group becomes a miss and is asked -of the track's fills, like one evicted before the reader judged it. A reader -never receives a group it already took twice, and slow readers under cache -pressure cannot make the relay refetch without bound. - -## Plan - -From [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171), where the -maintainer chose to land fills without this (2026-10-21). Today a reader -judges each sequence once: a group cached when judged and evicted later is -skipped silently. - -- Track evictions per reader. An ordered reader can judge again every - eviction at or above its next sequence. An arrival-order reader delivers - above its floor, so it must tell the groups it already walked past from - the ones still ahead, for example by keeping each evicted group's arrival - index. Bound what the track keeps by the slowest reader, not by the cache. -- Guard against thrash. Under pool pressure a refill evicts another group, - possibly the next one the same slow reader needs. Decide in the PR, with - options and a recommendation, how a reader stops refilling once its - refills only evict what it reads next: give up to the drop cursor, or rely - on its max delay. -- Test a slow ascending reader on a pooled track under bufferbloat: it - receives every group its budget covers, takes none twice, and refills stay - proportional to what it reads. - -Public API: none. Wire: none. - -## Related - -- [Unbounded groups](/quest/m1/unbounded-groups/README.md) - refills the parts of a group the cache sheds under pressure diff --git a/rs/moq-net/src/ledger.rs b/rs/moq-net/src/ledger.rs index 002968647b1..7a389ec5053 100644 --- a/rs/moq-net/src/ledger.rs +++ b/rs/moq-net/src/ledger.rs @@ -77,6 +77,18 @@ impl Changes { self.changed.insert(self.revision, run.start); } + /// Remove the runs last changed at or before `revision`, returning them. + pub fn trim(&mut self, revision: u64) -> Vec> { + let kept = self.changed.split_off(&(revision.saturating_add(1))); + std::mem::replace(&mut self.changed, kept) + .into_values() + .map(|start| { + let (end, _) = self.runs.remove(&start).expect("a changed run stays indexed"); + start..end + }) + .collect() + } + pub fn since(&self, revision: u64) -> impl Iterator> + '_ { self.changed .range((std::ops::Bound::Excluded(revision), std::ops::Bound::Unbounded)) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index da4102ad639..9637336124f 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -39,7 +39,7 @@ use fill::FillState; pub(crate) use fill::{GroupFill, Readers as FillReaders}; use std::{ - collections::{BTreeMap, BTreeSet, VecDeque}, + collections::{BTreeMap, BTreeSet, VecDeque, btree_map}, ops::{self, Bound, RangeBounds}, sync::Arc, sync::OnceLock, @@ -460,6 +460,14 @@ pub(crate) struct TrackState { // Bumped on every insert into `drops`, so a cursor rescans it only after a change. drops_revision: u64, drops_changes: ledger::Changes, + // Sequences evicted while servable, indexed by when each run last changed, so a reader + // that judged one cached judges it again. Kept only until every reader that judged + // before an eviction has seen it: the slowest reader bounds it, not the cache. + evictions: RangeSet, + eviction_changes: ledger::Changes, + // How many readers judged up to each eviction revision, the lowest being the slowest. + // Its own lock, so a reader moves through the read guard it judges under. + eviction_readers: Mutex>, // Whether the cache reflects the live feed: readers get nothing from it while it does // not, since how stale it is cannot be told; fetches still do. A track is live from @@ -934,6 +942,54 @@ impl TrackState { Some(slot) } + /// [`Self::uncache`] a servable group to free the cache and abort it with `err`, recording + /// it for the readers that judged it cached to judge again. + fn evict_group(&mut self, sequence: u64, err: Error) { + self.uncache(sequence).expect("a cached group").abort(err); + let Some(end) = sequence.checked_add(1) else { + return; + }; + self.evictions.insert(sequence..end); + let run = self + .evictions + .get(&sequence) + .cloned() + .expect("the eviction was just recorded"); + // The neighbours it coalesced with retire their entries. + for old in [run.start, end] { + self.eviction_changes.remove(old); + } + self.eviction_changes.insert(run); + // Readers registered after a run last changed judged it as it is: only one that + // judged earlier still needs it. + let slowest = self + .eviction_readers + .lock() + .unwrap_or_else(PoisonError::into_inner) + .first_key_value() + .map(|(revision, _)| *revision); + let seen = slowest.unwrap_or(self.eviction_changes.revision()); + for run in self.eviction_changes.trim(seen) { + self.evictions.remove(run); + } + } + + /// Move one reader's eviction registration from `from`, if it had one, to `to`. + pub(super) fn judged_evictions(&self, from: Option, to: Option) { + let mut readers = self.eviction_readers.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(from) = from + && let btree_map::Entry::Occupied(mut entry) = readers.entry(from) + { + *entry.get_mut() -= 1; + if *entry.get() == 0 { + entry.remove(); + } + } + if let Some(to) = to { + *readers.entry(to).or_default() += 1; + } + } + /// [`Self::uncache`] a group that another copy displaces rather than one that ends, so a /// teardown still ends the copies its readers hold. fn displace(&mut self, sequence: u64) { @@ -1756,7 +1812,7 @@ impl TrackState { // Take the group out of the cache and abort it, so any consumer // still reading surfaces `Error::Old` instead of blocking forever // on a frame that will never arrive. - self.uncache(sequence).unwrap().abort(Error::Old); + self.evict_group(sequence, Error::Old); } } @@ -2226,7 +2282,7 @@ impl TrackState { self.debt -= size; paid = paid.saturating_add(size); - self.uncache(sequence).unwrap().abort(Error::Evicted); + self.evict_group(sequence, Error::Evicted); } // No cold group left to pay with: shed the front of the groups that could not go // whole, the ones being read and then the live edge, so neither grows without bound. @@ -5904,6 +5960,15 @@ struct Cursor { misses: Option>, } +impl Drop for Cursor { + fn drop(&mut self) { + // The slowest reader bounds what the track keeps of its evictions. + if let Some(evicted) = self.misses.as_ref().and_then(|misses| misses.evicted()) { + self.state.read().judged_evictions(Some(evicted), None); + } + } +} + struct Peeked { group: group::Consumer, taken: Taken, @@ -6118,7 +6183,7 @@ impl Cursor { } #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - fn poll_drop(&mut self, waiter: &kio::Waiter) -> Poll<(ops::Range, StreamError)> { + fn poll_drop(&mut self, taken: &Delivered, waiter: &kio::Waiter) -> Poll<(ops::Range, StreamError)> { self.sync(); if self.drops.is_none() { let mut drops = Box::new(DropCursor::new(&self.state.read())); @@ -6132,7 +6197,9 @@ impl Cursor { if self.filter.refused.is_none() && let Poll::Ready(Ok(drift)) = self.poll_drift(&self.servable, waiter) { - self.probe(self.floor(), &drift, waiter); + // From where the reader reads next: an ordered one asks nothing below what it took. + let from = self.next_sequence.unwrap_or(u64::MAX).max(self.floor()); + self.probe(from, &drift, taken, waiter); } // A cap at or below the floor leaves nothing to report, not a reversed span. let floor = self.floor(); @@ -6514,7 +6581,8 @@ impl Cursor { /// Ask fills for what this cursor misses from `from`, and take in what they answered, /// parking `waiter` on the fills still out. Whatever it gives up goes to the drop cursor. - fn probe(&mut self, from: u64, drift: &Drift, waiter: &kio::Waiter) { + /// `taken` is what its subscriber handed out, which an eviction never makes a miss. + fn probe(&mut self, from: u64, drift: &Drift, taken: &Delivered, waiter: &kio::Waiter) { let (window, cap, revision) = (&self.filter.window, self.end_sequence, self.filter.revision); let asking = { let subscription = self.subscription.read(); @@ -6547,6 +6615,7 @@ impl Cursor { .map(|run| *run.start()..=(*run.end()).min(end)) .collect() }, + taken, drift, &mut given_up, waiter, @@ -6588,7 +6657,12 @@ impl Cursor { self.misses.as_ref().is_some_and(|misses| misses.is_waiting()) } - fn poll_recv_group(&mut self, take: bool, waiter: &kio::Waiter) -> Poll>> { + fn poll_recv_group( + &mut self, + take: bool, + taken: &Delivered, + waiter: &kio::Waiter, + ) -> Poll>> { if let Some(peeked) = self.peeked.take().filter(|peeked| take && !peeked.ordered) { // The reserved group is still the one handed out, but bounded by the ranges in // force now, which a `Control` may have replaced since the peek. @@ -6607,7 +6681,7 @@ impl Cursor { // One scan for the whole poll, so walking a backlog off stays linear in its size. let drift = ready!(self.poll_drift(&self.servable, waiter))?; self.reoffer(drift.budget); - self.probe(self.floor(), &drift, waiter); + self.probe(self.floor(), &drift, taken, waiter); self.poll_recv_with(take, drift, waiter) } @@ -6791,7 +6865,12 @@ impl Cursor { /// The lowest servable group past the last one returned, walking off everything the /// drift budget has convicted on the way. - fn poll_next_group(&mut self, take: bool, waiter: &kio::Waiter) -> Poll>> { + fn poll_next_group( + &mut self, + take: bool, + taken: &Delivered, + waiter: &kio::Waiter, + ) -> Poll>> { if let Some(peeked) = self.peeked.take().filter(|peeked| take && peeked.ordered) { // Bounded by the ranges in force now, as on the arrival path. self.sync(); @@ -6807,7 +6886,7 @@ impl Cursor { let floor = next.max(self.floor()); // One scan for the whole poll, so walking a backlog off stays linear in its size. let drift = ready!(self.poll_drift(&self.servable, waiter))?; - self.probe(floor, &drift, waiter); + self.probe(floor, &drift, taken, waiter); self.poll_next_with(take, floor, drift, waiter) } @@ -7152,8 +7231,8 @@ impl Subscriber { let res = match &mut self.inner { Inner::Plain(cursor) => loop { let res = match ordered { - true => cursor.poll_next_group(true, waiter), - false => cursor.poll_recv_group(true, waiter), + true => cursor.poll_next_group(true, &self.delivered, waiter), + false => cursor.poll_recv_group(true, &self.delivered, waiter), }; match res { Poll::Ready(Ok(Some(group))) if self.delivered.contains(group.sequence) => continue, @@ -7175,8 +7254,8 @@ impl Subscriber { match &mut self.inner { Inner::Plain(cursor) => loop { let group = ready!(match ordered { - true => cursor.poll_next_group(false, waiter), - false => cursor.poll_recv_group(false, waiter), + true => cursor.poll_next_group(false, &self.delivered, waiter), + false => cursor.poll_recv_group(false, &self.delivered, waiter), })?; let Some(group) = group else { return Poll::Ready(Ok(None)); @@ -7186,8 +7265,8 @@ impl Subscriber { } // A replay advances only this copy's cursor, as an ordinary read would. ready!(match ordered { - true => cursor.poll_next_group(true, waiter), - false => cursor.poll_recv_group(true, waiter), + true => cursor.poll_next_group(true, &self.delivered, waiter), + false => cursor.poll_recv_group(true, &self.delivered, waiter), })?; }, Inner::Resume(resume, _) => resume.poll_peek_group(ordered, &self.delivered, waiter), @@ -7534,7 +7613,7 @@ impl Subscriber { #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn poll_drop(&mut self, waiter: &kio::Waiter) -> Poll<(ops::Range, StreamError)> { match &mut self.inner { - Inner::Plain(cursor) => cursor.poll_drop(waiter), + Inner::Plain(cursor) => cursor.poll_drop(&self.delivered, waiter), Inner::Resume(resume, _) => resume.poll_drop(&self.delivered, waiter), } } @@ -18622,6 +18701,251 @@ mod test { assert_eq!(asked, [(4, 6), (0, 2)]); } + /// A group evicted after an arrival-order reader judged it cached, before it took it, is + /// asked of a fill like one evicted before: the reader still receives it, and a group it + /// took is never asked again. + #[test] + #[cfg(feature = "async")] + fn a_group_evicted_before_its_reader_took_it_is_refilled() { + let producer = track_producer("test", Info::timed()); + let dynamic = producer.requests(); + let mut subscriber = producer.subscribe(replay()); + for sequence in 0..3 { + write_group(&producer, sequence); + } + let waiter = kio::Waiter::noop(); + let Poll::Ready(Ok(Some(first))) = subscriber.poll_recv_group(&waiter) else { + panic!("group 0 is cached"); + }; + assert_eq!(first.sequence, 0); + evict(&producer, 0); + evict(&producer, 1); + assert_eq!(drain_groups(&mut subscriber), [2]); + let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) else { + panic!("group 1 is a miss now"); + }; + assert_eq!((fill.start().group, fill.end()), (1, 2), "group 0 was taken"); + assert_eq!(fill.next(), Some(1)); + write_filled(fill.accept(1, None), 1); + drop(fill); + assert_eq!(drain_groups(&mut subscriber), [1]); + assert!(dynamic.poll_requested_fill(&waiter).is_pending()); + assert!(drained_drops(&mut subscriber).is_empty(), "nothing was given up"); + } + + /// The track keeps an eviction run only while a reader that judged before its last change + /// has yet to see it: the slowest reader bounds the record, and none keeps nothing. + #[test] + #[cfg(feature = "async")] + fn the_slowest_reader_bounds_the_eviction_record() { + let producer = track_producer("test", Info::timed()); + let _dynamic = producer.requests(); + for sequence in 0..6 { + write_group(&producer, sequence); + } + let recorded = || { + let state = producer.state.read(); + state + .evictions + .iter() + .map(|run| (run.start, run.end)) + .collect::>() + }; + evict(&producer, 0); + assert!(recorded().is_empty(), "no reader judged group 0"); + + let mut fast = producer.subscribe(replay()); + let mut slow = producer.subscribe(replay()); + drain_groups(&mut fast); + drain_groups(&mut slow); + evict(&producer, 1); + assert_eq!(recorded(), [(1, 2)], "both readers still owe a look"); + drain_groups(&mut fast); + evict(&producer, 2); + assert_eq!(recorded(), [(1, 3)], "the slow reader has seen neither"); + drop(slow); + drain_groups(&mut fast); + evict(&producer, 4); + assert_eq!(recorded(), [(4, 5)], "the fast reader saw every earlier run"); + drop(fast); + evict(&producer, 3); + assert!(recorded().is_empty(), "no reader is left"); + } + + /// An ordered reader forwarding drops asks nothing below what it took, though its drop + /// cursor reports from the window's floor. + #[test] + #[cfg(feature = "async")] + fn an_ordered_readers_drop_poll_asks_nothing_it_took() { + let producer = track_producer("test", Info::timed()); + let dynamic = producer.requests(); + let mut subscriber = producer.subscribe(replay().with_order(Order::Ascending)); + for sequence in 0..3 { + write_group(&producer, sequence); + } + let waiter = kio::Waiter::noop(); + for expected in 0..2 { + let Poll::Ready(Ok(Some(group))) = subscriber.poll_next_group(&waiter) else { + panic!("group {expected} is cached"); + }; + assert_eq!(group.sequence, expected); + } + evict(&producer, 0); + evict(&producer, 1); + assert!(drained_drops(&mut subscriber).is_empty()); + assert!(dynamic.poll_requested_fill(&waiter).is_pending(), "both were taken"); + } + + /// An ordered reader waits on the refill of a group evicted at or above its next + /// sequence, and reads it in order. + #[test] + #[cfg(feature = "async")] + fn an_ordered_reader_refills_an_evicted_group_in_order() { + let producer = track_producer("test", Info::timed()); + let dynamic = producer.requests(); + let mut subscriber = producer.subscribe(replay().with_order(Order::Ascending)); + for sequence in 0..4 { + write_group(&producer, sequence); + } + let waiter = kio::Waiter::noop(); + let next = |subscriber: &mut Subscriber| match subscriber.poll_next_group(&kio::Waiter::noop()) { + Poll::Ready(Ok(Some(group))) => Some(group.sequence), + _ => None, + }; + assert_eq!(next(&mut subscriber), Some(0)); + evict(&producer, 1); + evict(&producer, 2); + assert_eq!(next(&mut subscriber), None, "group 1 comes before 3"); + let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) else { + panic!("groups 1 and 2 are misses now"); + }; + assert_eq!((fill.start().group, fill.end()), (1, 3)); + while let Some(sequence) = fill.next() { + write_filled(fill.accept(sequence, None), sequence); + } + drop(fill); + assert_eq!([next(&mut subscriber), next(&mut subscriber)], [Some(1), Some(2)]); + assert_eq!(next(&mut subscriber), Some(3)); + } + + /// A refill evicted again before its reader took it is given up rather than asked a + /// second time, so refills that evict each other cannot loop. + #[test] + #[cfg(feature = "async")] + fn a_refill_evicted_again_is_given_up() { + let producer = track_producer("test", Info::timed()); + let dynamic = producer.requests(); + let mut subscriber = producer.subscribe(replay()); + for sequence in 0..3 { + write_group(&producer, sequence); + } + let waiter = kio::Waiter::noop(); + assert!(matches!(subscriber.poll_recv_group(&waiter), Poll::Ready(Ok(Some(_))))); + evict(&producer, 1); + assert_eq!(drain_groups(&mut subscriber), [2]); + let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) else { + panic!("group 1 is a miss now"); + }; + assert_eq!(fill.next(), Some(1)); + write_filled(fill.accept(1, None), 1); + drop(fill); + evict(&producer, 1); + assert!(drain_groups(&mut subscriber).is_empty()); + assert!(dynamic.poll_requested_fill(&waiter).is_pending(), "refilled once"); + assert_eq!(drained_drops(&mut subscriber), [(1, 2)]); + } + + /// A slow ascending reader on a pooled track holding `cached` groups, taking `pace` of + /// every four its feed writes, as behind a bufferbloated link. Returns what it read, how + /// many groups the fills wrote, and what it gave up. + #[cfg(feature = "async")] + fn read_slowly(cached: usize, pace: u64) -> (Vec, u64, Vec<(u64, u64)>) { + let size = 10_000; + let (producer, _pool) = pooled_producer((cached * size) as u64); + let dynamic = producer.requests(); + let mut subscriber = producer.subscribe(replay().with_order(Order::Ascending)); + let write = |group: Result, sequence: u64| { + let mut group = group.unwrap(); + group + .write_frame(Timestamp::from_millis(sequence).unwrap(), vec![0u8; size]) + .unwrap(); + group.finish().unwrap(); + }; + let waiter = kio::Waiter::noop(); + let mut refilled = 0; + let mut serve = || { + while let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) { + while let Some(sequence) = fill.next() { + write(fill.accept(sequence, None), sequence); + refilled += 1; + } + } + }; + let (mut read, mut drops) = (Vec::new(), Vec::new()); + // A wire publisher forwards what the reader gives up as it reads. + let mut take = |subscriber: &mut Subscriber| { + let taken = match subscriber.poll_next_group(&kio::Waiter::noop()) { + Poll::Ready(Ok(Some(group))) => { + read.push(group.sequence); + true + } + _ => false, + }; + drops.extend(drained_drops(subscriber)); + taken + }; + for sequence in 0..GROUPS { + write(producer.create_group(group::Info { sequence }), sequence); + if sequence % 4 < pace { + take(&mut subscriber); + } + serve(); + } + // The feed pauses and the reader drains what it can, refills included. + loop { + serve(); + if !take(&mut subscriber) { + serve(); + if !take(&mut subscriber) { + break; + } + } + } + (read, refilled, drops) + } + + #[cfg(feature = "async")] + const GROUPS: u64 = 40; + + /// A backlog a little past the cache: what the reader judged and lost to eviction is + /// refilled, so it receives every group once, in order, refilling no more than it reads. + #[test] + #[cfg(feature = "async")] + fn a_slow_reader_on_a_pooled_track_receives_every_group() { + let (read, refilled, drops) = read_slowly(8, 3); + assert_eq!(read, (0..GROUPS).collect::>(), "every group once, in order"); + assert!(drops.is_empty(), "nothing given up: {drops:?}"); + assert!(refilled > 0, "the backlog outgrew the cache"); + assert!(refilled <= GROUPS, "{refilled} refills for {GROUPS} groups read"); + } + + /// A backlog far past the cache, where refills evict the groups the reader reads next: + /// each group is refilled at most once, and one lost again is given up and named, never + /// handed out twice. + #[test] + #[cfg(feature = "async")] + fn a_slow_reader_past_the_cache_refills_each_group_once() { + let (read, refilled, drops) = read_slowly(4, 2); + let mut seen = read.clone(); + seen.dedup(); + assert_eq!(seen, read, "none twice"); + assert!(read.is_sorted(), "in order"); + assert!(refilled <= GROUPS, "{refilled} refills for {GROUPS} groups"); + let lost: Vec = (0..GROUPS).filter(|sequence| !read.contains(sequence)).collect(); + let named: Vec = drops.iter().flat_map(|&(start, end)| start..end).collect(); + assert_eq!(lost, named, "every group lost is named dropped"); + } + /// A reader forgets the misses it gave up once its window moves past them, as it forgets /// what it judged, so a feed skipping every other group costs it only the gaps it can /// still read. @@ -19413,12 +19737,7 @@ mod test { /// Take group `sequence` out of the cache as eviction does. fn evict(producer: &Producer, sequence: u64) { - producer - .modify() - .unwrap() - .uncache(sequence) - .unwrap() - .abort(Error::Evicted); + producer.modify().unwrap().evict_group(sequence, Error::Evicted); } /// A rewound first frame is refused and leaves the group as it was, so the publisher can diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index 6bc2dc064ec..e1d25c8062d 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -7,8 +7,9 @@ //! (evicted, or a gap the feed skipped). A hole the ledger still waits for is deferred and //! judged again as the feed moves. //! -//! Each sequence is judged once, so a group evicted after the reader judged it cached is -//! not a miss: judging it again would refill a group an arrival-order reader already took. +//! Each sequence is judged once, except a group evicted after the reader judged it cached +//! and before it took it: judged again as a miss, once. A refill evicted again before it is +//! taken is given up, so a slow reader's refills cannot evict each other without bound. //! //! A start is no promise, since a group below it may still arrive late. A hole under it is //! asked at once, but a refusal is final only once the ledger gives up on it too; until @@ -24,6 +25,7 @@ use std::{ use rangemap::{RangeInclusiveSet, RangeSet}; +use super::super::delivered::Delivered; pub(super) use super::fill::Asking; use super::fill::{Ask, FillState, Join}; use super::{Drift, Order, Position, TrackState}; @@ -60,6 +62,10 @@ pub(super) struct Misses { filled: BTreeSet, /// Never asked again: named dropped to the reader. given_up: RangeSet, + /// The track's eviction revision this reader judged up to, registered with the track. + evicted: Option, + /// Judged again after an eviction, within the judged span: never refilled twice. + refilled: RangeSet, } /// A run of misses the reader gives up, for the drop cursor to name. @@ -190,8 +196,13 @@ impl Misses { &self.given_up } - /// The next answered group still cached in `order`, forgetting one evicted before it was - /// read. + /// The eviction revision this reader is registered at with its track, if any. + pub fn evicted(&self) -> Option { + self.evicted + } + + /// The next answered group still cached in `order`, giving up one evicted before it was + /// read: [`Self::judge`] asked a first eviction again, so this one was refilled already. pub fn next_filled( &mut self, state: &TrackState, @@ -351,7 +362,8 @@ impl Misses { /// Judge the holes in `target` the last probe has not, returning the runs to ask a /// fill for. Gives up a run already stale under this reader's budget. `named` is the - /// lowest group the reader's ranges name, which may reach below the feed's start. + /// lowest group the reader's ranges name, which may reach below the feed's start, and + /// `taken` what the reader handed out, which an eviction never makes a miss again. #[expect( clippy::too_many_arguments, reason = "one probe's inputs, borrowed apart from its cursor" @@ -362,6 +374,7 @@ impl Misses { key: (u64, u64, Option), named: Option, mut target: impl FnMut(RangeInclusive) -> RangeInclusiveSet, + taken: &Delivered, drift: &Drift, given_up: &mut GivenUp, waiter: &kio::Waiter, @@ -387,13 +400,15 @@ impl Misses { let ended = state.abort.is_some() || state.is_complete(); let ledger = state.arrivals.changes(); let drops = &state.drops_changes; + let evictions = &state.eviction_changes; // Every reader probes on every poll, mostly with nothing changed since the last, so // that answer costs neither the subscriptions' lock nor a clock read. let unchanged = self.at == Some(at) && self.budget == Some(drift.budget) && self.ended == ended && self.ledger_revision == ledger.revision() - && self.drops_revision == drops.revision(); + && self.drops_revision == drops.revision() + && self.evicted == Some(evictions.revision()); // The grace only times deferred gaps, so with none a new one has nothing to move. // It is judged once something else changes. if unchanged && self.deferred.runs.is_empty() { @@ -436,6 +451,7 @@ impl Misses { self.judged = clip(&self.judged); self.refused = clip(&self.refused); self.given_up = clip(&self.given_up); + self.refilled = clip(&self.refilled); for (&start, &(end, _)) in &self.deferred.runs { for run in span.overlapping(&(start..end)) { candidates.insert(start.max(run.start)..end.min(run.end)); @@ -451,7 +467,12 @@ impl Misses { // A moving read floor removes only its retired prefix. New production extends the // judged span; neither path visits older deferred gaps. if low > 0 { - for set in [&mut self.judged, &mut self.refused, &mut self.given_up] { + for set in [ + &mut self.judged, + &mut self.refused, + &mut self.given_up, + &mut self.refilled, + ] { set.remove(0..low); } self.deferred.remove(0..low); @@ -497,6 +518,43 @@ impl Misses { self.refused.remove(sequence..sequence + 1); } } + // What was judged cached and evicted since, before this reader took it, is a miss now. + let mut refills = RangeSet::new(); + if let Some(seen) = self.evicted + && seen != evictions.revision() + && low <= top + { + for run in evictions.since(seen) { + let run = run.start.max(low)..run.end.min(top + 1); + if run.is_empty() { + continue; + } + for part in target(run.start..=run.end - 1).iter() { + for gap in taken.gaps(*part.start()..*part.end() + 1) { + for judged in self.judged.overlapping(&gap) { + let judged = judged.start.max(gap.start)..judged.end.min(gap.end); + for fresh in self.refilled.gaps(&judged) { + refills.insert(fresh); + } + } + } + } + } + } + if self.evicted != Some(evictions.revision()) { + state.judged_evictions(self.evicted, Some(evictions.revision())); + self.evicted = Some(evictions.revision()); + } + for run in refills.iter() { + self.refilled.insert(run.clone()); + candidates.insert(run.clone()); + // A fill's answer evicted before it was taken is asked again, not given up. + let filled: Vec = self.filled.range(run.clone()).copied().collect(); + for sequence in filled { + self.filled.remove(&sequence); + self.release(sequence); + } + } self.ledger_revision = ledger.revision(); self.drops_revision = drops.revision(); self.budget = Some(drift.budget); @@ -857,6 +915,7 @@ mod tests { (0, 0, None), None, |span| [span].into_iter().collect(), + &Delivered::default(), &drift, &mut given_up, &waiter, @@ -896,6 +955,7 @@ mod tests { (0, 0, None), None, |_| panic!("the edge has not moved"), + &Delivered::default(), &drift, &mut given_up, &waiter, @@ -920,6 +980,7 @@ mod tests { (0, 0, None), None, |_| panic!("recreation extends no span"), + &Delivered::default(), &drift, &mut given_up, &waiter, @@ -961,6 +1022,7 @@ mod tests { (0, 0, None), None, |span| [span].into_iter().collect(), + &Delivered::default(), &drift, &mut given_up, &waiter, @@ -994,6 +1056,7 @@ mod tests { (0, 0, None), None, |_| panic!("the edge has not moved"), + &Delivered::default(), &drift, &mut given_up, &waiter, @@ -1030,6 +1093,7 @@ mod tests { (0, 0, None), None, |span| [span].into_iter().collect(), + &Delivered::default(), &drift, &mut given_up, &waiter @@ -1045,6 +1109,7 @@ mod tests { (0, 0, None), None, |_| panic!("an unchanged probe rebuilds no span"), + &Delivered::default(), &drift, &mut given_up, &waiter @@ -1065,6 +1130,7 @@ mod tests { (0, 0, None), None, |_| panic!("a late arrival extends no span"), + &Delivered::default(), &drift, &mut given_up, &waiter @@ -1081,6 +1147,7 @@ mod tests { (0, 0, None), None, |_| panic!("a due gap extends no span"), + &Delivered::default(), &drift, &mut given_up, &waiter, @@ -1117,7 +1184,16 @@ mod tests { let full = |span| [span].into_iter().collect(); assert!( misses - .judge(&state, (0, 0, None), None, full, &drift, &mut given_up, &waiter) + .judge( + &state, + (0, 0, None), + None, + full, + &Delivered::default(), + &drift, + &mut given_up, + &waiter + ) .is_empty() ); assert!(misses.deferred.runs.is_empty(), "a contiguous feed defers nothing"); @@ -1134,6 +1210,7 @@ mod tests { (0, 0, None), None, |_| panic!("nothing deferred: the new grace waits for a change"), + &Delivered::default(), &drift, &mut given_up, &waiter @@ -1151,7 +1228,16 @@ mod tests { let opened = state.cache.pool().now_wall(); assert!( misses - .judge(&state, (0, 0, None), None, full, &drift, &mut given_up, &waiter) + .judge( + &state, + (0, 0, None), + None, + full, + &Delivered::default(), + &drift, + &mut given_up, + &waiter + ) .is_empty() ); assert_eq!(misses.deferred.overlapping(0..5).collect::>(), vec![3..4]); From 3d311c8acc76994f9170b5be02840bd31c7da95f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 05:14:53 -0700 Subject: [PATCH 03/14] fix(net): keep evictions only within what a reader judged A reader that judged once and stalled pinned its revision, so evictions of ever newer sequences it never judged piled up. Each reader now registers the newest sequence it judged too, and the track keeps no eviction past every reader's newest. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track.rs | 121 +++++++++++++++++++++-------- rs/moq-net/src/model/track/miss.rs | 22 +++--- 2 files changed, 100 insertions(+), 43 deletions(-) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 9637336124f..ecf3c5346d8 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -461,13 +461,14 @@ pub(crate) struct TrackState { drops_revision: u64, drops_changes: ledger::Changes, // Sequences evicted while servable, indexed by when each run last changed, so a reader - // that judged one cached judges it again. Kept only until every reader that judged - // before an eviction has seen it: the slowest reader bounds it, not the cache. + // that judged one cached judges it again. Kept only within what a reader judged, until + // every reader that judged before an eviction has seen it: the slowest reader bounds + // it, not the cache, and a stalled one pins no eviction past what it judged. evictions: RangeSet, eviction_changes: ledger::Changes, - // How many readers judged up to each eviction revision, the lowest being the slowest. - // Its own lock, so a reader moves through the read guard it judges under. - eviction_readers: Mutex>, + // Where each reader judged up to. Its own lock, so a reader moves through the read guard + // it judges under. + eviction_readers: Mutex, // Whether the cache reflects the live feed: readers get nothing from it while it does // not, since how stale it is cannot be told; fetches still do. A track is live from @@ -946,47 +947,54 @@ impl TrackState { /// it for the readers that judged it cached to judge again. fn evict_group(&mut self, sequence: u64, err: Error) { self.uncache(sequence).expect("a cached group").abort(err); - let Some(end) = sequence.checked_add(1) else { - return; + let (slowest, top) = { + let readers = self.eviction_readers.lock().unwrap_or_else(PoisonError::into_inner); + (readers.slowest(), readers.top()) }; - self.evictions.insert(sequence..end); - let run = self - .evictions - .get(&sequence) - .cloned() - .expect("the eviction was just recorded"); - // The neighbours it coalesced with retire their entries. - for old in [run.start, end] { - self.eviction_changes.remove(old); + // No reader judged a group above every reader's newest, so none needs it again. + if let Some(end) = sequence.checked_add(1) + && top.is_some_and(|top| sequence <= top) + { + self.evictions.insert(sequence..end); + let run = self + .evictions + .get(&sequence) + .cloned() + .expect("the eviction was just recorded"); + // The neighbours it coalesced with retire their entries. + for old in [run.start, end] { + self.eviction_changes.remove(old); + } + self.eviction_changes.insert(run); } - self.eviction_changes.insert(run); // Readers registered after a run last changed judged it as it is: only one that // judged earlier still needs it. - let slowest = self - .eviction_readers - .lock() - .unwrap_or_else(PoisonError::into_inner) - .first_key_value() - .map(|(revision, _)| *revision); let seen = slowest.unwrap_or(self.eviction_changes.revision()); for run in self.eviction_changes.trim(seen) { self.evictions.remove(run); } + // A run past every reader's newest, once the reader that judged it left. + let above = top.map_or(0, |top| top.saturating_add(1)); + let past: Vec<_> = self + .evictions + .overlapping(&(above..u64::MAX)) + .filter(|run| run.start >= above) + .cloned() + .collect(); + for run in past { + self.eviction_changes.remove(run.start); + self.evictions.remove(run); + } } /// Move one reader's eviction registration from `from`, if it had one, to `to`. - pub(super) fn judged_evictions(&self, from: Option, to: Option) { + pub(super) fn judged_evictions(&self, from: Option, to: Option) { let mut readers = self.eviction_readers.lock().unwrap_or_else(PoisonError::into_inner); - if let Some(from) = from - && let btree_map::Entry::Occupied(mut entry) = readers.entry(from) - { - *entry.get_mut() -= 1; - if *entry.get() == 0 { - entry.remove(); - } + if let Some(from) = from { + readers.remove(from); } if let Some(to) = to { - *readers.entry(to).or_default() += 1; + readers.add(to); } } @@ -5911,6 +5919,45 @@ impl PresentationEdge { } } +/// Where a reader judged evictions up to: the track's eviction revision and the newest +/// sequence it judged. +pub(super) type Judged = (u64, u64); + +/// Where every reader judged evictions up to, counted. +#[derive(Default)] +struct EvictionReaders { + revisions: BTreeMap, + tops: BTreeMap, +} + +impl EvictionReaders { + fn add(&mut self, (revision, top): Judged) { + *self.revisions.entry(revision).or_default() += 1; + *self.tops.entry(top).or_default() += 1; + } + + fn remove(&mut self, (revision, top): Judged) { + for (counts, key) in [(&mut self.revisions, revision), (&mut self.tops, top)] { + if let btree_map::Entry::Occupied(mut entry) = counts.entry(key) { + *entry.get_mut() -= 1; + if *entry.get() == 0 { + entry.remove(); + } + } + } + } + + /// The lowest revision a reader judged up to. + fn slowest(&self) -> Option { + self.revisions.first_key_value().map(|(revision, _)| *revision) + } + + /// The newest sequence any reader judged. + fn top(&self) -> Option { + self.tops.last_key_value().map(|(top, _)| *top) + } +} + /// The read cursor behind a [`Subscriber`]. struct Cursor { state: kio::Consumer, @@ -18734,7 +18781,8 @@ mod test { } /// The track keeps an eviction run only while a reader that judged before its last change - /// has yet to see it: the slowest reader bounds the record, and none keeps nothing. + /// has yet to see it, and only within what a reader judged: the slowest reader bounds the + /// record, a stalled one pins nothing it never judged, and none keeps nothing. #[test] #[cfg(feature = "async")] fn the_slowest_reader_bounds_the_eviction_record() { @@ -18767,6 +18815,13 @@ mod test { drain_groups(&mut fast); evict(&producer, 4); assert_eq!(recorded(), [(4, 5)], "the fast reader saw every earlier run"); + // Stalled, it judged nothing past group 5: what the feed evicts beyond is not kept. + for sequence in (6..20).step_by(2) { + write_group(&producer, sequence); + write_group(&producer, sequence + 1); + evict(&producer, sequence); + } + assert_eq!(recorded(), [(4, 5)], "nothing past what the stalled reader judged"); drop(fast); evict(&producer, 3); assert!(recorded().is_empty(), "no reader is left"); diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index e1d25c8062d..b47207847b4 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -28,7 +28,7 @@ use rangemap::{RangeInclusiveSet, RangeSet}; use super::super::delivered::Delivered; pub(super) use super::fill::Asking; use super::fill::{Ask, FillState, Join}; -use super::{Drift, Order, Position, TrackState}; +use super::{Drift, Judged, Order, Position, TrackState}; use crate::Error; /// What a probe judges against: the span read, the feed's newest group, where the feed @@ -62,8 +62,9 @@ pub(super) struct Misses { filled: BTreeSet, /// Never asked again: named dropped to the reader. given_up: RangeSet, - /// The track's eviction revision this reader judged up to, registered with the track. - evicted: Option, + /// The track's eviction revision and newest sequence this reader judged up to, + /// registered with the track. + evicted: Option, /// Judged again after an eviction, within the judged span: never refilled twice. refilled: RangeSet, } @@ -196,8 +197,8 @@ impl Misses { &self.given_up } - /// The eviction revision this reader is registered at with its track, if any. - pub fn evicted(&self) -> Option { + /// Where this reader is registered with its track as having judged evictions up to. + pub fn evicted(&self) -> Option { self.evicted } @@ -408,7 +409,7 @@ impl Misses { && self.ended == ended && self.ledger_revision == ledger.revision() && self.drops_revision == drops.revision() - && self.evicted == Some(evictions.revision()); + && self.evicted.map(|(revision, _)| revision) == Some(evictions.revision()); // The grace only times deferred gaps, so with none a new one has nothing to move. // It is judged once something else changes. if unchanged && self.deferred.runs.is_empty() { @@ -520,7 +521,7 @@ impl Misses { } // What was judged cached and evicted since, before this reader took it, is a miss now. let mut refills = RangeSet::new(); - if let Some(seen) = self.evicted + if let Some((seen, _)) = self.evicted && seen != evictions.revision() && low <= top { @@ -541,9 +542,10 @@ impl Misses { } } } - if self.evicted != Some(evictions.revision()) { - state.judged_evictions(self.evicted, Some(evictions.revision())); - self.evicted = Some(evictions.revision()); + let judged = Some((evictions.revision(), top)); + if self.evicted != judged { + state.judged_evictions(self.evicted, judged); + self.evicted = judged; } for run in refills.iter() { self.refilled.insert(run.clone()); From d46b18e9e7cb98ba176ffd23def561de412a9ed1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 05:45:08 -0700 Subject: [PATCH 04/14] fix(net): keep an eviction run only while a reader that judged it lags A stalled reader pinned the lowest revision while an active one raised the newest judged sequence, so neither bound held alone. A trim now keeps a run only if a reader registered before its last change judged up to it, checked once the record doubles past what the last trim kept; with no reader left, the record clears. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/ledger.rs | 21 +++--- rs/moq-net/src/model/track.rs | 121 +++++++++++++++++++--------------- 2 files changed, 78 insertions(+), 64 deletions(-) diff --git a/rs/moq-net/src/ledger.rs b/rs/moq-net/src/ledger.rs index 7a389ec5053..645f7ac33ee 100644 --- a/rs/moq-net/src/ledger.rs +++ b/rs/moq-net/src/ledger.rs @@ -77,16 +77,17 @@ impl Changes { self.changed.insert(self.revision, run.start); } - /// Remove the runs last changed at or before `revision`, returning them. - pub fn trim(&mut self, revision: u64) -> Vec> { - let kept = self.changed.split_off(&(revision.saturating_add(1))); - std::mem::replace(&mut self.changed, kept) - .into_values() - .map(|start| { - let (end, _) = self.runs.remove(&start).expect("a changed run stays indexed"); - start..end - }) - .collect() + /// Every current run with the revision it last changed at, oldest change first. + pub fn by_revision(&self) -> impl Iterator)> + '_ { + self.changed.iter().map(|(revision, start)| { + let (end, _) = self.runs.get(start).expect("a changed run stays indexed"); + (*revision, *start..*end) + }) + } + + /// How many runs are indexed. + pub fn len(&self) -> usize { + self.runs.len() } pub fn since(&self, revision: u64) -> impl Iterator> + '_ { diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index ecf3c5346d8..44739038a08 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -461,11 +461,13 @@ pub(crate) struct TrackState { drops_revision: u64, drops_changes: ledger::Changes, // Sequences evicted while servable, indexed by when each run last changed, so a reader - // that judged one cached judges it again. Kept only within what a reader judged, until - // every reader that judged before an eviction has seen it: the slowest reader bounds - // it, not the cache, and a stalled one pins no eviction past what it judged. + // that judged one cached judges it again. A run is kept only while some reader judged up + // to it and has yet to see its last change: the readers bound it, not the cache, and a + // stalled one pins only evictions within what it judged. evictions: RangeSet, eviction_changes: ledger::Changes, + // The runs the last trim kept, so trimming waits for the record to double past it. + evictions_kept: usize, // Where each reader judged up to. Its own lock, so a reader moves through the read guard // it judges under. eviction_readers: Mutex, @@ -947,13 +949,10 @@ impl TrackState { /// it for the readers that judged it cached to judge again. fn evict_group(&mut self, sequence: u64, err: Error) { self.uncache(sequence).expect("a cached group").abort(err); - let (slowest, top) = { - let readers = self.eviction_readers.lock().unwrap_or_else(PoisonError::into_inner); - (readers.slowest(), readers.top()) - }; - // No reader judged a group above every reader's newest, so none needs it again. + let readers = self.eviction_readers.lock().unwrap_or_else(PoisonError::into_inner); + // Every reader registered before this change, so one judged up to it needs it. if let Some(end) = sequence.checked_add(1) - && top.is_some_and(|top| sequence <= top) + && readers.top().is_some_and(|top| sequence <= top) { self.evictions.insert(sequence..end); let run = self @@ -967,24 +966,39 @@ impl TrackState { } self.eviction_changes.insert(run); } - // Readers registered after a run last changed judged it as it is: only one that - // judged earlier still needs it. - let seen = slowest.unwrap_or(self.eviction_changes.revision()); - for run in self.eviction_changes.trim(seen) { - self.evictions.remove(run); + // With no reader left, nothing is owed. + if readers.top().is_none() { + drop(readers); + for (_, run) in self.eviction_changes.by_revision().collect::>() { + self.eviction_changes.remove(run.start); + } + self.evictions = RangeSet::new(); + self.evictions_kept = 0; + return; } - // A run past every reader's newest, once the reader that judged it left. - let above = top.map_or(0, |top| top.saturating_add(1)); - let past: Vec<_> = self - .evictions - .overlapping(&(above..u64::MAX)) - .filter(|run| run.start >= above) - .cloned() - .collect(); - for run in past { + if self.eviction_changes.len() <= 2 * self.evictions_kept { + return; + } + // Oldest change first, against the readers registered before it: a run stays only if + // one of them judged up to it. + let mut registered = readers.by_revision().peekable(); + let mut top = None; + let mut seen = Vec::new(); + for (revision, run) in self.eviction_changes.by_revision() { + while let Some((_, judged)) = registered.next_if(|(judged, _)| *judged < revision) { + top = top.max(Some(judged)); + } + if top.is_none_or(|top| top < run.start) { + seen.push(run); + } + } + drop(registered); + drop(readers); + for run in seen { self.eviction_changes.remove(run.start); self.evictions.remove(run); } + self.evictions_kept = self.eviction_changes.len(); } /// Move one reader's eviction registration from `from`, if it had one, to `to`. @@ -5926,18 +5940,18 @@ pub(super) type Judged = (u64, u64); /// Where every reader judged evictions up to, counted. #[derive(Default)] struct EvictionReaders { - revisions: BTreeMap, + judged: BTreeMap, tops: BTreeMap, } impl EvictionReaders { - fn add(&mut self, (revision, top): Judged) { - *self.revisions.entry(revision).or_default() += 1; - *self.tops.entry(top).or_default() += 1; + fn add(&mut self, judged: Judged) { + *self.judged.entry(judged).or_default() += 1; + *self.tops.entry(judged.1).or_default() += 1; } - fn remove(&mut self, (revision, top): Judged) { - for (counts, key) in [(&mut self.revisions, revision), (&mut self.tops, top)] { + fn remove(&mut self, judged: Judged) { + fn take(counts: &mut BTreeMap, key: K) { if let btree_map::Entry::Occupied(mut entry) = counts.entry(key) { *entry.get_mut() -= 1; if *entry.get() == 0 { @@ -5945,17 +5959,19 @@ impl EvictionReaders { } } } - } - - /// The lowest revision a reader judged up to. - fn slowest(&self) -> Option { - self.revisions.first_key_value().map(|(revision, _)| *revision) + take(&mut self.judged, judged); + take(&mut self.tops, judged.1); } /// The newest sequence any reader judged. fn top(&self) -> Option { self.tops.last_key_value().map(|(top, _)| *top) } + + /// Each registration's revision and newest sequence, lowest revision first. + fn by_revision(&self) -> impl Iterator + '_ { + self.judged.keys().copied() + } } /// The read cursor behind a [`Subscriber`]. @@ -18780,12 +18796,12 @@ mod test { assert!(drained_drops(&mut subscriber).is_empty(), "nothing was given up"); } - /// The track keeps an eviction run only while a reader that judged before its last change - /// has yet to see it, and only within what a reader judged: the slowest reader bounds the - /// record, a stalled one pins nothing it never judged, and none keeps nothing. + /// The track keeps an eviction run only while a reader that judged up to it has yet to + /// see its last change: a reader that judged once and stalled pins nothing past what it + /// judged while another reads on, and with no reader nothing is kept. #[test] #[cfg(feature = "async")] - fn the_slowest_reader_bounds_the_eviction_record() { + fn the_readers_bound_the_eviction_record() { let producer = track_producer("test", Info::timed()); let _dynamic = producer.requests(); for sequence in 0..6 { @@ -18802,27 +18818,24 @@ mod test { evict(&producer, 0); assert!(recorded().is_empty(), "no reader judged group 0"); - let mut fast = producer.subscribe(replay()); - let mut slow = producer.subscribe(replay()); - drain_groups(&mut fast); - drain_groups(&mut slow); + let mut stalled = producer.subscribe(replay()); + let mut active = producer.subscribe(replay()); + drain_groups(&mut stalled); + drain_groups(&mut active); evict(&producer, 1); - assert_eq!(recorded(), [(1, 2)], "both readers still owe a look"); - drain_groups(&mut fast); - evict(&producer, 2); - assert_eq!(recorded(), [(1, 3)], "the slow reader has seen neither"); - drop(slow); - drain_groups(&mut fast); - evict(&producer, 4); - assert_eq!(recorded(), [(4, 5)], "the fast reader saw every earlier run"); - // Stalled, it judged nothing past group 5: what the feed evicts beyond is not kept. - for sequence in (6..20).step_by(2) { + assert_eq!(recorded(), [(1, 2)], "both readers judged it"); + for sequence in (6..200).step_by(2) { write_group(&producer, sequence); write_group(&producer, sequence + 1); + drain_groups(&mut active); evict(&producer, sequence); + let runs = recorded(); + assert!(runs.contains(&(1, 2)), "the stalled reader has yet to see group 1"); + // A trim keeps group 1 and the active reader's latest, and waits for twice that. + assert!(runs.len() <= 4, "{} runs after group {sequence}", runs.len()); } - assert_eq!(recorded(), [(4, 5)], "nothing past what the stalled reader judged"); - drop(fast); + drop(stalled); + drop(active); evict(&producer, 3); assert!(recorded().is_empty(), "no reader is left"); } From 6e265b9cfa9c4db78535a9e9d3465898e11e7132 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 06:05:36 -0700 Subject: [PATCH 05/14] refactor(net): leave evictions in the inbox of each reader that judged them The shared eviction record kept runs by sequence span and revision, which a stalled reader could still pin with late groups it never judged cached. Each reader now registers an inbox; an eviction lands only in the inboxes of readers that judged that slot cached (cached by the count they judged at, inside their judged span), so a reader holds at most the slots it judged. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/ledger.rs | 13 -- rs/moq-net/src/model/track.rs | 193 +++++++---------------------- rs/moq-net/src/model/track/miss.rs | 137 +++++++++++++++----- 3 files changed, 147 insertions(+), 196 deletions(-) diff --git a/rs/moq-net/src/ledger.rs b/rs/moq-net/src/ledger.rs index 645f7ac33ee..002968647b1 100644 --- a/rs/moq-net/src/ledger.rs +++ b/rs/moq-net/src/ledger.rs @@ -77,19 +77,6 @@ impl Changes { self.changed.insert(self.revision, run.start); } - /// Every current run with the revision it last changed at, oldest change first. - pub fn by_revision(&self) -> impl Iterator)> + '_ { - self.changed.iter().map(|(revision, start)| { - let (end, _) = self.runs.get(start).expect("a changed run stays indexed"); - (*revision, *start..*end) - }) - } - - /// How many runs are indexed. - pub fn len(&self) -> usize { - self.runs.len() - } - pub fn since(&self, revision: u64) -> impl Iterator> + '_ { self.changed .range((std::ops::Bound::Excluded(revision), std::ops::Bound::Unbounded)) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 44739038a08..e394e15e415 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -39,7 +39,7 @@ use fill::FillState; pub(crate) use fill::{GroupFill, Readers as FillReaders}; use std::{ - collections::{BTreeMap, BTreeSet, VecDeque, btree_map}, + collections::{BTreeMap, BTreeSet, VecDeque}, ops::{self, Bound, RangeBounds}, sync::Arc, sync::OnceLock, @@ -460,17 +460,15 @@ pub(crate) struct TrackState { // Bumped on every insert into `drops`, so a cursor rescans it only after a change. drops_revision: u64, drops_changes: ledger::Changes, - // Sequences evicted while servable, indexed by when each run last changed, so a reader - // that judged one cached judges it again. A run is kept only while some reader judged up - // to it and has yet to see its last change: the readers bound it, not the cache, and a - // stalled one pins only evictions within what it judged. - evictions: RangeSet, - eviction_changes: ledger::Changes, - // The runs the last trim kept, so trimming waits for the record to double past it. - evictions_kept: usize, - // Where each reader judged up to. Its own lock, so a reader moves through the read guard - // it judges under. - eviction_readers: Mutex, + // Every reader's eviction inbox. Its own lock, so a reader registers through the read + // guard it judges under. + inboxes: Mutex, + // Bumped whenever an eviction lands in an inbox, so a reader with nothing new skips + // its own lock. + evictions: u64, + // Bumped as each group is cached, stamped on its slot: a reader judged a slot cached + // only if it was cached by the count the reader last judged at. + cached: u64, // Whether the cache reflects the live feed: readers get nothing from it while it does // not, since how stale it is cannot be told; fetches still do. A track is live from @@ -554,6 +552,9 @@ struct Slot { // the replacement and deliver it twice or at the wrong position. stamp: u32, + // [`TrackState::cached`] once this incarnation was cached. + cached: u64, + // Whether this incarnation came from the live publisher and can replace older // subscription content. Fetch-only backfill stays cached but never anchors drift. visible: bool, @@ -948,67 +949,15 @@ impl TrackState { /// [`Self::uncache`] a servable group to free the cache and abort it with `err`, recording /// it for the readers that judged it cached to judge again. fn evict_group(&mut self, sequence: u64, err: Error) { - self.uncache(sequence).expect("a cached group").abort(err); - let readers = self.eviction_readers.lock().unwrap_or_else(PoisonError::into_inner); - // Every reader registered before this change, so one judged up to it needs it. - if let Some(end) = sequence.checked_add(1) - && readers.top().is_some_and(|top| sequence <= top) - { - self.evictions.insert(sequence..end); - let run = self - .evictions - .get(&sequence) - .cloned() - .expect("the eviction was just recorded"); - // The neighbours it coalesced with retire their entries. - for old in [run.start, end] { - self.eviction_changes.remove(old); - } - self.eviction_changes.insert(run); - } - // With no reader left, nothing is owed. - if readers.top().is_none() { - drop(readers); - for (_, run) in self.eviction_changes.by_revision().collect::>() { - self.eviction_changes.remove(run.start); - } - self.evictions = RangeSet::new(); - self.evictions_kept = 0; - return; - } - if self.eviction_changes.len() <= 2 * self.evictions_kept { - return; - } - // Oldest change first, against the readers registered before it: a run stays only if - // one of them judged up to it. - let mut registered = readers.by_revision().peekable(); - let mut top = None; - let mut seen = Vec::new(); - for (revision, run) in self.eviction_changes.by_revision() { - while let Some((_, judged)) = registered.next_if(|(judged, _)| *judged < revision) { - top = top.max(Some(judged)); - } - if top.is_none_or(|top| top < run.start) { - seen.push(run); - } - } - drop(registered); - drop(readers); - for run in seen { - self.eviction_changes.remove(run.start); - self.evictions.remove(run); - } - self.evictions_kept = self.eviction_changes.len(); - } - - /// Move one reader's eviction registration from `from`, if it had one, to `to`. - pub(super) fn judged_evictions(&self, from: Option, to: Option) { - let mut readers = self.eviction_readers.lock().unwrap_or_else(PoisonError::into_inner); - if let Some(from) = from { - readers.remove(from); - } - if let Some(to) = to { - readers.add(to); + let slot = self.uncache(sequence).expect("a cached group"); + slot.abort(err); + let landed = self + .inboxes + .get_mut() + .unwrap_or_else(PoisonError::into_inner) + .evicted(sequence, slot.cached); + if landed { + self.evictions += 1; } } @@ -2110,11 +2059,13 @@ impl TrackState { if visible { self.record_arrival(sequence, now); } + self.cached += 1; let replaced = self.lookup.insert( sequence, Slot { group: group.clone(), stamp, + cached: self.cached, visible, arrived: now, head: None, @@ -5933,47 +5884,6 @@ impl PresentationEdge { } } -/// Where a reader judged evictions up to: the track's eviction revision and the newest -/// sequence it judged. -pub(super) type Judged = (u64, u64); - -/// Where every reader judged evictions up to, counted. -#[derive(Default)] -struct EvictionReaders { - judged: BTreeMap, - tops: BTreeMap, -} - -impl EvictionReaders { - fn add(&mut self, judged: Judged) { - *self.judged.entry(judged).or_default() += 1; - *self.tops.entry(judged.1).or_default() += 1; - } - - fn remove(&mut self, judged: Judged) { - fn take(counts: &mut BTreeMap, key: K) { - if let btree_map::Entry::Occupied(mut entry) = counts.entry(key) { - *entry.get_mut() -= 1; - if *entry.get() == 0 { - entry.remove(); - } - } - } - take(&mut self.judged, judged); - take(&mut self.tops, judged.1); - } - - /// The newest sequence any reader judged. - fn top(&self) -> Option { - self.tops.last_key_value().map(|(top, _)| *top) - } - - /// Each registration's revision and newest sequence, lowest revision first. - fn by_revision(&self) -> impl Iterator + '_ { - self.judged.keys().copied() - } -} - /// The read cursor behind a [`Subscriber`]. struct Cursor { state: kio::Consumer, @@ -6023,15 +5933,6 @@ struct Cursor { misses: Option>, } -impl Drop for Cursor { - fn drop(&mut self) { - // The slowest reader bounds what the track keeps of its evictions. - if let Some(evicted) = self.misses.as_ref().and_then(|misses| misses.evicted()) { - self.state.read().judged_evictions(Some(evicted), None); - } - } -} - struct Peeked { group: group::Consumer, taken: Taken, @@ -18796,48 +18697,40 @@ mod test { assert!(drained_drops(&mut subscriber).is_empty(), "nothing was given up"); } - /// The track keeps an eviction run only while a reader that judged up to it has yet to - /// see its last change: a reader that judged once and stalled pins nothing past what it - /// judged while another reads on, and with no reader nothing is kept. + /// An eviction waits only for a reader that judged the evicted group cached: a reader + /// that judged once and stalled holds no more than what was cached then, however many + /// groups the feed caches and evicts after, late lower ones included. #[test] #[cfg(feature = "async")] - fn the_readers_bound_the_eviction_record() { + fn an_eviction_waits_only_for_readers_that_judged_it_cached() { let producer = track_producer("test", Info::timed()); let _dynamic = producer.requests(); - for sequence in 0..6 { - write_group(&producer, sequence); - } - let recorded = || { - let state = producer.state.read(); - state - .evictions - .iter() - .map(|run| (run.start, run.end)) - .collect::>() - }; + write_group(&producer, 0); + write_group(&producer, 1); + write_group(&producer, 1 << 40); + let held = || producer.state.read().inboxes.lock().unwrap().held(); evict(&producer, 0); - assert!(recorded().is_empty(), "no reader judged group 0"); + assert!(held().is_empty(), "no reader judged group 0"); let mut stalled = producer.subscribe(replay()); let mut active = producer.subscribe(replay()); drain_groups(&mut stalled); drain_groups(&mut active); - evict(&producer, 1); - assert_eq!(recorded(), [(1, 2)], "both readers judged it"); - for sequence in (6..200).step_by(2) { + // Late groups below the stalled reader's newest, cached after it judged. + for sequence in (2..200).step_by(2) { write_group(&producer, sequence); - write_group(&producer, sequence + 1); drain_groups(&mut active); evict(&producer, sequence); - let runs = recorded(); - assert!(runs.contains(&(1, 2)), "the stalled reader has yet to see group 1"); - // A trim keeps group 1 and the active reader's latest, and waits for twice that. - assert!(runs.len() <= 4, "{} runs after group {sequence}", runs.len()); } + // The active reader judged each one cached, so it holds the last until it judges again. + assert_eq!(held(), [vec![], vec![(198, 199)]], "the stalled reader judged none"); + drain_groups(&mut active); + evict(&producer, 1); + assert_eq!(held(), [vec![(1, 2)], vec![(1, 2)]], "both judged group 1 cached"); + drain_groups(&mut active); + assert_eq!(held(), [vec![(1, 2)], vec![]], "the active reader took it in"); drop(stalled); - drop(active); - evict(&producer, 3); - assert!(recorded().is_empty(), "no reader is left"); + assert_eq!(held(), [vec![]], "a dropped reader holds nothing"); } /// An ordered reader forwarding drops asks nothing below what it took, though its drop diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index b47207847b4..3573e457a45 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -19,6 +19,7 @@ use std::{ collections::{BTreeMap, BTreeSet}, ops::{self, RangeInclusive}, + sync::{Arc, PoisonError}, task::Poll, time::Duration, }; @@ -28,8 +29,9 @@ use rangemap::{RangeInclusiveSet, RangeSet}; use super::super::delivered::Delivered; pub(super) use super::fill::Asking; use super::fill::{Ask, FillState, Join}; -use super::{Drift, Judged, Order, Position, TrackState}; +use super::{Drift, Order, Position, TrackState}; use crate::Error; +use crate::sync::Mutex; /// What a probe judges against: the span read, the feed's newest group, where the feed /// and readers start, and the drops revision. @@ -62,9 +64,10 @@ pub(super) struct Misses { filled: BTreeSet, /// Never asked again: named dropped to the reader. given_up: RangeSet, - /// The track's eviction revision and newest sequence this reader judged up to, - /// registered with the track. - evicted: Option, + /// Where the track leaves what it evicts that this reader judged cached, once it judged. + inbox: Option, + /// The track's eviction count this reader last emptied its inbox at. + evictions: u64, /// Judged again after an eviction, within the judged span: never refilled twice. refilled: RangeSet, } @@ -197,11 +200,6 @@ impl Misses { &self.given_up } - /// Where this reader is registered with its track as having judged evictions up to. - pub fn evicted(&self) -> Option { - self.evicted - } - /// The next answered group still cached in `order`, giving up one evicted before it was /// read: [`Self::judge`] asked a first eviction again, so this one was refilled already. pub fn next_filled( @@ -401,7 +399,6 @@ impl Misses { let ended = state.abort.is_some() || state.is_complete(); let ledger = state.arrivals.changes(); let drops = &state.drops_changes; - let evictions = &state.eviction_changes; // Every reader probes on every poll, mostly with nothing changed since the last, so // that answer costs neither the subscriptions' lock nor a clock read. let unchanged = self.at == Some(at) @@ -409,7 +406,7 @@ impl Misses { && self.ended == ended && self.ledger_revision == ledger.revision() && self.drops_revision == drops.revision() - && self.evicted.map(|(revision, _)| revision) == Some(evictions.revision()); + && self.evictions == state.evictions; // The grace only times deferred gaps, so with none a new one has nothing to move. // It is judged once something else changes. if unchanged && self.deferred.runs.is_empty() { @@ -519,34 +516,36 @@ impl Misses { self.refused.remove(sequence..sequence + 1); } } - // What was judged cached and evicted since, before this reader took it, is a miss now. + // What it judged cached and the track evicted since, before this reader took it, is a + // miss now. + let inbox = self.inbox.get_or_insert_with(|| { + let inbox = Inbox::default(); + state + .inboxes + .lock() + .unwrap_or_else(PoisonError::into_inner) + .register(inbox.clone()); + inbox + }); + let evicted = inbox.judge(state.cached, low, top); + self.evictions = state.evictions; let mut refills = RangeSet::new(); - if let Some((seen, _)) = self.evicted - && seen != evictions.revision() - && low <= top - { - for run in evictions.since(seen) { - let run = run.start.max(low)..run.end.min(top + 1); - if run.is_empty() { - continue; - } - for part in target(run.start..=run.end - 1).iter() { - for gap in taken.gaps(*part.start()..*part.end() + 1) { - for judged in self.judged.overlapping(&gap) { - let judged = judged.start.max(gap.start)..judged.end.min(gap.end); - for fresh in self.refilled.gaps(&judged) { - refills.insert(fresh); - } + for run in evicted.iter() { + let run = run.start.max(low)..run.end.min(top.saturating_add(1)); + if run.is_empty() { + continue; + } + for part in target(run.start..=run.end - 1).iter() { + for gap in taken.gaps(*part.start()..*part.end() + 1) { + for judged in self.judged.overlapping(&gap) { + let judged = judged.start.max(gap.start)..judged.end.min(gap.end); + for fresh in self.refilled.gaps(&judged) { + refills.insert(fresh); } } } } } - let judged = Some((evictions.revision(), top)); - if self.evicted != judged { - state.judged_evictions(self.evicted, judged); - self.evicted = judged; - } for run in refills.iter() { self.refilled.insert(run.clone()); candidates.insert(run.clone()); @@ -785,6 +784,78 @@ impl Misses { } } +/// What the track evicted that one reader judged cached, and what it judged: a slot cached +/// by the count it judged at, within the span it judged. Holds no more than the slots it +/// judged cached, so a stalled reader pins nothing cached after. +#[derive(Clone, Default)] +pub(super) struct Inbox(Arc>); + +#[derive(Default)] +struct Judging { + cached: u64, + span: Option>, + evicted: RangeSet, +} + +impl Inbox { + /// Take what was evicted since the last call, and judge `low..=top` from here on, with + /// every slot cached by `cached`. + fn judge(&self, cached: u64, low: u64, top: u64) -> RangeSet { + let mut judging = self.0.lock().unwrap_or_else(PoisonError::into_inner); + judging.cached = cached; + judging.span = Some(low..=top); + std::mem::take(&mut judging.evicted) + } +} + +/// Every reader's [`Inbox`], pruned of dropped readers once it doubles past what the last +/// prune kept. +#[derive(Default)] +pub(super) struct Inboxes { + inboxes: Vec, + kept: usize, +} + +impl Inboxes { + fn register(&mut self, inbox: Inbox) { + self.inboxes.push(inbox); + if self.inboxes.len() > 2 * self.kept.max(1) { + self.inboxes.retain(|inbox| Arc::strong_count(&inbox.0) > 1); + self.kept = self.inboxes.len(); + } + } + + /// What each live reader's inbox holds, in registration order. + #[cfg(test)] + pub(super) fn held(&self) -> Vec> { + self.inboxes + .iter() + .filter(|inbox| Arc::strong_count(&inbox.0) > 1) + .map(|inbox| { + let judging = inbox.0.lock().unwrap_or_else(PoisonError::into_inner); + judging.evicted.iter().map(|run| (run.start, run.end)).collect() + }) + .collect() + } + + /// Leave `sequence`, cached as count `cached`, with each reader that judged it cached. + /// Returns whether any did. + pub(super) fn evicted(&mut self, sequence: u64, cached: u64) -> bool { + let Some(end) = sequence.checked_add(1) else { + return false; + }; + let mut landed = false; + for inbox in &self.inboxes { + let mut judging = inbox.0.lock().unwrap_or_else(PoisonError::into_inner); + if cached <= judging.cached && judging.span.as_ref().is_some_and(|span| span.contains(&sequence)) { + judging.evicted.insert(sequence..end); + landed = true; + } + } + landed + } +} + impl TrackState { /// Split `hole`, which the cache holds nothing for, into what the live feed will never /// deliver and the gaps it still waits for, with when each stops being waited for. From 13b6b9492c283dc8867161e7d79d47702750a3b1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 06:14:22 -0700 Subject: [PATCH 06/14] test(net): gate the inbox accessor with its async test Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track/miss.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index 3573e457a45..ebe2c028296 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -826,7 +826,7 @@ impl Inboxes { } /// What each live reader's inbox holds, in registration order. - #[cfg(test)] + #[cfg(all(test, feature = "async"))] pub(super) fn held(&self) -> Vec> { self.inboxes .iter() From 2630b0378a47329ade8c7e2f7a7586b721e86c5e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 06:41:35 -0700 Subject: [PATCH 07/14] fix(net): prune a dropped reader's inbox on each eviction Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track.rs | 7 +++++++ rs/moq-net/src/model/track/miss.rs | 20 ++++++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index e394e15e415..d7f1d480d8d 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -18731,6 +18731,13 @@ mod test { assert_eq!(held(), [vec![(1, 2)], vec![]], "the active reader took it in"); drop(stalled); assert_eq!(held(), [vec![]], "a dropped reader holds nothing"); + write_group(&producer, 3); + evict(&producer, 3); + assert_eq!( + producer.state.read().inboxes.lock().unwrap().len(), + 1, + "an eviction prunes a dropped reader's inbox" + ); } /// An ordered reader forwarding drops asks nothing below what it took, though its drop diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index ebe2c028296..8e3fe0acb08 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -808,8 +808,8 @@ impl Inbox { } } -/// Every reader's [`Inbox`], pruned of dropped readers once it doubles past what the last -/// prune kept. +/// Every reader's [`Inbox`], pruned of dropped readers on each eviction, or once +/// registrations double it past what the last prune kept. #[derive(Default)] pub(super) struct Inboxes { inboxes: Vec, @@ -825,6 +825,12 @@ impl Inboxes { } } + /// How many inboxes are registered, live or not. + #[cfg(all(test, feature = "async"))] + pub(super) fn len(&self) -> usize { + self.inboxes.len() + } + /// What each live reader's inbox holds, in registration order. #[cfg(all(test, feature = "async"))] pub(super) fn held(&self) -> Vec> { @@ -845,13 +851,19 @@ impl Inboxes { return false; }; let mut landed = false; - for inbox in &self.inboxes { + // Every inbox is visited anyway, so a dropped reader's goes here too. + self.inboxes.retain(|inbox| { + if Arc::strong_count(&inbox.0) == 1 { + return false; + } let mut judging = inbox.0.lock().unwrap_or_else(PoisonError::into_inner); if cached <= judging.cached && judging.span.as_ref().is_some_and(|span| span.contains(&sequence)) { judging.evicted.insert(sequence..end); landed = true; } - } + true + }); + self.kept = self.inboxes.len(); landed } } From 88d6ebba290f37f297c882cfd2ea06a6317830b9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 31 Oct 2026 00:25:33 -0700 Subject: [PATCH 08/14] fix(net): give up a refill evicted again on either read order An ordered reader never looks at what a fill answered before reading past it, so a refill evicted again while it waited below was skipped unnamed and its fill join held. Settle it in judge instead, and count a fill's answer as seen cached so its eviction reaches the inbox. Add a benchmark sweeping readers by evicted groups. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/benches/track.rs | 46 ++++++++++++++++++++++++++++++ rs/moq-net/src/model/track.rs | 34 ++++++++++++++++++++++ rs/moq-net/src/model/track/miss.rs | 33 ++++++++++++++++++--- 3 files changed, 109 insertions(+), 4 deletions(-) diff --git a/rs/moq-net/benches/track.rs b/rs/moq-net/benches/track.rs index a5d55b421f6..45d9df045af 100644 --- a/rs/moq-net/benches/track.rs +++ b/rs/moq-net/benches/track.rs @@ -46,6 +46,11 @@ //! wall clock: each empty group is handed out like any other, so a walk over the rest of //! the run per group shows up as a slope over it. //! +//! `track_evict_fanout` writes groups into a full pool, each evicting one that every reader +//! judged cached, swept over the readers and the groups written between their reads: each +//! eviction leaves the group in every reader's inbox, so its cost grows with the readers, +//! and a cost that grows with the groups written beyond that shows up as a slope. +//! //! `track_fanout_frame` writes frames into an open group while every subscriber is parked //! on the next one, re-polling whichever reader the write woke: a frame wakes no track //! reader, so waking them shows up as a slope over the subscribers. @@ -552,6 +557,46 @@ fn bench_fanout(c: &mut Criterion) { group.finish(); } +/// Groups written between reads in `track_evict_fanout`, each evicting one. +const EVICTED: [usize; 3] = [1, 8, 64]; + +fn bench_evict_fanout(c: &mut Criterion) { + let mut group = c.benchmark_group("track_evict_fanout"); + for subscribers in FANOUT { + for evicted in EVICTED { + group.throughput(Throughput::Elements(evicted as u64)); + group.bench_function(BenchmarkId::new(format!("{subscribers}r"), evicted), |b| { + let mut fanout = Fanout::new(subscribers); + // Twice what the pool holds, so every write evicts. + for _ in 0..2 * CACHE_CAPACITY / PAYLOAD as u64 { + fanout.cycle(); + } + // Only the writes and the evictions they cause are timed, not the reads that + // judge each written group cached again. + b.iter_custom(|iterations| { + let mut elapsed = Duration::ZERO; + for _ in 0..iterations { + let start = Instant::now(); + for _ in 0..evicted { + let mut group = fanout.track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, fanout.payload.clone()).unwrap(); + group.finish().unwrap(); + } + elapsed += start.elapsed(); + for (subscriber, waiter) in fanout.subscribers.iter_mut().zip(&fanout.waiters) { + while let Poll::Ready(Ok(Some(group))) = subscriber.poll_recv_group(waiter) { + black_box(group); + } + } + } + elapsed + }); + }); + } + } + group.finish(); +} + /// Frames written per group in `track_fanout_frame`, after which the group rolls and every /// subscriber takes the next one: enough that the roll's cost per frame stays small. const FRAMES_PER_GROUP: usize = 256; @@ -1945,6 +1990,7 @@ criterion_group!( bench_budget_widening, bench_fanout, bench_fanout_frame, + bench_evict_fanout, bench_fetch_expiry, bench_fetch_budgets, bench_fetch_park, diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index eac242566d3..869d87bba75 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -18866,6 +18866,40 @@ mod test { assert_eq!(next(&mut subscriber), Some(3)); } + /// An ordered reader gives up a refill evicted again while it waits on a lower one, + /// naming it dropped and releasing its fill, rather than reading past it unnamed. + #[test] + #[cfg(feature = "async")] + fn an_ordered_reader_gives_up_a_refill_evicted_while_it_waits_below() { + let producer = track_producer("test", Info::timed()); + let dynamic = producer.requests(); + let mut subscriber = producer.subscribe(replay().with_order(Order::Ascending)); + for sequence in 0..4 { + write_group(&producer, sequence); + } + let waiter = kio::Waiter::noop(); + let next = |subscriber: &mut Subscriber| match subscriber.poll_next_group(&kio::Waiter::noop()) { + Poll::Ready(Ok(Some(group))) => Some(group.sequence), + _ => None, + }; + assert_eq!(next(&mut subscriber), Some(0)); + evict(&producer, 1); + evict(&producer, 2); + assert_eq!(next(&mut subscriber), None, "group 1 comes before 3"); + let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) else { + panic!("groups 1 and 2 are misses now"); + }; + write_filled(fill.accept(2, None), 2); + assert_eq!(next(&mut subscriber), None, "still waiting on group 1"); + evict(&producer, 2); + assert_eq!(next(&mut subscriber), None, "still waiting on group 1"); + assert_eq!(drained_drops(&mut subscriber), [(2, 3)]); + write_filled(fill.accept(1, None), 1); + drop(fill); + assert_eq!([next(&mut subscriber), next(&mut subscriber)], [Some(1), Some(3)]); + assert!(dynamic.poll_requested_fill(&waiter).is_pending(), "refilled once"); + } + /// A refill evicted again before its reader took it is given up rather than asked a /// second time, so refills that evict each other cannot loop. #[test] diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index 8e3fe0acb08..9378bcc219f 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -346,6 +346,10 @@ impl Misses { taken.remove(sequence..sequence + 1); } } + // Seen cached, though judging may not run again before it is evicted. + if let Some(inbox) = &self.inbox { + inbox.saw(state.cached); + } } Err((Error::Old, asked)) if looser(max_delay, asked) => retry.insert(groups), // A drop the track recorded is named by the drop cursor already. @@ -530,6 +534,7 @@ impl Misses { let evicted = inbox.judge(state.cached, low, top); self.evictions = state.evictions; let mut refills = RangeSet::new(); + let mut again = RangeSet::new(); for run in evicted.iter() { let run = run.start.max(low)..run.end.min(top.saturating_add(1)); if run.is_empty() { @@ -542,19 +547,33 @@ impl Misses { for fresh in self.refilled.gaps(&judged) { refills.insert(fresh); } + for refilled in self.refilled.overlapping(&judged) { + again.insert(refilled.start.max(judged.start)..refilled.end.min(judged.end)); + } } } } } - for run in refills.iter() { - self.refilled.insert(run.clone()); - candidates.insert(run.clone()); - // A fill's answer evicted before it was taken is asked again, not given up. + // Settled here rather than when read, since an ordered reader never looks at what a + // fill answered: it reads past a group gone from the cache. Still asked, its fill + // settles it; given up already, it was named. + for run in self.wanted.iter().chain(self.given_up.iter()) { + again.remove(run.clone()); + } + for (run, refill) in refills.iter().map(|run| (run, true)).chain(again.iter().map(|run| (run, false))) { + // A fill's answer evicted before it was taken is no longer held for this reader. let filled: Vec = self.filled.range(run.clone()).copied().collect(); for sequence in filled { self.filled.remove(&sequence); self.release(sequence); } + match refill { + true => { + self.refilled.insert(run.clone()); + candidates.insert(run.clone()); + } + false => self.give_up(run.clone(), given_up), + } } self.ledger_revision = ledger.revision(); self.drops_revision = drops.revision(); @@ -806,6 +825,12 @@ impl Inbox { judging.span = Some(low..=top); std::mem::take(&mut judging.evicted) } + + /// Count every slot cached by `cached` as judged cached, within the span judged last. + fn saw(&self, cached: u64) { + let mut judging = self.0.lock().unwrap_or_else(PoisonError::into_inner); + judging.cached = judging.cached.max(cached); + } } /// Every reader's [`Inbox`], pruned of dropped readers on each eviction, or once From 14298637acc60801aefbf9409af5a19e0910f496 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 31 Oct 2026 00:32:04 -0700 Subject: [PATCH 09/14] style(net): format the refill give-up Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track/miss.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index 9378bcc219f..38d7d4b7db7 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -560,7 +560,11 @@ impl Misses { for run in self.wanted.iter().chain(self.given_up.iter()) { again.remove(run.clone()); } - for (run, refill) in refills.iter().map(|run| (run, true)).chain(again.iter().map(|run| (run, false))) { + for (run, refill) in refills + .iter() + .map(|run| (run, true)) + .chain(again.iter().map(|run| (run, false))) + { // A fill's answer evicted before it was taken is no longer held for this reader. let filled: Vec = self.filled.range(run.clone()).copied().collect(); for sequence in filled { From 64c2911b4fe20e6169697a22fc410bf9b7cd66be Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 31 Oct 2026 01:53:10 -0700 Subject: [PATCH 10/14] fix(net): forget a refill once it is handed out Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track.rs | 5 +++++ rs/moq-net/src/model/track/miss.rs | 10 +++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 869d87bba75..ab3da86dc82 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -18765,6 +18765,11 @@ mod test { assert_eq!(drain_groups(&mut subscriber), [1]); assert!(dynamic.poll_requested_fill(&waiter).is_pending()); assert!(drained_drops(&mut subscriber).is_empty(), "nothing was given up"); + let Inner::Plain(cursor) = &subscriber.inner else { + unreachable!("a plain subscription"); + }; + let misses = cursor.misses.as_ref().expect("the reader judged its misses"); + assert!(misses.refilled().is_empty(), "a refill handed out is forgotten"); } /// An eviction waits only for a reader that judged the evicted group cached: a reader diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index 38d7d4b7db7..aec4a0e7e3c 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -200,6 +200,12 @@ impl Misses { &self.given_up } + /// What was refilled after an eviction and not handed out yet. + #[cfg(all(test, feature = "async"))] + pub fn refilled(&self) -> &RangeSet { + &self.refilled + } + /// The next answered group still cached in `order`, giving up one evicted before it was /// read: [`Self::judge`] asked a first eviction again, so this one was refilled already. pub fn next_filled( @@ -223,8 +229,10 @@ impl Misses { } } - /// Take `sequence` as handed out, if a fill answered it. + /// Take `sequence` as handed out, if a fill answered it. A refill handed out is never + /// judged again, so it is forgotten too. pub fn take_filled(&mut self, sequence: u64) { + self.refilled.remove(sequence..sequence + 1); if self.filled.remove(&sequence) { self.release(sequence); } From 6764deea6af9b705ebd1d7dd38dd13269b045d25 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 31 Oct 2026 02:21:32 -0700 Subject: [PATCH 11/14] fix(net): forget a refill handed out at the last sequence too Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track/miss.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index aec4a0e7e3c..fabcff2d8f6 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -232,7 +232,10 @@ impl Misses { /// Take `sequence` as handed out, if a fill answered it. A refill handed out is never /// judged again, so it is forgotten too. pub fn take_filled(&mut self, sequence: u64) { - self.refilled.remove(sequence..sequence + 1); + // Refills lie below the newest group, so never at the last sequence. + if let Some(end) = sequence.checked_add(1) { + self.refilled.remove(sequence..end); + } if self.filled.remove(&sequence) { self.release(sequence); } From 32e06e313c4c1292500c4be8f6416bed014bc4db Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 31 Oct 2026 02:52:34 -0700 Subject: [PATCH 12/14] docs(net): say why the last sequence needs no refill bookkeeping Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track/miss.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index fabcff2d8f6..82c482a3b45 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -232,7 +232,7 @@ impl Misses { /// Take `sequence` as handed out, if a fill answered it. A refill handed out is never /// judged again, so it is forgotten too. pub fn take_filled(&mut self, sequence: u64) { - // Refills lie below the newest group, so never at the last sequence. + // The last sequence is never judged, so never refilled. if let Some(end) = sequence.checked_add(1) { self.refilled.remove(sequence..end); } From c33f7f118f4e690e7e3092cbad7395740298aa58 Mon Sep 17 00:00:00 2001 From: quest fixer Date: Sat, 31 Oct 2026 12:03:38 -0700 Subject: [PATCH 13/14] test(net): follow Fill wanted rename in eviction tests --- rs/moq-net/src/model/track.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 7c83abc9b10..0d0f70049fc 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -19289,7 +19289,7 @@ mod test { panic!("group 1 is a miss now"); }; assert_eq!((fill.start().group, fill.end()), (1, 2), "group 0 was taken"); - assert_eq!(fill.next(), Some(1)); + assert_eq!(fill.wanted(), Some(1)); write_filled(fill.accept(1, None), 1); drop(fill); assert_eq!(drain_groups(&mut subscriber), [1]); @@ -19393,7 +19393,7 @@ mod test { panic!("groups 1 and 2 are misses now"); }; assert_eq!((fill.start().group, fill.end()), (1, 3)); - while let Some(sequence) = fill.next() { + while let Some(sequence) = fill.wanted() { write_filled(fill.accept(sequence, None), sequence); } drop(fill); @@ -19453,7 +19453,7 @@ mod test { let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) else { panic!("group 1 is a miss now"); }; - assert_eq!(fill.next(), Some(1)); + assert_eq!(fill.wanted(), Some(1)); write_filled(fill.accept(1, None), 1); drop(fill); evict(&producer, 1); @@ -19482,7 +19482,7 @@ mod test { let mut refilled = 0; let mut serve = || { while let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) { - while let Some(sequence) = fill.next() { + while let Some(sequence) = fill.wanted() { write(fill.accept(sequence, None), sequence); refilled += 1; } From 861f7343cae7d5657622890d7ce18c14e8ab97d0 Mon Sep 17 00:00:00 2001 From: quest fixer Date: Sat, 31 Oct 2026 23:22:36 -0700 Subject: [PATCH 14/14] fix(net): follow base removal of sync mutex re-export --- rs/moq-net/src/model/track/miss.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index 82c482a3b45..45f5b710329 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -19,7 +19,7 @@ use std::{ collections::{BTreeMap, BTreeSet}, ops::{self, RangeInclusive}, - sync::{Arc, PoisonError}, + sync::{Arc, Mutex, PoisonError}, task::Poll, time::Duration, }; @@ -31,7 +31,6 @@ pub(super) use super::fill::Asking; use super::fill::{Ask, FillState, Join}; use super::{Drift, Order, Position, TrackState}; use crate::Error; -use crate::sync::Mutex; /// What a probe judges against: the span read, the feed's newest group, where the feed /// and readers start, and the drops revision.