diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 7c370d5a628..2360abd4ecf 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -398,6 +398,8 @@ impl Consumer { groups_read: false, out: Vec::new(), drops: RangeMap::new(), + received: RangeSet::new(), + received_checked: None, reported_stale: RangeSet::new(), declared: None, declared_error: None, @@ -949,8 +951,8 @@ impl Subscriber { /// Take back every stale run this reader queued or the serving route's copy reported; /// see [`track::Subscriber::reoffer_stale`]. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(super) fn reoffer_stale(&mut self) -> Vec> { - self.reader().reoffer_stale() + pub(super) fn reoffer_stale(&mut self, delivered: &Delivered) -> Vec> { + self.reader().reoffer_stale(delivered) } /// Poll for keys the serving copy gave up that no copy may still deliver; see @@ -1114,9 +1116,14 @@ struct Reader { out: Vec<(u64, Weak<()>)>, /// Runs the serving copy gave up that the reader has yet to report; see /// [`Reader::poll_drop`]. - /// Each with whether it is a received stale run the serving copy held, which new terms - /// can supersede. - drops: RangeMap, + drops: RangeMap, + /// The part of [`Self::drops`] that is a received stale run the serving copy held, which + /// new terms can supersede. Kept apart so a poll costs these runs, not every queued one. + received: RangeSet, + /// The serving copy's [`track::Subscriber::stale_revision`] when [`Self::received`] was + /// last checked against it, so a poll rechecks those runs only after one may be gone. + /// `None` until the serving copy is first checked. + received_checked: Option, /// The stale runs reported since the last [`Reader::reoffer_stale`], whichever copy gave /// them up: an update reopens each, and the copy that reported it may no longer serve. reported_stale: RangeSet, @@ -1252,6 +1259,8 @@ impl Reader { self.generation = Some(generation); // What a former route gave up is no verdict on the one that serves now. self.drops = RangeMap::new(); + self.received = RangeSet::new(); + self.received_checked = None; let Some(track) = serving else { return }; if self.copies.last().is_some_and(|copy| copy.generation == generation) { return; @@ -1830,17 +1839,20 @@ impl Reader { /// judged again: a replaced copy may no longer deliver, so what it gave up is the new /// route's to deliver or give up. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - fn reoffer_stale(&mut self) -> Vec> { + fn reoffer_stale(&mut self, delivered: &Delivered) -> Vec> { let mut runs = std::mem::take(&mut self.reported_stale); let queued: Vec<_> = self .drops .iter() - .filter(|(_, (hole, _))| hole.stale_budget().is_some()) + .filter(|(_, hole)| hole.stale_budget().is_some()) .map(|(run, _)| run.clone()) .collect(); for run in queued { - self.drops.remove(run.clone()); - runs.insert(run); + self.remove_drop(run.clone()); + // A run delivered since it queued is owed nothing; see [`Self::poll_drop`]. + if !delivered.covers(run.clone()) { + runs.insert(run); + } } if let Poll::Ready(Some(sub)) = self.poll_serving(&kio::Waiter::noop()) { runs.extend(sub.reoffer_stale()); @@ -1860,20 +1872,22 @@ impl Reader { probe: bool, waiter: &kio::Waiter, ) -> Poll<(std::ops::Range, track::Hole)> { - let queued: Vec<_> = self - .drops - .iter() - .filter(|(_, (_, received))| *received) - .map(|(run, _)| run.clone()) - .collect(); let mut given_up = Vec::new(); let mut superseded = Vec::new(); - if let Poll::Ready(Some(sub)) = self.poll_serving(waiter) { + let ready = matches!(self.poll_serving(waiter), Poll::Ready(Some(_))); + // The serving copy's subscription, borrowed apart from the received runs it checks. + if let Some(Sub::Ready(sub)) = self.copies.last_mut().filter(|_| ready).map(|copy| &mut copy.sub) { // A received stale run queued here that new terms superseded since is unknown again: - // the copy no longer holds it, and its upstream re-offers or drops it again. - for run in queued { - superseded.extend(sub.stale_held(run.clone()).gaps(&run)); + // the copy no longer holds it, and its upstream re-offers or drops it again. Checked + // only once the copy let a stale run go, never on every poll: each poll reports one + // run, so a check per poll would cost the received runs squared to drain them. + let current = sub.stale_revision(); + if self.received_checked != Some(current) { + for run in self.received.iter() { + superseded.extend(sub.stale_held(run.clone()).gaps(run)); + } } + self.received_checked = Some(current); loop { let polled = match probe { true => sub.poll_drop(waiter), @@ -1886,21 +1900,24 @@ impl Reader { } } for run in superseded { - self.drops.remove(run); + self.remove_drop(run); } for (run, hole, received) in given_up { if !Error::Stream(hole.error()).retryable() { - self.drops.insert(run, (hole, received)); + match received { + true => self.received.insert(run.clone()), + false => self.received.remove(run.clone()), + } + self.drops.insert(run, hole); } } - let covered: Vec<_> = self - .drops - .iter() - .map(|(run, _)| run.clone()) - .filter(|run| delivered.covers(run.clone())) - .collect(); - for run in covered { - self.drops.remove(run); + // A delivered run goes once a scan reaches it, never in a pass over the whole queue: + // each poll reports one run, so such a pass would cost the queue squared to drain it, + // and a new reader can be handed one run per cached group and per gap between them. + while let Some(run) = self.drops.first_range_value().map(|(run, _)| run.clone()) + && delivered.covers(run.clone()) + { + self.remove_drop(run); } // The common case: nothing to report on one route, and the serving copy's own poll // above wakes this for the next run it gives up. A replaced copy is still judged: @@ -1924,7 +1941,12 @@ impl Reader { feeding.insert(from..until); } } - let next = self.drops.iter().find_map(|(run, (err, _))| { + let mut passed = Vec::new(); + let next = self.drops.iter().find_map(|(run, err)| { + if delivered.covers(run.clone()) { + passed.push(run.clone()); + return None; + } let run = run.start.max(start)..run.end.min(end.unwrap_or(u64::MAX)); let open = delivered .gaps(run) @@ -1932,12 +1954,15 @@ impl Reader { .find_map(|gap| feeding.gaps(&gap).next())?; Some((open, err.clone())) }); + for run in passed { + self.remove_drop(run); + } let Some((open, err)) = next else { return Poll::Pending; }; // Reported once. As on a plain cursor, the report is not final: a miss (a fill given // up) may still land, and is handed out if it does. - self.drops.remove(open.clone()); + self.remove_drop(open.clone()); match &err { err if err.stale_budget().is_some() => self.reported_stale.insert(open.clone()), // A skip or final drop is final, so no update reopens a stale report it replaced. @@ -1946,6 +1971,12 @@ impl Reader { Poll::Ready((open, err)) } + /// Forget `run` as queued, received or not. + fn remove_drop(&mut self, run: std::ops::Range) { + self.received.remove(run.clone()); + self.drops.remove(run); + } + /// The serving copy's subscription, once it answered; `None` once the track ended /// without one. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] diff --git a/rs/moq-net/src/model/skipped.rs b/rs/moq-net/src/model/skipped.rs index 74f0bb70474..a339eaeb961 100644 --- a/rs/moq-net/src/model/skipped.rs +++ b/rs/moq-net/src/model/skipped.rs @@ -3,7 +3,7 @@ use std::ops::RangeInclusive; use std::time::Duration; -use rangemap::RangeInclusiveMap; +use rangemap::{RangeInclusiveMap, RangeInclusiveSet}; /// The groups a reader skipped as stale and never handed out, with the budget each was /// judged under, as runs of consecutive keys. @@ -22,6 +22,10 @@ struct Runs { /// At most the narrowest budget any run was judged under, so a poll whose budget did /// not widen past it skips the scan. narrowest: Duration, + /// Keys whose verdict may have become final since [`Skipped::take_released`]: no + /// longer skipped, or never were. A reader settles only these, so a hand-out costs + /// what changed rather than every run, which sparse keys never merge. + released: RangeInclusiveSet, } impl Skipped { @@ -34,12 +38,27 @@ impl Skipped { skipped.runs.insert(key..=key, budget); } - /// Forget `key`, now handed out. + /// Forget `key`, now handed out or final. pub fn remove(&mut self, key: u64) { if let Some(skipped) = &mut self.0 - && !skipped.runs.is_empty() + && skipped.runs.contains_key(&key) { skipped.runs.remove(key..=key); + skipped.released.insert(key..=key); + } + } + + /// Mark `key` as one whose verdict may be final, whether or not it is skipped. + pub fn release(&mut self, key: u64) { + self.0.get_or_insert_default().released.insert(key..=key); + } + + /// The keys released since the last call: removed or pruned, or marked by + /// [`Self::release`]. Each may still be skipped again since. + pub fn take_released(&mut self) -> RangeInclusiveSet { + match &mut self.0 { + Some(skipped) => std::mem::take(&mut skipped.released), + None => RangeInclusiveSet::new(), } } @@ -111,7 +130,8 @@ impl Skipped { if !gone(&run) { break; } - skipped.runs.remove(run); + skipped.runs.remove(run.clone()); + skipped.released.insert(run); } if skipped.runs.len() > cached.saturating_mul(2) { let gone: Vec<_> = skipped @@ -121,7 +141,8 @@ impl Skipped { .filter(&mut gone) .collect(); for run in gone { - skipped.runs.remove(run); + skipped.runs.remove(run.clone()); + skipped.released.insert(run); } } } @@ -203,4 +224,25 @@ mod test { skipped.prune(0, |run| *run.start() != 5); assert_eq!(skipped.widened(WIDE), vec![5..=5]); } + + #[test] + fn released_holds_only_what_left_since_the_last_take() { + let mut skipped = Skipped::default(); + assert!(skipped.take_released().is_empty()); + for key in [10, 20, 30] { + skipped.insert(key, NARROW); + } + skipped.remove(20); + skipped.remove(25); + skipped.prune(3, |run| *run.start() < 20); + skipped.release(u64::MAX); + let released: Vec<_> = skipped.take_released().into_iter().collect(); + assert_eq!( + released, + [10..=10, 20..=20, u64::MAX..=u64::MAX], + "never-skipped 25 is not" + ); + assert!(skipped.take_released().is_empty(), "taken once"); + assert_eq!(skipped.lowest(), Some(30)); + } } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 7c47c639982..eeda631ad97 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -710,6 +710,9 @@ pub(crate) struct TrackState { // Bumped when `judged` changes, which settles received stale runs differently, so a // wire cursor scans the whole drop set again. judged_revision: u64, + // Bumped whenever a stale run leaves `drops` or another verdict replaces it, so a front's + // reader rechecks the received stale runs it queued only after one may be gone. + stale_revision: u64, drops_changes: ledger::Changes, // The pool tick past which each run of `drops` outlives the pool's expiry window, as of // when it was last given up, rounded up to a quarter of the window so runs given up @@ -1386,6 +1389,14 @@ impl TrackState { /// Keep the change index aligned with the range map, including the neighbours an /// insertion coalesces and the two pieces a removal can leave. fn change_drops(&mut self, groups: ops::Range, hole: Option) { + if hole.as_ref().is_none_or(|hole| hole.stale_budget().is_none()) + && self + .drops + .overlapping(&groups) + .any(|(_, held)| held.stale_budget().is_some()) + { + self.stale_revision += 1; + } let mut affected = groups.start.saturating_sub(1)..groups.end.saturating_add(1); let old: Vec<_> = self.drops.overlapping(&affected).map(|(run, _)| run.clone()).collect(); for run in &old { @@ -8577,18 +8588,25 @@ impl Cursor { /// accounts for below its floor. Only a group a wider budget may still offer stays /// written off, so the write-offs never outnumber the stale skips the cache holds. fn retire_written_off(&mut self) { + // Only a released key can settle: every other write-off is still skipped. Taken even + // with nothing written off, so the released keys never pile up. + let released = self.record.skipped.take_released(); // Most cursors never write a group off, and this runs on every hand-out. - if self.record.written_off.is_empty() { + if self.record.written_off.is_empty() || released.is_empty() { return; } // By run, so a long stale run costs one lookup rather than one per group. - let settled: Vec> = self - .record - .written_off - .iter() - .flat_map(|run| self.record.skipped.gaps(run.start..=run.end - 1)) - .map(|gap| *gap.start()..gap.end() + 1) - .collect(); + let mut settled: Vec> = Vec::new(); + for released in released.iter() { + let span = *released.start()..released.end().saturating_add(1); + for run in self.record.written_off.overlapping(&span) { + let (start, end) = (run.start.max(span.start), run.end.min(span.end)); + if start < end { + let gaps = self.record.skipped.gaps(start..=end - 1); + settled.extend(gaps.into_iter().map(|gap| *gap.start()..gap.end() + 1)); + } + } + } for run in settled { self.record.written_off.remove(run.clone()); self.record.delivered.insert_run(run); @@ -9224,6 +9242,8 @@ impl Cursor { // group by it, even once the group left the cache. if let Some(end) = key.checked_add(1) { self.record.written_off.insert(key..end); + // Settled by the next retire unless still skipped. + self.record.skipped.release(key); } } @@ -10448,7 +10468,7 @@ impl Subscriber { match &mut self.inner { Inner::Plain(cursor) => cursor.reoffer_stale(), Inner::Resume(resume, logical) => { - let mut runs: RangeSet = resume.reoffer_stale().into_iter().collect(); + let mut runs: RangeSet = resume.reoffer_stale(&logical.record.delivered).into_iter().collect(); // A stale run the logical record queued and has yet to report is owed again too, as // a plain cursor takes back its own: the new terms judge it anew. let queued: Vec<_> = logical @@ -10638,6 +10658,16 @@ impl Subscriber { } } + /// Bumped whenever a run [`Self::stale_held`] reported may no longer be held, so a caller + /// rechecks the runs it holds only after a change. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn stale_revision(&self) -> u64 { + match &self.inner { + Inner::Plain(cursor) => cursor.state.read().stale_revision, + Inner::Resume(..) => 0, + } + } + /// The budget this subscriber judges staleness under: its max delay, clamped to the /// track's max age bound. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))]