From bf0ec8025139d535509cc27119c3afec2bebad04 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 24 Oct 2026 22:39:44 -0700 Subject: [PATCH 01/14] chore(quest): claim quest/m1/resolved-demand.md Quest-Claim: d57037f45178d098 From 9998e7a512d01f5dd59e2ed432c1e5c9bf8d0f6b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 24 Oct 2026 22:55:29 -0700 Subject: [PATCH 02/14] fix(moq-net): re-offer cached groups a widened max age covers Checkpoint: the arrival cursor records the cached groups it skips as stale with the budget it judged them under, and a wider budget owes them again through the widening re-offer path, now in the subscription's order. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/mod.rs | 1 + rs/moq-net/src/model/resume.rs | 2 +- rs/moq-net/src/model/skipped.rs | 137 ++++++++++++++++++++ rs/moq-net/src/model/track.rs | 219 +++++++++++++++++++++++++------- 4 files changed, 310 insertions(+), 49 deletions(-) create mode 100644 rs/moq-net/src/model/skipped.rs diff --git a/rs/moq-net/src/model/mod.rs b/rs/moq-net/src/model/mod.rs index 48101f0b509..fb62ab941e4 100644 --- a/rs/moq-net/src/model/mod.rs +++ b/rs/moq-net/src/model/mod.rs @@ -21,6 +21,7 @@ mod bytes; mod clock; pub(crate) mod datagram; mod delivered; +mod skipped; mod expiry; // The DEFLATE codec behind compressed tracks. mod flate; diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 2db144d166c..e297053dca9 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -3081,7 +3081,7 @@ mod test { assert_eq!(drain(&mut sub, false), vec![4, 5]); let mut sub = sub.with_range(1..); - assert_eq!(drain(&mut sub, false), vec![1, 2, 3], "passed below the floor"); + assert_eq!(drain(&mut sub, false), vec![3, 2, 1], "passed below the floor"); let mut sub = sub.with_range(..); assert_eq!(drain(&mut sub, false), vec![0]); assert_eq!(drain(&mut sub, false), Vec::::new(), "nothing goes out twice"); diff --git a/rs/moq-net/src/model/skipped.rs b/rs/moq-net/src/model/skipped.rs new file mode 100644 index 00000000000..b4b3de9a2cb --- /dev/null +++ b/rs/moq-net/src/model/skipped.rs @@ -0,0 +1,137 @@ +//! The cached groups a reader gave up as stale, so a wider budget can offer them again. + +use std::ops::RangeInclusive; +use std::time::Duration; + +use rangemap::RangeInclusiveMap; + +/// The groups a reader skipped as stale and never handed out, with the budget each was +/// judged under, as runs of consecutive sequences. +/// +/// Only a widening reverses a stale verdict, so a run comes back once the budget grows +/// past the one it was judged under. A run is kept only while one of its groups is cached: +/// a group gone from the cache cannot be offered again, so its verdict is final. +#[derive(Default)] +pub(super) struct Skipped { + runs: RangeInclusiveMap, + /// At most the narrowest budget any run was judged under, so a poll whose budget did + /// not widen past it skips the scan. + narrowest: Duration, +} + +impl Skipped { + /// Record `sequence` as skipped under `budget`. + pub fn insert(&mut self, sequence: u64, budget: Duration) { + if self.runs.is_empty() || budget < self.narrowest { + self.narrowest = budget; + } + self.runs.insert(sequence..=sequence, budget); + } + + /// Forget `sequence`, now handed out. + pub fn remove(&mut self, sequence: u64) { + if !self.runs.is_empty() { + self.runs.remove(sequence..=sequence); + } + } + + /// Take the runs judged under a budget narrower than `budget`, lowest first. + pub fn widened(&mut self, budget: Duration) -> Vec> { + if self.runs.is_empty() || budget <= self.narrowest { + return Vec::new(); + } + let mut widened = Vec::new(); + let mut narrowest = None::; + for (run, judged) in self.runs.iter() { + match *judged < budget { + true => widened.push(run.clone()), + false => narrowest = Some(narrowest.map_or(*judged, |narrowest| narrowest.min(*judged))), + } + } + for run in &widened { + self.runs.remove(run.clone()); + } + self.narrowest = narrowest.unwrap_or_default(); + widened + } + + /// The lowest sequence skipped. + pub fn lowest(&self) -> Option { + self.runs.first_range_value().map(|(run, _)| *run.start()) + } + + /// Drop the lowest runs `gone` says no longer hold a cached group, and every such run + /// once the runs outnumber `cached` twice over, so each kept run holds a distinct cached + /// group and pruning costs O(1) per skip amortized. + pub fn prune(&mut self, cached: usize, mut gone: impl FnMut(&RangeInclusive) -> bool) { + while let Some((run, _)) = self.runs.first_range_value() { + let run = run.clone(); + if !gone(&run) { + break; + } + self.runs.remove(run); + } + if self.runs.len() > cached.saturating_mul(2) { + let gone: Vec<_> = self.runs.iter().map(|(run, _)| run.clone()).filter(&mut gone).collect(); + for run in gone { + self.runs.remove(run); + } + } + } + + /// How many runs it holds. + #[cfg(test)] + pub fn runs(&self) -> usize { + self.runs.len() + } +} + +#[cfg(test)] +mod test { + use super::*; + + const NARROW: Duration = Duration::from_secs(1); + const WIDE: Duration = Duration::from_secs(5); + + #[test] + fn a_widening_takes_only_the_runs_judged_narrower() { + let mut skipped = Skipped::default(); + skipped.insert(1, NARROW); + skipped.insert(2, NARROW); + skipped.insert(4, WIDE); + assert_eq!(skipped.runs(), 2); + assert_eq!(skipped.lowest(), Some(1)); + + assert!(skipped.widened(NARROW).is_empty(), "the same budget reverses nothing"); + assert_eq!(skipped.widened(WIDE), vec![1..=2]); + assert!(skipped.widened(WIDE).is_empty(), "a run judged at the budget stays"); + assert_eq!(skipped.widened(WIDE * 2), vec![4..=4]); + assert_eq!(skipped.lowest(), None); + } + + #[test] + fn a_hand_out_splits_a_run() { + let mut skipped = Skipped::default(); + for sequence in 3..=5 { + skipped.insert(sequence, NARROW); + } + skipped.remove(4); + assert_eq!(skipped.widened(WIDE), vec![3..=3, 5..=5]); + } + + #[test] + fn prune_drops_runs_with_nothing_cached() { + let mut skipped = Skipped::default(); + for sequence in [1, 3, 5, 7] { + skipped.insert(sequence, NARROW); + } + // The lowest runs go until one still holds a group. + skipped.prune(4, |run| *run.start() < 3); + assert_eq!(skipped.lowest(), Some(3)); + assert_eq!(skipped.runs(), 3); + + // More runs than twice the cache prunes every one that holds nothing. + skipped.prune(0, |run| *run.start() != 5); + assert_eq!(skipped.widened(WIDE), vec![5..=5]); + } +} diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 206482615fe..ad7e2f19cb1 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -19,6 +19,7 @@ use crate::{broadcast, cache, group, stats}; use super::Datagram; use super::delivered::Delivered; +use super::skipped::Skipped; use crate::ledger::{self, Ledger}; pub use super::subscription::{Anchor, Order, Position, Range, Subscription}; @@ -5141,6 +5142,8 @@ enum Inner { #[derive(Clone)] struct Drift { budget: Duration, + /// The budget is the track's max age bound, which no widening passes: its verdicts are final. + bounded: bool, edge: Edge, } @@ -5478,6 +5481,8 @@ struct Cursor { /// Groups the seek path has convicted but whose sequences no caller has committed /// past yet, keyed by sequence; see [`Self::commit_seek_stale`]. seek_pending: BTreeMap, + /// Cached groups the arrival path skipped as stale, which a wider budget owes again. + skipped: Skipped, /// The drops a wire publisher forwards, once it first polls for one. drops: Option>, /// What this cursor misses and asks fills for, once it first reads. @@ -5626,8 +5631,9 @@ struct Filter { admitted: RangeInclusiveSet, /// The floor, [`Self::revision`], and cap [`Self::admitted`] was taken at. admitted_at: Option<(u64, u64, Option)>, - /// The spans a widening newly admitted, scanned in the cache lowest first and handed - /// out ahead of the walk: each cached group in them is visited once. + /// The spans a widening newly covers, from the ranges, the floor, the cap, or the + /// budget, scanned in the cache in the subscription's order and handed out ahead of the + /// walk: each cached group in them is visited once. owed: RangeInclusiveSet, } @@ -5690,6 +5696,7 @@ impl Cursor { published: kio::Producer::new(published), stale: stats::Content::default(), seek_pending: BTreeMap::new(), + skipped: Skipped::default(), drops: None, misses: None, } @@ -5871,44 +5878,95 @@ impl Cursor { self.filter.admitted_at = Some(at); } - /// The lowest cached group still owed, consuming the spans scanned below it: only one - /// the walk could have read, never fetched backfill or a withheld group. The group - /// itself stays owed until [`Self::advance_arrival`] takes it, so a peek keeps it. + /// The next cached group still owed in the subscription's order, newest first unless + /// ascending, consuming the spans scanned on the way: only one the walk could have read, + /// never fetched backfill or a withheld group. The group itself stays owed until + /// [`Self::advance_arrival`] takes it, so a peek keeps it. fn next_owed(&mut self) -> Option { if self.filter.owed.is_empty() { return None; } + let ascending = self.subscription.read().order == Order::Ascending; let state = self.state.read(); // Nothing is read from the cache now, so leave what is owed until it is. if !state.readable() { return None; } let low = state.live_floor.unwrap_or(0); - while let Some(run) = self.filter.owed.iter().next().cloned() { + loop { + let run = match ascending { + true => self.filter.owed.first(), + false => self.filter.owed.last(), + } + .cloned()?; let from = (*run.start()).max(low); let found = (from <= *run.end()) .then(|| { - state - .lookup - .range(from..=*run.end()) - .map(|(_, slot)| slot) - .find(|slot| slot.is_read()) - .map(Slot::serving) + let mut slots = state.lookup.range(from..=*run.end()).map(|(_, slot)| slot); + match ascending { + true => slots.find(|slot| slot.is_read()), + false => slots.rev().find(|slot| slot.is_read()), + } + .map(Slot::serving) }) .flatten(); - match found { - Some(group) => { - if let Some(below) = group.sequence.checked_sub(1) - && *run.start() <= below - { - self.filter.owed.remove(*run.start()..=below); - } - return Some(group.clone()); + let Some(group) = found else { + self.filter.owed.remove(run); + continue; + }; + let scanned = match ascending { + true => group.sequence.checked_sub(1).map(|below| *run.start()..=below), + false => group.sequence.checked_add(1).map(|above| above..=*run.end()), + }; + if let Some(scanned) = scanned.filter(|scanned| !scanned.is_empty()) { + self.filter.owed.remove(scanned); + } + return Some(group.clone()); + } + } + + /// Record `sequence` as skipped stale under `budget`, so a wider one owes it again. + fn skip(&mut self, sequence: u64, budget: Duration) { + self.skipped.insert(sequence, budget); + self.prune_skipped(); + } + + /// Forget the skipped runs no cached group is left in: nothing can offer them again. + fn prune_skipped(&mut self) { + let state = self.state.read(); + self.skipped.prune(state.lookup.len(), |run| { + !state.lookup.range(run.clone()).any(|(_, slot)| slot.is_read()) + }); + } + + /// Owe again the groups this cursor skipped as stale under a budget narrower than + /// `budget`: only a widening reverses a stale verdict. Costs the skipped runs, never + /// the cache, and nothing when the budget did not widen. + fn reoffer(&mut self, budget: Duration) { + for run in self.skipped.widened(budget) { + // Judged again, so a drop not reported yet is withdrawn; a reported one stands. + if let Some(drops) = &mut self.drops { + let span = *run.start()..run.end().saturating_add(1); + let old: Vec<_> = drops + .pending + .overlapping(span.clone()) + .filter(|(_, err)| **err == StreamError::Old) + .map(|(range, _)| range.start.max(span.start)..range.end.min(span.end)) + .collect(); + for range in old { + drops.pending.remove(range); } - None => self.filter.owed.remove(run), + } + let admitted: Vec<_> = self + .filter + .admitted + .overlapping(&run) + .map(|admitted| (*admitted.start()).max(*run.start())..=(*admitted.end()).min(*run.end())) + .collect(); + for owed in admitted { + self.filter.owed.insert(owed); } } - None } // A helper to automatically apply Dropped if the state is closed without an error. @@ -5953,8 +6011,10 @@ impl Cursor { Poll::<()>::Pending }); self.poll(waiter, |state| { + let bound = state.max_age_bound(); Poll::Ready(Ok(Drift { - budget: clamp_max_delay(max_delay, state.max_age_bound()), + budget: clamp_max_delay(max_delay, bound), + bounded: bound.is_some_and(|bound| max_delay >= bound), edge: state.drift_edge(servable), })) }) @@ -5969,13 +6029,15 @@ impl Cursor { } /// Forget the gaps in `delivered` below each group past this cursor's budget now, - /// whether this track holds it or not: one landing there would be written off. + /// whether this track holds it or not: one landing there would be written off. A group + /// skipped as stale is not forgotten, since a wider budget owes it again. /// /// One edge for every group judged, so forgetting N gaps never resolves it N times. fn forget_past_budget(&self, delivered: &mut Delivered) { if delivered.above_gap().is_none() { return; } + let skipped = self.skipped.lowest(); let max_delay = self.subscription.read().budget(); let state = self.state.read(); let budget = clamp_max_delay(max_delay, state.max_age_bound()); @@ -6001,6 +6063,12 @@ impl Cursor { let last = delivered .last_before(shared) .expect("the oldest run is below its successor"); + if let Some(skipped) = skipped + && skipped < last + { + delivered.forget_below(skipped); + break; + } delivered.forget_below(last); } } @@ -6091,6 +6159,7 @@ impl Cursor { // force now, which a `Control` may have replaced since the peek. self.sync(); self.advance_arrival(peeked.group.sequence, peeked.taken); + self.skipped.remove(peeked.group.sequence); peeked.group.cache_refresh(); return Poll::Ready(Ok(Some(self.deliver(peeked.group)))); } @@ -6102,6 +6171,7 @@ impl Cursor { // One scan for the whole poll, so walking a backlog off stays linear in its size. let drift = ready!(self.poll_drift(&self.servable, waiter))?; + self.reoffer(drift.budget); self.probe(self.floor(), &drift, waiter); self.poll_recv_with(take, drift, waiter) } @@ -6115,9 +6185,10 @@ impl Cursor { waiter: &kio::Waiter, ) -> Poll>> { loop { - // Re-offer what a widening newly covers ahead of the arrival walk, lowest first: - // it is the oldest thing still owed. A group the walk has yet to reach comes - // again from it, and the subscriber's delivered set drops the repeat. + // Re-offer what a widening newly covers ahead of the arrival walk, in the + // subscription's order: it is the oldest thing still owed. A group the walk has + // yet to reach comes again from it, and the subscriber's delivered set drops the + // repeat. let owed = match self.next_owed() { Some(producer) => Some((producer, Taken::Owed)), // A fill answered what the walk never reaches: backfill is not in it. @@ -6197,9 +6268,13 @@ impl Cursor { { drops.pending.insert(consumer.sequence..end, StreamError::Old); } + if !drift.bounded { + self.skip(consumer.sequence, drift.budget); + } continue; } return Poll::Ready(Ok(Some(if take { + self.skipped.remove(consumer.sequence); self.deliver(consumer) } else { self.peeked = Some(Box::new(Peeked { @@ -6565,7 +6640,8 @@ impl Subscriber { /// a second route delivers too, is not handed out again. Once a group is past the /// budget below, the subscriber stops expecting every group before it, so a gap that /// old is never filled, which holds because a timed track refuses a group whose start - /// rewinds. + /// rewinds. The exception is a cached group it skipped as stale, which a wider budget + /// offers again. /// /// Groups are semi-reliable, and the [`Subscription::max_delay`] budget is the other /// thing (alongside eviction and a moving start) that decides which of them arrive: @@ -6583,8 +6659,9 @@ impl Subscriber { /// Honors the group range set by [`Self::with_range`]: a group beyond the cap is /// passed without blocking in-range groups that arrive behind it, and holds nothing /// here. Raising the cap, or widening the ranges, offers the groups it newly covers - /// that the cache still holds, lowest sequence first. Once the track finishes, a - /// reader with nothing left in range ends. + /// that the cache still holds, newest first unless [`Order::Ascending`]. Widening the + /// budget offers the cached groups it skipped as stale and now covers the same way. + /// Once the track finishes, a reader with nothing left in range ends. /// /// A group one of [`Subscription::ranges`] starts or ends inside reads only the frames /// they cover there. One missing the frames asked for reads [`Error::Lagged`] rather @@ -6669,6 +6746,9 @@ impl Subscriber { if ordered { self.delivered.forget_below(sequence); } + if let Inner::Plain(cursor) = &mut self.inner { + cursor.prune_skipped(); + } self.delivered.insert(sequence); let mut delivered = std::mem::take(&mut self.delivered); self.forget_past_budget(&mut delivered); @@ -6992,7 +7072,7 @@ impl Subscriber { /// empty range. Both bounds are assigned, in either direction: an omitted start is /// group 0 and an omitted end removes the cap. Lowering the floor or raising the cap /// offers the groups it newly covers that the cache still holds and this reader never - /// handed out, lowest first; a group is never handed out twice. Groups past the cap + /// handed out, in its [`Order`]; a group is never handed out twice. Groups past the cap /// stay cached, and lowering the cap below the cursor waits until it rises again or /// the track finishes. [`Self::poll_largest`] reports within these bounds. /// @@ -11660,6 +11740,7 @@ mod test { let state = producer.state.read(); let drift = Drift { budget: Duration::ZERO, + bounded: false, edge: state.drift_edge(&Servable::ALL), }; assert!( @@ -13539,7 +13620,7 @@ mod test { for ordered in [false, true] { for raise_floor in [false, true] { let producer = track_producer("test", Info::timed()); - let mut subscriber = producer.subscribe(replay()); + let mut subscriber = producer.subscribe(replay().with_order(Order::Ascending)); for sequence in [1, 2] { producer .create_group(group::Info { sequence }) @@ -13603,7 +13684,7 @@ mod test { #[test] fn peeking_keeps_a_re_offered_group_owed() { let producer = track_producer("test", Info::timed()); - let mut subscriber = producer.subscribe(replay()); + let mut subscriber = producer.subscribe(replay().with_order(Order::Ascending)); subscriber.set_range(..0); for sequence in [1, 2] { producer @@ -14137,7 +14218,7 @@ mod test { fn empty_local_cap_holds_while_another_subscriber_requests_everything() { let producer = track_producer("test", Info::timed()); let mut everything = producer.subscribe(replay()); - let mut empty = producer.subscribe(replay()); + let mut empty = producer.subscribe(replay().with_order(Order::Ascending)); empty.set_range((Bound::Unbounded, Position::group(0).group_end())); for s in 0..3 { @@ -17986,7 +18067,7 @@ mod test { assert_eq!(drain(&mut subscriber), vec![0, 1]); subscriber.update(replay().with_range(0..)).unwrap(); - assert_eq!(drain(&mut subscriber), vec![2, 3], "held back, now in range"); + assert_eq!(drain(&mut subscriber), vec![3, 2], "held back, now in range, newest first"); // The latest group is 3 now, so one back is 2, already read: only new groups follow. let mut late = producer.subscribe(replay().with_range(..1)); @@ -17996,14 +18077,14 @@ mod test { end: None, }])) .unwrap(); - assert_eq!(drain(&mut late), vec![2, 3]); + assert_eq!(drain(&mut late), vec![3, 2]); append_at(&mut producer, 4000); assert_eq!(drain(&mut late), vec![4]); } /// A narrow reader under an open aggregate passes the groups outside its range holding /// nothing for them, however many arrive, and widening its ranges still serves the - /// ones the cache holds in the new span, lowest first and once each. + /// ones the cache holds in the new span, newest first and once each. #[test] #[cfg(feature = "async")] fn a_narrow_reader_holds_nothing_for_excluded_groups() { @@ -18027,12 +18108,12 @@ mod test { narrow .update(replay().with_ranges([Range::groups(0..2), Range::groups(5..8)])) .unwrap(); - assert_eq!(drain(&mut narrow), vec![5, 6, 7]); + assert_eq!(drain(&mut narrow), vec![7, 6, 5]); narrow.set_range(..9); narrow.update(replay().with_range(0..)).unwrap(); - assert_eq!(drain(&mut narrow), vec![2, 3, 4, 8]); + assert_eq!(drain(&mut narrow), vec![8, 4, 3, 2]); narrow.set_range(..); - assert_eq!(drain(&mut narrow), (9..1000).collect::>()); + assert_eq!(drain(&mut narrow), (9..1000).rev().collect::>()); } /// Lowering the floor re-offers the cached groups the arrival walk passed below it, @@ -18050,11 +18131,11 @@ mod test { subscriber.update(replay().with_range(2..)).unwrap(); assert_eq!( drain(&mut subscriber), - vec![2, 3, 4], + vec![4, 3, 2], "passed below the floor, now in range" ); subscriber.update(replay().with_range(0..)).unwrap(); - assert_eq!(drain(&mut subscriber), vec![0, 1]); + assert_eq!(drain(&mut subscriber), vec![1, 0]); // Raising and lowering it again re-offers only what is cached in between, and // nothing already served goes out twice. @@ -18067,7 +18148,7 @@ mod test { } /// Lowering a reader's local floor offers the cached groups below the old one that it - /// passed over, lowest first, and never a group twice. + /// passed over, in its order, and never a group twice. #[test] #[cfg(feature = "async")] fn a_lowered_range_reoffers_the_unread_cached_groups() { @@ -18079,7 +18160,7 @@ mod test { assert_eq!(drain(&mut subscriber), vec![4, 5]); let mut subscriber = subscriber.with_range(1..); - assert_eq!(drain(&mut subscriber), vec![1, 2, 3], "passed below the floor"); + assert_eq!(drain(&mut subscriber), vec![3, 2, 1], "passed below the floor"); let mut subscriber = subscriber.with_range(..=1); assert_eq!(drain(&mut subscriber), vec![0], "the floor and the cap both assign"); let mut subscriber = subscriber.with_range(..); @@ -18489,10 +18570,14 @@ mod test { let mut groups: Vec<_> = (0..4) .map(|sequence| producer.create_group(group::Info { sequence }).unwrap()) .collect(); - let mut subscriber = producer.subscribe(replay().with_ranges([Range { - start: Anchor::Absolute(0), - end: Some(Anchor::Time(at(3500))), - }])); + let mut subscriber = producer.subscribe( + replay() + .with_order(Order::Ascending) + .with_ranges([Range { + start: Anchor::Absolute(0), + end: Some(Anchor::Time(at(3500))), + }]), + ); assert_eq!(drain(&mut subscriber), vec![0]); let parked = moq_clock::sim::spawn(async move { @@ -18618,4 +18703,42 @@ mod test { "the datagram took sequence 0" ); } + + /// A widened budget re-offers the cached groups the cursor skipped as stale, newest + /// first by default, and never a group it already handed out. + #[test] + #[cfg(feature = "async")] + fn a_widened_budget_reoffers_what_it_skipped_as_stale() { + let mut producer = track_producer("test", Info::timed()); + let narrow = Subscription::default().with_max_delay(Duration::from_millis(1500)); + let mut subscriber = producer.subscribe(narrow.clone()); + append_at(&mut producer, 0); + assert_eq!(drain(&mut subscriber), vec![0]); + for millis in [1000, 2000, 3000, 6000] { + append_at(&mut producer, millis); + } + // Groups 1 and 2 reach 2s and 3s, at least 1.5s behind the 6s edge. + assert_eq!(drain(&mut subscriber), vec![3, 4]); + + subscriber.update(narrow.with_max_delay(Duration::from_secs(10))).unwrap(); + assert_eq!(drain(&mut subscriber), vec![2, 1], "newest first, never 0, 3, or 4 again"); + assert_eq!(drain(&mut subscriber), Vec::::new()); + } + + /// An ascending subscription gets the re-offer lowest first. + #[test] + #[cfg(feature = "async")] + fn an_ascending_widening_reoffers_lowest_first() { + let mut producer = track_producer("test", Info::timed()); + for millis in [0, 1000, 2000, 5000] { + append_at(&mut producer, millis); + } + let narrow = Subscription::default() + .with_max_delay(Duration::from_millis(1500)) + .with_order(Order::Ascending); + let mut subscriber = producer.subscribe(narrow.clone()); + assert_eq!(drain(&mut subscriber), vec![2, 3]); + subscriber.update(narrow.with_max_delay(Duration::from_secs(10))).unwrap(); + assert_eq!(drain(&mut subscriber), vec![0, 1]); + } } From 4b2c30c9de7e4780be861e0f8a41989e739eb20d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 24 Oct 2026 23:49:56 -0700 Subject: [PATCH 03/14] fix(net): re-offer skipped groups over the wire on a widened max age The lite publisher and the JS track re-offer the cached groups a cursor skipped as stale once SUBSCRIBE_UPDATE widens the max age, newest first unless ascending. A drop already reported stays dropped; one still pending is withdrawn. Covers the relay, direct, and JS wire paths, with a bench swept over readers, cached groups, and skipped runs. Co-Authored-By: Claude Opus 5.5 --- js/net/src/internal.ts | 8 + js/net/src/lite/publisher.ts | 5 + js/net/src/lite/widened_max_age.test.ts | 69 ++++++++ js/net/src/track.test.ts | 24 +++ js/net/src/track.ts | 67 +++++++- quest/m1/resolved-demand.md | 7 + rs/moq-net/benches/track.rs | 97 +++++++++++ rs/moq-net/src/lite/publisher.rs | 42 +++++ rs/moq-net/src/model/mod.rs | 2 +- rs/moq-net/src/model/skipped.rs | 54 ++++-- rs/moq-net/src/model/track.rs | 115 +++++++++++-- rs/moq-net/tests/integration/main.rs | 1 + .../tests/integration/widened_max_age.rs | 159 ++++++++++++++++++ 13 files changed, 619 insertions(+), 31 deletions(-) create mode 100644 js/net/src/lite/widened_max_age.test.ts create mode 100644 rs/moq-net/tests/integration/widened_max_age.rs diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 311219b09bf..5fdff612948 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -463,6 +463,11 @@ export const hooks: { deliveredRuns: (subscriber: Subscriber) => number; /** Forget the gaps in another reader's `delivered` below each group past this subscriber's budget. */ forgetPastBudget: (subscriber: Subscriber, delivered: Delivered) => void; + /** + * Take `[start, end)` back out of what a subscriber's wire cursor reported dropped: a drop + * below where SUBSCRIBE_START resolved never goes out, so a wider budget may still serve it. + */ + unreport: (subscriber: Subscriber, start: number, end: number) => void; /** Whether this subscriber's budget gives up on `sequence`, judged by its successor; see {@link expireGroup}. */ staleAt: (subscriber: Subscriber, sequence: number) => Expiry; /** A group the subscriber's track still caches, as a fresh read handle. */ @@ -611,6 +616,9 @@ export const hooks: { forgetPastBudget: () => { throw new Error("track.ts not loaded"); }, + unreport: () => { + throw new Error("track.ts not loaded"); + }, tryRecvDatagram: () => { throw new Error("track.ts not loaded"); }, diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index 089effdb8b6..9b4d0c2e1b8 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -1385,6 +1385,11 @@ export class Publisher { if (range.start <= startSequence && range.end > startSequence) startSequence = Math.min(range.end, group.sequence); } + // What was given up below the start is never named, so a wider budget may serve it. + for (const range of skipped) { + if (range.start < startSequence) + hooks.unreport(track, range.start, Math.min(range.end, startSequence)); + } // A later group above an explicit floor is still delivered, so the cursor stays put. // A pre-06 subscribe that named no group is the exception: an absent // Group Start is the latest group, and this sequence becomes the floor. diff --git a/js/net/src/lite/widened_max_age.test.ts b/js/net/src/lite/widened_max_age.test.ts new file mode 100644 index 00000000000..982e2420c90 --- /dev/null +++ b/js/net/src/lite/widened_max_age.test.ts @@ -0,0 +1,69 @@ +// A subscriber widening its max age over the wire gets the cached groups the publisher skipped +// as stale, including those below where SUBSCRIBE_START resolved, and never one twice. + +import { expect, test } from "bun:test"; +import { Milli } from "@moq/clock"; +import { accept, connect } from "../connection/index.ts"; +import type { Consumer as GroupConsumer } from "../group.ts"; +import { createMockTransportPair } from "../mock.ts"; +import { Producer as OriginProducer } from "../origin.ts"; +import * as Path from "../path.ts"; +import { Timescale, Timestamp } from "../time.ts"; +import type { Subscriber as TrackSubscriber } from "../track.ts"; +import { wireOf } from "../wire.ts"; +import { ALPN_07_WIP, Version } from "./version.ts"; + +const url = new URL("https://localhost:4443/test"); + +/** Every group `reader` hands out until it goes quiet, and the read still waiting. */ +async function received( + reader: TrackSubscriber, + waiting?: Promise, +): Promise<{ got: number[]; waiting: Promise }> { + const got: number[] = []; + for (;;) { + const next = waiting ?? reader.recvGroup(); + waiting = next; + const quiet = new Promise<"quiet">((resolve) => setTimeout(() => resolve("quiet"), 100)); + const group = await Promise.race([next, quiet]); + if (group === "quiet") return { got: got.sort((a, b) => a - b), waiting: next }; + waiting = undefined; + if (!group) throw new Error("the track ended"); + got.push(group.sequence); + } +} + +test("a widened maxDelay re-offers what the publisher skipped below the start", async () => { + const pair = createMockTransportPair(ALPN_07_WIP); + const origin = new OriginProducer(); + const broadcast = origin.createBroadcast(Path.from("room")); + broadcast.announce(); + const producer = broadcast.createTrack("video", { timescale: Timescale.MILLI }); + // One group per second, so a budget picks how far back a reader reaches. + for (let sequence = 0; sequence < 10; sequence++) { + const group = producer.appendGroup(); + group.writeFrame({ payload: new Uint8Array([sequence]), timestamp: Timestamp.fromMillis(sequence * 1000) }); + group.close(); + } + + const [client, server] = await Promise.all([ + connect({ url, transport: pair.client }), + accept({ transport: pair.server, url, version: Version.DRAFT_07, publish: origin.consume() }), + ]); + const remote = wireOf(client).consume(Path.from("room")); + // Group 5 ends at 6s, a full 3s behind the 9s edge. + const reader = remote.track("video").subscribe({ maxDelay: Milli(2500) }); + const first = await received(reader); + expect(first.got).toEqual([6, 7, 8, 9]); + + reader.update({ maxDelay: Milli(6500) }); + const widened = await received(reader, first.waiting); + expect(widened.got).toEqual([2, 3, 4, 5]); + + reader.close(); + remote.close(); + broadcast.close(); + origin.close(); + client.abort(); + server.abort(); +}); diff --git a/js/net/src/track.test.ts b/js/net/src/track.test.ts index 98712920dd3..f861a1eff44 100644 --- a/js/net/src/track.test.ts +++ b/js/net/src/track.test.ts @@ -3625,3 +3625,27 @@ for (const abort of [false, true]) { producer.close(); }); } + +// A widened budget re-offers the cached groups the reader skipped as stale, and never one it +// already handed out. +test("a widened maxDelay re-offers what it skipped as stale", () => { + const producer = new TrackProducer("test").accept({ timescale: Timescale.MILLI }); + const subscriber = producer.subscribe({ maxDelay: Milli(1500) }); + stampGroup(producer, 0, 0); + expect(drainGroups(subscriber)).toEqual([0]); + for (const [sequence, ms] of [ + [1, 1000], + [2, 2000], + [3, 3000], + [4, 6000], + ]) { + stampGroup(producer, sequence, ms); + } + // Groups 1 and 2 reach 2s and 3s, at least 1.5s behind the 6s edge. + expect(drainGroups(subscriber)).toEqual([3, 4]); + + subscriber.update({ maxDelay: Milli(10_000) }); + expect(drainGroups(subscriber)).toEqual([1, 2]); + expect(drainGroups(subscriber)).toEqual([]); + producer.close(); +}); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 6f5062b84f9..7fc29e16f16 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -645,6 +645,9 @@ let reofferCached: ( end: number, offer: (sequence: number) => boolean, ) => void; +// Mirror the cached groups at `sequences` into a sink again, for a reader whose budget widened +// past the one it skipped them under; assigned in the producer's static block. +let reofferSequences: (producer: Producer, sink: TrackState, sequences: readonly number[]) => void; // Whether a group may still land on the producer's live feed; assigned in its static block. let mayLand: (producer: Producer, land: Land) => boolean; let sourceStart: (producer: Producer) => number | "pending" | undefined; @@ -868,6 +871,13 @@ export class Producer { producer.#mirror(entry, sink); } }; + reofferSequences = (producer, sink, sequences) => { + const held = new Set(sink.groups.peek().map((group) => group.sequence)); + for (const sequence of sequences) { + const entry = producer.#cached.get(sequence); + if (entry && !held.has(sequence)) producer.#mirror(entry, sink); + } + }; mayLand = (producer, land) => producer.#mayLand(land); } @@ -1801,6 +1811,8 @@ export class Producer { // One read pass's view of the budget and the live edge; see Subscriber.#drift. type Drift = { budget: number; + // The budget is the track's max age, which no widening passes: its verdicts are final. + bounded: boolean; presentation?: { sequence: number; timestamp: Timestamp }; end?: number; }; @@ -1832,6 +1844,12 @@ export class Subscriber { // Every group sequence this reader handed out, so a re-created group or one a second route // delivers is handed out once. #delivered = new Delivered(); + // Cached groups the arrival cursor skipped as stale, each with the budget it was judged + // under, so a wider budget offers them again. Only a widening reverses a stale verdict, and + // an entry is kept only while its group is cached, since nothing could offer it again. + #skipped = new Map(); + // At most the narrowest budget in #skipped, so a read whose budget did not widen skips the scan. + #narrowest = Number.POSITIVE_INFINITY; // This reader closed itself, so lowering its floor offers nothing: a producer's clean close // alone leaves the cache readable, but the reader leaving must stay terminal. #closed = false; @@ -1893,6 +1911,7 @@ export class Subscriber { const retained = info?.maxAge; return { budget: retained === undefined ? requested : Math.min(requested, retained), + bounded: retained !== undefined && requested >= retained, presentation, end, }; @@ -2075,22 +2094,63 @@ export class Subscriber { #handOut(sequence: number, ordered: boolean): void { if (ordered) this.#delivered.forgetBelow(sequence); this.#delivered.add(sequence); + this.#skipped.delete(sequence); if (this.#resume) this.#resume.forgetPastBudget(this.#delivered); else this.#forgetPastBudget(this.#delivered); } // Forget the gaps in `delivered` below each group past this subscriber's budget now, whether - // this track holds it or not: one landing there would be written off. + // this track holds it or not: one landing there would be written off. A group skipped as + // stale is not forgotten, since a wider budget offers it again. #forgetPastBudget(delivered: Delivered): void { if (delivered.aboveGap() === undefined) return; // One edge for every group judged, so forgetting N gaps never resolves it N times. const drift = this.#drift(); + let skipped: number | undefined; + for (const sequence of this.#skipped.keys()) skipped = Math.min(skipped ?? sequence, sequence); for (let oldest = delivered.aboveGap(); oldest !== undefined; oldest = delivered.aboveGap()) { if (!this.#staleAt(oldest, drift)) break; + if (skipped !== undefined && skipped < oldest) { + delivered.forgetBelow(skipped); + break; + } delivered.forgetBelow(oldest); } } + // Record `sequence` as skipped stale under `budget`, so a wider one offers it again. + #skip(sequence: number, budget: number): void { + this.#skipped.set(sequence, budget); + this.#narrowest = Math.min(this.#narrowest, budget); + // Each kept entry holds a distinct cached group, so pruning once they outnumber the + // cache twice over costs O(1) per skip amortized. + const producer = this.#state.producer; + if (!producer || this.#skipped.size <= 2 * this.#state.timeline.length) return; + for (const skipped of this.#skipped.keys()) { + if (!cachedHolds(producer, skipped)) this.#skipped.delete(skipped); + } + } + + // Offer again the cached groups skipped as stale under a budget narrower than `budget`, now + // that it widened. Costs the skipped groups, never the cache. + #reoffer(budget: number): void { + if (this.#skipped.size === 0 || budget <= this.#narrowest) return; + const sequences: number[] = []; + let narrowest = Number.POSITIVE_INFINITY; + for (const [sequence, judged] of this.#skipped) { + if (judged >= budget) { + narrowest = Math.min(narrowest, judged); + continue; + } + this.#skipped.delete(sequence); + // A sequence already reported dropped stays dropped. + if (runGaps(this.#reported, sequence, sequence + 1).length > 0) sequences.push(sequence); + } + this.#narrowest = narrowest; + const producer = this.#state.producer; + if (producer && !this.#closed) reofferSequences(producer, this.#state, sequences); + } + private constructor(name: string, state: TrackState) { this.name = name; this.#state = state; @@ -2115,6 +2175,9 @@ export class Subscriber { hooks.tryNextGroup = (subscriber) => subscriber.#tryNextGroup(); hooks.peekGroup = (subscriber, ordered) => subscriber.#peekGroup(ordered); hooks.forgetPastBudget = (subscriber, delivered) => subscriber.#forgetPastBudget(delivered); + hooks.unreport = (subscriber, start, end) => { + subscriber.#reported = removeRun(subscriber.#reported, start, end); + }; hooks.tryRecvDatagram = (subscriber) => subscriber.#tryDatagram(); hooks.deliveredRuns = (subscriber) => subscriber.#delivered.runs; hooks.datagramChanged = (subscriber, fn) => { @@ -2518,6 +2581,7 @@ export class Subscriber { } } const drift = this.#drift(); + this.#reoffer(drift.budget); for (;;) { // The buffer is sequence-sorted, so an in-range group that arrives behind a @@ -2543,6 +2607,7 @@ export class Subscriber { } if (this.#staleAt(group.sequence, drift)) { group.close(); + if (!drift.bounded) this.#skip(group.sequence, drift.budget); const dropped = report(group.sequence, group.sequence + 1, old()); if (dropped) return dropped; continue; diff --git a/quest/m1/resolved-demand.md b/quest/m1/resolved-demand.md index 0f4b16d90bc..0d6e86f5c19 100644 --- a/quest/m1/resolved-demand.md +++ b/quest/m1/resolved-demand.md @@ -19,6 +19,13 @@ copy's first one. ## Plan +In progress in [moq-dev/moq#6717](https://github.com/moq-dev/moq/pull/6717): +the re-offer landed in Rust and JS, with a skipped-run record whose lifetime +is tied to the cached groups it covers rather than the pool-billed record +below. Whether to keep that or build the full record (replacing `Delivered`, +absorbing `TrackRun::served` and `#reported`, `TooFarBehind` on eviction) is +the open decision in the PR. + Found while building SUBSCRIBE_ALL ([moq-dev/moq#5931](https://github.com/moq-dev/moq/pull/5931)), which closed unmerged; its listed-track workaround (sending the resolved start as diff --git a/rs/moq-net/benches/track.rs b/rs/moq-net/benches/track.rs index 057fff21af1..5d0bff74fd9 100644 --- a/rs/moq-net/benches/track.rs +++ b/rs/moq-net/benches/track.rs @@ -75,6 +75,11 @@ //! for the groups outside its ranges, so a widening scans only the span it newly covers, //! and a scan of the cache or the audience shows up as a slope. //! +//! `track_budget_widening` widens one reader's budget past the cached groups it skipped as +//! stale, each between two it handed out, swept over the readers beside it, the groups +//! cached, and the runs it skipped: a widening costs the runs it re-offers, so a scan of the +//! cache or the audience shows up as a slope. +//! //! `track_datagram_pressure` appends datagrams across tracks sharing a bounded pool, each //! holding a backlog for a reader that never reads, with small plain payloads and with //! compressed zeros that inflate far past their wire size: the pool bills both forms, so the @@ -1568,6 +1573,97 @@ fn bench_range_widening(c: &mut Criterion) { bench.finish(); } +/// One reader holding `runs` cached groups it skipped as stale, each between two it handed +/// out, in a cache of `cached` groups, beside `readers` idle readers; see +/// [`bench_budget_widening`]. +struct BudgetWidening { + _broadcast: broadcast::Producer, + _track: track::Producer, + reader: track::Subscriber, + _beside: Vec, + runs: u64, +} + +impl BudgetWidening { + /// A budget shorter than the 100ms between groups, and longer than wall-clock jitter. + const NARROW: Duration = Duration::from_millis(50); + + fn new(readers: usize, cached: u64, runs: u64) -> Self { + let broadcast = broadcast::Info::default().produce(); + let track = broadcast.create_track("bench", timed()).unwrap(); + let narrow = track::Subscription::default().with_max_delay(Self::NARROW); + let mut reader = track.subscribe(narrow.clone()); + let beside = (0..readers).map(|_| track.subscribe(narrow.clone())).collect(); + let waiter = kio::Waiter::noop(); + let mut write = |sequences: std::ops::Range| { + for sequence in sequences { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + group + .write_frame( + Timestamp::from_millis(100 * sequence).unwrap(), + Bytes::from_static(b"x"), + ) + .unwrap(); + group.finish().unwrap(); + } + while let Poll::Ready(Ok(Some(group))) = reader.poll_recv_group(&waiter) { + black_box(group); + } + }; + write(0..1); + // Read three at a time, the first is stale: its successor starts a full budget behind + // the third. The reader takes the other two, so each skip is a run of its own. + for run in 0..runs { + write(3 * run + 1..3 * run + 4); + } + for sequence in 3 * runs + 1..cached { + write(sequence..sequence + 1); + } + Self { + _broadcast: broadcast, + _track: track, + reader, + _beside: beside, + runs, + } + } + + fn widen(&mut self) { + let wide = track::Subscription::default().with_max_delay(AGED_BUDGET); + self.reader.update(wide).unwrap(); + let waiter = kio::Waiter::noop(); + let mut offered = 0; + while let Poll::Ready(Ok(Some(group))) = self.reader.poll_recv_group(&waiter) { + black_box(group); + offered += 1; + } + assert_eq!(offered, self.runs); + } +} + +/// One reader widening its budget past the cached groups it skipped as stale, swept over the +/// readers beside it, the groups cached, and the runs it skipped. Each re-offered group must +/// cost the same, so a scan of the cache or the audience shows up as a slope. +fn bench_budget_widening(c: &mut Criterion) { + let mut group = c.benchmark_group("track_budget_widening"); + group.sample_size(10); + for readers in [1usize, 64] { + for cached in WIDENING_CACHED { + for runs in [1u64, 16, 256].into_iter().filter(|runs| 4 * runs <= cached) { + group.throughput(Throughput::Elements(runs)); + group.bench_function(BenchmarkId::new(format!("{readers}r/{cached}c"), runs), |b| { + b.iter_batched_ref( + || BudgetWidening::new(readers, cached, runs), + BudgetWidening::widen, + BatchSize::PerIteration, + ); + }); + } + } + } + group.finish(); +} + /// Readers whose range ends at a time no group reaches yet, as groups land behind a missing /// sequence; see [`bench_time_end`]. struct TimeEnd { @@ -1726,6 +1822,7 @@ fn bench_fill(c: &mut Criterion) { criterion_group!( benches, bench_range_widening, + bench_budget_widening, bench_fanout, bench_fanout_frame, bench_fetch_expiry, diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index c6760a44464..9b19129ccb1 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -5860,6 +5860,48 @@ mod serve_group_test { drops } + /// A stale group already named in SUBSCRIBE_DROP stays dropped once SUBSCRIBE_UPDATE + /// widens the max age past it: the subscriber may have settled it already. + #[moq_clock::test] + async fn a_reported_stale_drop_survives_a_widened_max_age() { + for version in [Version::Lite05, Version::Lite06, Version::Lite07] { + let mut track = track::Producer::new( + Arc::new(broadcast::Info::default()), + "test", + crate::track::Info::timed(), + ); + let subscriber = + track.subscribe(track::Subscription::default().with_max_delay(Duration::from_millis(1500))); + let (mut run, mut writer, log) = lite_run(version, SinkSession::new(Log::default()), subscriber); + write_group(&mut track, 0, 0); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + // Groups 1 and 2 reach 2s and 3s, at least 1.5s behind the 6s edge. + for (sequence, millis) in [(1, 1000), (2, 2000), (3, 3000), (4, 6000)] { + write_group(&mut track, sequence, millis); + } + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + assert_eq!(run.ctx.outcomes.opened.load(Ordering::Relaxed), 3, "{version:?}"); + assert_eq!(drops(&mut writer, &log, version).await, [(1, 2)], "{version:?}"); + + run.update(lite::SubscribeUpdate { + wanted_group: None, + priority: 0, + max_delay: Duration::from_secs(10), + start_group: None, + end_group: None, + start_frame: 0, + end_frame: None, + order: Order::Descending, + }); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + assert_eq!( + run.ctx.outcomes.opened.load(Ordering::Relaxed), + 3, + "{version:?}: no stream for a dropped group" + ); + } + } + /// Lite-05 and later name every sequence below the end that the subscription never /// got once the track ends: one skipped between groups, and one past the last. #[moq_clock::test] diff --git a/rs/moq-net/src/model/mod.rs b/rs/moq-net/src/model/mod.rs index fb62ab941e4..2a169a6e2c0 100644 --- a/rs/moq-net/src/model/mod.rs +++ b/rs/moq-net/src/model/mod.rs @@ -21,8 +21,8 @@ mod bytes; mod clock; pub(crate) mod datagram; mod delivered; -mod skipped; mod expiry; +mod skipped; // The DEFLATE codec behind compressed tracks. mod flate; pub(crate) mod object; diff --git a/rs/moq-net/src/model/skipped.rs b/rs/moq-net/src/model/skipped.rs index b4b3de9a2cb..85331ee27fe 100644 --- a/rs/moq-net/src/model/skipped.rs +++ b/rs/moq-net/src/model/skipped.rs @@ -11,8 +11,13 @@ use rangemap::RangeInclusiveMap; /// Only a widening reverses a stale verdict, so a run comes back once the budget grows /// past the one it was judged under. A run is kept only while one of its groups is cached: /// a group gone from the cache cannot be offered again, so its verdict is final. +/// +/// Boxed on the first skip: most readers never skip one, and the cursor holding it is hot. +#[derive(Default)] +pub(super) struct Skipped(Option>); + #[derive(Default)] -pub(super) struct Skipped { +struct Runs { runs: RangeInclusiveMap, /// At most the narrowest budget any run was judged under, so a poll whose budget did /// not widen past it skips the scan. @@ -22,59 +27,74 @@ pub(super) struct Skipped { impl Skipped { /// Record `sequence` as skipped under `budget`. pub fn insert(&mut self, sequence: u64, budget: Duration) { - if self.runs.is_empty() || budget < self.narrowest { - self.narrowest = budget; + let skipped = self.0.get_or_insert_default(); + if skipped.runs.is_empty() || budget < skipped.narrowest { + skipped.narrowest = budget; } - self.runs.insert(sequence..=sequence, budget); + skipped.runs.insert(sequence..=sequence, budget); } /// Forget `sequence`, now handed out. pub fn remove(&mut self, sequence: u64) { - if !self.runs.is_empty() { - self.runs.remove(sequence..=sequence); + if let Some(skipped) = &mut self.0 + && !skipped.runs.is_empty() + { + skipped.runs.remove(sequence..=sequence); } } /// Take the runs judged under a budget narrower than `budget`, lowest first. pub fn widened(&mut self, budget: Duration) -> Vec> { - if self.runs.is_empty() || budget <= self.narrowest { + let Some(skipped) = self.0.as_mut().filter(|skipped| !skipped.runs.is_empty()) else { + return Vec::new(); + }; + if budget <= skipped.narrowest { return Vec::new(); } let mut widened = Vec::new(); let mut narrowest = None::; - for (run, judged) in self.runs.iter() { + for (run, judged) in skipped.runs.iter() { match *judged < budget { true => widened.push(run.clone()), false => narrowest = Some(narrowest.map_or(*judged, |narrowest| narrowest.min(*judged))), } } for run in &widened { - self.runs.remove(run.clone()); + skipped.runs.remove(run.clone()); } - self.narrowest = narrowest.unwrap_or_default(); + skipped.narrowest = narrowest.unwrap_or_default(); widened } /// The lowest sequence skipped. pub fn lowest(&self) -> Option { - self.runs.first_range_value().map(|(run, _)| *run.start()) + let skipped = self.0.as_ref()?; + skipped.runs.first_range_value().map(|(run, _)| *run.start()) } /// Drop the lowest runs `gone` says no longer hold a cached group, and every such run /// once the runs outnumber `cached` twice over, so each kept run holds a distinct cached /// group and pruning costs O(1) per skip amortized. pub fn prune(&mut self, cached: usize, mut gone: impl FnMut(&RangeInclusive) -> bool) { - while let Some((run, _)) = self.runs.first_range_value() { + let Some(skipped) = &mut self.0 else { + return; + }; + while let Some((run, _)) = skipped.runs.first_range_value() { let run = run.clone(); if !gone(&run) { break; } - self.runs.remove(run); + skipped.runs.remove(run); } - if self.runs.len() > cached.saturating_mul(2) { - let gone: Vec<_> = self.runs.iter().map(|(run, _)| run.clone()).filter(&mut gone).collect(); + if skipped.runs.len() > cached.saturating_mul(2) { + let gone: Vec<_> = skipped + .runs + .iter() + .map(|(run, _)| run.clone()) + .filter(&mut gone) + .collect(); for run in gone { - self.runs.remove(run); + skipped.runs.remove(run); } } } @@ -82,7 +102,7 @@ impl Skipped { /// How many runs it holds. #[cfg(test)] pub fn runs(&self) -> usize { - self.runs.len() + self.0.as_ref().map_or(0, |skipped| skipped.runs.len()) } } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index ad7e2f19cb1..66a67045030 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -18067,7 +18067,11 @@ mod test { assert_eq!(drain(&mut subscriber), vec![0, 1]); subscriber.update(replay().with_range(0..)).unwrap(); - assert_eq!(drain(&mut subscriber), vec![3, 2], "held back, now in range, newest first"); + assert_eq!( + drain(&mut subscriber), + vec![3, 2], + "held back, now in range, newest first" + ); // The latest group is 3 now, so one back is 2, already read: only new groups follow. let mut late = producer.subscribe(replay().with_range(..1)); @@ -18570,14 +18574,10 @@ mod test { let mut groups: Vec<_> = (0..4) .map(|sequence| producer.create_group(group::Info { sequence }).unwrap()) .collect(); - let mut subscriber = producer.subscribe( - replay() - .with_order(Order::Ascending) - .with_ranges([Range { - start: Anchor::Absolute(0), - end: Some(Anchor::Time(at(3500))), - }]), - ); + let mut subscriber = producer.subscribe(replay().with_order(Order::Ascending).with_ranges([Range { + start: Anchor::Absolute(0), + end: Some(Anchor::Time(at(3500))), + }])); assert_eq!(drain(&mut subscriber), vec![0]); let parked = moq_clock::sim::spawn(async move { @@ -18720,8 +18720,14 @@ mod test { // Groups 1 and 2 reach 2s and 3s, at least 1.5s behind the 6s edge. assert_eq!(drain(&mut subscriber), vec![3, 4]); - subscriber.update(narrow.with_max_delay(Duration::from_secs(10))).unwrap(); - assert_eq!(drain(&mut subscriber), vec![2, 1], "newest first, never 0, 3, or 4 again"); + subscriber + .update(narrow.with_max_delay(Duration::from_secs(10))) + .unwrap(); + assert_eq!( + drain(&mut subscriber), + vec![2, 1], + "newest first, never 0, 3, or 4 again" + ); assert_eq!(drain(&mut subscriber), Vec::::new()); } @@ -18738,7 +18744,92 @@ mod test { .with_order(Order::Ascending); let mut subscriber = producer.subscribe(narrow.clone()); assert_eq!(drain(&mut subscriber), vec![2, 3]); - subscriber.update(narrow.with_max_delay(Duration::from_secs(10))).unwrap(); + subscriber + .update(narrow.with_max_delay(Duration::from_secs(10))) + .unwrap(); assert_eq!(drain(&mut subscriber), vec![0, 1]); } + + /// How many runs of skipped groups `subscriber` holds. + fn skipped_runs(subscriber: &Subscriber) -> usize { + let Inner::Plain(cursor) = &subscriber.inner else { + unreachable!() + }; + cursor.skipped.runs() + } + + /// The groups handed out around a skipped one aging past the budget does not fold it in + /// with them: a widening still owes it. + #[test] + #[cfg(feature = "async")] + fn a_skipped_group_outlives_the_fold_of_its_neighbours() { + let mut producer = track_producer("test", Info::timed()); + let narrow = Subscription::default().with_max_delay(Duration::from_millis(1500)); + let mut subscriber = producer.subscribe(narrow.clone()); + append_at(&mut producer, 0); + assert_eq!(drain(&mut subscriber), vec![0]); + for millis in [1000, 2000, 3000, 6000] { + append_at(&mut producer, millis); + } + assert_eq!(drain(&mut subscriber), vec![3, 4]); + // Groups 3 and 4 are past the budget now, which would write off the gap below them. + for millis in [20_000, 40_000] { + append_at(&mut producer, millis); + assert_eq!(drain(&mut subscriber).len(), 1); + } + assert_eq!( + subscriber.delivered_runs(), + 2, + "folded up to the skipped groups, no further" + ); + + subscriber + .update(narrow.with_max_delay(Duration::from_secs(60))) + .unwrap(); + assert_eq!(drain(&mut subscriber), vec![2, 1]); + } + + /// A group skipped at the track's max age bound is final: no widening passes the bound, so + /// the cursor keeps no record of it and its gap folds as the groups above it age out. + #[test] + #[cfg(feature = "async")] + fn a_skip_at_the_max_age_bound_folds_into_the_floor() { + let mut producer = track_producer("test", Info::timed().with_max_age(Duration::from_millis(500))); + for millis in [0, 1000, 2000] { + append_at(&mut producer, millis); + } + let mut subscriber = producer.subscribe(Subscription::default().with_max_delay(Duration::from_secs(10))); + assert_eq!(drain(&mut subscriber), vec![1, 2]); + assert_eq!(skipped_runs(&subscriber), 0); + append_at(&mut producer, 3000); + assert_eq!(drain(&mut subscriber), vec![3]); + assert_eq!(subscriber.delivered_runs(), 1, "the gap at group 0 folded"); + } + + /// A skipped group gone from the cache can never be offered again, so the cursor forgets + /// it and a widening finds nothing. + #[test] + #[cfg(feature = "async")] + fn an_evicted_skip_is_forgotten() { + let mut producer = track_producer("test", Info::timed()); + let narrow = Subscription::default().with_max_delay(Duration::from_millis(1500)); + let mut subscriber = producer.subscribe(narrow.clone()); + append_at(&mut producer, 0); + assert_eq!(drain(&mut subscriber), vec![0]); + for millis in [1000, 2000, 3000, 6000] { + append_at(&mut producer, millis); + } + assert_eq!(drain(&mut subscriber), vec![3, 4]); + assert_eq!(skipped_runs(&subscriber), 1); + + evict(&producer, 1); + evict(&producer, 2); + append_at(&mut producer, 7000); + assert_eq!(drain(&mut subscriber), vec![5]); + assert_eq!(skipped_runs(&subscriber), 0); + subscriber + .update(narrow.with_max_delay(Duration::from_secs(60))) + .unwrap(); + assert_eq!(drain(&mut subscriber), Vec::::new()); + } } diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index 2c03e868c98..df5c8fd09f1 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -73,4 +73,5 @@ mod unannounce_release; mod untimed; mod wide_hop; mod wide_sequence; +mod widened_max_age; mod zero_copy; diff --git a/rs/moq-net/tests/integration/widened_max_age.rs b/rs/moq-net/tests/integration/widened_max_age.rs new file mode 100644 index 00000000000..579906df092 --- /dev/null +++ b/rs/moq-net/tests/integration/widened_max_age.rs @@ -0,0 +1,159 @@ +#![cfg(feature = "async")] + +//! A relay a narrow reader warmed serves a newcomer with a wider max age the groups a cold +//! relay would: the widened SUBSCRIBE_UPDATE has the publisher re-offer the cached groups +//! it skipped as stale, below where the copy started. + +use std::collections::BTreeSet; +use std::time::Duration; + +use moq_net::track::{Info, Range, Subscription}; +use moq_net::{Hop, Timescale, Timestamp, Version, origin, track}; + +use crate::support::harness::{peer, spawn}; + +const TIMEOUT: Duration = Duration::from_secs(10); + +/// Groups 0 through 9 present one per second, so a budget picks how far back a reader reaches. +const GROUPS: u64 = 10; + +/// Reaches groups 6 and up: group 5 ends at 6s, a full 3s behind the 9s edge. +const NARROW: Duration = Duration::from_millis(2500); + +/// Reaches groups 2 and up. +const WIDE: Duration = Duration::from_millis(6500); + +/// Only lite-07 resolves a reader's relative start at the publisher; older wires and +/// moq-transport fill the copy's backlog by range instead, which the budget does not gate. +const VERSION: &str = "moq-lite-07-wip"; + +fn origin(hop: u64) -> origin::Producer { + let (origin, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); + spawn(driver); + origin +} + +/// A publisher holding every group, a relay, and a viewer behind it. +struct Chain { + source: track::Producer, + viewer: origin::Producer, + _hold: Vec>, +} + +impl Chain { + /// Without `relay`, the viewer reads the publisher directly. + async fn new(relay: bool) -> Self { + let version: Version = VERSION.parse().unwrap(); + let publisher = origin(1); + let relay = relay.then(|| origin(2)); + let viewer = origin(3); + let mut links = Vec::new(); + match &relay { + Some(relay) => { + links.push(peer(version, &publisher, relay).await); + links.push(peer(version, relay, &viewer).await); + } + None => links.push(peer(version, &publisher, &viewer).await), + } + let broadcast = publisher.create_broadcast("live").unwrap(); + let source = broadcast + .create_track("video", Info::default().with_timescale(Timescale::MILLI)) + .unwrap(); + for sequence in 0..GROUPS { + let mut group = source.create_group(moq_net::group::Info { sequence }).unwrap(); + group + .write_frame(Timestamp::from_millis(sequence * 1000).unwrap(), &b"x"[..]) + .unwrap(); + group.finish().unwrap(); + } + broadcast.announce(Default::default()).unwrap(); + Self { + source, + viewer, + _hold: vec![ + Box::new(publisher), + Box::new(relay), + Box::new(links), + Box::new(broadcast), + ], + } + } + + /// Subscribe from the viewer with `budget`, reading from the oldest group still around. + async fn subscribe(&self, budget: Duration) -> track::Subscriber { + let consumer = self.viewer.consume(); + consumer.routed("live").await.unwrap(); + let remote = consumer.request_broadcast("live", None).await.unwrap(); + let subscription = Subscription::default() + .with_max_delay(budget) + .with_ranges([Range::oldest()]); + remote.track("video").unwrap().subscribe(subscription).await.unwrap() + } +} + +/// Every group `subscriber` receives until it goes quiet. +async fn received(subscriber: &mut track::Subscriber) -> BTreeSet { + let mut got = BTreeSet::new(); + while let Ok(group) = moq_clock::timeout(Duration::from_secs(1), subscriber.recv_group()).await { + let group = group.unwrap().expect("the track is live"); + assert!(got.insert(group.sequence), "group {} delivered twice", group.sequence); + } + got +} + +/// What the subscriber reads after widening its own max age, through a relay or not. +#[moq_clock::test] +async fn a_widened_reader_gets_what_was_skipped() { + for relay in [false, true] { + let chain = Chain::new(relay).await; + let mut reader = moq_clock::timeout(TIMEOUT, chain.subscribe(NARROW)).await.unwrap(); + let first = moq_clock::timeout(TIMEOUT, received(&mut reader)).await.unwrap(); + assert_eq!(first, (6..GROUPS).collect(), "relay={relay}"); + + let wide = Subscription::default() + .with_max_delay(WIDE) + .with_ranges([Range::oldest()]); + reader.update(wide).unwrap(); + let widened = moq_clock::timeout(TIMEOUT, received(&mut reader)).await.unwrap(); + assert_eq!(widened, (2..6).collect(), "relay={relay}: only what was skipped"); + } +} + +#[moq_clock::test] +async fn a_warm_relay_serves_a_wider_newcomer_what_a_cold_one_would() { + let cold = Chain::new(true).await; + let mut wide = moq_clock::timeout(TIMEOUT, cold.subscribe(WIDE)).await.unwrap(); + let want = moq_clock::timeout(TIMEOUT, received(&mut wide)).await.unwrap(); + assert_eq!(want, (2..GROUPS).collect(), "cold"); + + let warm = Chain::new(true).await; + let mut narrow = moq_clock::timeout(TIMEOUT, warm.subscribe(NARROW)).await.unwrap(); + let first = moq_clock::timeout(TIMEOUT, received(&mut narrow)).await.unwrap(); + assert_eq!(first, (6..GROUPS).collect(), "the narrow reader warms the relay"); + + let mut wide = moq_clock::timeout(TIMEOUT, warm.subscribe(WIDE)).await.unwrap(); + let got = moq_clock::timeout(TIMEOUT, received(&mut wide)).await.unwrap(); + assert_eq!(got, want, "warm"); + let late = moq_clock::timeout(TIMEOUT, received(&mut narrow)).await.unwrap(); + assert!(late.is_empty(), "the narrow reader still skips {late:?}"); +} + +/// A reader joining with the max age the relay already asks for changes nothing upstream. +#[moq_clock::test] +async fn the_same_max_age_sends_no_update() { + let mut chain = Chain::new(true).await; + let mut narrow = moq_clock::timeout(TIMEOUT, chain.subscribe(NARROW)).await.unwrap(); + moq_clock::timeout(TIMEOUT, received(&mut narrow)).await.unwrap(); + while moq_clock::timeout(Duration::from_secs(1), chain.source.subscription_changed()) + .await + .is_ok() + {} + + let mut same = moq_clock::timeout(TIMEOUT, chain.subscribe(NARROW)).await.unwrap(); + assert_eq!( + moq_clock::timeout(TIMEOUT, received(&mut same)).await.unwrap(), + (6..GROUPS).collect() + ); + let changed = moq_clock::timeout(Duration::from_secs(1), chain.source.subscription_changed()).await; + assert!(changed.is_err(), "the publisher saw {changed:?}"); +} From e33878582df2772cad0bcad0f01ceed69568bd44 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 08:38:08 -0700 Subject: [PATCH 04/14] quest: finish resolved-demand, defer the pool-billed record to drop-set Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 1 - quest/m1/drop-set.md | 76 +++++++++------ quest/m1/resolved-demand.md | 180 ------------------------------------ 3 files changed, 50 insertions(+), 207 deletions(-) delete mode 100644 quest/m1/resolved-demand.md diff --git a/quest/m1/README.md b/quest/m1/README.md index 84cf6e6c7b1..e04f03eebc0 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -113,7 +113,6 @@ Catalog HLS field. - [Announcement takeover](/quest/m1/announce-takeover.md) - a new independent route explicitly continues instances served through its predecessor, with shared Rust/JS semantics and lite-07 wire - [Subscribe ranges](/quest/m1/subscribe-ranges/README.md) - a lite-07 SUBSCRIBE asks for past and live ranges with absolute or relative ends, in either order, and replaces FETCH; merged ranges never starve a subscriber; relays fill misses by range, including over moq-transport - [Updates reach fills](/quest/m1/fill-updates.md) - a reader's new priority, delay, order, or earlier start reaches fills already queued or taken -- [A widened max age re-offers cached groups](/quest/m1/resolved-demand.md) - a warm relay serves a reader with a wider max age the cached groups a cold relay would, with no wire change - [Drop set](/quest/m1/drop-set.md) - each cursor judges missing groups against its own budget, so a narrow reader drops promptly and a widening re-offers groups already reported dropped on lite-07 - [Broadcast archives](/quest/m1/archive/README.md) - ANNOUNCE, TRACKS, and a SUBSCRIBE per track record every track under a prefix to any object_store, a host serves a stored broadcast back, and the catalog drops its archive pointer for an opt-in `hls` field - [Compressed tracks](/quest/m1/compress/README.md) - a track flagged `compress` is DEFLATEd per group by moq-net, forwarded compressed, and inflated once on the relay for peers that cannot signal it; `catalog.json.z` and moq-flate go away diff --git a/quest/m1/drop-set.md b/quest/m1/drop-set.md index 222fecffd40..fa30147c076 100644 --- a/quest/m1/drop-set.md +++ b/quest/m1/drop-set.md @@ -14,9 +14,17 @@ narrower budget is served after a widening, as a cold relay would. ## Plan -Decided by the maintainer on 2026-10-19, splitting this from -[A widened max age re-offers cached groups](/quest/m1/resolved-demand.md), -which keeps drops final for now. +Decided by the maintainer on 2026-10-19, splitting this from the widened +max age re-offer +([moq-dev/moq#6717](https://github.com/moq-dev/moq/pull/6717)), which keeps +drops final for now. That PR re-offers cached groups a cursor skipped as +stale, in the subscription's order (newest first unless ascending; `js/net` +lowest first, since its buffer is ascending), through the +[moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929) owed path. Its +skipped record (`Skipped` in `rs/moq-net/src/model/skipped.rs`, `#skipped` +in `js/net/src/track.ts`) keeps a run only while the cache holds one of its +groups. On 2026-10-25 the maintainer deferred the full pool-billed +disposition record to this quest. - **One run-length set per track.** The track state keeps the dropped and expired group runs within the subscription range. Each run carries its @@ -63,21 +71,42 @@ which keeps drops final for now. `TrackRun::poll_forward_drop` emits only recorded drops, cuts, or stale cached groups. A sequence the relay never cached is not judged until the track finishes. -- **It extends resolved-demand's per-cursor disposition record,** which - stays next to the per-track set: a group that arrives leaves the shared - set, so only the cursor's record remembers that it reported cached G - stale. A reported run now carries its reason and, for a stale one, the - budget it was judged under. On lite-07 a reported stale run is no longer - final: like a skipped one, it is re-offered when a widening covers it. +- **Each cursor keeps a disposition record,** next to the per-track set: a + group that arrives leaves the shared set, so only the cursor's record + remembers that it reported cached G stale. The record is one run-length + map per cursor, each run marked delivered, skipped as stale, or reported + in SUBSCRIBE_DROP, with its reason and, for a stale run, the budget and + range it was judged under. It replaces the skipped record, `Delivered` + (`rs/moq-net/src/model/delivered.rs`), whose floor folds stale gaps in + with delivered groups, `TrackRun::served` in + `rs/moq-net/src/lite/publisher.rs`, and the JS wire cursor's `#reported`. + Delivery is recorded, not inferred from the cursor position, since an + arrival-order or descending cursor delivers out of sequence order. On + lite-07 a reported stale run is no longer final: like a skipped one, it + is re-offered when a widening covers it. +- **The record is bounded by the pool, never by guessing.** Each run is + billed to the cache's `Pool` through a `Charge` at a fixed per-run cost + and is settled a run at a time, so a cursor's state and CPU track its + runs, never their span. The floor only rises over a final prefix: the + runs from the floor up to the lowest run that is not final. An unknown + hole is never final: a descending cursor can deliver H while a lower G + is still unknown. When the pool evicts a run inside the final prefix, it + folds into the floor; any other run cannot go without guessing, so the + subscription ends with `TooFarBehind` and the subscriber re-subscribes. - **A widening re-offers the dropped runs now inside its budget,** through - the re-offer path the resolved-demand quest adds, in the subscription's - order. Upstream, the relay's own widening asks for them again. -- **Only a widening reverses a stale verdict,** as resolved-demand - decides, local or received alike: no run tracks the group that convicted - it, and an abort or eviction of that group leaves the run standing (the - open question in resolved-demand). A received run is also reversed by - the upstream re-offering its group after a widening, since a later group - wins over its drop. Cuts and never-produced runs are unaffected. + the same re-offer path as skipped runs, in the subscription's order. + Upstream, the relay's own widening asks for them again. +- **Only a widening reverses a stale verdict,** local or received alike: no + run tracks the group that convicted it, and an abort or eviction of that + group leaves the run standing. So warm/cold equivalence after an edge + loss is not promised: when the newer group that made G stale is aborted + or evicted, a cold cursor may serve G while a warm one stays past it. + Watching for it would need both groups a verdict depends on (`drifted` + uses G's successor and a possibly later presentation edge), and a run + skipped at the max age bound would no longer be final. A received run is + also reversed by the upstream re-offering its group after a widening, + since a later group wins over its drop. Cuts and never-produced runs are + unaffected. - **A stale drop names its budget.** A SUBSCRIBE_UPDATE has no acknowledgement, so after a widening from B1 to B2, an in-flight B1 drop and the B2 re-drop look identical, and no guess settles a B2 reader. On @@ -139,11 +168,10 @@ drop and arrival), and JS run state grows no faster than the JS cache itself. The JS byte bound for both is [JS unbounded groups](/quest/m1/unbounded-groups/js.md). -The per-cursor record keeps resolved-demand's pool bound and floor rule, -with "final" narrowed: only a delivered run or a budget-independent drop (a -cut, a group never produced, an upstream or origin eviction or expiry, or any drop -reported over lite-05 or lite-06) -folds into the floor. A stale run reported on lite-07 within reach of the +Only a final run folds into a cursor's floor: a delivered run, a run +skipped at the track's max age bound (no widening reaches past it), or a +budget-independent drop (a cut, a group never produced, an upstream or +origin eviction or expiry, or any drop reported over lite-05 or lite-06). A stale run reported on lite-07 within reach of the max age bound is budget-qualified, so it is treated like an unknown hole: the floor never rises over it, and if the pool must evict it or a run above it, the cursor fails with `TooFarBehind`. Folding it would forgo the @@ -220,10 +248,6 @@ Public API: none. Wire: lite-07 SUBSCRIBE_DROP (unpublished) gains `Max Age` and the widening and ordering rules above; published versions are unchanged. -## Required - -- [A widened max age re-offers cached groups](/quest/m1/resolved-demand.md) - the re-offer path this extends to reported groups - ## Related - [Subscribe ranges](/quest/m1/subscribe-ranges/README.md) - fills the cache miss an evicted run leaves diff --git a/quest/m1/resolved-demand.md b/quest/m1/resolved-demand.md deleted file mode 100644 index 0d6e86f5c19..00000000000 --- a/quest/m1/resolved-demand.md +++ /dev/null @@ -1,180 +0,0 @@ -# [M] A widened max age re-offers cached groups - -## Goal - -A warm relay serves a new reader the same groups a cold one would. Today it -does not: once a reader asking for the oldest group (`Range::oldest`) holds -a copy, its upstream subscription resolved where that reader's max age put -it, say group 10. A newcomer whose wider max age covers groups 5 through 9, -with a floor at 5 or with no floor at all, gets nothing below 10. The relay -does send the wider max age upstream in SUBSCRIBE_UPDATE, but the upstream -cursor never re-offers the cached groups it already skipped as stale. The -reader silently starts later than it asked, and only through a warm relay. -Groups the upstream already reported dropped stay out of scope here (see -[Drop set](/quest/m1/drop-set.md)). - -An archive resuming at its last stored frame through a relay that a viewer -already warmed hits this, as does any reader with a wider max age than the -copy's first one. - -## Plan - -In progress in [moq-dev/moq#6717](https://github.com/moq-dev/moq/pull/6717): -the re-offer landed in Rust and JS, with a skipped-run record whose lifetime -is tied to the cached groups it covers rather than the pool-billed record -below. Whether to keep that or build the full record (replacing `Delivered`, -absorbing `TrackRun::served` and `#reported`, `TooFarBehind` on eviction) is -the open decision in the PR. - -Found while building SUBSCRIBE_ALL -([moq-dev/moq#5931](https://github.com/moq-dev/moq/pull/5931)), which -closed unmerged; its listed-track workaround (sending the resolved start as -an absolute floor) never reached main. - -Decided by the maintainer on 2026-10-19: re-offer when the max age widens, -with no wire change. - -- **The aggregate is the union of subscriber ranges**, as - [Subscribe ranges](/quest/m1/subscribe-ranges/README.md) already decided - ("Merging subscribers is the union of their ranges"). On a version with a - single floor, the union of a no-floor reader and a floor F is no floor, so - the relay sends no floor upstream and filters F locally, which the Rust and - JS aggregates already do. A floor on the wire - is a filter (the publisher "MUST NOT deliver an older one"), so sending one - would drop a late group the no-floor reader is still owed. -- **Why max age, not the floor.** An "oldest" reader resolves at S only - because the groups below S were stale under its max age, or missing. Age is - measured against the newest group, so it only grows. A newcomer is owed a - group below S only if its max age is wider, and a wider max age already - changes the aggregate (the max of every reader's) and goes upstream in - SUBSCRIBE_UPDATE. The missing piece is the publisher acting on it. -- **Today a widened max age does not re-offer.** Confirmed in - `rs/moq-net/src/model/track.rs`: the ordered cursor (`poll_next_group`) - holds a stale conviction in `seek_pending` and walks it back if the budget - widens, but only until it delivers a later group, which commits the skip - and moves `next_sequence` past it for good. The arrival-order path drops - the gaps past budget from `Delivered` (`forget_past_budget`) and never - revisits them. Only an uncommitted conviction can come back. -- **The fix.** When a subscriber's max age widens, re-offer the cached groups - at or above its floor that its cursor gave up as stale and that the new - budget covers, in the subscription's order: newest first when descending - (the default, so congestion sheds backlog rather than the live edge), and - lowest first only when ascending. This extends the re-offer - [moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929) landed for a - widened range (a group outside the ranges is parked and re-offered when they - widen); reuse that path rather than a second one. Only - groups this cursor never delivered come back, keeping #5929's rule that - nothing already served goes out twice; a group already delivered and since - evicted downstream is a cache miss, which the relay fills upstream by range - per [Subscribe ranges](/quest/m1/subscribe-ranges/README.md) (a miss below - the cache becomes a range request), not a re-offer. -- **Only a widening reverses a stale verdict,** a bigger budget or a wider - range, decided by the maintainer on 2026-10-19 ("Narrow, then merge"). - Nothing at the same terms re-offers a group the cursor gave up, so no - cursor tracks what convicted it. -- **Open question: warm/cold equivalence after an edge loss is not - promised.** When the newer group that made G stale is aborted or evicted, - a cold cursor may serve G while a warm one stays past it. Deferred - because watching for it needs more than this plan carries: a verdict - depends on two groups (`drifted` uses G's successor for `reach` and a - possibly later presentation edge in `live_above`, and losing either can - free G), and a run skipped at the max age bound would no longer be final. Likewise, - a stale drop in flight across an aggregate range change (lite-07, see - [Drop set](/quest/m1/drop-set.md)) may settle a reader whose own range - would not convict, since SUBSCRIBE_UPDATE has no acknowledgement and the - drop names only its `Max Age`; deferred for the same reason. A group - arriving after its drop was folded into a cursor's floor under pool - pressure is not delivered to that cursor ([Drop set](/quest/m1/drop-set.md)). -- **Each cursor keeps a disposition record,** as the maintainer first framed - it ("keep what was dropped within each cursor and within the track - state"). It is what makes a re-offer safe: the cursor must tell a group it - gave up as stale (re-offer it) from one it delivered (never resend it). - The record is one run-length map per cursor, each run marked delivered, - skipped as stale (with the budget and range it was judged under), or - reported in SUBSCRIBE_DROP (final for now, below). It replaces today's `Delivered` - (`rs/moq-net/src/model/delivered.rs`), whose floor (`forget_below`) folds - stale gaps in with delivered groups, and absorbs `TrackRun::served` in - `rs/moq-net/src/lite/publisher.rs` and the JS wire cursor's `#reported` in - `js/net/src/track.ts`. Delivery is recorded, not inferred from the cursor - position: an arrival-order or descending cursor delivers out of sequence - order, and #5929 re-offers below the position. The record only answers - "re-offer or not"; the serving and re-offer path stays #5929's. -- **The record is bounded by the pool, never by guessing.** Each run is - billed to the cache's `Pool` through a `Charge` at a fixed per-run cost - and is settled a run at a time, so a cursor's - state and CPU track its runs, never their span. The floor only rises over - a final prefix: the runs from the floor up to the lowest run that is not - final. Final means delivered, reported, or skipped as stale at the - track's max age bound: only a widening reverses a verdict, and no - widening reaches past the bound since every budget is clamped to it. A skipped stale run within reach is not final, and neither - is an unknown hole: a descending or arrival-order cursor can deliver H - while a lower G is still unknown, and folding H would silently treat G as - handled. When the pool evicts one of a cursor's runs: - - A run inside the final prefix folds into the floor, as `forget_below` - does today, and nothing is lost. - - Any other run cannot go without guessing, so the cursor fails: the - subscription ends with `TooFarBehind` (stream code 0x5, already meaning - a reader that fell too far behind and lost content), the subscriber - re-subscribes, and the cursor's whole record is released with it. - - `js/net` has no byte pool and gets no new cap: a cursor's runs expire - with the same idle window as the groups they sit between, under the same - fold-or-`TooFarBehind` rule, so there are at most about one per cached - sequence and the record grows no faster than the JS cache. The JS byte - bound is [JS unbounded groups](/quest/m1/unbounded-groups/js.md). -- **Drops are final for now**, decided by the maintainer on 2026-10-19. A - widening re-offers only groups this cursor never reported in - SUBSCRIBE_DROP; a reported group stays dropped, as today. That leaves a - known warm/cold gap: on lite-05 and lite-06 the publisher reports a stale - skip as soon as it gives it up, so a group already reported is not - re-offered, while a cold relay would serve it. Closing that gap needs - per-cursor judgement of missing sequences, planned in - [Drop set](/quest/m1/drop-set.md). -- **Cost, accepted.** Groups below a newcomer's floor that the wider budget - covers also come upstream, and the relay drops them locally. Bounded by the - max age and the cache. -- Rejected: sending min(S, floors) as the wire floor, which filters late - groups for the no-floor reader; a second upstream SUBSCRIBE bounded to the - gap, which costs a slot, puts two pulls into one copy, and misses the - no-floor case; and a new wire hint. - -Watch for: - -- **Every relay changes.** A widening now costs the publisher a scan of its - cached groups below the cursor. Keep it proportional to the groups - re-offered, not the cache, and make sure a reader joining with the same max - age causes no scan and no SUBSCRIBE_UPDATE. Extend `track_subscriber_churn` - in `rs/moq-net/benches/track.rs` with widening readers, swept over - subscriber count, cached groups, and runs per cursor record. -- **`js/net` has the same bug and is in scope.** `combineSubscriptions` in - `js/net/src/track.ts` already sends no floor when any subscriber has none, - and its publisher needs the same re-offer on a widened max age. Land a JS - regression test beside the Rust one. - -Test on mock time through a relay: a newcomer with a wider max age, with a -floor below the copy's start and with none, gets the groups upstream still -caches (fails on `main`); the same through a cold relay gives the same -answer; a late group below the copy's start still reaches the no-floor -reader; a reader joining with the same max age sends no SUBSCRIBE_UPDATE; and -a re-offered group is never delivered twice to one subscriber; and a widen, -shrink, evict, widen sequence at the relay gets the evicted groups back -through the range miss fill, not a re-offer; and a stale group already reported in -SUBSCRIBE_DROP stays dropped after a widening, never served after its drop; -and a cursor that delivered F, skipped its neighbour G as stale while G -stayed cached, and delivered H, gets only G after a widening that covers -all three, through a bigger budget or a wider range, never F or H again; -and a run skipped at the max age bound folds into the floor; and a -cursor whose final prefix is evicted under pool pressure folds it and -resends nothing; and a descending cursor that delivered H while a lower G -is still unknown or skipped within reach, under pool pressure that evicts -H's run, ends with `TooFarBehind` instead of skipping G. Cover the -warm-relay case in `js/net` too. - -Public API: none. Wire: no encoding change. A publisher now re-offers -cached groups it never reported dropped when a SUBSCRIBE_UPDATE widens the -max age or the range. - -## Related - -- [Drop set](/quest/m1/drop-set.md) - closes the gap this leaves for groups already reported dropped -- [JS unbounded groups](/quest/m1/unbounded-groups/js.md) - the JS byte bound for groups and the cursor runs between them From 263b317260ea9e5718129a9947c8c8c50047d925 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 09:42:25 -0700 Subject: [PATCH 05/14] fix(net): keep re-offered skips owed until handled, respect a raised floor A widening kept the skipped record only until it queued the re-offer, so a newer re-offered group handed out first could fold an older skip into the delivered floor. JS also found its buffer index before re-offering, letting a re-offered group below a raised floor through. The wire test no longer races a wall-clock timer. Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/widened_max_age.test.ts | 27 +++++-------- js/net/src/track.test.ts | 53 +++++++++++++++++++++++++ js/net/src/track.ts | 24 +++++++---- rs/moq-net/src/model/skipped.rs | 31 +++++++++------ rs/moq-net/src/model/track.rs | 36 ++++++++++++++++- 5 files changed, 132 insertions(+), 39 deletions(-) diff --git a/js/net/src/lite/widened_max_age.test.ts b/js/net/src/lite/widened_max_age.test.ts index 982e2420c90..6ca1b931699 100644 --- a/js/net/src/lite/widened_max_age.test.ts +++ b/js/net/src/lite/widened_max_age.test.ts @@ -4,7 +4,6 @@ import { expect, test } from "bun:test"; import { Milli } from "@moq/clock"; import { accept, connect } from "../connection/index.ts"; -import type { Consumer as GroupConsumer } from "../group.ts"; import { createMockTransportPair } from "../mock.ts"; import { Producer as OriginProducer } from "../origin.ts"; import * as Path from "../path.ts"; @@ -15,22 +14,15 @@ import { ALPN_07_WIP, Version } from "./version.ts"; const url = new URL("https://localhost:4443/test"); -/** Every group `reader` hands out until it goes quiet, and the read still waiting. */ -async function received( - reader: TrackSubscriber, - waiting?: Promise, -): Promise<{ got: number[]; waiting: Promise }> { +/** The next `count` groups `reader` hands out, in sequence order. */ +async function received(reader: TrackSubscriber, count: number): Promise { const got: number[] = []; - for (;;) { - const next = waiting ?? reader.recvGroup(); - waiting = next; - const quiet = new Promise<"quiet">((resolve) => setTimeout(() => resolve("quiet"), 100)); - const group = await Promise.race([next, quiet]); - if (group === "quiet") return { got: got.sort((a, b) => a - b), waiting: next }; - waiting = undefined; + while (got.length < count) { + const group = await reader.recvGroup(); if (!group) throw new Error("the track ended"); got.push(group.sequence); } + return got.sort((a, b) => a - b); } test("a widened maxDelay re-offers what the publisher skipped below the start", async () => { @@ -53,12 +45,13 @@ test("a widened maxDelay re-offers what the publisher skipped below the start", const remote = wireOf(client).consume(Path.from("room")); // Group 5 ends at 6s, a full 3s behind the 9s edge. const reader = remote.track("video").subscribe({ maxDelay: Milli(2500) }); - const first = await received(reader); - expect(first.got).toEqual([6, 7, 8, 9]); + expect(await received(reader, 4)).toEqual([6, 7, 8, 9]); reader.update({ maxDelay: Milli(6500) }); - const widened = await received(reader, first.waiting); - expect(widened.got).toEqual([2, 3, 4, 5]); + expect(await received(reader, 4)).toEqual([2, 3, 4, 5]); + // Ending the track flushes anything else the publisher would send: nothing, not even a repeat. + producer.close(); + expect(await reader.recvGroup()).toBeUndefined(); reader.close(); remote.close(); diff --git a/js/net/src/track.test.ts b/js/net/src/track.test.ts index a38e44dd1a3..2a6c0791f86 100644 --- a/js/net/src/track.test.ts +++ b/js/net/src/track.test.ts @@ -3635,3 +3635,56 @@ test("a widened maxDelay re-offers what it skipped as stale", () => { expect(drainGroups(subscriber)).toEqual([]); producer.close(); }); + +// A re-offer never hands out a group below a floor raised in the same update. +test("a widened maxDelay re-offers nothing below a raised floor", () => { + const producer = new TrackProducer("test").accept({ timescale: Timescale.MILLI }); + const subscriber = producer.subscribe({ maxDelay: Milli(1500) }); + stampGroup(producer, 0, 0); + expect(drainGroups(subscriber)).toEqual([0]); + for (const [sequence, ms] of [ + [1, 1000], + [2, 2000], + [3, 3000], + [4, 6000], + ]) { + stampGroup(producer, sequence, ms); + } + expect(drainGroups(subscriber)).toEqual([3, 4]); + + subscriber.update({ maxDelay: Milli(10_000), groups: { start: { included: 2 } } }); + expect(drainGroups(subscriber)).toEqual([2]); + producer.close(); +}); + +// A widening that covers only the newer of two skipped groups hands it out without folding the +// older one into the delivered floor: a further widening still re-offers it. +test("a partial maxDelay widening keeps the older skip owed", () => { + const producer = new TrackProducer("test").accept({ timescale: Timescale.MILLI }); + const subscriber = producer.subscribe({ maxDelay: Milli(1500) }); + stampGroup(producer, 0, 0); + expect(drainGroups(subscriber)).toEqual([0]); + for (const [sequence, ms] of [ + [1, 1000], + [2, 2000], + [3, 4000], + ]) { + stampGroup(producer, sequence, ms); + } + expect(drainGroups(subscriber)).toEqual([2, 3]); + for (const [sequence, ms] of [ + [4, 5000], + [5, 6000], + [6, 9000], + ]) { + stampGroup(producer, sequence, ms); + } + expect(drainGroups(subscriber)).toEqual([5, 6]); + + // Group 4 reaches 6s, 3s behind the 9s edge; group 1 reaches 2s, 7s behind. + subscriber.update({ maxDelay: Milli(4000) }); + expect(drainGroups(subscriber)).toEqual([4]); + subscriber.update({ maxDelay: Milli(10_000) }); + expect(drainGroups(subscriber)).toEqual([1]); + producer.close(); +}); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 0dbbcc98646..dca5691986e 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -2046,7 +2046,9 @@ export class Subscriber { } // Offer again the cached groups skipped as stale under a budget narrower than `budget`, now - // that it widened. Costs the skipped groups, never the cache. + // that it widened. Each stays recorded, judged under `budget`, until it is handed out or + // skipped again, so a newer one handed out first never folds it into #delivered. Costs the + // skipped groups, never the cache. #reoffer(budget: number): void { if (this.#skipped.size === 0 || budget <= this.#narrowest) return; const sequences: number[] = []; @@ -2056,9 +2058,14 @@ export class Subscriber { narrowest = Math.min(narrowest, judged); continue; } - this.#skipped.delete(sequence); // A sequence already reported dropped stays dropped. - if (runGaps(this.#reported, sequence, sequence + 1).length > 0) sequences.push(sequence); + if (runGaps(this.#reported, sequence, sequence + 1).length === 0) { + this.#skipped.delete(sequence); + continue; + } + this.#skipped.set(sequence, budget); + narrowest = budget; + sequences.push(sequence); } this.#narrowest = narrowest; const producer = this.#state.producer; @@ -2474,9 +2481,6 @@ export class Subscriber { }; const old = () => new StreamError(StreamCode.Old, { message: "group is past the budget" }); - // Keep unread groups below the floor until cache eviction: replacing the - // subscription can lower it, just as raising the cap re-offers held groups. - const index = timelineIndex(groups, start); if (dispositions) { // Rescanned only after it changed, so an idle poll costs nothing per drop held. // Out-of-window runs were skipped, so a moved window rescans too. @@ -2496,6 +2500,10 @@ export class Subscriber { } const drift = this.#drift(); this.#reoffer(drift.budget); + // Keep unread groups below the floor until cache eviction: replacing the + // subscription can lower it, just as raising the cap re-offers held groups. Found + // after the re-offer, which may insert below the floor. + const index = timelineIndex(groups, start); for (;;) { // The buffer is sequence-sorted, so an in-range group that arrives behind a @@ -2521,7 +2529,9 @@ export class Subscriber { } if (this.#staleAt(group.sequence, drift)) { group.close(); - if (!drift.bounded) this.#skip(group.sequence, drift.budget); + // A re-offered group stale at the bound is final. + if (drift.bounded) this.#skipped.delete(group.sequence); + else this.#skip(group.sequence, drift.budget); const dropped = report(group.sequence, group.sequence + 1, old()); if (dropped) return dropped; continue; diff --git a/rs/moq-net/src/model/skipped.rs b/rs/moq-net/src/model/skipped.rs index 85331ee27fe..9da57597780 100644 --- a/rs/moq-net/src/model/skipped.rs +++ b/rs/moq-net/src/model/skipped.rs @@ -43,7 +43,9 @@ impl Skipped { } } - /// Take the runs judged under a budget narrower than `budget`, lowest first. + /// The runs judged under a budget narrower than `budget`, lowest first, now judged + /// under `budget`: each stays until its group is handed out or skipped again, so a + /// newer group handed out first never folds it into the delivered floor. pub fn widened(&mut self, budget: Duration) -> Vec> { let Some(skipped) = self.0.as_mut().filter(|skipped| !skipped.runs.is_empty()) else { return Vec::new(); @@ -51,18 +53,17 @@ impl Skipped { if budget <= skipped.narrowest { return Vec::new(); } - let mut widened = Vec::new(); - let mut narrowest = None::; - for (run, judged) in skipped.runs.iter() { - match *judged < budget { - true => widened.push(run.clone()), - false => narrowest = Some(narrowest.map_or(*judged, |narrowest| narrowest.min(*judged))), - } - } + let widened: Vec<_> = skipped + .runs + .iter() + .filter(|(_, judged)| **judged < budget) + .map(|(run, _)| run.clone()) + .collect(); for run in &widened { - skipped.runs.remove(run.clone()); + skipped.runs.insert(run.clone(), budget); } - skipped.narrowest = narrowest.unwrap_or_default(); + // Every run left was judged under `budget` or wider. + skipped.narrowest = budget; widened } @@ -125,8 +126,12 @@ mod test { assert!(skipped.widened(NARROW).is_empty(), "the same budget reverses nothing"); assert_eq!(skipped.widened(WIDE), vec![1..=2]); assert!(skipped.widened(WIDE).is_empty(), "a run judged at the budget stays"); - assert_eq!(skipped.widened(WIDE * 2), vec![4..=4]); - assert_eq!(skipped.lowest(), None); + assert_eq!(skipped.lowest(), Some(1), "a widened run stays until it is handed out"); + // Rejudged under WIDE, the widened run now merges with the run already judged there. + assert_eq!(skipped.widened(WIDE * 2), vec![1..=2, 4..=4]); + skipped.remove(1); + skipped.remove(2); + assert_eq!(skipped.lowest(), Some(4)); } #[test] diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 89b5d4dc850..d2b3d986881 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -6165,8 +6165,10 @@ impl Cursor { { drops.pending.insert(consumer.sequence..end, StreamError::Old); } - if !drift.bounded { - self.skip(consumer.sequence, drift.budget); + match drift.bounded { + // A re-offered group stale at the bound is final. + true => self.skipped.remove(consumer.sequence), + false => self.skip(consumer.sequence, drift.budget), } continue; } @@ -18621,6 +18623,36 @@ mod test { assert_eq!(drain(&mut subscriber), vec![0, 1]); } + /// A widening that covers only the newer of two skipped groups hands it out without + /// folding the older one into the delivered floor: a further widening still owes it. + #[test] + #[cfg(feature = "async")] + fn a_partial_widening_keeps_the_older_skip_owed() { + let mut producer = track_producer("test", Info::timed()); + let narrow = Subscription::default().with_max_delay(Duration::from_millis(1500)); + let mut subscriber = producer.subscribe(narrow.clone()); + append_at(&mut producer, 0); + assert_eq!(drain(&mut subscriber), vec![0]); + for millis in [1000, 2000, 4000] { + append_at(&mut producer, millis); + } + assert_eq!(drain(&mut subscriber), vec![2, 3]); + for millis in [5000, 6000, 9000] { + append_at(&mut producer, millis); + } + assert_eq!(drain(&mut subscriber), vec![5, 6]); + + // Group 4 reaches 6s, 3s behind the 9s edge; group 1 reaches 2s, 7s behind. + subscriber + .update(narrow.clone().with_max_delay(Duration::from_secs(4))) + .unwrap(); + assert_eq!(drain(&mut subscriber), vec![4]); + subscriber + .update(narrow.with_max_delay(Duration::from_secs(10))) + .unwrap(); + assert_eq!(drain(&mut subscriber), vec![1]); + } + /// How many runs of skipped groups `subscriber` holds. fn skipped_runs(subscriber: &Subscriber) -> usize { let Inner::Plain(cursor) = &subscriber.inner else { From 84e19c17043192160d14ce0e34b2787455bfb7b9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 10:31:07 -0700 Subject: [PATCH 06/14] fix(net): wake a parked JS read when its budget widens Co-Authored-By: Claude Opus 5.5 --- js/net/src/track.test.ts | 22 ++++++++++++++++++++++ js/net/src/track.ts | 2 ++ 2 files changed, 24 insertions(+) diff --git a/js/net/src/track.test.ts b/js/net/src/track.test.ts index 7a3fc2572c0..9d046e88693 100644 --- a/js/net/src/track.test.ts +++ b/js/net/src/track.test.ts @@ -3636,6 +3636,28 @@ test("a widened maxDelay re-offers what it skipped as stale", () => { producer.close(); }); +// A read already parked when the budget widens wakes for the re-offer. +test("a widened maxDelay wakes a parked read", async () => { + const producer = new TrackProducer("test").accept({ timescale: Timescale.MILLI }); + const subscriber = producer.subscribe({ maxDelay: Milli(1500) }); + stampGroup(producer, 0, 0); + expect(drainGroups(subscriber)).toEqual([0]); + for (const [sequence, ms] of [ + [1, 1000], + [2, 2000], + [3, 6000], + ]) { + stampGroup(producer, sequence, ms); + } + // Group 1 reaches 2s, 4s behind the 6s edge. + expect(drainGroups(subscriber)).toEqual([2, 3]); + + const parked = subscriber.recvGroup(); + subscriber.update({ maxDelay: Milli(10_000) }); + expect((await parked)?.sequence).toBe(1); + producer.close(); +}); + // A re-offer never hands out a group below a floor raised in the same update. test("a widened maxDelay re-offers nothing below a raised floor", () => { const producer = new TrackProducer("test").accept({ timescale: Timescale.MILLI }); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 176f1c5bb8a..aff205c67c9 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -2615,6 +2615,8 @@ export class Subscriber { this.#state.drops.changed(fn), this.#state.startChanged.changed(fn), this.#cursor.changed(fn), + // A widened budget re-offers the groups it skipped as stale. + this.#state.update.changed(fn), this.#state.closed.changed(fn), this.#state.end.changed(fn), this.#state.received.changed(fn), From 261b7f25f7bb8d752fcdcfed1a32d2d74ccf9efd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 10:46:45 -0700 Subject: [PATCH 07/14] perf(net): skip the skipped-record work on hand-outs that need none Co-Authored-By: Claude Opus 5.5 --- js/net/src/track.ts | 7 ++++++- rs/moq-net/src/model/skipped.rs | 5 +++++ rs/moq-net/src/model/track.rs | 4 ++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/js/net/src/track.ts b/js/net/src/track.ts index aff205c67c9..b0df6439c2a 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -2020,10 +2020,15 @@ export class Subscriber { if (delivered.aboveGap() === undefined) return; // One edge for every group judged, so forgetting N gaps never resolves it N times. const drift = this.#drift(); + // Found once a gap is stale, so a hand-out that forgets nothing never scans #skipped. let skipped: number | undefined; - for (const sequence of this.#skipped.keys()) skipped = Math.min(skipped ?? sequence, sequence); + let scanned = false; for (let oldest = delivered.aboveGap(); oldest !== undefined; oldest = delivered.aboveGap()) { if (!this.#staleAt(oldest, drift)) break; + if (!scanned) { + scanned = true; + for (const sequence of this.#skipped.keys()) skipped = Math.min(skipped ?? sequence, sequence); + } if (skipped !== undefined && skipped < oldest) { delivered.forgetBelow(skipped); break; diff --git a/rs/moq-net/src/model/skipped.rs b/rs/moq-net/src/model/skipped.rs index 9da57597780..30b7634a898 100644 --- a/rs/moq-net/src/model/skipped.rs +++ b/rs/moq-net/src/model/skipped.rs @@ -67,6 +67,11 @@ impl Skipped { widened } + /// Whether no run is recorded. + pub fn is_empty(&self) -> bool { + self.0.as_ref().is_none_or(|skipped| skipped.runs.is_empty()) + } + /// The lowest sequence skipped. pub fn lowest(&self) -> Option { let skipped = self.0.as_ref()?; diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 06fb69bf8d3..665bbf773fb 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -5830,6 +5830,10 @@ impl Cursor { /// Forget the skipped runs no cached group is left in: nothing can offer them again. fn prune_skipped(&mut self) { + // Most cursors never skip, and this runs on every hand-out. + if self.skipped.is_empty() { + return; + } let state = self.state.read(); self.skipped.prune(state.lookup.len(), |run| { !state.lookup.range(run.clone()).any(|(_, slot)| slot.is_read()) From 5812f1061d5f1ad10a53ef49fafc8228550db2bc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 11:08:50 -0700 Subject: [PATCH 08/14] fix(net): forget an evicted JS skip before it holds delivered gaps Co-Authored-By: Claude Opus 5.5 --- js/net/src/track.test.ts | 31 +++++++++++++++++++++++++++++++ js/net/src/track.ts | 10 ++++++++-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/js/net/src/track.test.ts b/js/net/src/track.test.ts index 9d046e88693..2eec85927c3 100644 --- a/js/net/src/track.test.ts +++ b/js/net/src/track.test.ts @@ -3658,6 +3658,37 @@ test("a widened maxDelay wakes a parked read", async () => { producer.close(); }); +// A skip whose group the cache evicted can never be offered again, so it stops holding the +// delivered gaps above it. +test("an evicted stale skip lets the delivered gaps fold", () => { + const time = manualTime(1_000); + try { + const producer = new TrackProducer("test").accept({ timescale: Timescale.MILLI, maxAge: Milli(30_000) }); + const subscriber = producer.subscribe({ maxDelay: Milli(10) }); + stampGroup(producer, 0, 0); + expect(drainGroups(subscriber)).toEqual([0]); + // Group 1 reaches 5ms, past the 10ms budget behind the 30ms edge. + stampGroup(producer, 1, 1); + stampGroup(producer, 2, 5); + stampGroup(producer, 3, 30); + expect(drainGroups(subscriber)).toEqual([2, 3]); + + // Past the cache window, so the next write evicts group 1. + time.set(1_000 + 2 * CACHE_WINDOW_MS); + let most = 0; + for (let sequence = 4; sequence < 300; sequence++) { + // Every third group never arrives. + if (sequence % 3 === 1) continue; + stampGroup(producer, sequence, sequence + 30); + expect(drainGroups(subscriber)).toEqual([sequence]); + most = Math.max(most, hooks.deliveredRuns(subscriber)); + } + expect(most).toBeLessThan(10); + } finally { + time.installed.close(); + } +}); + // A re-offer never hands out a group below a floor raised in the same update. test("a widened maxDelay re-offers nothing below a raised floor", () => { const producer = new TrackProducer("test").accept({ timescale: Timescale.MILLI }); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index b0df6439c2a..5788811c27e 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -2020,14 +2020,20 @@ export class Subscriber { if (delivered.aboveGap() === undefined) return; // One edge for every group judged, so forgetting N gaps never resolves it N times. const drift = this.#drift(); - // Found once a gap is stale, so a hand-out that forgets nothing never scans #skipped. + // Found once a gap is stale, so a hand-out that forgets nothing never scans #skipped. A + // skip whose group left the cache can never be offered again, so it is forgotten here + // rather than holding the gaps above it forever. let skipped: number | undefined; let scanned = false; for (let oldest = delivered.aboveGap(); oldest !== undefined; oldest = delivered.aboveGap()) { if (!this.#staleAt(oldest, drift)) break; if (!scanned) { scanned = true; - for (const sequence of this.#skipped.keys()) skipped = Math.min(skipped ?? sequence, sequence); + const producer = this.#state.producer; + for (const sequence of this.#skipped.keys()) { + if (producer && !cachedHolds(producer, sequence)) this.#skipped.delete(sequence); + else skipped = Math.min(skipped ?? sequence, sequence); + } } if (skipped !== undefined && skipped < oldest) { delivered.forgetBelow(skipped); From d149156380558b4f0d169cc779597673659d662e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 11:29:31 -0700 Subject: [PATCH 09/14] fix(net): never fold groups a widened range still owes into the delivered floor Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track.rs | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 665bbf773fb..f2debd661d4 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -5931,14 +5931,16 @@ impl Cursor { /// Forget the gaps in `delivered` below each group past this cursor's budget now, /// whether this track holds it or not: one landing there would be written off. A group - /// skipped as stale is not forgotten, since a wider budget owes it again. + /// skipped as stale is not forgotten, since a wider budget owes it again, and neither is + /// one a widening still owes, which a newest-first re-offer has yet to judge. /// /// One edge for every group judged, so forgetting N gaps never resolves it N times. fn forget_past_budget(&self, delivered: &mut Delivered) { if delivered.above_gap().is_none() { return; } - let skipped = self.skipped.lowest(); + let owed = self.filter.owed.first().map(|run| *run.start()); + let skipped = self.skipped.lowest().into_iter().chain(owed).min(); let max_delay = self.subscription.read().budget(); let state = self.state.read(); let budget = clamp_max_delay(max_delay, state.max_age_bound()); @@ -18657,6 +18659,33 @@ mod test { assert_eq!(drain(&mut subscriber), vec![1]); } + /// Handing out the newest group a widened range owes never folds the older ones it still + /// owes into the delivered floor before they are judged: a later budget widening gets the + /// ones judged stale. + #[test] + #[cfg(feature = "async")] + fn a_newest_first_range_reoffer_keeps_the_older_owed_groups() { + let mut producer = track_producer("test", Info::timed()); + for millis in [0, 1000, 2000, 3000, 6000] { + append_at(&mut producer, millis); + } + let narrow = Subscription::default().with_max_delay(Duration::from_millis(1500)); + let mut subscriber = producer.subscribe(narrow.clone()); + subscriber.set_range(3..5); + assert_eq!(drain(&mut subscriber), vec![3, 4]); + for millis in [20_000, 40_000] { + append_at(&mut producer, millis); + } + + subscriber.set_range(0..); + // Groups 0 through 2 reach 3s at most, far behind the 40s edge. + assert_eq!(drain(&mut subscriber), vec![6, 5]); + subscriber + .update(narrow.with_max_delay(Duration::from_secs(60))) + .unwrap(); + assert_eq!(drain(&mut subscriber), vec![2, 1, 0]); + } + /// How many runs of skipped groups `subscriber` holds. fn skipped_runs(subscriber: &Subscriber) -> usize { let Inner::Plain(cursor) = &subscriber.inner else { From 67de1ec2a29ec5da2aee32942b3ace5b1fde9344 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 15:59:11 -0700 Subject: [PATCH 10/14] test(net): build the widened max age update with ranges Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/publisher.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index ada71669565..b0b50fc5b4e 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -6032,10 +6032,7 @@ mod serve_group_test { wanted_group: None, priority: 0, max_delay: Duration::from_secs(10), - start_group: None, - end_group: None, - start_frame: 0, - end_frame: None, + ranges: vec![lite::Range::span(None, 0, None, None)], order: Order::Descending, }); while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} From 7e256353cb1c22509fbf28a0dfe1428e15f029a8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 16:26:39 -0700 Subject: [PATCH 11/14] quest: drop the finished resolved-demand link from relay-demand-gaps Co-Authored-By: Claude Opus 5.5 --- quest/m1/relay-demand-gaps.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/quest/m1/relay-demand-gaps.md b/quest/m1/relay-demand-gaps.md index 6cd6a955cfe..356774f2131 100644 --- a/quest/m1/relay-demand-gaps.md +++ b/quest/m1/relay-demand-gaps.md @@ -17,7 +17,3 @@ per newcomer, so it scales with subscribers; cover it with a benchmark if it reproduces. Public API: none expected. Wire: none expected. - -## Related - -- [A widened max age re-offers cached groups](/quest/m1/resolved-demand.md) From f31a929f6e6dc93ca3d61278076da2652f965595 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 16:26:39 -0700 Subject: [PATCH 12/14] bench(net): drop scheduling domains from the schedule bench moq-dev/moq#6809 removed Schedule::open_domain and open_group's domain argument, which left this bench (it includes moq-quic's schedule.rs by path) failing to compile on main. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/benches/schedule.rs | 64 +++++++++++++++------------------- 1 file changed, 28 insertions(+), 36 deletions(-) diff --git a/rs/moq-net/benches/schedule.rs b/rs/moq-net/benches/schedule.rs index d52c73f6e29..6eec6e54dd1 100644 --- a/rs/moq-net/benches/schedule.rs +++ b/rs/moq-net/benches/schedule.rs @@ -14,8 +14,8 @@ //! //! `fanout` sweeps subscriptions against groups in flight per subscription, timing one frame, //! so a scheduling cost that grows with the table rather than the touched path is a slope. -//! `datagram` sweeps send groups, domains, and queued datagrams per group against one -//! datagram sent and one evicted, for the same reason. +//! `datagram` sweeps send groups and queued datagrams per group against one datagram sent +//! and one evicted, for the same reason. use std::collections::HashMap; use std::sync::{Arc, Mutex}; @@ -212,7 +212,7 @@ impl Sim { let groups = workload .tracks .iter() - .map(|track| schedule.open_group(track.priority.into(), None)) + .map(|track| schedule.open_group(track.priority.into())) .collect(); let mut sim = Self { kind, @@ -429,7 +429,7 @@ fn saturated(subscriptions: usize, streams: usize) -> (Schedule, Vec (Schedule, Vec) { +/// A full datagram budget: `groups` send groups at one priority, with `datagrams` queued in +/// each, one packet apiece. +fn datagrams(groups: usize, datagrams: usize) -> (Schedule, Vec) { let mut schedule = Schedule::new(); - let ids: Vec<_> = (0..groups) - .map(|_| { - let domain = domains.then(|| schedule.open_domain()); - schedule.open_group(0, domain) - }) - .collect(); + let ids: Vec<_> = (0..groups).map(|_| schedule.open_group(0)).collect(); let budget = groups * datagrams * PACKET; for seq in 0..datagrams { for &group in &ids { @@ -512,31 +507,28 @@ fn datagrams(groups: usize, datagrams: usize, domains: bool) -> (Schedule 100_000 { - continue; - } - let (mut schedule, ids) = datagrams(groups, queued, domains); - let budget = schedule.datagram_bytes(); - let mut seq = queued as i64; - let mut next = 0; - group.bench_with_input(BenchmarkId::new(format!("{name}_{groups}"), queued), &(), |b, ()| { - b.iter(|| { - // Round robin, so every group keeps its share of the budget. - schedule.push_datagram(Some(ids[next]), seq, (), PACKET, budget); - next = (next + 1) % ids.len(); - seq += 1; - }); - }); + // A datagram into a full budget, which evicts the oldest of the lowest-priority group. + let mut group = c.benchmark_group("schedule_datagram_evict_groups"); + for groups in [1, 16, 256, 4_096] { + for queued in [1, 16, 256] { + if groups * queued > 100_000 { + continue; } + let (mut schedule, ids) = datagrams(groups, queued); + let budget = schedule.datagram_bytes(); + let mut seq = queued as i64; + let mut next = 0; + group.bench_with_input(BenchmarkId::new(format!("groups_{groups}"), queued), &(), |b, ()| { + b.iter(|| { + // Round robin, so every group keeps its share of the budget. + schedule.push_datagram(Some(ids[next]), seq, (), PACKET, budget); + next = (next + 1) % ids.len(); + seq += 1; + }); + }); } - group.finish(); } + group.finish(); // A datagram sent and another queued in its place, through every round robin. let mut group = c.benchmark_group("schedule_datagram_send"); @@ -545,7 +537,7 @@ fn bench_datagram(c: &mut Criterion) { if groups * queued > 100_000 { continue; } - let (mut schedule, _) = datagrams(groups, queued, false); + let (mut schedule, _) = datagrams(groups, queued); let mut seq = queued as i64; group.bench_with_input(BenchmarkId::new(format!("groups_{groups}"), queued), &(), |b, ()| { b.iter(|| { From 2dada1af98aefaa88c4832e61df62b25c61e9a7d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 16:43:54 -0700 Subject: [PATCH 13/14] fix(net): unreport below-start skips on every JS START path The idle START path (a start past everything the track has) skipped the unreport the group path did, so a later widening could not re-offer a group given up below that start. Unreport where the skipped runs are settled instead, once for both paths, matching the Rust publisher. Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/publisher.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index 9b4d0c2e1b8..6fdce03851b 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -1210,6 +1210,9 @@ export class Publisher { for (;;) { if (startSequence !== undefined && skipped.length > 0) { for (const range of skipped) { + // What was given up below the start is never named, so a wider budget may serve it. + if (range.start < startSequence) + hooks.unreport(track, range.start, Math.min(range.end, startSequence)); if (!(await drop(Math.max(range.start, startSequence), range.end, range.error))) return false; } skipped.length = 0; @@ -1385,11 +1388,6 @@ export class Publisher { if (range.start <= startSequence && range.end > startSequence) startSequence = Math.min(range.end, group.sequence); } - // What was given up below the start is never named, so a wider budget may serve it. - for (const range of skipped) { - if (range.start < startSequence) - hooks.unreport(track, range.start, Math.min(range.end, startSequence)); - } // A later group above an explicit floor is still delivered, so the cursor stays put. // A pre-06 subscribe that named no group is the exception: an absent // Group Start is the latest group, and this sequence becomes the floor. From 8d943f0369be2aa0d85cb8d1fde6540773eeb055 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 19:00:36 -0700 Subject: [PATCH 14/14] test(net): announce the widened max age broadcast through its publication Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/widened_max_age.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/js/net/src/lite/widened_max_age.test.ts b/js/net/src/lite/widened_max_age.test.ts index 6ca1b931699..67a394013a2 100644 --- a/js/net/src/lite/widened_max_age.test.ts +++ b/js/net/src/lite/widened_max_age.test.ts @@ -28,8 +28,9 @@ async function received(reader: TrackSubscriber, count: number): Promise { const pair = createMockTransportPair(ALPN_07_WIP); const origin = new OriginProducer(); - const broadcast = origin.createBroadcast(Path.from("room")); - broadcast.announce(); + const publication = origin.createBroadcast(Path.from("room")); + publication.announce(); + const broadcast = publication.broadcast; const producer = broadcast.createTrack("video", { timescale: Timescale.MILLI }); // One group per second, so a budget picks how far back a reader reaches. for (let sequence = 0; sequence < 10; sequence++) {