From 18365ab1a97fb27a01ab064972b2aef51bd9ff15 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 09:25:35 -0800 Subject: [PATCH 01/17] chore(quest): claim quest/m1/drop-set.md Quest-Claim: e74455a093d50e5a From 21446a22f3938ccae7558eb76ab09ab97964aeed Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 10:58:54 -0800 Subject: [PATCH 02/17] wip(net): bill the track drop set to the cache pool Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/benches/track.rs | 39 +++++ rs/moq-net/src/fuzz.rs | 51 +++++++ rs/moq-net/src/model/cache.rs | 9 ++ rs/moq-net/src/model/track.rs | 271 +++++++++++++++++++++++++++++++--- 4 files changed, 347 insertions(+), 23 deletions(-) diff --git a/rs/moq-net/benches/track.rs b/rs/moq-net/benches/track.rs index 725b7a5baa2..058048e8120 100644 --- a/rs/moq-net/benches/track.rs +++ b/rs/moq-net/benches/track.rs @@ -16,6 +16,13 @@ //! `track_subscriber_churn_after_peak` runs the same churn after a departed audience, //! so entries walked in proportion to the old peak show up as a slope over it. //! +//! `track_subscriber_churn_drops` drops one more run into a fragmented drop set read by wire +//! cursors, each reporting it, swept over the readers, the runs the track shares, the runs +//! each reader's range holds and already reported, and the span of each run (built with the +//! `fuzz` feature, which reaches the crate-private drop cursor). A drop must cost each reader +//! the runs it touches, so a slope over the runs, what each reader reported, or the span +//! means a cursor walks the set. +//! //! `track_subscriber_join` measures a burst of viewers joining one track: each join //! registers in the subscription list, so any per-join walk of it shows up as a slope. //! @@ -905,6 +912,37 @@ fn bench_subscriber_churn_after_peak(c: &mut Criterion) { group.finish(); } +/// Readers of a fragmented drop set, its runs, the runs each reader reported, and each run's span. +#[cfg(feature = "fuzz")] +const DROP_READERS: [usize; 3] = [1, 16, 256]; +#[cfg(feature = "fuzz")] +const DROP_RUNS: [u64; 3] = [16, 1024, 65_536]; +#[cfg(feature = "fuzz")] +const DROP_SPAN: [u64; 2] = [1, 1 << 32]; + +#[cfg(feature = "fuzz")] +fn bench_subscriber_churn_drops(c: &mut Criterion) { + let mut group = c.benchmark_group("track_subscriber_churn_drops"); + group.throughput(Throughput::Elements(1)); + for readers in DROP_READERS { + for runs in DROP_RUNS { + for reported in [1, runs] { + for span in DROP_SPAN { + let id = format!("{readers}r/{runs}n/{reported}k/span_{span}"); + group.bench_function(BenchmarkId::from_parameter(id), |b| { + let mut set = moq_net::fuzz::DropSet::new(readers, runs, reported, span); + b.iter(|| set.drop_one()); + }); + } + } + } + } + group.finish(); +} + +#[cfg(not(feature = "fuzz"))] +fn bench_subscriber_churn_drops(_: &mut Criterion) {} + /// One join and its aggregate poll, excluding the departure between iterations. fn bench_subscriber_join(c: &mut Criterion) { let mut group = c.benchmark_group("track_subscriber_join"); @@ -2134,6 +2172,7 @@ criterion_group!( bench_gc, bench_subscriber_churn, bench_subscriber_churn_after_peak, + bench_subscriber_churn_drops, bench_subscriber_join, bench_range_demand, bench_subscription_ranges, diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index 20548c21a4c..1481d40695d 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -33,6 +33,57 @@ use crate::{ /// [`seeds`] is checked against. pub type Target = fn(&[u8]) -> bool; +/// A track holding a fragmented drop set, read by wire cursors, for the drop-set bench. +pub struct DropSet { + _broadcast: broadcast::Producer, + track: track::Producer, + readers: Vec, + next: u64, + span: u64, + waiter: kio::Waiter, +} + +impl DropSet { + /// `runs` disjoint dropped runs of `span` sequences each, read by `readers` wire cursors + /// whose ranges hold the newest `reported` of them, each already reported. + pub fn new(readers: usize, runs: u64, reported: u64, span: u64) -> Self { + let broadcast = broadcast::Producer::new(broadcast::Info::default()); + let track = broadcast.create_track("drops", track::Info::default()).unwrap(); + let stride = span + 1; + let from = runs.saturating_sub(reported) * stride; + let mut readers: Vec<_> = (0..readers) + .map(|_| track.subscribe(None).with_range(from..)) + .collect(); + for run in 0..runs { + track.drop_groups(run * stride..run * stride + span, crate::StreamError::Old); + } + let waiter = kio::Waiter::noop(); + for reader in &mut readers { + while reader.poll_drop(&waiter).is_ready() {} + } + Self { + _broadcast: broadcast, + track, + readers, + next: runs * stride, + span, + waiter, + } + } + + /// Drop one more run above the rest, and have every reader report it. + pub fn drop_one(&mut self) { + let start = self.next; + self.next += self.span + 1; + self.track + .drop_groups(start..start + self.span, crate::StreamError::Old); + for reader in &mut self.readers { + assert!(reader.poll_drop(&self.waiter).is_ready(), "the new run is owed"); + assert!(reader.poll_drop(&self.waiter).is_pending(), "nothing else is"); + } + } +} + /// Build a cached group with optional private object fields for model benchmarks. pub fn object_group(frames: usize, properties: bool) -> (broadcast::Producer, track::Producer, group::Producer) { let broadcast = broadcast::Producer::new(broadcast::Info::default()); diff --git a/rs/moq-net/src/model/cache.rs b/rs/moq-net/src/model/cache.rs index 2269c660bfa..97d5ed1f3d1 100644 --- a/rs/moq-net/src/model/cache.rs +++ b/rs/moq-net/src/model/cache.rs @@ -903,6 +903,15 @@ impl Track { } } + /// Open an empty bill against this account, for bookkeeping the track keeps beside its + /// groups, such as its drop set. + pub(super) fn extra(self: &Arc) -> Extra { + Extra { + track: Some(self.clone()), + bytes: AtomicU64::new(0), + } + } + /// Bill `bytes` of a failed group's head while a hand-off holds it for a successor route. /// /// Readers that stop polling can hold a hand-off indefinitely, so the head is billed diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index d9139965e18..fce82ac0d13 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -667,7 +667,21 @@ pub(crate) struct TrackState { // Bumped on every insert into `drops`, and when `judged` changes, so a cursor rescans // it only after a change. drops_revision: u64, + // Bumped when `judged` changes, which settles received stale runs differently, so a + // wire cursor scans the whole drop set again. + judged_revision: u64, drops_changes: ledger::Changes, + // The pool tick past which each run of `drops` outlives the pool's expiry window, as of + // when it was last given up, rounded up to a quarter of the window so runs given up + // together stay one entry. Empty when the pool has no expiry. + drops_born: RangeMap, + // At most the lowest tick in `drops_born`, so an expiry scan finds nothing to forget + // without walking it. + drops_oldest: Option, + // What `drops` and `drops_born` bill to the pool: [`RUN_OVERHEAD`] per entry. Paying + // eviction debt sheds the lowest runs, and the expiry scan forgets those older than the + // pool's window; either leaves their sequences unknown again. + drops_bill: cache::Extra, // How many copies the cache took in that the live feed didn't deliver: fetched backfill, // and fetched heads joining a live group, which may give it a start. Neither is in the // arrival ledger, so a time end parked on a gap looks again. @@ -948,6 +962,16 @@ impl Slot { pub(super) const CACHE_OVERHEAD: u64 = 2 * (2 * size_of::() + size_of::() + size_of::<(u64, Timestamp)>() + 2 * size_of::<(u64, u32)>()) as u64; +/// Heap the track keeps per run of its drop set: the run and its reason in `drops`, its two +/// entries in `drops_changes`, and its expiry in `drops_born`, billed per entry of either map. +/// Doubled for half-full `BTreeMap` nodes, as [`CACHE_OVERHEAD`] is. Billed to the pool, so a peer fragmenting its drops into runs costs +/// what the cache would hold, never more. +pub(super) const RUN_OVERHEAD: u64 = 2 + * (size_of::<(ops::Range, Hole)>() + + size_of::<(u64, (u64, u64))>() + + size_of::<(u64, u64)>() + + size_of::<(ops::Range, u64)>()) as u64; + /// Where the groups this hop holds start, for resolving time anchors: ordered lookups in /// the starts index and the cache, so no query walks more than the run it answers. impl Holdings for TrackState { @@ -1372,21 +1396,88 @@ impl TrackState { fn change_drops(&mut self, groups: ops::Range, hole: Option) { let mut affected = groups.start.saturating_sub(1)..groups.end.saturating_add(1); let old: Vec<_> = self.drops.overlapping(&affected).map(|(run, _)| run.clone()).collect(); - for run in old { + for run in &old { self.drops_changes.remove(run.start); affected.start = affected.start.min(run.start); affected.end = affected.end.max(run.end); } + let entries = |state: &Self, affected: &ops::Range| { + (state.drops.overlapping(affected).count() + state.drops_born.overlapping(affected).count()) as u64 + }; + let before = entries(self, &affected); match hole { - Some(hole) => self.drops.insert(groups, hole), - None => self.drops.remove(groups), + Some(hole) => { + self.drops.insert(groups.clone(), hole); + let max_ticks = self.cache.pool().expiry_ticks(); + if max_ticks != u64::MAX { + // Rounded up to a quarter of the window, so a run is forgotten no sooner + // than the window after it was given up. + let quarter = (max_ticks / 4).max(1); + let now = self.cache.pool().now(); + let expires = (now / quarter).saturating_add(1).saturating_mul(quarter).saturating_add(max_ticks); + self.drops_born.insert(groups, expires); + self.drops_oldest = Some(self.drops_oldest.map_or(expires, |oldest| oldest.min(expires))); + } + } + None => { + self.drops.remove(groups.clone()); + self.drops_born.remove(groups); + } } for (run, _) in self.drops.overlapping(&affected) { self.drops_changes.insert(run.clone()); } + let after = entries(self, &affected); + if after > before { + self.drops_bill.add((after - before) * RUN_OVERHEAD); + } else { + self.drops_bill.sub((before - after) * RUN_OVERHEAD); + } self.drops_revision += 1; } + /// Whether a run of the drop set may have outlived the pool's expiry window by `now`. + fn drops_expired(&self, now: u64) -> bool { + self.drops_oldest.is_some_and(|expires| now >= expires) + } + + /// Forget the runs of the drop set given up longer than the pool's expiry window before + /// `now`: their sequences are unknown again, as a shed run's are. Walks the set only once + /// its oldest run is due, which a quarter of the window separates from the next. + fn expire_drops(&mut self, now: u64) { + if !self.drops_expired(now) { + return; + } + let expired: Vec<_> = self + .drops_born + .iter() + .filter(|(_, expires)| now >= **expires) + .map(|(run, _)| run.clone()) + .collect(); + for run in expired { + self.change_drops(run, None); + } + self.drops_oldest = self.drops_born.iter().map(|(_, expires)| *expires).min(); + } + + /// Forget the lowest runs of the drop set until the debt is paid or `cap` bytes are + /// freed, returning the bytes freed. Their sequences are unknown again: a reader judges + /// them by its budget, or fills them by range. + fn shed_drops(&mut self, cap: u64) -> u64 { + let mut paid = 0u64; + while self.debt > 0 && paid < cap { + let Some(run) = self.drops.first_range_value().map(|(run, _)| run.clone()) else { + break; + }; + let billed = self.drops_bill.size(); + self.change_drops(run, None); + let freed = billed.saturating_sub(self.drops_bill.size()); + self.debt -= freed.min(self.debt); + paid = paid.saturating_add(freed); + } + paid + } + /// Remove an aborted slot, keeping a cut as a drop: reclaiming it must not erase a /// disposition a relay still has to forward. fn reclaim_aborted(&mut self, sequence: u64) { @@ -2115,9 +2206,11 @@ impl TrackState { } } - self.arrival - .front() - .is_some_and(|(sequence, stamp)| !self.is_current(*sequence, *stamp)) + self.drops_expired(scan.now) + || self + .arrival + .front() + .is_some_and(|(sequence, stamp)| !self.is_current(*sequence, *stamp)) || self .evict .front() @@ -2187,6 +2280,8 @@ impl TrackState { } } + self.expire_drops(scan.now); + // Trim dead leading arrival entries to advance the subscriber offset. An // entry is dead once its slot is gone or re-stamped by a newer incarnation. while let Some((sequence, stamp)) = self.arrival.front() { @@ -2366,7 +2461,10 @@ impl TrackState { ..Default::default() }); let cache = cache::Track::new(broadcast.pool.clone(), broadcast.frames.clone(), state.downgrade()); - state.write().ok().expect("a new track is open").cache = cache; + let mut new = state.write().ok().expect("a new track is open"); + new.drops_bill = cache.extra(); + new.cache = cache; + drop(new); state } @@ -2682,6 +2780,11 @@ impl TrackState { paid = paid.saturating_add(size); self.evict_group(sequence, Error::Evicted); } + // Then the drop set, oldest sequences first: losing a run costs a reader a judgment or + // a fill, never content being read. + if self.debt > 0 && paid < cap { + paid = paid.saturating_add(self.shed_drops(cap - paid)); + } // No cold group left to pay with: shed the front of the groups that could not go // whole, the ones being read and then the live edge, so neither grows without bound. fresh.extend(self.latest_group.filter(|latest| self.protects(*latest))); @@ -4124,6 +4227,7 @@ impl Producer { return RangeSet::new(); }; let cached = state.skip_groups(groups); + state.charge_debt(); let deferred = state.deferred(); drop(state); deferred.wake(); @@ -4143,6 +4247,7 @@ impl Producer { if state.judged != judged { state.judged = judged; state.drops_revision += 1; + state.judged_revision += 1; } state.supersede_stale() } @@ -4176,7 +4281,7 @@ impl Producer { .filter(|(_, slot)| slot.visible && !slot.group.is_aborted()) .map(|(&sequence, slot)| (sequence, slot.group.clone())) .collect(); - match hole.stale_budget() { + let recorded = match hole.stale_budget() { // A lite-07 publisher may serve a group after its stale drop, and the group's // stream may land first: the group wins, still arriving or not. Some(_) => { @@ -4197,7 +4302,9 @@ impl Producer { state.drop_groups(groups, hole), open.into_iter().map(|(_, group)| group).collect(), ), - } + }; + state.charge_debt(); + recorded } Err(_) => return RangeSet::new(), }; @@ -7368,8 +7475,9 @@ impl Pending { } } -/// The [`TrackState::drops_revision`], span, window revision, and the budget and coverage -/// that settle received stale runs (see [`Drift::settles`]) a [`DropCursor`] folded in. +/// The [`TrackState::judged_revision`], span, window revision, and the budget and coverage +/// that settle received stale runs (see [`Drift::settles`]) a [`DropCursor`] last scanned +/// the whole span under; between them it folds in only the runs that changed. type Scan = (u64, ops::Range, u64, Option<(Duration, Coverage)>); /// What a wire publisher's cursor gave up, found as it changes rather than by rescanning @@ -7384,6 +7492,9 @@ struct DropCursor { /// What `pending` last folded in; see [`Scan`]. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] scanned: Option, + /// The [`TrackState::drops_changes`] revision `pending` folded in. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + seen: u64, /// The budget and coverage of the last drift judged, which settle received stale runs /// until the next: a queue-only poll judges none, and must not unsettle them. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] @@ -7413,6 +7524,7 @@ impl DropCursor { cuts: state.cache.watch_cuts(), pending: RangeMap::new(), scanned: None, + seen: 0, drift: None, reported: RangeSet::new(), reported_stale: RangeSet::new(), @@ -7489,25 +7601,47 @@ impl DropCursor { self.cut(sequence, slot); } } - // Rescan only after the drops change, the window moves, or what settles a stale run - // does; out-of-window and unsettling runs were skipped and may be owed now. + // Rescan the span only after the window moves or what settles a stale run does, since + // out-of-window and unsettling runs were skipped and may be owed now; otherwise fold in + // only the runs that changed, so a drop costs each cursor the runs it touches. if let Some(drift) = within.drift { self.drift = Some((drift.budget, drift.coverage.clone())); } - let scan = (state.drops_revision, span.clone(), within.revision, self.drift.clone()); - if self.scanned.as_ref() != Some(&scan) { - self.scanned = Some(scan); - let stale: Vec<_> = self + let scan = (state.judged_revision, span.clone(), within.revision, self.drift.clone()); + let full = self.scanned.as_ref() != Some(&scan); + let changed: Vec> = match full { + false => state.drops_changes.since(self.seen).collect(), + true => { + self.scanned = Some(scan); + let received: Vec<_> = self + .pending + .iter() + .filter(|(_, pending)| pending.received) + .map(|(run, _)| run.clone()) + .collect(); + for run in received { + self.pending.remove(run); + } + vec![span.clone()] + } + }; + self.seen = state.drops_changes.revision(); + for changed in changed { + let changed = changed.start.max(span.start)..changed.end.min(span.end); + if changed.is_empty() { + continue; + } + let received: Vec<_> = self .pending - .iter() + .overlapping(&changed) .filter(|(_, pending)| pending.received) - .map(|(run, _)| run.clone()) + .map(|(run, _)| run.start.max(changed.start)..run.end.min(changed.end)) .collect(); - for run in stale { + for run in received { self.pending.remove(run); } - for (dropped, hole) in state.drops.overlapping(span.clone()) { - let run = dropped.start.max(span.start)..dropped.end.min(span.end); + for (dropped, hole) in state.drops.overlapping(&changed) { + let run = dropped.start.max(changed.start)..dropped.end.min(changed.end); // A received stale run that does not settle this reader is unknown to it. let settled = match &self.drift { Some((budget, coverage)) => settles(*budget, coverage, run.start, hole, state.judged.as_ref()), @@ -7523,6 +7657,8 @@ impl DropCursor { ); } } + } + if full { for (&sequence, slot) in state.lookup.range(span.clone()) { self.cut(sequence, slot); } @@ -7534,6 +7670,25 @@ impl DropCursor { .map(|(range, pending)| (range.clone(), pending.clone())) { self.pending.remove(range.clone()); + // A received run the track has since forgotten, shed or superseded, is unknown + // again: only the part the track still holds as it was folded in is owed. + let range = match pending.received { + true => { + let Some((held, _)) = state + .drops + .overlapping(&range) + .find(|(_, hole)| **hole == pending.hole) + else { + continue; + }; + let held = range.start.max(held.start)..range.end.min(held.end); + if held.end < range.end { + self.pending.insert(held.end..range.end, pending.clone()); + } + held + } + false => range, + }; // Only what this subscriber asked for is owed: the first run of its ranges in it, // the rest of the range waiting its turn. let Some(clipped) = window.clip(range.start.max(span.start)..range.end.min(span.end)) else { @@ -19709,8 +19864,10 @@ mod test { let settling = producer.state.downgrade().upgrade().expect("open"); drop(producer); - assert_eq!(pool.used(), 0, "the abrupt teardown still released the cache"); + // What remains is the drop the teardown recorded for the open group, held by the state. + assert_eq!(pool.used(), RUN_OVERHEAD, "the abrupt teardown still released the cache"); drop(settling); + assert_eq!(pool.used(), 0, "the state's drop set goes with it"); } /// A subscriber holding one cached group must not pin the whole track: a group @@ -23519,6 +23676,74 @@ mod test { /// Skips are kept and reported as runs: a huge `skip_groups` or append gap costs one /// entry and one report, whatever its length. + /// A run given up longer than the pool's expiry window ago is forgotten by the sweep, even + /// on a track that caches nothing, so a pool with no byte target still bounds the drop set; + /// a run given up since stays. + #[test] + fn the_sweep_forgets_runs_past_the_expiry_window() { + let producer = track_producer("test", Info::timed()); + let pool = producer.broadcast.pool.clone(); + let used = pool.used(); + producer.drop_groups(0..1, StreamError::Old); + producer.drop_groups(2..3, StreamError::Old); + elapse(&producer, cache::DEFAULT_EXPIRY / 2); + producer.drop_groups(4..5, StreamError::Old); + assert_eq!(pool.used(), used + 6 * RUN_OVERHEAD, "each run and its expiry are billed"); + + elapse(&producer, cache::DEFAULT_EXPIRY); + pool.sweep(); + let held: Vec<_> = producer.state.read().drops.iter().map(|(run, _)| run.clone()).collect(); + assert_eq!(held, [4..5], "only the run given up within the window stays"); + assert_eq!(pool.used(), used + 2 * RUN_OVERHEAD); + assert!(producer.state.read().given_up(0).is_none(), "a forgotten run is unknown again"); + + elapse(&producer, cache::DEFAULT_EXPIRY); + pool.sweep(); + assert!(producer.state.read().drops.is_empty()); + assert_eq!(pool.used(), used); + } + + /// A drop-only flood of disjoint runs bills the pool and sheds its lowest runs to stay + /// within it, with no cached group to evict; a shed run is unknown again, so a wire cursor + /// reports only the runs the track still holds. + #[test] + fn a_drop_flood_stays_within_the_pool() { + const RUNS: u64 = 1000; + let capacity = 64 * RUN_OVERHEAD; + let (producer, pool) = pooled_producer(capacity); + let waiter = kio::Waiter::noop(); + let mut subscriber = producer.subscribe(replay().with_range(0..)); + let mut reported = Vec::new(); + for run in 0..RUNS { + producer.drop_groups(2 * run..2 * run + 1, StreamError::Old); + assert!( + pool.used() <= capacity + 2 * RUN_OVERHEAD, + "run {run} took the pool to {}", + pool.used() + ); + // Midway, the cursor takes in every run held then, and reports the lowest; the + // rest are shed before it reports them. + if run == RUNS / 2 { + let Poll::Ready((first, _)) = subscriber.poll_drop(&waiter) else { + panic!("a run is owed"); + }; + reported.push(first); + } + } + let held: Vec<_> = producer.state.read().drops.iter().map(|(run, _)| run.clone()).collect(); + assert_eq!(pool.used(), held.len() as u64 * RUN_OVERHEAD); + let lowest = held.first().expect("the newest runs stay").start; + assert!(lowest > 0, "the lowest runs were shed first"); + assert!(held.windows(2).all(|pair| pair[1].start == pair[0].start + 2), "only a prefix was shed"); + assert_eq!(held.last().map(|run| run.start), Some(2 * (RUNS - 1))); + assert!(producer.state.read().given_up(0).is_none(), "a shed run is unknown again"); + + while let Poll::Ready((run, _)) = subscriber.poll_drop(&waiter) { + reported.push(run); + } + assert_eq!(reported[1..], held[..]); + } + #[test] #[cfg(feature = "async")] fn a_huge_skip_costs_one_run() { From 111035bba9c08f6af5d95dbea383c3d4b145e4a7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 12:05:21 -0800 Subject: [PATCH 03/17] wip(net): mirror the billed drop set in js/net, incremental cursor folds Co-Authored-By: Claude Opus 5.5 --- js/net/src/cache.test.ts | 68 +++++++- js/net/src/cache.ts | 44 ++++- js/net/src/drops.ts | 169 +++++++++++++++++++ js/net/src/internal.ts | 4 +- js/net/src/resume.ts | 2 +- js/net/src/track.ts | 253 +++++++++++++++++++++-------- rs/moq-net/src/fuzz.rs | 4 +- rs/moq-net/src/model/track.rs | 89 ++++++---- rs/moq-net/src/model/track/miss.rs | 62 ++++++- 9 files changed, 577 insertions(+), 118 deletions(-) create mode 100644 js/net/src/drops.ts diff --git a/js/net/src/cache.test.ts b/js/net/src/cache.test.ts index 7885e62b300..8e387da40f1 100644 --- a/js/net/src/cache.test.ts +++ b/js/net/src/cache.test.ts @@ -1,7 +1,7 @@ import { expect, test } from "bun:test"; import { Manual, Milli, Nano, yieldNow } from "@moq/clock"; import * as cache from "./cache.ts"; -import { StreamCode, TooFarBehind } from "./error.ts"; +import { StreamCode, StreamError, TooFarBehind } from "./error.ts"; import { hooks } from "./internal.ts"; import { Producer as TrackProducer } from "./track.ts"; @@ -274,3 +274,69 @@ test("the debt sheds the front of an open live edge", async () => { await expect(behind?.readFrame()).rejects.toThrow(); }); }); + +// Every run a fresh reader of `producer` is told was given up: what the track still holds. +function dropsHeld(producer: TrackProducer): bigint[] { + const reader = producer.subscribe({ delay: { max: null } }); + const held: bigint[] = []; + for (let run = hooks.tryDrop(reader); run; run = hooks.tryDrop(reader)) held.push(run.start); + reader.close(); + return held; +} + +const old = () => ({ error: new StreamError(StreamCode.Old) }); + +// A drop-only flood of disjoint runs bills the pool and sheds its lowest runs to stay within it, +// with no cached group to evict; a shed run is unknown again, so a reader reports only the runs +// the track still holds. Mirrors Rust `a_drop_flood_stays_within_the_pool`. +test("a drop flood stays within the pool", () => { + const runs = 1000n; + const capacity = 64 * cache.RUN_OVERHEAD; + const pool = new cache.Pool({ capacity }); + const producer = pooled(pool); + const reader = producer.subscribe({ delay: { max: null } }); + const reported: bigint[] = []; + for (let run = 0n; run < runs; run++) { + hooks.dropTrackGroups(producer, 2n * run, 2n * run + 1n, old()); + expect(pool.used).toBeLessThanOrEqual(capacity + 2 * cache.RUN_OVERHEAD); + // Midway, the reader takes in every run held then and reports the lowest; the rest are + // shed before it reports them. + if (run === runs / 2n) { + const first = hooks.tryDrop(reader); + if (!first) throw new Error("a run is owed"); + reported.push(first.start); + } + } + const held = dropsHeld(producer); + expect(pool.used).toBe(held.length * cache.RUN_OVERHEAD); + expect(held[0]).toBeGreaterThan(0n); + expect(held.every((start, i) => i === 0 || start === held[i - 1] + 2n)).toBe(true); + expect(held[held.length - 1]).toBe(2n * (runs - 1n)); + for (let run = hooks.tryDrop(reader); run; run = hooks.tryDrop(reader)) reported.push(run.start); + expect(reported.slice(1)).toEqual(held); +}); + +// A run given up longer than the pool's idle window ago is forgotten by the sweep, even on a +// track that caches nothing, and a run given up since stays. Mirrors Rust +// `the_sweep_forgets_runs_past_the_expiry_window`. +test("the sweep forgets runs past the idle window", async () => { + await withClock(async (clock) => { + const pool = new cache.Pool({ expiry: Milli(1_000) }); + const producer = pooled(pool); + hooks.dropTrackGroups(producer, 0n, 1n, old()); + hooks.dropTrackGroups(producer, 2n, 3n, old()); + await clock.advance(500); + hooks.dropTrackGroups(producer, 4n, 5n, old()); + // Each run and its expiry are billed. + expect(pool.used).toBe(3 * (cache.RUN_OVERHEAD + cache.BORN_OVERHEAD)); + + // The sweep at 1.5 s forgets the first two, given up at 0 s. + await clock.advance(1_000); + expect(dropsHeld(producer)).toEqual([4n]); + expect(pool.used).toBe(cache.RUN_OVERHEAD + cache.BORN_OVERHEAD); + + await clock.advance(1_000); + expect(dropsHeld(producer)).toEqual([]); + expect(pool.used).toBe(0); + }); +}); diff --git a/js/net/src/cache.ts b/js/net/src/cache.ts index de7b9b80bb1..93ff8eec3e4 100644 --- a/js/net/src/cache.ts +++ b/js/net/src/cache.ts @@ -30,6 +30,12 @@ export const DEFAULT_EXPIRY = Milli(30_000); export const ENTRY_OVERHEAD = 3 * 1024; /** Bookkeeping billed per subscriber mirror a cached group holds. */ export const MIRROR_OVERHEAD = 4 * 1024; +// Bun 1.4's heap grows about 334 bytes per disjoint run of a track's drop set, its change-log +// entry included, and about 53 more per entry of the expiry index a pool with an idle window keeps. +/** Bookkeeping billed per run of a track's drop set. */ +export const RUN_OVERHEAD = 336; +/** Bookkeeping billed per entry of a drop set's expiry index. */ +export const BORN_OVERHEAD = 64; // The coarse clock behind access stamps, as Rust's: the expiry window rounds up to it. const TICK = Nano.fromMilli(Milli(100)); @@ -227,6 +233,12 @@ export interface Owner { shed(entry: E, debt: number, cap: number): number; /** Something was evicted: settle what the cache's contents decide. */ settle(): void; + /** Forget the lowest runs of the track's drop set toward `debt`, freeing about `cap` at most, and return the bytes freed. */ + shedDrops(debt: number, cap: number): number; + /** Forget the runs of the drop set given up longer than the pool's idle window before tick `now`, returning whether any went. */ + expireDrops(now: number): boolean; + /** Whether the track holds a drop run, which keeps its account in the idle sweep. */ + holdsDrops(): boolean; } /** @@ -269,6 +281,18 @@ export class Track { this.#written += n; } + /** Bill `n` bytes the track keeps beside its groups, such as its drop set, which the idle sweep then visits. */ + bill(n: number): void { + this.pool.add(n); + this.wrote(n); + this.pool.register(this as Track); + } + + /** Release `n` bytes {@link bill} billed. */ + unbill(n: number): void { + this.pool.sub(n); + } + /** Whether the account caches any group. */ get empty(): boolean { return this.#live.size === 0; @@ -299,8 +323,10 @@ export class Track { if (this.#live.size === 0) { this.#order = []; this.#head = 0; - this.#debt = 0; - this.pool.unregister(this as Track); + if (!this.#owner.holdsDrops()) { + this.#debt = 0; + this.pool.unregister(this as Track); + } } } @@ -375,6 +401,13 @@ export class Track { evicted = true; this.#owner.evict(entry, "evicted"); } + // Then the drop set, oldest sequences first: losing a run costs a reader a judgment or a + // fill, never content being read. + if (this.#debt > 0 && paid < cap) { + const freed = this.#owner.shedDrops(this.#debt, cap - paid); + this.#debt -= Math.min(freed, this.#debt); + paid += freed; + } const edge = this.#owner.edge(); if (edge) fresh.push(edge); for (const entry of fresh) { @@ -397,8 +430,10 @@ export class Track { const order = this.#order; const head = this.#head; const len = order.length - head; - if (expiry === undefined || len === 0) return false; + if (expiry === undefined) return false; const now = this.pool.now(); + const forgot = this.#owner.expireDrops(now); + if (len === 0) return forgot; const start = full ? 0 : this.#cursor % len; const width = full ? len : Math.min(len, EVICT_SCAN); if (!full) this.#cursor = start + width; @@ -410,7 +445,7 @@ export class Track { } for (const entry of expired) this.#owner.evict(entry, "expired"); this.#compact(); - return expired.length > 0; + return expired.length > 0 || forgot; } // Skip the empty slots at the front of the order, and rebuild it once the slots before the @@ -441,6 +476,7 @@ export class Track { sweep(): void { if (this.evictExpired(true)) this.#owner.settle(); // A live edge alone never expires, so it needs no sweep to keep its track alive for. + if (this.#owner.holdsDrops()) return; for (const entry of this.#live.keys()) if (!this.#owner.protects(entry)) return; this.pool.unregister(this as Track); } diff --git a/js/net/src/drops.ts b/js/net/src/drops.ts new file mode 100644 index 00000000000..ccf7b14fbdc --- /dev/null +++ b/js/net/src/drops.ts @@ -0,0 +1,169 @@ +/** + * A track's drop set: the sequences its publisher gave up, with why, as runs. Mirrors the Rust + * `TrackState::drops` with its change index. + * + * Package-internal: not exported from the entrypoint. + * + * @module + */ +import { Signal } from "@moq/signals"; +import { type DropRun, type Run, sameHole } from "./internal.ts"; + +/** + * Sorted, disjoint runs, each replacing whatever it overlaps, with neighbours that say the same + * thing merged. Every change costs the runs it touches and a binary search, never the set. + */ +export class Runs { + #runs: R[] = []; + #same: (a: R, b: R) => boolean; + + constructor(same: (a: R, b: R) => boolean) { + this.#same = same; + } + + /** Every run, lowest first. */ + get all(): readonly R[] { + return this.#runs; + } + + /** How many runs are held. */ + get size(): number { + return this.#runs.length; + } + + /** The run holding `sequence`, if any. */ + at(sequence: bigint): R | undefined { + const run = this.#runs[this.#above(sequence)]; + return run && run.start <= sequence ? run : undefined; + } + + /** The runs overlapping `[start, end)`, lowest first. */ + overlapping(start: bigint, end: bigint): R[] { + const found: R[] = []; + for (let i = this.#above(start); i < this.#runs.length && this.#runs[i].start < end; i++) + found.push(this.#runs[i]); + return found; + } + + /** + * Set `[start, end)` to `run`, or clear it when `run` is undefined. Returns the runs taken out + * and the ones put in their place, the neighbours a merge or split touched included. + */ + set(start: bigint, end: bigint, run?: R): { removed: R[]; added: R[] } { + if (start >= end) return { removed: [], added: [] }; + const runs = this.#runs; + // Touching neighbours are taken out too, so an alike one merges. + let first = this.#above(start); + if (first > 0 && runs[first - 1].end === start) first--; + let last = first; + while (last < runs.length && runs[last].start <= end) last++; + const removed = runs.slice(first, last); + const pieces: R[] = []; + const lowest = removed[0]; + if (lowest && lowest.start < start) pieces.push({ ...lowest, end: start < lowest.end ? start : lowest.end }); + if (run) pieces.push({ ...run, start, end }); + const highest = removed[removed.length - 1]; + if (highest && highest.end > end) pieces.push({ ...highest, start: end > highest.start ? end : highest.start }); + const added: R[] = []; + for (const piece of pieces) { + const before = added[added.length - 1]; + if (before && before.end === piece.start && this.#same(before, piece)) before.end = piece.end; + else added.push(piece); + } + runs.splice(first, last - first, ...added); + return { removed, added }; + } + + // The index of the first run ending past `sequence`. + #above(sequence: bigint): number { + let lo = 0; + let hi = this.#runs.length; + while (lo < hi) { + const mid = (lo + hi) >>> 1; + if (this.#runs[mid].end <= sequence) lo = mid + 1; + else hi = mid; + } + return lo; + } +} + +// Dead entries the change log tolerates beyond twice the live runs before it is rebuilt. +const LOG_SLACK = 64; + +/** + * The sequences a track's publisher gave up, with why, shared by the track and every reader of + * it, plus an index of each run's last change, so a reader folds in only what changed since it + * last looked. + */ +export class DropSet { + readonly #runs = new Runs(sameHole); + // Each run's last change, oldest first. An entry whose run was since replaced is dead, and + // the log is rebuilt once dead entries clearly outnumber live ones. + #log: { revision: number; run: DropRun }[] = []; + #live = new Set(); + #revision = 0; + + /** Bumped on every change, for a reader to wake on. */ + readonly version = new Signal(0); + + /** Every run, lowest first. */ + get all(): readonly DropRun[] { + return this.#runs.all; + } + + /** How many runs are held. */ + get size(): number { + return this.#runs.size; + } + + /** The revision of the latest change, for {@link since}. */ + get revision(): number { + return this.#revision; + } + + /** The run holding `sequence`, if any. */ + at(sequence: bigint): DropRun | undefined { + return this.#runs.at(sequence); + } + + /** The runs overlapping `[start, end)`, lowest first. */ + overlapping(start: bigint, end: bigint): DropRun[] { + return this.#runs.overlapping(start, end); + } + + /** Give up `[start, end)` as `run` says, or forget it when `run` is undefined. */ + set(start: bigint, end: bigint, run?: Omit): void { + const { removed, added } = this.#runs.set(start, end, run && { ...run, start, end }); + if (removed.length === 0 && added.length === 0) return; + for (const old of removed) this.#live.delete(old); + for (const fresh of added) { + this.#live.add(fresh); + this.#log.push({ revision: ++this.#revision, run: fresh }); + } + if (added.length === 0) this.#revision++; + if (this.#log.length > 2 * this.#live.size + LOG_SLACK) { + this.#log = this.#log.filter((entry) => this.#live.has(entry.run)); + } + this.version.update((version) => version + 1); + } + + /** + * The runs changed since `revision`, as their current extent. A run forgotten outright leaves + * no entry, so a reader checks what it folded in against {@link at} before reporting it. + */ + since(revision: number): Run[] { + let lo = 0; + let hi = this.#log.length; + while (lo < hi) { + const mid = (lo + hi) >>> 1; + if (this.#log[mid].revision <= revision) lo = mid + 1; + else hi = mid; + } + const changed: Run[] = []; + for (let i = lo; i < this.#log.length; i++) { + const { run } = this.#log[i]; + if (this.#live.has(run)) changed.push({ start: run.start, end: run.end }); + } + return changed; + } +} diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index ae144d4f70f..b5da97dfcb5 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -223,7 +223,7 @@ export interface DropRun extends Run, Hole { } /** Whether two runs say the same thing, so they may merge. */ -function sameHole(a: DropRun, b: DropRun): boolean { +export function sameHole(a: DropRun, b: DropRun): boolean { return ( toStreamCode(a.error) === toStreamCode(b.error) && !a.skip === !b.skip && @@ -724,7 +724,7 @@ export const hooks: { /** Whether {@link cachedGroup} would find the group, without making a read handle. */ holdsGroup: (subscriber: Subscriber, sequence: bigint) => boolean; /** The error the subscriber's track gave `sequence` up with, if it did, and what changes it. */ - givenUp: (subscriber: Subscriber, sequence: bigint) => { error?: Error; changed: Getter }; + givenUp: (subscriber: Subscriber, sequence: bigint) => { error?: Error; changed: Getter[] }; /** * The parts of `[start, end)` the subscriber's track still holds as a received stale run: one * new terms superseded is gone. Mirrors the Rust crate-private `track::Subscriber::stale_held`. diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index a93a52a81b5..e98ccca94a0 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -1705,7 +1705,7 @@ class Recover implements GroupRecover { if (given.error && !routeFailed(given.error) && !copyFailed(serving)) { throw given.error; } - waits.push(given.changed); + waits.push(...given.changed); // A newer copy's track caches it: continue from there. if (!failed && generation !== current.generation) { const cached = hooks.cachedGroup(serving.sub, this.sequence, index); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 805bf6b2d20..39845e25cc7 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -8,6 +8,7 @@ import { Instant, Milli, Nano, now, sleepUntil } from "@moq/clock"; import { Derived, type Dispose, type GetPromise, type Getter, Once, Signal } from "@moq/signals"; import * as cache from "./cache.ts"; import { Delivered } from "./delivered.ts"; +import { DropSet, Runs } from "./drops.ts"; import { GroupTooLarge, ProtocolViolation, @@ -40,6 +41,7 @@ import { removeRun, runGaps, type StagedTrack, + sameHole, skipHole, staleHole, type TrackRequestOptions, @@ -697,9 +699,10 @@ class TrackState { // Sequences the upstream publisher gave up, with why: an incoming SUBSCRIBE_SKIP or // SUBSCRIBE_DROP, or a producer cut. Kept apart from the cache so reclaim cannot // erase a disposition a downstream publisher still has to forward, and as runs so a wide - // DROP costs one entry. Replaced rather than mutated, so a cursor can tell it changed by - // identity. - drops = new Signal([]); + // DROP costs one entry. A sink shares its producer's, which bills it to the track's account. + drops = new DropSet(); + // The runs this sink's own fills refused, which it reads as given up beside `drops`. + refused = new Signal([]); // Every group the ranges of a received copy's current upstream terms name, which its stale runs // were judged against; undefined when unknown, which counts as every group. judged?: readonly Run[]; @@ -879,8 +882,29 @@ function sameCoverage(a: Coverage, b: Coverage): boolean { * Whether the live edge `received` reached `final`: no sequence below it is left that was neither * written nor given up, as a declared skip or a drop. Mirrors the Rust `TrackState::reaches`. */ -function reaches(received: bigint, drops: readonly Run[], final: bigint): boolean { - return received >= final || runGaps(drops, received, final).length === 0; +// A run of the drop set and the pool tick it outlives the idle window at. +interface Born extends Run { + expires: number; +} + +// Whether `a` and `b` give a sequence up alike, however each was folded in. +function sameVerdict(a: DropRun, b: DropRun): boolean { + return sameHole({ ...a, received: undefined }, { ...b, received: undefined }); +} + +// The runs of `drops` and a reader's `refused` overlapping `[start, end)`, lowest first. +function givenUp(drops: DropSet, refused: readonly DropRun[], start: bigint, end: bigint): DropRun[] { + const found = drops.overlapping(start, end); + if (refused.length === 0) return found; + found.push(...refused.filter((run) => run.start < end && start < run.end)); + return found.sort((a, b) => (a.start < b.start ? -1 : a.start > b.start ? 1 : 0)); +} + +function reaches(received: bigint, drops: DropSet, final: bigint, refused: readonly Run[] = []): boolean { + if (received >= final) return true; + const given: Run[] = []; + for (const run of [...drops.overlapping(received, final), ...refused]) insertRun(given, run.start, run.end); + return runGaps(given, received, final).length === 0; } /** @@ -1029,6 +1053,14 @@ export class Producer { // This track's account against its broadcast's cache pool (a pool of its own when it has no // broadcast), which bills the cached groups and evicts them under its budget and idle window. #account: cache.Track; + // The pool tick past which each run of the drop set outlives the pool's idle window, as of when + // it was last given up; empty when the pool has no idle window. See `#setDrops`. + #dropsBorn = new Runs((a, b) => a.expires === b.expires); + // At most the lowest tick in `#dropsBorn`, so an idle scan finds nothing to forget without + // walking it. + #dropsOldest?: number; + // What the drop set and its expiry index bill to the account. + #dropsBilled = 0; // One independent downstream state per live subscriber. #sinks = new Set(); @@ -1076,7 +1108,10 @@ export class Producer { this.accept({ ...info, maxAge: info.maxAge ?? null }); return this.#createGroup(sequence, 0); }, - drop: (start, end, error) => this.#drop(start, end, { error }), + drop: (start, end, error) => { + this.#drop(start, end, { error }); + this.#chargeDebt(); + }, holds: (sequence) => cachedHolds(this, sequence), closed: () => this.#state.closed.peek() !== undefined, released: this.#state.closed, @@ -1098,12 +1133,67 @@ export class Producer { }, shed: (entry, debt, cap) => hooks.shedGroup(entry.group, debt, cap, entry.mirrors.values()), settle: () => this.#settle(), + shedDrops: (debt, cap) => this.#shedDrops(debt, cap), + expireDrops: (now) => this.#expireDrops(now), + holdsDrops: () => this.#state.drops.size > 0, }); } + // Give up `[start, end)` as `hole` says, or forget it without one, billing the drop set to the + // track's account and dating each run against the pool's idle window. Mirrors the Rust + // `TrackState::change_drops`. + #setDrops(start: bigint, end: bigint, hole?: Hole): void { + this.#state.drops.set(start, end, hole); + const pool = this.#account.pool; + const window = pool.expiryTicks; + if (!hole) this.#dropsBorn.set(start, end); + else if (window !== undefined) { + // Rounded up to a quarter of the window, so runs given up together stay one entry and a + // run is forgotten no sooner than the window after it was given up. + const quarter = Math.max(1, Math.floor(window / 4)); + const expires = (Math.floor(pool.now() / quarter) + 1) * quarter + window; + this.#dropsBorn.set(start, end, { start, end, expires }); + this.#dropsOldest = Math.min(this.#dropsOldest ?? expires, expires); + } + const billed = this.#state.drops.size * cache.RUN_OVERHEAD + this.#dropsBorn.size * cache.BORN_OVERHEAD; + if (billed > this.#dropsBilled) this.#account.bill(billed - this.#dropsBilled); + else this.#account.unbill(this.#dropsBilled - billed); + this.#dropsBilled = billed; + } + + // Forget the lowest runs of the drop set toward `debt`, at most about `cap` bytes, returning the + // bytes freed. Their sequences are unknown again: a reader judges them by its budget, or fills + // them by range. + #shedDrops(debt: number, cap: number): number { + let paid = 0; + while (paid < debt && paid < cap) { + const run = this.#state.drops.all[0]; + if (!run) break; + const billed = this.#dropsBilled; + this.#setDrops(run.start, run.end); + paid += billed - this.#dropsBilled; + } + return paid; + } + + // Forget the runs given up longer than the pool's idle window before tick `now`; walks the set + // only once its oldest run is due, which a quarter of the window separates from the next. + #expireDrops(now: number): boolean { + if (this.#dropsOldest === undefined || now < this.#dropsOldest) return false; + const expired = this.#dropsBorn.all + .filter((run) => now >= run.expires) + .map(({ start, end }) => ({ start, end })); + for (const run of expired) this.#setDrops(run.start, run.end); + this.#dropsOldest = undefined; + for (const run of this.#dropsBorn.all) + this.#dropsOldest = Math.min(this.#dropsOldest ?? run.expires, run.expires); + return expired.length > 0; + } + static { hooks.poolTrack = (producer, pool) => { - if (!producer.#account.empty) throw new Error("track already caches groups"); + if (!producer.#account.empty || producer.#state.drops.size > 0) + throw new Error("track already caches groups"); producer.#account = producer.#openAccount(pool); }; hooks.subscribing = (producer) => producer.#subscribing; @@ -1148,8 +1238,7 @@ export class Producer { // Without a declared start, the first arrival stands in for it, as the tail's does. const floor = producer.#wire?.served?.start ?? producer.#arrivals.first() ?? 0n; if (sequence < floor) return floor; - const dropped = producer.#state.drops.peek().find((run) => run.start <= sequence && sequence < run.end); - return dropped?.end ?? producer.#arrivals.runEnd(sequence); + return producer.#state.drops.at(sequence)?.end ?? producer.#arrivals.runEnd(sequence); }; hooks.producerLargest = (producer) => producer.#state.largest; hooks.joinProducer = (producer) => producer.#join(); @@ -1158,17 +1247,20 @@ export class Producer { hooks.releaseTrack = (producer) => producer.#close(undefined); hooks.trackReleased = (producer) => producer.#state.closed; hooks.trackFinished = (producer) => producer.#state.end.peek() !== undefined; - hooks.dropTrackGroups = (producer, start, end, hole) => producer.#drop(start, end, hole, true); + hooks.dropTrackGroups = (producer, start, end, hole) => { + const runs = producer.#drop(start, end, hole, true); + // A peer's drops bill the pool as its groups do, so they pay their debt the same way. + producer.#chargeDebt(); + return runs; + }; hooks.skipTrackGroups = (producer, start, end) => producer.#skipReceived(start, end); hooks.supersedeStale = (producer, judged) => { - const held = producer.#state.drops.peek().filter((run) => run.stale !== undefined); - for (const state of [producer.#state, ...producer.#sinks]) { - state.judged = judged; - const drops = state.drops.peek(); - if (drops.some((run) => run.stale !== undefined)) - state.drops.set(drops.filter((run) => run.stale === undefined)); - } - return held.map(({ start, end }) => ({ start, end })); + const held = producer.#state.drops.all + .filter((run) => run.stale !== undefined) + .map(({ start, end }) => ({ start, end })); + for (const state of [producer.#state, ...producer.#sinks]) state.judged = judged; + for (const run of held) producer.#setDrops(run.start, run.end); + return held; }; hooks.dropAnchors = (producer) => { for (const state of [producer.#state, ...producer.#sinks]) state.dropsAnchors.set(true); @@ -1266,11 +1358,11 @@ export class Producer { const end = declared === undefined ? land.end : min(land.end, finalSequence(declared)); const at = now(); const limit = Nano.fromMilli(land.grace); - const drops = this.#state.drops.peek(); + const drops = this.#state.drops; for (const gap of this.#arrivals.gaps(start, end)) { const folds = gap.since === undefined ? undefined : gap.since + limit; if (folds !== undefined && at >= folds) continue; - for (const open of runGaps(drops, gap.start, gap.end)) { + for (const open of runGaps(drops.overlapping(gap.start, gap.end), gap.start, gap.end)) { yield { ...open, folds }; } } @@ -1283,7 +1375,7 @@ export class Producer { // nothing produced. #drop(start: bigint, end: bigint, hole: Hole, open = false): Run[] { if (start >= end) return []; - const skipped = hole.skip ? [] : this.#state.drops.peek().filter((run) => run.skip); + const skipped = hole.skip ? [] : this.#state.drops.overlapping(start, end).filter((run) => run.skip); let runs = runGaps(skipped, start, end); // A received stale verdict leaves a group that arrived and was not aborted: a lite-07 // publisher may serve a group after its stale drop, and the group's stream may land first. @@ -1298,11 +1390,7 @@ export class Producer { // A fill no longer owes what the track gave up for good. A stale verdict settles only // the readers it covers, so a fill asked by any other is still owed. if (hole.stale === undefined) this.#fills.dropped(start, end); - for (const state of [this.#state, ...this.#sinks]) { - state.drops.update((drops) => - runs.reduce((drops, run) => insertDrop(drops, run.start, run.end, hole), drops), - ); - } + for (const run of runs) this.#setDrops(run.start, run.end, hole); if (open && !hole.skip && hole.stale === undefined) { for (const [sequence, entry] of this.#cached) { if (sequence >= start && sequence < end && entry.group.closed.peek() === undefined) { @@ -1320,12 +1408,7 @@ export class Producer { // A publisher may re-create a group it cut; the sequence is no longer given up. #undrop(sequence: bigint): void { - for (const state of [this.#state, ...this.#sinks]) { - const drops = state.drops.peek(); - if (runGaps(drops, sequence, sequence + 1n).length === 0) { - state.drops.set(removeRun(drops, sequence, sequence + 1n)); - } - } + if (this.#state.drops.at(sequence)) this.#setDrops(sequence, sequence + 1n); } /** @@ -1448,6 +1531,7 @@ export class Producer { // track still drains the buffered groups before seeing the end. #addSink(sink: TrackState): void { sink.producer = this; + sink.drops = this.#state.drops; const info = this.#state.info.peek(); if (info) sink.info.set(info); sink.largest.set(this.#state.largest.peek()); @@ -1503,7 +1587,6 @@ export class Producer { // Full, unlike a write's bounded scan, so the subscriber gets no group already idled out. this.#evictExpired(true); for (const entry of this.#cache) this.#mirror(entry, sink); - sink.drops.set(this.#state.drops.peek()); sink.judged = this.#state.judged; sink.received.set(this.#received); @@ -1873,12 +1956,13 @@ export class Producer { if (sequence >= start && sequence < end) insertRun(cached, sequence, sequence + 1n); } this.#skipGaps(cached, start, end); + this.#chargeDebt(); return cached; } // Skip what of `[start, end)` neither `kept` nor an earlier skip names, replacing any drop. #skipGaps(kept: readonly Run[], start: bigint, end: bigint): void { - const skipped = this.#state.drops.peek().filter((run) => run.skip); + const skipped = this.#state.drops.overlapping(start, end).filter((run) => run.skip); for (const gap of runGaps(kept, start, end)) { for (const open of runGaps(skipped, gap.start, gap.end)) { this.#drop(open.start, open.end, skipHole()); @@ -1911,8 +1995,7 @@ export class Producer { // Refuse a sequence the publisher declared skipped: a skip is final. #refuseSkipped(sequence: bigint): void { - const run = this.#state.drops.peek().find((run) => run.start <= sequence && sequence < run.end); - if (run?.skip) throw new Skipped(sequence); + if (this.#state.drops.at(sequence)?.skip) throw new Skipped(sequence); } /** @@ -2125,7 +2208,7 @@ export class Producer { this.#settle(); if (this.#state.largestEnded.peek() || this.#largestOpen) return; const end = this.#state.end.peek()?.at; - if (end === undefined || !reaches(this.#received, this.#state.drops.peek(), finalSequence(end))) return; + if (end === undefined || !reaches(this.#received, this.#state.drops, finalSequence(end))) return; for (const state of [this.#state, ...this.#sinks]) state.largestEnded.set(true); } @@ -2212,7 +2295,7 @@ export class Producer { const end = this.#state.end.peek()?.at; if (end === undefined) return false; const final = finalSequence(end); - if (!reaches(this.#received, this.#state.drops.peek(), final) || this.#owes(final)) return false; + if (!reaches(this.#received, this.#state.drops, final) || this.#owes(final)) return false; for (const { group } of this.#cache) if (group.sequence < final && group.closed.peek() !== null) return false; return true; } @@ -2307,7 +2390,9 @@ export class Subscriber { // persistent track state, so they are tracked here rather than removed for everyone. #pending: DropRun[] = []; #dropsSeen?: { - drops: readonly DropRun[]; + // The track's drops revision folded in: later changes fold in alone. + drops: number; + refused: readonly DropRun[]; start: bigint; end?: bigint; revision: number; @@ -2917,7 +3002,7 @@ export class Subscriber { maxDelay: () => this.#drift().budget, order: () => this.#state.update.peek()?.order ?? "descending", refuse: (start, end, error) => - this.#state.drops.update((drops) => insertDrop(drops, start, end, { error })), + this.#state.refused.update((runs) => insertDrop(runs, start, end, { error })), }; return this.#state.asker; } @@ -2949,20 +3034,19 @@ export class Subscriber { // still asked for, as the Rust `Misses::unclaimed` judges it. const budget = drift.budget; const coverage = this.#coverage(); - const drops = this.#state.drops.peek(); + const { drops, refused } = this.#state; const low = floor === undefined ? start : max(start, floor); const edge = min(end, max(this.#state.received.peek(), (fillEdge(producer) ?? -1n) + 1n)); const spans: Run[] = [{ start: low, end: edge }]; // The upstream accounted for a stale run, so the live feed brings none of it unless an // update supersedes it: asked even above the newest group received. - for (const dropped of drops) { + for (const dropped of drops.overlapping(max(low, edge), end)) { const span = { ...dropped, start: max(dropped.start, max(low, edge)), end: min(dropped.end, end) }; if (span.start < span.end && !settles(span, budget, coverage, this.#state.judged)) spans.push(span); } for (const run of spans.flatMap((span) => this.#wants(span.start, span.end))) { const gone = [...accounted]; - for (const dropped of drops) { - if (dropped.end <= run.start || run.end <= dropped.start) continue; + for (const dropped of givenUp(drops, refused.peek(), run.start, run.end)) { // Judged from where it meets the asked run, as the reader would read it. const clipped = { ...dropped, start: max(dropped.start, run.start) }; if (settles(clipped, budget, coverage, this.#state.judged)) @@ -3103,15 +3187,17 @@ export class Subscriber { // the reader still wants those groups, however the feed accounted for them. const budget = this.#drift().budget; const coverage = this.#coverage(); - const all = this.#state.drops.peek(); + const { drops: all, refused } = this.#state; for (const run of this.#window.runs(this.#window.floor() ?? 0n, end)) { let gaps = [run]; for (const delivered of taken) gaps = gaps.flatMap((gap) => delivered.gaps(gap.start, gap.end)); for (const gap of gaps) { // Each run judged from where it meets the gap, as the reader would read it. - const overlapping = all - .filter((run) => run.start < gap.end && gap.start < run.end) - .map((run) => ({ ...run, start: max(run.start, gap.start), end: min(run.end, gap.end) })); + const overlapping = givenUp(all, refused.peek(), gap.start, gap.end).map((run) => ({ + ...run, + start: max(run.start, gap.start), + end: min(run.end, gap.end), + })); const drops = overlapping.filter((run) => settles(run, budget, coverage, this.#state.judged)); const unknown = overlapping.filter((run) => !settles(run, budget, coverage, this.#state.judged)); for (const open of runGaps(drops, gap.start, gap.end)) { @@ -3243,15 +3329,17 @@ export class Subscriber { }; hooks.staleHeld = (subscriber, start, end) => { const held: Run[] = []; - for (const run of subscriber.#state.drops.peek()) { - if (run.stale === undefined || run.end <= start || run.start >= end) continue; - insertRun(held, max(run.start, start), min(run.end, end)); + for (const run of subscriber.#state.drops.overlapping(start, end)) { + if (run.stale !== undefined) insertRun(held, max(run.start, start), min(run.end, end)); } return held; }; hooks.givenUp = (subscriber, sequence) => { - const drops = subscriber.#state.drops; - return { error: dropAt(drops.peek(), sequence), changed: drops }; + const { drops, refused } = subscriber.#state; + return { + error: drops.at(sequence)?.error ?? dropAt(refused.peek(), sequence), + changed: [drops.version, refused], + }; }; hooks.mayLand = (subscriber, land) => { const producer = subscriber.#state.producer; @@ -3771,31 +3859,48 @@ export class Subscriber { } this.#settle(); const { start, end } = this.#cursor.peek(); - // Rescanned only after it changed, so an idle poll costs nothing per drop held. - // Out-of-window runs were skipped, so a moved window rescans too. - const drops = this.#state.drops.peek(); + // The whole span is folded in again only after the window, the budget, or what settles a + // stale run moved, since out-of-window and unsettling runs were skipped and may be owed + // now; otherwise only the runs that changed are, so a drop costs each reader the runs it + // touches. + const { drops, refused } = this.#state; const revision = this.#revision; const { budget } = this.#drift(); const coverage = this.#coverage(); const judged = this.#state.judged; const seen = this.#dropsSeen; - if ( + const full = !seen || - seen.drops !== drops || + seen.refused !== refused.peek() || seen.start !== start || seen.end !== end || seen.revision !== revision || seen.budget !== budget || !sameCoverage(seen.coverage, coverage) || - seen.judged !== judged - ) { - this.#dropsSeen = { drops, start, end, revision, budget, coverage, judged }; - // The track's runs are folded in afresh, so one it forgot, such as a stale run new terms - // superseded, or one that stopped settling this reader, leaves nothing behind. - this.#pending = this.#pending.filter((run) => !run.received); - for (const dropped of drops) { + seen.judged !== judged; + const changed: Run[] = full ? [{ start, end: end ?? UNBOUNDED }] : drops.since(seen.drops); + if (full) this.#pending = this.#pending.filter((run) => !run.received); + this.#dropsSeen = { + drops: drops.revision, + refused: refused.peek(), + start, + end, + revision, + budget, + coverage, + judged, + }; + for (const span of changed) { + const from = max(span.start, start); + const to = end === undefined ? span.end : min(span.end, end); + if (from >= to) continue; + if (!full) { + this.#pending = this.#pending.flatMap((run) => (run.received ? removeRun([run], from, to) : [run])); + } + const folded = full ? givenUp(drops, refused.peek(), from, to) : drops.overlapping(from, to); + for (const dropped of folded) { // Judged from where it meets this cursor's span, as Rust clips it. - const run = { ...dropped, start: max(dropped.start, start), end: min(dropped.end, end ?? dropped.end) }; + const run = { ...dropped, start: max(dropped.start, from), end: min(dropped.end, to) }; if (run.start >= run.end) continue; // A received stale run that does not settle this reader is unknown to it. if (!settles(run, budget, coverage, judged)) continue; @@ -3803,6 +3908,17 @@ export class Subscriber { } } for (let run = this.#pending.shift(); run; run = this.#pending.shift()) { + // A received run the track has since forgotten, shed or superseded, is unknown again: + // only the part it still holds as it was folded in is owed. + if (run.received) { + const folded = run; + const held = givenUp(drops, refused.peek(), run.start, run.end).find((given) => + sameVerdict(given, folded), + ); + if (!held) continue; + if (held.end < run.end) this.#pending.unshift({ ...run, start: held.end }); + run = { ...run, start: max(run.start, held.start), end: min(run.end, held.end) }; + } const dropped = this.#report(run.start, run.end, run); if (!dropped) continue; if (dropped.end < run.end) this.#pending.unshift({ ...run, start: dropped.end }); @@ -3836,7 +3952,7 @@ export class Subscriber { const end = this.#state.end.peek()?.at; return ( end !== undefined && - reaches(this.#state.received.peek(), this.#state.drops.peek(), finalSequence(end)) && + reaches(this.#state.received.peek(), this.#state.drops, finalSequence(end), this.#state.refused.peek()) && !this.#state.owed.peek() ); } @@ -3854,7 +3970,8 @@ export class Subscriber { const producer = this.#state.producer; const dispose = [ this.#state.groups.changed(fn), - this.#state.drops.changed(fn), + this.#state.drops.version.changed(fn), + this.#state.refused.changed(fn), this.#fillMoves.changed(fn), // A handler opening takes a miss nobody could ask before. ...(producer ? [fillsOf(producer).opened.changed(fn)] : []), diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index 1481d40695d..1bca6c4702d 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -51,9 +51,7 @@ impl DropSet { let track = broadcast.create_track("drops", track::Info::default()).unwrap(); let stride = span + 1; let from = runs.saturating_sub(reported) * stride; - let mut readers: Vec<_> = (0..readers) - .map(|_| track.subscribe(None).with_range(from..)) - .collect(); + let mut readers: Vec<_> = (0..readers).map(|_| track.subscribe(None).with_range(from..)).collect(); for run in 0..runs { track.drop_groups(run * stride..run * stride + span, crate::StreamError::Old); } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index fce82ac0d13..2bd24ebe311 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -678,7 +678,8 @@ pub(crate) struct TrackState { // At most the lowest tick in `drops_born`, so an expiry scan finds nothing to forget // without walking it. drops_oldest: Option, - // What `drops` and `drops_born` bill to the pool: [`RUN_OVERHEAD`] per entry. Paying + // What `drops` and `drops_born` bill to the pool: [`RUN_OVERHEAD`] and [`BORN_OVERHEAD`] + // per entry. Paying // eviction debt sheds the lowest runs, and the expiry scan forgets those older than the // pool's window; either leaves their sequences unknown again. drops_bill: cache::Extra, @@ -962,15 +963,16 @@ impl Slot { pub(super) const CACHE_OVERHEAD: u64 = 2 * (2 * size_of::() + size_of::() + size_of::<(u64, Timestamp)>() + 2 * size_of::<(u64, u32)>()) as u64; -/// Heap the track keeps per run of its drop set: the run and its reason in `drops`, its two -/// entries in `drops_changes`, and its expiry in `drops_born`, billed per entry of either map. -/// Doubled for half-full `BTreeMap` nodes, as [`CACHE_OVERHEAD`] is. Billed to the pool, so a peer fragmenting its drops into runs costs +/// Heap the track keeps per run of its drop set: the run and its reason in `drops`, and its +/// two entries in `drops_changes`. Doubled for half-full `BTreeMap` nodes, as +/// [`CACHE_OVERHEAD`] is. Billed to the pool, so a peer fragmenting its drops into runs costs /// what the cache would hold, never more. -pub(super) const RUN_OVERHEAD: u64 = 2 - * (size_of::<(ops::Range, Hole)>() - + size_of::<(u64, (u64, u64))>() - + size_of::<(u64, u64)>() - + size_of::<(ops::Range, u64)>()) as u64; +pub(super) const RUN_OVERHEAD: u64 = + 2 * (size_of::<(ops::Range, Hole)>() + size_of::<(u64, (u64, u64))>() + size_of::<(u64, u64)>()) as u64; + +/// Heap the track keeps per entry of its drop set's expiry index, `drops_born`, doubled as +/// [`RUN_OVERHEAD`] is. +pub(super) const BORN_OVERHEAD: u64 = 2 * size_of::<(ops::Range, u64)>() as u64; /// Where the groups this hop holds start, for resolving time anchors: ordered lookups in /// the starts index and the cache, so no query walks more than the run it answers. @@ -1402,7 +1404,8 @@ impl TrackState { affected.end = affected.end.max(run.end); } let entries = |state: &Self, affected: &ops::Range| { - (state.drops.overlapping(affected).count() + state.drops_born.overlapping(affected).count()) as u64 + state.drops.overlapping(affected).count() as u64 * RUN_OVERHEAD + + state.drops_born.overlapping(affected).count() as u64 * BORN_OVERHEAD }; let before = entries(self, &affected); match hole { @@ -1414,7 +1417,10 @@ impl TrackState { // than the window after it was given up. let quarter = (max_ticks / 4).max(1); let now = self.cache.pool().now(); - let expires = (now / quarter).saturating_add(1).saturating_mul(quarter).saturating_add(max_ticks); + let expires = (now / quarter) + .saturating_add(1) + .saturating_mul(quarter) + .saturating_add(max_ticks); self.drops_born.insert(groups, expires); self.drops_oldest = Some(self.drops_oldest.map_or(expires, |oldest| oldest.min(expires))); } @@ -1429,9 +1435,9 @@ impl TrackState { } let after = entries(self, &affected); if after > before { - self.drops_bill.add((after - before) * RUN_OVERHEAD); + self.drops_bill.add(after - before); } else { - self.drops_bill.sub((before - after) * RUN_OVERHEAD); + self.drops_bill.sub(before - after); } self.drops_revision += 1; } @@ -7674,10 +7680,7 @@ impl DropCursor { // again: only the part the track still holds as it was folded in is owed. let range = match pending.received { true => { - let Some((held, _)) = state - .drops - .overlapping(&range) - .find(|(_, hole)| **hole == pending.hole) + let Some((held, _)) = state.drops.overlapping(&range).find(|(_, hole)| **hole == pending.hole) else { continue; }; @@ -19864,8 +19867,13 @@ mod test { let settling = producer.state.downgrade().upgrade().expect("open"); drop(producer); - // What remains is the drop the teardown recorded for the open group, held by the state. - assert_eq!(pool.used(), RUN_OVERHEAD, "the abrupt teardown still released the cache"); + // What remains is the drop the teardown recorded for the open group and its expiry, held + // by the state. + assert_eq!( + pool.used(), + RUN_OVERHEAD + BORN_OVERHEAD, + "the abrupt teardown still released the cache" + ); drop(settling); assert_eq!(pool.used(), 0, "the state's drop set goes with it"); } @@ -23688,14 +23696,21 @@ mod test { producer.drop_groups(2..3, StreamError::Old); elapse(&producer, cache::DEFAULT_EXPIRY / 2); producer.drop_groups(4..5, StreamError::Old); - assert_eq!(pool.used(), used + 6 * RUN_OVERHEAD, "each run and its expiry are billed"); + assert_eq!( + pool.used(), + used + 3 * (RUN_OVERHEAD + BORN_OVERHEAD), + "each run and its expiry are billed" + ); elapse(&producer, cache::DEFAULT_EXPIRY); pool.sweep(); let held: Vec<_> = producer.state.read().drops.iter().map(|(run, _)| run.clone()).collect(); assert_eq!(held, [4..5], "only the run given up within the window stays"); - assert_eq!(pool.used(), used + 2 * RUN_OVERHEAD); - assert!(producer.state.read().given_up(0).is_none(), "a forgotten run is unknown again"); + assert_eq!(pool.used(), used + RUN_OVERHEAD + BORN_OVERHEAD); + assert!( + producer.state.read().given_up(0).is_none(), + "a forgotten run is unknown again" + ); elapse(&producer, cache::DEFAULT_EXPIRY); pool.sweep(); @@ -23709,7 +23724,7 @@ mod test { #[test] fn a_drop_flood_stays_within_the_pool() { const RUNS: u64 = 1000; - let capacity = 64 * RUN_OVERHEAD; + let capacity = 64 * (RUN_OVERHEAD + BORN_OVERHEAD); let (producer, pool) = pooled_producer(capacity); let waiter = kio::Waiter::noop(); let mut subscriber = producer.subscribe(replay().with_range(0..)); @@ -23717,7 +23732,7 @@ mod test { for run in 0..RUNS { producer.drop_groups(2 * run..2 * run + 1, StreamError::Old); assert!( - pool.used() <= capacity + 2 * RUN_OVERHEAD, + pool.used() <= capacity + 2 * (RUN_OVERHEAD + BORN_OVERHEAD), "run {run} took the pool to {}", pool.used() ); @@ -23731,12 +23746,19 @@ mod test { } } let held: Vec<_> = producer.state.read().drops.iter().map(|(run, _)| run.clone()).collect(); - assert_eq!(pool.used(), held.len() as u64 * RUN_OVERHEAD); + // Each disjoint run is an entry in the drop set and one in its expiry index. + assert_eq!(pool.used(), held.len() as u64 * (RUN_OVERHEAD + BORN_OVERHEAD)); let lowest = held.first().expect("the newest runs stay").start; assert!(lowest > 0, "the lowest runs were shed first"); - assert!(held.windows(2).all(|pair| pair[1].start == pair[0].start + 2), "only a prefix was shed"); + assert!( + held.windows(2).all(|pair| pair[1].start == pair[0].start + 2), + "only a prefix was shed" + ); assert_eq!(held.last().map(|run| run.start), Some(2 * (RUNS - 1))); - assert!(producer.state.read().given_up(0).is_none(), "a shed run is unknown again"); + assert!( + producer.state.read().given_up(0).is_none(), + "a shed run is unknown again" + ); while let Poll::Ready((run, _)) = subscriber.poll_drop(&waiter) { reported.push(run); @@ -24144,7 +24166,7 @@ mod test { /// A cut stays readable for as long as the group it was: the sweep that reclaims /// spent slots leaves it to a reader still behind, and once it expires like a group, - /// a relay still forwards it as a drop. + /// a relay still forwards it as a drop until that drop outlives the window too. #[test] #[cfg(feature = "async")] fn a_cut_outlives_the_sweep_until_it_expires() { @@ -24167,10 +24189,8 @@ mod test { assert_eq!(first.sequence, 0, "the sweep kept the cut"); assert!(matches!(first.read_frame().now_or_never(), Some(Err(Error::App(5))))); - for _ in 0..2 { - elapse(&producer, Duration::from_secs(20)); - pool.sweep(); - } + elapse(&producer, Duration::from_secs(20)); + pool.sweep(); assert!(!producer.state.read().lookup.contains_key(&0), "the cut expired"); let mut relay = producer.subscribe(replay()); let waiter = kio::Waiter::noop(); @@ -24178,6 +24198,11 @@ mod test { relay.poll_drop(&waiter), Poll::Ready((range, Hole::Drop { error: StreamError::App(5), .. })) if range == (0..1) )); + + // The drop it left ages out a window later like any run, unknown again. + elapse(&producer, Duration::from_secs(20)); + pool.sweep(); + assert!(producer.state.read().given_up(0).is_none()); } /// A cut above a reader's cap is handed out once the cap rises over it. diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index 5039d1e49e1..1f0148e293f 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -94,6 +94,12 @@ pub(super) struct Misses { evictions: u64, /// Judged again after an eviction, within the judged span: never refilled twice. refilled: RangeSet, + /// The last sequence of the highest received stale run above the newest group that this + /// reader cannot trust, as of [`Self::unknown_at`]; see [`Self::unknown`]. + unknown: Option, + /// The drops revision, judged revision, budget, and coverage [`Self::unknown`] was found + /// under: while the terms hold, only the runs changed since are looked at. + unknown_at: Option<(u64, u64, Duration, super::Coverage)>, } /// A run of misses the reader gives up, for the drop cursor to name, with the budget it was @@ -604,6 +610,54 @@ impl Misses { clippy::too_many_arguments, reason = "one probe's inputs, borrowed apart from its cursor" )] + /// The last sequence of the highest received stale run above the newest group that `drift` + /// does not settle. While the terms hold and the run it found still stands, only the runs + /// changed since are looked at, so a drop costs each reader the runs it touches. + fn unknown(&mut self, state: &TrackState, drift: &Drift) -> Option { + let above = |run: &ops::Range| state.newest_produced.is_none_or(|newest| run.end - 1 > newest); + let unsettled = |run: &ops::Range, hole| above(run) && !drift.settles(state, run.start, hole); + let revision = state.drops_changes.revision(); + let since = self + .unknown_at + .as_ref() + .filter(|(_, judged, budget, coverage)| { + *judged == state.judged_revision && *budget == drift.budget && *coverage == drift.coverage + }) + .map(|(seen, ..)| *seen) + // The newest group only rises, so the run found stays the highest unless it fell + // below it or changed. + .filter(|_| { + self.unknown.is_none_or(|top| { + state + .drops + .get_key_value(&top) + .is_some_and(|(run, hole)| unsettled(run, hole)) + }) + }); + self.unknown = match since { + Some(seen) => { + let mut unknown = self.unknown; + for changed in state.drops_changes.since(seen) { + for (run, hole) in state.drops.overlapping(&changed) { + if unsettled(run, hole) { + unknown = unknown.max(Some(run.end - 1)); + } + } + } + unknown + } + None => state + .drops + .iter() + .rev() + .take_while(|(run, _)| above(run)) + .find(|(run, hole)| !drift.settles(state, run.start, hole)) + .map(|(run, _)| run.end - 1), + }; + self.unknown_at = Some((revision, state.judged_revision, drift.budget, drift.coverage.clone())); + self.unknown + } + pub fn judge( &mut self, state: &TrackState, @@ -630,13 +684,7 @@ impl Misses { // it arrived: only an update brings it back otherwise, and a reader narrower than the // upstream's terms changes none. See `TrackState::unfed`. Only runs above the newest // group need a look. - let unknown = state - .drops - .iter() - .rev() - .take_while(|(run, _)| state.newest_produced.is_none_or(|newest| run.end - 1 > newest)) - .find(|(run, hole)| !drift.settles(state, run.start, hole)) - .map(|(run, _)| run.end - 1); + let unknown = self.unknown(state, drift); let Some(newest) = state.newest_produced.max(bound).max(unknown) else { return asks; }; From 043d28cdecaea106f3b04c8caa9e20d105ed248c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 12:20:18 -0800 Subject: [PATCH 04/17] fix(net): bill and bound the track drop set, and fold drops incrementally The track's drop set (Rust TrackState::drops, JS DropSet) bills each run to the track's pool account, sheds its lowest runs when paying eviction debt, and forgets runs past the pool's idle window. Wire cursors and the miss probe fold in only the runs changed since they last looked. JS sinks share the track's set instead of copying it. Co-Authored-By: Claude Opus 5.5 --- quest/m1/drop-set.md | 58 +++++++++++++++++------------- rs/moq-net/src/model/track.rs | 4 +-- rs/moq-net/src/model/track/miss.rs | 18 +++++----- 3 files changed, 45 insertions(+), 35 deletions(-) diff --git a/quest/m1/drop-set.md b/quest/m1/drop-set.md index e32605f983b..06727a74268 100644 --- a/quest/m1/drop-set.md +++ b/quest/m1/drop-set.md @@ -19,14 +19,29 @@ reason (`Hole` in `rs/moq-net/src/model/track/hole.rs`, only for readers it covers (`Drift::settles`, `settles` in `js/net/src/track.ts`) and forgetting stale runs when they send new terms (`supersede_stale`), and every lite-07 update re-offering what was reported -stale (`reoffer_stale`). It kept today's storage: `TrackState::drops`, -`DropCursor` with its `reported` and `reported_stale` sets, `Skipped`, -`Delivered`, `TrackRun::served`, the lite subscriber's record of the group -streams that finished since its last update (`Tail::finished`, JS -`Tail#finished`), and the JS `drops` signal, `#pending`, `#reported`, and -`#reportedStale`, none billed to anything. This quest is +stale (`reoffer_stale`). The per-track runs are bounded now (below), but +not the per-cursor storage: `DropCursor` with +its `reported`, `reported_stale`, `reported_received`, and `reported_skip` +sets, `Skipped`, `Delivered`, `TrackRun::served`, the lite subscriber's record +of the group streams that finished since its last update (`Tail::finished`, +JS `Tail#finished`), and the JS wire cursor's `#pending`, `#reported`, +`#reportedStale`, and `#reportedSkip`, none billed to anything. This quest is what remains: bounding them. +Done, the drop set's track half: `TrackState::drops` (Rust) and the shared +`DropSet` in `js/net/src/drops.ts`, which a JS track's sinks now share instead +of copying, bill each run (`RUN_OVERHEAD`) and each entry of their expiry index +(`BORN_OVERHEAD`) to the track's pool account. Paying eviction debt sheds the +lowest runs after the cold groups, and the expiry scan forgets a run given up +longer than the pool's idle window ago (rounded up to a quarter of it), so a +pool with no byte target still bounds the set. A shed or forgotten run is +unknown again. A JS reader's own fill refusals stay its own (`refused`). +`DropCursor::next`, the JS wire cursor, and `Misses::unknown` fold in only the +runs changed since they last looked (`drops_changes`, `DropSet::since`), and +check a folded run against the set before reporting it, so a drop costs each +reader the runs it touches: `track_subscriber_churn_drops` in +`rs/moq-net/benches/track.rs` stays flat over shared runs and span. + Decided 2026-11-03 (maintainer, /quest-unblock): `js/net` bounds run memory by porting Rust's per-broadcast `cache::Pool` to JS. One shared byte account per broadcast bills frames and drop-set runs (a fixed cost per run) under @@ -278,13 +293,10 @@ starts. publisher's `served`. Delivery is recorded, not inferred from the cursor position, since an arrival-order or descending cursor delivers out of sequence order. -- **Billed to the pool, never guessed.** Each track run and each cursor run - is billed through a `Charge` at a fixed per-run cost (in JS, through the - pool in `js/net/src/cache.ts`). Under pressure the pool - evicts the oldest track runs as it evicts groups, and a run older than the - cache's retention window is forgotten. An evicted track run's sequences are - unknown again: a reader judges them by `drifted` or fills them by range. - A relay's own eviction is never a run. +- **Billed to the pool, never guessed.** Each cursor run is billed through a + `Charge` at a fixed per-run cost (in JS, through the pool in + `js/net/src/cache.ts`), as the track's runs already are. A relay's own + eviction is never a run. - **The floor rises only over a final prefix:** the runs from the floor up to the lowest run that is not final. A final run is a delivered one, one skipped at the track's max age bound, a skip, a budget-independent drop, @@ -297,14 +309,12 @@ starts. `rs/moq-net/src/model/track/miss.rs` keeps per gap (from [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171), decided 2026-10-23). -- **CPU per drop.** Every SUBSCRIBE_DROP bumps `drops_revision`, and each - reader's `DropCursor::next` rescans the overlapping runs, so N fragmented - runs and R readers cost O(R*N) per 6-byte drop; JS `insertDrop` - (`js/net/src/internal.ts`) copies the whole run array once for the track - and once per sink. Keep the per-cursor walk incremental in both languages. - Extend `track_subscriber_churn` in `rs/moq-net/benches/track.rs` with a - fragmented set up to pool capacity, swept over subscriber count, shared - runs, runs per cursor record, and run span. +- **CPU per drop.** The per-cursor records are still `RangeSet`s and sorted + arrays that every report inserts into, so `track_subscriber_churn_drops` + slows once many readers each hold many reported runs (256 readers holding + 65,536 each: about 1 to 4 ms per drop against 0.16 ms with one each). JS + `#askFills` still walks every run above the edge on each probe while a fill + handler serves the track; mirror Rust's `Misses::unknown` memo there. Still to cover, beyond the memory work: @@ -342,9 +352,9 @@ Tests, on mock time through a relay, in Rust and mirrored in `js/net`: - Under pool pressure, delivered runs and budget-independent drops fold into the floor, while evicting a lite-07 stale report within reach, or a run above it, ends the cursor with `TooFarBehind`. -- A flood of fragmented runs, including a JS drop-only flood with no cached - groups, stays within the pool, evicting the oldest shared runs first, and - a sequence whose run was evicted is judged locally or filled by range. +- A sequence whose shared run was shed or forgotten is judged locally or + filled by range, through a relay (the flood bound itself is covered by + `a_drop_flood_stays_within_the_pool` and its JS mirror). - A single run spanning billions of sequences is settled by each cursor in one step, whether dropped, skipped by a wider reader, or evicted and judged again. diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 2bd24ebe311..55f841806f5 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -23704,8 +23704,8 @@ mod test { elapse(&producer, cache::DEFAULT_EXPIRY); pool.sweep(); - let held: Vec<_> = producer.state.read().drops.iter().map(|(run, _)| run.clone()).collect(); - assert_eq!(held, [4..5], "only the run given up within the window stays"); + let held: Vec<_> = producer.state.read().drops.iter().map(|(run, _)| run.start).collect(); + assert_eq!(held, [4], "only the run given up within the window stays"); assert_eq!(pool.used(), used + RUN_OVERHEAD + BORN_OVERHEAD); assert!( producer.state.read().given_up(0).is_none(), diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index 1f0148e293f..469a81976d0 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -601,15 +601,6 @@ impl Misses { } } - /// Judge the holes in `target` the last probe has not, returning the runs to ask a - /// fill for. Gives up a run already stale under this reader's budget. `named` is the - /// lowest group the reader's ranges name, which may reach below the feed's start, - /// `bound` the exclusive end of a window whose ranges all end, and `taken` what the - /// reader handed out, which an eviction never makes a miss again. - #[expect( - clippy::too_many_arguments, - reason = "one probe's inputs, borrowed apart from its cursor" - )] /// The last sequence of the highest received stale run above the newest group that `drift` /// does not settle. While the terms hold and the run it found still stands, only the runs /// changed since are looked at, so a drop costs each reader the runs it touches. @@ -658,6 +649,15 @@ impl Misses { self.unknown } + /// Judge the holes in `target` the last probe has not, returning the runs to ask a + /// fill for. Gives up a run already stale under this reader's budget. `named` is the + /// lowest group the reader's ranges name, which may reach below the feed's start, + /// `bound` the exclusive end of a window whose ranges all end, and `taken` what the + /// reader handed out, which an eviction never makes a miss again. + #[expect( + clippy::too_many_arguments, + reason = "one probe's inputs, borrowed apart from its cursor" + )] pub fn judge( &mut self, state: &TrackState, From 20e064774eddfaedf6ddec61cf7caf97a385e942 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 12:32:54 -0800 Subject: [PATCH 05/17] fix(net): a local track keeps the drops its declared end relies on Co-Authored-By: Claude Opus 5.5 --- js/net/src/cache.test.ts | 12 ++++++++--- js/net/src/track.ts | 22 ++++++++++++++------ quest/m1/drop-set.md | 4 +++- rs/moq-net/src/model/track.rs | 38 +++++++++++++++++++++++++++++++---- 4 files changed, 62 insertions(+), 14 deletions(-) diff --git a/js/net/src/cache.test.ts b/js/net/src/cache.test.ts index 8e387da40f1..5778f5d05fa 100644 --- a/js/net/src/cache.test.ts +++ b/js/net/src/cache.test.ts @@ -294,6 +294,7 @@ test("a drop flood stays within the pool", () => { const capacity = 64 * cache.RUN_OVERHEAD; const pool = new cache.Pool({ capacity }); const producer = pooled(pool); + hooks.receiveTrack(producer); const reader = producer.subscribe({ delay: { max: null } }); const reported: bigint[] = []; for (let run = 0n; run < runs; run++) { @@ -323,20 +324,25 @@ test("the sweep forgets runs past the idle window", async () => { await withClock(async (clock) => { const pool = new cache.Pool({ expiry: Milli(1_000) }); const producer = pooled(pool); + hooks.receiveTrack(producer); + // A local track keeps what it skipped above its newest group, which its end relies on. + const local = pooled(pool, "local"); + local.skipGroups(5n, 10n); hooks.dropTrackGroups(producer, 0n, 1n, old()); hooks.dropTrackGroups(producer, 2n, 3n, old()); await clock.advance(500); hooks.dropTrackGroups(producer, 4n, 5n, old()); // Each run and its expiry are billed. - expect(pool.used).toBe(3 * (cache.RUN_OVERHEAD + cache.BORN_OVERHEAD)); + expect(pool.used).toBe(4 * (cache.RUN_OVERHEAD + cache.BORN_OVERHEAD)); // The sweep at 1.5 s forgets the first two, given up at 0 s. await clock.advance(1_000); expect(dropsHeld(producer)).toEqual([4n]); - expect(pool.used).toBe(cache.RUN_OVERHEAD + cache.BORN_OVERHEAD); + expect(pool.used).toBe(2 * (cache.RUN_OVERHEAD + cache.BORN_OVERHEAD)); await clock.advance(1_000); expect(dropsHeld(producer)).toEqual([]); - expect(pool.used).toBe(0); + expect(dropsHeld(local)).toEqual([5n]); + expect(pool.used).toBe(cache.RUN_OVERHEAD + cache.BORN_OVERHEAD); }); }); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 39845e25cc7..5bcd7f951a9 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -1166,11 +1166,12 @@ export class Producer { // them by range. #shedDrops(debt: number, cap: number): number { let paid = 0; + const kept = this.#keptDrops(); while (paid < debt && paid < cap) { const run = this.#state.drops.all[0]; - if (!run) break; + if (!run || run.start >= kept) break; const billed = this.#dropsBilled; - this.#setDrops(run.start, run.end); + this.#setDrops(run.start, min(run.end, kept)); paid += billed - this.#dropsBilled; } return paid; @@ -1180,16 +1181,25 @@ export class Producer { // only once its oldest run is due, which a quarter of the window separates from the next. #expireDrops(now: number): boolean { if (this.#dropsOldest === undefined || now < this.#dropsOldest) return false; + const kept = this.#keptDrops(); const expired = this.#dropsBorn.all - .filter((run) => now >= run.expires) - .map(({ start, end }) => ({ start, end })); + .filter((run) => now >= run.expires && run.start < kept) + .map(({ start, end }) => ({ start, end: min(end, kept) })); for (const run of expired) this.#setDrops(run.start, run.end); this.#dropsOldest = undefined; - for (const run of this.#dropsBorn.all) - this.#dropsOldest = Math.min(this.#dropsOldest ?? run.expires, run.expires); + for (const run of this.#dropsBorn.all) { + if (run.start < kept) this.#dropsOldest = Math.min(this.#dropsOldest ?? run.expires, run.expires); + } return expired.length > 0; } + // Where the runs a reclaim may forget end: every run on a received copy, but only those below + // a local track's edge, since one above it is what lets its declared end be reached (see + // `reaches`). A local producer gives up only what it chose to. Mirrors the Rust `kept_drops`. + #keptDrops(): bigint { + return this.#wire ? UNBOUNDED : this.#received; + } + static { hooks.poolTrack = (producer, pool) => { if (!producer.#account.empty || producer.#state.drops.size > 0) diff --git a/quest/m1/drop-set.md b/quest/m1/drop-set.md index 06727a74268..5061427dd79 100644 --- a/quest/m1/drop-set.md +++ b/quest/m1/drop-set.md @@ -35,7 +35,9 @@ of copying, bill each run (`RUN_OVERHEAD`) and each entry of their expiry index lowest runs after the cold groups, and the expiry scan forgets a run given up longer than the pool's idle window ago (rounded up to a quarter of it), so a pool with no byte target still bounds the set. A shed or forgotten run is -unknown again. A JS reader's own fill refusals stay its own (`refused`). +unknown again. A local track keeps its runs from its newest group up, which +its declared end relies on (`kept_drops`); only a received copy's peer can +fragment drops, and all of those are reclaimable. A JS reader's own fill refusals stay its own (`refused`). `DropCursor::next`, the JS wire cursor, and `Misses::unknown` fold in only the runs changed since they last looked (`drops_changes`, `DropSet::since`), and check a folded run against the set before reporting it, so a drop costs each diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 55f841806f5..14b9d208bef 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1454,16 +1454,33 @@ impl TrackState { if !self.drops_expired(now) { return; } + let kept = self.kept_drops(); let expired: Vec<_> = self .drops_born .iter() - .filter(|(_, expires)| now >= **expires) - .map(|(run, _)| run.clone()) + .filter(|(run, expires)| now >= **expires && run.start < kept) + .map(|(run, _)| run.start..run.end.min(kept)) .collect(); for run in expired { self.change_drops(run, None); } - self.drops_oldest = self.drops_born.iter().map(|(_, expires)| *expires).min(); + self.drops_oldest = self + .drops_born + .iter() + .filter(|(run, _)| run.start < kept) + .map(|(_, expires)| *expires) + .min(); + } + + /// Where the runs a reclaim may forget end: every run on a received copy, whose tail + /// decides its end, but only those below a local track's edge, since one above it is what + /// lets its declared end be reached (see [`Self::reaches`]). A local producer gives up only + /// what it chose to. + fn kept_drops(&self) -> u64 { + match self.anchors.received() { + true => u64::MAX, + false => self.edge(), + } } /// Forget the lowest runs of the drop set until the debt is paid or `cap` bytes are @@ -1471,8 +1488,14 @@ impl TrackState { /// them by its budget, or fills them by range. fn shed_drops(&mut self, cap: u64) -> u64 { let mut paid = 0u64; + let kept = self.kept_drops(); while self.debt > 0 && paid < cap { - let Some(run) = self.drops.first_range_value().map(|(run, _)| run.clone()) else { + let Some(run) = self + .drops + .first_range_value() + .map(|(run, _)| run.start..run.end.min(kept)) + .filter(|run| !run.is_empty()) + else { break; }; let billed = self.drops_bill.size(); @@ -23690,7 +23713,11 @@ mod test { #[test] fn the_sweep_forgets_runs_past_the_expiry_window() { let producer = track_producer("test", Info::timed()); + producer.received(); let pool = producer.broadcast.pool.clone(); + // A local track keeps what it skipped above its newest group, which its end relies on. + let local = track_producer_pooled("local", pool.clone()); + local.skip_groups(5..10); let used = pool.used(); producer.drop_groups(0..1, StreamError::Old); producer.drop_groups(2..3, StreamError::Old); @@ -23716,6 +23743,7 @@ mod test { pool.sweep(); assert!(producer.state.read().drops.is_empty()); assert_eq!(pool.used(), used); + assert!(local.state.read().skipped(5), "a local skip above the edge stays"); } /// A drop-only flood of disjoint runs bills the pool and sheds its lowest runs to stay @@ -23726,6 +23754,8 @@ mod test { const RUNS: u64 = 1000; let capacity = 64 * (RUN_OVERHEAD + BORN_OVERHEAD); let (producer, pool) = pooled_producer(capacity); + // A peer's drops, which a received copy holds. + producer.received(); let waiter = kio::Waiter::noop(); let mut subscriber = producer.subscribe(replay().with_range(0..)); let mut reported = Vec::new(); From ebb880cd4aeede1efa24b9ca950ab21d417e3c4e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 18:01:58 -0800 Subject: [PATCH 06/17] quest: drop set keeps only the per-cursor record, still-to-cover items, and relay tests Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 2 +- quest/m1/drop-set.md | 332 ++++++------------------------------------- quest/m2/README.md | 2 +- 3 files changed, 43 insertions(+), 293 deletions(-) diff --git a/quest/m1/README.md b/quest/m1/README.md index 1159f5c1bad..31750426208 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -31,7 +31,7 @@ moves what remains of them once their claims merge. ## Required -- [Drop set](/quest/m1/drop-set.md) - the drop set's track and cursor runs are billed to the cache pool, so fragmenting drops costs what the pool holds; eviction folds final runs or ends the cursor with `TooFarBehind` +- [Drop set](/quest/m1/drop-set.md) - each cursor's disposition record is billed to the cache pool, as the track's drop runs are, so fragmenting drops costs what the pool holds; eviction folds final runs or ends the cursor with `TooFarBehind` - [Routing guide](/quest/m1/routing-guide.md) - concepts and practical outbound federation setup - [A reader on a failed copy continues on the held re-offer](/quest/m1/copy-recover.md) - a live reader keeps a group whose copy fails by continuing on the same track's re-offer at the same frame - [Group errors are cached by kind](/quest/m1/group-error-cache.md) - permanent group errors bind every reader, retryable ones are kept only for a bounded time, and a live SUBSCRIBE_DROP or FETCH answered `Old` no longer caches it for good diff --git a/quest/m1/drop-set.md b/quest/m1/drop-set.md index 5061427dd79..40c9a852870 100644 --- a/quest/m1/drop-set.md +++ b/quest/m1/drop-set.md @@ -3,302 +3,45 @@ ## Goal A hostile upstream fragmenting drops costs what the cache's byte pool would -hold, never more: the drop set's per-track runs and each cursor's -disposition record are billed to the pool at a fixed cost per run, in Rust -and `js/net`, and a cursor's CPU per drop tracks the runs it touches, never -their span or the set. Eviction keeps every promise a reader was made: a -final run folds into the cursor's floor, and losing any other ends the -subscription with `TooFarBehind`. +hold, never more: each cursor's disposition record is billed to the pool at a +fixed cost per run, in Rust and `js/net`, as the track's runs already are, and +a cursor's CPU per drop tracks the runs it touches, never their span or the +set. Eviction keeps every promise a reader was made: a final run folds into +the cursor's floor, and losing any other ends the subscription with +`TooFarBehind`. ## Plan -[moq-dev/moq#7670](https://github.com/moq-dev/moq/pull/7670) landed the -behavior: lite-07 SUBSCRIBE_SKIP and SUBSCRIBE_DROP with `Update`, a per-run -reason (`Hole` in `rs/moq-net/src/model/track/hole.rs`, -`Hole` in `js/net/src/internal.ts`), relays settling a received stale drop -only for readers it covers (`Drift::settles`, `settles` in -`js/net/src/track.ts`) and forgetting stale runs when they send new terms -(`supersede_stale`), and every lite-07 update re-offering what was reported -stale (`reoffer_stale`). The per-track runs are bounded now (below), but -not the per-cursor storage: `DropCursor` with -its `reported`, `reported_stale`, `reported_received`, and `reported_skip` -sets, `Skipped`, `Delivered`, `TrackRun::served`, the lite subscriber's record -of the group streams that finished since its last update (`Tail::finished`, -JS `Tail#finished`), and the JS wire cursor's `#pending`, `#reported`, -`#reportedStale`, and `#reportedSkip`, none billed to anything. This quest is -what remains: bounding them. - -Done, the drop set's track half: `TrackState::drops` (Rust) and the shared -`DropSet` in `js/net/src/drops.ts`, which a JS track's sinks now share instead -of copying, bill each run (`RUN_OVERHEAD`) and each entry of their expiry index -(`BORN_OVERHEAD`) to the track's pool account. Paying eviction debt sheds the -lowest runs after the cold groups, and the expiry scan forgets a run given up -longer than the pool's idle window ago (rounded up to a quarter of it), so a -pool with no byte target still bounds the set. A shed or forgotten run is -unknown again. A local track keeps its runs from its newest group up, which -its declared end relies on (`kept_drops`); only a received copy's peer can -fragment drops, and all of those are reclaimable. A JS reader's own fill refusals stay its own (`refused`). -`DropCursor::next`, the JS wire cursor, and `Misses::unknown` fold in only the -runs changed since they last looked (`drops_changes`, `DropSet::since`), and -check a folded run against the set before reporting it, so a drop costs each -reader the runs it touches: `track_subscriber_churn_drops` in -`rs/moq-net/benches/track.rs` stays flat over shared runs and span. - -Decided 2026-11-03 (maintainer, /quest-unblock): `js/net` bounds run memory -by porting Rust's per-broadcast `cache::Pool` to JS. One shared byte account -per broadcast bills frames and drop-set runs (a fixed cost per run) under -Rust's default budget, with eviction and the fold-into-floor or -`TooFarBehind` behavior below, so Rust and JS mirror each other with no new -setting. Reason: the 2026-11-02 audit found the existing 32 MiB limit in -`js/net/src/group.ts` bounds only one group's frame tail, and a mock-time -probe retained 64, 256, and 1,024 disjoint OLD runs with zero cached groups; -a shared pool bounds drop-only tracks the same way in both languages. Not -chosen: a per-track run cap; upstream backpressure. The pool port itself -landed on its own as `js/net/src/cache.ts` (split out on 2026-11-03), billing -cached groups; this quest bills the runs to it. - -Decided 2026-11-03 (maintainer, /quest-unblock), renamed and narrowed by the -`/quest-complete` decisions below: range terms carry an -explicit update id. Each lite-07 SUBSCRIBE and SUBSCRIBE_UPDATE carries an -increasing range-terms id, and a stale drop echoes the id it was judged -under. A relay treats a drop whose id has been superseded as unknown and -refills it by range, keeping no term history. Reason: a drop sent under an -old broad range can arrive after SUBSCRIBE_UPDATE narrows the aggregate at -the same budget (mock-time cursors over groups at 0, 1, and 2 seconds serve -group 2 under the open range and group 0 under a range capped before group -1), and `Max Age` alone cannot tell the two verdicts apart. Not chosen: a -conservative refill with bounded remembered ranges; treating every range -update as invalidating in-flight drops. - -Decided 2026-11-03 (maintainer, /quest-unblock), with `Max Age` dropped, -`Terms` renamed `Update`, and SKIP narrowed to declared sequences by the -`/quest-complete` decisions below: lite-07 -splits its hole message by whether the group ever existed. `SUBSCRIBE_SKIP {Group Start, -Group End}` names sequences the publisher never produced, such as skipped -sequence numbers or ones past a cut: it has no error code, ends no reader, -and is final. `SUBSCRIBE_DROP {Group Start, Group End, Error Code, Max Age, -Terms}` names groups that existed but will not be delivered: an aborted or -reset group, eviction or expiry with no copy left, and stale (OLD). It ends -any held reader with its code; a stale drop carries the `Max Age` and the -range-terms id it was judged under, so a widening can re-offer it. lite-06 -and older keep today's SUBSCRIBE_DROP on the wire. The split runs through -moq-net's Rust and JS model and API, whose drop/skip naming follows it, and -through the draft. Reason: a hole that never existed has no reader to end -and no budget to qualify, so one message conflated a final, code-free -verdict with a reversible, reader-ending one. Not chosen: stale in SKIP; a -single message with a reason enum; renaming only. - -Decided 2026-11-03 (maintainer, /quest-complete), changing the design -[moq-dev/moq#7670](https://github.com/moq-dev/moq/pull/7670) first shipped: - -- **`Terms` is renamed `Update`** on lite-07 SUBSCRIBE, SUBSCRIBE_UPDATE and - SUBSCRIBE_DROP: the number of the SUBSCRIBE or SUBSCRIBE_UPDATE in effect, - in the Rust and JS code too. -- **SUBSCRIBE_DROP drops `Max Age`:** it is {Group Start, Group End, Error - Code, Update}. The budget a stale (OLD) drop was judged under comes from - the subscriber's own record of what it sent with that Update, and a drop - whose Update is superseded is no verdict. Any SUBSCRIBE_UPDATE reopens every - outstanding OLD drop in the subscriber's tail accounting; the publisher - re-offers what the new terms cover and drops again, under the new Update, - what is still stale. This replaces reopening only when the max delay widens - past the drop's. A non-OLD drop is final, whatever its Update. -- **`track::Producer::drop_groups` is crate-private.** `skip_groups` stays - public (see the next decision). - -Decided 2026-11-03 (maintainer, /quest-complete), replacing the "eager SKIP" -decision above and the PR's `## Waiting` item on out-of-order creation: - -- ✅ **SKIP comes only from an explicit, public `track::Producer::skip_groups`**, - and from `append_group` (below). Reason: producers legitimately create - groups out of order (`claim_sequence` re-creating an aborted sequence, - moq-mux and moq-c `seek`, `finish()`'s documented "older groups can still - arrive"), so a gap tells moq-net nothing; only the producer knows a - sequence will never exist. moq-archive's replay is the in-tree consumer of - `skip_groups`, for the sequences up to a recorded end it holds nothing - for; a producer that numbers sparsely would call it too. moq-mux's live - encoders use `append_group`. The relay records a received SKIP with - `skip_groups` (below). -- ✅ **Both inferences are removed:** the eager skip of the gap in - `create_group`, and the automatic SKIP at finish for never-arrived - sequences below the final one (`TrackState::missed` and the publisher's - `drop_unserved`). An undeclared gap is judged stale against the reader's - budget, or named with DROP NOT_FOUND, as before this PR. -- ✅ **SKIP is final:** creating a group (or appending a group or datagram) - at a skipped sequence is refused with `Error::Skipped` (JS `Skipped`). - Re-creating a dropped or aborted sequence still works. -- ✅ **SKIP replaces every drop, final or stale** (maintainer, 2026-11-03, - `/quest-complete`, replacing the 2026-11-04 `/quest-iterate` rule that a - SKIP replaced only a stale OLD drop). "Never produced" is the most - authoritative fact about a sequence, so a received SKIP, `skip_groups`, or - `append_group`'s implicit skip overwrites an earlier DROP of any code (OLD, - aborted, evicted, NOT_FOUND), and a later DROP leaves a skip in place. - Readers that drop already ended stay ended, and nothing is re-delivered; the - run's reason becomes Skip, so every later reader, relay and downstream - report sees SKIP, and a relay forwards the SKIP even over a DROP it already - forwarded (`DropCursor::reported_skip`, `TrackRun::dropped`, and their JS - mirrors). A cached group is never overwritten: a received SKIP naming one - leaves the group, logs a protocol error, and keeps the session, since the - draft requires no more. A received SKIP or final DROP also takes its groups - out of the subscriber's stale record, so the next update doesn't owe them - again. Not chosen: SKIP replacing only a stale OLD drop. -- ✅ **A group or final drop replaces a lite-07 stale drop already sent** - (agent decision from review, 2026-11-04, `/quest-iterate`; the maintainer - chose to keep it and retire the stale report at the 2026-11-03 round-4 - checkpoint, `/quest-complete`). A front that switches routes can hand out - the new route's group, or its final verdict, for a sequence the old route - reported stale, before the update that reopens it, so the publisher serves - the group, or sends the final drop, over the OLD it sent. Its own expired - streams' OLD drops count the same. Whatever replaces a stale report - retires it: a group handed out, a SKIP, or a final drop takes the sequence - out of the reader's stale record (`DropCursor::reported_stale`, the - resumed `Reader::reported_stale`, JS `#reportedStale`) and the publisher's - (`TrackRun::dropped_stale`, `expired`), so the next SUBSCRIBE_UPDATE - neither sends OLD for it again nor names it at the end. On the receiving - side a Group Stream wins over a stale drop of its sequence in either order: - an OLD that lands after the stream leaves the group, still arriving or - not, or finished under the same update and since evicted (agent decision - 26, 2026-11-05, for review), and an update owes again only the stale runs - the track still holds. - The end sweep's NOT_FOUND never replaces a stale drop, so an update can - still reach an ended track's history. Not chosen: reverting the - replacement; merging with the stale report left in place. -- ✅ **`append_group` is an implicit skip** (refinement, same session): it - means "the next group", so the producer declares it won't go back, and it - skips every sequence below it that it never produced, from just past the - previous append (before the first append, from the lowest group created), - as if `skip_groups` were called on them: a drop there becomes a skip, and - they are then final. Today - `append_group` leaves no gap of its own, so this only fires on holes earlier - `create_group` calls left below the head; under time addressing it names - the span between consecutive starts. `create_group` stays non-committal, - so out-of-order creation (`claim_sequence` retries, `seek`) keeps working. - There is still no inference at finish. -- ✅ **`skip_groups` stays public** and is documented as the explicit source - of SKIP; `drop_groups` stays crate-private. Rust and JS mirror each other - (`skipGroups`, `appendGroup`). -- ✅ **Draft:** SKIP names only sequences the publisher declared it will never - produce, and it is final. -- ✅ **Run-length tracking** (refinement, same session): skipped and dropped - groups are kept as runs, never one entry per sequence, in Rust and JS: the - track's hole set, the implicit `append_group` skip, `skip_groups`, and what - the publisher reports. A huge `skip_groups` or append gap costs one run and - one SUBSCRIBE_SKIP (`a_huge_skip_costs_one_run`, - `lite07_skips_a_huge_run_at_once`, and their JS mirrors). -- ✅ **`create_group` refuses a skipped sequence and allows a dropped one** - (refinement, same session): a group stream wins over its drop, which fills - answering a miss and `claim_sequence` retries of an aborted sequence need. -- ✅ **No `create_group` gap error, for now** (maintainer, same session): - gaps stay legal, and `append_group` or `skip_groups` declares them. No - principled bound exists in the model: the cache and the budget bound bytes - and time, not sequence counts, a gap costs one run whatever its length, - and under time addressing every group leaves a gap between starts. Not - chosen: a debug log on a gap; a configurable size limit (an arbitrary - knob). -- Not chosen: eager skip on `create_group` with a late group winning over - it (the shipped design, which inferred SKIP from a gap that out-of-order - creation makes meaningless); refusing every `create_group` below the last - group (broke `claim_sequence`, `seek` and 85 moq-net tests); keeping the - finish-time inference for a local producer (a gap at finish is no more a - declaration than one mid-stream). - -Decided 2026-11-05 (maintainer, /quest-complete): a duplicate Group Stream -waits until the first copy settles. On lite-07 the OLD drop for an expired -group, and the stream the next update re-offers for it, can both reach the -subscriber before the original stream's reset, which used to refuse the -re-offer as a duplicate and lose the group once the reset landed. The -subscriber now holds a stream whose sequence another copy still arriving -holds, unread, and discards it if that copy finishes or takes the group if it -fails (Rust `GroupRecv`'s `Hold` wait over `track::Received::Held`, JS -`runGroup` over `receiveGroup`'s `arriving`). The held stream keeps the -lowest receive priority, so it holds at most its stream credit and is the -first a full receive window evicts. A reader already handed the failed copy -sees it fail, since no sequence is handed out twice; the cache, fills, and -readers that had not reached it get the re-offer. Not chosen: letting the -duplicate replace the copy still arriving; accepting the loss. - -Mirror each case in `js/net`. -In `js/net`, a group a sink buffers keeps its source wanted only while -`Subscriber.#reading` says its reader will still read it -([moq-dev/moq#7948](https://github.com/moq-dev/moq/pull/7948)), so a change to -how a cursor judges a missing or stale group keeps that verdict in step. - -Decided 2026-11-05 (maintainer, /quest-plan): the agent decisions -[moq-dev/moq#7670](https://github.com/moq-dev/moq/pull/7670) listed for -review stand as recorded, numbered as in its body: - -- ✅ **14:** a Group Stream or final drop the publisher's reader reaches - replaces a stale drop it already sent (above). -- ✅ **16:** SUBSCRIBE_UPDATE carries `Update` after the Range edit and before - `Wanted Group`; SUBSCRIBE_SKIP keeps 0x3 beside SUBSCRIBE_STARTED at 0x4. -- ✅ **17:** an OLD drop records no stale run on groups a range fill asked - for, which recovery asks for again. Every drop the subscriber accounts, - and every SKIP, answers the fills owing those groups; an OLD drop under a - superseded `Update` answers none. -- ✅ **18:** the JS covered-FIN reopen loop and its stale-run supersede go, - as main deleted the loop. -- ✅ **19:** a final OLD drop, such as one relayed from lite-06, goes out on - lite-07 as the stale drop its code names, recorded and dropped again on - each update. -- ✅ **20:** an update judges again the re-offers still waiting for stream - credit, and a verdict sent for a queued re-offer keeps it from opening. -- ✅ **21:** a SUBSCRIBE_DROP naming an `Update` later than the last one sent - is a PROTOCOL_VIOLATION. -- ✅ **22:** a received stale run is judged again for a reader whose terms - changed, so a widened budget asks a fill for it. -- ✅ **23:** recording a stale verdict answers no outstanding fill (JS, as - Rust), `skipGroups` checks its bounds, and a fill cannot cache a skipped - sequence. -- ✅ **25:** a relay cursor's stale report that an upstream's received stale - run decided stays the upstream's to judge after an update - (`DropCursor::reported_received`, JS `#reportedReceived`). - -Decided 2026-11-06 (maintainer, /quest-complete), replacing agent decision 27 -of [moq-dev/moq#7670](https://github.com/moq-dev/moq/pull/7670), which asked a -fill for every unknown stale run instead: - -- ✅ **27: a stale verdict accepts every range.** A relay keeps every group - its current upstream terms' ranges name (`TrackState::judged`, JS - `judged`), taking a bound only the publisher resolves at its widest. A - received stale run settles a reader no wider than the budget it was judged - under that reads every one of those groups above the run's start, however - many ranges they span, since a group's verdict looks only above it. A - reader with the upstream's own ranges is settled with nothing received. A - reader narrower than the upstream's ranges still treats the run as unknown, - and asks a fill for it even above the newest group received, since its - ranges add nothing to the upstream's terms and no update brings the run - back. Not chosen: asking a fill for every unknown stale run without the - multi-range verdict; reversing the fix and tracking the hang as a - follow-up. -- ✅ **28 (2026-11-06, maintainer, /quest-complete): keep the fill for - readers narrower than the verdict's ranges**, with no fill-handler gate. A - verdict judged on the union of readers' ranges is valid only for readers - that read every group it names above the run's start; a narrower reader - still needs groups the upstream already dropped, so only a fill can - deliver them. Not chosen: re-judging per reader (it would still need the - fill); separate upstream subscriptions per reader shape (they give up - aggregation); deferring the narrow case as a follow-up. - -Decided 2026-10-27 (maintainer): this runs after the range count cost work, -which finished in [moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542), -and before [Rust groups by start](/quest/m2/time-addressing/model.md), which -requires it because runs keyed by group sequence convert mechanically to -starts. +The track's drop set is billed and bounded, and every reader folds it in +incrementally ([moq-dev/moq#8294](https://github.com/moq-dev/moq/pull/8294)). +The per-cursor storage is not billed to anything: `DropCursor`'s `pending` +and its `reported`, `reported_stale`, `reported_received`, and +`reported_skip` sets, `Skipped`, `Delivered`, `TrackRun`'s `served`, +`dropped`, `dropped_stale`, and `expired`, the lite subscriber's record of +the group streams that finished since its last update (`Tail::finished`, JS +`Tail#finished`), and the JS wire cursor's `#pending`, `#skipped`, +`#reported`, `#reportedStale`, `#reportedReceived`, and `#reportedSkip`. +This quest bounds them, keeping the behavior +[moq-dev/moq#7670](https://github.com/moq-dev/moq/pull/7670) and +moq-dev/moq#8294 landed and the decisions recorded there. + +Decided 2026-10-27 (maintainer): this runs before +[Rust groups by start](/quest/m2/time-addressing/model.md), which requires it +because runs keyed by group sequence convert mechanically to starts. - **One disposition record per cursor,** a run-length map whose runs are delivered, skipped as stale, or reported dropped or skipped, with the reason and, for a stale run, the budget it was judged under. It replaces `Skipped` (`rs/moq-net/src/model/skipped.rs`), `Delivered` (`rs/moq-net/src/model/delivered.rs`, whose floor folds stale gaps in with - delivered groups), `DropCursor`'s `reported` and `reported_stale`, - `TrackRun::served` in `rs/moq-net/src/lite/publisher.rs`, and the JS - wire cursor's `#reported` and `#reportedStale`, `#skipped`, and the - publisher's `served`. Delivery is recorded, not inferred from the cursor - position, since an arrival-order or descending cursor delivers out of - sequence order. + delivered groups), `DropCursor`'s `reported*` sets, `TrackRun::served` in + `rs/moq-net/src/lite/publisher.rs`, and their JS mirrors. Delivery is + recorded, not inferred from the cursor position, since an arrival-order or + descending cursor delivers out of sequence order. - **Billed to the pool, never guessed.** Each cursor run is billed through a `Charge` at a fixed per-run cost (in JS, through the pool in - `js/net/src/cache.ts`), as the track's runs already are. A relay's own - eviction is never a run. + `js/net/src/cache.ts`), as the track's runs are. A relay's own eviction is + never a run. - **The floor rises only over a final prefix:** the runs from the floor up to the lowest run that is not final. A final run is a delivered one, one skipped at the track's max age bound, a skip, a budget-independent drop, @@ -311,12 +54,19 @@ starts. `rs/moq-net/src/model/track/miss.rs` keeps per gap (from [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171), decided 2026-10-23). -- **CPU per drop.** The per-cursor records are still `RangeSet`s and sorted - arrays that every report inserts into, so `track_subscriber_churn_drops` - slows once many readers each hold many reported runs (256 readers holding - 65,536 each: about 1 to 4 ms per drop against 0.16 ms with one each). JS - `#askFills` still walks every run above the edge on each probe while a fill - handler serves the track; mirror Rust's `Misses::unknown` memo there. +- **CPU per drop.** Every report inserts into the per-cursor `RangeSet`s and + sorted arrays, so `track_subscriber_churn_drops` in + `rs/moq-net/benches/track.rs` slows once many readers each hold many + reported runs (256 readers holding 65,536 each: about 1 to 4 ms per drop + against 0.16 ms with one each). JS `#askFills` still walks every run above + the edge on each probe while a fill handler serves the track; mirror Rust's + `Misses::unknown` memo there. + +Mirror each case in `js/net`. +In `js/net`, a group a sink buffers keeps its source wanted only while +`Subscriber.#reading` says its reader will still read it +([moq-dev/moq#7948](https://github.com/moq-dev/moq/pull/7948)), so a change to +how a cursor judges a missing or stale group keeps that verdict in step. Still to cover, beyond the memory work: diff --git a/quest/m2/README.md b/quest/m2/README.md index 1ae9ab6e886..36254851194 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -186,7 +186,7 @@ quests that require them. ## Required -- [Drop set](/quest/m1/drop-set.md) - the drop set's track and cursor runs are billed to the cache pool, so fragmenting drops costs what the pool holds; eviction folds final runs or ends the cursor with `TooFarBehind` (held in m1 by an open claim) +- [Drop set](/quest/m1/drop-set.md) - each cursor's disposition record is billed to the cache pool, as the track's drop runs are, so fragmenting drops costs what the pool holds; eviction folds final runs or ends the cursor with `TooFarBehind` (held in m1 by an open claim) - [Subscribe ranges](/quest/m2/subscribe-ranges/README.md) - a lite-07 SUBSCRIBE asks for past and live ranges with absolute or relative ends, in either order, and replaces FETCH for multi-group reads (single-group FETCH stays); merged ranges never starve a subscriber; relays fill misses by range, including over moq-transport - [Address groups by timestamp](/quest/m2/time-addressing/README.md) - group start timestamps replace sequence numbers in the moq-net API and lite-07 wire, while old subscribers, old publishers, and published JS clients keep working - [Cluster routing](/quest/m2/cluster-routing/README.md) - prove shared routing contracts, implement per-origin routes, and connect outbound local clusters From 0dbf96afcd5d1a240b6e1b22e05a4758762b4d88 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 20:55:31 -0800 Subject: [PATCH 07/17] fix(net): a kept drop run is reclaimable once its edge passes or its track closes, and a group clears a reader's refusal Co-Authored-By: Claude Opus 5.5 --- js/net/src/cache.test.ts | 25 +++++++++++++++++++++ js/net/src/cache.ts | 4 ++-- js/net/src/fill.test.ts | 18 +++++++++++++++ js/net/src/track.ts | 32 ++++++++++++++++++++++----- rs/moq-net/src/model/track.rs | 41 ++++++++++++++++++++++++++++++++--- 5 files changed, 109 insertions(+), 11 deletions(-) diff --git a/js/net/src/cache.test.ts b/js/net/src/cache.test.ts index 5778f5d05fa..e4558659d5d 100644 --- a/js/net/src/cache.test.ts +++ b/js/net/src/cache.test.ts @@ -346,3 +346,28 @@ test("the sweep forgets runs past the idle window", async () => { expect(pool.used).toBe(cache.RUN_OVERHEAD + cache.BORN_OVERHEAD); }); }); + +// A local skip kept above the track's edge is dated once the edge passes it, or once the track +// closes, so an expiry scan past its window forgets it with no other drop to rearm it, and its +// bill leaves the pool. Mirrors Rust `a_kept_local_skip_expires_once_the_edge_passes_it`; Rust +// frees a closed track's drops with its state. +test("a kept local skip expires once the edge passes it or the track closes", async () => { + await withClock(async (clock) => { + const pool = new cache.Pool({ expiry: Milli(1_000) }); + const passed = pooled(pool); + const closed = pooled(pool, "closed"); + passed.skipGroups(0n, 10n); + closed.skipGroups(0n, 10n); + await clock.advance(3_000); + expect(dropsHeld(passed)).toEqual([0n]); + expect(dropsHeld(closed)).toEqual([0n]); + + // The write's own expiry scan finds the skip. + passed.createGroup(10n).finish(); + expect(dropsHeld(passed)).toEqual([]); + + const used = pool.used; + closed.close(); + expect(pool.used).toBe(used - cache.RUN_OVERHEAD - cache.BORN_OVERHEAD); + }); +}); diff --git a/js/net/src/cache.ts b/js/net/src/cache.ts index 93ff8eec3e4..3fc863f6452 100644 --- a/js/net/src/cache.ts +++ b/js/net/src/cache.ts @@ -237,7 +237,7 @@ export interface Owner { shedDrops(debt: number, cap: number): number; /** Forget the runs of the drop set given up longer than the pool's idle window before tick `now`, returning whether any went. */ expireDrops(now: number): boolean; - /** Whether the track holds a drop run, which keeps its account in the idle sweep. */ + /** Whether the track holds a drop run a reclaim may forget, which keeps its account in the idle sweep. */ holdsDrops(): boolean; } @@ -285,7 +285,7 @@ export class Track { bill(n: number): void { this.pool.add(n); this.wrote(n); - this.pool.register(this as Track); + if (this.#owner.holdsDrops()) this.pool.register(this as Track); } /** Release `n` bytes {@link bill} billed. */ diff --git a/js/net/src/fill.test.ts b/js/net/src/fill.test.ts index ac9e8a627f4..75da4cff371 100644 --- a/js/net/src/fill.test.ts +++ b/js/net/src/fill.test.ts @@ -306,6 +306,24 @@ test("fill: a refusal fails the readers waiting now, and a later reader asks aga broadcast.close(); }); +test("fill: a group arriving after a refusal reaches the reader the refusal named", async () => { + const { broadcast, track } = resumed(5n); + const requests = track.requests(); + const waiting = wireOf(broadcast).fetchGroup("video", 2n); + const fill = await requests.requestedFill(); + if (!fill) throw new Error("no fill"); + fill.refuseGroups(2n, 3n, new StreamError(StreamCode.Cancel, { message: "storage busy" })); + // Before the waiting fetch looks again. + const group = track.createGroup(2n); + group.writeFrame({ payload: enc.encode("late") }); + group.finish(); + fill.close(); + expect(await frames(await waiting)).toEqual(["late"]); + + requests.close(); + broadcast.close(); +}); + test("fill: a fill names the most urgent priority and loosest budget among its readers", async () => { const { broadcast, track } = resumed(5n); const requests = track.requests(); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 5bcd7f951a9..721ba948abf 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -1135,7 +1135,7 @@ export class Producer { settle: () => this.#settle(), shedDrops: (debt, cap) => this.#shedDrops(debt, cap), expireDrops: (now) => this.#expireDrops(now), - holdsDrops: () => this.#state.drops.size > 0, + holdsDrops: () => (this.#state.drops.all[0]?.start ?? UNBOUNDED) < this.#keptDrops(), }); } @@ -1193,11 +1193,22 @@ export class Producer { return expired.length > 0; } - // Where the runs a reclaim may forget end: every run on a received copy, but only those below - // a local track's edge, since one above it is what lets its declared end be reached (see - // `reaches`). A local producer gives up only what it chose to. Mirrors the Rust `kept_drops`. + // Where the runs a reclaim may forget end: every run on a received copy, but only those below an + // open local track's edge, since one above it is what lets its declared end be reached (see + // `reaches`). A local producer gives up only what it chose to. Mirrors the Rust `kept_drops`, + // except that a closed track's runs all go here too: Rust frees them with the state's last + // handle, but here the pool holds the account until the sweep forgets them. #keptDrops(): bigint { - return this.#wire ? UNBOUNDED : this.#received; + return this.#wire || this.#state.closed.peek() !== undefined ? UNBOUNDED : this.#received; + } + + // The runs from `from` up to `#keptDrops` were kept and are now reclaimable: date the oldest so + // the next idle scan finds it, and have the sweep visit the track for them. + #rearmDrops(from: bigint): void { + const runs = this.#dropsBorn.overlapping(from, this.#keptDrops()); + for (const run of runs) this.#dropsOldest = Math.min(this.#dropsOldest ?? run.expires, run.expires); + if (this.#state.drops.overlapping(from, this.#keptDrops()).length > 0) + this.#account.pool.register(this.#account); } static { @@ -1416,9 +1427,14 @@ export class Producer { return runs; } - // A publisher may re-create a group it cut; the sequence is no longer given up. + // A publisher may re-create a group it cut, and a group a reader's fill refused may still + // arrive; the sequence is no longer given up. #undrop(sequence: bigint): void { if (this.#state.drops.at(sequence)) this.#setDrops(sequence, sequence + 1n); + for (const sink of this.#sinks) { + const refused = sink.refused.peek(); + if (dropAt(refused, sequence)) sink.refused.set(removeRun(refused, sequence, sequence + 1n)); + } } /** @@ -1863,7 +1879,9 @@ export class Producer { // asks for. #advance(sequence: bigint, answers: boolean): void { const next = this.#received; + const kept = this.#keptDrops(); this.#received = max(next, sequence + 1n); + if (this.#keptDrops() > kept) this.#rearmDrops(kept); this.#advanceLargest({ group: sequence, frame: 0 }); this.#endLargest(); if (!this.#refresh || sequence < next) return; @@ -2269,7 +2287,9 @@ export class Producer { // Nobody will finish these, so a subscriber that has not taken one yet never sees it. // Not evicted: a reader already holding one keeps its frames and sees the abort. const open = [...this.#cache].filter((entry) => entry.group.closed.peek() === undefined); + const kept = this.#keptDrops(); closeTrackState(this.#state, abort); + if (this.#keptDrops() > kept) this.#rearmDrops(kept); if (this.#state.outcome.peek() === undefined) this.#state.outcome.set(abort ?? this.#state.end.peek()?.error ?? null); // Closing a copy never supplies the missing tail of a group. Already finished groups diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 14b9d208bef..8a097ab5507 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1475,7 +1475,7 @@ impl TrackState { /// Where the runs a reclaim may forget end: every run on a received copy, whose tail /// decides its end, but only those below a local track's edge, since one above it is what /// lets its declared end be reached (see [`Self::reaches`]). A local producer gives up only - /// what it chose to. + /// what it chose to. A closed track keeps them until its last handle frees the state. fn kept_drops(&self) -> u64 { match self.anchors.received() { true => u64::MAX, @@ -1483,6 +1483,19 @@ impl TrackState { } } + /// The runs from `from` up to [`Self::kept_drops`] were kept and are now reclaimable: date + /// the oldest of them, which the expiry scan skipped, so the next scan finds it. + fn rearm_drops(&mut self, from: u64) { + let due = self + .drops_born + .overlapping(&(from..self.kept_drops())) + .map(|(_, expires)| *expires) + .min(); + if let Some(due) = due { + self.drops_oldest = Some(self.drops_oldest.map_or(due, |oldest| oldest.min(due))); + } + } + /// Forget the lowest runs of the drop set until the debt is paid or `cap` bytes are /// freed, returning the bytes freed. Their sequences are unknown again: a reader judges /// them by its budget, or fills them by range. @@ -2505,7 +2518,11 @@ impl TrackState { /// The demand wakes it owes wait in [`Self::deferred`] for the caller to fire once its guard /// is released. fn advance(&mut self, sequence: u64, answers: bool) { + let kept = self.kept_drops(); self.max_sequence = Some(self.max_sequence.map_or(sequence, |max| max.max(sequence))); + if self.kept_drops() > kept { + self.rearm_drops(kept); + } self.cache.advance_largest(Position::group(sequence)); let wakes = self.subscriptions.lock().wrote(sequence, answers); if !wakes.is_empty() { @@ -23705,8 +23722,6 @@ mod test { write_group(&producer, 9); } - /// Skips are kept and reported as runs: a huge `skip_groups` or append gap costs one - /// entry and one report, whatever its length. /// A run given up longer than the pool's expiry window ago is forgotten by the sweep, even /// on a track that caches nothing, so a pool with no byte target still bounds the drop set; /// a run given up since stays. @@ -23796,6 +23811,26 @@ mod test { assert_eq!(reported[1..], held[..]); } + /// A local skip kept above the track's edge is dated once the edge passes it, so the next + /// expiry scan past its window forgets it with no other drop to rearm it. + #[test] + fn a_kept_local_skip_expires_once_the_edge_passes_it() { + let producer = track_producer("test", Info::timed()); + let pool = producer.broadcast.pool.clone(); + producer.skip_groups(0..10); + elapse(&producer, cache::DEFAULT_EXPIRY * 2); + pool.sweep(); + assert!(producer.state.read().skipped(0), "a skip above the edge stays"); + + // The write's own expiry scan finds the skip, with nothing else to rearm it. + let _group = producer.create_group(group::Info { sequence: 10 }).unwrap(); + let state = producer.state.read(); + assert!(!state.skipped(0), "the edge passed the skip"); + assert_eq!(state.drops_bill.size(), 0, "its bill went with it"); + } + + /// Skips are kept and reported as runs: a huge `skip_groups` or append gap costs one + /// entry and one report, whatever its length. #[test] #[cfg(feature = "async")] fn a_huge_skip_costs_one_run() { From c8d20672d24b4ae76f8c6e686bcf90f6da1622d6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 21:35:38 -0800 Subject: [PATCH 08/17] fix(js/net): a track drop or skip wins over a reader's own refusal A reader's fill refusal and a later track verdict on the same sequences folded in sort order, so the refusal's error hid a declared skip. Clip the refusals to what the track's drops leave, as Rust's Misses does. Co-Authored-By: Claude Opus 5.5 --- js/net/src/fill.test.ts | 21 +++++++++++++++++++++ js/net/src/track.ts | 8 ++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/js/net/src/fill.test.ts b/js/net/src/fill.test.ts index 75da4cff371..9720c7e88e5 100644 --- a/js/net/src/fill.test.ts +++ b/js/net/src/fill.test.ts @@ -324,6 +324,27 @@ test("fill: a group arriving after a refusal reaches the reader the refusal name broadcast.close(); }); +test("fill: the track's skip of a run a reader's fill refused is what the reader reports", async () => { + const { broadcast, track } = resumed(4n); + const requests = track.requests(); + const reader = track.subscribe({ + ranges: [{ start: { absolute: 0n }, end: { absolute: 3n } }], + delay: { max: null }, + }); + const first = reader.recvGroup(); + const fill = await requests.requestedFill(); + if (!fill) throw new Error("no fill"); + fill.refuseGroups(0n, 2n, new StreamError(StreamCode.Cancel, { message: "storage busy" })); + // The publisher declares the refused run never produced; its verdict wins, as in Rust. + track.skipGroups(0n, 2n); + expect(hooks.tryDrop(reader)).toMatchObject({ kind: "drop", start: 0n, end: 2n, skip: true }); + + fill.close(); + requests.close(); + broadcast.close(); + await first.catch(() => undefined); +}); + test("fill: a fill names the most urgent priority and loosest budget among its readers", async () => { const { broadcast, track } = resumed(5n); const requests = track.requests(); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 721ba948abf..b2608589aae 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -892,11 +892,15 @@ function sameVerdict(a: DropRun, b: DropRun): boolean { return sameHole({ ...a, received: undefined }, { ...b, received: undefined }); } -// The runs of `drops` and a reader's `refused` overlapping `[start, end)`, lowest first. +// The runs of `drops` and a reader's `refused` overlapping `[start, end)`, lowest first; a +// track drop or skip hides the refusals it covers. function givenUp(drops: DropSet, refused: readonly DropRun[], start: bigint, end: bigint): DropRun[] { const found = drops.overlapping(start, end); if (refused.length === 0) return found; - found.push(...refused.filter((run) => run.start < end && start < run.end)); + // The track's verdict wins where it covers the reader's own refusal, as in Rust. + let own = refused.filter((run) => run.start < end && start < run.end); + for (const run of found) own = removeRun(own, run.start, run.end); + found.push(...own); return found.sort((a, b) => (a.start < b.start ? -1 : a.start > b.start ? 1 : 0)); } From 36663c63e5920ea2f1ec8785b8f9858cb8d0cce0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 21:47:19 -0800 Subject: [PATCH 09/17] fix(js/net): only a drop that settles the reader hides its refusal A stale verdict the reader's budget or coverage rejects is no verdict for it, so it must not clip the reader's own fill refusal, as Rust's Misses::unclaimed gates on drift.settles. Co-Authored-By: Claude Opus 5.5 --- js/net/src/fill.test.ts | 21 +++++++++++++++++++++ js/net/src/track.ts | 36 +++++++++++++++++++++++++----------- 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/js/net/src/fill.test.ts b/js/net/src/fill.test.ts index 9720c7e88e5..8bc663e5de8 100644 --- a/js/net/src/fill.test.ts +++ b/js/net/src/fill.test.ts @@ -813,6 +813,27 @@ test("fill: a stale run that does not settle the reader is still asked for", asy broadcast.close(); }); +test("fill: a stale run that does not settle the reader leaves its own refusal reported", async () => { + const { broadcast, track } = resumed(2n); + const requests = track.requests(); + hooks.dropTrackGroups(track, 0n, 2n, staleHole(60_000)); + const reader = track.subscribe({ + ranges: [{ start: { absolute: 0n }, end: { absolute: 2n } }], + delay: { max: Milli(500) }, + }); + const first = reader.recvGroup(); + const fill = await requests.requestedFill(); + if (!fill) throw new Error("no fill"); + const refusal = new StreamError(StreamCode.Cancel, { message: "storage busy" }); + fill.refuseGroups(0n, 2n, refusal); + expect(hooks.tryDrop(reader)).toMatchObject({ kind: "drop", start: 0n, end: 2n, error: refusal }); + + fill.close(); + requests.close(); + broadcast.close(); + await first.catch(() => undefined); +}); + // The upstream judged 0..2 and 4..6 under 500ms, and gave up both of them as stale. function judgedTwice() { const broadcast = new BroadcastProducer(); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index b2608589aae..672c287638c 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -892,14 +892,22 @@ function sameVerdict(a: DropRun, b: DropRun): boolean { return sameHole({ ...a, received: undefined }, { ...b, received: undefined }); } -// The runs of `drops` and a reader's `refused` overlapping `[start, end)`, lowest first; a -// track drop or skip hides the refusals it covers. -function givenUp(drops: DropSet, refused: readonly DropRun[], start: bigint, end: bigint): DropRun[] { +// The runs of `drops` and a reader's `refused` overlapping `[start, end)`, lowest first. A drop +// that `settles` the reader, judged from where it meets `start`, hides the refusals it covers: +// the track's verdict wins, as in Rust, but a stale one the reader rejects is no verdict for it. +function givenUp( + drops: DropSet, + refused: readonly DropRun[], + start: bigint, + end: bigint, + settles: (run: DropRun) => boolean, +): DropRun[] { const found = drops.overlapping(start, end); if (refused.length === 0) return found; - // The track's verdict wins where it covers the reader's own refusal, as in Rust. let own = refused.filter((run) => run.start < end && start < run.end); - for (const run of found) own = removeRun(own, run.start, run.end); + for (const run of found) { + if (settles({ ...run, start: max(run.start, start) })) own = removeRun(own, run.start, run.end); + } found.push(...own); return found.sort((a, b) => (a.start < b.start ? -1 : a.start > b.start ? 1 : 0)); } @@ -3080,7 +3088,9 @@ export class Subscriber { } for (const run of spans.flatMap((span) => this.#wants(span.start, span.end))) { const gone = [...accounted]; - for (const dropped of givenUp(drops, refused.peek(), run.start, run.end)) { + for (const dropped of givenUp(drops, refused.peek(), run.start, run.end, (run) => + settles(run, budget, coverage, this.#state.judged), + )) { // Judged from where it meets the asked run, as the reader would read it. const clipped = { ...dropped, start: max(dropped.start, run.start) }; if (settles(clipped, budget, coverage, this.#state.judged)) @@ -3227,7 +3237,9 @@ export class Subscriber { for (const delivered of taken) gaps = gaps.flatMap((gap) => delivered.gaps(gap.start, gap.end)); for (const gap of gaps) { // Each run judged from where it meets the gap, as the reader would read it. - const overlapping = givenUp(all, refused.peek(), gap.start, gap.end).map((run) => ({ + const overlapping = givenUp(all, refused.peek(), gap.start, gap.end, (run) => + settles(run, budget, coverage, this.#state.judged), + ).map((run) => ({ ...run, start: max(run.start, gap.start), end: min(run.end, gap.end), @@ -3931,7 +3943,9 @@ export class Subscriber { if (!full) { this.#pending = this.#pending.flatMap((run) => (run.received ? removeRun([run], from, to) : [run])); } - const folded = full ? givenUp(drops, refused.peek(), from, to) : drops.overlapping(from, to); + const folded = full + ? givenUp(drops, refused.peek(), from, to, (run) => settles(run, budget, coverage, judged)) + : drops.overlapping(from, to); for (const dropped of folded) { // Judged from where it meets this cursor's span, as Rust clips it. const run = { ...dropped, start: max(dropped.start, from), end: min(dropped.end, to) }; @@ -3946,9 +3960,9 @@ export class Subscriber { // only the part it still holds as it was folded in is owed. if (run.received) { const folded = run; - const held = givenUp(drops, refused.peek(), run.start, run.end).find((given) => - sameVerdict(given, folded), - ); + const held = givenUp(drops, refused.peek(), run.start, run.end, (run) => + settles(run, budget, coverage, judged), + ).find((given) => sameVerdict(given, folded)); if (!held) continue; if (held.end < run.end) this.#pending.unshift({ ...run, start: held.end }); run = { ...run, start: max(run.start, held.start), end: min(run.end, held.end) }; From ca0de2b3e05ef4be5df1d1ad072255a902ac3204 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 22:20:09 -0800 Subject: [PATCH 10/17] fix(moq-net): hold the drop-set bench's run and record counts steady Each measured drop added a run to the track and to every reader's record, so across warm-up and sampling the configured cardinalities grew without bound. The fixture now rebuilds the track (untimed) once its runs grow by a sixteenth, and otherwise resubscribes the readers once their records do, followed by one untimed warm drop so the timed one carries no first-drop cost. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/benches/track.rs | 12 ++++++- rs/moq-net/src/fuzz.rs | 69 +++++++++++++++++++++++++++---------- 2 files changed, 62 insertions(+), 19 deletions(-) diff --git a/rs/moq-net/benches/track.rs b/rs/moq-net/benches/track.rs index 058048e8120..6edf7367edd 100644 --- a/rs/moq-net/benches/track.rs +++ b/rs/moq-net/benches/track.rs @@ -931,7 +931,17 @@ fn bench_subscriber_churn_drops(c: &mut Criterion) { let id = format!("{readers}r/{runs}n/{reported}k/span_{span}"); group.bench_function(BenchmarkId::from_parameter(id), |b| { let mut set = moq_net::fuzz::DropSet::new(readers, runs, reported, span); - b.iter(|| set.drop_one()); + b.iter_custom(|iterations| { + let mut elapsed = Duration::ZERO; + for _ in 0..iterations { + // Untimed: restore the runs and records each drop adds. + set.hold(); + let start = Instant::now(); + set.drop_one(); + elapsed += start.elapsed(); + } + elapsed + }); }); } } diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index 1bca6c4702d..533304d3cae 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -38,8 +38,11 @@ pub struct DropSet { _broadcast: broadcast::Producer, track: track::Producer, readers: Vec, + shape: (usize, u64, u64, u64), + /// Runs added since the track was built, and since the readers subscribed. + added: u64, + fresh: u64, next: u64, - span: u64, waiter: kio::Waiter, } @@ -49,36 +52,66 @@ impl DropSet { pub fn new(readers: usize, runs: u64, reported: u64, span: u64) -> Self { let broadcast = broadcast::Producer::new(broadcast::Info::default()); let track = broadcast.create_track("drops", track::Info::default()).unwrap(); - let stride = span + 1; - let from = runs.saturating_sub(reported) * stride; - let mut readers: Vec<_> = (0..readers).map(|_| track.subscribe(None).with_range(from..)).collect(); - for run in 0..runs { - track.drop_groups(run * stride..run * stride + span, crate::StreamError::Old); + // One run short: subscribing adds it. + let built = runs.saturating_sub(1); + for run in 0..built { + track.drop_groups(run * (span + 1)..run * (span + 1) + span, crate::StreamError::Old); } - let waiter = kio::Waiter::noop(); - for reader in &mut readers { - while reader.poll_drop(&waiter).is_ready() {} - } - Self { + let mut set = Self { _broadcast: broadcast, track, - readers, - next: runs * stride, - span, - waiter, + readers: Vec::new(), + shape: (readers, runs, reported, span), + added: 0, + fresh: 0, + next: built * (span + 1), + waiter: kio::Waiter::noop(), + }; + set.subscribe(); + set.added = 0; + set + } + + /// Replace every reader with one holding the newest `reported` runs, each reported. + /// + /// The newest run is dropped after they subscribe, so the first drop a caller times + /// does not carry the one-off cost of a fresh track or cursor. + fn subscribe(&mut self) { + let (readers, _, reported, span) = self.shape; + self.readers.clear(); + let from = self.next.saturating_sub(reported.saturating_sub(1) * (span + 1)); + for _ in 0..readers { + let mut reader = self.track.subscribe(None).with_range(from..); + while reader.poll_drop(&self.waiter).is_ready() {} + self.readers.push(reader); + } + self.drop_one(); + self.fresh = 0; + } + + /// Restore the counts `new` was given before either grows by more than a sixteenth: + /// rebuild the track once its runs have, otherwise resubscribe once the readers' have. + pub fn hold(&mut self) { + let (readers, runs, reported, span) = self.shape; + if self.added >= (runs / 16).max(1) { + *self = Self::new(readers, runs, reported, span); + } else if self.fresh >= (reported / 16).max(1) { + self.subscribe(); } } /// Drop one more run above the rest, and have every reader report it. pub fn drop_one(&mut self) { + let span = self.shape.3; let start = self.next; - self.next += self.span + 1; - self.track - .drop_groups(start..start + self.span, crate::StreamError::Old); + self.next += span + 1; + self.track.drop_groups(start..start + span, crate::StreamError::Old); for reader in &mut self.readers { assert!(reader.poll_drop(&self.waiter).is_ready(), "the new run is owed"); assert!(reader.poll_drop(&self.waiter).is_pending(), "nothing else is"); } + self.added += 1; + self.fresh += 1; } } From 101be34462ee9b1faccd6ac675bff6902878d2d6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 22:42:08 -0800 Subject: [PATCH 11/17] fix(moq-net): address review of the drop set's billing and folds - A fill source's refusals settle the pool's debt like any drop, so a source refusing in fragments pays for the runs it records. - JS forgets expired runs in one pass: runs given up in the same quarter of the idle window expire together, and one splice each was quadratic. - The JS wire cursor's incremental fold keeps the reader's own pending refusals over a changed span, as the full fold does. - A resumed group's verdict lets a stale drop hide the reader's own refusal only when it settles that reader. - The quest records two gaps forgetting a run leaves: a reader that already judged it, and a received copy's tail completion. Co-Authored-By: Claude Opus 5.5 --- js/net/src/drops.test.ts | 35 +++++++++++++++++++++++++ js/net/src/drops.ts | 41 +++++++++++++++++++++++++++++- js/net/src/fill.test.ts | 24 +++++++++++++++++ js/net/src/track.ts | 28 +++++++++++++++----- quest/m1/drop-set.md | 18 +++++++++++-- rs/moq-net/src/model/track.rs | 31 ++++++++++++++++++++++ rs/moq-net/src/model/track/fill.rs | 4 +++ 7 files changed, 172 insertions(+), 9 deletions(-) create mode 100644 js/net/src/drops.test.ts diff --git a/js/net/src/drops.test.ts b/js/net/src/drops.test.ts new file mode 100644 index 00000000000..ff47931e9aa --- /dev/null +++ b/js/net/src/drops.test.ts @@ -0,0 +1,35 @@ +import { expect, test } from "bun:test"; +import { Runs } from "./drops.ts"; +import type { Run } from "./internal.ts"; + +type Tagged = Run & { tag: number }; + +function filled(): Runs { + const runs = new Runs((a, b) => a.tag === b.tag); + for (let i = 0n; i < 8n; i++) runs.set(i * 10n, i * 10n + 5n, { start: 0n, end: 0n, tag: Number(i % 2n) }); + return runs; +} + +test("drops: clearing ranges in one pass matches clearing each", () => { + // Whole runs, a run split in two, a prefix, a suffix, and a range spanning runs and gaps. + const ranges: Run[] = [ + { start: 0n, end: 5n }, + { start: 11n, end: 13n }, + { start: 20n, end: 22n }, + { start: 24n, end: 31n }, + { start: 42n, end: 65n }, + ]; + const each = filled(); + for (const { start, end } of ranges) each.set(start, end); + const bulk = filled(); + const { removed, added } = bulk.clear(ranges); + expect(bulk.all).toEqual(each.all); + expect(removed.map((run) => run.start)).toEqual([0n, 10n, 20n, 30n, 40n, 50n, 60n]); + expect(added.map((run) => [run.start, run.end])).toEqual([ + [10n, 11n], + [13n, 15n], + [22n, 24n], + [31n, 35n], + [40n, 42n], + ]); +}); diff --git a/js/net/src/drops.ts b/js/net/src/drops.ts index ccf7b14fbdc..3cff4ae6c2a 100644 --- a/js/net/src/drops.ts +++ b/js/net/src/drops.ts @@ -74,6 +74,37 @@ export class Runs { return { removed, added }; } + /** + * Clear each of `ranges`, sorted and disjoint, in one pass over the set rather than one + * {@link set} each. Returns what {@link set} would. + */ + clear(ranges: readonly Run[]): { removed: R[]; added: R[] } { + const removed: R[] = []; + const added: R[] = []; + const kept: R[] = []; + let next = 0; + for (const run of this.#runs) { + while (next < ranges.length && ranges[next].end <= run.start) next++; + if (next === ranges.length || ranges[next].start >= run.end) { + kept.push(run); + continue; + } + removed.push(run); + // What the ranges leave of it. A removal never meets an alike neighbour, so nothing merges. + const pieces: R[] = []; + let from = run.start; + for (let i = next; i < ranges.length && ranges[i].start < run.end; i++) { + if (ranges[i].start > from) pieces.push({ ...run, start: from, end: ranges[i].start }); + if (ranges[i].end > from) from = ranges[i].end; + } + if (from < run.end) pieces.push({ ...run, start: from, end: run.end }); + kept.push(...pieces); + added.push(...pieces); + } + this.#runs = kept; + return { removed, added }; + } + // The index of the first run ending past `sequence`. #above(sequence: bigint): number { let lo = 0; @@ -133,7 +164,15 @@ export class DropSet { /** Give up `[start, end)` as `run` says, or forget it when `run` is undefined. */ set(start: bigint, end: bigint, run?: Omit): void { - const { removed, added } = this.#runs.set(start, end, run && { ...run, start, end }); + this.#apply(this.#runs.set(start, end, run && { ...run, start, end })); + } + + /** Forget each of `ranges`, sorted and disjoint, in one pass. */ + clear(ranges: readonly Run[]): void { + this.#apply(this.#runs.clear(ranges)); + } + + #apply({ removed, added }: { removed: DropRun[]; added: DropRun[] }): void { if (removed.length === 0 && added.length === 0) return; for (const old of removed) this.#live.delete(old); for (const fresh of added) { diff --git a/js/net/src/fill.test.ts b/js/net/src/fill.test.ts index 8bc663e5de8..b04aa2215b6 100644 --- a/js/net/src/fill.test.ts +++ b/js/net/src/fill.test.ts @@ -834,6 +834,30 @@ test("fill: a stale run that does not settle the reader leaves its own refusal r await first.catch(() => undefined); }); +test("fill: a stale run that does not settle the reader keeps its pending refusals", async () => { + const { broadcast, track } = resumed(6n); + const requests = track.requests(); + const reader = track.subscribe({ + ranges: [{ start: { absolute: 0n }, end: { absolute: 6n } }], + delay: { max: Milli(500) }, + }); + const first = reader.recvGroup(); + const fill = await requests.requestedFill(); + if (!fill) throw new Error("no fill"); + const refusal = new StreamError(StreamCode.Cancel, { message: "storage busy" }); + fill.refuseGroups(0n, 1n, refusal); + fill.refuseGroups(3n, 4n, refusal); + expect(hooks.tryDrop(reader)).toMatchObject({ start: 0n, end: 1n, error: refusal }); + // Folded in as a change, over the refusal still to report. + hooks.dropTrackGroups(track, 2n, 5n, staleHole(60_000)); + expect(hooks.tryDrop(reader)).toMatchObject({ start: 3n, end: 4n, error: refusal }); + + fill.close(); + requests.close(); + broadcast.close(); + await first.catch(() => undefined); +}); + // The upstream judged 0..2 and 4..6 under 500ms, and gave up both of them as stale. function judgedTwice() { const broadcast = new BroadcastProducer(); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index e3c515a1fd3..254c153e3e3 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -1146,6 +1146,11 @@ export class Producer { this.#dropsBorn.set(start, end, { start, end, expires }); this.#dropsOldest = Math.min(this.#dropsOldest ?? expires, expires); } + this.#billDrops(); + } + + // Bill the drop set and its expiry index to the account as they now stand. + #billDrops(): void { const billed = this.#state.drops.size * cache.RUN_OVERHEAD + this.#dropsBorn.size * cache.BORN_OVERHEAD; if (billed > this.#dropsBilled) this.#account.bill(billed - this.#dropsBilled); else this.#account.unbill(this.#dropsBilled - billed); @@ -1176,7 +1181,12 @@ export class Producer { const expired = this.#dropsBorn.all .filter((run) => now >= run.expires && run.start < kept) .map(({ start, end }) => ({ start, end: min(end, kept) })); - for (const run of expired) this.#setDrops(run.start, run.end); + // In one pass: runs given up in the same quarter of the window expire together. + if (expired.length > 0) { + this.#state.drops.clear(expired); + this.#dropsBorn.clear(expired); + this.#billDrops(); + } this.#dropsOldest = undefined; for (const run of this.#dropsBorn.all) { if (run.start < kept) this.#dropsOldest = Math.min(this.#dropsOldest ?? run.expires, run.expires); @@ -3360,9 +3370,16 @@ export class Subscriber { return held; }; hooks.givenUp = (subscriber, sequence) => { - const { drops, refused } = subscriber.#state; + const { drops, refused, judged } = subscriber.#state; + const drop = drops.at(sequence); + const own = dropAt(refused.peek(), sequence); + // As in `givenUp`: only a drop that settles the reader hides its own refusal. + const wins = + drop && + (own === undefined || + settles({ ...drop, start: sequence }, subscriber.#drift().budget, subscriber.#coverage(), judged)); return { - error: drops.at(sequence)?.error ?? dropAt(refused.peek(), sequence), + error: wins ? drop.error : own, changed: [drops.version, refused], }; }; @@ -3921,9 +3938,8 @@ export class Subscriber { if (!full) { this.#pending = this.#pending.flatMap((run) => (run.received ? removeRun([run], from, to) : [run])); } - const folded = full - ? givenUp(drops, refused.peek(), from, to, (run) => settles(run, budget, coverage, judged)) - : drops.overlapping(from, to); + // The reader's own refusals too: clearing the span above took out those it covered. + const folded = givenUp(drops, refused.peek(), from, to, (run) => settles(run, budget, coverage, judged)); for (const dropped of folded) { // Judged from where it meets this cursor's span, as Rust clips it. const run = { ...dropped, start: max(dropped.start, from), end: min(dropped.end, to) }; diff --git a/quest/m1/drop-set.md b/quest/m1/drop-set.md index 40c9a852870..b3a2f1ba0f6 100644 --- a/quest/m1/drop-set.md +++ b/quest/m1/drop-set.md @@ -57,8 +57,8 @@ because runs keyed by group sequence convert mechanically to starts. - **CPU per drop.** Every report inserts into the per-cursor `RangeSet`s and sorted arrays, so `track_subscriber_churn_drops` in `rs/moq-net/benches/track.rs` slows once many readers each hold many - reported runs (256 readers holding 65,536 each: about 1 to 4 ms per drop - against 0.16 ms with one each). JS `#askFills` still walks every run above + reported runs (256 readers holding 65,536 each: about 0.75 ms per drop + against 0.18 ms with one each). JS `#askFills` still walks every run above the edge on each probe while a fill handler serves the track; mirror Rust's `Misses::unknown` memo there. @@ -88,6 +88,20 @@ Still to cover, beyond the memory work: as a budget-independent drop, including OLD judged under that fill's own terms. Record it as a stale run, so it settles only the readers it covers. +- **A forgotten run a reader already judged.** Found 2026-11-09 by review + on moq-dev/moq#8294: `Misses::judge` marks a dropped run judged, and only a + change of terms rejudges what it settled, through the runs the track still + holds. A run shed or expired after a reader judged it, but before the reader + read past it, becomes a hole that reader never asks for or gives up, so an + ordered reader behind it can wait on it. Rejudge what a forgotten run + leaves, without a full rescan per shed. +- **A received copy's tail completion.** Found the same day: once a received + copy's declared end is reached, `owes` counts a gap the drop set covered as + delivered, so forgetting that run can reopen the tail after a reader saw + the end, and under an unbounded budget hold it. Keep the completion once + reached, rather than keep the runs, which a peer declaring a far end would + make unbounded. + Tests, on mock time through a relay, in Rust and mirrored in `js/net`: - After a widening from B1 to B2 for a group the upstream no longer holds, an diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 8a097ab5507..82e229138d9 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -23761,6 +23761,37 @@ mod test { assert!(local.state.read().skipped(5), "a local skip above the edge stays"); } + /// A source refusing a fill in disjoint groups pays for the runs it records, as a drop does. + #[test] + #[cfg(feature = "async")] + fn a_fragmented_fill_refusal_stays_within_the_pool() { + const RUNS: u64 = 1000; + const START: u64 = 2 * RUNS; + let capacity = 64 * (RUN_OVERHEAD + BORN_OVERHEAD); + let (mut producer, pool) = pooled_producer(capacity); + let dynamic = producer.requests(); + let mut subscriber = producer.subscribe(replay().with_range(0..)); + producer.start_at(START).unwrap(); + write_group(&producer, START); + assert_eq!(drain_groups(&mut subscriber), [START]); + let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&kio::Waiter::noop()) else { + panic!("the miss below the start is asked"); + }; + // Nothing arrives once the feed ends, so a refusal is a verdict on the track. + producer.finish().unwrap(); + for run in 0..RUNS { + fill.drop_groups(2 * run..2 * run + 1); + } + let held = producer.state.read().drops.len() as u64; + assert!(held > 0, "the refusals are recorded"); + assert!(held < RUNS, "the lowest runs were shed"); + assert!( + pool.used() <= capacity + 2 * (RUN_OVERHEAD + BORN_OVERHEAD) + 1024, + "{}", + pool.used() + ); + } + /// A drop-only flood of disjoint runs bills the pool and sheds its lowest runs to stay /// within it, with no cached group to evict; a shed run is unknown again, so a wire cursor /// reports only the runs the track still holds. diff --git a/rs/moq-net/src/model/track/fill.rs b/rs/moq-net/src/model/track/fill.rs index dcc8d7b369b..f3ba94b7242 100644 --- a/rs/moq-net/src/model/track/fill.rs +++ b/rs/moq-net/src/model/track/fill.rs @@ -1470,6 +1470,10 @@ impl TrackState { for groups in dropped.iter() { self.drop_groups(groups.clone(), super::Hole::drop(err.clone())); } + // The runs are billed, so a source refusing fills in fragments pays for them. + if !dropped.is_empty() { + self.charge_debt(); + } } } From 96a994fe1ecffd5c75601af9a636bdb91f61ec98 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 23:31:14 -0800 Subject: [PATCH 12/17] fix(moq-net): keep a reached end reached when its drops are reclaimed A received copy counts a dropped gap below its end as delivered, so forgetting that run under pool pressure or past the idle window made a reader that already saw the end wait on the gap again. The end now latches as reached before a reclaim forgets runs. JS bulk expiry appends a split run's pieces one by one, since a spread of a run split by many ranges can exceed the argument limit. Co-Authored-By: Claude Opus 5.5 --- js/net/src/drops.ts | 7 ++-- quest/m1/drop-set.md | 6 ---- rs/moq-net/src/model/track.rs | 60 ++++++++++++++++++++++++++++++++--- 3 files changed, 61 insertions(+), 12 deletions(-) diff --git a/js/net/src/drops.ts b/js/net/src/drops.ts index 3cff4ae6c2a..d741f65d9a1 100644 --- a/js/net/src/drops.ts +++ b/js/net/src/drops.ts @@ -98,8 +98,11 @@ export class Runs { if (ranges[i].end > from) from = ranges[i].end; } if (from < run.end) pieces.push({ ...run, start: from, end: run.end }); - kept.push(...pieces); - added.push(...pieces); + // One by one: a run split by many ranges would overflow a spread's arguments. + for (const piece of pieces) { + kept.push(piece); + added.push(piece); + } } this.#runs = kept; return { removed, added }; diff --git a/quest/m1/drop-set.md b/quest/m1/drop-set.md index b3a2f1ba0f6..fb5533e65cb 100644 --- a/quest/m1/drop-set.md +++ b/quest/m1/drop-set.md @@ -95,12 +95,6 @@ Still to cover, beyond the memory work: read past it, becomes a hole that reader never asks for or gives up, so an ordered reader behind it can wait on it. Rejudge what a forgotten run leaves, without a full rescan per shed. -- **A received copy's tail completion.** Found the same day: once a received - copy's declared end is reached, `owes` counts a gap the drop set covered as - delivered, so forgetting that run can reopen the tail after a reader saw - the end, and under an unbounded budget hold it. Keep the completion once - reached, rather than keep the runs, which a peer declaring a far end would - make unbounded. Tests, on mock time through a relay, in Rust and mirrored in `js/net`: diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 82e229138d9..2a03ffa4ed3 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -407,6 +407,10 @@ struct End { /// clean end stands for good. Atomic so reporting it latches through a read guard; see /// [`TrackState::latch_settled`]. settled: AtomicBool, + /// Whether the end was reached when a reclaim forgot runs of the drop set, which keeps it + /// reached: a received copy's tail counts a dropped gap as delivered, so forgetting the + /// run would make a reader that saw the end wait on it again. + reached: AtomicBool, } impl End { @@ -1454,6 +1458,7 @@ impl TrackState { if !self.drops_expired(now) { return; } + self.latch_reached(); let kept = self.kept_drops(); let expired: Vec<_> = self .drops_born @@ -1472,6 +1477,15 @@ impl TrackState { .min(); } + /// Keep a reached end reached before a reclaim forgets runs; see [`End::reached`]. + fn latch_reached(&self) { + if let Outcome::Ending(end) = &self.outcome + && self.is_complete() + { + end.reached.store(true, Ordering::Relaxed); + } + } + /// Where the runs a reclaim may forget end: every run on a received copy, whose tail /// decides its end, but only those below a local track's edge, since one above it is what /// lets its declared end be reached (see [`Self::reaches`]). A local producer gives up only @@ -1502,6 +1516,9 @@ impl TrackState { fn shed_drops(&mut self, cap: u64) -> u64 { let mut paid = 0u64; let kept = self.kept_drops(); + if self.debt > 0 && cap > 0 { + self.latch_reached(); + } while self.debt > 0 && paid < cap { let Some(run) = self .drops @@ -3040,6 +3057,7 @@ impl TrackState { error, tail, settled: AtomicBool::new(settled), + reached: AtomicBool::new(false), }); self.cache.declare_end(); self.cache.wakes().arrived(fin..); @@ -3091,10 +3109,13 @@ impl TrackState { /// still open, so the track was cut off rather than ended. fn is_complete(&self) -> bool { match &self.outcome { - Outcome::Ending(end) => end - .at - .final_sequence() - .is_some_and(|fin| self.reaches(fin) && !self.awaits_tail(fin)), + Outcome::Ending(end) => { + end.reached.load(Ordering::Relaxed) + || end + .at + .final_sequence() + .is_some_and(|fin| self.reaches(fin) && !self.awaits_tail(fin)) + } // A declared end the last producer dropped short of. Outcome::Sealed(_) => true, // An abort wins unless the end had already settled: a group below it was still open. @@ -23761,6 +23782,37 @@ mod test { assert!(local.state.read().skipped(5), "a local skip above the edge stays"); } + /// A received copy whose end a dropped gap let it reach stays ended once that run expires, + /// even for a reader whose budget outlasts the run, which would otherwise wait on the gap. + #[test] + #[cfg(feature = "async")] + fn a_reached_end_outlives_its_expired_drops() { + let patient = || { + Subscription::default() + .with_range(0..) + .with_delay(Delay::default().with_max(Duration::from_secs(3600))) + }; + let pool = cache::Pool::new(cache::Config::default().with_expiry(Duration::from_secs(10))); + let mut producer = track_producer_pooled("test", pool.clone()); + producer.received(); + let mut subscriber = producer.subscribe(patient()); + write_group(&producer, 0); + producer.drop_groups(1..2, StreamError::Old); + write_group(&producer, 2); + producer.finish_at(Position::group(3), None).unwrap(); + drain_groups(&mut subscriber); + let waiter = kio::Waiter::noop(); + assert!(matches!(subscriber.poll_recv_group(&waiter), Poll::Ready(Ok(None)))); + + elapse(&producer, Duration::from_secs(40)); + pool.sweep(); + assert!(producer.state.read().drops.is_empty(), "the run expired"); + assert!(matches!(subscriber.poll_recv_group(&waiter), Poll::Ready(Ok(None)))); + let mut late = producer.subscribe(patient()); + drain_groups(&mut late); + assert!(matches!(late.poll_recv_group(&waiter), Poll::Ready(Ok(None)))); + } + /// A source refusing a fill in disjoint groups pays for the runs it records, as a drop does. #[test] #[cfg(feature = "async")] From dc0a7e8a9a34ee403f988f45248b0761f0fdd4f0 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 10:20:46 -0800 Subject: [PATCH 13/17] fix(net): account a reclaimed drop run in the arrivals, and rejudge forgotten runs A reclaim (shed or expiry) now accounts the forgotten run in the arrivals ledger, as an evicted group stays, instead of latching End::reached: the tail never owes it again, so a refined end and abort settlement keep a reached end. A run touching no accounted one takes the gap below it, so a fragment flood never grows the ledger. Misses::judge folds a bounded forget log to judge again what it judged inside a forgotten run. Co-Authored-By: Claude Opus 5.5 --- js/net/src/cache.test.ts | 38 +++++- js/net/src/ledger.ts | 5 + js/net/src/track.ts | 30 ++++- quest/m1/drop-set.md | 11 +- rs/moq-net/src/ledger.rs | 5 + rs/moq-net/src/model/track.rs | 198 ++++++++++++++++++++++------- rs/moq-net/src/model/track/miss.rs | 79 ++++++++++++ 7 files changed, 306 insertions(+), 60 deletions(-) diff --git a/js/net/src/cache.test.ts b/js/net/src/cache.test.ts index e4558659d5d..03c1f379be3 100644 --- a/js/net/src/cache.test.ts +++ b/js/net/src/cache.test.ts @@ -2,7 +2,7 @@ import { expect, test } from "bun:test"; import { Manual, Milli, Nano, yieldNow } from "@moq/clock"; import * as cache from "./cache.ts"; import { StreamCode, StreamError, TooFarBehind } from "./error.ts"; -import { hooks } from "./internal.ts"; +import { hooks, staleHole } from "./internal.ts"; import { Producer as TrackProducer } from "./track.ts"; // Run `fn` on a manual clock, stepped in milliseconds. @@ -371,3 +371,39 @@ test("a kept local skip expires once the edge passes it or the track closes", as expect(pool.used).toBe(used - cache.RUN_OVERHEAD - cache.BORN_OVERHEAD); }); }); + +// A received stale run the pool forgets is accounted for, as an evicted group is: no reader waits +// on it from the live feed, so a reader that wants it asks a fill. Mirrors Rust +// `a_reader_asks_a_fill_for_a_forgotten_run_it_judged`. +test("a forgotten stale run is asked of a fill", async () => { + await withClock(async (clock) => { + const pool = new cache.Pool({ expiry: Milli(1_000) }); + const producer = pooled(pool); + hooks.receiveTrack(producer); + const requests = producer.requests(); + producer.createGroup(0n).finish(); + hooks.dropTrackGroups(producer, 1n, 3n, staleHole(500)); + producer.createGroup(3n).finish(); + await clock.advance(2_000); + + const reader = producer.subscribe({ + ranges: [{ start: { absolute: 1n } }], + delay: { max: Milli(60_000) }, + order: "ascending", + }); + // The live group first, in arrival order; the next read asks for what it misses. + expect((await reader.recvGroup())?.sequence).toBe(3n); + const next = reader.recvGroup(); + const asked: bigint[][] = []; + for (;;) { + const fill = await Promise.race([requests.requestedFill(), yieldNow().then(() => undefined)]); + if (!fill) break; + asked.push([fill.start, fill.end]); + } + expect(asked).toEqual([[1n, 3n]]); + + reader.close(); + requests.close(); + await next.catch(() => undefined); + }); +}); diff --git a/js/net/src/ledger.ts b/js/net/src/ledger.ts index 0044625ba3e..9b3db33ba53 100644 --- a/js/net/src/ledger.ts +++ b/js/net/src/ledger.ts @@ -122,6 +122,11 @@ export class Ledger { return run !== undefined && sequence < run.end ? run.end : undefined; } + /** The exclusive end of the highest run starting at or below `sequence`. */ + endBelow(sequence: bigint): bigint | undefined { + return this.#runs.getPairOrNextLower(sequence)?.[1].end; + } + /** Whether every sequence in `[start, end)` is accounted for. */ covers(start: bigint, end: bigint): boolean { if (start >= end) return true; diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 254c153e3e3..5321095983f 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -1077,8 +1077,8 @@ export class Producer { #wire?: Wire; // The misses its readers asked of a {@link Requests} handle. #fills: FillState; - // The sequences the live feed delivered, as a group: what a pending tail is - // judged by. Eviction never touches it. A gap folds away a subscription's grace after a + // The sequences the live feed delivered, as a group or a drop, and the stale runs a reclaim + // forgot: what a pending tail is judged by. Eviction never touches it. A gap folds away a subscription's grace after a // later arrival opened it, as the session's tail does, so this holds only the gaps still // worth waiting for. #arrivals = new Ledger(); @@ -1167,7 +1167,9 @@ export class Producer { const run = this.#state.drops.all[0]; if (!run || run.start >= kept) break; const billed = this.#dropsBilled; - this.#setDrops(run.start, min(run.end, kept)); + const end = min(run.end, kept); + this.#setDrops(run.start, end); + this.#forgot([{ start: run.start, end }]); paid += billed - this.#dropsBilled; } return paid; @@ -1186,6 +1188,7 @@ export class Producer { this.#state.drops.clear(expired); this.#dropsBorn.clear(expired); this.#billDrops(); + this.#forgot(expired); } this.#dropsOldest = undefined; for (const run of this.#dropsBorn.all) { @@ -1194,6 +1197,27 @@ export class Producer { return expired.length > 0; } + // Account `forgotten`, runs a reclaim took out of the drop set, in the arrivals as an evicted + // group stays: the publisher accounted for them, so neither the tail nor a reader waits on + // them again, and a reader that wants one asks a fill or judges it by its budget. A reclaim + // never adds a run to the arrivals: a run touching no accounted one takes the gap below it + // too, given up early as a fold would. Nothing below the tail's floor is accounted, since + // accounting it would lower the floor onto gaps nobody waited for. Mirrors the Rust + // `TrackState::forget_drops`. + #forgot(forgotten: readonly Run[]): void { + const feed = this.#feedStart(); + for (const run of forgotten) { + const floor = (feed === "pending" ? this.#wire?.served?.requested : feed) ?? this.#arrivals.first() ?? 0n; + let start = max(run.start, floor); + if (start >= run.end) continue; + // The highest run starting at or below the run's end touches it when it ends at or past + // its start; otherwise it is the run below. + const below = this.#arrivals.endBelow(run.end); + if (below === undefined || below < start) start = below === undefined ? floor : max(below, floor); + this.#arrive(start, run.end); + } + } + // Where the runs a reclaim may forget end: every run on a received copy, but only those below an // open local track's edge, since one above it is what lets its declared end be reached (see // `reaches`). A local producer gives up only what it chose to. Mirrors the Rust `kept_drops`, diff --git a/quest/m1/drop-set.md b/quest/m1/drop-set.md index fb5533e65cb..0164b616415 100644 --- a/quest/m1/drop-set.md +++ b/quest/m1/drop-set.md @@ -60,7 +60,8 @@ because runs keyed by group sequence convert mechanically to starts. reported runs (256 readers holding 65,536 each: about 0.75 ms per drop against 0.18 ms with one each). JS `#askFills` still walks every run above the edge on each probe while a fill handler serves the track; mirror Rust's - `Misses::unknown` memo there. + `Misses::unknown` memo there, and have the memo judge again what the track + forgets, as Rust's `Misses::judge` folds the track's `Forgotten` log. Mirror each case in `js/net`. In `js/net`, a group a sink buffers keeps its source wanted only while @@ -88,14 +89,6 @@ Still to cover, beyond the memory work: as a budget-independent drop, including OLD judged under that fill's own terms. Record it as a stale run, so it settles only the readers it covers. -- **A forgotten run a reader already judged.** Found 2026-11-09 by review - on moq-dev/moq#8294: `Misses::judge` marks a dropped run judged, and only a - change of terms rejudges what it settled, through the runs the track still - holds. A run shed or expired after a reader judged it, but before the reader - read past it, becomes a hole that reader never asks for or gives up, so an - ordered reader behind it can wait on it. Rejudge what a forgotten run - leaves, without a full rescan per shed. - Tests, on mock time through a relay, in Rust and mirrored in `js/net`: - After a widening from B1 to B2 for a group the upstream no longer holds, an diff --git a/rs/moq-net/src/ledger.rs b/rs/moq-net/src/ledger.rs index 8d8bbe2da34..ab4cfb67a78 100644 --- a/rs/moq-net/src/ledger.rs +++ b/rs/moq-net/src/ledger.rs @@ -251,6 +251,11 @@ impl Ledger { .map(|(_, run)| run.end) } + /// The exclusive end of the highest accounted run starting at or below `sequence`. + pub fn end_below(&self, sequence: u64) -> Option { + self.runs.range(..=sequence).next_back().map(|(_, run)| run.end) + } + /// The accounted runs overlapping `groups`, clipped to it, lowest first. pub fn accounted(&self, groups: Range) -> impl Iterator> + '_ { // From the run holding the first group, if one does. diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 2a03ffa4ed3..2e5d57aa21c 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -407,10 +407,6 @@ struct End { /// clean end stands for good. Atomic so reporting it latches through a read guard; see /// [`TrackState::latch_settled`]. settled: AtomicBool, - /// Whether the end was reached when a reclaim forgot runs of the drop set, which keeps it - /// reached: a received copy's tail counts a dropped gap as delivered, so forgetting the - /// run would make a reader that saw the end wait on it again. - reached: AtomicBool, } impl End { @@ -634,8 +630,8 @@ pub(crate) struct TrackState { // last group does not lose where the track ended. outcome: Outcome, - // The sequences the live feed delivered: what a pending tail - // is judged by. Fetched copies never count, and eviction never touches it. A gap folds + // The sequences the live feed delivered, and the drops a reclaim forgot: what a pending + // tail is judged by. Fetched copies never count, and eviction never touches it. A gap folds // away a subscription's grace after a later arrival opened it, as the session's tail // does, so this holds only the gaps still worth waiting for. arrivals: Ledger, @@ -687,6 +683,8 @@ pub(crate) struct TrackState { // eviction debt sheds the lowest runs, and the expiry scan forgets those older than the // pool's window; either leaves their sequences unknown again. drops_bill: cache::Extra, + // The runs reclaimed from `drops`, for readers that judged them to judge again. + forgotten: miss::Forgotten, // How many copies the cache took in that the live feed didn't deliver: fetched backfill, // and fetched heads joining a live group, which may give it a start. Neither is in the // arrival ledger, so a time end parked on a gap looks again. @@ -1458,7 +1456,6 @@ impl TrackState { if !self.drops_expired(now) { return; } - self.latch_reached(); let kept = self.kept_drops(); let expired: Vec<_> = self .drops_born @@ -1467,7 +1464,7 @@ impl TrackState { .map(|(run, _)| run.start..run.end.min(kept)) .collect(); for run in expired { - self.change_drops(run, None); + self.forget_drops(run); } self.drops_oldest = self .drops_born @@ -1477,15 +1474,6 @@ impl TrackState { .min(); } - /// Keep a reached end reached before a reclaim forgets runs; see [`End::reached`]. - fn latch_reached(&self) { - if let Outcome::Ending(end) = &self.outcome - && self.is_complete() - { - end.reached.store(true, Ordering::Relaxed); - } - } - /// Where the runs a reclaim may forget end: every run on a received copy, whose tail /// decides its end, but only those below a local track's edge, since one above it is what /// lets its declared end be reached (see [`Self::reaches`]). A local producer gives up only @@ -1516,9 +1504,6 @@ impl TrackState { fn shed_drops(&mut self, cap: u64) -> u64 { let mut paid = 0u64; let kept = self.kept_drops(); - if self.debt > 0 && cap > 0 { - self.latch_reached(); - } while self.debt > 0 && paid < cap { let Some(run) = self .drops @@ -1529,7 +1514,7 @@ impl TrackState { break; }; let billed = self.drops_bill.size(); - self.change_drops(run, None); + self.forget_drops(run); let freed = billed.saturating_sub(self.drops_bill.size()); self.debt -= freed.min(self.debt); paid = paid.saturating_add(freed); @@ -1537,6 +1522,36 @@ impl TrackState { paid } + /// Forget `run` of the drop set to reclaim it, accounting it in the arrivals as an evicted + /// group stays: the publisher accounted for it, so neither the tail nor a reader waits on + /// it again, and a reader that wants it asks a fill or judges it by its budget. + /// + /// A reclaim never adds a run to the arrivals, so forgetting a flood of fragments costs + /// nothing there: a run touching no accounted one takes the gap below it too, given up + /// early as a fold would. Nothing below the tail's floor is accounted, since nothing waits + /// on it and accounting it would lower the floor onto gaps nobody waited for. + fn forget_drops(&mut self, run: ops::Range) { + self.change_drops(run.clone(), None); + self.forgotten.record(run.clone()); + let floor = self.tail_floor(); + let start = run.start.max(floor); + if start >= run.end { + return; + } + // The highest accounted run starting at or below the run's end touches it when it ends + // at or past its start; otherwise it is the run below. + let start = match self.arrivals.end_below(run.end) { + Some(end) if end >= start => start, + below => below.map_or(floor, |end| end.max(floor)), + }; + let now = self.cache.pool().now_wall(); + // Only a first run takes this as the age of the gap below it, which nothing opened yet: + // the tail's, so an end already reached stays reached. + let since = self.outcome.end().and_then(|end| end.tail).map_or(now, |tail| tail.min(now)); + self.arrivals.account(start..run.end, since); + self.cache.wakes().arrived(start..run.end); + } + /// Remove an aborted slot, keeping a cut as a drop: reclaiming it must not erase a /// disposition a relay still has to forward. fn reclaim_aborted(&mut self, sequence: u64) { @@ -3057,7 +3072,6 @@ impl TrackState { error, tail, settled: AtomicBool::new(settled), - reached: AtomicBool::new(false), }); self.cache.declare_end(); self.cache.wakes().arrived(fin..); @@ -3109,13 +3123,10 @@ impl TrackState { /// still open, so the track was cut off rather than ended. fn is_complete(&self) -> bool { match &self.outcome { - Outcome::Ending(end) => { - end.reached.load(Ordering::Relaxed) - || end - .at - .final_sequence() - .is_some_and(|fin| self.reaches(fin) && !self.awaits_tail(fin)) - } + Outcome::Ending(end) => end + .at + .final_sequence() + .is_some_and(|fin| self.reaches(fin) && !self.awaits_tail(fin)), // A declared end the last producer dropped short of. Outcome::Sealed(_) => true, // An abort wins unless the end had already settled: a group below it was still open. @@ -3190,19 +3201,16 @@ impl TrackState { let grace = self.arrival_grace(); // Fetched backfill never reaches a reader in arrival order, so only the live feed's // arrivals count. Without a declared start, the first one stands in for it. - let floor = match self.start { - Start::Declared(start) | Start::Resolved(start) | Start::Pending(Some(start)) => start, - // Where the feed starts is still unknown, so everything below the end not yet - // delivered or dropped is owed. - Start::Pending(None) => { - let missing = self - .arrivals - .gaps(0..fin) - .any(|(gap, _)| self.drops.gaps(&gap).next().is_some()); - return missing.then(|| active.checked_add(grace)); - } - Start::Unknown => self.arrivals.first().unwrap_or(0), - }; + // Where the feed starts is still unknown, so everything below the end not yet + // delivered or dropped is owed. + if self.start == Start::Pending(None) { + let missing = self + .arrivals + .gaps(0..fin) + .any(|(gap, _)| self.drops.gaps(&gap).next().is_some()); + return missing.then(|| active.checked_add(grace)); + } + let floor = self.tail_floor(); // Nothing opened the gap above the last arrival, so its grace runs from `active`. self.arrivals .gaps(floor.min(fin)..fin) @@ -3212,6 +3220,15 @@ impl TrackState { .reduce(|a, b| a.zip(b).map(|(a, b)| a.max(b))) } + /// The lowest sequence a received copy's tail owes: where the feed starts, or before a + /// start is declared, its first arrival. + fn tail_floor(&self) -> u64 { + match self.start { + Start::Pending(None) => 0, + start => start.sequence().or(self.arrivals.first()).unwrap_or(0), + } + } + /// Wake `waiter` when a received copy's declared end stops waiting for its tail. fn watch_tail(&self, waiter: &kio::Waiter) { if let Some(Some(at)) = self.final_sequence().and_then(|fin| self.tail_deadline(fin)) @@ -23787,11 +23804,6 @@ mod test { #[test] #[cfg(feature = "async")] fn a_reached_end_outlives_its_expired_drops() { - let patient = || { - Subscription::default() - .with_range(0..) - .with_delay(Delay::default().with_max(Duration::from_secs(3600))) - }; let pool = cache::Pool::new(cache::Config::default().with_expiry(Duration::from_secs(10))); let mut producer = track_producer_pooled("test", pool.clone()); producer.received(); @@ -23813,6 +23825,94 @@ mod test { assert!(matches!(late.poll_recv_group(&waiter), Poll::Ready(Ok(None)))); } + /// A received copy that reached its end through a dropped gap, after that run expires. + #[cfg(feature = "async")] + fn reached_then_expired() -> (Producer, Subscriber, cache::Pool) { + let pool = cache::Pool::new(cache::Config::default().with_expiry(Duration::from_secs(10))); + let mut producer = track_producer_pooled("test", pool.clone()); + producer.received(); + let mut subscriber = producer.subscribe(patient()); + write_group(&producer, 0); + producer.drop_groups(1..2, StreamError::Old); + write_group(&producer, 2); + producer.finish_at(Position::group(3), None).unwrap(); + drain_groups(&mut subscriber); + let waiter = kio::Waiter::noop(); + assert!(matches!(subscriber.poll_recv_group(&waiter), Poll::Ready(Ok(None)))); + elapse(&producer, Duration::from_secs(40)); + pool.sweep(); + assert!(producer.state.read().drops.is_empty(), "the run expired"); + (producer, subscriber, pool) + } + + /// A subscription whose budget outlasts any run, so it would wait on a forgotten one. + #[cfg(feature = "async")] + fn patient() -> Subscription { + Subscription::default() + .with_range(0..) + .with_delay(Delay::default().with_max(Duration::from_secs(3600))) + } + + /// A reached end refined mid-group by a peer that named the boundary first stays reached + /// after its dropped gap is forgotten, since the forgotten run stays accounted for. + #[test] + #[cfg(feature = "async")] + fn a_refined_end_stays_reached_after_its_drops_expire() { + let (mut producer, _subscriber, _pool) = reached_then_expired(); + producer.finish_at(Position { group: 2, frame: 1 }, None).unwrap(); + let mut late = producer.subscribe(patient()); + drain_groups(&mut late); + assert!(matches!( + late.poll_recv_group(&kio::Waiter::noop()), + Poll::Ready(Ok(None)) + )); + } + + /// A reached end settles an abort after its dropped gap is forgotten: readers end cleanly. + #[test] + #[cfg(feature = "async")] + fn an_abort_after_a_reached_end_drops_expire_ends_readers_cleanly() { + let (producer, mut subscriber, _pool) = reached_then_expired(); + producer.abort(Error::Cancel).unwrap(); + assert!(matches!( + subscriber.poll_recv_group(&kio::Waiter::noop()), + Poll::Ready(Ok(None)) + )); + } + + /// A reader that judged a dropped run before the pool forgot it, and has not read past it, + /// asks a fill for it once it is unknown again. + #[test] + #[cfg(feature = "async")] + fn a_reader_asks_a_fill_for_a_forgotten_run_it_judged() { + let pool = cache::Pool::new(cache::Config::default().with_expiry(Duration::from_secs(10))); + let producer = track_producer_pooled("test", pool.clone()); + producer.received(); + let dynamic = producer.requests(); + let mut subscriber = producer.subscribe(patient()); + write_group(&producer, 0); + producer.drop_groups(1..2, StreamError::Old); + write_group(&producer, 2); + let waiter = kio::Waiter::noop(); + let Poll::Ready(Ok(Some(first))) = subscriber.poll_recv_group(&waiter) else { + panic!("group 0 is handed out"); + }; + assert_eq!(first.sequence, 0); + assert!( + dynamic.poll_requested_fill(&waiter).is_pending(), + "a dropped run asks no fill" + ); + + elapse(&producer, Duration::from_secs(40)); + pool.sweep(); + assert!(producer.state.read().drops.is_empty(), "the run expired"); + let _ = subscriber.poll_recv_group(&waiter); + let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) else { + panic!("the forgotten run is asked of a fill"); + }; + assert_eq!(fill.start().group..fill.end(), 1..2); + } + /// A source refusing a fill in disjoint groups pays for the runs it records, as a drop does. #[test] #[cfg(feature = "async")] @@ -23883,6 +23983,10 @@ mod test { "only a prefix was shed" ); assert_eq!(held.last().map(|run| run.start), Some(2 * (RUNS - 1))); + assert!( + !producer.state.read().arrivals.has_gaps(), + "the shed runs are accounted for without fragmenting the arrivals" + ); assert!( producer.state.read().given_up(0).is_none(), "a shed run is unknown again" diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index 469a81976d0..28292146a5f 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -100,6 +100,8 @@ pub(super) struct Misses { /// The drops revision, judged revision, budget, and coverage [`Self::unknown`] was found /// under: while the terms hold, only the runs changed since are looked at. unknown_at: Option<(u64, u64, Duration, super::Coverage)>, + /// The [`Forgotten`] revision this reader judged again up to. + forgotten: u64, } /// A run of misses the reader gives up, for the drop cursor to name, with the budget it was @@ -144,6 +146,44 @@ impl Wake for JoinWake { } } +/// How many runs [`Forgotten`] keeps apart before it coalesces the oldest. +const FORGOTTEN_KEPT: usize = 4; + +/// The runs a track reclaimed from its drop set, by revision, for each reader to judge again +/// what it judged inside them. Past [`FORGOTTEN_KEPT`] runs the oldest two coalesce into the +/// span covering both under the newer revision, so this stays a few entries however much is +/// forgotten, and a reader that fell behind judges a wider span again, never a narrower one. +#[derive(Default)] +pub(super) struct Forgotten { + runs: std::collections::VecDeque<(u64, ops::Range)>, + revision: u64, +} + +impl Forgotten { + /// Record `run` as forgotten. + pub fn record(&mut self, run: ops::Range) { + self.revision += 1; + self.runs.push_back((self.revision, run)); + if self.runs.len() > FORGOTTEN_KEPT + && let Some((_, oldest)) = self.runs.pop_front() + && let Some((_, next)) = self.runs.front_mut() + { + *next = oldest.start.min(next.start)..oldest.end.max(next.end); + } + } + + /// The runs forgotten after `revision`, possibly wider, with the revision they bring a + /// reader to. + fn since(&self, revision: u64) -> (impl Iterator> + '_, u64) { + let runs = self + .runs + .iter() + .filter(move |(at, _)| *at > revision) + .map(|(_, run)| run.clone()); + (runs, self.revision) + } +} + /// The current deferred runs, indexed by sequence and by their next judgement deadline. /// Splits remove the old age entry before publishing either piece; no stale timers remain. #[derive(Default)] @@ -846,6 +886,22 @@ impl Misses { candidates.insert(run.clone()); } } + // A run the track forgot is unknown again, so what this reader judged inside it and has + // not taken is judged afresh: asked of a fill, or given up by its budget. + let (forgotten, revision) = state.forgotten.since(self.forgotten); + for run in forgotten { + let run = run.start.max(low)..run.end.min(top.saturating_add(1)); + if run.is_empty() { + continue; + } + for judged in self.judged.overlapping(&run) { + let judged = judged.start.max(run.start)..judged.end.min(run.end); + for fresh in taken.gaps(judged) { + candidates.insert(fresh); + } + } + } + self.forgotten = revision; // Current ledger runs are indexed by their last mutation. Coalescing retires the // old entries, so a reader that was idle needs no shared mutation history. for changed in ledger @@ -2509,6 +2565,29 @@ mod tests { ); } + /// The forget log stays a few entries, and a reader behind its coalesced entries judges a + /// span covering every run it missed, while a current one judges only what is new. + #[test] + fn the_forget_log_coalesces_into_spans_covering_what_a_reader_missed() { + let mut forgotten = Forgotten::default(); + for run in 0..10u64 { + forgotten.record(10 * run..10 * run + 1); + } + assert_eq!(forgotten.runs.len(), FORGOTTEN_KEPT); + let (runs, revision) = forgotten.since(0); + let mut missed = RangeSet::new(); + for run in runs { + missed.insert(run); + } + for run in 0..10u64 { + assert!(missed.contains(&(10 * run)), "run {run} is judged again"); + } + assert_eq!(revision, 10); + let (runs, _) = forgotten.since(9); + assert_eq!(runs.collect::>(), [90..91]); + assert_eq!(forgotten.since(10).0.count(), 0); + } + #[test] fn deferred_splits_and_replacements_retire_the_old_age_entries() { let mut deferred = Deferred::default(); From c9e13cbc0f285e6ab285fc1c4f17ff4f0fc59e45 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 10:32:46 -0800 Subject: [PATCH 14/17] style: format the reclaim accounting Co-Authored-By: Claude Opus 5.5 --- js/net/src/track.ts | 6 +++--- rs/moq-net/src/model/track.rs | 6 +++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 5321095983f..533040adecb 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -1078,9 +1078,9 @@ export class Producer { // The misses its readers asked of a {@link Requests} handle. #fills: FillState; // The sequences the live feed delivered, as a group or a drop, and the stale runs a reclaim - // forgot: what a pending tail is judged by. Eviction never touches it. A gap folds away a subscription's grace after a - // later arrival opened it, as the session's tail does, so this holds only the gaps still - // worth waiting for. + // forgot: what a pending tail is judged by. Eviction never touches it. A gap folds away a + // subscription's grace after a later arrival opened it, as the session's tail does, so this + // holds only the gaps still worth waiting for. #arrivals = new Ledger(); // What this track's own publisher produced since its last append, which skips every // sequence below it that never arrived and was not given up: the producer moved on. The diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 2e5d57aa21c..1b2138ab7c6 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1547,7 +1547,11 @@ impl TrackState { let now = self.cache.pool().now_wall(); // Only a first run takes this as the age of the gap below it, which nothing opened yet: // the tail's, so an end already reached stays reached. - let since = self.outcome.end().and_then(|end| end.tail).map_or(now, |tail| tail.min(now)); + let since = self + .outcome + .end() + .and_then(|end| end.tail) + .map_or(now, |tail| tail.min(now)); self.arrivals.account(start..run.end, since); self.cache.wakes().arrived(start..run.end); } From 2bf8648fb39f51ead3e290be1770590007b0bfa8 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:06:39 -0800 Subject: [PATCH 15/17] test(moq-net): satisfy clippy in the forget log test Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track/miss.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index 28292146a5f..9e0ddb5d3a2 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -2584,7 +2584,9 @@ mod tests { } assert_eq!(revision, 10); let (runs, _) = forgotten.since(9); - assert_eq!(runs.collect::>(), [90..91]); + let runs: Vec<_> = runs.collect(); + assert_eq!(runs.len(), 1); + assert_eq!(runs[0], 90..91); assert_eq!(forgotten.since(10).0.count(), 0); } From baf9e096f2dc9dfaa1bb10d6b25ce1a5998ca00c Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:37:17 -0800 Subject: [PATCH 16/17] fix(net): a received end's tail decides it in JS too, and drop billing costs O(1) JS reaches() now returns true for a tailed end, as Rust's does, so a forgotten skip that reached a received copy's end cannot reopen it; the tail stays set once the subscription ends, which owes nothing instead. Rust change_drops bills from the maps' lengths rather than counting a coalesced run's expiry entries on every change. Co-Authored-By: Claude Opus 5.5 --- js/net/src/cache.test.ts | 30 ++++++++++++++++++++++++ js/net/src/track.ts | 43 ++++++++++++++++++----------------- rs/moq-net/src/model/track.rs | 12 +++++----- 3 files changed, 58 insertions(+), 27 deletions(-) diff --git a/js/net/src/cache.test.ts b/js/net/src/cache.test.ts index 03c1f379be3..0281262ea58 100644 --- a/js/net/src/cache.test.ts +++ b/js/net/src/cache.test.ts @@ -407,3 +407,33 @@ test("a forgotten stale run is asked of a fill", async () => { await next.catch(() => undefined); }); }); + +// A received copy that reached its end through a skip stays ended once the pool forgets the skip, +// since the forgotten run stays accounted for. Mirrors Rust +// `a_reached_end_outlives_its_expired_drops`. +test("a reached end outlives its forgotten skip", async () => { + await withClock(async (clock) => { + const pool = new cache.Pool({ expiry: Milli(1_000) }); + const producer = pooled(pool); + hooks.receiveTrack(producer); + producer.createGroup(0n).finish(); + hooks.skipTrackGroups(producer, 1n, 3n); + hooks.endTrack(producer, { group: 3n, frame: 0 }); + const done = (reader: ReturnType) => { + for (let recv = hooks.tryRecvGroup(reader, false); ; recv = hooks.tryRecvGroup(reader, false)) { + if (recv.kind !== "group") return recv.kind; + recv.group.close(); + } + }; + const reader = producer.subscribe({ delay: { max: Milli(60_000) } }); + expect(done(reader)).toBe("done"); + + await clock.advance(3_000); + expect(dropsHeld(producer)).toEqual([]); + expect(done(reader)).toBe("done"); + const late = producer.subscribe({ delay: { max: Milli(60_000) } }); + expect(done(late)).toBe("done"); + reader.close(); + late.close(); + }); +}); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 533040adecb..3af0dabe0a2 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -747,8 +747,9 @@ type End = { error?: Error; // A received end, declared ahead of the group streams below it that QUIC may deliver in any // order: readers wait for a hole below it to fill, or for the track to close, rather than end - // at the boundary with groups still to arrive. Cleared once the subscription feeding the copy - // ends (see `hooks.cacheTrack`). Mirrors the Rust `End::tail`. + // at the boundary with groups still to arrive. Once the subscription feeding the copy ends (see + // `hooks.cacheTrack`), nothing is owed and readers end at the boundary. Mirrors the Rust + // `End::tail`. tail: boolean; }; @@ -895,7 +896,12 @@ function givenUp( return found.sort((a, b) => (a.start < b.start ? -1 : a.start > b.start ? 1 : 0)); } -function reaches(received: bigint, drops: DropSet, final: bigint, refused: readonly Run[] = []): boolean { +// Whether the live edge reached `end`, counting a gap up to it that was wholly given up, or a +// received end's tail decides it instead: what it misses below is owed only while the feed may +// still deliver it. Mirrors the Rust `TrackState::reaches`. +function reaches(received: bigint, drops: DropSet, end: End, refused: readonly Run[] = []): boolean { + if (end.tail) return true; + const final = finalSequence(end.at); if (received >= final) return true; const given: Run[] = []; for (const run of [...drops.overlapping(received, final), ...refused]) insertRun(given, run.start, run.end); @@ -1247,14 +1253,9 @@ export class Producer { resubscribe = (producer) => producer.#updateSubscription(); hooks.cacheTrack = (producer) => { producer.#wire = undefined; - // Nothing feeds the tail anymore, so readers end at the boundary. - const end = producer.#state.end.peek(); - const untailed = end?.tail ? { ...end, tail: false } : undefined; - for (const state of [producer.#state, ...producer.#sinks]) { - if (untailed) state.end.set(untailed); - state.owed.set(false); - } - producer.#settle(); + // Nothing feeds the tail anymore, so it owes nothing and readers end at the boundary. + for (const state of [producer.#state, ...producer.#sinks]) state.owed.set(false); + producer.#endLargest(); }; hooks.refuseLive = (producer, edge, error) => { for (const sink of producer.#sinks) { @@ -2139,7 +2140,6 @@ export class Producer { if (declared.at.frame !== 0 || end.frame === 0 || finalSequence(end) !== declared.at.group) { throw new ProtocolViolation(`conflicting track end: ${end.group}/${end.frame}`); } - // A tail `cacheTrack` cleared stays cleared: no subscription feeds it anymore. this.#declareEnd(end, error, declared.tail || tail); } this.#owe(); @@ -2169,7 +2169,7 @@ export class Producer { if (!end?.tail) return; const owed = this.#owes(finalSequence(end.at)); for (const state of [this.#state, ...this.#sinks]) state.owed.set(owed); - this.#settle(); + this.#endLargest(); } // Settle the outcome cleanly once the declared end is reached and settled. Also run before a @@ -2223,7 +2223,7 @@ export class Producer { } #owes(final: bigint): boolean { - if (!this.#state.end.peek()?.tail) return false; + if (!this.#wire || !this.#state.end.peek()?.tail) return false; const feed = this.#feedStart(); const floor = (feed === "pending" ? this.#wire?.served?.requested : feed) ?? this.#arrivals.first(); if (floor === undefined) return final > 0n; @@ -2260,8 +2260,9 @@ export class Producer { #endLargest(): void { this.#settle(); if (this.#state.largestEnded.peek() || this.#largestOpen) return; - const end = this.#state.end.peek()?.at; - if (end === undefined || !reaches(this.#received, this.#state.drops, finalSequence(end))) return; + const end = this.#state.end.peek(); + if (end === undefined || !reaches(this.#received, this.#state.drops, end) || this.#owes(finalSequence(end.at))) + return; for (const state of [this.#state, ...this.#sinks]) state.largestEnded.set(true); } @@ -2347,10 +2348,10 @@ export class Producer { // abort over a group the tail still owed cut it off, so readers get the error rather than // a clean end without it. Mirrors the Rust `is_settled`. #settled(): boolean { - const end = this.#state.end.peek()?.at; + const end = this.#state.end.peek(); if (end === undefined) return false; - const final = finalSequence(end); - if (!reaches(this.#received, this.#state.drops, final) || this.#owes(final)) return false; + const final = finalSequence(end.at); + if (!reaches(this.#received, this.#state.drops, end) || this.#owes(final)) return false; for (const { group } of this.#cache) if (group.sequence < final && group.closed.peek() !== null) return false; return true; } @@ -4015,10 +4016,10 @@ export class Subscriber { // owes below it. Reader completion does not settle the tail: open streams remain // accounted for by the wire layer, and an abort still wins on the next read. #complete(): boolean { - const end = this.#state.end.peek()?.at; + const end = this.#state.end.peek(); return ( end !== undefined && - reaches(this.#state.received.peek(), this.#state.drops, finalSequence(end), this.#state.refused.peek()) && + reaches(this.#state.received.peek(), this.#state.drops, end, this.#state.refused.peek()) && !this.#state.owed.peek() ); } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 1b2138ab7c6..17a40b2db07 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1405,11 +1405,11 @@ impl TrackState { affected.start = affected.start.min(run.start); affected.end = affected.end.max(run.end); } - let entries = |state: &Self, affected: &ops::Range| { - state.drops.overlapping(affected).count() as u64 * RUN_OVERHEAD - + state.drops_born.overlapping(affected).count() as u64 * BORN_OVERHEAD - }; - let before = entries(self, &affected); + // Whole-map lengths: a coalesced run can span many expiry entries, which a count over + // the affected span would scan on every change. + let entries = + |state: &Self| state.drops.len() as u64 * RUN_OVERHEAD + state.drops_born.len() as u64 * BORN_OVERHEAD; + let before = entries(self); match hole { Some(hole) => { self.drops.insert(groups.clone(), hole); @@ -1435,7 +1435,7 @@ impl TrackState { for (run, _) in self.drops.overlapping(&affected) { self.drops_changes.insert(run.clone()); } - let after = entries(self, &affected); + let after = entries(self); if after > before { self.drops_bill.add(after - before); } else { From 3d8272e37d1da6719a78da4f8eff6fd302e8d04e Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:33:27 -0800 Subject: [PATCH 17/17] test(moq-net): finish the fill-refusal test's track through a shared handle Main's Producer::finish now consumes the producer; the test reads its state after. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 048e79d90f9..30dc6846b62 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -24121,7 +24121,7 @@ mod test { panic!("the miss below the start is asked"); }; // Nothing arrives once the feed ends, so a refusal is a verdict on the track. - producer.finish().unwrap(); + producer.finish_shared().unwrap(); for run in 0..RUNS { fill.drop_groups(2 * run..2 * run + 1); }