From 8ef012aceefae858b141a2166a0391d9377b8fd6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 16:21:11 -0800 Subject: [PATCH 01/37] chore(quest): claim quest/m1/drop-set.md Quest-Claim: e27be1fd24fac6bd From b267c9decf56df96a5ee3f87fee44a3134e4133a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 17:03:26 -0800 Subject: [PATCH 02/37] fix(net): a failed fetch below the end no longer holds it open A fetch-only copy (Rust) or fill answer (JS) that aborts below a declared end promised nothing of the feed's, so the end settles at once instead of once the expiry scan reclaims it. Also covers the JS idle-START unreport. Co-Authored-By: Claude Opus 5.5 --- js/net/src/fill.test.ts | 29 ++++++++++++++++ js/net/src/track.test.ts | 26 ++++++++++++++ js/net/src/track.ts | 16 +++++++-- quest/m1/drop-set.md | 11 ++---- rs/moq-net/src/model/track.rs | 64 +++++++++++++++++++++++++++++++++-- 5 files changed, 133 insertions(+), 13 deletions(-) diff --git a/js/net/src/fill.test.ts b/js/net/src/fill.test.ts index ffd85098ca3..5779de411de 100644 --- a/js/net/src/fill.test.ts +++ b/js/net/src/fill.test.ts @@ -203,6 +203,35 @@ test("fill: a fetch cancelled while its fill awaits a reply withdraws the fill", broadcast.close(); }); +// As in Rust: a failed fill answer holds nothing the feed promised, so it never holds the end. +test("fill: an answer aborted below the end does not hold the end open", async () => { + for (const parked of [false, true]) { + const { broadcast, track } = resumed(1n); + const requests = track.requests(); + const fetching = wireOf(broadcast).fetchGroup("video", 0n); + const fill = await requests.requestedFill(); + if (!fill) throw new Error("no fill"); + const group = fill.accept(0n); + group.writeFrame({ payload: enc.encode("backfill") }); + if (parked) { + track.finishAt({ group: 2n, frame: 0 }); + await yieldNow(); + expect(track.closed.peek()).toBeUndefined(); + } + group.abort(new StreamError(StreamCode.Cancel)); + fill.close(); + await fetching.catch(() => undefined); + if (!parked) track.finishAt({ group: 2n, frame: 0 }); + await yieldNow(); + expect(track.closed.peek()).toBeNull(); + track.abort(new Error("boom")); + expect(track.closed.peek()).toBeNull(); + + requests.close(); + broadcast.close(); + } +}); + test("fill: a reader reaching back for history gets the misses filled in order", async () => { const { broadcast, track } = resumed(4n); const requests = track.requests(); diff --git a/js/net/src/track.test.ts b/js/net/src/track.test.ts index c19fa9c5811..997f7c85c9d 100644 --- a/js/net/src/track.test.ts +++ b/js/net/src/track.test.ts @@ -4414,6 +4414,32 @@ for (const reported of [false, true]) { }); } +// The lite publisher unreports a stale skip below an idle SUBSCRIBE_START, which never went out. +// An update between that START and the first group then neither reopens nor drops it again, +// while a widening still re-offers its cached group. +test("an update after an idle start leaves an unreported stale skip to a widening", () => { + const producer = new TrackProducer("test").accept({ timescale: Timescale.MILLI, maxAge: null }); + const subscriber = producer.subscribe({ delay: { max: Duration.millis(1500) } }); + expect(hooks.tryDrop(subscriber)).toBeUndefined(); + stampGroup(producer, 0n, 0); + stampGroup(producer, 1n, 1000); + stampGroup(producer, 2n, 6000); + // Groups 0 and 1 reach 1s and 6s; only group 0 is 1.5s behind the 6s edge. + expect(drainGroups(subscriber)).toEqual([1n, 2n]); + expect(hooks.tryDrop(subscriber)).toMatchObject({ start: 0n, end: 1n }); + + // SUBSCRIBE_START resolved at group 1 with the skip still held for it. + hooks.unreport(subscriber, 0n, 1n); + // An update under the same budget reopens nothing below the start. + expect(hooks.reofferStale(subscriber)).toEqual([]); + expect(hooks.tryDrop(subscriber)).toBeUndefined(); + + subscriber.update({ delay: { max: Duration.millis(10_000) } }); + expect(drainGroups(subscriber)).toEqual([0n]); + expect(hooks.tryDrop(subscriber)).toBeUndefined(); + producer.close(); +}); + // A widening that covers only the newer of two skipped groups hands it out without folding the // older one into the delivered floor: a further widening still re-offers it. test("a partial maxDelay widening keeps the older skip owed", () => { diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 221c7a78eb4..6135ccee02f 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -810,6 +810,7 @@ function after(a: Position, b: Position): boolean { // `arrived` is when the group joined this track, which its mirrors age a predecessor from. // `bound` is its first frame's timestamp once checked to fit between its neighbors', which // bounds theirs; `unverified` marks one accepted below the eviction watermark instead. +// `fetched` marks one a fill answered, which the feed never promised. type CachedGroup = { group: GroupProducer; charge: cache.Charge; @@ -817,6 +818,7 @@ type CachedGroup = { arrived: Instant; bound?: Timestamp; unverified?: boolean; + fetched?: boolean; }; // A group's checked start, kept by sequence. @@ -1116,7 +1118,10 @@ export class Producer { name, accept: (sequence, info) => { if (info !== undefined && this.#state.info.peek() === undefined) this.accept(info); - return this.#createGroup(sequence, 0); + const group = this.#createGroup(sequence, 0); + const entry = this.#cached.get(sequence); + if (entry) entry.fetched = true; + return group; }, drop: (start, end, error) => { this.#drop(start, end, { error }); @@ -2400,13 +2405,18 @@ export class Producer { // Whether the declared end was reached, the pending tail owes nothing below it, and every // cached group below it finished, so the track already holds everything it promised. An // abort over a group the tail still owed cut it off, so readers get the error rather than - // a clean end without it. Mirrors the Rust `is_settled`. + // a clean end without it. A failed fill answer holds nothing open, since the feed never + // promised it. Mirrors the Rust `is_settled`. #settled(): boolean { const end = this.#state.end.peek(); if (end === undefined) return false; const final = finalSequence(end.at); if (!reaches(this.#received, this.#state.drops, end) || this.#owes(final)) return false; - for (const { group } of this.#cache) if (group.sequence < final && group.closed.peek() !== null) return false; + for (const { group, fetched } of this.#cache) { + if (group.sequence >= final) continue; + const closed = group.closed.peek(); + if (closed !== null && !(fetched && closed instanceof Error)) return false; + } return true; } diff --git a/quest/m1/drop-set.md b/quest/m1/drop-set.md index c361ab869db..477fec3dbd6 100644 --- a/quest/m1/drop-set.md +++ b/quest/m1/drop-set.md @@ -76,14 +76,6 @@ how a cursor judges a missing or stale group keeps that verdict in step. Still to cover, beyond the memory work: -- **Idle START unreport test.** [moq-dev/moq#6717](https://github.com/moq-dev/moq/pull/6717) - made the JS lite publisher unreport stale skips below an idle - SUBSCRIBE_START, without a test: the case needs an update between START - and the first group. Cover it deterministically through the track hooks. -- **Cancelled fetches hold the end.** Found 2026-10-28 by - [moq-dev/moq#7149](https://github.com/moq-dev/moq/pull/7149): a cancelled - fetch below a track's end holds the end open until the cache expiry scan - removes it. Settle it with a run reason. - **Untimed fills below a resolved start.** Noted 2026-10-31 from [moq-dev/moq#7235](https://github.com/moq-dev/moq/pull/7235): a lite-07 copy with no media time asks a fill below the start even for a reader whose @@ -93,6 +85,9 @@ Still to cover, beyond the memory work: (`rs/moq-net/src/model/track/fill.rs`) records what a fill's source gave up as a budget-independent drop, including OLD judged under that fill's own terms. Record it as a stale run, so it settles only the readers it covers. + Check first that a case is left: `Inner::old_binds_everyone` already + records an OLD only when the fill asked at or past the max age bound, which + the floor rule above counts as final. Tests, on mock time through a relay, in Rust and mirrored in `js/net`: diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index c02e50adb9b..8c57f8bf022 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -867,9 +867,10 @@ impl Slot { /// nothing it promised a reader is still in flight. A fetched head serving the group to /// its end is judged in place of the live copy beside it, which may still be written or /// may have been cut or evicted. Once no head serves, an aborted live copy holds the end - /// open: a session that died mid-group delivered less than the end promised. + /// open: a session that died mid-group delivered less than the end promised. A failed + /// fetch-only copy holds nothing open, since the feed never promised it. fn is_settled(&self) -> bool { - self.serving().is_settled() + (!self.visible && self.is_aborted()) || self.serving().is_settled() } /// Every copy this slot holds, the live group first. @@ -19312,6 +19313,65 @@ mod test { assert!(matches!(consumer.poll_complete(&noop), Poll::Ready(Ok(())))); } + /// A fetch below the end that was cancelled, or failed, before the live feed delivered its + /// group holds nothing the feed promised, so the end settles at once rather than once the + /// expiry scan reclaims its slot, and a later abort keeps that clean end. + #[kio::test] + #[cfg(feature = "async")] + async fn an_aborted_fetch_below_the_end_does_not_hold_it() { + use std::sync::atomic::{AtomicBool, Ordering}; + + for (err, parked) in [(Error::Cancel, false), (Error::Cancel, true), (Error::App(5), true)] { + let mut producer = track_producer("test", Info::timed()); + let dynamic = producer.requests(); + let consumer = producer.consume(); + let noop = kio::Waiter::noop(); + // The feed delivered group 1 and never group 0, which a fetch asks of the source. + producer + .create_group(group::Info { sequence: 1 }) + .unwrap() + .finish() + .unwrap(); + + let fetch = consumer.fetch_group(0, None); + let mut fetched = dynamic + .requested_fill_group() + .await + .unwrap() + .accept(Info::timed()) + .unwrap(); + fetched + .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"backfill")) + .unwrap(); + let woken = Arc::new(AtomicBool::new(false)); + let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); + if parked { + producer.finish_at(Position::group(2), None).unwrap(); + assert!( + producer.poll_closed(&waiter).is_pending(), + "{err:?}: the fetch still writes" + ); + } + fetched.abort(err.clone()).unwrap(); + drop(fetch); + if parked { + assert!( + woken.load(Ordering::SeqCst), + "{err:?}: the fetch failing wakes the wait" + ); + } else { + producer.finish_at(Position::group(2), None).unwrap(); + } + assert!( + matches!(producer.poll_closed(&noop), Poll::Ready(Ok(()))), + "{err:?} parked={parked}: the end settles" + ); + producer.clone().abort(Error::Timeout).unwrap(); + assert!(matches!(producer.poll_closed(&noop), Poll::Ready(Ok(()))), "{err:?}"); + assert!(matches!(consumer.poll_complete(&noop), Poll::Ready(Ok(()))), "{err:?}"); + } + } + /// A fetched head still writing holds the largest stream open after the live copy /// aborted and the track ended, so a later head frame is reported rather than lost. #[kio::test] From d3592aed6f3f94345cadca2804357c8ca2767762 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 17:38:51 -0800 Subject: [PATCH 03/37] refactor(net): one record per reader holds what it handed out, skipped, and reported Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/mod.rs | 1 + rs/moq-net/src/model/record.rs | 76 +++++++ rs/moq-net/src/model/track.rs | 360 +++++++++++++++------------------ 3 files changed, 241 insertions(+), 196 deletions(-) create mode 100644 rs/moq-net/src/model/record.rs diff --git a/rs/moq-net/src/model/mod.rs b/rs/moq-net/src/model/mod.rs index 76a6bffd228..087344834dd 100644 --- a/rs/moq-net/src/model/mod.rs +++ b/rs/moq-net/src/model/mod.rs @@ -27,6 +27,7 @@ mod delivered; mod expiry; // The early exit a read nothing woke takes. mod idle; +mod record; mod skipped; // How far a group's writer reads ahead of its fastest reader. pub(crate) mod pace; diff --git a/rs/moq-net/src/model/record.rs b/rs/moq-net/src/model/record.rs new file mode 100644 index 00000000000..aa5daea7489 --- /dev/null +++ b/rs/moq-net/src/model/record.rs @@ -0,0 +1,76 @@ +//! What one reader did with each group sequence, as runs: handed out, skipped as stale, given +//! up, and what its wire was told. + +use rangemap::{RangeMap, RangeSet}; + +use super::delivered::Delivered; +use super::skipped::Skipped; +use super::track::Hole; + +/// One reader's record of every sequence it settled, as runs. +#[derive(Default)] +pub(super) struct Record { + /// The groups handed out, so none is handed out twice: not when its sequence is re-created, + /// nor when another route delivers it too. + pub delivered: Delivered, + /// Cached groups skipped as stale, which a wider budget owes again. + pub skipped: Skipped, + /// Groups skipped as stale, which account for a bounded window as handed out would, until a + /// wider budget owes them again. Kept under an open window too, which a reader may narrow to + /// a bounded one. + pub written_off: RangeSet, + /// Sequences given up and not reported yet, with why: stale skips, cuts, and upstream skips + /// and drops. + pub pending: RangeMap, + /// Every sequence already reported, with how. + pub reported: RangeMap, +} + +/// A run a reader has yet to report, and whether it was copied from the track's runs, which a +/// rescan folds in afresh: one the track forgot, such as a stale run new terms superseded, +/// leaves nothing behind. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct Pending { + pub hole: Hole, + pub received: bool, +} + +impl Pending { + /// A verdict this reader reached itself. + pub fn own(hole: Hole) -> Self { + Self { hole, received: false } + } +} + +/// How a reader reported a run. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Report { + /// Dropped for good, or stale until its group was handed out: no update reopens it. + Dropped, + /// Dropped as stale, which an update reopens. `received` when an upstream's stale run + /// decided it, which is the upstream's to judge again then. + Stale { received: bool }, + /// Skipped, which replaces a drop. + Skip, +} + +impl Report { + /// How `hole` reports, as it came from upstream when `received`. + pub fn of(hole: &Hole, received: bool) -> Self { + match hole { + Hole::Skip => Self::Skip, + hole if hole.stale_budget().is_some() => Self::Stale { received }, + _ => Self::Dropped, + } + } + + /// Whether this report stops `hole` going out over it: a skip goes out over a drop, and + /// a final drop over a stale one, since each replaces what it goes out over. + pub fn blocks(self, hole: &Hole) -> bool { + match hole { + Hole::Skip => self == Self::Skip, + Hole::Drop { stale: None, .. } => !matches!(self, Self::Stale { .. }), + Hole::Drop { .. } => true, + } + } +} diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 8c57f8bf022..6bab9ccaef9 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -19,7 +19,7 @@ use std::sync::Mutex; use super::delivered::Delivered; use super::idle; -use super::skipped::Skipped; +use super::record::{Pending, Record, Report}; use crate::ledger::{self, Ledger}; pub(crate) use super::subscription::Resolved; @@ -4846,7 +4846,6 @@ impl Producer { broadcast, info, inner: Inner::Plain(Cursor::new(self.state.consume(), subscription)), - delivered: Delivered::default(), // A producer-side (in-process) subscribe is not egress: stay untagged. stats: stats::Scope::default(), _stats_sub: stats::Subscription::default(), @@ -6408,7 +6407,10 @@ impl Subscribing { let inner = match resume { Some(resume) => Inner::Resume( Box::new(resume.subscribe(self.subscription.clone())), - self.state.clone(), + Logical { + state: self.state.clone(), + record: Record::default(), + }, ), None => Inner::Plain(Cursor::new(self.state.clone(), self.subscription.clone())), }; @@ -6417,7 +6419,6 @@ impl Subscribing { broadcast: self.broadcast.clone(), info, inner, - delivered: Delivered::default(), stats: self.stats.clone(), _stats_sub: self.stats.subscribe(), largest: Reported::default(), @@ -6801,9 +6802,6 @@ pub struct Subscriber { broadcast: Arc, info: Info, inner: Inner, - /// The groups handed out, so none is handed out twice: not when its sequence is - /// re-created, nor when another route delivers it too. - delivered: Delivered, // Egress stats scope, used to meter the groups this subscriber reads. Empty // (no-op) for an untagged track. stats: stats::Scope, @@ -6851,10 +6849,13 @@ enum Inner { /// A front's logical track, read straight from its routes' copies; see /// [`super::resume`]. Holds the logical track so it counts as a reader. /// Boxed: the plain cursor is the hot path. - Resume( - Box, - #[allow(dead_code, reason = "held so the logical track counts as a reader")] kio::Consumer, - ), + Resume(Box, Logical), +} + +/// A front's logical track, held so it counts as a reader, and what its reader settled. +struct Logical { + state: kio::Consumer, + record: Record, } /// A group a [`Subscriber`] reserved without handing it out; see @@ -7627,8 +7628,8 @@ struct Cursor { /// Groups the seek path has convicted but whose sequences no caller has committed /// past yet, keyed by sequence; see [`Self::commit_seek_stale`]. seek_pending: BTreeMap, - /// Cached groups the arrival path skipped as stale, which a wider budget owes again. - skipped: Skipped, + /// What this cursor handed out, skipped, gave up, and reported. + record: Record, /// The drops a wire publisher forwards, once it first polls for one. drops: Option>, /// What this cursor misses and asks fills for, once it first reads. @@ -7690,22 +7691,6 @@ struct Within<'a> { drift: Option<&'a Drift>, } -/// A run a [`DropCursor`] has yet to report, and whether it was copied from the track's -/// runs, which a rescan folds in afresh: one the track forgot, such as a stale run new terms -/// superseded, leaves nothing behind. -#[derive(Clone, Debug, PartialEq, Eq)] -struct Pending { - hole: Hole, - received: bool, -} - -impl Pending { - /// A verdict this cursor reached itself. - fn own(hole: Hole) -> Self { - Self { hole, received: false } - } -} - /// The [`TrackState::judged_revision`], span, window revision, and the budget and coverage /// that settle received stale runs (see [`Drift::settles`]) a [`DropCursor`] last scanned /// the whole span under; between them it folds in only the runs that changed. @@ -7717,56 +7702,16 @@ struct DropCursor { /// Every group the publisher cuts. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] cuts: super::aborts::Watch, - /// Sequences given up and not reported yet, with why: stale skips, cuts, and upstream - /// skips and drops. - pending: RangeMap, - /// What `pending` last folded in; see [`Scan`]. + /// What [`Record::pending`] last folded in; see [`Scan`]. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] scanned: Option, - /// The [`TrackState::drops_changes`] revision `pending` folded in. + /// The [`TrackState::drops_changes`] revision [`Record::pending`] folded in. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] seen: u64, /// The budget and coverage of the last drift judged, which settle received stale runs /// until the next: a queue-only poll judges none, and must not unsettle them. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] drift: Option<(Duration, Coverage)>, - /// Every sequence this cursor already reported, with how. - #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - reported: RangeMap, -} - -/// How a [`DropCursor`] reported a run. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Report { - /// Dropped for good, or stale until its group was handed out: no update reopens it. - Dropped, - /// Dropped as stale, which an update reopens; see [`Subscriber::reoffer_stale`]. - /// `received` when an upstream's stale run decided it, which is the upstream's to judge - /// again then. - Stale { received: bool }, - /// Skipped, which replaces a drop. - Skip, -} - -impl Report { - /// How `hole` reports, as it came from upstream when `received`. - fn of(hole: &Hole, received: bool) -> Self { - match hole { - Hole::Skip => Self::Skip, - hole if hole.stale_budget().is_some() => Self::Stale { received }, - _ => Self::Dropped, - } - } - - /// Whether this report stops `hole` going out over it: a skip goes out over a drop, and - /// a final drop over a stale one, since each replaces what it goes out over. - fn blocks(self, hole: &Hole) -> bool { - match hole { - Hole::Skip => self == Self::Skip, - Hole::Drop { stale: None, .. } => !matches!(self, Self::Stale { .. }), - Hole::Drop { .. } => true, - } - } } #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] @@ -7775,34 +7720,32 @@ impl DropCursor { // Watch before the first scan, so a cut landing between the two is still seen. Self { cuts: state.cache.watch_cuts(), - pending: RangeMap::new(), scanned: None, seen: 0, drift: None, - reported: RangeMap::new(), } } /// Take back every stale run, reported or not: an update reopens their verdict, so they /// are judged again, and reported again under the new update if still stale. Also returns /// the part an upstream's received stale run decided. - fn take_stale(&mut self) -> (Vec>, RangeSet) { + fn take_stale(&mut self, record: &mut Record) -> (Vec>, RangeSet) { let mut stale = RangeSet::new(); let mut received = RangeSet::new(); - let queued: Vec<_> = self + let queued: Vec<_> = record .pending .iter() .filter(|(_, pending)| pending.hole.stale_budget().is_some()) .map(|(run, pending)| (run.clone(), pending.received)) .collect(); for (run, from_upstream) in queued { - self.pending.remove(run.clone()); + record.pending.remove(run.clone()); if from_upstream { received.insert(run.clone()); } stale.insert(run); } - let reported: Vec<_> = self + let reported: Vec<_> = record .reported .iter() .filter_map(|(run, report)| match report { @@ -7811,7 +7754,7 @@ impl DropCursor { }) .collect(); for (run, from_upstream) in reported { - self.reported.remove(run.clone()); + record.reported.remove(run.clone()); if from_upstream { received.insert(run.clone()); } @@ -7824,43 +7767,49 @@ impl DropCursor { /// Retire a stale report on `sequence` once its group is handed out: the group won over /// the drop, so no update reopens it. - fn retire(&mut self, sequence: u64) { + fn retire(record: &mut Record, sequence: u64) { if let Some(end) = sequence.checked_add(1) - && matches!(self.reported.get(&sequence), Some(Report::Stale { .. })) + && matches!(record.reported.get(&sequence), Some(Report::Stale { .. })) { - self.reported.insert(sequence..end, Report::Dropped); + record.reported.insert(sequence..end, Report::Dropped); } } /// Record a run this cursor's fills gave up on as a miss, as `hole` says; a reason /// already pending, such as the track's own drop, is kept. - fn give_up(&mut self, run: ops::Range, hole: Hole) { - let gaps: Vec<_> = self.pending.gaps(&run).collect(); + fn give_up(record: &mut Record, run: ops::Range, hole: Hole) { + let gaps: Vec<_> = record.pending.gaps(&run).collect(); for gap in gaps { - self.pending.insert(gap, Pending::own(hole.clone())); + record.pending.insert(gap, Pending::own(hole.clone())); } } /// Record `slot`, cached at `sequence`, if it holds a cut no fetched copy covers: a /// copy still serving the group's end delivers it, so it is never also dropped. - fn cut(&mut self, sequence: u64, slot: &Slot) { + fn cut(record: &mut Record, sequence: u64, slot: &Slot) { if slot.visible && !slot.serves_end() && let Some(err) = slot.group.cut_error() && let Some(end) = sequence.checked_add(1) { - self.pending + record.pending .insert(sequence..end, Pending::own(Hole::drop(StreamError::from(err)))); } } /// The lowest unreported run of `within` given up, now marked reported, with why. - fn next(&mut self, state: &TrackState, cut: BTreeSet, within: &Within<'_>) -> Option<(ops::Range, Hole)> { + fn next( + &mut self, + record: &mut Record, + state: &TrackState, + cut: BTreeSet, + within: &Within<'_>, + ) -> Option<(ops::Range, Hole)> { let (span, window) = (&within.span, within.window); // A cut already reclaimed from the cache is in the track's drops instead. for sequence in cut { if let Some(slot) = state.lookup.get(&sequence) { - self.cut(sequence, slot); + Self::cut(record, sequence, slot); } } // Rescan the span only after the window moves or what settles a stale run does, since @@ -7875,14 +7824,14 @@ impl DropCursor { false => state.drops_changes.since(self.seen).collect(), true => { self.scanned = Some(scan); - let received: Vec<_> = self + let received: Vec<_> = record .pending .iter() .filter(|(_, pending)| pending.received) .map(|(run, _)| run.clone()) .collect(); for run in received { - self.pending.remove(run); + record.pending.remove(run); } vec![span.clone()] } @@ -7893,14 +7842,14 @@ impl DropCursor { if changed.is_empty() { continue; } - let received: Vec<_> = self + let received: Vec<_> = record .pending .overlapping(&changed) .filter(|(_, pending)| pending.received) .map(|(run, _)| run.start.max(changed.start)..run.end.min(changed.end)) .collect(); for run in received { - self.pending.remove(run); + record.pending.remove(run); } for (dropped, hole) in state.drops.overlapping(&changed) { let run = dropped.start.max(changed.start)..dropped.end.min(changed.end); @@ -7910,7 +7859,7 @@ impl DropCursor { None => hole.stale_budget().is_none(), }; if settled { - self.pending.insert( + record.pending.insert( run, Pending { hole: hole.clone(), @@ -7922,16 +7871,16 @@ impl DropCursor { } if full { for (&sequence, slot) in state.lookup.range(span.clone()) { - self.cut(sequence, slot); + Self::cut(record, sequence, slot); } } - while let Some((range, pending)) = self + while let Some((range, pending)) = record .pending .iter() .next() .map(|(range, pending)| (range.clone(), pending.clone())) { - self.pending.remove(range.clone()); + record.pending.remove(range.clone()); // A received run the track has since forgotten, shed or superseded, is unknown // again: only the part the track still holds as it was folded in is owed. let range = match pending.received { @@ -7942,7 +7891,7 @@ impl DropCursor { }; let held = range.start.max(held.start)..range.end.min(held.end); if held.end < range.end { - self.pending.insert(held.end..range.end, pending.clone()); + record.pending.insert(held.end..range.end, pending.clone()); } held } @@ -7954,9 +7903,9 @@ impl DropCursor { continue; }; if clipped.end < range.end { - self.pending.insert(clipped.end..range.end, pending.clone()); + record.pending.insert(clipped.end..range.end, pending.clone()); } - let blocked: RangeSet = self + let blocked: RangeSet = record .reported .overlapping(&clipped) .filter(|(_, report)| report.blocks(&pending.hole)) @@ -7964,10 +7913,10 @@ impl DropCursor { .collect(); if let Some(gap) = blocked.gaps(&clipped).next() { if gap.end < range.end { - self.pending.insert(gap.end..range.end, pending.clone()); + record.pending.insert(gap.end..range.end, pending.clone()); } // Replacing what it goes out over: a final report is never reopened by an update. - self.reported + record.reported .insert(gap.clone(), Report::of(&pending.hole, pending.received)); return Some((gap, pending.hole)); } @@ -7997,10 +7946,6 @@ struct Filter { owed: RangeInclusiveSet, /// The time starts [`Self::window`] looks up from the track's history, each asked once. lookups: Vec, - /// Groups skipped as stale, which account for a bounded window as handed out would, - /// until a wider budget owes them again. Kept under an open window too, which a reader - /// may narrow to a bounded one. - written_off: RangeSet, } impl Cursor { @@ -8028,7 +7973,6 @@ impl Cursor { admitted_at: None, owed: RangeInclusiveSet::new(), lookups: Vec::new(), - written_off: RangeSet::new(), }; (filter, state.cache.clone()) }; @@ -8047,7 +7991,7 @@ impl Cursor { published: kio::Producer::new(published), stale: stats::Content::default(), seek_pending: BTreeMap::new(), - skipped: Skipped::default(), + record: Record::default(), drops: None, misses: None, heads: None, @@ -8058,16 +8002,15 @@ impl Cursor { /// The next run given up, judging the live edge first only when `probe`. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - fn poll_drop(&mut self, taken: &Delivered, probe: bool, waiter: &kio::Waiter) -> Poll<(ops::Range, Hole)> { + fn poll_drop(&mut self, probe: bool, waiter: &kio::Waiter) -> Poll<(ops::Range, Hole)> { self.sync(); if self.drops.is_none() { - let mut drops = Box::new(DropCursor::new(&self.state.read())); + self.drops = Some(Box::new(DropCursor::new(&self.state.read()))); if let Some(misses) = &self.misses { for run in misses.given_up().iter() { - drops.give_up(run.clone(), Hole::missed(None)); + DropCursor::give_up(&mut self.record, run.clone(), Hole::missed(None)); } } - self.drops = Some(drops); } let mut judged = None; if probe @@ -8076,7 +8019,7 @@ impl Cursor { { // From where the reader reads next: an ordered one asks nothing below what it took. let from = self.next_sequence.max(self.floor()); - self.probe(from, &drift, taken, waiter); + self.probe(from, &drift, waiter); judged = Some(drift); } // A cap at or below the floor leaves nothing to report, not a reversed span. @@ -8094,12 +8037,13 @@ impl Cursor { // A cut closes its group without modifying the track state, so its own watch wakes // this. Taken before reading the track, so a cut after the read still wakes. let cut = drops.cuts.poll(waiter); + let record = &mut self.record; match self.state.poll(waiter, |state| { - drops.next(state, cut, &within).map_or(Poll::Pending, Poll::Ready) + drops.next(record, state, cut, &within).map_or(Poll::Pending, Poll::Ready) }) { Poll::Ready(Ok(range)) => Poll::Ready(range), Poll::Ready(Err(state)) => drops - .next(&state, BTreeSet::new(), &within) + .next(record, &state, BTreeSet::new(), &within) .map_or(Poll::Pending, Poll::Ready), Poll::Pending => Poll::Pending, } @@ -8221,7 +8165,8 @@ impl Cursor { /// handed out (in any of `taken`), dropped, skipped as stale, or given up as unfillable. /// An open window, or one with a fill still out, always has more, and so does one its /// cap keeps a group out of, since a reader may raise a cap again. - fn is_exhausted(&self, taken: &[&Delivered]) -> bool { + fn is_exhausted(&self, logical: &[&Delivered]) -> bool { + let taken: Vec<&Delivered> = std::iter::once(&self.record.delivered).chain(logical.iter().copied()).collect(); let Some(end) = self.filter.window.end() else { return false; }; @@ -8236,7 +8181,7 @@ impl Cursor { |gap: ops::Range| { let mut left = RangeSet::new(); left.insert(gap.clone()); - for run in self.filter.written_off.overlapping(&gap) { + for run in self.record.written_off.overlapping(&gap) { left.remove(run.clone()); } if let Some(given_up) = given_up { @@ -8262,7 +8207,7 @@ impl Cursor { .take_while(|run| *run.start() < end) .all(|run| { let run = *run.start()..run.end().checked_add(1).map_or(end, |next| next.min(end)); - Delivered::gaps_in_all(taken, run).into_iter().all(&accounted) + Delivered::gaps_in_all(&taken, run).into_iter().all(&accounted) }) } @@ -8356,22 +8301,22 @@ impl Cursor { /// left the cache or was judged at the bound, into `taken`, which compacts what it /// accounts for below its floor. Only a group a wider budget may still offer stays /// written off, so the write-offs never outnumber the stale skips the cache holds. - fn retire_written_off(&mut self, taken: &mut Delivered) { + fn retire_written_off(&mut self) { // Most cursors never write a group off, and this runs on every hand-out. - if self.filter.written_off.is_empty() { + if self.record.written_off.is_empty() { return; } // By run, so a long stale run costs one lookup rather than one per group. let settled: Vec> = self - .filter + .record .written_off .iter() - .flat_map(|run| self.skipped.gaps(run.start..=run.end - 1)) + .flat_map(|run| self.record.skipped.gaps(run.start..=run.end - 1)) .map(|gap| *gap.start()..gap.end() + 1) .collect(); for run in settled { - self.filter.written_off.remove(run.clone()); - taken.insert_run(run); + self.record.written_off.remove(run.clone()); + self.record.delivered.insert_run(run); } } @@ -8418,18 +8363,18 @@ impl Cursor { /// Record `sequence` as skipped stale under `budget`, so a wider one owes it again. fn skip(&mut self, sequence: u64, budget: Duration) { - self.skipped.insert(sequence, budget); + self.record.skipped.insert(sequence, budget); self.prune_skipped(); } /// Forget the skipped runs no cached group is left in: nothing can offer them again. fn prune_skipped(&mut self) { // Most cursors never skip, and this runs on every hand-out. - if self.skipped.is_empty() { + if self.record.skipped.is_empty() { return; } let state = self.state.read(); - self.skipped.prune(state.lookup.len(), |run| { + self.record.skipped.prune(state.lookup.len(), |run| { !state.lookup.range(run.clone()).any(|(_, slot)| slot.is_read()) }); } @@ -8438,7 +8383,7 @@ impl Cursor { /// `budget`: only a widening reverses a stale verdict it has not reported. Costs the /// skipped runs, never the cache, and nothing when the budget did not widen. fn reoffer(&mut self, budget: Duration) { - for run in self.skipped.widened(budget) { + for run in self.record.skipped.widened(budget) { self.reowe(run); } } @@ -8451,7 +8396,7 @@ impl Cursor { let Some(drops) = &mut self.drops else { return Vec::new(); }; - let (runs, received) = drops.take_stale(); + let (runs, received) = drops.take_stale(&mut self.record); // What neither a skip nor a miss offers again: a group written off at the track's max // age bound, which no budget reverses. It is still stale under the new terms. What an // upstream's stale run decided is the upstream's to judge again: the next scan folds @@ -8469,7 +8414,7 @@ impl Cursor { misses.retry(run.clone()); } if let Some(last) = run.end.checked_sub(1) { - for skipped in self.skipped.within(run.start..=last) { + for skipped in self.record.skipped.within(run.start..=last) { kept.remove(*skipped.start()..skipped.end().saturating_add(1)); self.reowe(skipped); } @@ -8479,9 +8424,9 @@ impl Cursor { let state = self.state.read(); clamp_max_delay(self.subscription.read().budget(), state.max_age_bound()) }; - if let Some(drops) = &mut self.drops { + if self.drops.is_some() { for run in kept.iter() { - drops.give_up(run.clone(), Hole::stale(budget)); + DropCursor::give_up(&mut self.record, run.clone(), Hole::stale(budget)); } } runs @@ -8491,16 +8436,17 @@ impl Cursor { /// reported yet; a reported one stands. fn reowe(&mut self, run: ops::RangeInclusive) { let span = *run.start()..run.end().saturating_add(1); - self.filter.written_off.remove(span.clone()); - if let Some(drops) = &mut self.drops { - let old: Vec<_> = drops + self.record.written_off.remove(span.clone()); + if self.drops.is_some() { + let old: Vec<_> = self + .record .pending .overlapping(span.clone()) .filter(|(_, pending)| pending.hole.stale_budget().is_some()) .map(|(range, _)| range.start.max(span.start)..range.end.min(span.end)) .collect(); for range in old { - drops.pending.remove(range); + self.record.pending.remove(range); } } for owed in self.filter.admitted.overlapping(run) { @@ -8625,6 +8571,24 @@ impl Cursor { }) } + /// Record `sequence` as handed out. An ordered read never goes back, and any reader + /// stops expecting what is past its budget, so only the gaps within it cost memory. + fn hand_out(&mut self, sequence: u64, ordered: bool) { + if ordered { + self.record.delivered.forget_below(sequence); + } + self.prune_skipped(); + self.record.delivered.insert(sequence); + // The group won over any stale report on it. + if self.drops.is_some() { + DropCursor::retire(&mut self.record, sequence); + } + let mut delivered = std::mem::take(&mut self.record.delivered); + self.forget_past_budget(&mut delivered); + self.record.delivered = delivered; + self.retire_written_off(); + } + /// Forget the gaps in `delivered` below each group past this cursor's budget now, /// whether this track holds it or not: one landing there would be written off. A group /// skipped as stale is not forgotten, since a wider budget owes it again, and neither is @@ -8636,7 +8600,7 @@ impl Cursor { return; } let owed = self.filter.owed.first().map(|run| *run.start()); - let skipped = self.skipped.lowest().into_iter().chain(owed).min(); + let skipped = self.record.skipped.lowest().into_iter().chain(owed).min(); let max_delay = self.subscription.read().budget(); let state = self.state.read(); let budget = clamp_max_delay(max_delay, state.max_age_bound()); @@ -8764,8 +8728,9 @@ impl Cursor { /// A live reader that takes each group as it arrives never misses one, so most probes /// find nothing to ask and hold no fill: those skip the fills, the subscription, and /// the answers entirely, paying only for the judgement that the new groups arrived. - fn probe(&mut self, from: u64, drift: &Drift, taken: &Delivered, waiter: &kio::Waiter) { + fn probe(&mut self, from: u64, drift: &Drift, waiter: &kio::Waiter) { self.look_up(waiter); + let taken = &self.record.delivered; let (window, cap, revision) = (&self.filter.window, self.end_sequence, self.filter.revision); let misses = self.misses.get_or_insert_with(Box::default); let mut given_up = Vec::new(); @@ -8823,9 +8788,9 @@ impl Cursor { if misses.is_waiting() { misses.expire(&self.state.read(), drift, &mut given_up, waiter); } - if let Some(drops) = &mut self.drops { + if self.drops.is_some() { for (run, stale) in given_up { - drops.give_up(run.clone(), Hole::missed(stale)); + DropCursor::give_up(&mut self.record, run.clone(), Hole::missed(stale)); } } // A refusal taken in may move a time end walking history past it, widening what @@ -8842,9 +8807,9 @@ impl Cursor { let order = self.subscription.read().order; let mut given_up = Vec::new(); let group = misses.next_filled(&self.state.read(), order, &mut given_up); - if let Some(drops) = &mut self.drops { + if self.drops.is_some() { for (run, stale) in given_up { - drops.give_up(run.clone(), Hole::missed(stale)); + DropCursor::give_up(&mut self.record, run.clone(), Hole::missed(stale)); } } group @@ -8865,13 +8830,13 @@ impl Cursor { return self.advance_sequence(group); }; self.advance_arrival(group.sequence, taken); - self.skipped.remove(group.sequence); + self.record.skipped.remove(group.sequence); // Taking is the cache access a peek is not. group.cache_refresh(); self.deliver(group) } - fn poll_peek_recv(&mut self, taken: &Delivered, waiter: &kio::Waiter) -> Poll>> { + fn poll_peek_recv(&mut self, waiter: &kio::Waiter) -> Poll>> { self.sync(); if let Some(err) = &self.filter.refused { return Poll::Ready(Err(err.clone())); @@ -8881,7 +8846,7 @@ impl Cursor { // One scan for the whole poll, so walking a backlog off stays linear in its size. let drift = ready!(self.poll_drift(&self.servable, waiter))?; self.reoffer(drift.budget); - self.probe(self.floor(), &drift, taken, waiter); + self.probe(self.floor(), &drift, waiter); if let Some(misses) = &mut self.misses { misses.pick_up(); } @@ -8977,22 +8942,22 @@ impl Cursor { /// skipped for a wider budget to offer again, unless the budget is at its bound. fn write_off_stale(&mut self, sequence: u64, content: stats::Content, drift: &Drift) { self.stale.add(content); - if let Some(drops) = &mut self.drops + if self.drops.is_some() && let Some(end) = sequence.checked_add(1) { - drops + self.record .pending .insert(sequence..end, Pending::own(Hole::stale(drift.budget))); } match drift.bounded { // A re-offered group stale at the bound is final. - true => self.skipped.remove(sequence), + true => self.record.skipped.remove(sequence), false => self.skip(sequence, drift.budget), } // Whatever the window is now: narrowed to a bounded one later, it accounts for this // group by it, even once the group left the cache. if let Some(end) = sequence.checked_add(1) { - self.filter.written_off.insert(sequence..end); + self.record.written_off.insert(sequence..end); } } @@ -9130,9 +9095,9 @@ impl Cursor { misses.give_up(run, &mut runs); } // A head the cursor found missing belonged to a group that existed. - if let Some(drops) = &mut self.drops { + if self.drops.is_some() { for (run, _) in runs { - drops.give_up(run, Hole::drop(StreamError::NotFound)); + DropCursor::give_up(&mut self.record, run, Hole::drop(StreamError::NotFound)); } } } @@ -9173,7 +9138,7 @@ impl Cursor { /// The lowest servable group past the last one returned, walking off everything the /// drift budget has convicted on the way. - fn poll_peek_next(&mut self, taken: &Delivered, waiter: &kio::Waiter) -> Poll>> { + fn poll_peek_next(&mut self, waiter: &kio::Waiter) -> Poll>> { let next = self.next_sequence; self.sync(); if let Some(err) = &self.filter.refused { @@ -9182,7 +9147,7 @@ impl Cursor { let floor = next.max(self.floor()); // One scan for the whole poll, so walking a backlog off stays linear in its size. let drift = ready!(self.poll_drift(&self.servable, waiter))?; - self.probe(floor, &drift, taken, waiter); + self.probe(floor, &drift, waiter); if let Some(misses) = &mut self.misses { misses.pick_up(); } @@ -9476,7 +9441,7 @@ impl Subscriber { pub(crate) fn consume(&self) -> Consumer { let state = match &self.inner { Inner::Plain(cursor) => cursor.state.weak(), - Inner::Resume(_, state) => state.weak(), + Inner::Resume(_, logical) => logical.state.weak(), }; Consumer::new(self.name.clone(), state) } @@ -9600,14 +9565,14 @@ impl Subscriber { let mut idle = idle::Idle::start(&mut cursor.idle, waiter); let polled = loop { let peeked = match ordered { - true => cursor.poll_peek_next(&self.delivered, idle.waiter()), - false => cursor.poll_peek_recv(&self.delivered, idle.waiter()), + true => cursor.poll_peek_next(idle.waiter()), + false => cursor.poll_peek_recv(idle.waiter()), }; let peeked = match peeked { Poll::Pending => { // A read that only skipped since its last hand-out settles its write-offs too. - cursor.retire_written_off(&mut self.delivered); - break match !self.parks && cursor.is_exhausted(&[&self.delivered]) { + cursor.retire_written_off(); + break match !self.parks && cursor.is_exhausted(&[]) { true => Poll::Ready(Ok(None)), false => Poll::Pending, }; @@ -9616,7 +9581,7 @@ impl Subscriber { Poll::Ready(Ok(None)) => break Poll::Ready(Ok(None)), Poll::Ready(Ok(Some(peeked))) => peeked, }; - if !self.delivered.contains(peeked.group.sequence) { + if !cursor.record.delivered.contains(peeked.group.sequence) { break Poll::Ready(Ok(Some(Reserved::Plain(peeked)))); } // A replay advances only this copy's cursor, as an ordinary read would. @@ -9625,8 +9590,8 @@ impl Subscriber { idle.finish(&mut cursor.idle, walk, polled.is_pending()); polled } - Inner::Resume(resume, _) => resume - .poll_peek_group(ordered, self.parks, &self.delivered, waiter) + Inner::Resume(resume, logical) => resume + .poll_peek_group(ordered, self.parks, &logical.record.delivered, waiter) .map(|res| res.map(|candidate| candidate.map(|candidate| Reserved::Resume(Box::new(candidate))))), } } @@ -9634,13 +9599,15 @@ impl Subscriber { /// Whether this route copy's bounded window has nothing left for the logical readers that /// handed out `logical`: what this copy took itself, wrote off, or gave up counts too. pub(super) fn exhausted_for(&mut self, logical: &[&Delivered], waiter: &kio::Waiter) -> bool { - let mut taken = Vec::with_capacity(logical.len() + 1); - taken.push(&self.delivered); - taken.extend_from_slice(logical); match &mut self.inner { - Inner::Plain(cursor) => cursor.is_exhausted(&taken), + Inner::Plain(cursor) => cursor.is_exhausted(logical), // A copy that is another front's logical track judges across its own copies. - Inner::Resume(resume, _) => resume.exhausted(&taken, waiter), + Inner::Resume(resume, own) => { + let mut taken = Vec::with_capacity(logical.len() + 1); + taken.push(&own.record.delivered); + taken.extend_from_slice(logical); + resume.exhausted(&taken, waiter) + } } } @@ -9700,14 +9667,14 @@ impl Subscriber { pub(super) fn skip_ordered(&mut self, floor: u64) { match &mut self.inner { Inner::Plain(cursor) => cursor.skip_ordered(floor), - Inner::Resume(resume, _) => { + Inner::Resume(resume, logical) => { // What it handed out moves, which nothing it parked on notifies. - if !self.delivered.covers(0..floor) { + if !logical.record.delivered.covers(0..floor) { resume.disarm(); } // Its retention decisions also need the containing reader's past prefix. // This records what is no longer owed, without handing a group out. - self.delivered.forget_below(floor); + logical.record.delivered.forget_below(floor); } } } @@ -9715,24 +9682,17 @@ impl Subscriber { /// Record `sequence` as handed out. An ordered read never goes back, and any reader /// stops expecting what is past its budget, so only the gaps within it cost memory. fn hand_out(&mut self, sequence: u64, ordered: bool) { - if ordered { - self.delivered.forget_below(sequence); - } - if let Inner::Plain(cursor) = &mut self.inner { - cursor.prune_skipped(); - } - self.delivered.insert(sequence); - // The group won over any stale report on it; a resumed reader retires its own. - if let Inner::Plain(cursor) = &mut self.inner - && let Some(drops) = &mut cursor.drops - { - drops.retire(sequence); - } - let mut delivered = std::mem::take(&mut self.delivered); - self.forget_past_budget(&mut delivered); - self.delivered = delivered; - if let Inner::Plain(cursor) = &mut self.inner { - cursor.retire_written_off(&mut self.delivered); + match &mut self.inner { + Inner::Plain(cursor) => cursor.hand_out(sequence, ordered), + // A resumed reader retires its own stale reports. + Inner::Resume(resume, logical) => { + let delivered = &mut logical.record.delivered; + if ordered { + delivered.forget_below(sequence); + } + delivered.insert(sequence); + resume.forget_past_budget(delivered); + } } } @@ -9748,14 +9708,22 @@ impl Subscriber { /// How many runs of handed-out groups this subscriber remembers. #[cfg(test)] pub(super) fn delivered_runs(&self) -> usize { - self.delivered.runs() + self.record().delivered.runs() + } + + /// What this reader settled. + fn record(&self) -> &Record { + match &self.inner { + Inner::Plain(cursor) => &cursor.record, + Inner::Resume(_, logical) => &logical.record, + } } /// How many runs of groups written off as stale this subscriber remembers. #[cfg(test)] fn written_off_runs(&self) -> usize { match &self.inner { - Inner::Plain(cursor) => cursor.filter.written_off.iter().count(), + Inner::Plain(cursor) => cursor.record.written_off.iter().count(), Inner::Resume(..) => 0, } } @@ -10015,8 +9983,8 @@ impl Subscriber { #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] fn poll_drop_probing(&mut self, probe: bool, waiter: &kio::Waiter) -> Poll<(ops::Range, Hole)> { match &mut self.inner { - Inner::Plain(cursor) => cursor.poll_drop(&self.delivered, probe, waiter), - Inner::Resume(resume, _) => resume.poll_drop(&self.delivered, probe, waiter), + Inner::Plain(cursor) => cursor.poll_drop(probe, waiter), + Inner::Resume(resume, logical) => resume.poll_drop(&logical.record.delivered, probe, waiter), } } @@ -10122,7 +10090,7 @@ impl Subscriber { cursor.end_sequence = end; cursor.update_servable(); } - Inner::Resume(resume, _) => resume.set_range(start, end, &self.delivered), + Inner::Resume(resume, logical) => resume.set_range(start, end, &logical.record.delivered), } self.largest.bounds_moved(); } @@ -10315,8 +10283,8 @@ impl Subscriber { pub(crate) fn join(&self) { match &self.inner { Inner::Plain(cursor) => TrackState::join(cursor.state.read()), - Inner::Resume(_, state) => { - let routes = state.read().routes.clone(); + Inner::Resume(_, logical) => { + let routes = logical.state.read().routes.clone(); if let Some(copy) = routes.and_then(|routes| routes.serving()) { copy.join(); } @@ -17327,7 +17295,7 @@ mod test { subscriber.skip_ordered(u64::MAX); subscriber.skip_ordered(0); assert_eq!(peek(&mut subscriber, true), Poll::Ready(None)); - assert_eq!(subscriber.delivered.runs(), 0, "skips are not deliveries"); + assert_eq!(subscriber.record().delivered.runs(), 0, "skips are not deliveries"); assert_eq!(subscriber.take_stale().groups, 0, "duplicates are not stale skips"); let Inner::Plain(cursor) = &subscriber.inner else { unreachable!() @@ -23082,7 +23050,7 @@ mod test { write_untimed_group(&producer, 14); assert_eq!(drain_groups(&mut subscriber), [14]); assert_eq!(subscriber.delivered_runs(), 4, "floor through 8, then 10, 12 and 14"); - assert!(!subscriber.delivered.contains(9), "the next gap is still remembered"); + assert!(!subscriber.record().delivered.contains(9), "the next gap is still remembered"); write_untimed_group(&producer, 11); assert_eq!( drain_groups(&mut subscriber), @@ -25189,7 +25157,7 @@ mod test { let Inner::Plain(cursor) = &subscriber.inner else { unreachable!() }; - cursor.skipped.runs() + cursor.record.skipped.runs() } /// The groups handed out around a skipped one aging past the budget does not fold it in From d73d5d83185a4dd095f31e0b1791061a9ff899f6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 17:52:38 -0800 Subject: [PATCH 04/37] refactor(net): a publisher's wire record is its reader's record TrackRun's served, dropped, dropped_stale, and expired sets fold into the reader's Record: one reported map says what the wire was told of each run, which the cursor's drops and the publisher's groups both consult. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/publisher.rs | 199 ++++++++++++++----------------- rs/moq-net/src/model/mod.rs | 2 +- rs/moq-net/src/model/record.rs | 193 +++++++++++++++++++++++++++--- rs/moq-net/src/model/track.rs | 45 ++++--- 4 files changed, 284 insertions(+), 155 deletions(-) diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 315408aa43f..cc7eed2a6bc 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -1,6 +1,7 @@ use crate::driver::Transport; use crate::transport::poll::Session as _; +use crate::model::record::Report; use crate::model::track::Hole; use crate::{SessionError, datagram, frame, group, origin, track}; use rangemap::{RangeMap, RangeSet}; @@ -5073,10 +5074,9 @@ struct TrackRun { skipped: RangeMap, // The number of the SUBSCRIBE (0) or latest SUBSCRIBE_UPDATE, which a drop echoes. update: u64, - // Lite-07 groups whose stream expired as stale, which every update drops again under its - // number: the subscriber reopens each stale drop then, and the stream already reset. - expired: RangeSet, - // The expired runs an update owes a drop again, sent as ranges on the next poll. + // The expired runs an update owes a drop again, sent as ranges on the next poll. Lite-07 + // groups whose stream expired as stale are dropped again under every update's number: the + // subscriber reopens each stale drop then, and the stream already reset. redrop: RangeSet, // The expired groups an update re-offers, still cached and in range, each served again // on its own stream as stream credit allows. @@ -5085,18 +5085,6 @@ struct TrackRun { // Lite-07 with every range bounded: once every group in its ranges is given up, it is // answered without a group to resolve its start, rather than left unanswered. bounded: bool, - // Lite05+ names every sequence below the end that this subscription never - // got with SUBSCRIBE_DROP, so the subscriber settles without waiting out its grace - // for a group that will not come. The sequences served or dropped. A gap that never arrived stays until the end sweep names it: this side - // cannot tell when the subscriber stopped waiting for it. - // `None` on other versions. - served: Option>, - // Lite-07: the part of `served` named in SUBSCRIBE_DROP, never served. A skip replaces a - // drop, so it is sent over one of these, but never over a group served. - dropped: RangeSet, - // Lite-07: the part of `dropped` named stale (OLD), which a final drop or a group replaces, - // as one a route the front switched to delivers after the update that reopens it. - dropped_stale: RangeSet, /// The one group waiting for stream credit. The rest of the backlog waits in the cache, /// which bills, ages, and evicts it, and the cursor's budget skips whatever went stale /// meanwhile: a machine per parked group would hold state for as long as the peer @@ -5132,9 +5120,8 @@ impl TrackRun { // The FIN of a covered subscription is the wire's to send, below. let track = track.with_range((Bound::Included(floor), cap)).parked(); - let emit_range = ctx.version.has_track_stream(); // Lite03 and Lite04 declare no end, so neither side could settle on a drop. - let served = emit_range.then(RangeSet::new); + let emit_range = ctx.version.has_track_stream(); let bounded = bounds.ends(ctx.version); let initial = bounds.ranges.len() as u64; @@ -5156,13 +5143,9 @@ impl TrackRun { first: None, skipped: RangeMap::new(), update: 0, - expired: RangeSet::new(), redrop: RangeSet::new(), reoffered: Default::default(), end_sent: false, - served, - dropped: RangeSet::new(), - dropped_stale: RangeSet::new(), opening: None, children: kio::Tasks::new(), idle: None, @@ -5294,12 +5277,12 @@ impl TrackRun { // rest is dropped again, by run, on the next poll. A run an earlier update left to // drop again is judged again too, since these terms may serve it, and so is a group // one re-offered that has not opened yet, since they may no longer admit it. + let mut expired = self.track.record_mut().expired(); for group in std::mem::take(&mut self.reoffered) { - self.expired.insert(group.sequence..group.sequence + 1); + expired.insert(group.sequence..group.sequence + 1); } - let pending = std::mem::take(&mut self.redrop); - self.expired.extend(pending.iter().cloned()); - for run in std::mem::take(&mut self.expired).iter() { + expired.extend(std::mem::take(&mut self.redrop).iter().cloned()); + for run in expired.iter() { let (groups, gone) = self.track.reoffer_groups(run.clone()); self.owe(groups.iter().map(|group| group.sequence..group.sequence + 1).collect()); self.reoffered.extend(groups); @@ -5470,7 +5453,6 @@ impl TrackRun { let hole = match err { StreamError::Old => { if self.ctx.version.has_skip() { - self.expired.insert(sequence..sequence + 1); self.sent_stale(sequence..sequence + 1); } Hole::stale(self.track.budget()) @@ -5485,12 +5467,8 @@ impl TrackRun { let hole = Hole::stale(self.track.budget()); for run in std::mem::take(&mut self.redrop).iter() { writer.buffer(&self.hole(run.clone(), &hole))?; - self.expired.insert(run.clone()); - self.sent_stale(run.clone()); // A re-offer an update owed again is given up now, so the end sweep never names it. - if let Some(served) = &mut self.served { - served.insert(run.clone()); - } + self.sent_stale(run.clone()); } return Poll::Ready(Ok(ControlFlow::Continue(()))); } @@ -5561,7 +5539,7 @@ impl TrackRun { && let Some(group) = self.reoffered.pop_front() { let sequence = group.sequence; - if self.served.as_ref().is_some_and(|served| served.contains(&sequence)) { + if self.emit_range && self.track.record_mut().told(sequence) { return Poll::Ready(Ok(ControlFlow::Continue(()))); } match position_group(group, &self.frames) { @@ -5588,9 +5566,10 @@ impl TrackRun { // group handed out after one is served, as one a route the front switched to // delivers: the update that reopens the drop would never re-offer it. let sequence = group.sequence; - if self.dropped_stale.contains(&sequence) { + let record = self.track.record_mut(); + if self.emit_range && record.told_stale(sequence) { self.replace_stale(sequence..sequence + 1); - } else if self.served.as_ref().is_some_and(|served| served.contains(&sequence)) { + } else if self.emit_range && record.told(sequence) { return Poll::Ready(Ok(ControlFlow::Continue(()))); } let Some(group) = position_group(group, &self.frames) else { @@ -5711,68 +5690,79 @@ impl TrackRun { /// re-offers a group it still holds and drops again, under the new update, one it cannot /// serve. Earlier versions' drops are final, so their cursor takes nothing back. fn owe(&mut self, reopened: Vec>) { - if let Some(served) = &mut self.served { + if self.emit_range { + let record = self.track.record_mut(); for run in reopened { - self.dropped.remove(run.clone()); - self.dropped_stale.remove(run.clone()); - served.remove(run); + record.untell(run); } } } /// Record `sequence` as served, for [`Self::drop_unserved`]. fn mark_served(&mut self, sequence: u64) { - let Some(served) = &mut self.served else { - return; - }; - served.insert(sequence..sequence + 1); + if self.emit_range { + self.track.record_mut().tell(sequence..sequence + 1, Report::Settled); + } } - /// Forward a verdict the cursor reached on `range`, as [`Self::drop_range`] does. On lite-07 - /// it also replaces what it overrides: a skip any drop already sent, and a drop the wire - /// names final (any code but OLD) a stale one, which the next update would otherwise reopen. A reader the earlier drop ended stays - /// ended. + /// Forward a verdict the cursor reached on `range`, which its record already took in, + /// replacing what it goes out over: on lite-07, a skip any drop already sent, and a final + /// drop a stale one, which the next update would otherwise reopen. A reader the earlier drop + /// ended stays ended. Returns whether any was buffered. fn forward_range( &mut self, range: Range, hole: &Hole, writer: &mut Writer, ) -> Result { - if self.ctx.version.has_skip() && self.served.is_some() { - let replaces = match hole { - Hole::Skip => Some(&self.dropped), - Hole::Drop { .. } if hole.error() == StreamError::Old => None, - Hole::Drop { .. } => Some(&self.dropped_stale), - }; - let replaced: Vec<_> = replaces - .into_iter() - .flat_map(|runs| runs.overlapping(&range)) - .map(|run| run.start.max(range.start)..run.end.min(range.end)) - .collect(); - for run in replaced { - self.replace_stale(run.clone()); - if let Some(served) = &mut self.served { - served.remove(run); - } - } + if !self.emit_range || range.is_empty() { + return Ok(false); + } + self.redrop.remove(range.clone()); + let report = self.report(hole); + self.track.record_mut().tell(range.clone(), report); + writer.buffer(&self.hole(range, hole))?; + Ok(true) + } + + /// What the wire is told when `hole` goes out. + fn report(&self, hole: &Hole) -> Report { + match self.ctx.version.has_skip() { + // Every drop is final before lite-07. + false => Report::Settled, + true => match hole { + Hole::Skip => Report::Skip, + // The wire names every OLD drop stale, which the next update reopens. One given up + // as final, such as one relayed from a version whose drops are all final, is + // dropped again by each update, as an expired stream is. + Hole::Drop { .. } if hole.error() == StreamError::Old => Report::Stale { + received: false, + expired: hole.stale_budget().is_none(), + taken: false, + }, + Hole::Drop { .. } => Report::Dropped, + }, } - self.drop_range(range, hole, writer) } - /// Record `run` as named stale (OLD) in a lite-07 drop, which a later verdict or group - /// replaces; see [`Self::dropped_stale`]. + /// Record `run` as named stale (OLD) in a lite-07 drop that each update drops again, which + /// a later verdict or group replaces. fn sent_stale(&mut self, run: Range) { - self.dropped.insert(run.clone()); - self.dropped_stale.insert(run); + self.track.record_mut().tell( + run, + Report::Stale { + received: false, + expired: true, + taken: false, + }, + ); } /// Forget that `run` went out as a drop: a verdict or group replaces it. A stream of this /// subscription that expired there is no longer owed again on an update, since what /// replaced it settles it. fn replace_stale(&mut self, run: Range) { - self.dropped.remove(run.clone()); - self.dropped_stale.remove(run.clone()); - self.expired.remove(run.clone()); + self.track.record_mut().replace_drops(run.clone()); self.redrop.remove(run); } @@ -5784,39 +5774,13 @@ impl TrackRun { hole: &Hole, writer: &mut Writer, ) -> Result { - let Some(served) = &mut self.served else { + if !self.emit_range || range.is_empty() { return Ok(false); - }; - if range.is_empty() { - return Ok(false); - } - let gaps: RangeSet = served.gaps(&range).collect(); - if self.ctx.version.has_skip() { - for gap in gaps.iter() { - match hole { - Hole::Skip => { - self.dropped.remove(gap.clone()); - self.dropped_stale.remove(gap.clone()); - } - // The wire names every OLD drop stale, which the next update reopens. One the - // cursor gave up as final, such as one relayed from a version whose drops are - // all final, is dropped again by each update, as an expired stream is. - Hole::Drop { .. } if hole.error() == StreamError::Old => { - self.dropped.insert(gap.clone()); - self.dropped_stale.insert(gap.clone()); - if hole.stale_budget().is_none() { - self.expired.insert(gap.clone()); - } - } - Hole::Drop { .. } => { - self.dropped.insert(gap.clone()); - self.dropped_stale.remove(gap.clone()); - } - } - } } - served.extend(gaps.iter().cloned()); + let gaps = self.track.record_mut().untold(range); + let report = self.report(hole); for gap in gaps.iter() { + self.track.record_mut().tell(gap.clone(), report); let msg = self.hole(gap.clone(), hole); writer.buffer(&msg)?; } @@ -5960,8 +5924,10 @@ impl TrackRun { starts: self.starts(start), }))?; // The subscriber now owes itself everything from the start, so a range given up - // above it is named as it would have been after the start. + // above it is named as it would have been after the start. What was given up below it is + // never named, so a wider budget may still serve it. for (range, hole) in skipped.iter() { + self.track.record_mut().untell(range.start..range.end.min(start)); self.forward_range(range.start.max(start)..range.end, hole, writer)?; } Ok(()) @@ -7395,6 +7361,19 @@ mod serve_group_test { } /// A subscription's run loop on `version`, from group 0, and the log of its subscribe stream. + /// Forward the parts of `range` a cursor would, which its record lets out over what it + /// already reported. + fn forward( + run: &mut TrackRun, + range: Range, + hole: &Hole, + writer: &mut Writer, + ) { + while let Some(gap) = run.track.record_mut().report(range.clone(), hole, false) { + run.forward_range(gap, hole, writer).unwrap(); + } + } + fn lite_run( version: Version, session: SinkSession, @@ -8180,9 +8159,8 @@ mod serve_group_test { while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} assert!(run.start.is_some(), "{version}: started"); let stale = Hole::stale(Duration::from_secs(1)); - run.forward_range(1..3, &stale, &mut writer).unwrap(); - run.forward_range(1..2, &Hole::drop(StreamError::Cancel), &mut writer) - .unwrap(); + forward(&mut run, 1..3, &stale, &mut writer); + forward(&mut run, 1..2, &Hole::drop(StreamError::Cancel), &mut writer); write_group(&mut track, 2, 2); while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} @@ -8259,8 +8237,7 @@ mod serve_group_test { // Group 1's stream expired, as a blocked transport would leave it. run.ctx.outcomes.dropped.lock().unwrap().push((1, StreamError::Old)); while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} - run.forward_range(1..2, &Hole::drop(StreamError::Cancel), &mut writer) - .unwrap(); + forward(&mut run, 1..2, &Hole::drop(StreamError::Cancel), &mut writer); run.update(lite::SubscribeUpdate { wanted_group: None, priority: 1, @@ -8569,8 +8546,7 @@ mod serve_group_test { match narrow { true => run.update(update(1, Some(1), 2)), false => { - run.forward_range(0..1, &Hole::drop(StreamError::Cancel), &mut writer) - .unwrap(); + forward(&mut run, 0..1, &Hole::drop(StreamError::Cancel), &mut writer); } } while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} @@ -8647,8 +8623,7 @@ mod serve_group_test { let (mut run, mut writer, log) = lite07_run(SinkSession::new(Log::default()), subscriber); write_group(&mut track, 0, 0); while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} - run.forward_range(1..2, &Hole::drop(StreamError::Old), &mut writer) - .unwrap(); + forward(&mut run, 1..2, &Hole::drop(StreamError::Old), &mut writer); for update in [1, 2] { run.update(lite::SubscribeUpdate { wanted_group: None, diff --git a/rs/moq-net/src/model/mod.rs b/rs/moq-net/src/model/mod.rs index 087344834dd..5da7b75d4f7 100644 --- a/rs/moq-net/src/model/mod.rs +++ b/rs/moq-net/src/model/mod.rs @@ -27,7 +27,7 @@ mod delivered; mod expiry; // The early exit a read nothing woke takes. mod idle; -mod record; +pub(crate) mod record; mod skipped; // How far a group's writer reads ahead of its fastest reader. pub(crate) mod pace; diff --git a/rs/moq-net/src/model/record.rs b/rs/moq-net/src/model/record.rs index aa5daea7489..6345c35fa49 100644 --- a/rs/moq-net/src/model/record.rs +++ b/rs/moq-net/src/model/record.rs @@ -9,28 +9,28 @@ use super::track::Hole; /// One reader's record of every sequence it settled, as runs. #[derive(Default)] -pub(super) struct Record { +pub(crate) struct Record { /// The groups handed out, so none is handed out twice: not when its sequence is re-created, /// nor when another route delivers it too. - pub delivered: Delivered, + pub(super) delivered: Delivered, /// Cached groups skipped as stale, which a wider budget owes again. - pub skipped: Skipped, + pub(super) skipped: Skipped, /// Groups skipped as stale, which account for a bounded window as handed out would, until a /// wider budget owes them again. Kept under an open window too, which a reader may narrow to /// a bounded one. - pub written_off: RangeSet, + pub(super) written_off: RangeSet, /// Sequences given up and not reported yet, with why: stale skips, cuts, and upstream skips /// and drops. - pub pending: RangeMap, + pub(super) pending: RangeMap, /// Every sequence already reported, with how. - pub reported: RangeMap, + pub(super) reported: RangeMap, } /// A run a reader has yet to report, and whether it was copied from the track's runs, which a /// rescan folds in afresh: one the track forgot, such as a stale run new terms superseded, /// leaves nothing behind. #[derive(Clone, Debug, PartialEq, Eq)] -pub(super) struct Pending { +pub(crate) struct Pending { pub hole: Hole, pub received: bool, } @@ -42,14 +42,25 @@ impl Pending { } } -/// How a reader reported a run. +/// How a reader reported a run, and what its wire was told. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(super) enum Report { - /// Dropped for good, or stale until its group was handed out: no update reopens it. +pub(crate) enum Report { + /// Served on a Group Stream, or given up on a version whose drops are all final: nothing + /// goes out over it. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + Settled, + /// Dropped for good: no update reopens it, though a skip still goes out over it. Dropped, - /// Dropped as stale, which an update reopens. `received` when an upstream's stale run - /// decided it, which is the upstream's to judge again then. - Stale { received: bool }, + /// Dropped as stale (OLD), which an update reopens. + Stale { + /// An upstream's stale run decided it, which is the upstream's to judge again then. + received: bool, + /// Its Group Stream expired as stale, or it was dropped OLD for good: its publisher + /// drops it again under each update rather than this reader reopening it. + expired: bool, + /// Its group was handed out after the drop, which the group won: no update reopens it. + taken: bool, + }, /// Skipped, which replaces a drop. Skip, } @@ -59,18 +70,162 @@ impl Report { pub fn of(hole: &Hole, received: bool) -> Self { match hole { Hole::Skip => Self::Skip, - hole if hole.stale_budget().is_some() => Self::Stale { received }, + hole if hole.stale_budget().is_some() => Self::Stale { + received, + expired: false, + taken: false, + }, _ => Self::Dropped, } } /// Whether this report stops `hole` going out over it: a skip goes out over a drop, and - /// a final drop over a stale one, since each replaces what it goes out over. + /// a final drop over a stale one, since each replaces what it goes out over. An OLD drop + /// never goes out over a stale one, which already said OLD. pub fn blocks(self, hole: &Hole) -> bool { - match hole { - Hole::Skip => self == Self::Skip, - Hole::Drop { stale: None, .. } => !matches!(self, Self::Stale { .. }), - Hole::Drop { .. } => true, + match self { + Self::Settled | Self::Skip => true, + Self::Dropped | Self::Stale { taken: true, .. } => !matches!(hole, Hole::Skip), + Self::Stale { .. } => hole.error() == crate::StreamError::Old, + } + } + + /// Whether an update reopens it: a stale drop nothing settled since. + pub fn reopens(self) -> bool { + matches!( + self, + Self::Stale { + expired: false, + taken: false, + .. + } + ) + } +} + +impl Record { + /// Report the first part of `range` nothing already reported blocks, as `hole` says, + /// replacing what it goes out over. `None` when every part is blocked. + pub fn report(&mut self, range: std::ops::Range, hole: &Hole, received: bool) -> Option> { + let blocked: RangeSet = self + .reported + .overlapping(&range) + .filter(|(_, report)| report.blocks(hole)) + .map(|(run, _)| run.clone()) + .collect(); + let gap = blocked.gaps(&range).next()?; + self.reported.insert(gap.clone(), Report::of(hole, received)); + Some(gap) + } + + /// The lowest pending run's first part nothing already reported blocks, now reported, with + /// why; the rest of that run waits its turn. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub fn next_pending(&mut self) -> Option<(std::ops::Range, Hole)> { + while let Some((range, pending)) = self.pending.iter().next().map(|(range, pending)| (range.clone(), pending.clone())) { + self.pending.remove(range.clone()); + if let Some(gap) = self.report(range.clone(), &pending.hole, pending.received) { + if gap.end < range.end { + self.pending.insert(gap.end..range.end, pending.clone()); + } + return Some((gap, pending.hole)); + } + } + None + } + + /// Mark a stale report on `sequence` settled by its group being handed out. + pub fn retire(&mut self, sequence: u64) { + if let Some(end) = sequence.checked_add(1) + && let Some(Report::Stale { received, expired, .. }) = self.reported.get(&sequence).copied() + { + self.reported.insert( + sequence..end, + Report::Stale { + received, + expired, + taken: true, + }, + ); } } } + +/// What a publisher told the wire about a reader's sequences. +#[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] +impl Record { + /// Whether the wire was told of `sequence`: served, or given up. + pub fn told(&self, sequence: u64) -> bool { + self.reported.contains_key(&sequence) + } + + /// Whether `sequence` went out as a stale drop, which a group still replaces. + pub fn told_stale(&self, sequence: u64) -> bool { + matches!(self.reported.get(&sequence), Some(Report::Stale { .. })) + } + + /// The parts of `range` the wire was told nothing of. + pub fn untold(&self, range: std::ops::Range) -> RangeSet { + self.reported.gaps(&range).collect() + } + + /// Record that the wire was told `report` of `range`. A stale report keeps the parts an + /// upstream's stale run decided, which stay the upstream's to judge again. + pub fn tell(&mut self, range: std::ops::Range, report: Report) { + if range.is_empty() { + return; + } + let received: Vec<_> = match report { + Report::Stale { received: false, .. } => self + .reported + .overlapping(&range) + .filter(|(_, report)| matches!(report, Report::Stale { received: true, .. })) + .map(|(run, _)| run.start.max(range.start)..run.end.min(range.end)) + .collect(), + _ => Vec::new(), + }; + self.reported.insert(range, report); + if let Report::Stale { expired, taken, .. } = report { + for run in received { + self.reported.insert( + run, + Report::Stale { + received: true, + expired, + taken, + }, + ); + } + } + } + + /// Forget what the wire was told of `range`, which is owed again. + pub fn untell(&mut self, range: std::ops::Range) { + if !range.is_empty() { + self.reported.remove(range); + } + } + + /// Forget the drops of `range` the wire was told, which a verdict or group replaces. + pub fn replace_drops(&mut self, range: std::ops::Range) { + let drops: Vec<_> = self + .reported + .overlapping(&range) + .filter(|(_, report)| matches!(report, Report::Dropped | Report::Stale { .. })) + .map(|(run, _)| run.start.max(range.start)..run.end.min(range.end)) + .collect(); + for run in drops { + self.reported.remove(run); + } + } + + /// The runs whose Group Stream expired as stale, or that went out OLD for good, which + /// each update drops again. + pub fn expired(&self) -> RangeSet { + self.reported + .iter() + .filter(|(_, report)| matches!(report, Report::Stale { expired: true, .. })) + .map(|(run, _)| run.clone()) + .collect() + } +} diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 6bab9ccaef9..6f15b16890c 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -7749,7 +7749,7 @@ impl DropCursor { .reported .iter() .filter_map(|(run, report)| match report { - Report::Stale { received } => Some((run.clone(), *received)), + Report::Stale { received, .. } if report.reopens() => Some((run.clone(), *received)), _ => None, }) .collect(); @@ -7765,16 +7765,6 @@ impl DropCursor { (stale.into_iter().collect(), received) } - /// Retire a stale report on `sequence` once its group is handed out: the group won over - /// the drop, so no update reopens it. - fn retire(record: &mut Record, sequence: u64) { - if let Some(end) = sequence.checked_add(1) - && matches!(record.reported.get(&sequence), Some(Report::Stale { .. })) - { - record.reported.insert(sequence..end, Report::Dropped); - } - } - /// Record a run this cursor's fills gave up on as a miss, as `hole` says; a reason /// already pending, such as the track's own drop, is kept. fn give_up(record: &mut Record, run: ops::Range, hole: Hole) { @@ -7905,19 +7895,10 @@ impl DropCursor { if clipped.end < range.end { record.pending.insert(clipped.end..range.end, pending.clone()); } - let blocked: RangeSet = record - .reported - .overlapping(&clipped) - .filter(|(_, report)| report.blocks(&pending.hole)) - .map(|(run, _)| run.clone()) - .collect(); - if let Some(gap) = blocked.gaps(&clipped).next() { + if let Some(gap) = record.report(clipped, &pending.hole, pending.received) { if gap.end < range.end { record.pending.insert(gap.end..range.end, pending.clone()); } - // Replacing what it goes out over: a final report is never reopened by an update. - record.reported - .insert(gap.clone(), Report::of(&pending.hole, pending.received)); return Some((gap, pending.hole)); } } @@ -8581,7 +8562,7 @@ impl Cursor { self.record.delivered.insert(sequence); // The group won over any stale report on it. if self.drops.is_some() { - DropCursor::retire(&mut self.record, sequence); + self.record.retire(sequence); } let mut delivered = std::mem::take(&mut self.record.delivered); self.forget_past_budget(&mut delivered); @@ -9712,6 +9693,7 @@ impl Subscriber { } /// What this reader settled. + #[cfg(test)] fn record(&self) -> &Record { match &self.inner { Inner::Plain(cursor) => &cursor.record, @@ -9719,6 +9701,15 @@ impl Subscriber { } } + /// What this reader settled, for its publisher to record what its wire was told. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn record_mut(&mut self) -> &mut Record { + match &mut self.inner { + Inner::Plain(cursor) => &mut cursor.record, + Inner::Resume(_, logical) => &mut logical.record, + } + } + /// How many runs of groups written off as stale this subscriber remembers. #[cfg(test)] fn written_off_runs(&self) -> usize { @@ -9984,7 +9975,15 @@ impl Subscriber { fn poll_drop_probing(&mut self, probe: bool, waiter: &kio::Waiter) -> Poll<(ops::Range, Hole)> { match &mut self.inner { Inner::Plain(cursor) => cursor.poll_drop(probe, waiter), - Inner::Resume(resume, logical) => resume.poll_drop(&logical.record.delivered, probe, waiter), + // Each copy reports to its own route; the logical record keeps one route's run from + // going out over what another's already settled. + Inner::Resume(resume, logical) => loop { + if let Some(next) = logical.record.next_pending() { + return Poll::Ready(next); + } + let (range, hole) = std::task::ready!(resume.poll_drop(&logical.record.delivered, probe, waiter)); + logical.record.pending.insert(range, Pending::own(hole)); + }, } } From 3a3ced42ac21af7df2c033fe8b434725c81e382d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 18:11:49 -0800 Subject: [PATCH 05/37] feat(net): a reader's record is billed to the pool and folds or ends under its debt Each run of a reader's record bills a fixed cost to its track's account. A debt the track cannot pay with its groups, arrivals, and drop set falls on its readers: each folds its final prefix into a floor, and one that still owes a promise (a stale report an update reopens, a run given up and not reported, an unknown hole) ends with TooFarBehind if it holds more than a cached group's overhead and at least the average reader. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/cache.rs | 121 ++++++++++++ rs/moq-net/src/model/delivered.rs | 10 +- rs/moq-net/src/model/record.rs | 297 +++++++++++++++++++++++++++++- rs/moq-net/src/model/skipped.rs | 17 +- rs/moq-net/src/model/track.rs | 110 ++++++++++- 5 files changed, 544 insertions(+), 11 deletions(-) diff --git a/rs/moq-net/src/model/cache.rs b/rs/moq-net/src/model/cache.rs index dfff13e923e..a839183f1bc 100644 --- a/rs/moq-net/src/model/cache.rs +++ b/rs/moq-net/src/model/cache.rs @@ -724,6 +724,18 @@ pub(super) struct Track { // One finished local group's encoder, kept until another group needs it or the // account drops. Raw codec only: an Extra here would keep this account alive. encoder: Mutex>, + + // What this track's readers' records bill, how many bill anything, and the eviction debt + // the track could not pay itself, which those readers pay; see [`Reader`]. + readers: Readers, +} + +/// What a track's readers bill for their records, and the debt they owe. +#[derive(Default)] +struct Readers { + bytes: AtomicU64, + count: AtomicU64, + debt: AtomicU64, } /// The greatest location a track has written or its source reported, followed by the track @@ -831,6 +843,7 @@ impl Track { largest: Default::default(), declared: AtomicBool::new(false), encoder: Mutex::default(), + readers: Readers::default(), }); if let Some(key) = track.pool.register(&track) { let _ = track.sweep.set(key); @@ -913,6 +926,27 @@ impl Track { } } + /// Open an empty bill against this account for one reader's record. + pub(super) fn reader(self: &Arc) -> Reader { + Reader { + track: self.clone(), + bytes: 0, + } + } + + /// Hand the part of `debt` this track could not pay to its readers, up to what their + /// records bill beyond what they already owe. + pub(super) fn owe_readers(&self, debt: &mut u64) { + let readers = &self.readers; + let mut shifted = 0; + let _ = readers.debt.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |owed| { + let room = readers.bytes.load(Ordering::Relaxed).saturating_sub(owed); + shifted = room.min(*debt); + (shifted > 0).then_some(owed + shifted) + }); + *debt -= shifted; + } + /// Bill `bytes` of a failed group's head while a hand-off holds it for a successor route. /// /// Readers that stop polling can hold a hand-off indefinitely, so the head is billed @@ -954,6 +988,20 @@ impl Track { self.settle_inner(now, false); } + /// Settle eviction debt now if the pool is over its capacity, without waiting for the + /// write threshold: a reader's record grows by runs, never by enough bytes to cross it. + /// Takes the track lock: call with none held. + pub(super) fn charge_over(&self) { + if self.pool.capacity().is_none_or(|capacity| self.pool.used() <= capacity) { + return; + } + if let Some(state) = self.state.upgrade() + && let Ok(mut state) = state.write() + { + state.charge_debt(); + } + } + /// Settle from [`Pool::sweep`], dating activity and expiring every idle candidate. pub(super) fn sweep(&self) { self.settle_inner(None, true); @@ -1299,6 +1347,79 @@ impl Drop for Charge { } } +/// What one reader's record bills to its track's account, and its share of the debt the +/// track could not pay. Released on drop. +pub(super) struct Reader { + track: Arc, + bytes: u64, +} + +impl Reader { + /// Bill `bytes` in all, settling the track's debt after a growth: call with no track lock + /// held. + pub(super) fn set(&mut self, bytes: u64) { + if bytes == self.bytes { + return; + } + let readers = &self.track.readers; + match (self.bytes, bytes) { + (0, _) => readers.count.fetch_add(1, Ordering::Relaxed), + (_, 0) => readers.count.fetch_sub(1, Ordering::Relaxed), + _ => 0, + }; + if bytes > self.bytes { + let grown = bytes - self.bytes; + readers.bytes.fetch_add(grown, Ordering::Relaxed); + self.track.pool.add(grown); + self.track.written.fetch_add(grown, Ordering::Relaxed); + self.bytes = bytes; + self.track.charge_over(); + } else { + let shrunk = self.bytes - bytes; + readers.bytes.fetch_sub(shrunk, Ordering::Relaxed); + self.track.pool.sub(shrunk); + self.bytes = bytes; + } + } + + /// Bytes currently billed. + pub(super) fn size(&self) -> u64 { + self.bytes + } + + /// The debt the track's readers owe. + pub(super) fn debt(&self) -> u64 { + self.track.readers.debt.load(Ordering::Relaxed) + } + + /// Pay up to `bytes` of the readers' debt, as a record that shrank by that much. + pub(super) fn pay(&self, bytes: u64) { + let _ = self + .track + .readers + .debt + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |debt| Some(debt.saturating_sub(bytes))); + } + + /// Whether this record is one to end when folding leaves a debt: it bills more than a + /// cached group's overhead, so ending it frees more than evicting a group would, and at + /// least its readers' average, so a reader holding little never ends for one holding much. + pub(super) fn ends(&self) -> bool { + let readers = &self.track.readers; + let count = readers.count.load(Ordering::Relaxed); + self.bytes > ENTRY_OVERHEAD + && u128::from(self.bytes) * u128::from(count) >= u128::from(readers.bytes.load(Ordering::Relaxed)) + } +} + +impl Drop for Reader { + fn drop(&mut self) { + let bytes = self.bytes; + self.set(0); + self.pay(bytes); + } +} + /// Bytes a group keeps beside its cached frames, billed to the same pool and account as /// its [`Charge`] but reachable outside the group's write lock: a compressed group's /// inflated copy, which its readers build, and its codec state. Released on [`Self::clear`] or drop. diff --git a/rs/moq-net/src/model/delivered.rs b/rs/moq-net/src/model/delivered.rs index d652550051c..fd39fcb714b 100644 --- a/rs/moq-net/src/model/delivered.rs +++ b/rs/moq-net/src/model/delivered.rs @@ -137,8 +137,16 @@ impl Delivered { self.runs.range(..sequence).next_back().map(|(start, _)| *start) } + /// One past the last sequence of the run holding `sequence`, if it was handed out. + pub fn run_end(&self, sequence: u64) -> Option { + self.runs + .range(..=sequence) + .next_back() + .filter(|(_, last)| sequence <= **last) + .map(|(_, last)| last + 1) + } + /// How many runs it holds: one per gap, plus the floor. - #[cfg(test)] pub fn runs(&self) -> usize { self.runs.len() } diff --git a/rs/moq-net/src/model/record.rs b/rs/moq-net/src/model/record.rs index 6345c35fa49..d29f8fc7202 100644 --- a/rs/moq-net/src/model/record.rs +++ b/rs/moq-net/src/model/record.rs @@ -1,11 +1,21 @@ //! What one reader did with each group sequence, as runs: handed out, skipped as stale, given //! up, and what its wire was told. +use std::ops::Range; +use std::sync::Arc; + use rangemap::{RangeMap, RangeSet}; +use super::cache; use super::delivered::Delivered; use super::skipped::Skipped; use super::track::Hole; +use crate::Error; + +/// Heap a record keeps per run: its widest entry, doubled for half-full `BTreeMap` nodes as +/// the cache's own overheads are. Billed to the pool, so a peer fragmenting what a reader +/// settles costs what the cache would hold, never more. +pub(super) const RUN_OVERHEAD: u64 = 2 * size_of::<(Range, Pending)>() as u64; /// One reader's record of every sequence it settled, as runs. #[derive(Default)] @@ -24,6 +34,13 @@ pub(crate) struct Record { pub(super) pending: RangeMap, /// Every sequence already reported, with how. pub(super) reported: RangeMap, + /// Everything below was settled for good and folded away under pool pressure: handed + /// out, or reported final, so none is handed out or reported again. + floor: u64, + /// What the runs above the floor bill to the track's account, opened on the first run. + bill: Option, + /// The pool needed a run this reader was promised, so it ended. + failed: bool, } /// A run a reader has yet to report, and whether it was copied from the track's runs, which a @@ -90,6 +107,15 @@ impl Report { } } + /// Whether it settles its run for good, so a fold may forget it: served, skipped, dropped + /// for good, or stale with its group handed out since. + fn is_final(self) -> bool { + match self { + Self::Settled | Self::Skip | Self::Dropped => true, + Self::Stale { taken, .. } => taken, + } + } + /// Whether an update reopens it: a stale drop nothing settled since. pub fn reopens(self) -> bool { matches!( @@ -106,7 +132,11 @@ impl Report { impl Record { /// Report the first part of `range` nothing already reported blocks, as `hole` says, /// replacing what it goes out over. `None` when every part is blocked. - pub fn report(&mut self, range: std::ops::Range, hole: &Hole, received: bool) -> Option> { + pub fn report(&mut self, range: Range, hole: &Hole, received: bool) -> Option> { + let range = range.start.max(self.floor)..range.end; + if range.is_empty() { + return None; + } let blocked: RangeSet = self .reported .overlapping(&range) @@ -121,7 +151,7 @@ impl Record { /// The lowest pending run's first part nothing already reported blocks, now reported, with /// why; the rest of that run waits its turn. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub fn next_pending(&mut self) -> Option<(std::ops::Range, Hole)> { + pub fn next_pending(&mut self) -> Option<(Range, Hole)> { while let Some((range, pending)) = self.pending.iter().next().map(|(range, pending)| (range.clone(), pending.clone())) { self.pending.remove(range.clone()); if let Some(gap) = self.report(range.clone(), &pending.hole, pending.received) { @@ -156,7 +186,7 @@ impl Record { impl Record { /// Whether the wire was told of `sequence`: served, or given up. pub fn told(&self, sequence: u64) -> bool { - self.reported.contains_key(&sequence) + sequence < self.floor || self.reported.contains_key(&sequence) } /// Whether `sequence` went out as a stale drop, which a group still replaces. @@ -165,13 +195,17 @@ impl Record { } /// The parts of `range` the wire was told nothing of. - pub fn untold(&self, range: std::ops::Range) -> RangeSet { - self.reported.gaps(&range).collect() + pub fn untold(&self, range: Range) -> RangeSet { + let range = range.start.max(self.floor)..range.end; + match range.is_empty() { + true => RangeSet::new(), + false => self.reported.gaps(&range).collect(), + } } /// Record that the wire was told `report` of `range`. A stale report keeps the parts an /// upstream's stale run decided, which stay the upstream's to judge again. - pub fn tell(&mut self, range: std::ops::Range, report: Report) { + pub fn tell(&mut self, range: Range, report: Report) { if range.is_empty() { return; } @@ -200,14 +234,14 @@ impl Record { } /// Forget what the wire was told of `range`, which is owed again. - pub fn untell(&mut self, range: std::ops::Range) { + pub fn untell(&mut self, range: Range) { if !range.is_empty() { self.reported.remove(range); } } /// Forget the drops of `range` the wire was told, which a verdict or group replaces. - pub fn replace_drops(&mut self, range: std::ops::Range) { + pub fn replace_drops(&mut self, range: Range) { let drops: Vec<_> = self .reported .overlapping(&range) @@ -229,3 +263,250 @@ impl Record { .collect() } } + +/// What a record bills, and the debt its track left to its readers. +impl Record { + /// How many runs it holds above its floor, each billed [`RUN_OVERHEAD`]: the handed-out + /// prefix is one run that bills nothing. + fn runs(&self) -> u64 { + (self.delivered.runs().saturating_sub(1) + + self.skipped.runs() + + self.written_off.len() + + self.pending.len() + + self.reported.len()) as u64 + } + + /// Whether the pool needed a run this reader was promised, which ended it. + pub(super) fn failed(&self) -> bool { + self.failed + } + + /// Bill this record to `account`, opened on its first run, and pay what it can of the + /// debt the track left its readers by folding what it settled for good into its floor. + /// Ends the reader with [`Error::Lagged`] when a debt is left that only a run it was + /// promised could pay, and its record is one to end (see [`cache::Reader::ends`]). + /// + /// Call with no track lock held: a growing bill settles the track's debt. + pub(super) fn settle(&mut self, account: impl FnOnce() -> Arc) -> Result<(), Error> { + if self.failed { + return Err(Error::Lagged); + } + let bytes = self.runs() * RUN_OVERHEAD; + let bill = match &mut self.bill { + Some(bill) => bill, + None if bytes == 0 => return Ok(()), + None => self.bill.insert(account().reader()), + }; + bill.set(bytes); + if bill.debt() == 0 { + return Ok(()); + } + self.fold(); + let bytes = self.runs() * RUN_OVERHEAD; + let bill = self.bill.as_mut().expect("opened above"); + let freed = bill.size().saturating_sub(bytes); + bill.set(bytes); + bill.pay(freed); + if bill.debt() > 0 && bill.ends() { + self.fail(); + return Err(Error::Lagged); + } + Ok(()) + } + + /// Raise the floor over the final prefix: each run from it up to the lowest one that is + /// not final. Final are the groups handed out, the reports nothing reopens, and this + /// reader's own stale skips, which a wider budget would offer again only while the pool + /// still held their groups. Not final are the reports an update reopens, the runs given + /// up and not reported yet, and the sequences nothing settled. + fn fold(&mut self) { + let mut floor = self.floor; + loop { + let next = match self.reported.get_key_value(&floor) { + Some((run, report)) if report.is_final() => run.end, + Some(_) => break, + None if self.pending.contains_key(&floor) => break, + None => { + let local = [ + self.delivered.run_end(floor), + self.written_off.get(&floor).map(|run| run.end), + self.skipped.run_end(floor), + ]; + let Some(end) = local.into_iter().flatten().max() else { + break; + }; + // A report or a run given up inside it is judged on its own. + let span = floor..end; + [ + self.reported.overlapping(&span).next().map(|(run, _)| run.start), + self.pending.overlapping(&span).next().map(|(run, _)| run.start), + ] + .into_iter() + .flatten() + .fold(end, |end, start| end.min(start.max(floor))) + } + }; + if next <= floor { + break; + } + floor = next; + } + if floor == self.floor { + return; + } + self.floor = floor; + self.delivered.forget_below(floor); + self.skipped.forget_below(floor); + self.written_off.remove(0..floor); + self.pending.remove(0..floor); + self.reported.remove(0..floor); + } + + /// End this reader: forget every run, releasing its bill against the debt. + fn fail(&mut self) { + let bill = self.bill.take(); + *self = Self { + failed: true, + ..Self::default() + }; + drop(bill); + } +} + +#[cfg(test)] +mod test { + use std::time::Duration; + + use super::*; + use crate::StreamError; + + fn account() -> (cache::Pool, Arc) { + let pool = cache::Pool::unbounded(); + let track = cache::Track::new(pool.clone(), Default::default(), kio::Weak::new()); + (pool, track) + } + + /// Leave the track's readers `debt` to pay. + fn owe(track: &cache::Track, mut debt: u64) { + track.owe_readers(&mut debt); + } + + /// Each run past the handed-out prefix bills a fixed cost, released when the record goes. + #[test] + fn a_record_bills_each_run() { + let (pool, track) = account(); + let mut record = Record::default(); + for sequence in 0..4 { + record.delivered.insert(sequence); + } + record.settle(|| track.clone()).unwrap(); + assert_eq!(pool.used(), 0, "a lone prefix bills nothing"); + for sequence in (10..1_000).step_by(2) { + record.reported.insert(sequence..sequence + 1, Report::Dropped); + } + record.settle(|| track.clone()).unwrap(); + assert_eq!(pool.used(), 495 * RUN_OVERHEAD); + drop(record); + assert_eq!(pool.used(), 0); + } + + /// Under a debt its track could not pay, a record folds what is final into its floor: + /// handed-out groups and reports nothing reopens, however many sequences a run spans. + /// Folded sequences count as handed out and reported. + #[test] + fn a_debt_folds_the_final_prefix() { + let (pool, track) = account(); + let mut record = Record::default(); + record.delivered.insert(0); + record.reported.insert(1..u64::MAX / 2, Report::Dropped); + record.delivered.insert(u64::MAX / 2); + record.reported.insert(u64::MAX / 2 + 1..u64::MAX / 2 + 2, Report::Skip); + record.settle(|| track.clone()).unwrap(); + assert_eq!(pool.used(), 3 * RUN_OVERHEAD); + owe(&track, pool.used()); + record.settle(|| track.clone()).unwrap(); + assert_eq!(pool.used(), 0, "all of it folded"); + assert!(record.told(7) && record.delivered.contains(7), "a folded drop stays dropped"); + assert_eq!(record.report(0..u64::MAX / 2 + 2, &Hole::Skip, false), None); + let mut untold = RangeSet::new(); + untold.insert(u64::MAX / 2 + 2..u64::MAX / 2 + 3); + assert_eq!(record.untold(0..u64::MAX / 2 + 3), untold); + } + + /// A debt that needs a run this reader was promised ends it with `TooFarBehind`: a stale + /// report an update reopens, or a run above one. What is final below it still folds. + #[test] + fn a_debt_on_a_promise_ends_the_reader() { + for promise in [ + Report::Stale { + received: false, + expired: false, + taken: false, + }, + Report::Stale { + received: false, + expired: true, + taken: false, + }, + ] { + let (pool, track) = account(); + let mut record = Record::default(); + record.delivered.insert(0); + record.reported.insert(1..2, Report::Dropped); + record.reported.insert(2..3, promise); + // Enough runs above it that ending the reader frees more than evicting a group. + for sequence in (5..1_000).step_by(2) { + record.reported.insert(sequence..sequence + 1, Report::Dropped); + } + record.settle(|| track.clone()).unwrap(); + owe(&track, pool.used()); + let err = record.settle(|| track.clone()).unwrap_err(); + assert_eq!(StreamError::from(&err), StreamError::TooFarBehind, "{promise:?}"); + assert_eq!(pool.used(), 0, "{promise:?}: its record went with it"); + assert!(record.failed()); + assert!(record.settle(|| track.clone()).is_err(), "{promise:?}: it stays ended"); + } + } + + /// A run given up and not reported yet, or a sequence nothing settled, is no fold's to + /// forget either; a debt the fold pays leaves the reader running. + #[test] + fn a_debt_the_fold_pays_leaves_the_reader() { + let (pool, track) = account(); + let mut record = Record::default(); + record.delivered.insert(0); + record.reported.insert(1..2, Report::Dropped); + record.pending.insert(2..3, Pending::own(Hole::drop(StreamError::NotFound))); + record.skipped.insert(4, Duration::from_secs(1)); + record.settle(|| track.clone()).unwrap(); + owe(&track, RUN_OVERHEAD); + record.settle(|| track.clone()).unwrap(); + assert_eq!(pool.used(), 2 * RUN_OVERHEAD, "the drop folded"); + assert!(!record.told(2), "the run given up is still owed"); + } + + /// Of a track's readers, one holding little never ends for one holding much, nor one + /// holding less than a cached group costs. + #[test] + fn a_debt_ends_only_a_reader_holding_much() { + let (pool, track) = account(); + let stale = Report::Stale { + received: false, + expired: false, + taken: false, + }; + let mut small = Record::default(); + small.reported.insert(1..2, stale); + let mut large = Record::default(); + for sequence in (1..1_000).step_by(2) { + large.reported.insert(sequence..sequence + 1, stale); + } + small.settle(|| track.clone()).unwrap(); + large.settle(|| track.clone()).unwrap(); + owe(&track, RUN_OVERHEAD); + small.settle(|| track.clone()).unwrap(); + assert!(large.settle(|| track.clone()).is_err()); + assert_eq!(pool.used(), RUN_OVERHEAD); + small.settle(|| track.clone()).unwrap(); + } +} diff --git a/rs/moq-net/src/model/skipped.rs b/rs/moq-net/src/model/skipped.rs index e726c33773d..50cb1721415 100644 --- a/rs/moq-net/src/model/skipped.rs +++ b/rs/moq-net/src/model/skipped.rs @@ -126,8 +126,23 @@ impl Skipped { } } + /// One past the last sequence of the run holding `sequence`, if it was skipped. + pub fn run_end(&self, sequence: u64) -> Option { + let skipped = self.0.as_ref()?; + skipped.runs.get_key_value(&sequence).map(|(run, _)| run.end().saturating_add(1)) + } + + /// Forget every run below `floor`, which a fold settled. + pub fn forget_below(&mut self, floor: u64) { + if let Some(skipped) = &mut self.0 + && let Some(below) = floor.checked_sub(1) + && !skipped.runs.is_empty() + { + skipped.runs.remove(0..=below); + } + } + /// How many runs it holds. - #[cfg(test)] pub fn runs(&self) -> usize { self.0.as_ref().map_or(0, |skipped| skipped.runs.len()) } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 6f15b16890c..0612334028a 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -2872,8 +2872,12 @@ impl TrackState { let mut paid = 0u64; let mut scanned = 0usize; let mut fresh = Vec::new(); + // Whether every cold group was offered, so a debt left at the end is one the track + // itself cannot pay. + let mut swept = true; for _ in 0..self.evict.len() { if self.debt == 0 || paid >= cap || scanned >= EVICT_SCAN { + swept = self.debt == 0 || paid >= cap; break; } let Some((sequence, stamp)) = self.evict.pop_front() else { @@ -2957,6 +2961,11 @@ impl TrackState { paid = paid.saturating_add(freed); } } + // Last, the readers' records: nothing cheaper is left, so each reader folds what it + // settled for good, and one that cannot ends rather than forgo a promise. + if swept && self.debt > 0 && paid < cap { + self.cache.owe_readers(&mut self.debt); + } paid } @@ -9508,6 +9517,18 @@ impl Subscriber { /// The next group in arrival order, or in sequence order for `ordered`, skipping any /// already handed out. fn poll_group(&mut self, ordered: bool, waiter: &kio::Waiter) -> Poll>> { + let res = self.poll_group_unsettled(ordered, waiter); + // What the read settled is billed now, and pays what it can of its readers' debt. + match self.settle_record() { + Ok(()) => res, + Err(err) => Poll::Ready(Err(err)), + } + } + + fn poll_group_unsettled(&mut self, ordered: bool, waiter: &kio::Waiter) -> Poll>> { + if self.record().failed() { + return Poll::Ready(Err(Error::Lagged)); + } // Ahead of the meter: a read that parks again counts only what a peek skipped. let walk = idle::Walk::new(ordered); match &self.inner { @@ -9692,8 +9713,17 @@ impl Subscriber { self.record().delivered.runs() } + /// Bill what this reader settled, and pay what it can of the debt its track left its + /// readers; [`Error::Lagged`] once that needed a run it was promised. + fn settle_record(&mut self) -> Result<()> { + let (state, record) = match &mut self.inner { + Inner::Plain(cursor) => (&cursor.state, &mut cursor.record), + Inner::Resume(_, logical) => (&logical.state, &mut logical.record), + }; + record.settle(|| state.read().cache.clone()) + } + /// What this reader settled. - #[cfg(test)] fn record(&self) -> &Record { match &self.inner { Inner::Plain(cursor) => &cursor.record, @@ -9973,6 +10003,11 @@ impl Subscriber { #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] fn poll_drop_probing(&mut self, probe: bool, waiter: &kio::Waiter) -> Poll<(ops::Range, Hole)> { + // What a publisher told its wire since is billed now. A reader the pool ended reports + // nothing more: its reads fail with the reason. + if self.settle_record().is_err() { + return Poll::Pending; + } match &mut self.inner { Inner::Plain(cursor) => cursor.poll_drop(probe, waiter), // Each copy reports to its own route; the logical record keeps one route's run from @@ -10138,6 +10173,8 @@ impl Subscriber { // sources even while nobody reads. Inner::Resume(resume, _) => resume.mirror().sync(), } + // A reader the pool ended reports it on its next read. + let _ = self.settle_record(); Ok(()) } @@ -24161,6 +24198,75 @@ mod test { ); } + /// A flood a reader reported, fragmented into runs alternating with upstream skips, + /// bills its record too. Once the pool shrinks under it and its track has shed what it + /// can, the reader folds the final runs into its floor and stays within the pool, while + /// one holding stale reports an update would reopen ends with `TooFarBehind` rather than + /// forgo the re-offer. + #[test] + fn a_reported_flood_folds_or_ends_the_reader() { + const RUNS: u64 = 1000; + let budget = Duration::from_secs(1); + for stale in [false, true] { + let (producer, pool) = pooled_producer(1 << 30); + producer.received(); + let waiter = kio::Waiter::noop(); + let mut subscriber = producer.subscribe( + Subscription::default() + .with_delay(crate::track::Delay::default().with_max(budget)) + .with_range(0..), + ); + let hole = match stale { + true => Hole::stale(budget), + false => Hole::drop(StreamError::Cancel), + }; + let mut reported = RangeSet::new(); + let flood = |run: u64| { + producer.give_up(2 * run..2 * run + 1, hole.clone()); + producer.give_up(2 * run + 1..2 * run + 2, Hole::Skip); + }; + for run in 0..RUNS { + flood(run); + } + while let Poll::Ready((run, _)) = subscriber.poll_drop(&waiter) { + assert!(!reported.overlaps(&run), "stale={stale}: {run:?} reported twice"); + reported.insert(run); + } + assert_eq!(reported.iter().map(|run| run.end - run.start).sum::(), 2 * RUNS, "stale={stale}"); + assert!(subscriber.poll_recv_group(&waiter).is_pending(), "stale={stale}"); + let used = pool.used(); + assert!(used >= 2 * RUNS * super::super::record::RUN_OVERHEAD, "stale={stale}: the record is billed"); + + // The flood goes on into a smaller pool, whose debt each drop accrues. + let capacity = 64 * (RUN_OVERHEAD + BORN_OVERHEAD); + pool.resize(capacity); + let mut read = Poll::Pending; + for run in RUNS..2 * RUNS { + flood(run); + read = subscriber.poll_recv_group(&waiter); + if read.is_ready() { + break; + } + while let Poll::Ready((run, _)) = subscriber.poll_drop(&waiter) { + assert!(!reported.overlaps(&run), "{run:?}, folded, was reported again"); + reported.insert(run); + } + } + match stale { + true => assert!( + matches!(read, Poll::Ready(Err(Error::Lagged))), + "the stale reports could not fold" + ), + false => assert!(read.is_pending(), "the reader runs on"), + } + assert!( + pool.used() <= capacity + 4 * (RUN_OVERHEAD + BORN_OVERHEAD), + "stale={stale}: the pool holds {} past {capacity}", + pool.used() + ); + } + } + /// A drop-only flood of disjoint runs bills the pool and sheds its lowest runs to stay /// within it, with no cached group to evict; a shed run is unknown again, so a wire cursor /// reports only the runs the track still holds. @@ -24212,6 +24318,8 @@ mod test { while let Poll::Ready((run, _)) = subscriber.poll_drop(&waiter) { reported.push(run); } + // Billing the reader's record shed the track's lowest runs before it reported them. + let held: Vec<_> = producer.state.read().drops.iter().map(|(run, _)| run.clone()).collect(); assert_eq!(reported[1..], held[..]); } From 8b40da8b66137448ec118e81c40a02f2c7193a56 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 18:16:15 -0800 Subject: [PATCH 06/37] feat(net): bill a reader's given-up misses and a tail's finished streams A reader's given-up misses bill with its record and are forgotten below its folded floor. A lite subscription's record of the streams finished since its last update bills to its copy's account. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/subscriber.rs | 13 +++++++ rs/moq-net/src/model/record.rs | 56 +++++++++++++++++++++--------- rs/moq-net/src/model/track.rs | 47 ++++++++++++++++++++++--- rs/moq-net/src/model/track/miss.rs | 14 ++++++++ rs/moq-net/src/tail.rs | 23 ++++++++++++ 5 files changed, 131 insertions(+), 22 deletions(-) diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index ab10fbfd8a2..1109417b178 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -286,6 +286,9 @@ impl Drop for ParkWait { impl TrackEntry { fn new(copy: Option, tail: kio::Producer) -> Self { + if let Some(copy) = © { + Self::bill_tail(&tail, copy); + } Self { copy, parked: None, @@ -317,7 +320,17 @@ impl TrackEntry { } /// Give the copy to the subscription, opening the groups of the streams parked on it. + /// Bill what the tail keeps to the copy's account. + fn bill_tail(tail: &kio::Producer, copy: &Accepted) { + // Opened before the tail's lock, which is never held across the track's. + let bill = copy.producer.bill(); + if let Ok(mut tail) = tail.write() { + tail.bill(bill); + } + } + fn accept(&mut self, copy: Accepted, version: Version, now: crate::time::Instant) { + Self::bill_tail(&self.tail, ©); self.copy = Some(copy); let Some(parked) = self.parked.clone() else { return; diff --git a/rs/moq-net/src/model/record.rs b/rs/moq-net/src/model/record.rs index d29f8fc7202..c4823ce649d 100644 --- a/rs/moq-net/src/model/record.rs +++ b/rs/moq-net/src/model/record.rs @@ -264,6 +264,23 @@ impl Record { } } +/// Runs a reader keeps beside its record, billed with it and settled by its fold: the +/// misses it gave up, which it never asks a fill for again. +pub(super) trait Beside { + /// How many runs it holds. + fn runs(&self) -> u64; + /// Forget every run below `floor`, which the record folded. + fn forget_below(&mut self, floor: u64); +} + +impl Beside for () { + fn runs(&self) -> u64 { + 0 + } + + fn forget_below(&mut self, _floor: u64) {} +} + /// What a record bills, and the debt its track left to its readers. impl Record { /// How many runs it holds above its floor, each billed [`RUN_OVERHEAD`]: the handed-out @@ -287,11 +304,15 @@ impl Record { /// promised could pay, and its record is one to end (see [`cache::Reader::ends`]). /// /// Call with no track lock held: a growing bill settles the track's debt. - pub(super) fn settle(&mut self, account: impl FnOnce() -> Arc) -> Result<(), Error> { + pub(super) fn settle( + &mut self, + account: impl FnOnce() -> Arc, + beside: &mut dyn Beside, + ) -> Result<(), Error> { if self.failed { return Err(Error::Lagged); } - let bytes = self.runs() * RUN_OVERHEAD; + let bytes = (self.runs() + beside.runs()) * RUN_OVERHEAD; let bill = match &mut self.bill { Some(bill) => bill, None if bytes == 0 => return Ok(()), @@ -302,7 +323,8 @@ impl Record { return Ok(()); } self.fold(); - let bytes = self.runs() * RUN_OVERHEAD; + beside.forget_below(self.floor); + let bytes = (self.runs() + beside.runs()) * RUN_OVERHEAD; let bill = self.bill.as_mut().expect("opened above"); let freed = bill.size().saturating_sub(bytes); bill.set(bytes); @@ -399,12 +421,12 @@ mod test { for sequence in 0..4 { record.delivered.insert(sequence); } - record.settle(|| track.clone()).unwrap(); + record.settle(|| track.clone(), &mut ()).unwrap(); assert_eq!(pool.used(), 0, "a lone prefix bills nothing"); for sequence in (10..1_000).step_by(2) { record.reported.insert(sequence..sequence + 1, Report::Dropped); } - record.settle(|| track.clone()).unwrap(); + record.settle(|| track.clone(), &mut ()).unwrap(); assert_eq!(pool.used(), 495 * RUN_OVERHEAD); drop(record); assert_eq!(pool.used(), 0); @@ -421,10 +443,10 @@ mod test { record.reported.insert(1..u64::MAX / 2, Report::Dropped); record.delivered.insert(u64::MAX / 2); record.reported.insert(u64::MAX / 2 + 1..u64::MAX / 2 + 2, Report::Skip); - record.settle(|| track.clone()).unwrap(); + record.settle(|| track.clone(), &mut ()).unwrap(); assert_eq!(pool.used(), 3 * RUN_OVERHEAD); owe(&track, pool.used()); - record.settle(|| track.clone()).unwrap(); + record.settle(|| track.clone(), &mut ()).unwrap(); assert_eq!(pool.used(), 0, "all of it folded"); assert!(record.told(7) && record.delivered.contains(7), "a folded drop stays dropped"); assert_eq!(record.report(0..u64::MAX / 2 + 2, &Hole::Skip, false), None); @@ -458,13 +480,13 @@ mod test { for sequence in (5..1_000).step_by(2) { record.reported.insert(sequence..sequence + 1, Report::Dropped); } - record.settle(|| track.clone()).unwrap(); + record.settle(|| track.clone(), &mut ()).unwrap(); owe(&track, pool.used()); - let err = record.settle(|| track.clone()).unwrap_err(); + let err = record.settle(|| track.clone(), &mut ()).unwrap_err(); assert_eq!(StreamError::from(&err), StreamError::TooFarBehind, "{promise:?}"); assert_eq!(pool.used(), 0, "{promise:?}: its record went with it"); assert!(record.failed()); - assert!(record.settle(|| track.clone()).is_err(), "{promise:?}: it stays ended"); + assert!(record.settle(|| track.clone(), &mut ()).is_err(), "{promise:?}: it stays ended"); } } @@ -478,9 +500,9 @@ mod test { record.reported.insert(1..2, Report::Dropped); record.pending.insert(2..3, Pending::own(Hole::drop(StreamError::NotFound))); record.skipped.insert(4, Duration::from_secs(1)); - record.settle(|| track.clone()).unwrap(); + record.settle(|| track.clone(), &mut ()).unwrap(); owe(&track, RUN_OVERHEAD); - record.settle(|| track.clone()).unwrap(); + record.settle(|| track.clone(), &mut ()).unwrap(); assert_eq!(pool.used(), 2 * RUN_OVERHEAD, "the drop folded"); assert!(!record.told(2), "the run given up is still owed"); } @@ -501,12 +523,12 @@ mod test { for sequence in (1..1_000).step_by(2) { large.reported.insert(sequence..sequence + 1, stale); } - small.settle(|| track.clone()).unwrap(); - large.settle(|| track.clone()).unwrap(); + small.settle(|| track.clone(), &mut ()).unwrap(); + large.settle(|| track.clone(), &mut ()).unwrap(); owe(&track, RUN_OVERHEAD); - small.settle(|| track.clone()).unwrap(); - assert!(large.settle(|| track.clone()).is_err()); + small.settle(|| track.clone(), &mut ()).unwrap(); + assert!(large.settle(|| track.clone(), &mut ()).is_err()); assert_eq!(pool.used(), RUN_OVERHEAD); - small.settle(|| track.clone()).unwrap(); + small.settle(|| track.clone(), &mut ()).unwrap(); } } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 0612334028a..4e29bebacb7 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -4016,6 +4016,28 @@ pub struct Producer { stats: stats::Scope, } +/// Bytes a session keeps for a track beside its cache, billed to the track's account and +/// released on drop. +pub(crate) struct Bill(cache::Extra); + +#[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] +impl Bill { + /// Bill `bytes` in all. + pub(crate) fn set(&self, bytes: u64) { + let size = self.0.size(); + match bytes > size { + true => self.0.add(bytes - size), + false => self.0.sub(size - bytes), + } + } +} + +impl std::fmt::Debug for Bill { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("Bill").field(&self.0.size()).finish() + } +} + impl Producer { /// The immutable publisher priority committed when this track was accepted. #[cfg(test)] @@ -4908,6 +4930,13 @@ impl Producer { self.prev_subscription = seen; } + /// Open a bill against this track's account, for what a session keeps for it beside the + /// cache, such as its tail's record of finished streams. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn bill(&self) -> Bill { + Bill(self.state.read().cache.extra()) + } + /// How long a subscriber session's tail waits for a late group: `budget`, bounded by how /// long the publisher keeps one. The aggregate asked upstream is not bounded by it. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] @@ -9716,11 +9745,19 @@ impl Subscriber { /// Bill what this reader settled, and pay what it can of the debt its track left its /// readers; [`Error::Lagged`] once that needed a run it was promised. fn settle_record(&mut self) -> Result<()> { - let (state, record) = match &mut self.inner { - Inner::Plain(cursor) => (&cursor.state, &mut cursor.record), - Inner::Resume(_, logical) => (&logical.state, &mut logical.record), - }; - record.settle(|| state.read().cache.clone()) + match &mut self.inner { + Inner::Plain(cursor) => { + let state = &cursor.state; + match cursor.misses.as_deref_mut() { + Some(misses) => cursor.record.settle(|| state.read().cache.clone(), misses), + None => cursor.record.settle(|| state.read().cache.clone(), &mut ()), + } + } + Inner::Resume(_, logical) => { + let state = &logical.state; + logical.record.settle(|| state.read().cache.clone(), &mut ()) + } + } } /// What this reader settled. diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index d507b170375..876c4d2a9a7 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -1632,6 +1632,20 @@ fn looser(ours: Option, asked: Option) -> bool { } } +/// The misses a reader gave up, billed with its record and forgotten below its floor once it +/// folds: it hands out nothing there, so never asks a fill for it again. +impl super::super::record::Beside for Misses { + fn runs(&self) -> u64 { + self.given_up.len() as u64 + } + + fn forget_below(&mut self, floor: u64) { + if floor > 0 { + self.given_up.remove(0..floor); + } + } +} + #[cfg(test)] mod tests { use super::super::{Delay, Servable, Subscription}; diff --git a/rs/moq-net/src/tail.rs b/rs/moq-net/src/tail.rs index a5cb88c1778..d6c6671d236 100644 --- a/rs/moq-net/src/tail.rs +++ b/rs/moq-net/src/tail.rs @@ -34,8 +34,16 @@ pub(crate) struct Tail { // that update names one only if it was sent before the publisher served the group, so the // group won over it, even once the cache evicted it. finished: RangeSet, + // What `finished` bills to the track's account, [`FINISHED_OVERHEAD`] per run, once the + // subscription has a copy to bill. + bill: Option, } +/// Heap `finished` keeps per run, doubled for half-full `BTreeMap` nodes as the cache's own +/// overheads are. Billed, so a publisher fragmenting the streams it finishes between updates +/// costs what the cache would hold. +const FINISHED_OVERHEAD: u64 = 2 * size_of::>() as u64; + impl Default for Tail { fn default() -> Self { Self::new(GRACE) @@ -51,6 +59,19 @@ impl Tail { active: 0, ended: false, finished: RangeSet::new(), + bill: None, + } + } + + /// Bill what this tail keeps to `bill`, the subscription's copy's account. + pub fn bill(&mut self, bill: crate::model::track::Bill) { + self.bill = Some(bill); + self.rebill(); + } + + fn rebill(&self) { + if let Some(bill) = &self.bill { + bill.set(self.finished.len() as u64 * FINISHED_OVERHEAD); } } @@ -119,6 +140,7 @@ impl Tail { /// one is no verdict, so none needs them. pub fn updated(&mut self) { self.finished = RangeSet::new(); + self.rebill(); } /// Whether a data stream whose header arrived is still being read. @@ -167,6 +189,7 @@ impl Reading { && let Ok(mut state) = tail.write() { state.finished.insert(group..group.saturating_add(1)); + state.rebill(); } } From a57149a672ac75a0fb639637d55934513fc153c2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 18:26:43 -0800 Subject: [PATCH 07/37] feat(net): bill each js/net reader's record and fold or end it under its debt Mirrors the Rust record: the lite publisher's served, dropped, dropped-stale, and expired runs move into the reader's own reported runs, a resumed reader's copies report through them, each reader bills its record to its track's account, and a debt the track cannot pay folds its final prefix or ends it with TooFarBehind. A tail's finished streams bill too. The flood tests now drive pressure with groups the reader reads, so no unknown hole lands below its reports. Co-Authored-By: Claude Opus 5.5 --- js/net/src/cache.test.ts | 61 ++++++- js/net/src/cache.ts | 103 ++++++++++- js/net/src/delivered.ts | 6 + js/net/src/internal.ts | 18 +- js/net/src/lite/publisher.ts | 133 ++++++-------- js/net/src/lite/subscriber.ts | 2 + js/net/src/tail.ts | 19 ++ js/net/src/track.test.ts | 2 +- js/net/src/track.ts | 316 +++++++++++++++++++++++++++++---- rs/moq-net/src/model/record.rs | 17 ++ rs/moq-net/src/model/track.rs | 45 +++-- 11 files changed, 584 insertions(+), 138 deletions(-) diff --git a/js/net/src/cache.test.ts b/js/net/src/cache.test.ts index ae80005a821..f57f239b1e4 100644 --- a/js/net/src/cache.test.ts +++ b/js/net/src/cache.test.ts @@ -309,17 +309,74 @@ test("a drop flood stays within the pool", () => { reported.push(first.start); } } + const used = pool.used; + // A probe's record bills too: an unbounded pool lets it read every run without shedding one. + pool.resize(); const held = dropsHeld(producer); + pool.resize(capacity); // Each drop is accounted in the arrivals too, whose gaps are folded before any run is shed. expect(hooks.arrivalRuns(producer)).toBeLessThanOrEqual(2); - expect(pool.used).toBe(held.length * cache.RUN_OVERHEAD + (hooks.arrivalRuns(producer) - 1) * ARRIVAL_OVERHEAD); + expect(used).toBe(held.length * cache.RUN_OVERHEAD + (hooks.arrivalRuns(producer) - 1) * ARRIVAL_OVERHEAD); expect(held[0]).toBeGreaterThan(0n); expect(held.every((start, i) => i === 0 || start === held[i - 1] + 2n)).toBe(true); expect(held[held.length - 1]).toBe(2n * (runs - 1n)); for (let run = hooks.tryDrop(reader); run; run = hooks.tryDrop(reader)) reported.push(run.start); - expect(reported.slice(1)).toEqual(held); + // Billing the reader's record shed the track's lowest runs before it reported them. + pool.resize(); + expect(reported.slice(1)).toEqual(dropsHeld(producer)); }); +// A flood a reader reported, fragmented into runs alternating with upstream skips, bills its +// record too. Once the pool shrinks under it and its track has shed what it can, the reader folds +// the final runs and the groups it read into its floor and stays within the pool, while one +// holding stale reports an update would reopen ends with TooFarBehind rather than forgo the +// re-offer. Mirrors Rust +// `a_reported_flood_folds_or_ends_the_reader`. +for (const stale of [false, true]) { + test(`a reported flood ${stale ? "ends" : "folds into"} the reader`, () => { + const runs = 1000n; + const budget = 1000; + const pool = new cache.Pool({ capacity: 1 << 30 }); + const producer = pooled(pool); + hooks.receiveTrack(producer); + const reader = producer.subscribe({ delay: { max: Duration.millis(budget) } }); + const hole = stale ? staleHole(budget) : { error: new StreamError(StreamCode.Cancel) }; + const flood = (run: bigint) => { + hooks.dropTrackGroups(producer, 2n * run, 2n * run + 1n, hole); + producer.skipGroups(2n * run + 1n, 2n * run + 2n); + }; + for (let run = 0n; run < runs; run++) flood(run); + let reported = 0n; + for (let run = hooks.tryDrop(reader); run; run = hooks.tryDrop(reader)) reported += run.end - run.start; + expect(reported).toBe(2n * runs); + expect(reader.tryRecvGroup()).toBeUndefined(); + expect(pool.used).toBeGreaterThanOrEqual(Number(2n * runs) * cache.READER_RUN_OVERHEAD); + + // The pool shrinks, and groups the reader reads above the flood accrue its debt: the track + // sheds its own drop set first, then the reader pays. + const capacity = 64 * cache.RUN_OVERHEAD; + pool.resize(capacity); + let error: unknown; + for (let sequence = 2n * runs; sequence < 2n * runs + 64n && error === undefined; sequence++) { + const group = producer.createGroup(sequence); + group.writeFrame({ payload: new Uint8Array(16 * 1024) }); + group.finish(); + try { + expect(reader.tryRecvGroup()?.sequence).toBe(sequence); + } catch (err) { + error = err; + } + } + if (stale) expect(error).toBeInstanceOf(TooFarBehind); + else { + expect(error).toBeUndefined(); + expect(hooks.tryDrop(reader)).toBeUndefined(); + } + expect(pool.used).toBeLessThanOrEqual(capacity + 2 * (16 * 1024 + cache.ENTRY_OVERHEAD)); + reader.close(); + }); +} + // A peer fragmenting its arrivals under a grace that never folds a gap bills each run to the pool, // which folds the oldest gaps early to stay within it. Mirrors Rust `an_arrival_flood_stays_within_the_pool`. test("an arrival flood stays within the pool", () => { diff --git a/js/net/src/cache.ts b/js/net/src/cache.ts index aa11b0cb6bf..40facc3f970 100644 --- a/js/net/src/cache.ts +++ b/js/net/src/cache.ts @@ -36,6 +36,10 @@ export const MIRROR_OVERHEAD = 4 * 1024; export const RUN_OVERHEAD = 336; /** Bookkeeping billed per entry of a drop set's expiry index. */ export const BORN_OVERHEAD = 64; +// A reader's record keeps an object per run in a sorted array, about what a drop run costs +// without its change-log entry. +/** Bookkeeping billed per run of a reader's record. */ +export const READER_RUN_OVERHEAD = 256; // The coarse clock behind access stamps, as Rust's: the expiry window rounds up to it. const TICK = Duration.millis(100); @@ -267,6 +271,9 @@ export class Track { // Where the next bounded idle scan starts, rotating so fresh groups at the front do not // starve the ones behind them. #cursor = 0; + // What this track's readers' records bill, how many bill anything, and the debt the track + // could not pay itself, which those readers pay; see {@link Reader}. + #readers = { bytes: 0, count: 0, debt: 0 }; constructor(pool: Pool, owner: Owner) { this.pool = pool; @@ -295,6 +302,11 @@ export class Track { this.pool.sub(n); } + /** Open an empty bill for one reader's record. */ + reader(): Reader { + return new Reader(this as Track, this.#readers); + } + /** Whether the account caches any group. */ get empty(): boolean { return this.#live.size === 0; @@ -373,9 +385,15 @@ export class Track { let paid = 0; let scanned = 0; let evicted = false; + // Whether every cold group was offered, so a debt left at the end is one the track + // itself cannot pay. + let swept = true; const fresh: E[] = []; for (let i = this.#order.length - this.#head; i > 0; i--) { - if (this.#debt === 0 || paid >= cap || scanned >= EVICT_SCAN) break; + if (this.#debt === 0 || paid >= cap || scanned >= EVICT_SCAN) { + swept = this.#debt === 0 || paid >= cap; + break; + } // An eviction that emptied the account reset the order. if (this.#head >= this.#order.length) break; const slot = this.#order[this.#head++]; @@ -426,10 +444,28 @@ export class Track { this.#debt -= Math.min(freed, this.#debt); paid += freed; } + // Last, the readers' records: nothing cheaper is left, so each reader folds what it + // settled for good, and one that cannot ends rather than forgo a promise. + if (swept && this.#debt > 0 && paid < cap) { + const readers = this.#readers; + const shifted = Math.min(this.#debt, Math.max(0, readers.bytes - readers.debt)); + readers.debt += shifted; + this.#debt -= shifted; + } this.#compact(); return evicted; } + /** + * Pay the eviction debt now if the pool is over its capacity, without waiting for the write + * threshold: a reader's record grows by runs, never by enough bytes to cross it. + */ + chargeOver(): void { + const capacity = this.pool.capacity; + if (capacity === undefined || this.pool.used <= capacity) return; + if (this.chargeDebt()) this.#owner.settle(); + } + /** * Evict the groups idle past the pool's window, never the live edge: every one when `full`, * or else a bounded scan from a rotating cursor. Returns whether it evicted a group. @@ -503,6 +539,71 @@ export class Track { let makeCharge: (track: Track) => Charge; +/** + * What one reader's record bills to its track's account, and its share of the debt the track + * could not pay. Mirrors the Rust `cache::Reader`; {@link close} releases it. + */ +export class Reader { + #track: Track; + #readers: { bytes: number; count: number; debt: number }; + #bytes = 0; + + constructor(track: Track, readers: { bytes: number; count: number; debt: number }) { + this.#track = track; + this.#readers = readers; + } + + /** Bytes currently billed. */ + get size(): number { + return this.#bytes; + } + + /** The debt the track's readers owe. */ + get debt(): number { + return this.#readers.debt; + } + + /** Bill `bytes` in all, paying the track's debt after a growth. */ + set(bytes: number): void { + const before = this.#bytes; + if (bytes === before) return; + const readers = this.#readers; + if (before === 0) readers.count++; + else if (bytes === 0) readers.count--; + readers.bytes += bytes - before; + this.#bytes = bytes; + if (bytes > before) { + this.#track.pool.add(bytes - before); + this.#track.wrote(bytes - before); + this.#track.chargeOver(); + } else { + this.#track.pool.sub(before - bytes); + } + } + + /** Pay up to `bytes` of the readers' debt, as a record that shrank by that much. */ + pay(bytes: number): void { + this.#readers.debt = Math.max(0, this.#readers.debt - bytes); + } + + /** + * Whether this record is one to end when folding leaves a debt: it bills more than a cached + * group's overhead, so ending it frees more than evicting a group would, and at least its + * readers' average, so a reader holding little never ends for one holding much. + */ + get ends(): boolean { + const readers = this.#readers; + return this.#bytes > ENTRY_OVERHEAD && this.#bytes * readers.count >= readers.bytes; + } + + /** Release the bill, paying the readers' debt with it. */ + close(): void { + const bytes = this.#bytes; + this.set(0); + this.pay(bytes); + } +} + /** * What one cached group bills to its track's account: its bytes, overhead included, and its * access stamp, which the pool's mean samples while the group is evictable. Cleared once the diff --git a/js/net/src/delivered.ts b/js/net/src/delivered.ts index f180e5645df..74bb6c35ad2 100644 --- a/js/net/src/delivered.ts +++ b/js/net/src/delivered.ts @@ -105,6 +105,12 @@ export class Delivered { return first[0] === 0n ? this.#runs[1]?.[0] : first[0]; } + /** One past the last sequence of the run holding `sequence`, if it was handed out. */ + runEnd(sequence: bigint): bigint | undefined { + const index = this.#below(sequence); + return index >= 0 && sequence <= this.#runs[index][1] ? this.#runs[index][1] + 1n : undefined; + } + /** How many runs it holds: one per gap, plus the floor. */ get runs(): number { return this.#runs.length; diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index b774debbce9..80f4c2ff156 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -30,6 +30,7 @@ import type { Request, Subscriber, Subscription, + WireRecord, } from "./track.ts"; import { max, min } from "./util/sequence.ts"; import * as Varint from "./varint.ts"; @@ -706,10 +707,16 @@ export const hooks: { /** Forget the gaps in another reader's `delivered` below each group past this subscriber's budget. */ forgetPastBudget: (subscriber: Subscriber, delivered: Delivered) => void; /** - * Take `[start, end)` back out of what a subscriber's wire cursor reported dropped: a drop - * below where SUBSCRIBE_START resolved never goes out, so a wider budget may still serve it. + * What a lite publisher told its wire about a subscriber's sequences, kept in the + * subscriber's own record. Mirrors the Rust crate-private `track::Subscriber::record_mut`. */ - unreport: (subscriber: Subscriber, start: bigint, end: bigint) => void; + wire: (subscriber: Subscriber) => WireRecord; + /** + * Open a bill against a track's cache account for what a session keeps for it beside the + * cache, such as its tail's record of finished streams: each call bills that many bytes in + * all. Mirrors the Rust crate-private `track::Producer::bill`. + */ + billTrack: (producer: Producer) => (bytes: number) => void; /** * Re-offer what a subscriber gave up as stale through dispositions, as a wire whose every update * reopens its stale drops does: each run is judged again under the new terms. Returns the runs @@ -1014,7 +1021,10 @@ export const hooks: { forgetPastBudget: () => { throw new Error("track.ts not loaded"); }, - unreport: () => { + wire: () => { + throw new Error("track.ts not loaded"); + }, + billTrack: () => { throw new Error("track.ts not loaded"); }, supersedeStale: () => { diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index fb18179d911..625e7658e1e 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -31,7 +31,6 @@ import { insertRun, presented, type Run, - removeRun, runGaps, } from "../internal.ts"; import type { Consumer as OriginConsumer } from "../origin.ts"; @@ -42,6 +41,7 @@ import { rescale } from "../rescale.ts"; import { concat, encodeVarint, type Reader, type Stream, type StreamVersion, Writer } from "../stream.ts"; import { floorAt, type Timescale, Timestamp } from "../time.ts"; import type * as track from "../track.ts"; +import type { Reported } from "../track.ts"; import { untilAborted } from "../util/abort.ts"; import { nextChange } from "../util/change.ts"; import { GivenUp } from "../util/given_up.ts"; @@ -1041,10 +1041,9 @@ export class Publisher { let controls: SubscriptionControls | undefined; // Lite-05 and later name every sequence below the end that this subscription never // got with SUBSCRIBE_DROP, so the subscriber settles without waiting out its grace - // for a group that will not come. The group sequences served or - // dropped. A gap that never arrived stays until the end sweep names it: this side - // cannot tell when the subscriber stopped waiting for it. Undefined on other versions. - const served: Run[] | undefined = hasTrackStream(this.version) ? [] : undefined; + // for a group that will not come. A gap that never arrived stays until the end sweep + // names it: this side cannot tell when the subscriber stopped waiting for it. + const named = hasTrackStream(this.version); try { // Every draft waits for the track's info, as Rust does: it surfaces a rejected track @@ -1167,7 +1166,7 @@ export class Publisher { stamp, publisher, compress, - served, + named, floor, bounds, update: msg.update, @@ -1337,7 +1336,9 @@ export class Publisher { /** The track's publisher priority, breaking ties between subscriptions. */ publisher: number; compress: boolean; - served: Run[] | undefined; + // Whether the wire names what it never served (lite-05 and later), recording what it + // told in the reader's own record. + named: boolean; // The read cursor's floor; see {@link cursorFloor}. floor: bigint; bounds: FrameBounds; @@ -1347,7 +1348,10 @@ export class Publisher { initial: bigint; }, ): Promise { - const { sub, broadcast, stamp, publisher, compress, served, bounds, initial } = serving; + const { sub, broadcast, stamp, publisher, compress, named, bounds, initial } = serving; + // What the wire was told of each sequence: served, or given up, and how. Kept in the + // reader's own record, which its drops consult too. Undefined before lite-05. + const record = named ? hooks.wire(track) : undefined; let updateNumber = serving.update; // The read cursor's floor; the subscription's ranges cap it. let floor = serving.floor; @@ -1430,73 +1434,45 @@ export class Publisher { // Whether lite-07 names `hole` stale: an OLD drop, which the next update reopens. const sentStale = (hole: Hole) => !hole.skip && toStreamCode(hole.error) === StreamCode.Old; - // Lite-07: the part of `served` named in SUBSCRIBE_DROP, never served. A skip replaces a - // drop, so it is sent over one of these, but never over a group served. - let dropped: Run[] = []; - // Lite-07: the part of `dropped` named stale (OLD), which a final drop or a group replaces, - // as one a route the front switched to delivers after the update that reopens it. - let droppedStale: Run[] = []; - // Lite-07 groups whose stream expired as stale, which every update drops again under its - // number: the subscriber reopens each stale drop then, and the stream already reset. - const expired: Run[] = []; + // What the wire is told when `hole` goes out. Mirrors the Rust `TrackRun::report`. + const reportOf = (hole: Hole): Omit => { + // Every drop is final before lite-07. + if (!hasSkip(this.version)) return { kind: "settled" }; + if (hole.skip) return { kind: "skip" }; + // The wire names every OLD drop stale, which the next update reopens. One given up as + // final, such as one relayed from a version whose drops are all final, is dropped again + // by each update, as an expired stream is. + if (sentStale(hole)) return { kind: "stale", ...(hole.stale === undefined ? { expired: true } : {}) }; + return { kind: "dropped" }; + }; // Give up, as `hole` says, whatever part of `[start, end)` this subscription has not been // served or given up yet. const drop = async (start: bigint, end: bigint, hole: Hole): Promise => { - if (!served) return true; - const gaps = runGaps(served, start, end); - if (hasSkip(this.version)) { - for (const gap of gaps) { - if (hole.skip) { - dropped = removeRun(dropped, gap.start, gap.end); - droppedStale = removeRun(droppedStale, gap.start, gap.end); - continue; - } - insertRun(dropped, gap.start, gap.end); - // The wire names every OLD drop stale, which the next update reopens. One the - // reader gave up as final, such as one relayed from a version whose drops are all - // final, is dropped again by each update, as an expired stream is. - if (!sentStale(hole)) { - droppedStale = removeRun(droppedStale, gap.start, gap.end); - continue; - } - insertRun(droppedStale, gap.start, gap.end); - if (hole.stale === undefined) insertRun(expired, gap.start, gap.end); - } - } - for (const gap of gaps) { - insertRun(served, gap.start, gap.end); + if (!record) return true; + const report = reportOf(hole); + for (const gap of record.untold(start, end)) { + record.tell(gap.start, gap.end, report); const msg = holeMessage(gap.start, gap.end, hole); if (!(await controls.response(encodeSubscribeResponse(stream, msg, this.version)))) return false; } return true; }; - // Forward a verdict the reader reached on `[start, end)`, as `drop` does. On lite-07 it also - // replaces what it overrides: a skip any drop already sent, and a final (non-OLD) drop a stale one, - // which the next update would otherwise reopen. A reader the earlier drop ended stays ended. + // Forward a verdict the reader reached on `[start, end)`, which its record already took in, + // replacing what it goes out over: on lite-07, a skip any drop already sent, and a final + // drop a stale one, which the next update would otherwise reopen. A reader the earlier drop + // ended stays ended. const forward = async (start: bigint, end: bigint, hole: Hole): Promise => { - if (served && hasSkip(this.version)) { - const replaces = hole.skip ? dropped : sentStale(hole) ? [] : droppedStale; - for (const run of [...replaces]) { - const from = max(run.start, start); - const to = min(run.end, end); - if (from >= to) continue; - replaceStale(from, to); - served.splice(0, served.length, ...removeRun(served, from, to)); - } - } - return drop(start, end, hole); + if (!record || start >= end) return true; + record.tell(start, end, reportOf(hole)); + const msg = holeMessage(start, end, hole); + return controls.response(encodeSubscribeResponse(stream, msg, this.version)); }; - // Forget that `[start, end)` went out as a drop: a verdict or group replaces it. A stream of - // this subscription that expired there is no longer owed again on an update, since what - // replaced it settles it. - const replaceStale = (start: bigint, end: bigint) => { - dropped = removeRun(dropped, start, end); - droppedStale = removeRun(droppedStale, start, end); - expired.splice(0, expired.length, ...removeRun(expired, start, end)); - }; + // Record `[start, end)` as named stale (OLD) in a lite-07 drop that each update drops + // again, which a later verdict or group replaces. + const toldStale = (start: bigint, end: bigint) => record?.tell(start, end, { kind: "stale", expired: true }); // The track's exclusive final boundary. A Rust subscriber feeds SUBSCRIBE_FINISH // straight into finish_at, so it must name the track's boundary (which counts @@ -1563,8 +1539,7 @@ export class Publisher { if (startSequence !== undefined && skipped.length > 0) { for (const range of skipped) { // What was given up below the start is never named, so a wider budget may serve it. - if (range.start < startSequence) - hooks.unreport(track, range.start, min(range.end, startSequence)); + if (range.start < startSequence) record?.untell(range.start, min(range.end, startSequence)); if (!(await forward(max(range.start, startSequence), range.end, range))) return false; } skipped.length = 0; @@ -1572,14 +1547,11 @@ export class Publisher { if (pendingDrops.length > 0) { for (const { start, end, error } of pendingDrops.splice(0)) { // A stream expired under this subscription's budget is a stale verdict, which - // a later verdict or group replaces as any stale drop sent. - if (hasSkip(this.version) && toStreamCode(error) === StreamCode.Old) { - insertRun(expired, start, end); - insertRun(dropped, start, end); - insertRun(droppedStale, start, end); - } - // A re-offer an update owed again is given up now, so the end sweep never names it. - if (served) insertRun(served, start, end); + // a later verdict or group replaces as any stale drop sent, and each update drops + // again. A re-offer an update owed again is given up now, so the end sweep never + // names it. + if (hasSkip(this.version) && toStreamCode(error) === StreamCode.Old) toldStale(start, end); + else record?.tell(start, end, reportOf({ error })); const msg = SubscribeDrop.given(start, end, error, this.version, updateNumber); if (!(await controls.response(encodeSubscribeResponse(stream, { drop: msg }, this.version)))) return false; @@ -1646,12 +1618,11 @@ export class Publisher { // (applied as the update was decoded): what was reported stale is offered // again, and dropped again under this update's number if still stale. if (hasSkip(this.version)) { - const owe = (start: bigint, end: bigint) => { - if (served) served.splice(0, served.length, ...removeRun(served, start, end)); - dropped = removeRun(dropped, start, end); - droppedStale = removeRun(droppedStale, start, end); - }; + const owe = (start: bigint, end: bigint) => record?.untell(start, end); for (const run of hooks.reofferStale(track)) owe(run.start, run.end); + // What each update drops again: the stale runs whose stream expired, or + // that went out OLD for good. + const expired: Run[] = record?.expired() ?? []; // A stale drop not sent yet, such as one an earlier update left to drop // again, is judged again under these terms too. for (const pending of pendingDrops.filter( @@ -1853,9 +1824,9 @@ export class Publisher { // A lite-07 stale drop is no final verdict, and a Group Stream wins over it, so a group // handed out after one is served, as one a route the front switched to delivers: the // update that reopens the drop would never re-offer it. - if (runGaps(droppedStale, group.sequence, group.sequence + 1n).length === 0) { - replaceStale(group.sequence, group.sequence + 1n); - } else if (served && runGaps(served, group.sequence, group.sequence + 1n).length === 0) { + if (record?.toldStale(group.sequence)) { + record.replaceDrops(group.sequence, group.sequence + 1n); + } else if (record?.told(group.sequence)) { group.close(); continue; } @@ -1897,7 +1868,7 @@ export class Publisher { ) return false; } - if (served) insertRun(served, group.sequence, group.sequence + 1n); + record?.tell(group.sequence, group.sequence + 1n, { kind: "settled" }); const options: RunGroup = { sub, diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 4d419ffa7e5..39fd344bef7 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -1418,6 +1418,7 @@ export class Subscriber { stream.abort(e); } finally { answering?.removeEventListener("abort", replaced); + entry.tail.close(); entry.held?.close(); entry.datagramDemand?.(); if (entry.datagrams) { @@ -1572,6 +1573,7 @@ export class Subscriber { // for a late group anyway, and no longer than the publisher keeps one fresh. tail: new Tail({ grace: () => hooks.tailGrace(producer), + bill: hooks.billTrack(producer), }), requested: msg.startGroup, floor: msg.startGroup, diff --git a/js/net/src/tail.ts b/js/net/src/tail.ts index b00b3f8e4f6..be612c519c0 100644 --- a/js/net/src/tail.ts +++ b/js/net/src/tail.ts @@ -1,5 +1,6 @@ import { type Duration, now, sleep } from "@moq/clock"; import { type GetPromise, Signal } from "@moq/signals"; +import { READER_RUN_OVERHEAD } from "./cache.ts"; import { insertRun, type Run, runGaps } from "./internal.ts"; import { GRACE, Ledger } from "./ledger.ts"; @@ -7,6 +8,8 @@ import { GRACE, Ledger } from "./ledger.ts"; export interface TailOptions { /** How long to wait for a group that cannot be accounted for. Read each time it is needed. */ grace?: () => Duration; + /** Bill the bytes the tail keeps, in all, to the subscription's copy's account. */ + bill?: (bytes: number) => void; } /** @@ -34,9 +37,23 @@ export class Tail { // update names one only if it was sent before the publisher served the group, so the group won // over it, even once the cache evicted it. #finished: Run[] = []; + // What `#finished` bills, `READER_RUN_OVERHEAD` per run, so a publisher fragmenting the + // streams it finishes between updates costs what the cache would hold. + #bill?: (bytes: number) => void; constructor(options: TailOptions = {}) { this.#grace = options.grace ?? (() => GRACE); + this.#bill = options.bill; + } + + #rebill(): void { + this.#bill?.(this.#finished.length * READER_RUN_OVERHEAD); + } + + /** Release what the tail bills, once its subscription ended. */ + close(): void { + this.#finished = []; + this.#rebill(); } /** Group streams whose header arrived, whether they finished or were reset. */ @@ -97,6 +114,7 @@ export class Tail { /** Record that the group stream for `sequence` arrived whole. */ finish(sequence: bigint): void { insertRun(this.#finished, sequence, sequence + 1n); + this.#rebill(); } /** The parts of `[start, end)` whose stream did not finish since the last update was sent. */ @@ -110,6 +128,7 @@ export class Tail { */ updated(): void { this.#finished = []; + this.#rebill(); } /** Re-check what {@link settle} waits on, after something outside the ledger changed. */ diff --git a/js/net/src/track.test.ts b/js/net/src/track.test.ts index 997f7c85c9d..7738ac5977e 100644 --- a/js/net/src/track.test.ts +++ b/js/net/src/track.test.ts @@ -4429,7 +4429,7 @@ test("an update after an idle start leaves an unreported stale skip to a widenin expect(hooks.tryDrop(subscriber)).toMatchObject({ start: 0n, end: 1n }); // SUBSCRIBE_START resolved at group 1 with the skip still held for it. - hooks.unreport(subscriber, 0n, 1n); + hooks.wire(subscriber).untell(0n, 1n); // An update under the same budget reopens nothing below the start. expect(hooks.reofferStale(subscriber)).toEqual([]); expect(hooks.tryDrop(subscriber)).toBeUndefined(); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 6135ccee02f..fff9a61ff31 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -852,6 +852,8 @@ let resubscribe: (producer: Producer) => void; let owedRuns: (producer: Producer, start: bigint, end: bigint) => Iterable; let owedGroup: (producer: Producer, start: bigint, end: bigint) => { sequence: bigint; folds?: Instant } | undefined; let fillEdge: (producer: Producer) => bigint | undefined; +// The producer's cache account, which its readers' records bill to. +let accountOf: (producer: Producer) => cache.Track; let sourceStart: (producer: Producer) => bigint | "pending" | undefined; // The lowest group a reader naming `named` asks a fill for; see `Producer.#fillFloor`. let fillFloor: (producer: Producer, named: bigint | undefined) => bigint | undefined; @@ -1298,6 +1300,15 @@ export class Producer { producer.#account = producer.#openAccount(pool); }; hooks.arrivalRuns = (producer) => producer.#arrivals.size; + accountOf = (producer) => producer.#account; + hooks.billTrack = (producer) => { + let billed = 0; + return (bytes) => { + if (bytes > billed) producer.#account.bill(bytes - billed); + else producer.#account.unbill(billed - bytes); + billed = bytes; + }; + }; hooks.subscribing = (producer) => producer.#subscribing; hooks.requested = (producer) => producer.#state.update; resubscribe = (producer) => producer.#updateSubscription(); @@ -2456,9 +2467,64 @@ type Successor = { sequence: bigint; arrived: Instant; fate: () => Error | undef // out, so no update reopens it), dropped as stale, which an update reopens, or skipped, which // replaces a drop. `received` when an upstream's stale run decided it, which is the upstream's to // judge again then. Mirrors the Rust `Report`. -interface Reported extends Run { - kind: "dropped" | "stale" | "skip"; +/** + * How a reader reported a run, and what its wire was told. Mirrors the Rust `record::Report`: + * `settled` is a group served, or a drop on a version whose drops are all final. + */ +export interface Reported extends Run { + kind: "settled" | "dropped" | "stale" | "skip"; + /** A stale run an upstream decided, which is the upstream's to judge again. */ received?: boolean; + /** A stale run whose Group Stream expired, or that went out OLD for good: its publisher drops it again under each update. */ + expired?: boolean; + /** A stale run whose group was handed out after the drop, which the group won: no update reopens it. */ + taken?: boolean; +} + +const sameReport = (a: Reported, b: Reported) => + a.kind === b.kind && !a.received === !b.received && !a.expired === !b.expired && !a.taken === !b.taken; + +// Whether `reported` settles its run for good, so a fold may forget it. +const finalReport = (reported: Reported) => reported.kind !== "stale" || reported.taken === true; + +// Whether an update reopens `reported`: a stale drop nothing settled since. +const reopens = (reported: Reported) => reported.kind === "stale" && !reported.expired && !reported.taken; + +// Whether `reported` stops `hole` going out over it: a skip goes out over a drop, and a final +// drop over a stale one, since each replaces what it goes out over. An OLD drop never goes out +// over a stale one, which already said OLD. Mirrors the Rust `Report::blocks`. +function blocksHole(reported: Reported, hole: Hole): boolean { + switch (reported.kind) { + case "settled": + case "skip": + return true; + case "dropped": + return !hole.skip; + case "stale": + return reported.taken ? !hole.skip : !hole.skip && toStreamCode(hole.error) === StreamCode.Old; + } +} + +/** + * What a lite publisher told its wire about a reader's sequences, kept in the reader's own + * record so the cursor's drops and the publisher's groups consult one map. Mirrors the + * publisher half of the Rust `record::Record`. + */ +export interface WireRecord { + /** Whether the wire was told of `sequence`: served, or given up. */ + told(sequence: bigint): boolean; + /** Whether `sequence` went out as a stale drop, which a group still replaces. */ + toldStale(sequence: bigint): boolean; + /** The parts of `[start, end)` the wire was told nothing of. */ + untold(start: bigint, end: bigint): Run[]; + /** Record that the wire was told `report` of `[start, end)`. */ + tell(start: bigint, end: bigint, report: Omit): void; + /** Forget what the wire was told of `[start, end)`, which is owed again. */ + untell(start: bigint, end: bigint): void; + /** Forget the drops of `[start, end)` the wire was told, which a verdict or group replaces. */ + replaceDrops(start: bigint, end: bigint): void; + /** The runs each update drops again: stale ones whose stream expired, or that went out OLD for good. */ + expired(): Run[]; } /** @@ -2556,7 +2622,14 @@ export class Subscriber { coverage?: Coverage; judged?: readonly Run[]; }; - #reported = new Runs((a, b) => a.kind === b.kind && a.received === b.received); + #reported = new Runs(sameReport); + // Everything below was settled for good and folded away under pool pressure: handed out, or + // reported final, so none is handed out or reported again. Mirrors the Rust `Record::floor`. + #folded = 0n; + // What the record bills to the track's account, opened on its first run. + #bill?: cache.Reader; + // The pool needed a run this reader was promised, so it ended with TooFarBehind. + #lagged = false; // The group the frame-level helpers are currently draining, acquired through the // sequence cursor so frame reads and {@link Ordered.nextGroup} share one floor. #frameGroup?: GroupConsumer; @@ -2811,8 +2884,9 @@ export class Subscriber { this.#skipped.delete(sequence); // The group won over any stale report on it, so no update reopens it. A resumed reader // retires its own. - if (this.#reported.at(sequence)?.kind === "stale") - this.#reported.set(sequence, sequence + 1n, { start: sequence, end: sequence + 1n, kind: "dropped" }); + const stale = this.#reported.at(sequence); + if (stale?.kind === "stale") + this.#reported.set(sequence, sequence + 1n, { ...stale, start: sequence, end: sequence + 1n, taken: true }); // Another copy of it buffered here is never handed out. this.#judge(sequence, sequence + 1n); if (this.#resume) this.#resume.forgetPastBudget(this.#delivered); @@ -2933,6 +3007,132 @@ export class Subscriber { if (producer && !this.#left) reofferSequences(producer, this.#state, sequences); } + // The publisher's view of this reader's record. + #wireRecord(): WireRecord { + const reported = this.#reported; + return { + told: (sequence) => sequence < this.#folded || reported.at(sequence) !== undefined, + toldStale: (sequence) => reported.at(sequence)?.kind === "stale", + untold: (start, end) => { + const from = max(start, this.#folded); + return from < end ? runGaps(reported.overlapping(from, end), from, end) : []; + }, + tell: (start, end, report) => { + if (start >= end) return; + // A stale report keeps the parts an upstream decided, which stay its to judge again. + const received = + report.kind === "stale" && !report.received + ? reported.overlapping(start, end).filter((run) => run.kind === "stale" && run.received) + : []; + reported.set(start, end, { ...report, start, end } as Reported); + for (const run of received) { + const from = max(run.start, start); + const to = min(run.end, end); + reported.set(from, to, { ...report, start: from, end: to, received: true } as Reported); + } + }, + untell: (start, end) => { + reported.set(start, end); + }, + replaceDrops: (start, end) => { + const drops = reported + .overlapping(start, end) + .filter((run) => run.kind === "dropped" || run.kind === "stale"); + for (const run of drops) reported.set(max(run.start, start), min(run.end, end)); + }, + expired: () => + reported.all + .filter((run) => run.kind === "stale" && run.expired) + .map(({ start, end }) => ({ start, end })), + }; + } + + // How many runs the record holds above its floor, each billed `cache.READER_RUN_OVERHEAD`: + // the handed-out prefix is one run that bills nothing. + #recordRuns(): number { + return ( + Math.max(0, this.#delivered.runs - 1) + + this.#skipped.size + + this.#pending.length + + this.#reported.size + + this.#givenUp.length + ); + } + + // Bill this reader's record, and pay what it can of the debt its track left its readers by + // folding what it settled for good into its floor. Returns TooFarBehind once that needed a + // run it was promised and its record is one to end. Mirrors the Rust `Record::settle`. + #settleRecord(): TooFarBehind | undefined { + if (this.#lagged) return new TooFarBehind(); + const bytes = this.#recordRuns() * cache.READER_RUN_OVERHEAD; + if (!this.#bill) { + const producer = this.#state.producer; + if (bytes === 0 || !producer) return undefined; + this.#bill = accountOf(producer).reader(); + } + const bill = this.#bill; + bill.set(bytes); + if (bill.debt === 0) return undefined; + this.#fold(); + const folded = this.#recordRuns() * cache.READER_RUN_OVERHEAD; + const freed = Math.max(0, bill.size - folded); + bill.set(folded); + bill.pay(freed); + if (bill.debt > 0 && bill.ends) { + this.#lagged = true; + bill.close(); + this.#bill = undefined; + this.#pending = []; + this.#reported = new Runs(sameReport); + this.#skipped.clear(); + this.#givenUp = []; + return new TooFarBehind(); + } + return undefined; + } + + // Raise the floor over the final prefix: each run from it up to the lowest one that is not + // final. Final are the groups handed out, the reports nothing reopens, and this reader's own + // stale skips, which a wider budget would offer again only while the pool still held their + // groups. Not final are the reports an update reopens, the runs given up and not reported + // yet, and the sequences nothing settled. The runs copied from the track's drop set go + // first: the next scan folds in again whatever the track still holds. Mirrors the Rust + // `Record::fold`. + #fold(): void { + if (this.#pending.some((run) => run.received)) { + this.#pending = this.#pending.filter((run) => !run.received); + this.#dropsSeen = undefined; + } + let floor = this.#folded; + for (;;) { + const reported = this.#reported.at(floor); + let next: bigint; + if (reported) { + if (!finalReport(reported)) break; + next = reported.end; + } else { + if (this.#pending.some((run) => run.start <= floor && floor < run.end)) break; + let end = this.#delivered.runEnd(floor); + if (end === undefined && this.#skipped.has(floor)) end = floor + 1n; + if (end === undefined) break; + // A report or a run given up inside it is judged on its own. + const [inside] = this.#reported.overlapping(floor, end); + if (inside) end = min(end, max(inside.start, floor)); + for (const run of this.#pending) if (run.start > floor && run.start < end) end = run.start; + next = end; + } + if (next <= floor) break; + floor = next; + } + if (floor === this.#folded) return; + this.#folded = floor; + this.#delivered.forgetBelow(floor); + for (const sequence of this.#skipped.keys()) if (sequence < floor) this.#skipped.delete(sequence); + this.#pending = this.#pending.flatMap((run) => removeRun([run], 0n, floor)); + this.#reported.set(0n, floor); + this.#givenUp = removeRun(this.#givenUp, 0n, floor); + } + // Owe again every run this cursor gave up as stale, reported or not: an update reopens the // verdict. A group still cached is offered again as a widened skip is, and whatever is still // stale under the new terms is given up and reported again. Returns the runs taken back. A @@ -2941,7 +3141,7 @@ export class Subscriber { if (this.#resume) return this.#resume.reofferStale(); const taken: Run[] = []; const received: Run[] = []; - const stale = this.#reported.all.filter((run) => run.kind === "stale"); + const stale = this.#reported.all.filter(reopens); for (const run of stale) { insertRun(taken, run.start, run.end); if (run.received) insertRun(received, run.start, run.end); @@ -3439,9 +3639,7 @@ export class Subscriber { return subscriber.#peekGroup(ordered); }; hooks.forgetPastBudget = (subscriber, delivered) => subscriber.#forgetPastBudget(delivered); - hooks.unreport = (subscriber, start, end) => { - subscriber.#reported.set(start, end); - }; + hooks.wire = (subscriber) => subscriber.#wireRecord(); hooks.reofferStale = (subscriber) => subscriber.#reofferStale(); hooks.reofferGroups = (subscriber, start, end) => subscriber.#reofferGroups(start, end); hooks.deliveredRuns = (subscriber) => subscriber.#delivered.runs; @@ -3809,6 +4007,8 @@ export class Subscriber { })); } this.#left = true; + this.#bill?.close(); + this.#bill = undefined; this.#state.waitingOrdered = false; // Settle if we're first (the producer may already have); either way drop anything // still buffered. Groups parked at the withRange cap deliberately outlive a clean @@ -3898,6 +4098,14 @@ export class Subscriber { // Package-internal synchronous half of recvGroup. The lite publisher uses this so applying // control state, popping the group, and positioning its frames are one JavaScript turn. #tryRecvGroup(dispositions = false): Recv { + const recv = this.#tryRecvUnsettled(dispositions); + // What the read settled is billed now, and pays what it can of its readers' debt. + const lagged = this.#settleRecord(); + return lagged ? { kind: "error", error: lagged } : recv; + } + + #tryRecvUnsettled(dispositions: boolean): Recv { + if (this.#lagged) return { kind: "error", error: new TooFarBehind() }; if (this.#resume) return this.#tryResumed(false, dispositions); const refused = this.#settle(); if (refused) return { kind: "error", error: refused }; @@ -3978,35 +4186,49 @@ export class Subscriber { return this.#parked(groups) ? { kind: "boundary" } : { kind: "done" }; } - // The first part of `[from, to)` this cursor has not reported yet, now reported, with why. A - // skip goes out over a drop already reported, and a final drop over a stale one, since each - // replaces what it goes out over. + // The first part of `[from, to)` this cursor has not reported yet within its window, now + // reported, with why. #report(from: bigint, to: bigint, hole: Hole & { received?: boolean }): (Recv & { kind: "drop" }) | undefined { const { start, end } = this.#cursor.peek(); - const kind = hole.skip ? "skip" : hole.stale !== undefined ? "stale" : "dropped"; - const blocks = (reported: Reported) => - kind === "skip" ? reported.kind === "skip" : kind === "dropped" ? reported.kind !== "stale" : true; for (const run of this.#window.runs(max(from, start), min(to, end ?? to))) { - const blocked = this.#reported.overlapping(run.start, run.end).filter(blocks); - const [gap] = runGaps(blocked, run.start, run.end); - if (!gap) continue; - // Replacing what it goes out over: a final report is never reopened by an update. - this.#reported.set(gap.start, gap.end, { - ...gap, - kind, - ...(kind === "stale" && hole.received ? { received: true } : {}), - }); - return { kind: "drop", ...gap, error: hole.error, skip: hole.skip, stale: hole.stale }; + const dropped = this.#reportRun(run.start, run.end, hole); + if (dropped) return dropped; } return undefined; } + // The first part of `[from, to)` nothing already reported blocks, now reported, with why, + // replacing what it goes out over. Mirrors the Rust `Record::report`. + #reportRun(from: bigint, to: bigint, hole: Hole & { received?: boolean }): (Recv & { kind: "drop" }) | undefined { + const start = max(from, this.#folded); + if (start >= to) return undefined; + const blocked = this.#reported.overlapping(start, to).filter((reported) => blocksHole(reported, hole)); + const [gap] = runGaps(blocked, start, to); + if (!gap) return undefined; + const kind = hole.skip ? "skip" : hole.stale !== undefined ? "stale" : "dropped"; + this.#reported.set(gap.start, gap.end, { + ...gap, + kind, + ...(kind === "stale" && hole.received ? { received: true } : {}), + }); + return { kind: "drop", ...gap, error: hole.error, skip: hole.skip, stale: hole.stale }; + } + // The next run this cursor gave up and has not reported, without taking a group. From the // first call on, a read without dispositions keeps its budget skips for it too. A resumed // one reports what its routes' copies gave up. #tryDrop(): (Recv & { kind: "drop" }) | undefined { + // What a publisher told its wire since is billed now. A reader the pool ended reports + // nothing more: its reads fail with the reason. + if (this.#settleRecord()) return undefined; if (this.#resume) { - return this.#resume.tryDrop(this.#delivered); + for (;;) { + const queued = this.#nextResumedDrop(); + if (queued) return queued; + const next = this.#resume.tryDrop(this.#delivered); + if (!next) return undefined; + this.#queueResumedDrop(next); + } } this.#settle(); const { start, end } = this.#cursor.peek(); @@ -4091,10 +4313,39 @@ export class Subscriber { // A clean end leaves what the copies hold readable, as a direct reader's buffer is. const closed = this.#state.closed.peek(); if (closed instanceof Error) return { kind: "error", error: closed }; - const recv = resume.tryGroup(ordered, this.#delivered, dispositions); - if (recv.kind === "group") this.#handOut(recv.group.sequence, ordered); - else if (recv.kind === "idle" && this.#exhausted()) return { kind: "done" }; - return recv; + for (;;) { + if (dispositions) { + const queued = this.#nextResumedDrop(); + if (queued) return queued; + } + const recv = resume.tryGroup(ordered, this.#delivered, dispositions); + if (recv.kind === "drop") { + this.#queueResumedDrop(recv); + continue; + } + if (recv.kind === "group") this.#handOut(recv.group.sequence, ordered); + else if (recv.kind === "idle" && this.#exhausted()) return { kind: "done" }; + return recv; + } + } + + // Queue a run a resumed reader's copy gave up. Each copy reports to its own route; this + // reader's record keeps one route's run from going out over what another's already settled. + #queueResumedDrop(dropped: Recv & { kind: "drop" }): void { + const { start, end, error, skip, stale } = dropped; + this.#pending.push({ start, end, error, skip, stale }); + } + + // The lowest queued run's first part nothing already reported blocks, now reported; the rest + // of that run waits its turn. Mirrors the Rust `Record::next_pending`. + #nextResumedDrop(): (Recv & { kind: "drop" }) | undefined { + for (let run = this.#pending.shift(); run; run = this.#pending.shift()) { + const dropped = this.#reportRun(run.start, run.end, run); + if (!dropped) continue; + if (dropped.end < run.end) this.#pending.unshift({ ...run, start: dropped.end }); + return dropped; + } + return undefined; } // The live edge reached the end, with no hole a wire subscription's pending tail still @@ -4229,7 +4480,10 @@ export class Subscriber { // The synchronous half of #nextGroup. #tryNextGroup(): Recv { - const recv = this.#resume ? this.#tryResumed(true) : this.#tryNextBuffered(); + if (this.#lagged) return { kind: "error", error: new TooFarBehind() }; + let recv = this.#resume ? this.#tryResumed(true) : this.#tryNextBuffered(); + const lagged = this.#settleRecord(); + if (lagged) recv = { kind: "error", error: lagged }; // One cursor: the frame helpers must not keep draining a group this read just // moved past, or interleaved reads would run backwards. if (recv.kind === "group" && this.#frameGroup && this.#frameGroup.sequence < recv.group.sequence) { diff --git a/rs/moq-net/src/model/record.rs b/rs/moq-net/src/model/record.rs index c4823ce649d..f0ce53f1503 100644 --- a/rs/moq-net/src/model/record.rs +++ b/rs/moq-net/src/model/record.rs @@ -41,6 +41,9 @@ pub(crate) struct Record { bill: Option, /// The pool needed a run this reader was promised, so it ended. failed: bool, + /// A fold forgot the runs copied from the track's drop set, which the next drop scan + /// folds in again in full. + pub(super) rescan: bool, } /// A run a reader has yet to report, and whether it was copied from the track's runs, which a @@ -342,6 +345,20 @@ impl Record { /// still held their groups. Not final are the reports an update reopens, the runs given /// up and not reported yet, and the sequences nothing settled. fn fold(&mut self) { + // The runs copied from the track's drop set go first: the next scan folds in again + // whatever the track still holds. + let received: Vec<_> = self + .pending + .iter() + .filter(|(_, pending)| pending.received) + .map(|(run, _)| run.clone()) + .collect(); + if !received.is_empty() { + for run in received { + self.pending.remove(run); + } + self.rescan = true; + } let mut floor = self.floor; loop { let next = match self.reported.get_key_value(&floor) { diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 4e29bebacb7..60d738cc92a 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -7847,7 +7847,7 @@ impl DropCursor { self.drift = Some((drift.budget, drift.coverage.clone())); } let scan = (state.judged_revision, span.clone(), within.revision, self.drift.clone()); - let full = self.scanned.as_ref() != Some(&scan); + let full = std::mem::take(&mut record.rescan) || self.scanned.as_ref() != Some(&scan); let changed: Vec> = match full { false => state.drops_changes.since(self.seen).collect(), true => { @@ -24237,9 +24237,9 @@ mod test { /// A flood a reader reported, fragmented into runs alternating with upstream skips, /// bills its record too. Once the pool shrinks under it and its track has shed what it - /// can, the reader folds the final runs into its floor and stays within the pool, while - /// one holding stale reports an update would reopen ends with `TooFarBehind` rather than - /// forgo the re-offer. + /// can, the reader folds the final runs and the groups it read into its floor and stays + /// within the pool, while one holding stale reports an update would reopen ends with + /// `TooFarBehind` rather than forgo the re-offer. #[test] fn a_reported_flood_folds_or_ends_the_reader() { const RUNS: u64 = 1000; @@ -24274,30 +24274,39 @@ mod test { let used = pool.used(); assert!(used >= 2 * RUNS * super::super::record::RUN_OVERHEAD, "stale={stale}: the record is billed"); - // The flood goes on into a smaller pool, whose debt each drop accrues. + // The pool shrinks, and groups the reader reads above the flood accrue its debt: the + // track sheds its own drop set first, then the reader pays. let capacity = 64 * (RUN_OVERHEAD + BORN_OVERHEAD); pool.resize(capacity); - let mut read = Poll::Pending; - for run in RUNS..2 * RUNS { - flood(run); - read = subscriber.poll_recv_group(&waiter); - if read.is_ready() { - break; - } - while let Poll::Ready((run, _)) = subscriber.poll_drop(&waiter) { - assert!(!reported.overlaps(&run), "{run:?}, folded, was reported again"); - reported.insert(run); + let mut read = None; + for sequence in 2 * RUNS..2 * RUNS + 64 { + let mut group = producer.create_group(group::Info { sequence }).unwrap(); + group + .write_frame(Timestamp::from_millis(sequence).unwrap(), vec![0u8; 16 * 1024]) + .unwrap(); + group.finish().unwrap(); + match subscriber.poll_recv_group(&waiter) { + Poll::Ready(Ok(Some(group))) => assert_eq!(group.sequence, sequence, "stale={stale}"), + res => { + read = Some(res.map(|res| res.map(|_| ()))); + break; + } } } match stale { true => assert!( - matches!(read, Poll::Ready(Err(Error::Lagged))), + matches!(read, Some(Poll::Ready(Err(Error::Lagged)))), "the stale reports could not fold" ), - false => assert!(read.is_pending(), "the reader runs on"), + false => { + assert!(read.is_none(), "the reader runs on"); + while let Poll::Ready((run, _)) = subscriber.poll_drop(&waiter) { + assert!(!reported.overlaps(&run), "{run:?}, folded, was reported again"); + } + } } assert!( - pool.used() <= capacity + 4 * (RUN_OVERHEAD + BORN_OVERHEAD), + pool.used() <= capacity + 2 * (16 * 1024 + cache::ENTRY_OVERHEAD), "stale={stale}: the pool holds {} past {capacity}", pool.used() ); From ccba2f023a7c7348f30145717f7a16efde20afc9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 18:28:41 -0800 Subject: [PATCH 08/37] perf(net): js/net memoizes the stale runs a reader cannot trust #askFills walked every drop run above the edge on each probe while a fill handler served the track. It now keeps the unsettled runs, judging again only what the drop set changed or forgot since, as Rust's Misses::unknown does. Co-Authored-By: Claude Opus 5.5 --- js/net/src/fill.test.ts | 33 +++++++++++++++++++++++++++++ js/net/src/track.ts | 46 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 76 insertions(+), 3 deletions(-) diff --git a/js/net/src/fill.test.ts b/js/net/src/fill.test.ts index 5779de411de..b3a3fc62390 100644 --- a/js/net/src/fill.test.ts +++ b/js/net/src/fill.test.ts @@ -845,6 +845,39 @@ test("fill: a stale run that does not settle the reader is still asked for", asy broadcast.close(); }); +// The stale runs above the newest group a reader cannot trust are memoized, and what the track +// gives up after a probe is judged on the next: a run given up later is asked too. Mirrors the +// Rust `Misses::unknown`. +test("fill: a stale run given up after a probe is asked on the next", async () => { + const { broadcast, track } = resumed(2n); + const requests = track.requests(); + hooks.dropTrackGroups(track, 5n, 7n, staleHole(60_000)); + const reader = track.subscribe({ + ranges: [{ start: { absolute: 2n }, end: { absolute: 20n } }], + delay: { max: Duration.millis(500) }, + }); + expect(reader.tryRecvGroup()?.sequence).toBe(2n); + reader.tryRecvGroup(); + const asked = async () => { + const fills: bigint[][] = []; + for (;;) { + const fill = await Promise.race([requests.requestedFill(), yieldNow().then(() => undefined)]); + if (!fill) return fills; + fills.push([fill.start, fill.end]); + fill.close(); + } + }; + expect(await asked()).toEqual([[5n, 7n]]); + + hooks.dropTrackGroups(track, 9n, 10n, staleHole(60_000)); + reader.tryRecvGroup(); + expect(await asked()).toEqual([[9n, 10n]]); + + reader.close(); + requests.close(); + broadcast.close(); +}); + test("fill: a stale run that does not settle the reader leaves its own refusal reported", async () => { const { broadcast, track } = resumed(2n); const requests = track.requests(); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index fff9a61ff31..16a4f55c687 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -2652,6 +2652,15 @@ export class Subscriber { // Fill-owed runs this reader gave up once its budget convicted them: never wanted again, so // the fills withdraw them and their handlers hear, as the Rust `Misses` gives up a stale run. #givenUp: Run[] = []; + // The received stale runs that do not settle this reader, and the terms they were judged + // under; see #unknownRuns. + #unknown?: { + drops: number; + judged: readonly Run[] | undefined; + budget: number; + coverage: Coverage; + runs: DropRun[]; + }; // The last read waited on a fill, so the staleness deadline it armed wakes the next one: a // fill that never answers cannot pin a reader in either order. #fillWait = false; @@ -3384,9 +3393,12 @@ export class Subscriber { const edge = min(end, max(this.#state.received.peek(), (fillEdge(producer) ?? -1n) + 1n)); const spans: Run[] = [{ start: low, end: edge }]; // The upstream accounted for a stale run, so the live feed brings none of it unless an - // update supersedes it: asked even above the newest group received. - for (const dropped of drops.overlapping(max(low, edge), end)) { - const span = { ...dropped, start: max(dropped.start, max(low, edge)), end: min(dropped.end, end) }; + // update supersedes it: asked even above the newest group received. Only the runs that + // do not settle this reader are walked, never every run above the edge. + const above = max(low, edge); + for (const dropped of this.#unknownRuns(budget, coverage)) { + if (dropped.end <= above || dropped.start >= end) continue; + const span = { ...dropped, start: max(dropped.start, above), end: min(dropped.end, end) }; if (span.start < span.end && !settles(span, budget, coverage, this.#state.judged)) spans.push(span); } for (const run of spans.flatMap((span) => this.#wants(span.start, span.end))) { @@ -3439,6 +3451,34 @@ export class Subscriber { return owed; } + // The received stale runs that do not settle this reader under `budget` and `coverage`, + // lowest first. While the terms hold, only what changed since is judged again: a run the + // track changed or forgot leaves the memo, and the runs the change left are judged anew, so a + // drop costs each probe the runs it touches. A later start only narrows what a run asks the + // reader to read, so a run that settles here settles clipped too. Mirrors the Rust + // `Misses::unknown`. + #unknownRuns(budget: number, coverage: Coverage): readonly DropRun[] { + const { drops, judged } = this.#state; + const unsettled = (run: DropRun) => !settles(run, budget, coverage, judged); + const memo = this.#unknown; + let runs: DropRun[]; + if (memo && memo.judged === judged && memo.budget === budget && sameCoverage(memo.coverage, coverage)) { + if (memo.drops === drops.revision) return memo.runs; + runs = memo.runs.filter((run) => drops.at(run.start) === run); + for (const changed of drops.since(memo.drops)) { + for (const run of drops.overlapping(changed.start, changed.end)) { + if (!unsettled(run) || runs.includes(run)) continue; + const at = runs.findIndex((held) => held.start > run.start); + runs.splice(at < 0 ? runs.length : at, 0, run); + } + } + } else { + runs = drops.all.filter(unsettled); + } + this.#unknown = { drops: drops.revision, judged, budget, coverage, runs }; + return runs; + } + // Ask a fill for a fetch's group at once, written yet or not, as the Rust `fetch_group` queues // one; a group the live feed writes meanwhile answers it. `false` with no handler to ask; `"pending"` until the track is accepted. #askFetch(sequence: bigint): boolean | "pending" { From 628251a9c390b520d0952eeecf853c737ff032c6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 18:45:56 -0800 Subject: [PATCH 09/37] chore(net): format, and keep the JS wire record types internal Co-Authored-By: Claude Opus 5.5 --- js/net/src/track.ts | 4 ++++ rs/moq-net/src/model/cache.rs | 4 +++- rs/moq-net/src/model/record.rs | 21 +++++++++++++++++---- rs/moq-net/src/model/skipped.rs | 5 ++++- rs/moq-net/src/model/track.rs | 27 +++++++++++++++++++++------ 5 files changed, 49 insertions(+), 12 deletions(-) diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 16a4f55c687..ad506b895d1 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -2470,6 +2470,8 @@ type Successor = { sequence: bigint; arrived: Instant; fate: () => Error | undef /** * How a reader reported a run, and what its wire was told. Mirrors the Rust `record::Report`: * `settled` is a group served, or a drop on a version whose drops are all final. + * + * @internal */ export interface Reported extends Run { kind: "settled" | "dropped" | "stale" | "skip"; @@ -2509,6 +2511,8 @@ function blocksHole(reported: Reported, hole: Hole): boolean { * What a lite publisher told its wire about a reader's sequences, kept in the reader's own * record so the cursor's drops and the publisher's groups consult one map. Mirrors the * publisher half of the Rust `record::Record`. + * + * @internal */ export interface WireRecord { /** Whether the wire was told of `sequence`: served, or given up. */ diff --git a/rs/moq-net/src/model/cache.rs b/rs/moq-net/src/model/cache.rs index a839183f1bc..df324784a59 100644 --- a/rs/moq-net/src/model/cache.rs +++ b/rs/moq-net/src/model/cache.rs @@ -1398,7 +1398,9 @@ impl Reader { .track .readers .debt - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |debt| Some(debt.saturating_sub(bytes))); + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |debt| { + Some(debt.saturating_sub(bytes)) + }); } /// Whether this record is one to end when folding leaves a debt: it bills more than a diff --git a/rs/moq-net/src/model/record.rs b/rs/moq-net/src/model/record.rs index f0ce53f1503..a7dc664819f 100644 --- a/rs/moq-net/src/model/record.rs +++ b/rs/moq-net/src/model/record.rs @@ -155,7 +155,12 @@ impl Record { /// why; the rest of that run waits its turn. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub fn next_pending(&mut self) -> Option<(Range, Hole)> { - while let Some((range, pending)) = self.pending.iter().next().map(|(range, pending)| (range.clone(), pending.clone())) { + while let Some((range, pending)) = self + .pending + .iter() + .next() + .map(|(range, pending)| (range.clone(), pending.clone())) + { self.pending.remove(range.clone()); if let Some(gap) = self.report(range.clone(), &pending.hole, pending.received) { if gap.end < range.end { @@ -465,7 +470,10 @@ mod test { owe(&track, pool.used()); record.settle(|| track.clone(), &mut ()).unwrap(); assert_eq!(pool.used(), 0, "all of it folded"); - assert!(record.told(7) && record.delivered.contains(7), "a folded drop stays dropped"); + assert!( + record.told(7) && record.delivered.contains(7), + "a folded drop stays dropped" + ); assert_eq!(record.report(0..u64::MAX / 2 + 2, &Hole::Skip, false), None); let mut untold = RangeSet::new(); untold.insert(u64::MAX / 2 + 2..u64::MAX / 2 + 3); @@ -503,7 +511,10 @@ mod test { assert_eq!(StreamError::from(&err), StreamError::TooFarBehind, "{promise:?}"); assert_eq!(pool.used(), 0, "{promise:?}: its record went with it"); assert!(record.failed()); - assert!(record.settle(|| track.clone(), &mut ()).is_err(), "{promise:?}: it stays ended"); + assert!( + record.settle(|| track.clone(), &mut ()).is_err(), + "{promise:?}: it stays ended" + ); } } @@ -515,7 +526,9 @@ mod test { let mut record = Record::default(); record.delivered.insert(0); record.reported.insert(1..2, Report::Dropped); - record.pending.insert(2..3, Pending::own(Hole::drop(StreamError::NotFound))); + record + .pending + .insert(2..3, Pending::own(Hole::drop(StreamError::NotFound))); record.skipped.insert(4, Duration::from_secs(1)); record.settle(|| track.clone(), &mut ()).unwrap(); owe(&track, RUN_OVERHEAD); diff --git a/rs/moq-net/src/model/skipped.rs b/rs/moq-net/src/model/skipped.rs index 50cb1721415..6d142aa5f2c 100644 --- a/rs/moq-net/src/model/skipped.rs +++ b/rs/moq-net/src/model/skipped.rs @@ -129,7 +129,10 @@ impl Skipped { /// One past the last sequence of the run holding `sequence`, if it was skipped. pub fn run_end(&self, sequence: u64) -> Option { let skipped = self.0.as_ref()?; - skipped.runs.get_key_value(&sequence).map(|(run, _)| run.end().saturating_add(1)) + skipped + .runs + .get_key_value(&sequence) + .map(|(run, _)| run.end().saturating_add(1)) } /// Forget every run below `floor`, which a fold settled. diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 60d738cc92a..4ab11ad9767 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -7820,7 +7820,8 @@ impl DropCursor { && let Some(err) = slot.group.cut_error() && let Some(end) = sequence.checked_add(1) { - record.pending + record + .pending .insert(sequence..end, Pending::own(Hole::drop(StreamError::from(err)))); } } @@ -8058,7 +8059,9 @@ impl Cursor { let cut = drops.cuts.poll(waiter); let record = &mut self.record; match self.state.poll(waiter, |state| { - drops.next(record, state, cut, &within).map_or(Poll::Pending, Poll::Ready) + drops + .next(record, state, cut, &within) + .map_or(Poll::Pending, Poll::Ready) }) { Poll::Ready(Ok(range)) => Poll::Ready(range), Poll::Ready(Err(state)) => drops @@ -8185,7 +8188,9 @@ impl Cursor { /// An open window, or one with a fill still out, always has more, and so does one its /// cap keeps a group out of, since a reader may raise a cap again. fn is_exhausted(&self, logical: &[&Delivered]) -> bool { - let taken: Vec<&Delivered> = std::iter::once(&self.record.delivered).chain(logical.iter().copied()).collect(); + let taken: Vec<&Delivered> = std::iter::once(&self.record.delivered) + .chain(logical.iter().copied()) + .collect(); let Some(end) = self.filter.window.end() else { return false; }; @@ -23123,7 +23128,10 @@ mod test { write_untimed_group(&producer, 14); assert_eq!(drain_groups(&mut subscriber), [14]); assert_eq!(subscriber.delivered_runs(), 4, "floor through 8, then 10, 12 and 14"); - assert!(!subscriber.record().delivered.contains(9), "the next gap is still remembered"); + assert!( + !subscriber.record().delivered.contains(9), + "the next gap is still remembered" + ); write_untimed_group(&producer, 11); assert_eq!( drain_groups(&mut subscriber), @@ -24269,10 +24277,17 @@ mod test { assert!(!reported.overlaps(&run), "stale={stale}: {run:?} reported twice"); reported.insert(run); } - assert_eq!(reported.iter().map(|run| run.end - run.start).sum::(), 2 * RUNS, "stale={stale}"); + assert_eq!( + reported.iter().map(|run| run.end - run.start).sum::(), + 2 * RUNS, + "stale={stale}" + ); assert!(subscriber.poll_recv_group(&waiter).is_pending(), "stale={stale}"); let used = pool.used(); - assert!(used >= 2 * RUNS * super::super::record::RUN_OVERHEAD, "stale={stale}: the record is billed"); + assert!( + used >= 2 * RUNS * super::super::record::RUN_OVERHEAD, + "stale={stale}: the record is billed" + ); // The pool shrinks, and groups the reader reads above the flood accrue its debt: the // track sheds its own drop set first, then the reader pays. From 200e649c48e6d5bd08ac80cf716ca0b4ac3a1a0f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 19:24:29 -0800 Subject: [PATCH 10/37] test(net): relay-level drop record tests, and name final drops held below the start A publisher that resolves its start past runs it gave up before the first group now names the final ones all the same, in Rust and js/net: only a stale run is left unnamed for a wider budget. A relay whose reader reached below the start otherwise filled them from upstream, got a final app-error drop it refuses, and left a later reader of the range waiting forever. Adds rs/moq-net/tests/integration/drop_record.rs and js/net/src/lite/drop_record.test.ts, the quest's relay-level cases through a publisher, a relay, and a viewer. Cases that expose bugs main already has stay ignored or skipped for now. Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/drop_record.test.ts | 711 ++++++++++++++++++++ js/net/src/lite/publisher.test.ts | 12 +- js/net/src/lite/publisher.ts | 10 +- rs/moq-net/src/lite/publisher.rs | 14 +- rs/moq-net/tests/integration/drop_record.rs | 646 ++++++++++++++++++ rs/moq-net/tests/integration/main.rs | 1 + 6 files changed, 1386 insertions(+), 8 deletions(-) create mode 100644 js/net/src/lite/drop_record.test.ts create mode 100644 rs/moq-net/tests/integration/drop_record.rs diff --git a/js/net/src/lite/drop_record.test.ts b/js/net/src/lite/drop_record.test.ts new file mode 100644 index 00000000000..ab9e9a30cce --- /dev/null +++ b/js/net/src/lite/drop_record.test.ts @@ -0,0 +1,711 @@ +// How each reader's disposition record settles drops, skips, and evictions through a relay: a +// publisher's session feeds a relay's received copy, which serves a viewer's session from it, +// all on mock transports and a manual clock. Mirrors rs/moq-net/tests/integration/drop_record.rs. + +import { afterEach, expect, mock, spyOn, test } from "bun:test"; +import { Duration, Elapsed, type Manual, timeout } from "@moq/clock"; +import * as cache from "../cache.ts"; +import { accept, connect } from "../connection/index.ts"; +import { StreamCode, StreamError, TooFarBehind, toStreamCode } from "../error.ts"; +import { hooks, type Recv, staleHole } from "../internal.ts"; +import { createMockTransportPair, manual } from "../mock.ts"; +import { Producer as OriginProducer } from "../origin.ts"; +import * as Path from "../path.ts"; +import type { Reader } from "../stream.ts"; +import { Timescale, Timestamp } from "../time.ts"; +import type { Subscription, Producer as TrackProducer, Subscriber as TrackSubscriber } from "../track.ts"; +import { wireOf } from "../wire.ts"; +import { SubscribeDrop, SubscribeSkip, SubscribeStart, SubscribeUpdate } from "./subscribe.ts"; +import { ALPN_06, ALPN_07_WIP, Version } from "./version.ts"; + +const url = new URL("https://localhost:4443/test"); +const path = Path.from("room"); + +afterEach(() => mock.restore()); + +// A hop of the chain: the publisher to the relay, or the relay to the viewer. +type Hop = "up" | "down"; + +// A SUBSCRIBE_DROP or SUBSCRIBE_SKIP a subscriber decoded, and on which hop. +interface Heard { + hop: Hop; + kind: "drop" | "skip"; + start: bigint; + end: bigint; + error?: number; + update?: bigint; +} + +/** + * Listen to every subscribe response the chain decodes. A hop is named by the order its first + * SUBSCRIBE_START arrives: the relay's upstream START always comes first, since its downstream + * START waits on it. A drop matching `hold` is kept from its subscriber until released, as one + * still in flight. + */ +function tap() { + const hops = new Map(); + const heard: Heard[] = []; + const updates: SubscribeUpdate[] = []; + const held: (() => void)[] = []; + let hold: ((heard: Heard) => boolean) | undefined; + + const start = SubscribeStart.decode.bind(SubscribeStart); + spyOn(SubscribeStart, "decode").mockImplementation(async (r, version) => { + if (!hops.has(r)) hops.set(r, hops.size === 0 ? "up" : "down"); + return start(r, version); + }); + const drop = SubscribeDrop.decode.bind(SubscribeDrop); + spyOn(SubscribeDrop, "decode").mockImplementation(async (r, version) => { + const msg = await drop(r, version); + const entry: Heard = { + hop: hops.get(r) ?? "down", + kind: "drop", + start: msg.start, + end: msg.end, + error: msg.error, + update: msg.update, + }; + heard.push(entry); + if (hold?.(entry)) await new Promise((release) => held.push(release)); + return msg; + }); + const skip = SubscribeSkip.decode.bind(SubscribeSkip); + spyOn(SubscribeSkip, "decode").mockImplementation(async (r, version) => { + const msg = await skip(r, version); + heard.push({ hop: hops.get(r) ?? "down", kind: "skip", start: msg.start, end: msg.end }); + return msg; + }); + const update = SubscribeUpdate.decodeMaybe.bind(SubscribeUpdate); + spyOn(SubscribeUpdate, "decodeMaybe").mockImplementation(async (r, version) => { + const msg = await update(r, version); + if (msg) updates.push(msg); + return msg; + }); + + return { + heard, + updates, + held, + /** The drops and skips heard on `hop` from entry `from` on, as `[kind, start, end, code, update]`. */ + on(hop: Hop, from = 0) { + return heard + .slice(from) + .filter((entry) => entry.hop === hop) + .map((entry) => [entry.kind, entry.start, entry.end, entry.error, entry.update]); + }, + hold(fn: ((heard: Heard) => boolean) | undefined) { + hold = fn; + }, + /** Hand every held drop to its subscriber. */ + release() { + for (const release of held.splice(0)) release(); + }, + }; +} + +/** Every track's cache pool, in the order the tracks open. */ +function pools(): { producer: TrackProducer; pool: cache.Pool }[] { + const opened: { producer: TrackProducer; pool: cache.Pool }[] = []; + const original = hooks.poolTrack; + spyOn(hooks, "poolTrack").mockImplementation((producer, pool) => { + opened.push({ producer, pool }); + original(producer, pool); + }); + return opened; +} + +/** + * A publisher's session on `up` to a relay, and the relay's session on `down` to a viewer. The + * relay serves the viewer from its received copy of the publisher's broadcast, as a JS relay does. + * The publisher's track caches into a pool that never ages a group out, so it still holds what + * the relay and the viewer evict on the manual clock, unless a test resizes it. + */ +async function chain(up: Version, down: Version) { + const alpn = (version: Version) => (version === Version.DRAFT_07 ? ALPN_07_WIP : ALPN_06); + const upstream = createMockTransportPair(alpn(up)); + const downstream = createMockTransportPair(alpn(down)); + const source = new OriginProducer(); + const relay = new OriginProducer(); + const [publisher, ingest, serve, viewer] = await Promise.all([ + connect({ url, transport: upstream.client, publish: source.consume() }), + accept({ transport: upstream.server, url, version: up }), + accept({ transport: downstream.server, url, version: down, publish: relay.consume() }), + connect({ url, transport: downstream.client }), + ]); + const sessions = [publisher, ingest, serve, viewer]; + let ended = false; + for (const session of sessions) { + void session.closed.then(() => { + ended = true; + }); + } + + const claim = relay.claim(path); + const serving = (async () => { + for await (const request of claim.requested()) request.serve(wireOf(ingest).consume(request.path)); + })(); + const broadcast = source.route(path).broadcast; + const track = broadcast.createTrack("video", { timescale: Timescale.MILLI, maxAge: null }); + const pool = new cache.Pool(); + hooks.poolTrack(track, pool); + const remote = wireOf(viewer).consume(path); + + return { + track, + pool, + remote, + /** Write group `sequence` presenting at `millis`, with `size` bytes of payload. */ + write(sequence: bigint, millis: number, size = 1) { + const group = track.createGroup(sequence); + group.writeFrame({ payload: new Uint8Array(size), timestamp: Timestamp.fromMillis(millis) }); + group.finish(); + }, + /** How many FETCHes the relay sent upstream, and the viewer sent the relay. */ + fetches() { + return { up: upstream.server.sendStreams.bidi.length, down: downstream.client.sendStreams.bidi.length }; + }, + /** Whether any session closed. */ + ended: () => ended, + /** Wait until the publisher's track has a subscriber through the relay. */ + async reached() { + const demand = track.demand(); + while (demand.wanted.peek() === undefined) await demand.wanted.changed(); + }, + async close() { + remote.close(); + broadcast.close(); + claim.close(); + await serving; + for (const session of sessions) session.abort(); + source.close(); + relay.close(); + }, + }; +} + +/** The next thing `reader` settles: a group, a drop or skip, its end, or its error. */ +async function next(reader: TrackSubscriber): Promise { + for (;;) { + const recv = hooks.tryRecvGroup(reader, true); + if (recv.kind !== "idle" && recv.kind !== "boundary") return recv; + await new Promise((resolve) => { + const dispose = hooks.groupChanged(reader, () => { + dispose(); + resolve(); + }); + }); + } +} + +/** `work`'s result, or undefined once `ms` of manual time passed without it. */ +async function within(clock: Manual, work: Promise, ms = 1000): Promise { + try { + return await clock.drive(timeout(Duration.millis(ms), work)); + } catch (err) { + if (err instanceof Elapsed) return undefined; + throw err; + } +} + +/** Drain pending work, then step manual time 10 ms at a time, until `ready` holds. */ +async function until(clock: Manual, ready: () => boolean): Promise { + await clock.advance(Duration.ZERO); + for (let i = 0; i < 1000 && !ready(); i++) await clock.advance(Duration.millis(10)); + expect(ready()).toBe(true); +} + +/** What a read settled, in a comparable shape. */ +function settled(recv: Recv | undefined): unknown[] { + if (!recv) return ["nothing"]; + switch (recv.kind) { + case "group": + return ["group", recv.group.sequence]; + case "drop": + return [recv.skip ? "skip" : "drop", recv.start, recv.end, toStreamCode(recv.error)]; + case "error": + return ["error", recv.error]; + default: + return [recv.kind]; + } +} + +/** Everything `reader` settles until it goes quiet for a second of manual time, or ends. */ +async function drain(clock: Manual, reader: TrackSubscriber): Promise { + const got: unknown[][] = []; + for (;;) { + const recv = await within(clock, next(reader)); + if (!recv) return got; + got.push(settled(recv)); + if (recv.kind === "done" || recv.kind === "error") return got; + } +} + +/** The sequences of the groups among `rows`, in order. */ +function groupsOf(rows: unknown[][]): unknown[] { + return rows + .filter((row) => row[0] === "group") + .map((row) => row[1] as bigint) + .sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); +} + +const WIDE: Subscription = { delay: { max: null }, order: "ascending", ranges: [{ start: { absolute: 0n } }] }; + +// Groups 1 to 4 after group 0 at 0s: 1 and 2 end 3s and 4s behind the 6s edge, and 3 starts 3s +// behind it, so a 1.5s budget gives up 1 and 2 and still takes 3. +function behindTheEdge(write: (sequence: bigint, millis: number) => void) { + for (const [sequence, millis] of [ + [1n, 1000], + [2n, 2000], + [3n, 3000], + [4n, 6000], + ] as const) + write(sequence, millis); +} + +// A reader with a 1.5s budget is reported groups 1 and 2 stale through the relay, and those OLD +// drops are still in flight to the viewer when it widens to 10s, which the publisher, having +// evicted both, can only drop again. With `prompt`, that re-drop goes out under the widening +// before the track ends; otherwise the publisher's end sweep names it. +async function widening(prompt: boolean): Promise { + using clock = manual(); + const wire = tap(); + const chained = await chain(Version.DRAFT_07, Version.DRAFT_07); + const reader = chained.remote.track("video").subscribe({ delay: { max: Duration.millis(1500) } }); + await chained.reached(); + chained.write(0n, 0); + expect(settled(await next(reader))).toEqual(["group", 0n]); + + wire.hold((heard) => heard.hop === "down" && heard.update === 0n); + behindTheEdge(chained.write); + expect([settled(await next(reader)), settled(await next(reader))]).toEqual([ + ["group", 3n], + ["group", 4n], + ]); + await until(clock, () => wire.held.length > 0); + expect(wire.on("up")).toEqual([ + ["drop", 1n, 2n, StreamCode.Old, 0n], + ["drop", 2n, 3n, StreamCode.Old, 0n], + ]); + + // The publisher no longer holds them: it evicts everything but the live edge. + chained.pool.resize(1); + chained.write(5n, 6100); + expect(settled(await next(reader))).toEqual(["group", 5n]); + + // The widening reaches both hops before the old drops land, which then settle nothing. + reader.update({ delay: { max: Duration.millis(10_000) } }); + await until(clock, () => wire.updates.length >= 2); + expect(wire.updates.map((update) => update.update)).toEqual([1n, 1n]); + wire.hold((heard) => heard.hop === "down" && heard.update === 1n); + wire.release(); + await until(clock, () => wire.on("down").length >= 2); + expect(hooks.staleHeld(reader, 1n, 3n)).toEqual([]); + expect(hooks.tryRecvGroup(reader, true).kind).toBe("idle"); + + // The re-drop under the widening's own number settles the widened reader for good. + if (!prompt) chained.track.finish(); + await until(clock, () => wire.held.length > 0); + expect(wire.on("down")).toEqual([ + ["drop", 1n, 2n, StreamCode.Old, 0n], + ["drop", 2n, 3n, StreamCode.Old, 0n], + ["drop", 1n, 3n, StreamCode.NotFound, 1n], + ]); + expect(hooks.tryRecvGroup(reader, true).kind).toBe("idle"); + wire.hold(undefined); + wire.release(); + expect(settled(await next(reader))).toEqual(["drop", 1n, 3n, StreamCode.NotFound]); + if (prompt) chained.track.finish(); + expect(await drain(clock, reader)).toEqual([["done"]]); + expect(chained.ended()).toBe(false); + + reader.close(); + await chained.close(); +} + +// Mirrors Rust `after_a_widening_the_old_updates_drop_settles_no_reader` in +// rs/moq-net/tests/integration/drop_record.rs. +test("a drop under the narrower update settles no reader once the widening is sent, and the end's re-drop under it does", () => + widening(false)); + +// Mirrors Rust `after_a_widening_the_old_updates_drop_settles_no_reader` in +// rs/moq-net/tests/integration/drop_record.rs. +// Skipped: the publisher's reader re-offers an evicted stale group from its skip record, so nothing drops it again until the end sweep. +test.skip("a widening drops again at once what the publisher evicted", () => widening(true)); + +// Readers on the viewer while the publisher still holds groups 0 to 5 and the relay and the +// viewer evicted all but group 5: `owed` subscribed before the eviction and read nothing, `fresh` +// subscribes after it, and `widened` gave groups 1 and 2 up as stale and widens after it. Each +// is filled through the relay and never told a group was dropped. +async function relayEvicted(readers: ("owed" | "fresh" | "widened")[]): Promise { + using clock = manual(); + const wire = tap(); + const chained = await chain(Version.DRAFT_07, Version.DRAFT_07); + // The first reader takes every group, so the relay's copy received them all. + const first = chained.remote.track("video").subscribe(WIDE); + const owed = chained.remote.track("video").subscribe(WIDE); + await chained.reached(); + for (let sequence = 0n; sequence < 5n; sequence++) chained.write(sequence, Number(sequence) * 1000); + const read: unknown[][] = []; + for (let i = 0; i < 5; i++) read.push(settled(await next(first))); + expect(groupsOf(read)).toEqual([0n, 1n, 2n, 3n, 4n]); + // Half a second gives up groups 1 and 2 as stale, on the viewer alone. + const widened = chained.remote.track("video").subscribe({ delay: { max: Duration.millis(500) } }); + const stale = await drain(clock, widened); + expect(stale.filter((row) => row[0] === "drop")).toEqual([ + ["drop", 1n, 2n, StreamCode.Old], + ["drop", 2n, 3n, StreamCode.Old], + ]); + expect(groupsOf(stale)).toEqual([0n, 3n, 4n]); + + // The relay and the viewer age out everything but the live edge; the publisher keeps it all. + await clock.advance(cache.DEFAULT_EXPIRY.add(Duration.millis(10_000))); + chained.write(5n, 5000); + expect(settled(await next(first))).toEqual(["group", 5n]); + const before = chained.fetches(); + + const all = [0n, 1n, 2n, 3n, 4n, 5n]; + for (const name of readers) { + let reader: TrackSubscriber; + let expected = all; + if (name === "owed") reader = owed; + else if (name === "fresh") reader = chained.remote.track("video").subscribe(WIDE); + else { + reader = widened; + reader.update(WIDE); + expected = [1n, 2n, 5n]; + } + const rows = await drain(clock, reader); + expect(rows.filter((row) => row[0] !== "group")).toEqual([]); + expect(groupsOf(rows)).toEqual(expected); + reader.close(); + } + // Filled through the relay from the publisher, and never named dropped on either hop. + const after = chained.fetches(); + expect(after.up).toBeGreaterThan(before.up); + expect(after.down).toBeGreaterThan(before.down); + expect(wire.heard).toEqual([]); + expect(chained.ended()).toBe(false); + + for (const reader of [first, owed, widened]) reader.close(); + await chained.close(); +} + +// Mirrors Rust `a_group_the_relay_evicted_is_filled_never_dropped` in +// rs/moq-net/tests/integration/drop_record.rs. +test("a group the relay evicted while the publisher holds it is filled for a new and an owed reader", () => + relayEvicted(["fresh", "owed"])); + +// Mirrors Rust `a_group_the_relay_evicted_is_filled_never_dropped` in +// rs/moq-net/tests/integration/drop_record.rs. +// Skipped: the widened reader re-offers its evicted stale groups from its skip record, so their fill lags seconds and group 5 never arrives. +test.skip("a group the relay evicted while the publisher holds it is filled for a widened reader", () => + relayEvicted(["widened"])); + +// Mirrors Rust `an_evicted_drop_is_forwarded_unchanged_and_never_reoffered` in +// rs/moq-net/tests/integration/drop_record.rs. +test("an origin's evicted drops cross the relay unchanged, and no update re-offers them", async () => { + using clock = manual(); + const wire = tap(); + const chained = await chain(Version.DRAFT_07, Version.DRAFT_07); + const reader = chained.remote.track("video").subscribe({ delay: { max: Duration.millis(2000) } }); + await chained.reached(); + const evicted = () => ({ error: new StreamError(StreamCode.Evicted, { message: "group evicted from the cache" }) }); + chained.write(0n, 0); + expect(settled(await next(reader))).toEqual(["group", 0n]); + // Group 1 the origin's cache evicted, and group 3 its retention expired: both EVICTED, never OLD. + hooks.dropTrackGroups(chained.track, 1n, 2n, evicted()); + chained.write(2n, 200); + hooks.dropTrackGroups(chained.track, 3n, 4n, evicted()); + chained.write(4n, 400); + const got: unknown[][] = []; + for (let i = 0; i < 4; i++) got.push(settled(await next(reader))); + expect(groupsOf(got)).toEqual([2n, 4n]); + expect(got.filter((row) => row[0] !== "group")).toEqual([ + ["drop", 1n, 2n, StreamCode.Evicted], + ["drop", 3n, 4n, StreamCode.Evicted], + ]); + for (const hop of ["up", "down"] as const) { + expect(wire.on(hop)).toEqual([ + ["drop", 1n, 2n, StreamCode.Evicted, 0n], + ["drop", 3n, 4n, StreamCode.Evicted, 0n], + ]); + } + + // A widening and a narrowing each reach the publisher, and neither drops nor offers them again. + const heard = wire.heard.length; + reader.update({ delay: { max: Duration.millis(60_000) } }); + await until(clock, () => wire.updates.length >= 2); + reader.update({ delay: { max: Duration.millis(1000) } }); + await until(clock, () => wire.updates.length >= 4); + chained.track.finish(); + expect(await drain(clock, reader)).toEqual([["done"]]); + expect(wire.heard.slice(heard)).toEqual([]); + expect(chained.ended()).toBe(false); + + reader.close(); + await chained.close(); +}); + +// Mirrors Rust `a_lite06_hop_keeps_a_reported_group_dropped` in +// rs/moq-net/tests/integration/drop_record.rs. +for (const down of [Version.DRAFT_06, Version.DRAFT_07]) { + const name = down === Version.DRAFT_07 ? "lite-07" : "lite-06"; + test(`through a lite-06 hop, a reported group stays dropped for every ${name} reader, and a shrink sends no drops`, async () => { + using clock = manual(); + const wire = tap(); + const chained = await chain(Version.DRAFT_06, down); + const reader = chained.remote.track("video").subscribe({ delay: { max: Duration.millis(1500) } }); + await chained.reached(); + chained.write(0n, 0); + expect(settled(await next(reader))).toEqual(["group", 0n]); + behindTheEdge(chained.write); + // Lite-06 names no code, so every drop it carries reads as OLD, and is final. + const got: unknown[][] = []; + for (let i = 0; i < 4; i++) got.push(settled(await next(reader))); + expect(got.filter((row) => row[0] === "drop")).toEqual([ + ["drop", 1n, 2n, StreamCode.Old], + ["drop", 2n, 3n, StreamCode.Old], + ]); + expect(wire.on("up")).toEqual([ + ["drop", 1n, 2n, 0, 0n], + ["drop", 2n, 3n, 0, 0n], + ]); + + // A widened reader and a new one each find groups 1 and 2 still dropped, though the + // publisher holds them. + reader.update({ delay: { max: null } }); + const fresh = chained.remote.track("video").subscribe(WIDE); + expect(groupsOf(await drain(clock, fresh))).toEqual([0n, 3n, 4n]); + expect(groupsOf(await drain(clock, reader))).toEqual([]); + + // A shrink sends no drops upstream, nor any new one downstream. + fresh.close(); + const heard = wire.heard.length; + const updates = wire.updates.length; + reader.update({ delay: { max: Duration.millis(500) } }); + await until(clock, () => wire.updates.length >= updates + 2); + await drain(clock, reader); + expect(wire.on("up", heard)).toEqual([]); + for (const row of wire.on("down", heard)) expect([row[1], row[2]]).toEqual([1n, 3n]); + chained.track.finish(); + expect(await drain(clock, reader)).toEqual([["done"]]); + expect(chained.ended()).toBe(false); + + reader.close(); + await chained.close(); + }); +} + +// Mirrors Rust `pool_pressure_folds_final_runs_and_ends_a_stale_reader` in +// rs/moq-net/tests/integration/drop_record.rs. +for (const stale of [false, true]) { + const outcome = stale + ? "holding stale reports within reach ends with TooFarBehind" + : "folds what it settled for good"; + test(`under pool pressure on the relay, a reader ${outcome}`, async () => { + using clock = manual(); + const opened = pools(); + const chained = await chain(Version.DRAFT_07, Version.DRAFT_07); + const budget = 1000; + const reader = chained.remote.track("video").subscribe({ delay: { max: Duration.millis(budget) } }); + // The viewer's copy opens as the reader subscribes, then the relay's as the request reaches it. + await chained.reached(); + const relay = opened[3]; + expect(opened.slice(0, 2).every((entry) => entry.producer === chained.track)).toBe(true); + expect(relay.producer).not.toBe(chained.track); + chained.write(0n, 0); + expect(settled(await next(reader))).toEqual(["group", 0n]); + + // A flood of drops alternating with skips, which the relay forwards and its reader records. + const runs = 200n; + const hole = () => (stale ? staleHole(budget) : { error: new StreamError(StreamCode.Cancel) }); + for (let run = 0n; run < runs; run++) { + hooks.dropTrackGroups(chained.track, 2n * run + 1n, 2n * run + 2n, hole()); + chained.track.skipGroups(2n * run + 2n, 2n * run + 3n); + } + let reported = 0n; + while (reported < 2n * runs) { + const recv = await next(reader); + expect(recv.kind).toBe("drop"); + if (recv.kind === "drop") reported += recv.end - recv.start; + } + expect(relay.pool.used).toBeGreaterThanOrEqual(Number(2n * runs) * cache.READER_RUN_OVERHEAD); + + // The relay's pool shrinks, and the groups its reader serves accrue the debt: its track + // sheds what it can first, then the reader pays. + const capacity = 64 * cache.RUN_OVERHEAD; + relay.pool.resize(capacity); + let error: unknown; + let groups = 0; + for (let sequence = 2n * runs + 1n; sequence < 2n * runs + 65n && error === undefined; sequence++) { + chained.write(sequence, budget + Number(sequence), 16 * 1024); + const recv = await within(clock, next(reader)); + if (recv?.kind === "error") error = recv.error; + else { + expect(settled(recv)).toEqual(["group", sequence]); + groups++; + } + } + if (stale) { + expect(error).toBeInstanceOf(TooFarBehind); + } else { + expect(error).toBeUndefined(); + expect(groups).toBe(64); + expect(relay.pool.used).toBeLessThanOrEqual(capacity + 2 * (16 * 1024 + cache.ENTRY_OVERHEAD)); + } + expect(chained.ended()).toBe(false); + + reader.close(); + await chained.close(); + }); +} + +// Groups 1 and 2 are judged stale for the relay's subscription, under a 1.5s budget, a run the +// relay's copy shares with all its readers, then the relay `lost` that run: it aged out past the +// pool's idle window, or the pool shed it under pressure. The readers in `readers` subscribe next: +// a narrow one never takes groups 1 and 2, and a wide one is filled them through the relay from +// the publisher, which still holds them. +async function lostRun(lost: "forgot" | "shed", readers: ("wide" | "narrow")[]): Promise { + using clock = manual(); + const opened = pools(); + const wire = tap(); + const chained = await chain(Version.DRAFT_07, Version.DRAFT_07); + const reader = chained.remote.track("video").subscribe({ delay: { max: Duration.millis(1500) } }); + await chained.reached(); + const relay = opened[3]; + chained.write(0n, 0); + expect(settled(await next(reader))).toEqual(["group", 0n]); + behindTheEdge(chained.write); + const got: unknown[][] = []; + for (let i = 0; i < 4; i++) got.push(settled(await next(reader))); + expect(got.filter((row) => row[0] === "drop")).toEqual([ + ["drop", 1n, 2n, StreamCode.Old], + ["drop", 2n, 3n, StreamCode.Old], + ]); + + // Past the pool's idle window the relay's copy and the viewer's forget the run; a pool too + // small for one large group sheds the relay's. + if (lost === "forgot") await clock.advance(cache.DEFAULT_EXPIRY.mul(2)); + else relay.pool.resize(cache.RUN_OVERHEAD); + chained.write(5n, 6100, 64 * 1024); + expect(settled(await next(reader))).toEqual(["group", 5n]); + const before = chained.fetches(); + const heard = wire.heard.length; + + for (const name of readers) { + if (name === "wide") { + const wide = chained.remote.track("video").subscribe(WIDE); + const filled = await drain(clock, wide); + expect(filled.filter((row) => row[0] !== "group")).toEqual([]); + expect(groupsOf(filled)).toEqual(expect.arrayContaining([1n, 2n])); + expect(chained.fetches().up).toBeGreaterThan(before.up); + wide.close(); + } else { + // Judged stale by its own budget, or filled and then judged. + const narrow = chained.remote.track("video").subscribe({ + delay: { max: Duration.millis(1500) }, + ranges: [{ start: { absolute: 1n } }], + }); + const judged = await drain(clock, narrow); + expect(groupsOf(judged)).not.toContain(1n); + expect(groupsOf(judged)).not.toContain(2n); + narrow.close(); + } + } + // Nothing a narrow budget gave up is named dropped on the wire, where it would settle every + // reader, wide ones too. + if (readers.includes("wide")) expect(wire.heard.slice(heard)).toEqual([]); + expect(chained.ended()).toBe(false); + + reader.close(); + await chained.close(); +} + +for (const lost of ["forgot", "shed"] as const) { + // Mirrors Rust `a_shed_run_is_judged_or_filled_through_the_relay` in + // rs/moq-net/tests/integration/drop_record.rs. + test(`a sequence whose run the relay ${lost} is judged by a narrow reader`, () => lostRun(lost, ["narrow"])); + + // Mirrors Rust `a_shed_run_is_judged_or_filled_through_the_relay` in + // rs/moq-net/tests/integration/drop_record.rs. + // Skipped: the relay refills the run by FETCH under the narrow budget and records the stale answer as a final NotFound drop for every reader. + test.skip(`a sequence whose run the relay ${lost} is filled for a wide reader`, () => + lostRun(lost, ["narrow", "wide"])); +} + +const BILLIONS = 1n << 40n; + +// Mirrors Rust `a_run_of_billions_settles_in_one_step` in +// rs/moq-net/tests/integration/drop_record.rs. +test("a run of billions of sequences is settled by each reader in one step, dropped or skipped", async () => { + using clock = manual(); + const wire = tap(); + const chained = await chain(Version.DRAFT_07, Version.DRAFT_07); + const narrow = chained.remote.track("video").subscribe({ delay: { max: Duration.millis(1500) } }); + const wide = chained.remote.track("video").subscribe({ delay: { max: null }, order: "ascending" }); + await chained.reached(); + chained.write(0n, 0); + for (const reader of [narrow, wide]) expect(settled(await next(reader))).toEqual(["group", 0n]); + const before = chained.fetches(); + + hooks.dropTrackGroups(chained.track, 1n, 1n + BILLIONS, { error: new StreamError(StreamCode.Cancel) }); + chained.track.skipGroups(1n + BILLIONS, 1n + 2n * BILLIONS); + chained.write(1n + 2n * BILLIONS, 1000); + for (const reader of [narrow, wide]) { + const got: unknown[][] = []; + for (let i = 0; i < 3; i++) got.push(settled(await within(clock, next(reader)))); + expect(got).toEqual([ + ["drop", 1n, 1n + BILLIONS, StreamCode.Cancel], + ["skip", 1n + BILLIONS, 1n + 2n * BILLIONS, StreamCode.NotFound], + ["group", 1n + 2n * BILLIONS], + ]); + } + for (const hop of ["up", "down"] as const) { + expect(wire.on(hop)).toEqual([ + ["drop", 1n, 1n + BILLIONS, StreamCode.Cancel, 0n], + ["skip", 1n + BILLIONS, 1n + 2n * BILLIONS, undefined, undefined], + ]); + } + expect(chained.fetches()).toEqual(before); + + for (const reader of [narrow, wide]) reader.close(); + await chained.close(); +}); + +// Mirrors Rust `a_run_of_billions_settles_in_one_step` in +// rs/moq-net/tests/integration/drop_record.rs. +for (const judged of ["reopened by a widening", "forgotten by the relay"] as const) { + // Skipped: a run judged again is filled one lite FETCH per sequence from its top down, so a run of billions never settles. + test.skip(`a run of billions ${judged} is judged again in one step`, async () => { + using clock = manual(); + const chained = await chain(Version.DRAFT_07, Version.DRAFT_07); + const reader = chained.remote.track("video").subscribe({ delay: { max: Duration.millis(1500) } }); + // An unbounded reader keeps the relay's subscription unbounded too. + const wide = + judged === "forgotten by the relay" + ? chained.remote.track("video").subscribe({ delay: { max: null } }) + : undefined; + await chained.reached(); + chained.write(0n, 0); + expect(settled(await next(reader))).toEqual(["group", 0n]); + if (judged === "reopened by a widening") + hooks.dropTrackGroups(chained.track, 1n, 1n + BILLIONS, staleHole(1500)); + else chained.track.skipGroups(1n, 1n + BILLIONS); + chained.write(1n + BILLIONS, 1000); + const expected = judged === "reopened by a widening" ? StreamCode.Old : StreamCode.NotFound; + expect(settled(await next(reader))[3]).toBe(expected); + expect(settled(await next(reader))).toEqual(["group", 1n + BILLIONS]); + if (wide) await drain(clock, wide); + const before = chained.fetches(); + + if (judged === "reopened by a widening") reader.update({ delay: { max: null } }); + else await clock.advance(cache.DEFAULT_EXPIRY.mul(2)); + const after = await drain(clock, reader); + expect(after.length).toBeLessThanOrEqual(1); + expect(chained.fetches().up - before.up).toBeLessThanOrEqual(1); + + reader.close(); + wide?.close(); + await chained.close(); + }); +} diff --git a/js/net/src/lite/publisher.test.ts b/js/net/src/lite/publisher.test.ts index 423ab41f438..2fc43b0ff7f 100644 --- a/js/net/src/lite/publisher.test.ts +++ b/js/net/src/lite/publisher.test.ts @@ -1794,9 +1794,15 @@ for (const skipped of ["failed", "stale"] as const) { if (!("start" in response)) throw new Error("expected SUBSCRIBE_START"); expect(response.start.starts[0]).toBe(6n); if (skipped === "failed") { - const dropped = await decodeSubscribeResponse(sub.client.reader, Version.DRAFT_06); - if (!("drop" in dropped)) throw new Error("expected SUBSCRIBE_DROP"); - expect([dropped.drop.start, dropped.drop.end]).toEqual([7n, 8n]); + // The failed group below the start is final, so it is named all the same. + for (const run of [ + [5n, 6n], + [7n, 8n], + ]) { + const dropped = await decodeSubscribeResponse(sub.client.reader, Version.DRAFT_06); + if (!("drop" in dropped)) throw new Error("expected SUBSCRIBE_DROP"); + expect([dropped.drop.start, dropped.drop.end]).toEqual(run); + } } expect(await sub.servedSequence()).toBe(6n); } finally { diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index 625e7658e1e..02187cce2ca 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -1538,8 +1538,14 @@ export class Publisher { for (;;) { if (startSequence !== undefined && skipped.length > 0) { for (const range of skipped) { - // What was given up below the start is never named, so a wider budget may serve it. - if (range.start < startSequence) record?.untell(range.start, min(range.end, startSequence)); + // A stale run below the start is never named, so a wider budget may serve it. A + // final one is named all the same: no budget brings it back, and a subscriber + // filling below the start (a relay whose reader reaches lower) must still learn + // its fate. Mirrors the Rust `TrackRun::resolve`. + const below = min(range.end, startSequence); + if (range.stale !== undefined) { + if (range.start < below) record?.untell(range.start, below); + } else if (!(await forward(range.start, below, range))) return false; if (!(await forward(max(range.start, startSequence), range.end, range))) return false; } skipped.length = 0; diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index cc7eed2a6bc..8d6e1ae5efb 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -5924,10 +5924,18 @@ impl TrackRun { starts: self.starts(start), }))?; // The subscriber now owes itself everything from the start, so a range given up - // above it is named as it would have been after the start. What was given up below it is - // never named, so a wider budget may still serve it. + // above it is named as it would have been after the start. A stale run below it is + // never named, so a wider budget may still serve it. A final one is named all the same: + // no budget brings it back, and a subscriber filling below the start (a relay whose + // reader reaches lower) must still learn its fate. for (range, hole) in skipped.iter() { - self.track.record_mut().untell(range.start..range.end.min(start)); + let below = range.start..range.end.min(start); + match hole.stale_budget() { + Some(_) => self.track.record_mut().untell(below), + None => { + self.forward_range(below, hole, writer)?; + } + } self.forward_range(range.start.max(start)..range.end, hole, writer)?; } Ok(()) diff --git a/rs/moq-net/tests/integration/drop_record.rs b/rs/moq-net/tests/integration/drop_record.rs new file mode 100644 index 00000000000..cc8b3d54e21 --- /dev/null +++ b/rs/moq-net/tests/integration/drop_record.rs @@ -0,0 +1,646 @@ +#![cfg(feature = "async")] + +//! How each reader's disposition record settles drops, skips, and evictions through a relay: +//! a publisher origin, a relay origin, and a viewer origin joined with lite sessions. + +use std::collections::BTreeSet; +use std::time::Duration; + +use moq_net::track::{Delay, Info, Range, Subscription}; +use moq_net::{Timescale, Timestamp, Version, cache, origin, track}; + +use crate::support::harness::{MockConnectOptions, MockPair, connect_mock, spawn}; + +const TIMEOUT: Duration = Duration::from_secs(10); + +/// Groups 0 through 9 present one per second. +const GROUPS: u64 = 10; + +/// Reaches groups 6 and up: group 5 ends at 6s, a full 3s behind the 9s edge. +const NARROW: Duration = Duration::from_millis(2500); + +const LITE07: &str = "moq-lite-07-wip"; + +fn origin(hop: u64, pool: cache::Pool) -> origin::Producer { + let mut config = origin::Config::new(std::num::NonZeroU64::new(hop).unwrap()); + config.pool = pool; + let (origin, driver) = origin::Producer::new(config); + spawn(driver); + origin.with_proxy(true) +} + +/// A millisecond track with no publisher max age, so only a reader's budget skips its groups. +fn timed() -> Info { + Info::default().with_timescale(Timescale::MILLI).with_max_age(None) +} + +/// Write group `sequence` presented at `millis`, and finish it. +fn write(source: &track::Producer, sequence: u64, millis: u64) { + let mut group = source.create_group(moq_net::group::Info { sequence }).unwrap(); + group + .write_frame(Timestamp::from_millis(millis).unwrap(), &b"x"[..]) + .unwrap(); + group.finish().unwrap(); +} + +/// A publisher, a relay subscribing to it, and a viewer subscribing to the relay. +struct Chain { + source: track::Producer, + relay: origin::Producer, + viewer: origin::Producer, + /// The relay's session to the publisher: the client is the relay. + upstream: MockPair, + /// The viewer's session to the relay: the server is the relay. + downstream: MockPair, + _hold: Vec>, +} + +impl Chain { + /// Each hop's wire, and each origin's pool, publisher first. + async fn new(versions: [&str; 2], pools: [cache::Pool; 3]) -> Self { + Self::with_credit(versions, pools, None).await + } + + /// [`Self::new`], with the publisher granted `credit` uni streams open toward the relay. + async fn with_credit(versions: [&str; 2], pools: [cache::Pool; 3], credit: Option) -> Self { + let [publisher, relay, viewer] = pools; + let publisher = origin(1, publisher); + let relay = origin(2, relay); + let viewer = origin(3, viewer); + let mut options = MockConnectOptions::new(versions[0].parse::().unwrap()); + options.server_publish = Some(publisher.consume()); + options.client_subscribe = Some(relay.clone()); + options.transport.max_concurrent_uni_streams = credit; + let upstream = connect_mock(options).await; + let mut options = MockConnectOptions::new(versions[1].parse::().unwrap()); + options.server_publish = Some(relay.consume()); + options.client_subscribe = Some(viewer.clone()); + let downstream = connect_mock(options).await; + let broadcast = publisher + .route("live", moq_net::origin::RouteOptions::hidden()) + .unwrap(); + let source = broadcast.broadcast().create_track("video", timed()).unwrap(); + broadcast.announce().unwrap(); + Self { + source, + relay, + viewer, + upstream, + downstream, + _hold: vec![Box::new(publisher), Box::new(broadcast)], + } + } + + /// Groups 0 through 9, one per second. + fn populate(&self) { + for sequence in 0..GROUPS { + write(&self.source, sequence, sequence * 1000); + } + } + + /// The relay's SUBSCRIBE and FETCH streams to the publisher, by their stream types 2 and 3. + fn asked_upstream(&self) -> (usize, usize) { + asked(&self.upstream) + } + + /// The SUBSCRIBE and FETCH streams on both hops. + fn asked(&self) -> [(usize, usize); 2] { + [asked(&self.upstream), asked(&self.downstream)] + } + + async fn track(origin: &origin::Producer) -> track::Consumer { + let remote = kio::clock::timeout(TIMEOUT, origin.consume().follow("live", None).unwrap().available()) + .await + .expect("resolve timeout") + .unwrap() + .expect("the broadcast resolves"); + remote.track("video").unwrap() + } + + /// No session on either hop has closed. + fn assert_open(&self, name: &str) { + for (hop, pair) in [("upstream", &self.upstream), ("downstream", &self.downstream)] { + assert_eq!( + pair.client_transport.close_reason(), + None, + "{name}: the {hop} session closed" + ); + } + } + + /// Subscribe from the viewer. + async fn subscribe(&self, subscription: Subscription) -> track::Subscriber { + let track = Self::track(&self.viewer).await; + kio::clock::timeout(TIMEOUT, track.subscribe(subscription)) + .await + .expect("subscribe timeout") + .unwrap() + } + + /// Subscribe on the relay's own cache. + async fn subscribe_relay(&self, subscription: Subscription) -> track::Subscriber { + let track = Self::track(&self.relay).await; + kio::clock::timeout(TIMEOUT, track.subscribe(subscription)) + .await + .expect("subscribe timeout") + .unwrap() + } +} + +/// The SUBSCRIBE and FETCH streams `pair`'s client opened, by their stream types 2 and 3. +fn asked(pair: &MockPair) -> (usize, usize) { + let types = pair.client_transport.bidi_types(); + let count = |kind| types.iter().filter(|opened| **opened == kind).count(); + (count(2), count(3)) +} + +/// A reader of `ranges` with max delay `budget`. +fn reading(budget: impl Into>, ranges: impl IntoIterator) -> Subscription { + Subscription::default() + .with_delay(Delay::default().with_max(budget)) + .with_ranges(ranges) +} + +/// Every group `subscriber` receives until it goes quiet for a second, or ends. +async fn received(subscriber: &mut track::Subscriber) -> BTreeSet { + let mut got = BTreeSet::new(); + while let Ok(group) = kio::clock::timeout(Duration::from_secs(1), subscriber.recv_group()).await { + let Some(group) = group.expect("the reader runs on") else { + break; + }; + assert!(got.insert(group.sequence), "group {} delivered twice", group.sequence); + } + got +} + +/// A pool with no byte target that keeps every group for good. +fn forever() -> cache::Pool { + cache::Pool::unbounded() +} + +/// A pool that reclaims what nobody touched for `expiry`. +fn expiring(expiry: Duration) -> cache::Pool { + cache::Pool::new(cache::Config::default().with_expiry(expiry)) +} + +/// A group a relay evicted while its publisher still holds it is never reported dropped: a +/// new reader, a reader that widens its budget, and a reader still owed it each fill it by +/// range from the publisher. +#[kio::test] +async fn a_relay_eviction_is_filled_by_range_for_every_reader() { + const EXPIRY: Duration = Duration::from_secs(10); + kio::clock::timeout(TIMEOUT * 10, async { + let chain = Chain::new([LITE07, LITE07], [forever(), expiring(EXPIRY), expiring(EXPIRY)]).await; + chain.populate(); + + // A reader with no budget warms the relay with every group, then leaves. + let mut warm = chain.subscribe(reading(None, [Range::oldest()])).await; + assert_eq!(received(&mut warm).await, (0..GROUPS).collect()); + drop(warm); + + let mut widened = chain.subscribe(reading(NARROW, [Range::oldest()])).await; + assert_eq!(received(&mut widened).await, (6..GROUPS).collect()); + // Owed every group from 0, on the relay's own cache, and reading none of them yet. + let mut owed = chain.subscribe_relay(reading(None, [Range::groups(0..)])).await; + + // Nobody touches them, so the relay and the viewer reclaim all but the live edge. Every + // group is stale by the wall clock now, so only a reader with no budget wants them. + kio::clock::sleep(EXPIRY * 3).await; + let (subscribes, _) = chain.asked_upstream(); + + widened.update(reading(None, [Range::oldest()])).unwrap(); + assert_eq!(received(&mut widened).await, (0..6).collect(), "the widened reader"); + + let mut new = chain.subscribe(reading(None, [Range::groups(1..4)])).await; + assert_eq!(received(&mut new).await, (1..4).collect(), "the new reader"); + + assert_eq!(received(&mut owed).await, (0..GROUPS).collect(), "the owed reader"); + let (filled, fetches) = chain.asked_upstream(); + assert!( + filled > subscribes, + "the relay filled what it evicted from the publisher" + ); + assert_eq!(fetches, 0, "every fill rides a subscription"); + }) + .await + .expect("timed out"); +} + +/// What settles the run between group 0 and the far group in [`a_huge_run_settles_in_one_step`]. +#[derive(Clone, Copy, Debug)] +enum Huge { + /// The publisher never produces it, which its end names as one drop. + Dropped, + /// The publisher skips it. + Skipped, + /// Dropped, then forgotten by the relay and the viewer, so a later reader asks again. + Forgotten, +} + +/// A single run spanning 2^40 sequences is settled by each cursor in one step, whether it is +/// dropped, skipped, or forgotten and judged again: every reader, narrow or wide, gets the +/// groups on either side and ends at once, on mock time, rather than walking the run. +#[kio::test] +async fn a_huge_run_settles_in_one_step() { + const FAR: u64 = 1 << 40; + const EXPIRY: Duration = Duration::from_secs(10); + for huge in [Huge::Dropped, Huge::Skipped, Huge::Forgotten] { + kio::clock::timeout(TIMEOUT * 10, async { + let chain = Chain::new([LITE07, LITE07], [forever(), expiring(EXPIRY), expiring(EXPIRY)]).await; + let everything = || reading(None, [Range::groups(0..)]); + let mut narrow = chain.subscribe(reading(NARROW, [Range::groups(0..)])).await; + let mut wide = chain.subscribe(everything()).await; + write(&chain.source, 0, 0); + if let Huge::Skipped = huge { + chain.source.skip_groups(1..FAR); + } + // Presented just after group 0, since its sequence in milliseconds overflows. + write(&chain.source, FAR, 1); + let _finished = chain.source.clone().finish().unwrap(); + + let started = kio::clock::now(); + for (name, reader) in [("narrow", &mut narrow), ("wide", &mut wide)] { + assert_eq!(drain(reader).await, [0, FAR].into(), "{huge:?}: the {name} reader"); + } + assert!( + kio::clock::now() - started < Duration::from_secs(1), + "{huge:?}: ended after {:?}", + kio::clock::now() - started + ); + + if let Huge::Forgotten = huge { + drop((narrow, wide)); + kio::clock::sleep(EXPIRY * 3).await; + let (subscribes, _) = chain.asked_upstream(); + let started = kio::clock::now(); + let mut late = chain.subscribe(everything()).await; + assert_eq!(drain(&mut late).await, [0, FAR].into(), "the later reader"); + assert!( + kio::clock::now() - started < Duration::from_secs(1), + "judged again after {:?}", + kio::clock::now() - started + ); + assert!(chain.asked_upstream().0 > subscribes, "the relay asked again"); + } + }) + .await + .unwrap_or_else(|_| panic!("{huge:?}: timed out")); + } +} + +/// Every group `subscriber` receives until the track ends cleanly. +async fn drain(subscriber: &mut track::Subscriber) -> BTreeSet { + let mut got = BTreeSet::new(); + while let Some(group) = kio::clock::timeout(TIMEOUT, subscriber.recv_group()) + .await + .expect("the reader stalled") + .expect("the reader runs on") + { + assert!(got.insert(group.sequence), "group {} delivered twice", group.sequence); + } + got +} + +const LITE06: &str = "moq-lite-06"; + +/// Through a lite-06 hop, a group the publisher reported dropped stays dropped for every +/// reader, whatever budget it widens to, with no session error. Lite-07 end to end, the same +/// stale report is reopened by the widening update instead. A shrink sends no drops: what an +/// update narrows away is still there for a later reader. +#[kio::test] +async fn a_lite06_hop_keeps_a_reported_group_dropped() { + for versions in [[LITE07, LITE06], [LITE06, LITE06], [LITE07, LITE07]] { + reported_drop_through(versions).await; + } +} + +/// [`a_lite06_hop_keeps_a_reported_group_dropped`] with the lite-06 hop upstream of a lite-07 +/// one: the relay forwards the final drop as a stale one, the viewer's widening update reopens +/// it and fetches the group, and the relay fetches it from the publisher instead of answering +/// from the drop it holds. +#[kio::test] +#[ignore = "a relay fetches upstream a group its copy holds as a final lite-06 drop"] +async fn a_lite06_upstream_keeps_a_reported_group_dropped_for_a_lite07_viewer() { + reported_drop_through([LITE06, LITE07]).await; +} + +async fn reported_drop_through(versions: [&str; 2]) { + // Reaches groups 5 and up by media time. + const B1: Duration = Duration::from_millis(3500); + // Reaches every group, by media time and by the wall clock this test spends. + const B2: Duration = Duration::from_secs(60); + let name = format!("{} then {}", versions[0], versions[1]); + kio::clock::timeout(TIMEOUT * 10, async { + let chain = Chain::new(versions, [forever(), forever(), forever()]).await; + for sequence in (0..GROUPS).filter(|sequence| *sequence != 7) { + write(&chain.source, sequence, sequence * 1000); + } + let mut first = chain.subscribe(reading(B1, [Range::oldest()])).await; + assert_eq!(received(&mut first).await, [5, 6, 8, 9].into(), "{name}"); + + // Group 7 lands once its successor is older than B1 by the wall clock, so the + // publisher reports it stale under B1. + kio::clock::sleep(B1 + Duration::from_secs(1)).await; + write(&chain.source, 7, 7000); + assert_eq!(received(&mut first).await, BTreeSet::new(), "{name}"); + + // Only lite-07 end to end leaves the stale report open to a widening update. + let want: BTreeSet = match versions.contains(&LITE06) { + true => (0..GROUPS).filter(|sequence| *sequence != 7).collect(), + false => (0..GROUPS).collect(), + }; + first.update(reading(B2, [Range::oldest()])).unwrap(); + assert_eq!( + received(&mut first).await, + &want - &[5, 6, 8, 9].into(), + "{name}: the widened reader" + ); + let mut second = chain.subscribe(reading(B2, [Range::oldest()])).await; + assert_eq!(received(&mut second).await, want, "{name}: the later reader"); + + let _finished = chain.source.clone().finish().unwrap(); + for (reader, which) in [(&mut first, "first"), (&mut second, "second")] { + assert_eq!(drain(reader).await, BTreeSet::new(), "{name}: the {which} reader ends"); + } + chain.assert_open(&name); + }) + .await + .unwrap_or_else(|_| panic!("{name}: timed out")); + + // The shrink reaches the publisher before any group it narrows away exists. + kio::clock::timeout(TIMEOUT * 10, async { + let chain = Chain::new(versions, [forever(), forever(), forever()]).await; + let mut shrunk = chain.subscribe(reading(None, [Range::groups(0..)])).await; + shrunk.update(reading(None, [Range::groups(8..)])).unwrap(); + kio::clock::sleep(Duration::from_secs(1)).await; + chain.populate(); + assert_eq!(received(&mut shrunk).await, [8, 9].into(), "{name}: the shrunk reader"); + + let mut later = chain.subscribe(reading(None, [Range::groups(0..)])).await; + assert_eq!( + received(&mut later).await, + (0..GROUPS).collect(), + "{name}: the later reader" + ); + chain.assert_open(&name); + }) + .await + .unwrap_or_else(|_| panic!("{name}: shrink timed out")); +} + +/// An origin's own eviction and retention expiry of a group it still owed a subscriber is a +/// final drop (EVICTED, not OLD), which the relay forwards as it is: the viewer's reader settles +/// on it and ends at once, and neither a widening update nor a later reader reopens it, so +/// nothing asks for the group again on either hop. +#[kio::test] +async fn an_origin_eviction_drop_is_forwarded_final() { + const EXPIRY: Duration = Duration::from_secs(10); + const PAYLOAD: usize = 16 * 1024; + for expiry in [true, false] { + kio::clock::timeout(TIMEOUT * 10, async { + let pool = match expiry { + true => expiring(EXPIRY), + false => cache::Pool::new(cache::Config::default().with_capacity(4 * PAYLOAD as u64)), + }; + // One stream of credit: group 0's, held from the relay, so the rest queue. + let chain = Chain::with_credit([LITE07, LITE07], [pool, forever(), forever()], Some(1)).await; + let mut reader = chain.subscribe(reading(None, [Range::groups(0..)])).await; + chain.upstream.server_transport.hold_unis(); + let write = |sequence: u64| { + let mut group = chain.source.create_group(moq_net::group::Info { sequence }).unwrap(); + group + .write_frame(Timestamp::from_millis(sequence * 1000).unwrap(), vec![0u8; PAYLOAD]) + .unwrap(); + group.finish().unwrap(); + }; + for sequence in 0..4 { + write(sequence); + } + let last = match expiry { + // Nobody reads the queued groups, so the origin reclaims all but the newest. + true => { + kio::clock::sleep(EXPIRY * 3).await; + 3 + } + // The newer groups push the queued ones out. + false => { + for sequence in 4..8 { + write(sequence); + } + 7 + } + }; + chain.upstream.server_transport.release_unis(); + let got = received(&mut reader).await; + assert!(got.contains(&last), "expiry={expiry}: {got:?}"); + assert!(got.len() <= last as usize, "expiry={expiry}: nothing evicted: {got:?}"); + let finished = chain.source.clone().finish().unwrap(); + let started = kio::clock::now(); + assert_eq!(drain(&mut reader).await, BTreeSet::new(), "expiry={expiry}"); + assert!( + kio::clock::now() - started < Duration::from_secs(1), + "expiry={expiry}: the evicted groups held the end for {:?}", + kio::clock::now() - started + ); + + let asked = chain.asked(); + reader + .update(reading(Duration::from_secs(3600), [Range::groups(0..)])) + .unwrap(); + let mut later = chain.subscribe(reading(None, [Range::groups(0..)])).await; + assert_eq!(drain(&mut later).await, got, "expiry={expiry}: the later reader"); + assert_eq!( + drain(&mut reader).await, + BTreeSet::new(), + "expiry={expiry}: the updated reader" + ); + assert_eq!( + chain.asked(), + asked, + "expiry={expiry}: an evicted group was asked again" + ); + drop(finished); + }) + .await + .unwrap_or_else(|_| panic!("expiry={expiry}: timed out")); + } +} + +/// After a widening from max delay B1 to B2 for a group the publisher no longer holds, its +/// stale drop under B1, still in flight when the relay sends the B2 update, settles no reader; +/// the drop the publisher sends again under the B2 update settles the B2 reader, over lite-07. +#[kio::test] +async fn a_drop_under_a_superseded_budget_settles_no_reader() { + // Reaches groups 5 and up by media time. + const B1: Duration = Duration::from_millis(3500); + // Reaches every group, by media time and by the wall clock this test spends. + const B2: Duration = Duration::from_secs(60); + // The publisher reclaims an unread group well within one trip to the relay and back. + const EXPIRY: Duration = Duration::from_secs(2); + // One way between the publisher and the relay. + const LATENCY: Duration = Duration::from_secs(10); + kio::clock::timeout(TIMEOUT * 10, async { + let chain = Chain::new([LITE07, LITE07], [expiring(EXPIRY), forever(), forever()]).await; + for sequence in (0..GROUPS).filter(|sequence| *sequence != 7) { + write(&chain.source, sequence, sequence * 1000); + } + let mut reader = chain.subscribe(reading(B1, [Range::oldest()])).await; + assert_eq!(received(&mut reader).await, [5, 6, 8, 9].into()); + + // Group 7 lands once its successor is older than B1 by the wall clock: the publisher + // drops it as stale under B1 at once, and reclaims it soon after, unread. + kio::clock::sleep(B1 + Duration::from_secs(1)).await; + chain.upstream.client_transport.set_latency(LATENCY); + let sent = kio::clock::now(); + write(&chain.source, 7, 7000); + let _finished = chain.source.clone().finish().unwrap(); + + // The B2 update leaves the relay while the B1 drop is halfway there. + kio::clock::sleep(LATENCY / 2).await; + // Any request the relay opens upstream from now on never arrives, so only the + // subscription can settle group 7. + chain.upstream.client_transport.hold_bidis(); + reader.update(reading(B2, [Range::oldest()])).unwrap(); + let ended = kio::clock::timeout(LATENCY * 10, reader.recv_group()) + .await + .expect("the reader never settled group 7") + .unwrap() + .map(|group| group.sequence); + assert_eq!(ended, None, "group 7 was reclaimed"); + // The B1 drop reached the relay at one latency, the B2 update the publisher at one and + // a half, and its drop the relay at two and a half. + let elapsed = kio::clock::now() - sent; + assert!( + elapsed >= LATENCY * 5 / 2 && elapsed < LATENCY * 5 / 2 + Duration::from_secs(1), + "settled after {elapsed:?}" + ); + }) + .await + .expect("timed out"); +} + +/// A run dropped before the track's first group is settled through a relay for a later +/// reader whose range covers it: the reader gets the group after it and ends, as one reading the +/// relay's own cache does. +#[kio::test] +async fn a_run_dropped_before_the_first_group_settles_a_later_range() { + let chain = Chain::new([LITE07, LITE07], [forever(), forever(), forever()]).await; + let mut reader = chain.subscribe(reading(None, [Range::groups(0..)])).await; + for sequence in 0..2 { + let group = chain.source.create_group(moq_net::group::Info { sequence }).unwrap(); + group.abort(moq_net::Error::App(1)).unwrap(); + } + write(&chain.source, 2, 0); + assert_eq!(received(&mut reader).await, [2].into()); + + let mut viewer = chain.subscribe(reading(None, [Range::groups(0..3)])).await; + assert_eq!(drain(&mut viewer).await, [2].into(), "on the viewer"); + let mut relayed = chain.subscribe_relay(reading(None, [Range::groups(0..3)])).await; + assert_eq!(drain(&mut relayed).await, [2].into(), "on the relay"); +} + +/// Under pool pressure on the relay, a reader whose record holds only delivered runs and +/// budget-independent drops folds them into its floor and reads on, while one holding lite-07 +/// stale reports within reach of the max age bound ends rather than forgo the re-offer: with +/// `Lagged` on the relay itself, and `TooFarBehind` over the wire. +#[kio::test] +async fn relay_pool_pressure_folds_final_runs_and_ends_stale_reports() { + /// The relay's readers each record this many disjoint dropped runs. + const RUNS: u64 = 200; + /// The groups that land on the relay at once once its pool shrinks. + const BURST: u64 = 1000; + let budget = Duration::from_secs(2); + for stale in [false, true] { + kio::clock::timeout(TIMEOUT * 10, async { + let pool = cache::Pool::new(cache::Config::default().with_expiry(None)); + let chain = Chain::new([LITE07, LITE07], [forever(), pool.clone(), forever()]).await; + let mut local = chain.subscribe_relay(reading(budget, [Range::groups(0..)])).await; + let mut remote = chain.subscribe(reading(budget, [Range::groups(0..)])).await; + let odd = write_odd(&chain, RUNS); + for (reader, name) in [(&mut local, "local"), (&mut remote, "remote")] { + assert_eq!(received(reader).await, odd, "stale={stale}: the {name} reader"); + } + settle_even(&chain, RUNS, stale.then_some(budget * 2)).await; + for (reader, name) in [(&mut local, "local"), (&mut remote, "remote")] { + assert_eq!( + received(reader).await, + BTreeSet::new(), + "stale={stale}: the {name} reader" + ); + } + + // The pool shrinks, and a burst of groups accrues the relay's debt: its own groups + // and drop set go first, then its readers' records. + pool.resize(4 * 1024); + for sequence in 2 * RUNS..2 * RUNS + BURST { + write(&chain.source, sequence, sequence); + } + let mut read = [0, 0]; + let mut ended = [None, None]; + for (index, reader) in [&mut local, &mut remote].into_iter().enumerate() { + while let Ok(group) = kio::clock::timeout(Duration::from_secs(1), reader.recv_group()).await { + match group { + Ok(Some(_)) => read[index] += 1, + Ok(None) => panic!("stale={stale}: reader {index} ended cleanly"), + Err(err) => { + ended[index] = Some(err); + break; + } + } + } + } + match stale { + false => { + assert!(ended.iter().all(Option::is_none), "the final runs fold: {ended:?}"); + // Under this much pressure the relay may evict a burst group before serving it, + // so only the local reader's count is exact. + assert_eq!(read[0], BURST, "the local reader reads on"); + assert!(read[1] > 0, "the remote reader reads on"); + } + true => { + let [local, remote] = ended; + assert!(matches!(local, Some(moq_net::Error::Lagged)), "local: {local:?}"); + assert!( + matches!(remote, Some(moq_net::Error::Stream(moq_net::StreamError::TooFarBehind))), + "remote: {remote:?}" + ); + } + } + chain.assert_open(&format!("stale={stale}")); + }) + .await + .unwrap_or_else(|_| panic!("stale={stale}: timed out")); + } +} + +/// Write the odd sequences below `2 * runs`, presented a millisecond apart so only the wall +/// clock makes one stale, and return them. +fn write_odd(chain: &Chain, runs: u64) -> BTreeSet { + for run in 0..runs { + write(&chain.source, 2 * run + 1, 2 * run + 1); + } + (0..runs).map(|run| 2 * run + 1).collect() +} + +/// Settle the even sequences below `2 * runs` as disjoint runs. With `stale`, each group lands +/// that long after its successor, so a reader with a shorter budget drops it as stale; without, +/// each is aborted, a drop no budget changes. +async fn settle_even(chain: &Chain, runs: u64, stale: Option) { + match stale { + Some(late) => { + kio::clock::sleep(late).await; + for run in 0..runs { + write(&chain.source, 2 * run, 2 * run); + } + } + None => { + for run in 0..runs { + let group = chain + .source + .create_group(moq_net::group::Info { sequence: 2 * run }) + .unwrap(); + group.abort(moq_net::Error::App(1)).unwrap(); + } + } + } +} diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index e87fc5e9185..f98d0bbebad 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -21,6 +21,7 @@ mod control_stats; mod datagram; mod dial_split_horizon; mod drained_claim; +mod drop_record; mod empty_group; mod errored_end; mod extensions; From afc72537f3fa5f2a963b92d1c9d950dc78b287c2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 21:29:48 -0800 Subject: [PATCH 11/37] fix(net): keep final drops dropped, re-judge evicted stale runs, and fill runs by range Fixes the bugs the relay-level drop record tests found, in Rust and js/net: - A relay serving a FETCH for a peer answers a sequence its copy gave up for good (a skip, or a drop under no budget) from that drop instead of asking upstream (Rust `poll_fetch_cached` judges_drops; JS `givenUp`). A resumed group's continuation still goes past a drop. - A read that hands out a group keeps it above its record's fold, so its publisher does not take it as told already and close it silently. - JS: an update re-judges a reopened stale run whose groups were evicted, and drops it again at once or asks a fill, as Rust does. - JS: a fill's OLD answer stays OLD and is refused only to readers no more tolerant than its budget (`fill.rejectGroups`, mirroring Rust `Inner::reject`). - JS: a lite-07 fill with no budget asks for the whole run on one bounded SUBSCRIBE instead of one FETCH per group; an evicted refill is given up. Co-Authored-By: Claude Opus 5.5 --- js/net/src/broadcast.test.ts | 11 +- js/net/src/broadcast.ts | 14 +- js/net/src/fill.ts | 42 +++- js/net/src/internal.ts | 47 ++++ js/net/src/lite/copy_fill.test.ts | 10 +- js/net/src/lite/drop_record.test.ts | 107 ++++++--- js/net/src/lite/publisher.ts | 9 +- js/net/src/lite/subscriber.ts | 239 ++++++++++++++++++-- js/net/src/resume.ts | 9 +- js/net/src/track.ts | 122 ++++++++-- js/net/src/wire.ts | 6 + rs/moq-net/src/model/record.rs | 56 +++-- rs/moq-net/src/model/track.rs | 71 ++++-- rs/moq-net/tests/integration/drop_record.rs | 1 - 14 files changed, 620 insertions(+), 124 deletions(-) diff --git a/js/net/src/broadcast.test.ts b/js/net/src/broadcast.test.ts index 4522f30ef07..3b433682514 100644 --- a/js/net/src/broadcast.test.ts +++ b/js/net/src/broadcast.test.ts @@ -1,7 +1,7 @@ import { expect, test } from "bun:test"; import { Duration, Manual, yieldNow } from "@moq/clock"; import { Consumer as BroadcastConsumer, Producer as BroadcastProducer } from "./broadcast.ts"; -import { NotFound, StreamCode } from "./error.ts"; +import { NotFound, StreamCode, toStreamCode } from "./error.ts"; import { hooks } from "./internal.ts"; import { textFrame } from "./mock.ts"; import { Timescale, Timestamp } from "./time.ts"; @@ -666,9 +666,12 @@ test("fetch maxDelay tightens the publisher's maxAge without changing omission", expect(omitted.sequence).toBe(0n); omitted.close(); - await expect(wireOf(broadcast).fetchGroup("video", 0n, { maxDelay: Duration.millis(0) })).rejects.toBeInstanceOf( - NotFound, - ); + // Stale under its own budget, which says OLD as the Rust `fetch_group` does, so a relay asking + // for a narrow reader refuses it to that reader alone. + const stale = await wireOf(broadcast) + .fetchGroup("video", 0n, { maxDelay: Duration.millis(0) }) + .catch((err: unknown) => err); + expect(toStreamCode(stale)).toBe(StreamCode.Old); broadcast.close(); }); diff --git a/js/net/src/broadcast.ts b/js/net/src/broadcast.ts index 5fb8106363f..e75f54823ce 100644 --- a/js/net/src/broadcast.ts +++ b/js/net/src/broadcast.ts @@ -6,7 +6,7 @@ import { type GetPromise, Once, Signal } from "@moq/signals"; import * as cache from "./cache.ts"; import * as datagram from "./datagram.ts"; -import { NotFound, StreamCode, StreamError, toStreamCode } from "./error.ts"; +import { NotFound, StreamCode, StreamError } from "./error.ts"; import type { Consumer as GroupConsumer } from "./group.ts"; import type * as Instance from "./instance.ts"; import { hooks } from "./internal.ts"; @@ -288,6 +288,12 @@ function largest(state: BroadcastState, name: string): track.Position | undefine return producer && hooks.cachedLargest(producer); } +// The error `name`'s cache gave `sequence` up for good with, without subscribing. +function givenUp(state: BroadcastState, name: string, sequence: bigint): Error | undefined { + const producer = lookup(state, name); + return producer && hooks.givenUpGroup(producer, sequence); +} + // Fetch one group as a one-range subscription: the cached group at once, or else one the track's // {@link track.Requests} handler answers, asked at once whether the group was written yet or not, // as the Rust `fetch_group` does. A group the producer writes meanwhile answers it too. With no @@ -327,8 +333,8 @@ async function fetchGroup( void recv.group.closed.then(() => subscriber.close()); return recv.group; case "drop": - // A fill's refusal says why; anything else given up is just gone. - if (toStreamCode(recv.error) === StreamCode.Old) throw new NotFound(`group ${sequence}`); + // Its own reason, so a stale verdict stays OLD: a relay's fill asking under this + // budget refuses it only to readers no more tolerant (see `fill.rejectGroups`). throw recv.error; case "error": throw recv.error; @@ -393,6 +399,7 @@ export class Producer { subscribe: (name, options) => subscribe(this.#state, name, options), resolveTrackInfo: (name, hold) => resolveTrackInfo(this.#state, name, hold), largest: (name) => largest(this.#state, name), + givenUp: (name, sequence) => givenUp(this.#state, name, sequence), fetchGroup: (name, sequence, options) => fetchGroup(this.#state, name, sequence, options), requested: () => requested(this.#state), datagrams: (name) => datagrams(this.#state, name), @@ -555,6 +562,7 @@ export class Consumer { subscribe: (name, options) => subscribe(this.#state, name, options), resolveTrackInfo: (name, hold) => resolveTrackInfo(this.#state, name, hold), largest: (name) => largest(this.#state, name), + givenUp: (name, sequence) => givenUp(this.#state, name, sequence), fetchGroup: (name, sequence, options) => fetchGroup(this.#state, name, sequence, options), requested: () => requested(this.#state), datagrams: (name) => datagrams(this.#state, name), diff --git a/js/net/src/fill.ts b/js/net/src/fill.ts index b757c144aa4..87c040a4404 100644 --- a/js/net/src/fill.ts +++ b/js/net/src/fill.ts @@ -12,7 +12,7 @@ */ import { Duration } from "@moq/clock"; import { type GetPromise, type Getter, Signal } from "@moq/signals"; -import { NotFound } from "./error.ts"; +import { NotFound, StreamCode, toStreamCode } from "./error.ts"; import type { Producer as GroupProducer } from "./group.ts"; import { insertRun, NO_MAX_DELAY, runGaps } from "./internal.ts"; import type { Info, Order } from "./track.ts"; @@ -56,15 +56,19 @@ export interface Asker { maxDelay(): number; /** Which end it reads first. */ order(): Order; - /** Stop waiting for `[start, end)`, which a fill refused with `error`, without asking again. */ - refuse(start: bigint, end: bigint, error: Error): void; + /** + * Stop waiting for `[start, end)`, which a fill refused with `error`, without asking again. + * With `stale`, the source judged them too old under that budget in milliseconds: a verdict + * on this reader only while its own budget is no wider, so a widening asks again. + */ + refuse(start: bigint, end: bigint, error: Error, stale?: number): void; } // The fill operations a handle runs against the state it came from; assigned in its static block. let fillOps: { accept(state: FillState, pending: Pending, sequence: bigint, info: Info | undefined): GroupProducer; - drop(state: FillState, pending: Pending, start: bigint, end: bigint): void; - refuse(state: FillState, pending: Pending, start: bigint, end: bigint, error: Error): void; + drop(state: FillState, pending: Pending, start: bigint, end: bigint, error?: Error): void; + refuse(state: FillState, pending: Pending, start: bigint, end: bigint, error: Error, stale?: number): void; settle(state: FillState, pending: Pending): void; name(state: FillState): string; }; @@ -311,16 +315,17 @@ export class FillState { // Landing in the cache answers it, as `arrived` records. return state.#track.accept(sequence, info); }, - drop: (state, pending, start, end) => { + drop: (state, pending, start, end, error) => { for (const run of pending.owed(start, end)) { - state.#track.drop(run.start, run.end, new NotFound(`group ${run.start}`)); + state.#track.drop(run.start, run.end, error ?? new NotFound(`group ${run.start}`)); } state.#resolve(pending, start, end); }, - refuse: (state, pending, start, end, error) => { + refuse: (state, pending, start, end, error, stale) => { for (const gap of pending.unresolved(start, end)) { for (const asker of pending.askers) { - for (const run of asker.wants(gap.start, gap.end)) asker.refuse(run.start, run.end, error); + for (const run of asker.wants(gap.start, gap.end)) + asker.refuse(run.start, run.end, error, stale); } } state.#resolve(pending, start, end); @@ -343,6 +348,16 @@ function subtract(gap: Run, run: Run): Run[] { export let wantedRuns: (fill: Fill) => Run[]; /** Wait for a change to the groups or readers of a fill. @internal */ export let fillChanged: (fill: Fill) => Promise; +/** + * Answer `[start, end)` of `fill` with its source's refusal of a request for them. An OLD one + * says every older group of the fill is stale too, since a timed track's starts never rewind: + * with `stale`, the budget in milliseconds the request carried, it is refused to the readers + * waiting now, a verdict only on those no more tolerant, so a wider reader asks again; without, + * the publisher's max age judged it, which binds every reader, so the track records it dropped. + * Any other refusal answers only `[start, end)`, for now. Mirrors the Rust `Fill::reject_groups`. + * @internal + */ +export let rejectGroups: (fill: Fill, start: bigint, end: bigint, error: Error, stale: number | undefined) => void; // Constructs the public handles from within this module; assigned in their static blocks. let makeFill: (state: FillState, pending: Pending) => Fill; @@ -450,6 +465,15 @@ export class Fill { makeFill = (state, pending) => new Fill(state, pending); wantedRuns = (fill) => fill.#pending.owed(); fillChanged = (fill) => Promise.resolve(fill.#pending.revision.changed()); + rejectGroups = (fill, start, end, error, stale) => { + if (toStreamCode(error) !== StreamCode.Old) { + fill.refuseGroups(start, end, error); + } else if (stale === undefined) { + fillOps.drop(fill.#state, fill.#pending, fill.start, end, error); + } else { + fillOps.refuse(fill.#state, fill.#pending, fill.start, end, error, stale); + } + }; } /** The most urgent delivery priority among the readers waiting on it now. */ diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 80f4c2ff156..9da84ad18e5 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -263,6 +263,43 @@ export function insertDrop( return merged; } +/** + * The parts of sorted, disjoint `runs` that sorted, disjoint `cut` leaves, each keeping its run's + * fields, as {@link removeRun} of each cut would, in one pass. + */ +export function subtractRuns(runs: readonly R[], cut: readonly Run[]): R[] { + const kept: R[] = []; + let first = 0; + for (const run of runs) { + while (first < cut.length && cut[first].end <= run.start) first++; + let from = run.start; + for (let i = first; i < cut.length && cut[i].start < run.end; i++) { + if (cut[i].start > from) kept.push({ ...run, start: from, end: cut[i].start }); + from = max(from, cut[i].end); + } + if (from < run.end) kept.push({ ...run, start: from, end: run.end }); + } + return kept; +} + +/** + * `drops` with each of the sorted, disjoint `runs` inserted as {@link insertDrop} inserts it, in + * one pass: folding in a span of many runs costs the runs, not their square. + */ +export function insertDrops(drops: readonly DropRun[], runs: readonly DropRun[]): DropRun[] { + if (runs.length === 0) return [...drops]; + const next = subtractRuns(drops, runs); + next.push(...runs); + next.sort((a, b) => (a.start < b.start ? -1 : a.start > b.start ? 1 : 0)); + const merged: DropRun[] = []; + for (const run of next) { + const last = merged[merged.length - 1]; + if (last && last.end === run.start && sameHole(last, run)) last.end = run.end; + else merged.push({ ...run }); + } + return merged; +} + /** The error `drops` gave `sequence` up with, if any. */ export function dropAt(drops: readonly DropRun[], sequence: bigint): Error | undefined { return drops.find((run) => run.start <= sequence && sequence < run.end)?.error; @@ -616,6 +653,13 @@ export const hooks: { stageRequest: (request: Request, info: InfoInit) => StagedTrack; /** The newest frame a producer's cache holds, read without subscribing; assigned by `track.ts`. */ cachedLargest: (producer: Producer) => Position | undefined; + /** + * The error a producer gave `sequence` up for good with, when it holds no group for it: a + * skip, or a drop under no budget, which none of its readers fills. A stale drop stays + * fillable by a wider budget. Mirrors the Rust `poll_fetch_cached` `judges_drops`; assigned + * by `track.ts`. + */ + givenUpGroup: (producer: Producer, sequence: bigint) => Error | undefined; /** The publisher priority from a demanded track's committed {@link Info}; assigned by `track.ts`. */ demandPriority: (demand: Demand) => number; /** Seed a received track from the publisher's reported edge; assigned by `track.ts`. */ @@ -979,6 +1023,9 @@ export const hooks: { cachedLargest: () => { throw new Error("track.ts not loaded"); }, + givenUpGroup: () => { + throw new Error("track.ts not loaded"); + }, demandPriority: () => { throw new Error("track.ts not loaded"); }, diff --git a/js/net/src/lite/copy_fill.test.ts b/js/net/src/lite/copy_fill.test.ts index d36665bfcb0..efdad8f9c57 100644 --- a/js/net/src/lite/copy_fill.test.ts +++ b/js/net/src/lite/copy_fill.test.ts @@ -1,6 +1,7 @@ // A received copy asks its publisher for the groups its readers miss, over lite FETCH, as the Rust -// lite subscriber's `FillServeRun` does: below where the copy's feed started, by the copy's start -// rule, and inside the copy, once it evicted what it received. +// lite subscriber's `FillServeRun` does, or on lite-07 for a reader with no budget, by run on a +// SUBSCRIBE of its own, as its `Recovery` does: below where the copy's feed started, by the copy's +// start rule, and inside the copy, once it evicted what it received. import { expect, test } from "bun:test"; import { Duration, Elapsed, Manual, sleep, timeout } from "@moq/clock"; import { DEFAULT_EXPIRY } from "../cache.ts"; @@ -209,7 +210,8 @@ test.each([Lite.ALPN_05, Lite.ALPN_06, Lite.ALPN_07_WIP])( expect((await first.recvGroup())?.sequence).toBe(4n); const before = peer.fetches(); - // The same ranges, so nothing changes upstream: only a FETCH brings what the copy lost. + // The same ranges, so nothing changes upstream: only a fill brings what the copy lost, a + // FETCH per group, or on lite-07 one SUBSCRIBE for the run, its groups and its miss. const again = peer.remote.track("video").subscribe(history); const got = await take(again, 4); expect(sorted(got)).toEqual([ @@ -218,7 +220,7 @@ test.each([Lite.ALPN_05, Lite.ALPN_06, Lite.ALPN_07_WIP])( [3n, "old 3"], [4n, "live 4"], ]); - expect(peer.fetches()).toBe(before + 4); + expect(peer.fetches()).toBe(before + (alpn === Lite.ALPN_07_WIP ? 1 : 4)); first.close(); again.close(); } finally { diff --git a/js/net/src/lite/drop_record.test.ts b/js/net/src/lite/drop_record.test.ts index ab9e9a30cce..df94cf290c8 100644 --- a/js/net/src/lite/drop_record.test.ts +++ b/js/net/src/lite/drop_record.test.ts @@ -183,27 +183,38 @@ async function chain(up: Version, down: Version) { }; } -/** The next thing `reader` settles: a group, a drop or skip, its end, or its error. */ -async function next(reader: TrackSubscriber): Promise { +/** + * The next thing `reader` settles: a group, a drop or skip, its end, or its error. Once `signal` + * aborts it stops reading and never resolves, so a read given up takes nothing from the reader. + */ +async function next(reader: TrackSubscriber, signal?: AbortSignal): Promise { for (;;) { + if (signal?.aborted) return new Promise(() => {}); const recv = hooks.tryRecvGroup(reader, true); if (recv.kind !== "idle" && recv.kind !== "boundary") return recv; await new Promise((resolve) => { - const dispose = hooks.groupChanged(reader, () => { + const wake = () => { dispose(); + signal?.removeEventListener("abort", wake); resolve(); - }); + }; + const dispose = hooks.groupChanged(reader, wake); + signal?.addEventListener("abort", wake); }); } } -/** `work`'s result, or undefined once `ms` of manual time passed without it. */ -async function within(clock: Manual, work: Promise, ms = 1000): Promise { +/** The next thing `reader` settles, or undefined once `ms` of manual time passed without it. */ +async function within(clock: Manual, reader: TrackSubscriber, ms = 1000): Promise { + const controller = new AbortController(); try { - return await clock.drive(timeout(Duration.millis(ms), work)); + return await clock.drive(timeout(Duration.millis(ms), next(reader, controller.signal))); } catch (err) { if (err instanceof Elapsed) return undefined; throw err; + } finally { + // A read still pending would otherwise take the next group from whoever reads after. + controller.abort(); } } @@ -233,7 +244,7 @@ function settled(recv: Recv | undefined): unknown[] { async function drain(clock: Manual, reader: TrackSubscriber): Promise { const got: unknown[][] = []; for (;;) { - const recv = await within(clock, next(reader)); + const recv = await within(clock, reader); if (!recv) return got; got.push(settled(recv)); if (recv.kind === "done" || recv.kind === "error") return got; @@ -329,8 +340,7 @@ test("a drop under the narrower update settles no reader once the widening is se // Mirrors Rust `after_a_widening_the_old_updates_drop_settles_no_reader` in // rs/moq-net/tests/integration/drop_record.rs. -// Skipped: the publisher's reader re-offers an evicted stale group from its skip record, so nothing drops it again until the end sweep. -test.skip("a widening drops again at once what the publisher evicted", () => widening(true)); +test("a widening drops again at once what the publisher evicted", () => widening(true)); // Readers on the viewer while the publisher still holds groups 0 to 5 and the relay and the // viewer evicted all but group 5: `owed` subscribed before the eviction and read nothing, `fresh` @@ -348,14 +358,17 @@ async function relayEvicted(readers: ("owed" | "fresh" | "widened")[]): Promise< const read: unknown[][] = []; for (let i = 0; i < 5; i++) read.push(settled(await next(first))); expect(groupsOf(read)).toEqual([0n, 1n, 2n, 3n, 4n]); - // Half a second gives up groups 1 and 2 as stale, on the viewer alone. + // Half a second gives up groups 1 and 2 as stale, on the viewer alone. Group 0 is the + // reader's start or stale too, depending on which group reached it first. const widened = chained.remote.track("video").subscribe({ delay: { max: Duration.millis(500) } }); const stale = await drain(clock, widened); - expect(stale.filter((row) => row[0] === "drop")).toEqual([ - ["drop", 1n, 2n, StreamCode.Old], - ["drop", 2n, 3n, StreamCode.Old], - ]); - expect(groupsOf(stale)).toEqual([0n, 3n, 4n]); + const given = stale.filter((row) => row[0] === "drop"); + expect(given.every((row) => row[3] === StreamCode.Old && row[2] === (row[1] as bigint) + 1n)).toBe(true); + const givenUp = given.map((row) => row[1] as bigint); + expect(givenUp).toEqual(expect.arrayContaining([1n, 2n])); + const accounted = [...givenUp, ...(groupsOf(stale) as bigint[])]; + expect(accounted.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))).toEqual([0n, 1n, 2n, 3n, 4n]); + expect(groupsOf(stale)).toEqual(expect.arrayContaining([3n, 4n])); // The relay and the viewer age out everything but the live edge; the publisher keeps it all. await clock.advance(cache.DEFAULT_EXPIRY.add(Duration.millis(10_000))); @@ -372,7 +385,7 @@ async function relayEvicted(readers: ("owed" | "fresh" | "widened")[]): Promise< else { reader = widened; reader.update(WIDE); - expected = [1n, 2n, 5n]; + expected = [...givenUp, 5n]; } const rows = await drain(clock, reader); expect(rows.filter((row) => row[0] !== "group")).toEqual([]); @@ -397,8 +410,7 @@ test("a group the relay evicted while the publisher holds it is filled for a new // Mirrors Rust `a_group_the_relay_evicted_is_filled_never_dropped` in // rs/moq-net/tests/integration/drop_record.rs. -// Skipped: the widened reader re-offers its evicted stale groups from its skip record, so their fill lags seconds and group 5 never arrives. -test.skip("a group the relay evicted while the publisher holds it is filled for a widened reader", () => +test("a group the relay evicted while the publisher holds it is filled for a widened reader", () => relayEvicted(["widened"])); // Mirrors Rust `an_evicted_drop_is_forwarded_unchanged_and_never_reoffered` in @@ -496,6 +508,42 @@ for (const down of [Version.DRAFT_06, Version.DRAFT_07]) { }); } +// Mirrors Rust `a_lite06_upstream_keeps_a_reported_group_dropped_for_a_lite07_viewer` in +// rs/moq-net/tests/integration/drop_record.rs. +test("through a lite-06 upstream, a group reported dropped as it lands stays dropped for a lite-07 viewer", async () => { + using clock = manual(); + const chained = await chain(Version.DRAFT_06, Version.DRAFT_07); + for (let sequence = 0n; sequence < 10n; sequence++) { + if (sequence !== 7n) chained.write(sequence, Number(sequence) * 1000); + } + // Reaches groups 5 and up by media time. + const first = chained.remote.track("video").subscribe({ delay: { max: Duration.millis(3500) }, ranges: [{}] }); + await chained.reached(); + expect(groupsOf(await drain(clock, first))).toEqual([5n, 6n, 8n, 9n]); + + // Group 7 lands once its successor is older than 3.5s by the clock, so the lite-06 + // publisher reports it dropped, which lite-06 makes final. + await clock.advance(Duration.millis(4500)); + chained.write(7n, 7000); + expect(groupsOf(await drain(clock, first))).toEqual([]); + + // Reaches every group: the widened reader fills 0 to 4, and group 7 alone stays dropped, + // for it and for a later reader, rather than being fetched from the publisher. + const all = [0n, 1n, 2n, 3n, 4n, 5n, 6n, 8n, 9n]; + first.update({ delay: { max: Duration.millis(60_000) }, ranges: [{}] }); + expect(groupsOf(await drain(clock, first))).toEqual([0n, 1n, 2n, 3n, 4n]); + const second = chained.remote.track("video").subscribe({ delay: { max: Duration.millis(60_000) }, ranges: [{}] }); + expect(groupsOf(await drain(clock, second))).toEqual(all); + + chained.track.finish(); + for (const reader of [first, second]) expect(groupsOf(await drain(clock, reader))).toEqual([]); + expect(chained.ended()).toBe(false); + + first.close(); + second.close(); + await chained.close(); +}); + // Mirrors Rust `pool_pressure_folds_final_runs_and_ends_a_stale_reader` in // rs/moq-net/tests/integration/drop_record.rs. for (const stale of [false, true]) { @@ -539,7 +587,7 @@ for (const stale of [false, true]) { let groups = 0; for (let sequence = 2n * runs + 1n; sequence < 2n * runs + 65n && error === undefined; sequence++) { chained.write(sequence, budget + Number(sequence), 16 * 1024); - const recv = await within(clock, next(reader)); + const recv = await within(clock, reader); if (recv?.kind === "error") error = recv.error; else { expect(settled(recv)).toEqual(["group", sequence]); @@ -564,7 +612,8 @@ for (const stale of [false, true]) { // relay's copy shares with all its readers, then the relay `lost` that run: it aged out past the // pool's idle window, or the pool shed it under pressure. The readers in `readers` subscribe next: // a narrow one never takes groups 1 and 2, and a wide one is filled them through the relay from -// the publisher, which still holds them. +// the publisher, which still holds them: fetched once the viewer's copy forgot the run too, and +// re-offered over both subscriptions by the widening update while it still holds it stale. async function lostRun(lost: "forgot" | "shed", readers: ("wide" | "narrow")[]): Promise { using clock = manual(); const opened = pools(); @@ -584,11 +633,14 @@ async function lostRun(lost: "forgot" | "shed", readers: ("wide" | "narrow")[]): ]); // Past the pool's idle window the relay's copy and the viewer's forget the run; a pool too - // small for one large group sheds the relay's. + // small for one large group sheds the relay's. The pool then grows back, so the relay holds + // the groups it is filled next until its readers take them: one too small to hold any would + // see a refill evicted again, which its reader gives up and names dropped. if (lost === "forgot") await clock.advance(cache.DEFAULT_EXPIRY.mul(2)); else relay.pool.resize(cache.RUN_OVERHEAD); chained.write(5n, 6100, 64 * 1024); expect(settled(await next(reader))).toEqual(["group", 5n]); + if (lost === "shed") relay.pool.resize(); const before = chained.fetches(); const heard = wire.heard.length; @@ -598,7 +650,7 @@ async function lostRun(lost: "forgot" | "shed", readers: ("wide" | "narrow")[]): const filled = await drain(clock, wide); expect(filled.filter((row) => row[0] !== "group")).toEqual([]); expect(groupsOf(filled)).toEqual(expect.arrayContaining([1n, 2n])); - expect(chained.fetches().up).toBeGreaterThan(before.up); + if (lost === "forgot") expect(chained.fetches().up).toBeGreaterThan(before.up); wide.close(); } else { // Judged stale by its own budget, or filled and then judged. @@ -628,9 +680,7 @@ for (const lost of ["forgot", "shed"] as const) { // Mirrors Rust `a_shed_run_is_judged_or_filled_through_the_relay` in // rs/moq-net/tests/integration/drop_record.rs. - // Skipped: the relay refills the run by FETCH under the narrow budget and records the stale answer as a final NotFound drop for every reader. - test.skip(`a sequence whose run the relay ${lost} is filled for a wide reader`, () => - lostRun(lost, ["narrow", "wide"])); + test(`a sequence whose run the relay ${lost} is filled for a wide reader`, () => lostRun(lost, ["narrow", "wide"])); } const BILLIONS = 1n << 40n; @@ -653,7 +703,7 @@ test("a run of billions of sequences is settled by each reader in one step, drop chained.write(1n + 2n * BILLIONS, 1000); for (const reader of [narrow, wide]) { const got: unknown[][] = []; - for (let i = 0; i < 3; i++) got.push(settled(await within(clock, next(reader)))); + for (let i = 0; i < 3; i++) got.push(settled(await within(clock, reader))); expect(got).toEqual([ ["drop", 1n, 1n + BILLIONS, StreamCode.Cancel], ["skip", 1n + BILLIONS, 1n + 2n * BILLIONS, StreamCode.NotFound], @@ -675,8 +725,7 @@ test("a run of billions of sequences is settled by each reader in one step, drop // Mirrors Rust `a_run_of_billions_settles_in_one_step` in // rs/moq-net/tests/integration/drop_record.rs. for (const judged of ["reopened by a widening", "forgotten by the relay"] as const) { - // Skipped: a run judged again is filled one lite FETCH per sequence from its top down, so a run of billions never settles. - test.skip(`a run of billions ${judged} is judged again in one step`, async () => { + test(`a run of billions ${judged} is judged again in one step`, async () => { using clock = manual(); const chained = await chain(Version.DRAFT_07, Version.DRAFT_07); const reader = chained.remote.track("video").subscribe({ delay: { max: Duration.millis(1500) } }); diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index 02187cce2ca..84497242b8f 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -1277,9 +1277,12 @@ export class Publisher { } finally { hold.abort(); } - // A group the track does not cache goes to its request handler at once, written yet - // or not, as in Rust; with none open it is NotFound. A pending answer holds this - // FETCH's slot until the requester leaves. + // A group the track gave up for good is refused with that drop, as its own readers + // are, rather than asked upstream again. One it does not cache goes to its request + // handler at once, written yet or not, as in Rust; with none open it is NotFound. A + // pending answer holds this FETCH's slot until the requester leaves. + const refused = wireOf(front).givenUp(msg.track, msg.group); + if (refused) throw refused; group = await front.track(msg.track).fetchGroup(msg.group, { priority: msg.priority, maxDelay: msg.maxDelay === undefined ? undefined : Duration.millis(msg.maxDelay), diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 39fd344bef7..b3cd7311ad1 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -11,6 +11,7 @@ import type * as datagram from "../datagram.ts"; import * as DatagramStream from "../datagram_stream.ts"; import { error, + NotFound, ProtocolViolation, reason, SessionCode, @@ -21,6 +22,7 @@ import { toStreamCode, unauthorized, } from "../error.ts"; +import { fillChanged, rejectGroups, wantedRuns } from "../fill.ts"; import * as netGroup from "../group.ts"; import { Cost, type Hop, MAX_HOPS, routesEqual, UNKNOWN_HOP, type WireRoute } from "../hop.ts"; import * as Instance from "../instance.ts"; @@ -154,6 +156,14 @@ interface DatagramEntry { ended?: boolean; } +// How a received copy's groups are read, and the publisher's max age, which bounds every reader. +interface Framing { + compress: boolean; + timescale: number; + // Undefined while the copy has no info, and null for no bound. + maxAge?: Duration | null; +} + interface SubscribeEntry { datagrams?: DatagramEntry; /** Release the stream demand held by readers of the legacy datagram track. */ @@ -216,6 +226,8 @@ interface SubscribeEntry { // Called at the SUBSCRIBE's first response (lite-05+). Until then a SUBSCRIBE without an // instance may be answered by whichever stream won the path, so a replacement refuses it. answered?: () => void; + // A fill's own subscription, whose groups are cancelled once nobody wants them, as a FETCH's. + filling?: boolean; } /** @@ -315,6 +327,11 @@ export class Subscriber { // group closes. #fetches = new Map }>(); + // The runs a lite-07 copy's subscription reported stale under an update it has since + // superseded: its publisher re-offers them, or drops them again under the new number, so a + // fill waits on the subscription for them rather than FETCHing a copy the re-offer duplicates. + #reoffers = new WeakMap>(); + // The peer's PROBE estimates, written as they arrive (Lite03+ only). #probe?: Signal; @@ -1418,6 +1435,9 @@ export class Subscriber { stream.abort(e); } finally { answering?.removeEventListener("abort", replaced); + // Nothing more is re-offered on this subscription. + this.#reoffers.get(producer)?.set([]); + this.#reoffers.delete(producer); entry.tail.close(); entry.held?.close(); entry.datagramDemand?.(); @@ -1933,12 +1953,14 @@ export class Subscriber { // Rust `FillServeRun` does. async #runFills(broadcast: Path.Valid, consumed: Consumed, entry: SubscribeEntry): Promise { const producer = entry.track; - const framing = { compress: entry.compress, timescale: entry.timescale }; const requests = producer.requests(); const jobs = new Set>(); try { + // Accepted before its fills run, so this only reads what TRACK_INFO said. + const maxAge = (await producer.info().catch(() => undefined))?.maxAge; + const framing = { compress: entry.compress, timescale: entry.timescale, maxAge }; for (let fill = await requests.requestedFill(); fill; fill = await requests.requestedFill()) { - const job = this.#serveFill(broadcast, consumed, producer.name, framing, fill); + const job = this.#serveFill(broadcast, consumed, producer, framing, fill); jobs.add(job); void job.then(() => jobs.delete(job)); } @@ -1952,24 +1974,203 @@ export class Subscriber { async #serveFill( broadcast: Path.Valid, consumed: Consumed, - name: string, - framing: { compress: boolean; timescale: number }, + producer: track.Producer, + framing: Framing, fill: track.Fill, ): Promise { try { let last: bigint | undefined; for (let sequence = fill.wanted(); sequence !== undefined; sequence = fill.wanted()) { + // The subscription re-offers it, or drops it again, either of which answers the fill. + const reoffers = this.#reoffers.get(producer); + const reoffered = reoffers?.peek() ?? []; + if (reoffers && reoffered.some((run) => run.start <= sequence && sequence < run.end)) { + await race([fillChanged(fill), reoffers.changed()]); + continue; + } + // Lite-07 asks a fill with no budget for the whole owed run around it at once, up to a + // re-offered group, so the publisher answers each run it gave up with one verdict + // however many groups it spans. A budget is judged against the whole track, which a + // FETCH does and a bounded range does not, so a bounded fill FETCHes a group at a time. + if (hasRanges(this.version) && fill.maxDelay === null) { + let { start, end } = wantedRuns(fill).find( + (run) => run.start <= sequence && sequence < run.end, + ) ?? { start: sequence, end: sequence + 1n }; + for (const run of reoffered) { + if (run.end <= sequence) start = max(start, run.end); + else end = min(end, run.start); + } + await this.#fillRun(broadcast, consumed, producer, framing, fill, { start, end }); + continue; + } // Every fetch answers its group, so the same one coming back is a bug: closing the // fill names it dropped rather than fetching it forever. if (sequence === last) break; last = sequence; - await this.#fillGroup(broadcast, consumed, name, framing, fill, sequence); + await this.#fillGroup(broadcast, consumed, producer.name, framing, fill, sequence); } } finally { fill.close(); } } + // Ask `run` of `fill`, which has no budget, on a lite-07 SUBSCRIBE of its own bounded to the + // run, until the fill owes none of it, as the Rust `Recovery` asks: its groups land in the copy + // as a subscription's do, and the publisher names what it gives up by run. Whatever the + // subscription leaves owed when it ends is refused, so the next run is asked, not this one. + async #fillRun( + broadcast: Path.Valid, + consumed: Consumed, + producer: track.Producer, + framing: Framing, + fill: track.Fill, + run: Run, + ): Promise { + const owes = () => wantedRuns(fill).some((owed) => owed.start < run.end && run.start < owed.end); + const id = this.#subscribeNext++; + const ranges: track.Range[] = [{ start: { absolute: run.start }, end: { absolute: run.end } }]; + const msg = this.#withRanges( + this.#subscribeMessage( + id, + broadcast, + consumed, + producer.name, + { priority: fill.priority, order: fill.order, delay: { max: null } }, + ranges, + ), + ranges, + undefined, + ); + const entry: SubscribeEntry = { + track: producer, + compress: framing.compress, + timescale: framing.timescale, + tail: new Tail({ grace: () => hooks.tailGrace(producer), bill: hooks.billTrack(producer) }), + requested: msg.startGroup, + floor: msg.startGroup, + asked: ranges, + live: new LiveRanges(this.version, msg.ranges ?? []), + starts: new Map(), + update: 0n, + sent: 0, + filling: true, + }; + const left = new AbortController(); + const signal = consumed.valid ? AbortSignal.any([consumed.valid, left.signal]) : left.signal; + let failed: Error = new StreamError(StreamCode.Cancel, { message: "fill subscription ended" }); + let draining = Promise.resolve(); + this.#subscribes.set(id, entry); + try { + while ( + (await this.#auth?.ready(broadcast, "subscribe", { + signal: consumed.valid, + until: () => fillChanged(fill), + })) === false + ) { + if (!owes()) return; + } + if (this.#denied(broadcast)) throw unauthorized(broadcast); + const stream = await this.#exchange( + { sendOrder: sendOrder({ priority: fill.priority }), version: this.version }, + async (stream) => { + await stream.writer.u53(StreamId.Subscribe); + await msg.encode(stream.writer, this.version); + return stream; + }, + signal, + new Deadline(), + ); + let ended = false; + const responses = this.#fillResponses(stream, entry, fill, run).then( + () => { + ended = true; + }, + // A reset says why the rest is refused. + (err: unknown) => { + ended = true; + failed = error(err); + }, + ); + try { + while (owes() && !ended) await race([fillChanged(fill), responses]); + } catch (err) { + stream.abort(error(err)); + throw err; + } + if (ended) { + stream.close(); + } else { + // Done asking. Its streams are read to their end, or until nobody wants their group, + // as a FETCH's is, and only then FINned, which ends whatever the publisher still serves. + draining = (async () => { + await race([entry.tail.settle(() => true, hooks.trackReleased(producer)), responses]); + stream.writer.close(); + await responses; + stream.close(); + })(); + } + } catch (err) { + failed = error(err); + } finally { + left.abort(); + if (owes()) fill.refuseGroups(run.start, run.end, failed); + void draining.then(() => { + this.#subscribes.delete(id); + entry.tail.close(); + }); + } + } + + // Read a fill subscription's responses to its FIN: answer the fill with what the publisher + // names given up, judged as the Rust `RangeFills::refuse` does. + async #fillResponses(stream: Stream, entry: SubscribeEntry, fill: track.Fill, run: Run): Promise { + for (;;) { + const resp = await decodeSubscribeResponseMaybe(stream.reader, this.version); + if (!resp) return; + if ("start" in resp) { + // The range starts past what it skips, which it will not deliver: refused for this fill + // alone, since another request may still get them. + const start = resp.start.starts[0] ?? run.end; + if (run.start < start) + fill.refuseGroups(run.start, min(start, run.end), new NotFound(`group ${run.start}`)); + } else if ("drop" in resp) { + const { start, end } = resp.drop; + entry.tail.account(start, end); + const cause = resp.drop.cause(this.version); + // Only a group its publisher never had is lost for every reader, and asked with no + // budget, an OLD one is past the publisher's max age, which binds every reader too. + if (toStreamCode(cause) === StreamCode.NotFound) fill.dropGroups(start, end); + else rejectGroups(fill, start, end, cause, undefined); + } else if ("skip" in resp) { + // Never produced: final, as the subscription feeding the copy records it. + const { start, end } = resp.skip; + entry.tail.account(start, end); + hooks.skipTrackGroups(entry.track, start, end); + } else if ("finish" in resp) { + // Nothing is delivered at or past the track's end. + const end = max(resp.finish.group, run.start); + if (end < run.end) fill.refuseGroups(end, run.end, new NotFound(`group ${end}`)); + } + } + } + + // The budget an OLD answer to a request carrying `asked` judged, or undefined when it + // reached the publisher's max age, which binds every reader, as the Rust + // `Inner::old_binds_everyone` judges. No budget leaves only the max age. + #stale(framing: Framing, asked: number | undefined): number | undefined { + const bound = framing.maxAge?.asMillis(); + return asked === undefined || (bound !== undefined && asked >= bound) ? undefined : asked; + } + + // Settle `[start, end)` of what `producer`'s subscription re-offers: a stream or a verdict + // under the latest update answered it. + #reoffered(producer: track.Producer, start: bigint, end: bigint): void { + const reoffers = this.#reoffers.get(producer); + const runs = reoffers?.peek(); + if (!runs?.some((run) => run.start < end && start < run.end)) return; + reoffers?.set(removeRun(runs, start, end)); + } + // FETCH group `sequence` of `fill` and write it into the copy once the publisher answers: lite // has no FETCH_OK, so a publisher without the group resets the stream, which refuses the group // to the readers waiting now, as the Rust `GroupFill::reject` does. A fetch nobody waits on any @@ -1978,7 +2179,7 @@ export class Subscriber { broadcast: Path.Valid, consumed: Consumed, name: string, - framing: { compress: boolean; timescale: number }, + framing: Framing, fill: track.Fill, sequence: bigint, ): Promise { @@ -1986,7 +2187,7 @@ export class Subscriber { void fill .unwanted(sequence) .then(() => cancel.abort(new StreamError(StreamCode.Cancel, { message: "cancel" }))); - const stream = await this.#requestFill(broadcast, consumed, name, fill, sequence, cancel.signal); + const stream = await this.#requestFill(broadcast, consumed, name, framing, fill, sequence, cancel.signal); if (!stream) { cancel.abort(); return; @@ -2021,11 +2222,14 @@ export class Subscriber { // Open the FETCH for group `sequence` of `fill` once the session's grant admits it, and wait // for the publisher's answer. A refusal answers the readers waiting now, unless the fill's // budget widened since the FETCH carried it: a reader that joined meanwhile may reach what - // the narrower one could not, so it asks again. Undefined once refused, or once nobody waits. + // the narrower one could not, so it asks again. An OLD one judged the budget the FETCH + // carried, which binds only readers no more tolerant, unless it reached the publisher's max + // age, as the Rust `Inner::reject` judges. Undefined once refused, or once nobody waits. async #requestFill( broadcast: Path.Valid, consumed: Consumed, name: string, + framing: Framing, fill: track.Fill, sequence: bigint, cancel: AbortSignal, @@ -2075,7 +2279,7 @@ export class Subscriber { if (cancel.aborted || !fill.isWanted(sequence)) return undefined; if (maxDelay !== undefined && (fill.maxDelay?.asMillis() ?? Number.POSITIVE_INFINITY) > maxDelay) continue; - fill.refuseGroups(sequence, sequence + 1n, error(err)); + rejectGroups(fill, sequence, sequence + 1n, error(err), this.#stale(framing, maxDelay)); return undefined; } } @@ -2192,6 +2396,7 @@ export class Subscriber { // publisher re-offers it or drops it again under the new number, so it is still owed, // and a reader that wants it fills it meanwhile. if (stale && update < entry.update) continue; + this.#reoffered(entry.track, start, end); entry.tail.account(start, end); // Judged under the budget this subscriber sent with that update. const hole = stale ? staleHole(entry.maxDelay ?? 0) : { error }; @@ -2210,6 +2415,7 @@ export class Subscriber { // replaces any drop of the same groups, stale or final, so no update reopens them; a // reader a drop already ended stays ended. const { start, end } = resp.skip; + this.#reoffered(entry.track, start, end); entry.tail.account(start, end); entry.stale = removeRun(entry.stale ?? [], start, end); // A group that arrived was produced after all, so the skip contradicts it: it keeps @@ -2400,6 +2606,14 @@ export class Subscriber { if (start < end) entry.tail.unaccount(start, end); } } + // Re-offered whether or not the copy still holds the stale verdict. + if (entry.stale?.length) { + const reoffers = this.#reoffers.get(track) ?? new Signal([]); + this.#reoffers.set(track, reoffers); + const runs = [...reoffers.peek()]; + for (const run of entry.stale) insertRun(runs, run.start, run.end); + reoffers.set(runs); + } entry.stale = undefined; // Encoded apart from the write, so a bound the wire can't carry still fails the track. const encoded = await Writer.collect(this.version, (w) => update.encode(w, this.version)); @@ -2492,6 +2706,7 @@ export class Subscriber { const { track, timescale, tail } = entry; let producer: netGroup.Producer | undefined; const read = tail.open(group.sequence); + this.#reoffered(track, group.sequence, group.sequence + 1n); try { // The publisher contradicted its own end, which no later group can repair. lite-05 @@ -2515,10 +2730,8 @@ export class Subscriber { } producer = hooks.receiveGroup(track, { sequence: group.sequence, start: group.frameStart }); - await readFrames(stream, producer, { - scale: timescale, - compress: entry.compress, - }); + const reading = readFrames(stream, producer, { scale: timescale, compress: entry.compress }); + await (entry.filling ? untilAbandoned(producer, reading) : reading); producer.finish(); // Only a group that arrived whole, not one a drop or the track ended meanwhile, and only diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 960c47ff77e..4af788ed8cd 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -2148,7 +2148,7 @@ export class Front { } async fetchGroup(name: string, sequence: bigint, options?: track.FetchGroupOptions): Promise { - const fetched = await this.#fetch(name, sequence, options); + const fetched = await this.#fetch(name, sequence, options, undefined, undefined, true); let current = fetched.source; // Too old and evicted judge a subscription's delivery, not whether the group exists, so // neither is a verdict on a fetch: one answered from a group still arriving over a @@ -2195,13 +2195,16 @@ export class Front { // Fetch a group from whichever route serves the track, asking the next one when a route // fails, and never `avoid`. The source comes back retained for as long as the group is read. - // Once no route will come, it ends with `ended`, or as not found. + // Once no route will come, it ends with `ended`, or as not found. With `judgesDrops`, the + // first fetch, a sequence the route's copy gave up for good is refused with that drop rather + // than asked again; a continuation goes past a drop, as a resumed group does. async #fetch( name: string, sequence: bigint, options: track.FetchGroupOptions | undefined, avoid?: Source, ended?: Error, + judgesDrops = false, ): Promise<{ group: GroupConsumer; source: Source }> { for (;;) { const source = this.#source; @@ -2215,6 +2218,8 @@ export class Front { const retired = this.#retire.signal; const signal = options?.signal ? AbortSignal.any([options.signal, retired]) : retired; try { + const refused = judgesDrops ? source.wire.givenUp(name, sequence) : undefined; + if (refused) throw refused; // Asked alongside the check that the source claims the track's recorded properties, // rather than a round trip after it: only handing the group out waits for the check. const fetchGroup = source.wire.fetchGroup; diff --git a/js/net/src/track.ts b/js/net/src/track.ts index ad506b895d1..0e3d93cc081 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -31,6 +31,7 @@ import { type Hole, hooks, insertDrop, + insertDrops, insertRun, type Land, maxDelayOf, @@ -44,6 +45,7 @@ import { sameHole, skipHole, staleHole, + subtractRuns, type TrackRequestOptions, } from "./internal.ts"; import { RUN_OVERHEAD as ARRIVAL_OVERHEAD, grace, Ledger } from "./ledger.ts"; @@ -902,11 +904,9 @@ function givenUp( ): DropRun[] { const found = drops.overlapping(start, end); if (refused.length === 0) return found; - let own = refused.filter((run) => run.start < end && start < run.end); - for (const run of found) { - if (settles({ ...run, start: max(run.start, start) })) own = removeRun(own, run.start, run.end); - } - found.push(...own); + const own = refused.filter((run) => run.start < end && start < run.end); + const hiding = found.filter((run) => settles({ ...run, start: max(run.start, start) })); + found.push(...subtractRuns(own, hiding)); return found.sort((a, b) => (a.start < b.start ? -1 : a.start > b.start ? 1 : 0)); } @@ -1381,6 +1381,11 @@ export class Producer { for (const state of [producer.#state, ...producer.#sinks]) state.startChanged.update((rev) => rev + 1); }; hooks.cachedLargest = (producer) => producer.#cachedLargest(); + hooks.givenUpGroup = (producer, sequence) => { + if (cachedHolds(producer, sequence)) return undefined; + const drop = producer.#state.drops.at(sequence); + return drop && (drop.skip || drop.stale === undefined) ? drop.error : undefined; + }; hooks.reportedLargest = (producer, largest) => producer.#advanceObject(largest); hooks.reportedGroup = (producer, group) => producer.#advanceLargest({ group, frame: 0 }); objectLargest = (producer) => producer.#objectLargest; @@ -3074,8 +3079,9 @@ export class Subscriber { // Bill this reader's record, and pay what it can of the debt its track left its readers by // folding what it settled for good into its floor. Returns TooFarBehind once that needed a - // run it was promised and its record is one to end. Mirrors the Rust `Record::settle`. - #settleRecord(): TooFarBehind | undefined { + // run it was promised and its record is one to end; `handed` stays above the fold (see + // #fold). Mirrors the Rust `Record::settle`. + #settleRecord(handed?: bigint): TooFarBehind | undefined { if (this.#lagged) return new TooFarBehind(); const bytes = this.#recordRuns() * cache.READER_RUN_OVERHEAD; if (!this.#bill) { @@ -3086,7 +3092,7 @@ export class Subscriber { const bill = this.#bill; bill.set(bytes); if (bill.debt === 0) return undefined; - this.#fold(); + this.#fold(handed); const folded = this.#recordRuns() * cache.READER_RUN_OVERHEAD; const freed = Math.max(0, bill.size - folded); bill.set(folded); @@ -3110,8 +3116,9 @@ export class Subscriber { // groups. Not final are the reports an update reopens, the runs given up and not reported // yet, and the sequences nothing settled. The runs copied from the track's drop set go // first: the next scan folds in again whatever the track still holds. Mirrors the Rust - // `Record::fold`. - #fold(): void { + // `Record::fold`. It stops below `handed`, a group this read hands out that its publisher has + // yet to tell its wire of. + #fold(handed?: bigint): void { if (this.#pending.some((run) => run.received)) { this.#pending = this.#pending.filter((run) => !run.received); this.#dropsSeen = undefined; @@ -3134,6 +3141,7 @@ export class Subscriber { for (const run of this.#pending) if (run.start > floor && run.start < end) end = run.start; next = end; } + if (handed !== undefined && floor <= handed && handed < next) next = handed; if (next <= floor) break; floor = next; } @@ -3147,8 +3155,9 @@ export class Subscriber { } // Owe again every run this cursor gave up as stale, reported or not: an update reopens the - // verdict. A group still cached is offered again as a widened skip is, and whatever is still - // stale under the new terms is given up and reported again. Returns the runs taken back. A + // verdict. A group still cached is offered again as a widened skip is, a missing one is missed + // again (asked of a fill, or dropped as not found when none can serve it), and whatever is + // still stale under the new terms is given up and reported again. Returns the runs taken back. A // resumed reader takes back what it queued and what its serving route's copy reported. #reofferStale(): Run[] { if (this.#resume) return this.#resume.reofferStale(); @@ -3168,11 +3177,18 @@ export class Subscriber { this.#pending = this.#pending.filter((run) => run.stale === undefined); // A received stale run is folded in again by the next scan, if the track still holds it. this.#dropsSeen = undefined; - const sequences = [...this.#skipped.keys()].filter((sequence) => - taken.some((run) => run.start <= sequence && sequence < run.end), - ); + const producer = this.#state.producer; + const sequences: bigint[] = []; + for (const sequence of this.#skipped.keys()) { + if (!taken.some((run) => run.start <= sequence && sequence < run.end)) continue; + // A skip whose group left the cache can never be offered again: forgotten, so the + // run is judged again below, as the Rust `prune_skipped` forgets it first. + if (producer && !cachedHolds(producer, sequence)) this.#skipped.delete(sequence); + else sequences.push(sequence); + } // What no skip offers again, such as a group written off at the track's max age bound, - // which no budget reverses, is still stale under the new terms: reported again. What an + // which no budget reverses, is still stale under the new terms: reported again, as is a + // missing group they still convict. What an // upstream's stale run decided is the upstream's to judge again: the scan above folds it // in again if the track still holds it, and it stays owed if new terms superseded it. // Mirrors Rust. @@ -3180,14 +3196,57 @@ export class Subscriber { for (const sequence of sequences) kept = removeRun(kept, sequence, sequence + 1n); for (const decided of received) kept = removeRun(kept, decided.start, decided.end); if (kept.length > 0) { - const hole = staleHole(this.#drift().budget); - for (const run of kept) this.#pending = insertDrop(this.#pending, run.start, run.end, hole); + const drift = this.#drift(); + const stale = staleHole(drift.budget); + for (const run of kept) { + for (const part of this.#missed(run)) { + // A miss the new terms still convict is stale again; one a fill can still bring + // stays owed, asked afresh; the rest no copy left here can serve. + if (!part.missed || this.#staleAt(part.end - 1n, drift)) { + this.#pending = insertDrop(this.#pending, part.start, part.end, stale); + } else if (!this.#retry(part)) { + const hole = { error: new StreamError(StreamCode.NotFound, { message: "group not found" }) }; + this.#pending = insertDrop(this.#pending, part.start, part.end, hole); + } + } + } } - const producer = this.#state.producer; if (sequences.length > 0 && producer && !this.#left) reofferSequences(producer, this.#state, sequences); return taken; } + // `run` split into its cached groups, which no skip offers again and so stay stale, and the + // gaps between them, which the cache no longer holds: misses, judged as the Rust `Misses` + // judges a gap, by its newest sequence. + #missed(run: Run): (Run & { missed: boolean })[] { + const producer = this.#state.producer; + if (!producer) return [{ ...run, missed: false }]; + const parts: (Run & { missed: boolean })[] = []; + let next = run.start; + // A failed copy holds nothing to read, so it is missed like an evicted one. + const held = cachedIn(producer, run.start, run.end).filter((sequence) => cachedHolds(producer, sequence)); + for (const sequence of held.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))) { + if (next < sequence) parts.push({ start: next, end: sequence, missed: true }); + parts.push({ start: sequence, end: sequence + 1n, missed: false }); + next = sequence + 1n; + } + if (next < run.end) parts.push({ start: next, end: run.end, missed: true }); + return parts; + } + + // Ask a fill again for `run`, which an update reopened: no longer judged, so the next read + // asks it or gives it up afresh, as the Rust `Misses::retry`. False when no fill can answer. + #retry(run: Run): boolean { + const producer = this.#state.producer; + if (!producer || this.#left) return false; + const fills = fillsOf(producer); + if (!fills.serving && !fills.active) return false; + this.#givenUp = removeRun(this.#givenUp, run.start, run.end); + this.#asked = removeRun(this.#asked, run.start, run.end); + this.#refilled = removeRun(this.#refilled, run.start, run.end); + return true; + } + // Fresh reads of the cached groups in `[start, end)` this cursor admits and no longer judges // stale, each expiring against the budget, and the runs still given up. A resumed reader asks // its serving route's copy. Mirrors the Rust `reoffer_groups`. @@ -3359,8 +3418,8 @@ export class Subscriber { priority: () => this.#state.update.peek()?.priority ?? 0, maxDelay: () => this.#drift().budget, order: () => this.#state.update.peek()?.order ?? "descending", - refuse: (start, end, error) => - this.#state.refused.update((runs) => insertDrop(runs, start, end, { error })), + refuse: (start, end, error, stale) => + this.#state.refused.update((runs) => insertDrop(runs, start, end, { error, stale })), }; return this.#state.asker; } @@ -3524,9 +3583,19 @@ export class Subscriber { for (const gap of runGaps(this.#asked, start, end)) { for (const run of fills.ask(gap.start, gap.end, asker)) insertRun(this.#asked, run.start, run.end); } + // Refilled and missed again, unless a fill still brings it: given up and named, never + // asked a third time, as the Rust `Misses` gives up a refill evicted again. + const lost = intersect(this.#refilled, start, end).flatMap((run) => + runGaps(fills.covered(run.start, run.end, asker), run.start, run.end), + ); for (const refill of again) { for (const run of fills.ask(refill.start, refill.end, asker)) insertRun(this.#refilled, run.start, run.end); } + for (const run of lost) { + insertRun(this.#givenUp, run.start, run.end); + asker.refuse(run.start, run.end, new StreamError(StreamCode.NotFound, { message: "group evicted again" })); + } + if (lost.length > 0) this.#fillsMoved(); } // Whether a held group waits on the local cap rising: one past it may still be read. A @@ -4143,8 +4212,10 @@ export class Subscriber { // control state, popping the group, and positioning its frames are one JavaScript turn. #tryRecvGroup(dispositions = false): Recv { const recv = this.#tryRecvUnsettled(dispositions); - // What the read settled is billed now, and pays what it can of its readers' debt. - const lagged = this.#settleRecord(); + // What the read settled is billed now, and pays what it can of its readers' debt. The group + // it hands out stays above the fold until the next read, so the publisher tells its wire of + // it first rather than taking it for one already served. + const lagged = this.#settleRecord(recv.kind === "group" ? recv.group.sequence : undefined); return lagged ? { kind: "error", error: lagged } : recv; } @@ -4316,14 +4387,17 @@ export class Subscriber { } // The reader's own refusals too: clearing the span above took out those it covered. const folded = givenUp(drops, refused.peek(), from, to, (run) => settles(run, budget, coverage, judged)); + const received: DropRun[] = []; for (const dropped of folded) { // Judged from where it meets this cursor's span, as Rust clips it. const run = { ...dropped, start: max(dropped.start, from), end: min(dropped.end, to) }; if (run.start >= run.end) continue; // A received stale run that does not settle this reader is unknown to it. if (!settles(run, budget, coverage, judged)) continue; - this.#pending = insertDrop(this.#pending, run.start, run.end, { ...run, received: true }); + received.push({ ...run, received: true }); } + // Sorted and disjoint: the settling drops and the refusals they leave never overlap. + this.#pending = insertDrops(this.#pending, received); } for (let run = this.#pending.shift(); run; run = this.#pending.shift()) { // A received run the track has since forgotten, shed or superseded, is unknown again: diff --git a/js/net/src/wire.ts b/js/net/src/wire.ts index 52c09ad2d29..aea61922ff2 100644 --- a/js/net/src/wire.ts +++ b/js/net/src/wire.ts @@ -31,6 +31,12 @@ export interface Broadcast { resolveTrackInfo(name: string, hold?: AbortSignal): Promise; /** The newest frame the track's cache holds, read without subscribing; `undefined` while it holds none. */ largest(name: string): track.Position | undefined; + /** + * The error the track's cache gave `sequence` up for good with, read without subscribing, so + * a request served for a peer asks nothing upstream for it; `undefined` while a fill may still + * bring it. + */ + givenUp(name: string, sequence: bigint): Error | undefined; /** Fetch one group; absent when the session's version has no FETCH, so callers never ask. */ fetchGroup?(name: string, sequence: bigint, options?: track.FetchGroupOptions): Promise; requested(): Promise; diff --git a/rs/moq-net/src/model/record.rs b/rs/moq-net/src/model/record.rs index a7dc664819f..77ddf3c07fa 100644 --- a/rs/moq-net/src/model/record.rs +++ b/rs/moq-net/src/model/record.rs @@ -316,6 +316,7 @@ impl Record { &mut self, account: impl FnOnce() -> Arc, beside: &mut dyn Beside, + handed: Option, ) -> Result<(), Error> { if self.failed { return Err(Error::Lagged); @@ -330,7 +331,7 @@ impl Record { if bill.debt() == 0 { return Ok(()); } - self.fold(); + self.fold(handed); beside.forget_below(self.floor); let bytes = (self.runs() + beside.runs()) * RUN_OVERHEAD; let bill = self.bill.as_mut().expect("opened above"); @@ -349,7 +350,7 @@ impl Record { /// reader's own stale skips, which a wider budget would offer again only while the pool /// still held their groups. Not final are the reports an update reopens, the runs given /// up and not reported yet, and the sequences nothing settled. - fn fold(&mut self) { + fn fold(&mut self, handed: Option) { // The runs copied from the track's drop set go first: the next scan folds in again // whatever the track still holds. let received: Vec<_> = self @@ -390,6 +391,12 @@ impl Record { .fold(end, |end, start| end.min(start.max(floor))) } }; + // The group this read hands out stays above: its publisher has yet to tell its wire + // of it, and would take one below the floor as told already. + let next = match handed { + Some(handed) if floor <= handed && handed < next => handed, + _ => next, + }; if next <= floor { break; } @@ -443,12 +450,12 @@ mod test { for sequence in 0..4 { record.delivered.insert(sequence); } - record.settle(|| track.clone(), &mut ()).unwrap(); + record.settle(|| track.clone(), &mut (), None).unwrap(); assert_eq!(pool.used(), 0, "a lone prefix bills nothing"); for sequence in (10..1_000).step_by(2) { record.reported.insert(sequence..sequence + 1, Report::Dropped); } - record.settle(|| track.clone(), &mut ()).unwrap(); + record.settle(|| track.clone(), &mut (), None).unwrap(); assert_eq!(pool.used(), 495 * RUN_OVERHEAD); drop(record); assert_eq!(pool.used(), 0); @@ -465,10 +472,10 @@ mod test { record.reported.insert(1..u64::MAX / 2, Report::Dropped); record.delivered.insert(u64::MAX / 2); record.reported.insert(u64::MAX / 2 + 1..u64::MAX / 2 + 2, Report::Skip); - record.settle(|| track.clone(), &mut ()).unwrap(); + record.settle(|| track.clone(), &mut (), None).unwrap(); assert_eq!(pool.used(), 3 * RUN_OVERHEAD); owe(&track, pool.used()); - record.settle(|| track.clone(), &mut ()).unwrap(); + record.settle(|| track.clone(), &mut (), None).unwrap(); assert_eq!(pool.used(), 0, "all of it folded"); assert!( record.told(7) && record.delivered.contains(7), @@ -480,6 +487,23 @@ mod test { assert_eq!(record.untold(0..u64::MAX / 2 + 3), untold); } + /// The group a read hands out stays above the fold: its publisher has yet to tell its wire + /// of it, and takes a sequence below the floor as told already. + #[test] + fn a_debt_folds_short_of_the_handed_group() { + let (pool, track) = account(); + let mut record = Record::default(); + record.delivered.insert(0); + record.reported.insert(1..3, Report::Dropped); + record.delivered.insert(3); + record.reported.insert(5..6, Report::Skip); + record.settle(|| track.clone(), &mut (), None).unwrap(); + owe(&track, pool.used()); + record.settle(|| track.clone(), &mut (), Some(3)).unwrap(); + assert!(record.told(2), "the drops below it fold"); + assert!(!record.told(3), "the handed group stays to be told"); + } + /// A debt that needs a run this reader was promised ends it with `TooFarBehind`: a stale /// report an update reopens, or a run above one. What is final below it still folds. #[test] @@ -505,14 +529,14 @@ mod test { for sequence in (5..1_000).step_by(2) { record.reported.insert(sequence..sequence + 1, Report::Dropped); } - record.settle(|| track.clone(), &mut ()).unwrap(); + record.settle(|| track.clone(), &mut (), None).unwrap(); owe(&track, pool.used()); - let err = record.settle(|| track.clone(), &mut ()).unwrap_err(); + let err = record.settle(|| track.clone(), &mut (), None).unwrap_err(); assert_eq!(StreamError::from(&err), StreamError::TooFarBehind, "{promise:?}"); assert_eq!(pool.used(), 0, "{promise:?}: its record went with it"); assert!(record.failed()); assert!( - record.settle(|| track.clone(), &mut ()).is_err(), + record.settle(|| track.clone(), &mut (), None).is_err(), "{promise:?}: it stays ended" ); } @@ -530,9 +554,9 @@ mod test { .pending .insert(2..3, Pending::own(Hole::drop(StreamError::NotFound))); record.skipped.insert(4, Duration::from_secs(1)); - record.settle(|| track.clone(), &mut ()).unwrap(); + record.settle(|| track.clone(), &mut (), None).unwrap(); owe(&track, RUN_OVERHEAD); - record.settle(|| track.clone(), &mut ()).unwrap(); + record.settle(|| track.clone(), &mut (), None).unwrap(); assert_eq!(pool.used(), 2 * RUN_OVERHEAD, "the drop folded"); assert!(!record.told(2), "the run given up is still owed"); } @@ -553,12 +577,12 @@ mod test { for sequence in (1..1_000).step_by(2) { large.reported.insert(sequence..sequence + 1, stale); } - small.settle(|| track.clone(), &mut ()).unwrap(); - large.settle(|| track.clone(), &mut ()).unwrap(); + small.settle(|| track.clone(), &mut (), None).unwrap(); + large.settle(|| track.clone(), &mut (), None).unwrap(); owe(&track, RUN_OVERHEAD); - small.settle(|| track.clone(), &mut ()).unwrap(); - assert!(large.settle(|| track.clone(), &mut ()).is_err()); + small.settle(|| track.clone(), &mut (), None).unwrap(); + assert!(large.settle(|| track.clone(), &mut (), None).is_err()); assert_eq!(pool.used(), RUN_OVERHEAD); - small.settle(|| track.clone(), &mut ()).unwrap(); + small.settle(|| track.clone(), &mut (), None).unwrap(); } } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 4ab11ad9767..f71f382cb1a 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -2204,8 +2204,14 @@ impl TrackState { /// Resolve a one-shot fetch from the track side: the cached group, or an [`Error`] /// once it can never be served. A missing group is a failure ([`Error::NotFound`]), not an /// end-of-stream. The handler side (a refusal, or no [`Requests`] at all) lives - /// in the track's fills; [`Fetching`] polls both. - fn poll_fetch_cached(&self, sequence: u64, frames: group::Bounds) -> Poll> { + /// in the track's fills; [`Fetching`] polls both. `judges_drops` answers a sequence given + /// up for good from the drop set, as this track's readers are answered. + fn poll_fetch_cached( + &self, + sequence: u64, + frames: group::Bounds, + judges_drops: bool, + ) -> Poll> { // A group aborted between the lookup and the consume (an abandoned fetch cut // short) is a miss, not a hit that fails on its first read. if let Some(group) = self.covering_range(sequence, frames) @@ -2227,6 +2233,16 @@ impl TrackState { return Poll::Ready(Err(Error::NotFound)); } + // Given up for good: skipped, or dropped under no budget, which no reader of this + // track fills, so a request asks nothing upstream for it either. A stale drop stays + // fillable by a wider budget. + if judges_drops + && let Some(hole) = self.drops.get(&sequence) + && hole.stale_budget().is_none() + { + return Poll::Ready(Err(Error::Stream(hole.error()))); + } + Poll::Pending } @@ -6156,10 +6172,13 @@ impl Consumer { options: options.into().unwrap_or_default(), end: None, }, + false, ) } - /// Fill a wire request, widening a cold or compressed group to a reusable prefix. + /// Fill a wire request, widening a cold or compressed group to a reusable prefix. It asks + /// on a reader's behalf, so a sequence the serving copy gave up for good is answered from + /// that drop rather than asked upstream again, as the copy's own readers are. pub(crate) fn fetch_range(&self, sequence: u64, mut request: FetchOptions) -> kio::Pending { let state = self.resolve(None).0; { @@ -6175,10 +6194,10 @@ impl Consumer { request.end = None; } } - self.fetching(sequence, request) + self.fetching(sequence, request, true) } - fn fetching(&self, sequence: u64, request: FetchOptions) -> kio::Pending { + fn fetching(&self, sequence: u64, request: FetchOptions, judges_drops: bool) -> kio::Pending { let FetchOptions { options, end: frame_end, @@ -6202,6 +6221,7 @@ impl Consumer { priority: options.priority, max_delay: options.max_delay, judges_drift: true, + judges_drops, result: None, hit: Some(Box::new(group)), stats: self.stats.clone(), @@ -6223,6 +6243,7 @@ impl Consumer { priority: options.priority, max_delay: options.max_delay, judges_drift: true, + judges_drops, result: None, hit: None, stats: self.stats.clone(), @@ -6245,6 +6266,7 @@ impl Consumer { end: frame_end, }, self.stats.clone(), + judges_drops, )) } @@ -6583,6 +6605,10 @@ pub struct Fetching { // Whether this fetch refuses a group a newer cached one superseded, judged across the // whole cache. A cursor's head fill leaves it to the cursor, which judges its window. judges_drift: bool, + // Whether this fetch, a request on a reader's behalf, refuses a sequence the track gave up + // for good. A group already partly read is continued past a drop, which judged only the + // subscription. + judges_drops: bool, // The fill this fetch joined; `None` on a cache hit, or when no handler existed to // queue on. Boxed: a miss already allocates, and the handle stays small. result: Option>, @@ -6605,6 +6631,7 @@ impl Fetching { sequence: u64, request: FetchOptions, stats: stats::Scope, + judges_drops: bool, ) -> Self { let frames = group::Bounds { start: request.options.frame_start, @@ -6612,7 +6639,10 @@ impl Fetching { }; let (fills, cached) = { let state = state.read(); - (state.fills.clone(), state.poll_fetch_cached(sequence, frames)) + ( + state.fills.clone(), + state.poll_fetch_cached(sequence, frames, judges_drops), + ) }; let result = match cached.is_pending() { true => queue(&fills, sequence, &request), @@ -6633,6 +6663,7 @@ impl Fetching { priority: request.options.priority, max_delay: request.options.max_delay, judges_drift: true, + judges_drops, result, hit, stats, @@ -6734,7 +6765,9 @@ impl Fetching { start: frame_start, end: self.frame_end, }; - let cached = |waiter: &kio::Waiter| match state.poll(waiter, |state| state.poll_fetch_cached(sequence, frames)) + let judges_drops = self.judges_drops; + let cached = |waiter: &kio::Waiter| match state + .poll(waiter, |state| state.poll_fetch_cached(sequence, frames, judges_drops)) { Poll::Ready(Ok(res)) => Poll::Ready(res.map(resolve)), Poll::Ready(Err(closed)) => Poll::Ready(Err(closed.outcome.aborted().cloned().unwrap_or(Error::Dropped))), @@ -9024,7 +9057,8 @@ impl Cursor { }; // The cursor judges the answer against its window when it hands it out. let mut fetching = - Fetching::cached_or_queued(self.state.clone(), sequence, request, stats::Scope::default()).unjudged(); + Fetching::cached_or_queued(self.state.clone(), sequence, request, stats::Scope::default(), false) + .unjudged(); let heads = self.heads.get_or_insert_default(); match kio::Task::poll(&mut fetching, waiter) { Poll::Pending => { @@ -9552,8 +9586,13 @@ impl Subscriber { /// already handed out. fn poll_group(&mut self, ordered: bool, waiter: &kio::Waiter) -> Poll>> { let res = self.poll_group_unsettled(ordered, waiter); - // What the read settled is billed now, and pays what it can of its readers' debt. - match self.settle_record() { + // What the read settled is billed now, and pays what it can of its readers' debt. The + // group it hands out stays above the fold until its reader accounts for it. + let handed = match &res { + Poll::Ready(Ok(Some(group))) => Some(group.sequence), + _ => None, + }; + match self.settle_record(handed) { Ok(()) => res, Err(err) => Poll::Ready(Err(err)), } @@ -9749,18 +9788,18 @@ impl Subscriber { /// Bill what this reader settled, and pay what it can of the debt its track left its /// readers; [`Error::Lagged`] once that needed a run it was promised. - fn settle_record(&mut self) -> Result<()> { + fn settle_record(&mut self, handed: Option) -> Result<()> { match &mut self.inner { Inner::Plain(cursor) => { let state = &cursor.state; match cursor.misses.as_deref_mut() { - Some(misses) => cursor.record.settle(|| state.read().cache.clone(), misses), - None => cursor.record.settle(|| state.read().cache.clone(), &mut ()), + Some(misses) => cursor.record.settle(|| state.read().cache.clone(), misses, handed), + None => cursor.record.settle(|| state.read().cache.clone(), &mut (), handed), } } Inner::Resume(_, logical) => { let state = &logical.state; - logical.record.settle(|| state.read().cache.clone(), &mut ()) + logical.record.settle(|| state.read().cache.clone(), &mut (), handed) } } } @@ -10047,7 +10086,7 @@ impl Subscriber { fn poll_drop_probing(&mut self, probe: bool, waiter: &kio::Waiter) -> Poll<(ops::Range, Hole)> { // What a publisher told its wire since is billed now. A reader the pool ended reports // nothing more: its reads fail with the reason. - if self.settle_record().is_err() { + if self.settle_record(None).is_err() { return Poll::Pending; } match &mut self.inner { @@ -10216,7 +10255,7 @@ impl Subscriber { Inner::Resume(resume, _) => resume.mirror().sync(), } // A reader the pool ended reports it on its next read. - let _ = self.settle_record(); + let _ = self.settle_record(None); Ok(()) } diff --git a/rs/moq-net/tests/integration/drop_record.rs b/rs/moq-net/tests/integration/drop_record.rs index cc8b3d54e21..40a42ce31c9 100644 --- a/rs/moq-net/tests/integration/drop_record.rs +++ b/rs/moq-net/tests/integration/drop_record.rs @@ -319,7 +319,6 @@ async fn a_lite06_hop_keeps_a_reported_group_dropped() { /// it and fetches the group, and the relay fetches it from the publisher instead of answering /// from the drop it holds. #[kio::test] -#[ignore = "a relay fetches upstream a group its copy holds as a final lite-06 drop"] async fn a_lite06_upstream_keeps_a_reported_group_dropped_for_a_lite07_viewer() { reported_drop_through([LITE06, LITE07]).await; } From d300ccdff0ddd59e2fabe54982d2491f34b111d2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 21:29:57 -0800 Subject: [PATCH 12/37] quest: delete the drop set quest, complete Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 11 +- quest/m1/copy-recover.md | 6 +- quest/m1/drop-runs-prune.md | 6 +- quest/m1/drop-set.md | 119 ------------------ quest/m1/explicit-state/README.md | 2 +- quest/m1/group-error-cache.md | 5 +- quest/m1/stream-credit/takeover.md | 6 +- quest/m2/README.md | 1 - quest/m2/group-drops.md | 5 +- quest/m2/recut-held.md | 4 +- quest/m2/subscribe-ranges/js-fills.md | 3 +- quest/m2/subscribe-ranges/lite07-fetch.md | 6 +- .../m2/subscribe-ranges/start-bookkeeping.md | 8 +- quest/m2/time-addressing/model.md | 3 +- quest/m3/lite-tail-drop-count.md | 4 - 15 files changed, 19 insertions(+), 170 deletions(-) delete mode 100644 quest/m1/drop-set.md diff --git a/quest/m1/README.md b/quest/m1/README.md index 8671bdee8e0..b8079e6945e 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -23,15 +23,14 @@ additive. video-vaapi moved up from later work because a downstream iGPU transcode trial needs it now. Held here by open claims: a quest's branch is its path, so a claimed quest -cannot move until its PR merges, and an epic moves with its children. These -breaking epics and quests stay in m1 for now and are listed in m2's -`Required`, so the cut still waits on them: drop-set and -stream-credit. The [re-cut follow-up](/quest/m2/recut-held.md) -moves what remains of them once their claims merge. +cannot move until its PR merges, and an epic moves with its children. This +breaking epic stays in m1 for now and is listed in m2's +`Required`, so the cut still waits on it: stream-credit. The +[re-cut follow-up](/quest/m2/recut-held.md) moves what remains of it once +its claims merge. ## Required -- [Drop set](/quest/m1/drop-set.md) - each cursor's disposition record is billed to the cache pool, as the track's drop runs are, so fragmenting drops costs what the pool holds; eviction folds final runs or ends the cursor with `TooFarBehind` - [Routing guide](/quest/m1/routing-guide.md) - concepts and practical outbound federation setup - [A reader on a failed copy continues on the held re-offer](/quest/m1/copy-recover.md) - a live reader keeps a group whose copy fails by continuing on the same track's re-offer at the same frame - [Group errors are cached by kind](/quest/m1/group-error-cache.md) - permanent group errors bind every reader, retryable ones are kept only for a bounded time, and a live SUBSCRIBE_DROP or FETCH answered `Old` no longer caches it for good diff --git a/quest/m1/copy-recover.md b/quest/m1/copy-recover.md index a67c5c1079e..aeabf7e4f7c 100644 --- a/quest/m1/copy-recover.md +++ b/quest/m1/copy-recover.md @@ -20,11 +20,7 @@ copy that fails continues on the re-offer with no duplicate or missing frame. Decided 2026-11-05 (maintainer, /quest-audit): ✅ the wait on [moq-dev/moq#7670](https://github.com/moq-dev/moq/pull/7670), Drop set's own -PR, becomes a `Required` link to [Drop set](/quest/m1/drop-set.md), since +PR, becomes a `Required` link to Drop set ([moq-dev/moq#8476](https://github.com/moq-dev/moq/pull/8476)), since #7670 adds the held re-offer (its decision 24) this builds on. Public API: none expected. Wire: none. - -## Required - -- [Drop set](/quest/m1/drop-set.md) - adds the held re-offer this builds on diff --git a/quest/m1/drop-runs-prune.md b/quest/m1/drop-runs-prune.md index 7151a038a7f..3369a2b314a 100644 --- a/quest/m1/drop-runs-prune.md +++ b/quest/m1/drop-runs-prune.md @@ -6,8 +6,4 @@ A track's record of dropped groups stays proportional to the window readers can ## Plan -Found by the 2026-11-07 invariant audit (js-4). Runs below the cache/arrivals floor are forgotten, as `Delivered.forgetBelow` does, and JS inserts in place. This waits on [Drop set](/quest/m1/drop-set.md), which redesigns these runs with their reason and budget; prune that representation rather than today's. - -## Required - -- [Drop set](/quest/m1/drop-set.md) - drop runs carry their reason and budget, the structure this quest prunes +Found by the 2026-11-07 invariant audit (js-4). Runs below the cache/arrivals floor are forgotten, as `Delivered.forgetBelow` does, and JS inserts in place. Drop set ([moq-dev/moq#8476](https://github.com/moq-dev/moq/pull/8476)), which billed each reader's disposition record to the pool, left the track's own drop runs as they were; prune them in place. diff --git a/quest/m1/drop-set.md b/quest/m1/drop-set.md deleted file mode 100644 index 477fec3dbd6..00000000000 --- a/quest/m1/drop-set.md +++ /dev/null @@ -1,119 +0,0 @@ -# [L] Drop set - -## Goal - -A hostile upstream fragmenting drops costs what the cache's byte pool would -hold, never more: each cursor's disposition record is billed to the pool at a -fixed cost per run, in Rust and `js/net`, as the track's runs already are, and -a cursor's CPU per drop tracks the runs it touches, never their span or the -set. Eviction keeps every promise a reader was made: a final run folds into -the cursor's floor, and losing any other ends the subscription with -`TooFarBehind`. - -## Plan - -The track's drop set is billed and bounded, and every reader folds it in -incrementally ([moq-dev/moq#8294](https://github.com/moq-dev/moq/pull/8294)). -So are the track's arrivals: each gap is billed, and paying eviction debt -folds the oldest gaps early ([moq-dev/moq#8440](https://github.com/moq-dev/moq/pull/8440)). -The per-cursor storage is not billed to anything: `DropCursor`'s `pending` -and its `reported` record (one `RangeMap`), `Skipped`, -`Delivered`, `TrackRun`'s `served`, `dropped`, `dropped_stale`, and -`expired`, the lite subscriber's record of the group streams that finished -since its last update (`Tail::finished`, JS `Tail#finished`), and the JS wire -cursor's `#pending`, `#skipped`, and `#reported` (one `Runs`). -This quest bounds them, keeping the behavior -[moq-dev/moq#7670](https://github.com/moq-dev/moq/pull/7670) and -moq-dev/moq#8294 landed and the decisions recorded there. - -Decided 2026-10-27 (maintainer): this runs before -[Rust groups by start](/quest/m2/time-addressing/model.md), which requires it -because runs keyed by group sequence convert mechanically to starts. - -- **One disposition record per cursor,** a run-length map whose runs are - delivered, skipped as stale, or reported dropped or skipped, with the - reason and, for a stale run, the budget it was judged under. It replaces - `Skipped` (`rs/moq-net/src/model/skipped.rs`), `Delivered` - (`rs/moq-net/src/model/delivered.rs`, whose floor folds stale gaps in with - delivered groups), `DropCursor`'s `reported` record, whose `Report` kinds - carry over, `TrackRun::served` in `rs/moq-net/src/lite/publisher.rs`, and - their JS mirrors. Delivery is - recorded, not inferred from the cursor position, since an arrival-order or - descending cursor delivers out of sequence order. -- **Billed to the pool, never guessed.** Each cursor run is billed through a - `Charge` at a fixed per-run cost (in JS, through the pool in - `js/net/src/cache.ts`), as the track's runs are. A relay's own eviction is - never a run. A cursor pays only the debt its track could not: the track - pays with its groups, its arrivals' oldest gaps, and its drop set first, so - a relay running at capacity never ends a reader whose lowest run is an - unknown hole while cheaper bookkeeping is left to shed. -- **The floor rises only over a final prefix:** the runs from the floor up to - the lowest run that is not final. A final run is a delivered one, one - skipped at the track's max age bound, a skip, a budget-independent drop, - or any drop reported over lite-05 or lite-06. A lite-07 stale report within - reach of the max age bound, or an unknown hole, is not: when the pool must - evict it or a run above it, the cursor fails with `TooFarBehind` rather - than forgo the re-offer. A drop folded into the floor is final for that - cursor, so its group arriving late is not delivered there. -- **Per-reader misses** fold into the cursor's runs too: the given-up misses - `rs/moq-net/src/model/track/miss.rs` keeps per gap (from - [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171), decided - 2026-10-23). -- **CPU per drop.** Every report inserts into the per-cursor records, so - `track_subscriber_churn_drops` in - `rs/moq-net/benches/track.rs` slows once many readers each hold many - reported runs (256 readers holding 65,536 each: about 0.75 ms per drop - against 0.18 ms with one each). JS `#askFills` still walks every run above - the edge on each probe while a fill handler serves the track; mirror Rust's - `Misses::unknown` memo there, and have the memo judge again what the track - forgets, as Rust's `Misses::judge` folds the track's `Forgotten` log. - -Mirror each case in `js/net`. -In `js/net`, a group a sink buffers keeps its source wanted only while -`Subscriber.#reading` says its reader will still read it -([moq-dev/moq#7948](https://github.com/moq-dev/moq/pull/7948)), so a change to -how a cursor judges a missing or stale group keeps that verdict in step. - -Still to cover, beyond the memory work: - -- **Untimed fills below a resolved start.** Noted 2026-10-31 from - [moq-dev/moq#7235](https://github.com/moq-dev/moq/pull/7235): a lite-07 - copy with no media time asks a fill below the start even for a reader whose - budget the publisher already exceeded. If the extra FETCH shows up in relay - traffic, bound it with a run below the start carrying its budget. -- **A fill's stale verdict is final for everyone.** `TrackState::drop_unfed` - (`rs/moq-net/src/model/track/fill.rs`) records what a fill's source gave up - as a budget-independent drop, including OLD judged under that fill's own - terms. Record it as a stale run, so it settles only the readers it covers. - Check first that a case is left: `Inner::old_binds_everyone` already - records an OLD only when the fill asked at or past the max age bound, which - the floor rule above counts as final. - -Tests, on mock time through a relay, in Rust and mirrored in `js/net`: - -- After a widening from B1 to B2 for a group the upstream no longer holds, an - in-flight drop under the B1 update settles no reader once the B2 update is - sent, and the re-drop under the B2 update settles the B2 reader, over the - wire. -- A group a relay evicted while the upstream still holds it is never reported - dropped: a new, a widened, and an owed reader each fill it by range. -- Rust and `js/net` encode and decode an origin's eviction and retention - expiry drop (EVICTED, not OLD) across languages; a relay forwards them - unchanged, and no update re-offers either. -- Through a lite-06 hop, a reported group stays dropped for every reader, - with no session error; a shrink sends no drops. -- Under pool pressure, delivered runs and budget-independent drops fold into - the floor, while evicting a lite-07 stale report within reach, or a run - above it, ends the cursor with `TooFarBehind`. -- A sequence whose shared run was shed or forgotten is judged locally or - filled by range, through a relay (the flood bound itself is covered by - `a_drop_flood_stays_within_the_pool` and its JS mirror). -- A single run spanning billions of sequences is settled by each cursor in - one step, whether dropped, skipped by a wider reader, or evicted and judged - again. - -Public API: none. Wire: none. - -## Related - -- [Subscribe ranges](/quest/m2/subscribe-ranges/README.md) - fills the cache miss an evicted run leaves diff --git a/quest/m1/explicit-state/README.md b/quest/m1/explicit-state/README.md index c6882297806..05f188a6307 100644 --- a/quest/m1/explicit-state/README.md +++ b/quest/m1/explicit-state/README.md @@ -65,7 +65,7 @@ Out of scope: the lite `TrackRun` start shortcut that never sends its pending drops (`rs/moq-net/src/lite/publisher.rs`, the "start past everything" branch). It belongs to [Delete the old start bookkeeping](/quest/m2/subscribe-ranges/start-bookkeeping.md), -which collapses that state and waits on [Drop set](/quest/m1/drop-set.md). +which collapses that state and waits on Drop set ([moq-dev/moq#8476](https://github.com/moq-dev/moq/pull/8476)). ## Required diff --git a/quest/m1/group-error-cache.md b/quest/m1/group-error-cache.md index 691f801a927..8f9831c05c8 100644 --- a/quest/m1/group-error-cache.md +++ b/quest/m1/group-error-cache.md @@ -57,7 +57,7 @@ The work: already has over a new constant: for `Old`, the budget it was judged under (a widening already re-asks), or the publisher's max age. The bound must also hold up against a hostile peer: what the cache holds is - sized by the pool, as the [drop set](/quest/m1/drop-set.md) bills its + sized by the pool, as the drop set ([moq-dev/moq#8476](https://github.com/moq-dev/moq/pull/8476)) bills its runs. - **Fix both main paths** under the rule: a live SUBSCRIBE_DROP records only a permanent error, and `Fill::reject` stops recording `Old` as lost @@ -77,9 +77,6 @@ The work: Public API: possibly a classification query on `Error` (Rust) and its JS mirror, if a caller outside moq-net needs it; otherwise none. Wire: none. -## Required - -- [Drop set](/quest/m1/drop-set.md) - its per-cursor runs and their reasons are what this bounds; its PR, [moq-dev/moq#7670](https://github.com/moq-dev/moq/pull/7670), rewrites the same recording paths ## Related diff --git a/quest/m1/stream-credit/takeover.md b/quest/m1/stream-credit/takeover.md index 08ce2bea5a7..a3d87c95197 100644 --- a/quest/m1/stream-credit/takeover.md +++ b/quest/m1/stream-credit/takeover.md @@ -34,12 +34,8 @@ a group that lost its stream resumes without loss. Decided 2026-11-05 (maintainer, /quest-audit): ✅ the wait on [moq-dev/moq#7670](https://github.com/moq-dev/moq/pull/7670), Drop set's own -PR, becomes a `Required` link to [Drop set](/quest/m1/drop-set.md), since +PR, becomes a `Required` link to Drop set ([moq-dev/moq#8476](https://github.com/moq-dev/moq/pull/8476)), since resuming a reset group sends a second stream for the same group, which #7670's hold-the-duplicate rule rewrites. Public API: none. Wire: lite-07 group stream takeover. - -## Required - -- [Drop set](/quest/m1/drop-set.md) - its hold-the-duplicate rule governs a resumed group's second stream diff --git a/quest/m2/README.md b/quest/m2/README.md index 017490c24d6..5fdceb72fab 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -191,7 +191,6 @@ quests that require them. ## Required -- [Drop set](/quest/m1/drop-set.md) - each cursor's disposition record is billed to the cache pool, as the track's drop runs are, so fragmenting drops costs what the pool holds; eviction folds final runs or ends the cursor with `TooFarBehind` (held in m1 by an open claim) - [Subscribe ranges](/quest/m2/subscribe-ranges/README.md) - a lite-07 SUBSCRIBE asks for past and live ranges with absolute or relative ends, in either order, and replaces FETCH for multi-group reads (single-group FETCH stays); merged ranges never starve a subscriber; relays fill misses by range, including over moq-transport - [Address groups by timestamp](/quest/m2/time-addressing/README.md) - group start timestamps replace sequence numbers in the moq-net API and lite-07 wire, while old subscribers, old publishers, and published JS clients keep working - [Cluster routing](/quest/m2/cluster-routing/README.md) - prove shared routing contracts, implement per-origin routes, and connect outbound local clusters diff --git a/quest/m2/group-drops.md b/quest/m2/group-drops.md index adb977da2d0..c46847b7eeb 100644 --- a/quest/m2/group-drops.md +++ b/quest/m2/group-drops.md @@ -55,7 +55,7 @@ every run; the 256-stream cell sometimes does. for a named consumer (for example a player or container reader that skips gaps), per the API audit rule that new surface needs one; name it in the PR or leave the mirror out. -- **Order with the [drop set](/quest/m1/drop-set.md).** The drop set +- **Order with the drop set ([moq-dev/moq#8476](https://github.com/moq-dev/moq/pull/8476)).** The drop set replaces `poll_drop`'s `Delivered` argument and the skipped record, and edits the lite-07 SUBSCRIBE_DROP wording. Publish the API against its final disposition record, so the public shape doesn't change twice. @@ -64,7 +64,7 @@ every run; the 256-stream cell sometimes does. missing group. Decided 2026-11-05 (maintainer, /quest-audit): ✅ this waits for -[Drop set](/quest/m1/drop-set.md), since both rewrite the lite-07 +Drop set ([moq-dev/moq#8476](https://github.com/moq-dev/moq/pull/8476)), since both rewrite the lite-07 SUBSCRIBE_DROP draft wording and the drop signal's `Delivered` argument. Decided 2026-11-07 (maintainer, /quest-plan from the API sweep of open PRs and remaining quests): ✅ key the drop signal by group start and land after @@ -97,7 +97,6 @@ Wire: possibly lite-07 (SUBSCRIBE_DROP for a stopped group), decided here. ## Required -- [Drop set](/quest/m1/drop-set.md) - owns the per-cursor drop runs this reports - [Address groups by timestamp](/quest/m2/time-addressing/model.md) - the drop range is keyed by start ## Related diff --git a/quest/m2/recut-held.md b/quest/m2/recut-held.md index da386c2b14f..945f46ecd2c 100644 --- a/quest/m2/recut-held.md +++ b/quest/m2/recut-held.md @@ -3,8 +3,8 @@ ## Goal The epics that open claims held in place during the 2026-11-05 milestone -re-cut sit where the re-cut's rules put them. What remains of drop-set -and stream-credit moves from m1 to m2; and the +re-cut sit where the re-cut's rules put them. What remains of +stream-credit moves from m1 to m2; and the probe epic moves from m2 to m3. The "held" notes in the m1, m2, and cut gate READMEs are removed, along with each held entry's extra link from m2's or the gate's `Required`. diff --git a/quest/m2/subscribe-ranges/js-fills.md b/quest/m2/subscribe-ranges/js-fills.md index c9ec518b99e..b74bd590bb4 100644 --- a/quest/m2/subscribe-ranges/js-fills.md +++ b/quest/m2/subscribe-ranges/js-fills.md @@ -61,7 +61,7 @@ through a JS relay between Rust peers (`just test relay-fill`). Public API: `Fill` gains `time` and a frame start, matching Rust's `Fill::time` and `Fill::start`. Wire: none. -Decided 2026-11-05 (maintainer, /quest-plan): this waits for [Drop set](/quest/m1/drop-set.md), since its head-fill +Decided 2026-11-05 (maintainer, /quest-plan): this waits for Drop set ([moq-dev/moq#8476](https://github.com/moq-dev/moq/pull/8476)), since its head-fill half rewrites the same JS fill and miss code as [moq-dev/moq#7670](https://github.com/moq-dev/moq/pull/7670) and Drop set, which also redefine the stale-drop judgment it relies on. Not chosen: @@ -74,5 +74,4 @@ subscription. ## Required -- [Drop set](/quest/m1/drop-set.md) - redefines the JS stale-drop judgment and fill code this rewrites - [Lite-07 FETCH is single-group only](/quest/m2/subscribe-ranges/lite07-fetch.md) - moves JS multi-group fills onto the subscription in the same `#runFills` diff --git a/quest/m2/subscribe-ranges/lite07-fetch.md b/quest/m2/subscribe-ranges/lite07-fetch.md index b7c5ae30568..3f5e1e749ce 100644 --- a/quest/m2/subscribe-ranges/lite07-fetch.md +++ b/quest/m2/subscribe-ranges/lite07-fetch.md @@ -63,13 +63,9 @@ on lite-07 (`track.Consumer.fetchGroup` in JS). - Run `just test interop --all`. Decided 2026-11-05 (maintainer, /quest-audit): ✅ this waits for -[Drop set](/quest/m1/drop-set.md), since both rewrite the JS fill code +Drop set ([moq-dev/moq#8476](https://github.com/moq-dev/moq/pull/8476)), since both rewrite the JS fill code (`js/net/src/lite/subscriber.ts` `#runFills`) and the same lite-07 draft sections. Public API: none expected. Wire: lite-07 (unpublished) SUBSCRIBE anchors lose their frame. - -## Required - -- [Drop set](/quest/m1/drop-set.md) - rewrites the JS fill code and lite-07 draft sections this changes diff --git a/quest/m2/subscribe-ranges/start-bookkeeping.md b/quest/m2/subscribe-ranges/start-bookkeeping.md index 6591c529870..1e22b410a4e 100644 --- a/quest/m2/subscribe-ranges/start-bookkeeping.md +++ b/quest/m2/subscribe-ranges/start-bookkeeping.md @@ -8,7 +8,7 @@ the per-version start branches collapse into one start state that only translates into SUBSCRIBE_START per version, and on lite-07 the `requested == start` gate is gone, since a fresh START per update makes it unnecessary. What a cursor is owed or was served is out of scope: it stays -with [Drop set](/quest/m1/drop-set.md). +with Drop set ([moq-dev/moq#8476](https://github.com/moq-dev/moq/pull/8476)). ## Plan @@ -22,7 +22,7 @@ folding it into Drop set, or doing it now on main. Decided 2026-11-05 (maintainer, /quest-audit): ✅ the wait on [moq-dev/moq#7670](https://github.com/moq-dev/moq/pull/7670), Drop set's own -PR, becomes a `Required` link to [Drop set](/quest/m1/drop-set.md), since +PR, becomes a `Required` link to Drop set ([moq-dev/moq#8476](https://github.com/moq-dev/moq/pull/8476)), since #7670 rewrites the same lite publisher state (about 1,250 lines of `lite/publisher.rs`). @@ -60,7 +60,3 @@ three wire generations of "start". Public API: none expected. Wire: lite-07 (unpublished) may send a START per update; published versions are unchanged. - -## Required - -- [Drop set](/quest/m1/drop-set.md) - rewrites the same lite publisher state diff --git a/quest/m2/time-addressing/model.md b/quest/m2/time-addressing/model.md index e7e01574d5d..6776dcdc9f9 100644 --- a/quest/m2/time-addressing/model.md +++ b/quest/m2/time-addressing/model.md @@ -57,7 +57,7 @@ a producer would need per-track memory of its last number. wire, Lite-07 ranges ([moq-dev/moq#7872](https://github.com/moq-dev/moq/pull/7872), merged 2026-11-05) and [Delete the old start bookkeeping](/quest/m2/subscribe-ranges/start-bookkeeping.md), - not the whole epic. [Drop set](/quest/m1/drop-set.md) lands first too: its + not the whole epic. Drop set ([moq-dev/moq#8476](https://github.com/moq-dev/moq/pull/8476)) lands first too: its runs keyed by group sequence convert mechanically to starts. - **Old relays** (decided 2026-10-26, maintainer): a released relay fills a hole over lite-05/06 with one FETCH per integer ID, so sparse IDs cost it @@ -220,4 +220,3 @@ Wire: no format change; legacy and lite-07-wip group IDs become start ticks. ## Required - [Delete the old start bookkeeping](/quest/m2/subscribe-ranges/start-bookkeeping.md) - the lite cursor state lives in the model before the cut rekeys it -- [Drop set](/quest/m1/drop-set.md) - its dropped and expired runs convert mechanically from sequences to starts diff --git a/quest/m3/lite-tail-drop-count.md b/quest/m3/lite-tail-drop-count.md index 11b5a9bc934..d516641e28e 100644 --- a/quest/m3/lite-tail-drop-count.md +++ b/quest/m3/lite-tail-drop-count.md @@ -14,7 +14,3 @@ Decided 2026-10-30 (maintainer, follow-up planning): m2. Found by the audit on [ - The IETF tail keeps its grace until the moq-transport gap-signal extension (planned in [moq-dev/moq#7425](https://github.com/moq-dev/moq/pull/7425)) lands, since moq-transport can't signal gaps. Public API: none. Wire: none. - -## Related - -- [Drop set](/quest/m1/drop-set.md) - the per-cursor drop reporting this counts From 79363025ff07ca01aea96bd6b7bbccb9053e2f4b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 22:42:36 -0800 Subject: [PATCH 13/37] fix(net): a lite-06 drop stays fetchable by a wider budget Reverts the FETCH refusal for a relay's budget-independent drops, in Rust and js/net: a lite-06 OLD drop is final for the subscription that heard it, but was judged under that subscription's budget, and the relay drills' FETCH recovery requires the publisher to serve the group to a wider reader. The relay-level test now asserts that a widened lite-07 viewer behind a lite-06 hop recovers the group. Also makes the JS relay eviction test's setup tolerant of which group a narrow reader starts at. Co-Authored-By: Claude Opus 5.5 --- js/net/src/broadcast.ts | 8 ---- js/net/src/internal.ts | 10 ----- js/net/src/lite/drop_record.test.ts | 16 +++---- js/net/src/lite/publisher.ts | 9 ++-- js/net/src/resume.ts | 9 +--- js/net/src/track.ts | 5 --- js/net/src/wire.ts | 6 --- rs/moq-net/src/model/track.rs | 46 ++++----------------- rs/moq-net/tests/integration/drop_record.rs | 14 ++++--- 9 files changed, 29 insertions(+), 94 deletions(-) diff --git a/js/net/src/broadcast.ts b/js/net/src/broadcast.ts index e75f54823ce..b89179eb34e 100644 --- a/js/net/src/broadcast.ts +++ b/js/net/src/broadcast.ts @@ -288,12 +288,6 @@ function largest(state: BroadcastState, name: string): track.Position | undefine return producer && hooks.cachedLargest(producer); } -// The error `name`'s cache gave `sequence` up for good with, without subscribing. -function givenUp(state: BroadcastState, name: string, sequence: bigint): Error | undefined { - const producer = lookup(state, name); - return producer && hooks.givenUpGroup(producer, sequence); -} - // Fetch one group as a one-range subscription: the cached group at once, or else one the track's // {@link track.Requests} handler answers, asked at once whether the group was written yet or not, // as the Rust `fetch_group` does. A group the producer writes meanwhile answers it too. With no @@ -399,7 +393,6 @@ export class Producer { subscribe: (name, options) => subscribe(this.#state, name, options), resolveTrackInfo: (name, hold) => resolveTrackInfo(this.#state, name, hold), largest: (name) => largest(this.#state, name), - givenUp: (name, sequence) => givenUp(this.#state, name, sequence), fetchGroup: (name, sequence, options) => fetchGroup(this.#state, name, sequence, options), requested: () => requested(this.#state), datagrams: (name) => datagrams(this.#state, name), @@ -562,7 +555,6 @@ export class Consumer { subscribe: (name, options) => subscribe(this.#state, name, options), resolveTrackInfo: (name, hold) => resolveTrackInfo(this.#state, name, hold), largest: (name) => largest(this.#state, name), - givenUp: (name, sequence) => givenUp(this.#state, name, sequence), fetchGroup: (name, sequence, options) => fetchGroup(this.#state, name, sequence, options), requested: () => requested(this.#state), datagrams: (name) => datagrams(this.#state, name), diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 9da84ad18e5..fbe40e54a61 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -653,13 +653,6 @@ export const hooks: { stageRequest: (request: Request, info: InfoInit) => StagedTrack; /** The newest frame a producer's cache holds, read without subscribing; assigned by `track.ts`. */ cachedLargest: (producer: Producer) => Position | undefined; - /** - * The error a producer gave `sequence` up for good with, when it holds no group for it: a - * skip, or a drop under no budget, which none of its readers fills. A stale drop stays - * fillable by a wider budget. Mirrors the Rust `poll_fetch_cached` `judges_drops`; assigned - * by `track.ts`. - */ - givenUpGroup: (producer: Producer, sequence: bigint) => Error | undefined; /** The publisher priority from a demanded track's committed {@link Info}; assigned by `track.ts`. */ demandPriority: (demand: Demand) => number; /** Seed a received track from the publisher's reported edge; assigned by `track.ts`. */ @@ -1023,9 +1016,6 @@ export const hooks: { cachedLargest: () => { throw new Error("track.ts not loaded"); }, - givenUpGroup: () => { - throw new Error("track.ts not loaded"); - }, demandPriority: () => { throw new Error("track.ts not loaded"); }, diff --git a/js/net/src/lite/drop_record.test.ts b/js/net/src/lite/drop_record.test.ts index df94cf290c8..cdd307ef151 100644 --- a/js/net/src/lite/drop_record.test.ts +++ b/js/net/src/lite/drop_record.test.ts @@ -508,9 +508,9 @@ for (const down of [Version.DRAFT_06, Version.DRAFT_07]) { }); } -// Mirrors Rust `a_lite06_upstream_keeps_a_reported_group_dropped_for_a_lite07_viewer` in +// Mirrors Rust `a_lite06_upstream_drop_is_fetched_for_a_widened_lite07_viewer` in // rs/moq-net/tests/integration/drop_record.rs. -test("through a lite-06 upstream, a group reported dropped as it lands stays dropped for a lite-07 viewer", async () => { +test("through a lite-06 upstream, a group reported dropped as it lands is fetched for a widened lite-07 viewer", async () => { using clock = manual(); const chained = await chain(Version.DRAFT_06, Version.DRAFT_07); for (let sequence = 0n; sequence < 10n; sequence++) { @@ -522,16 +522,18 @@ test("through a lite-06 upstream, a group reported dropped as it lands stays dro expect(groupsOf(await drain(clock, first))).toEqual([5n, 6n, 8n, 9n]); // Group 7 lands once its successor is older than 3.5s by the clock, so the lite-06 - // publisher reports it dropped, which lite-06 makes final. + // publisher reports it dropped, final for the relay's subscription but judged under its + // budget: the publisher still holds the group. await clock.advance(Duration.millis(4500)); chained.write(7n, 7000); expect(groupsOf(await drain(clock, first))).toEqual([]); - // Reaches every group: the widened reader fills 0 to 4, and group 7 alone stays dropped, - // for it and for a later reader, rather than being fetched from the publisher. - const all = [0n, 1n, 2n, 3n, 4n, 5n, 6n, 8n, 9n]; + // Reaches every group: the relay forwards the drop to the lite-07 viewer as a stale one, so + // the widening reopens it, and the widened reader fills 0 to 4 and 7 from the publisher, as + // a later reader does. + const all = [0n, 1n, 2n, 3n, 4n, 5n, 6n, 7n, 8n, 9n]; first.update({ delay: { max: Duration.millis(60_000) }, ranges: [{}] }); - expect(groupsOf(await drain(clock, first))).toEqual([0n, 1n, 2n, 3n, 4n]); + expect(groupsOf(await drain(clock, first))).toEqual([0n, 1n, 2n, 3n, 4n, 7n]); const second = chained.remote.track("video").subscribe({ delay: { max: Duration.millis(60_000) }, ranges: [{}] }); expect(groupsOf(await drain(clock, second))).toEqual(all); diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index 84497242b8f..02187cce2ca 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -1277,12 +1277,9 @@ export class Publisher { } finally { hold.abort(); } - // A group the track gave up for good is refused with that drop, as its own readers - // are, rather than asked upstream again. One it does not cache goes to its request - // handler at once, written yet or not, as in Rust; with none open it is NotFound. A - // pending answer holds this FETCH's slot until the requester leaves. - const refused = wireOf(front).givenUp(msg.track, msg.group); - if (refused) throw refused; + // A group the track does not cache goes to its request handler at once, written yet + // or not, as in Rust; with none open it is NotFound. A pending answer holds this + // FETCH's slot until the requester leaves. group = await front.track(msg.track).fetchGroup(msg.group, { priority: msg.priority, maxDelay: msg.maxDelay === undefined ? undefined : Duration.millis(msg.maxDelay), diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 4af788ed8cd..960c47ff77e 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -2148,7 +2148,7 @@ export class Front { } async fetchGroup(name: string, sequence: bigint, options?: track.FetchGroupOptions): Promise { - const fetched = await this.#fetch(name, sequence, options, undefined, undefined, true); + const fetched = await this.#fetch(name, sequence, options); let current = fetched.source; // Too old and evicted judge a subscription's delivery, not whether the group exists, so // neither is a verdict on a fetch: one answered from a group still arriving over a @@ -2195,16 +2195,13 @@ export class Front { // Fetch a group from whichever route serves the track, asking the next one when a route // fails, and never `avoid`. The source comes back retained for as long as the group is read. - // Once no route will come, it ends with `ended`, or as not found. With `judgesDrops`, the - // first fetch, a sequence the route's copy gave up for good is refused with that drop rather - // than asked again; a continuation goes past a drop, as a resumed group does. + // Once no route will come, it ends with `ended`, or as not found. async #fetch( name: string, sequence: bigint, options: track.FetchGroupOptions | undefined, avoid?: Source, ended?: Error, - judgesDrops = false, ): Promise<{ group: GroupConsumer; source: Source }> { for (;;) { const source = this.#source; @@ -2218,8 +2215,6 @@ export class Front { const retired = this.#retire.signal; const signal = options?.signal ? AbortSignal.any([options.signal, retired]) : retired; try { - const refused = judgesDrops ? source.wire.givenUp(name, sequence) : undefined; - if (refused) throw refused; // Asked alongside the check that the source claims the track's recorded properties, // rather than a round trip after it: only handing the group out waits for the check. const fetchGroup = source.wire.fetchGroup; diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 0e3d93cc081..0cf545998e8 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -1381,11 +1381,6 @@ export class Producer { for (const state of [producer.#state, ...producer.#sinks]) state.startChanged.update((rev) => rev + 1); }; hooks.cachedLargest = (producer) => producer.#cachedLargest(); - hooks.givenUpGroup = (producer, sequence) => { - if (cachedHolds(producer, sequence)) return undefined; - const drop = producer.#state.drops.at(sequence); - return drop && (drop.skip || drop.stale === undefined) ? drop.error : undefined; - }; hooks.reportedLargest = (producer, largest) => producer.#advanceObject(largest); hooks.reportedGroup = (producer, group) => producer.#advanceLargest({ group, frame: 0 }); objectLargest = (producer) => producer.#objectLargest; diff --git a/js/net/src/wire.ts b/js/net/src/wire.ts index aea61922ff2..52c09ad2d29 100644 --- a/js/net/src/wire.ts +++ b/js/net/src/wire.ts @@ -31,12 +31,6 @@ export interface Broadcast { resolveTrackInfo(name: string, hold?: AbortSignal): Promise; /** The newest frame the track's cache holds, read without subscribing; `undefined` while it holds none. */ largest(name: string): track.Position | undefined; - /** - * The error the track's cache gave `sequence` up for good with, read without subscribing, so - * a request served for a peer asks nothing upstream for it; `undefined` while a fill may still - * bring it. - */ - givenUp(name: string, sequence: bigint): Error | undefined; /** Fetch one group; absent when the session's version has no FETCH, so callers never ask. */ fetchGroup?(name: string, sequence: bigint, options?: track.FetchGroupOptions): Promise; requested(): Promise; diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index f71f382cb1a..f3b355db7e3 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -2204,14 +2204,8 @@ impl TrackState { /// Resolve a one-shot fetch from the track side: the cached group, or an [`Error`] /// once it can never be served. A missing group is a failure ([`Error::NotFound`]), not an /// end-of-stream. The handler side (a refusal, or no [`Requests`] at all) lives - /// in the track's fills; [`Fetching`] polls both. `judges_drops` answers a sequence given - /// up for good from the drop set, as this track's readers are answered. - fn poll_fetch_cached( - &self, - sequence: u64, - frames: group::Bounds, - judges_drops: bool, - ) -> Poll> { + /// in the track's fills; [`Fetching`] polls both. + fn poll_fetch_cached(&self, sequence: u64, frames: group::Bounds) -> Poll> { // A group aborted between the lookup and the consume (an abandoned fetch cut // short) is a miss, not a hit that fails on its first read. if let Some(group) = self.covering_range(sequence, frames) @@ -2233,16 +2227,6 @@ impl TrackState { return Poll::Ready(Err(Error::NotFound)); } - // Given up for good: skipped, or dropped under no budget, which no reader of this - // track fills, so a request asks nothing upstream for it either. A stale drop stays - // fillable by a wider budget. - if judges_drops - && let Some(hole) = self.drops.get(&sequence) - && hole.stale_budget().is_none() - { - return Poll::Ready(Err(Error::Stream(hole.error()))); - } - Poll::Pending } @@ -6172,7 +6156,6 @@ impl Consumer { options: options.into().unwrap_or_default(), end: None, }, - false, ) } @@ -6194,10 +6177,10 @@ impl Consumer { request.end = None; } } - self.fetching(sequence, request, true) + self.fetching(sequence, request) } - fn fetching(&self, sequence: u64, request: FetchOptions, judges_drops: bool) -> kio::Pending { + fn fetching(&self, sequence: u64, request: FetchOptions) -> kio::Pending { let FetchOptions { options, end: frame_end, @@ -6221,7 +6204,6 @@ impl Consumer { priority: options.priority, max_delay: options.max_delay, judges_drift: true, - judges_drops, result: None, hit: Some(Box::new(group)), stats: self.stats.clone(), @@ -6243,7 +6225,6 @@ impl Consumer { priority: options.priority, max_delay: options.max_delay, judges_drift: true, - judges_drops, result: None, hit: None, stats: self.stats.clone(), @@ -6266,7 +6247,6 @@ impl Consumer { end: frame_end, }, self.stats.clone(), - judges_drops, )) } @@ -6605,10 +6585,6 @@ pub struct Fetching { // Whether this fetch refuses a group a newer cached one superseded, judged across the // whole cache. A cursor's head fill leaves it to the cursor, which judges its window. judges_drift: bool, - // Whether this fetch, a request on a reader's behalf, refuses a sequence the track gave up - // for good. A group already partly read is continued past a drop, which judged only the - // subscription. - judges_drops: bool, // The fill this fetch joined; `None` on a cache hit, or when no handler existed to // queue on. Boxed: a miss already allocates, and the handle stays small. result: Option>, @@ -6631,7 +6607,6 @@ impl Fetching { sequence: u64, request: FetchOptions, stats: stats::Scope, - judges_drops: bool, ) -> Self { let frames = group::Bounds { start: request.options.frame_start, @@ -6639,10 +6614,7 @@ impl Fetching { }; let (fills, cached) = { let state = state.read(); - ( - state.fills.clone(), - state.poll_fetch_cached(sequence, frames, judges_drops), - ) + (state.fills.clone(), state.poll_fetch_cached(sequence, frames)) }; let result = match cached.is_pending() { true => queue(&fills, sequence, &request), @@ -6663,7 +6635,6 @@ impl Fetching { priority: request.options.priority, max_delay: request.options.max_delay, judges_drift: true, - judges_drops, result, hit, stats, @@ -6765,9 +6736,7 @@ impl Fetching { start: frame_start, end: self.frame_end, }; - let judges_drops = self.judges_drops; - let cached = |waiter: &kio::Waiter| match state - .poll(waiter, |state| state.poll_fetch_cached(sequence, frames, judges_drops)) + let cached = |waiter: &kio::Waiter| match state.poll(waiter, |state| state.poll_fetch_cached(sequence, frames)) { Poll::Ready(Ok(res)) => Poll::Ready(res.map(resolve)), Poll::Ready(Err(closed)) => Poll::Ready(Err(closed.outcome.aborted().cloned().unwrap_or(Error::Dropped))), @@ -9057,8 +9026,7 @@ impl Cursor { }; // The cursor judges the answer against its window when it hands it out. let mut fetching = - Fetching::cached_or_queued(self.state.clone(), sequence, request, stats::Scope::default(), false) - .unjudged(); + Fetching::cached_or_queued(self.state.clone(), sequence, request, stats::Scope::default()).unjudged(); let heads = self.heads.get_or_insert_default(); match kio::Task::poll(&mut fetching, waiter) { Poll::Pending => { diff --git a/rs/moq-net/tests/integration/drop_record.rs b/rs/moq-net/tests/integration/drop_record.rs index 40a42ce31c9..2b9d6b26241 100644 --- a/rs/moq-net/tests/integration/drop_record.rs +++ b/rs/moq-net/tests/integration/drop_record.rs @@ -315,11 +315,12 @@ async fn a_lite06_hop_keeps_a_reported_group_dropped() { } /// [`a_lite06_hop_keeps_a_reported_group_dropped`] with the lite-06 hop upstream of a lite-07 -/// one: the relay forwards the final drop as a stale one, the viewer's widening update reopens -/// it and fetches the group, and the relay fetches it from the publisher instead of answering -/// from the drop it holds. +/// one: a lite-06 drop is final for the relay's subscription, which no update re-offers, but +/// it was judged under that subscription's budget. The relay forwards it as a stale one, the +/// viewer's widening update reopens it, and the group is fetched from the publisher, which +/// still holds it, as the relay drills' FETCH recovery requires. #[kio::test] -async fn a_lite06_upstream_keeps_a_reported_group_dropped_for_a_lite07_viewer() { +async fn a_lite06_upstream_drop_is_fetched_for_a_widened_lite07_viewer() { reported_drop_through([LITE06, LITE07]).await; } @@ -343,8 +344,9 @@ async fn reported_drop_through(versions: [&str; 2]) { write(&chain.source, 7, 7000); assert_eq!(received(&mut first).await, BTreeSet::new(), "{name}"); - // Only lite-07 end to end leaves the stale report open to a widening update. - let want: BTreeSet = match versions.contains(&LITE06) { + // A lite-07 viewer hop leaves the stale report open to a widening update, which fills + // the group from upstream; over a lite-06 viewer hop the drop is final. + let want: BTreeSet = match versions[1] == LITE06 { true => (0..GROUPS).filter(|sequence| *sequence != 7).collect(), false => (0..GROUPS).collect(), }; From 33db1bba9e9d793180e9293cfb3c92a314783d71 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 23:32:26 -0800 Subject: [PATCH 14/37] fix(net): fold a tail's finished streams and a reader's refused fills under debt - A subscription's tail bills its finished streams as a reader's record, so the pool can hand it debt, and under debt keeps only the newest run: a stale drop under the update then stands as the publisher judged it. Rust and js/net. - js/net: a reader's refused fills are billed with its record and forgotten below its floor, as Rust's `Misses::given_up` is. - js/net: a range fill's setup stops once nobody wants its run, so a stalled exchange never sends a subscription nobody asked for anymore. Co-Authored-By: Claude Opus 5.5 --- js/net/src/cache.test.ts | 25 +++++++++++++ js/net/src/internal.ts | 8 ++-- js/net/src/lite/subscriber.ts | 9 +++++ js/net/src/tail.ts | 24 +++++++++--- js/net/src/track.ts | 15 +++----- rs/moq-net/src/model/track.rs | 69 ++++++++++++++++++++++++++++++----- rs/moq-net/src/tail.rs | 21 +++++++++-- 7 files changed, 139 insertions(+), 32 deletions(-) diff --git a/js/net/src/cache.test.ts b/js/net/src/cache.test.ts index f57f239b1e4..de4523a8776 100644 --- a/js/net/src/cache.test.ts +++ b/js/net/src/cache.test.ts @@ -4,6 +4,7 @@ import * as cache from "./cache.ts"; import { StreamCode, StreamError, TooFarBehind } from "./error.ts"; import { hooks, staleHole } from "./internal.ts"; import { RUN_OVERHEAD as ARRIVAL_OVERHEAD } from "./ledger.ts"; +import { Tail } from "./tail.ts"; import { Producer as TrackProducer } from "./track.ts"; // Run `fn` on a manual clock, stepped in milliseconds. @@ -377,6 +378,30 @@ for (const stale of [false, true]) { }); } +// A subscription's tail, recording a fragmented run of finished streams between updates, bills it to +// its track's account. Under the pool's debt it forgets them and stays within the pool. Mirrors Rust +// `a_tail_forgets_its_finished_streams_under_debt`. +test("a tail forgets its finished streams under debt", () => { + const runs = 4000n; + const pool = new cache.Pool({ capacity: 1 << 30 }); + const producer = pooled(pool); + hooks.receiveTrack(producer); + const tail = new Tail({ bill: hooks.billTrack(producer) }); + for (let run = 0n; run < runs; run++) tail.finish(2n * run); + expect(tail.unfinished(0n, 2n * runs).length).toBe(Number(runs)); + expect(pool.used).toBeGreaterThanOrEqual(Number(runs) * cache.READER_RUN_OVERHEAD); + + // The pool shrinks, and groups written above them accrue the track's debt, which its own groups + // cannot pay: the tail pays it by forgetting what finished. + const capacity = 64 * cache.RUN_OVERHEAD; + pool.resize(capacity); + for (let i = 0; i < 64; i++) finished(producer, 16 * 1024); + tail.finish(2n * runs); + expect(tail.unfinished(0n, 2n * runs + 1n).length).toBeLessThanOrEqual(2); + expect(pool.used).toBeLessThanOrEqual(capacity + 2 * (16 * 1024 + cache.ENTRY_OVERHEAD)); + tail.close(); +}); + // A peer fragmenting its arrivals under a grace that never folds a gap bills each run to the pool, // which folds the oldest gaps early to stay within it. Mirrors Rust `an_arrival_flood_stays_within_the_pool`. test("an arrival flood stays within the pool", () => { diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index fbe40e54a61..9dca1af037c 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -8,7 +8,7 @@ import type { Duration, Instant } from "@moq/clock"; import type { Dispose, GetPromise, Getter } from "@moq/signals"; import type { Consumer as BroadcastConsumer, Producer as BroadcastProducer } from "./broadcast.ts"; -import type { Charge, Pool } from "./cache.ts"; +import type { Reader as CacheReader, Charge, Pool } from "./cache.ts"; import type * as datagram from "./datagram.ts"; import type { Delivered } from "./delivered.ts"; import { StreamCode, StreamError, toStreamCode } from "./error.ts"; @@ -750,10 +750,10 @@ export const hooks: { wire: (subscriber: Subscriber) => WireRecord; /** * Open a bill against a track's cache account for what a session keeps for it beside the - * cache, such as its tail's record of finished streams: each call bills that many bytes in - * all. Mirrors the Rust crate-private `track::Producer::bill`. + * cache, such as its tail's record of finished streams, billed as a reader's record is, so + * the pool can hand it debt. Mirrors the Rust crate-private `track::Producer::bill`. */ - billTrack: (producer: Producer) => (bytes: number) => void; + billTrack: (producer: Producer) => CacheReader; /** * Re-offer what a subscriber gave up as stale through dispositions, as a wire whose every update * reopens its stale drops does: each run is judged again under the new terms. Returns the runs diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 0613973f4cb..f351f0e11d6 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -2070,6 +2070,15 @@ export class Subscriber { }; const left = new AbortController(); const signal = consumed.valid ? AbortSignal.any([consumed.valid, left.signal]) : left.signal; + // Setup stops once nobody wants the run, as a FETCH's does: a stalled exchange (no stream + // credit, or a blocked write) never sends a subscription nobody asked for anymore. + const leaving = new Promise((resolve) => + left.signal.addEventListener("abort", () => resolve(), { once: true }), + ); + void (async () => { + while (!left.signal.aborted && owes()) await race([fillChanged(fill), leaving]); + left.abort(); + })(); let failed: Error = new StreamError(StreamCode.Cancel, { message: "fill subscription ended" }); let draining = Promise.resolve(); this.#subscribes.set(id, entry); diff --git a/js/net/src/tail.ts b/js/net/src/tail.ts index be612c519c0..5d62bcf8946 100644 --- a/js/net/src/tail.ts +++ b/js/net/src/tail.ts @@ -1,6 +1,6 @@ import { type Duration, now, sleep } from "@moq/clock"; import { type GetPromise, Signal } from "@moq/signals"; -import { READER_RUN_OVERHEAD } from "./cache.ts"; +import { READER_RUN_OVERHEAD, type Reader } from "./cache.ts"; import { insertRun, type Run, runGaps } from "./internal.ts"; import { GRACE, Ledger } from "./ledger.ts"; @@ -8,8 +8,8 @@ import { GRACE, Ledger } from "./ledger.ts"; export interface TailOptions { /** How long to wait for a group that cannot be accounted for. Read each time it is needed. */ grace?: () => Duration; - /** Bill the bytes the tail keeps, in all, to the subscription's copy's account. */ - bill?: (bytes: number) => void; + /** Bills the bytes the tail keeps to the subscription's copy's account, as a reader's record. */ + bill?: Reader; } /** @@ -39,21 +39,33 @@ export class Tail { #finished: Run[] = []; // What `#finished` bills, `READER_RUN_OVERHEAD` per run, so a publisher fragmenting the // streams it finishes between updates costs what the cache would hold. - #bill?: (bytes: number) => void; + #bill?: Reader; constructor(options: TailOptions = {}) { this.#grace = options.grace ?? (() => GRACE); this.#bill = options.bill; } + // Bill what `#finished` keeps, and under the pool's debt forget all but its newest run: a stale + // drop under this update then stands as the publisher judged it, which a wider budget still + // reverses. The newest run stays, so streams finishing in sequence grow nothing to charge again. + // Mirrors the Rust `Tail::rebill`. #rebill(): void { - this.#bill?.(this.#finished.length * READER_RUN_OVERHEAD); + const bill = this.#bill; + if (!bill) return; + bill.set(this.#finished.length * READER_RUN_OVERHEAD); + if (bill.debt > 0 && this.#finished.length > 1) { + const freed = bill.size - READER_RUN_OVERHEAD; + this.#finished = this.#finished.slice(-1); + bill.set(READER_RUN_OVERHEAD); + bill.pay(freed); + } } /** Release what the tail bills, once its subscription ended. */ close(): void { this.#finished = []; - this.#rebill(); + this.#bill?.close(); } /** Group streams whose header arrived, whether they finished or were reset. */ diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 0cf545998e8..eaac84bcca6 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -1301,14 +1301,7 @@ export class Producer { }; hooks.arrivalRuns = (producer) => producer.#arrivals.size; accountOf = (producer) => producer.#account; - hooks.billTrack = (producer) => { - let billed = 0; - return (bytes) => { - if (bytes > billed) producer.#account.bill(bytes - billed); - else producer.#account.unbill(billed - bytes); - billed = bytes; - }; - }; + hooks.billTrack = (producer) => producer.#account.reader(); hooks.subscribing = (producer) => producer.#subscribing; hooks.requested = (producer) => producer.#state.update; resubscribe = (producer) => producer.#updateSubscription(); @@ -3068,7 +3061,8 @@ export class Subscriber { this.#skipped.size + this.#pending.length + this.#reported.size + - this.#givenUp.length + this.#givenUp.length + + this.#state.refused.peek().length ); } @@ -3147,6 +3141,9 @@ export class Subscriber { this.#pending = this.#pending.flatMap((run) => removeRun([run], 0n, floor)); this.#reported.set(0n, floor); this.#givenUp = removeRun(this.#givenUp, 0n, floor); + // The fills it refused, as the Rust `Misses::given_up` is forgotten below the floor. + if (this.#state.refused.peek().some((run) => run.start < floor)) + this.#state.refused.update((runs) => removeRun(runs, 0n, floor)); } // Owe again every run this cursor gave up as stale, reported or not: an update reopens the diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index f3b355db7e3..c44c2d15ca0 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -4016,19 +4016,30 @@ pub struct Producer { stats: stats::Scope, } -/// Bytes a session keeps for a track beside its cache, billed to the track's account and -/// released on drop. -pub(crate) struct Bill(cache::Extra); +/// Bytes a session keeps for a track beside its cache, billed to the track's account as a +/// reader's record is, so the pool can hand it debt, and released on drop. +pub(crate) struct Bill(cache::Reader); #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] impl Bill { /// Bill `bytes` in all. - pub(crate) fn set(&self, bytes: u64) { - let size = self.0.size(); - match bytes > size { - true => self.0.add(bytes - size), - false => self.0.sub(size - bytes), - } + pub(crate) fn set(&mut self, bytes: u64) { + self.0.set(bytes); + } + + /// What this bill holds. + pub(crate) fn size(&self) -> u64 { + self.0.size() + } + + /// The debt the track left its readers, which this bill pays by shrinking. + pub(crate) fn debt(&self) -> u64 { + self.0.debt() + } + + /// Pay up to `bytes` of that debt, as a bill that shrank by that much. + pub(crate) fn pay(&self, bytes: u64) { + self.0.pay(bytes); } } @@ -4934,7 +4945,7 @@ impl Producer { /// cache, such as its tail's record of finished streams. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn bill(&self) -> Bill { - Bill(self.state.read().cache.extra()) + Bill(self.state.read().cache.reader()) } /// How long a subscriber session's tail waits for a late group: `budget`, bounded by how @@ -24250,6 +24261,44 @@ mod test { ); } + /// A subscription's tail, recording a fragmented run of finished streams between updates, + /// bills it to its track's account. Under the pool's debt it forgets them, which leaves a + /// stale drop under the update to stand as the publisher judged it, and stays within the + /// pool. + #[cfg(feature = "async")] + #[kio::test] + async fn a_tail_forgets_its_finished_streams_under_debt() { + const RUNS: u64 = 4000; + let (producer, pool) = pooled_producer(1 << 30); + producer.received(); + let tail = kio::Producer::new(crate::tail::Tail::default()); + if let Ok(mut state) = tail.write() { + state.bill(producer.bill()); + } + let now = crate::time::sim().now(); + let finish = |sequence: u64| crate::tail::Reading::open(&tail, Some(sequence), now).finish(); + for run in 0..RUNS { + finish(2 * run); + } + assert_eq!(tail.read().finished(0..2 * RUNS).count() as u64, RUNS); + assert!(pool.used() >= RUNS * 16, "the finished streams are billed"); + + // The pool shrinks, and groups written above them accrue the track's debt, which its own + // groups cannot pay: the tail pays it by forgetting what finished. + let capacity = 64 * (RUN_OVERHEAD + BORN_OVERHEAD); + pool.resize(capacity); + let mut producer = producer; + for _ in 0..64 { + finished_group(&mut producer, 16 * 1024); + } + finish(2 * RUNS); + assert!( + tail.read().finished(0..2 * RUNS + 1).count() <= 1, + "forgotten under debt" + ); + assert!(pool.used() <= capacity, "within the pool: {}", pool.used()); + } + /// A flood a reader reported, fragmented into runs alternating with upstream skips, /// bills its record too. Once the pool shrinks under it and its track has shed what it /// can, the reader folds the final runs and the groups it read into its floor and stays diff --git a/rs/moq-net/src/tail.rs b/rs/moq-net/src/tail.rs index d6c6671d236..de340dcbb1e 100644 --- a/rs/moq-net/src/tail.rs +++ b/rs/moq-net/src/tail.rs @@ -69,9 +69,24 @@ impl Tail { self.rebill(); } - fn rebill(&self) { - if let Some(bill) = &self.bill { - bill.set(self.finished.len() as u64 * FINISHED_OVERHEAD); + /// Bill what `finished` keeps, and under the pool's debt forget all but its newest run: a + /// stale drop under this update then stands as the publisher judged it, which a wider budget + /// still reverses. The newest run stays, so streams finishing in sequence grow nothing to + /// charge again. + fn rebill(&mut self) { + let Some(bill) = &mut self.bill else { + return; + }; + bill.set(self.finished.len() as u64 * FINISHED_OVERHEAD); + if bill.debt() > 0 + && self.finished.len() > 1 + && let Some(newest) = self.finished.iter().next_back().cloned() + { + let freed = bill.size() - FINISHED_OVERHEAD; + self.finished = RangeSet::new(); + self.finished.insert(newest); + bill.set(FINISHED_OVERHEAD); + bill.pay(freed); } } From 41318508e9a6d7b2587d752452504f5d557af563 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 01:48:59 -0800 Subject: [PATCH 15/37] fix(net): pay reader debt on every shrink, bill resumed JS readers, and drop folded re-offers as evicted A reader's bill now pays its track's reader debt with whatever it frees, and the debt is forgiven once the pool is back within capacity. A JS front's reader bills against its serving copy's account, and a closed tail never bills again. A route copy's record settles on each peek, and a resumed reader retires a delivered group's stale report. A lagged reader ends its subscription even while a group waits for stream credit, and closes the group a lag replaced. A queued re-offer, or a stream expiring OLD, that the record already folded goes out as a final EVICTED drop rather than silence. A fill subscription's groups count as fetched, so an abandoned one holds no end open. Co-Authored-By: Claude Opus 5.5 --- js/net/src/cache.test.ts | 32 +++++++++++++++++++++++++ js/net/src/cache.ts | 17 ++++++------- js/net/src/internal.ts | 14 +++++++++-- js/net/src/lite/publisher.ts | 25 +++++++++++++++++-- js/net/src/lite/subscriber.ts | 6 ++++- js/net/src/resume.ts | 6 +++++ js/net/src/tail.ts | 4 ++-- js/net/src/track.ts | 41 ++++++++++++++++++++++++-------- rs/moq-net/src/lite/publisher.rs | 16 +++++++++++++ rs/moq-net/src/model/cache.rs | 28 ++++++++++------------ rs/moq-net/src/model/record.rs | 27 ++++++++++++++++++--- rs/moq-net/src/model/resume.rs | 7 +++++- rs/moq-net/src/model/track.rs | 21 +++++++--------- rs/moq-net/src/tail.rs | 2 -- 14 files changed, 188 insertions(+), 58 deletions(-) diff --git a/js/net/src/cache.test.ts b/js/net/src/cache.test.ts index de4523a8776..368bc957aa1 100644 --- a/js/net/src/cache.test.ts +++ b/js/net/src/cache.test.ts @@ -402,6 +402,38 @@ test("a tail forgets its finished streams under debt", () => { tail.close(); }); +// Whatever a bill frees pays its readers' debt, however it shrank, and the debt is forgiven once +// the pool is back within its capacity. A closed tail never bills again. Mirrors Rust +// `a_debt_is_paid_by_any_shrink_and_forgiven_without_pressure`. +test("a reader's debt is paid by any shrink and forgiven without pressure", () => { + const unit = cache.READER_RUN_OVERHEAD; + const pool = new cache.Pool({ capacity: 1 << 30 }); + const producer = pooled(pool); + const bill = hooks.billTrack(producer); + bill.set(4 * unit); + pool.resize(2 * unit); + // Nothing else is cached, so the debt the track accrues falls to its readers. + producer.createGroup(0n).abort(new Error("gone")); + bill.set(5 * unit); + const owed = bill.debt; + expect(owed).toBeGreaterThan(unit); + bill.set(4 * unit); + expect(bill.debt).toBe(owed - unit); + pool.resize(1 << 30); + expect(bill.debt).toBe(0); + pool.resize(2 * unit); + expect(bill.debt).toBe(0); + bill.close(); + + const tail = new Tail({ bill: hooks.billTrack(producer) }); + tail.finish(0n); + tail.finish(2n); + tail.close(); + const used = pool.used; + tail.finish(4n); + expect(pool.used).toBe(used); +}); + // A peer fragmenting its arrivals under a grace that never folds a gap bills each run to the pool, // which folds the oldest gaps early to stay within it. Mirrors Rust `an_arrival_flood_stays_within_the_pool`. test("an arrival flood stays within the pool", () => { diff --git a/js/net/src/cache.ts b/js/net/src/cache.ts index 40facc3f970..350105f3e06 100644 --- a/js/net/src/cache.ts +++ b/js/net/src/cache.ts @@ -558,8 +558,14 @@ export class Reader { return this.#bytes; } - /** The debt the track's readers owe. */ + /** + * The debt the track's readers owe, forgiven once the pool is back within its capacity: + * nothing needs the room any more, as the track forgives its own debt. + */ get debt(): number { + const pool = this.#track.pool; + const capacity = pool.capacity; + if (capacity === undefined || pool.used <= capacity) this.#readers.debt = 0; return this.#readers.debt; } @@ -577,15 +583,12 @@ export class Reader { this.#track.wrote(bytes - before); this.#track.chargeOver(); } else { + // Whatever a record frees pays toward its readers' debt, however it shrank. + readers.debt = Math.max(0, readers.debt - (before - bytes)); this.#track.pool.sub(before - bytes); } } - /** Pay up to `bytes` of the readers' debt, as a record that shrank by that much. */ - pay(bytes: number): void { - this.#readers.debt = Math.max(0, this.#readers.debt - bytes); - } - /** * Whether this record is one to end when folding leaves a debt: it bills more than a cached * group's overhead, so ending it frees more than evicting a group would, and at least its @@ -598,9 +601,7 @@ export class Reader { /** Release the bill, paying the readers' debt with it. */ close(): void { - const bytes = this.#bytes; this.set(0); - this.pay(bytes); } } diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 9dca1af037c..5922f74a5f1 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -490,6 +490,8 @@ export interface ResumeSource { * route change keeps, and those the serving copy reported. */ reofferStale(): Run[]; + /** Open a bill for the reader's record against the serving copy's account, undefined without one. */ + bill(): CacheReader | undefined; /** Let go of every copy. */ close(abort?: Error): void; } @@ -610,8 +612,11 @@ export const hooks: { * ask below it. Assigned by `track.ts`. */ receiveTrack: (producer: Producer, starts?: FillStarts) => void; - /** Create a received group at its wire offset before publishing it to readers. Assigned by `track.ts`. */ - receiveGroup: (producer: Producer, group: { sequence: bigint; start: number }) => GroupProducer; + /** + * Create a received group at its wire offset before publishing it to readers; `fill` marks one a + * fill's subscription answered, which the feed never promised. Assigned by `track.ts`. + */ + receiveGroup: (producer: Producer, group: { sequence: bigint; start: number; fill?: boolean }) => GroupProducer; /** How the copy of `sequence` still arriving ends, if one is, for a duplicate to wait on. Assigned by `track.ts`. */ arrivingGroup: (producer: Producer, sequence: bigint) => GetPromise | undefined; /** The producer's largest reported location and its subsequent changes. */ @@ -754,6 +759,8 @@ export const hooks: { * the pool can hand it debt. Mirrors the Rust crate-private `track::Producer::bill`. */ billTrack: (producer: Producer) => CacheReader; + /** Open a bill against the account of the track `subscriber` reads, undefined while it has none. */ + billSubscriber: (subscriber: Subscriber) => CacheReader | undefined; /** * Re-offer what a subscriber gave up as stale through dispositions, as a wire whose every update * reopens its stale drops does: each run is judged again under the new terms. Returns the runs @@ -1064,6 +1071,9 @@ export const hooks: { billTrack: () => { throw new Error("track.ts not loaded"); }, + billSubscriber: () => { + throw new Error("track.ts not loaded"); + }, supersedeStale: () => { throw new Error("track.ts not loaded"); }, diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index 02187cce2ca..7f5c35b68ee 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -1551,12 +1551,17 @@ export class Publisher { skipped.length = 0; } if (pendingDrops.length > 0) { - for (const { start, end, error } of pendingDrops.splice(0)) { + for (const { start, end, error: failed } of pendingDrops.splice(0)) { + let error = failed; + const old = hasSkip(this.version) && toStreamCode(error) === StreamCode.Old; + // The pool folded the run while its stream was open, which is final for this + // reader: no update would re-offer a stale drop on it. Mirrors the Rust `TrackRun`. + if (old && record?.folded(end - 1n)) error = new StreamError(StreamCode.Evicted); // A stream expired under this subscription's budget is a stale verdict, which // a later verdict or group replaces as any stale drop sent, and each update drops // again. A re-offer an update owed again is given up now, so the end sweep never // names it. - if (hasSkip(this.version) && toStreamCode(error) === StreamCode.Old) toldStale(start, end); + else if (old) toldStale(start, end); else record?.tell(start, end, reportOf({ error })); const msg = SubscribeDrop.given(start, end, error, this.version, updateNumber); if (!(await controls.response(encodeSubscribeResponse(stream, { drop: msg }, this.version)))) @@ -1830,6 +1835,22 @@ export class Publisher { // A lite-07 stale drop is no final verdict, and a Group Stream wins over it, so a group // handed out after one is served, as one a route the front switched to delivers: the // update that reopens the drop would never re-offer it. + // The pool folded a re-offered group while it waited, which is final for this reader: + // tell the wire it is gone, since an update re-offered it. Mirrors the Rust `TrackRun`. + if (again && record?.folded(group.sequence)) { + group.close(); + const evicted = new StreamError(StreamCode.Evicted); + const msg = SubscribeDrop.given( + group.sequence, + group.sequence + 1n, + evicted, + this.version, + updateNumber, + ); + if (!(await controls.response(encodeSubscribeResponse(stream, { drop: msg }, this.version)))) + return false; + continue; + } if (record?.toldStale(group.sequence)) { record.replaceDrops(group.sequence, group.sequence + 1n); } else if (record?.told(group.sequence)) { diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index f351f0e11d6..5df0c540389 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -2760,7 +2760,11 @@ export class Subscriber { await race([settled, stream.closed.then(() => settled)]); arriving = hooks.arrivingGroup(track, group.sequence); } - producer = hooks.receiveGroup(track, { sequence: group.sequence, start: group.frameStart }); + producer = hooks.receiveGroup(track, { + sequence: group.sequence, + start: group.frameStart, + fill: entry.filling, + }); const reading = readFrames(stream, producer, { scale: timescale, compress: entry.compress }); await (entry.filling ? untilAbandoned(producer, reading) : reading); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 960c47ff77e..d02160e8661 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -48,6 +48,7 @@ import { Duration, Instant, now, sleepUntil } from "@moq/clock"; import { type Dispose, type Getter, race, Signal } from "@moq/signals"; import * as broadcast from "./broadcast.ts"; +import type { Reader as CacheReader } from "./cache.ts"; import { Delivered } from "./delivered.ts"; import { NotFound, SessionError, StreamCode, StreamError, sessionClosed } from "./error.ts"; import type { Consumer as GroupConsumer } from "./group.ts"; @@ -744,6 +745,11 @@ class Reader implements ResumeSource { * replaced copy may no longer deliver, so what it gave up is the new route's to deliver or * give up. Mirrors the Rust `Reader::reoffer_stale`. */ + bill(): CacheReader | undefined { + const serving = this.#serving(); + return serving && hooks.billSubscriber(serving.sub); + } + reofferStale(): Run[] { const taken = this.#reportedStale; this.#reportedStale = []; diff --git a/js/net/src/tail.ts b/js/net/src/tail.ts index 5d62bcf8946..6bac68ce20b 100644 --- a/js/net/src/tail.ts +++ b/js/net/src/tail.ts @@ -55,17 +55,17 @@ export class Tail { if (!bill) return; bill.set(this.#finished.length * READER_RUN_OVERHEAD); if (bill.debt > 0 && this.#finished.length > 1) { - const freed = bill.size - READER_RUN_OVERHEAD; this.#finished = this.#finished.slice(-1); bill.set(READER_RUN_OVERHEAD); - bill.pay(freed); } } /** Release what the tail bills, once its subscription ended. */ close(): void { this.#finished = []; + // A stream still in flight finishes after this: with no bill, it can never bill again. this.#bill?.close(); + this.#bill = undefined; } /** Group streams whose header arrived, whether they finished or were reset. */ diff --git a/js/net/src/track.ts b/js/net/src/track.ts index eaac84bcca6..e9a1e2f5b4d 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -1327,7 +1327,12 @@ export class Producer { hooks.receiveTrack = (producer, starts = {}) => { producer.#wire = { starts, served: producer.#wire?.served }; }; - hooks.receiveGroup = (producer, { sequence, start }) => producer.#createGroup(sequence, start); + hooks.receiveGroup = (producer, { sequence, start, fill }) => { + const group = producer.#createGroup(sequence, start); + const entry = fill ? producer.#cached.get(sequence) : undefined; + if (entry) entry.fetched = true; + return group; + }; hooks.arrivingGroup = (producer, sequence) => { const closed = producer.#cached.get(sequence)?.group.closed; return closed?.peek() === undefined ? closed : undefined; @@ -2510,6 +2515,8 @@ function blocksHole(reported: Reported, hole: Hole): boolean { export interface WireRecord { /** Whether the wire was told of `sequence`: served, or given up. */ told(sequence: bigint): boolean; + /** Whether the pool folded `sequence` into the record's floor, final for this reader. */ + folded(sequence: bigint): boolean; /** Whether `sequence` went out as a stale drop, which a group still replaces. */ toldStale(sequence: bigint): boolean; /** The parts of `[start, end)` the wire was told nothing of. */ @@ -3018,6 +3025,7 @@ export class Subscriber { const reported = this.#reported; return { told: (sequence) => sequence < this.#folded || reported.at(sequence) !== undefined, + folded: (sequence) => sequence < this.#folded, toldStale: (sequence) => reported.at(sequence)?.kind === "stale", untold: (start, end) => { const from = max(start, this.#folded); @@ -3053,6 +3061,14 @@ export class Subscriber { }; } + // A bill against the account of the track this reads: a resumed reader's serving copy, as the + // Rust logical track's account bills its routes' readers. + #openBill(): cache.Reader | undefined { + if (this.#resume) return this.#resume.bill(); + const producer = this.#state.producer; + return producer && accountOf(producer).reader(); + } + // How many runs the record holds above its floor, each billed `cache.READER_RUN_OVERHEAD`: // the handed-out prefix is one run that bills nothing. #recordRuns(): number { @@ -3074,18 +3090,15 @@ export class Subscriber { if (this.#lagged) return new TooFarBehind(); const bytes = this.#recordRuns() * cache.READER_RUN_OVERHEAD; if (!this.#bill) { - const producer = this.#state.producer; - if (bytes === 0 || !producer) return undefined; - this.#bill = accountOf(producer).reader(); + if (bytes === 0) return undefined; + this.#bill = this.#openBill(); } const bill = this.#bill; + if (!bill) return undefined; bill.set(bytes); if (bill.debt === 0) return undefined; this.#fold(handed); - const folded = this.#recordRuns() * cache.READER_RUN_OVERHEAD; - const freed = Math.max(0, bill.size - folded); - bill.set(folded); - bill.pay(freed); + bill.set(this.#recordRuns() * cache.READER_RUN_OVERHEAD); if (bill.debt > 0 && bill.ends) { this.#lagged = true; bill.close(); @@ -3804,6 +3817,7 @@ export class Subscriber { const producer = subscriber.#state.producer; return producer !== undefined && mayLand(producer, land); }; + hooks.billSubscriber = (subscriber) => subscriber.#openBill(); hooks.resumeSubscriber = (name, subscription, source) => { const state = new TrackState(subscription ?? Subscription.default()); const subscriber = new Subscriber(name, state); @@ -4208,7 +4222,10 @@ export class Subscriber { // it hands out stays above the fold until the next read, so the publisher tells its wire of // it first rather than taking it for one already served. const lagged = this.#settleRecord(recv.kind === "group" ? recv.group.sequence : undefined); - return lagged ? { kind: "error", error: lagged } : recv; + if (!lagged) return recv; + // Nobody takes the group the lag replaced, so it lets go of its source here. + if (recv.kind === "group") recv.group.close(); + return { kind: "error", error: lagged }; } #tryRecvUnsettled(dispositions: boolean): Recv { @@ -4593,7 +4610,11 @@ export class Subscriber { if (this.#lagged) return { kind: "error", error: new TooFarBehind() }; let recv = this.#resume ? this.#tryResumed(true) : this.#tryNextBuffered(); const lagged = this.#settleRecord(); - if (lagged) recv = { kind: "error", error: lagged }; + if (lagged) { + // Nobody takes the group the lag replaced, so it lets go of its source here. + if (recv.kind === "group") recv.group.close(); + recv = { kind: "error", error: lagged }; + } // One cursor: the frame helpers must not keep draining a group this read just // moved past, or interleaved reads would run backwards. if (recv.kind === "group" && this.#frameGroup && this.#frameGroup.sequence < recv.group.sequence) { diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 8d6e1ae5efb..e5726960cc8 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -5451,6 +5451,11 @@ impl TrackRun { for (sequence, err) in dropped { // A stream expired under this subscription's budget is a stale verdict. let hole = match err { + // The pool folded the group while its stream was open, which is final for this + // reader: no update would re-offer a stale drop on it. + StreamError::Old if self.ctx.version.has_skip() && self.track.record_mut().folded(sequence) => { + Hole::drop(StreamError::Evicted) + } StreamError::Old => { if self.ctx.version.has_skip() { self.sent_stale(sequence..sequence + 1); @@ -5539,6 +5544,12 @@ impl TrackRun { && let Some(group) = self.reoffered.pop_front() { let sequence = group.sequence; + // The pool folded it while it waited, which is final for this reader: tell the wire + // it is gone, since an update re-offered it. + if self.emit_range && self.track.record_mut().folded(sequence) { + writer.buffer(&self.hole(sequence..sequence + 1, &Hole::drop(StreamError::Evicted)))?; + return Poll::Ready(Ok(ControlFlow::Continue(()))); + } if self.emit_range && self.track.record_mut().told(sequence) { return Poll::Ready(Ok(ControlFlow::Continue(()))); } @@ -5675,6 +5686,11 @@ impl TrackRun { false => self.track.poll_queued_drop(waiter), }; let Poll::Ready((range, hole)) = polled else { + // A reader the pool ended reports no drop: end the subscription now, rather than once + // a group waiting on stream credit reads the failure. + if self.track.lagged() { + return Err(Error::Lagged); + } return Ok(false); }; match self.start { diff --git a/rs/moq-net/src/model/cache.rs b/rs/moq-net/src/model/cache.rs index df324784a59..8527ee463e3 100644 --- a/rs/moq-net/src/model/cache.rs +++ b/rs/moq-net/src/model/cache.rs @@ -1375,8 +1375,12 @@ impl Reader { self.bytes = bytes; self.track.charge_over(); } else { + // Whatever a record frees pays toward its readers' debt, however it shrank. let shrunk = self.bytes - bytes; readers.bytes.fetch_sub(shrunk, Ordering::Relaxed); + let _ = readers.debt.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |debt| { + (debt > 0).then(|| debt.saturating_sub(shrunk)) + }); self.track.pool.sub(shrunk); self.bytes = bytes; } @@ -1387,20 +1391,16 @@ impl Reader { self.bytes } - /// The debt the track's readers owe. + /// The debt the track's readers owe, forgiven once the pool is back within its capacity: + /// nothing needs the room any more, as the track forgives its own debt. pub(super) fn debt(&self) -> u64 { - self.track.readers.debt.load(Ordering::Relaxed) - } - - /// Pay up to `bytes` of the readers' debt, as a record that shrank by that much. - pub(super) fn pay(&self, bytes: u64) { - let _ = self - .track - .readers - .debt - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |debt| { - Some(debt.saturating_sub(bytes)) - }); + let readers = &self.track.readers; + let pool = &self.track.pool; + if pool.capacity().is_none_or(|capacity| pool.used() <= capacity) { + readers.debt.store(0, Ordering::Relaxed); + return 0; + } + readers.debt.load(Ordering::Relaxed) } /// Whether this record is one to end when folding leaves a debt: it bills more than a @@ -1416,9 +1416,7 @@ impl Reader { impl Drop for Reader { fn drop(&mut self) { - let bytes = self.bytes; self.set(0); - self.pay(bytes); } } diff --git a/rs/moq-net/src/model/record.rs b/rs/moq-net/src/model/record.rs index 77ddf3c07fa..d3a0a164caa 100644 --- a/rs/moq-net/src/model/record.rs +++ b/rs/moq-net/src/model/record.rs @@ -197,6 +197,11 @@ impl Record { sequence < self.floor || self.reported.contains_key(&sequence) } + /// Whether the pool folded `sequence` into this record's floor, final for this reader. + pub fn folded(&self, sequence: u64) -> bool { + sequence < self.floor + } + /// Whether `sequence` went out as a stale drop, which a group still replaces. pub fn told_stale(&self, sequence: u64) -> bool { matches!(self.reported.get(&sequence), Some(Report::Stale { .. })) @@ -335,9 +340,7 @@ impl Record { beside.forget_below(self.floor); let bytes = (self.runs() + beside.runs()) * RUN_OVERHEAD; let bill = self.bill.as_mut().expect("opened above"); - let freed = bill.size().saturating_sub(bytes); bill.set(bytes); - bill.pay(freed); if bill.debt() > 0 && bill.ends() { self.fail(); return Err(Error::Lagged); @@ -431,8 +434,9 @@ mod test { use super::*; use crate::StreamError; + /// An account on a pool that any bill puts over its capacity, so a debt it is owed stands. fn account() -> (cache::Pool, Arc) { - let pool = cache::Pool::unbounded(); + let pool = cache::Pool::new(cache::Config::default().with_capacity(1)); let track = cache::Track::new(pool.clone(), Default::default(), kio::Weak::new()); (pool, track) } @@ -585,4 +589,21 @@ mod test { assert_eq!(pool.used(), RUN_OVERHEAD); small.settle(|| track.clone(), &mut (), None).unwrap(); } + + /// Whatever a bill frees pays its readers' debt, however it shrank, and the debt is forgiven + /// once the pool is back within its capacity: nothing needs the room any more. + #[test] + fn a_debt_is_paid_by_any_shrink_and_forgiven_without_pressure() { + let pool = cache::Pool::new(cache::Config::default().with_capacity(2 * RUN_OVERHEAD)); + let track = cache::Track::new(pool.clone(), Default::default(), kio::Weak::new()); + let mut bill = track.reader(); + bill.set(4 * RUN_OVERHEAD); + owe(&track, 3 * RUN_OVERHEAD); + bill.set(3 * RUN_OVERHEAD); + assert_eq!(bill.debt(), 2 * RUN_OVERHEAD, "the shrink paid what it freed"); + bill.set(2 * RUN_OVERHEAD); + assert_eq!(bill.debt(), 0, "within its capacity, the pool forgives the rest"); + bill.set(5 * RUN_OVERHEAD); + assert_eq!(bill.debt(), 0, "a later pressure owes nothing of it"); + } } diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 120b565bace..2d06d5875aa 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -1514,7 +1514,12 @@ impl Reader { if ordered { sub.skip_ordered(floor); } - let res = sub.poll_peek_group(ordered, waiter); + // The copy's own record bills to its track as a direct reader's does, including + // what a peek skipped before it parked. + let mut res = sub.poll_peek_group(ordered, waiter); + if let Err(err) = sub.settle_record(None) { + res = Poll::Ready(Err(err)); + } match res { Poll::Pending => break, Poll::Ready(Ok(Some(reserved))) => { diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index c44c2d15ca0..9ce8de77cc5 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -4027,20 +4027,10 @@ impl Bill { self.0.set(bytes); } - /// What this bill holds. - pub(crate) fn size(&self) -> u64 { - self.0.size() - } - /// The debt the track left its readers, which this bill pays by shrinking. pub(crate) fn debt(&self) -> u64 { self.0.debt() } - - /// Pay up to `bytes` of that debt, as a bill that shrank by that much. - pub(crate) fn pay(&self, bytes: u64) { - self.0.pay(bytes); - } } impl std::fmt::Debug for Bill { @@ -9738,7 +9728,6 @@ impl Subscriber { fn hand_out(&mut self, sequence: u64, ordered: bool) { match &mut self.inner { Inner::Plain(cursor) => cursor.hand_out(sequence, ordered), - // A resumed reader retires its own stale reports. Inner::Resume(resume, logical) => { let delivered = &mut logical.record.delivered; if ordered { @@ -9746,6 +9735,8 @@ impl Subscriber { } delivered.insert(sequence); resume.forget_past_budget(delivered); + // The group won over any stale report its publisher made on it. + logical.record.retire(sequence); } } } @@ -9767,7 +9758,7 @@ impl Subscriber { /// Bill what this reader settled, and pay what it can of the debt its track left its /// readers; [`Error::Lagged`] once that needed a run it was promised. - fn settle_record(&mut self, handed: Option) -> Result<()> { + pub(super) fn settle_record(&mut self, handed: Option) -> Result<()> { match &mut self.inner { Inner::Plain(cursor) => { let state = &cursor.state; @@ -9791,6 +9782,12 @@ impl Subscriber { } } + /// Whether the pool ended this reader, needing a run it was promised. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn lagged(&self) -> bool { + self.record().failed() + } + /// What this reader settled, for its publisher to record what its wire was told. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn record_mut(&mut self) -> &mut Record { diff --git a/rs/moq-net/src/tail.rs b/rs/moq-net/src/tail.rs index de340dcbb1e..42930edfc6a 100644 --- a/rs/moq-net/src/tail.rs +++ b/rs/moq-net/src/tail.rs @@ -82,11 +82,9 @@ impl Tail { && self.finished.len() > 1 && let Some(newest) = self.finished.iter().next_back().cloned() { - let freed = bill.size() - FINISHED_OVERHEAD; self.finished = RangeSet::new(); self.finished.insert(newest); bill.set(FINISHED_OVERHEAD); - bill.pay(freed); } } From 3578dba368d649bbbe39bdce1f65668dcb101f8e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 03:51:00 -0800 Subject: [PATCH 16/37] fix(net): a JS routed reader's bill follows its serving route A resumed reader opened its bill once, on the copy serving it then, and kept charging that route's pool after a route change. Each settle now asks for the bill on the serving copy's account and moves the record there when the route changed, closing the replaced route's bill. Rust bills the logical track's account, which a route change keeps. Co-Authored-By: Claude Opus 5.5 --- js/net/src/cache.ts | 5 +++++ js/net/src/internal.ts | 14 ++++++++++---- js/net/src/resume.test.ts | 29 +++++++++++++++++++++++++++++ js/net/src/resume.ts | 11 ++++++----- js/net/src/track.ts | 25 +++++++++++++++---------- 5 files changed, 65 insertions(+), 19 deletions(-) diff --git a/js/net/src/cache.ts b/js/net/src/cache.ts index 350105f3e06..7a8c25f022d 100644 --- a/js/net/src/cache.ts +++ b/js/net/src/cache.ts @@ -558,6 +558,11 @@ export class Reader { return this.#bytes; } + /** Whether this bills `account`. */ + bills(account: unknown): boolean { + return this.#track === account; + } + /** * The debt the track's readers owe, forgiven once the pool is back within its capacity: * nothing needs the room any more, as the track forgives its own debt. diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 5922f74a5f1..b55fa9bc78c 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -490,8 +490,11 @@ export interface ResumeSource { * route change keeps, and those the serving copy reported. */ reofferStale(): Run[]; - /** Open a bill for the reader's record against the serving copy's account, undefined without one. */ - bill(): CacheReader | undefined; + /** + * The bill for the reader's record on the serving copy's account: `current` while it bills that + * account, or else a new one. `current` without a serving copy. + */ + bill(current: CacheReader | undefined): CacheReader | undefined; /** Let go of every copy. */ close(abort?: Error): void; } @@ -759,8 +762,11 @@ export const hooks: { * the pool can hand it debt. Mirrors the Rust crate-private `track::Producer::bill`. */ billTrack: (producer: Producer) => CacheReader; - /** Open a bill against the account of the track `subscriber` reads, undefined while it has none. */ - billSubscriber: (subscriber: Subscriber) => CacheReader | undefined; + /** + * The bill on the account of the track `subscriber` reads: `current` while it bills that account, + * or else a new one; undefined while it has none. + */ + billSubscriber: (subscriber: Subscriber, current: CacheReader | undefined) => CacheReader | undefined; /** * Re-offer what a subscriber gave up as stale through dispositions, as a wire whose every update * reopens its stale drops does: each run is judged again under the new terms. Returns the runs diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 4062653572a..2f8a5627ddb 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -1,6 +1,7 @@ import { expect, spyOn, test } from "bun:test"; import { Duration, Manual, yieldNow } from "@moq/clock"; import { Consumer as BroadcastConsumer, Producer as BroadcastProducer } from "./broadcast.ts"; +import * as cache from "./cache.ts"; import { NotFound, SessionCode, SessionError, StreamCode, StreamError } from "./error.ts"; import { type Consumer as GroupConsumer, Producer as GroupProducer } from "./group.ts"; import { WireRoute } from "./hop.ts"; @@ -3684,3 +3685,31 @@ test("a group an inner front hands out already failed still continues on the out a.close(); b.close(); }); + +// A routed reader's record bills the pool of the copy serving it, and follows a route change: the +// replaced route's pool stops carrying it, so the pool serving the reader is the one whose debt can +// fold it. +test("a routed reader's record bills the serving route's pool across a route change", async () => { + const { a, b, active, swap, close } = await setup(); + const pools = { a: new cache.Pool({ capacity: 1 << 30 }), b: new cache.Pool({ capacity: 1 << 30 }) }; + const ta = copy(a); + const tb = copy(b); + hooks.poolTrack(ta.producer, pools.a); + hooks.poolTrack(tb.producer, pools.b); + const sub = active.track("seq").subscribe({ delay: { max: null } }); + // Every other group, so the reader's record holds a run per gap. + const runs = 64n; + for (let group = 0n; group < runs; group++) ta.write(2n * group); + expect(drain(sub).length).toBe(Number(runs)); + + await swap(); + tb.write(2n * runs); + const before = { a: pools.a.used, b: pools.b.used }; + expect(drain(sub)).toEqual([2n * runs]); + const bill = Number(runs - 1n) * cache.READER_RUN_OVERHEAD; + expect(before.a - pools.a.used).toBeGreaterThanOrEqual(bill); + expect(pools.b.used - before.b).toBeGreaterThanOrEqual(bill); + + sub.close(); + close(); +}); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index d02160e8661..418e0d2e727 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -737,6 +737,12 @@ class Reader implements ResumeSource { return { groups: groups.map((group) => this.#handOut(serving, group)), givenUp }; } + /** The bill on the serving copy's account, whose pool holds what this reader reads. */ + bill(current: CacheReader | undefined): CacheReader | undefined { + const serving = this.#serving(); + return serving ? hooks.billSubscriber(serving.sub, current) : current; + } + /** * Take back every stale run: those queued here, those this reader reported, and those the * serving copy reported, which its cursor owes again. What this reader reported counts @@ -745,11 +751,6 @@ class Reader implements ResumeSource { * replaced copy may no longer deliver, so what it gave up is the new route's to deliver or * give up. Mirrors the Rust `Reader::reoffer_stale`. */ - bill(): CacheReader | undefined { - const serving = this.#serving(); - return serving && hooks.billSubscriber(serving.sub); - } - reofferStale(): Run[] { const taken = this.#reportedStale; this.#reportedStale = []; diff --git a/js/net/src/track.ts b/js/net/src/track.ts index e9a1e2f5b4d..3e3f1362aaa 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -3061,12 +3061,15 @@ export class Subscriber { }; } - // A bill against the account of the track this reads: a resumed reader's serving copy, as the - // Rust logical track's account bills its routes' readers. - #openBill(): cache.Reader | undefined { - if (this.#resume) return this.#resume.bill(); + // The bill on the account of the track this reads: `current` while it bills that account, or + // else a new one. A resumed reader's follows its serving copy across route changes, so the pool + // that holds what it reads is the one its debt comes from. + #billOn(current: cache.Reader | undefined): cache.Reader | undefined { + if (this.#resume) return this.#resume.bill(current); const producer = this.#state.producer; - return producer && accountOf(producer).reader(); + if (!producer) return current; + const account = accountOf(producer); + return current?.bills(account) ? current : account.reader(); } // How many runs the record holds above its floor, each billed `cache.READER_RUN_OVERHEAD`: @@ -3089,11 +3092,13 @@ export class Subscriber { #settleRecord(handed?: bigint): TooFarBehind | undefined { if (this.#lagged) return new TooFarBehind(); const bytes = this.#recordRuns() * cache.READER_RUN_OVERHEAD; - if (!this.#bill) { - if (bytes === 0) return undefined; - this.#bill = this.#openBill(); + if (!this.#bill && bytes === 0) return undefined; + const bill = this.#billOn(this.#bill); + if (bill !== this.#bill) { + // The serving route changed: the replaced route's pool stops carrying this record. + this.#bill?.close(); + this.#bill = bill; } - const bill = this.#bill; if (!bill) return undefined; bill.set(bytes); if (bill.debt === 0) return undefined; @@ -3817,7 +3822,7 @@ export class Subscriber { const producer = subscriber.#state.producer; return producer !== undefined && mayLand(producer, land); }; - hooks.billSubscriber = (subscriber) => subscriber.#openBill(); + hooks.billSubscriber = (subscriber, current) => subscriber.#billOn(current); hooks.resumeSubscriber = (name, subscription, source) => { const state = new TrackState(subscription ?? Subscription.default()); const subscriber = new Subscriber(name, state); From 6fec48938f70c0f6decb784384ccd2abfadf8d57 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 05:47:04 -0800 Subject: [PATCH 17/37] fix(net): fetched groups owe nothing for the feed, and re-offers keep final reports JS: a fill's answer no longer counts as a live-feed arrival, as Rust records arrivals only for visible groups, so a failed answer can't settle an end the feed still owes; one that fails rechecks the end, as Rust's poll_outcome does. Untelling a run keeps its final reports, so no stale verdict goes out over a skip. A resumed reader's update takes back the stale runs its logical record queued and has yet to report, as a plain cursor does. Co-Authored-By: Claude Opus 5.5 --- js/net/src/fill.test.ts | 23 ++++++++++++++++++ js/net/src/track.ts | 43 ++++++++++++++++++++-------------- rs/moq-net/src/model/record.rs | 31 +++++++++++++++++++++--- rs/moq-net/src/model/track.rs | 18 +++++++++++++- 4 files changed, 93 insertions(+), 22 deletions(-) diff --git a/js/net/src/fill.test.ts b/js/net/src/fill.test.ts index b3a3fc62390..45da7791ee6 100644 --- a/js/net/src/fill.test.ts +++ b/js/net/src/fill.test.ts @@ -232,6 +232,29 @@ test("fill: an answer aborted below the end does not hold the end open", async ( } }); +// A fill's answer is no feed arrival: one that fails leaves the gap the live feed still owes holding +// the end, as the Rust track records arrivals only for the groups its feed delivers. +test("fill: a failed answer to a gap the feed owes still holds the end", async () => { + const broadcast = new BroadcastProducer(); + const track = broadcast.createTrack("video", { maxAge: null }); + hooks.receiveTrack(track); + track.subscribe({ delay: { max: null } }); + for (const sequence of [0n, 2n]) hooks.receiveGroup(track, { sequence, start: 0 }).finish(); + const requests = track.requests(); + const fetching = wireOf(broadcast).fetchGroup("video", 1n); + const fill = await requests.requestedFill(); + if (!fill) throw new Error("no fill"); + fill.accept(1n).abort(new StreamError(StreamCode.Cancel)); + fill.close(); + await fetching.catch(() => undefined); + hooks.endTrack(track, { group: 3n, frame: 0 }); + await yieldNow(); + expect(track.closed.peek()).toBeUndefined(); + + requests.close(); + broadcast.close(); +}); + test("fill: a reader reaching back for history gets the misses filled in order", async () => { const { broadcast, track } = resumed(4n); const requests = track.requests(); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 3e3f1362aaa..cec3df4cf2a 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -1120,10 +1120,7 @@ export class Producer { name, accept: (sequence, info) => { if (info !== undefined && this.#state.info.peek() === undefined) this.accept(info); - const group = this.#createGroup(sequence, 0); - const entry = this.#cached.get(sequence); - if (entry) entry.fetched = true; - return group; + return this.#createGroup(sequence, 0, true); }, drop: (start, end, error) => { this.#drop(start, end, { error }); @@ -1327,12 +1324,7 @@ export class Producer { hooks.receiveTrack = (producer, starts = {}) => { producer.#wire = { starts, served: producer.#wire?.served }; }; - hooks.receiveGroup = (producer, { sequence, start, fill }) => { - const group = producer.#createGroup(sequence, start); - const entry = fill ? producer.#cached.get(sequence) : undefined; - if (entry) entry.fetched = true; - return group; - }; + hooks.receiveGroup = (producer, { sequence, start, fill }) => producer.#createGroup(sequence, start, fill); hooks.arrivingGroup = (producer, sequence) => { const closed = producer.#cached.get(sequence)?.group.closed; return closed?.peek() === undefined ? closed : undefined; @@ -1898,7 +1890,9 @@ export class Producer { // Retain a source group and fan it out to every live sink; its frame 0 is checked // against this track when it lands. - #publish(group: GroupProducer): void { + // `fetched` marks a group a fill answered, which the live feed never delivered: it accounts + // for nothing the feed owes, as the Rust `insert` records arrivals only for visible groups. + #publish(group: GroupProducer, fetched = false): void { // Before the group is cached, so the write creating it can never evict it. this.#chargeDebt(); this.#settle(); @@ -1907,6 +1901,7 @@ export class Producer { charge: this.#account.charge(), mirrors: new Map(), arrived: now(), + fetched: fetched || undefined, }; hooks.chargeGroup(group, entry.charge); this.#undrop(group.sequence); @@ -1931,8 +1926,8 @@ export class Producer { this.#cache.add(entry); this.#cached.set(group.sequence, entry); this.#fills.arrived(group.sequence); - this.#advance(group.sequence, true); - this.#arrive(group.sequence); + this.#advance(group.sequence, !fetched); + if (!fetched) this.#arrive(group.sequence); this.#produce(group.sequence); // A newer group demotes the live edge to an evictable one. const edge = this.#edgeEntry(); @@ -1952,6 +1947,9 @@ export class Producer { for (const sink of this.#sinks) if (sink.waitingOrdered) sink.timelineChanged.update((revision) => revision + 1); if (closed === null) this.#finishLargest(group.sequence); + // A failed fill answer holds no end open (see #settled), so the end may settle now, as the + // Rust `poll_outcome` judges again once any copy closes. + else if (fetched) this.#settle(); }); // Give held mirrors the new live edge before pruning their timeline entry, // so their latency guard can preserve a terminal expiry verdict. @@ -2130,7 +2128,7 @@ export class Producer { return this.#createGroup(sequence, 0); } - #createGroup(sequence: bigint, start: number): GroupProducer { + #createGroup(sequence: bigint, start: number, fetched = false): GroupProducer { this.#writable(sequence); // A skip says the sequence will never be produced, so it is final. A drop is not: a // publisher may re-create a group it cut or aborted. @@ -2150,7 +2148,7 @@ export class Producer { hooks.startGroup(group, start); this.#bind(group); this.#cap(group); - this.#publish(group); + this.#publish(group, fetched); return group; } @@ -2523,7 +2521,7 @@ export interface WireRecord { untold(start: bigint, end: bigint): Run[]; /** Record that the wire was told `report` of `[start, end)`. */ tell(start: bigint, end: bigint, report: Omit): void; - /** Forget what the wire was told of `[start, end)`, which is owed again. */ + /** Forget what the wire was told of `[start, end)` that an update may reopen, which is owed again. */ untell(start: bigint, end: bigint): void; /** Forget the drops of `[start, end)` the wire was told, which a verdict or group replaces. */ replaceDrops(start: bigint, end: bigint): void; @@ -3046,7 +3044,9 @@ export class Subscriber { } }, untell: (start, end) => { - reported.set(start, end); + // A final report stays: nothing reopens it, and a stale verdict must never go out over it. + const reopened = reported.overlapping(start, end).filter((run) => !finalReport(run)); + for (const run of reopened) reported.set(max(run.start, start), min(run.end, end)); }, replaceDrops: (start, end) => { const drops = reported @@ -3170,7 +3170,14 @@ export class Subscriber { // still stale under the new terms is given up and reported again. Returns the runs taken back. A // resumed reader takes back what it queued and what its serving route's copy reported. #reofferStale(): Run[] { - if (this.#resume) return this.#resume.reofferStale(); + if (this.#resume) { + const taken = this.#resume.reofferStale(); + // A stale run this reader queued and has yet to report is owed again too, as the plain + // path below takes back its own: the new terms judge it anew. Mirrors Rust. + for (const run of this.#pending) if (run.stale !== undefined) insertRun(taken, run.start, run.end); + this.#pending = this.#pending.filter((run) => run.stale === undefined); + return taken; + } const taken: Run[] = []; const received: Run[] = []; const stale = this.#reported.all.filter(reopens); diff --git a/rs/moq-net/src/model/record.rs b/rs/moq-net/src/model/record.rs index d3a0a164caa..791b7f896b9 100644 --- a/rs/moq-net/src/model/record.rs +++ b/rs/moq-net/src/model/record.rs @@ -246,10 +246,17 @@ impl Record { } } - /// Forget what the wire was told of `range`, which is owed again. + /// Forget what the wire was told of `range` that an update may reopen, which is owed again. pub fn untell(&mut self, range: Range) { - if !range.is_empty() { - self.reported.remove(range); + // A final report stays: nothing reopens it, and a stale verdict must never go out over it. + let reopened: Vec<_> = self + .reported + .overlapping(&range) + .filter(|(_, report)| !report.is_final()) + .map(|(run, _)| run.start.max(range.start)..run.end.min(range.end)) + .collect(); + for run in reopened { + self.reported.remove(run); } } @@ -606,4 +613,22 @@ mod test { bill.set(5 * RUN_OVERHEAD); assert_eq!(bill.debt(), 0, "a later pressure owes nothing of it"); } + + /// Owing a run again takes back only what an update may reopen: a final report inside it + /// stays, so no stale verdict goes out over it. + #[test] + fn untelling_a_run_keeps_its_final_reports() { + let stale = Report::Stale { + received: false, + expired: false, + taken: false, + }; + let mut record = Record::default(); + record.tell(0..2, stale); + record.tell(2..3, Report::Skip); + record.tell(3..5, stale); + record.untell(0..5); + assert!(!record.told(0) && !record.told(4), "the stale runs are owed again"); + assert!(record.told(2), "the skip stays"); + } } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 9ce8de77cc5..574e1a2ed8f 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -10169,7 +10169,23 @@ impl Subscriber { pub(crate) fn reoffer_stale(&mut self) -> Vec> { match &mut self.inner { Inner::Plain(cursor) => cursor.reoffer_stale(), - Inner::Resume(resume, _) => resume.reoffer_stale(), + Inner::Resume(resume, logical) => { + let mut runs: RangeSet = resume.reoffer_stale().into_iter().collect(); + // A stale run the logical record queued and has yet to report is owed again too, as + // a plain cursor takes back its own: the new terms judge it anew. + let queued: Vec<_> = logical + .record + .pending + .iter() + .filter(|(_, pending)| pending.hole.stale_budget().is_some()) + .map(|(run, _)| run.clone()) + .collect(); + for run in queued { + logical.record.pending.remove(run.clone()); + runs.insert(run); + } + runs.into_iter().collect() + } } } From 6dfa2ce4a9b620f33c9e58abc3ac2441c08d4cf2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 05:57:33 -0800 Subject: [PATCH 18/37] fix(net): a failed fill answer reaches no feed arrival through its drop Co-Authored-By: Claude Opus 5.5 --- js/net/src/fill.test.ts | 10 ++++++++-- js/net/src/track.ts | 9 +++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/js/net/src/fill.test.ts b/js/net/src/fill.test.ts index 45da7791ee6..3fb70a4f51c 100644 --- a/js/net/src/fill.test.ts +++ b/js/net/src/fill.test.ts @@ -235,6 +235,12 @@ test("fill: an answer aborted below the end does not hold the end open", async ( // A fill's answer is no feed arrival: one that fails leaves the gap the live feed still owes holding // the end, as the Rust track records arrivals only for the groups its feed delivers. test("fill: a failed answer to a gap the feed owes still holds the end", async () => { + for (const error of [new StreamError(StreamCode.Cancel), new Error("archive failed")]) { + await failedAnswerHoldsTheEnd(error); + } +}); + +async function failedAnswerHoldsTheEnd(error: Error) { const broadcast = new BroadcastProducer(); const track = broadcast.createTrack("video", { maxAge: null }); hooks.receiveTrack(track); @@ -244,7 +250,7 @@ test("fill: a failed answer to a gap the feed owes still holds the end", async ( const fetching = wireOf(broadcast).fetchGroup("video", 1n); const fill = await requests.requestedFill(); if (!fill) throw new Error("no fill"); - fill.accept(1n).abort(new StreamError(StreamCode.Cancel)); + fill.accept(1n).abort(error); fill.close(); await fetching.catch(() => undefined); hooks.endTrack(track, { group: 3n, frame: 0 }); @@ -253,7 +259,7 @@ test("fill: a failed answer to a gap the feed owes still holds the end", async ( requests.close(); broadcast.close(); -}); +} test("fill: a reader reaching back for history gets the misses filled in order", async () => { const { broadcast, track } = resumed(4n); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index cec3df4cf2a..5f3beca90ed 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -1471,7 +1471,7 @@ export class Producer { // also ends the groups still arriving in it with its error, so their readers settle now // rather than once the stream's reset lands; one that finished here stays whole. A skip names // nothing produced. - #drop(start: bigint, end: bigint, hole: Hole, open = false): Run[] { + #drop(start: bigint, end: bigint, hole: Hole, open = false, arrives = true): Run[] { if (start >= end) return []; const skipped = hole.skip ? [] : this.#state.drops.overlapping(start, end).filter((run) => run.skip); let runs = runGaps(skipped, start, end); @@ -1497,8 +1497,9 @@ export class Producer { } } // A pending tail owes nothing the publisher gave up. A stale verdict is not receipt: new - // terms may supersede it, leaving those groups unknown again, as in Rust. - if (hole.stale === undefined) this.#arrive(start, end); + // terms may supersede it, leaving those groups unknown again, as in Rust. Nor is a failed + // fill answer (`arrives` unset), which the feed never delivered. + if (hole.stale === undefined && arrives) this.#arrive(start, end); // Giving up the last sequences below the end reaches it, as writing them would. this.#endLargest(); return runs; @@ -1912,7 +1913,7 @@ export class Producer { // A re-created group already replaced this one. const current = this.#cached.get(group.sequence); if (current && current.group !== group) return; - this.#drop(group.sequence, group.sequence + 1n, { error: closed }); + this.#drop(group.sequence, group.sequence + 1n, { error: closed }, false, !fetched); }); const sequence = group.sequence; if (this.#unchecked()) { From 1e299eb09be77396e79c21b832f5fadff1864404 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 08:07:41 -0800 Subject: [PATCH 19/37] fix(net): a reclaimed failed fill answer accounts nothing for the feed JS marks a failed fill answer's drop as a fill's, which neither its drop nor a later reclaim of it, by expiry or debt, accounts as a feed arrival: the gap stays owed until the feed delivers it or its grace passes. Rust records no drop for a failed fill answer. Tests cover that, and a resumed reader taking back the stale runs it queued, in Rust and JS. Co-Authored-By: Claude Opus 5.5 --- js/net/src/cache.test.ts | 21 ++++++++++++++++ js/net/src/internal.ts | 5 +++- js/net/src/resume.test.ts | 44 ++++++++++++++++++++++++++++++++++ js/net/src/track.ts | 25 +++++++++++++------ rs/moq-net/src/model/resume.rs | 38 +++++++++++++++++++++++++++++ 5 files changed, 125 insertions(+), 8 deletions(-) diff --git a/js/net/src/cache.test.ts b/js/net/src/cache.test.ts index 368bc957aa1..38125f91f51 100644 --- a/js/net/src/cache.test.ts +++ b/js/net/src/cache.test.ts @@ -591,6 +591,27 @@ test("a forgotten stale run is asked of a fill", async () => { }); }); +// A failed fill answer's drop accounts nothing the feed owes, even once the pool forgets it: the gap +// stays owed until the feed delivers it or the tail's grace passes, as Rust records no drop for it. +test("a forgotten failed fill answer leaves its gap owed", async () => { + await withClock(async (clock) => { + const pool = new cache.Pool({ expiry: Duration.millis(1_000) }); + const producer = pooled(pool); + hooks.receiveTrack(producer); + const reader = producer.subscribe({ delay: { max: Duration.millis(60_000) } }); + for (const sequence of [0n, 2n]) hooks.receiveGroup(producer, { sequence, start: 0 }).finish(); + hooks.receiveGroup(producer, { sequence: 1n, start: 0, fill: true }).abort(new Error("archive failed")); + hooks.endTrack(producer, { group: 3n, frame: 0 }); + await yieldNow(); + expect(dropsHeld(producer).length).toBe(1); + + await clock.advance(3_000); + expect(dropsHeld(producer)).toEqual([]); + expect(producer.closed.peek()).toBeUndefined(); + reader.close(); + }); +}); + // A received copy that reached its end through a skip stays ended once the pool forgets the skip, // since the forgotten run stays accounted for. Mirrors Rust // `a_reached_end_outlives_its_expired_drops`. diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index b55fa9bc78c..f52a42e55dd 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -226,6 +226,8 @@ export function staleHole(budget: number): Hole { export interface DropRun extends Run, Hole { /** Copied into a cursor from the track's runs, which it folds in afresh on each change. */ received?: boolean; + /** A failed fill answer, which the live feed never accounted for: reclaiming it accounts nothing. */ + fill?: boolean; } /** Whether two runs say the same thing, so they may merge. */ @@ -234,7 +236,8 @@ export function sameHole(a: DropRun, b: DropRun): boolean { toStreamCode(a.error) === toStreamCode(b.error) && !a.skip === !b.skip && a.stale === b.stale && - !a.received === !b.received + !a.received === !b.received && + !a.fill === !b.fill ); } diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 2f8a5627ddb..be37d471d92 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -3594,6 +3594,50 @@ test("a resumed reader takes back its stale reports", () => { a.close(); }); +// A stale run the reader queued and has yet to report is taken back by an update too, as a plain +// reader's is: the new terms judge it anew, rather than an old verdict going out under the new +// update number. The update widens past the verdict, so nothing goes out at all. Mirrors Rust +// `a_resumed_reader_takes_back_its_queued_stale_runs`, which checks the queue directly. +test("a resumed reader takes back its queued stale runs", async () => { + const clock = new Manual(); + const installed = clock.install(); + try { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const ta = copy(a); + const tb = copy(b); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ delay: { max: Duration.millis(1_000) } }); + ta.write(0n); + expect(drain(sub)).toEqual([0n]); + hooks.dropTrackGroups(ta.producer, 2n, 3n, staleHole(1_000)); + expect(hooks.tryDrop(sub)).toMatchObject({ kind: "drop", start: 2n, end: 3n }); + + // The new route's stale run around the one already reported goes out in parts once the old + // route can no longer deliver it: the part below it first, with the rest queued. + front.serve(b.consume()); + await settle(); + hooks.dropTrackGroups(tb.producer, 1n, 5n, staleHole(1_000)); + tb.write(5n); + expect(drain(sub)).toEqual([5n]); + await clock.advance(Duration.millis(1_000)); + expect(hooks.tryDrop(sub)).toMatchObject({ kind: "drop", start: 1n, end: 2n }); + + // An update that widens the budget past the verdict: nothing is stale under it. + sub.update({ delay: { max: Duration.millis(60_000) } }); + expect(hooks.reofferStale(sub)).toEqual([{ start: 1n, end: 5n }]); + await settle(); + expect(hooks.tryDrop(sub)).toBeUndefined(); + sub.close(); + front.consumer.close(); + a.close(); + b.close(); + } finally { + installed.close(); + } +}); + // A stale run one route reported is still taken back once another serves: the update reopened it // downstream, and the new route's copy knows nothing of it, so the groups it delivers there // afterwards are owed rather than kept reported. Mirrors Rust. diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 5f3beca90ed..4cff5e13590 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -1157,7 +1157,7 @@ export class Producer { // Give up `[start, end)` as `hole` says, or forget it without one, billing the drop set to the // track's account and dating each run against the pool's idle window. Mirrors the Rust // `TrackState::change_drops`. - #setDrops(start: bigint, end: bigint, hole?: Hole): void { + #setDrops(start: bigint, end: bigint, hole?: Hole & { fill?: boolean }): void { this.#state.drops.set(start, end, hole); const pool = this.#account.pool; const window = pool.expiryTicks; @@ -1193,7 +1193,7 @@ export class Producer { const billed = this.#dropsBilled; const end = min(run.end, kept); this.#setDrops(run.start, end); - this.#forgot([{ start: run.start, end }]); + if (!run.fill) this.#forgot([{ start: run.start, end }]); paid += billed - this.#dropsBilled; } return paid; @@ -1239,10 +1239,17 @@ export class Producer { .map(({ start, end }) => ({ start, end: min(end, kept) })); // In one pass: runs given up in the same quarter of the window expire together. if (expired.length > 0) { + // A failed fill answer's run accounts nothing: the feed never delivered it. + const accounted = expired.flatMap(({ start, end }) => + this.#state.drops + .overlapping(start, end) + .filter((run) => !run.fill) + .map((run) => ({ start: max(run.start, start), end: min(run.end, end) })), + ); this.#state.drops.clear(expired); this.#dropsBorn.clear(expired); this.#billDrops(); - this.#forgot(expired); + this.#forgot(accounted); } this.#dropsOldest = undefined; for (const run of this.#dropsBorn.all) { @@ -1471,7 +1478,7 @@ export class Producer { // also ends the groups still arriving in it with its error, so their readers settle now // rather than once the stream's reset lands; one that finished here stays whole. A skip names // nothing produced. - #drop(start: bigint, end: bigint, hole: Hole, open = false, arrives = true): Run[] { + #drop(start: bigint, end: bigint, hole: Hole & { fill?: boolean }, open = false): Run[] { if (start >= end) return []; const skipped = hole.skip ? [] : this.#state.drops.overlapping(start, end).filter((run) => run.skip); let runs = runGaps(skipped, start, end); @@ -1498,8 +1505,8 @@ export class Producer { } // A pending tail owes nothing the publisher gave up. A stale verdict is not receipt: new // terms may supersede it, leaving those groups unknown again, as in Rust. Nor is a failed - // fill answer (`arrives` unset), which the feed never delivered. - if (hole.stale === undefined && arrives) this.#arrive(start, end); + // fill answer (`fill`), which the feed never delivered. + if (hole.stale === undefined && !hole.fill) this.#arrive(start, end); // Giving up the last sequences below the end reaches it, as writing them would. this.#endLargest(); return runs; @@ -1913,7 +1920,11 @@ export class Producer { // A re-created group already replaced this one. const current = this.#cached.get(group.sequence); if (current && current.group !== group) return; - this.#drop(group.sequence, group.sequence + 1n, { error: closed }, false, !fetched); + this.#drop( + group.sequence, + group.sequence + 1n, + fetched ? { error: closed, fill: true } : { error: closed }, + ); }); const sequence = group.sequence; if (this.#unchecked()) { diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 2d06d5875aa..aee3619e1f3 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -3932,6 +3932,44 @@ mod test { assert_eq!(sub.reoffer_stale(), Vec::>::new()); } + /// A stale run the logical record queued and has yet to report is taken back by an update + /// too, as a plain cursor's is: the new terms judge it anew, rather than an old verdict going + /// out under the new update number. + #[kio::test] + async fn a_resumed_reader_takes_back_its_queued_stale_runs() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let budget = Duration::from_secs(1); + let mut sub = subscribe(&logical, budget); + write(&a, 0); + assert_eq!(drain(&mut sub, false), [0]); + a.give_up(2..3, track::Hole::stale(budget)); + assert_eq!(dropped(&mut sub), [(2..3, StreamError::Old)]); + + // The new route's stale run around the one already reported goes out in parts once the + // old route can no longer deliver it: the part below it first, with the rest queued. + let b = copy(); + routes.serve(b.consume()); + b.give_up(1..5, track::Hole::stale(budget)); + write(&b, 5); + assert_eq!(drain(&mut sub, false), [5]); + kio::clock::sleep(budget).await; + let Poll::Ready((first, _)) = sub.poll_drop(&kio::Waiter::noop()) else { + panic!("the new route's stale run goes out"); + }; + assert_eq!(first, 1..2); + assert!(sub.record_mut().pending.contains_key(&3), "the rest waits its turn"); + + let taken: RangeSet = sub.reoffer_stale().into_iter().collect(); + assert!(taken.contains(&3) && taken.contains(&4), "{taken:?}"); + assert!( + sub.record_mut().pending.is_empty(), + "nothing queued goes out under the new terms" + ); + } + /// A stale run one route reported is still taken back once another serves: the update /// reopened it downstream, and the new route's copy knows nothing of it, so the groups it /// delivers there afterwards are owed rather than kept reported. From 6d0c224ada83a324573a7233f2109ca0aed7198c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 10:19:46 -0800 Subject: [PATCH 20/37] fix(net): a failed fill answer is no verdict and is served to no reader A failed fetched group recorded a drop and stayed buffered in every sink, so `#landing` took it for the feed's verdict and an ordered reader was handed the failed copy and moved past a group the feed still owed. JS now mirrors Rust: a failed fill answer records no drop and no arrival, and leaves the cache and every sink at once, as Rust serves no failed fetch-only slot. With nothing recorded, every drop-set reader is correct by construction, so the `fill` marker on drop runs goes away. Co-Authored-By: Claude Opus 5.5 --- js/net/src/cache.test.ts | 4 +++- js/net/src/fill.test.ts | 18 +++++++++++++--- js/net/src/internal.ts | 5 +---- js/net/src/track.ts | 40 +++++++++++++++++------------------ rs/moq-net/src/model/track.rs | 34 +++++++++++++++++++++++++++++ 5 files changed, 73 insertions(+), 28 deletions(-) diff --git a/js/net/src/cache.test.ts b/js/net/src/cache.test.ts index 38125f91f51..69f92e7676e 100644 --- a/js/net/src/cache.test.ts +++ b/js/net/src/cache.test.ts @@ -603,7 +603,9 @@ test("a forgotten failed fill answer leaves its gap owed", async () => { hooks.receiveGroup(producer, { sequence: 1n, start: 0, fill: true }).abort(new Error("archive failed")); hooks.endTrack(producer, { group: 3n, frame: 0 }); await yieldNow(); - expect(dropsHeld(producer).length).toBe(1); + // A failed fill answer says nothing of the feed, so it records no drop, as in Rust. + expect(dropsHeld(producer)).toEqual([]); + expect(producer.closed.peek()).toBeUndefined(); await clock.advance(3_000); expect(dropsHeld(producer)).toEqual([]); diff --git a/js/net/src/fill.test.ts b/js/net/src/fill.test.ts index 3fb70a4f51c..7591991de4c 100644 --- a/js/net/src/fill.test.ts +++ b/js/net/src/fill.test.ts @@ -233,8 +233,9 @@ test("fill: an answer aborted below the end does not hold the end open", async ( }); // A fill's answer is no feed arrival: one that fails leaves the gap the live feed still owes holding -// the end, as the Rust track records arrivals only for the groups its feed delivers. -test("fill: a failed answer to a gap the feed owes still holds the end", async () => { +// the end and the ordered reader, as the Rust track records arrivals only for the groups its feed +// delivers and serves no failed fetch-only slot. Mirrors Rust `a_failed_fill_answer_holds_an_ordered_reader`. +test("fill: a failed answer to a gap the feed owes holds the end and the ordered reader", async () => { for (const error of [new StreamError(StreamCode.Cancel), new Error("archive failed")]) { await failedAnswerHoldsTheEnd(error); } @@ -244,8 +245,10 @@ async function failedAnswerHoldsTheEnd(error: Error) { const broadcast = new BroadcastProducer(); const track = broadcast.createTrack("video", { maxAge: null }); hooks.receiveTrack(track); - track.subscribe({ delay: { max: null } }); + const reader = track.subscribe({ delay: { max: null } }); + const ordered = reader.ordered(); for (const sequence of [0n, 2n]) hooks.receiveGroup(track, { sequence, start: 0 }).finish(); + expect((await ordered.nextGroup())?.sequence).toBe(0n); const requests = track.requests(); const fetching = wireOf(broadcast).fetchGroup("video", 1n); const fill = await requests.requestedFill(); @@ -257,6 +260,15 @@ async function failedAnswerHoldsTheEnd(error: Error) { await yieldNow(); expect(track.closed.peek()).toBeUndefined(); + // The failed answer is no verdict on group 1, so an ordered reader still waits for it rather + // than moving on to group 2, and gets it once the feed delivers it. + const next = ordered.nextGroup(); + expect(await Promise.race([next.then(() => "moved"), yieldNow().then(() => "waiting")])).toBe("waiting"); + hooks.receiveGroup(track, { sequence: 1n, start: 0 }).finish(); + expect((await next)?.sequence).toBe(1n); + expect((await ordered.nextGroup())?.sequence).toBe(2n); + + reader.close(); requests.close(); broadcast.close(); } diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index f52a42e55dd..b55fa9bc78c 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -226,8 +226,6 @@ export function staleHole(budget: number): Hole { export interface DropRun extends Run, Hole { /** Copied into a cursor from the track's runs, which it folds in afresh on each change. */ received?: boolean; - /** A failed fill answer, which the live feed never accounted for: reclaiming it accounts nothing. */ - fill?: boolean; } /** Whether two runs say the same thing, so they may merge. */ @@ -236,8 +234,7 @@ export function sameHole(a: DropRun, b: DropRun): boolean { toStreamCode(a.error) === toStreamCode(b.error) && !a.skip === !b.skip && a.stale === b.stale && - !a.received === !b.received && - !a.fill === !b.fill + !a.received === !b.received ); } diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 4cff5e13590..edfac92dfcb 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -1157,7 +1157,7 @@ export class Producer { // Give up `[start, end)` as `hole` says, or forget it without one, billing the drop set to the // track's account and dating each run against the pool's idle window. Mirrors the Rust // `TrackState::change_drops`. - #setDrops(start: bigint, end: bigint, hole?: Hole & { fill?: boolean }): void { + #setDrops(start: bigint, end: bigint, hole?: Hole): void { this.#state.drops.set(start, end, hole); const pool = this.#account.pool; const window = pool.expiryTicks; @@ -1193,7 +1193,7 @@ export class Producer { const billed = this.#dropsBilled; const end = min(run.end, kept); this.#setDrops(run.start, end); - if (!run.fill) this.#forgot([{ start: run.start, end }]); + this.#forgot([{ start: run.start, end }]); paid += billed - this.#dropsBilled; } return paid; @@ -1239,17 +1239,10 @@ export class Producer { .map(({ start, end }) => ({ start, end: min(end, kept) })); // In one pass: runs given up in the same quarter of the window expire together. if (expired.length > 0) { - // A failed fill answer's run accounts nothing: the feed never delivered it. - const accounted = expired.flatMap(({ start, end }) => - this.#state.drops - .overlapping(start, end) - .filter((run) => !run.fill) - .map((run) => ({ start: max(run.start, start), end: min(run.end, end) })), - ); this.#state.drops.clear(expired); this.#dropsBorn.clear(expired); this.#billDrops(); - this.#forgot(accounted); + this.#forgot(expired); } this.#dropsOldest = undefined; for (const run of this.#dropsBorn.all) { @@ -1478,7 +1471,7 @@ export class Producer { // also ends the groups still arriving in it with its error, so their readers settle now // rather than once the stream's reset lands; one that finished here stays whole. A skip names // nothing produced. - #drop(start: bigint, end: bigint, hole: Hole & { fill?: boolean }, open = false): Run[] { + #drop(start: bigint, end: bigint, hole: Hole, open = false): Run[] { if (start >= end) return []; const skipped = hole.skip ? [] : this.#state.drops.overlapping(start, end).filter((run) => run.skip); let runs = runGaps(skipped, start, end); @@ -1504,9 +1497,8 @@ export class Producer { } } // A pending tail owes nothing the publisher gave up. A stale verdict is not receipt: new - // terms may supersede it, leaving those groups unknown again, as in Rust. Nor is a failed - // fill answer (`fill`), which the feed never delivered. - if (hole.stale === undefined && !hole.fill) this.#arrive(start, end); + // terms may supersede it, leaving those groups unknown again, as in Rust. + if (hole.stale === undefined) this.#arrive(start, end); // Giving up the last sequences below the end reaches it, as writing them would. this.#endLargest(); return runs; @@ -1916,15 +1908,23 @@ export class Producer { group.closed.then((closed) => { // An aborted live edge is no longer protected, so it may expire like any other. if (closed instanceof Error) this.#account.demote(entry); - if (!(closed instanceof Error) || HOLES.has(toStreamCode(closed))) return; + if (!(closed instanceof Error)) return; // A re-created group already replaced this one. const current = this.#cached.get(group.sequence); if (current && current.group !== group) return; - this.#drop( - group.sequence, - group.sequence + 1n, - fetched ? { error: closed, fill: true } : { error: closed }, - ); + // A failed fill answer says nothing of the feed, so it gives nothing up: the track's drops + // and arrivals hold only the feed's verdicts, as Rust records nothing for a failed fill. + // No reader is handed it either, as Rust serves no failed fetch-only slot, so its sequence + // is unknown again: owed while the feed owes it, else judged by budget or filled again. + if (fetched) { + if (current) { + this.#evict(entry); + this.#uncache(entry); + } + return; + } + if (HOLES.has(toStreamCode(closed))) return; + this.#drop(group.sequence, group.sequence + 1n, { error: closed }); }); const sequence = group.sequence; if (this.#unchecked()) { diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 574e1a2ed8f..36be4418eef 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -14948,6 +14948,40 @@ mod test { groups.assert_no_group(); } + /// A failed fill answer is no verdict on a gap the feed owes: an ordered reader neither takes + /// the failed copy nor moves past it, and gets the feed's group once it lands. Mirrors JS + /// `fill: a failed answer to a gap the feed owes holds the end and the ordered reader`. + #[test] + #[cfg(feature = "async")] + fn a_failed_fill_answer_holds_an_ordered_reader() { + for error in [Error::Cancel, Error::App(1)] { + let producer = received_producer(None); + let dynamic = producer.requests(); + let consumer = producer.consume(); + let mut ordered = producer.subscribe(replay()).ordered(); + put(&producer, 0, None); + put(&producer, 2, None); + assert_eq!(drain_ordered(&mut ordered), [0]); + + let pending = consumer.fetch_group(1, None); + let req = dynamic + .requested_fill_group() + .now_or_never() + .expect("fill request is ready") + .unwrap(); + req.accept(None).unwrap().abort(error.clone()).unwrap(); + drop(pending); + assert!(drain_ordered(&mut ordered).is_empty(), "{error}: group 1 is still owed"); + + put(&producer, 1, None); + assert_eq!( + drain_ordered(&mut ordered), + [1, 2], + "{error}: the feed's group lands in order" + ); + } + } + /// The live edge is resolved once per poll and applied to every group that poll /// walks off, so it has to be revalidated before it convicts anything: an eviction /// in between would otherwise discard a group on the strength of content that is no From 846cc1c450ebab516249e78fba227398d2c62118 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 10:29:04 -0800 Subject: [PATCH 21/37] fix(net): a failed fill answer leaves held readers their frames and failure Taking a failed fill answer out of the cache through `#evict` also evicted mirrors a reader already held, clearing their buffered frames and replacing the fill's failure with an eviction. It now only unlinks the entry, releasing the mirrors still unread in a sink, so a held reader keeps what it buffered. Co-Authored-By: Claude Opus 5.5 --- js/net/src/fill.test.ts | 24 ++++++++++++++++++++++++ js/net/src/track.ts | 10 ++++++---- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/js/net/src/fill.test.ts b/js/net/src/fill.test.ts index 7591991de4c..73ca82c984d 100644 --- a/js/net/src/fill.test.ts +++ b/js/net/src/fill.test.ts @@ -273,6 +273,30 @@ async function failedAnswerHoldsTheEnd(error: Error) { broadcast.close(); } +// A failed fill answer leaves the cache, but a reader already holding it keeps what it buffered and +// sees the fill's own failure, never a cache eviction. +test("fill: a held answer that fails keeps its frames and its failure", async () => { + const broadcast = new BroadcastProducer(); + const track = broadcast.createTrack("video", { maxAge: null }); + const requests = track.requests(); + const fetching = wireOf(broadcast).fetchGroup("video", 1n); + const fill = await requests.requestedFill(); + if (!fill) throw new Error("no fill"); + const answer = fill.accept(1n); + answer.writeFrame({ payload: enc.encode("fill 1") }); + const group = await fetching; + const failure = new Error("archive failed"); + answer.abort(failure); + fill.close(); + await yieldNow(); + + expect(dec.decode((await group.readFrame())?.payload)).toBe("fill 1"); + await expect(group.readFrame()).rejects.toBe(failure); + + requests.close(); + broadcast.close(); +}); + test("fill: a reader reaching back for history gets the misses filled in order", async () => { const { broadcast, track } = resumed(4n); const requests = track.requests(); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index edfac92dfcb..7151011bd2d 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -1916,11 +1916,13 @@ export class Producer { // and arrivals hold only the feed's verdicts, as Rust records nothing for a failed fill. // No reader is handed it either, as Rust serves no failed fetch-only slot, so its sequence // is unknown again: owed while the feed owes it, else judged by budget or filled again. + // A reader already holding it keeps its frames and the fill's own failure. if (fetched) { - if (current) { - this.#evict(entry); - this.#uncache(entry); - } + if (!current) return; + const unread = [...entry.mirrors].filter(([sink, mirror]) => sink.groups.peek().includes(mirror)); + this.#unlink(entry); + this.#uncache(entry); + for (const [, mirror] of unread) hooks.releaseGroupReader(mirror, undefined); return; } if (HOLES.has(toStreamCode(closed))) return; From c2f8e816d4cd21f4751bc1e60b3203af083b6324 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 10:38:31 -0800 Subject: [PATCH 22/37] fix(net): a failed fill answer stays cached and billed, withheld from delivery Uncaching a failed fill answer cleared its whole charge while a reader still held its frames, so retained buffers escaped the pool. As Rust keeps a spent fetch-only slot until its scan reclaims it, the entry now stays cached, billed and reclaimable: `#withhold` takes it out of every sink that has not read it, and `#mirror` gives it to no new sink. Co-Authored-By: Claude Opus 5.5 --- js/net/src/cache.test.ts | 19 +++++++++++++++++++ js/net/src/track.ts | 31 +++++++++++++++++++++++-------- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/js/net/src/cache.test.ts b/js/net/src/cache.test.ts index 69f92e7676e..ed552b852c0 100644 --- a/js/net/src/cache.test.ts +++ b/js/net/src/cache.test.ts @@ -614,6 +614,25 @@ test("a forgotten failed fill answer leaves its gap owed", async () => { }); }); +// A failed fill answer a reader still holds stays billed, and reclaimable by the pool, while the +// reader keeps its frames: it leaves delivery, not the cache, as Rust keeps a spent fetch-only slot +// until its scan reclaims it. +test("a held failed fill answer stays billed", async () => { + const pool = new cache.Pool({}); + const producer = pooled(pool); + hooks.receiveTrack(producer); + const reader = producer.subscribe({ delay: { max: Duration.millis(60_000) } }); + const answer = hooks.receiveGroup(producer, { sequence: 1n, start: 0, fill: true }); + answer.writeFrame({ payload: new Uint8Array(10_000) }); + const group = await reader.recvGroup(); + expect(group?.sequence).toBe(1n); + answer.abort(new Error("archive failed")); + await yieldNow(); + expect(pool.used).toBeGreaterThanOrEqual(10_000); + expect((await group?.readFrame())?.payload.byteLength).toBe(10_000); + reader.close(); +}); + // A received copy that reached its end through a skip stays ended once the pool forgets the skip, // since the forgotten run stays accounted for. Mirrors Rust // `a_reached_end_outlives_its_expired_drops`. diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 7151011bd2d..b6650f36882 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -1721,6 +1721,7 @@ export class Producer { // source is written and keeps its own read cursor; frame bytes are shared by // reference. Tracked on the entry so eviction can drop it from the sink. #mirror(entry: CachedGroup, sink: TrackState): void { + if (entry.fetched && entry.group.closed.peek() instanceof Error) return; const dst = entry.group.mirror(); hooks.arriveGroup(dst, entry.arrived); if (entry.unverified) hooks.unverifyGroup(dst); @@ -1752,6 +1753,25 @@ export class Producer { for (const mirror of mirrors) mirror.close(); } + // Take a failed fill answer out of every sink that has not read it, as Rust serves no failed + // fetch-only slot; #mirror gives it to no new sink either. The entry stays cached, billed and + // reclaimable, as Rust keeps a spent slot until its scan reclaims it, so a reader already + // holding it keeps its frames and the fill's own failure while the pool still counts them. + #withhold(entry: CachedGroup): void { + for (const [sink, mirror] of entry.mirrors) { + const index = sink.groups.peek().indexOf(mirror); + if (index < 0) continue; + sink.groups.mutate((groups) => { + groups.splice(index, 1); + }); + const at = timelineIndex(sink.timeline, mirror.sequence); + if (sink.timeline[at] === mirror) sink.timeline.splice(at, 1); + entry.mirrors.delete(sink); + entry.charge.sub(cache.MIRROR_OVERHEAD); + hooks.releaseGroupReader(mirror, undefined); + } + } + // Take a cached group's mirrors out of every sink, so a subscriber can no longer // receive them. A reader already holding one keeps it as is. #unlink(entry: CachedGroup): void { @@ -1914,15 +1934,10 @@ export class Producer { if (current && current.group !== group) return; // A failed fill answer says nothing of the feed, so it gives nothing up: the track's drops // and arrivals hold only the feed's verdicts, as Rust records nothing for a failed fill. - // No reader is handed it either, as Rust serves no failed fetch-only slot, so its sequence - // is unknown again: owed while the feed owes it, else judged by budget or filled again. - // A reader already holding it keeps its frames and the fill's own failure. + // No reader is handed it either (see #withhold), so its sequence is unknown again: owed + // while the feed owes it, else judged by budget or filled again. if (fetched) { - if (!current) return; - const unread = [...entry.mirrors].filter(([sink, mirror]) => sink.groups.peek().includes(mirror)); - this.#unlink(entry); - this.#uncache(entry); - for (const [, mirror] of unread) hooks.releaseGroupReader(mirror, undefined); + if (current) this.#withhold(entry); return; } if (HOLES.has(toStreamCode(closed))) return; From 7f97d9e9968a7cbe22e46109b4273ef202f281a1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 18:58:57 -0800 Subject: [PATCH 23/37] test(net): cover the late-subscriber guard and assert a pending ordered read Co-Authored-By: Claude Opus 5.5 --- js/net/src/fill.test.ts | 9 +++++++++ rs/moq-net/src/model/track.rs | 6 +++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/js/net/src/fill.test.ts b/js/net/src/fill.test.ts index 73ca82c984d..a4548f821bf 100644 --- a/js/net/src/fill.test.ts +++ b/js/net/src/fill.test.ts @@ -260,6 +260,13 @@ async function failedAnswerHoldsTheEnd(error: Error) { await yieldNow(); expect(track.closed.peek()).toBeUndefined(); + // A subscriber arriving after the failure is not handed the failed answer either. + const late = track.subscribe({ delay: { max: null } }); + const lateOrdered = late.ordered(); + expect((await lateOrdered.nextGroup())?.sequence).toBe(0n); + const lateNext = lateOrdered.nextGroup(); + expect(await Promise.race([lateNext.then(() => "moved"), yieldNow().then(() => "waiting")])).toBe("waiting"); + // The failed answer is no verdict on group 1, so an ordered reader still waits for it rather // than moving on to group 2, and gets it once the feed delivers it. const next = ordered.nextGroup(); @@ -267,8 +274,10 @@ async function failedAnswerHoldsTheEnd(error: Error) { hooks.receiveGroup(track, { sequence: 1n, start: 0 }).finish(); expect((await next)?.sequence).toBe(1n); expect((await ordered.nextGroup())?.sequence).toBe(2n); + expect((await lateNext)?.sequence).toBe(1n); reader.close(); + late.close(); requests.close(); broadcast.close(); } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 36be4418eef..f7db4d93e79 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -14971,7 +14971,11 @@ mod test { .unwrap(); req.accept(None).unwrap().abort(error.clone()).unwrap(); drop(pending); - assert!(drain_ordered(&mut ordered).is_empty(), "{error}: group 1 is still owed"); + let waiter = kio::Waiter::noop(); + assert!( + ordered.poll_next_group(&waiter).is_pending(), + "{error}: group 1 is still owed" + ); put(&producer, 1, None); assert_eq!( From 78c7d4426a11cfe381f9f40d16a6f8638a645ec2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 19:15:54 -0800 Subject: [PATCH 24/37] fix(net): a fill's budget refusal is a stale verdict a widening reopens JS no longer spends the one eviction refill on it, and Rust keeps its budget instead of giving the run up for good. Co-Authored-By: Claude Opus 5.5 --- js/net/src/fill.test.ts | 28 ++++++++++++++++++++++++ js/net/src/track.ts | 11 ++++++++-- rs/moq-net/src/model/track.rs | 34 ++++++++++++++++++++++++++++++ rs/moq-net/src/model/track/miss.rs | 3 +++ 4 files changed, 74 insertions(+), 2 deletions(-) diff --git a/js/net/src/fill.test.ts b/js/net/src/fill.test.ts index a4548f821bf..452378d376f 100644 --- a/js/net/src/fill.test.ts +++ b/js/net/src/fill.test.ts @@ -4,6 +4,7 @@ import { Producer as BroadcastProducer } from "./broadcast.ts"; import { accept as acceptSession, connect as connectSession } from "./connection/index.ts"; import { NotFound, StreamCode, StreamError, toStreamCode } from "./error.ts"; import type { Consumer as GroupConsumer } from "./group.ts"; +import { rejectGroups } from "./fill.ts"; import { hooks, staleHole } from "./internal.ts"; import * as Lite from "./lite/index.ts"; import { createMockTransportPair } from "./mock.ts"; @@ -753,6 +754,33 @@ test("fill: a group no reader reads any more stays withdrawn, and widening back broadcast.close(); }); +test("fill: a refusal under an old budget is asked afresh on each widening, never spending the eviction refill", async () => { + const { broadcast, track } = resumed(5n); + const requests = track.requests(); + const terms = (max: number): Subscription => ({ + ranges: [{ start: { absolute: 0n }, end: { absolute: 5n } }], + delay: { max: Duration.millis(max) }, + order: "ascending", + }); + const reader = track.subscribe(terms(60_000)); + void drain(reader); + const old = new StreamError(StreamCode.Old, { message: "old" }); + // Each widening past the last refusal's budget asks a new fill for the run. + for (const budget of [60_000, 120_000, 240_000]) { + const fill = await requests.requestedFill(); + expect([fill?.start, fill?.end]).toEqual([0n, 5n]); + if (!fill) throw new Error("no fill"); + rejectGroups(fill, 0n, 5n, old, budget); + fill.close(); + reader.update(terms(budget * 2)); + } + expect(await requests.requestedFill()).toMatchObject({ start: 0n, end: 5n }); + + reader.close(); + requests.close(); + broadcast.close(); +}); + test("fill: what a reader's sequence cursor passes, a fill no longer owes it", async () => { const { broadcast, track } = resumed(5n); const requests = track.requests(); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index b6650f36882..abae883f621 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -3464,8 +3464,15 @@ export class Subscriber { priority: () => this.#state.update.peek()?.priority ?? 0, maxDelay: () => this.#drift().budget, order: () => this.#state.update.peek()?.order ?? "descending", - refuse: (start, end, error, stale) => - this.#state.refused.update((runs) => insertDrop(runs, start, end, { error, stale })), + refuse: (start, end, error, stale) => { + // A budget's verdict is no eviction: a widening asks it afresh, never through the + // one refill an evicted answer is allowed, as the Rust `Misses::resolve` retries it. + if (stale !== undefined) { + this.#asked = removeRun(this.#asked, start, end); + this.#refilled = removeRun(this.#refilled, start, end); + } + this.#state.refused.update((runs) => insertDrop(runs, start, end, { error, stale })); + }, }; return this.#state.asker; } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index e39fbc20abc..cf4c29d8bcf 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -23895,6 +23895,40 @@ mod test { assert_eq!((fill.start().group, fill.end()), (0, 3)); } + /// A fill's OLD answer under the budget it was asked with is a stale verdict on its reader, + /// which each update reopens: a new fill is asked every time, never given up. Mirrors JS. + #[test] + #[cfg(feature = "async")] + #[expect(clippy::single_range_in_vec_init, reason = "the reopened runs are a list, here one")] + fn a_fills_old_answer_is_asked_again_after_each_update() { + let mut producer = track_producer("test", Info::timed()); + let dynamic = producer.requests(); + producer.start_at(3).unwrap(); + write_group(&producer, 3); + let terms = |secs| { + Subscription::default() + .with_delay(Delay::default().with_max(Duration::from_secs(secs))) + .with_ranges([Range::groups(0..)]) + }; + let mut subscriber = producer.subscribe(terms(60)); + let waiter = kio::Waiter::noop(); + assert_eq!(drain_groups(&mut subscriber), [3]); + for secs in [60, 120, 240] { + assert!(subscriber.poll_drop(&waiter).is_pending()); + let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) else { + panic!("a fill is asked under {secs}s"); + }; + assert_eq!((fill.start().group, fill.end()), (0, 3)); + fill.reject_groups(0..3, Error::Old); + let stale = Hole::stale(Duration::from_secs(secs)); + assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((0..3, stale))); + subscriber.update(terms(secs * 2)).unwrap(); + assert_eq!(subscriber.reoffer_stale(), [0..3]); + } + assert!(subscriber.poll_drop(&waiter).is_pending()); + assert!(matches!(dynamic.poll_requested_fill(&waiter), Poll::Ready(Ok(_)))); + } + /// Every group a set of ranges names, as a received copy's upstream terms judge them. #[cfg(feature = "async")] fn judged(runs: impl IntoIterator) -> Option> { diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index 876c4d2a9a7..a86acf0cbb3 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -611,6 +611,9 @@ impl Misses { } } Err((Error::Old, asked)) if looser(max_delay, asked) => retry.insert(groups), + // Stale under the budget it was asked with, which this reader's is no wider + // than: a stale verdict an update reopens, never a refusal spent for good. + Err((Error::Old, Some(asked))) => self.give_up_stale(groups, asked, given_up), // A drop the track recorded is named by the drop cursor already. Err(_) => self.settle(&state, groups, given_up), } From 5c05774eb0c007e762ee65f4da3f023441743d7c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 19:18:26 -0800 Subject: [PATCH 25/37] fix(net): withhold a failed fill answer as it aborts A subscribed reader that reads in the same turn could take it, since the withhold ran in a promise reaction. The fetch that asked keeps it, as its outcome. Co-Authored-By: Claude Opus 5.5 --- js/net/src/fill.test.ts | 27 +++++++++++++++++++++++++++ js/net/src/group.ts | 9 +++++++++ js/net/src/internal.ts | 5 +++++ js/net/src/track.ts | 23 +++++++++++++++++------ 4 files changed, 58 insertions(+), 6 deletions(-) diff --git a/js/net/src/fill.test.ts b/js/net/src/fill.test.ts index 452378d376f..3070cc4c99d 100644 --- a/js/net/src/fill.test.ts +++ b/js/net/src/fill.test.ts @@ -283,6 +283,33 @@ async function failedAnswerHoldsTheEnd(error: Error) { broadcast.close(); } +// Withheld as it fails, so a read in the same turn cannot take it either. +test("fill: a failed answer is withheld from a subscribed reader that reads at once", async () => { + for (const dispositions of [false, true]) { + const broadcast = new BroadcastProducer(); + const track = broadcast.createTrack("video", { maxAge: null }); + hooks.receiveTrack(track); + hooks.receiveGroup(track, { sequence: 2n, start: 0 }).finish(); + const reader = track.subscribe({ ranges: [{ start: { absolute: 0n } }], delay: { max: null } }); + const requests = track.requests(); + const fetching = wireOf(broadcast).fetchGroup("video", 1n); + const fill = await requests.requestedFill(); + if (!fill) throw new Error("no fill"); + fill.accept(1n).abort(new Error("archive failed")); + const seen: unknown[] = []; + for (let recv = hooks.tryRecvGroup(reader, dispositions); recv.kind !== "idle"; ) { + seen.push(recv.kind === "group" ? recv.group.sequence : recv); + recv = hooks.tryRecvGroup(reader, dispositions); + } + expect(seen).toEqual([2n]); + fill.close(); + await fetching.catch(() => undefined); + reader.close(); + requests.close(); + broadcast.close(); + } +}); + // A failed fill answer leaves the cache, but a reader already holding it keeps what it buffered and // sees the fill's own failure, never a cache eviction. test("fill: a held answer that fails keeps its frames and its failure", async () => { diff --git a/js/net/src/group.ts b/js/net/src/group.ts index 86b0489e593..9c304c50550 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -258,6 +258,9 @@ export class Producer { // timestamp against its neighbors' starts. #admits?: ((start: Timestamp) => () => void)[]; + // Called as the group aborts, before any signal subscriber runs (see `hooks.onGroupAbort`). + #aborted?: ((error: Error) => void)[]; + constructor(sequence: bigint) { if (checkSequence(sequence) >= RESERVED) throw new RangeError(`group sequence out of range: ${sequence}`); this.#state = new GroupState(sequence); @@ -328,6 +331,10 @@ export class Producer { group.#admits ??= []; group.#admits.push(admit); }; + hooks.onGroupAbort = (group, fn) => { + group.#aborted ??= []; + group.#aborted.push(fn); + }; hooks.overRefresh = (group, ratio) => { const state = group.#state; if (state.offset !== 0) return false; @@ -693,6 +700,8 @@ export class Producer { if (mirror.closed.peek() === undefined) mirror.closed.set(outcome); this.#unmirror(mirror); } + if (outcome) for (const fn of this.#aborted ?? []) fn(outcome); + this.#aborted = undefined; } } diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index d0d44344f9d..d4fcdf5ce13 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -859,6 +859,8 @@ export const hooks: { * the group joined accepted it. */ admitGroupStart: (group: GroupProducer, admit: (start: Timestamp) => () => void) => void; + /** Call `fn` as a group aborts, before its readers can look; assigned by `group.ts`. */ + onGroupAbort: (group: GroupProducer, fn: (error: Error) => void) => void; /** Whether a group's bytes after its first frame exceed `ratio` times the first frame. */ overRefresh: (group: GroupProducer, ratio: number) => boolean; /** Keep a mirror of a group accepted unverified below its track's eviction watermark out of media-time expiry. */ @@ -1198,6 +1200,9 @@ export const hooks: { admitGroupStart: () => { throw new Error("group.ts not loaded"); }, + onGroupAbort: () => { + throw new Error("group.ts not loaded"); + }, overRefresh: () => { throw new Error("group.ts not loaded"); }, diff --git a/js/net/src/track.ts b/js/net/src/track.ts index abae883f621..af0aaffd63a 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -682,6 +682,11 @@ class TrackState { producer?: Producer; /** This sink's reader as the producer's fills know it, once it asked one. */ asker?: Asker; + /** + * A one-group fetch, once it asked: a failed fill answer is its outcome, as a Rust fetch hears + * its fill's, so the producer's `#withhold` leaves it this sink. + */ + fetch?: true; /** * Whether this sink's reader will still read group `sequence`: only a buffered group it will * read keeps its source wanted. Set once the reader exists. @@ -1759,6 +1764,7 @@ export class Producer { // holding it keeps its frames and the fill's own failure while the pool still counts them. #withhold(entry: CachedGroup): void { for (const [sink, mirror] of entry.mirrors) { + if (sink.fetch) continue; const index = sink.groups.peek().indexOf(mirror); if (index < 0) continue; sink.groups.mutate((groups) => { @@ -1925,6 +1931,15 @@ export class Producer { }; hooks.chargeGroup(group, entry.charge); this.#undrop(group.sequence); + // No reader is handed a failed fill answer (see #withhold), withheld as it aborts so a + // read that follows at once cannot take it: its sequence is unknown again, owed while the + // feed owes it, else judged by budget or filled again. + if (fetched) { + hooks.onGroupAbort(group, (error) => { + // An eviction takes the mirrors out itself. + if (!EVICTED.has(error) && this.#cached.get(group.sequence) === entry) this.#withhold(entry); + }); + } group.closed.then((closed) => { // An aborted live edge is no longer protected, so it may expire like any other. if (closed instanceof Error) this.#account.demote(entry); @@ -1934,12 +1949,7 @@ export class Producer { if (current && current.group !== group) return; // A failed fill answer says nothing of the feed, so it gives nothing up: the track's drops // and arrivals hold only the feed's verdicts, as Rust records nothing for a failed fill. - // No reader is handed it either (see #withhold), so its sequence is unknown again: owed - // while the feed owes it, else judged by budget or filled again. - if (fetched) { - if (current) this.#withhold(entry); - return; - } + if (fetched) return; if (HOLES.has(toStreamCode(closed))) return; this.#drop(group.sequence, group.sequence + 1n, { error: closed }); }); @@ -3601,6 +3611,7 @@ export class Subscriber { const producer = this.#state.producer; if (!producer) return false; if (this.#state.info.peek() === undefined && this.#state.closed.peek() === undefined) return "pending"; + this.#state.fetch = true; if (cachedHolds(producer, sequence)) return true; // Joins a fill already bringing it, handler or not; asks a new one only while one is open. const asked = fillsOf(producer).ask(sequence, sequence + 1n, this.#askerOf()); From 98b5caf7480f9c200e8a47b94a6153bdeb2e5608 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 19:24:38 -0800 Subject: [PATCH 26/37] perf(net): a drop updates a reader's unknown-run memo by what it changed The drop set now reports the runs it replaced or forgot, so the memo no longer re-checks every run it holds on each new drop, and the probe starts at the edge rather than walking the runs below it. Co-Authored-By: Claude Opus 5.5 --- js/net/src/drops.test.ts | 23 ++++++++++++++- js/net/src/drops.ts | 25 ++++++++++++++++ js/net/src/track.ts | 61 +++++++++++++++++++++++++++------------- 3 files changed, 89 insertions(+), 20 deletions(-) diff --git a/js/net/src/drops.test.ts b/js/net/src/drops.test.ts index ff47931e9aa..ba67b88977d 100644 --- a/js/net/src/drops.test.ts +++ b/js/net/src/drops.test.ts @@ -1,5 +1,6 @@ import { expect, test } from "bun:test"; -import { Runs } from "./drops.ts"; +import { DropSet, Runs } from "./drops.ts"; +import { NotFound } from "./error.ts"; import type { Run } from "./internal.ts"; type Tagged = Run & { tag: number }; @@ -33,3 +34,23 @@ test("drops: clearing ranges in one pass matches clearing each", () => { [40n, 42n], ]); }); + +test("drops: what a change replaced or forgot is reported since its revision, until it is cleared", () => { + const drops = new DropSet(); + const hole = { error: new NotFound("gone") }; + drops.set(0n, 10n, hole); + const before = drops.revision; + drops.set(4n, 6n); + drops.set(20n, 30n, hole); + const extents = (runs: Run[] | undefined) => runs?.map((run) => [run.start, run.end]); + expect(extents(drops.removedSince(before))).toEqual([[0n, 10n]]); + expect(drops.removedSince(drops.revision)).toEqual([]); + // Removals far outnumbering the live runs are cleared, so an older revision rebuilds instead. + const recent = drops.revision; + for (let i = 0; i < 100; i++) { + drops.set(40n, 41n, hole); + drops.set(40n, 41n); + } + expect(drops.removedSince(recent)).toBeUndefined(); + expect(drops.removedSince(drops.revision)).toEqual([]); +}); diff --git a/js/net/src/drops.ts b/js/net/src/drops.ts index d741f65d9a1..d6aaea063da 100644 --- a/js/net/src/drops.ts +++ b/js/net/src/drops.ts @@ -135,6 +135,10 @@ export class DropSet { // the log is rebuilt once dead entries clearly outnumber live ones. #log: { revision: number; run: DropRun }[] = []; #live = new Set(); + // The extent of each run replaced or forgotten, oldest first, kept for {@link removedSince} + // back to `#removedFrom`: cleared once it clearly outnumbers the live runs. + #removed: { revision: number; start: bigint; end: bigint }[] = []; + #removedFrom = 0; #revision = 0; /** Bumped on every change, for a reader to wake on. */ @@ -183,9 +187,14 @@ export class DropSet { this.#log.push({ revision: ++this.#revision, run: fresh }); } if (added.length === 0) this.#revision++; + for (const old of removed) this.#removed.push({ revision: this.#revision, start: old.start, end: old.end }); if (this.#log.length > 2 * this.#live.size + LOG_SLACK) { this.#log = this.#log.filter((entry) => this.#live.has(entry.run)); } + if (this.#removed.length > 2 * this.#live.size + LOG_SLACK) { + this.#removed = []; + this.#removedFrom = this.#revision; + } this.version.update((version) => version + 1); } @@ -208,4 +217,20 @@ export class DropSet { } return changed; } + + /** + * The extents of the runs replaced or forgotten since `revision`, oldest first, or undefined + * once that is further back than the set remembers. + */ + removedSince(revision: number): Run[] | undefined { + if (revision < this.#removedFrom) return undefined; + let lo = 0; + let hi = this.#removed.length; + while (lo < hi) { + const mid = (lo + hi) >>> 1; + if (this.#removed[mid].revision <= revision) lo = mid + 1; + else hi = mid; + } + return this.#removed.slice(lo); + } } diff --git a/js/net/src/track.ts b/js/net/src/track.ts index af0aaffd63a..d67e5ad1326 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -682,11 +682,6 @@ class TrackState { producer?: Producer; /** This sink's reader as the producer's fills know it, once it asked one. */ asker?: Asker; - /** - * A one-group fetch, once it asked: a failed fill answer is its outcome, as a Rust fetch hears - * its fill's, so the producer's `#withhold` leaves it this sink. - */ - fetch?: true; /** * Whether this sink's reader will still read group `sequence`: only a buffered group it will * read keeps its source wanted. Set once the reader exists. @@ -715,7 +710,8 @@ class TrackState { dropsAnchors = new Signal(false); // Frames wake the sequence cursor only while an owed hole holds it. waitingOrdered = false; - // One-shot group reads need FETCH, even when metadata has not reported a live edge. + // One-shot group reads need FETCH, even when metadata has not reported a live edge. A failed + // fill answer is a fetch's outcome, as a Rust fetch hears its fill's, so it is not withheld. fetch = false; // Bumped when a wire subscription declares or resolves where its live feed starts, so a // publisher holding its first group for that answer wakes. @@ -897,6 +893,18 @@ function sameVerdict(a: DropRun, b: DropRun): boolean { return sameHole({ ...a, received: undefined }, { ...b, received: undefined }); } +// The index of the first of `runs`, sorted and disjoint, ending past `sequence`. +function endingPast(runs: readonly Run[], sequence: bigint): number { + let lo = 0; + let hi = runs.length; + while (lo < hi) { + const mid = (lo + hi) >>> 1; + if (runs[mid].end <= sequence) lo = mid + 1; + else hi = mid; + } + return lo; +} + // The runs of `drops` and a reader's `refused` overlapping `[start, end)`, lowest first. A drop // that `settles` the reader, judged from where it meets `start`, hides the refusals it covers: // the track's verdict wins, as in Rust, but a stale one the reader rejects is no verdict for it. @@ -1764,6 +1772,7 @@ export class Producer { // holding it keeps its frames and the fill's own failure while the pool still counts them. #withhold(entry: CachedGroup): void { for (const [sink, mirror] of entry.mirrors) { + // A fetch's own outcome. if (sink.fetch) continue; const index = sink.groups.peek().indexOf(mirror); if (index < 0) continue; @@ -3522,8 +3531,10 @@ export class Subscriber { // update supersedes it: asked even above the newest group received. Only the runs that // do not settle this reader are walked, never every run above the edge. const above = max(low, edge); - for (const dropped of this.#unknownRuns(budget, coverage)) { - if (dropped.end <= above || dropped.start >= end) continue; + const unknown = this.#unknownRuns(budget, coverage); + for (let i = endingPast(unknown, above); i < unknown.length; i++) { + const dropped = unknown[i]; + if (dropped.start >= end) break; const span = { ...dropped, start: max(dropped.start, above), end: min(dropped.end, end) }; if (span.start < span.end && !settles(span, budget, coverage, this.#state.judged)) spans.push(span); } @@ -3579,28 +3590,41 @@ export class Subscriber { // The received stale runs that do not settle this reader under `budget` and `coverage`, // lowest first. While the terms hold, only what changed since is judged again: a run the - // track changed or forgot leaves the memo, and the runs the change left are judged anew, so a - // drop costs each probe the runs it touches. A later start only narrows what a run asks the + // track replaced or forgot leaves the memo, and the runs the change left are judged anew, so a + // drop costs the memo the runs it touches. Once the set no longer remembers what it removed + // that far back, the memo is rebuilt, at most once per a live set's worth of removals. A later start only narrows what a run asks the // reader to read, so a run that settles here settles clipped too. Mirrors the Rust // `Misses::unknown`. #unknownRuns(budget: number, coverage: Coverage): readonly DropRun[] { const { drops, judged } = this.#state; const unsettled = (run: DropRun) => !settles(run, budget, coverage, judged); const memo = this.#unknown; - let runs: DropRun[]; - if (memo && memo.judged === judged && memo.budget === budget && sameCoverage(memo.coverage, coverage)) { + const removed = + memo && + memo.judged === judged && + memo.budget === budget && + sameCoverage(memo.coverage, coverage) && + drops.removedSince(memo.drops); + if (memo && removed) { if (memo.drops === drops.revision) return memo.runs; - runs = memo.runs.filter((run) => drops.at(run.start) === run); + // Updated in place: its one caller is done with the last answer. + const runs = memo.runs; + for (const gone of removed) { + const from = endingPast(runs, gone.start); + let to = from; + while (to < runs.length && runs[to].start < gone.end) to++; + runs.splice(from, to - from); + } for (const changed of drops.since(memo.drops)) { for (const run of drops.overlapping(changed.start, changed.end)) { - if (!unsettled(run) || runs.includes(run)) continue; - const at = runs.findIndex((held) => held.start > run.start); - runs.splice(at < 0 ? runs.length : at, 0, run); + const at = endingPast(runs, run.start); + if (unsettled(run) && runs[at] !== run) runs.splice(at, 0, run); } } - } else { - runs = drops.all.filter(unsettled); + memo.drops = drops.revision; + return runs; } + const runs = drops.all.filter(unsettled); this.#unknown = { drops: drops.revision, judged, budget, coverage, runs }; return runs; } @@ -3611,7 +3635,6 @@ export class Subscriber { const producer = this.#state.producer; if (!producer) return false; if (this.#state.info.peek() === undefined && this.#state.closed.peek() === undefined) return "pending"; - this.#state.fetch = true; if (cachedHolds(producer, sequence)) return true; // Joins a fill already bringing it, handler or not; asks a new one only while one is open. const asked = fillsOf(producer).ask(sequence, sequence + 1n, this.#askerOf()); From 0a9abac669cef23344b16c59e65a8c73d6180219 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 19:37:08 -0800 Subject: [PATCH 27/37] test(net): model the pool folding a subscription's record and tail Under pressure the publisher's record folds a group and names it EVICTED, and the tail forgets finished runs. The tail's fold breaks QuietIsSettled, kept as an open finding in Pressure.cfg. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-subscribe/SKILL.md | 10 ++- rs/moq-net/verify/subscribe/Pressure.cfg | 16 +++++ rs/moq-net/verify/subscribe/StreamWins.cfg | 2 + .../verify/subscribe/SubscribeAccounting.cfg | 3 + .../verify/subscribe/SubscribeAccounting.tla | 64 ++++++++++++++----- .../subscribe/SubscribeAccounting.view.ts | 4 ++ 6 files changed, 81 insertions(+), 18 deletions(-) create mode 100644 rs/moq-net/verify/subscribe/Pressure.cfg diff --git a/.claude/skills/verify-subscribe/SKILL.md b/.claude/skills/verify-subscribe/SKILL.md index 95f60bd7e1c..fb00191c667 100644 --- a/.claude/skills/verify-subscribe/SKILL.md +++ b/.claude/skills/verify-subscribe/SKILL.md @@ -5,7 +5,7 @@ description: Model-check how a lite-07 subscription accounts for each group (`rs # Verify subscription accounting -`rs/moq-net/verify/subscribe/SubscribeAccounting.tla` models one group of one answered lite-07 subscription between the publisher's `TrackRun` and the subscriber's `TrackServe`. The draft promises that every sequence in range gets a Group Stream, a SUBSCRIBE_SKIP, or a SUBSCRIBE_DROP; that a stale (OLD) drop holds only under the update it names and the next SUBSCRIBE_UPDATE reopens it; and that a Group Stream wins over a stale drop, even one it overtook. Drops and skips travel in order on the Subscribe Stream, each Group Stream travels on its own, a reset can lose a stream's header, and a reset sent after the FIN may arrive after every byte. Each model step is one poll of the code that handles one event. The track cursor's verdicts (hand the group out, give it up stale or final, skip it) and whether the cache still holds a fresh copy on an update are free choices. TLC checks every ordering under the bounds in the config. +`rs/moq-net/verify/subscribe/SubscribeAccounting.tla` models one group of one answered lite-07 subscription between the publisher's `TrackRun` and the subscriber's `TrackServe`. The draft promises that every sequence in range gets a Group Stream, a SUBSCRIBE_SKIP, or a SUBSCRIBE_DROP; that a stale (OLD) drop holds only under the update it names and the next SUBSCRIBE_UPDATE reopens it; and that a Group Stream wins over a stale drop, even one it overtook. Drops and skips travel in order on the Subscribe Stream, each Group Stream travels on its own, a reset can lose a stream's header, and a reset sent after the FIN may arrive after every byte. Each model step is one poll of the code that handles one event. The track cursor's verdicts (hand the group out, give it up stale or final, skip it) and whether the cache still holds a fresh copy on an update are free choices. With `Pressure`, so is when the cache pool's debt folds what each side keeps of the group: the publisher's record and the subscriber's tail. TLC checks every ordering under the bounds in the config. ## Rules @@ -23,9 +23,10 @@ Each rule is an INVARIANT; the harm says why it matters. ```sh just rs tla moq-net/verify/subscribe/SubscribeAccounting.tla just rs tla moq-net/verify/subscribe/SubscribeAccounting.tla -config rs/moq-net/verify/subscribe/StreamWins.cfg +just rs tla moq-net/verify/subscribe/SubscribeAccounting.tla -config rs/moq-net/verify/subscribe/Pressure.cfg ``` -The first checks every rule but `StreamWins`: about 3.0M states, 20 to 30 seconds on 32 cores, and it passes at the sync commit. The second fails at the sync commit on the open finding below. `just test tla rs/moq-net/verify/subscribe` runs every config here, as Nightly does, and fails if a config with an `\* Open finding:` line passes. One group is enough: groups share only the order of the Subscribe Stream, and a second group multiplies the states (60M and more) without adding a race. +The first checks every rule but `StreamWins`: about 3.0M states, 20 to 30 seconds on 32 cores, and it passes at the sync commit. The second and third fail on the open findings below; under `Pressure`, `NoViolation` and `ServedOnce` still hold (about 6.8M states). `just test tla rs/moq-net/verify/subscribe` runs every config here, as Nightly does, and fails if a config with an `\* Open finding:` line passes. One group is enough: groups share only the order of the Subscribe Stream, and a second group multiplies the states (60M and more) without adding a race. Read a counterexample with `/statechart`, passing it the spec, a config, and `SubscribeAccounting.view.ts`, which labels what each side holds for the group, which update each side is on, and each step as the event that happens. It gives the shortest path to each broken rule in those words. @@ -33,6 +34,10 @@ Read a counterexample with `/statechart`, passing it the spec, a config, and `Su `StreamWins.cfg` fails. `GroupServe` in `GroupState::Closed` has written every frame and the FIN, and resets with OLD when the group goes stale before the acknowledgement; `settle_failed` then has `TrackRun::poll_step` name it in a stale drop under the update in effect, and record it in `expired`. The subscriber may already hold every byte. It ignores that drop only while the group is in `Tail::finished`, which `buffer_update` clears through `Tail::updated`, or while its copy still caches the group (`track::Producer::give_up` keeps an open group). So with an update sent between reading the FIN and the drop's arrival, and the copy having evicted the group, the subscriber records it stale and owes it again on the next update, while that update makes the publisher serve the whole group again (`reoffer_groups`) or drop it again under every later update. The draft says the publisher, having served a group, "neither serves it again nor names it in another drop". `residual_a_stale_unacknowledged_fin_drops_and_resends_a_served_group` in `lite/publisher.rs` replays the publisher's half, and `a_finished_group_wins_over_a_late_stale_drop_once_evicted` in `lite/subscriber.rs` pins the subscriber's half. Keeping `Tail::finished` across updates passes every rule in the model (about 1.9M states). Once it is fixed, flip the residual test, move `StreamWins` into `SubscribeAccounting.cfg`, and delete `StreamWins.cfg`. +## Open finding: under pressure the tail owes a group it read whole + +`Pressure.cfg` fails `QuietIsSettled`. Under the pool's debt, `Tail::rebill` forgets every finished run but the newest. A stale drop the group's stream overtook (the cursor gave the group up stale, then handed it out over the drop) then finds no finished run, and, with the copy having evicted the group, `apply_response` records it stale. The next update's `buffer_update` owes it again, while the publisher, having served it, never sends it again, so the subscriber waits out its grace at the end. The publisher's fold alone (`Record::folded`) passes every rule. Once it is fixed, move `Pressure = TRUE` into `SubscribeAccounting.cfg` and delete `Pressure.cfg`. + ## Reading a failure A counterexample is a path through the model, not yet a bug. Replay it on the code: the publisher's half with `lite07_run`, `write_group`, `run.update`, `SinkSession::with_unacked_fin`, and `drop_updates` in `lite/publisher.rs` tests; the subscriber's half with `lite07_serving`, `apply_response`, `buffer_update`, and `GroupRecv` in `lite/subscriber.rs` tests. If the code does not follow the path, the model is wrong: fix the model to match the code. If it does, report the path and the rule in the code's words. Do not weaken a rule, add a constraint, or shrink the bounds until the check passes. @@ -52,6 +57,7 @@ Sync commit: `103052ed074a87f00269ecb0d639905482ecfb6f`. Run `git log 103052ed07 | `rs/moq-net/src/lite/subscriber.rs` `ServeLoop::apply_response` (`Drop`, `Skip`) | `OnDrop`, `OnSkip`, `SubRecv` | | `rs/moq-net/src/lite/subscriber.rs` `buffer_update`; `rs/moq-net/src/model/track.rs` `supersede_stale` | `SubUpdate` | | `rs/moq-net/src/tail.rs` `Reading::open`, `Reading::finish`, `Tail::finished`, `updated`, `account`, `unaccount` | `acct`, `finished` | +| `rs/moq-net/src/tail.rs` `Tail::rebill`; `rs/moq-net/src/model/record.rs` `Record::folded` | `TailFold`, `PubFold`, `folded` | | `rs/moq-net/src/model/track.rs` `Producer::give_up`, `skip_received` | `rec`, `cached` | ## Not covered diff --git a/rs/moq-net/verify/subscribe/Pressure.cfg b/rs/moq-net/verify/subscribe/Pressure.cfg new file mode 100644 index 00000000000..264b7c4235c --- /dev/null +++ b/rs/moq-net/verify/subscribe/Pressure.cfg @@ -0,0 +1,16 @@ +\* Open finding: under the pool's debt the tail forgets that a group's stream +\* finished, so a stale drop it overtook records it stale, and the next update owes +\* a group the subscriber read whole and the publisher never sends again. Once it is +\* fixed, move Pressure = TRUE into SubscribeAccounting.cfg and delete this file. +CONSTANTS + Groups = {0} + MaxUpdates = 4 + MaxStreams = 4 + Pressure = TRUE + +SPECIFICATION Spec + +INVARIANTS + NoViolation + ServedOnce + QuietIsSettled diff --git a/rs/moq-net/verify/subscribe/StreamWins.cfg b/rs/moq-net/verify/subscribe/StreamWins.cfg index bd829855150..6fdd0051a94 100644 --- a/rs/moq-net/verify/subscribe/StreamWins.cfg +++ b/rs/moq-net/verify/subscribe/StreamWins.cfg @@ -1,9 +1,11 @@ \* Open finding: a group read whole is recorded stale. Once it is fixed, move \* StreamWins into SubscribeAccounting.cfg and delete this file. +\* Without pressure, which breaks it by design (see Pressure.cfg). CONSTANTS Groups = {0} MaxUpdates = 4 MaxStreams = 4 + Pressure = FALSE SPECIFICATION Spec diff --git a/rs/moq-net/verify/subscribe/SubscribeAccounting.cfg b/rs/moq-net/verify/subscribe/SubscribeAccounting.cfg index 8c06a739270..8a4fafbdc14 100644 --- a/rs/moq-net/verify/subscribe/SubscribeAccounting.cfg +++ b/rs/moq-net/verify/subscribe/SubscribeAccounting.cfg @@ -2,10 +2,13 @@ \* order of the Subscribe Stream, so one group holds every race the model has, and a \* second multiplies the states without adding one. \* StreamWins lives in StreamWins.cfg while its finding is open. +\* Pressure, which lets the pool fold what each side keeps, is in Pressure.cfg while +\* its finding is open. CONSTANTS Groups = {0} MaxUpdates = 4 MaxStreams = 4 + Pressure = FALSE SPECIFICATION Spec diff --git a/rs/moq-net/verify/subscribe/SubscribeAccounting.tla b/rs/moq-net/verify/subscribe/SubscribeAccounting.tla index 212fa168e1d..73a61170a80 100644 --- a/rs/moq-net/verify/subscribe/SubscribeAccounting.tla +++ b/rs/moq-net/verify/subscribe/SubscribeAccounting.tla @@ -17,6 +17,8 @@ (* cursor's verdicts (hand a group out, give it up stale or final, skip *) (* it) and whether the cache still holds a fresh copy on an update are *) (* free choices, standing in for the cache, the clock, and the routes. *) +(* Under `Pressure`, so is when the pool's debt folds what each side keeps *) +(* of the group: the publisher's record and the subscriber's tail. *) (* *) (* Scope: the subscription is already answered (SUBSCRIBE_OK) and covers *) (* every group; ranges, starts, fills, and the tail's grace are left out. *) @@ -26,7 +28,8 @@ EXTENDS Integers, Sequences, FiniteSets CONSTANTS Groups, \* group sequences in range MaxUpdates, \* SUBSCRIBE_UPDATEs the subscriber sends - MaxStreams \* Group Streams the publisher opens + MaxStreams, \* Group Streams the publisher opens + Pressure \* the cache pool may hand the readers debt, so each side folds \* A Group Stream: the publisher's side (`GroupState`), and the subscriber's. \* pub: "open" (serving), "fin" (FIN sent, awaiting its acknowledgement), @@ -50,6 +53,7 @@ VARIABLES redrop, \* `redrop`: expired runs an update owes a drop again reoffered, \* `reoffered`: expired groups an update serves again outq, \* `GroupOutcomes::dropped`: failed streams not yet named + folded, \* the record folded the group into its floor (`Record::folded`) \* The wire. streams, \* every Group Stream opened, in order resp, \* the Subscribe Stream, publisher to subscriber @@ -64,7 +68,7 @@ VARIABLES got, \* history: the subscriber read the group to its FIN violation \* the subscriber closed the session with a PROTOCOL_VIOLATION -pubVars == <> +pubVars == <> subVars == <> vars == <> @@ -73,7 +77,7 @@ Init == /\ late = [g \in Groups |-> FALSE] /\ pup = 0 /\ served = {} /\ dropped = {} /\ dstale = {} /\ expired = {} - /\ redrop = {} /\ reoffered = {} /\ outq = <<>> + /\ redrop = {} /\ reoffered = {} /\ outq = <<>> /\ folded = {} /\ streams = <<>> /\ resp = <<>> /\ ctrl = <<>> /\ sup = 0 /\ acct = {} /\ finished = {} /\ sstale = {} @@ -107,6 +111,7 @@ Serve(g) == \* the next stream ahead of the cursor's. CursorHand(g) == /\ outq = <<>> /\ redrop = {} /\ reoffered = {} + /\ g \notin folded /\ \/ cur[g] = "pending" /\ UNCHANGED late \/ cur[g] \in {"stale", "final"} /\ ~late[g] /\ late' = [late EXCEPT ![g] = TRUE] /\ cur' = [cur EXCEPT ![g] = "handed"] @@ -117,7 +122,7 @@ CursorHand(g) == THEN UNCHANGED <> ELSE /\ Serve(g) /\ UNCHANGED <> - /\ UNCHANGED <> + /\ UNCHANGED <> \* `poll_forward_drop`: the cursor gives g up stale ("old"), final ("evicted"), or \* declares it skipped, after `poll_step` named failed streams and dropped again. A later @@ -147,7 +152,7 @@ CursorGiveUp(g, kind) == /\ resp' = IF ~named THEN resp ELSE IF kind = "skip" THEN Append(resp, SkipMsg(g)) ELSE Append(resp, DropMsg(g, kind, pup)) - /\ UNCHANGED <> + /\ UNCHANGED <> \* A serving stream writes its last frame and FINs (`GroupState::Closed`). StreamFinAt(i) == @@ -164,7 +169,7 @@ StreamFailAt(i, err) == ![i].wasFin = (streams[i].pub = "fin")] /\ outq' = Append(outq, <>) /\ UNCHANGED <> + folded, resp, ctrl, subVars>> \* The peer acknowledged everything up to the FIN. StreamAckAt(i) == @@ -174,11 +179,12 @@ StreamAckAt(i) == /\ UNCHANGED <> \* `poll_step` names the failed streams: a stale expiry is a stale drop the next -\* update judges again, anything else a final one. +\* update judges again, anything else a final one. A stale expiry of a group the +\* record folded while its stream was open is final too: no update re-offers it. PubOutcomes == /\ outq # <<>> /\ LET g == Head(outq)[1] - err == Head(outq)[2] + err == IF Head(outq)[2] = "old" /\ g \in folded THEN "evicted" ELSE Head(outq)[2] IN /\ outq' = Tail(outq) /\ resp' = Append(resp, DropMsg(g, err, pup)) /\ IF err = "old" @@ -186,7 +192,7 @@ PubOutcomes == /\ dropped' = dropped \cup {g} /\ dstale' = dstale \cup {g} ELSE UNCHANGED <> - /\ UNCHANGED <> + /\ UNCHANGED <> \* `poll_step` drops again, under the new number, what an update could not re-offer. PubRedrop == @@ -199,18 +205,34 @@ PubRedrop == /\ dropped' = dropped \cup {g} /\ dstale' = dstale \cup {g} /\ served' = served \cup {g} - /\ UNCHANGED <> + /\ UNCHANGED <> -\* `poll_track` serves a re-offered group, unless a verdict since accounted for it. +\* `poll_track` serves a re-offered group, unless a verdict since accounted for it. One +\* the record folded while it waited is named in a final drop instead. PubReoffer == /\ outq = <<>> /\ redrop = {} /\ reoffered # {} /\ LET g == CHOOSE x \in reoffered : \A y \in reoffered : x <= y IN /\ reoffered' = reoffered \ {g} - /\ IF g \in served - THEN UNCHANGED <> - ELSE Serve(g) - /\ UNCHANGED <> + /\ CASE g \in folded -> + /\ resp' = Append(resp, DropMsg(g, "evicted", pup)) + /\ UNCHANGED <> + [] g \in served -> UNCHANGED <> + [] OTHER -> Serve(g) /\ UNCHANGED resp + /\ UNCHANGED <> + +\* The pool hands the reader debt and its record folds the group into its floor: a group +\* handed out or named final, once nothing about it is still to tell. The fold stops at a +\* stale report the next update may reopen and at a failed stream not yet named. +PubFold(g) == + /\ Pressure + /\ g \notin folded + /\ cur[g] \in {"handed", "final", "skip"} + /\ g \notin dstale + /\ \A i \in 1..Len(outq) : outq[i][1] # g + /\ folded' = folded \cup {g} + /\ UNCHANGED <> \* `TrackRun::update` applies a SUBSCRIBE_UPDATE: the cursor's stale verdicts reopen \* (`reoffer_stale`, then `owe`), and each expired stream is judged again: served @@ -231,7 +253,7 @@ PubUpdate == /\ served' = served \ owed /\ dropped' = dropped \ owed /\ dstale' = dstale \ owed - /\ UNCHANGED <> + /\ UNCHANGED <> ----------------------------------------------------------------------------- (* Subscriber *) @@ -305,6 +327,14 @@ Evict(g) == /\ cached' = [cached EXCEPT ![g] = "evicted"] /\ UNCHANGED <> +\* The pool hands the tail debt and `Tail::rebill` forgets every finished run but the +\* newest: one group stands for any of them, since a later group may have finished. +TailFold(g) == + /\ Pressure + /\ g \in finished + /\ finished' = finished \ {g} + /\ UNCHANGED <> + \* `apply_response` for SUBSCRIBE_DROP. A stale drop under a replaced update is no \* verdict. Otherwise the tail accounts for the group; a stale drop records it in the \* copy unless its stream finished under this update or the copy holds it @@ -401,11 +431,13 @@ Next == \/ PubRedrop \/ PubReoffer \/ PubUpdate + \/ \E g \in Groups : PubFold(g) \/ SubHeader \/ SubFin \/ SubReset \/ SubResetHeld \/ \E g \in Groups : Evict(g) + \/ \E g \in Groups : TailFold(g) \/ SubRecv \/ SubUpdate \/ Done diff --git a/rs/moq-net/verify/subscribe/SubscribeAccounting.view.ts b/rs/moq-net/verify/subscribe/SubscribeAccounting.view.ts index fbb0a662644..d28ef11b430 100644 --- a/rs/moq-net/verify/subscribe/SubscribeAccounting.view.ts +++ b/rs/moq-net/verify/subscribe/SubscribeAccounting.view.ts @@ -83,6 +83,10 @@ export const step = (action: string, from: Vars, to: Vars) => { return "the publisher serves it again"; case "PubUpdate": return "the update reaches the publisher"; + case "PubFold": + return "the publisher's record folds the group under the pool's debt"; + case "TailFold": + return "the subscriber's tail forgets the group finished, under the pool's debt"; case "SubHeader": return "a Group Stream's header arrives"; case "SubFin": From d498d51a0c367906bc7b06c52c4f9f334cbc7714 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 20:09:58 -0800 Subject: [PATCH 28/37] style(net): sort the fill test's imports Co-Authored-By: Claude Opus 5.5 --- js/net/src/fill.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/net/src/fill.test.ts b/js/net/src/fill.test.ts index 3070cc4c99d..7e5212e38c5 100644 --- a/js/net/src/fill.test.ts +++ b/js/net/src/fill.test.ts @@ -3,8 +3,8 @@ import { Duration, Manual, yieldNow } from "@moq/clock"; import { Producer as BroadcastProducer } from "./broadcast.ts"; import { accept as acceptSession, connect as connectSession } from "./connection/index.ts"; import { NotFound, StreamCode, StreamError, toStreamCode } from "./error.ts"; -import type { Consumer as GroupConsumer } from "./group.ts"; import { rejectGroups } from "./fill.ts"; +import type { Consumer as GroupConsumer } from "./group.ts"; import { hooks, staleHole } from "./internal.ts"; import * as Lite from "./lite/index.ts"; import { createMockTransportPair } from "./mock.ts"; From 288ba3edd8163e0740c5ec60e6623c651a71d0eb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 21:30:54 -0800 Subject: [PATCH 29/37] test(net): count the updated reader's asks after the later reader's own Since moq-dev/moq#8485, a finished track's copies are torn down, so the later reader subscribes afresh on each hop. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/tests/integration/drop_record.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/tests/integration/drop_record.rs b/rs/moq-net/tests/integration/drop_record.rs index 2b9d6b26241..38572511a36 100644 --- a/rs/moq-net/tests/integration/drop_record.rs +++ b/rs/moq-net/tests/integration/drop_record.rs @@ -444,12 +444,14 @@ async fn an_origin_eviction_drop_is_forwarded_final() { kio::clock::now() - started ); + // A later reader may subscribe afresh once the finished track's copies are gone, + // so only the update is held to asking nothing again. + let mut later = chain.subscribe(reading(None, [Range::groups(0..)])).await; + assert_eq!(drain(&mut later).await, got, "expiry={expiry}: the later reader"); let asked = chain.asked(); reader .update(reading(Duration::from_secs(3600), [Range::groups(0..)])) .unwrap(); - let mut later = chain.subscribe(reading(None, [Range::groups(0..)])).await; - assert_eq!(drain(&mut later).await, got, "expiry={expiry}: the later reader"); assert_eq!( drain(&mut reader).await, BTreeSet::new(), From 5f3977d09d0102a306f61eadcd8efc1692b43c9f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 21:38:40 -0800 Subject: [PATCH 30/37] quest: a JS fill probe costs what changed, not every unknown run Co-Authored-By: Claude Opus 5.5 --- quest/m1/fill-probe-cost.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 quest/m1/fill-probe-cost.md diff --git a/quest/m1/fill-probe-cost.md b/quest/m1/fill-probe-cost.md new file mode 100644 index 00000000000..e709db4af3f --- /dev/null +++ b/quest/m1/fill-probe-cost.md @@ -0,0 +1,13 @@ +# [M] A JS fill probe costs what changed, not every unknown run + +## Goal + +A JS reader's fill probe (`#askFills` in `js/net/src/track.ts`) costs the runs that changed since its last probe, as the Rust `Misses::judge` does, rather than calling `#askMiss` on every received stale run it does not trust and every hole below the edge on each probe. +A publisher fragmenting stale drops then costs each reader linear work over the flood, not quadratic or worse. + +## Plan + +- Found on [moq-dev/moq#8476](https://github.com/moq-dev/moq/pull/8476), present on `main` before it: with N unsettled stale runs above the received edge, each probe walks all N, and each `#askMiss` scans `#asked`, `#refilled`, and the fills linearly. A bench of one probe per new drop took 0.35 s at N = 250, 2.5 s at 500, and 19 s at 1000. +- Recommended shape: judge each sequence once, as the Rust `Misses` keeps `judged`, and judge again only what an eviction, a refusal, a widening, or a moved window reopened. +- Keep the groups asked the same as today: the JS memo asks fills above the live edge per run, which the Rust grace deferral handles differently. +- Measure with a bench swept over the run count and the readers, as fan-out work requires. From 6c60a9fe832b5aca645db3ff49a44dd925f3b63e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 21:39:21 -0800 Subject: [PATCH 31/37] quest: say where the probe cost was measured Co-Authored-By: Claude Opus 5.5 --- quest/m1/fill-probe-cost.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quest/m1/fill-probe-cost.md b/quest/m1/fill-probe-cost.md index e709db4af3f..6f6344fb9b5 100644 --- a/quest/m1/fill-probe-cost.md +++ b/quest/m1/fill-probe-cost.md @@ -7,7 +7,7 @@ A publisher fragmenting stale drops then costs each reader linear work over the ## Plan -- Found on [moq-dev/moq#8476](https://github.com/moq-dev/moq/pull/8476), present on `main` before it: with N unsettled stale runs above the received edge, each probe walks all N, and each `#askMiss` scans `#asked`, `#refilled`, and the fills linearly. A bench of one probe per new drop took 0.35 s at N = 250, 2.5 s at 500, and 19 s at 1000. +- Found on [moq-dev/moq#8476](https://github.com/moq-dev/moq/pull/8476); `main` has the same probe shape: with N unsettled stale runs above the received edge, each probe walks all N, and each `#askMiss` scans `#asked`, `#refilled`, and the fills linearly. A bench of one probe per new drop took 0.35 s at N = 250, 2.5 s at 500, and 19 s at 1000. - Recommended shape: judge each sequence once, as the Rust `Misses` keeps `judged`, and judge again only what an eviction, a refusal, a widening, or a moved window reopened. - Keep the groups asked the same as today: the JS memo asks fills above the live edge per run, which the Rust grace deferral handles differently. - Measure with a bench swept over the run count and the readers, as fan-out work requires. From b672764412114bf1078acbca5cc12ad708088d7f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 21:52:12 -0800 Subject: [PATCH 32/37] fix(net): an update owes nothing a tail's fold forgot finishing Under the pool's debt the tail folds its older finished runs into one span. A stale drop there may be one its stream overtook, so the next update owes none of it again, and the subscriber no longer waits out its grace for a group it read whole. The model now checks every rule under pressure, and its sync commit moves to the branch's base. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-subscribe/SKILL.md | 13 +++---- js/net/src/cache.test.ts | 4 +++ js/net/src/lite/subscriber.ts | 6 ++-- js/net/src/tail.ts | 30 +++++++++++----- rs/moq-net/src/lite/subscriber.rs | 12 +++++-- rs/moq-net/src/model/track.rs | 6 ++++ rs/moq-net/src/tail.rs | 36 +++++++++++++------ rs/moq-net/verify/subscribe/Pressure.cfg | 16 --------- rs/moq-net/verify/subscribe/StreamWins.cfg | 8 +++-- .../verify/subscribe/SubscribeAccounting.cfg | 7 ++-- .../verify/subscribe/SubscribeAccounting.tla | 28 ++++++++------- 11 files changed, 100 insertions(+), 66 deletions(-) delete mode 100644 rs/moq-net/verify/subscribe/Pressure.cfg diff --git a/.claude/skills/verify-subscribe/SKILL.md b/.claude/skills/verify-subscribe/SKILL.md index fb00191c667..5ad2064107c 100644 --- a/.claude/skills/verify-subscribe/SKILL.md +++ b/.claude/skills/verify-subscribe/SKILL.md @@ -23,20 +23,15 @@ Each rule is an INVARIANT; the harm says why it matters. ```sh just rs tla moq-net/verify/subscribe/SubscribeAccounting.tla just rs tla moq-net/verify/subscribe/SubscribeAccounting.tla -config rs/moq-net/verify/subscribe/StreamWins.cfg -just rs tla moq-net/verify/subscribe/SubscribeAccounting.tla -config rs/moq-net/verify/subscribe/Pressure.cfg ``` -The first checks every rule but `StreamWins`: about 3.0M states, 20 to 30 seconds on 32 cores, and it passes at the sync commit. The second and third fail on the open findings below; under `Pressure`, `NoViolation` and `ServedOnce` still hold (about 6.8M states). `just test tla rs/moq-net/verify/subscribe` runs every config here, as Nightly does, and fails if a config with an `\* Open finding:` line passes. One group is enough: groups share only the order of the Subscribe Stream, and a second group multiplies the states (60M and more) without adding a race. +The first checks every rule but `StreamWins` under `Pressure`: about 13.1M states, a minute on 4 workers, and it passes at the sync commit. The second checks `StreamWins` without `Pressure` and fails on the open finding below. `just test tla rs/moq-net/verify/subscribe` runs every config here, as Nightly does, and fails if a config with an `\* Open finding:` line passes. One group is enough: groups share only the order of the Subscribe Stream, and a second group multiplies the states (60M and more) without adding a race. Read a counterexample with `/statechart`, passing it the spec, a config, and `SubscribeAccounting.view.ts`, which labels what each side holds for the group, which update each side is on, and each step as the event that happens. It gives the shortest path to each broken rule in those words. ## Open finding: a group read whole is named in a stale drop -`StreamWins.cfg` fails. `GroupServe` in `GroupState::Closed` has written every frame and the FIN, and resets with OLD when the group goes stale before the acknowledgement; `settle_failed` then has `TrackRun::poll_step` name it in a stale drop under the update in effect, and record it in `expired`. The subscriber may already hold every byte. It ignores that drop only while the group is in `Tail::finished`, which `buffer_update` clears through `Tail::updated`, or while its copy still caches the group (`track::Producer::give_up` keeps an open group). So with an update sent between reading the FIN and the drop's arrival, and the copy having evicted the group, the subscriber records it stale and owes it again on the next update, while that update makes the publisher serve the whole group again (`reoffer_groups`) or drop it again under every later update. The draft says the publisher, having served a group, "neither serves it again nor names it in another drop". `residual_a_stale_unacknowledged_fin_drops_and_resends_a_served_group` in `lite/publisher.rs` replays the publisher's half, and `a_finished_group_wins_over_a_late_stale_drop_once_evicted` in `lite/subscriber.rs` pins the subscriber's half. Keeping `Tail::finished` across updates passes every rule in the model (about 1.9M states). Once it is fixed, flip the residual test, move `StreamWins` into `SubscribeAccounting.cfg`, and delete `StreamWins.cfg`. - -## Open finding: under pressure the tail owes a group it read whole - -`Pressure.cfg` fails `QuietIsSettled`. Under the pool's debt, `Tail::rebill` forgets every finished run but the newest. A stale drop the group's stream overtook (the cursor gave the group up stale, then handed it out over the drop) then finds no finished run, and, with the copy having evicted the group, `apply_response` records it stale. The next update's `buffer_update` owes it again, while the publisher, having served it, never sends it again, so the subscriber waits out its grace at the end. The publisher's fold alone (`Record::folded`) passes every rule. Once it is fixed, move `Pressure = TRUE` into `SubscribeAccounting.cfg` and delete `Pressure.cfg`. +`StreamWins.cfg` fails. `GroupServe` in `GroupState::Closed` has written every frame and the FIN, and resets with OLD when the group goes stale before the acknowledgement; `settle_failed` then has `TrackRun::poll_step` name it in a stale drop under the update in effect, and record it in `expired`. The subscriber may already hold every byte. It ignores that drop only while the group is in `Tail::finished`, which `buffer_update` clears through `Tail::updated`, or while its copy still caches the group (`track::Producer::give_up` keeps an open group). So with an update sent between reading the FIN and the drop's arrival, and the copy having evicted the group, the subscriber records it stale and owes it again on the next update, while that update makes the publisher serve the whole group again (`reoffer_groups`) or drop it again under every later update. The draft says the publisher, having served a group, "neither serves it again nor names it in another drop". `residual_a_stale_unacknowledged_fin_drops_and_resends_a_served_group` in `lite/publisher.rs` replays the publisher's half, and `a_finished_group_wins_over_a_late_stale_drop_once_evicted` in `lite/subscriber.rs` pins the subscriber's half. Keeping `Tail::finished` across updates passes every rule in the model without `Pressure` (about 1.9M states). Under it, `Tail::rebill` still forgets a finished run by design: the overtaken drop records the group stale, though the update owes none of the span it forgot again. Once it is fixed, flip the residual test and drop the `\* Open finding:` line from `StreamWins.cfg`, which stays without `Pressure`. ## Reading a failure @@ -44,7 +39,7 @@ A counterexample is a path through the model, not yet a bug. Replay it on the co ## Coverage -Sync commit: `103052ed074a87f00269ecb0d639905482ecfb6f`. Run `git log 103052ed07.. -- rs/moq-net/src/lite/publisher.rs rs/moq-net/src/lite/subscriber.rs rs/moq-net/src/tail.rs rs/moq-net/src/model/track.rs` to see what changed since. After a change to a covered function, mirror it in the model, rerun both configs, and move the sync commit forward only once the model describes the code at the new commit. `just check` fails when a function or type the table names no longer exists in its file (`sh/coverage-tables.py`), so each Code cell starts with a repo-relative path, then the symbols in it. +Sync commit: `d9a88d2e3685f170e197c4e1623f26ccaa2c4c63`. Run `git log d9a88d2e36.. -- rs/moq-net/src/lite/publisher.rs rs/moq-net/src/lite/subscriber.rs rs/moq-net/src/tail.rs rs/moq-net/src/model/track.rs` to see what changed since. After a change to a covered function, mirror it in the model, rerun both configs, and move the sync commit forward only once the model describes the code at the new commit. `just check` fails when a function or type the table names no longer exists in its file (`sh/coverage-tables.py`), so each Code cell starts with a repo-relative path, then the symbols in it. | Code | Model | | --- | --- | @@ -57,7 +52,7 @@ Sync commit: `103052ed074a87f00269ecb0d639905482ecfb6f`. Run `git log 103052ed07 | `rs/moq-net/src/lite/subscriber.rs` `ServeLoop::apply_response` (`Drop`, `Skip`) | `OnDrop`, `OnSkip`, `SubRecv` | | `rs/moq-net/src/lite/subscriber.rs` `buffer_update`; `rs/moq-net/src/model/track.rs` `supersede_stale` | `SubUpdate` | | `rs/moq-net/src/tail.rs` `Reading::open`, `Reading::finish`, `Tail::finished`, `updated`, `account`, `unaccount` | `acct`, `finished` | -| `rs/moq-net/src/tail.rs` `Tail::rebill`; `rs/moq-net/src/model/record.rs` `Record::folded` | `TailFold`, `PubFold`, `folded` | +| `rs/moq-net/src/tail.rs` `Tail::rebill`, `updated`; `rs/moq-net/src/lite/subscriber.rs` `buffer_update`; `rs/moq-net/src/model/record.rs` `Record::folded` | `TailFold`, `forgot`, `PubFold`, `folded` | | `rs/moq-net/src/model/track.rs` `Producer::give_up`, `skip_received` | `rec`, `cached` | ## Not covered diff --git a/js/net/src/cache.test.ts b/js/net/src/cache.test.ts index ed552b852c0..63b9755660b 100644 --- a/js/net/src/cache.test.ts +++ b/js/net/src/cache.test.ts @@ -399,6 +399,10 @@ test("a tail forgets its finished streams under debt", () => { tail.finish(2n * runs); expect(tail.unfinished(0n, 2n * runs + 1n).length).toBeLessThanOrEqual(2); expect(pool.used).toBeLessThanOrEqual(capacity + 2 * (16 * 1024 + cache.ENTRY_OVERHEAD)); + // The update owes none of what the fold forgot again, since a stale drop there may be one its + // stream overtook. + expect(tail.updated()).toEqual({ start: 0n, end: 2n * runs - 1n }); + expect(tail.updated()).toBeUndefined(); tail.close(); }); diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 5df0c540389..3616708a27f 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -36,6 +36,7 @@ import { minDelayOf, type Run, removeRun, + runGaps, scopeCaptures, scopeHead, scopeOverlaps, @@ -2625,7 +2626,8 @@ export class Subscriber { // unknown again: a reader fills them rather than trust an old verdict. entry.update = update.update; entry.maxDelay = update.maxDelay; - entry.tail.updated(); + // What a fold forgot finishing may have overtaken its drop, so none of it is owed again. + const folded = entry.tail.updated(); const held = hooks.supersedeStale(entry.track, judged(ranges)); // A lite-07 publisher re-offers every group it reported stale on any update, and drops // again under the new number what is still stale, so the tail owes every stale run again. @@ -2635,7 +2637,7 @@ export class Subscriber { for (const reopened of held) { const start = max(run.start, reopened.start); const end = min(run.end, reopened.end); - if (start < end) entry.tail.unaccount(start, end); + for (const part of runGaps(folded ? [folded] : [], start, end)) entry.tail.unaccount(part.start, part.end); } } // Re-offered whether or not the copy still holds the stale verdict. diff --git a/js/net/src/tail.ts b/js/net/src/tail.ts index 6bac68ce20b..56181d5c304 100644 --- a/js/net/src/tail.ts +++ b/js/net/src/tail.ts @@ -37,6 +37,9 @@ export class Tail { // update names one only if it was sent before the publisher served the group, so the group won // over it, even once the cache evicted it. #finished: Run[] = []; + // The span of the finished runs a fold under the pool's debt forgot since the last update. A + // stale drop in it may be one its stream overtook, so the update owes none of it again. + #folded?: Run; // What `#finished` bills, `READER_RUN_OVERHEAD` per run, so a publisher fragmenting the // streams it finishes between updates costs what the cache would hold. #bill?: Reader; @@ -46,23 +49,30 @@ export class Tail { this.#bill = options.bill; } - // Bill what `#finished` keeps, and under the pool's debt forget all but its newest run: a stale - // drop under this update then stands as the publisher judged it, which a wider budget still - // reverses. The newest run stays, so streams finishing in sequence grow nothing to charge again. - // Mirrors the Rust `Tail::rebill`. + // Bill what `#finished` keeps, and under the pool's debt fold all but its newest run into + // `#folded`: a stale drop under this update then stands as the publisher judged it, but the + // update owes none of the folded span again. The newest run stays, so streams finishing in + // sequence grow nothing to charge again. Mirrors the Rust `Tail::rebill`. #rebill(): void { const bill = this.#bill; if (!bill) return; - bill.set(this.#finished.length * READER_RUN_OVERHEAD); + bill.set((this.#finished.length + (this.#folded ? 1 : 0)) * READER_RUN_OVERHEAD); if (bill.debt > 0 && this.#finished.length > 1) { + const start = this.#finished[0].start; + const end = this.#finished[this.#finished.length - 2].end; + const folded = this.#folded; + this.#folded = folded + ? { start: folded.start < start ? folded.start : start, end: folded.end > end ? folded.end : end } + : { start, end }; this.#finished = this.#finished.slice(-1); - bill.set(READER_RUN_OVERHEAD); + bill.set(2 * READER_RUN_OVERHEAD); } } /** Release what the tail bills, once its subscription ended. */ close(): void { this.#finished = []; + this.#folded = undefined; // A stream still in flight finishes after this: with no bill, it can never bill again. this.#bill?.close(); this.#bill = undefined; @@ -136,11 +146,15 @@ export class Tail { /** * Forget which streams finished, for an update being sent: a stale drop under an earlier one is - * no verdict, so none needs them. + * no verdict, so none needs them. Returns the span a fold forgot since the last update, which + * the update owes none of again. */ - updated(): void { + updated(): Run | undefined { this.#finished = []; + const folded = this.#folded; + this.#folded = undefined; this.#rebill(); + return folded; } /** Re-check what {@link settle} waits on, after something outside the ledger changed. */ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 943a81d8d97..927635b992b 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -10542,10 +10542,18 @@ fn buffer_update( let held = serving.supersede_stale(judged(active.live.iter().map(|(_, range)| range))); let recorded = std::mem::take(&mut active.stale); if let Ok(mut tail) = active.tail.write() { - tail.updated(); + // What a fold forgot finishing may have overtaken its drop, so none of it is owed again. + let folded = tail.updated(); for run in recorded.iter() { for reopened in held.overlapping(run) { - tail.unaccount(reopened.start.max(run.start)..reopened.end.min(run.end), now); + let mut owed = RangeSet::new(); + owed.insert(reopened.start.max(run.start)..reopened.end.min(run.end)); + if let Some(folded) = &folded { + owed.remove(folded.clone()); + } + for owed in owed.iter() { + tail.unaccount(owed.clone(), now); + } } } } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index cf4c29d8bcf..da215452fb9 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -24383,6 +24383,12 @@ mod test { "forgotten under debt" ); assert!(pool.used() <= capacity, "within the pool: {}", pool.used()); + // The update owes none of what the fold forgot again, since a stale drop there may be + // one its stream overtook. + if let Ok(mut state) = tail.write() { + assert_eq!(state.updated(), Some(0..2 * RUNS - 1)); + assert_eq!(state.updated(), None, "each update starts over"); + } } /// A flood a reader reported, fragmented into runs alternating with upstream skips, diff --git a/rs/moq-net/src/tail.rs b/rs/moq-net/src/tail.rs index 42930edfc6a..a1a7ca0c466 100644 --- a/rs/moq-net/src/tail.rs +++ b/rs/moq-net/src/tail.rs @@ -34,6 +34,9 @@ pub(crate) struct Tail { // that update names one only if it was sent before the publisher served the group, so the // group won over it, even once the cache evicted it. finished: RangeSet, + // The span of the finished runs a fold under the pool's debt forgot since the last update. + // A stale drop in it may be one its stream overtook, so the update owes none of it again. + folded: Option>, // What `finished` bills to the track's account, [`FINISHED_OVERHEAD`] per run, once the // subscription has a copy to bill. bill: Option, @@ -59,6 +62,7 @@ impl Tail { active: 0, ended: false, finished: RangeSet::new(), + folded: None, bill: None, } } @@ -69,22 +73,31 @@ impl Tail { self.rebill(); } - /// Bill what `finished` keeps, and under the pool's debt forget all but its newest run: a - /// stale drop under this update then stands as the publisher judged it, which a wider budget - /// still reverses. The newest run stays, so streams finishing in sequence grow nothing to - /// charge again. + /// Bill what `finished` keeps, and under the pool's debt fold all but its newest run into + /// `folded`: a stale drop under this update then stands as the publisher judged it, but the + /// update owes none of the folded span again. The newest run stays, so streams finishing in + /// sequence grow nothing to charge again. fn rebill(&mut self) { let Some(bill) = &mut self.bill else { return; }; - bill.set(self.finished.len() as u64 * FINISHED_OVERHEAD); + let runs = self.finished.len() + usize::from(self.folded.is_some()); + bill.set(runs as u64 * FINISHED_OVERHEAD); if bill.debt() > 0 && self.finished.len() > 1 - && let Some(newest) = self.finished.iter().next_back().cloned() - { + && let (Some(first), Some(older), Some(newest)) = ( + self.finished.iter().next().cloned(), + self.finished.iter().nth_back(1).cloned(), + self.finished.iter().next_back().cloned(), + ) { + let span = match self.folded.take() { + Some(folded) => folded.start.min(first.start)..folded.end.max(older.end), + None => first.start..older.end, + }; + self.folded = Some(span); self.finished = RangeSet::new(); self.finished.insert(newest); - bill.set(FINISHED_OVERHEAD); + bill.set(2 * FINISHED_OVERHEAD); } } @@ -150,10 +163,13 @@ impl Tail { } /// Forget which streams finished, for an update being sent: a stale drop under an earlier - /// one is no verdict, so none needs them. - pub fn updated(&mut self) { + /// one is no verdict, so none needs them. Returns the span a fold forgot since the last + /// update, which the update owes none of again. + pub fn updated(&mut self) -> Option> { self.finished = RangeSet::new(); + let folded = self.folded.take(); self.rebill(); + folded } /// Whether a data stream whose header arrived is still being read. diff --git a/rs/moq-net/verify/subscribe/Pressure.cfg b/rs/moq-net/verify/subscribe/Pressure.cfg deleted file mode 100644 index 264b7c4235c..00000000000 --- a/rs/moq-net/verify/subscribe/Pressure.cfg +++ /dev/null @@ -1,16 +0,0 @@ -\* Open finding: under the pool's debt the tail forgets that a group's stream -\* finished, so a stale drop it overtook records it stale, and the next update owes -\* a group the subscriber read whole and the publisher never sends again. Once it is -\* fixed, move Pressure = TRUE into SubscribeAccounting.cfg and delete this file. -CONSTANTS - Groups = {0} - MaxUpdates = 4 - MaxStreams = 4 - Pressure = TRUE - -SPECIFICATION Spec - -INVARIANTS - NoViolation - ServedOnce - QuietIsSettled diff --git a/rs/moq-net/verify/subscribe/StreamWins.cfg b/rs/moq-net/verify/subscribe/StreamWins.cfg index 6fdd0051a94..8374ed56b7b 100644 --- a/rs/moq-net/verify/subscribe/StreamWins.cfg +++ b/rs/moq-net/verify/subscribe/StreamWins.cfg @@ -1,6 +1,8 @@ -\* Open finding: a group read whole is recorded stale. Once it is fixed, move -\* StreamWins into SubscribeAccounting.cfg and delete this file. -\* Without pressure, which breaks it by design (see Pressure.cfg). +\* Open finding: a group read whole is recorded stale. Once it is fixed, drop +\* this line. +\* Without pressure: under the pool's debt the tail forgets a finished stream by +\* design, so a stale drop it overtook still records the group stale, though the +\* update owes it nothing again. CONSTANTS Groups = {0} MaxUpdates = 4 diff --git a/rs/moq-net/verify/subscribe/SubscribeAccounting.cfg b/rs/moq-net/verify/subscribe/SubscribeAccounting.cfg index 8a4fafbdc14..506587db28a 100644 --- a/rs/moq-net/verify/subscribe/SubscribeAccounting.cfg +++ b/rs/moq-net/verify/subscribe/SubscribeAccounting.cfg @@ -1,14 +1,13 @@ \* One group, four SUBSCRIBE_UPDATEs, and four Group Streams. Groups share only the \* order of the Subscribe Stream, so one group holds every race the model has, and a \* second multiplies the states without adding one. -\* StreamWins lives in StreamWins.cfg while its finding is open. -\* Pressure, which lets the pool fold what each side keeps, is in Pressure.cfg while -\* its finding is open. +\* StreamWins lives in StreamWins.cfg, without pressure, while its finding is open. +\* Pressure lets the pool fold what each side keeps at any point. CONSTANTS Groups = {0} MaxUpdates = 4 MaxStreams = 4 - Pressure = FALSE + Pressure = TRUE SPECIFICATION Spec diff --git a/rs/moq-net/verify/subscribe/SubscribeAccounting.tla b/rs/moq-net/verify/subscribe/SubscribeAccounting.tla index 73a61170a80..819cfc697b2 100644 --- a/rs/moq-net/verify/subscribe/SubscribeAccounting.tla +++ b/rs/moq-net/verify/subscribe/SubscribeAccounting.tla @@ -62,6 +62,7 @@ VARIABLES sup, \* `SubStream::update`: the latest number sent acct, \* the tail's ledger: groups accounted for finished, \* `Tail::finished`: groups whose stream finished since the last update + forgot, \* `Tail::folded`: the span a fold forgot since the last update sstale, \* `SubStream::stale`: runs accounted by a stale drop under `sup` rec, \* the copy's record: "none", "stale", "final", or "skip" cached, \* the copy's group: "none", "open", "whole", "aborted", "evicted" @@ -69,7 +70,7 @@ VARIABLES violation \* the subscriber closed the session with a PROTOCOL_VIOLATION pubVars == <> -subVars == <> +subVars == <> vars == <> Init == @@ -80,7 +81,7 @@ Init == /\ redrop = {} /\ reoffered = {} /\ outq = <<>> /\ folded = {} /\ streams = <<>> /\ resp = <<>> /\ ctrl = <<>> /\ sup = 0 - /\ acct = {} /\ finished = {} /\ sstale = {} + /\ acct = {} /\ finished = {} /\ forgot = {} /\ sstale = {} /\ rec = [g \in Groups |-> "none"] /\ cached = [g \in Groups |-> "none"] /\ got = {} @@ -281,7 +282,7 @@ SubHeaderAt(i) == /\ streams' = [streams EXCEPT ![i].sub = "header"] /\ cached' = [cached EXCEPT ![g] = "open"] /\ rec' = [rec EXCEPT ![g] = IF @ = "stale" THEN "none" ELSE @] - /\ UNCHANGED <> + /\ UNCHANGED <> \* The stream filling the copy is read to its FIN (`Reading::finish`); once every byte \* arrived, a reset sent after the FIN changes nothing. A held stream of the group then @@ -298,7 +299,7 @@ SubFinAt(i) == /\ finished' = finished \cup {g} /\ cached' = [cached EXCEPT ![g] = "whole"] /\ got' = got \cup {g} - /\ UNCHANGED <> + /\ UNCHANGED <> \* The reset of the stream filling the copy arrives: the group's readers end with its \* error, and the first held stream of the group opens it again (`TrackEntry::reopen`). @@ -312,7 +313,7 @@ SubResetAt(i) == ELSE IF j \in next THEN [streams[j] EXCEPT !.sub = "header"] ELSE streams[j]] /\ cached' = [cached EXCEPT ![g] = IF next = {} THEN "aborted" ELSE "open"] - /\ UNCHANGED <> + /\ UNCHANGED <> \* A held stream's own reset arrives. SubResetHeldAt(i) == @@ -325,14 +326,16 @@ SubResetHeldAt(i) == Evict(g) == /\ cached[g] = "whole" /\ cached' = [cached EXCEPT ![g] = "evicted"] - /\ UNCHANGED <> + /\ UNCHANGED <> -\* The pool hands the tail debt and `Tail::rebill` forgets every finished run but the -\* newest: one group stands for any of them, since a later group may have finished. +\* The pool hands the tail debt and `Tail::rebill` folds every finished run but the newest +\* into the span it forgot: one group stands for any of them, since a later group may have +\* finished, and the span takes in the gaps between them, so g need not have finished. TailFold(g) == /\ Pressure - /\ g \in finished + /\ g \notin forgot /\ finished' = finished \ {g} + /\ forgot' = forgot \cup {g} /\ UNCHANGED <> \* `apply_response` for SUBSCRIBE_DROP. A stale drop under a replaced update is no @@ -375,11 +378,11 @@ SubRecv == /\ resp' = Tail(resp) /\ LET m == Head(resp) IN IF m.type = "drop" THEN OnDrop(m) ELSE OnSkip(m) - /\ UNCHANGED <> + /\ UNCHANGED <> \* `buffer_update`: the copy forgets every stale record (`supersede_stale`), the tail \* owes again each stale run this subscription accounted that the copy still held, -\* and `Tail::updated` forgets which streams finished. +\* save the span a fold forgot, and `Tail::updated` forgets which streams finished. SubUpdate == /\ sup < MaxUpdates /\ ~violation @@ -387,9 +390,10 @@ SubUpdate == IN /\ sup' = sup + 1 /\ ctrl' = Append(ctrl, sup + 1) /\ rec' = [g \in Groups |-> IF rec[g] = "stale" THEN "none" ELSE rec[g]] - /\ acct' = acct \ (sstale \cap held) + /\ acct' = acct \ ((sstale \cap held) \ forgot) /\ sstale' = {} /\ finished' = {} + /\ forgot' = {} /\ UNCHANGED <> ----------------------------------------------------------------------------- From 2828cc6f2f38b460fdf7a883f9c160116a8d3382 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 22:25:02 -0800 Subject: [PATCH 33/37] style(net): the tail's folded span uses the sequence min and max Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/subscriber.ts | 3 ++- js/net/src/tail.ts | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 3616708a27f..c0b00267bff 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -2637,7 +2637,8 @@ export class Subscriber { for (const reopened of held) { const start = max(run.start, reopened.start); const end = min(run.end, reopened.end); - for (const part of runGaps(folded ? [folded] : [], start, end)) entry.tail.unaccount(part.start, part.end); + for (const part of runGaps(folded ? [folded] : [], start, end)) + entry.tail.unaccount(part.start, part.end); } } // Re-offered whether or not the copy still holds the stale verdict. diff --git a/js/net/src/tail.ts b/js/net/src/tail.ts index 56181d5c304..8119d173467 100644 --- a/js/net/src/tail.ts +++ b/js/net/src/tail.ts @@ -3,6 +3,7 @@ import { type GetPromise, Signal } from "@moq/signals"; import { READER_RUN_OVERHEAD, type Reader } from "./cache.ts"; import { insertRun, type Run, runGaps } from "./internal.ts"; import { GRACE, Ledger } from "./ledger.ts"; +import { max, min } from "./util/sequence.ts"; /** How a {@link Tail} measures its grace. */ export interface TailOptions { @@ -61,9 +62,7 @@ export class Tail { const start = this.#finished[0].start; const end = this.#finished[this.#finished.length - 2].end; const folded = this.#folded; - this.#folded = folded - ? { start: folded.start < start ? folded.start : start, end: folded.end > end ? folded.end : end } - : { start, end }; + this.#folded = folded ? { start: min(folded.start, start), end: max(folded.end, end) } : { start, end }; this.#finished = this.#finished.slice(-1); bill.set(2 * READER_RUN_OVERHEAD); } From 3499d64247e31b7d81e9f3d5f1fe9beb67babe07 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 22:47:51 -0800 Subject: [PATCH 34/37] fix(net): a tail's fold spares only the stale drops its ledger accounted first The maintainer picked option 1 for how the subscriber's tail folds its finished streams under pool pressure. Instead of the next update owing nothing inside the folded span, a stale drop is judged when it arrives: a group in the span that the ledger already accounted (its header arrived, or the grace gave up on it) stays out of what the next update owes; any other drop is owed as before, so a gap the publisher re-serves after a widening is still waited for. `Tail::overtaken` (JS `Tail.overtaken`) reports those groups, `Tail::fold` splits the fold out of `rebill` for tests, and `SubscribeAccounting.tla` moves the check from `SubUpdate` to `OnDrop`. `SubscribeAccounting.cfg` passes with `Pressure = TRUE` (13.05M distinct states, TLC 1.8.0 on Java 21). Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-subscribe/SKILL.md | 6 +- js/net/src/cache.test.ts | 11 +- js/net/src/lite/subscriber.test.ts | 13 + js/net/src/lite/subscriber.ts | 14 +- js/net/src/tail.ts | 54 ++-- rs/moq-net/src/lite/subscriber.rs | 47 ++- rs/moq-net/src/model/track.rs | 15 +- rs/moq-net/src/tail.rs | 66 +++-- rs/moq-net/verify/subscribe/StreamWins.cfg | 2 +- .../verify/subscribe/SubscribeAccounting.tla | 13 +- .../SubscribeAccounting_TTrace_1794465460.bin | Bin 0 -> 858 bytes .../SubscribeAccounting_TTrace_1794465460.tla | 275 ++++++++++++++++++ 12 files changed, 439 insertions(+), 77 deletions(-) create mode 100644 rs/moq-net/verify/subscribe/SubscribeAccounting_TTrace_1794465460.bin create mode 100644 rs/moq-net/verify/subscribe/SubscribeAccounting_TTrace_1794465460.tla diff --git a/.claude/skills/verify-subscribe/SKILL.md b/.claude/skills/verify-subscribe/SKILL.md index 5ad2064107c..18aceaa782f 100644 --- a/.claude/skills/verify-subscribe/SKILL.md +++ b/.claude/skills/verify-subscribe/SKILL.md @@ -31,7 +31,7 @@ Read a counterexample with `/statechart`, passing it the spec, a config, and `Su ## Open finding: a group read whole is named in a stale drop -`StreamWins.cfg` fails. `GroupServe` in `GroupState::Closed` has written every frame and the FIN, and resets with OLD when the group goes stale before the acknowledgement; `settle_failed` then has `TrackRun::poll_step` name it in a stale drop under the update in effect, and record it in `expired`. The subscriber may already hold every byte. It ignores that drop only while the group is in `Tail::finished`, which `buffer_update` clears through `Tail::updated`, or while its copy still caches the group (`track::Producer::give_up` keeps an open group). So with an update sent between reading the FIN and the drop's arrival, and the copy having evicted the group, the subscriber records it stale and owes it again on the next update, while that update makes the publisher serve the whole group again (`reoffer_groups`) or drop it again under every later update. The draft says the publisher, having served a group, "neither serves it again nor names it in another drop". `residual_a_stale_unacknowledged_fin_drops_and_resends_a_served_group` in `lite/publisher.rs` replays the publisher's half, and `a_finished_group_wins_over_a_late_stale_drop_once_evicted` in `lite/subscriber.rs` pins the subscriber's half. Keeping `Tail::finished` across updates passes every rule in the model without `Pressure` (about 1.9M states). Under it, `Tail::rebill` still forgets a finished run by design: the overtaken drop records the group stale, though the update owes none of the span it forgot again. Once it is fixed, flip the residual test and drop the `\* Open finding:` line from `StreamWins.cfg`, which stays without `Pressure`. +`StreamWins.cfg` fails. `GroupServe` in `GroupState::Closed` has written every frame and the FIN, and resets with OLD when the group goes stale before the acknowledgement; `settle_failed` then has `TrackRun::poll_step` name it in a stale drop under the update in effect, and record it in `expired`. The subscriber may already hold every byte. It ignores that drop only while the group is in `Tail::finished`, which `buffer_update` clears through `Tail::updated`, or while its copy still caches the group (`track::Producer::give_up` keeps an open group). So with an update sent between reading the FIN and the drop's arrival, and the copy having evicted the group, the subscriber records it stale and owes it again on the next update, while that update makes the publisher serve the whole group again (`reoffer_groups`) or drop it again under every later update. The draft says the publisher, having served a group, "neither serves it again nor names it in another drop". `residual_a_stale_unacknowledged_fin_drops_and_resends_a_served_group` in `lite/publisher.rs` replays the publisher's half, and `a_finished_group_wins_over_a_late_stale_drop_once_evicted` in `lite/subscriber.rs` pins the subscriber's half. Keeping `Tail::finished` across updates passes every rule in the model without `Pressure` (about 1.9M states). Under it, `Tail::rebill` still forgets a finished run by design: the overtaken drop records the group stale, though `Tail::overtaken` keeps it out of what the next update owes, since the ledger accounted the group before the drop. Once it is fixed, flip the residual test and drop the `\* Open finding:` line from `StreamWins.cfg`, which stays without `Pressure`. ## Reading a failure @@ -52,9 +52,9 @@ Sync commit: `d9a88d2e3685f170e197c4e1623f26ccaa2c4c63`. Run `git log d9a88d2e36 | `rs/moq-net/src/lite/subscriber.rs` `ServeLoop::apply_response` (`Drop`, `Skip`) | `OnDrop`, `OnSkip`, `SubRecv` | | `rs/moq-net/src/lite/subscriber.rs` `buffer_update`; `rs/moq-net/src/model/track.rs` `supersede_stale` | `SubUpdate` | | `rs/moq-net/src/tail.rs` `Reading::open`, `Reading::finish`, `Tail::finished`, `updated`, `account`, `unaccount` | `acct`, `finished` | -| `rs/moq-net/src/tail.rs` `Tail::rebill`, `updated`; `rs/moq-net/src/lite/subscriber.rs` `buffer_update`; `rs/moq-net/src/model/record.rs` `Record::folded` | `TailFold`, `forgot`, `PubFold`, `folded` | +| `rs/moq-net/src/tail.rs` `Tail::rebill`, `overtaken`, `updated`; `rs/moq-net/src/lite/subscriber.rs` `ServeLoop::apply_response`; `rs/moq-net/src/model/record.rs` `Record::folded` | `TailFold`, `forgot`, `PubFold`, `folded` | | `rs/moq-net/src/model/track.rs` `Producer::give_up`, `skip_received` | `rec`, `cached` | ## Not covered -The subscription starts answered and covers the group, so SUBSCRIBE_OK, SUBSCRIBE_STARTED, ranges and Range IDs, fills (`RangeFills`), floors, and SUBSCRIBE_FINISH with `drop_unserved` are left out. The tail's grace and the ledger's folding are left out: `QuietIsSettled` asks for the publisher's answer, never the grace. So are priority and order (`Steer`), the Subscribe Stream's FIN and cancellation, sessions closing, versions before lite-07, relays forwarding drops downstream, datagrams, and the copy's readers (`poll_drop`). A route switch appears only as the cursor handing out once more a group it gave up. FETCH is left out: one request and one response, ended by a FIN or a reset, with nothing to race. A pass covers one group, four updates, and four Group Streams. +The subscription starts answered and covers the group, so SUBSCRIBE_OK, SUBSCRIBE_STARTED, ranges and Range IDs, fills (`RangeFills`), floors, and SUBSCRIBE_FINISH with `drop_unserved` are left out. The tail's grace and the ledger's folding are left out, so a group `Tail::overtaken` reports is one whose header arrived: `QuietIsSettled` asks for the publisher's answer, never the grace. So are priority and order (`Steer`), the Subscribe Stream's FIN and cancellation, sessions closing, versions before lite-07, relays forwarding drops downstream, datagrams, and the copy's readers (`poll_drop`). A route switch appears only as the cursor handing out once more a group it gave up. FETCH is left out: one request and one response, ended by a FIN or a reset, with nothing to race. A pass covers one group, four updates, and four Group Streams. diff --git a/js/net/src/cache.test.ts b/js/net/src/cache.test.ts index 63b9755660b..cc3e44c40c0 100644 --- a/js/net/src/cache.test.ts +++ b/js/net/src/cache.test.ts @@ -399,10 +399,13 @@ test("a tail forgets its finished streams under debt", () => { tail.finish(2n * runs); expect(tail.unfinished(0n, 2n * runs + 1n).length).toBeLessThanOrEqual(2); expect(pool.used).toBeLessThanOrEqual(capacity + 2 * (16 * 1024 + cache.ENTRY_OVERHEAD)); - // The update owes none of what the fold forgot again, since a stale drop there may be one its - // stream overtook. - expect(tail.updated()).toEqual({ start: 0n, end: 2n * runs - 1n }); - expect(tail.updated()).toBeUndefined(); + // A stale drop inside what the fold forgot is owed again only where the ledger had not accounted + // for it: there a stream may have finished and overtaken the drop. + tail.open(2n)(); + expect(tail.overtaken(0n, 4n)).toEqual([{ start: 2n, end: 3n }]); + expect(tail.overtaken(2n * runs, 2n * runs + 1n)).toEqual([]); + tail.updated(); + expect(tail.overtaken(0n, 4n)).toEqual([]); tail.close(); }); diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index f5acf8b215d..68aa3ee91b4 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -2706,6 +2706,7 @@ test("a finished group wins over a late stale drop once evicted on lite-07", asy track: Producer; tail: { covers(start: bigint, end: bigint): boolean; + fold(): boolean; }; }; let entry: Entry | undefined; @@ -2767,6 +2768,18 @@ test("a finished group wins over a late stale drop once evicted on lite-07", asy await send(old(2n, 2n)); await update(); expect(entry.tail.covers(2n, 3n)).toBe(false); + + // Under the pool's debt the tail folds what finished, gaps and all. A group in the span whose + // stream arrived before its stale drop still wins over it, as does a gap the grace gave up + // on; a gap there the drop accounted first is owed again. + for (const sequence of [3n, 5n]) await arrive(sequence); + await evict(); + for (const sequence of [11n, 13n, 15n]) await arrive(sequence); + expect(entry.tail.fold()).toBe(true); + await send(old(3n, 3n), old(4n, 3n), old(12n, 3n)); + await update(); + expect(entry.tail.covers(3n, 5n)).toBe(true); + expect(entry.tail.covers(12n, 13n)).toBe(false); } finally { responses.mockRestore(); track.close(); diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index c0b00267bff..af3ec8b35da 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -36,7 +36,6 @@ import { minDelayOf, type Run, removeRun, - runGaps, scopeCaptures, scopeHead, scopeOverlaps, @@ -2430,6 +2429,9 @@ export class Subscriber { // and a reader that wants it fills it meanwhile. if (stale && update < entry.update) continue; this.#reoffered(entry.track, start, end); + // Asked before this drop is accounted: what a fold forgot finishing that the ledger + // already accounted may be a stream this drop overtook. + const overtaken = stale ? entry.tail.overtaken(start, end) : []; entry.tail.account(start, end); // Judged under the budget this subscriber sent with that update. const hole = stale ? staleHole(entry.maxDelay ?? 0) : { error }; @@ -2440,8 +2442,10 @@ export class Subscriber { // A final drop is never reopened, even where it replaces a stale one. A sequence // already skipped stays skipped, and one whose group arrived keeps it, so no update // reopens either. + // Owed again by the next update, save what a stream may have won. + const owed = overtaken.reduce((runs, run) => removeRun(runs, run.start, run.end), recorded); entry.stale = stale - ? recorded.reduce((runs, run) => insertDrop(runs, run.start, run.end, hole), entry.stale ?? []) + ? owed.reduce((runs, run) => insertDrop(runs, run.start, run.end, hole), entry.stale ?? []) : removeRun(entry.stale ?? [], start, end); } else if ("skip" in resp) { // Never produced, so nothing is waited for and no reader ends. A skip is final and @@ -2626,8 +2630,7 @@ export class Subscriber { // unknown again: a reader fills them rather than trust an old verdict. entry.update = update.update; entry.maxDelay = update.maxDelay; - // What a fold forgot finishing may have overtaken its drop, so none of it is owed again. - const folded = entry.tail.updated(); + entry.tail.updated(); const held = hooks.supersedeStale(entry.track, judged(ranges)); // A lite-07 publisher re-offers every group it reported stale on any update, and drops // again under the new number what is still stale, so the tail owes every stale run again. @@ -2637,8 +2640,7 @@ export class Subscriber { for (const reopened of held) { const start = max(run.start, reopened.start); const end = min(run.end, reopened.end); - for (const part of runGaps(folded ? [folded] : [], start, end)) - entry.tail.unaccount(part.start, part.end); + if (start < end) entry.tail.unaccount(start, end); } } // Re-offered whether or not the copy still holds the stale verdict. diff --git a/js/net/src/tail.ts b/js/net/src/tail.ts index 8119d173467..02dd3e31650 100644 --- a/js/net/src/tail.ts +++ b/js/net/src/tail.ts @@ -39,7 +39,8 @@ export class Tail { // over it, even once the cache evicted it. #finished: Run[] = []; // The span of the finished runs a fold under the pool's debt forgot since the last update. A - // stale drop in it may be one its stream overtook, so the update owes none of it again. + // stale drop there for a group the ledger already accounted may be one its stream overtook, so + // the next update owes none of it again. #folded?: Run; // What `#finished` bills, `READER_RUN_OVERHEAD` per run, so a publisher fragmenting the // streams it finishes between updates costs what the cache would hold. @@ -50,22 +51,28 @@ export class Tail { this.#bill = options.bill; } - // Bill what `#finished` keeps, and under the pool's debt fold all but its newest run into - // `#folded`: a stale drop under this update then stands as the publisher judged it, but the - // update owes none of the folded span again. The newest run stays, so streams finishing in - // sequence grow nothing to charge again. Mirrors the Rust `Tail::rebill`. + // Bill what `#finished` keeps, and under the pool's debt fold it. Mirrors the Rust `Tail::rebill`. #rebill(): void { const bill = this.#bill; if (!bill) return; bill.set((this.#finished.length + (this.#folded ? 1 : 0)) * READER_RUN_OVERHEAD); - if (bill.debt > 0 && this.#finished.length > 1) { - const start = this.#finished[0].start; - const end = this.#finished[this.#finished.length - 2].end; - const folded = this.#folded; - this.#folded = folded ? { start: min(folded.start, start), end: max(folded.end, end) } : { start, end }; - this.#finished = this.#finished.slice(-1); - bill.set(2 * READER_RUN_OVERHEAD); - } + if (bill.debt > 0 && this.fold()) bill.set(2 * READER_RUN_OVERHEAD); + } + + /** + * Fold all but the newest finished run into the span a fold forgot, returning whether any was: + * a stale drop under this update then stands as the publisher judged it, but the update owes + * none of what it names that {@link overtaken} reports. The newest run stays, so streams + * finishing in sequence grow nothing to charge again. + */ + fold(): boolean { + if (this.#finished.length < 2) return false; + const start = this.#finished[0].start; + const end = this.#finished[this.#finished.length - 2].end; + const folded = this.#folded; + this.#folded = folded ? { start: min(folded.start, start), end: max(folded.end, end) } : { start, end }; + this.#finished = this.#finished.slice(-1); + return true; } /** Release what the tail bills, once its subscription ended. */ @@ -143,17 +150,28 @@ export class Tail { return runGaps(this.#finished, start, end); } + /** + * The parts of `[start, end)` inside the span a fold forgot that the ledger accounts for, asked + * before a stale drop of them is accounted: the header of each arrived, or the grace gave up on + * it, before the drop, so the drop may be one a finished stream overtook. + */ + overtaken(start: bigint, end: bigint): Run[] { + const folded = this.#folded; + if (!folded) return []; + const from = max(start, folded.start); + const to = min(end, folded.end); + if (from >= to) return []; + return runGaps(this.#ledger.gaps(from, to), from, to); + } + /** * Forget which streams finished, for an update being sent: a stale drop under an earlier one is - * no verdict, so none needs them. Returns the span a fold forgot since the last update, which - * the update owes none of again. + * no verdict, so none needs them. */ - updated(): Run | undefined { + updated(): void { this.#finished = []; - const folded = this.#folded; this.#folded = undefined; this.#rebill(); - return folded; } /** Re-check what {@link settle} waits on, after something outside the ledger changed. */ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 927635b992b..8cbbce1efd3 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -5679,6 +5679,21 @@ mod tests { !covers(2..3), "a drop under a later update is the publisher's to re-offer" ); + + // Under the pool's debt the tail folds what finished, gaps and all. A group in the span + // whose stream arrived before its stale drop still wins over it; a gap there the drop + // accounted first is owed again. + for sequence in [3, 5, 7] { + arrive(&mut serving, sequence).await; + } + if let Ok(mut state) = tail.write() { + assert!(state.fold()); + } + serving.apply_response(&serve, old(3, 3)).unwrap(); + serving.apply_response(&serve, old(4, 3)).unwrap(); + update(&mut serving); + assert!(covers(3..4), "the folded group that arrived stays accounted for"); + assert!(!covers(4..5), "the folded gap is the publisher's to re-offer"); } /// A SKIP, or any final drop, that follows an OLD drop of the same groups replaces it: an @@ -10542,18 +10557,10 @@ fn buffer_update( let held = serving.supersede_stale(judged(active.live.iter().map(|(_, range)| range))); let recorded = std::mem::take(&mut active.stale); if let Ok(mut tail) = active.tail.write() { - // What a fold forgot finishing may have overtaken its drop, so none of it is owed again. - let folded = tail.updated(); + tail.updated(); for run in recorded.iter() { for reopened in held.overlapping(run) { - let mut owed = RangeSet::new(); - owed.insert(reopened.start.max(run.start)..reopened.end.min(run.end)); - if let Some(folded) = &folded { - owed.remove(folded.clone()); - } - for owed in owed.iter() { - tail.unaccount(owed.clone(), now); - } + tail.unaccount(reopened.start.max(run.start)..reopened.end.min(run.end), now); } } } @@ -11946,6 +11953,12 @@ impl ServeLoop { // the publisher re-offers it or drops it again under the new number, so it is // still owed, and a reader that wants it fills it by range meanwhile. if !(stale && dropped.update < active.update) { + // Asked before this drop is accounted: what a fold forgot finishing that the + // ledger already accounted may be a stream this drop overtook. + let overtaken: Vec<_> = match stale { + true => active.tail.read().overtaken(groups.clone()).collect(), + false => Vec::new(), + }; if let Ok(mut tail) = active.tail.write() { tail.account(groups.clone(), serve.subscriber.runtime.now()); } @@ -11965,12 +11978,18 @@ impl ServeLoop { for finished in active.tail.read().finished(groups.clone()) { given.remove(finished); } + let mut owed = rangemap::RangeSet::new(); for given in given.iter() { - let recorded = self - .serving - .give_up(given.clone(), track::Hole::stale(active.max_delay)); - active.stale.extend(recorded); + owed.extend( + self.serving + .give_up(given.clone(), track::Hole::stale(active.max_delay)), + ); + } + // Owed again by the next update, save what a stream may have won. + for overtaken in overtaken { + owed.remove(overtaken); } + active.stale.extend(owed); } // Final, so no update reopens it, even where it replaces a stale drop. false => { diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index da215452fb9..ddd992be98b 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -24383,12 +24383,19 @@ mod test { "forgotten under debt" ); assert!(pool.used() <= capacity, "within the pool: {}", pool.used()); - // The update owes none of what the fold forgot again, since a stale drop there may be - // one its stream overtook. + // A stale drop inside what the fold forgot is owed again only where the ledger had not + // accounted for it: there a stream may have finished and overtaken the drop. + let overtaken: Vec<_> = tail.read().overtaken(0..4).collect(); + assert_eq!(overtaken, vec![0..1, 2..3], "the gap between them is still owed"); + assert_eq!( + tail.read().overtaken(2 * RUNS..2 * RUNS + 1).count(), + 0, + "the newest run stays" + ); if let Ok(mut state) = tail.write() { - assert_eq!(state.updated(), Some(0..2 * RUNS - 1)); - assert_eq!(state.updated(), None, "each update starts over"); + state.updated(); } + assert_eq!(tail.read().overtaken(0..4).count(), 0, "each update starts over"); } /// A flood a reader reported, fragmented into runs alternating with upstream skips, diff --git a/rs/moq-net/src/tail.rs b/rs/moq-net/src/tail.rs index a1a7ca0c466..3d4072ce5df 100644 --- a/rs/moq-net/src/tail.rs +++ b/rs/moq-net/src/tail.rs @@ -35,7 +35,8 @@ pub(crate) struct Tail { // group won over it, even once the cache evicted it. finished: RangeSet, // The span of the finished runs a fold under the pool's debt forgot since the last update. - // A stale drop in it may be one its stream overtook, so the update owes none of it again. + // A stale drop there for a group the ledger already accounted may be one its stream + // overtook, so the next update owes none of it again. folded: Option>, // What `finished` bills to the track's account, [`FINISHED_OVERHEAD`] per run, once the // subscription has a copy to bill. @@ -73,10 +74,7 @@ impl Tail { self.rebill(); } - /// Bill what `finished` keeps, and under the pool's debt fold all but its newest run into - /// `folded`: a stale drop under this update then stands as the publisher judged it, but the - /// update owes none of the folded span again. The newest run stays, so streams finishing in - /// sequence grow nothing to charge again. + /// Bill what `finished` keeps, and under the pool's debt [`Self::fold`] it. fn rebill(&mut self) { let Some(bill) = &mut self.bill else { return; @@ -84,23 +82,35 @@ impl Tail { let runs = self.finished.len() + usize::from(self.folded.is_some()); bill.set(runs as u64 * FINISHED_OVERHEAD); if bill.debt() > 0 - && self.finished.len() > 1 - && let (Some(first), Some(older), Some(newest)) = ( - self.finished.iter().next().cloned(), - self.finished.iter().nth_back(1).cloned(), - self.finished.iter().next_back().cloned(), - ) { - let span = match self.folded.take() { - Some(folded) => folded.start.min(first.start)..folded.end.max(older.end), - None => first.start..older.end, - }; - self.folded = Some(span); - self.finished = RangeSet::new(); - self.finished.insert(newest); + && self.fold() + && let Some(bill) = &mut self.bill + { bill.set(2 * FINISHED_OVERHEAD); } } + /// Fold all but the newest finished run into `folded`, returning whether any was: a stale + /// drop under this update then stands as the publisher judged it, but the update owes none + /// of what it names that [`Self::overtaken`] reports. The newest run stays, so streams + /// finishing in sequence grow nothing to charge again. + pub fn fold(&mut self) -> bool { + let (Some(first), Some(older), Some(newest)) = ( + self.finished.iter().next().cloned(), + self.finished.iter().nth_back(1).cloned(), + self.finished.iter().next_back().cloned(), + ) else { + return false; + }; + let span = match self.folded.take() { + Some(folded) => folded.start.min(first.start)..folded.end.max(older.end), + None => first.start..older.end, + }; + self.folded = Some(span); + self.finished = RangeSet::new(); + self.finished.insert(newest); + true + } + /// Record that the publisher's FIN ended the track. pub fn end(&mut self) { self.ended = true; @@ -162,14 +172,24 @@ impl Tail { self.finished.overlapping(groups).cloned() } + /// The runs of `groups` inside the span a fold forgot that the ledger accounts for, asked + /// before a stale drop of `groups` is accounted: the header of each arrived, or the grace gave + /// up on it, before the drop, so the drop may be one a finished stream overtook. + pub fn overtaken(&self, groups: Range) -> impl Iterator> + '_ { + let span = self + .folded + .as_ref() + .map(|folded| groups.start.max(folded.start)..groups.end.min(folded.end)) + .filter(|span| !span.is_empty()); + span.into_iter().flat_map(|span| self.ledger.accounted(span)) + } + /// Forget which streams finished, for an update being sent: a stale drop under an earlier - /// one is no verdict, so none needs them. Returns the span a fold forgot since the last - /// update, which the update owes none of again. - pub fn updated(&mut self) -> Option> { + /// one is no verdict, so none needs them. + pub fn updated(&mut self) { self.finished = RangeSet::new(); - let folded = self.folded.take(); + self.folded = None; self.rebill(); - folded } /// Whether a data stream whose header arrived is still being read. diff --git a/rs/moq-net/verify/subscribe/StreamWins.cfg b/rs/moq-net/verify/subscribe/StreamWins.cfg index 8374ed56b7b..3f73e53a54d 100644 --- a/rs/moq-net/verify/subscribe/StreamWins.cfg +++ b/rs/moq-net/verify/subscribe/StreamWins.cfg @@ -2,7 +2,7 @@ \* this line. \* Without pressure: under the pool's debt the tail forgets a finished stream by \* design, so a stale drop it overtook still records the group stale, though the -\* update owes it nothing again. +\* next update owes it nothing again, since the ledger accounted it first. CONSTANTS Groups = {0} MaxUpdates = 4 diff --git a/rs/moq-net/verify/subscribe/SubscribeAccounting.tla b/rs/moq-net/verify/subscribe/SubscribeAccounting.tla index 819cfc697b2..72e1d93b5e8 100644 --- a/rs/moq-net/verify/subscribe/SubscribeAccounting.tla +++ b/rs/moq-net/verify/subscribe/SubscribeAccounting.tla @@ -330,7 +330,8 @@ Evict(g) == \* The pool hands the tail debt and `Tail::rebill` folds every finished run but the newest \* into the span it forgot: one group stands for any of them, since a later group may have -\* finished, and the span takes in the gaps between them, so g need not have finished. +\* finished, and the span takes in the gaps between them, so g need not have finished or +\* even arrived. TailFold(g) == /\ Pressure /\ g \notin forgot @@ -342,6 +343,9 @@ TailFold(g) == \* verdict. Otherwise the tail accounts for the group; a stale drop records it in the \* copy unless its stream finished under this update or the copy holds it \* (`give_up` keeps an open group), and a final drop records it and ends its readers. +\* A stale drop of a group inside the span a fold forgot that the ledger already +\* accounted (`Tail::overtaken`) may be one a finished stream overtook, so the next +\* update owes it nothing again. OnDrop(m) == LET g == m.g stale == m.err = "old" @@ -354,9 +358,10 @@ OnDrop(m) == THEN LET recorded == g \notin finished /\ cached[g] \notin {"open", "whole"} /\ rec[g] # "skip" + owed == recorded /\ ~(g \in forgot /\ g \in acct) IN /\ acct' = acct \cup {g} /\ rec' = IF recorded THEN [rec EXCEPT ![g] = "stale"] ELSE rec - /\ sstale' = IF recorded THEN sstale \cup {g} ELSE sstale + /\ sstale' = IF owed THEN sstale \cup {g} ELSE sstale /\ UNCHANGED <> ELSE /\ acct' = acct \cup {g} /\ sstale' = sstale \ {g} @@ -382,7 +387,7 @@ SubRecv == \* `buffer_update`: the copy forgets every stale record (`supersede_stale`), the tail \* owes again each stale run this subscription accounted that the copy still held, -\* save the span a fold forgot, and `Tail::updated` forgets which streams finished. +\* and `Tail::updated` forgets which streams finished and what a fold forgot. SubUpdate == /\ sup < MaxUpdates /\ ~violation @@ -390,7 +395,7 @@ SubUpdate == IN /\ sup' = sup + 1 /\ ctrl' = Append(ctrl, sup + 1) /\ rec' = [g \in Groups |-> IF rec[g] = "stale" THEN "none" ELSE rec[g]] - /\ acct' = acct \ ((sstale \cap held) \ forgot) + /\ acct' = acct \ (sstale \cap held) /\ sstale' = {} /\ finished' = {} /\ forgot' = {} diff --git a/rs/moq-net/verify/subscribe/SubscribeAccounting_TTrace_1794465460.bin b/rs/moq-net/verify/subscribe/SubscribeAccounting_TTrace_1794465460.bin new file mode 100644 index 0000000000000000000000000000000000000000..739832abe7106c847f008bee570ef12412c0b0c1 GIT binary patch literal 858 zcmV-g1Eu^QiwFP!00000|Gid;QWHTCeH%7}L*zznkMcfB)WRx}%ez2CMFl|xwOo=3 z#ERLtyMTU~-{e=s?%kPX7m^@WAyv>jJv}||b-$S*hfcdztNm^PtoMk1T75ro$|@`0 z;(y215`WS4*4-fX&$G?&Z@Z`5Vo`aj5Kk>e7Ir@H%DKolBV+OBH@$XPmZgI;Qw*TS z-?^%)6iStK34X<6$!5Eo!FQeDtzeKf{$|xIwiaZe1bWJ^>cBZ3w~4>txz%a{*0P|L z$zrKe@;4GQAJ}%tx2N(caN`=BU2!W)BH!?!Q)EN|%+cI4wnxK)D{m_S<^|hs*+TuOmXc*%V%sChoP2(#536i@C%c`iV6uCqKCkjZMI;?LO*N`^vOCa>8JMKUQDH0V%UgbbmDU=y;0 zRzi-@MrbE=q_)4Ww{P8WDx84OML0_64g&~1gkC})p`S277$gi4juB4eV2Q4>0%Anx zu-0R|quRwJ)?}NzhNa7TlYlW3pgd0k%s5;tQoK)6V_M7T`2Lbyt}M!2q{h6y(aHwm})s~%zGz(pe5(V?S){E=b8 zCxS3W7$;1~>?C0-&4h68uqZIN95S&G9vVx-4R9#30|v>Diy~50W@!i^*jpz00g?L{Qv*} literal 0 HcmV?d00001 diff --git a/rs/moq-net/verify/subscribe/SubscribeAccounting_TTrace_1794465460.tla b/rs/moq-net/verify/subscribe/SubscribeAccounting_TTrace_1794465460.tla new file mode 100644 index 00000000000..9fbfa54d39b --- /dev/null +++ b/rs/moq-net/verify/subscribe/SubscribeAccounting_TTrace_1794465460.tla @@ -0,0 +1,275 @@ +---- MODULE SubscribeAccounting_TTrace_1794465460 ---- +EXTENDS SubscribeAccounting, Sequences, TLCExt, Toolbox, Naturals, TLC + +_expression == + LET SubscribeAccounting_TEExpression == INSTANCE SubscribeAccounting_TEExpression + IN SubscribeAccounting_TEExpression!expression +---- + +_trace == + LET SubscribeAccounting_TETrace == INSTANCE SubscribeAccounting_TETrace + IN SubscribeAccounting_TETrace!trace +---- + +_inv == + ~( + TLCGet("level") = Len(_TETrace) + /\ + cur = ((0 :> "handed")) + /\ + redrop = ({}) + /\ + ctrl = (<<>>) + /\ + resp = (<<>>) + /\ + streams = (<<[g |-> 0, pub |-> "reset", err |-> "old", wasFin |-> TRUE, sub |-> "fin"]>>) + /\ + dropped = ({0}) + /\ + finished = ({}) + /\ + pup = (1) + /\ + got = ({0}) + /\ + sstale = ({0}) + /\ + sup = (1) + /\ + rec = ((0 :> "stale")) + /\ + expired = ({0}) + /\ + late = ((0 :> FALSE)) + /\ + reoffered = ({}) + /\ + outq = (<<>>) + /\ + violation = (FALSE) + /\ + served = ({0}) + /\ + cached = ((0 :> "evicted")) + /\ + dstale = ({0}) + /\ + forgot = ({}) + /\ + folded = ({}) + /\ + acct = ({0}) + ) +---- + +_init == + /\ sstale = _TETrace[1].sstale + /\ forgot = _TETrace[1].forgot + /\ reoffered = _TETrace[1].reoffered + /\ ctrl = _TETrace[1].ctrl + /\ cur = _TETrace[1].cur + /\ outq = _TETrace[1].outq + /\ sup = _TETrace[1].sup + /\ resp = _TETrace[1].resp + /\ dropped = _TETrace[1].dropped + /\ finished = _TETrace[1].finished + /\ rec = _TETrace[1].rec + /\ streams = _TETrace[1].streams + /\ violation = _TETrace[1].violation + /\ late = _TETrace[1].late + /\ served = _TETrace[1].served + /\ expired = _TETrace[1].expired + /\ acct = _TETrace[1].acct + /\ got = _TETrace[1].got + /\ redrop = _TETrace[1].redrop + /\ cached = _TETrace[1].cached + /\ pup = _TETrace[1].pup + /\ dstale = _TETrace[1].dstale + /\ folded = _TETrace[1].folded +---- + +_next == + /\ \E i,j \in DOMAIN _TETrace: + /\ \/ /\ j = i + 1 + /\ i = TLCGet("level") + /\ sstale = _TETrace[i].sstale + /\ sstale' = _TETrace[j].sstale + /\ forgot = _TETrace[i].forgot + /\ forgot' = _TETrace[j].forgot + /\ reoffered = _TETrace[i].reoffered + /\ reoffered' = _TETrace[j].reoffered + /\ ctrl = _TETrace[i].ctrl + /\ ctrl' = _TETrace[j].ctrl + /\ cur = _TETrace[i].cur + /\ cur' = _TETrace[j].cur + /\ outq = _TETrace[i].outq + /\ outq' = _TETrace[j].outq + /\ sup = _TETrace[i].sup + /\ sup' = _TETrace[j].sup + /\ resp = _TETrace[i].resp + /\ resp' = _TETrace[j].resp + /\ dropped = _TETrace[i].dropped + /\ dropped' = _TETrace[j].dropped + /\ finished = _TETrace[i].finished + /\ finished' = _TETrace[j].finished + /\ rec = _TETrace[i].rec + /\ rec' = _TETrace[j].rec + /\ streams = _TETrace[i].streams + /\ streams' = _TETrace[j].streams + /\ violation = _TETrace[i].violation + /\ violation' = _TETrace[j].violation + /\ late = _TETrace[i].late + /\ late' = _TETrace[j].late + /\ served = _TETrace[i].served + /\ served' = _TETrace[j].served + /\ expired = _TETrace[i].expired + /\ expired' = _TETrace[j].expired + /\ acct = _TETrace[i].acct + /\ acct' = _TETrace[j].acct + /\ got = _TETrace[i].got + /\ got' = _TETrace[j].got + /\ redrop = _TETrace[i].redrop + /\ redrop' = _TETrace[j].redrop + /\ cached = _TETrace[i].cached + /\ cached' = _TETrace[j].cached + /\ pup = _TETrace[i].pup + /\ pup' = _TETrace[j].pup + /\ dstale = _TETrace[i].dstale + /\ dstale' = _TETrace[j].dstale + /\ folded = _TETrace[i].folded + /\ folded' = _TETrace[j].folded + +\* Uncomment the ASSUME below to write the states of the error trace +\* to the given file in Json format. Note that you can pass any tuple +\* to `JsonSerialize`. For example, a sub-sequence of _TETrace. + \* ASSUME + \* LET J == INSTANCE Json + \* IN J!JsonSerialize("SubscribeAccounting_TTrace_1794465460.json", _TETrace) + +============================================================================= + + Note that you can extract this module `SubscribeAccounting_TEExpression` + to a dedicated file to reuse `expression` (the module in the + dedicated `SubscribeAccounting_TEExpression.tla` file takes precedence + over the module `SubscribeAccounting_TEExpression` below). + +---- MODULE SubscribeAccounting_TEExpression ---- +EXTENDS SubscribeAccounting, Sequences, TLCExt, Toolbox, Naturals, TLC + +expression == + [ + \* To hide variables of the `SubscribeAccounting` spec from the error trace, + \* remove the variables below. The trace will be written in the order + \* of the fields of this record. + sstale |-> sstale + ,forgot |-> forgot + ,reoffered |-> reoffered + ,ctrl |-> ctrl + ,cur |-> cur + ,outq |-> outq + ,sup |-> sup + ,resp |-> resp + ,dropped |-> dropped + ,finished |-> finished + ,rec |-> rec + ,streams |-> streams + ,violation |-> violation + ,late |-> late + ,served |-> served + ,expired |-> expired + ,acct |-> acct + ,got |-> got + ,redrop |-> redrop + ,cached |-> cached + ,pup |-> pup + ,dstale |-> dstale + ,folded |-> folded + + \* Put additional constant-, state-, and action-level expressions here: + \* ,_stateNumber |-> _TEPosition + \* ,_sstaleUnchanged |-> sstale = sstale' + + \* Format the `sstale` variable as Json value. + \* ,_sstaleJson |-> + \* LET J == INSTANCE Json + \* IN J!ToJson(sstale) + + \* Lastly, you may build expressions over arbitrary sets of states by + \* leveraging the _TETrace operator. For example, this is how to + \* count the number of times a spec variable changed up to the current + \* state in the trace. + \* ,_sstaleModCount |-> + \* LET F[s \in DOMAIN _TETrace] == + \* IF s = 1 THEN 0 + \* ELSE IF _TETrace[s].sstale # _TETrace[s-1].sstale + \* THEN 1 + F[s-1] ELSE F[s-1] + \* IN F[_TEPosition - 1] + ] + +============================================================================= + + + +Parsing and semantic processing can take forever if the trace below is long. + In this case, it is advised to uncomment the module below to deserialize the + trace from a generated binary file. + +\* +\*---- MODULE SubscribeAccounting_TETrace ---- +\*EXTENDS SubscribeAccounting, IOUtils, TLC +\* +\*trace == IODeserialize("SubscribeAccounting_TTrace_1794465460.bin", TRUE) +\* +\*============================================================================= +\* + +---- MODULE SubscribeAccounting_TETrace ---- +EXTENDS SubscribeAccounting, TLC + +trace == + << + ([cur |-> (0 :> "pending"),redrop |-> {},ctrl |-> <<>>,resp |-> <<>>,streams |-> <<>>,dropped |-> {},finished |-> {},pup |-> 0,got |-> {},sstale |-> {},sup |-> 0,rec |-> (0 :> "none"),expired |-> {},late |-> (0 :> FALSE),reoffered |-> {},outq |-> <<>>,violation |-> FALSE,served |-> {},cached |-> (0 :> "none"),dstale |-> {},forgot |-> {},folded |-> {},acct |-> {}]), + ([cur |-> (0 :> "handed"),redrop |-> {},ctrl |-> <<>>,resp |-> <<>>,streams |-> <<[g |-> 0, pub |-> "open", err |-> "", wasFin |-> FALSE, sub |-> "none"]>>,dropped |-> {},finished |-> {},pup |-> 0,got |-> {},sstale |-> {},sup |-> 0,rec |-> (0 :> "none"),expired |-> {},late |-> (0 :> FALSE),reoffered |-> {},outq |-> <<>>,violation |-> FALSE,served |-> {0},cached |-> (0 :> "none"),dstale |-> {},forgot |-> {},folded |-> {},acct |-> {}]), + ([cur |-> (0 :> "handed"),redrop |-> {},ctrl |-> <<>>,resp |-> <<>>,streams |-> <<[g |-> 0, pub |-> "fin", err |-> "", wasFin |-> FALSE, sub |-> "none"]>>,dropped |-> {},finished |-> {},pup |-> 0,got |-> {},sstale |-> {},sup |-> 0,rec |-> (0 :> "none"),expired |-> {},late |-> (0 :> FALSE),reoffered |-> {},outq |-> <<>>,violation |-> FALSE,served |-> {0},cached |-> (0 :> "none"),dstale |-> {},forgot |-> {},folded |-> {},acct |-> {}]), + ([cur |-> (0 :> "handed"),redrop |-> {},ctrl |-> <<>>,resp |-> <<>>,streams |-> <<[g |-> 0, pub |-> "reset", err |-> "old", wasFin |-> TRUE, sub |-> "none"]>>,dropped |-> {},finished |-> {},pup |-> 0,got |-> {},sstale |-> {},sup |-> 0,rec |-> (0 :> "none"),expired |-> {},late |-> (0 :> FALSE),reoffered |-> {},outq |-> <<<<0, "old">>>>,violation |-> FALSE,served |-> {0},cached |-> (0 :> "none"),dstale |-> {},forgot |-> {},folded |-> {},acct |-> {}]), + ([cur |-> (0 :> "handed"),redrop |-> {},ctrl |-> <<>>,resp |-> <<>>,streams |-> <<[g |-> 0, pub |-> "reset", err |-> "old", wasFin |-> TRUE, sub |-> "header"]>>,dropped |-> {},finished |-> {},pup |-> 0,got |-> {},sstale |-> {},sup |-> 0,rec |-> (0 :> "none"),expired |-> {},late |-> (0 :> FALSE),reoffered |-> {},outq |-> <<<<0, "old">>>>,violation |-> FALSE,served |-> {0},cached |-> (0 :> "open"),dstale |-> {},forgot |-> {},folded |-> {},acct |-> {0}]), + ([cur |-> (0 :> "handed"),redrop |-> {},ctrl |-> <<>>,resp |-> <<>>,streams |-> <<[g |-> 0, pub |-> "reset", err |-> "old", wasFin |-> TRUE, sub |-> "fin"]>>,dropped |-> {},finished |-> {0},pup |-> 0,got |-> {0},sstale |-> {},sup |-> 0,rec |-> (0 :> "none"),expired |-> {},late |-> (0 :> FALSE),reoffered |-> {},outq |-> <<<<0, "old">>>>,violation |-> FALSE,served |-> {0},cached |-> (0 :> "whole"),dstale |-> {},forgot |-> {},folded |-> {},acct |-> {0}]), + ([cur |-> (0 :> "handed"),redrop |-> {},ctrl |-> <<>>,resp |-> <<>>,streams |-> <<[g |-> 0, pub |-> "reset", err |-> "old", wasFin |-> TRUE, sub |-> "fin"]>>,dropped |-> {},finished |-> {0},pup |-> 0,got |-> {0},sstale |-> {},sup |-> 0,rec |-> (0 :> "none"),expired |-> {},late |-> (0 :> FALSE),reoffered |-> {},outq |-> <<<<0, "old">>>>,violation |-> FALSE,served |-> {0},cached |-> (0 :> "evicted"),dstale |-> {},forgot |-> {},folded |-> {},acct |-> {0}]), + ([cur |-> (0 :> "handed"),redrop |-> {},ctrl |-> <<1>>,resp |-> <<>>,streams |-> <<[g |-> 0, pub |-> "reset", err |-> "old", wasFin |-> TRUE, sub |-> "fin"]>>,dropped |-> {},finished |-> {},pup |-> 0,got |-> {0},sstale |-> {},sup |-> 1,rec |-> (0 :> "none"),expired |-> {},late |-> (0 :> FALSE),reoffered |-> {},outq |-> <<<<0, "old">>>>,violation |-> FALSE,served |-> {0},cached |-> (0 :> "evicted"),dstale |-> {},forgot |-> {},folded |-> {},acct |-> {0}]), + ([cur |-> (0 :> "handed"),redrop |-> {},ctrl |-> <<>>,resp |-> <<>>,streams |-> <<[g |-> 0, pub |-> "reset", err |-> "old", wasFin |-> TRUE, sub |-> "fin"]>>,dropped |-> {},finished |-> {},pup |-> 1,got |-> {0},sstale |-> {},sup |-> 1,rec |-> (0 :> "none"),expired |-> {},late |-> (0 :> FALSE),reoffered |-> {},outq |-> <<<<0, "old">>>>,violation |-> FALSE,served |-> {0},cached |-> (0 :> "evicted"),dstale |-> {},forgot |-> {},folded |-> {},acct |-> {0}]), + ([cur |-> (0 :> "handed"),redrop |-> {},ctrl |-> <<>>,resp |-> <<[g |-> 0, err |-> "old", u |-> 1, type |-> "drop"]>>,streams |-> <<[g |-> 0, pub |-> "reset", err |-> "old", wasFin |-> TRUE, sub |-> "fin"]>>,dropped |-> {0},finished |-> {},pup |-> 1,got |-> {0},sstale |-> {},sup |-> 1,rec |-> (0 :> "none"),expired |-> {0},late |-> (0 :> FALSE),reoffered |-> {},outq |-> <<>>,violation |-> FALSE,served |-> {0},cached |-> (0 :> "evicted"),dstale |-> {0},forgot |-> {},folded |-> {},acct |-> {0}]), + ([cur |-> (0 :> "handed"),redrop |-> {},ctrl |-> <<>>,resp |-> <<>>,streams |-> <<[g |-> 0, pub |-> "reset", err |-> "old", wasFin |-> TRUE, sub |-> "fin"]>>,dropped |-> {0},finished |-> {},pup |-> 1,got |-> {0},sstale |-> {0},sup |-> 1,rec |-> (0 :> "stale"),expired |-> {0},late |-> (0 :> FALSE),reoffered |-> {},outq |-> <<>>,violation |-> FALSE,served |-> {0},cached |-> (0 :> "evicted"),dstale |-> {0},forgot |-> {},folded |-> {},acct |-> {0}]) + >> +---- + + +============================================================================= + +---- CONFIG SubscribeAccounting_TTrace_1794465460 ---- +CONSTANTS + Groups = { 0 } + MaxUpdates = 4 + MaxStreams = 4 + Pressure = FALSE + +INVARIANT + _inv + +CHECK_DEADLOCK + \* CHECK_DEADLOCK off because of PROPERTY or INVARIANT above. + FALSE + +INIT + _init + +NEXT + _next + +CONSTANT + _TETrace <- _trace + +ALIAS + _expression +============================================================================= +\* Generated on Wed Nov 11 22:37:40 PST 2026 \ No newline at end of file From 4692f94cca9a34d1099f2422c5b450cb509128ec Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 22:47:53 -0800 Subject: [PATCH 35/37] chore: drop TLC trace files committed by mistake Co-Authored-By: Claude Opus 5.5 --- .../SubscribeAccounting_TTrace_1794465460.bin | Bin 858 -> 0 bytes .../SubscribeAccounting_TTrace_1794465460.tla | 275 ------------------ 2 files changed, 275 deletions(-) delete mode 100644 rs/moq-net/verify/subscribe/SubscribeAccounting_TTrace_1794465460.bin delete mode 100644 rs/moq-net/verify/subscribe/SubscribeAccounting_TTrace_1794465460.tla diff --git a/rs/moq-net/verify/subscribe/SubscribeAccounting_TTrace_1794465460.bin b/rs/moq-net/verify/subscribe/SubscribeAccounting_TTrace_1794465460.bin deleted file mode 100644 index 739832abe7106c847f008bee570ef12412c0b0c1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 858 zcmV-g1Eu^QiwFP!00000|Gid;QWHTCeH%7}L*zznkMcfB)WRx}%ez2CMFl|xwOo=3 z#ERLtyMTU~-{e=s?%kPX7m^@WAyv>jJv}||b-$S*hfcdztNm^PtoMk1T75ro$|@`0 z;(y215`WS4*4-fX&$G?&Z@Z`5Vo`aj5Kk>e7Ir@H%DKolBV+OBH@$XPmZgI;Qw*TS z-?^%)6iStK34X<6$!5Eo!FQeDtzeKf{$|xIwiaZe1bWJ^>cBZ3w~4>txz%a{*0P|L z$zrKe@;4GQAJ}%tx2N(caN`=BU2!W)BH!?!Q)EN|%+cI4wnxK)D{m_S<^|hs*+TuOmXc*%V%sChoP2(#536i@C%c`iV6uCqKCkjZMI;?LO*N`^vOCa>8JMKUQDH0V%UgbbmDU=y;0 zRzi-@MrbE=q_)4Ww{P8WDx84OML0_64g&~1gkC})p`S277$gi4juB4eV2Q4>0%Anx zu-0R|quRwJ)?}NzhNa7TlYlW3pgd0k%s5;tQoK)6V_M7T`2Lbyt}M!2q{h6y(aHwm})s~%zGz(pe5(V?S){E=b8 zCxS3W7$;1~>?C0-&4h68uqZIN95S&G9vVx-4R9#30|v>Diy~50W@!i^*jpz00g?L{Qv*} diff --git a/rs/moq-net/verify/subscribe/SubscribeAccounting_TTrace_1794465460.tla b/rs/moq-net/verify/subscribe/SubscribeAccounting_TTrace_1794465460.tla deleted file mode 100644 index 9fbfa54d39b..00000000000 --- a/rs/moq-net/verify/subscribe/SubscribeAccounting_TTrace_1794465460.tla +++ /dev/null @@ -1,275 +0,0 @@ ----- MODULE SubscribeAccounting_TTrace_1794465460 ---- -EXTENDS SubscribeAccounting, Sequences, TLCExt, Toolbox, Naturals, TLC - -_expression == - LET SubscribeAccounting_TEExpression == INSTANCE SubscribeAccounting_TEExpression - IN SubscribeAccounting_TEExpression!expression ----- - -_trace == - LET SubscribeAccounting_TETrace == INSTANCE SubscribeAccounting_TETrace - IN SubscribeAccounting_TETrace!trace ----- - -_inv == - ~( - TLCGet("level") = Len(_TETrace) - /\ - cur = ((0 :> "handed")) - /\ - redrop = ({}) - /\ - ctrl = (<<>>) - /\ - resp = (<<>>) - /\ - streams = (<<[g |-> 0, pub |-> "reset", err |-> "old", wasFin |-> TRUE, sub |-> "fin"]>>) - /\ - dropped = ({0}) - /\ - finished = ({}) - /\ - pup = (1) - /\ - got = ({0}) - /\ - sstale = ({0}) - /\ - sup = (1) - /\ - rec = ((0 :> "stale")) - /\ - expired = ({0}) - /\ - late = ((0 :> FALSE)) - /\ - reoffered = ({}) - /\ - outq = (<<>>) - /\ - violation = (FALSE) - /\ - served = ({0}) - /\ - cached = ((0 :> "evicted")) - /\ - dstale = ({0}) - /\ - forgot = ({}) - /\ - folded = ({}) - /\ - acct = ({0}) - ) ----- - -_init == - /\ sstale = _TETrace[1].sstale - /\ forgot = _TETrace[1].forgot - /\ reoffered = _TETrace[1].reoffered - /\ ctrl = _TETrace[1].ctrl - /\ cur = _TETrace[1].cur - /\ outq = _TETrace[1].outq - /\ sup = _TETrace[1].sup - /\ resp = _TETrace[1].resp - /\ dropped = _TETrace[1].dropped - /\ finished = _TETrace[1].finished - /\ rec = _TETrace[1].rec - /\ streams = _TETrace[1].streams - /\ violation = _TETrace[1].violation - /\ late = _TETrace[1].late - /\ served = _TETrace[1].served - /\ expired = _TETrace[1].expired - /\ acct = _TETrace[1].acct - /\ got = _TETrace[1].got - /\ redrop = _TETrace[1].redrop - /\ cached = _TETrace[1].cached - /\ pup = _TETrace[1].pup - /\ dstale = _TETrace[1].dstale - /\ folded = _TETrace[1].folded ----- - -_next == - /\ \E i,j \in DOMAIN _TETrace: - /\ \/ /\ j = i + 1 - /\ i = TLCGet("level") - /\ sstale = _TETrace[i].sstale - /\ sstale' = _TETrace[j].sstale - /\ forgot = _TETrace[i].forgot - /\ forgot' = _TETrace[j].forgot - /\ reoffered = _TETrace[i].reoffered - /\ reoffered' = _TETrace[j].reoffered - /\ ctrl = _TETrace[i].ctrl - /\ ctrl' = _TETrace[j].ctrl - /\ cur = _TETrace[i].cur - /\ cur' = _TETrace[j].cur - /\ outq = _TETrace[i].outq - /\ outq' = _TETrace[j].outq - /\ sup = _TETrace[i].sup - /\ sup' = _TETrace[j].sup - /\ resp = _TETrace[i].resp - /\ resp' = _TETrace[j].resp - /\ dropped = _TETrace[i].dropped - /\ dropped' = _TETrace[j].dropped - /\ finished = _TETrace[i].finished - /\ finished' = _TETrace[j].finished - /\ rec = _TETrace[i].rec - /\ rec' = _TETrace[j].rec - /\ streams = _TETrace[i].streams - /\ streams' = _TETrace[j].streams - /\ violation = _TETrace[i].violation - /\ violation' = _TETrace[j].violation - /\ late = _TETrace[i].late - /\ late' = _TETrace[j].late - /\ served = _TETrace[i].served - /\ served' = _TETrace[j].served - /\ expired = _TETrace[i].expired - /\ expired' = _TETrace[j].expired - /\ acct = _TETrace[i].acct - /\ acct' = _TETrace[j].acct - /\ got = _TETrace[i].got - /\ got' = _TETrace[j].got - /\ redrop = _TETrace[i].redrop - /\ redrop' = _TETrace[j].redrop - /\ cached = _TETrace[i].cached - /\ cached' = _TETrace[j].cached - /\ pup = _TETrace[i].pup - /\ pup' = _TETrace[j].pup - /\ dstale = _TETrace[i].dstale - /\ dstale' = _TETrace[j].dstale - /\ folded = _TETrace[i].folded - /\ folded' = _TETrace[j].folded - -\* Uncomment the ASSUME below to write the states of the error trace -\* to the given file in Json format. Note that you can pass any tuple -\* to `JsonSerialize`. For example, a sub-sequence of _TETrace. - \* ASSUME - \* LET J == INSTANCE Json - \* IN J!JsonSerialize("SubscribeAccounting_TTrace_1794465460.json", _TETrace) - -============================================================================= - - Note that you can extract this module `SubscribeAccounting_TEExpression` - to a dedicated file to reuse `expression` (the module in the - dedicated `SubscribeAccounting_TEExpression.tla` file takes precedence - over the module `SubscribeAccounting_TEExpression` below). - ----- MODULE SubscribeAccounting_TEExpression ---- -EXTENDS SubscribeAccounting, Sequences, TLCExt, Toolbox, Naturals, TLC - -expression == - [ - \* To hide variables of the `SubscribeAccounting` spec from the error trace, - \* remove the variables below. The trace will be written in the order - \* of the fields of this record. - sstale |-> sstale - ,forgot |-> forgot - ,reoffered |-> reoffered - ,ctrl |-> ctrl - ,cur |-> cur - ,outq |-> outq - ,sup |-> sup - ,resp |-> resp - ,dropped |-> dropped - ,finished |-> finished - ,rec |-> rec - ,streams |-> streams - ,violation |-> violation - ,late |-> late - ,served |-> served - ,expired |-> expired - ,acct |-> acct - ,got |-> got - ,redrop |-> redrop - ,cached |-> cached - ,pup |-> pup - ,dstale |-> dstale - ,folded |-> folded - - \* Put additional constant-, state-, and action-level expressions here: - \* ,_stateNumber |-> _TEPosition - \* ,_sstaleUnchanged |-> sstale = sstale' - - \* Format the `sstale` variable as Json value. - \* ,_sstaleJson |-> - \* LET J == INSTANCE Json - \* IN J!ToJson(sstale) - - \* Lastly, you may build expressions over arbitrary sets of states by - \* leveraging the _TETrace operator. For example, this is how to - \* count the number of times a spec variable changed up to the current - \* state in the trace. - \* ,_sstaleModCount |-> - \* LET F[s \in DOMAIN _TETrace] == - \* IF s = 1 THEN 0 - \* ELSE IF _TETrace[s].sstale # _TETrace[s-1].sstale - \* THEN 1 + F[s-1] ELSE F[s-1] - \* IN F[_TEPosition - 1] - ] - -============================================================================= - - - -Parsing and semantic processing can take forever if the trace below is long. - In this case, it is advised to uncomment the module below to deserialize the - trace from a generated binary file. - -\* -\*---- MODULE SubscribeAccounting_TETrace ---- -\*EXTENDS SubscribeAccounting, IOUtils, TLC -\* -\*trace == IODeserialize("SubscribeAccounting_TTrace_1794465460.bin", TRUE) -\* -\*============================================================================= -\* - ----- MODULE SubscribeAccounting_TETrace ---- -EXTENDS SubscribeAccounting, TLC - -trace == - << - ([cur |-> (0 :> "pending"),redrop |-> {},ctrl |-> <<>>,resp |-> <<>>,streams |-> <<>>,dropped |-> {},finished |-> {},pup |-> 0,got |-> {},sstale |-> {},sup |-> 0,rec |-> (0 :> "none"),expired |-> {},late |-> (0 :> FALSE),reoffered |-> {},outq |-> <<>>,violation |-> FALSE,served |-> {},cached |-> (0 :> "none"),dstale |-> {},forgot |-> {},folded |-> {},acct |-> {}]), - ([cur |-> (0 :> "handed"),redrop |-> {},ctrl |-> <<>>,resp |-> <<>>,streams |-> <<[g |-> 0, pub |-> "open", err |-> "", wasFin |-> FALSE, sub |-> "none"]>>,dropped |-> {},finished |-> {},pup |-> 0,got |-> {},sstale |-> {},sup |-> 0,rec |-> (0 :> "none"),expired |-> {},late |-> (0 :> FALSE),reoffered |-> {},outq |-> <<>>,violation |-> FALSE,served |-> {0},cached |-> (0 :> "none"),dstale |-> {},forgot |-> {},folded |-> {},acct |-> {}]), - ([cur |-> (0 :> "handed"),redrop |-> {},ctrl |-> <<>>,resp |-> <<>>,streams |-> <<[g |-> 0, pub |-> "fin", err |-> "", wasFin |-> FALSE, sub |-> "none"]>>,dropped |-> {},finished |-> {},pup |-> 0,got |-> {},sstale |-> {},sup |-> 0,rec |-> (0 :> "none"),expired |-> {},late |-> (0 :> FALSE),reoffered |-> {},outq |-> <<>>,violation |-> FALSE,served |-> {0},cached |-> (0 :> "none"),dstale |-> {},forgot |-> {},folded |-> {},acct |-> {}]), - ([cur |-> (0 :> "handed"),redrop |-> {},ctrl |-> <<>>,resp |-> <<>>,streams |-> <<[g |-> 0, pub |-> "reset", err |-> "old", wasFin |-> TRUE, sub |-> "none"]>>,dropped |-> {},finished |-> {},pup |-> 0,got |-> {},sstale |-> {},sup |-> 0,rec |-> (0 :> "none"),expired |-> {},late |-> (0 :> FALSE),reoffered |-> {},outq |-> <<<<0, "old">>>>,violation |-> FALSE,served |-> {0},cached |-> (0 :> "none"),dstale |-> {},forgot |-> {},folded |-> {},acct |-> {}]), - ([cur |-> (0 :> "handed"),redrop |-> {},ctrl |-> <<>>,resp |-> <<>>,streams |-> <<[g |-> 0, pub |-> "reset", err |-> "old", wasFin |-> TRUE, sub |-> "header"]>>,dropped |-> {},finished |-> {},pup |-> 0,got |-> {},sstale |-> {},sup |-> 0,rec |-> (0 :> "none"),expired |-> {},late |-> (0 :> FALSE),reoffered |-> {},outq |-> <<<<0, "old">>>>,violation |-> FALSE,served |-> {0},cached |-> (0 :> "open"),dstale |-> {},forgot |-> {},folded |-> {},acct |-> {0}]), - ([cur |-> (0 :> "handed"),redrop |-> {},ctrl |-> <<>>,resp |-> <<>>,streams |-> <<[g |-> 0, pub |-> "reset", err |-> "old", wasFin |-> TRUE, sub |-> "fin"]>>,dropped |-> {},finished |-> {0},pup |-> 0,got |-> {0},sstale |-> {},sup |-> 0,rec |-> (0 :> "none"),expired |-> {},late |-> (0 :> FALSE),reoffered |-> {},outq |-> <<<<0, "old">>>>,violation |-> FALSE,served |-> {0},cached |-> (0 :> "whole"),dstale |-> {},forgot |-> {},folded |-> {},acct |-> {0}]), - ([cur |-> (0 :> "handed"),redrop |-> {},ctrl |-> <<>>,resp |-> <<>>,streams |-> <<[g |-> 0, pub |-> "reset", err |-> "old", wasFin |-> TRUE, sub |-> "fin"]>>,dropped |-> {},finished |-> {0},pup |-> 0,got |-> {0},sstale |-> {},sup |-> 0,rec |-> (0 :> "none"),expired |-> {},late |-> (0 :> FALSE),reoffered |-> {},outq |-> <<<<0, "old">>>>,violation |-> FALSE,served |-> {0},cached |-> (0 :> "evicted"),dstale |-> {},forgot |-> {},folded |-> {},acct |-> {0}]), - ([cur |-> (0 :> "handed"),redrop |-> {},ctrl |-> <<1>>,resp |-> <<>>,streams |-> <<[g |-> 0, pub |-> "reset", err |-> "old", wasFin |-> TRUE, sub |-> "fin"]>>,dropped |-> {},finished |-> {},pup |-> 0,got |-> {0},sstale |-> {},sup |-> 1,rec |-> (0 :> "none"),expired |-> {},late |-> (0 :> FALSE),reoffered |-> {},outq |-> <<<<0, "old">>>>,violation |-> FALSE,served |-> {0},cached |-> (0 :> "evicted"),dstale |-> {},forgot |-> {},folded |-> {},acct |-> {0}]), - ([cur |-> (0 :> "handed"),redrop |-> {},ctrl |-> <<>>,resp |-> <<>>,streams |-> <<[g |-> 0, pub |-> "reset", err |-> "old", wasFin |-> TRUE, sub |-> "fin"]>>,dropped |-> {},finished |-> {},pup |-> 1,got |-> {0},sstale |-> {},sup |-> 1,rec |-> (0 :> "none"),expired |-> {},late |-> (0 :> FALSE),reoffered |-> {},outq |-> <<<<0, "old">>>>,violation |-> FALSE,served |-> {0},cached |-> (0 :> "evicted"),dstale |-> {},forgot |-> {},folded |-> {},acct |-> {0}]), - ([cur |-> (0 :> "handed"),redrop |-> {},ctrl |-> <<>>,resp |-> <<[g |-> 0, err |-> "old", u |-> 1, type |-> "drop"]>>,streams |-> <<[g |-> 0, pub |-> "reset", err |-> "old", wasFin |-> TRUE, sub |-> "fin"]>>,dropped |-> {0},finished |-> {},pup |-> 1,got |-> {0},sstale |-> {},sup |-> 1,rec |-> (0 :> "none"),expired |-> {0},late |-> (0 :> FALSE),reoffered |-> {},outq |-> <<>>,violation |-> FALSE,served |-> {0},cached |-> (0 :> "evicted"),dstale |-> {0},forgot |-> {},folded |-> {},acct |-> {0}]), - ([cur |-> (0 :> "handed"),redrop |-> {},ctrl |-> <<>>,resp |-> <<>>,streams |-> <<[g |-> 0, pub |-> "reset", err |-> "old", wasFin |-> TRUE, sub |-> "fin"]>>,dropped |-> {0},finished |-> {},pup |-> 1,got |-> {0},sstale |-> {0},sup |-> 1,rec |-> (0 :> "stale"),expired |-> {0},late |-> (0 :> FALSE),reoffered |-> {},outq |-> <<>>,violation |-> FALSE,served |-> {0},cached |-> (0 :> "evicted"),dstale |-> {0},forgot |-> {},folded |-> {},acct |-> {0}]) - >> ----- - - -============================================================================= - ----- CONFIG SubscribeAccounting_TTrace_1794465460 ---- -CONSTANTS - Groups = { 0 } - MaxUpdates = 4 - MaxStreams = 4 - Pressure = FALSE - -INVARIANT - _inv - -CHECK_DEADLOCK - \* CHECK_DEADLOCK off because of PROPERTY or INVARIANT above. - FALSE - -INIT - _init - -NEXT - _next - -CONSTANT - _TETrace <- _trace - -ALIAS - _expression -============================================================================= -\* Generated on Wed Nov 11 22:37:40 PST 2026 \ No newline at end of file From 89ff1b1753988b35580912cdb30cf06e78aefbc2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 22:48:08 -0800 Subject: [PATCH 36/37] quest: a FETCH of a failed group keeps its frames reclaimable Co-Authored-By: Claude Opus 5.5 --- quest/m1/fetch-mirror-owner.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 quest/m1/fetch-mirror-owner.md diff --git a/quest/m1/fetch-mirror-owner.md b/quest/m1/fetch-mirror-owner.md new file mode 100644 index 00000000000..6a7a54b1cf2 --- /dev/null +++ b/quest/m1/fetch-mirror-owner.md @@ -0,0 +1,12 @@ +# [S] A FETCH of a failed group keeps its frames reclaimable + +## Goal + +A JS FETCH caller holding a failed group (`fetchGroup` in `js/net/src/broadcast.ts`) keeps that group's mirror registered in its cache entry (`entry.mirrors` in `js/net/src/track.ts`) until the caller drains or releases it. +The unread frames it holds then stay billed to the pool and reachable by its eviction, like any other reader's. + +## Plan + +- Found in review of [moq-dev/moq#8476](https://github.com/moq-dev/moq/pull/8476); `main` has the same shape. `fetchGroup` closes its subscription once the returned group closes, and closing the sink deletes its mirror from `entry.mirrors`. A group that fails closes at once, so the frames its caller has not read yet leave the entry's reach while the caller still holds them. This holds for any failed group, including a live group the publisher cuts. +- Tie the mirror's release to the caller's group consumer finishing (drained or closed by the caller), not to the group's own close. +- Test it on mock time: a pool under debt evicts the unread frames of a failed group a FETCH caller still holds. From 79e39da584100267e359b91c8aa3c93fa013c17e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 23:09:22 -0800 Subject: [PATCH 37/37] fix(net): a closed JS reader bills nothing, and drop runs append without spreading A read still pending when a reader closes no longer opens a new bill for its record. insertDrops and the refused-run lookup append runs one by one, so a peer controlling how many there are cannot push past an engine's argument limit. Co-Authored-By: Claude Opus 5.5 --- js/net/src/cache.test.ts | 19 +++++++++++++++++++ js/net/src/internal.ts | 3 ++- js/net/src/track.ts | 4 +++- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/js/net/src/cache.test.ts b/js/net/src/cache.test.ts index cc3e44c40c0..df22fc88c58 100644 --- a/js/net/src/cache.test.ts +++ b/js/net/src/cache.test.ts @@ -378,6 +378,25 @@ for (const stale of [false, true]) { }); } +// A reader that closes while a read waits releases its record's bill for good: the read that wakes +// after the close bills nothing again. +test("a closed reader bills nothing from a read still pending", async () => { + const pool = new cache.Pool({ capacity: 1 << 30 }); + const producer = pooled(pool); + hooks.receiveTrack(producer); + const reader = producer.subscribe({ delay: { max: null } }); + for (let run = 0n; run < 64n; run++) { + hooks.dropTrackGroups(producer, 2n * run, 2n * run + 1n, { error: new StreamError(StreamCode.Cancel) }); + producer.skipGroups(2n * run + 1n, 2n * run + 2n); + } + while (hooks.tryDrop(reader)); + const billed = pool.used; + const read = reader.recvGroup(); + reader.close(); + await read; + expect(pool.used).toBeLessThan(billed - 32 * cache.READER_RUN_OVERHEAD); +}); + // A subscription's tail, recording a fragmented run of finished streams between updates, bills it to // its track's account. Under the pool's debt it forgets them and stays within the pool. Mirrors Rust // `a_tail_forgets_its_finished_streams_under_debt`. diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index d4fcdf5ce13..6081081370c 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -289,7 +289,8 @@ export function subtractRuns(runs: readonly R[], cut: readonly Ru export function insertDrops(drops: readonly DropRun[], runs: readonly DropRun[]): DropRun[] { if (runs.length === 0) return [...drops]; const next = subtractRuns(drops, runs); - next.push(...runs); + // Appended one by one: a peer controls how many runs there are, past any argument limit. + for (const run of runs) next.push(run); next.sort((a, b) => (a.start < b.start ? -1 : a.start > b.start ? 1 : 0)); const merged: DropRun[] = []; for (const run of next) { diff --git a/js/net/src/track.ts b/js/net/src/track.ts index d67e5ad1326..cbf920d49f0 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -919,7 +919,7 @@ function givenUp( if (refused.length === 0) return found; const own = refused.filter((run) => run.start < end && start < run.end); const hiding = found.filter((run) => settles({ ...run, start: max(run.start, start) })); - found.push(...subtractRuns(own, hiding)); + for (const run of subtractRuns(own, hiding)) found.push(run); return found.sort((a, b) => (a.start < b.start ? -1 : a.start > b.start ? 1 : 0)); } @@ -3139,6 +3139,8 @@ export class Subscriber { // #fold). Mirrors the Rust `Record::settle`. #settleRecord(handed?: bigint): TooFarBehind | undefined { if (this.#lagged) return new TooFarBehind(); + // A closed reader released its bill for good; a read still pending must not open another. + if (this.#left) return undefined; const bytes = this.#recordRuns() * cache.READER_RUN_OVERHEAD; if (!this.#bill && bytes === 0) return undefined; const bill = this.#billOn(this.#bill);