From 00f646d4103eba88e232e36a8716a25b774f2f0d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 17:33:55 -0800 Subject: [PATCH 1/3] chore(quest): claim quest/m1/precut-fixes/tail-ledger-bound.md Quest-Claim: f5c4bb03f4d680f9 From e3ffd26729ad00552289163e9020b9cd79363db7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 17:41:46 -0800 Subject: [PATCH 2/3] fix(net): a subscription's drop ledger is billed and folds under memory pressure The session's tail ledger recorded every range a publisher's SUBSCRIBE_DROP and SUBSCRIBE_SKIP named, unbilled, and with no max age and no max delay its grace never passed, so alternating single-group drops grew relay memory for the life of the subscription. Bill the ledger's gaps to the copy's cache account as a reader's record is, and fold the oldest gaps early while the pool owes the readers a debt. Co-Authored-By: Claude Sonnet 5.5 --- quest/m1/precut-fixes/tail-ledger-bound.md | 40 --------- rs/moq-net/src/lite/subscriber.rs | 17 ++-- rs/moq-net/src/model/cache.rs | 14 +++- rs/moq-net/src/model/track.rs | 6 ++ rs/moq-net/src/tail.rs | 94 +++++++++++++++++++++- 5 files changed, 119 insertions(+), 52 deletions(-) delete mode 100644 quest/m1/precut-fixes/tail-ledger-bound.md diff --git a/quest/m1/precut-fixes/tail-ledger-bound.md b/quest/m1/precut-fixes/tail-ledger-bound.md deleted file mode 100644 index 0ef24621ea7..00000000000 --- a/quest/m1/precut-fixes/tail-ledger-bound.md +++ /dev/null @@ -1,40 +0,0 @@ -# [S] A subscription's drop ledger is billed and folds under memory pressure - -## Goal - -The runs a publisher's SUBSCRIBE_DROP and SUBSCRIBE_SKIP messages add to a -relay subscription's tail ledger are charged to that copy's account and fold -away under cache-pool pressure, so a publisher sending alternating -single-group drops cannot grow relay memory for the life of a subscription -that has no max age and no max delay. - -## Plan - -Found by the 2026-11-17 pre-cut correctness audit; confirmed from source. -`Tail::account` (`rs/moq-net/src/tail.rs`) records any range the peer names, -and `rebill` charges only `finished`, never the ledger's runs. With no track -max age, no `cache.duration`, and no viewer max delay, the grace is -`Duration::MAX`, so `ledger.rs` never folds a run: each 4 to 6 wire bytes of -drop leaves about 160 B on the relay. On `release` the ledger was unbilled -too, but an unset max age gave a 1 s grace. - -Recommended: bill the ledger's runs to the copy's account and fold the -oldest gap with `Ledger::fold_oldest` under pressure, as the track's own -arrivals ledger already does (rejected: a cap on runs, which the review -rules refuse). This holds whatever -[the tail drop count](/quest/m3/lite-tail-drop-count.md) later decides about -the grace. - -Test: apply N disjoint drops on a track with no max age and no max delay, -advance the simulated clock, apply pool pressure, and assert the ledger -folds. - -Public API: none. Wire: none. - -Caused by, for the fix PR's `## Caused by`: - -- [moq-dev/moq#7597](https://github.com/moq-dev/moq/pull/7597) - made an unset max age an unbounded grace - -## Related - -- [Tail drop count](/quest/m3/lite-tail-drop-count.md) - removes the grace from deciding the tail diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 36f45b791e9..1f165663a47 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -11197,7 +11197,7 @@ impl TrackServe { let widened = (changed && demand.is_some()) .then(|| subscription.start().or(active.started.flatten()).unwrap_or(0)..u64::MAX); if let Ok(mut tail) = active.tail.write() { - tail.set_grace(producer.tail_grace(subscription.budget())); + tail.follow(producer, subscription.budget()); let now = self.subscriber.runtime.now(); for groups in lowered.into_iter().chain(widened) { tail.demand(groups, now); @@ -11387,11 +11387,12 @@ impl TrackServe { tracing::debug!(id, broadcast = %self.subscriber.log_path(&self.path), track = %self.name, "subscribe started"); // A pipelined SUBSCRIBE has no copy, so no max age, until it is adopted. - let grace = match © { - Some(copy) => copy.producer.tail_grace(subscription.budget()), - None => tail::grace(subscription.budget()), - }; - let tail = kio::Producer::new(Tail::new(grace)); + let tail = kio::Producer::new(Tail::new(tail::grace(subscription.budget()))); + if let Some(copy) = © + && let Ok(mut tail) = tail.write() + { + tail.follow(©.producer, subscription.budget()); + } let mut entry = TrackEntry::new(copy, tail.clone()); let version = self.subscriber.version; if version.has_datagrams() && !version.has_datagram_tracks() { @@ -12764,7 +12765,7 @@ impl ServeLoop { PipelinedState::Open(active) => &active.tail, }; if let Ok(mut tail) = tail.write() { - tail.set_grace(self.serving.tail_grace(pipelined.pref.budget())); + tail.follow(&self.serving, pipelined.pref.budget()); } self.serving.seen_subscription(Some(pipelined.pref)); match pipelined.state { @@ -13069,7 +13070,7 @@ impl ServeLoop { if !serve.subscriber.version.has_track_stream() { tail.end(); } - tail.set_grace(serving.tail_grace(budget.unwrap_or_default())); + tail.follow(serving, budget.unwrap_or_default()); tail.expire(serve.subscriber.runtime.now()); } Settle::new(&serve.subscriber.runtime, active.tail.consume()) diff --git a/rs/moq-net/src/model/cache.rs b/rs/moq-net/src/model/cache.rs index fefec59f182..c5e5a92599c 100644 --- a/rs/moq-net/src/model/cache.rs +++ b/rs/moq-net/src/model/cache.rs @@ -1367,7 +1367,7 @@ impl Drop for Charge { /// What one reader's record bills to its track's account, and its share of the debt the /// track could not pay. Released on drop. -pub(super) struct Reader { +pub(crate) struct Reader { track: Arc, bytes: u64, } @@ -1375,7 +1375,7 @@ pub(super) struct Reader { impl Reader { /// Bill `bytes` in all, settling the track's debt after a growth: call with no track lock /// held. - pub(super) fn set(&mut self, bytes: u64) { + pub(crate) fn set(&mut self, bytes: u64) { if bytes == self.bytes { return; } @@ -1406,7 +1406,7 @@ impl Reader { /// The debt the track's readers owe, forgiven once the pool is back within its capacity: /// nothing needs the room any more, as the track forgives its own debt. - pub(super) fn debt(&self) -> u64 { + pub(crate) fn debt(&self) -> u64 { let readers = &self.track.readers; let pool = &self.track.pool; if pool.capacity().is_none_or(|capacity| pool.used() <= capacity) { @@ -1427,6 +1427,14 @@ impl Reader { } } +impl std::fmt::Debug for Reader { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Reader") + .field("bytes", &self.bytes) + .finish_non_exhaustive() + } +} + impl Drop for Reader { fn drop(&mut self) { self.set(0); diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 4a8f327b4cb..b0901609fe6 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -5273,6 +5273,12 @@ impl Producer { ledger::grace(clamp_max_delay(budget, self.state.read().max_age_bound())) } + /// An empty bill against this track's cache account, for a subscription's tail ledger. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn tail_bill(&self) -> cache::Reader { + self.state.read().cache.reader() + } + /// Create a [`Requests`] handle that serves on-demand fetches of uncached /// (old) groups. Most producers never need this; a relay creates one to fetch /// past groups from upstream. diff --git a/rs/moq-net/src/tail.rs b/rs/moq-net/src/tail.rs index 05f2f1600cc..7f83c38a81e 100644 --- a/rs/moq-net/src/tail.rs +++ b/rs/moq-net/src/tail.rs @@ -14,14 +14,20 @@ use std::{ops::Range, task::Poll, time::Duration}; pub(crate) use crate::ledger::{GRACE, grace}; use crate::{ - ledger::Ledger, + cache, + ledger::{Ledger, RUN_OVERHEAD}, time::{Deadline, Instant}, + track, }; /// The data streams a subscription has received, and the groups they account for. #[derive(Debug)] pub(crate) struct Tail { ledger: Ledger, + // What the ledger's gaps bill to the copy's account, once it has one. A publisher's drops + // and skips name any range, so under pool pressure the oldest gaps fold early rather + // than grow with a grace that never passes. + bill: Option, streams: u64, // Streams whose header arrived and that are still being read. active: u64, @@ -41,6 +47,7 @@ impl Tail { pub fn new(grace: Duration) -> Self { Self { ledger: Ledger::new(grace), + bill: None, streams: 0, active: 0, ended: false, @@ -62,6 +69,30 @@ impl Tail { self.ledger.set_grace(grace); } + /// Follow the copy `track`: wait on its max age and `budget`, and bill the ledger to its + /// account. + pub fn follow(&mut self, track: &track::Producer, budget: Duration) { + self.set_grace(track.tail_grace(budget)); + if self.bill.is_none() { + self.bill = Some(track.tail_bill()); + self.rebill(); + } + } + + /// Bill the ledger's gaps, then pay what the pool is owed from them: fold the oldest gaps + /// early, giving up on them as the grace passing would, until the debt is paid or none is + /// left. The run they split is the subscription's own, so an in-order feed bills nothing. + fn rebill(&mut self) { + let Some(bill) = &mut self.bill else { + return; + }; + let bytes = |ledger: &Ledger| ledger.len().saturating_sub(1) as u64 * RUN_OVERHEAD; + bill.set(bytes(&self.ledger)); + while bill.debt() > 0 && self.ledger.fold_oldest().is_some() { + bill.set(bytes(&self.ledger)); + } + } + /// Record groups as accounted for without a stream: dropped, or sent as a datagram. pub fn account(&mut self, groups: Range, now: Instant) { if groups.is_empty() { @@ -79,6 +110,7 @@ impl Tail { /// Fold every gap older than the grace into the runs around it. pub fn expire(&mut self, now: Instant) { self.ledger.expire(now); + self.rebill(); } /// See [`Ledger::covers`]. @@ -259,4 +291,64 @@ mod tests { drop(reading); assert_eq!(tail.read().active, 0); } + + /// A copy with no max age and a subscription with no max delay never ages a gap out, so a + /// publisher's alternating single-group drops bill the pool instead, and the oldest gaps + /// fold once it is short of room. + #[kio::test] + async fn drops_are_billed_and_fold_under_pressure() { + const DROPS: u64 = 1000; + let runtime = crate::time::sim(); + let capacity = 64 * RUN_OVERHEAD; + let pool = cache::Pool::new(cache::Config::default().with_capacity(capacity)); + let broadcast = crate::broadcast::Info { + pool: pool.clone(), + ..Default::default() + }; + let track = track::Producer::new(std::sync::Arc::new(broadcast), "test", track::Info::timed()); + let base = pool.used(); + + let mut tail = Tail::default(); + tail.follow(&track, Duration::MAX); + assert_eq!(tail.ledger.grace(), Duration::MAX, "nothing ever ages a gap out"); + let now = runtime.now(); + for drop in 0..DROPS { + tail.account(2 * drop..2 * drop + 1, now); + } + assert!(tail.ledger.len() < DROPS as usize, "the oldest gaps folded"); + assert!( + pool.used() <= base + capacity + 2 * RUN_OVERHEAD, + "the ledger stayed within the pool: {}", + pool.used() + ); + assert_eq!( + pool.used(), + base + (tail.ledger.len() - 1) as u64 * RUN_OVERHEAD, + "what is left is billed" + ); + + drop(tail); + assert_eq!(pool.used(), base, "the bill went with the tail"); + } + + /// Without pressure nothing folds early: the pool bills the gaps and keeps them. + #[kio::test] + async fn drops_are_kept_while_the_pool_has_room() { + let runtime = crate::time::sim(); + let pool = cache::Pool::unbounded(); + let broadcast = crate::broadcast::Info { + pool: pool.clone(), + ..Default::default() + }; + let track = track::Producer::new(std::sync::Arc::new(broadcast), "test", track::Info::timed()); + let base = pool.used(); + + let mut tail = Tail::default(); + tail.follow(&track, Duration::MAX); + for drop in 0..100 { + tail.account(2 * drop..2 * drop + 1, runtime.now()); + } + assert_eq!(tail.ledger.len(), 100); + assert_eq!(pool.used(), base + 99 * RUN_OVERHEAD); + } } From 8fa385851c5a6cd6899d713aa739ce5889a8dd7a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 20:25:42 -0800 Subject: [PATCH 3/3] fix(net): a tail's bill pays the readers' debt but never joins the mean that ends a record Co-Authored-By: Claude Sonnet 5.5 --- rs/moq-net/src/model/cache.rs | 52 +++++++++++++++++++++++++--------- rs/moq-net/src/model/record.rs | 26 +++++++++++++++++ rs/moq-net/src/model/track.rs | 2 +- rs/moq-net/src/tail.rs | 3 +- 4 files changed, 67 insertions(+), 16 deletions(-) diff --git a/rs/moq-net/src/model/cache.rs b/rs/moq-net/src/model/cache.rs index c5e5a92599c..600ba402c2c 100644 --- a/rs/moq-net/src/model/cache.rs +++ b/rs/moq-net/src/model/cache.rs @@ -725,16 +725,20 @@ pub(super) struct Track { // account drops. Raw codec only: an Extra here would keep this account alive. encoder: Mutex>, - // What this track's readers' records bill, how many bill anything, and the eviction debt - // the track could not pay itself, which those readers pay; see [`Reader`]. + // What this track's readers' records bill, how many bill anything, the tail ledgers' bills, + // and the eviction debt the track could not pay itself, which those readers pay; see + // [`Reader`]. readers: Readers, } /// What a track's readers bill for their records, and the debt they owe. #[derive(Default)] struct Readers { + // Records only: the mean [`Reader::ends`] compares against, which a tail never skews. bytes: AtomicU64, count: AtomicU64, + // Tail ledgers: room for the debt like the records, but never one to end. + tails: AtomicU64, debt: AtomicU64, } @@ -949,16 +953,28 @@ impl Track { Reader { track: self.clone(), bytes: 0, + record: true, + } + } + + /// Open an empty bill against this account for a subscription's tail ledger: it pays the + /// readers' debt by folding, but never ends, since a publisher's drops are final. + pub(super) fn tail(self: &Arc) -> Reader { + Reader { + track: self.clone(), + bytes: 0, + record: false, } } /// Hand the part of `debt` this track could not pay to its readers, up to what their - /// records bill beyond what they already owe. + /// records and tails bill beyond what they already owe. pub(super) fn owe_readers(&self, debt: &mut u64) { let readers = &self.readers; let mut shifted = 0; let _ = readers.debt.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |owed| { - let room = readers.bytes.load(Ordering::Relaxed).saturating_sub(owed); + let billed = readers.bytes.load(Ordering::Relaxed) + readers.tails.load(Ordering::Relaxed); + let room = billed.saturating_sub(owed); shifted = room.min(*debt); (shifted > 0).then_some(owed + shifted) }); @@ -1365,11 +1381,15 @@ impl Drop for Charge { } } -/// What one reader's record bills to its track's account, and its share of the debt the -/// track could not pay. Released on drop. +/// What one reader's record, or one subscription's tail ledger, bills to its track's account, +/// and its share of the debt the track could not pay. Released on drop. +/// +/// Only a record can [`Self::ends`]; a tail pays by folding, so it stays out of the mean +/// that decides which record to end. pub(crate) struct Reader { track: Arc, bytes: u64, + record: bool, } impl Reader { @@ -1380,14 +1400,17 @@ impl Reader { return; } let readers = &self.track.readers; - match (self.bytes, bytes) { - (0, _) => readers.count.fetch_add(1, Ordering::Relaxed), - (_, 0) => readers.count.fetch_sub(1, Ordering::Relaxed), - _ => 0, - }; + let billed = if self.record { &readers.bytes } else { &readers.tails }; + if self.record { + match (self.bytes, bytes) { + (0, _) => readers.count.fetch_add(1, Ordering::Relaxed), + (_, 0) => readers.count.fetch_sub(1, Ordering::Relaxed), + _ => 0, + }; + } if bytes > self.bytes { let grown = bytes - self.bytes; - readers.bytes.fetch_add(grown, Ordering::Relaxed); + billed.fetch_add(grown, Ordering::Relaxed); self.track.pool.add(grown); self.track.written.fetch_add(grown, Ordering::Relaxed); self.bytes = bytes; @@ -1395,7 +1418,7 @@ impl Reader { } else { // Whatever a record frees pays toward its readers' debt, however it shrank. let shrunk = self.bytes - bytes; - readers.bytes.fetch_sub(shrunk, Ordering::Relaxed); + billed.fetch_sub(shrunk, Ordering::Relaxed); let _ = readers.debt.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |debt| { (debt > 0).then(|| debt.saturating_sub(shrunk)) }); @@ -1422,7 +1445,8 @@ impl Reader { pub(super) fn ends(&self) -> bool { let readers = &self.track.readers; let count = readers.count.load(Ordering::Relaxed); - self.bytes > ENTRY_OVERHEAD + self.record + && self.bytes > ENTRY_OVERHEAD && u128::from(self.bytes) * u128::from(count) >= u128::from(readers.bytes.load(Ordering::Relaxed)) } } diff --git a/rs/moq-net/src/model/record.rs b/rs/moq-net/src/model/record.rs index fe4659ae396..c83f40a84c6 100644 --- a/rs/moq-net/src/model/record.rs +++ b/rs/moq-net/src/model/record.rs @@ -465,6 +465,32 @@ mod test { small.settle(|| track.clone(), &mut (), None).unwrap(); } + /// A tail's bill pays the readers' debt but is no record to end: it never joins the mean + /// that picks which record ends, whether it is small or large. + #[test] + fn a_tail_bill_never_skews_which_record_ends() { + let (_pool, track) = account(); + let mut records = [track.reader(), track.reader()]; + records[0].set(1_536); + records[1].set(2_880); + let mut tail = track.tail(); + tail.set(RUN_OVERHEAD); + assert!( + !records[0].ends(), + "the mean is the records' 2208 B, not the tail's share" + ); + assert!(records[1].ends()); + assert!(!tail.ends(), "a tail never ends"); + tail.set(100_000); + assert!( + records[1].ends(), + "a large tail does not lift the mean over every record" + ); + drop(tail); + assert!(records[1].ends()); + assert!(!records[0].ends()); + } + /// Whatever a bill frees pays its readers' debt, however it shrank, and the debt is forgiven /// once the pool is back within its capacity: nothing needs the room any more. #[test] diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index b0901609fe6..ee8c5c403ed 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -5276,7 +5276,7 @@ impl Producer { /// An empty bill against this track's cache account, for a subscription's tail ledger. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn tail_bill(&self) -> cache::Reader { - self.state.read().cache.reader() + self.state.read().cache.tail() } /// Create a [`Requests`] handle that serves on-demand fetches of uncached diff --git a/rs/moq-net/src/tail.rs b/rs/moq-net/src/tail.rs index 7f83c38a81e..d79a99f98cc 100644 --- a/rs/moq-net/src/tail.rs +++ b/rs/moq-net/src/tail.rs @@ -70,7 +70,8 @@ impl Tail { } /// Follow the copy `track`: wait on its max age and `budget`, and bill the ledger to its - /// account. + /// account. Call it with no lock on `track`'s state held, as [`Self::account`] and + /// [`Self::expire`] must be: billing the ledger can take that lock to settle the debt. pub fn follow(&mut self, track: &track::Producer, budget: Duration) { self.set_grace(track.tail_grace(budget)); if self.bill.is_none() {