From 2823618257e2949fde419ec30274d74a62b57d13 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 28 Sep 2026 06:56:43 -0700 Subject: [PATCH 1/2] fix(net): forget spliced tracks unread for the linger, finished ones included A relay's front removed a finished or aborted logical track from its machine, so the idle linger never ran for it and the broadcast kept it (and the source copy's cache) for the life of the front. Tracks now stay in the machine until unread for TRACK_IDLE_LINGER, then are removed from the broadcast. The same covers parked tracks and lookups whose reader left before the front saw them. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/broadcast.rs | 32 ++++- rs/moq-net/src/model/front.rs | 231 +++++++++++++++++++++++++----- rs/moq-net/src/model/origin.rs | 104 ++++++++++++-- rs/moq-net/src/model/resume.rs | 5 + 4 files changed, 317 insertions(+), 55 deletions(-) diff --git a/rs/moq-net/src/model/broadcast.rs b/rs/moq-net/src/model/broadcast.rs index 3b7f90306e0..ef40c6e9d4f 100644 --- a/rs/moq-net/src/model/broadcast.rs +++ b/rs/moq-net/src/model/broadcast.rs @@ -419,6 +419,29 @@ impl Producer { } } + /// Remove the spliced track `producer` from under `name`, unless it has a reader. + /// + /// Returns false only when a reader holds it: lookups hand out readers under the + /// same lock, so one arriving after the caller decided is never cut off. Anything + /// else (removed, or already replaced under the name) is gone as far as the caller + /// is concerned. + pub(crate) fn forget_spliced(&self, name: &str, producer: &super::resume::Producer) -> bool { + let mut state = self.state.lock(); + let Some(spliced) = state.spliced.as_mut() else { + return true; + }; + match spliced.tracks.get(name) { + Some(current) if current.is_clone(producer) => { + if current.is_used() { + return false; + } + spliced.tracks.remove(name); + true + } + _ => true, + } + } + /// Create a consumer of this one publisher's broadcast. /// /// A view of this broadcast object, not of its path: a new publisher at the same @@ -814,11 +837,12 @@ impl Consumer { // the time, not a property of the name: a publisher that had not yet // created the track may have it now. Drop it so this request reaches a // source again, exactly as the plain lookup below reclaims a closed - // entry. A *finished* one stays, since its cache is still readable. + // entry. A *finished* one stays, since its cache is still readable, + // until the front forgets it after going unread for its linger. // - // So a name, once finished, never comes back here: a publisher that - // finishes a track and publishes it again is serving new content, not - // resuming this one, and a subscriber has to re-read the catalog and + // So a name, once finished, is never spliced onto again: a publisher + // that finishes a track and publishes it again is serving new content, + // not resuming this one, and a subscriber has to re-read the catalog and // re-initialize rather than be spliced onto it. Resuming the same // content across routes is the transparent case, and that is what // `resume::Producer` already does. Publish new content under a new diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index b56136b02a8..5c1d2365828 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -53,8 +53,9 @@ pub(super) enum Event { Resolved { route: u64, result: Result }, /// A source closed: it will never serve again. SourceClosed { source: u64 }, - /// The spliced broadcast handed out a new logical track to serve. - TrackAssigned { track: Arc }, + /// The spliced broadcast handed out a new logical track to serve. It has no + /// reader until [`Event::Used`] says so. + TrackAssigned { track: Arc, now: Instant }, /// A source answered a track query: its copy's metadata, or a refusal. /// `closing` is whether the source has begun closing, in which case a /// refusal is not held against it: it is on its way out. @@ -80,6 +81,9 @@ pub(super) enum Event { Unused { track: Arc, now: Instant }, /// The armed deadline passed. Deadline { now: Instant }, + /// The driver let go of a track the machine asked it to [`Action::Forget`]. + /// Not fed when a reader arrived first: [`Event::Used`] follows instead. + Forgotten { track: Arc }, /// The origin is tearing down. Closed, } @@ -105,9 +109,10 @@ pub(super) enum Action { /// Drop the source copy of `track` but keep the delivered groups spliced, /// so resume stays seamless while nobody reads. Park { track: Arc }, - /// Drop the source copy of `track` and every delivered group: the linger expired - /// unread, or the source is local and keeps its own cache. - Release { track: Arc }, + /// Remove `track` from the broadcast and drop everything behind it, unless a reader + /// arrived meanwhile: it went unread for the linger, or the source is local and + /// keeps its own cache. Feed back [`Event::Forgotten`] once it is gone. + Forget { track: Arc }, /// The logical track completed. Finish { track: Arc }, /// The logical track failed for good. @@ -174,8 +179,8 @@ enum TrackState { Querying { source: u64 }, /// A source's copy is spliced in and being served. Spliced { source: u64 }, - /// Nobody reads it: the copy was dropped, the delivered groups stay - /// spliced until the linger expires. + /// Nobody reads it: the copy was dropped, and whatever the track delivered + /// stays spliced until the linger expires and the track is forgotten. Parked { since: Instant }, } @@ -189,6 +194,9 @@ struct Track { refusal: Option, /// Whether anyone reads the track: a copy is only spliced in for a reader. used: bool, + /// The track finished or aborted: nothing is spliced again, and it stays only + /// until it goes unread for the linger. + ended: bool, } /// One front's state; see the module docs. @@ -213,7 +221,7 @@ pub(super) struct Front { /// Immutable track metadata, fixed by the first copy of each track. Every /// source of the broadcast must serve the same content. info: BTreeMap, track::Info>, - /// How long an unread track keeps its delivered groups spliced. + /// How long an unread track stays before it is forgotten. linger: Duration, /// The deadline last armed, so a step only re-arms on change. armed: Option, @@ -222,7 +230,7 @@ pub(super) struct Front { impl Front { /// A front with no source and no tracks; `linger` is how long an unread - /// track keeps its delivered groups spliced. + /// track stays before it is forgotten. pub(super) fn new(linger: Duration) -> Self { Self { identity: Identity::Undetermined, @@ -278,15 +286,17 @@ impl Front { Event::SourceClosed { source } => self.source_closed(source, &mut actions), // A fresh logical track, even under a name served before: an earlier // verdict belonged to that request, and a later one asks afresh. The - // broadcast's track metadata is what persists. - Event::TrackAssigned { track } => { + // broadcast's track metadata is what persists. Unread until a reader + // shows up, so a lookup whose reader left early is forgotten too. + Event::TrackAssigned { track, now } => { self.tracks.insert( track, Track { - state: TrackState::Idle, + state: TrackState::Parked { since: now }, refused: HashSet::new(), refusal: None, used: false, + ended: false, }, ); } @@ -306,6 +316,9 @@ impl Front { Event::Used { track } => self.used(track, &mut actions), Event::Unused { track, now } => self.unused(track, now, &mut actions), Event::Deadline { now } => self.deadline(now, &mut actions), + Event::Forgotten { track } => { + self.tracks.remove(&track); + } Event::Closed => self.end(Error::Dropped, &mut actions), } if !self.ended { @@ -406,7 +419,7 @@ impl Front { } // A read track is never parked, so idle is the only state to re-query. for (name, track) in &mut self.tracks { - if track.used && track.state == TrackState::Idle { + if track.used && !track.ended && track.state == TrackState::Idle { track.state = TrackState::Querying { source }; actions.push(Action::Query { track: name.clone(), @@ -519,11 +532,11 @@ impl Front { return; } match result { - // Over for good: the track leaves the machine, so a name served again - // later starts afresh and a long-lived front does not keep a stub per - // name it ever served. + // Over for good: readers drain what it delivered, and it is forgotten + // once unread for the linger, like any other track. Ok(()) => { - self.tracks.remove(&name); + track.state = TrackState::Idle; + track.ended = true; actions.push(Action::Finish { track: name }); } // Died mid-serve after delivering: normal failover, re-splice from @@ -560,12 +573,12 @@ impl Front { return; }; let track = self.tracks.get_mut(&name).expect("dispatching a known track"); - if !track.used || track.state != TrackState::Idle { + if !track.used || track.ended || track.state != TrackState::Idle { return; } if track.refused.contains(&source) { let err = track.refusal.clone().unwrap_or(Error::NotFound); - self.tracks.remove(&name); + track.ended = true; actions.push(Action::Abort { track: name, err }); return; } @@ -594,11 +607,11 @@ impl Front { track.used = false; match track.state { // A local source keeps its own cache, so a warm copy would only be a staler - // duplicate of it: drop the copy outright, and a returning reader re-splices - // the source and reads its cache against the real live edge. + // duplicate of it: forget the track outright, and a returning reader + // re-splices the source and reads its cache against the real live edge. TrackState::Spliced { .. } if self.identity == Identity::Local => { track.state = TrackState::Idle; - actions.push(Action::Release { track: name }); + actions.push(Action::Forget { track: name }); } // Drop the copy so the source goes idle at once; the delivered // groups stay spliced for the linger. @@ -606,21 +619,22 @@ impl Front { track.state = TrackState::Parked { since: now }; actions.push(Action::Park { track: name }); } - TrackState::Querying { .. } => track.state = TrackState::Idle, - _ => {} + TrackState::Parked { .. } => {} + TrackState::Idle | TrackState::Querying { .. } => track.state = TrackState::Parked { since: now }, } } fn deadline(&mut self, now: Instant, actions: &mut Vec) { // The driver's deadline fired and cleared itself: re-arm whatever is still - // parked, even if nothing was due yet, or it would never be released. + // parked, even if nothing was due yet, or it would never be forgotten. + // Idle until the driver confirms: a reader that arrived first keeps it. self.armed = None; for (name, track) in &mut self.tracks { if let TrackState::Parked { since } = track.state && since + self.linger <= now { track.state = TrackState::Idle; - actions.push(Action::Release { track: name.clone() }); + actions.push(Action::Forget { track: name.clone() }); } } } @@ -705,13 +719,23 @@ mod tests { }), &[Action::Resolve], ); - assert_actions(front.step(Event::TrackAssigned { track: name("video") }), &[]); + let t0 = Instant::now(); assert_actions( - front.step(Event::Used { track: name("video") }), - &[Action::Query { + front.step(Event::TrackAssigned { track: name("video"), - source, - }], + now: t0, + }), + &[Action::Arm { at: Some(t0 + LINGER) }], + ); + assert_actions( + front.step(Event::Used { track: name("video") }), + &[ + Action::Query { + track: name("video"), + source, + }, + Action::Arm { at: None }, + ], ); assert_actions( front.step(Event::TrackInfo { @@ -970,7 +994,10 @@ mod tests { #[test] fn a_source_refusing_a_track_aborts_it_and_nothing_else() { let mut front = serving(remote(1, 10), 100); - front.step(Event::TrackAssigned { track: name("audio") }); + front.step(Event::TrackAssigned { + track: name("audio"), + now: Instant::now(), + }); front.step(Event::Used { track: name("audio") }); assert_actions( front.step(Event::TrackInfo { @@ -990,7 +1017,10 @@ mod tests { #[test] fn a_closing_source_refusal_is_not_a_verdict() { let mut front = serving(remote(1, 10), 100); - front.step(Event::TrackAssigned { track: name("audio") }); + front.step(Event::TrackAssigned { + track: name("audio"), + now: Instant::now(), + }); front.step(Event::Used { track: name("audio") }); assert_actions( front.step(Event::TrackInfo { @@ -1100,7 +1130,7 @@ mod tests { } #[test] - fn unread_track_parks_then_releases_after_the_linger() { + fn unread_track_parks_then_is_forgotten_after_the_linger() { let mut front = serving(remote(1, 10), 100); let t0 = Instant::now(); assert_actions( @@ -1123,9 +1153,26 @@ mod tests { // nothing re-arms. assert_actions( front.step(Event::Deadline { now: t0 + LINGER }), - &[Action::Release { track: name("video") }], + &[Action::Forget { track: name("video") }], + ); + assert_actions(front.step(Event::Forgotten { track: name("video") }), &[]); + assert!(!front.tracks.contains_key(&name("video"))); + } + + /// A reader that looked the track up before the driver could forget it keeps + /// it: the driver feeds `Used` instead of `Forgotten`, and the track re-splices. + #[test] + fn a_reader_racing_the_forget_keeps_the_track() { + let mut front = serving(remote(1, 10), 100); + let t0 = Instant::now(); + front.step(Event::Unused { + track: name("video"), + now: t0, + }); + assert_actions( + front.step(Event::Deadline { now: t0 + LINGER }), + &[Action::Forget { track: name("video") }], ); - // A returning reader re-splices from the serving source. assert_actions( front.step(Event::Used { track: name("video") }), &[Action::Query { @@ -1135,6 +1182,98 @@ mod tests { ); } + /// A finished track stays for readers still draining it and for the linger + /// after the last one, is never spliced again, and is then forgotten. + #[test] + fn a_finished_track_lingers_then_is_forgotten() { + let mut front = serving(remote(1, 10), 100); + assert_actions( + front.step(Event::TrackEnded { + track: name("video"), + source: 100, + closing: false, + result: Ok(()), + delivered: true, + }), + &[Action::Finish { track: name("video") }], + ); + let t0 = Instant::now(); + assert_actions( + front.step(Event::Unused { + track: name("video"), + now: t0, + }), + &[Action::Arm { at: Some(t0 + LINGER) }], + ); + // A returning reader reads what it holds: no query, and the linger waits. + assert_actions( + front.step(Event::Used { track: name("video") }), + &[Action::Arm { at: None }], + ); + // A replacement source does not re-splice it either. + front.step(Event::Selected { + best: Some(remote(2, 10)), + serving_closing: false, + }); + assert_actions( + front.step(Event::Resolved { + route: 2, + result: Ok(200), + }), + &[Action::Detach { source: 100 }], + ); + let t1 = t0 + LINGER; + front.step(Event::Unused { + track: name("video"), + now: t1, + }); + assert_actions( + front.step(Event::Deadline { now: t1 + LINGER }), + &[Action::Forget { track: name("video") }], + ); + front.step(Event::Forgotten { track: name("video") }); + assert!(!front.tracks.contains_key(&name("video"))); + } + + /// An aborted track lingers the same way rather than staying for the life of + /// the front. + #[test] + fn an_aborted_track_lingers_then_is_forgotten() { + let mut front = serving(remote(1, 10), 100); + front.step(Event::TrackEnded { + track: name("video"), + source: 100, + closing: false, + result: Err(Error::Dropped), + delivered: false, + }); + let t0 = Instant::now(); + assert_actions( + front.step(Event::Unused { + track: name("video"), + now: t0, + }), + &[Action::Arm { at: Some(t0 + LINGER) }], + ); + assert_actions( + front.step(Event::Deadline { now: t0 + LINGER }), + &[Action::Forget { track: name("video") }], + ); + } + + /// A local source keeps its own cache, so an unread track is forgotten at once. + #[test] + fn an_unread_local_track_is_forgotten_at_once() { + let mut front = serving(local(1), 100); + assert_actions( + front.step(Event::Unused { + track: name("video"), + now: Instant::now(), + }), + &[Action::Forget { track: name("video") }], + ); + } + #[test] fn a_returning_reader_cancels_the_linger() { let mut front = serving(remote(1, 10), 100); @@ -1155,11 +1294,21 @@ mod tests { ); } + /// A track whose reader left before the front saw it is never spliced, and is + /// forgotten after the linger rather than kept as a stub. #[test] fn unread_track_is_never_spliced() { let mut front = serving(remote(1, 10), 100); - assert_actions(front.step(Event::TrackAssigned { track: name("audio") }), &[]); - assert_eq!(front.tracks[&name("audio")].state, TrackState::Idle); + let t0 = Instant::now(); + front.step(Event::TrackAssigned { + track: name("audio"), + now: t0, + }); + assert_eq!(front.tracks[&name("audio")].state, TrackState::Parked { since: t0 }); + assert_actions( + front.step(Event::Deadline { now: t0 + LINGER }), + &[Action::Forget { track: name("audio") }], + ); } #[test] @@ -1237,7 +1386,10 @@ mod tests { }), }, Event::SourceClosed { source: 100 }, - Event::TrackAssigned { track: name("v") }, + Event::TrackAssigned { + track: name("v"), + now: t0, + }, Event::Used { track: name("v") }, Event::Unused { track: name("v"), @@ -1263,6 +1415,7 @@ mod tests { delivered: false, }, Event::Deadline { now: t0 + LINGER }, + Event::Forgotten { track: name("v") }, ]; fn walk(front: &Front, alphabet: &[Event], depth: usize, sequences: &mut usize) { diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 8643499979a..9f8d1e05ce9 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -1986,7 +1986,8 @@ impl Drop for DriverState { /// Within the window a returning viewer, or the next of a run of back-to-back /// fetches, reads the groups the front already cached: no second round trip for /// `TRACK_INFO`. Groups past that cached edge cost a fresh source splice. After -/// the window, the cached segment is released. +/// the window the track is forgotten, finished and aborted ones included, so a +/// long-lived broadcast only holds the tracks read recently. /// /// Sized above the fetch cadence of a segmented consumer: HLS polls every /// `TARGETDURATION` seconds, commonly 6 or 10, so a shorter window would drop the @@ -2193,6 +2194,18 @@ struct TrackIo { used: bool, } +impl TrackIo { + /// Let go of every source handle once the logical track ended: its segments keep + /// what readers drain, and only its demand is still watched, until it is forgotten. + fn end(&mut self) { + self.staged = None; + self.query = None; + self.copy = None; + self.warm = None; + self.head = None; + } +} + /// Drives one front: feeds the world's events to a [`Front`] and performs the /// actions it returns, until the front ends. The decisions live in the machine; /// this only waits and executes, so nothing here decides anything twice. @@ -2441,24 +2454,27 @@ async fn run_front(task: FrontTask) { } io.warm = warm; } - Action::Release { track: name } => { - let Some(io) = tracks.get_mut(&name) else { continue }; - // A local source releases straight from the spliced copy. - io.copy = None; - io.warm = None; - io.head = None; - if io.resume.release().is_err() { - tracks.remove(&name); + Action::Forget { track: name } => { + // A reader that looked the track up since the machine decided keeps + // it; its demand edge feeds `Used` next. + if let Some(io) = tracks.get(&name) + && !broadcast.forget_spliced(&name, &io.resume) + { + continue; } + tracks.remove(&name); + events.push_back(Event::Forgotten { track: name }); } Action::Finish { track: name } => { - if let Some(mut io) = tracks.remove(&name) { + if let Some(io) = tracks.get_mut(&name) { + io.end(); let _ = io.resume.finish(); } } Action::Abort { track: name, err } => { - if let Some(mut io) = tracks.remove(&name) { + if let Some(io) = tracks.get_mut(&name) { tracing::debug!(name = %name, %err, "aborting track"); + io.end(); let _ = io.resume.abort(err); } } @@ -2567,7 +2583,10 @@ async fn run_front(task: FrontTask) { used: false, }, ); - Event::TrackAssigned { track: name } + Event::TrackAssigned { + track: name, + now: timers.now(), + } } Step::Resolved(route, result) => { upstream = None; @@ -5373,6 +5392,67 @@ mod tests { assert_eq!(&payload[..], b"snapshot"); } + /// A finished track stays readable from the front while it is read and for the + /// linger after, then leaves the broadcast so it stops pinning its cache: the next + /// reader asks the source afresh. + #[tokio::test(start_paused = true)] + async fn finished_track_is_forgotten_after_the_linger() { + let (_server, _upstream, mut dynamic, resolved) = served_front().await; + + let track = resolved.track("catalog").unwrap(); + let subscribing = tokio::spawn(async move { track.subscribe(None).await }); + let request = tokio::time::timeout(Duration::from_secs(1), dynamic.requested_track()) + .await + .expect("the front asked the source") + .expect("request"); + let source = request.resolving_start().accept(None); + let mut group = source.create_group(0u64.into()).unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"snapshot".as_ref()).unwrap(); + group.finish().unwrap(); + source.finish().unwrap(); + let mut subscription = subscribing.await.unwrap().expect("subscribe"); + assert_eq!( + next_group(&mut subscription) + .await + .unwrap() + .expect("the catalog") + .sequence, + 0 + ); + assert!(next_group(&mut subscription).await.unwrap().is_none()); + drop(subscription); + drop(source); + + // Within the linger, a returning reader gets the finished track from the front. + let mut subscription = resolved.track("catalog").unwrap().subscribe(None).await.unwrap(); + assert_eq!( + next_group(&mut subscription) + .await + .unwrap() + .expect("the catalog") + .sequence, + 0 + ); + assert!(next_group(&mut subscription).await.unwrap().is_none()); + drop(subscription); + assert!( + tokio::time::timeout(Duration::from_secs(1), dynamic.requested_track()) + .await + .is_err(), + "a finished track within the linger asked the source again" + ); + + // Paused time runs the front's earlier deadline before this sleep returns. + tokio::time::sleep(TRACK_IDLE_LINGER).await; + + let track = resolved.track("catalog").unwrap(); + let _subscribing = tokio::spawn(async move { track.subscribe(None).await }); + tokio::time::timeout(Duration::from_secs(1), dynamic.requested_track()) + .await + .expect("the finished track outlived the linger") + .expect("request"); + } + /// A group that stays open for good (a JSON log in group 0) survives a park: the /// returning reader gets the frames delivered before it from the warm cache, and the /// re-splice asks the source only for the frames after them, across repeated parks. diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 0d9b8ba0386..c7dd92e8034 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -518,6 +518,11 @@ impl Producer { self.state.read().abort.is_some() } + /// Whether `other` produces the same logical track. + pub(crate) fn is_clone(&self, other: &Self) -> bool { + self.state.same_channel(&other.state) + } + /// Create a read handle for the logical track. pub fn consume(&self) -> Consumer { Consumer { From 962917798347300d77aa6c866801102cee51ee39 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 28 Sep 2026 11:29:18 -0700 Subject: [PATCH 2/2] fix(net): feed Used when a reader keeps a track from being forgotten The demand poll is level-triggered, so a reader that blocked the forget and left before the next poll left the track unread with no linger armed. Also assert the aborted track leaves the machine once forgotten. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/front.rs | 2 ++ rs/moq-net/src/model/origin.rs | 10 ++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index 5c1d2365828..4f0ed264427 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -1259,6 +1259,8 @@ mod tests { front.step(Event::Deadline { now: t0 + LINGER }), &[Action::Forget { track: name("video") }], ); + front.step(Event::Forgotten { track: name("video") }); + assert!(!front.tracks.contains_key(&name("video"))); } /// A local source keeps its own cache, so an unread track is forgotten at once. diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 9f8d1e05ce9..1e3752e0de0 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -2456,10 +2456,16 @@ async fn run_front(task: FrontTask) { } Action::Forget { track: name } => { // A reader that looked the track up since the machine decided keeps - // it; its demand edge feeds `Used` next. - if let Some(io) = tracks.get(&name) + // it. Feed its `Used` edge here: the demand poll only sees the + // current level, so a reader gone before the next poll would + // otherwise leave the track unread with no linger armed. + if let Some(io) = tracks.get_mut(&name) && !broadcast.forget_spliced(&name, &io.resume) { + if !io.used { + io.used = true; + events.push_back(Event::Used { track: name }); + } continue; } tracks.remove(&name);