diff --git a/rs/moq-net/src/model/broadcast.rs b/rs/moq-net/src/model/broadcast.rs index 3b7f90306e0..ef40c6e9d4f 100644 --- a/rs/moq-net/src/model/broadcast.rs +++ b/rs/moq-net/src/model/broadcast.rs @@ -419,6 +419,29 @@ impl Producer { } } + /// Remove the spliced track `producer` from under `name`, unless it has a reader. + /// + /// Returns false only when a reader holds it: lookups hand out readers under the + /// same lock, so one arriving after the caller decided is never cut off. Anything + /// else (removed, or already replaced under the name) is gone as far as the caller + /// is concerned. + pub(crate) fn forget_spliced(&self, name: &str, producer: &super::resume::Producer) -> bool { + let mut state = self.state.lock(); + let Some(spliced) = state.spliced.as_mut() else { + return true; + }; + match spliced.tracks.get(name) { + Some(current) if current.is_clone(producer) => { + if current.is_used() { + return false; + } + spliced.tracks.remove(name); + true + } + _ => true, + } + } + /// Create a consumer of this one publisher's broadcast. /// /// A view of this broadcast object, not of its path: a new publisher at the same @@ -814,11 +837,12 @@ impl Consumer { // the time, not a property of the name: a publisher that had not yet // created the track may have it now. Drop it so this request reaches a // source again, exactly as the plain lookup below reclaims a closed - // entry. A *finished* one stays, since its cache is still readable. + // entry. A *finished* one stays, since its cache is still readable, + // until the front forgets it after going unread for its linger. // - // So a name, once finished, never comes back here: a publisher that - // finishes a track and publishes it again is serving new content, not - // resuming this one, and a subscriber has to re-read the catalog and + // So a name, once finished, is never spliced onto again: a publisher + // that finishes a track and publishes it again is serving new content, + // not resuming this one, and a subscriber has to re-read the catalog and // re-initialize rather than be spliced onto it. Resuming the same // content across routes is the transparent case, and that is what // `resume::Producer` already does. Publish new content under a new diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index b56136b02a8..4f0ed264427 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -53,8 +53,9 @@ pub(super) enum Event { Resolved { route: u64, result: Result }, /// A source closed: it will never serve again. SourceClosed { source: u64 }, - /// The spliced broadcast handed out a new logical track to serve. - TrackAssigned { track: Arc }, + /// The spliced broadcast handed out a new logical track to serve. It has no + /// reader until [`Event::Used`] says so. + TrackAssigned { track: Arc, now: Instant }, /// A source answered a track query: its copy's metadata, or a refusal. /// `closing` is whether the source has begun closing, in which case a /// refusal is not held against it: it is on its way out. @@ -80,6 +81,9 @@ pub(super) enum Event { Unused { track: Arc, now: Instant }, /// The armed deadline passed. Deadline { now: Instant }, + /// The driver let go of a track the machine asked it to [`Action::Forget`]. + /// Not fed when a reader arrived first: [`Event::Used`] follows instead. + Forgotten { track: Arc }, /// The origin is tearing down. Closed, } @@ -105,9 +109,10 @@ pub(super) enum Action { /// Drop the source copy of `track` but keep the delivered groups spliced, /// so resume stays seamless while nobody reads. Park { track: Arc }, - /// Drop the source copy of `track` and every delivered group: the linger expired - /// unread, or the source is local and keeps its own cache. - Release { track: Arc }, + /// Remove `track` from the broadcast and drop everything behind it, unless a reader + /// arrived meanwhile: it went unread for the linger, or the source is local and + /// keeps its own cache. Feed back [`Event::Forgotten`] once it is gone. + Forget { track: Arc }, /// The logical track completed. Finish { track: Arc }, /// The logical track failed for good. @@ -174,8 +179,8 @@ enum TrackState { Querying { source: u64 }, /// A source's copy is spliced in and being served. Spliced { source: u64 }, - /// Nobody reads it: the copy was dropped, the delivered groups stay - /// spliced until the linger expires. + /// Nobody reads it: the copy was dropped, and whatever the track delivered + /// stays spliced until the linger expires and the track is forgotten. Parked { since: Instant }, } @@ -189,6 +194,9 @@ struct Track { refusal: Option, /// Whether anyone reads the track: a copy is only spliced in for a reader. used: bool, + /// The track finished or aborted: nothing is spliced again, and it stays only + /// until it goes unread for the linger. + ended: bool, } /// One front's state; see the module docs. @@ -213,7 +221,7 @@ pub(super) struct Front { /// Immutable track metadata, fixed by the first copy of each track. Every /// source of the broadcast must serve the same content. info: BTreeMap, track::Info>, - /// How long an unread track keeps its delivered groups spliced. + /// How long an unread track stays before it is forgotten. linger: Duration, /// The deadline last armed, so a step only re-arms on change. armed: Option, @@ -222,7 +230,7 @@ pub(super) struct Front { impl Front { /// A front with no source and no tracks; `linger` is how long an unread - /// track keeps its delivered groups spliced. + /// track stays before it is forgotten. pub(super) fn new(linger: Duration) -> Self { Self { identity: Identity::Undetermined, @@ -278,15 +286,17 @@ impl Front { Event::SourceClosed { source } => self.source_closed(source, &mut actions), // A fresh logical track, even under a name served before: an earlier // verdict belonged to that request, and a later one asks afresh. The - // broadcast's track metadata is what persists. - Event::TrackAssigned { track } => { + // broadcast's track metadata is what persists. Unread until a reader + // shows up, so a lookup whose reader left early is forgotten too. + Event::TrackAssigned { track, now } => { self.tracks.insert( track, Track { - state: TrackState::Idle, + state: TrackState::Parked { since: now }, refused: HashSet::new(), refusal: None, used: false, + ended: false, }, ); } @@ -306,6 +316,9 @@ impl Front { Event::Used { track } => self.used(track, &mut actions), Event::Unused { track, now } => self.unused(track, now, &mut actions), Event::Deadline { now } => self.deadline(now, &mut actions), + Event::Forgotten { track } => { + self.tracks.remove(&track); + } Event::Closed => self.end(Error::Dropped, &mut actions), } if !self.ended { @@ -406,7 +419,7 @@ impl Front { } // A read track is never parked, so idle is the only state to re-query. for (name, track) in &mut self.tracks { - if track.used && track.state == TrackState::Idle { + if track.used && !track.ended && track.state == TrackState::Idle { track.state = TrackState::Querying { source }; actions.push(Action::Query { track: name.clone(), @@ -519,11 +532,11 @@ impl Front { return; } match result { - // Over for good: the track leaves the machine, so a name served again - // later starts afresh and a long-lived front does not keep a stub per - // name it ever served. + // Over for good: readers drain what it delivered, and it is forgotten + // once unread for the linger, like any other track. Ok(()) => { - self.tracks.remove(&name); + track.state = TrackState::Idle; + track.ended = true; actions.push(Action::Finish { track: name }); } // Died mid-serve after delivering: normal failover, re-splice from @@ -560,12 +573,12 @@ impl Front { return; }; let track = self.tracks.get_mut(&name).expect("dispatching a known track"); - if !track.used || track.state != TrackState::Idle { + if !track.used || track.ended || track.state != TrackState::Idle { return; } if track.refused.contains(&source) { let err = track.refusal.clone().unwrap_or(Error::NotFound); - self.tracks.remove(&name); + track.ended = true; actions.push(Action::Abort { track: name, err }); return; } @@ -594,11 +607,11 @@ impl Front { track.used = false; match track.state { // A local source keeps its own cache, so a warm copy would only be a staler - // duplicate of it: drop the copy outright, and a returning reader re-splices - // the source and reads its cache against the real live edge. + // duplicate of it: forget the track outright, and a returning reader + // re-splices the source and reads its cache against the real live edge. TrackState::Spliced { .. } if self.identity == Identity::Local => { track.state = TrackState::Idle; - actions.push(Action::Release { track: name }); + actions.push(Action::Forget { track: name }); } // Drop the copy so the source goes idle at once; the delivered // groups stay spliced for the linger. @@ -606,21 +619,22 @@ impl Front { track.state = TrackState::Parked { since: now }; actions.push(Action::Park { track: name }); } - TrackState::Querying { .. } => track.state = TrackState::Idle, - _ => {} + TrackState::Parked { .. } => {} + TrackState::Idle | TrackState::Querying { .. } => track.state = TrackState::Parked { since: now }, } } fn deadline(&mut self, now: Instant, actions: &mut Vec) { // The driver's deadline fired and cleared itself: re-arm whatever is still - // parked, even if nothing was due yet, or it would never be released. + // parked, even if nothing was due yet, or it would never be forgotten. + // Idle until the driver confirms: a reader that arrived first keeps it. self.armed = None; for (name, track) in &mut self.tracks { if let TrackState::Parked { since } = track.state && since + self.linger <= now { track.state = TrackState::Idle; - actions.push(Action::Release { track: name.clone() }); + actions.push(Action::Forget { track: name.clone() }); } } } @@ -705,13 +719,23 @@ mod tests { }), &[Action::Resolve], ); - assert_actions(front.step(Event::TrackAssigned { track: name("video") }), &[]); + let t0 = Instant::now(); assert_actions( - front.step(Event::Used { track: name("video") }), - &[Action::Query { + front.step(Event::TrackAssigned { track: name("video"), - source, - }], + now: t0, + }), + &[Action::Arm { at: Some(t0 + LINGER) }], + ); + assert_actions( + front.step(Event::Used { track: name("video") }), + &[ + Action::Query { + track: name("video"), + source, + }, + Action::Arm { at: None }, + ], ); assert_actions( front.step(Event::TrackInfo { @@ -970,7 +994,10 @@ mod tests { #[test] fn a_source_refusing_a_track_aborts_it_and_nothing_else() { let mut front = serving(remote(1, 10), 100); - front.step(Event::TrackAssigned { track: name("audio") }); + front.step(Event::TrackAssigned { + track: name("audio"), + now: Instant::now(), + }); front.step(Event::Used { track: name("audio") }); assert_actions( front.step(Event::TrackInfo { @@ -990,7 +1017,10 @@ mod tests { #[test] fn a_closing_source_refusal_is_not_a_verdict() { let mut front = serving(remote(1, 10), 100); - front.step(Event::TrackAssigned { track: name("audio") }); + front.step(Event::TrackAssigned { + track: name("audio"), + now: Instant::now(), + }); front.step(Event::Used { track: name("audio") }); assert_actions( front.step(Event::TrackInfo { @@ -1100,7 +1130,7 @@ mod tests { } #[test] - fn unread_track_parks_then_releases_after_the_linger() { + fn unread_track_parks_then_is_forgotten_after_the_linger() { let mut front = serving(remote(1, 10), 100); let t0 = Instant::now(); assert_actions( @@ -1123,9 +1153,26 @@ mod tests { // nothing re-arms. assert_actions( front.step(Event::Deadline { now: t0 + LINGER }), - &[Action::Release { track: name("video") }], + &[Action::Forget { track: name("video") }], + ); + assert_actions(front.step(Event::Forgotten { track: name("video") }), &[]); + assert!(!front.tracks.contains_key(&name("video"))); + } + + /// A reader that looked the track up before the driver could forget it keeps + /// it: the driver feeds `Used` instead of `Forgotten`, and the track re-splices. + #[test] + fn a_reader_racing_the_forget_keeps_the_track() { + let mut front = serving(remote(1, 10), 100); + let t0 = Instant::now(); + front.step(Event::Unused { + track: name("video"), + now: t0, + }); + assert_actions( + front.step(Event::Deadline { now: t0 + LINGER }), + &[Action::Forget { track: name("video") }], ); - // A returning reader re-splices from the serving source. assert_actions( front.step(Event::Used { track: name("video") }), &[Action::Query { @@ -1135,6 +1182,100 @@ mod tests { ); } + /// A finished track stays for readers still draining it and for the linger + /// after the last one, is never spliced again, and is then forgotten. + #[test] + fn a_finished_track_lingers_then_is_forgotten() { + let mut front = serving(remote(1, 10), 100); + assert_actions( + front.step(Event::TrackEnded { + track: name("video"), + source: 100, + closing: false, + result: Ok(()), + delivered: true, + }), + &[Action::Finish { track: name("video") }], + ); + let t0 = Instant::now(); + assert_actions( + front.step(Event::Unused { + track: name("video"), + now: t0, + }), + &[Action::Arm { at: Some(t0 + LINGER) }], + ); + // A returning reader reads what it holds: no query, and the linger waits. + assert_actions( + front.step(Event::Used { track: name("video") }), + &[Action::Arm { at: None }], + ); + // A replacement source does not re-splice it either. + front.step(Event::Selected { + best: Some(remote(2, 10)), + serving_closing: false, + }); + assert_actions( + front.step(Event::Resolved { + route: 2, + result: Ok(200), + }), + &[Action::Detach { source: 100 }], + ); + let t1 = t0 + LINGER; + front.step(Event::Unused { + track: name("video"), + now: t1, + }); + assert_actions( + front.step(Event::Deadline { now: t1 + LINGER }), + &[Action::Forget { track: name("video") }], + ); + front.step(Event::Forgotten { track: name("video") }); + assert!(!front.tracks.contains_key(&name("video"))); + } + + /// An aborted track lingers the same way rather than staying for the life of + /// the front. + #[test] + fn an_aborted_track_lingers_then_is_forgotten() { + let mut front = serving(remote(1, 10), 100); + front.step(Event::TrackEnded { + track: name("video"), + source: 100, + closing: false, + result: Err(Error::Dropped), + delivered: false, + }); + let t0 = Instant::now(); + assert_actions( + front.step(Event::Unused { + track: name("video"), + now: t0, + }), + &[Action::Arm { at: Some(t0 + LINGER) }], + ); + assert_actions( + front.step(Event::Deadline { now: t0 + LINGER }), + &[Action::Forget { track: name("video") }], + ); + front.step(Event::Forgotten { track: name("video") }); + assert!(!front.tracks.contains_key(&name("video"))); + } + + /// A local source keeps its own cache, so an unread track is forgotten at once. + #[test] + fn an_unread_local_track_is_forgotten_at_once() { + let mut front = serving(local(1), 100); + assert_actions( + front.step(Event::Unused { + track: name("video"), + now: Instant::now(), + }), + &[Action::Forget { track: name("video") }], + ); + } + #[test] fn a_returning_reader_cancels_the_linger() { let mut front = serving(remote(1, 10), 100); @@ -1155,11 +1296,21 @@ mod tests { ); } + /// A track whose reader left before the front saw it is never spliced, and is + /// forgotten after the linger rather than kept as a stub. #[test] fn unread_track_is_never_spliced() { let mut front = serving(remote(1, 10), 100); - assert_actions(front.step(Event::TrackAssigned { track: name("audio") }), &[]); - assert_eq!(front.tracks[&name("audio")].state, TrackState::Idle); + let t0 = Instant::now(); + front.step(Event::TrackAssigned { + track: name("audio"), + now: t0, + }); + assert_eq!(front.tracks[&name("audio")].state, TrackState::Parked { since: t0 }); + assert_actions( + front.step(Event::Deadline { now: t0 + LINGER }), + &[Action::Forget { track: name("audio") }], + ); } #[test] @@ -1237,7 +1388,10 @@ mod tests { }), }, Event::SourceClosed { source: 100 }, - Event::TrackAssigned { track: name("v") }, + Event::TrackAssigned { + track: name("v"), + now: t0, + }, Event::Used { track: name("v") }, Event::Unused { track: name("v"), @@ -1263,6 +1417,7 @@ mod tests { delivered: false, }, Event::Deadline { now: t0 + LINGER }, + Event::Forgotten { track: name("v") }, ]; fn walk(front: &Front, alphabet: &[Event], depth: usize, sequences: &mut usize) { diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index bfb20e19b58..064367f5884 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -1996,7 +1996,8 @@ impl Drop for DriverState { /// Within the window a returning viewer, or the next of a run of back-to-back /// fetches, reads the groups the front already cached: no second round trip for /// `TRACK_INFO`. Groups past that cached edge cost a fresh source splice. After -/// the window, the cached segment is released. +/// the window the track is forgotten, finished and aborted ones included, so a +/// long-lived broadcast only holds the tracks read recently. /// /// Sized above the fetch cadence of a segmented consumer: HLS polls every /// `TARGETDURATION` seconds, commonly 6 or 10, so a shorter window would drop the @@ -2203,6 +2204,18 @@ struct TrackIo { used: bool, } +impl TrackIo { + /// Let go of every source handle once the logical track ended: its segments keep + /// what readers drain, and only its demand is still watched, until it is forgotten. + fn end(&mut self) { + self.staged = None; + self.query = None; + self.copy = None; + self.warm = None; + self.head = None; + } +} + /// Drives one front: feeds the world's events to a [`Front`] and performs the /// actions it returns, until the front ends. The decisions live in the machine; /// this only waits and executes, so nothing here decides anything twice. @@ -2451,24 +2464,33 @@ async fn run_front(task: FrontTask) { } io.warm = warm; } - Action::Release { track: name } => { - let Some(io) = tracks.get_mut(&name) else { continue }; - // A local source releases straight from the spliced copy. - io.copy = None; - io.warm = None; - io.head = None; - if io.resume.release().is_err() { - tracks.remove(&name); + Action::Forget { track: name } => { + // A reader that looked the track up since the machine decided keeps + // it. Feed its `Used` edge here: the demand poll only sees the + // current level, so a reader gone before the next poll would + // otherwise leave the track unread with no linger armed. + if let Some(io) = tracks.get_mut(&name) + && !broadcast.forget_spliced(&name, &io.resume) + { + if !io.used { + io.used = true; + events.push_back(Event::Used { track: name }); + } + continue; } + tracks.remove(&name); + events.push_back(Event::Forgotten { track: name }); } Action::Finish { track: name } => { - if let Some(mut io) = tracks.remove(&name) { + if let Some(io) = tracks.get_mut(&name) { + io.end(); let _ = io.resume.finish(); } } Action::Abort { track: name, err } => { - if let Some(mut io) = tracks.remove(&name) { + if let Some(io) = tracks.get_mut(&name) { tracing::debug!(name = %name, %err, "aborting track"); + io.end(); let _ = io.resume.abort(err); } } @@ -2577,7 +2599,10 @@ async fn run_front(task: FrontTask) { used: false, }, ); - Event::TrackAssigned { track: name } + Event::TrackAssigned { + track: name, + now: timers.now(), + } } Step::Resolved(route, result) => { upstream = None; @@ -5407,6 +5432,67 @@ mod tests { assert_eq!(&payload[..], b"snapshot"); } + /// A finished track stays readable from the front while it is read and for the + /// linger after, then leaves the broadcast so it stops pinning its cache: the next + /// reader asks the source afresh. + #[tokio::test(start_paused = true)] + async fn finished_track_is_forgotten_after_the_linger() { + let (_server, _upstream, mut dynamic, resolved) = served_front().await; + + let track = resolved.track("catalog").unwrap(); + let subscribing = tokio::spawn(async move { track.subscribe(None).await }); + let request = tokio::time::timeout(Duration::from_secs(1), dynamic.requested_track()) + .await + .expect("the front asked the source") + .expect("request"); + let source = request.resolving_start().accept(None); + let mut group = source.create_group(0u64.into()).unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"snapshot".as_ref()).unwrap(); + group.finish().unwrap(); + source.finish().unwrap(); + let mut subscription = subscribing.await.unwrap().expect("subscribe"); + assert_eq!( + next_group(&mut subscription) + .await + .unwrap() + .expect("the catalog") + .sequence, + 0 + ); + assert!(next_group(&mut subscription).await.unwrap().is_none()); + drop(subscription); + drop(source); + + // Within the linger, a returning reader gets the finished track from the front. + let mut subscription = resolved.track("catalog").unwrap().subscribe(None).await.unwrap(); + assert_eq!( + next_group(&mut subscription) + .await + .unwrap() + .expect("the catalog") + .sequence, + 0 + ); + assert!(next_group(&mut subscription).await.unwrap().is_none()); + drop(subscription); + assert!( + tokio::time::timeout(Duration::from_secs(1), dynamic.requested_track()) + .await + .is_err(), + "a finished track within the linger asked the source again" + ); + + // Paused time runs the front's earlier deadline before this sleep returns. + tokio::time::sleep(TRACK_IDLE_LINGER).await; + + let track = resolved.track("catalog").unwrap(); + let _subscribing = tokio::spawn(async move { track.subscribe(None).await }); + tokio::time::timeout(Duration::from_secs(1), dynamic.requested_track()) + .await + .expect("the finished track outlived the linger") + .expect("request"); + } + /// A group that stays open for good (a JSON log in group 0) survives a park: the /// returning reader gets the frames delivered before it from the warm cache, and the /// re-splice asks the source only for the frames after them, across repeated parks. diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 0d9b8ba0386..c7dd92e8034 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -518,6 +518,11 @@ impl Producer { self.state.read().abort.is_some() } + /// Whether `other` produces the same logical track. + pub(crate) fn is_clone(&self, other: &Self) -> bool { + self.state.same_channel(&other.state) + } + /// Create a read handle for the logical track. pub fn consume(&self) -> Consumer { Consumer {