diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index 93eae6197f9..7b88d82edca 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -75,6 +75,9 @@ pub(super) enum Event { result: Result<(), Error>, delivered: bool, }, + /// A copy detached while spliced, still feeding its segment, ended. Only fed while + /// nothing has replaced it. + Drained { track: Arc, result: Result<(), Error> }, /// A reader arrived on the track. Used { track: Arc }, /// The last reader left the track. @@ -196,6 +199,10 @@ struct Track { refusal: Option, /// Whether anyone reads the track: a copy is only spliced in for a reader. used: bool, + /// A copy detached while spliced still feeds the track's segment until a + /// replacement splices in or [`Event::Drained`] reports its end, so a source + /// refusing the track does not end it before then. + draining: bool, /// The track finished or aborted: nothing is spliced again, and it stays only /// until it goes unread for the linger. ended: bool, @@ -298,6 +305,7 @@ impl Front { refused: HashSet::new(), refusal: None, used: false, + draining: false, ended: false, }, ); @@ -315,6 +323,7 @@ impl Front { result, delivered, } => self.track_ended(track, source, closing, result, delivered, &mut actions), + Event::Drained { track, result } => self.drained(track, result, &mut actions), Event::Used { track } => self.used(track, &mut actions), Event::Unused { track, now } => self.unused(track, now, &mut actions), Event::Deadline { now } => self.deadline(now, &mut actions), @@ -409,6 +418,7 @@ impl Front { for track in self.tracks.values_mut() { if matches!(track.state, TrackState::Querying { source: s } | TrackState::Spliced { source: s } if s == old) { + track.draining |= track.state == (TrackState::Spliced { source: old }); track.state = TrackState::Idle; } } @@ -461,6 +471,7 @@ impl Front { for track in self.tracks.values_mut() { if matches!(track.state, TrackState::Querying { source: s } | TrackState::Spliced { source: s } if s == source) { + track.draining |= track.state == (TrackState::Spliced { source }); track.state = TrackState::Idle; } } @@ -496,6 +507,8 @@ impl Front { || expected.max_age != info.max_age || expected.priority != info.priority => { + // At once: a detached copy's remaining groups are not waited for. + track.draining = false; Err(Error::Unsupported) } Some(_) => Ok(()), @@ -509,6 +522,7 @@ impl Front { match verdict { Ok(()) => { track.state = TrackState::Spliced { source }; + track.draining = false; actions.push(Action::Splice { track: name.clone(), source, @@ -579,6 +593,10 @@ impl Front { return; } if track.refused.contains(&source) { + // The detached copy may still deliver the rest, and its end is the track's. + if track.draining { + return; + } let err = track.refusal.clone().unwrap_or(Error::NotFound); track.ended = true; actions.push(Action::Abort { track: name, err }); @@ -591,6 +609,28 @@ impl Front { actions.push(Action::Query { track: name, source }); } + fn drained(&mut self, name: Arc, result: Result<(), Error>, actions: &mut Vec) { + let Some(track) = self.tracks.get_mut(&name) else { + return; + }; + track.draining = false; + if track.ended || matches!(track.state, TrackState::Spliced { .. }) { + return; + } + match result { + // The content is over: nothing a replacement could add. + Ok(()) => { + if !matches!(track.state, TrackState::Parked { .. }) { + track.state = TrackState::Idle; + } + track.ended = true; + actions.push(Action::Finish { track: name }); + } + // Stalled until a replacement splices in, or ended now if every source refused. + Err(_) => self.redispatch(name, actions), + } + } + fn used(&mut self, name: Arc, actions: &mut Vec) { let Some(track) = self.tracks.get_mut(&name) else { return; @@ -1051,6 +1091,55 @@ mod tests { ); } + /// The serving copy detaches while still open, because its source closed or a + /// better route replaced it, and the replacement refuses the track: the detached + /// copy's end, not the refusal, ends it. + #[test] + fn a_refusal_waits_for_the_detached_copy() { + for (closed, end, expected) in [true, false].into_iter().flat_map(|closed| { + [ + (closed, Ok(()), Action::Finish { track: name("video") }), + ( + closed, + Err(Error::Dropped), + Action::Abort { + track: name("video"), + err: Error::Unroutable, + }, + ), + ] + }) { + let mut front = serving(remote(1, 10), 100); + if closed { + front.step(Event::SourceClosed { source: 100 }); + } + front.step(Event::Selected { + best: Some(remote(2, 10)), + serving_closing: false, + }); + front.step(Event::Resolved { + route: 2, + result: Ok(200), + }); + assert_actions( + front.step(Event::TrackInfo { + track: name("video"), + source: 200, + closing: false, + result: Err(Error::Unroutable), + }), + &[], + ); + assert_actions( + front.step(Event::Drained { + track: name("video"), + result: end, + }), + &[expected], + ); + } + } + #[test] fn a_copy_dying_after_delivering_resplices() { let mut front = serving(remote(1, 10), 100); diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 767a75a05fb..a21e5666059 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -2723,6 +2723,9 @@ struct TrackIo { /// The spliced copy, once it died: its segment keeps ending readers with its error /// unless a takeover follows, which retires it. dead: Option, + /// Whether `dead` was detached while still open, so its end is reported to the + /// machine as [`Event::Drained`]. + draining: bool, /// Whether the track had a reader as of the last demand edge. used: bool, } @@ -2739,6 +2742,7 @@ impl TrackIo { self.prior = None; self.held = None; self.dead = None; + self.draining = false; } /// Hold the spliced copy's newest `group`. Past the prior warm copy's edge, that @@ -2802,6 +2806,7 @@ async fn run_front(task: FrontTask) { SourceClosed(u64), Info(Arc, u64, Result), Ended(Arc, u64, Result<(), Error>), + Drained(Arc, Result<(), Error>), Held(Arc, group::Producer), Demand(Arc), Deadline, @@ -2955,6 +2960,7 @@ async fn run_front(task: FrontTask) { for io in tracks.values_mut() { if io.copy.as_ref().is_some_and(|(s, _)| *s == source) { io.bury(); + io.draining = true; } if io.query.as_ref().is_some_and(|(s, ..)| *s == source) { io.query = None; @@ -3019,6 +3025,7 @@ async fn run_front(task: FrontTask) { // edge the copy is asked to advance. io.edge = io.resume.resume_position(); io.held = None; + io.draining = false; io.copy = Some((source, copy)); } Action::Park { track: name } => { @@ -3033,6 +3040,7 @@ async fn run_front(task: FrontTask) { io.prior = None; io.held = None; io.dead = None; + io.draining = false; let parked = match &warm { Some(warm) => io.resume.park(&warm.track), None => io.resume.release(), @@ -3147,6 +3155,12 @@ async fn run_front(task: FrontTask) { { return Poll::Ready(Step::Ended(name.clone(), *source, result)); } + if io.draining + && let Some(dead) = &io.dead + && let Poll::Ready(result) = dead.poll_complete(waiter) + { + return Poll::Ready(Step::Drained(name.clone(), result)); + } if let Some((_, copy)) = &io.copy && let Poll::Ready(group) = copy.poll_latest_group(io.held.as_ref().map(|held| held.0.sequence), waiter) @@ -3184,6 +3198,7 @@ async fn run_front(task: FrontTask) { prior: None, held: None, dead: None, + draining: false, used: false, }, ); @@ -3253,6 +3268,11 @@ async fn run_front(task: FrontTask) { delivered, } } + Step::Drained(name, result) => { + let Some(io) = tracks.get_mut(&name) else { continue }; + io.draining = false; + Event::Drained { track: name, result } + } Step::Held(name, group) => { if let Some(io) = tracks.get_mut(&name) { io.hold(group); @@ -8257,6 +8277,48 @@ mod tests { } } + /// The serving route is withdrawn while its copy of the track is still open, and the + /// next route for the same publisher no longer has the broadcast: its refusal must not + /// end the track, which takes the incumbent's remaining groups and clean end. + #[tokio::test] + async fn a_refused_replacement_keeps_a_withdrawn_incumbent_open() { + let (mut rig, incumbent, source) = ResumeRig::new(&[9, 2]).await; + let standby = rig.standby(&[9, 5, 3]); + + // The peer withdraws the broadcast: the route retracts and its source closes, + // while the subscription already in flight stays open. + incumbent.withdrawn(); + source.close(); + + let request = queued(&standby).await; + let successor = broadcast::Info::new().produce(); + let mut handler = successor.dynamic(); + request.accept(&successor); + let track = tokio::time::timeout(Duration::from_secs(1), handler.requested_track()) + .await + .expect("the front asked the standby") + .expect("request"); + track.reject(Error::Unroutable); + for _ in 0..10 { + tokio::task::yield_now().await; + } + + let mut group = rig.incumbent_track.append_group().unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"last".as_ref()).unwrap(); + group.finish().unwrap(); + let mut group = next_group(&mut rig.subscription) + .await + .expect("the incumbent keeps serving") + .expect("track open"); + assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"last"); + + rig.incumbent_track.finish().unwrap(); + let end = next_group(&mut rig.subscription) + .await + .map(|group| group.map(|group| group.sequence)); + assert!(matches!(end, Ok(None)), "the incumbent's clean end, not {end:?}"); + } + #[tokio::test] async fn different_first_hop_ends_the_subscription() { let (mut rig, incumbent, source) = ResumeRig::new(&[10]).await;