From 83506a30e2eeab3d55904913260d2d8c58d02293 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 13:33:28 -0800 Subject: [PATCH 1/2] fix(net): a refused replacement never ends a track a withdrawn copy still feeds When a route is withdrawn while a track it serves is still open, the front detaches that copy and asks the next route for the same publisher. If that route no longer has the broadcast (its own withdrawal has not arrived yet), it refuses the track, and the front aborted the track with the refusal even though the detached copy was about to deliver its remaining groups and a clean end. The front now remembers that a detached copy still feeds the track and defers a refusal until that copy ends: a clean end finishes the track, an error falls back to the refusal. Different track metadata is still refused at once. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/front.rs | 83 ++++++++++++++++++++++++++++++++++ rs/moq-net/src/model/origin.rs | 62 +++++++++++++++++++++++++ 2 files changed, 145 insertions(+) diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index 93eae6197f9..0dbf1dae0af 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -75,6 +75,9 @@ pub(super) enum Event { result: Result<(), Error>, delivered: bool, }, + /// A copy detached while spliced, still feeding its segment, ended. Only fed while + /// nothing has replaced it. + Drained { track: Arc, result: Result<(), Error> }, /// A reader arrived on the track. Used { track: Arc }, /// The last reader left the track. @@ -196,6 +199,10 @@ struct Track { refusal: Option, /// Whether anyone reads the track: a copy is only spliced in for a reader. used: bool, + /// A copy detached while spliced still feeds the track's segment until a + /// replacement splices in or [`Event::Drained`] reports its end, so a source + /// refusing the track does not end it before then. + draining: bool, /// The track finished or aborted: nothing is spliced again, and it stays only /// until it goes unread for the linger. ended: bool, @@ -298,6 +305,7 @@ impl Front { refused: HashSet::new(), refusal: None, used: false, + draining: false, ended: false, }, ); @@ -315,6 +323,7 @@ impl Front { result, delivered, } => self.track_ended(track, source, closing, result, delivered, &mut actions), + Event::Drained { track, result } => self.drained(track, result, &mut actions), Event::Used { track } => self.used(track, &mut actions), Event::Unused { track, now } => self.unused(track, now, &mut actions), Event::Deadline { now } => self.deadline(now, &mut actions), @@ -409,6 +418,7 @@ impl Front { for track in self.tracks.values_mut() { if matches!(track.state, TrackState::Querying { source: s } | TrackState::Spliced { source: s } if s == old) { + track.draining |= track.state == (TrackState::Spliced { source: old }); track.state = TrackState::Idle; } } @@ -461,6 +471,7 @@ impl Front { for track in self.tracks.values_mut() { if matches!(track.state, TrackState::Querying { source: s } | TrackState::Spliced { source: s } if s == source) { + track.draining |= track.state == (TrackState::Spliced { source }); track.state = TrackState::Idle; } } @@ -496,6 +507,8 @@ impl Front { || expected.max_age != info.max_age || expected.priority != info.priority => { + // At once: a detached copy's remaining groups are not waited for. + track.draining = false; Err(Error::Unsupported) } Some(_) => Ok(()), @@ -509,6 +522,7 @@ impl Front { match verdict { Ok(()) => { track.state = TrackState::Spliced { source }; + track.draining = false; actions.push(Action::Splice { track: name.clone(), source, @@ -579,6 +593,10 @@ impl Front { return; } if track.refused.contains(&source) { + // The detached copy may still deliver the rest, and its end is the track's. + if track.draining { + return; + } let err = track.refusal.clone().unwrap_or(Error::NotFound); track.ended = true; actions.push(Action::Abort { track: name, err }); @@ -591,6 +609,28 @@ impl Front { actions.push(Action::Query { track: name, source }); } + fn drained(&mut self, name: Arc, result: Result<(), Error>, actions: &mut Vec) { + let Some(track) = self.tracks.get_mut(&name) else { + return; + }; + track.draining = false; + if track.ended || matches!(track.state, TrackState::Spliced { .. }) { + return; + } + match result { + // The content is over: nothing a replacement could add. + Ok(()) => { + if !matches!(track.state, TrackState::Parked { .. }) { + track.state = TrackState::Idle; + } + track.ended = true; + actions.push(Action::Finish { track: name }); + } + // Stalled until a replacement splices in, or ended now if every source refused. + Err(_) => self.redispatch(name, actions), + } + } + fn used(&mut self, name: Arc, actions: &mut Vec) { let Some(track) = self.tracks.get_mut(&name) else { return; @@ -1051,6 +1091,49 @@ mod tests { ); } + /// The serving source closes while its copy is still open, and the replacement + /// refuses the track: the detached copy's end, not the refusal, ends it. + #[test] + fn a_refusal_waits_for_the_detached_copy() { + for (end, expected) in [ + (Ok(()), Action::Finish { track: name("video") }), + ( + Err(Error::Dropped), + Action::Abort { + track: name("video"), + err: Error::Unroutable, + }, + ), + ] { + let mut front = serving(remote(1, 10), 100); + front.step(Event::SourceClosed { source: 100 }); + front.step(Event::Selected { + best: Some(remote(2, 10)), + serving_closing: false, + }); + front.step(Event::Resolved { + route: 2, + result: Ok(200), + }); + assert_actions( + front.step(Event::TrackInfo { + track: name("video"), + source: 200, + closing: false, + result: Err(Error::Unroutable), + }), + &[], + ); + assert_actions( + front.step(Event::Drained { + track: name("video"), + result: end, + }), + &[expected], + ); + } + } + #[test] fn a_copy_dying_after_delivering_resplices() { let mut front = serving(remote(1, 10), 100); diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 767a75a05fb..a21e5666059 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -2723,6 +2723,9 @@ struct TrackIo { /// The spliced copy, once it died: its segment keeps ending readers with its error /// unless a takeover follows, which retires it. dead: Option, + /// Whether `dead` was detached while still open, so its end is reported to the + /// machine as [`Event::Drained`]. + draining: bool, /// Whether the track had a reader as of the last demand edge. used: bool, } @@ -2739,6 +2742,7 @@ impl TrackIo { self.prior = None; self.held = None; self.dead = None; + self.draining = false; } /// Hold the spliced copy's newest `group`. Past the prior warm copy's edge, that @@ -2802,6 +2806,7 @@ async fn run_front(task: FrontTask) { SourceClosed(u64), Info(Arc, u64, Result), Ended(Arc, u64, Result<(), Error>), + Drained(Arc, Result<(), Error>), Held(Arc, group::Producer), Demand(Arc), Deadline, @@ -2955,6 +2960,7 @@ async fn run_front(task: FrontTask) { for io in tracks.values_mut() { if io.copy.as_ref().is_some_and(|(s, _)| *s == source) { io.bury(); + io.draining = true; } if io.query.as_ref().is_some_and(|(s, ..)| *s == source) { io.query = None; @@ -3019,6 +3025,7 @@ async fn run_front(task: FrontTask) { // edge the copy is asked to advance. io.edge = io.resume.resume_position(); io.held = None; + io.draining = false; io.copy = Some((source, copy)); } Action::Park { track: name } => { @@ -3033,6 +3040,7 @@ async fn run_front(task: FrontTask) { io.prior = None; io.held = None; io.dead = None; + io.draining = false; let parked = match &warm { Some(warm) => io.resume.park(&warm.track), None => io.resume.release(), @@ -3147,6 +3155,12 @@ async fn run_front(task: FrontTask) { { return Poll::Ready(Step::Ended(name.clone(), *source, result)); } + if io.draining + && let Some(dead) = &io.dead + && let Poll::Ready(result) = dead.poll_complete(waiter) + { + return Poll::Ready(Step::Drained(name.clone(), result)); + } if let Some((_, copy)) = &io.copy && let Poll::Ready(group) = copy.poll_latest_group(io.held.as_ref().map(|held| held.0.sequence), waiter) @@ -3184,6 +3198,7 @@ async fn run_front(task: FrontTask) { prior: None, held: None, dead: None, + draining: false, used: false, }, ); @@ -3253,6 +3268,11 @@ async fn run_front(task: FrontTask) { delivered, } } + Step::Drained(name, result) => { + let Some(io) = tracks.get_mut(&name) else { continue }; + io.draining = false; + Event::Drained { track: name, result } + } Step::Held(name, group) => { if let Some(io) = tracks.get_mut(&name) { io.hold(group); @@ -8257,6 +8277,48 @@ mod tests { } } + /// The serving route is withdrawn while its copy of the track is still open, and the + /// next route for the same publisher no longer has the broadcast: its refusal must not + /// end the track, which takes the incumbent's remaining groups and clean end. + #[tokio::test] + async fn a_refused_replacement_keeps_a_withdrawn_incumbent_open() { + let (mut rig, incumbent, source) = ResumeRig::new(&[9, 2]).await; + let standby = rig.standby(&[9, 5, 3]); + + // The peer withdraws the broadcast: the route retracts and its source closes, + // while the subscription already in flight stays open. + incumbent.withdrawn(); + source.close(); + + let request = queued(&standby).await; + let successor = broadcast::Info::new().produce(); + let mut handler = successor.dynamic(); + request.accept(&successor); + let track = tokio::time::timeout(Duration::from_secs(1), handler.requested_track()) + .await + .expect("the front asked the standby") + .expect("request"); + track.reject(Error::Unroutable); + for _ in 0..10 { + tokio::task::yield_now().await; + } + + let mut group = rig.incumbent_track.append_group().unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"last".as_ref()).unwrap(); + group.finish().unwrap(); + let mut group = next_group(&mut rig.subscription) + .await + .expect("the incumbent keeps serving") + .expect("track open"); + assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"last"); + + rig.incumbent_track.finish().unwrap(); + let end = next_group(&mut rig.subscription) + .await + .map(|group| group.map(|group| group.sequence)); + assert!(matches!(end, Ok(None)), "the incumbent's clean end, not {end:?}"); + } + #[tokio::test] async fn different_first_hop_ends_the_subscription() { let (mut rig, incumbent, source) = ResumeRig::new(&[10]).await; From 861b3b23750d5af6de00cc71230ddb645b828f3d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 23:52:35 -0800 Subject: [PATCH 2/2] test(net): cover a refused replacement attached over a live serving copy Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/front.rs | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index 0dbf1dae0af..7b88d82edca 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -1091,22 +1091,28 @@ mod tests { ); } - /// The serving source closes while its copy is still open, and the replacement - /// refuses the track: the detached copy's end, not the refusal, ends it. + /// The serving copy detaches while still open, because its source closed or a + /// better route replaced it, and the replacement refuses the track: the detached + /// copy's end, not the refusal, ends it. #[test] fn a_refusal_waits_for_the_detached_copy() { - for (end, expected) in [ - (Ok(()), Action::Finish { track: name("video") }), - ( - Err(Error::Dropped), - Action::Abort { - track: name("video"), - err: Error::Unroutable, - }, - ), - ] { + for (closed, end, expected) in [true, false].into_iter().flat_map(|closed| { + [ + (closed, Ok(()), Action::Finish { track: name("video") }), + ( + closed, + Err(Error::Dropped), + Action::Abort { + track: name("video"), + err: Error::Unroutable, + }, + ), + ] + }) { let mut front = serving(remote(1, 10), 100); - front.step(Event::SourceClosed { source: 100 }); + if closed { + front.step(Event::SourceClosed { source: 100 }); + } front.step(Event::Selected { best: Some(remote(2, 10)), serving_closing: false,