From 248aa8795e6626951a0af04659e0ecc1bc869e55 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 17:09:00 -0800 Subject: [PATCH 1/6] fix(moq-net): never stitch two broadcasts on release A front resumed a broadcast through another route to the same publisher when its source closed, and spliced a better route in while serving. A publisher that re-creates a broadcast at the same path restarts its group numbers, so the relay stitched the new broadcast onto the old one's sequence and viewers waited on groups that never came. A front now ends when its source closes or when another route becomes best; readers re-request and get a fresh front. Nothing splices a second source into a front, and a relay always reads through the route it advertises. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/front.rs | 220 ++------------- rs/moq-net/src/model/origin.rs | 498 ++++++++------------------------- 2 files changed, 151 insertions(+), 567 deletions(-) diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index 7b88d82edca..e031fcbdfdd 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -352,13 +352,17 @@ impl Front { Some(candidate) if Some(candidate.route) == serving_route => { self.upstream = None; } - // Already asking through it. - Some(candidate) if self.upstream == Some(candidate.route) => {} - // A local newcomer must not be spliced into a broadcast whose - // publisher is on the way out: end, and a fresh front serves it. - Some(candidate) if candidate.local && self.identity == Identity::Local && serving_closing => { + // Another route is now best. It may carry another broadcast under the same + // path and group numbers (a local newcomer, a publisher back with its + // numbering restarted), so it is never spliced into this one. Keeping the + // old source instead would read through a route other than the one this + // relay advertises. The front ends, and readers re-request through the + // table's choice. + Some(_) if self.serving.is_some() => { self.end(Error::Dropped, actions); } + // Already asking through it. + Some(candidate) if self.upstream == Some(candidate.route) => {} // A better (or replacement) qualifying route: request through it. Some(candidate) => { self.upstream = Some(candidate.route); @@ -476,12 +480,10 @@ impl Front { } } self.last_err = Some(Error::Dropped); - match self.identity { - // A local publisher ending ends its broadcast; a newcomer at the - // path gets a fresh one. An anonymous source can never be resumed. - Identity::Local | Identity::Anonymous { .. } | Identity::Undetermined => self.end(Error::Dropped, actions), - Identity::Publisher(_) => actions.push(Action::Reselect), - } + // The source's broadcast is over. Another route, even to the same publisher, + // may carry a different broadcast under the same path and group numbers, so + // nothing resumes it: the front ends and a newcomer gets a fresh one. + self.end(Error::Dropped, actions); } fn track_info( @@ -827,46 +829,34 @@ mod tests { ); } + /// A better route to the same publisher is never spliced in, since it may carry + /// another broadcast under the same group numbers: the front ends instead. #[test] - fn better_route_takes_over_and_resplices() { + fn a_better_route_ends_the_front() { let mut front = serving(remote(1, 10), 100); assert_actions( front.step(Event::Selected { best: Some(remote(2, 10)), serving_closing: false, }), - &[Action::Request { route: 2 }], - ); - assert_actions( - front.step(Event::Resolved { - route: 2, - result: Ok(200), - }), - &[ - Action::Detach { source: 100 }, - Action::Query { - track: name("video"), - source: 200, - }, - ], + &[Action::End { err: Error::Dropped }], ); - assert_eq!(front.serving, Some((200, 2))); } #[test] fn a_stale_resolution_is_let_go() { - let mut front = serving(remote(1, 10), 100); + let mut front = Front::new(LINGER); front.step(Event::Selected { best: Some(remote(2, 10)), serving_closing: false, }); - // The table moved back before the request landed. + // The table moved on before the request landed. assert_actions( front.step(Event::Selected { best: Some(remote(1, 10)), serving_closing: false, }), - &[], + &[Action::Request { route: 1 }], ); assert_actions( front.step(Event::Resolved { @@ -877,31 +867,16 @@ mod tests { ); } + /// A source closing ends the front, even with another route to its publisher: + /// nothing resumes a broadcast through another source. #[test] - fn dead_source_reselects_through_the_same_publisher() { + fn a_dead_source_ends_the_front() { let mut front = serving(remote(1, 10), 100); assert_actions( front.step(Event::SourceClosed { source: 100 }), - &[Action::Detach { source: 100 }, Action::Reselect], - ); - assert!(front.refused_routes().contains(&1)); - assert_actions( - front.step(Event::Selected { - best: Some(remote(3, 10)), - serving_closing: false, - }), - &[Action::Request { route: 3 }], - ); - assert_actions( - front.step(Event::Resolved { - route: 3, - result: Ok(300), - }), - &[Action::Query { - track: name("video"), - source: 300, - }], + &[Action::Detach { source: 100 }, Action::End { err: Error::Dropped }], ); + assert!(front.ended()); } /// A live source does not keep a front alive once its route is gone: nothing @@ -944,19 +919,6 @@ mod tests { ); } - #[test] - fn dead_source_with_no_replacement_ends() { - let mut front = serving(remote(1, 10), 100); - front.step(Event::SourceClosed { source: 100 }); - assert_actions( - front.step(Event::Selected { - best: None, - serving_closing: false, - }), - &[Action::End { err: Error::Dropped }], - ); - } - #[test] fn anonymous_source_never_resumes() { let candidate = Candidate { @@ -991,27 +953,6 @@ mod tests { ); } - #[test] - fn standing_refusal_while_serving_skips_the_refuser() { - let mut front = serving(remote(1, 10), 100); - front.step(Event::Selected { - best: Some(remote(2, 10)), - serving_closing: false, - }); - assert_actions( - front.step(Event::Resolved { - route: 2, - result: Err(Refusal { - err: Error::NotFound, - standing: true, - }), - }), - &[Action::Reselect], - ); - assert!(front.refused_routes().contains(&2)); - assert_eq!(front.serving, Some((100, 1))); - } - #[test] fn retracted_route_is_not_a_refusal() { let mut front = Front::new(LINGER); @@ -1074,70 +1015,6 @@ mod tests { &[], ); assert!(front.tracks[&name("audio")].refused.is_empty()); - // The replacement is asked afresh. - front.step(Event::SourceClosed { source: 100 }); - front.step(Event::Selected { - best: Some(remote(2, 10)), - serving_closing: false, - }); - let actions = front.step(Event::Resolved { - route: 2, - result: Ok(200), - }); - assert!( - actions - .iter() - .any(|action| matches!(action, Action::Query { track, source: 200 } if track.as_ref() == "audio")) - ); - } - - /// The serving copy detaches while still open, because its source closed or a - /// better route replaced it, and the replacement refuses the track: the detached - /// copy's end, not the refusal, ends it. - #[test] - fn a_refusal_waits_for_the_detached_copy() { - for (closed, end, expected) in [true, false].into_iter().flat_map(|closed| { - [ - (closed, Ok(()), Action::Finish { track: name("video") }), - ( - closed, - Err(Error::Dropped), - Action::Abort { - track: name("video"), - err: Error::Unroutable, - }, - ), - ] - }) { - let mut front = serving(remote(1, 10), 100); - if closed { - front.step(Event::SourceClosed { source: 100 }); - } - front.step(Event::Selected { - best: Some(remote(2, 10)), - serving_closing: false, - }); - front.step(Event::Resolved { - route: 2, - result: Ok(200), - }); - assert_actions( - front.step(Event::TrackInfo { - track: name("video"), - source: 200, - closing: false, - result: Err(Error::Unroutable), - }), - &[], - ); - assert_actions( - front.step(Event::Drained { - track: name("video"), - result: end, - }), - &[expected], - ); - } } #[test] @@ -1191,35 +1068,6 @@ mod tests { ); } - #[test] - fn incompatible_copy_is_refused() { - let mut front = serving(remote(1, 10), 100); - front.step(Event::Selected { - best: Some(remote(2, 10)), - serving_closing: false, - }); - front.step(Event::Resolved { - route: 2, - result: Ok(200), - }); - let other = track::Info { - max_age: Duration::from_secs(1), - ..track::Info::default() - }; - assert_actions( - front.step(Event::TrackInfo { - track: name("video"), - source: 200, - closing: false, - result: Ok(other), - }), - &[Action::Abort { - track: name("video"), - err: Error::Unsupported, - }], - ); - } - #[test] fn unread_track_parks_then_is_forgotten_after_the_linger() { let mut front = serving(remote(1, 10), 100); @@ -1301,18 +1149,6 @@ mod tests { front.step(Event::Used { track: name("video") }), &[Action::Arm { at: None }], ); - // A replacement source does not re-splice it either. - front.step(Event::Selected { - best: Some(remote(2, 10)), - serving_closing: false, - }); - assert_actions( - front.step(Event::Resolved { - route: 2, - result: Ok(200), - }), - &[Action::Detach { source: 100 }], - ); let t1 = t0 + LINGER; front.step(Event::Unused { track: name("video"), @@ -1404,8 +1240,10 @@ mod tests { ); } + /// A local newcomer is a new broadcast: the front ends so a fresh one serves it, + /// rather than splicing it into the incumbent's tracks. #[test] - fn local_newcomer_takes_over_a_live_incumbent() { + fn a_local_newcomer_ends_the_front() { let mut front = serving(local(1), 100); assert_eq!(front.identity, Identity::Local); assert_eq!(front.pin(), Pin::Local); @@ -1414,7 +1252,7 @@ mod tests { best: Some(local(2)), serving_closing: false, }), - &[Action::Request { route: 2 }], + &[Action::End { err: Error::Dropped }], ); } diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index a21e5666059..4f8436bd050 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -6537,154 +6537,6 @@ mod tests { } } - /// A route that ends abruptly releases its open groups, and the takeover asks the - /// next route only for the frames past the break. The front keeps the head it - /// spliced, so a reader arriving afterwards still gets the whole group: whether the - /// old route dies before the new one takes over (a reconnect) or after (a cheaper - /// route preempting it). A reader joining mid-outage gets it too, and the reader - /// already mid-group is not handed it a second time. - #[tokio::test] - async fn a_takeover_keeps_the_open_group_head_for_later_readers() { - tokio::time::pause(); - for dies_first in [true, false] { - takeover_keeps_the_open_group_head(dies_first).await; - } - } - - async fn takeover_keeps_the_open_group_head(dies_first: bool) { - let producer = origin(1).produce(); - let first_server = producer - .dynamic("room/alice", Route::default().with_hops(hops(&[10])).with_cost(5)) - .unwrap(); - let pending = producer.consume().request_broadcast("room/alice"); - let upstream = broadcast::Info::new().produce(); - let mut dynamic = upstream.dynamic(); - queued(&first_server).await.accept(&upstream); - let resolved = pending.await.unwrap(); - - async fn read(group: &mut group::Consumer) -> Vec { - let frame = tokio::time::timeout(Duration::from_secs(1), group.read_frame()) - .await - .expect("frame") - .unwrap() - .expect("group ended"); - frame.payload.to_vec() - } - - // A reader is mid-way through the open group when the route changes. - let track = resolved.track("log").unwrap(); - let subscribing = tokio::spawn(async move { track.subscribe(None).await }); - let source = dynamic.requested_track().await.unwrap().accept(None); - let mut group = source.create_group(0u64.into()).unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"a".as_ref()).unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"b".as_ref()).unwrap(); - let mut subscription = subscribing.await.unwrap().unwrap(); - let mut reading = subscription.recv_group().await.unwrap().unwrap(); - assert_eq!(read(&mut reading).await, b"a"); - assert_eq!(read(&mut reading).await, b"b"); - - // Either the first route's session ends and reconnects, announcing the same route - // afresh while the old one is still held, or a cheaper route preempts the live one. - let mut first = Some((group, source, upstream, dynamic)); - let mut joining = None; - if dies_first { - drop(first.take()); - // The reader sees its route die and stalls. Polled in place: yielding would let - // the front give up on a path with no route left before the replacement lands. - assert!(futures::FutureExt::now_or_never(subscription.recv_group()).is_none()); - let track = resolved.track("log").unwrap(); - joining = Some(tokio::spawn(async move { - let mut joined = track.subscribe(None).await.unwrap(); - let group = joined.recv_group().await.unwrap().expect("track ended"); - (joined, group) - })); - } - let cost = if dies_first { 5 } else { 0 }; - let replacement_server = producer - .dynamic("room/alice", Route::default().with_hops(hops(&[10])).with_cost(cost)) - .unwrap(); - let replacement = broadcast::Info::new().produce(); - let mut replacement_dynamic = replacement.dynamic(); - queued(&replacement_server).await.accept(&replacement); - let mut next = tokio::time::timeout(Duration::from_secs(1), replacement_dynamic.requested_track()) - .await - .expect("the front asked the replacement") - .unwrap() - .resolving_start() - .accept(None); - next.start_at(0).unwrap(); - let mut next_group = next.create_group(0u64.into()).unwrap(); - next_group.start_at(2).unwrap(); - next_group.write_frame(crate::Timestamp::ZERO, b"c".as_ref()).unwrap(); - assert_eq!(read(&mut reading).await, b"c", "dies_first={dies_first}"); - - // A preempted route still owns its open group: its writer and other readers carry on. - if let Some((group, ..)) = &mut first { - let mut independent = group.consume(); - group.write_frame(crate::Timestamp::ZERO, b"x".as_ref()).unwrap(); - for expect in [b"a", b"b", b"x"] { - assert_eq!(read(&mut independent).await, expect); - } - group.finish().unwrap(); - } - drop(first); - - let track = resolved.track("log").unwrap(); - let mut later = tokio::time::timeout(Duration::from_secs(1), track.subscribe(None)) - .await - .expect("subscribe") - .unwrap(); - let mut late = tokio::time::timeout(Duration::from_secs(1), later.recv_group()) - .await - .unwrap_or_else(|_| panic!("dies_first={dies_first}: the later reader never got the group")) - .unwrap() - .expect("track ended"); - assert_eq!(late.sequence, 0); - for expect in [b"a", b"b", b"c"] { - assert_eq!(read(&mut late).await, expect, "dies_first={dies_first}"); - } - let mut joined = None; - if let Some(joining) = joining { - let (subscription, mut group) = tokio::time::timeout(Duration::from_secs(1), joining) - .await - .expect("the reader joining mid-outage never got the group") - .unwrap(); - assert_eq!(group.sequence, 0); - for expect in [b"a", b"b", b"c"] { - assert_eq!(read(&mut group).await, expect); - } - joined = Some((subscription, group)); - } - - // The reader that was mid-group already has it. - assert!( - tokio::time::timeout(Duration::from_secs(1), subscription.recv_group()) - .await - .is_err(), - "dies_first={dies_first}: the group was handed out twice" - ); - - // The continuation still flows to both. - next_group.write_frame(crate::Timestamp::ZERO, b"d".as_ref()).unwrap(); - assert_eq!(read(&mut reading).await, b"d"); - assert_eq!(read(&mut late).await, b"d"); - if let Some((_, group)) = &mut joined { - assert_eq!(read(group).await, b"d"); - } - next_group.finish().unwrap(); - drop(( - joined, - reading, - late, - subscription, - later, - next, - replacement, - replacement_server, - first_server, - )); - } - /// A returning reader is handed the parked open group spliced onto the source's /// continuation, which starts past the warm head. The group's end stays unknown /// until the source finishes it, even before a frame of it is read: readers drop a @@ -6754,91 +6606,6 @@ mod tests { assert_eq!(reading.finished().await.unwrap(), 2); } - #[tokio::test] - async fn warm_head_survives_another_takeover_before_park() { - tokio::time::pause(); - let producer = origin(1).produce(); - let _server = producer - .dynamic("room/alice", Route::default().with_hops(hops(&[10])).with_cost(5)) - .unwrap(); - let pending = producer.consume().request_broadcast("room/alice"); - let upstream = broadcast::Info::new().produce(); - let mut dynamic = upstream.dynamic(); - queued(&_server).await.accept(&upstream); - let resolved = pending.await.unwrap(); - - let track = resolved.track("log").unwrap(); - let subscribing = tokio::spawn(async move { track.subscribe(None).await }); - let source = dynamic.requested_track().await.unwrap().accept(None); - let mut group = source.create_group(0u64.into()).unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"a".as_ref()).unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"b".as_ref()).unwrap(); - let mut subscription = subscribing.await.unwrap().unwrap(); - let mut reading = subscription.recv_group().await.unwrap().unwrap(); - assert_eq!(&reading.read_frame().await.unwrap().unwrap().payload[..], b"a"); - drop(reading); - drop(subscription); - source.unused().await.unwrap(); - drop(group); - drop(source); - - let track = resolved.track("log").unwrap(); - let subscribing = tokio::spawn(async move { track.subscribe(None).await }); - let mut resumed = dynamic.requested_track().await.unwrap().resolving_start().accept(None); - resumed.start_at(0).unwrap(); - let mut subscription = subscribing.await.unwrap().unwrap(); - let mut reading = subscription.recv_group().await.unwrap().unwrap(); - assert_eq!(&reading.read_frame().await.unwrap().unwrap().payload[..], b"a"); - - let replacement_server = producer - .dynamic("room/alice", Route::default().with_hops(hops(&[10]))) - .unwrap(); - let replacement = broadcast::Info::new().produce(); - let mut replacement_dynamic = replacement.dynamic(); - queued(&replacement_server).await.accept(&replacement); - let mut source = replacement_dynamic - .requested_track() - .await - .unwrap() - .resolving_start() - .accept(None); - source.start_at(0).unwrap(); - let mut group = source.create_group(0u64.into()).unwrap(); - group.start_at(2).unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"c".as_ref()).unwrap(); - let mut continuation = reading.clone(); - continuation.start_at(2); - let frame = tokio::time::timeout(Duration::from_secs(1), continuation.read_frame()) - .await - .expect("replacement frame") - .unwrap() - .unwrap(); - assert_eq!(&frame.payload[..], b"c"); - drop(continuation); - assert_eq!(&reading.read_frame().await.unwrap().unwrap().payload[..], b"b"); - assert_eq!(&reading.read_frame().await.unwrap().unwrap().payload[..], b"c"); - drop(reading); - drop(subscription); - source.unused().await.unwrap(); - drop(group); - drop(source); - - let track = resolved.track("log").unwrap(); - let subscribing = tokio::spawn(async move { track.subscribe(None).await }); - let mut next = replacement_dynamic - .requested_track() - .await - .unwrap() - .resolving_start() - .accept(None); - next.start_at(0).unwrap(); - let mut subscription = subscribing.await.unwrap().unwrap(); - let mut reading = subscription.recv_group().await.unwrap().unwrap(); - for expected in [b"a", b"b", b"c"] { - assert_eq!(&reading.read_frame().await.unwrap().unwrap().payload[..], expected); - } - } - /// The same holds for a reader returning to a parked track: its warm cache /// does not stand in for the copy it is waiting on. #[tokio::test] @@ -7426,9 +7193,10 @@ mod tests { drop(incumbent); drop(source); - let replacement = broadcast::Info::new().produce(); - queued(&echo).await.accept(&replacement); - resolved.assert_not_closed(); + // The front ends with its source rather than asking through the peer. + settle(|| peer.get_broadcast("room/alice").is_none()).await; + assert!(echo.poll_requested_broadcast(&kio::Waiter::noop()).is_pending()); + drop(resolved); view.assert_next_ended("room"); } @@ -8184,33 +7952,41 @@ mod tests { } } - /// Accept the front's re-request on `server` with a source carrying the same - /// content stream (the delivered group plus its successor) and prove the - /// rig's subscription resumes onto it: the successor group is delivered on - /// the same subscription, at the group boundary. - async fn assert_resumes(rig: &mut ResumeRig, server: &Dynamic) { + /// The front ends, never splicing in `server`'s copy, and a new request gets a + /// fresh front through `server`. + async fn assert_ends_then_serves(rig: &mut ResumeRig, server: &Dynamic) { + tokio::time::timeout(Duration::from_secs(10), rig.resolved.closed()) + .await + .expect("the front ends"); + let consumer = rig.producer.consume(); + + let pending = consumer.request_broadcast("room/alice"); let request = queued(server).await; let replacement = broadcast::Info::new().produce(); let track = replacement.create_track("video", None).unwrap(); - // The same content: group 0 was already delivered through the old route, - // so the splice resumes at group 1. let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"before".as_ref()).unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"new".as_ref()).unwrap(); group.finish().unwrap(); request.accept(&replacement); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"resumed".as_ref()).unwrap(); - group.finish().unwrap(); - - let mut group = rig - .subscription - .recv_group() + let resolved = pending.await.expect("resolves through the new route"); + assert!(!resolved.is_clone(&rig.resolved), "joined the ended front"); + let mut fresh = resolved + .track("video") + .unwrap() + .subscribe(None) + .await + .expect("subscribe"); + let mut group = next_group(&mut fresh) .await - .expect("subscription survives the failover") + .expect("recv group") .expect("track ended early"); - let frame = group.read_frame().await.expect("read frame").expect("frame"); - assert_eq!(&frame.payload[..], b"resumed"); + assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"new"); + + // The old subscription never gets the new route's groups. + rig.incumbent_track.finish().unwrap(); + let end = next_group(&mut rig.subscription).await; + assert!(!matches!(end, Ok(Some(_))), "the new route was spliced in"); } /// The driver's completion contract: it resolves once every producer handle @@ -8227,96 +8003,63 @@ mod tests { drop(consumer); } + /// A publisher back behind the same first hop may be a new broadcast that + /// restarts its group numbers, as a re-created one does: it is never stitched + /// onto the old one, so a later request gets the new broadcast from its start. #[tokio::test] - async fn remote_source_resumes_through_same_first_hop() { - let (mut rig, incumbent, source) = ResumeRig::new(&[10]).await; - let standby_server = rig.standby(&[10, 20]); - - // The serving route dies: retraction plus source abort, like a session. - drop(incumbent); - drop(source); - - // The standby shares the first hop, so the subscription resumes there. - assert_resumes(&mut rig, &standby_server).await; - } - - /// A source claiming the same content cannot change immutable track metadata: - /// the successor is refused instead of the subscriber's samples being read on - /// a different grid, and the verdict outlives the aborted logical track. - #[tokio::test] - async fn incompatible_successor_is_refused() { - for replacement in [ - track::Info::default().with_timescale(crate::Timescale::MICRO), - track::Info::default().with_priority(7), - track::Info::default().with_max_age(Duration::from_secs(7)), - ] { - let (mut rig, incumbent, source) = ResumeRig::new(&[10]).await; - let standby_server = rig.standby(&[10, 20]); - drop(incumbent); - drop(source); - - // The standby shares the first hop, so the front re-requests through it, - // but its copy of the track is on another grid. - let request = queued(&standby_server).await; - let successor = broadcast::Info::new().produce(); - let track = successor.create_track("video", replacement).unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"before".as_ref()).unwrap(); + async fn a_returning_publisher_is_never_stitched() { + let (rig, incumbent, source) = ResumeRig::new(&[10]).await; + let ResumeRig { + producer, + mut subscription, + incumbent_track, + .. + } = rig; + // The old broadcast reached group 3. + for _ in 1..=3 { + let mut group = incumbent_track.append_group().unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"old".as_ref()).unwrap(); group.finish().unwrap(); - request.accept(&successor); - - assert!( - matches!(rig.subscription.recv_group().await, Err(Error::Unsupported)), - "the subscription must abort rather than resume onto incompatible metadata" - ); - - // Reopening the aborted logical track must not forget the broadcast's metadata. - let reopened = rig.resolved.track("video").unwrap(); - assert!(matches!(reopened.query().await, Err(Error::Unsupported))); - assert!(matches!(reopened.subscribe(None).await, Err(Error::Unsupported))); + subscription.recv_group().await.unwrap().unwrap(); } - } - - /// The serving route is withdrawn while its copy of the track is still open, and the - /// next route for the same publisher no longer has the broadcast: its refusal must not - /// end the track, which takes the incumbent's remaining groups and clean end. - #[tokio::test] - async fn a_refused_replacement_keeps_a_withdrawn_incumbent_open() { - let (mut rig, incumbent, source) = ResumeRig::new(&[9, 2]).await; - let standby = rig.standby(&[9, 5, 3]); - - // The peer withdraws the broadcast: the route retracts and its source closes, - // while the subscription already in flight stays open. - incumbent.withdrawn(); - source.close(); + // The publisher is back before the old front noticed it left. + let server = producer + .dynamic("room", Route::default().with_hops(hops(&[10]))) + .unwrap(); + drop(incumbent); + drop(source); + drop(incumbent_track); - let request = queued(&standby).await; - let successor = broadcast::Info::new().produce(); - let mut handler = successor.dynamic(); - request.accept(&successor); - let track = tokio::time::timeout(Duration::from_secs(1), handler.requested_track()) + // The old broadcast's readers end with it rather than wait on another source. + let ended = tokio::time::timeout(Duration::from_secs(10), subscription.recv_group()) .await - .expect("the front asked the standby") - .expect("request"); - track.reject(Error::Unroutable); - for _ in 0..10 { - tokio::task::yield_now().await; - } + .expect("the old subscription waits on the returning publisher"); + assert!(ended.is_err(), "the old subscription ends"); - let mut group = rig.incumbent_track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"last".as_ref()).unwrap(); + let consumer = producer.consume(); + let pending = consumer.request_broadcast("room/alice"); + let request = queued(&server).await; + let replacement = broadcast::Info::new().produce(); + let track = replacement.create_track("video", None).unwrap(); + let mut group = track.append_group().unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"new".as_ref()).unwrap(); group.finish().unwrap(); - let mut group = next_group(&mut rig.subscription) - .await - .expect("the incumbent keeps serving") - .expect("track open"); - assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"last"); + request.accept(&replacement); - rig.incumbent_track.finish().unwrap(); - let end = next_group(&mut rig.subscription) + let resolved = pending.await.expect("resolves through the returning publisher"); + let mut fresh = resolved + .track("video") + .unwrap() + .subscribe(None) + .await + .expect("subscribe"); + let mut group = fresh + .recv_group() .await - .map(|group| group.map(|group| group.sequence)); - assert!(matches!(end, Ok(None)), "the incumbent's clean end, not {end:?}"); + .expect("recv group") + .expect("track ended early"); + let frame = group.read_frame().await.expect("read frame").expect("frame"); + assert_eq!(&frame.payload[..], b"new"); } #[tokio::test] @@ -8408,20 +8151,17 @@ mod tests { } } - /// A named front whose route moves to a new publisher resumes through another route - /// from its own publisher, never through the updated one. + /// A named front whose route moves to a new publisher ends rather than splicing + /// in another route from its own publisher, and a new request gets a fresh front + /// through the new publisher, now the shortest route. #[tokio::test] - async fn a_first_hop_update_resumes_through_the_same_publisher() { + async fn a_first_hop_update_ends_the_front() { let (mut rig, incumbent, _source) = ResumeRig::new(&[10]).await; - let standby_server = rig.standby(&[10, 20]); + let _standby = rig.standby(&[10, 20]); incumbent.update(Route::default().with_hops(hops(&[11]))).unwrap(); - assert_resumes(&mut rig, &standby_server).await; - assert!( - incumbent.poll_requested_broadcast(&kio::Waiter::noop()).is_pending(), - "the front never asks the new publisher" - ); + assert_ends_then_serves(&mut rig, &incumbent).await; } #[tokio::test] @@ -8572,39 +8312,39 @@ mod tests { assert_eq!(&frame.payload[..], b"steady"); } + /// The serving route drains, repriced to the ceiling while its session keeps + /// serving: the front ends rather than stitching the standby's copy in, and a + /// new request gets a fresh front through the standby. #[tokio::test] - async fn drain_reprice_migrates_before_the_session_dies() { - let (mut rig, incumbent, source) = ResumeRig::new(&[10]).await; + async fn drain_reprice_ends_the_front() { + let (mut rig, incumbent, _source) = ResumeRig::new(&[10]).await; let standby_server = rig.standby(&[10, 20]); - // The serving route drains: repriced to the ceiling while its session - // keeps serving. The front migrates to the standby without waiting for - // the death. incumbent .update(Route::default().with_hops(hops(&[10])).with_cost(Cost::DRAIN)) .unwrap(); - assert_resumes(&mut rig, &standby_server).await; - - // The drained source outlived the migration. - drop(incumbent); - drop(source); + assert_ends_then_serves(&mut rig, &standby_server).await; } #[tokio::test] - async fn local_sources_splice_newest_first() { + async fn a_second_local_source_gets_a_fresh_front() { let producer = origin(1).produce(); let consumer = producer.consume(); let first = producer.publish("room/alice", Route::default()).unwrap(); let resolved = consumer.request_broadcast("room/alice").await.expect("resolves"); - // A second source at the same path joins the same front. + // A second source at the same path is a new broadcast: the first's front + // ends, and a new request gets a fresh one rather than a splice. let second = producer.publish("room/alice", Route::default()).unwrap(); + tokio::time::timeout(Duration::from_secs(10), resolved.closed()) + .await + .expect("the first front ends"); let again = consumer.request_broadcast("room/alice").await.expect("resolves"); - assert!(again.is_clone(&resolved)); + assert!(!again.is_clone(&resolved)); - // Losing one source keeps the front alive; losing both closes it. + // The first source leaving leaves the second's front; losing both closes it. first.close(); settle(|| consumer.get_broadcast("room/alice").is_some()).await; second.close(); @@ -8970,11 +8710,10 @@ mod tests { assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"fetched"); } - /// A newer local source wins dispatch the moment it attaches, but one whose copy - /// of the track carries different metadata is refused: the incumbent keeps - /// serving, and the refusal is never retried once the incumbent leaves. + /// A newer local source is a new broadcast, whatever its track metadata: the + /// incumbent's front ends rather than splicing it in, and a new request gets it. #[tokio::test] - async fn incompatible_local_source_keeps_the_incumbent() { + async fn a_local_newcomer_ends_the_incumbent_front() { let producer = origin(1).produce(); let consumer = producer.consume(); @@ -8993,26 +8732,33 @@ mod tests { let mut group = subscription.recv_group().await.unwrap().unwrap(); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"before"); - // The newest source is dispatched the track, and refused for its metadata. let second = producer.publish("room/alice", Route::default()).unwrap(); - let _incompatible = second + let newcomer = second .create_track("video", track::Info::default().with_timescale(crate::Timescale::MICRO)) .unwrap(); - for _ in 0..10 { - tokio::task::yield_now().await; - } + tokio::time::timeout(Duration::from_secs(10), resolved.closed()) + .await + .expect("the incumbent's front ends"); - // Still spliced to the incumbent, still delivering. - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"still".as_ref()).unwrap(); + let fresh = consumer.request_broadcast("room/alice").await.expect("resolves"); + assert!(!fresh.is_clone(&resolved)); + let mut fresh_subscription = fresh + .track("video") + .unwrap() + .subscribe(None) + .await + .expect("the newcomer's metadata applies"); + let mut group = newcomer.append_group().unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"new".as_ref()).unwrap(); group.finish().unwrap(); - let mut group = subscription.recv_group().await.unwrap().unwrap(); - assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"still"); + let mut group = fresh_subscription.recv_group().await.unwrap().unwrap(); + assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"new"); - // The incumbent leaving exhausts the table: the refusal is never retried. - drop(track); - first.close(); - assert!(matches!(subscription.recv_group().await, Err(Error::Unsupported))); + // The incumbent's reader never gets the newcomer's groups. + track.finish().unwrap(); + let end = next_group(&mut subscription).await; + assert!(!matches!(end, Ok(Some(_))), "the newcomer was spliced in"); + drop(first); } #[tokio::test] From dfe0da6d4a2b7116b00e1e573cd0f5664da115bb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 17:18:21 -0800 Subject: [PATCH 2/6] fix(moq-net): a request joins a front only through its own route A front never changes routes now; it ends. A request arriving after another route became best, before the driver ended the front, joined the front about to end. Admission now checks the route the front's source came through, not just its publisher. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/origin.rs | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 4f8436bd050..b1b94cef580 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -1215,9 +1215,10 @@ struct RemoteFront { /// The front's spliced broadcast, weak: dead once the front ends, so a /// later request re-creates the front instead of joining a corpse. broadcast: broadcast::WeakConsumer, - /// Which routes serve the front's content, fixed by its first source. A - /// request joins the front only while the best route is one of them. - pin: kio::Lock, + /// Which routes serve the front's content, and the route its source came + /// through, fixed by its first source. A request joins the front only while + /// the best route is that route: the front never changes routes, it ends. + pin: kio::Lock<(Pin, Option)>, } /// The last route a cursor observed at a presented prefix. @@ -2685,7 +2686,7 @@ struct FrontTask { /// Resolves the requesters parked on the front's channel. request: kio::Producer, /// Published for requesters once the first source fixes it; see [`RemoteFront::pin`]. - pin: kio::Lock, + pin: kio::Lock<(Pin, Option)>, timers: Clock, } @@ -2886,7 +2887,7 @@ async fn run_front(task: FrontTask) { continue; }; front.identify(candidate); - *pin.lock() = front.pin(); + *pin.lock() = (front.pin(), Some(route)); if let Some(source) = source { let id = next_source; next_source += 1; @@ -4717,16 +4718,16 @@ impl Consumer { // Join the live front for this path and horizon, if any: its watcher // resolves (or already resolved) the request channel with the front's // spliced broadcast, so repeat requests share one upstream - // subscription. Only while the best route still serves the front's - // content, though: once a different publisher wins (a cheaper route), a - // newcomer gets a fresh front from it, and the old front keeps serving the - // readers it has, since other content can't be spliced into it. + // subscription. Only while the best route is still the one the front's + // source came through, though: once another route wins, even from the + // same publisher, the front is about to end rather than splice it in, so + // a newcomer gets a fresh front from it at once. let key = (absolute.clone(), horizon); if let Some(front) = state.fronts.get(&key) { - let pin = *front.pin.lock(); + let (pin, route) = *front.pin.lock(); let current = state .best_route(&absolute.as_path(), horizon, Pin::Any, &HashSet::new()) - .is_some_and(|entry| entry.qualifies(pin)); + .is_some_and(|entry| entry.qualifies(pin) && route.is_none_or(|route| entry.id == route)); if current { let pending = Requesting::queued(front.request.consume()) .with_path(requested) @@ -4748,7 +4749,7 @@ impl Consumer { let request = kio::Producer::::default(); let consumer = request.consume(); let watch = state.watch(&self.shared, &absolute); - let pin = kio::Lock::new(Pin::Any); + let pin = kio::Lock::new((Pin::Any, None)); state.fronts.insert( key, RemoteFront { @@ -8335,14 +8336,15 @@ mod tests { let first = producer.publish("room/alice", Route::default()).unwrap(); let resolved = consumer.request_broadcast("room/alice").await.expect("resolves"); - // A second source at the same path is a new broadcast: the first's front - // ends, and a new request gets a fresh one rather than a splice. + // A second source at the same path is a new broadcast: a request right + // away, before the driver has ended the first's front, gets a fresh one + // rather than joining the front about to end. let second = producer.publish("room/alice", Route::default()).unwrap(); + let again = consumer.request_broadcast("room/alice").await.expect("resolves"); + assert!(!again.is_clone(&resolved)); tokio::time::timeout(Duration::from_secs(10), resolved.closed()) .await .expect("the first front ends"); - let again = consumer.request_broadcast("room/alice").await.expect("resolves"); - assert!(!again.is_clone(&resolved)); // The first source leaving leaves the second's front; losing both closes it. first.close(); From 8a6d46f6e3fd8535c4fa44118f3c68efb2d47054 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 17:49:43 -0800 Subject: [PATCH 3/6] fix(moq-net): an ended front bounces its path's announce A front that ends while a route still serves its path changed nothing the announce cursors could see, so a browser or downstream relay holding the ended broadcast never requested a fresh one: it stalled offline, or a downstream relay re-spliced the new broadcast at the old position. The front now emits Bounce before End when it served, and the origin re-announces the exact path (a retraction, then the route again) to every cursor presenting it. Covering-prefix routes are left alone. Failover across relays is no longer seamless: the GOAWAY cluster tests now assert the old subscription ends and a fresh request reads the new leg, and the cluster doc says so. Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/cluster.md | 15 +++-- rs/moq-net/src/model/front.rs | 34 +++++++--- rs/moq-net/src/model/origin.rs | 97 ++++++++++++++++++++++++++++ rs/moq-net/tests/rejoin.rs | 22 +++++-- rs/moq-relay/tests/goaway_cluster.rs | 90 ++++++++++++++++---------- 5 files changed, 206 insertions(+), 52 deletions(-) diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index 3f2cfb3396c..44a6fc47c72 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -32,12 +32,15 @@ follow the current best route immediately; only the announcement waits. For publisher that a peer still hunting offers, instead of announcing the broadcast again. -Failover routes must carry copies of the same broadcast. For each track, the -relay requires matching timescale, retention window, publisher priority, and -group ordering. A source with different properties is refused before its groups -are spliced in. If no compatible source remains, the track fails with -`Unsupported`. New immutable properties require a new track name or broadcast -identity. +A relay never splices a second route into a broadcast it serves. Two routes +from the same publisher can carry different broadcasts under one path, such as +one the publisher re-created with its group numbers starting over. So when the +serving route closes, or another route becomes the best, the relay ends the +broadcast and re-announces its path: a retraction, then the route again. +Subscribers then request the broadcast afresh through the current best route, +which costs viewers a brief restart on every failover, a relay drain included. +A path announced only through a covering prefix is not re-announced, since that +would restart the prefix's other broadcasts too. A route whose original publisher (its first hop) changes is updated in place on both wire protocols, so the broadcast never briefly vanishes downstream. diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index e031fcbdfdd..c5659661e06 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -122,6 +122,11 @@ pub(super) enum Action { Abort { track: Arc, err: Error }, /// Arm (or clear) the deadline the front wants to be woken at. Arm { at: Option }, + /// Re-announce the path to its announce consumers, a retraction then the + /// route again. Precedes the [`Action::End`] of a front that served while a + /// route may still serve the path: the table alone shows those consumers no + /// change, so nothing else tells them the broadcast they hold has ended. + Bounce, /// The front is over: reject the parked requesters with `err`, leave each /// read track to end with the copy it is spliced from or still waiting on, /// abort the rest with `err`, and drop every source. @@ -359,6 +364,7 @@ impl Front { // relay advertises. The front ends, and readers re-request through the // table's choice. Some(_) if self.serving.is_some() => { + actions.push(Action::Bounce); self.end(Error::Dropped, actions); } // Already asking through it. @@ -466,8 +472,7 @@ impl Front { return; } // A standing route can outlive the source it produced. Asking it again - // would re-request the broadcast that just ended; another route to the - // same publisher may still resume it. + // would re-request the broadcast that just ended. self.refused.insert(route); self.serving = None; self.serving_closing = false; @@ -483,6 +488,7 @@ impl Front { // The source's broadcast is over. Another route, even to the same publisher, // may carry a different broadcast under the same path and group numbers, so // nothing resumes it: the front ends and a newcomer gets a fresh one. + actions.push(Action::Bounce); self.end(Error::Dropped, actions); } @@ -839,7 +845,7 @@ mod tests { best: Some(remote(2, 10)), serving_closing: false, }), - &[Action::End { err: Error::Dropped }], + &[Action::Bounce, Action::End { err: Error::Dropped }], ); } @@ -874,7 +880,11 @@ mod tests { let mut front = serving(remote(1, 10), 100); assert_actions( front.step(Event::SourceClosed { source: 100 }), - &[Action::Detach { source: 100 }, Action::End { err: Error::Dropped }], + &[ + Action::Detach { source: 100 }, + Action::Bounce, + Action::End { err: Error::Dropped }, + ], ); assert!(front.ended()); } @@ -930,7 +940,11 @@ mod tests { assert_eq!(front.pin(), Pin::Route(1)); assert_actions( front.step(Event::SourceClosed { source: 100 }), - &[Action::Detach { source: 100 }, Action::End { err: Error::Dropped }], + &[ + Action::Detach { source: 100 }, + Action::Bounce, + Action::End { err: Error::Dropped }, + ], ); } @@ -1252,7 +1266,7 @@ mod tests { best: Some(local(2)), serving_closing: false, }), - &[Action::End { err: Error::Dropped }], + &[Action::Bounce, Action::End { err: Error::Dropped }], ); } @@ -1264,7 +1278,7 @@ mod tests { best: Some(local(2)), serving_closing: true, }), - &[Action::End { err: Error::Dropped }], + &[Action::Bounce, Action::End { err: Error::Dropped }], ); } @@ -1273,7 +1287,11 @@ mod tests { let mut front = serving(local(1), 100); assert_actions( front.step(Event::SourceClosed { source: 100 }), - &[Action::Detach { source: 100 }, Action::End { err: Error::Dropped }], + &[ + Action::Detach { source: 100 }, + Action::Bounce, + Action::End { err: Error::Dropped }, + ], ); } diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index b1b94cef580..e1e2ead85f5 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -1322,6 +1322,19 @@ impl TableCursor { && self.named.as_ref().is_none_or(|(mount, _)| mount.names(&entry.prefix)) } + /// Re-deliver the route at this presented prefix as a retraction then an + /// announce, if it was announced at exactly `path`. See [`OriginState::bounce`]. + fn bounce(&mut self, presented: &PathOwned, path: &Path) { + let Some(current) = self.current.get(presented).filter(|current| current.prefix == *path) else { + return; + }; + if let Ok(mut state) = self.state.write() { + let at = self.under.join(presented); + state.apply_unannounce(at.clone(), current.meta.clone(), current.captures.clone()); + state.apply_announce(at, current.meta.clone(), current.captures.clone()); + } + } + /// Deliver a changed winner at this presented prefix. fn update(&mut self, presented: &PathOwned, best: Option<&RouteEntry>) { let Some(entry) = best else { @@ -3083,6 +3096,7 @@ async fn run_front(task: FrontTask) { } } Action::Arm { at } => deadline.set(at), + Action::Bounce => shared.lock().bounce(&path.as_path()), Action::End { err } => { if let Ok(mut pending) = request.write() { pending.resolved.get_or_insert(Err(err.clone())); @@ -3784,6 +3798,26 @@ impl OriginState { routes.poke_below(prefix); } + /// Re-announce `path` on every cursor presenting a route at exactly `path`: + /// a retraction, then the same route again. A front at `path` is ending while + /// a route may still serve it, and the cursors would otherwise see no change, + /// so a consumer holding the ended broadcast would never request a fresh one. + /// A cursor seeing `path` only through a covering prefix is left alone, since + /// a bounce there would restart the prefix's other broadcasts too. + fn bounce(&mut self, path: &Path) { + let Ok(claim) = prefix_claim(path) else { + return; + }; + for id in self.routes.cursors_touching(path) { + let Some(cursor) = self.cursors.get_mut(&id) else { + continue; + }; + if let Some(presented) = cursor.presented(path, &claim) { + cursor.bounce(&presented, path); + } + } + } + /// Register a [`Watch`] on the routes covering `path`. fn watch(&mut self, shared: &kio::Shared, path: &Path) -> Watch { let id = self.next_watch; @@ -8063,6 +8097,69 @@ mod tests { assert_eq!(&frame.payload[..], b"new"); } + /// A front that ends while a route still serves its path re-announces the path, + /// a retraction then the route again, so a consumer holding the ended broadcast + /// (a browser, a downstream relay) requests a fresh one. Whether the source + /// closed under a route that stays or another route won, the table alone shows + /// the announce cursors no change. + #[tokio::test] + async fn an_ended_front_bounces_its_announce() { + for source_closes in [true, false] { + let producer = origin(1).produce(); + let consumer = producer.consume(); + let mut view = consumer.announced(); + let incumbent = producer + .dynamic("room/alice", Route::default().with_hops(hops(&[10])).with_cost(5)) + .unwrap(); + view.assert_next_active("room/alice"); + + let pending = consumer.request_broadcast("room/alice"); + let source = broadcast::Info::new().produce(); + queued(&incumbent).await.accept(&source); + let resolved = pending.await.expect("resolves"); + + // The same publisher is back on an identical route, which the cursor never + // sees, before the old one leaves; or a cheaper route from it wins. + let cost = if source_closes { 5 } else { 0 }; + let other = producer + .dynamic("room/alice", Route::default().with_hops(hops(&[10])).with_cost(cost)) + .unwrap(); + let _incumbent = if source_closes { + drop(incumbent); + drop(source); + None + } else { + // The cheaper route's metadata waits out the update hold, and the bounce + // below takes its place. + Some((incumbent, source)) + }; + + tokio::time::timeout(Duration::from_secs(10), resolved.closed()) + .await + .expect("the front ends"); + view.assert_next_ended("room/alice"); + view.assert_next_active("room/alice"); + view.assert_next_wait(); + drop(other); + } + } + + /// A front at a path served only through a covering prefix bounces nothing: + /// re-announcing the prefix would restart its other broadcasts too. + #[tokio::test] + async fn a_covering_route_is_never_bounced() { + let (rig, incumbent, source) = ResumeRig::new(&[10]).await; + let mut view = rig.producer.consume().announced(); + view.assert_next_active("room"); + let _returned = rig.standby(&[10]); + drop(incumbent); + drop(source); + tokio::time::timeout(Duration::from_secs(10), rig.resolved.closed()) + .await + .expect("the front ends"); + view.assert_next_wait(); + } + #[tokio::test] async fn different_first_hop_ends_the_subscription() { let (mut rig, incumbent, source) = ResumeRig::new(&[10]).await; diff --git a/rs/moq-net/tests/rejoin.rs b/rs/moq-net/tests/rejoin.rs index 77b2c58bc13..d7536331bdc 100644 --- a/rs/moq-net/tests/rejoin.rs +++ b/rs/moq-net/tests/rejoin.rs @@ -207,9 +207,10 @@ async fn rejoined_quiet_catalog_survives_cache_expiry() { /// A quiet track's latest group outlives the cache window across a publisher takeover. /// -/// The publisher reconnects while a reader holds the catalog: the relay retires the old -/// upstream copy and resumes from the new one past the cached group, which must stay -/// protected as the live edge. +/// The publisher reconnects while a reader holds the catalog. The relay never stitches +/// the new connection's copy into the old front: the holder's subscription ends, and a +/// later reader gets a fresh front whose latest catalog must stay protected as the +/// live edge. #[tokio::test(start_paused = true)] async fn taken_over_quiet_catalog_survives_cache_expiry() { for version in ["moq-lite-05", "moq-lite-06"] { @@ -276,7 +277,7 @@ async fn quiet_catalog_survives_cache_expiry(version: Version, takeover: bool) { }; tokio::time::sleep(moq_net::cache::DEFAULT_EXPIRY * 2 + Duration::from_secs(2)).await; - let (mut later, _later_remote, _later_pair) = session(5).await; + let (mut later, later_remote, _later_pair) = session(5).await; let mut group = tokio::time::timeout(Duration::from_secs(1), later.recv_group()) .await .unwrap_or_else(|_| panic!("{version} takeover={takeover}: late reader lost the latest catalog")) @@ -285,6 +286,19 @@ async fn quiet_catalog_survives_cache_expiry(version: Version, takeover: bool) { assert_eq!(read_all(&mut group).await.unwrap(), [b"catalog"], "{version}"); drop(group); + // After a takeover the holder read the old front, which ended; the later reader + // holds the fresh one. + let (mut holder, holder_remote) = if takeover { + let end = tokio::time::timeout(Duration::from_secs(1), holder.recv_group()).await; + assert!( + !matches!(end, Ok(Ok(Some(_)))), + "{version}: the takeover was stitched into the old front" + ); + (later, later_remote) + } else { + (holder, holder_remote) + }; + // A newer catalog supersedes it, so the old one expires like any other group. let mut newer = track.append_group().unwrap(); newer diff --git a/rs/moq-relay/tests/goaway_cluster.rs b/rs/moq-relay/tests/goaway_cluster.rs index 3e72ecf9050..7a0bc5f3f2b 100644 --- a/rs/moq-relay/tests/goaway_cluster.rs +++ b/rs/moq-relay/tests/goaway_cluster.rs @@ -250,9 +250,9 @@ async fn spawn_upstream( } /// An upstream GOAWAY with a redirect migrates the cluster dial to the sibling. -/// The draining session keeps serving through the handover window, and the path -/// never retracts on the cluster origin (route re-pricing and the sibling's -/// announce surface as metadata updates, not churn). +/// The draining session keeps serving through the handover window. The front +/// never stitches the sibling's copy in: it ends and bounces the path (a +/// retraction, then the route again), which ends announced. async fn cluster_migrates_on_upstream_goaway_inner() { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); @@ -311,9 +311,7 @@ async fn cluster_migrates_on_upstream_goaway_inner() { b"g0"[..] ); - // Watch for retraction during the swap: the migration must never - // unannounce the path (metadata updates are expected: the drain - // re-prices the old route and the sibling announces its own). + // Watch the path through the swap: it may bounce, but ends announced. let mut announcements = cluster.origin.consume().announced(); let first = announcements.next().await.expect("initial announce"); assert_eq!(first.prefix.as_str(), "cam"); @@ -349,16 +347,17 @@ async fn cluster_migrates_on_upstream_goaway_inner() { // handover window at the latest). session_a.closed().await; - // No retraction leaked to the origin during the whole swap. Metadata - // updates (the drain re-pricing, the sibling's route) are expected and - // harmless; an inactive event means the path flapped. + // The front ended rather than stitch the sibling's copy in, so the path may + // have bounced, but it never stays retracted. + let mut active = true; loop { match tokio::time::timeout(Duration::from_millis(500), announcements.next()).await { Err(_) => break, - Ok(Some(update)) if update.kind.is_active() => continue, - Ok(event) => panic!("migration must not retract the path on the cluster origin: {event:?}"), + Ok(Some(update)) => active = update.kind.is_active(), + Ok(None) => panic!("the cluster origin's announcements ended"), } } + assert!(active, "the migration left the path retracted on the cluster origin"); cluster_run.abort(); }) @@ -442,11 +441,11 @@ async fn spawn_relay_with_upstream( /// gated: MID-B's first inbound connection can only be that reconnect). /// 3. The draining MID-A leg keeps serving through the handover window, so /// every group published across the swap arrives exactly once. -/// 4. Once the old leg closes, the SAME subscription keeps flowing: both legs' -/// routes share TOP as their first hop, so BOTTOM's front re-splices onto -/// the MID-B leg at a group boundary instead of ending the subscription. -/// 5. No GOAWAY leaks to the subscriber's own session, and the path never -/// retracts under the subscriber. +/// 4. Once the old leg closes, BOTTOM's front ends rather than stitch the MID-B +/// leg's copy in, even though both legs name TOP as the publisher: the +/// subscription ends, and a fresh request reads the MID-B leg. +/// 5. No GOAWAY leaks to the subscriber's own session, and the path ends +/// announced under the subscriber (it may bounce). async fn cluster_diamond_goaway_seamless_failover_inner() { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); @@ -523,7 +522,7 @@ async fn cluster_diamond_goaway_seamless_failover_inner() { .expect("origin closed before the announce"); assert_eq!(first.prefix.as_str(), "diamond"); - let bc = within("broadcast resolves on the subscriber origin", async { + let bc_old = within("broadcast resolves on the subscriber origin", async { let consumer = sub_origin.consume(); consumer.routed("diamond").await?; consumer.request_broadcast("diamond").await.ok() @@ -541,7 +540,8 @@ async fn cluster_diamond_goaway_seamless_failover_inner() { // must preserve is that every group arrives, exactly once, with its frames intact. let mut sub = within( "subscribe to the video track", - bc.track("video") + bc_old + .track("video") .expect("track handle") .subscribe(moq_net::track::Subscription::default().with_max_age(Duration::from_secs(60))), ) @@ -617,10 +617,9 @@ async fn cluster_diamond_goaway_seamless_failover_inner() { // BOTTOM, positively proving the new leg carries the resubscribe. drop(mid_a_upstream); - // The subscription was served through the MID-A leg, but its close does not - // end it: both legs' routes name TOP as their first hop, so BOTTOM's front - // re-splices onto the MID-B leg at a group boundary and the subscription - // rides through. Keep publishing and keep reading the SAME subscription. + // The subscription was served through the MID-A leg, and its close ends it: + // BOTTOM never stitches the MID-B leg's copy in. Keep publishing, and read + // through a fresh request, as the bounce tells a subscriber to. const POST_DRAIN_LAST: u64 = LAST_GROUP + 40; let post_publisher = tokio::spawn(async move { for seq in (LAST_GROUP + 1)..=POST_DRAIN_LAST { @@ -635,19 +634,41 @@ async fn cluster_diamond_goaway_seamless_failover_inner() { } }); - // Three distinct post-drain groups on the same subscription, every frame - // verified, proves the new leg carries it: MID-A is severed, so they can - // only have flowed TOP -> MID-B -> BOTTOM. + // The old subscription ends without a post-drain group. + while let Ok(Some(group)) = within("the old subscription ends", sub.recv_group()).await { + assert!( + group.sequence <= LAST_GROUP, + "group {} reached the old subscription after the drain", + group.sequence + ); + } + let bc = within("a fresh request resolves through the MID-B leg", async { + let consumer = sub_origin.consume(); + loop { + if let Ok(bc) = consumer.request_broadcast("diamond").await + && !bc.is_clone(&bc_old) + { + return bc; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await; + let mut sub = within( + "subscribe through the MID-B leg", + bc.track("video").expect("track handle").subscribe(None), + ) + .await + .expect("subscribe"); + + // Three distinct post-drain groups, every frame verified, proves the new leg + // carries it: MID-A is severed, so they can only have flowed TOP -> MID-B -> + // BOTTOM. let mut post = BTreeSet::new(); while post.iter().filter(|seq| **seq > LAST_GROUP).count() < 3 { collect_group(&mut sub, &mut post, FRAMES_PER_GROUP, "post-drain (MID-B leg only)").await; } post_publisher.abort(); - // The splice must not re-deliver anything the old leg already served. - assert!( - post.iter().all(|seq| *seq > LAST_GROUP), - "the splice re-delivered pre-failover groups: {post:?}" - ); // ── no GOAWAY cascade to the downstream subscriber ─────────────────── assert!( @@ -666,15 +687,16 @@ async fn cluster_diamond_goaway_seamless_failover_inner() { "downstream subscriber received a GOAWAY (the relay should absorb it): {leaked:?}" ); - // ── announcement stability: the path never retracted under the swap ── - // Metadata updates (route re-pricing, the new leg's hops) are expected. + // ── announcement: the path may bounce under the swap, but ends announced ── + let mut active = true; loop { match tokio::time::timeout(Duration::from_millis(500), announcements.next()).await { Err(_) => break, - Ok(Some(update)) if update.kind.is_active() => continue, - Ok(event) => panic!("failover must not retract the path under the subscriber: {event:?}"), + Ok(Some(update)) => active = update.kind.is_active(), + Ok(None) => panic!("the subscriber's announcements ended"), } } + assert!(active, "the failover left the path retracted under the subscriber"); } /// Receive the next group, assert every frame's payload and the exact frame count From 83590a3258d1b1d92a4eadccb1cdeb948fd974a9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 18:00:37 -0800 Subject: [PATCH 4/6] fix(moq-net): a peer's copy dying mid-serve ends its track As main does for a lite-06 (minted) instance: asking the peer again could reach another broadcast under the same path, since the peer's front may have ended and a fresh one taken over, so the track ends with the error and a returning reader is not re-spliced. A local broadcast's copy still re-splices, as on main, since it is the one broadcast published here. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/front.rs | 42 +++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index c5659661e06..a5607f15ecc 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -563,12 +563,22 @@ impl Front { track.ended = true; actions.push(Action::Finish { track: name }); } - // Died mid-serve after delivering: normal failover, re-splice from - // the serving source at the first missing group. - Err(_) if delivered => { + // Died mid-serve after delivering from a broadcast published here: the + // source is that one broadcast, so re-splice from it at the first missing + // group, as for a track its publisher re-created. + Err(_) if delivered && self.identity == Identity::Local => { track.state = TrackState::Idle; self.redispatch(name, actions); } + // Died mid-serve after delivering from a peer. Asking the source again + // could reach another broadcast under the same path (the peer's front may + // have ended and a fresh one taken over), so the track ends with the error + // instead of re-splicing, and a returning reader is not re-spliced either. + Err(err) if delivered => { + track.state = TrackState::Idle; + track.ended = true; + actions.push(Action::Abort { track: name, err }); + } // Died before producing anything: a refusal. Err(err) => self.refuse(name, source, closing, err, actions), } @@ -1031,9 +1041,33 @@ mod tests { assert!(front.tracks[&name("audio")].refused.is_empty()); } + /// A peer's copy dying after delivering ends the track: asking the peer again could + /// reach another broadcast under the same path. A returning reader is not + /// re-spliced either. #[test] - fn a_copy_dying_after_delivering_resplices() { + fn a_peer_copy_dying_after_delivering_ends_the_track() { let mut front = serving(remote(1, 10), 100); + assert_actions( + front.step(Event::TrackEnded { + track: name("video"), + source: 100, + closing: false, + result: Err(Error::Dropped), + delivered: true, + }), + &[Action::Abort { + track: name("video"), + err: Error::Dropped, + }], + ); + assert_actions(front.step(Event::Used { track: name("video") }), &[]); + } + + /// A local copy dying after delivering re-splices: the source is the one broadcast + /// published here, as when its publisher re-creates the track. + #[test] + fn a_local_copy_dying_after_delivering_resplices() { + let mut front = serving(local(1), 100); assert_actions( front.step(Event::TrackEnded { track: name("video"), From 88f137135a9db69f725a043d7f99cbe6a046fd91 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 18:25:08 -0800 Subject: [PATCH 5/6] fix(moq-net): bounce a route renamed in place, and say a front serves one source A route renamed in place (its first hop changed, path unchanged) shows the announce cursors only an update, so a front ending on it left the old broadcast announced active. The front now bounces in that case too. The front, origin, and request_broadcast docs still described resuming through routes sharing a first hop; they now say a front serves one source and ends with a bounce. moq-tokio's route migration test asserts the new contract: the old subscription ends and a fresh request reads the new route. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/front.rs | 30 +++++++++++------- rs/moq-net/src/model/origin.rs | 45 +++++++++++++++++++++------ rs/moq-tokio/tests/broadcast.rs | 55 ++++++++++++++++++++++++++------- 3 files changed, 98 insertions(+), 32 deletions(-) diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index a5607f15ecc..1df0ea83f13 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -2,8 +2,10 @@ //! //! A [`Front`] is one path served through a spliced broadcast: it picks the //! source it serves from and splices each logical track from that source's -//! copy, re-splicing when the source is replaced and refusing what a source -//! will not serve. Everything here is plain data: [`Front::step`] takes one +//! copy, refusing what the source will not serve. A front serves one source for +//! its life: when that source ends or another route becomes best, the front +//! ends and bounces the path's announcement rather than splicing in a source +//! that may be another broadcast under the same path and group numbers. Everything here is plain data: [`Front::step`] takes one //! [`Event`] and returns the [`Action`]s to perform, without locks or waiters. //! The origin's driver task feeds events from the world (the route table, the //! sources, the tracks, the clock) and executes the actions; see @@ -140,17 +142,17 @@ pub(super) enum Identity { /// covering route, and whoever serves it fixes the identity. Undetermined, /// The first source was a broadcast published on this origin. A newer local - /// publisher takes over while the incumbent is live; once the incumbent is - /// closing the front ends instead, so a newcomer gets a fresh broadcast - /// rather than being spliced into one that is over. + /// publisher ends the front, so the newcomer gets a fresh broadcast rather + /// than being spliced into the incumbent's. Local, /// The serving route's first hop was absent or [`Hop::UNKNOWN`], which /// identifies nobody: the front cannot resume through any other route, so /// its source ending, `route` leaving the table, or `route` gaining a first /// hop ends it. Anonymous { route: u64 }, - /// The first hop of the serving route. Routes sharing it are the same - /// origin reached another way and safe to resume through. + /// The first hop of the serving route. Routes sharing it are the same origin + /// reached another way, so they qualify while the first request resolves; + /// once a source serves, any route change still ends the front. Publisher(Hop), } @@ -375,14 +377,20 @@ impl Front { actions.push(Action::Request { route: candidate.route }); } // Nothing qualifies: the front is over, even with a live source. Its - // route left the table and nothing with the same content replaced it, - // so it serves nobody new; a later announcement gets a fresh front. + // route left the table or now names another publisher, and nothing with + // the same content replaced it, so it serves nobody new; a later + // announcement gets a fresh front. A route still at the path (renamed in + // place) shows the announce cursors only an update, so a front that + // served bounces it like any other end. None => { self.upstream = None; let err = match self.identity { Identity::Undetermined => self.last_err.take().unwrap_or(Error::Unroutable), _ => self.last_err.take().unwrap_or(Error::Dropped), }; + if self.resolved { + actions.push(Action::Bounce); + } self.end(err, actions); } } @@ -909,7 +917,7 @@ mod tests { best: None, serving_closing: false, }), - &[Action::End { err: Error::Dropped }], + &[Action::Bounce, Action::End { err: Error::Dropped }], ); } @@ -935,7 +943,7 @@ mod tests { best: None, serving_closing: false, }), - &[Action::End { err: Error::Dropped }], + &[Action::Bounce, Action::End { err: Error::Dropped }], ); } diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index e1e2ead85f5..a70f2188351 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -3664,9 +3664,9 @@ struct OriginState { cursors: HashMap, // The remotely-served fronts; see [`FrontKey`]. Each is a spliced broadcast - // whose watcher task materializes it from the best covering route and - // re-splices it through routes sharing its first hop, so a route change the - // identity survives is invisible to subscribers. Weak, so a front dies with + // whose watcher task materializes it from the best covering route and serves + // that one source until it ends or another route becomes best, when the + // front ends and bounces the path's announcement. Weak, so a front dies with // its watcher and a later request re-creates it. fronts: WeakCache, @@ -4693,12 +4693,11 @@ impl Consumer { /// the best announced route covering it (the most specific prefix, then the /// cheapest, a broadcast published on this origin winning ties) and /// materializes it, from the broadcast itself or from the peer that - /// announced the route. When its serving source dies or a better qualifying - /// route appears, the front re-splices through the best route sharing its - /// first hop at a group boundary, invisibly to subscribers. A change that - /// does not preserve the first hop ends the broadcast instead, as does its - /// route retracting with no replacement, and the next request re-serves the - /// path. Tracks already in flight carry on to their own end. + /// announced the route. A front never splices a second source in: when its + /// serving source dies, another route becomes best, or its route retracts or + /// is renamed, the broadcast ends and the path's announcement is bounced + /// (ended, then announced again where a route remains), and the next request + /// re-serves the path. Tracks already in flight carry on to their own end. /// /// The returned future fails with [`Error::Unroutable`] at once when no /// announced route covers the path, including a broadcast created on this @@ -8144,6 +8143,34 @@ mod tests { } } + /// A route renamed in place to another publisher leaves the front's pin with + /// nothing to select, so the front ends while the path stays announced, and it + /// bounces like any other end. + #[tokio::test] + async fn a_route_renamed_in_place_bounces_its_announce() { + let producer = origin(1).produce(); + let consumer = producer.consume(); + let mut view = consumer.announced(); + let server = producer + .dynamic("room/alice", Route::default().with_hops(hops(&[10]))) + .unwrap(); + view.assert_next_active("room/alice"); + let pending = consumer.request_broadcast("room/alice"); + // Held open, so the front ends for its route, not for a closed source. + let source = broadcast::Info::new().produce(); + queued(&server).await.accept(&source); + let resolved = pending.await.expect("resolves"); + + server.update(Route::default().with_hops(hops(&[11]))).unwrap(); + tokio::time::timeout(Duration::from_secs(10), resolved.closed()) + .await + .expect("the front ends"); + view.assert_next_ended("room/alice"); + view.assert_next_active("room/alice"); + view.assert_next_wait(); + drop(source); + } + /// A front at a path served only through a covering prefix bounces nothing: /// re-announcing the prefix would restart its other broadcasts too. #[tokio::test] diff --git a/rs/moq-tokio/tests/broadcast.rs b/rs/moq-tokio/tests/broadcast.rs index c1c0b410e3f..95e50a849e0 100644 --- a/rs/moq-tokio/tests/broadcast.rs +++ b/rs/moq-tokio/tests/broadcast.rs @@ -979,13 +979,15 @@ async fn read_payloads(sub: &mut moq_net::track::Subscriber, count: usize) -> Ve payloads } -/// A path stays routed across two publisher sessions announcing it. +/// A path stays routed across two publisher sessions announcing it, and a +/// subscription never rides from one to the other. /// /// The client connects to two servers announcing the same route. The preferred -/// (cheaper) route serves the track; when that session dies, the broadcast -/// re-splices through the standby at a group boundary, without the path ever -/// being retracted: both routes name the same first hop, so they are the same -/// origin reached different ways and the subscription rides the failover. +/// (cheaper) route serves the track. When that session dies, the broadcast ends +/// rather than splice the standby's copy in, even though both routes name the +/// same first hop: the publisher may have re-created the broadcast with its +/// group numbers starting over. The path bounces, and a fresh request reads the +/// standby. #[tracing_test::traced_test] #[tokio::test] async fn broadcast_route_migration() { @@ -1110,13 +1112,41 @@ async fn broadcast_route_migration() { .expect("subscribe failed"); assert_eq!(read_payloads(&mut sub, 1).await, ["a1"]); - // Kill the serving session. Both routes name the same first hop, so the - // broadcast re-splices through the standby at the group boundary and the - // SAME subscription carries B's continuation. + // Kill the serving session. The old subscription ends without any of B's + // groups, and a fresh request resolves through the standby. drop(session_a); + let end = tokio::time::timeout(TIMEOUT, sub.recv_group()) + .await + .expect("the old subscription ends"); + assert!( + !matches!(end, Ok(Some(_))), + "the standby was spliced into the old subscription" + ); + let fresh = tokio::time::timeout(TIMEOUT, async { + loop { + if let Ok(fresh) = sub_consumer.request_broadcast("test").await + && !fresh.is_clone(&broadcast) + { + return fresh; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("a fresh request resolves through the standby"); + let mut sub = fresh + .track("video") + .unwrap() + .subscribe( + moq_net::track::Subscription::default() + .with_start(moq_net::track::Position::group(2)) + .with_max_age(Duration::from_secs(10)), + ) + .await + .expect("subscribe failed"); assert_eq!(read_payloads(&mut sub, 2).await, ["b2", "b3"]); - // The new subscription is live: what B produces from here on flows through it. + // The fresh subscription is live: what B produces from here on flows through it. { let mut group = standby_track .create_group(moq_net::group::Info { sequence: 4 }) @@ -1126,11 +1156,12 @@ async fn broadcast_route_migration() { } assert_eq!(read_payloads(&mut sub, 1).await, ["b4"]); - // The path was never retracted across the swap: any events delivered are - // active metadata updates (the standby's chain taking over). + // The path may have bounced across the swap, but it ends announced. + let mut active = true; while let Some(update) = announcements.try_next() { - assert!(update.kind.is_active(), "failover must not retract the route"); + active = update.kind.is_active(); } + assert!(active, "the failover left the route retracted"); handle_a.await.expect("server a panicked").expect("server a failed"); drop(_session_b); From 94dd3acc08fed70de561be0220b519aad87bebe7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 18:36:15 -0800 Subject: [PATCH 6/6] docs(moq-net): request_broadcast's body says a front serves one source Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/origin.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index a70f2188351..9f97639cc00 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -4772,8 +4772,8 @@ impl Consumer { // A route covers the path: mint the front and hand its watcher the // request. The watcher materializes the path from the best covering - // route, resolves the channel, and re-splices the front through - // routes sharing its first hop for as long as one serves. + // route, resolves the channel, and serves that one source until it ends + // or another route becomes best, when the front bounces the path and ends. let broadcast = broadcast::Producer::new_spliced(broadcast::Info { pool: self.pool.clone(), cache_duration: self.cache_duration,