diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index 3f2cfb3396c..44a6fc47c72 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -32,12 +32,15 @@ follow the current best route immediately; only the announcement waits. For publisher that a peer still hunting offers, instead of announcing the broadcast again. -Failover routes must carry copies of the same broadcast. For each track, the -relay requires matching timescale, retention window, publisher priority, and -group ordering. A source with different properties is refused before its groups -are spliced in. If no compatible source remains, the track fails with -`Unsupported`. New immutable properties require a new track name or broadcast -identity. +A relay never splices a second route into a broadcast it serves. Two routes +from the same publisher can carry different broadcasts under one path, such as +one the publisher re-created with its group numbers starting over. So when the +serving route closes, or another route becomes the best, the relay ends the +broadcast and re-announces its path: a retraction, then the route again. +Subscribers then request the broadcast afresh through the current best route, +which costs viewers a brief restart on every failover, a relay drain included. +A path announced only through a covering prefix is not re-announced, since that +would restart the prefix's other broadcasts too. A route whose original publisher (its first hop) changes is updated in place on both wire protocols, so the broadcast never briefly vanishes downstream. diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index 7b88d82edca..1df0ea83f13 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -2,8 +2,10 @@ //! //! A [`Front`] is one path served through a spliced broadcast: it picks the //! source it serves from and splices each logical track from that source's -//! copy, re-splicing when the source is replaced and refusing what a source -//! will not serve. Everything here is plain data: [`Front::step`] takes one +//! copy, refusing what the source will not serve. A front serves one source for +//! its life: when that source ends or another route becomes best, the front +//! ends and bounces the path's announcement rather than splicing in a source +//! that may be another broadcast under the same path and group numbers. Everything here is plain data: [`Front::step`] takes one //! [`Event`] and returns the [`Action`]s to perform, without locks or waiters. //! The origin's driver task feeds events from the world (the route table, the //! sources, the tracks, the clock) and executes the actions; see @@ -122,6 +124,11 @@ pub(super) enum Action { Abort { track: Arc, err: Error }, /// Arm (or clear) the deadline the front wants to be woken at. Arm { at: Option }, + /// Re-announce the path to its announce consumers, a retraction then the + /// route again. Precedes the [`Action::End`] of a front that served while a + /// route may still serve the path: the table alone shows those consumers no + /// change, so nothing else tells them the broadcast they hold has ended. + Bounce, /// The front is over: reject the parked requesters with `err`, leave each /// read track to end with the copy it is spliced from or still waiting on, /// abort the rest with `err`, and drop every source. @@ -135,17 +142,17 @@ pub(super) enum Identity { /// covering route, and whoever serves it fixes the identity. Undetermined, /// The first source was a broadcast published on this origin. A newer local - /// publisher takes over while the incumbent is live; once the incumbent is - /// closing the front ends instead, so a newcomer gets a fresh broadcast - /// rather than being spliced into one that is over. + /// publisher ends the front, so the newcomer gets a fresh broadcast rather + /// than being spliced into the incumbent's. Local, /// The serving route's first hop was absent or [`Hop::UNKNOWN`], which /// identifies nobody: the front cannot resume through any other route, so /// its source ending, `route` leaving the table, or `route` gaining a first /// hop ends it. Anonymous { route: u64 }, - /// The first hop of the serving route. Routes sharing it are the same - /// origin reached another way and safe to resume through. + /// The first hop of the serving route. Routes sharing it are the same origin + /// reached another way, so they qualify while the first request resolves; + /// once a source serves, any route change still ends the front. Publisher(Hop), } @@ -352,27 +359,38 @@ impl Front { Some(candidate) if Some(candidate.route) == serving_route => { self.upstream = None; } - // Already asking through it. - Some(candidate) if self.upstream == Some(candidate.route) => {} - // A local newcomer must not be spliced into a broadcast whose - // publisher is on the way out: end, and a fresh front serves it. - Some(candidate) if candidate.local && self.identity == Identity::Local && serving_closing => { + // Another route is now best. It may carry another broadcast under the same + // path and group numbers (a local newcomer, a publisher back with its + // numbering restarted), so it is never spliced into this one. Keeping the + // old source instead would read through a route other than the one this + // relay advertises. The front ends, and readers re-request through the + // table's choice. + Some(_) if self.serving.is_some() => { + actions.push(Action::Bounce); self.end(Error::Dropped, actions); } + // Already asking through it. + Some(candidate) if self.upstream == Some(candidate.route) => {} // A better (or replacement) qualifying route: request through it. Some(candidate) => { self.upstream = Some(candidate.route); actions.push(Action::Request { route: candidate.route }); } // Nothing qualifies: the front is over, even with a live source. Its - // route left the table and nothing with the same content replaced it, - // so it serves nobody new; a later announcement gets a fresh front. + // route left the table or now names another publisher, and nothing with + // the same content replaced it, so it serves nobody new; a later + // announcement gets a fresh front. A route still at the path (renamed in + // place) shows the announce cursors only an update, so a front that + // served bounces it like any other end. None => { self.upstream = None; let err = match self.identity { Identity::Undetermined => self.last_err.take().unwrap_or(Error::Unroutable), _ => self.last_err.take().unwrap_or(Error::Dropped), }; + if self.resolved { + actions.push(Action::Bounce); + } self.end(err, actions); } } @@ -462,8 +480,7 @@ impl Front { return; } // A standing route can outlive the source it produced. Asking it again - // would re-request the broadcast that just ended; another route to the - // same publisher may still resume it. + // would re-request the broadcast that just ended. self.refused.insert(route); self.serving = None; self.serving_closing = false; @@ -476,12 +493,11 @@ impl Front { } } self.last_err = Some(Error::Dropped); - match self.identity { - // A local publisher ending ends its broadcast; a newcomer at the - // path gets a fresh one. An anonymous source can never be resumed. - Identity::Local | Identity::Anonymous { .. } | Identity::Undetermined => self.end(Error::Dropped, actions), - Identity::Publisher(_) => actions.push(Action::Reselect), - } + // The source's broadcast is over. Another route, even to the same publisher, + // may carry a different broadcast under the same path and group numbers, so + // nothing resumes it: the front ends and a newcomer gets a fresh one. + actions.push(Action::Bounce); + self.end(Error::Dropped, actions); } fn track_info( @@ -555,12 +571,22 @@ impl Front { track.ended = true; actions.push(Action::Finish { track: name }); } - // Died mid-serve after delivering: normal failover, re-splice from - // the serving source at the first missing group. - Err(_) if delivered => { + // Died mid-serve after delivering from a broadcast published here: the + // source is that one broadcast, so re-splice from it at the first missing + // group, as for a track its publisher re-created. + Err(_) if delivered && self.identity == Identity::Local => { track.state = TrackState::Idle; self.redispatch(name, actions); } + // Died mid-serve after delivering from a peer. Asking the source again + // could reach another broadcast under the same path (the peer's front may + // have ended and a fresh one taken over), so the track ends with the error + // instead of re-splicing, and a returning reader is not re-spliced either. + Err(err) if delivered => { + track.state = TrackState::Idle; + track.ended = true; + actions.push(Action::Abort { track: name, err }); + } // Died before producing anything: a refusal. Err(err) => self.refuse(name, source, closing, err, actions), } @@ -827,46 +853,34 @@ mod tests { ); } + /// A better route to the same publisher is never spliced in, since it may carry + /// another broadcast under the same group numbers: the front ends instead. #[test] - fn better_route_takes_over_and_resplices() { + fn a_better_route_ends_the_front() { let mut front = serving(remote(1, 10), 100); assert_actions( front.step(Event::Selected { best: Some(remote(2, 10)), serving_closing: false, }), - &[Action::Request { route: 2 }], + &[Action::Bounce, Action::End { err: Error::Dropped }], ); - assert_actions( - front.step(Event::Resolved { - route: 2, - result: Ok(200), - }), - &[ - Action::Detach { source: 100 }, - Action::Query { - track: name("video"), - source: 200, - }, - ], - ); - assert_eq!(front.serving, Some((200, 2))); } #[test] fn a_stale_resolution_is_let_go() { - let mut front = serving(remote(1, 10), 100); + let mut front = Front::new(LINGER); front.step(Event::Selected { best: Some(remote(2, 10)), serving_closing: false, }); - // The table moved back before the request landed. + // The table moved on before the request landed. assert_actions( front.step(Event::Selected { best: Some(remote(1, 10)), serving_closing: false, }), - &[], + &[Action::Request { route: 1 }], ); assert_actions( front.step(Event::Resolved { @@ -877,31 +891,20 @@ mod tests { ); } + /// A source closing ends the front, even with another route to its publisher: + /// nothing resumes a broadcast through another source. #[test] - fn dead_source_reselects_through_the_same_publisher() { + fn a_dead_source_ends_the_front() { let mut front = serving(remote(1, 10), 100); assert_actions( front.step(Event::SourceClosed { source: 100 }), - &[Action::Detach { source: 100 }, Action::Reselect], - ); - assert!(front.refused_routes().contains(&1)); - assert_actions( - front.step(Event::Selected { - best: Some(remote(3, 10)), - serving_closing: false, - }), - &[Action::Request { route: 3 }], - ); - assert_actions( - front.step(Event::Resolved { - route: 3, - result: Ok(300), - }), - &[Action::Query { - track: name("video"), - source: 300, - }], + &[ + Action::Detach { source: 100 }, + Action::Bounce, + Action::End { err: Error::Dropped }, + ], ); + assert!(front.ended()); } /// A live source does not keep a front alive once its route is gone: nothing @@ -914,7 +917,7 @@ mod tests { best: None, serving_closing: false, }), - &[Action::End { err: Error::Dropped }], + &[Action::Bounce, Action::End { err: Error::Dropped }], ); } @@ -940,20 +943,7 @@ mod tests { best: None, serving_closing: false, }), - &[Action::End { err: Error::Dropped }], - ); - } - - #[test] - fn dead_source_with_no_replacement_ends() { - let mut front = serving(remote(1, 10), 100); - front.step(Event::SourceClosed { source: 100 }); - assert_actions( - front.step(Event::Selected { - best: None, - serving_closing: false, - }), - &[Action::End { err: Error::Dropped }], + &[Action::Bounce, Action::End { err: Error::Dropped }], ); } @@ -968,7 +958,11 @@ mod tests { assert_eq!(front.pin(), Pin::Route(1)); assert_actions( front.step(Event::SourceClosed { source: 100 }), - &[Action::Detach { source: 100 }, Action::End { err: Error::Dropped }], + &[ + Action::Detach { source: 100 }, + Action::Bounce, + Action::End { err: Error::Dropped }, + ], ); } @@ -991,27 +985,6 @@ mod tests { ); } - #[test] - fn standing_refusal_while_serving_skips_the_refuser() { - let mut front = serving(remote(1, 10), 100); - front.step(Event::Selected { - best: Some(remote(2, 10)), - serving_closing: false, - }); - assert_actions( - front.step(Event::Resolved { - route: 2, - result: Err(Refusal { - err: Error::NotFound, - standing: true, - }), - }), - &[Action::Reselect], - ); - assert!(front.refused_routes().contains(&2)); - assert_eq!(front.serving, Some((100, 1))); - } - #[test] fn retracted_route_is_not_a_refusal() { let mut front = Front::new(LINGER); @@ -1074,75 +1047,35 @@ mod tests { &[], ); assert!(front.tracks[&name("audio")].refused.is_empty()); - // The replacement is asked afresh. - front.step(Event::SourceClosed { source: 100 }); - front.step(Event::Selected { - best: Some(remote(2, 10)), - serving_closing: false, - }); - let actions = front.step(Event::Resolved { - route: 2, - result: Ok(200), - }); - assert!( - actions - .iter() - .any(|action| matches!(action, Action::Query { track, source: 200 } if track.as_ref() == "audio")) - ); } - /// The serving copy detaches while still open, because its source closed or a - /// better route replaced it, and the replacement refuses the track: the detached - /// copy's end, not the refusal, ends it. + /// A peer's copy dying after delivering ends the track: asking the peer again could + /// reach another broadcast under the same path. A returning reader is not + /// re-spliced either. #[test] - fn a_refusal_waits_for_the_detached_copy() { - for (closed, end, expected) in [true, false].into_iter().flat_map(|closed| { - [ - (closed, Ok(()), Action::Finish { track: name("video") }), - ( - closed, - Err(Error::Dropped), - Action::Abort { - track: name("video"), - err: Error::Unroutable, - }, - ), - ] - }) { - let mut front = serving(remote(1, 10), 100); - if closed { - front.step(Event::SourceClosed { source: 100 }); - } - front.step(Event::Selected { - best: Some(remote(2, 10)), - serving_closing: false, - }); - front.step(Event::Resolved { - route: 2, - result: Ok(200), - }); - assert_actions( - front.step(Event::TrackInfo { - track: name("video"), - source: 200, - closing: false, - result: Err(Error::Unroutable), - }), - &[], - ); - assert_actions( - front.step(Event::Drained { - track: name("video"), - result: end, - }), - &[expected], - ); - } + fn a_peer_copy_dying_after_delivering_ends_the_track() { + let mut front = serving(remote(1, 10), 100); + assert_actions( + front.step(Event::TrackEnded { + track: name("video"), + source: 100, + closing: false, + result: Err(Error::Dropped), + delivered: true, + }), + &[Action::Abort { + track: name("video"), + err: Error::Dropped, + }], + ); + assert_actions(front.step(Event::Used { track: name("video") }), &[]); } + /// A local copy dying after delivering re-splices: the source is the one broadcast + /// published here, as when its publisher re-creates the track. #[test] - fn a_copy_dying_after_delivering_resplices() { - let mut front = serving(remote(1, 10), 100); + fn a_local_copy_dying_after_delivering_resplices() { + let mut front = serving(local(1), 100); assert_actions( front.step(Event::TrackEnded { track: name("video"), @@ -1191,35 +1124,6 @@ mod tests { ); } - #[test] - fn incompatible_copy_is_refused() { - let mut front = serving(remote(1, 10), 100); - front.step(Event::Selected { - best: Some(remote(2, 10)), - serving_closing: false, - }); - front.step(Event::Resolved { - route: 2, - result: Ok(200), - }); - let other = track::Info { - max_age: Duration::from_secs(1), - ..track::Info::default() - }; - assert_actions( - front.step(Event::TrackInfo { - track: name("video"), - source: 200, - closing: false, - result: Ok(other), - }), - &[Action::Abort { - track: name("video"), - err: Error::Unsupported, - }], - ); - } - #[test] fn unread_track_parks_then_is_forgotten_after_the_linger() { let mut front = serving(remote(1, 10), 100); @@ -1301,18 +1205,6 @@ mod tests { front.step(Event::Used { track: name("video") }), &[Action::Arm { at: None }], ); - // A replacement source does not re-splice it either. - front.step(Event::Selected { - best: Some(remote(2, 10)), - serving_closing: false, - }); - assert_actions( - front.step(Event::Resolved { - route: 2, - result: Ok(200), - }), - &[Action::Detach { source: 100 }], - ); let t1 = t0 + LINGER; front.step(Event::Unused { track: name("video"), @@ -1404,8 +1296,10 @@ mod tests { ); } + /// A local newcomer is a new broadcast: the front ends so a fresh one serves it, + /// rather than splicing it into the incumbent's tracks. #[test] - fn local_newcomer_takes_over_a_live_incumbent() { + fn a_local_newcomer_ends_the_front() { let mut front = serving(local(1), 100); assert_eq!(front.identity, Identity::Local); assert_eq!(front.pin(), Pin::Local); @@ -1414,7 +1308,7 @@ mod tests { best: Some(local(2)), serving_closing: false, }), - &[Action::Request { route: 2 }], + &[Action::Bounce, Action::End { err: Error::Dropped }], ); } @@ -1426,7 +1320,7 @@ mod tests { best: Some(local(2)), serving_closing: true, }), - &[Action::End { err: Error::Dropped }], + &[Action::Bounce, Action::End { err: Error::Dropped }], ); } @@ -1435,7 +1329,11 @@ mod tests { let mut front = serving(local(1), 100); assert_actions( front.step(Event::SourceClosed { source: 100 }), - &[Action::Detach { source: 100 }, Action::End { err: Error::Dropped }], + &[ + Action::Detach { source: 100 }, + Action::Bounce, + Action::End { err: Error::Dropped }, + ], ); } diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index a21e5666059..9f97639cc00 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -1215,9 +1215,10 @@ struct RemoteFront { /// The front's spliced broadcast, weak: dead once the front ends, so a /// later request re-creates the front instead of joining a corpse. broadcast: broadcast::WeakConsumer, - /// Which routes serve the front's content, fixed by its first source. A - /// request joins the front only while the best route is one of them. - pin: kio::Lock, + /// Which routes serve the front's content, and the route its source came + /// through, fixed by its first source. A request joins the front only while + /// the best route is that route: the front never changes routes, it ends. + pin: kio::Lock<(Pin, Option)>, } /// The last route a cursor observed at a presented prefix. @@ -1321,6 +1322,19 @@ impl TableCursor { && self.named.as_ref().is_none_or(|(mount, _)| mount.names(&entry.prefix)) } + /// Re-deliver the route at this presented prefix as a retraction then an + /// announce, if it was announced at exactly `path`. See [`OriginState::bounce`]. + fn bounce(&mut self, presented: &PathOwned, path: &Path) { + let Some(current) = self.current.get(presented).filter(|current| current.prefix == *path) else { + return; + }; + if let Ok(mut state) = self.state.write() { + let at = self.under.join(presented); + state.apply_unannounce(at.clone(), current.meta.clone(), current.captures.clone()); + state.apply_announce(at, current.meta.clone(), current.captures.clone()); + } + } + /// Deliver a changed winner at this presented prefix. fn update(&mut self, presented: &PathOwned, best: Option<&RouteEntry>) { let Some(entry) = best else { @@ -2685,7 +2699,7 @@ struct FrontTask { /// Resolves the requesters parked on the front's channel. request: kio::Producer, /// Published for requesters once the first source fixes it; see [`RemoteFront::pin`]. - pin: kio::Lock, + pin: kio::Lock<(Pin, Option)>, timers: Clock, } @@ -2886,7 +2900,7 @@ async fn run_front(task: FrontTask) { continue; }; front.identify(candidate); - *pin.lock() = front.pin(); + *pin.lock() = (front.pin(), Some(route)); if let Some(source) = source { let id = next_source; next_source += 1; @@ -3082,6 +3096,7 @@ async fn run_front(task: FrontTask) { } } Action::Arm { at } => deadline.set(at), + Action::Bounce => shared.lock().bounce(&path.as_path()), Action::End { err } => { if let Ok(mut pending) = request.write() { pending.resolved.get_or_insert(Err(err.clone())); @@ -3649,9 +3664,9 @@ struct OriginState { cursors: HashMap, // The remotely-served fronts; see [`FrontKey`]. Each is a spliced broadcast - // whose watcher task materializes it from the best covering route and - // re-splices it through routes sharing its first hop, so a route change the - // identity survives is invisible to subscribers. Weak, so a front dies with + // whose watcher task materializes it from the best covering route and serves + // that one source until it ends or another route becomes best, when the + // front ends and bounces the path's announcement. Weak, so a front dies with // its watcher and a later request re-creates it. fronts: WeakCache, @@ -3783,6 +3798,26 @@ impl OriginState { routes.poke_below(prefix); } + /// Re-announce `path` on every cursor presenting a route at exactly `path`: + /// a retraction, then the same route again. A front at `path` is ending while + /// a route may still serve it, and the cursors would otherwise see no change, + /// so a consumer holding the ended broadcast would never request a fresh one. + /// A cursor seeing `path` only through a covering prefix is left alone, since + /// a bounce there would restart the prefix's other broadcasts too. + fn bounce(&mut self, path: &Path) { + let Ok(claim) = prefix_claim(path) else { + return; + }; + for id in self.routes.cursors_touching(path) { + let Some(cursor) = self.cursors.get_mut(&id) else { + continue; + }; + if let Some(presented) = cursor.presented(path, &claim) { + cursor.bounce(&presented, path); + } + } + } + /// Register a [`Watch`] on the routes covering `path`. fn watch(&mut self, shared: &kio::Shared, path: &Path) -> Watch { let id = self.next_watch; @@ -4658,12 +4693,11 @@ impl Consumer { /// the best announced route covering it (the most specific prefix, then the /// cheapest, a broadcast published on this origin winning ties) and /// materializes it, from the broadcast itself or from the peer that - /// announced the route. When its serving source dies or a better qualifying - /// route appears, the front re-splices through the best route sharing its - /// first hop at a group boundary, invisibly to subscribers. A change that - /// does not preserve the first hop ends the broadcast instead, as does its - /// route retracting with no replacement, and the next request re-serves the - /// path. Tracks already in flight carry on to their own end. + /// announced the route. A front never splices a second source in: when its + /// serving source dies, another route becomes best, or its route retracts or + /// is renamed, the broadcast ends and the path's announcement is bounced + /// (ended, then announced again where a route remains), and the next request + /// re-serves the path. Tracks already in flight carry on to their own end. /// /// The returned future fails with [`Error::Unroutable`] at once when no /// announced route covers the path, including a broadcast created on this @@ -4717,16 +4751,16 @@ impl Consumer { // Join the live front for this path and horizon, if any: its watcher // resolves (or already resolved) the request channel with the front's // spliced broadcast, so repeat requests share one upstream - // subscription. Only while the best route still serves the front's - // content, though: once a different publisher wins (a cheaper route), a - // newcomer gets a fresh front from it, and the old front keeps serving the - // readers it has, since other content can't be spliced into it. + // subscription. Only while the best route is still the one the front's + // source came through, though: once another route wins, even from the + // same publisher, the front is about to end rather than splice it in, so + // a newcomer gets a fresh front from it at once. let key = (absolute.clone(), horizon); if let Some(front) = state.fronts.get(&key) { - let pin = *front.pin.lock(); + let (pin, route) = *front.pin.lock(); let current = state .best_route(&absolute.as_path(), horizon, Pin::Any, &HashSet::new()) - .is_some_and(|entry| entry.qualifies(pin)); + .is_some_and(|entry| entry.qualifies(pin) && route.is_none_or(|route| entry.id == route)); if current { let pending = Requesting::queued(front.request.consume()) .with_path(requested) @@ -4738,8 +4772,8 @@ impl Consumer { // A route covers the path: mint the front and hand its watcher the // request. The watcher materializes the path from the best covering - // route, resolves the channel, and re-splices the front through - // routes sharing its first hop for as long as one serves. + // route, resolves the channel, and serves that one source until it ends + // or another route becomes best, when the front bounces the path and ends. let broadcast = broadcast::Producer::new_spliced(broadcast::Info { pool: self.pool.clone(), cache_duration: self.cache_duration, @@ -4748,7 +4782,7 @@ impl Consumer { let request = kio::Producer::::default(); let consumer = request.consume(); let watch = state.watch(&self.shared, &absolute); - let pin = kio::Lock::new(Pin::Any); + let pin = kio::Lock::new((Pin::Any, None)); state.fronts.insert( key, RemoteFront { @@ -6537,154 +6571,6 @@ mod tests { } } - /// A route that ends abruptly releases its open groups, and the takeover asks the - /// next route only for the frames past the break. The front keeps the head it - /// spliced, so a reader arriving afterwards still gets the whole group: whether the - /// old route dies before the new one takes over (a reconnect) or after (a cheaper - /// route preempting it). A reader joining mid-outage gets it too, and the reader - /// already mid-group is not handed it a second time. - #[tokio::test] - async fn a_takeover_keeps_the_open_group_head_for_later_readers() { - tokio::time::pause(); - for dies_first in [true, false] { - takeover_keeps_the_open_group_head(dies_first).await; - } - } - - async fn takeover_keeps_the_open_group_head(dies_first: bool) { - let producer = origin(1).produce(); - let first_server = producer - .dynamic("room/alice", Route::default().with_hops(hops(&[10])).with_cost(5)) - .unwrap(); - let pending = producer.consume().request_broadcast("room/alice"); - let upstream = broadcast::Info::new().produce(); - let mut dynamic = upstream.dynamic(); - queued(&first_server).await.accept(&upstream); - let resolved = pending.await.unwrap(); - - async fn read(group: &mut group::Consumer) -> Vec { - let frame = tokio::time::timeout(Duration::from_secs(1), group.read_frame()) - .await - .expect("frame") - .unwrap() - .expect("group ended"); - frame.payload.to_vec() - } - - // A reader is mid-way through the open group when the route changes. - let track = resolved.track("log").unwrap(); - let subscribing = tokio::spawn(async move { track.subscribe(None).await }); - let source = dynamic.requested_track().await.unwrap().accept(None); - let mut group = source.create_group(0u64.into()).unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"a".as_ref()).unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"b".as_ref()).unwrap(); - let mut subscription = subscribing.await.unwrap().unwrap(); - let mut reading = subscription.recv_group().await.unwrap().unwrap(); - assert_eq!(read(&mut reading).await, b"a"); - assert_eq!(read(&mut reading).await, b"b"); - - // Either the first route's session ends and reconnects, announcing the same route - // afresh while the old one is still held, or a cheaper route preempts the live one. - let mut first = Some((group, source, upstream, dynamic)); - let mut joining = None; - if dies_first { - drop(first.take()); - // The reader sees its route die and stalls. Polled in place: yielding would let - // the front give up on a path with no route left before the replacement lands. - assert!(futures::FutureExt::now_or_never(subscription.recv_group()).is_none()); - let track = resolved.track("log").unwrap(); - joining = Some(tokio::spawn(async move { - let mut joined = track.subscribe(None).await.unwrap(); - let group = joined.recv_group().await.unwrap().expect("track ended"); - (joined, group) - })); - } - let cost = if dies_first { 5 } else { 0 }; - let replacement_server = producer - .dynamic("room/alice", Route::default().with_hops(hops(&[10])).with_cost(cost)) - .unwrap(); - let replacement = broadcast::Info::new().produce(); - let mut replacement_dynamic = replacement.dynamic(); - queued(&replacement_server).await.accept(&replacement); - let mut next = tokio::time::timeout(Duration::from_secs(1), replacement_dynamic.requested_track()) - .await - .expect("the front asked the replacement") - .unwrap() - .resolving_start() - .accept(None); - next.start_at(0).unwrap(); - let mut next_group = next.create_group(0u64.into()).unwrap(); - next_group.start_at(2).unwrap(); - next_group.write_frame(crate::Timestamp::ZERO, b"c".as_ref()).unwrap(); - assert_eq!(read(&mut reading).await, b"c", "dies_first={dies_first}"); - - // A preempted route still owns its open group: its writer and other readers carry on. - if let Some((group, ..)) = &mut first { - let mut independent = group.consume(); - group.write_frame(crate::Timestamp::ZERO, b"x".as_ref()).unwrap(); - for expect in [b"a", b"b", b"x"] { - assert_eq!(read(&mut independent).await, expect); - } - group.finish().unwrap(); - } - drop(first); - - let track = resolved.track("log").unwrap(); - let mut later = tokio::time::timeout(Duration::from_secs(1), track.subscribe(None)) - .await - .expect("subscribe") - .unwrap(); - let mut late = tokio::time::timeout(Duration::from_secs(1), later.recv_group()) - .await - .unwrap_or_else(|_| panic!("dies_first={dies_first}: the later reader never got the group")) - .unwrap() - .expect("track ended"); - assert_eq!(late.sequence, 0); - for expect in [b"a", b"b", b"c"] { - assert_eq!(read(&mut late).await, expect, "dies_first={dies_first}"); - } - let mut joined = None; - if let Some(joining) = joining { - let (subscription, mut group) = tokio::time::timeout(Duration::from_secs(1), joining) - .await - .expect("the reader joining mid-outage never got the group") - .unwrap(); - assert_eq!(group.sequence, 0); - for expect in [b"a", b"b", b"c"] { - assert_eq!(read(&mut group).await, expect); - } - joined = Some((subscription, group)); - } - - // The reader that was mid-group already has it. - assert!( - tokio::time::timeout(Duration::from_secs(1), subscription.recv_group()) - .await - .is_err(), - "dies_first={dies_first}: the group was handed out twice" - ); - - // The continuation still flows to both. - next_group.write_frame(crate::Timestamp::ZERO, b"d".as_ref()).unwrap(); - assert_eq!(read(&mut reading).await, b"d"); - assert_eq!(read(&mut late).await, b"d"); - if let Some((_, group)) = &mut joined { - assert_eq!(read(group).await, b"d"); - } - next_group.finish().unwrap(); - drop(( - joined, - reading, - late, - subscription, - later, - next, - replacement, - replacement_server, - first_server, - )); - } - /// A returning reader is handed the parked open group spliced onto the source's /// continuation, which starts past the warm head. The group's end stays unknown /// until the source finishes it, even before a frame of it is read: readers drop a @@ -6754,91 +6640,6 @@ mod tests { assert_eq!(reading.finished().await.unwrap(), 2); } - #[tokio::test] - async fn warm_head_survives_another_takeover_before_park() { - tokio::time::pause(); - let producer = origin(1).produce(); - let _server = producer - .dynamic("room/alice", Route::default().with_hops(hops(&[10])).with_cost(5)) - .unwrap(); - let pending = producer.consume().request_broadcast("room/alice"); - let upstream = broadcast::Info::new().produce(); - let mut dynamic = upstream.dynamic(); - queued(&_server).await.accept(&upstream); - let resolved = pending.await.unwrap(); - - let track = resolved.track("log").unwrap(); - let subscribing = tokio::spawn(async move { track.subscribe(None).await }); - let source = dynamic.requested_track().await.unwrap().accept(None); - let mut group = source.create_group(0u64.into()).unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"a".as_ref()).unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"b".as_ref()).unwrap(); - let mut subscription = subscribing.await.unwrap().unwrap(); - let mut reading = subscription.recv_group().await.unwrap().unwrap(); - assert_eq!(&reading.read_frame().await.unwrap().unwrap().payload[..], b"a"); - drop(reading); - drop(subscription); - source.unused().await.unwrap(); - drop(group); - drop(source); - - let track = resolved.track("log").unwrap(); - let subscribing = tokio::spawn(async move { track.subscribe(None).await }); - let mut resumed = dynamic.requested_track().await.unwrap().resolving_start().accept(None); - resumed.start_at(0).unwrap(); - let mut subscription = subscribing.await.unwrap().unwrap(); - let mut reading = subscription.recv_group().await.unwrap().unwrap(); - assert_eq!(&reading.read_frame().await.unwrap().unwrap().payload[..], b"a"); - - let replacement_server = producer - .dynamic("room/alice", Route::default().with_hops(hops(&[10]))) - .unwrap(); - let replacement = broadcast::Info::new().produce(); - let mut replacement_dynamic = replacement.dynamic(); - queued(&replacement_server).await.accept(&replacement); - let mut source = replacement_dynamic - .requested_track() - .await - .unwrap() - .resolving_start() - .accept(None); - source.start_at(0).unwrap(); - let mut group = source.create_group(0u64.into()).unwrap(); - group.start_at(2).unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"c".as_ref()).unwrap(); - let mut continuation = reading.clone(); - continuation.start_at(2); - let frame = tokio::time::timeout(Duration::from_secs(1), continuation.read_frame()) - .await - .expect("replacement frame") - .unwrap() - .unwrap(); - assert_eq!(&frame.payload[..], b"c"); - drop(continuation); - assert_eq!(&reading.read_frame().await.unwrap().unwrap().payload[..], b"b"); - assert_eq!(&reading.read_frame().await.unwrap().unwrap().payload[..], b"c"); - drop(reading); - drop(subscription); - source.unused().await.unwrap(); - drop(group); - drop(source); - - let track = resolved.track("log").unwrap(); - let subscribing = tokio::spawn(async move { track.subscribe(None).await }); - let mut next = replacement_dynamic - .requested_track() - .await - .unwrap() - .resolving_start() - .accept(None); - next.start_at(0).unwrap(); - let mut subscription = subscribing.await.unwrap().unwrap(); - let mut reading = subscription.recv_group().await.unwrap().unwrap(); - for expected in [b"a", b"b", b"c"] { - assert_eq!(&reading.read_frame().await.unwrap().unwrap().payload[..], expected); - } - } - /// The same holds for a reader returning to a parked track: its warm cache /// does not stand in for the copy it is waiting on. #[tokio::test] @@ -7426,9 +7227,10 @@ mod tests { drop(incumbent); drop(source); - let replacement = broadcast::Info::new().produce(); - queued(&echo).await.accept(&replacement); - resolved.assert_not_closed(); + // The front ends with its source rather than asking through the peer. + settle(|| peer.get_broadcast("room/alice").is_none()).await; + assert!(echo.poll_requested_broadcast(&kio::Waiter::noop()).is_pending()); + drop(resolved); view.assert_next_ended("room"); } @@ -8184,33 +7986,41 @@ mod tests { } } - /// Accept the front's re-request on `server` with a source carrying the same - /// content stream (the delivered group plus its successor) and prove the - /// rig's subscription resumes onto it: the successor group is delivered on - /// the same subscription, at the group boundary. - async fn assert_resumes(rig: &mut ResumeRig, server: &Dynamic) { + /// The front ends, never splicing in `server`'s copy, and a new request gets a + /// fresh front through `server`. + async fn assert_ends_then_serves(rig: &mut ResumeRig, server: &Dynamic) { + tokio::time::timeout(Duration::from_secs(10), rig.resolved.closed()) + .await + .expect("the front ends"); + let consumer = rig.producer.consume(); + + let pending = consumer.request_broadcast("room/alice"); let request = queued(server).await; let replacement = broadcast::Info::new().produce(); let track = replacement.create_track("video", None).unwrap(); - // The same content: group 0 was already delivered through the old route, - // so the splice resumes at group 1. let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"before".as_ref()).unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"new".as_ref()).unwrap(); group.finish().unwrap(); request.accept(&replacement); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"resumed".as_ref()).unwrap(); - group.finish().unwrap(); - - let mut group = rig - .subscription - .recv_group() + let resolved = pending.await.expect("resolves through the new route"); + assert!(!resolved.is_clone(&rig.resolved), "joined the ended front"); + let mut fresh = resolved + .track("video") + .unwrap() + .subscribe(None) + .await + .expect("subscribe"); + let mut group = next_group(&mut fresh) .await - .expect("subscription survives the failover") + .expect("recv group") .expect("track ended early"); - let frame = group.read_frame().await.expect("read frame").expect("frame"); - assert_eq!(&frame.payload[..], b"resumed"); + assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"new"); + + // The old subscription never gets the new route's groups. + rig.incumbent_track.finish().unwrap(); + let end = next_group(&mut rig.subscription).await; + assert!(!matches!(end, Ok(Some(_))), "the new route was spliced in"); } /// The driver's completion contract: it resolves once every producer handle @@ -8227,96 +8037,154 @@ mod tests { drop(consumer); } + /// A publisher back behind the same first hop may be a new broadcast that + /// restarts its group numbers, as a re-created one does: it is never stitched + /// onto the old one, so a later request gets the new broadcast from its start. #[tokio::test] - async fn remote_source_resumes_through_same_first_hop() { - let (mut rig, incumbent, source) = ResumeRig::new(&[10]).await; - let standby_server = rig.standby(&[10, 20]); - - // The serving route dies: retraction plus source abort, like a session. + async fn a_returning_publisher_is_never_stitched() { + let (rig, incumbent, source) = ResumeRig::new(&[10]).await; + let ResumeRig { + producer, + mut subscription, + incumbent_track, + .. + } = rig; + // The old broadcast reached group 3. + for _ in 1..=3 { + let mut group = incumbent_track.append_group().unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"old".as_ref()).unwrap(); + group.finish().unwrap(); + subscription.recv_group().await.unwrap().unwrap(); + } + // The publisher is back before the old front noticed it left. + let server = producer + .dynamic("room", Route::default().with_hops(hops(&[10]))) + .unwrap(); drop(incumbent); drop(source); + drop(incumbent_track); - // The standby shares the first hop, so the subscription resumes there. - assert_resumes(&mut rig, &standby_server).await; + // The old broadcast's readers end with it rather than wait on another source. + let ended = tokio::time::timeout(Duration::from_secs(10), subscription.recv_group()) + .await + .expect("the old subscription waits on the returning publisher"); + assert!(ended.is_err(), "the old subscription ends"); + + let consumer = producer.consume(); + let pending = consumer.request_broadcast("room/alice"); + let request = queued(&server).await; + let replacement = broadcast::Info::new().produce(); + let track = replacement.create_track("video", None).unwrap(); + let mut group = track.append_group().unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"new".as_ref()).unwrap(); + group.finish().unwrap(); + request.accept(&replacement); + + let resolved = pending.await.expect("resolves through the returning publisher"); + let mut fresh = resolved + .track("video") + .unwrap() + .subscribe(None) + .await + .expect("subscribe"); + let mut group = fresh + .recv_group() + .await + .expect("recv group") + .expect("track ended early"); + let frame = group.read_frame().await.expect("read frame").expect("frame"); + assert_eq!(&frame.payload[..], b"new"); } - /// A source claiming the same content cannot change immutable track metadata: - /// the successor is refused instead of the subscriber's samples being read on - /// a different grid, and the verdict outlives the aborted logical track. + /// A front that ends while a route still serves its path re-announces the path, + /// a retraction then the route again, so a consumer holding the ended broadcast + /// (a browser, a downstream relay) requests a fresh one. Whether the source + /// closed under a route that stays or another route won, the table alone shows + /// the announce cursors no change. #[tokio::test] - async fn incompatible_successor_is_refused() { - for replacement in [ - track::Info::default().with_timescale(crate::Timescale::MICRO), - track::Info::default().with_priority(7), - track::Info::default().with_max_age(Duration::from_secs(7)), - ] { - let (mut rig, incumbent, source) = ResumeRig::new(&[10]).await; - let standby_server = rig.standby(&[10, 20]); - drop(incumbent); - drop(source); + async fn an_ended_front_bounces_its_announce() { + for source_closes in [true, false] { + let producer = origin(1).produce(); + let consumer = producer.consume(); + let mut view = consumer.announced(); + let incumbent = producer + .dynamic("room/alice", Route::default().with_hops(hops(&[10])).with_cost(5)) + .unwrap(); + view.assert_next_active("room/alice"); - // The standby shares the first hop, so the front re-requests through it, - // but its copy of the track is on another grid. - let request = queued(&standby_server).await; - let successor = broadcast::Info::new().produce(); - let track = successor.create_track("video", replacement).unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"before".as_ref()).unwrap(); - group.finish().unwrap(); - request.accept(&successor); + let pending = consumer.request_broadcast("room/alice"); + let source = broadcast::Info::new().produce(); + queued(&incumbent).await.accept(&source); + let resolved = pending.await.expect("resolves"); - assert!( - matches!(rig.subscription.recv_group().await, Err(Error::Unsupported)), - "the subscription must abort rather than resume onto incompatible metadata" - ); + // The same publisher is back on an identical route, which the cursor never + // sees, before the old one leaves; or a cheaper route from it wins. + let cost = if source_closes { 5 } else { 0 }; + let other = producer + .dynamic("room/alice", Route::default().with_hops(hops(&[10])).with_cost(cost)) + .unwrap(); + let _incumbent = if source_closes { + drop(incumbent); + drop(source); + None + } else { + // The cheaper route's metadata waits out the update hold, and the bounce + // below takes its place. + Some((incumbent, source)) + }; - // Reopening the aborted logical track must not forget the broadcast's metadata. - let reopened = rig.resolved.track("video").unwrap(); - assert!(matches!(reopened.query().await, Err(Error::Unsupported))); - assert!(matches!(reopened.subscribe(None).await, Err(Error::Unsupported))); + tokio::time::timeout(Duration::from_secs(10), resolved.closed()) + .await + .expect("the front ends"); + view.assert_next_ended("room/alice"); + view.assert_next_active("room/alice"); + view.assert_next_wait(); + drop(other); } } - /// The serving route is withdrawn while its copy of the track is still open, and the - /// next route for the same publisher no longer has the broadcast: its refusal must not - /// end the track, which takes the incumbent's remaining groups and clean end. + /// A route renamed in place to another publisher leaves the front's pin with + /// nothing to select, so the front ends while the path stays announced, and it + /// bounces like any other end. #[tokio::test] - async fn a_refused_replacement_keeps_a_withdrawn_incumbent_open() { - let (mut rig, incumbent, source) = ResumeRig::new(&[9, 2]).await; - let standby = rig.standby(&[9, 5, 3]); - - // The peer withdraws the broadcast: the route retracts and its source closes, - // while the subscription already in flight stays open. - incumbent.withdrawn(); - source.close(); - - let request = queued(&standby).await; - let successor = broadcast::Info::new().produce(); - let mut handler = successor.dynamic(); - request.accept(&successor); - let track = tokio::time::timeout(Duration::from_secs(1), handler.requested_track()) - .await - .expect("the front asked the standby") - .expect("request"); - track.reject(Error::Unroutable); - for _ in 0..10 { - tokio::task::yield_now().await; - } + async fn a_route_renamed_in_place_bounces_its_announce() { + let producer = origin(1).produce(); + let consumer = producer.consume(); + let mut view = consumer.announced(); + let server = producer + .dynamic("room/alice", Route::default().with_hops(hops(&[10]))) + .unwrap(); + view.assert_next_active("room/alice"); + let pending = consumer.request_broadcast("room/alice"); + // Held open, so the front ends for its route, not for a closed source. + let source = broadcast::Info::new().produce(); + queued(&server).await.accept(&source); + let resolved = pending.await.expect("resolves"); - let mut group = rig.incumbent_track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"last".as_ref()).unwrap(); - group.finish().unwrap(); - let mut group = next_group(&mut rig.subscription) + server.update(Route::default().with_hops(hops(&[11]))).unwrap(); + tokio::time::timeout(Duration::from_secs(10), resolved.closed()) .await - .expect("the incumbent keeps serving") - .expect("track open"); - assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"last"); + .expect("the front ends"); + view.assert_next_ended("room/alice"); + view.assert_next_active("room/alice"); + view.assert_next_wait(); + drop(source); + } - rig.incumbent_track.finish().unwrap(); - let end = next_group(&mut rig.subscription) + /// A front at a path served only through a covering prefix bounces nothing: + /// re-announcing the prefix would restart its other broadcasts too. + #[tokio::test] + async fn a_covering_route_is_never_bounced() { + let (rig, incumbent, source) = ResumeRig::new(&[10]).await; + let mut view = rig.producer.consume().announced(); + view.assert_next_active("room"); + let _returned = rig.standby(&[10]); + drop(incumbent); + drop(source); + tokio::time::timeout(Duration::from_secs(10), rig.resolved.closed()) .await - .map(|group| group.map(|group| group.sequence)); - assert!(matches!(end, Ok(None)), "the incumbent's clean end, not {end:?}"); + .expect("the front ends"); + view.assert_next_wait(); } #[tokio::test] @@ -8408,20 +8276,17 @@ mod tests { } } - /// A named front whose route moves to a new publisher resumes through another route - /// from its own publisher, never through the updated one. + /// A named front whose route moves to a new publisher ends rather than splicing + /// in another route from its own publisher, and a new request gets a fresh front + /// through the new publisher, now the shortest route. #[tokio::test] - async fn a_first_hop_update_resumes_through_the_same_publisher() { + async fn a_first_hop_update_ends_the_front() { let (mut rig, incumbent, _source) = ResumeRig::new(&[10]).await; - let standby_server = rig.standby(&[10, 20]); + let _standby = rig.standby(&[10, 20]); incumbent.update(Route::default().with_hops(hops(&[11]))).unwrap(); - assert_resumes(&mut rig, &standby_server).await; - assert!( - incumbent.poll_requested_broadcast(&kio::Waiter::noop()).is_pending(), - "the front never asks the new publisher" - ); + assert_ends_then_serves(&mut rig, &incumbent).await; } #[tokio::test] @@ -8572,39 +8437,40 @@ mod tests { assert_eq!(&frame.payload[..], b"steady"); } + /// The serving route drains, repriced to the ceiling while its session keeps + /// serving: the front ends rather than stitching the standby's copy in, and a + /// new request gets a fresh front through the standby. #[tokio::test] - async fn drain_reprice_migrates_before_the_session_dies() { - let (mut rig, incumbent, source) = ResumeRig::new(&[10]).await; + async fn drain_reprice_ends_the_front() { + let (mut rig, incumbent, _source) = ResumeRig::new(&[10]).await; let standby_server = rig.standby(&[10, 20]); - // The serving route drains: repriced to the ceiling while its session - // keeps serving. The front migrates to the standby without waiting for - // the death. incumbent .update(Route::default().with_hops(hops(&[10])).with_cost(Cost::DRAIN)) .unwrap(); - assert_resumes(&mut rig, &standby_server).await; - - // The drained source outlived the migration. - drop(incumbent); - drop(source); + assert_ends_then_serves(&mut rig, &standby_server).await; } #[tokio::test] - async fn local_sources_splice_newest_first() { + async fn a_second_local_source_gets_a_fresh_front() { let producer = origin(1).produce(); let consumer = producer.consume(); let first = producer.publish("room/alice", Route::default()).unwrap(); let resolved = consumer.request_broadcast("room/alice").await.expect("resolves"); - // A second source at the same path joins the same front. + // A second source at the same path is a new broadcast: a request right + // away, before the driver has ended the first's front, gets a fresh one + // rather than joining the front about to end. let second = producer.publish("room/alice", Route::default()).unwrap(); let again = consumer.request_broadcast("room/alice").await.expect("resolves"); - assert!(again.is_clone(&resolved)); + assert!(!again.is_clone(&resolved)); + tokio::time::timeout(Duration::from_secs(10), resolved.closed()) + .await + .expect("the first front ends"); - // Losing one source keeps the front alive; losing both closes it. + // The first source leaving leaves the second's front; losing both closes it. first.close(); settle(|| consumer.get_broadcast("room/alice").is_some()).await; second.close(); @@ -8970,11 +8836,10 @@ mod tests { assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"fetched"); } - /// A newer local source wins dispatch the moment it attaches, but one whose copy - /// of the track carries different metadata is refused: the incumbent keeps - /// serving, and the refusal is never retried once the incumbent leaves. + /// A newer local source is a new broadcast, whatever its track metadata: the + /// incumbent's front ends rather than splicing it in, and a new request gets it. #[tokio::test] - async fn incompatible_local_source_keeps_the_incumbent() { + async fn a_local_newcomer_ends_the_incumbent_front() { let producer = origin(1).produce(); let consumer = producer.consume(); @@ -8993,26 +8858,33 @@ mod tests { let mut group = subscription.recv_group().await.unwrap().unwrap(); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"before"); - // The newest source is dispatched the track, and refused for its metadata. let second = producer.publish("room/alice", Route::default()).unwrap(); - let _incompatible = second + let newcomer = second .create_track("video", track::Info::default().with_timescale(crate::Timescale::MICRO)) .unwrap(); - for _ in 0..10 { - tokio::task::yield_now().await; - } + tokio::time::timeout(Duration::from_secs(10), resolved.closed()) + .await + .expect("the incumbent's front ends"); - // Still spliced to the incumbent, still delivering. - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"still".as_ref()).unwrap(); + let fresh = consumer.request_broadcast("room/alice").await.expect("resolves"); + assert!(!fresh.is_clone(&resolved)); + let mut fresh_subscription = fresh + .track("video") + .unwrap() + .subscribe(None) + .await + .expect("the newcomer's metadata applies"); + let mut group = newcomer.append_group().unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"new".as_ref()).unwrap(); group.finish().unwrap(); - let mut group = subscription.recv_group().await.unwrap().unwrap(); - assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"still"); + let mut group = fresh_subscription.recv_group().await.unwrap().unwrap(); + assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"new"); - // The incumbent leaving exhausts the table: the refusal is never retried. - drop(track); - first.close(); - assert!(matches!(subscription.recv_group().await, Err(Error::Unsupported))); + // The incumbent's reader never gets the newcomer's groups. + track.finish().unwrap(); + let end = next_group(&mut subscription).await; + assert!(!matches!(end, Ok(Some(_))), "the newcomer was spliced in"); + drop(first); } #[tokio::test] diff --git a/rs/moq-net/tests/rejoin.rs b/rs/moq-net/tests/rejoin.rs index 77b2c58bc13..d7536331bdc 100644 --- a/rs/moq-net/tests/rejoin.rs +++ b/rs/moq-net/tests/rejoin.rs @@ -207,9 +207,10 @@ async fn rejoined_quiet_catalog_survives_cache_expiry() { /// A quiet track's latest group outlives the cache window across a publisher takeover. /// -/// The publisher reconnects while a reader holds the catalog: the relay retires the old -/// upstream copy and resumes from the new one past the cached group, which must stay -/// protected as the live edge. +/// The publisher reconnects while a reader holds the catalog. The relay never stitches +/// the new connection's copy into the old front: the holder's subscription ends, and a +/// later reader gets a fresh front whose latest catalog must stay protected as the +/// live edge. #[tokio::test(start_paused = true)] async fn taken_over_quiet_catalog_survives_cache_expiry() { for version in ["moq-lite-05", "moq-lite-06"] { @@ -276,7 +277,7 @@ async fn quiet_catalog_survives_cache_expiry(version: Version, takeover: bool) { }; tokio::time::sleep(moq_net::cache::DEFAULT_EXPIRY * 2 + Duration::from_secs(2)).await; - let (mut later, _later_remote, _later_pair) = session(5).await; + let (mut later, later_remote, _later_pair) = session(5).await; let mut group = tokio::time::timeout(Duration::from_secs(1), later.recv_group()) .await .unwrap_or_else(|_| panic!("{version} takeover={takeover}: late reader lost the latest catalog")) @@ -285,6 +286,19 @@ async fn quiet_catalog_survives_cache_expiry(version: Version, takeover: bool) { assert_eq!(read_all(&mut group).await.unwrap(), [b"catalog"], "{version}"); drop(group); + // After a takeover the holder read the old front, which ended; the later reader + // holds the fresh one. + let (mut holder, holder_remote) = if takeover { + let end = tokio::time::timeout(Duration::from_secs(1), holder.recv_group()).await; + assert!( + !matches!(end, Ok(Ok(Some(_)))), + "{version}: the takeover was stitched into the old front" + ); + (later, later_remote) + } else { + (holder, holder_remote) + }; + // A newer catalog supersedes it, so the old one expires like any other group. let mut newer = track.append_group().unwrap(); newer diff --git a/rs/moq-relay/tests/goaway_cluster.rs b/rs/moq-relay/tests/goaway_cluster.rs index 3e72ecf9050..7a0bc5f3f2b 100644 --- a/rs/moq-relay/tests/goaway_cluster.rs +++ b/rs/moq-relay/tests/goaway_cluster.rs @@ -250,9 +250,9 @@ async fn spawn_upstream( } /// An upstream GOAWAY with a redirect migrates the cluster dial to the sibling. -/// The draining session keeps serving through the handover window, and the path -/// never retracts on the cluster origin (route re-pricing and the sibling's -/// announce surface as metadata updates, not churn). +/// The draining session keeps serving through the handover window. The front +/// never stitches the sibling's copy in: it ends and bounces the path (a +/// retraction, then the route again), which ends announced. async fn cluster_migrates_on_upstream_goaway_inner() { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); @@ -311,9 +311,7 @@ async fn cluster_migrates_on_upstream_goaway_inner() { b"g0"[..] ); - // Watch for retraction during the swap: the migration must never - // unannounce the path (metadata updates are expected: the drain - // re-prices the old route and the sibling announces its own). + // Watch the path through the swap: it may bounce, but ends announced. let mut announcements = cluster.origin.consume().announced(); let first = announcements.next().await.expect("initial announce"); assert_eq!(first.prefix.as_str(), "cam"); @@ -349,16 +347,17 @@ async fn cluster_migrates_on_upstream_goaway_inner() { // handover window at the latest). session_a.closed().await; - // No retraction leaked to the origin during the whole swap. Metadata - // updates (the drain re-pricing, the sibling's route) are expected and - // harmless; an inactive event means the path flapped. + // The front ended rather than stitch the sibling's copy in, so the path may + // have bounced, but it never stays retracted. + let mut active = true; loop { match tokio::time::timeout(Duration::from_millis(500), announcements.next()).await { Err(_) => break, - Ok(Some(update)) if update.kind.is_active() => continue, - Ok(event) => panic!("migration must not retract the path on the cluster origin: {event:?}"), + Ok(Some(update)) => active = update.kind.is_active(), + Ok(None) => panic!("the cluster origin's announcements ended"), } } + assert!(active, "the migration left the path retracted on the cluster origin"); cluster_run.abort(); }) @@ -442,11 +441,11 @@ async fn spawn_relay_with_upstream( /// gated: MID-B's first inbound connection can only be that reconnect). /// 3. The draining MID-A leg keeps serving through the handover window, so /// every group published across the swap arrives exactly once. -/// 4. Once the old leg closes, the SAME subscription keeps flowing: both legs' -/// routes share TOP as their first hop, so BOTTOM's front re-splices onto -/// the MID-B leg at a group boundary instead of ending the subscription. -/// 5. No GOAWAY leaks to the subscriber's own session, and the path never -/// retracts under the subscriber. +/// 4. Once the old leg closes, BOTTOM's front ends rather than stitch the MID-B +/// leg's copy in, even though both legs name TOP as the publisher: the +/// subscription ends, and a fresh request reads the MID-B leg. +/// 5. No GOAWAY leaks to the subscriber's own session, and the path ends +/// announced under the subscriber (it may bounce). async fn cluster_diamond_goaway_seamless_failover_inner() { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); @@ -523,7 +522,7 @@ async fn cluster_diamond_goaway_seamless_failover_inner() { .expect("origin closed before the announce"); assert_eq!(first.prefix.as_str(), "diamond"); - let bc = within("broadcast resolves on the subscriber origin", async { + let bc_old = within("broadcast resolves on the subscriber origin", async { let consumer = sub_origin.consume(); consumer.routed("diamond").await?; consumer.request_broadcast("diamond").await.ok() @@ -541,7 +540,8 @@ async fn cluster_diamond_goaway_seamless_failover_inner() { // must preserve is that every group arrives, exactly once, with its frames intact. let mut sub = within( "subscribe to the video track", - bc.track("video") + bc_old + .track("video") .expect("track handle") .subscribe(moq_net::track::Subscription::default().with_max_age(Duration::from_secs(60))), ) @@ -617,10 +617,9 @@ async fn cluster_diamond_goaway_seamless_failover_inner() { // BOTTOM, positively proving the new leg carries the resubscribe. drop(mid_a_upstream); - // The subscription was served through the MID-A leg, but its close does not - // end it: both legs' routes name TOP as their first hop, so BOTTOM's front - // re-splices onto the MID-B leg at a group boundary and the subscription - // rides through. Keep publishing and keep reading the SAME subscription. + // The subscription was served through the MID-A leg, and its close ends it: + // BOTTOM never stitches the MID-B leg's copy in. Keep publishing, and read + // through a fresh request, as the bounce tells a subscriber to. const POST_DRAIN_LAST: u64 = LAST_GROUP + 40; let post_publisher = tokio::spawn(async move { for seq in (LAST_GROUP + 1)..=POST_DRAIN_LAST { @@ -635,19 +634,41 @@ async fn cluster_diamond_goaway_seamless_failover_inner() { } }); - // Three distinct post-drain groups on the same subscription, every frame - // verified, proves the new leg carries it: MID-A is severed, so they can - // only have flowed TOP -> MID-B -> BOTTOM. + // The old subscription ends without a post-drain group. + while let Ok(Some(group)) = within("the old subscription ends", sub.recv_group()).await { + assert!( + group.sequence <= LAST_GROUP, + "group {} reached the old subscription after the drain", + group.sequence + ); + } + let bc = within("a fresh request resolves through the MID-B leg", async { + let consumer = sub_origin.consume(); + loop { + if let Ok(bc) = consumer.request_broadcast("diamond").await + && !bc.is_clone(&bc_old) + { + return bc; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await; + let mut sub = within( + "subscribe through the MID-B leg", + bc.track("video").expect("track handle").subscribe(None), + ) + .await + .expect("subscribe"); + + // Three distinct post-drain groups, every frame verified, proves the new leg + // carries it: MID-A is severed, so they can only have flowed TOP -> MID-B -> + // BOTTOM. let mut post = BTreeSet::new(); while post.iter().filter(|seq| **seq > LAST_GROUP).count() < 3 { collect_group(&mut sub, &mut post, FRAMES_PER_GROUP, "post-drain (MID-B leg only)").await; } post_publisher.abort(); - // The splice must not re-deliver anything the old leg already served. - assert!( - post.iter().all(|seq| *seq > LAST_GROUP), - "the splice re-delivered pre-failover groups: {post:?}" - ); // ── no GOAWAY cascade to the downstream subscriber ─────────────────── assert!( @@ -666,15 +687,16 @@ async fn cluster_diamond_goaway_seamless_failover_inner() { "downstream subscriber received a GOAWAY (the relay should absorb it): {leaked:?}" ); - // ── announcement stability: the path never retracted under the swap ── - // Metadata updates (route re-pricing, the new leg's hops) are expected. + // ── announcement: the path may bounce under the swap, but ends announced ── + let mut active = true; loop { match tokio::time::timeout(Duration::from_millis(500), announcements.next()).await { Err(_) => break, - Ok(Some(update)) if update.kind.is_active() => continue, - Ok(event) => panic!("failover must not retract the path under the subscriber: {event:?}"), + Ok(Some(update)) => active = update.kind.is_active(), + Ok(None) => panic!("the subscriber's announcements ended"), } } + assert!(active, "the failover left the path retracted under the subscriber"); } /// Receive the next group, assert every frame's payload and the exact frame count diff --git a/rs/moq-tokio/tests/broadcast.rs b/rs/moq-tokio/tests/broadcast.rs index c1c0b410e3f..95e50a849e0 100644 --- a/rs/moq-tokio/tests/broadcast.rs +++ b/rs/moq-tokio/tests/broadcast.rs @@ -979,13 +979,15 @@ async fn read_payloads(sub: &mut moq_net::track::Subscriber, count: usize) -> Ve payloads } -/// A path stays routed across two publisher sessions announcing it. +/// A path stays routed across two publisher sessions announcing it, and a +/// subscription never rides from one to the other. /// /// The client connects to two servers announcing the same route. The preferred -/// (cheaper) route serves the track; when that session dies, the broadcast -/// re-splices through the standby at a group boundary, without the path ever -/// being retracted: both routes name the same first hop, so they are the same -/// origin reached different ways and the subscription rides the failover. +/// (cheaper) route serves the track. When that session dies, the broadcast ends +/// rather than splice the standby's copy in, even though both routes name the +/// same first hop: the publisher may have re-created the broadcast with its +/// group numbers starting over. The path bounces, and a fresh request reads the +/// standby. #[tracing_test::traced_test] #[tokio::test] async fn broadcast_route_migration() { @@ -1110,13 +1112,41 @@ async fn broadcast_route_migration() { .expect("subscribe failed"); assert_eq!(read_payloads(&mut sub, 1).await, ["a1"]); - // Kill the serving session. Both routes name the same first hop, so the - // broadcast re-splices through the standby at the group boundary and the - // SAME subscription carries B's continuation. + // Kill the serving session. The old subscription ends without any of B's + // groups, and a fresh request resolves through the standby. drop(session_a); + let end = tokio::time::timeout(TIMEOUT, sub.recv_group()) + .await + .expect("the old subscription ends"); + assert!( + !matches!(end, Ok(Some(_))), + "the standby was spliced into the old subscription" + ); + let fresh = tokio::time::timeout(TIMEOUT, async { + loop { + if let Ok(fresh) = sub_consumer.request_broadcast("test").await + && !fresh.is_clone(&broadcast) + { + return fresh; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("a fresh request resolves through the standby"); + let mut sub = fresh + .track("video") + .unwrap() + .subscribe( + moq_net::track::Subscription::default() + .with_start(moq_net::track::Position::group(2)) + .with_max_age(Duration::from_secs(10)), + ) + .await + .expect("subscribe failed"); assert_eq!(read_payloads(&mut sub, 2).await, ["b2", "b3"]); - // The new subscription is live: what B produces from here on flows through it. + // The fresh subscription is live: what B produces from here on flows through it. { let mut group = standby_track .create_group(moq_net::group::Info { sequence: 4 }) @@ -1126,11 +1156,12 @@ async fn broadcast_route_migration() { } assert_eq!(read_payloads(&mut sub, 1).await, ["b4"]); - // The path was never retracted across the swap: any events delivered are - // active metadata updates (the standby's chain taking over). + // The path may have bounced across the swap, but it ends announced. + let mut active = true; while let Some(update) = announcements.try_next() { - assert!(update.kind.is_active(), "failover must not retract the route"); + active = update.kind.is_active(); } + assert!(active, "the failover left the route retracted"); handle_a.await.expect("server a panicked").expect("server a failed"); drop(_session_b);