diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index b3b139ed31e..671c3c9dc9f 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -519,8 +519,8 @@ pub struct Route { /// What pulling content via this route costs, accumulated per link: lower wins /// among routes of the same anonymity, with ties broken by a broadcast published - /// on this origin, then hop length, then a deterministic hash, and finally the - /// most recently announced route. See [`Cost`]. + /// on this origin, then hop length, then a deterministic hash of a route a cluster + /// peer forwarded, and finally the most recently announced route. See [`Cost`]. pub cost: Cost, /// The announcing session's declared or assigned identity. @@ -648,13 +648,21 @@ fn fnv_key(name: &str, origins: impl IntoIterator) -> u64 { /// of the prefix and chain so every node converges on the same winner, and finally /// the newest announcement, so a reconnect under an otherwise identical route wins /// the moment it lands instead of after the transport retires the old session. +/// +/// Only a route a cluster peer forwarded is hashed. One that entered here is ranked +/// by no other node, and a restarted publisher announces under a new first hop (its +/// own random id, or this relay's stamp), so a hash would keep its dead session +/// winning half the time until the transport times out. fn route_order(prefix: &Path, entry: &RouteEntry) -> (bool, Cost, bool, usize, u64, Reverse) { ( entry.is_anonymous(), entry.cost, !entry.local, entry.hops.len(), - fnv_key(prefix.as_str(), entry.hops.iter().copied()), + match entry.peer { + true => fnv_key(prefix.as_str(), entry.hops.iter().copied()), + false => 0, + }, Reverse(entry.id), ) } @@ -7353,6 +7361,34 @@ mod tests { } } + /// A publisher that reconnects before its old session times out announces under a + /// new first hop: a random id when it names itself, or this relay's stamp when it + /// does not. The new session wins whichever chain hashes lower. + #[tokio::test(start_paused = true)] + async fn a_reconnect_under_a_new_hop_wins_over_the_old_session() { + for (old, new) in [ + (&[10][..], &[11][..]), + (&[11], &[10]), + (&[10, 0], &[11, 0]), + (&[11, 0], &[10, 0]), + ] { + let producer = origin(1).produce(); + let mut announced = producer.consume().announced(); + tokio::time::sleep(Duration::from_millis(1)).await; + + let _old = producer.dynamic("room", Route::default().with_hops(hops(old))).unwrap(); + assert_eq!(announced.assert_next_active("room").hops, hops(old)); + + let _new = producer.dynamic("room", Route::default().with_hops(hops(new))).unwrap(); + let update = tokio::time::timeout(DEFAULT_UPDATE_HOLD * 2, announced.next()) + .await + .unwrap_or_else(|_| panic!("{new:?} never took over from {old:?}")) + .unwrap(); + assert_eq!(update.kind, AnnounceKind::Updated); + assert_eq!(update.route.hops, hops(new)); + } + } + #[tokio::test] async fn withdrawn_route_no_longer_covers_requests() { let producer = origin(1).produce(); diff --git a/rs/moq-net/tests/restart_lingering.rs b/rs/moq-net/tests/restart_lingering.rs new file mode 100644 index 00000000000..55463ec6c81 --- /dev/null +++ b/rs/moq-net/tests/restart_lingering.rs @@ -0,0 +1,76 @@ +//! A publisher restarting while its old session lingers on the relay. +//! +//! A publisher that dies without a close (SIGKILL, a dropped uplink) leaves its session on +//! the relay until the transport idles it out. The restarted process announces the same +//! path under a new first hop, so the two routes tie on everything but the hash. + +mod support; + +use std::time::Duration; + +use moq_net::{Hop, Timestamp, Version, origin}; +use support::harness::{MockConnectOptions, MockPair, connect_mock}; + +const TEST_TIMEOUT: Duration = Duration::from_secs(10); + +fn produce_origin(hop: u64) -> origin::Producer { + let (producer, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); + tokio::spawn(support::harness::run(driver)); + producer +} + +/// Connect `publisher` to `relay` as a client publishing into it. +async fn link(version: Version, publisher: &origin::Producer, relay: &origin::Producer) -> MockPair { + let mut options = MockConnectOptions::new(version); + options.client_publish = Some(publisher.consume()); + options.server_subscribe = Some(relay.clone()); + connect_mock(options).await +} + +/// The restart wins at once, whichever chain hashes lower, so a viewer's first request +/// reaches it instead of waiting on the dead session. +async fn restart_while_the_old_session_lingers(version: Version, old_hop: u64, new_hop: u64) { + let relay = produce_origin(3); + let mut announced = relay.consume().announced(); + + let old = produce_origin(old_hop); + let old_cam = old.create_broadcast("cam").unwrap(); + let _old_video = old_cam.create_track("video", None).unwrap(); + old_cam.announce(Default::default()).unwrap(); + let old_session = link(version, &old, &relay).await; + announced.next().await.expect("the old route announces"); + + // The old process dies without a close: its session falls silent. + old_session.client_transport.set_latency(Duration::from_secs(3600)); + old_session.server_transport.set_latency(Duration::from_secs(3600)); + + let new = produce_origin(new_hop); + let new_cam = new.create_broadcast("cam").unwrap(); + let new_video = new_cam.create_track("video", None).unwrap(); + new_cam.announce(Default::default()).unwrap(); + let _new_session = link(version, &new, &relay).await; + tokio::time::sleep(origin::DEFAULT_UPDATE_HOLD * 2).await; + + let cam = relay.consume().request_broadcast("cam").await.unwrap(); + let mut video = cam.track("video").unwrap().subscribe(None).await.unwrap(); + let mut group = new_video.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, b"new:0".as_ref()).unwrap(); + let mut group = video.recv_group().await.unwrap().expect("a group"); + let frame = group.read_frame().await.unwrap().expect("a frame"); + assert_eq!(frame.payload.as_ref(), b"new:0", "{version} ({old_hop} -> {new_hop})"); +} + +#[tokio::test(start_paused = true)] +async fn restart_while_the_old_session_lingers_wins() { + for version in ["moq-lite-06", "moq-lite-07-wip", "moq-transport-22"] { + for (old_hop, new_hop) in [(1, 4), (4, 1)] { + let version: Version = version.parse().unwrap(); + tokio::time::timeout( + TEST_TIMEOUT, + restart_while_the_old_session_lingers(version, old_hop, new_hop), + ) + .await + .unwrap_or_else(|_| panic!("{version} ({old_hop} -> {new_hop}): subscribe hung")); + } + } +}