Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 39 additions & 3 deletions rs/moq-net/src/model/origin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -519,8 +519,8 @@ pub struct Route {

/// What pulling content via this route costs, accumulated per link: lower wins
/// among routes of the same anonymity, with ties broken by a broadcast published
/// on this origin, then hop length, then a deterministic hash, and finally the
/// most recently announced route. See [`Cost`].
/// on this origin, then hop length, then a deterministic hash of a route a cluster
/// peer forwarded, and finally the most recently announced route. See [`Cost`].
pub cost: Cost,

/// The announcing session's declared or assigned identity.
Expand Down Expand Up @@ -648,13 +648,21 @@ fn fnv_key(name: &str, origins: impl IntoIterator<Item = Hop>) -> u64 {
/// of the prefix and chain so every node converges on the same winner, and finally
/// the newest announcement, so a reconnect under an otherwise identical route wins
/// the moment it lands instead of after the transport retires the old session.
///
/// Only a route a cluster peer forwarded is hashed. One that entered here is ranked
/// by no other node, and a restarted publisher announces under a new first hop (its
/// own random id, or this relay's stamp), so a hash would keep its dead session
/// winning half the time until the transport times out.
fn route_order(prefix: &Path, entry: &RouteEntry) -> (bool, Cost, bool, usize, u64, Reverse<u64>) {
(
entry.is_anonymous(),
entry.cost,
!entry.local,
entry.hops.len(),
fnv_key(prefix.as_str(), entry.hops.iter().copied()),
match entry.peer {
true => fnv_key(prefix.as_str(), entry.hops.iter().copied()),
false => 0,
},
Reverse(entry.id),
)
}
Expand Down Expand Up @@ -7353,6 +7361,34 @@ mod tests {
}
}

/// A publisher that reconnects before its old session times out announces under a
/// new first hop: a random id when it names itself, or this relay's stamp when it
/// does not. The new session wins whichever chain hashes lower.
#[tokio::test(start_paused = true)]
async fn a_reconnect_under_a_new_hop_wins_over_the_old_session() {
for (old, new) in [
(&[10][..], &[11][..]),
(&[11], &[10]),
(&[10, 0], &[11, 0]),
(&[11, 0], &[10, 0]),
] {
let producer = origin(1).produce();
let mut announced = producer.consume().announced();
tokio::time::sleep(Duration::from_millis(1)).await;

let _old = producer.dynamic("room", Route::default().with_hops(hops(old))).unwrap();
assert_eq!(announced.assert_next_active("room").hops, hops(old));

let _new = producer.dynamic("room", Route::default().with_hops(hops(new))).unwrap();
let update = tokio::time::timeout(DEFAULT_UPDATE_HOLD * 2, announced.next())
.await
.unwrap_or_else(|_| panic!("{new:?} never took over from {old:?}"))
.unwrap();
assert_eq!(update.kind, AnnounceKind::Updated);
assert_eq!(update.route.hops, hops(new));
}
}

#[tokio::test]
async fn withdrawn_route_no_longer_covers_requests() {
let producer = origin(1).produce();
Expand Down
76 changes: 76 additions & 0 deletions rs/moq-net/tests/restart_lingering.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
//! A publisher restarting while its old session lingers on the relay.
//!
//! A publisher that dies without a close (SIGKILL, a dropped uplink) leaves its session on
//! the relay until the transport idles it out. The restarted process announces the same
//! path under a new first hop, so the two routes tie on everything but the hash.

mod support;

use std::time::Duration;

use moq_net::{Hop, Timestamp, Version, origin};
use support::harness::{MockConnectOptions, MockPair, connect_mock};

const TEST_TIMEOUT: Duration = Duration::from_secs(10);

fn produce_origin(hop: u64) -> origin::Producer {
let (producer, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap()));
tokio::spawn(support::harness::run(driver));
producer
}

/// Connect `publisher` to `relay` as a client publishing into it.
async fn link(version: Version, publisher: &origin::Producer, relay: &origin::Producer) -> MockPair {
let mut options = MockConnectOptions::new(version);
options.client_publish = Some(publisher.consume());
options.server_subscribe = Some(relay.clone());
connect_mock(options).await
}

/// The restart wins at once, whichever chain hashes lower, so a viewer's first request
/// reaches it instead of waiting on the dead session.
async fn restart_while_the_old_session_lingers(version: Version, old_hop: u64, new_hop: u64) {
let relay = produce_origin(3);
let mut announced = relay.consume().announced();

let old = produce_origin(old_hop);
let old_cam = old.create_broadcast("cam").unwrap();
let _old_video = old_cam.create_track("video", None).unwrap();
old_cam.announce(Default::default()).unwrap();
let old_session = link(version, &old, &relay).await;
announced.next().await.expect("the old route announces");

// The old process dies without a close: its session falls silent.
old_session.client_transport.set_latency(Duration::from_secs(3600));
old_session.server_transport.set_latency(Duration::from_secs(3600));

let new = produce_origin(new_hop);
let new_cam = new.create_broadcast("cam").unwrap();
let new_video = new_cam.create_track("video", None).unwrap();
new_cam.announce(Default::default()).unwrap();
let _new_session = link(version, &new, &relay).await;
tokio::time::sleep(origin::DEFAULT_UPDATE_HOLD * 2).await;

let cam = relay.consume().request_broadcast("cam").await.unwrap();
let mut video = cam.track("video").unwrap().subscribe(None).await.unwrap();
let mut group = new_video.append_group().unwrap();
group.write_frame(Timestamp::ZERO, b"new:0".as_ref()).unwrap();
let mut group = video.recv_group().await.unwrap().expect("a group");
let frame = group.read_frame().await.unwrap().expect("a frame");
assert_eq!(frame.payload.as_ref(), b"new:0", "{version} ({old_hop} -> {new_hop})");
}

#[tokio::test(start_paused = true)]
async fn restart_while_the_old_session_lingers_wins() {
for version in ["moq-lite-06", "moq-lite-07-wip", "moq-transport-22"] {
for (old_hop, new_hop) in [(1, 4), (4, 1)] {
let version: Version = version.parse().unwrap();
tokio::time::timeout(
TEST_TIMEOUT,
restart_while_the_old_session_lingers(version, old_hop, new_hop),
)
.await
.unwrap_or_else(|_| panic!("{version} ({old_hop} -> {new_hop}): subscribe hung"));
}
}
}
Loading