From 4a9005c3c58ce72df21421df042ec8e660217cf1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 22:29:44 -0800 Subject: [PATCH] test(moq-net): a relay restarting under its old hop id strands nothing On the production graph, a relay with a fixed cluster id drains, its publisher moves to a sibling under the same origin hop, and the relay comes back under its old hop id, closing cleanly or with the close lost until the sessions idle out. Viewers watch from every relay throughout; afterwards every relay serves a new request, including a track no viewer holds open. The production graph moves to the shared test support. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/tests/mesh_withdraw.rs | 39 +--- rs/moq-net/tests/relay_restart.rs | 243 +++++++++++++++++++++++++ rs/moq-net/tests/support/mod.rs | 1 + rs/moq-net/tests/support/production.rs | 37 ++++ 4 files changed, 285 insertions(+), 35 deletions(-) create mode 100644 rs/moq-net/tests/relay_restart.rs create mode 100644 rs/moq-net/tests/support/production.rs diff --git a/rs/moq-net/tests/mesh_withdraw.rs b/rs/moq-net/tests/mesh_withdraw.rs index bf7f0338023..aff0b24d18a 100644 --- a/rs/moq-net/tests/mesh_withdraw.rs +++ b/rs/moq-net/tests/mesh_withdraw.rs @@ -12,7 +12,10 @@ use std::{collections::HashMap, time::Duration}; use tokio::sync::mpsc; use moq_net::{Hop, Hops, Version, announce, broadcast, origin}; -use support::harness::{MockPair, peer, peer_with_latency}; +use support::{ + harness::{MockPair, peer, peer_with_latency}, + production::{PRODUCTION, PRODUCTION_LATENCY_MS}, +}; fn produce_origin(hop: u64) -> origin::Producer { produce_capped(hop, origin::HopLimit::default()) @@ -151,40 +154,6 @@ async fn partial_mesh_withdraw_then_republish() { let _broadcasts = mesh.publish(100, 5).await; } -/// A 34-relay partial mesh: moq.pro's production graph on 2026-09-30, where -/// relays in neighboring PoPs peer and most reach a publisher's relay through -/// others. -#[rustfmt::skip] -const PRODUCTION: &[(usize, usize)] = &[ - (0, 20), (0, 22), (0, 29), (1, 3), (1, 12), (1, 14), (1, 21), (2, 20), (2, 26), (2, 32), (3, 8), - (3, 11), (3, 15), (3, 16), (3, 17), (3, 18), (3, 21), (3, 22), (3, 23), (3, 24), (3, 29), (3, 30), - (3, 31), (4, 7), (4, 20), (4, 22), (4, 25), (4, 29), (5, 20), (5, 27), (5, 33), (6, 23), (6, 24), - (6, 30), (6, 31), (7, 22), (7, 25), (7, 29), (8, 15), (8, 17), (8, 21), (8, 23), (8, 30), (9, 19), - (9, 22), (9, 23), (9, 28), (9, 29), (9, 30), (10, 19), (10, 20), (10, 22), (10, 23), (10, 26), - (10, 27), (10, 28), (10, 29), (10, 30), (10, 32), (10, 33), (11, 15), (11, 16), (11, 21), - (12, 14), (12, 27), (12, 33), (13, 14), (13, 27), (13, 33), (14, 26), (14, 27), (14, 32), - (14, 33), (15, 21), (16, 18), (16, 21), (17, 21), (17, 24), (17, 31), (18, 21), (18, 22), - (18, 23), (18, 24), (18, 29), (18, 30), (18, 31), (19, 22), (19, 23), (19, 26), (19, 28), - (19, 29), (19, 30), (19, 32), (20, 22), (20, 26), (20, 27), (20, 29), (20, 32), (20, 33), - (21, 22), (21, 23), (21, 24), (21, 29), (21, 30), (21, 31), (22, 23), (22, 24), (22, 25), - (22, 26), (22, 27), (22, 28), (22, 29), (22, 30), (22, 31), (22, 32), (22, 33), (23, 24), - (23, 28), (23, 29), (23, 30), (23, 31), (24, 29), (24, 30), (24, 31), (25, 29), (26, 27), - (26, 28), (26, 29), (26, 32), (26, 33), (27, 29), (27, 32), (27, 33), (28, 29), (28, 30), - (28, 32), (29, 30), (29, 31), (29, 32), (29, 33), (30, 31), (32, 33), -]; - -/// One-way latency in ms for each [`PRODUCTION`] link, from the great-circle -/// distance between its PoPs at 200 km/ms over a path 1.5 times as long, plus 1 ms. -#[rustfmt::skip] -const PRODUCTION_LATENCY_MS: &[u64] = &[ - 10, 7, 7, 50, 30, 51, 50, 9, 13, 13, 5, 4, 6, 8, 3, 4, 1, 50, 6, 4, 50, 6, 4, 50, 63, 58, 50, 58, - 16, 5, 5, 10, 10, 10, 10, 12, 1, 12, 6, 3, 5, 8, 8, 5, 4, 43, 5, 4, 43, 6, 11, 8, 49, 22, 23, 6, - 8, 49, 22, 23, 3, 5, 4, 41, 103, 103, 60, 91, 91, 59, 64, 59, 64, 6, 9, 8, 3, 6, 6, 4, 48, 4, 3, - 48, 4, 3, 5, 44, 26, 1, 5, 44, 26, 15, 21, 18, 15, 21, 18, 50, 6, 4, 50, 6, 4, 45, 47, 12, 29, 30, - 5, 1, 45, 47, 29, 30, 2, 44, 45, 1, 2, 47, 2, 1, 12, 9, 26, 29, 1, 9, 30, 9, 1, 5, 44, 26, 45, 47, - 29, 30, 2, 9, -]; - /// One withdrawal on a partial mesh retracts everywhere once. A relay two hops /// from the publisher's hears only that its neighbor withdrew, and must not /// fall back to, and re-advertise, another path derived from the same diff --git a/rs/moq-net/tests/relay_restart.rs b/rs/moq-net/tests/relay_restart.rs new file mode 100644 index 00000000000..f6676be2606 --- /dev/null +++ b/rs/moq-net/tests/relay_restart.rs @@ -0,0 +1,243 @@ +//! A relay restarting under its old hop id on the production graph. +//! +//! A relay with a fixed cluster id drains: its publisher reconnects to a sibling +//! relay, keeping its own origin hop, and its old session closes. The relay's peers +//! lose it when the drain ends, cleanly or with the close never arriving, and it +//! comes back under the same hop id. Viewers watch from every relay throughout. +//! Afterwards every relay serves a new request, including for a track no viewer +//! holds open, whose subscription must travel the whole chain to the publisher. + +mod support; + +use std::{ + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, +}; + +use moq_net::{Error, Hop, Timestamp, Version, origin, track}; +use support::{ + harness::{MockConnectOptions, MockPair, connect_mock, peer_with_latency}, + production::{PRODUCTION, PRODUCTION_LATENCY_MS}, +}; + +/// Bound on a request that should succeed, on the paused clock. +const WAIT: Duration = Duration::from_secs(30); + +/// How long a session whose close never arrived lingers before its transport idles it out. +const IDLE: Duration = Duration::from_secs(30); + +/// The publisher's origin hop, kept across its move. +const PUBLISHER: u64 = 100; + +/// The publisher's relay, a hub, and its sibling in the same PoP. +const FROM: usize = 3; +const TO: usize = 21; + +/// A relay far from the publisher that restarts afterwards, as a manual restart +/// to clear a strand would. +const FAR: usize = 13; + +fn produce_origin(hop: u64) -> origin::Producer { + let (producer, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); + tokio::spawn(support::harness::run(driver)); + producer +} + +/// Connect `publisher` to `relay` as a client publishing into it. +async fn publish(version: Version, publisher: &origin::Producer, relay: &origin::Producer) -> MockPair { + let mut options = MockConnectOptions::new(version); + options.client_publish = Some(publisher.consume()); + options.server_subscribe = Some(relay.clone()); + connect_mock(options).await +} + +/// End a session: closed now when `clean`, otherwise silent until it idles out. +fn end(pair: &MockPair, clean: bool) { + if clean { + pair.client.abort(Error::Cancel); + pair.server.abort(Error::Cancel); + } else { + pair.client_transport.set_latency(Duration::from_secs(3600)); + pair.server_transport.set_latency(Duration::from_secs(3600)); + } +} + +/// Write one frame in a fresh group. +fn write(track: &track::Producer, n: u64, payload: &str) { + let mut group = track.append_group().unwrap(); + group + .write_frame(Timestamp::from_millis(n * 1000).unwrap(), payload.as_bytes().to_vec()) + .unwrap(); + group.finish().unwrap(); +} + +/// Keep the publisher live: one group a second, its payload the group's count. +fn ticker(video: track::Producer) -> (Arc, tokio::task::JoinHandle<()>) { + let count = Arc::new(AtomicU64::new(0)); + let task = tokio::spawn({ + let count = count.clone(); + async move { + loop { + let n = count.fetch_add(1, Ordering::Relaxed) + 1; + write(&video, n, &n.to_string()); + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + }); + (count, task) +} + +/// A player at `relay` that keeps watching `demo`, subscribing again a second after +/// its track ends. +fn viewer(relay: origin::Producer) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + loop { + if let Ok(demo) = relay.consume().request_broadcast("demo").await + && let Ok(track) = demo.track("video") + && let Ok(mut video) = track.subscribe(None).await + { + while let Ok(Some(mut group)) = video.recv_group().await { + while let Ok(Some(_)) = group.read_frame().await {} + } + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + }) +} + +/// Request `demo` at `relay`, read a video group newer than `after`, then the +/// catalog, which no viewer holds open. +async fn serve(relay: &origin::Producer, after: u64) -> Result<(), Error> { + let demo = relay.consume().request_broadcast("demo").await?; + let mut video = demo.track("video")?.subscribe(None).await?; + loop { + let mut group = video.recv_group().await?.ok_or(Error::Cancel)?; + let Some(frame) = group.read_frame().await? else { + continue; + }; + if std::str::from_utf8(&frame.payload).unwrap().parse::().unwrap() > after { + break; + } + } + let mut catalog = demo.track("catalog")?.subscribe(None).await?; + let mut group = catalog.recv_group().await?.ok_or(Error::Cancel)?; + group.read_frame().await?.ok_or(Error::Cancel)?; + Ok(()) +} + +/// The production graph's relays, one link per [`PRODUCTION`] edge, and a viewer on each. +struct Fleet { + version: Version, + nodes: Vec, + links: Vec, + viewers: Vec>, +} + +impl Fleet { + async fn new(version: Version) -> Self { + let nodes: Vec<_> = (1..=34).map(produce_origin).collect(); + let mut links = Vec::new(); + for (&(a, b), &ms) in PRODUCTION.iter().zip(PRODUCTION_LATENCY_MS) { + links.push(peer_with_latency(version, &nodes[a], &nodes[b], Duration::from_millis(ms)).await); + } + let viewers = nodes.iter().cloned().map(viewer).collect(); + Self { + version, + nodes, + links, + viewers, + } + } + + /// End `relay`'s links with its drain, then bring it back under its old hop id + /// and peer it again. Returns the old links, for an unclean end to idle out. + async fn restart(&mut self, relay: usize, clean: bool) -> Vec { + let touching: Vec<_> = (0..PRODUCTION.len()) + .filter(|&i| PRODUCTION[i].0 == relay || PRODUCTION[i].1 == relay) + .collect(); + for &i in &touching { + end(&self.links[i], clean); + } + self.viewers[relay].abort(); + tokio::time::sleep(Duration::from_secs(5)).await; + + self.nodes[relay] = produce_origin(relay as u64 + 1); + self.viewers[relay] = viewer(self.nodes[relay].clone()); + let mut old = Vec::new(); + for i in touching { + let (a, b) = PRODUCTION[i]; + let latency = Duration::from_millis(PRODUCTION_LATENCY_MS[i]); + let new = peer_with_latency(self.version, &self.nodes[a], &self.nodes[b], latency).await; + old.push(std::mem::replace(&mut self.links[i], new)); + } + old + } + + /// Every relay serves a new request for `demo`. + async fn assert_serves(&self, step: &str, count: &AtomicU64) { + let after = count.load(Ordering::Relaxed); + let mut requests = tokio::task::JoinSet::new(); + for (relay, node) in self.nodes.iter().enumerate() { + let node = node.clone(); + requests.spawn(async move { (relay, tokio::time::timeout(WAIT, serve(&node, after)).await) }); + } + let mut stranded: Vec<_> = requests + .join_all() + .await + .into_iter() + .filter_map(|(relay, outcome)| match outcome { + Ok(Ok(())) => None, + Ok(Err(err)) => Some((relay, err.to_string())), + Err(_) => Some((relay, "hung".to_string())), + }) + .collect(); + stranded.sort(); + assert!(stranded.is_empty(), "{step}: {stranded:?}"); + } +} + +async fn restart_under_the_same_hop(clean: bool) { + let version: Version = "moq-lite-06".parse().unwrap(); + let mut fleet = Fleet::new(version).await; + + let publisher = produce_origin(PUBLISHER); + let demo = publisher.create_broadcast("demo").unwrap(); + let catalog = demo.create_track("catalog", None).unwrap(); + write(&catalog, 0, "catalog"); + let (count, _ticker) = ticker(demo.create_track("video", None).unwrap()); + demo.announce(Default::default()).unwrap(); + let mut from = Some(publish(version, &publisher, &fleet.nodes[FROM]).await); + tokio::time::sleep(Duration::from_secs(5)).await; + fleet.assert_serves("start", &count).await; + + // The relay drains: the publisher moves to its sibling, then its old session ends. + let _to = publish(version, &publisher, &fleet.nodes[TO]).await; + tokio::time::sleep(origin::DEFAULT_UPDATE_HOLD * 2).await; + end(from.as_ref().unwrap(), clean); + + for (relay, step) in [(FROM, "publisher's relay restarted"), (FAR, "far relay restarted")] { + let mut old = fleet.restart(relay, clean).await; + old.extend(from.take()); + tokio::time::sleep(IDLE).await; + for pair in &old { + end(pair, true); + } + tokio::time::sleep(Duration::from_secs(10)).await; + fleet.assert_serves(step, &count).await; + } +} + +#[tokio::test(start_paused = true)] +async fn clean_restart_under_the_same_hop() { + restart_under_the_same_hop(true).await; +} + +/// The close never reaches the peers, so each holds a silent session to the old +/// relay, under the same hop id as the new one, until it idles out. +#[tokio::test(start_paused = true)] +async fn unclean_restart_under_the_same_hop() { + restart_under_the_same_hop(false).await; +} diff --git a/rs/moq-net/tests/support/mod.rs b/rs/moq-net/tests/support/mod.rs index 44c6b336839..3a6252bc9ca 100644 --- a/rs/moq-net/tests/support/mod.rs +++ b/rs/moq-net/tests/support/mod.rs @@ -2,3 +2,4 @@ pub mod harness; pub mod mock; +pub mod production; diff --git a/rs/moq-net/tests/support/production.rs b/rs/moq-net/tests/support/production.rs new file mode 100644 index 00000000000..022e3b7325f --- /dev/null +++ b/rs/moq-net/tests/support/production.rs @@ -0,0 +1,37 @@ +//! A production relay graph, shared by the mesh tests. + +#![allow(dead_code)] + +/// A 34-relay partial mesh: a production CDN's graph on 2026-09-30, where +/// relays in neighboring PoPs peer and most reach a publisher's relay through +/// others. +#[rustfmt::skip] +pub const PRODUCTION: &[(usize, usize)] = &[ + (0, 20), (0, 22), (0, 29), (1, 3), (1, 12), (1, 14), (1, 21), (2, 20), (2, 26), (2, 32), (3, 8), + (3, 11), (3, 15), (3, 16), (3, 17), (3, 18), (3, 21), (3, 22), (3, 23), (3, 24), (3, 29), (3, 30), + (3, 31), (4, 7), (4, 20), (4, 22), (4, 25), (4, 29), (5, 20), (5, 27), (5, 33), (6, 23), (6, 24), + (6, 30), (6, 31), (7, 22), (7, 25), (7, 29), (8, 15), (8, 17), (8, 21), (8, 23), (8, 30), (9, 19), + (9, 22), (9, 23), (9, 28), (9, 29), (9, 30), (10, 19), (10, 20), (10, 22), (10, 23), (10, 26), + (10, 27), (10, 28), (10, 29), (10, 30), (10, 32), (10, 33), (11, 15), (11, 16), (11, 21), + (12, 14), (12, 27), (12, 33), (13, 14), (13, 27), (13, 33), (14, 26), (14, 27), (14, 32), + (14, 33), (15, 21), (16, 18), (16, 21), (17, 21), (17, 24), (17, 31), (18, 21), (18, 22), + (18, 23), (18, 24), (18, 29), (18, 30), (18, 31), (19, 22), (19, 23), (19, 26), (19, 28), + (19, 29), (19, 30), (19, 32), (20, 22), (20, 26), (20, 27), (20, 29), (20, 32), (20, 33), + (21, 22), (21, 23), (21, 24), (21, 29), (21, 30), (21, 31), (22, 23), (22, 24), (22, 25), + (22, 26), (22, 27), (22, 28), (22, 29), (22, 30), (22, 31), (22, 32), (22, 33), (23, 24), + (23, 28), (23, 29), (23, 30), (23, 31), (24, 29), (24, 30), (24, 31), (25, 29), (26, 27), + (26, 28), (26, 29), (26, 32), (26, 33), (27, 29), (27, 32), (27, 33), (28, 29), (28, 30), + (28, 32), (29, 30), (29, 31), (29, 32), (29, 33), (30, 31), (32, 33), +]; + +/// One-way latency in ms for each [`PRODUCTION`] link, from the great-circle +/// distance between its PoPs at 200 km/ms over a path 1.5 times as long, plus 1 ms. +#[rustfmt::skip] +pub const PRODUCTION_LATENCY_MS: &[u64] = &[ + 10, 7, 7, 50, 30, 51, 50, 9, 13, 13, 5, 4, 6, 8, 3, 4, 1, 50, 6, 4, 50, 6, 4, 50, 63, 58, 50, 58, + 16, 5, 5, 10, 10, 10, 10, 12, 1, 12, 6, 3, 5, 8, 8, 5, 4, 43, 5, 4, 43, 6, 11, 8, 49, 22, 23, 6, + 8, 49, 22, 23, 3, 5, 4, 41, 103, 103, 60, 91, 91, 59, 64, 59, 64, 6, 9, 8, 3, 6, 6, 4, 48, 4, 3, + 48, 4, 3, 5, 44, 26, 1, 5, 44, 26, 15, 21, 18, 15, 21, 18, 50, 6, 4, 50, 6, 4, 45, 47, 12, 29, 30, + 5, 1, 45, 47, 29, 30, 2, 44, 45, 1, 2, 47, 2, 1, 12, 9, 26, 29, 1, 9, 30, 9, 1, 5, 44, 26, 45, 47, + 29, 30, 2, 9, +];