diff --git a/rs/moq-cli/tests/integration/import.rs b/rs/moq-cli/tests/integration/import.rs index 9b8bcaea962..1ca85b12d42 100644 --- a/rs/moq-cli/tests/integration/import.rs +++ b/rs/moq-cli/tests/integration/import.rs @@ -219,6 +219,135 @@ async fn importers_sharing_an_epoch_publish_identical_timestamps() { ); } +/// Two importers of one path under one epoch, started together through the two relays of a +/// cluster, so neither is first: each relay hears the other's broadcast as a route with the same +/// epoch. Both stay live to their input's end, and each relay serves its own importer's broadcast +/// on the timeline the epoch places, every frame both carry at the same timestamp. +#[tokio::test] +async fn importers_co_started_across_a_cluster_publish_one_timeline() { + let _ = moq_tokio::crypto::install_default(); + let first = moq_relay::test_relay().await.expect("first relay"); + let second = moq_relay::test_relay_peered(&first.url, &first.fingerprint) + .await + .expect("second relay"); + let endpoints = [ + (first.url.clone(), first.fingerprint.clone()), + (second.url.clone(), second.fingerprint.clone()), + ]; + let mut origins = Vec::new(); + for fixture in [first, second] { + origins.push(fixture.relay.cluster().origin.clone()); + let ready = fixture.relay.ready(); + tokio::spawn(fixture.relay.run()); + ready.wait().await.expect("relay ready"); + } + let probe = origins[0].create_broadcast("probe").expect("probe broadcast"); + probe.announce(Default::default()).expect("announce the probe"); + tokio::time::timeout(TIMEOUT, origins[1].consume().routed_broadcast("probe")) + .await + .expect("the second relay never heard the first") + .expect("probe resolves"); + drop(probe); + + let epoch = moq_net::Epoch::mint(); + let global = ["--epoch", epoch.as_str()]; + // The second takes the source from a third of the way in, at the next PAT and PMT, so each + // relay's broadcast shows whose it is. + let join = BBB.len() / 188 / 3 * 188; + let mut children = endpoints + .each_ref() + .map(|(url, fingerprint)| import_as(url.as_str(), fingerprint, &global, "demo")); + let mut stdins: Vec<_> = children + .iter_mut() + .map(|child| child.stdin.take().expect("stdin")) + .collect(); + let (first_input, second_input) = stdins.split_at_mut(1); + let (written, other) = tokio::join!(first_input[0].write_all(BBB), second_input[0].write_all(&BBB[join..])); + written.and(other).expect("write stdin"); + + // Each relay's own importer, not the route its peer forwards. + let mut published = Vec::new(); + for (child, origin) in children.iter_mut().zip(&origins) { + let broadcast = running( + child, + "the broadcast", + origin.consume().local().routed_broadcast("demo"), + ) + .await + .expect("announced broadcast resolves"); + let mut catalogs = running( + child, + "the catalog subscription", + hang::catalog::Catalog::<()>::subscribe(&broadcast), + ) + .await + .expect("subscribe to the catalog"); + let catalog = running(child, "the first catalog", catalogs.next()) + .await + .expect("catalog read") + .expect("a catalog"); + let video = catalog + .video + .renditions + .keys() + .next() + .expect("a video rendition") + .clone(); + let subscription = moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(60)); + let track = running( + child, + "the video subscription", + broadcast.track(&video).expect("track").subscribe(subscription), + ) + .await + .expect("subscribe to the video"); + published.push((catalog.clock, track)); + } + // Together: a relay may resume a subscription on its peer's route, which ends only with + // the other input. + drop(stdins); + + let mut timelines = Vec::new(); + for (clock, mut track) in published { + let mut frames = std::collections::HashSet::new(); + while let Some(mut group) = tokio::time::timeout(TIMEOUT, track.recv_group()) + .await + .expect("the video never ended") + .expect("video group") + { + while let Some(frame) = group.read_frame().await.expect("video frame") { + let frame = hang::container::Frame::decode(frame.payload).expect("a timestamped frame"); + frames.insert((frame.timestamp.as_micros(), frame.payload)); + } + } + timelines.push((clock, frames)); + } + for (child, name) in children.iter_mut().zip(["first", "second"]) { + let status = tokio::time::timeout(TIMEOUT, child.wait()) + .await + .expect("moq never exited") + .expect("wait for moq"); + assert!(status.success(), "the {name} importer exited with {status}"); + } + + let [(first_clock, first), (second_clock, second)] = &timelines[..] else { + unreachable!("two relays"); + }; + assert!(first_clock.is_some(), "the epoch places the root clock"); + assert_eq!(second_clock, first_clock); + assert!( + !second.is_empty() && second.len() < first.len(), + "each relay serves its own importer: {} of {}", + second.len(), + first.len() + ); + let moved: Vec<_> = second.difference(first).map(|(micros, _)| *micros).collect(); + assert!( + moved.is_empty(), + "frames the second published at other timestamps: {moved:?}" + ); +} + /// About one second of CBR HEVC and Opus: shorter than the two-second mux-rate window, so /// the catalog is only released at EOF. const SHORT: &[u8] = include_bytes!("../../../moq-mux/src/container/ts/test_data/bbb_cbr.ts") diff --git a/rs/moq-relay/src/acme/tests.rs b/rs/moq-relay/src/acme/tests.rs index 4f94bc82434..50ab49f9a80 100644 --- a/rs/moq-relay/src/acme/tests.rs +++ b/rs/moq-relay/src/acme/tests.rs @@ -124,6 +124,7 @@ impl Directory { self.state.issued.borrow().clone() } + #[cfg(all(target_os = "linux", feature = "_quic"))] async fn wait_issued(&self, count: usize) -> Vec> { let mut issued = self.state.issued.subscribe(); tokio::time::timeout(TIMEOUT, issued.wait_for(|issued| issued.len() >= count)) @@ -424,6 +425,7 @@ impl Running { } /// Wait for HTTPS to serve `leaf`, which deploys just after the directory hands it out. + #[cfg(all(target_os = "linux", feature = "_quic"))] async fn wait_served(&self, directory: &Directory, leaf: &CertificateDer<'static>) { tokio::time::timeout(TIMEOUT, async { while self.served(directory, DOMAIN).await != *leaf { diff --git a/rs/moq-relay/src/test_support.rs b/rs/moq-relay/src/test_support.rs index 7303f2355e4..29267be1c20 100644 --- a/rs/moq-relay/src/test_support.rs +++ b/rs/moq-relay/src/test_support.rs @@ -20,13 +20,29 @@ pub struct TestRelay { /// Bind a public test relay on ephemeral loopback ports without TOML or port probes. pub async fn test_relay() -> anyhow::Result { + bind(config()?).await +} + +/// Bind a test relay like [`test_relay`] that dials another as its cluster peer, given that +/// one's [`TestRelay::url`] and [`TestRelay::fingerprint`]. +pub async fn test_relay_peered(peer: &url::Url, fingerprint: &str) -> anyhow::Result { + let mut config = config()?; + config.connect.tls.fingerprint = vec![fingerprint.to_string()]; + config.cluster.connect = vec![crate::cluster::Peer::new(peer.to_string())]; + bind(config).await +} + +fn config() -> anyhow::Result { let mut config = Config::default(); config.drain_timeout = std::time::Duration::ZERO; config.listen.bind = Some("127.0.0.1:0".parse()?); config.listen.tls.generate = vec!["localhost".into()]; config.web.http.listen = Some("127.0.0.1:0".parse()?); config.auth.public = vec![moq_auth::Pattern::all()]; + Ok(config) +} +async fn bind(config: Config) -> anyhow::Result { let relay = Relay::load(config).await?; let quic = relay.quic_addr()?; let http = relay.web_addrs().http.expect("HTTP listener was configured");