diff --git a/rs/moq-net/src/model/frame.rs b/rs/moq-net/src/model/frame.rs index fa92cfaeefd..d7f5bcb2df4 100644 --- a/rs/moq-net/src/model/frame.rs +++ b/rs/moq-net/src/model/frame.rs @@ -779,6 +779,10 @@ impl Consumer { if self.expired { return res; } + // Read to its end: no route is needed for it any more. + if matches!(res, Poll::Ready(Ok(_))) && self.read_idx >= self.size() { + self.recover = None; + } let Some(recover) = self.recover.as_mut() else { return res; }; diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index c958c4b7641..953e19bb42d 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -196,12 +196,19 @@ impl Consumer { Subscriber { reader, subscription } } - /// Fetch one group from whichever route serves the track. - pub(super) fn fetch_group(&self, sequence: u64, options: group::Fetch) -> Fetching { + /// Fetch one group from whichever route serves the track. `logical` is the logical + /// track's state, which the fetched group holds while it is read. + pub(super) fn fetch_group( + &self, + sequence: u64, + options: group::Fetch, + logical: kio::Consumer, + ) -> Fetching { Fetching { route: self.clone(), sequence, options, + logical, pending: None, } } @@ -590,6 +597,7 @@ impl Reader { generation: copy.generation, copy: copy.track.clone(), lease: Some(copy.lease.clone()), + logical: None, fetch: None, }) } @@ -878,6 +886,10 @@ pub(super) struct Recover { copy: track::Consumer, /// Keeps a replaced copy subscribed while this group is read from it. lease: Option>, + /// A fetch's logical track, held while the group is read. A fetch is no subscription, + /// so without it the last subscriber leaving parks the track, dropping the copy that + /// still feeds an open group, and the rest of the group never comes. + logical: Option>, /// A fetch of the rest of the group from the serving route, by its generation. fetch: Option<(u64, Fetch)>, } @@ -898,6 +910,7 @@ impl Clone for Recover { generation: self.generation, copy: self.copy.clone(), lease: self.lease.clone(), + logical: self.logical.clone(), // A clone reads in parallel, so it asks for itself. fetch: None, } @@ -1104,6 +1117,7 @@ pub(super) struct Fetching { route: Consumer, sequence: u64, options: group::Fetch, + logical: kio::Consumer, /// The fetch in flight, and the route generation it went to. pending: Option<(u64, kio::Pending)>, } @@ -1152,6 +1166,7 @@ impl kio::Task for Fetching { generation, copy: serving.clone(), lease: None, + logical: Some(self.logical.clone()), fetch: None, }))), // The route failed: ask the next one, unless none will come. @@ -2190,10 +2205,11 @@ mod test { let a = copy(); let dynamic = a.dynamic(); routes.serve(a.consume()); - let mut fetching = routes.consume().fetch_group(0, group::Fetch::default()); + let logical = logical(&routes); + let mut fetching = logical.consume().fetch_group(0, None); let wake = Arc::new(Wake::default()); let waiter = kio::Waiter::new(wake.clone().into()); - assert!(kio::Task::poll(&mut fetching, &waiter).is_pending()); + assert!(kio::Task::poll(&mut *fetching, &waiter).is_pending()); let _request = dynamic.requested_group().now_or_never().unwrap().unwrap(); wake.0.store(false, std::sync::atomic::Ordering::Relaxed); let b = copy(); @@ -2202,7 +2218,7 @@ mod test { group.finish().unwrap(); routes.serve(b.consume()); assert!(wake.0.load(std::sync::atomic::Ordering::Relaxed)); - assert!(matches!(kio::Task::poll(&mut fetching, &waiter), Poll::Ready(Ok(_)))); + assert!(matches!(kio::Task::poll(&mut *fetching, &waiter), Poll::Ready(Ok(_)))); } #[test] @@ -2425,7 +2441,9 @@ mod test { let a = copy(); let _open = a.create_group(group::Info { sequence: 0 }).unwrap(); routes.serve(a.consume()); - let mut reading = kio::Pending::new(routes.consume().fetch_group(0, group::Fetch::default())) + let mut reading = logical(&routes) + .consume() + .fetch_group(0, None) .now_or_never() .unwrap() .unwrap(); @@ -2448,7 +2466,9 @@ mod test { let mut arriving = a.create_group(group::Info { sequence: 0 }).unwrap(); arriving.write_frame(ts(0), b"a".as_ref()).unwrap(); - let mut fetched = kio::Pending::new(routes.consume().fetch_group(0, group::Fetch::default())) + let mut fetched = logical(&routes) + .consume() + .fetch_group(0, None) .now_or_never() .unwrap() .unwrap(); @@ -2467,6 +2487,28 @@ mod test { assert_eq!(read(&mut fetched), None); } + /// A fetched frame read to its end no longer holds the logical track, so a caller + /// keeping the handle does not stop the track from parking. + #[test] + #[cfg(feature = "async")] + fn a_read_frame_releases_the_track() { + let routes = Producer::new(); + let a = copy(); + routes.serve(a.consume()); + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + + let logical = logical(&routes); + let demand = logical.demand(); + let mut fetched = logical.consume().fetch_group(0, None).now_or_never().unwrap().unwrap(); + assert!(demand.is_used(), "an open fetched group holds the track"); + let mut frame = fetched.next_frame().now_or_never().unwrap().unwrap().unwrap(); + drop(fetched); + assert!(demand.is_used(), "an unread frame holds the track"); + assert_eq!(frame.read_all().now_or_never().unwrap().unwrap(), b"a".as_ref()); + assert!(!demand.is_used(), "a read frame does not"); + } + /// A rerouted fetch keeps its own age budget: the new route is asked with it, and a /// read waiting on that route gives up once the group falls past it there. #[test] @@ -2480,7 +2522,9 @@ mod test { let budget = Duration::from_millis(500); let options = group::Fetch::default().with_max_delay(budget); - let mut fetched = kio::Pending::new(routes.consume().fetch_group(0, options)) + let mut fetched = logical(&routes) + .consume() + .fetch_group(0, options) .now_or_never() .unwrap() .unwrap(); @@ -2674,12 +2718,13 @@ mod test { let a = copy(); let _dynamic = a.dynamic(); routes.serve(a.consume()); - let mut fetching = routes.consume().fetch_group(0, group::Fetch::default()); + let logical = logical(&routes); + let mut fetching = logical.consume().fetch_group(0, None); let waiter = kio::Waiter::noop(); - assert!(kio::Task::poll(&mut fetching, &waiter).is_pending()); + assert!(kio::Task::poll(&mut *fetching, &waiter).is_pending()); routes.end(Err(Error::Unroutable)); assert!(matches!( - kio::Task::poll(&mut fetching, &waiter), + kio::Task::poll(&mut *fetching, &waiter), Poll::Ready(Err(Error::Unroutable)) )); } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 75dfa3981ed..478270f7552 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -3761,7 +3761,7 @@ impl Consumer { result: None, hit: None, stats: self.stats.clone(), - resume: Some(Box::new(resume.fetch_group(sequence, options))), + resume: Some(Box::new(resume.fetch_group(sequence, options, state.clone()))), }); } @@ -4248,7 +4248,7 @@ impl kio::Task for Fetching { max_delay: self.max_delay, frame_start: self.frame_start, }; - self.resume = Some(Box::new(routes.fetch_group(self.sequence, options))); + self.resume = Some(Box::new(routes.fetch_group(self.sequence, options, self.state.clone()))); } if let Some(resume) = &mut self.resume { let group = ready!(kio::Task::poll(&mut **resume, waiter))?; diff --git a/rs/moq-net/tests/integration/catalog_churn.rs b/rs/moq-net/tests/integration/catalog_churn.rs new file mode 100644 index 00000000000..31c944f87c2 --- /dev/null +++ b/rs/moq-net/tests/integration/catalog_churn.rs @@ -0,0 +1,460 @@ +#![cfg(feature = "async")] + +//! Seeded churn on a catalog: one open group at a time, a snapshot then deltas, read +//! on `R` by subscribers and short fetches that come and go while `R`'s routes to the +//! publisher `P` (through `A`, through `B`, and direct) appear and disappear. +//! +//! A reader may fail loudly when its route goes (its client asks again), but it must +//! never wait forever: every subscriber gets each group it reads from the snapshot +//! on, and every fetch gets the whole group written so far. A failing seed is +//! shrunk to the ops it needs, which the report names. + +use crate::support; + +use std::time::Duration; + +use futures::{StreamExt, channel::mpsc}; +use moq_net::{Error, Hop, Timestamp, Version, origin, track}; +use support::harness::{MockConnectOptions, MockPair, connect_mock}; + +const SEEDS: u64 = 32; +const STEPS: usize = 60; + +/// How long any one request or read may take, in virtual time. +const READ_TIMEOUT: Duration = Duration::from_secs(10); + +/// How long a reader short of its frames at a quiet point gets to prove it is only late. +const STUCK_AFTER: Duration = Duration::from_secs(120); + +fn produce_origin(hop: u64) -> origin::Producer { + let (producer, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); + support::harness::spawn(driver); + producer +} + +async fn link(version: Version, from: &origin::Producer, to: &origin::Producer) -> MockPair { + let mut options = MockConnectOptions::new(version); + options.server_publish = Some(from.consume()); + options.client_subscribe = Some(to.clone()); + connect_mock(options).await +} + +/// xorshift64, so a seed replays the same ops anywhere. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } +} + +#[derive(Clone, Copy, Debug)] +enum Op { + /// Append a delta to the open group. + Write, + /// Finish the group and open the next with a new snapshot. + NewGroup, + /// A subscriber joins. + Join, + /// A short reader fetches the latest group, as an HTTP request does. + Fetch, + /// A subscriber leaves, picked by this value. + Leave(u64), + /// Connect the route, or drop it if another route stays up. + ToggleA, + ToggleB, + ToggleDirect, + Sleep(Duration), + /// Settle, then require every subscriber to hold what it should. + Check, +} + +fn generate(seed: u64) -> Vec { + let mut rng = Rng(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1); + let mut ops = Vec::new(); + for _ in 0..STEPS { + ops.push(match rng.below(100) { + 0..25 => Op::Write, + 25..30 => Op::NewGroup, + 30..45 => Op::Join, + 45..55 => Op::Fetch, + 55..70 => Op::Leave(rng.next()), + 70..80 => Op::ToggleA, + 80..90 => Op::ToggleB, + _ => Op::ToggleDirect, + }); + // Mostly back to back, sometimes past the relays' 30 s idle linger. + let pause = match rng.below(6) { + 0 | 1 => continue, + 2 => Duration::from_millis(50), + 3 => Duration::from_millis(500), + 4 => Duration::from_secs(5), + _ => Duration::from_secs(31), + }; + ops.push(Op::Sleep(pause)); + if pause >= Duration::from_millis(500) { + ops.push(Op::Check); + } + } + ops.push(Op::Check); + ops +} + +/// How a seed failed: `stuck` (a reader never got what it should have), `late` (only +/// after a quiet point), or `wrong` (it got something it should not have). +struct Failure { + kind: &'static str, + detail: String, +} + +fn stuck(detail: String) -> Failure { + Failure { kind: "stuck", detail } +} + +fn late(detail: String) -> Failure { + Failure { kind: "late", detail } +} + +fn wrong(detail: String) -> Failure { + Failure { kind: "wrong", detail } +} + +/// A subscriber's frames as `group:payload`, or the error that ended it. +struct Reader { + id: usize, + rx: mpsc::UnboundedReceiver>, + /// Every frame written since it joined, from the snapshot of the group then open. + expected: Vec, + seen: Vec, + /// Owns the subscription, so dropping the reader leaves the track. + task: moq_clock::sim::JoinHandle<()>, +} + +impl Drop for Reader { + fn drop(&mut self) { + self.task.abort(); + } +} + +impl Reader { + fn new(id: usize, mut sub: track::Subscriber, expected: Vec) -> Self { + let (tx, rx) = mpsc::unbounded(); + let task = moq_clock::sim::spawn(async move { + loop { + let mut group = match sub.recv_group().await { + Ok(Some(group)) => group, + Ok(None) => return, + Err(err) => { + let _ = tx.unbounded_send(Err(err)); + return; + } + }; + loop { + let frame = match group.read_frame().await { + Ok(Some(frame)) => frame, + Ok(None) => break, + Err(err) => { + let _ = tx.unbounded_send(Err(err)); + return; + } + }; + let frame = format!("{}:{}", group.sequence, String::from_utf8_lossy(&frame.payload)); + if tx.unbounded_send(Ok(frame)).is_err() { + return; + } + } + } + }); + Self { + id, + rx, + expected, + seen: Vec::new(), + task, + } + } + + /// Take what arrived; `false` once the subscription failed or ended. + fn drain(&mut self) -> bool { + loop { + match self.rx.try_recv() { + Ok(Ok(frame)) => self.seen.push(frame), + Ok(Err(_)) => return false, + Err(mpsc::TryRecvError::Empty) => return true, + Err(mpsc::TryRecvError::Closed) => return false, + } + } + } + + /// Compare what it saw with what it should see. A joiner may first get groups older + /// than the newest at its join (`R` had not seen the newer one yet), and may skip + /// whole groups superseded before it resolved, but each group it gets starts at the + /// snapshot and continues in order. + fn judge(&self) -> Verdict { + let group = |frame: &String| frame.split(':').next().unwrap().parse::().unwrap(); + let joined = self.expected.first().map(group); + let mut seen = self.seen.as_slice(); + while let (Some(first), Some(joined)) = (seen.first(), joined) { + let sequence = group(first); + if sequence >= joined { + break; + } + if !first.ends_with(":snapshot") { + return Verdict::Wrong; + } + let older = seen.iter().take_while(|frame| group(frame) == sequence).count(); + seen = &seen[older..]; + } + let mut expected = self.expected.as_slice(); + if let Some(first) = seen.first() { + let skipped = expected.iter().take_while(|frame| group(frame) < group(first)).count(); + expected = &expected[skipped..]; + } + match (seen == expected, expected.starts_with(seen)) { + (true, _) => Verdict::Done, + (false, true) => Verdict::Short, + (false, false) => Verdict::Wrong, + } + } + + fn report(&self) -> String { + format!("r{} saw {:?}, expected {:?}", self.id, self.seen, self.expected) + } +} + +#[derive(PartialEq)] +enum Verdict { + Done, + Short, + Wrong, +} + +struct Topology { + version: Version, + publisher: origin::Producer, + relay_a: origin::Producer, + relay_b: origin::Producer, + subscriber: origin::Producer, + routes: [Option; 3], + _upstream: Vec, +} + +impl Topology { + /// Connect route `index` (`A`, `B`, direct), or drop it if another stays up. + async fn toggle(&mut self, index: usize) { + if let Some(pair) = self.routes[index].take() { + match self.routes.iter().any(Option::is_some) { + true => { + pair.server.abort(Error::Cancel); + pair.client.abort(Error::Cancel); + } + false => self.routes[index] = Some(pair), + } + return; + } + let from = [&self.relay_a, &self.relay_b, &self.publisher][index]; + self.routes[index] = Some(link(self.version, from, &self.subscriber).await); + } + + /// A fresh request for the broadcast; `None` when refused, as with no route right now. + async fn request(&self, step: usize) -> Result, Failure> { + let request = self.subscriber.consume().request_broadcast("live", None); + match moq_clock::timeout(READ_TIMEOUT, request).await { + Ok(result) => Ok(result.ok()), + Err(_) => Err(stuck(format!("{step}: request hung"))), + } + } +} + +async fn execute(version: Version, ops: &[Op]) -> Result<(), Failure> { + let publisher = produce_origin(1); + let relay_a = produce_origin(2); + let relay_b = produce_origin(3); + let subscriber = produce_origin(4); + + let broadcast = publisher.create_broadcast("live").unwrap(); + let info = track::Info::default().with_timescale(moq_net::Timescale::MILLI); + let track = broadcast.create_track("catalog.json", info).unwrap(); + broadcast + .announce(origin::Route::default().with_epoch(moq_net::Epoch::mint())) + .unwrap(); + + let upstream = vec![ + link(version, &publisher, &relay_a).await, + link(version, &publisher, &relay_b).await, + ]; + let a_to_r = link(version, &relay_a, &subscriber).await; + subscriber.consume().routed("live").await.unwrap(); + let mut topology = Topology { + version, + publisher, + relay_a, + relay_b, + subscriber, + routes: [Some(a_to_r), None, None], + _upstream: upstream, + }; + + let mut group = track.append_group().unwrap(); + // The open group as written so far, as `group:payload`. + let mut written: Vec = Vec::new(); + let mut readers: Vec = Vec::new(); + let mut joined = 0; + let mut clock = 0; + let mut write = + |group: &mut moq_net::group::Producer, written: &mut Vec, readers: &mut [Reader], payload: &str| { + clock += 10; + group + .write_frame(Timestamp::from_millis(clock).unwrap(), payload.as_bytes()) + .unwrap(); + let frame = format!("{}:{payload}", group.sequence); + for reader in readers { + reader.expected.push(frame.clone()); + } + written.push(frame); + }; + write(&mut group, &mut written, &mut readers, "snapshot"); + + for (step, op) in ops.iter().enumerate() { + match *op { + Op::Write => write(&mut group, &mut written, &mut readers, &format!("d{step}")), + Op::NewGroup => { + group.finish().unwrap(); + group = track.append_group().unwrap(); + written.clear(); + write(&mut group, &mut written, &mut readers, "snapshot"); + // Let it reach `R`, so the newest group at the next join is unambiguous. + moq_clock::sleep(Duration::from_millis(500)).await; + } + Op::Join => { + let Some(remote) = topology.request(step).await? else { + continue; + }; + let subscribe = remote.track("catalog.json").unwrap().subscribe(None); + match moq_clock::timeout(READ_TIMEOUT, subscribe).await { + Ok(Ok(sub)) => readers.push(Reader::new(joined, sub, written.clone())), + Ok(Err(_)) => {} + Err(_) => return Err(stuck(format!("{step}: subscribe hung"))), + } + joined += 1; + } + Op::Fetch => { + let Some(remote) = topology.request(step).await? else { + continue; + }; + let fetch = remote.track("catalog.json").unwrap().fetch_group(group.sequence, None); + let mut fetched = match moq_clock::timeout(READ_TIMEOUT, fetch).await { + Ok(Ok(fetched)) => fetched, + Ok(Err(_)) => continue, + Err(_) => return Err(stuck(format!("{step}: fetch hung"))), + }; + let mut got = Vec::new(); + while got.len() < written.len() { + match moq_clock::timeout(READ_TIMEOUT, fetched.read_frame()).await { + Ok(Ok(Some(frame))) => got.push(format!( + "{}:{}", + fetched.sequence, + String::from_utf8_lossy(&frame.payload) + )), + Ok(Ok(None)) => return Err(wrong(format!("{step}: fetch ended at {got:?}"))), + Ok(Err(_)) => break, + Err(_) => return Err(stuck(format!("{step}: fetched {got:?}, then hung"))), + } + } + if !written.starts_with(&got) { + return Err(wrong(format!("{step}: fetched {got:?}, wrote {written:?}"))); + } + } + Op::Leave(pick) => { + if !readers.is_empty() { + readers.remove((pick % readers.len() as u64) as usize); + } + } + Op::ToggleA => topology.toggle(0).await, + Op::ToggleB => topology.toggle(1).await, + Op::ToggleDirect => topology.toggle(2).await, + Op::Sleep(pause) => moq_clock::sleep(pause).await, + Op::Check => { + moq_clock::sleep(Duration::from_millis(500)).await; + let mut failed = Vec::new(); + for (index, reader) in readers.iter_mut().enumerate() { + let open = reader.drain(); + match reader.judge() { + Verdict::Wrong => return Err(wrong(format!("{step}: {}", reader.report()))), + // Failed loudly: its client asks again. + _ if !open => failed.push(index), + Verdict::Done => {} + Verdict::Short => { + let _ = moq_clock::timeout(STUCK_AFTER, async { + while reader.judge() == Verdict::Short { + match reader.rx.next().await { + Some(Ok(frame)) => reader.seen.push(frame), + _ => break, + } + } + }) + .await; + return Err(match reader.judge() { + Verdict::Done => late(format!("{step}: r{}", reader.id)), + _ => stuck(format!("{step}: {}", reader.report())), + }); + } + } + } + for index in failed.into_iter().rev() { + readers.remove(index); + } + } + } + } + Ok(()) +} + +/// Drop ops one at a time while the same kind of failure persists. +async fn shrink(version: Version, mut ops: Vec, kind: &str) -> Vec { + let mut index = 0; + while index < ops.len() { + let mut candidate = ops.clone(); + candidate.remove(index); + match execute(version, &candidate).await { + Err(failure) if failure.kind == kind => ops = candidate, + _ => index += 1, + } + } + ops +} + +async fn churn(version: &str) { + let version: Version = version.parse().unwrap(); + let mut failures = Vec::new(); + for seed in 0..SEEDS { + let ops = generate(seed); + if let Err(failure) = execute(version, &ops).await { + let small = shrink(version, ops, failure.kind).await; + failures.push(format!( + "seed {seed}: {} {}\n shrunk to {small:?}", + failure.kind, failure.detail + )); + } + } + assert!(failures.is_empty(), "{version}:\n{}", failures.join("\n")); +} + +/// Routes carry the publisher's epoch, so subscriptions resume across them. +#[moq_clock::test] +async fn catalog_churn_lite_07() { + churn("moq-lite-07-wip").await; +} + +/// No epoch on the wire: a route change ends subscriptions, which must fail loudly. +#[moq_clock::test] +async fn catalog_churn_lite_06() { + churn("moq-lite-06").await; +} diff --git a/rs/moq-net/tests/integration/catalog_handover.rs b/rs/moq-net/tests/integration/catalog_handover.rs new file mode 100644 index 00000000000..8039b5eb11f --- /dev/null +++ b/rs/moq-net/tests/integration/catalog_handover.rs @@ -0,0 +1,522 @@ +#![cfg(feature = "async")] + +//! A catalog's one open group (a snapshot at frame 0, deltas after) survives its +//! route changing in the middle of that group, end to end over real sessions. +//! +//! A publisher `P` is pulled by two relays `A` and `B` carrying its epoch, and `R` +//! reads through `A` until the route changes mid-group. The reader already on `R` +//! continues with the next delta, and every reader after the change (a fresh one, a +//! returning one, and a burst of short ones) still starts from the snapshot. The +//! replacement's copy of the group was opened mid-group for the continuing reader, +//! so a reader that starts at frame 0 must not be refused as lagged and left with +//! no catalog. + +use crate::support; + +use std::time::Duration; + +use futures::{StreamExt, channel::mpsc}; +use moq_net::{Error, Hop, Timestamp, Version, broadcast, origin, track}; +use support::harness::{MockConnectOptions, MockPair, connect_mock}; + +const TEST_TIMEOUT: Duration = Duration::from_secs(300); + +/// How long any one read may take, in virtual time. +const READ_TIMEOUT: Duration = Duration::from_secs(10); + +/// Short readers in each burst, re-reading the catalog and leaving. +const BURST: usize = 4; + +/// When the returning reader comes back: twice a relay's 30 s idle linger. +const RETURN_AFTER: Duration = Duration::from_secs(62); + +fn timed() -> track::Info { + track::Info::default().with_timescale(moq_net::Timescale::MILLI) +} + +fn produce_origin(hop: u64) -> origin::Producer { + let (producer, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); + support::harness::spawn(driver); + producer +} + +async fn link(version: Version, from: &origin::Producer, to: &origin::Producer, cost: Option) -> MockPair { + let mut options = MockConnectOptions::new(version); + options.server_publish = Some(from.consume()); + options.client_subscribe = Some(to.clone()); + options.cost = cost; + connect_mock(options).await +} + +fn abort(pair: MockPair) { + pair.server.abort(Error::Cancel); + pair.client.abort(Error::Cancel); +} + +/// Let every task settle. Time is paused, so this returns once the runtime is idle. +async fn settle() { + moq_clock::sleep(Duration::from_millis(500)).await; +} + +/// How the serving route changes. +#[derive(Clone, Copy, Debug)] +enum Trigger { + /// The session `A -> R` dies; `B` takes over. + Disconnect, + /// `A` loses `P` and withdraws the broadcast while its session to `R` stays up. + Unannounce, + /// `R` gains a direct route to `P` that beats `A`. + BetterRoute, + /// The session `A -> R` dies with no standby, then reconnects. The broadcast closes + /// with its last route, so a viewer's subscription ends rather than resuming. + Reconnect, +} + +/// Who holds the catalog on `R` while the route changes. +#[derive(Clone, Copy, Debug)] +enum Holder { + /// A viewer stays subscribed and must see the next delta exactly once. + Viewer, + /// Nobody: the last reader left, so the change lands on a parked track. + Parked, +} + +struct Topology { + version: Version, + publisher: origin::Producer, + relay_a: origin::Producer, + relay_b: origin::Producer, + subscriber: origin::Producer, + group: moq_net::group::Producer, + p_to_a: Option, + a_to_r: Option, + links: Vec, + _broadcast: broadcast::Producer, + _track: track::Producer, + /// Every payload written to the open group so far. + written: Vec, +} + +impl Topology { + async fn new(version: Version) -> Self { + let publisher = produce_origin(1); + let relay_a = produce_origin(2); + let relay_b = produce_origin(3); + let subscriber = produce_origin(4); + + let broadcast = publisher.create_broadcast("live").unwrap(); + let track = broadcast.create_track("catalog.json", timed()).unwrap(); + broadcast + .announce(origin::Route::default().with_epoch(moq_net::Epoch::mint())) + .unwrap(); + let group = track.append_group().unwrap(); + + let p_to_a = link(version, &publisher, &relay_a, None).await; + let p_to_b = link(version, &publisher, &relay_b, None).await; + let a_to_r = link(version, &relay_a, &subscriber, None).await; + subscriber.consume().routed("live").await.unwrap(); + + let mut topology = Self { + version, + publisher, + relay_a, + relay_b, + subscriber, + group, + p_to_a: Some(p_to_a), + a_to_r: Some(a_to_r), + links: vec![p_to_b], + _broadcast: broadcast, + _track: track, + written: Vec::new(), + }; + topology.write("snapshot"); + topology + } + + fn write(&mut self, payload: &str) { + let at = Timestamp::from_millis(self.written.len() as u64 * 10).unwrap(); + self.group.write_frame(at, payload.as_bytes()).unwrap(); + self.written.push(payload.to_string()); + } + + /// Connect `B -> R`, priced worse than `A` so it only stands by. + async fn standby(&mut self) { + let b_to_r = link(self.version, &self.relay_b, &self.subscriber, Some(1_000)).await; + self.links.push(b_to_r); + settle().await; + } + + async fn trigger(&mut self, trigger: Trigger) { + match trigger { + Trigger::Disconnect => abort(self.a_to_r.take().unwrap()), + Trigger::Unannounce => abort(self.p_to_a.take().unwrap()), + Trigger::BetterRoute => { + let direct = link(self.version, &self.publisher, &self.subscriber, None).await; + self.links.push(direct); + } + Trigger::Reconnect => { + abort(self.a_to_r.take().unwrap()); + settle().await; + let a_to_r = link(self.version, &self.relay_a, &self.subscriber, None).await; + self.a_to_r = Some(a_to_r); + } + } + settle().await; + } + + /// A new reader on `R`: a fresh broadcast request and a floorless subscription. + async fn reader(&self) -> Reader { + let remote = moq_clock::timeout(READ_TIMEOUT, self.subscriber.consume().request_broadcast("live", None)) + .await + .expect("request hung") + .expect("request refused"); + let sub = moq_clock::timeout(READ_TIMEOUT, remote.track("catalog.json").unwrap().subscribe(None)) + .await + .expect("subscribe hung") + .expect("subscribe refused"); + Reader::new(sub) + } +} + +/// Reports every frame a subscription delivers, as `group:payload`, or the error that +/// ended it. +struct Reader { + rx: mpsc::UnboundedReceiver>, + /// Owns the subscription, so dropping the reader leaves the track. + task: moq_clock::sim::JoinHandle<()>, +} + +impl Drop for Reader { + fn drop(&mut self) { + self.task.abort(); + } +} + +impl Reader { + fn new(mut sub: track::Subscriber) -> Self { + let (tx, rx) = mpsc::unbounded(); + let task = moq_clock::sim::spawn(async move { + loop { + let mut group = match sub.recv_group().await { + Ok(Some(group)) => group, + Ok(None) => return, + Err(err) => { + let _ = tx.unbounded_send(Err(format!("track: {err}"))); + return; + } + }; + loop { + let sent = match group.read_frame().await { + Ok(Some(frame)) => tx.unbounded_send(Ok(format!( + "{}:{}", + group.sequence, + String::from_utf8_lossy(&frame.payload) + ))), + Ok(None) => break, + Err(err) => { + let _ = tx.unbounded_send(Err(format!("group {} frame: {err}", group.sequence))); + break; + } + }; + if sent.is_err() { + return; + } + } + } + }); + Self { rx, task } + } + + /// The next `n` frames, failing on an error or a hang. + async fn take(&mut self, n: usize, who: &str) -> Vec { + let mut frames = Vec::new(); + while frames.len() < n { + match moq_clock::timeout(READ_TIMEOUT, self.rx.next()).await { + Ok(Some(Ok(frame))) => frames.push(frame), + Ok(Some(Err(err))) => panic!("{who}: failed after {frames:?}: {err}"), + Ok(None) => panic!("{who}: ended after {frames:?}"), + Err(_) => panic!("{who}: hung after {frames:?}"), + } + } + frames + } + + /// Expect the whole open group so far, from the snapshot. + async fn expect_all(&mut self, topology: &Topology, who: &str) { + let expected: Vec = topology.written.iter().map(|payload| format!("0:{payload}")).collect(); + assert_eq!(self.take(expected.len(), who).await, expected, "{who}"); + } + + /// The subscription fails or ends, rather than hanging or delivering more. + async fn expect_end(&mut self, who: &str) { + match moq_clock::timeout(READ_TIMEOUT, self.rx.next()).await { + Ok(None | Some(Err(_))) => {} + Ok(Some(Ok(frame))) => panic!("{who}: delivered {frame:?} past its route"), + Err(_) => panic!("{who}: hung instead of ending"), + } + } + + /// Nothing more has arrived: no duplicate frame or group trails. + fn expect_quiet(&mut self, who: &str) { + if let Ok(extra) = self.rx.try_recv() { + panic!("{who}: trailing delivery {extra:?}"); + } + } +} + +/// Short readers that each read the catalog from its snapshot and leave. +async fn burst(topology: &Topology, label: &str) { + for i in 0..BURST { + let who = format!("{label} reader {i}"); + let mut reader = topology.reader().await; + reader.expect_all(topology, &who).await; + } +} + +async fn handover(version: &str, trigger: Trigger, holder: Holder) { + let version: Version = version.parse().unwrap(); + let mut topology = Topology::new(version).await; + let tag = format!("{version} {trigger:?} {holder:?}"); + + let mut viewer = topology.reader().await; + viewer.expect_all(&topology, &format!("{tag}: viewer")).await; + topology.write("delta1"); + assert_eq!(viewer.take(1, &tag).await, ["0:delta1"], "{tag}: viewer"); + + // Readers come and go inside the linger, re-splicing the catalog. + burst(&topology, &format!("{tag}: before")).await; + + if !matches!(trigger, Trigger::Reconnect) { + topology.standby().await; + } + let mut viewer = match holder { + Holder::Viewer => Some(viewer), + Holder::Parked => { + drop(viewer); + settle().await; + None + } + }; + + topology.trigger(trigger).await; + if let (Trigger::Reconnect, Some(mut ended)) = (trigger, viewer.take()) { + ended.expect_end(&format!("{tag}: viewer")).await; + } + topology.write("delta2"); + + if let Some(viewer) = &mut viewer { + assert_eq!(viewer.take(1, &tag).await, ["0:delta2"], "{tag}: viewer continues"); + } + + // A fresh reader after the change starts from the snapshot. + let mut fresh = topology.reader().await; + fresh.expect_all(&topology, &format!("{tag}: fresh")).await; + + // Short readers again, then one that leaves and returns after the next delta. + burst(&topology, &format!("{tag}: after")).await; + drop(fresh); + settle().await; + topology.write("delta3"); + if let Some(viewer) = &mut viewer { + assert_eq!(viewer.take(1, &tag).await, ["0:delta3"], "{tag}: viewer continues"); + } + // Returning well past the relays' idle linger, so no copy of the head is warm. + moq_clock::sleep(RETURN_AFTER).await; + let mut returning = topology.reader().await; + returning.expect_all(&topology, &format!("{tag}: returning")).await; + + // Everyone still reading follows the live edge, exactly once. + topology.write("delta4"); + assert_eq!(returning.take(1, &tag).await, ["0:delta4"], "{tag}: returning"); + if let Some(viewer) = &mut viewer { + assert_eq!(viewer.take(1, &tag).await, ["0:delta4"], "{tag}: viewer"); + } + settle().await; + returning.expect_quiet(&format!("{tag}: returning")); + if let Some(viewer) = &mut viewer { + viewer.expect_quiet(&format!("{tag}: viewer")); + } +} + +/// What happens on `R` before a reader joins, long after, with nothing in between. +#[derive(Clone, Copy, Debug)] +enum Before { + /// A viewer stays while a direct route beats `A` mid-group. + RouteChange, + /// A reader leaves, so the catalog parks, and another resumes it. + Resume, +} + +/// The minimal sequences the release-line splice left a late reader stuck on: it +/// subscribes, and nothing ever arrives. +async fn late_reader(version: &str, before: Before) { + let version: Version = version.parse().unwrap(); + let mut topology = Topology::new(version).await; + let tag = format!("{version} {before:?}"); + + let mut first = topology.reader().await; + first.expect_all(&topology, &format!("{tag}: first")).await; + let _holder = match before { + Before::RouteChange => { + topology.trigger(Trigger::BetterRoute).await; + first + } + Before::Resume => { + settle().await; + drop(first); + moq_clock::sleep(Duration::from_secs(5)).await; + let mut resumed = topology.reader().await; + resumed.expect_all(&topology, &format!("{tag}: resumed")).await; + resumed + } + }; + + moq_clock::sleep(RETURN_AFTER).await; + let mut late = topology.reader().await; + late.expect_all(&topology, &format!("{tag}: late")).await; +} + +/// A short reader fetches the open group just after the last subscriber left, while +/// the newest delta is still on its way: the fetch must deliver it, not stop at the +/// frames the departing subscription had cached. +async fn fetch_after_leave(version: &str, pause: Duration) { + let version: Version = version.parse().unwrap(); + let mut topology = Topology::new(version).await; + let tag = format!("{version} {pause:?}"); + + let mut reader = topology.reader().await; + reader.expect_all(&topology, &format!("{tag}: reader")).await; + topology.write("delta1"); + drop(reader); + moq_clock::sleep(pause).await; + + let remote = topology + .subscriber + .consume() + .request_broadcast("live", None) + .await + .unwrap(); + let fetch = remote.track("catalog.json").unwrap().fetch_group(0, None); + let mut group = moq_clock::timeout(READ_TIMEOUT, fetch) + .await + .unwrap_or_else(|_| panic!("{tag}: fetch hung")) + .unwrap_or_else(|err| panic!("{tag}: fetch refused: {err}")); + let mut got = Vec::new(); + while got.len() < topology.written.len() { + match moq_clock::timeout(READ_TIMEOUT, group.read_frame()).await { + Ok(Ok(Some(frame))) => got.push(String::from_utf8_lossy(&frame.payload).to_string()), + Ok(Ok(None)) => panic!("{tag}: fetch ended after {got:?}"), + Ok(Err(err)) => panic!("{tag}: fetch failed after {got:?}: {err}"), + Err(_) => panic!("{tag}: fetch hung after {got:?}"), + } + } + assert_eq!(got, topology.written, "{tag}"); +} + +macro_rules! handover_tests { + ($($name:ident: $version:literal,)*) => { + $( + mod $name { + use super::*; + + async fn run(trigger: Trigger, holder: Holder) { + moq_clock::timeout(TEST_TIMEOUT, handover($version, trigger, holder)) + .await + .expect("timed out"); + } + + #[moq_clock::test] + async fn disconnect_with_viewer() { + run(Trigger::Disconnect, Holder::Viewer).await; + } + + #[moq_clock::test] + async fn disconnect_parked() { + run(Trigger::Disconnect, Holder::Parked).await; + } + + #[moq_clock::test] + async fn unannounce_with_viewer() { + run(Trigger::Unannounce, Holder::Viewer).await; + } + + #[moq_clock::test] + async fn unannounce_parked() { + run(Trigger::Unannounce, Holder::Parked).await; + } + + #[moq_clock::test] + async fn better_route_with_viewer() { + run(Trigger::BetterRoute, Holder::Viewer).await; + } + + #[moq_clock::test] + async fn better_route_parked() { + run(Trigger::BetterRoute, Holder::Parked).await; + } + + #[moq_clock::test] + async fn reconnect_with_viewer() { + run(Trigger::Reconnect, Holder::Viewer).await; + } + + #[moq_clock::test] + async fn reconnect_parked() { + run(Trigger::Reconnect, Holder::Parked).await; + } + + #[moq_clock::test] + async fn late_reader_after_route_change() { + moq_clock::timeout(TEST_TIMEOUT, late_reader($version, Before::RouteChange)) + .await + .expect("timed out"); + } + + #[moq_clock::test] + async fn late_reader_after_resume() { + moq_clock::timeout(TEST_TIMEOUT, late_reader($version, Before::Resume)) + .await + .expect("timed out"); + } + } + )* + }; +} + +// Only an epoch lets `R` resume onto another route; older wires end the subscription +// with its route instead (see `route_change`). +handover_tests! { + lite_07: "moq-lite-07-wip", +} + +macro_rules! fetch_tests { + ($($name:ident: $version:literal,)*) => { + $( + mod $name { + use super::*; + + #[moq_clock::test] + async fn fetch_right_after_leave() { + moq_clock::timeout(TEST_TIMEOUT, fetch_after_leave($version, Duration::ZERO)) + .await + .expect("timed out"); + } + + #[moq_clock::test] + async fn fetch_after_leave_settles() { + moq_clock::timeout(TEST_TIMEOUT, fetch_after_leave($version, Duration::from_millis(500))) + .await + .expect("timed out"); + } + } + )* + }; +} + +// A fetch needs no epoch. moq-transport is left out: its copy goes idle on the leave +// and still answers from the open group it can no longer feed. +fetch_tests! { + fetch_lite_06: "moq-lite-06", + fetch_lite_07: "moq-lite-07-wip", +} diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index 458ccd8e1c9..13a819d8b2e 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -11,6 +11,8 @@ mod announce_to_serve; mod auth; mod broadcast_close; mod catalog_acquisition; +mod catalog_churn; +mod catalog_handover; mod datagram; mod dial_split_horizon; mod drained_claim;