From b142b6ddc425751ff78fd7db347afb1207ec9392 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 16:56:08 -0700 Subject: [PATCH 1/3] fix(net): a fetch keeps its track read while it reads an open group A fetch answered from the serving copy's cache is no subscription, so the last subscriber leaving parked the logical track and dropped the copy that still fed the open group: the fetch read the cached frames and then waited forever. The fetched group now holds the logical track until it is read. Its track therefore ends with the front's verdict when its route leaves, `Unroutable` rather than `Dropped`. Adds catalog regressions over the in-memory transport: an open catalog group through mid-group route changes (disconnect, withdrawal, a better route, a reconnect) with a viewer staying or the track parked, then fresh, returning, and bursts of short readers; late readers after a route change or a park and resume; a fetch right after the last subscriber leaves; and a seeded churn of subscribers, fetches, and routes that shrinks a failing seed to the ops it needs. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/resume.rs | 41 +- rs/moq-net/src/model/track.rs | 4 +- rs/moq-net/tests/integration/auth.rs | 9 +- rs/moq-net/tests/integration/catalog_churn.rs | 460 +++++++++++++++ .../tests/integration/catalog_handover.rs | 522 ++++++++++++++++++ rs/moq-net/tests/integration/main.rs | 2 + 6 files changed, 1023 insertions(+), 15 deletions(-) create mode 100644 rs/moq-net/tests/integration/catalog_churn.rs create mode 100644 rs/moq-net/tests/integration/catalog_handover.rs diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 186c3f0dec6..7ab3f413546 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -174,12 +174,19 @@ impl Consumer { Subscriber { reader, subscription } } - /// Fetch one group from whichever route serves the track. - pub(crate) fn fetch_group(&self, sequence: u64, options: group::Fetch) -> Fetching { + /// Fetch one group from whichever route serves the track. `logical` is the logical + /// track's state, which the fetched group holds while it is read. + pub(crate) fn fetch_group( + &self, + sequence: u64, + options: group::Fetch, + logical: kio::Consumer, + ) -> Fetching { Fetching { route: self.clone(), sequence, options, + logical, pending: None, } } @@ -552,6 +559,7 @@ impl Reader { generation: copy.generation, copy: copy.track.clone(), lease: Some(copy.lease.clone()), + logical: None, fetch: None, }) } @@ -835,6 +843,10 @@ pub(crate) struct Recover { copy: track::Consumer, /// Keeps a replaced copy subscribed while this group is read from it. lease: Option>, + /// A fetch's logical track, held while the group is read. A fetch is no subscription, + /// so without it the last subscriber leaving parks the track, dropping the copy that + /// still feeds an open group, and the rest of the group never comes. + logical: Option>, /// A fetch of the rest of the group from the serving route, by its generation. fetch: Option<(u64, Fetch)>, } @@ -854,6 +866,7 @@ impl Clone for Recover { generation: self.generation, copy: self.copy.clone(), lease: self.lease.clone(), + logical: self.logical.clone(), // A clone reads in parallel, so it asks for itself. fetch: None, } @@ -1053,6 +1066,7 @@ pub(crate) struct Fetching { route: Consumer, sequence: u64, options: group::Fetch, + logical: kio::Consumer, /// The fetch in flight, and the route generation it went to. pending: Option<(u64, kio::Pending)>, } @@ -1100,6 +1114,7 @@ impl kio::Task for Fetching { generation, copy: serving.clone(), lease: None, + logical: Some(self.logical.clone()), fetch: None, }))), // The route failed: ask the next one, unless none will come. @@ -2065,10 +2080,11 @@ mod test { let a = copy(); let dynamic = a.dynamic(); routes.serve(a.consume()); - let mut fetching = routes.consume().fetch_group(0, group::Fetch::default()); + let logical = logical(&routes); + let mut fetching = logical.consume().fetch_group(0, None); let wake = Arc::new(Wake::default()); let waiter = kio::Waiter::new(wake.clone().into()); - assert!(kio::Task::poll(&mut fetching, &waiter).is_pending()); + assert!(kio::Task::poll(&mut *fetching, &waiter).is_pending()); let _request = dynamic.requested_group().now_or_never().unwrap().unwrap(); wake.0.store(false, std::sync::atomic::Ordering::Relaxed); let b = copy(); @@ -2077,7 +2093,7 @@ mod test { group.finish().unwrap(); routes.serve(b.consume()); assert!(wake.0.load(std::sync::atomic::Ordering::Relaxed)); - assert!(matches!(kio::Task::poll(&mut fetching, &waiter), Poll::Ready(Ok(_)))); + assert!(matches!(kio::Task::poll(&mut *fetching, &waiter), Poll::Ready(Ok(_)))); } #[test] @@ -2300,7 +2316,9 @@ mod test { let a = copy(); let _open = a.create_group(group::Info { sequence: 0 }).unwrap(); routes.serve(a.consume()); - let mut reading = kio::Pending::new(routes.consume().fetch_group(0, group::Fetch::default())) + let mut reading = logical(&routes) + .consume() + .fetch_group(0, None) .now_or_never() .unwrap() .unwrap(); @@ -2323,7 +2341,9 @@ mod test { let mut arriving = a.create_group(group::Info { sequence: 0 }).unwrap(); arriving.write_frame(ts(0), b"a".as_ref()).unwrap(); - let mut fetched = kio::Pending::new(routes.consume().fetch_group(0, group::Fetch::default())) + let mut fetched = logical(&routes) + .consume() + .fetch_group(0, None) .now_or_never() .unwrap() .unwrap(); @@ -2514,12 +2534,13 @@ mod test { let a = copy(); let _dynamic = a.dynamic(); routes.serve(a.consume()); - let mut fetching = routes.consume().fetch_group(0, group::Fetch::default()); + let logical = logical(&routes); + let mut fetching = logical.consume().fetch_group(0, None); let waiter = kio::Waiter::noop(); - assert!(kio::Task::poll(&mut fetching, &waiter).is_pending()); + assert!(kio::Task::poll(&mut *fetching, &waiter).is_pending()); routes.end(Err(Error::Unroutable)); assert!(matches!( - kio::Task::poll(&mut fetching, &waiter), + kio::Task::poll(&mut *fetching, &waiter), Poll::Ready(Err(Error::Unroutable)) )); } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index d0335618cff..f12059299c3 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -3203,7 +3203,7 @@ impl Consumer { result: None, hit: None, stats: self.stats.clone(), - resume: Some(Box::new(resume.fetch_group(sequence, options))), + resume: Some(Box::new(resume.fetch_group(sequence, options, state.clone()))), }); } @@ -3604,7 +3604,7 @@ impl kio::Task for Fetching { let options = group::Fetch::default() .with_priority(self.priority) .with_frame_start(self.frame_start); - self.resume = Some(Box::new(routes.fetch_group(self.sequence, options))); + self.resume = Some(Box::new(routes.fetch_group(self.sequence, options, self.state.clone()))); } if let Some(resume) = &mut self.resume { let group = ready!(kio::Task::poll(&mut **resume, waiter))?; diff --git a/rs/moq-net/tests/integration/auth.rs b/rs/moq-net/tests/integration/auth.rs index abe9cd518a2..2010d8bd6de 100644 --- a/rs/moq-net/tests/integration/auth.rs +++ b/rs/moq-net/tests/integration/auth.rs @@ -1347,10 +1347,13 @@ async fn a_narrowing_resets_a_fetch_in_flight(version: &'static str) { Err(err) => break err, } }; - // The narrowing also retracts the route, and a fetch-only track whose route - // leaves ends `Dropped` locally, so the reader may see either. The wire carries + // The narrowing also retracts the route, and the fetch's track ends `Unroutable` + // locally when its route leaves, so the reader may see either. The wire carries // the revocation regardless. - assert!(unauthorized(&err) || matches!(err, moq_net::Error::Dropped), "{err:?}"); + assert!( + unauthorized(&err) || matches!(err, moq_net::Error::Unroutable), + "{err:?}" + ); let resets = pair.server_transport.resets(); assert!(resets.contains(&StreamError::Unauthorized.to_code()), "{resets:x?}"); drop(group); diff --git a/rs/moq-net/tests/integration/catalog_churn.rs b/rs/moq-net/tests/integration/catalog_churn.rs new file mode 100644 index 00000000000..5d6c48ec738 --- /dev/null +++ b/rs/moq-net/tests/integration/catalog_churn.rs @@ -0,0 +1,460 @@ +#![cfg(feature = "async")] + +//! Seeded churn on a catalog: one open group at a time, a snapshot then deltas, read +//! on `R` by subscribers and short fetches that come and go while `R`'s routes to the +//! publisher `P` (through `A`, through `B`, and direct) appear and disappear. +//! +//! A reader may fail loudly when its route goes (its client asks again), but it must +//! never wait forever: every subscriber gets each group it reads from the snapshot +//! on, and every fetch gets the whole group written so far. A failing seed is +//! shrunk to the ops it needs, which the report names. + +use crate::support; + +use std::time::Duration; + +use futures::{StreamExt, channel::mpsc}; +use moq_net::{Error, Hop, Timestamp, Version, origin, track}; +use support::harness::{MockConnectOptions, MockPair, connect_mock}; + +const SEEDS: u64 = 32; +const STEPS: usize = 60; + +/// How long any one request or read may take, in virtual time. +const READ_TIMEOUT: Duration = Duration::from_secs(10); + +/// How long a reader short of its frames at a quiet point gets to prove it is only late. +const STUCK_AFTER: Duration = Duration::from_secs(120); + +fn produce_origin(hop: u64) -> origin::Producer { + let (producer, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); + support::harness::spawn(driver); + producer +} + +async fn link(version: Version, from: &origin::Producer, to: &origin::Producer) -> MockPair { + let mut options = MockConnectOptions::new(version); + options.server_publish = Some(from.consume()); + options.client_subscribe = Some(to.clone()); + connect_mock(options).await +} + +/// xorshift64, so a seed replays the same ops anywhere. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } +} + +#[derive(Clone, Copy, Debug)] +enum Op { + /// Append a delta to the open group. + Write, + /// Finish the group and open the next with a new snapshot. + NewGroup, + /// A subscriber joins. + Join, + /// A short reader fetches the latest group, as an HTTP request does. + Fetch, + /// A subscriber leaves, picked by this value. + Leave(u64), + /// Connect the route, or drop it if another route stays up. + ToggleA, + ToggleB, + ToggleDirect, + Sleep(Duration), + /// Settle, then require every subscriber to hold what it should. + Check, +} + +fn generate(seed: u64) -> Vec { + let mut rng = Rng(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1); + let mut ops = Vec::new(); + for _ in 0..STEPS { + ops.push(match rng.below(100) { + 0..25 => Op::Write, + 25..30 => Op::NewGroup, + 30..45 => Op::Join, + 45..55 => Op::Fetch, + 55..70 => Op::Leave(rng.next()), + 70..80 => Op::ToggleA, + 80..90 => Op::ToggleB, + _ => Op::ToggleDirect, + }); + // Mostly back to back, sometimes past the relays' 30 s idle linger. + let pause = match rng.below(6) { + 0 | 1 => continue, + 2 => Duration::from_millis(50), + 3 => Duration::from_millis(500), + 4 => Duration::from_secs(5), + _ => Duration::from_secs(31), + }; + ops.push(Op::Sleep(pause)); + if pause >= Duration::from_millis(500) { + ops.push(Op::Check); + } + } + ops.push(Op::Check); + ops +} + +/// How a seed failed: `stuck` (a reader never got what it should have), `late` (only +/// after a quiet point), or `wrong` (it got something it should not have). +struct Failure { + kind: &'static str, + detail: String, +} + +fn stuck(detail: String) -> Failure { + Failure { kind: "stuck", detail } +} + +fn late(detail: String) -> Failure { + Failure { kind: "late", detail } +} + +fn wrong(detail: String) -> Failure { + Failure { kind: "wrong", detail } +} + +/// A subscriber's frames as `group:payload`, or the error that ended it. +struct Reader { + id: usize, + rx: mpsc::UnboundedReceiver>, + /// Every frame written since it joined, from the snapshot of the group then open. + expected: Vec, + seen: Vec, + /// Owns the subscription, so dropping the reader leaves the track. + task: moq_net_sim::JoinHandle<()>, +} + +impl Drop for Reader { + fn drop(&mut self) { + self.task.abort(); + } +} + +impl Reader { + fn new(id: usize, mut sub: track::Subscriber, expected: Vec) -> Self { + let (tx, rx) = mpsc::unbounded(); + let task = moq_net_sim::spawn(async move { + loop { + let mut group = match sub.recv_group().await { + Ok(Some(group)) => group, + Ok(None) => return, + Err(err) => { + let _ = tx.unbounded_send(Err(err)); + return; + } + }; + loop { + let frame = match group.read_frame().await { + Ok(Some(frame)) => frame, + Ok(None) => break, + Err(err) => { + let _ = tx.unbounded_send(Err(err)); + return; + } + }; + let frame = format!("{}:{}", group.sequence, String::from_utf8_lossy(&frame.payload)); + if tx.unbounded_send(Ok(frame)).is_err() { + return; + } + } + } + }); + Self { + id, + rx, + expected, + seen: Vec::new(), + task, + } + } + + /// Take what arrived; `false` once the subscription failed or ended. + fn drain(&mut self) -> bool { + loop { + match self.rx.try_recv() { + Ok(Ok(frame)) => self.seen.push(frame), + Ok(Err(_)) => return false, + Err(mpsc::TryRecvError::Empty) => return true, + Err(mpsc::TryRecvError::Closed) => return false, + } + } + } + + /// Compare what it saw with what it should see. A joiner may first get groups older + /// than the newest at its join (`R` had not seen the newer one yet), and may skip + /// whole groups superseded before it resolved, but each group it gets starts at the + /// snapshot and continues in order. + fn judge(&self) -> Verdict { + let group = |frame: &String| frame.split(':').next().unwrap().parse::().unwrap(); + let joined = self.expected.first().map(group); + let mut seen = self.seen.as_slice(); + while let (Some(first), Some(joined)) = (seen.first(), joined) { + let sequence = group(first); + if sequence >= joined { + break; + } + if !first.ends_with(":snapshot") { + return Verdict::Wrong; + } + let older = seen.iter().take_while(|frame| group(frame) == sequence).count(); + seen = &seen[older..]; + } + let mut expected = self.expected.as_slice(); + if let Some(first) = seen.first() { + let skipped = expected.iter().take_while(|frame| group(frame) < group(first)).count(); + expected = &expected[skipped..]; + } + match (seen == expected, expected.starts_with(seen)) { + (true, _) => Verdict::Done, + (false, true) => Verdict::Short, + (false, false) => Verdict::Wrong, + } + } + + fn report(&self) -> String { + format!("r{} saw {:?}, expected {:?}", self.id, self.seen, self.expected) + } +} + +#[derive(PartialEq)] +enum Verdict { + Done, + Short, + Wrong, +} + +struct Topology { + version: Version, + publisher: origin::Producer, + relay_a: origin::Producer, + relay_b: origin::Producer, + subscriber: origin::Producer, + routes: [Option; 3], + _upstream: Vec, +} + +impl Topology { + /// Connect route `index` (`A`, `B`, direct), or drop it if another stays up. + async fn toggle(&mut self, index: usize) { + if let Some(pair) = self.routes[index].take() { + match self.routes.iter().any(Option::is_some) { + true => { + pair.server.abort(Error::Cancel); + pair.client.abort(Error::Cancel); + } + false => self.routes[index] = Some(pair), + } + return; + } + let from = [&self.relay_a, &self.relay_b, &self.publisher][index]; + self.routes[index] = Some(link(self.version, from, &self.subscriber).await); + } + + /// A fresh request for the broadcast; `None` when refused, as with no route right now. + async fn request(&self, step: usize) -> Result, Failure> { + let request = self.subscriber.consume().request_broadcast("live", None); + match moq_net_sim::timeout(READ_TIMEOUT, request).await { + Ok(result) => Ok(result.ok()), + Err(_) => Err(stuck(format!("{step}: request hung"))), + } + } +} + +async fn execute(version: Version, ops: &[Op]) -> Result<(), Failure> { + let publisher = produce_origin(1); + let relay_a = produce_origin(2); + let relay_b = produce_origin(3); + let subscriber = produce_origin(4); + + let broadcast = publisher.create_broadcast("live").unwrap(); + let info = track::Info::default().with_timescale(moq_net::Timescale::MILLI); + let track = broadcast.create_track("catalog.json", info).unwrap(); + broadcast + .announce(origin::Route::default().with_epoch(moq_net::Epoch::mint())) + .unwrap(); + + let upstream = vec![ + link(version, &publisher, &relay_a).await, + link(version, &publisher, &relay_b).await, + ]; + let a_to_r = link(version, &relay_a, &subscriber).await; + subscriber.consume().routed("live").await.unwrap(); + let mut topology = Topology { + version, + publisher, + relay_a, + relay_b, + subscriber, + routes: [Some(a_to_r), None, None], + _upstream: upstream, + }; + + let mut group = track.append_group().unwrap(); + // The open group as written so far, as `group:payload`. + let mut written: Vec = Vec::new(); + let mut readers: Vec = Vec::new(); + let mut joined = 0; + let mut clock = 0; + let mut write = + |group: &mut moq_net::group::Producer, written: &mut Vec, readers: &mut [Reader], payload: &str| { + clock += 10; + group + .write_frame(Timestamp::from_millis(clock).unwrap(), payload.as_bytes()) + .unwrap(); + let frame = format!("{}:{payload}", group.sequence); + for reader in readers { + reader.expected.push(frame.clone()); + } + written.push(frame); + }; + write(&mut group, &mut written, &mut readers, "snapshot"); + + for (step, op) in ops.iter().enumerate() { + match *op { + Op::Write => write(&mut group, &mut written, &mut readers, &format!("d{step}")), + Op::NewGroup => { + group.finish().unwrap(); + group = track.append_group().unwrap(); + written.clear(); + write(&mut group, &mut written, &mut readers, "snapshot"); + // Let it reach `R`, so the newest group at the next join is unambiguous. + moq_net_sim::sleep(Duration::from_millis(500)).await; + } + Op::Join => { + let Some(remote) = topology.request(step).await? else { + continue; + }; + let subscribe = remote.track("catalog.json").unwrap().subscribe(None); + match moq_net_sim::timeout(READ_TIMEOUT, subscribe).await { + Ok(Ok(sub)) => readers.push(Reader::new(joined, sub, written.clone())), + Ok(Err(_)) => {} + Err(_) => return Err(stuck(format!("{step}: subscribe hung"))), + } + joined += 1; + } + Op::Fetch => { + let Some(remote) = topology.request(step).await? else { + continue; + }; + let fetch = remote.track("catalog.json").unwrap().fetch_group(group.sequence, None); + let mut fetched = match moq_net_sim::timeout(READ_TIMEOUT, fetch).await { + Ok(Ok(fetched)) => fetched, + Ok(Err(_)) => continue, + Err(_) => return Err(stuck(format!("{step}: fetch hung"))), + }; + let mut got = Vec::new(); + while got.len() < written.len() { + match moq_net_sim::timeout(READ_TIMEOUT, fetched.read_frame()).await { + Ok(Ok(Some(frame))) => got.push(format!( + "{}:{}", + fetched.sequence, + String::from_utf8_lossy(&frame.payload) + )), + Ok(Ok(None)) => return Err(wrong(format!("{step}: fetch ended at {got:?}"))), + Ok(Err(_)) => break, + Err(_) => return Err(stuck(format!("{step}: fetched {got:?}, then hung"))), + } + } + if !written.starts_with(&got) { + return Err(wrong(format!("{step}: fetched {got:?}, wrote {written:?}"))); + } + } + Op::Leave(pick) => { + if !readers.is_empty() { + readers.remove((pick % readers.len() as u64) as usize); + } + } + Op::ToggleA => topology.toggle(0).await, + Op::ToggleB => topology.toggle(1).await, + Op::ToggleDirect => topology.toggle(2).await, + Op::Sleep(pause) => moq_net_sim::sleep(pause).await, + Op::Check => { + moq_net_sim::sleep(Duration::from_millis(500)).await; + let mut failed = Vec::new(); + for (index, reader) in readers.iter_mut().enumerate() { + let open = reader.drain(); + match reader.judge() { + Verdict::Wrong => return Err(wrong(format!("{step}: {}", reader.report()))), + // Failed loudly: its client asks again. + _ if !open => failed.push(index), + Verdict::Done => {} + Verdict::Short => { + let _ = moq_net_sim::timeout(STUCK_AFTER, async { + while reader.judge() == Verdict::Short { + match reader.rx.next().await { + Some(Ok(frame)) => reader.seen.push(frame), + _ => break, + } + } + }) + .await; + return Err(match reader.judge() { + Verdict::Done => late(format!("{step}: r{}", reader.id)), + _ => stuck(format!("{step}: {}", reader.report())), + }); + } + } + } + for index in failed.into_iter().rev() { + readers.remove(index); + } + } + } + } + Ok(()) +} + +/// Drop ops one at a time while the same kind of failure persists. +async fn shrink(version: Version, mut ops: Vec, kind: &str) -> Vec { + let mut index = 0; + while index < ops.len() { + let mut candidate = ops.clone(); + candidate.remove(index); + match execute(version, &candidate).await { + Err(failure) if failure.kind == kind => ops = candidate, + _ => index += 1, + } + } + ops +} + +async fn churn(version: &str) { + let version: Version = version.parse().unwrap(); + let mut failures = Vec::new(); + for seed in 0..SEEDS { + let ops = generate(seed); + if let Err(failure) = execute(version, &ops).await { + let small = shrink(version, ops, failure.kind).await; + failures.push(format!( + "seed {seed}: {} {}\n shrunk to {small:?}", + failure.kind, failure.detail + )); + } + } + assert!(failures.is_empty(), "{version}:\n{}", failures.join("\n")); +} + +/// Routes carry the publisher's epoch, so subscriptions resume across them. +#[moq_net_sim::test] +async fn catalog_churn_lite_07() { + churn("moq-lite-07-wip").await; +} + +/// No epoch on the wire: a route change ends subscriptions, which must fail loudly. +#[moq_net_sim::test] +async fn catalog_churn_lite_06() { + churn("moq-lite-06").await; +} diff --git a/rs/moq-net/tests/integration/catalog_handover.rs b/rs/moq-net/tests/integration/catalog_handover.rs new file mode 100644 index 00000000000..fd15d916f42 --- /dev/null +++ b/rs/moq-net/tests/integration/catalog_handover.rs @@ -0,0 +1,522 @@ +#![cfg(feature = "async")] + +//! A catalog's one open group (a snapshot at frame 0, deltas after) survives its +//! route changing in the middle of that group, end to end over real sessions. +//! +//! A publisher `P` is pulled by two relays `A` and `B` carrying its epoch, and `R` +//! reads through `A` until the route changes mid-group. The reader already on `R` +//! continues with the next delta, and every reader after the change (a fresh one, a +//! returning one, and a burst of short ones) still starts from the snapshot. The +//! replacement's copy of the group was opened mid-group for the continuing reader, +//! so a reader that starts at frame 0 must not be refused as lagged and left with +//! no catalog. + +use crate::support; + +use std::time::Duration; + +use futures::{StreamExt, channel::mpsc}; +use moq_net::{Error, Hop, Timestamp, Version, broadcast, origin, track}; +use support::harness::{MockConnectOptions, MockPair, connect_mock}; + +const TEST_TIMEOUT: Duration = Duration::from_secs(300); + +/// How long any one read may take, in virtual time. +const READ_TIMEOUT: Duration = Duration::from_secs(10); + +/// Short readers in each burst, re-reading the catalog and leaving. +const BURST: usize = 4; + +/// When the returning reader comes back: twice a relay's 30 s idle linger. +const RETURN_AFTER: Duration = Duration::from_secs(62); + +fn timed() -> track::Info { + track::Info::default().with_timescale(moq_net::Timescale::MILLI) +} + +fn produce_origin(hop: u64) -> origin::Producer { + let (producer, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); + support::harness::spawn(driver); + producer +} + +async fn link(version: Version, from: &origin::Producer, to: &origin::Producer, cost: Option) -> MockPair { + let mut options = MockConnectOptions::new(version); + options.server_publish = Some(from.consume()); + options.client_subscribe = Some(to.clone()); + options.cost = cost; + connect_mock(options).await +} + +fn abort(pair: MockPair) { + pair.server.abort(Error::Cancel); + pair.client.abort(Error::Cancel); +} + +/// Let every task settle. Time is paused, so this returns once the runtime is idle. +async fn settle() { + moq_net_sim::sleep(Duration::from_millis(500)).await; +} + +/// How the serving route changes. +#[derive(Clone, Copy, Debug)] +enum Trigger { + /// The session `A -> R` dies; `B` takes over. + Disconnect, + /// `A` loses `P` and withdraws the broadcast while its session to `R` stays up. + Unannounce, + /// `R` gains a direct route to `P` that beats `A`. + BetterRoute, + /// The session `A -> R` dies with no standby, then reconnects. The broadcast closes + /// with its last route, so a viewer's subscription ends rather than resuming. + Reconnect, +} + +/// Who holds the catalog on `R` while the route changes. +#[derive(Clone, Copy, Debug)] +enum Holder { + /// A viewer stays subscribed and must see the next delta exactly once. + Viewer, + /// Nobody: the last reader left, so the change lands on a parked track. + Parked, +} + +struct Topology { + version: Version, + publisher: origin::Producer, + relay_a: origin::Producer, + relay_b: origin::Producer, + subscriber: origin::Producer, + group: moq_net::group::Producer, + p_to_a: Option, + a_to_r: Option, + links: Vec, + _broadcast: broadcast::Producer, + _track: track::Producer, + /// Every payload written to the open group so far. + written: Vec, +} + +impl Topology { + async fn new(version: Version) -> Self { + let publisher = produce_origin(1); + let relay_a = produce_origin(2); + let relay_b = produce_origin(3); + let subscriber = produce_origin(4); + + let broadcast = publisher.create_broadcast("live").unwrap(); + let track = broadcast.create_track("catalog.json", timed()).unwrap(); + broadcast + .announce(origin::Route::default().with_epoch(moq_net::Epoch::mint())) + .unwrap(); + let group = track.append_group().unwrap(); + + let p_to_a = link(version, &publisher, &relay_a, None).await; + let p_to_b = link(version, &publisher, &relay_b, None).await; + let a_to_r = link(version, &relay_a, &subscriber, None).await; + subscriber.consume().routed("live").await.unwrap(); + + let mut topology = Self { + version, + publisher, + relay_a, + relay_b, + subscriber, + group, + p_to_a: Some(p_to_a), + a_to_r: Some(a_to_r), + links: vec![p_to_b], + _broadcast: broadcast, + _track: track, + written: Vec::new(), + }; + topology.write("snapshot"); + topology + } + + fn write(&mut self, payload: &str) { + let at = Timestamp::from_millis(self.written.len() as u64 * 10).unwrap(); + self.group.write_frame(at, payload.as_bytes()).unwrap(); + self.written.push(payload.to_string()); + } + + /// Connect `B -> R`, priced worse than `A` so it only stands by. + async fn standby(&mut self) { + let b_to_r = link(self.version, &self.relay_b, &self.subscriber, Some(1_000)).await; + self.links.push(b_to_r); + settle().await; + } + + async fn trigger(&mut self, trigger: Trigger) { + match trigger { + Trigger::Disconnect => abort(self.a_to_r.take().unwrap()), + Trigger::Unannounce => abort(self.p_to_a.take().unwrap()), + Trigger::BetterRoute => { + let direct = link(self.version, &self.publisher, &self.subscriber, None).await; + self.links.push(direct); + } + Trigger::Reconnect => { + abort(self.a_to_r.take().unwrap()); + settle().await; + let a_to_r = link(self.version, &self.relay_a, &self.subscriber, None).await; + self.a_to_r = Some(a_to_r); + } + } + settle().await; + } + + /// A new reader on `R`: a fresh broadcast request and a floorless subscription. + async fn reader(&self) -> Reader { + let remote = moq_net_sim::timeout(READ_TIMEOUT, self.subscriber.consume().request_broadcast("live", None)) + .await + .expect("request hung") + .expect("request refused"); + let sub = moq_net_sim::timeout(READ_TIMEOUT, remote.track("catalog.json").unwrap().subscribe(None)) + .await + .expect("subscribe hung") + .expect("subscribe refused"); + Reader::new(sub) + } +} + +/// Reports every frame a subscription delivers, as `group:payload`, or the error that +/// ended it. +struct Reader { + rx: mpsc::UnboundedReceiver>, + /// Owns the subscription, so dropping the reader leaves the track. + task: moq_net_sim::JoinHandle<()>, +} + +impl Drop for Reader { + fn drop(&mut self) { + self.task.abort(); + } +} + +impl Reader { + fn new(mut sub: track::Subscriber) -> Self { + let (tx, rx) = mpsc::unbounded(); + let task = moq_net_sim::spawn(async move { + loop { + let mut group = match sub.recv_group().await { + Ok(Some(group)) => group, + Ok(None) => return, + Err(err) => { + let _ = tx.unbounded_send(Err(format!("track: {err}"))); + return; + } + }; + loop { + let sent = match group.read_frame().await { + Ok(Some(frame)) => tx.unbounded_send(Ok(format!( + "{}:{}", + group.sequence, + String::from_utf8_lossy(&frame.payload) + ))), + Ok(None) => break, + Err(err) => { + let _ = tx.unbounded_send(Err(format!("group {} frame: {err}", group.sequence))); + break; + } + }; + if sent.is_err() { + return; + } + } + } + }); + Self { rx, task } + } + + /// The next `n` frames, failing on an error or a hang. + async fn take(&mut self, n: usize, who: &str) -> Vec { + let mut frames = Vec::new(); + while frames.len() < n { + match moq_net_sim::timeout(READ_TIMEOUT, self.rx.next()).await { + Ok(Some(Ok(frame))) => frames.push(frame), + Ok(Some(Err(err))) => panic!("{who}: failed after {frames:?}: {err}"), + Ok(None) => panic!("{who}: ended after {frames:?}"), + Err(_) => panic!("{who}: hung after {frames:?}"), + } + } + frames + } + + /// Expect the whole open group so far, from the snapshot. + async fn expect_all(&mut self, topology: &Topology, who: &str) { + let expected: Vec = topology.written.iter().map(|payload| format!("0:{payload}")).collect(); + assert_eq!(self.take(expected.len(), who).await, expected, "{who}"); + } + + /// The subscription fails or ends, rather than hanging or delivering more. + async fn expect_end(&mut self, who: &str) { + match moq_net_sim::timeout(READ_TIMEOUT, self.rx.next()).await { + Ok(None | Some(Err(_))) => {} + Ok(Some(Ok(frame))) => panic!("{who}: delivered {frame:?} past its route"), + Err(_) => panic!("{who}: hung instead of ending"), + } + } + + /// Nothing more has arrived: no duplicate frame or group trails. + fn expect_quiet(&mut self, who: &str) { + if let Ok(extra) = self.rx.try_recv() { + panic!("{who}: trailing delivery {extra:?}"); + } + } +} + +/// Short readers that each read the catalog from its snapshot and leave. +async fn burst(topology: &Topology, label: &str) { + for i in 0..BURST { + let who = format!("{label} reader {i}"); + let mut reader = topology.reader().await; + reader.expect_all(topology, &who).await; + } +} + +async fn handover(version: &str, trigger: Trigger, holder: Holder) { + let version: Version = version.parse().unwrap(); + let mut topology = Topology::new(version).await; + let tag = format!("{version} {trigger:?} {holder:?}"); + + let mut viewer = topology.reader().await; + viewer.expect_all(&topology, &format!("{tag}: viewer")).await; + topology.write("delta1"); + assert_eq!(viewer.take(1, &tag).await, ["0:delta1"], "{tag}: viewer"); + + // Readers come and go inside the linger, re-splicing the catalog. + burst(&topology, &format!("{tag}: before")).await; + + if !matches!(trigger, Trigger::Reconnect) { + topology.standby().await; + } + let mut viewer = match holder { + Holder::Viewer => Some(viewer), + Holder::Parked => { + drop(viewer); + settle().await; + None + } + }; + + topology.trigger(trigger).await; + if let (Trigger::Reconnect, Some(mut ended)) = (trigger, viewer.take()) { + ended.expect_end(&format!("{tag}: viewer")).await; + } + topology.write("delta2"); + + if let Some(viewer) = &mut viewer { + assert_eq!(viewer.take(1, &tag).await, ["0:delta2"], "{tag}: viewer continues"); + } + + // A fresh reader after the change starts from the snapshot. + let mut fresh = topology.reader().await; + fresh.expect_all(&topology, &format!("{tag}: fresh")).await; + + // Short readers again, then one that leaves and returns after the next delta. + burst(&topology, &format!("{tag}: after")).await; + drop(fresh); + settle().await; + topology.write("delta3"); + if let Some(viewer) = &mut viewer { + assert_eq!(viewer.take(1, &tag).await, ["0:delta3"], "{tag}: viewer continues"); + } + // Returning well past the relays' idle linger, so no copy of the head is warm. + moq_net_sim::sleep(RETURN_AFTER).await; + let mut returning = topology.reader().await; + returning.expect_all(&topology, &format!("{tag}: returning")).await; + + // Everyone still reading follows the live edge, exactly once. + topology.write("delta4"); + assert_eq!(returning.take(1, &tag).await, ["0:delta4"], "{tag}: returning"); + if let Some(viewer) = &mut viewer { + assert_eq!(viewer.take(1, &tag).await, ["0:delta4"], "{tag}: viewer"); + } + settle().await; + returning.expect_quiet(&format!("{tag}: returning")); + if let Some(viewer) = &mut viewer { + viewer.expect_quiet(&format!("{tag}: viewer")); + } +} + +/// What happens on `R` before a reader joins, long after, with nothing in between. +#[derive(Clone, Copy, Debug)] +enum Before { + /// A viewer stays while a direct route beats `A` mid-group. + RouteChange, + /// A reader leaves, so the catalog parks, and another resumes it. + Resume, +} + +/// The minimal sequences the release-line splice left a late reader stuck on: it +/// subscribes, and nothing ever arrives. +async fn late_reader(version: &str, before: Before) { + let version: Version = version.parse().unwrap(); + let mut topology = Topology::new(version).await; + let tag = format!("{version} {before:?}"); + + let mut first = topology.reader().await; + first.expect_all(&topology, &format!("{tag}: first")).await; + let _holder = match before { + Before::RouteChange => { + topology.trigger(Trigger::BetterRoute).await; + first + } + Before::Resume => { + settle().await; + drop(first); + moq_net_sim::sleep(Duration::from_secs(5)).await; + let mut resumed = topology.reader().await; + resumed.expect_all(&topology, &format!("{tag}: resumed")).await; + resumed + } + }; + + moq_net_sim::sleep(RETURN_AFTER).await; + let mut late = topology.reader().await; + late.expect_all(&topology, &format!("{tag}: late")).await; +} + +/// A short reader fetches the open group just after the last subscriber left, while +/// the newest delta is still on its way: the fetch must deliver it, not stop at the +/// frames the departing subscription had cached. +async fn fetch_after_leave(version: &str, pause: Duration) { + let version: Version = version.parse().unwrap(); + let mut topology = Topology::new(version).await; + let tag = format!("{version} {pause:?}"); + + let mut reader = topology.reader().await; + reader.expect_all(&topology, &format!("{tag}: reader")).await; + topology.write("delta1"); + drop(reader); + moq_net_sim::sleep(pause).await; + + let remote = topology + .subscriber + .consume() + .request_broadcast("live", None) + .await + .unwrap(); + let fetch = remote.track("catalog.json").unwrap().fetch_group(0, None); + let mut group = moq_net_sim::timeout(READ_TIMEOUT, fetch) + .await + .unwrap_or_else(|_| panic!("{tag}: fetch hung")) + .unwrap_or_else(|err| panic!("{tag}: fetch refused: {err}")); + let mut got = Vec::new(); + while got.len() < topology.written.len() { + match moq_net_sim::timeout(READ_TIMEOUT, group.read_frame()).await { + Ok(Ok(Some(frame))) => got.push(String::from_utf8_lossy(&frame.payload).to_string()), + Ok(Ok(None)) => panic!("{tag}: fetch ended after {got:?}"), + Ok(Err(err)) => panic!("{tag}: fetch failed after {got:?}: {err}"), + Err(_) => panic!("{tag}: fetch hung after {got:?}"), + } + } + assert_eq!(got, topology.written, "{tag}"); +} + +macro_rules! handover_tests { + ($($name:ident: $version:literal,)*) => { + $( + mod $name { + use super::*; + + async fn run(trigger: Trigger, holder: Holder) { + moq_net_sim::timeout(TEST_TIMEOUT, handover($version, trigger, holder)) + .await + .expect("timed out"); + } + + #[moq_net_sim::test] + async fn disconnect_with_viewer() { + run(Trigger::Disconnect, Holder::Viewer).await; + } + + #[moq_net_sim::test] + async fn disconnect_parked() { + run(Trigger::Disconnect, Holder::Parked).await; + } + + #[moq_net_sim::test] + async fn unannounce_with_viewer() { + run(Trigger::Unannounce, Holder::Viewer).await; + } + + #[moq_net_sim::test] + async fn unannounce_parked() { + run(Trigger::Unannounce, Holder::Parked).await; + } + + #[moq_net_sim::test] + async fn better_route_with_viewer() { + run(Trigger::BetterRoute, Holder::Viewer).await; + } + + #[moq_net_sim::test] + async fn better_route_parked() { + run(Trigger::BetterRoute, Holder::Parked).await; + } + + #[moq_net_sim::test] + async fn reconnect_with_viewer() { + run(Trigger::Reconnect, Holder::Viewer).await; + } + + #[moq_net_sim::test] + async fn reconnect_parked() { + run(Trigger::Reconnect, Holder::Parked).await; + } + + #[moq_net_sim::test] + async fn late_reader_after_route_change() { + moq_net_sim::timeout(TEST_TIMEOUT, late_reader($version, Before::RouteChange)) + .await + .expect("timed out"); + } + + #[moq_net_sim::test] + async fn late_reader_after_resume() { + moq_net_sim::timeout(TEST_TIMEOUT, late_reader($version, Before::Resume)) + .await + .expect("timed out"); + } + } + )* + }; +} + +// Only an epoch lets `R` resume onto another route; older wires end the subscription +// with its route instead (see `route_change`). +handover_tests! { + lite_07: "moq-lite-07-wip", +} + +macro_rules! fetch_tests { + ($($name:ident: $version:literal,)*) => { + $( + mod $name { + use super::*; + + #[moq_net_sim::test] + async fn fetch_right_after_leave() { + moq_net_sim::timeout(TEST_TIMEOUT, fetch_after_leave($version, Duration::ZERO)) + .await + .expect("timed out"); + } + + #[moq_net_sim::test] + async fn fetch_after_leave_settles() { + moq_net_sim::timeout(TEST_TIMEOUT, fetch_after_leave($version, Duration::from_millis(500))) + .await + .expect("timed out"); + } + } + )* + }; +} + +// A fetch needs no epoch. moq-transport is left out: its copy goes idle on the leave +// and still answers from the open group it can no longer feed. +fetch_tests! { + fetch_lite_06: "moq-lite-06", + fetch_lite_07: "moq-lite-07-wip", +} diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index d155bb218f5..fa594666e04 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -11,6 +11,8 @@ mod announce_to_serve; mod auth; mod broadcast_close; mod catalog_acquisition; +mod catalog_churn; +mod catalog_handover; mod datagram; mod dial_split_horizon; mod drained_claim; From bd61bb5fb8cd92654d8f95443b56b27b2a925fd0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 20:43:18 -0700 Subject: [PATCH 2/3] fix(net): a fetched frame read to its end releases its track A frame handed out from a fetched group cloned the group's recovery state, including the logical track it holds, and kept it after its payload was fully read. A caller retaining the frame kept the track in use, so it never parked. Drop the recovery state once the payload is read, as a group does once it is read to its end. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/frame.rs | 4 ++++ rs/moq-net/src/model/resume.rs | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/rs/moq-net/src/model/frame.rs b/rs/moq-net/src/model/frame.rs index fa92cfaeefd..d7f5bcb2df4 100644 --- a/rs/moq-net/src/model/frame.rs +++ b/rs/moq-net/src/model/frame.rs @@ -779,6 +779,10 @@ impl Consumer { if self.expired { return res; } + // Read to its end: no route is needed for it any more. + if matches!(res, Poll::Ready(Ok(_))) && self.read_idx >= self.size() { + self.recover = None; + } let Some(recover) = self.recover.as_mut() else { return res; }; diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 389332fccfa..8857dce3af2 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -2379,6 +2379,28 @@ mod test { assert_eq!(read(&mut fetched), None); } + /// A fetched frame read to its end no longer holds the logical track, so a caller + /// keeping the handle does not stop the track from parking. + #[test] + #[cfg(feature = "async")] + fn a_read_frame_releases_the_track() { + let routes = Producer::new(); + let a = copy(); + routes.serve(a.consume()); + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + + let logical = logical(&routes); + let demand = logical.demand(); + let mut fetched = logical.consume().fetch_group(0, None).now_or_never().unwrap().unwrap(); + assert!(demand.is_used(), "an open fetched group holds the track"); + let mut frame = fetched.next_frame().now_or_never().unwrap().unwrap().unwrap(); + drop(fetched); + assert!(demand.is_used(), "an unread frame holds the track"); + assert_eq!(frame.read_all().now_or_never().unwrap().unwrap(), b"a".as_ref()); + assert!(!demand.is_used(), "a read frame does not"); + } + /// A rerouted fetch keeps its own age budget: the new route is asked with it, and a /// read waiting on that route gives up once the group falls past it there. #[test] From b6c337b415fbbff88cd1575fc4048261699c6ce0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 20:46:26 -0700 Subject: [PATCH 3/3] test(net): move the catalog regressions onto moq-clock Main replaced moq-net-sim with moq-clock's simulator (#5681). Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/tests/integration/catalog_churn.rs | 24 ++++---- .../tests/integration/catalog_handover.rs | 60 +++++++++---------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/rs/moq-net/tests/integration/catalog_churn.rs b/rs/moq-net/tests/integration/catalog_churn.rs index 5d6c48ec738..31c944f87c2 100644 --- a/rs/moq-net/tests/integration/catalog_churn.rs +++ b/rs/moq-net/tests/integration/catalog_churn.rs @@ -134,7 +134,7 @@ struct Reader { expected: Vec, seen: Vec, /// Owns the subscription, so dropping the reader leaves the track. - task: moq_net_sim::JoinHandle<()>, + task: moq_clock::sim::JoinHandle<()>, } impl Drop for Reader { @@ -146,7 +146,7 @@ impl Drop for Reader { impl Reader { fn new(id: usize, mut sub: track::Subscriber, expected: Vec) -> Self { let (tx, rx) = mpsc::unbounded(); - let task = moq_net_sim::spawn(async move { + let task = moq_clock::sim::spawn(async move { loop { let mut group = match sub.recv_group().await { Ok(Some(group)) => group, @@ -266,7 +266,7 @@ impl Topology { /// A fresh request for the broadcast; `None` when refused, as with no route right now. async fn request(&self, step: usize) -> Result, Failure> { let request = self.subscriber.consume().request_broadcast("live", None); - match moq_net_sim::timeout(READ_TIMEOUT, request).await { + match moq_clock::timeout(READ_TIMEOUT, request).await { Ok(result) => Ok(result.ok()), Err(_) => Err(stuck(format!("{step}: request hung"))), } @@ -331,14 +331,14 @@ async fn execute(version: Version, ops: &[Op]) -> Result<(), Failure> { written.clear(); write(&mut group, &mut written, &mut readers, "snapshot"); // Let it reach `R`, so the newest group at the next join is unambiguous. - moq_net_sim::sleep(Duration::from_millis(500)).await; + moq_clock::sleep(Duration::from_millis(500)).await; } Op::Join => { let Some(remote) = topology.request(step).await? else { continue; }; let subscribe = remote.track("catalog.json").unwrap().subscribe(None); - match moq_net_sim::timeout(READ_TIMEOUT, subscribe).await { + match moq_clock::timeout(READ_TIMEOUT, subscribe).await { Ok(Ok(sub)) => readers.push(Reader::new(joined, sub, written.clone())), Ok(Err(_)) => {} Err(_) => return Err(stuck(format!("{step}: subscribe hung"))), @@ -350,14 +350,14 @@ async fn execute(version: Version, ops: &[Op]) -> Result<(), Failure> { continue; }; let fetch = remote.track("catalog.json").unwrap().fetch_group(group.sequence, None); - let mut fetched = match moq_net_sim::timeout(READ_TIMEOUT, fetch).await { + let mut fetched = match moq_clock::timeout(READ_TIMEOUT, fetch).await { Ok(Ok(fetched)) => fetched, Ok(Err(_)) => continue, Err(_) => return Err(stuck(format!("{step}: fetch hung"))), }; let mut got = Vec::new(); while got.len() < written.len() { - match moq_net_sim::timeout(READ_TIMEOUT, fetched.read_frame()).await { + match moq_clock::timeout(READ_TIMEOUT, fetched.read_frame()).await { Ok(Ok(Some(frame))) => got.push(format!( "{}:{}", fetched.sequence, @@ -380,9 +380,9 @@ async fn execute(version: Version, ops: &[Op]) -> Result<(), Failure> { Op::ToggleA => topology.toggle(0).await, Op::ToggleB => topology.toggle(1).await, Op::ToggleDirect => topology.toggle(2).await, - Op::Sleep(pause) => moq_net_sim::sleep(pause).await, + Op::Sleep(pause) => moq_clock::sleep(pause).await, Op::Check => { - moq_net_sim::sleep(Duration::from_millis(500)).await; + moq_clock::sleep(Duration::from_millis(500)).await; let mut failed = Vec::new(); for (index, reader) in readers.iter_mut().enumerate() { let open = reader.drain(); @@ -392,7 +392,7 @@ async fn execute(version: Version, ops: &[Op]) -> Result<(), Failure> { _ if !open => failed.push(index), Verdict::Done => {} Verdict::Short => { - let _ = moq_net_sim::timeout(STUCK_AFTER, async { + let _ = moq_clock::timeout(STUCK_AFTER, async { while reader.judge() == Verdict::Short { match reader.rx.next().await { Some(Ok(frame)) => reader.seen.push(frame), @@ -448,13 +448,13 @@ async fn churn(version: &str) { } /// Routes carry the publisher's epoch, so subscriptions resume across them. -#[moq_net_sim::test] +#[moq_clock::test] async fn catalog_churn_lite_07() { churn("moq-lite-07-wip").await; } /// No epoch on the wire: a route change ends subscriptions, which must fail loudly. -#[moq_net_sim::test] +#[moq_clock::test] async fn catalog_churn_lite_06() { churn("moq-lite-06").await; } diff --git a/rs/moq-net/tests/integration/catalog_handover.rs b/rs/moq-net/tests/integration/catalog_handover.rs index fd15d916f42..8039b5eb11f 100644 --- a/rs/moq-net/tests/integration/catalog_handover.rs +++ b/rs/moq-net/tests/integration/catalog_handover.rs @@ -55,7 +55,7 @@ fn abort(pair: MockPair) { /// Let every task settle. Time is paused, so this returns once the runtime is idle. async fn settle() { - moq_net_sim::sleep(Duration::from_millis(500)).await; + moq_clock::sleep(Duration::from_millis(500)).await; } /// How the serving route changes. @@ -167,11 +167,11 @@ impl Topology { /// A new reader on `R`: a fresh broadcast request and a floorless subscription. async fn reader(&self) -> Reader { - let remote = moq_net_sim::timeout(READ_TIMEOUT, self.subscriber.consume().request_broadcast("live", None)) + let remote = moq_clock::timeout(READ_TIMEOUT, self.subscriber.consume().request_broadcast("live", None)) .await .expect("request hung") .expect("request refused"); - let sub = moq_net_sim::timeout(READ_TIMEOUT, remote.track("catalog.json").unwrap().subscribe(None)) + let sub = moq_clock::timeout(READ_TIMEOUT, remote.track("catalog.json").unwrap().subscribe(None)) .await .expect("subscribe hung") .expect("subscribe refused"); @@ -184,7 +184,7 @@ impl Topology { struct Reader { rx: mpsc::UnboundedReceiver>, /// Owns the subscription, so dropping the reader leaves the track. - task: moq_net_sim::JoinHandle<()>, + task: moq_clock::sim::JoinHandle<()>, } impl Drop for Reader { @@ -196,7 +196,7 @@ impl Drop for Reader { impl Reader { fn new(mut sub: track::Subscriber) -> Self { let (tx, rx) = mpsc::unbounded(); - let task = moq_net_sim::spawn(async move { + let task = moq_clock::sim::spawn(async move { loop { let mut group = match sub.recv_group().await { Ok(Some(group)) => group, @@ -232,7 +232,7 @@ impl Reader { async fn take(&mut self, n: usize, who: &str) -> Vec { let mut frames = Vec::new(); while frames.len() < n { - match moq_net_sim::timeout(READ_TIMEOUT, self.rx.next()).await { + match moq_clock::timeout(READ_TIMEOUT, self.rx.next()).await { Ok(Some(Ok(frame))) => frames.push(frame), Ok(Some(Err(err))) => panic!("{who}: failed after {frames:?}: {err}"), Ok(None) => panic!("{who}: ended after {frames:?}"), @@ -250,7 +250,7 @@ impl Reader { /// The subscription fails or ends, rather than hanging or delivering more. async fn expect_end(&mut self, who: &str) { - match moq_net_sim::timeout(READ_TIMEOUT, self.rx.next()).await { + match moq_clock::timeout(READ_TIMEOUT, self.rx.next()).await { Ok(None | Some(Err(_))) => {} Ok(Some(Ok(frame))) => panic!("{who}: delivered {frame:?} past its route"), Err(_) => panic!("{who}: hung instead of ending"), @@ -322,7 +322,7 @@ async fn handover(version: &str, trigger: Trigger, holder: Holder) { assert_eq!(viewer.take(1, &tag).await, ["0:delta3"], "{tag}: viewer continues"); } // Returning well past the relays' idle linger, so no copy of the head is warm. - moq_net_sim::sleep(RETURN_AFTER).await; + moq_clock::sleep(RETURN_AFTER).await; let mut returning = topology.reader().await; returning.expect_all(&topology, &format!("{tag}: returning")).await; @@ -365,14 +365,14 @@ async fn late_reader(version: &str, before: Before) { Before::Resume => { settle().await; drop(first); - moq_net_sim::sleep(Duration::from_secs(5)).await; + moq_clock::sleep(Duration::from_secs(5)).await; let mut resumed = topology.reader().await; resumed.expect_all(&topology, &format!("{tag}: resumed")).await; resumed } }; - moq_net_sim::sleep(RETURN_AFTER).await; + moq_clock::sleep(RETURN_AFTER).await; let mut late = topology.reader().await; late.expect_all(&topology, &format!("{tag}: late")).await; } @@ -389,7 +389,7 @@ async fn fetch_after_leave(version: &str, pause: Duration) { reader.expect_all(&topology, &format!("{tag}: reader")).await; topology.write("delta1"); drop(reader); - moq_net_sim::sleep(pause).await; + moq_clock::sleep(pause).await; let remote = topology .subscriber @@ -398,13 +398,13 @@ async fn fetch_after_leave(version: &str, pause: Duration) { .await .unwrap(); let fetch = remote.track("catalog.json").unwrap().fetch_group(0, None); - let mut group = moq_net_sim::timeout(READ_TIMEOUT, fetch) + let mut group = moq_clock::timeout(READ_TIMEOUT, fetch) .await .unwrap_or_else(|_| panic!("{tag}: fetch hung")) .unwrap_or_else(|err| panic!("{tag}: fetch refused: {err}")); let mut got = Vec::new(); while got.len() < topology.written.len() { - match moq_net_sim::timeout(READ_TIMEOUT, group.read_frame()).await { + match moq_clock::timeout(READ_TIMEOUT, group.read_frame()).await { Ok(Ok(Some(frame))) => got.push(String::from_utf8_lossy(&frame.payload).to_string()), Ok(Ok(None)) => panic!("{tag}: fetch ended after {got:?}"), Ok(Err(err)) => panic!("{tag}: fetch failed after {got:?}: {err}"), @@ -421,61 +421,61 @@ macro_rules! handover_tests { use super::*; async fn run(trigger: Trigger, holder: Holder) { - moq_net_sim::timeout(TEST_TIMEOUT, handover($version, trigger, holder)) + moq_clock::timeout(TEST_TIMEOUT, handover($version, trigger, holder)) .await .expect("timed out"); } - #[moq_net_sim::test] + #[moq_clock::test] async fn disconnect_with_viewer() { run(Trigger::Disconnect, Holder::Viewer).await; } - #[moq_net_sim::test] + #[moq_clock::test] async fn disconnect_parked() { run(Trigger::Disconnect, Holder::Parked).await; } - #[moq_net_sim::test] + #[moq_clock::test] async fn unannounce_with_viewer() { run(Trigger::Unannounce, Holder::Viewer).await; } - #[moq_net_sim::test] + #[moq_clock::test] async fn unannounce_parked() { run(Trigger::Unannounce, Holder::Parked).await; } - #[moq_net_sim::test] + #[moq_clock::test] async fn better_route_with_viewer() { run(Trigger::BetterRoute, Holder::Viewer).await; } - #[moq_net_sim::test] + #[moq_clock::test] async fn better_route_parked() { run(Trigger::BetterRoute, Holder::Parked).await; } - #[moq_net_sim::test] + #[moq_clock::test] async fn reconnect_with_viewer() { run(Trigger::Reconnect, Holder::Viewer).await; } - #[moq_net_sim::test] + #[moq_clock::test] async fn reconnect_parked() { run(Trigger::Reconnect, Holder::Parked).await; } - #[moq_net_sim::test] + #[moq_clock::test] async fn late_reader_after_route_change() { - moq_net_sim::timeout(TEST_TIMEOUT, late_reader($version, Before::RouteChange)) + moq_clock::timeout(TEST_TIMEOUT, late_reader($version, Before::RouteChange)) .await .expect("timed out"); } - #[moq_net_sim::test] + #[moq_clock::test] async fn late_reader_after_resume() { - moq_net_sim::timeout(TEST_TIMEOUT, late_reader($version, Before::Resume)) + moq_clock::timeout(TEST_TIMEOUT, late_reader($version, Before::Resume)) .await .expect("timed out"); } @@ -496,16 +496,16 @@ macro_rules! fetch_tests { mod $name { use super::*; - #[moq_net_sim::test] + #[moq_clock::test] async fn fetch_right_after_leave() { - moq_net_sim::timeout(TEST_TIMEOUT, fetch_after_leave($version, Duration::ZERO)) + moq_clock::timeout(TEST_TIMEOUT, fetch_after_leave($version, Duration::ZERO)) .await .expect("timed out"); } - #[moq_net_sim::test] + #[moq_clock::test] async fn fetch_after_leave_settles() { - moq_net_sim::timeout(TEST_TIMEOUT, fetch_after_leave($version, Duration::from_millis(500))) + moq_clock::timeout(TEST_TIMEOUT, fetch_after_leave($version, Duration::from_millis(500))) .await .expect("timed out"); }