diff --git a/quest/m1/README.md b/quest/m1/README.md index 3f93058e5c5..99fec91df07 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -172,7 +172,6 @@ removed). - [App-limited BBR3 estimate ages](/quest/m1/bbr-app-limited.md) - an app-limited BBR3 estimate falls within seconds after the path drops - [Bindings name every session code](/quest/m1/ffi-malformed-auth-kind.md) - MALFORMED_AUTH_TOKEN and its siblings reach moq-ffi, moq-c, and the wrappers - [Unbounded groups](/quest/m1/unbounded-groups/README.md) - groups and frames lose their size caps; the cache sheds parts of a group under pressure and refills the holes in place -- [Relay demand gaps](/quest/m1/relay-demand-gaps.md) - pre-07 backlog, IETF warm-relay newcomer, and double upstream pulls are fixed or ruled out - [Segment cuts hold bounded state](/quest/m1/segmenter-cuts.md) - the HLS segmenter keeps one pending cut, so `styp` input without sync samples cannot grow it - [Next Object on a cold Rust copy](/quest/m1/next-object-cold-copy.md) - a join after an upstream group report starts at the first frame received, never skipping - [Container consumers buffer by backpressure](/quest/m1/container-backpressure.md) - Rust and JS container consumers decode ahead only a byte budget, so a hostile publisher cannot grow them diff --git a/quest/m1/relay-demand-gaps.md b/quest/m1/relay-demand-gaps.md deleted file mode 100644 index 356774f2131..00000000000 --- a/quest/m1/relay-demand-gaps.md +++ /dev/null @@ -1,19 +0,0 @@ -# [M] Relay demand gaps left by the widened-age re-offer - -## Goal - -Three gaps found by [moq-dev/moq#6717](https://github.com/moq-dev/moq/pull/6717) -are fixed or ruled out, each with a test: - -- On lite before 07, a reader with a narrow max age receives the whole backlog. -- On moq-transport, a newcomer on a warm relay gets only the newest group. -- A relay newcomer may pull one group upstream twice: once from its range fill - and once from the publisher's re-offer. - -## Plan - -Reproduce each through a relay first. The third is wasted upstream bandwidth -per newcomer, so it scales with subscribers; cover it with a benchmark if it -reproduces. - -Public API: none expected. Wire: none expected. diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 3e02d35be12..10b1690a63f 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -7719,6 +7719,9 @@ impl ServeLoop { if serve.subscriber.version.resolves_anchors() { serving.anchored(); } + if serve.subscriber.version.fills_from_start() { + serving.fills_from_start(); + } let serving = serving.commit(); Self { edge: serving.consume(), diff --git a/rs/moq-net/src/lite/version.rs b/rs/moq-net/src/lite/version.rs index dd057d9e3cb..c3c424f2fff 100644 --- a/rs/moq-net/src/lite/version.rs +++ b/rs/moq-net/src/lite/version.rs @@ -219,6 +219,13 @@ impl Version { ) } + /// Whether a copy's fills stop at the start its publisher resolved from the budget. + /// Lite-06 resolves the start, but drops media time and sends FETCH without a max + /// delay, so nothing judges a group below that start against a reader's budget. + pub(crate) fn fills_from_start(self) -> bool { + self.resolves_start() && !self.has_media_time() && !self.has_fetch_max_delay() + } + /// Whether subscription completion waits for the subscriber's FIN. pub(crate) fn waits_for_subscriber_fin(self) -> bool { !matches!( diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 740a498034d..e2e3c3a126c 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -444,6 +444,10 @@ pub(crate) struct TrackState { anchored: bool, // This copy can translate only anchors its cache has already settled. cached_anchors: bool, + // Whether readers ask no fills below `start_sequence`: its publisher resolved it from the + // budget, but neither this copy's timestamps nor a fill can judge an older group against + // a reader's budget. A widened budget reaches lower groups through the feed's re-offer. + fills_from_start: bool, // Sequences the upstream publisher gave up, with the error each was given up with: an // incoming SUBSCRIBE_DROP or a producer cut. Kept apart from the cache so reclaiming a // slot cannot erase a disposition a relay must forward, and as ranges so a wide DROP @@ -3250,6 +3254,15 @@ impl Producer { } } + /// Have this received copy's readers ask no fills below the start its publisher resolved + /// from the budget, for a wire whose fills and timestamps cannot bound them by it. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn fills_from_start(&self) { + if let Ok(mut state) = self.modify() { + state.fills_from_start = true; + } + } + /// Record that the publisher gave up `groups` with `err`: none will arrive, so a reader /// waiting on one moves past it, a group still arriving in them ends with `err`, the /// declared end does not wait for them, and a relay serving this track forwards the drop. @@ -17193,9 +17206,19 @@ mod test { // The backfill serves by sequence, but never in arrival order. assert!(consumer.peek_group(1).is_some()); + let mut walked = Vec::new(); + let mut index = 0; + while let Poll::Ready(Ok(Some((group, next)))) = producer.state.read().poll_recv_group(index, 0) { + walked.push(group.sequence); + index = next + 1; + } + assert_eq!(walked, [0, 2]); + // A reader takes it as the fill answer for the group the feed gave up, as it would + // have asked a fill for it, and the stale arrival entry adds nothing. let mut subscriber = producer.subscribe(replay()); - assert_eq!(subscriber.assert_group().sequence, 0); - assert_eq!(subscriber.assert_group().sequence, 2); + let mut read: Vec<_> = (0..3).map(|_| subscriber.assert_group().sequence).collect(); + read.sort(); + assert_eq!(read, [0, 1, 2]); subscriber.assert_no_group(); } @@ -17769,6 +17792,66 @@ mod test { ); } + /// A copy whose fills cannot carry a reader's budget asks none below the start its + /// publisher resolved from that budget, and names nothing there dropped: what a wider + /// budget reaches comes through the feed's re-offer, which a reader takes as it arrives. + #[test] + #[cfg(feature = "async")] + fn no_fill_below_a_start_resolved_from_the_budget() { + let write = |producer: &Producer, sequence| { + let mut group = producer.create_group(group::Info { sequence }).unwrap(); + group.write_frame(None, &b"x"[..]).unwrap(); + group.finish().unwrap(); + }; + let mut producer = track_producer("test", Info::default()); + producer.fills_from_start(); + let dynamic = producer.requests(); + let mut subscriber = producer.subscribe(replay().with_range(0..)); + producer.request_start(None).unwrap(); + producer.start_at(2).unwrap(); + write(&producer, 2); + write(&producer, 3); + let waiter = kio::Waiter::noop(); + assert_eq!(drain_groups(&mut subscriber), [2, 3]); + assert!( + dynamic.poll_requested_fill(&waiter).is_pending(), + "below the start is no miss" + ); + assert!(drained_drops(&mut subscriber).is_empty()); + + // The re-offer of a widened budget. + write(&producer, 1); + assert_eq!(drain_groups(&mut subscriber), [1]); + assert!(dynamic.poll_requested_fill(&waiter).is_pending()); + } + + /// A reader arriving after another's fill cached the groups under the start reads them + /// from the cache: the arrival walk never reaches a backfill, and no fill is asked again. + #[test] + #[cfg(feature = "async")] + fn a_newcomer_reads_the_backfill_another_fill_cached() { + let (producer, dynamic, mut first) = started_late(); + let waiter = kio::Waiter::noop(); + while let Poll::Ready(Ok(Some(_))) = first.poll_recv_group(&waiter) {} + let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) else { + panic!("below the start is a miss"); + }; + while let Some(sequence) = fill.next() { + write_filled(fill.accept(sequence, None), sequence); + } + let mut newcomer = producer.subscribe(replay().with_range(0..)); + let mut read = Vec::new(); + while let Poll::Ready(Ok(Some(group))) = newcomer.poll_recv_group(&waiter) { + read.push(group.sequence); + } + read.sort(); + assert_eq!(read, [0, 1, 2, 3]); + assert!( + dynamic.poll_requested_fill(&waiter).is_pending(), + "the cache answers it" + ); + } + /// Groups a fill answered before the reader came back are handed out in the reader's /// order, so a warm cache keeps a descending reader newest first. #[test] diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index f50ce641a41..8b7e4c6bbd6 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -491,7 +491,24 @@ impl Misses { self.judged.insert(run.clone()); } let start = state.start_sequence.unwrap_or(0); + // Below the start its publisher resolved from the budget, such a copy cannot tell a + // group a reader's budget reaches, so it leaves them to the feed's re-offer. + if state.fills_from_start && start > 0 { + candidates.remove(0..start); + } for run in candidates.iter() { + // Backfill another reader's fill cached is no hole, yet the arrival walk never + // reaches it: this reader takes it as filled where it would ask for a hole. + for held in backfill(state, run.clone()).iter() { + let (unfed, awaited) = state.unfed(held.clone(), now); + for run in unfed.iter() { + self.filled.extend(run.clone()); + } + for (run, folds) in awaited { + self.filled.extend(run.start..run.end.min(start)); + self.deferred.insert(run.start.max(start)..run.end, folds); + } + } for hole in holes(state, run.clone()) { for hole in self.unclaimed(state, hole) { let (unfed, awaited) = state.unfed(hole, now); @@ -722,6 +739,16 @@ impl TrackState { } } +/// The runs of `run` the cache holds as backfill: servable, but out of the arrival walk. +fn backfill(state: &TrackState, run: ops::Range) -> RangeSet { + state + .lookup + .range(run) + .filter(|(_, slot)| !slot.is_walked() && slot.serves_end()) + .map(|(&sequence, _)| sequence..sequence + 1) + .collect() +} + /// The runs of `run` the cache holds nothing servable for. pub(super) fn holes(state: &TrackState, run: ops::Range) -> Vec> { let mut holes = Vec::new(); diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index da3b6acc2b0..553d4914c28 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -71,6 +71,7 @@ mod tracks; mod tracks_ietf; mod unannounce_release; mod untimed; +mod warm_backfill; mod wide_hop; mod wide_sequence; mod widened_max_age; diff --git a/rs/moq-net/tests/integration/warm_backfill.rs b/rs/moq-net/tests/integration/warm_backfill.rs new file mode 100644 index 00000000000..aeedd1ca90e --- /dev/null +++ b/rs/moq-net/tests/integration/warm_backfill.rs @@ -0,0 +1,90 @@ +#![cfg(feature = "async")] + +//! A relay that backfilled a reader's backlog serves the same backlog to the next reader +//! from its cache. +//! +//! Where the upstream starts the copy at its live edge, as moq-transport and moq-lite 05 +//! do, the backlog below that start arrives through fills, outside the live feed's arrival +//! walk. A newcomer finds those groups cached, so it neither walks nor asks for them, and +//! read only the live edge until it took the cached backfill as filled. + +use std::collections::BTreeSet; +use std::time::Duration; + +use moq_net::track::{Info, Subscription}; +use moq_net::{Hop, Timescale, Timestamp, Version, origin, track}; + +use crate::support::harness::{peer, spawn}; + +const TIMEOUT: Duration = Duration::from_secs(10); + +const GROUPS: u64 = 10; + +/// Reaches every group, however long the readers take. +const BUDGET: Duration = Duration::from_secs(100); + +const VERSIONS: &[&str] = &[ + "moq-lite-05", + "moq-lite-06", + "moq-lite-07-wip", + "moq-transport-18", + "moq-transport-22", +]; + +fn origin(hop: u64) -> origin::Producer { + let (origin, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); + spawn(driver); + origin +} + +async fn subscribe(relay: &origin::Producer) -> track::Subscriber { + let consumer = relay.consume(); + consumer.routed("live").await.unwrap(); + let remote = consumer.request_broadcast("live", None).await.unwrap(); + remote + .track("video") + .unwrap() + .subscribe(Subscription::default().with_max_delay(BUDGET)) + .await + .unwrap() +} + +/// Every group `subscriber` receives until it goes quiet. +async fn received(subscriber: &mut track::Subscriber) -> BTreeSet { + let mut got = BTreeSet::new(); + while let Ok(group) = moq_clock::timeout(Duration::from_secs(1), subscriber.recv_group()).await { + let group = group.unwrap().expect("the track is live"); + assert!(got.insert(group.sequence), "group {} delivered twice", group.sequence); + } + got +} + +#[moq_clock::test] +async fn a_newcomer_on_a_warm_relay_gets_the_backfill() { + for &version in VERSIONS { + let publisher = origin(1); + let relay = origin(2); + let _link = peer(version.parse::().unwrap(), &publisher, &relay).await; + let broadcast = publisher.create_broadcast("live").unwrap(); + let source = broadcast + .create_track("video", Info::default().with_timescale(Timescale::MILLI)) + .unwrap(); + for sequence in 0..GROUPS { + let mut group = source.create_group(moq_net::group::Info { sequence }).unwrap(); + group + .write_frame(Timestamp::from_millis(sequence * 1000).unwrap(), &b"x"[..]) + .unwrap(); + group.finish().unwrap(); + } + broadcast.announce(Default::default()).unwrap(); + + let mut first = moq_clock::timeout(TIMEOUT, subscribe(&relay)).await.unwrap(); + let cold = moq_clock::timeout(TIMEOUT, received(&mut first)).await.unwrap(); + assert_eq!(cold, (0..GROUPS).collect(), "{version}: cold"); + + // The same subscription changes nothing upstream, so only the cache can serve it. + let mut second = moq_clock::timeout(TIMEOUT, subscribe(&relay)).await.unwrap(); + let warm = moq_clock::timeout(TIMEOUT, received(&mut second)).await.unwrap(); + assert_eq!(warm, cold, "{version}: warm"); + } +} diff --git a/rs/moq-net/tests/integration/widened_max_age.rs b/rs/moq-net/tests/integration/widened_max_age.rs index 579906df092..c664ff65e80 100644 --- a/rs/moq-net/tests/integration/widened_max_age.rs +++ b/rs/moq-net/tests/integration/widened_max_age.rs @@ -3,6 +3,9 @@ //! A relay a narrow reader warmed serves a newcomer with a wider max age the groups a cold //! relay would: the widened SUBSCRIBE_UPDATE has the publisher re-offer the cached groups //! it skipped as stale, below where the copy started. +//! +//! Lite-06 relies on that re-offer alone: its copy is untimed and its FETCH carries no max +//! age, so a reader asks no fills below the start the publisher resolved from the budget. use std::collections::BTreeSet; use std::time::Duration; @@ -10,7 +13,7 @@ use std::time::Duration; use moq_net::track::{Info, Range, Subscription}; use moq_net::{Hop, Timescale, Timestamp, Version, origin, track}; -use crate::support::harness::{peer, spawn}; +use crate::support::harness::{MockConnectOptions, connect_mock, peer, spawn}; const TIMEOUT: Duration = Duration::from_secs(10); @@ -23,8 +26,12 @@ const NARROW: Duration = Duration::from_millis(2500); /// Reaches groups 2 and up. const WIDE: Duration = Duration::from_millis(6500); -/// Only lite-07 resolves a reader's relative start at the publisher; older wires and -/// moq-transport fill the copy's backlog by range instead, which the budget does not gate. +/// The wires whose publisher resolves the start from the budget. Lite-05 and moq-transport +/// start at the live edge and fill the backlog by range instead, which the budget does not gate. +const VERSIONS: &[&str] = &["moq-lite-06", "moq-lite-07-wip"]; + +/// The wire for the tests on one version: only lite-07 FETCHes below the start, so only it +/// can pull a group twice. const VERSION: &str = "moq-lite-07-wip"; fn origin(hop: u64) -> origin::Producer { @@ -42,8 +49,8 @@ struct Chain { impl Chain { /// Without `relay`, the viewer reads the publisher directly. - async fn new(relay: bool) -> Self { - let version: Version = VERSION.parse().unwrap(); + async fn new(version: &str, relay: bool) -> Self { + let version: Version = version.parse().unwrap(); let publisher = origin(1); let relay = relay.then(|| origin(2)); let viewer = origin(3); @@ -104,44 +111,64 @@ async fn received(subscriber: &mut track::Subscriber) -> BTreeSet { /// What the subscriber reads after widening its own max age, through a relay or not. #[moq_clock::test] async fn a_widened_reader_gets_what_was_skipped() { - for relay in [false, true] { - let chain = Chain::new(relay).await; - let mut reader = moq_clock::timeout(TIMEOUT, chain.subscribe(NARROW)).await.unwrap(); - let first = moq_clock::timeout(TIMEOUT, received(&mut reader)).await.unwrap(); - assert_eq!(first, (6..GROUPS).collect(), "relay={relay}"); - - let wide = Subscription::default() - .with_max_delay(WIDE) - .with_ranges([Range::oldest()]); - reader.update(wide).unwrap(); - let widened = moq_clock::timeout(TIMEOUT, received(&mut reader)).await.unwrap(); - assert_eq!(widened, (2..6).collect(), "relay={relay}: only what was skipped"); + for &version in VERSIONS { + for relay in [false, true] { + let chain = Chain::new(version, relay).await; + let mut reader = moq_clock::timeout(TIMEOUT, chain.subscribe(NARROW)).await.unwrap(); + let first = moq_clock::timeout(TIMEOUT, received(&mut reader)).await.unwrap(); + assert_eq!(first, (6..GROUPS).collect(), "{version} relay={relay}"); + + let wide = Subscription::default() + .with_max_delay(WIDE) + .with_ranges([Range::oldest()]); + reader.update(wide).unwrap(); + let widened = moq_clock::timeout(TIMEOUT, received(&mut reader)).await.unwrap(); + assert_eq!( + widened, + (2..6).collect(), + "{version} relay={relay}: only what was skipped" + ); + } } } #[moq_clock::test] async fn a_warm_relay_serves_a_wider_newcomer_what_a_cold_one_would() { - let cold = Chain::new(true).await; - let mut wide = moq_clock::timeout(TIMEOUT, cold.subscribe(WIDE)).await.unwrap(); - let want = moq_clock::timeout(TIMEOUT, received(&mut wide)).await.unwrap(); - assert_eq!(want, (2..GROUPS).collect(), "cold"); - - let warm = Chain::new(true).await; - let mut narrow = moq_clock::timeout(TIMEOUT, warm.subscribe(NARROW)).await.unwrap(); - let first = moq_clock::timeout(TIMEOUT, received(&mut narrow)).await.unwrap(); - assert_eq!(first, (6..GROUPS).collect(), "the narrow reader warms the relay"); - - let mut wide = moq_clock::timeout(TIMEOUT, warm.subscribe(WIDE)).await.unwrap(); - let got = moq_clock::timeout(TIMEOUT, received(&mut wide)).await.unwrap(); - assert_eq!(got, want, "warm"); - let late = moq_clock::timeout(TIMEOUT, received(&mut narrow)).await.unwrap(); - assert!(late.is_empty(), "the narrow reader still skips {late:?}"); + for &version in VERSIONS { + let cold = Chain::new(version, true).await; + let mut wide = moq_clock::timeout(TIMEOUT, cold.subscribe(WIDE)).await.unwrap(); + let want = moq_clock::timeout(TIMEOUT, received(&mut wide)).await.unwrap(); + assert_eq!(want, (2..GROUPS).collect(), "{version}: cold"); + + let warm = Chain::new(version, true).await; + let mut narrow = moq_clock::timeout(TIMEOUT, warm.subscribe(NARROW)).await.unwrap(); + let first = moq_clock::timeout(TIMEOUT, received(&mut narrow)).await.unwrap(); + assert_eq!( + first, + (6..GROUPS).collect(), + "{version}: the narrow reader warms the relay" + ); + + // The widened max age lowers the start the publisher resolved, and its re-offer + // serves what lies between. + let mut wide = moq_clock::timeout(TIMEOUT, warm.subscribe(WIDE)).await.unwrap(); + let got = moq_clock::timeout(TIMEOUT, received(&mut wide)).await.unwrap(); + assert_eq!(got, want, "{version}: warm"); + // A lite-06 copy has no media time to find the re-offer stale by, and each group in + // it arrives just now, so the narrow reader takes it too, once per widening. + let late = moq_clock::timeout(TIMEOUT, received(&mut narrow)).await.unwrap(); + let reoffered = match version { + "moq-lite-06" => (2..6).collect(), + _ => BTreeSet::new(), + }; + assert_eq!(late, reoffered, "{version}: what the narrow reader takes late"); + } } /// A reader joining with the max age the relay already asks for changes nothing upstream. #[moq_clock::test] async fn the_same_max_age_sends_no_update() { - let mut chain = Chain::new(true).await; + let mut chain = Chain::new(VERSION, true).await; let mut narrow = moq_clock::timeout(TIMEOUT, chain.subscribe(NARROW)).await.unwrap(); moq_clock::timeout(TIMEOUT, received(&mut narrow)).await.unwrap(); while moq_clock::timeout(Duration::from_secs(1), chain.source.subscription_changed()) @@ -157,3 +184,73 @@ async fn the_same_max_age_sends_no_update() { let changed = moq_clock::timeout(Duration::from_secs(1), chain.source.subscription_changed()).await; assert!(changed.is_err(), "the publisher saw {changed:?}"); } + +/// The groups `registry`'s sessions sent as publisher. +fn sent_groups(registry: &moq_net::stats::Registry) -> u64 { + registry + .snapshot() + .traffic() + .into_iter() + .filter(|(_, role, _)| *role == moq_net::stats::Role::Publisher) + .map(|(_, _, traffic)| traffic.groups) + .sum() +} + +/// Only the newcomer that widens the relay's max age pulls a re-offer upstream; the ones +/// after it are served from what the relay cached, however many there are. +#[moq_clock::test] +async fn wider_newcomers_after_the_first_pull_nothing_upstream() { + let version: Version = VERSION.parse().unwrap(); + let publisher = origin(1); + let relay = origin(2); + // The publisher accepts the relay, so its session meters what goes upstream. + let registry = moq_net::stats::Registry::new(Default::default()); + let mut options = MockConnectOptions::new(version); + options.client_publish = Some(relay.consume().with_hidden(true)); + options.client_subscribe = Some(relay.clone()); + options.server_publish = Some(publisher.consume().with_hidden(true)); + options.server_subscribe = Some(publisher.clone()); + options.server_stats = registry + .tier(moq_net::stats::Tier::new(moq_net::Path::from_static("upstream"))) + .session(moq_net::Path::from_static("")); + let _upstream = connect_mock(options).await; + let broadcast = publisher.create_broadcast("live").unwrap(); + let source = broadcast + .create_track("video", Info::default().with_timescale(Timescale::MILLI)) + .unwrap(); + for sequence in 0..GROUPS { + let mut group = source.create_group(moq_net::group::Info { sequence }).unwrap(); + group + .write_frame(Timestamp::from_millis(sequence * 1000).unwrap(), &b"x"[..]) + .unwrap(); + group.finish().unwrap(); + } + broadcast.announce(Default::default()).unwrap(); + + let mut links = Vec::new(); + let mut readers = Vec::new(); + let mut join = async |budget| { + let viewer = origin(3); + links.push(peer(version, &relay, &viewer).await); + let consumer = viewer.consume(); + consumer.routed("live").await.unwrap(); + let remote = consumer.request_broadcast("live", None).await.unwrap(); + let subscription = Subscription::default() + .with_max_delay(budget) + .with_ranges([Range::oldest()]); + let mut reader = remote.track("video").unwrap().subscribe(subscription).await.unwrap(); + let got = received(&mut reader).await; + readers.push((viewer, reader)); + got + }; + let narrow = moq_clock::timeout(TIMEOUT, join(NARROW)).await.unwrap(); + assert_eq!(narrow, (6..GROUPS).collect()); + let wide = moq_clock::timeout(TIMEOUT, join(WIDE)).await.unwrap(); + assert_eq!(wide, (2..GROUPS).collect()); + let widened = sent_groups(®istry); + for newcomer in 0..3 { + let got = moq_clock::timeout(TIMEOUT, join(WIDE)).await.unwrap(); + assert_eq!(got, wide, "newcomer {newcomer}"); + } + assert_eq!(sent_groups(®istry), widened, "later newcomers pull nothing upstream"); +}