diff --git a/quest/m1/README.md b/quest/m1/README.md index d0d397df169..d8a2a91013b 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -17,6 +17,7 @@ transport, benchmark tooling); worktrees isolate commits, not semantics. ## Required +- [Late joiner history](/quest/m1/relay-late-joiner-history.md) - a subscriber joining a relay's track from group 0 later still receives the cached finished group below the live one - [Cluster routing](/quest/m1/cluster-routing.md) - an announcement says where a broadcast originates, not how to reach it, and a relay hears only the prefixes its clients asked for - [Track tail interop](/quest/m1/track-tail-interop.md) - a Rust publisher ending a track with a group in flight is read to its end by the JS subscriber, and the reverse, in `just test interop` - [Go cancel test](/quest/m1/go-origin-gc.md) - the Go request-cancel test keeps its origin alive, so the collector can't close it mid-test diff --git a/quest/m1/relay-late-joiner-history.md b/quest/m1/relay-late-joiner-history.md new file mode 100644 index 00000000000..bd041dc43ce --- /dev/null +++ b/quest/m1/relay-late-joiner-history.md @@ -0,0 +1,44 @@ +# [M] Late joiner history + +## Goal + +A second subscriber that joins a relay's track from group 0 while a first +subscriber still holds it receives every group the relay has cached, +including a finished group older than the open live one. A regression test +that joins late through a relay fails before the fix. + +## Plan + +Found downstream on moq.pro: its billing rollup tracks replay a finished +history group followed by an open live group, and a second browser opening +the Cost page on the same edge intermittently loses the older group, so +prior-period usage rows are missing. Production dashboards are affected, +not just tests. #4387 fixed the first viewer's version of this; what it +leaves is the viewer that joins later. + +What we saw: + +- With #4387 applied, moq.pro's `e2e/tests/rollup-history.spec.ts + --repeat-each 12` still fails about a quarter of runs, always on the + second viewer. The edge logs `serving group` only for the newer group on + that viewer's subscription, while the first viewer got both. +- A late-joiner variant of `rs/moq-net/tests/history_groups.rs` showed the + relay's late cursor spliced across two segments with its floor at the + newer group, so the cached older group sat below it. The segment churn + comes through `Action::Query`, `Splice`, `Park`, and `Release` in + `rs/moq-net/src/model/origin.rs`, triggered by the TRACK_INFO request that + precedes each subscription and then goes idle. +- That variant is not a faithful reproduction yet: under paused time the + origin's linger timers fire during the test's idle waits, so it also failed + for lite-03 and the IETF drafts, which the real stack does not. A + reproduction needs to keep the relay's track warm without auto-advancing + past the linger, or drive the timers explicitly. + +Start from how a newly spliced or parked segment derives its floor for a +subscriber that asked for group 0, and whether a warm copy's cached groups +below the new segment's first group stay reachable. The +[splice edge cases](/quest/m1/splice-edges.md) touch the same code. + +## Related + +- [Splice edge cases](/quest/m1/splice-edges.md) - other spliced-track cases that lose or mis-judge groups diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index df6a175afcc..995a50578de 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -1,7 +1,7 @@ use crate::runtime::Timers as _; use crate::{SessionError, announce, frame, group, origin, track}; use std::{ - collections::HashMap, + collections::{BTreeSet, HashMap}, ops::Bound, sync::{ Arc, @@ -2214,6 +2214,10 @@ struct TrackRun { // exclusive final sequence (which may be ahead of the live edge). emit_range: bool, start_sent: bool, + // The first servable group, held until the source resolves where its feed starts. + first: Option, + // Groups skipped for a missing head before the start resolved, which it must not name. + skipped: BTreeSet, end_sent: bool, // Lite07+ sends SUBSCRIBE_END with the stream count instead of as soon as the // boundary is known, once every group below it has opened its stream. @@ -2250,6 +2254,8 @@ impl TrackRun { track_priority_tx, emit_range, start_sent: false, + first: None, + skipped: BTreeSet::new(), end_sent: false, count_streams, datagrams, @@ -2300,6 +2306,27 @@ impl TrackRun { continue; } + // The first group waits for the source to resolve where its feed starts; datagrams + // keep flowing meanwhile. + if let Some(group) = self.first.take() { + match self.track.poll_start(waiter) { + Poll::Ready(source) => { + self.start(group, source, stream)?; + continue; + } + Poll::Pending => { + self.first = Some(group); + if self.datagrams + && let Poll::Ready(Some(datagram)) = self.track.poll_recv_datagram(waiter)? + { + self.ctx.serve_datagram(datagram); + continue; + } + return Poll::Pending; + } + } + } + // One cursor drives the whole subscription: poll the cap-aware arrival-order // group and, when enabled, the next best-effort datagram. Groups are polled // first so a datagram burst can't starve them; datagrams flow whenever no @@ -2308,45 +2335,20 @@ impl TrackRun { if let Poll::Ready(res) = poll_recv_next(&mut self.track, self.datagrams, emit_boundary, waiter) { match res? { Recv::Group(mut group) => { - let sequence = group.sequence; if !position_group(&mut group, self.start_frame, self.end_frame) { // Its head is gone, and this subscriber didn't ask for a // partial group. Skip it rather than open a stream that can // only be reset; the next servable group resolves the start. - tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, sequence, "skipping group with a missing head"); + tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, sequence = group.sequence, "skipping group with a missing head"); + if self.emit_range && !self.start_sent { + self.skipped.insert(group.sequence); + } continue; } - if self.emit_range && !self.start_sent { - self.start_sent = true; - // Only the group: the subscriber derives the start frame from - // its own request (see `lite::SubscribeStart`). - stream - .writer - .buffer(&lite::SubscribeResponse::Start(lite::SubscribeStart { - group: sequence, - }))?; - // SUBSCRIBE_OK is an implicit drop of everything below the - // resolved start (the subscriber records it as a permanent - // miss), so a lower group arriving late must not be served - // after all. A widening SUBSCRIBE_UPDATE re-lowers the floor, - // renegotiating the resolved start along with the demand. - self.track.start_at(sequence); + match self.emit_range && !self.start_sent { + true => self.first = Some(group), + false => self.serve(group), } - - let frame_start = group.index(); - tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, sequence, "serving group"); - - // Use the latest priority for new groups so SUBSCRIBE_UPDATE applies to them too. - let current_priority = self.ctx.track_priority_current(); - // The subscribe id scopes the group tie-break: one queue serves every - // subscription on the session, and only groups of the same one may be - // ranked against each other by sequence. - let handle = self - .ctx - .priority - .insert(Priority::new(current_priority, self.ctx.id, sequence)); - self.children - .push(GroupServe::new(self.ctx.clone(), sequence, frame_start, handle, group)); } Recv::Datagram(datagram) => self.ctx.serve_datagram(datagram), Recv::Boundary(group) => { @@ -2383,6 +2385,63 @@ impl TrackRun { } } +impl TrackRun { + /// Send SUBSCRIBE_START for the first servable group, then serve it. + /// + /// A relay caches groups in upstream arrival order, and a newer group's stream can + /// beat an older one, so the first group here need not be the oldest the source + /// serves. `source` is where the source's feed starts, raised to this cursor's floor, + /// so the resolved start is the lower of the two: resolving from the later group would + /// drop the older one for good. + fn start( + &mut self, + group: group::Consumer, + source: Option, + stream: &mut Stream, + ) -> Result<(), Error> { + let mut start = source.map_or(group.sequence, |source| source.min(group.sequence)); + // A skipped group is never served, so it cannot be where the feed starts. This stops + // at the held group at the latest, since it was not skipped. + while self.skipped.contains(&start) { + start += 1; + } + self.skipped.clear(); + self.start_sent = true; + // Only the group: the subscriber derives the start frame from its own request + // (see `lite::SubscribeStart`). + stream + .writer + .buffer(&lite::SubscribeResponse::Start(lite::SubscribeStart { group: start }))?; + // SUBSCRIBE_START is an implicit drop of everything below the resolved start (the + // subscriber records it as a permanent miss), so a lower group arriving late must + // not be served after all. A widening SUBSCRIBE_UPDATE re-lowers the floor, + // renegotiating the resolved start along with the demand. Raised, not assigned: an + // update that landed while the group was held may already have raised it past. + self.track.raise_start_to(start); + self.serve(group); + Ok(()) + } + + /// Open a group machine for `group`. + fn serve(&mut self, group: group::Consumer) { + let sequence = group.sequence; + let frame_start = group.index(); + tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, sequence, "serving group"); + + // Use the latest priority for new groups so SUBSCRIBE_UPDATE applies to them too. + let current_priority = self.ctx.track_priority_current(); + // The subscribe id scopes the group tie-break: one queue serves every + // subscription on the session, and only groups of the same one may be + // ranked against each other by sequence. + let handle = self + .ctx + .priority + .insert(Priority::new(current_priority, self.ctx.id, sequence)); + self.children + .push(GroupServe::new(self.ctx.clone(), sequence, frame_start, handle, group)); + } +} + /// Serves one group on its own unidirectional stream: the header, then every /// frame, applying queue and SUBSCRIBE_UPDATE priority changes as they land. struct GroupServe { @@ -3102,6 +3161,155 @@ mod serve_group_test { assert_eq!(*log.writes.lock().unwrap(), [0, 1, 0, 1, 2, 3, 2]); } + /// A lite-07 run over a relay's track, whose upstream subscription still waits on the + /// source's SUBSCRIBE_START, on a subscribe stream the test can push updates onto. + struct RelayRun { + run: TrackRun, + stream: Stream, + session: ScriptedSession, + opens: Arc, + } + + impl RelayRun { + /// Subscribe to `track` from `start_group`. + async fn new(track: &mut track::Producer, start_group: u64) -> Self { + track.request_start(Some(0)).unwrap(); + let subscription = track::Subscription::default() + .with_start(track::Position::group(start_group)) + .with_max_age(Duration::from_secs(30)); + let subscriber = track.subscribe(subscription); + + let mut session = ScriptedSession::new(Vec::new()); + let (send, recv) = futures::future::poll_fn(|cx| { + ::poll_open_bi(&mut session, cx) + }) + .await + .unwrap(); + let stream = Stream:: { + writer: Writer::new(send, Version::Lite07), + reader: crate::coding::Reader::new(recv, Version::Lite07), + }; + let track_priority = kio::Producer::new(0u8); + let opens = Arc::::default(); + let ctx = Subscription { + session: session.clone(), + id: 0, + track_name: "test".into(), + priority: PriorityQueue::default(), + track_priority: track_priority.consume(), + track_priority_seen: 0, + version: Version::Lite07, + timescale: Some(crate::Timescale::default()), + opens: opens.clone(), + }; + let bounds = Bounds { + start_group: Some(start_group), + start_frame: 0, + end_group: None, + end_frame: None, + }; + Self { + run: TrackRun::new(ctx, subscriber, bounds, track_priority), + stream, + session, + opens, + } + } + + /// Drive the run until it parks, which it must. + fn settle(&mut self) { + let Self { run, stream, .. } = self; + let res = kio::wait(|waiter| run.poll(stream, waiter)).now_or_never(); + assert!(res.is_none(), "the run ended"); + } + + /// How many group streams the run opened. + fn opened(&self) -> u64 { + self.opens.opened.load(Ordering::Relaxed) + } + + /// Whether the first thing written was SUBSCRIBE_START at `group`. + fn started_at(&self, group: u64) -> bool { + let start = lite::SubscribeResponse::Start(lite::SubscribeStart { group }); + let start = start.encode_bytes(Version::Lite07).unwrap(); + self.session.log.writes.lock().unwrap().starts_with(&start) + } + } + + /// A SUBSCRIBE_UPDATE landing while the first group waits on the source's start keeps + /// the floor it raised: resolving the start from the held group must not lower it. + #[tokio::test] + async fn held_first_group_keeps_an_updated_floor() { + let mut track = track::Producer::new(Arc::new(broadcast::Info::default()), "test", None); + let mut relay = RelayRun::new(&mut track, 0).await; + + write_group(&mut track, 5, 0); + relay.settle(); + + // The subscriber moves its start past the held group before the source resolves. + let update = lite::SubscribeUpdate { + priority: 0, + max_age: Duration::ZERO, + start_group: Some(7), + end_group: None, + start_frame: 0, + end_frame: None, + }; + relay.session.push(&update.encode_bytes(Version::Lite07).unwrap()); + relay.settle(); + + track.start_at(0).unwrap(); + relay.settle(); + assert_eq!(relay.opened(), 1, "the held group is served"); + + // Group 6 is below the updated floor. + write_group(&mut track, 6, 6); + relay.settle(); + assert_eq!(relay.opened(), 1, "served a group below the floor"); + } + + /// A source whose feed starts below the subscriber's floor serves the floor's group, so + /// a newer group arriving first must not resolve the start past it. + #[tokio::test] + async fn held_first_group_resolves_to_the_floor_under_the_source() { + let mut track = track::Producer::new(Arc::new(broadcast::Info::default()), "test", None); + let mut relay = RelayRun::new(&mut track, 5).await; + + write_group(&mut track, 6, 6); + relay.settle(); + track.start_at(0).unwrap(); + relay.settle(); + assert_eq!(relay.opened(), 1, "the held group is served"); + + write_group(&mut track, 5, 5); + relay.settle(); + assert_eq!(relay.opened(), 2, "dropped the floor's group"); + assert!(relay.started_at(5)); + } + + /// A group skipped for a missing head before the start resolves is never served, so + /// SUBSCRIBE_START must not name it, even where the source's feed starts. + #[tokio::test] + async fn held_first_group_starts_past_a_skipped_head() { + let mut track = track::Producer::new(Arc::new(broadcast::Info::default()), "test", None); + let mut relay = RelayRun::new(&mut track, 5).await; + + // Group 5's first frame is gone. + let mut headless = track.create_group(group::Info { sequence: 5 }).unwrap(); + headless.start_at(1).unwrap(); + headless + .write_frame(Timestamp::from_millis(5).unwrap(), b"x".as_slice()) + .unwrap(); + headless.finish().unwrap(); + write_group(&mut track, 6, 6); + relay.settle(); + + track.start_at(5).unwrap(); + relay.settle(); + assert_eq!(relay.opened(), 1, "the held group is served"); + assert!(relay.started_at(6), "named the skipped group"); + } + /// A track that ends without a group still ends the subscription, with no stream owed. #[tokio::test] async fn lite07_end_counts_zero_streams() { diff --git a/rs/moq-net/src/lite/test_transport.rs b/rs/moq-net/src/lite/test_transport.rs index cfac22a22e7..a49d263aef9 100644 --- a/rs/moq-net/src/lite/test_transport.rs +++ b/rs/moq-net/src/lite/test_transport.rs @@ -674,6 +674,12 @@ impl ScriptedSession { } } + /// Append to the shared script: the peer sending more on a stream it already opened. + /// Nothing is woken, so the test re-polls the reader itself. + pub fn push(&self, bytes: &[u8]) { + self.script.lock().unwrap().extend_from_slice(bytes); + } + /// Answer each stream from `scripts`, but only once the gate opens: a peer that /// replies normally and is simply out of stream credit until then. pub fn gated_open(scripts: Vec>, gate: kio::Consumer) -> Self { diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index c7dd92e8034..4669dd10234 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -2261,6 +2261,26 @@ impl Subscriber { } } + /// Poll for where the source of the first live segment reaching this cursor's floor + /// starts, raised to the floor, once resolved; see [`track::Subscriber::poll_start`]. + /// A segment ending at or below the floor, or one that already ended, serves nothing + /// more here, so its source is not waited on. Like [`Self::poll_final`], this only + /// resolves the subscription and consumes no groups. + pub(crate) fn poll_start(&mut self, waiter: &kio::Waiter) -> Poll> { + self.poll_sync(waiter); + let floor = self.min_sequence; + for seg in &mut self.segments { + if seg.last_group().is_some_and(|last| last <= floor) { + continue; + } + ready!(Self::poll_activate(seg, &self.last_prefs, floor, waiter)); + if let SubState::Active(sub) = &mut seg.sub { + return Poll::Ready(ready!(sub.poll_start(waiter)).map(|start| start.max(floor))); + } + } + Poll::Ready(None) + } + /// Wait for the final segment's track to end: its group count when it /// finished, its error when it died, and `None` when there is no segment. Earlier segments don't /// decide the end. Only the subscription is resolved here: consuming groups, or @@ -2488,6 +2508,58 @@ mod test { recv_pending(&mut sub); } + /// A segment that ends at or below the cursor's floor serves it nothing, so its + /// source's unresolved start must not hold up the start of the segment that does. + #[tokio::test] + async fn poll_start_skips_a_segment_below_the_floor() { + let (mut track_a, consumer_a) = track_pair("a"); + let (mut track_b, consumer_b) = track_pair("b"); + track_a.request_start(Some(0)).unwrap(); + track_b.request_start(Some(5)).unwrap(); + + let mut producer = Producer::new(); + producer.switch(&consumer_a, None).unwrap(); + producer.switch(&consumer_b, Position::group(5)).unwrap(); + + let mut sub = producer.consume().subscribe(replay()); + sub.start_at(10); + assert!( + kio::wait(|waiter| sub.poll_start(waiter)).now_or_never().is_none(), + "B's source has not resolved yet" + ); + + // A's source never resolves: it serves nothing at or above the floor. + track_b.start_at(12).unwrap(); + let start = kio::wait(|waiter| sub.poll_start(waiter)).now_or_never(); + assert_eq!(start, Some(Some(12)), "waited on a segment below the floor"); + } + + /// A segment that already ended serves nothing more, so the start comes from the next + /// one rather than falling back to whichever group arrives first. + #[tokio::test] + async fn poll_start_skips_an_ended_segment() { + let (track_a, consumer_a) = track_pair("a"); + let (mut track_b, consumer_b) = track_pair("b"); + track_b.request_start(Some(5)).unwrap(); + + let mut producer = Producer::new(); + producer.switch(&consumer_a, None).unwrap(); + producer.switch(&consumer_b, Position::group(5)).unwrap(); + track_a.abort(Error::Cancel).unwrap(); + + let mut sub = producer.consume().subscribe(replay()); + // Reading drains A to its end, leaving nothing for it to serve. + recv_pending(&mut sub); + assert!( + kio::wait(|waiter| sub.poll_start(waiter)).now_or_never().is_none(), + "B's source has not resolved yet" + ); + + track_b.start_at(5).unwrap(); + let start = kio::wait(|waiter| sub.poll_start(waiter)).now_or_never(); + assert_eq!(start, Some(Some(5))); + } + #[tokio::test] async fn demand_reflects_boundaries() { let (track_a, consumer_a) = track_pair("a"); diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index e15fd8786af..10aac201e1e 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -2490,21 +2490,23 @@ impl Consumer { /// whatever it last declared, since nothing will resolve it anymore. pub(crate) fn poll_start(&self, waiter: &kio::Waiter) -> Poll> { match &self.inner { - ConsumerKind::Plain(state) => { - let res = state.poll(waiter, |state| match state.start_pending && state.abort.is_none() { - true => Poll::Pending, - false => Poll::Ready(state.start_sequence), - }); - match res { - Poll::Ready(Ok(start)) => Poll::Ready(start), - Poll::Ready(Err(state)) => Poll::Ready(state.start_sequence), - Poll::Pending => Poll::Pending, - } - } + ConsumerKind::Plain(state) => Self::poll_state_start(state, waiter), ConsumerKind::Spliced(_) => Poll::Ready(None), } } + /// [`Self::poll_start`] over one track's state, shared with [`Subscriber::poll_start`]. + fn poll_state_start(state: &kio::Consumer, waiter: &kio::Waiter) -> Poll> { + let res = state.poll(waiter, |state| match state.start_pending && state.abort.is_none() { + true => Poll::Pending, + false => Poll::Ready(state.start_sequence), + }); + match ready!(res) { + Ok(start) => Poll::Ready(start), + Err(state) => Poll::Ready(state.start_sequence), + } + } + /// The newest group, when it is already cached: resolved synchronously, without /// counting as a fetch or a delivery. The IETF publisher snapshots its frame count to /// resolve Largest Object; a group that is not immediately available reads as no edge. @@ -4006,6 +4008,19 @@ impl Subscriber { } } + /// Poll for where the source's feed starts, raised to this cursor's floor, once + /// resolved; see [`Consumer::poll_start`]. A feed starting below the floor serves the + /// floor's group too. `None` when the source declares none. + pub(crate) fn poll_start(&mut self, waiter: &kio::Waiter) -> Poll> { + match &mut self.inner { + SubscriberKind::Plain(plain) => { + let start = ready!(Consumer::poll_state_start(&plain.state, waiter)); + Poll::Ready(start.map(|start| start.max(plain.min_sequence))) + } + SubscriberKind::Spliced(spliced) => spliced.poll_start(waiter), + } + } + /// Poll for the track's declared final sequence, without blocking. pub fn poll_finished(&mut self, waiter: &kio::Waiter) -> Poll> { match &mut self.inner { diff --git a/rs/moq-net/tests/history_groups.rs b/rs/moq-net/tests/history_groups.rs new file mode 100644 index 00000000000..be515f4f94e --- /dev/null +++ b/rs/moq-net/tests/history_groups.rs @@ -0,0 +1,173 @@ +//! A fresh subscription from group 0 receives a finished older group alongside the open +//! newer one, however their streams race into a relay. +//! +//! A relay caches groups in upstream arrival order, and QUIC does not order streams, so +//! the newer group can land first. Resolving the relayed subscription's start from it +//! would drop the older group for good. The mock holds the publisher's group streams and +//! releases them newest first to make that race deterministic. + +mod support; + +use std::time::Duration; + +use moq_net::track::{Info, Position, Subscription}; +use moq_net::{Hop, Timestamp, Version}; +use support::harness::{MockConnectOptions, connect_mock}; + +const TIMEOUT: Duration = Duration::from_secs(10); + +/// A budget no group outlives, on both the publisher and the subscriber. +const FOREVER: Duration = Duration::from_millis((1 << 53) - 1); + +const VERSIONS: &[&str] = &[ + "moq-lite-03", + "moq-lite-05", + "moq-lite-07-wip", + "moq-transport-14", + "moq-transport-17", + "moq-transport-22", +]; + +fn produce_origin(hop: u64) -> moq_net::origin::Producer { + let (producer, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::new(Hop::new(hop).unwrap())); + tokio::spawn(support::harness::run(driver)); + producer +} + +/// Publish a finished group 5 (three frames) then an open group 6 (two frames), optionally +/// through a relay and with their streams delivered newest first, and return each group's +/// sequence and the frames one subscriber read from it. +async fn round(version: &str, relay: bool, newest_first: bool) -> Vec<(u64, usize)> { + let version: Version = version.parse().unwrap(); + let publisher = produce_origin(1); + let broadcast = publisher.create_broadcast("bcast").unwrap(); + let track = broadcast + .create_track("history", Info::default().with_max_age(FOREVER)) + .unwrap(); + broadcast.announce(Default::default()).unwrap(); + + let relay_origin = produce_origin(2); + let upstream = match relay { + true => { + let mut options = MockConnectOptions::new(version); + options.server_publish = Some(publisher.consume()); + options.client_subscribe = Some(relay_origin.clone()); + Some(connect_mock(options).await) + } + false => None, + }; + + let subscriber = produce_origin(3); + let mut options = MockConnectOptions::new(version); + options.server_publish = Some(match relay { + true => relay_origin.consume(), + false => publisher.consume(), + }); + options.client_subscribe = Some(subscriber.clone()); + let pair = connect_mock(options).await; + + let consumer = subscriber.consume(); + tokio::time::timeout(TIMEOUT, consumer.routed("bcast")) + .await + .expect("announce timeout") + .expect("routed"); + let remote = tokio::time::timeout(TIMEOUT, consumer.request_broadcast("bcast")) + .await + .expect("resolve timeout") + .expect("broadcast resolves"); + + let reader = tokio::spawn(async move { + let subscription = Subscription::default() + .with_max_age(FOREVER) + .with_start(Position::group(0)); + let mut sub = remote + .track("history") + .unwrap() + .subscribe(subscription) + .await + .expect("subscribe"); + + // Group 6 never ends, so read until every frame arrived or nothing more does. + let mut got = Vec::new(); + while got.iter().map(|(_, frames)| frames).sum::() < 5 { + let Ok(next) = tokio::time::timeout(TIMEOUT, sub.recv_group()).await else { + break; + }; + let Some(mut group) = next.expect("recv_group") else { + break; + }; + let mut frames = 0; + while let Ok(frame) = tokio::time::timeout(TIMEOUT, group.read_frame()).await { + match frame.expect("read_frame") { + Some(_) => frames += 1, + None => break, + } + } + got.push((group.sequence, frames)); + } + got.sort(); + got + }); + + tokio::time::timeout(TIMEOUT, track.used()) + .await + .expect("no subscriber appeared") + .unwrap(); + + // The hop the publisher's group streams cross first: into the relay, when there is one. + let first_hop = upstream + .as_ref() + .map_or(&pair.server_transport, |up| &up.server_transport); + if newest_first { + first_hop.hold_unis(); + } + + let mut old = track.create_group(moq_net::group::Info { sequence: 5 }).unwrap(); + for _ in 0..3 { + old.write_frame(Timestamp::now(), &b"old"[..]).unwrap(); + } + old.finish().unwrap(); + let mut live = track.create_group(moq_net::group::Info { sequence: 6 }).unwrap(); + for _ in 0..2 { + live.write_frame(Timestamp::now(), &b"live"[..]).unwrap(); + } + + if newest_first { + // Paused time advances only once every task is idle: both streams are open. + tokio::time::sleep(Duration::from_millis(10)).await; + first_hop.release_unis_reversed(); + } + + let got = tokio::time::timeout(TIMEOUT * 3, reader) + .await + .expect("reader hung") + .expect("reader panicked"); + drop(( + live, + track, + pair, + upstream, + relay_origin, + broadcast, + publisher, + subscriber, + )); + got +} + +#[tokio::test] +async fn a_fresh_subscriber_receives_the_finished_older_group() { + tokio::time::pause(); + let mut failures = Vec::new(); + for version in VERSIONS { + for relay in [false, true] { + for newest_first in [false, true] { + let got = round(version, relay, newest_first).await; + if got != [(5, 3), (6, 2)] { + failures.push(format!("{version} relay={relay} newest_first={newest_first}: {got:?}")); + } + } + } + } + assert!(failures.is_empty(), "{failures:#?}"); +} diff --git a/rs/moq-net/tests/support/mock.rs b/rs/moq-net/tests/support/mock.rs index 7043f020d25..c23cfbb5bff 100644 --- a/rs/moq-net/tests/support/mock.rs +++ b/rs/moq-net/tests/support/mock.rs @@ -532,6 +532,22 @@ impl MockSession { } } + /// Deliver the held uni streams newest first, and stop holding. + pub fn release_unis_reversed(&self) { + for stream in self + .side + .held + .lock() + .unwrap() + .take() + .unwrap_or_default() + .into_iter() + .rev() + { + let _ = self.side.peer_uni.try_push(stream); + } + } + /// Lose the held uni streams, as if each were reset before its header arrived, and /// stop holding. pub fn drop_unis(&self) {