From a34c849a6c3fb6ed964dffba64a230d15cf06de6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 7 Sep 2026 15:57:46 -0700 Subject: [PATCH 1/4] chore: claim quest/m0/resume-info-newest Co-Authored-By: Claude Opus 5 From 2b9fa85dec04dbbc9a718d5839a3ba8211d95b79 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 7 Sep 2026 16:58:18 -0700 Subject: [PATCH 2/4] fix(moq-net): resolve a spliced track's info from the newest segment `resume::Consumer::poll_info` resolved `track::Info` from segment zero while every data read routes to the newest segment. When a track is re-served on a live path (a source dies and a standby takes over, or a publisher reconnects), those are different generations, so a subscriber was handed the predecessor's metadata alongside the successor's frames. `timescale` is the damaging field. On moq-lite the model converts each frame timestamp into its own track's timescale and the raw value goes straight onto the wire, so TRACK_INFO advertising the predecessor's scale silently rescales every frame rather than failing. `priority` and `latency_max` were wrong but survivable. Resolve from the newest segment instead, and park on the segment list while its info is still pending so a takeover that supersedes it re-resolves against the successor rather than answering with a generation nobody will read. This is the Rust half of the defect #3422 fixed in `js/net`. No wire or API change, so no draft moves. Co-Authored-By: Claude Opus 5 --- quest/m0/README.md | 1 - quest/m0/resume-info-newest.md | 47 ------------------ rs/moq-net/src/model/origin.rs | 63 ++++++++++++++++++++++++ rs/moq-net/src/model/resume.rs | 90 +++++++++++++++++++++++++++------- rs/moq-net/src/model/track.rs | 3 +- 5 files changed, 136 insertions(+), 68 deletions(-) delete mode 100644 quest/m0/resume-info-newest.md diff --git a/quest/m0/README.md b/quest/m0/README.md index dd38c78d83c..ab81d231abf 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -23,7 +23,6 @@ regression test per Root Cause First. - [Publisher priority](/quest/m0/3534-ietf-publisher-priority.md) - moq-net: a track's publisher priority survives a moq-transport hop instead of being flattened to 0 - [TRACK_STATUS refusal](/quest/m0/3492-ietf-track-status-refusal.md) - moq-net: TRACK_STATUS gets a NOT_SUPPORTED refusal instead of a silent drop - [Connect auth race](/quest/m0/3532-connect-auth-race.md) - moq-native: a 403 on the WebSocket arm no longer fails a connect whose QUIC arm is still in flight; moq-ffi can disable the fallback -- [Resume info](/quest/m0/resume-info-newest.md) - moq-net: resume reports segment zero's track info, so a replaced broadcast rescales timestamps on the predecessor's timescale - [TS restart stall](/quest/m0/3533-ts-export-restart-stall.md) - moq export ts: a content restart on a continuous timeline no longer fences video and primary audio for good - [uring all-features](/quest/m0/uring-all-features-build.md) - moq-uring does not compile with `--all-features`, so the nightly features gate fails on it - [Failure artifacts](/quest/m0/qa-failure-artifacts.md) - a failing harness run keeps its run directory and a Playwright trace, and CI uploads them diff --git a/quest/m0/resume-info-newest.md b/quest/m0/resume-info-newest.md deleted file mode 100644 index a9eaf7db4cd..00000000000 --- a/quest/m0/resume-info-newest.md +++ /dev/null @@ -1,47 +0,0 @@ -# [M] moq-net: resume resolves track info from segment zero, not the newest - -## Goal - -`resume::Consumer::poll_info` reports the track info of the generation whose -frames a reader will actually receive. A track republished on a live path -serves the successor's `priority`, `maxAge`, and `timescale`, not the -predecessor's. - -Boundaries: this is the Rust half of the same defect `js/net` fixed by keying -its `TRACK_INFO` cache on the routing front. The wire format does not change, -so no draft moves. - -## Plan - -`resume::Consumer::poll_info` (`rs/moq-net/src/model/resume.rs`) resolves info -from **segment zero**, while every data read routes to the **newest** segment. -When a broadcast is replaced on a path, those are different generations, so a -subscriber can be handed the predecessor's metadata alongside the successor's -frames. - -`timescale` is the damaging field: the reader converts frame timestamps with -it, so a mismatched generation silently rescales every timestamp rather than -failing. `priority` and `maxAge` are wrong but survivable. - -`origin.rs`'s `run_front` already re-resolves the successor's info on -`Step::Splice` and then discards it, which is the natural place for the fix to -hang off. - -### How it was found - -Isolated while fixing the `js/net` half. Against a real relay, a publisher -wrote 1234 ticks on a successor's MICRO grid and the subscriber received -`1000us`: the JS fix alone was not enough, because the relay's own stale -metadata rescaled the frames. With the Rust side still wrong, the same -mismatch is reachable from any client. - -### Not covered by an existing tracker - -[#2991](/quest/m1/2991-net-coalesce-dynamic-tracks-and-preserve-sequences-across.md) -is about sequence continuity across replacement, not info resolution. #2610's -epoch remedy was removed from the draft by #3225, so there is no wire-level -generation marker to lean on; the fix is local. - -## Related - -- [#2991](/quest/m1/2991-net-coalesce-dynamic-tracks-and-preserve-sequences-across.md) - sequence continuity across the same replacement diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 4f2455a3dfd..7eab642bee2 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -3305,6 +3305,7 @@ impl AnnounceConsumer { #[cfg(test)] mod tests { + use crate::Timescale; use crate::coding::Decode; use crate::group; @@ -4182,6 +4183,68 @@ mod tests { sub.assert_not_closed(); } + /// A track's immutable properties belong to whichever source is serving it. + /// The successor picks its own timescale, and a reader handed the + /// predecessor's would silently rescale every frame it is about to receive. + #[tokio::test] + async fn test_track_info_follows_the_serving_source() { + tokio::time::pause(); + + let origin = Origin::random().produce(); + let consumer = origin.consume(); + + // Both routes share the first hop: interchangeable content. + let hops_a = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap(); + let hops_b = OriginList::try_from(vec![Origin::new(1).unwrap(), Origin::new(3).unwrap()]).unwrap(); + + let source_a = origin.create_broadcast("test", announce().with_hops(hops_a)).unwrap(); + let mut dynamic_a = source_a.dynamic(); + settle().await; + settle().await; + let broadcast = consumer.request_broadcast("test").await.unwrap(); + + let source_b = origin.create_broadcast("test", announce().with_hops(hops_b)).unwrap(); + let mut dynamic_b = source_b.dynamic(); + settle().await; + settle().await; + + // Held for the whole test: the reader's handle is what keeps the logical + // track spliced across the failover instead of releasing it. + let track = broadcast.track("video").unwrap(); + + // A is dispatched the track and serves it on the default grid. + let querying = track.info(); + let producer = accept_track(&mut dynamic_a, "video").await; + let info = tokio::time::timeout(std::time::Duration::from_secs(1), querying) + .await + .expect("timed out resolving the first source's info") + .unwrap(); + assert_eq!(info.timescale, Timescale::default()); + + // A dies; B re-serves the same track on a microsecond grid. + producer.abort(Error::Dropped).unwrap(); + source_a.abort(Error::Dropped).unwrap(); + drop(dynamic_a); + settle().await; + + let request = tokio::time::timeout(std::time::Duration::from_secs(1), dynamic_b.requested_track()) + .await + .expect("timed out waiting for a track request") + .expect("source closed"); + let _producer = request.accept(track::Info::default().with_timescale(Timescale::MICRO)); + settle().await; + + let info = tokio::time::timeout(std::time::Duration::from_secs(1), track.info()) + .await + .expect("timed out resolving the successor's info") + .unwrap(); + assert_eq!( + info.timescale, + Timescale::MICRO, + "info must come from the source now serving the track" + ); + } + /// Failover restores *every* subscribed track, not just one. A single-track /// takeover leaves a partial recovery indistinguishable from a whole one: /// the subscriber survives and keeps reading, but a track that is never diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index fd8c8e0d151..a450935699d 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -457,26 +457,29 @@ impl Consumer { } } - /// Poll for the track's [`track::Info`], resolved from the first segment. + /// Poll for the track's [`track::Info`], resolved from the newest segment. + /// + /// The newest segment is the generation a reader is served: every future group + /// is written there and fetches route there. Resolving from an older one hands + /// a subscriber the predecessor's `timescale`, which silently rescales the + /// successor's frame timestamps rather than failing. /// /// Stays pending until a segment exists and its track's info is known (the /// serving session may not have accepted it yet). pub fn poll_info(&self, waiter: &kio::Waiter) -> Poll> { - // Wait for the first segment (or a terminal state), then poll its info. - let track = match ready!(self.state.poll(waiter, |state| { - if state.abort.is_some() || !state.segments.is_empty() { - Poll::Ready( - state - .abort - .clone() - .map_or_else(|| Ok(state.segments[0].track.clone()), Err), - ) - } else { - Poll::Pending - } - })) { + // Wait for a segment (or a terminal state), then poll the newest one's info. + // The track is copied out of the guard: resolving it takes the underlying + // track's own lock. + let track = match ready!( + self.state + .poll(waiter, |state| match (&state.abort, state.segments.last()) { + (Some(err), _) => Poll::Ready(Err(err.clone())), + (None, Some(segment)) => Poll::Ready(Ok(segment.track.clone())), + (None, None) => Poll::Pending, + }) + ) { Ok(res) => res?, - Err(state) => match (&state.abort, state.segments.first()) { + Err(state) => match (&state.abort, state.segments.last()) { (Some(err), _) => return Poll::Ready(Err(err.clone())), (None, Some(segment)) => segment.track.clone(), // Closed without ever getting a segment: nothing will resolve this. @@ -484,10 +487,19 @@ impl Consumer { }, }; - track.info().poll_ok(waiter) + if let Poll::Ready(info) = track.info().poll_ok(waiter) { + return Poll::Ready(info); + } + + // The newest segment has not been accepted yet, so park on the segment list + // too: a takeover superseding it must re-resolve against the successor rather + // than answer with a generation nobody will read. Ready means the producer is + // gone and this segment is the last one, leaving only its info to wait on. + let _ = self.state.poll(waiter, |_| Poll::<()>::Pending); + Poll::Pending } - /// Return the track's [`track::Info`], resolved from the first segment. + /// Return the track's [`track::Info`], resolved from the newest segment. #[cfg(test)] pub async fn info(&self) -> Result { kio::wait(|waiter| self.poll_info(waiter)).await @@ -1250,7 +1262,11 @@ mod test { use std::sync::Arc; fn track_pair(name: &str) -> (track::Producer, track::Consumer) { - let producer = track::Producer::new(Arc::new(broadcast::Info::default()), name, None); + track_pair_info(name, None) + } + + fn track_pair_info(name: &str, info: impl Into>) -> (track::Producer, track::Consumer) { + let producer = track::Producer::new(Arc::new(broadcast::Info::default()), name, info.into()); let consumer = producer.consume(); (producer, consumer) } @@ -1472,8 +1488,14 @@ mod test { } #[tokio::test] - async fn info_from_first_segment() { + async fn info_from_newest_segment() { let (_track_a, consumer_a) = track_pair("a"); + let (_track_b, consumer_b) = track_pair_info( + "b", + track::Info::default() + .with_timescale(crate::Timescale::MICRO) + .with_priority(7), + ); let mut producer = Producer::new(); let consumer = producer.consume(); @@ -1484,6 +1506,36 @@ mod test { producer.switch(&consumer_a, None).unwrap(); let info = consumer.info().now_or_never().unwrap().unwrap(); assert_eq!(info.timescale, crate::Timescale::default()); + + // A track republished on the path serves the successor's generation, so its + // info is the answer: the predecessor's timescale would silently rescale + // every frame timestamp the reader is about to receive. + producer.switch(&consumer_b, 1).unwrap(); + let info = consumer.info().now_or_never().unwrap().unwrap(); + assert_eq!(info.timescale, crate::Timescale::MICRO); + assert_eq!(info.priority, 7); + } + + #[tokio::test] + async fn info_waits_for_a_takeover_superseding_an_unaccepted_segment() { + // An unaccepted request: it has a consumer, but no info yet. + let request = track::Request::new(Arc::new(broadcast::Info::default()), "a"); + let consumer_a = request.consume(); + let (_track_b, consumer_b) = + track_pair_info("b", track::Info::default().with_timescale(crate::Timescale::MICRO)); + + let mut producer = Producer::new(); + let consumer = producer.consume(); + producer.switch(&consumer_a, None).unwrap(); + + let info = consumer.info(); + let mut info = std::pin::pin!(info); + assert!(futures::poll!(info.as_mut()).is_pending(), "info should wait"); + + // The takeover supersedes it, so the answer comes from the successor rather + // than staying parked on a generation nobody will read. + producer.switch(&consumer_b, 1).unwrap(); + assert_eq!(info.await.unwrap().timescale, crate::Timescale::MICRO); } #[tokio::test] diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 4a5549104cb..4a43537297d 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1982,7 +1982,8 @@ impl Subscribing { })) } SubscribingKind::Spliced(resume) => { - // Resolved from the first segment's track. The publisher's latency + // Resolved from the newest segment's track: the generation whose + // groups this subscription will carry. The publisher's latency // window is applied to each per-session aggregate, not here. let info = ready!(resume.poll_info(waiter))?; From b9d5522f63cedae5794be54e1475b1bb89860f56 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 9 Sep 2026 13:20:50 -0700 Subject: [PATCH 3/4] fix(moq-net): revalidate metadata across concurrent takeover Co-Authored-By: GPT-6 --- rs/moq-net/src/model/resume.rs | 94 ++++++++++++++++++++++++---------- 1 file changed, 68 insertions(+), 26 deletions(-) diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index a450935699d..85a9e0d6189 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -467,36 +467,46 @@ impl Consumer { /// Stays pending until a segment exists and its track's info is known (the /// serving session may not have accepted it yet). pub fn poll_info(&self, waiter: &kio::Waiter) -> Poll> { - // Wait for a segment (or a terminal state), then poll the newest one's info. - // The track is copied out of the guard: resolving it takes the underlying - // track's own lock. - let track = match ready!( - self.state - .poll(waiter, |state| match (&state.abort, state.segments.last()) { + loop { + // Poll the underlying track outside the resume lock, then validate the + // generation under that lock before returning or registering a waiter. + let (epoch, track) = match ready!(self.state.poll(waiter, |state| { + match (&state.abort, state.segments.last()) { (Some(err), _) => Poll::Ready(Err(err.clone())), - (None, Some(segment)) => Poll::Ready(Ok(segment.track.clone())), + (None, Some(segment)) => Poll::Ready(Ok((state.epoch, segment.track.clone()))), (None, None) => Poll::Pending, - }) - ) { - Ok(res) => res?, - Err(state) => match (&state.abort, state.segments.last()) { - (Some(err), _) => return Poll::Ready(Err(err.clone())), - (None, Some(segment)) => segment.track.clone(), - // Closed without ever getting a segment: nothing will resolve this. - (None, None) => return Poll::Ready(Err(Error::Dropped)), - }, - }; + } + })) { + Ok(res) => res?, + Err(state) => match (&state.abort, state.segments.last()) { + (Some(err), _) => return Poll::Ready(Err(err.clone())), + (None, Some(segment)) => (state.epoch, segment.track.clone()), + (None, None) => return Poll::Ready(Err(Error::Dropped)), + }, + }; - if let Poll::Ready(info) = track.info().poll_ok(waiter) { - return Poll::Ready(info); + let info = track.info().poll_ok(waiter); + #[cfg(test)] + test::INFO_POLLED.with(|hook| { + if let Some(hook) = hook.take() { + hook(); + } + }); + match self.state.poll(waiter, |state| { + if state.epoch != epoch { + Poll::Ready(false) + } else if info.is_ready() { + Poll::Ready(true) + } else { + // Registration and validation share the lock, so a switch cannot + // slip between them and leave us waiting on a retired segment. + Poll::Pending + } + }) { + Poll::Ready(Ok(false)) => continue, + _ => return info, + } } - - // The newest segment has not been accepted yet, so park on the segment list - // too: a takeover superseding it must re-resolve against the successor rather - // than answer with a generation nobody will read. Ready means the producer is - // gone and this segment is the last one, leaving only its info to wait on. - let _ = self.state.poll(waiter, |_| Poll::<()>::Pending); - Poll::Pending } /// Return the track's [`track::Info`], resolved from the newest segment. @@ -1261,6 +1271,38 @@ mod test { use futures::FutureExt; use std::sync::Arc; + thread_local! { + pub(super) static INFO_POLLED: std::cell::RefCell>> = const { std::cell::RefCell::new(None) }; + } + + #[tokio::test] + async fn info_rechecks_a_concurrent_takeover() { + for accepted in [false, true] { + let request = track::Request::new(Arc::new(broadcast::Info::default()), "a"); + let consumer_a = request.consume(); + let mut request = Some(request); + let _track_a = accepted.then(|| request.take().unwrap().accept(track::Info::default())); + let (_track_b, consumer_b) = + track_pair_info("b", track::Info::default().with_timescale(crate::Timescale::MICRO)); + let mut producer = Producer::new(); + let consumer = producer.consume(); + producer.switch(&consumer_a, None).unwrap(); + + // Force the switch between the underlying poll and generation validation. + INFO_POLLED.with(|hook| { + hook.replace(Some(Box::new(move || { + producer.switch(&consumer_b, 1).unwrap(); + }))); + }); + let info = consumer + .info() + .now_or_never() + .expect("takeover must not lose a wakeup") + .unwrap(); + assert_eq!(info.timescale, crate::Timescale::MICRO); + } + } + fn track_pair(name: &str) -> (track::Producer, track::Consumer) { track_pair_info(name, None) } From 64ccc3317e8f9bfa5b81f44c1896799a3b73775c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 9 Sep 2026 14:52:07 -0700 Subject: [PATCH 4/4] fix(moq-net): reject incompatible copies during failover Preserve immutable metadata for each broadcast track and reject incompatible routes before splicing. Keep healthy copies eligible and retain the contract across idle release and reopening. Co-Authored-By: GPT-6 --- doc/bin/relay/cluster.md | 7 + rs/moq-net/src/model/origin.rs | 375 ++++++++++++++++++++------------- rs/moq-net/src/model/resume.rs | 146 +++---------- rs/moq-net/src/model/track.rs | 3 +- 4 files changed, 260 insertions(+), 271 deletions(-) diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index 49da79b92a4..2a1fd9d259c 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -16,6 +16,13 @@ same way so the cluster converges instead of flapping. Both wire protocols carry it: natively on moq-lite, and via the [cluster extension](/draft/moq-cluster) on moq-transport 17+. +Failover routes must carry copies of the same broadcast. For each track, the +relay requires matching timescale, retention window, publisher priority, and +group ordering. A source with different properties is refused before its groups +are spliced in. If no compatible source remains, the track fails with +`Unsupported`. New immutable properties require a new track name or broadcast +identity. + ## Topology List the peers each relay dials. That's the whole topology. diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 7eab642bee2..0a7d75910d3 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -1427,6 +1427,9 @@ struct FrontState { /// an exact tie toward the newest source. next_route: u64, routes: Vec, + /// Immutable track metadata, retained across idle release and aborted attempts. + /// Every route of this broadcast must serve the same content. + track_info: HashMap, track::Info>, /// Peers this front is exposed to, refcounted by live [`ExclusionGuard`]s: those /// reading it through the shared broadcast, and those we merely advertise it to. /// The resolve-time check only proves the table is clean for a requester at that @@ -1453,6 +1456,31 @@ struct FrontState { } impl FrontState { + /// Admit only copies with the broadcast's established track properties. + fn accept_track_info(&mut self, name: &Arc, info: track::Info) -> Result<(), Error> { + if self.closed { + return Err(Error::Closed); + } + if let Some(expected) = self.track_info.get(name) { + let track::Info { + timescale, + latency_max, + priority, + ordered, + } = info; + if timescale != expected.timescale + || latency_max != expected.latency_max + || priority != expected.priority + || ordered != expected.ordered + { + return Err(Error::Unsupported); + } + } else { + self.track_info.insert(name.clone(), info); + } + Ok(()) + } + /// The one selection primitive every picker goes through: the best route by /// [`route_order`] among those surviving `keep`. With `untainted`, the pick /// also steers away from routes that flow through a peer currently reading @@ -1987,6 +2015,7 @@ fn attach_source( publisher, next_route: 1, excluded: HashMap::new(), + track_info: HashMap::new(), routes: vec![FrontRoute { id: 0, route, @@ -2133,8 +2162,9 @@ async fn run_front( /// Serves one spliced logical track: splices in the best source's copy of the /// track, re-splicing on handover or failure, until the track completes or the -/// front closes. A refusal (a source rejecting the track, or its copy dying -/// before delivering anything) is authoritative and never retried: the refuser +/// front closes. A refusal (a source rejecting the track, returning incompatible +/// metadata, or dying before delivering anything) is authoritative and never +/// retried: the refuser /// is skipped for this track so a joining standby cannot kill a subscription /// the incumbent is serving, and once every attached source has refused, the /// track aborts with the last refusal's error. The verdict belongs to this @@ -2375,9 +2405,12 @@ async fn serve_track(state: kio::Producer, name: Arc, mut resum None => continue, // A copy that is already aborted can't be spliced; // its error is the source's answer for the track. - Some(Ok(_)) => match track.poll_complete(&kio::Waiter::noop()) { + Some(Ok(info)) => match track.poll_complete(&kio::Waiter::noop()) { Poll::Ready(Err(err)) => Err(err), - _ => Ok(track), + _ => match state.write() { + Ok(mut state) => state.accept_track_info(&name, info).map(|()| track), + Err(_) => Err(Error::Dropped), + }, }, Some(Err(err)) => Err(err), } @@ -3340,6 +3373,7 @@ mod tests { publisher: routes.first().and_then(|r| r.hops.iter().next().copied()), next_route: routes.len() as u64, excluded: HashMap::new(), + track_info: HashMap::new(), routes: routes .into_iter() .enumerate() @@ -4183,66 +4217,89 @@ mod tests { sub.assert_not_closed(); } - /// A track's immutable properties belong to whichever source is serving it. - /// The successor picks its own timescale, and a reader handed the - /// predecessor's would silently rescale every frame it is about to receive. + /// A source claiming the same content cannot change immutable track metadata. #[tokio::test] - async fn test_track_info_follows_the_serving_source() { + async fn test_failover_rejects_different_track_info() { tokio::time::pause(); - let origin = Origin::random().produce(); - let consumer = origin.consume(); + for replacement in [ + track::Info::default().with_timescale(Timescale::MICRO), + track::Info::default().with_priority(7), + track::Info::default().with_ordered(true), + track::Info::default().with_latency_max(Duration::from_secs(7)), + ] { + let origin = Origin::random().produce(); + let consumer = origin.consume(); - // Both routes share the first hop: interchangeable content. - let hops_a = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap(); - let hops_b = OriginList::try_from(vec![Origin::new(1).unwrap(), Origin::new(3).unwrap()]).unwrap(); + // Both routes share the first hop: interchangeable content. + let hops_a = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap(); + let hops_b = OriginList::try_from(vec![Origin::new(1).unwrap(), Origin::new(3).unwrap()]).unwrap(); - let source_a = origin.create_broadcast("test", announce().with_hops(hops_a)).unwrap(); - let mut dynamic_a = source_a.dynamic(); - settle().await; - settle().await; - let broadcast = consumer.request_broadcast("test").await.unwrap(); + let source_a = origin.create_broadcast("test", announce().with_hops(hops_a)).unwrap(); + let mut dynamic_a = source_a.dynamic(); + settle().await; + settle().await; + let broadcast = consumer.request_broadcast("test").await.unwrap(); - let source_b = origin.create_broadcast("test", announce().with_hops(hops_b)).unwrap(); - let mut dynamic_b = source_b.dynamic(); - settle().await; - settle().await; + let source_b = origin.create_broadcast("test", announce().with_hops(hops_b)).unwrap(); + let mut dynamic_b = source_b.dynamic(); + settle().await; + settle().await; - // Held for the whole test: the reader's handle is what keeps the logical - // track spliced across the failover instead of releasing it. - let track = broadcast.track("video").unwrap(); + // Held for the whole test: the reader's handle is what keeps the logical + // track spliced across the failover instead of releasing it. + let track = broadcast.track("video").unwrap(); - // A is dispatched the track and serves it on the default grid. - let querying = track.info(); - let producer = accept_track(&mut dynamic_a, "video").await; - let info = tokio::time::timeout(std::time::Duration::from_secs(1), querying) - .await - .expect("timed out resolving the first source's info") - .unwrap(); - assert_eq!(info.timescale, Timescale::default()); + // A is dispatched the track and serves it on the default grid. + let querying = track.info(); + let mut producer = accept_track(&mut dynamic_a, "video").await; + let info = tokio::time::timeout(std::time::Duration::from_secs(1), querying) + .await + .expect("timed out resolving the first source's info") + .unwrap(); + assert_eq!(info.timescale, Timescale::default()); - // A dies; B re-serves the same track on a microsecond grid. - producer.abort(Error::Dropped).unwrap(); - source_a.abort(Error::Dropped).unwrap(); - drop(dynamic_a); - settle().await; + let mut reader = track.subscribe(None).await.unwrap(); + producer + .create_group(group::Info { sequence: 0 }) + .unwrap() + .finish() + .unwrap(); + assert_eq!(reader.recv_group().await.unwrap().unwrap().sequence, 0); - let request = tokio::time::timeout(std::time::Duration::from_secs(1), dynamic_b.requested_track()) - .await - .expect("timed out waiting for a track request") - .expect("source closed"); - let _producer = request.accept(track::Info::default().with_timescale(Timescale::MICRO)); - settle().await; + // Establish a subscriber but do not read its cached predecessor group yet. + let mut joining = track.subscribe(None).await.unwrap(); - let info = tokio::time::timeout(std::time::Duration::from_secs(1), track.info()) - .await - .expect("timed out resolving the successor's info") - .unwrap(); - assert_eq!( - info.timescale, - Timescale::MICRO, - "info must come from the source now serving the track" - ); + // A dies; B claims the same content with incompatible metadata. + source_a.abort(Error::Dropped).unwrap(); + drop(dynamic_a); + settle().await; + + let request = tokio::time::timeout(std::time::Duration::from_secs(1), dynamic_b.requested_track()) + .await + .expect("timed out waiting for a track request") + .expect("source closed"); + let mut successor = request.accept(replacement); + successor + .create_group(group::Info { sequence: 1 }) + .unwrap() + .finish() + .unwrap(); + settle().await; + + assert!(matches!(track.info().await, Err(Error::Unsupported))); + let cached = joining.recv_group().await.unwrap().unwrap(); + assert_eq!(cached.sequence, 0); + assert_eq!(cached.timescale(), info.timescale); + assert!(matches!(joining.recv_group().await, Err(Error::Unsupported))); + assert!(matches!(track.subscribe(None).await, Err(Error::Unsupported))); + assert!(matches!(reader.recv_group().await, Err(Error::Unsupported))); + assert!(matches!(track.fetch_group(1, None).await, Err(Error::Unsupported))); + + // Reopening an aborted logical track must not forget the broadcast's metadata. + let reopened = broadcast.track("video").unwrap(); + assert!(matches!(reopened.info().await, Err(Error::Unsupported))); + } } /// Failover restores *every* subscribed track, not just one. A single-track @@ -4716,63 +4773,76 @@ mod tests { /// re-requesting the track (and its info) every linger. #[tokio::test(start_paused = true)] async fn test_idle_track_releases_without_respinning() { - let origin = Info::new(Origin::random()).produce(); - let consumer = origin.consume(); - - let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap(); - let source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap(); - let mut dynamic = source.dynamic(); - settle().await; - let broadcast = consumer.request_broadcast("test").await.unwrap(); + for incompatible in [false, true] { + let origin = Info::new(Origin::random()).produce(); + let consumer = origin.consume(); - let subscribing = broadcast.track("video").unwrap().subscribe(None); - let producer = accept_track(&mut dynamic, "video").await; - settle().await; - let sub = subscribing.await.unwrap(); + let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap(); + let source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap(); + let mut dynamic = source.dynamic(); + settle().await; + let broadcast = consumer.request_broadcast("test").await.unwrap(); - // The reader leaves, but the copy stays warm inside the window so a viewer - // coming back (or a follow-up fetch) reuses it. - drop(sub); - tokio::time::sleep(TRACK_IDLE_LINGER / 2).await; - settle().await; - assert!( - producer.poll_unused(&kio::Waiter::noop()).is_pending(), - "the copy must stay spliced inside the linger", - ); + let subscribing = broadcast.track("video").unwrap().subscribe(None); + let producer = accept_track(&mut dynamic, "video").await; + settle().await; + let sub = subscribing.await.unwrap(); - // Past the window the segment is released, so the serving session sees its - // copy go unused and can drop it (along with the track info). - tokio::time::sleep(TRACK_IDLE_LINGER).await; - settle().await; - assert!( - producer.poll_unused(&kio::Waiter::noop()).is_ready(), - "an idle copy must be released after the linger", - ); + // The reader leaves, but the copy stays warm inside the window so a viewer + // coming back (or a follow-up fetch) reuses it. + drop(sub); + tokio::time::sleep(TRACK_IDLE_LINGER / 2).await; + settle().await; + assert!( + producer.poll_unused(&kio::Waiter::noop()).is_pending(), + "the copy must stay spliced inside the linger", + ); - // The anti-spin property: the release must not re-arm the splice. Ungated, - // the loop re-attaches the copy immediately and drops it again every linger, - // re-requesting the track (and its info) from the session each time it dies. - for _ in 0..3 { + // Past the window the segment is released, so the serving session sees its + // copy go unused and can drop it (along with the track info). tokio::time::sleep(TRACK_IDLE_LINGER).await; settle().await; assert!( producer.poll_unused(&kio::Waiter::noop()).is_ready(), - "an unread copy must stay released, not be re-spliced", + "an idle copy must be released after the linger", ); - } - assert!( - dynamic.requested_track().now_or_never().is_none(), - "an unread track must not be re-requested", - ); - drop(producer); - // A returning reader re-splices: the origin asks the source for a fresh copy. - let subscribing = broadcast.track("video").unwrap().subscribe(None); - let mut producer = accept_track(&mut dynamic, "video").await; - settle().await; - let mut sub = subscribing.await.unwrap(); - producer.append_group().unwrap(); - assert_eq!(sub.assert_group().sequence, 0); + // The anti-spin property: the release must not re-arm the splice. Ungated, + // the loop re-attaches the copy immediately and drops it again every linger, + // re-requesting the track (and its info) from the session each time it dies. + for _ in 0..3 { + tokio::time::sleep(TRACK_IDLE_LINGER).await; + settle().await; + assert!( + producer.poll_unused(&kio::Waiter::noop()).is_ready(), + "an unread copy must stay released, not be re-spliced", + ); + } + assert!( + dynamic.requested_track().now_or_never().is_none(), + "an unread track must not be re-requested", + ); + drop(producer); + + // A returning reader re-splices: the origin asks the source for a fresh copy. + let subscribing = broadcast.track("video").unwrap().subscribe(None); + let request = dynamic.requested_track().await.unwrap(); + let info = track::Info::default().with_timescale(if incompatible { + Timescale::MICRO + } else { + Timescale::MILLI + }); + let mut producer = request.accept(info); + if incompatible { + assert!(matches!(subscribing.await, Err(Error::Unsupported))); + continue; + } + + settle().await; + let mut sub = subscribing.await.unwrap(); + producer.append_group().unwrap(); + assert_eq!(sub.assert_group().sequence, 0); + } } /// Back-to-back fetches reuse the source's copy: only the first asks the source @@ -5847,62 +5917,69 @@ mod tests { /// refused, so the subscription aborts and the consumer's next request asks /// afresh. #[tokio::test] - async fn test_standby_missing_track_keeps_incumbent() { + async fn test_standby_refusal_keeps_incumbent() { tokio::time::pause(); + for incompatible in [false, true] { + let origin = Origin::random().produce(); + let consumer = origin.consume(); - let origin = Origin::random().produce(); - let consumer = origin.consume(); - - let publisher = Origin::new(1).unwrap(); - let peer = Origin::new(5).unwrap(); - let via_peer = OriginList::try_from(vec![publisher, peer]).unwrap(); - let local = OriginList::try_from(vec![publisher]).unwrap(); + let publisher = Origin::new(1).unwrap(); + let peer = Origin::new(5).unwrap(); + let via_peer = OriginList::try_from(vec![publisher, peer]).unwrap(); + let local = OriginList::try_from(vec![publisher]).unwrap(); - // Carrying via the peer, with a live subscriber mid-stream. - let source_remote = origin - .create_broadcast("test", announce().with_hops(via_peer).with_cost(2)) - .unwrap(); - let mut dynamic_remote = source_remote.dynamic(); - settle().await; - settle().await; - let broadcast = consumer.request_broadcast("test").await.unwrap(); - let subscribing = broadcast.track("audio").unwrap().subscribe(None); - let mut producer_remote = accept_track(&mut dynamic_remote, "audio").await; - settle().await; - let mut sub = subscribing.await.unwrap(); - producer_remote.append_group().unwrap(); - assert_eq!(sub.assert_group().sequence, 0); + // Carrying via the peer, with a live subscriber mid-stream. + let source_remote = origin + .create_broadcast("test", announce().with_hops(via_peer).with_cost(2)) + .unwrap(); + let mut dynamic_remote = source_remote.dynamic(); + settle().await; + settle().await; + let broadcast = consumer.request_broadcast("test").await.unwrap(); + let subscribing = broadcast.track("audio").unwrap().subscribe(None); + let mut producer_remote = accept_track(&mut dynamic_remote, "audio").await; + settle().await; + let mut sub = subscribing.await.unwrap(); + producer_remote.append_group().unwrap(); + assert_eq!(sub.assert_group().sequence, 0); - // The standby joins and wins dispatch, but has not created "audio" yet. - // Its refusal must cost the incumbent nothing. - let source_local = origin.create_broadcast("test", announce().with_hops(local)).unwrap(); - let mut dynamic_local = source_local.dynamic(); - settle().await; - let request = dynamic_local.requested_track().await.unwrap(); - assert_eq!(request.name(), "audio"); - request.reject(Error::NotFound); - settle().await; + // The standby wins dispatch but either lacks audio or serves incompatible + // metadata. Its refusal must cost the incumbent nothing. + let source_local = origin.create_broadcast("test", announce().with_hops(local)).unwrap(); + let mut dynamic_local = source_local.dynamic(); + settle().await; + let request = dynamic_local.requested_track().await.unwrap(); + assert_eq!(request.name(), "audio"); + let incompatible_track = if incompatible { + Some(request.accept(track::Info::default().with_timescale(Timescale::MICRO))) + } else { + request.reject(Error::NotFound); + None + }; + settle().await; - // Still spliced to the incumbent, still delivering. - producer_remote.append_group().unwrap(); - assert_eq!(sub.assert_group().sequence, 1); - sub.assert_not_closed(); + // Still spliced to the incumbent, still delivering. + producer_remote.append_group().unwrap(); + assert_eq!(sub.assert_group().sequence, 1); + sub.assert_not_closed(); - // The incumbent leaving exhausts the table (the standby's refusal is - // never retried): the subscription aborts. - source_remote.abort(Error::Dropped).unwrap(); - settle().await; - settle().await; - sub.assert_closed(); - dynamic_local.assert_no_request(); + // The incumbent leaving exhausts the table (the standby's refusal is + // never retried): the subscription aborts. + source_remote.abort(Error::Dropped).unwrap(); + settle().await; + settle().await; + sub.assert_closed(); + dynamic_local.assert_no_request(); - // A fresh consumer request asks the standby anew, which has the track now. - let retry = broadcast.track("audio").unwrap().subscribe(None); - let mut producer_local = accept_track(&mut dynamic_local, "audio").await; - settle().await; - let mut sub = retry.await.expect("a fresh request must reach the standby"); - producer_local.create_group(group::Info { sequence: 2 }).unwrap(); - assert_eq!(sub.assert_group().sequence, 2); + drop(incompatible_track); + // A fresh consumer request asks the standby anew, which has the track now. + let retry = broadcast.track("audio").unwrap().subscribe(None); + let mut producer_local = accept_track(&mut dynamic_local, "audio").await; + settle().await; + let mut sub = retry.await.expect("a fresh request must reach the standby"); + producer_local.create_group(group::Info { sequence: 2 }).unwrap(); + assert_eq!(sub.assert_group().sequence, 2); + } } /// A refused track aborts, but the verdict is not cached: it belongs to the diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 85a9e0d6189..fd8c8e0d151 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -457,59 +457,37 @@ impl Consumer { } } - /// Poll for the track's [`track::Info`], resolved from the newest segment. - /// - /// The newest segment is the generation a reader is served: every future group - /// is written there and fetches route there. Resolving from an older one hands - /// a subscriber the predecessor's `timescale`, which silently rescales the - /// successor's frame timestamps rather than failing. + /// Poll for the track's [`track::Info`], resolved from the first segment. /// /// Stays pending until a segment exists and its track's info is known (the /// serving session may not have accepted it yet). pub fn poll_info(&self, waiter: &kio::Waiter) -> Poll> { - loop { - // Poll the underlying track outside the resume lock, then validate the - // generation under that lock before returning or registering a waiter. - let (epoch, track) = match ready!(self.state.poll(waiter, |state| { - match (&state.abort, state.segments.last()) { - (Some(err), _) => Poll::Ready(Err(err.clone())), - (None, Some(segment)) => Poll::Ready(Ok((state.epoch, segment.track.clone()))), - (None, None) => Poll::Pending, - } - })) { - Ok(res) => res?, - Err(state) => match (&state.abort, state.segments.last()) { - (Some(err), _) => return Poll::Ready(Err(err.clone())), - (None, Some(segment)) => (state.epoch, segment.track.clone()), - (None, None) => return Poll::Ready(Err(Error::Dropped)), - }, - }; - - let info = track.info().poll_ok(waiter); - #[cfg(test)] - test::INFO_POLLED.with(|hook| { - if let Some(hook) = hook.take() { - hook(); - } - }); - match self.state.poll(waiter, |state| { - if state.epoch != epoch { - Poll::Ready(false) - } else if info.is_ready() { - Poll::Ready(true) - } else { - // Registration and validation share the lock, so a switch cannot - // slip between them and leave us waiting on a retired segment. - Poll::Pending - } - }) { - Poll::Ready(Ok(false)) => continue, - _ => return info, + // Wait for the first segment (or a terminal state), then poll its info. + let track = match ready!(self.state.poll(waiter, |state| { + if state.abort.is_some() || !state.segments.is_empty() { + Poll::Ready( + state + .abort + .clone() + .map_or_else(|| Ok(state.segments[0].track.clone()), Err), + ) + } else { + Poll::Pending } - } + })) { + Ok(res) => res?, + Err(state) => match (&state.abort, state.segments.first()) { + (Some(err), _) => return Poll::Ready(Err(err.clone())), + (None, Some(segment)) => segment.track.clone(), + // Closed without ever getting a segment: nothing will resolve this. + (None, None) => return Poll::Ready(Err(Error::Dropped)), + }, + }; + + track.info().poll_ok(waiter) } - /// Return the track's [`track::Info`], resolved from the newest segment. + /// Return the track's [`track::Info`], resolved from the first segment. #[cfg(test)] pub async fn info(&self) -> Result { kio::wait(|waiter| self.poll_info(waiter)).await @@ -1271,44 +1249,8 @@ mod test { use futures::FutureExt; use std::sync::Arc; - thread_local! { - pub(super) static INFO_POLLED: std::cell::RefCell>> = const { std::cell::RefCell::new(None) }; - } - - #[tokio::test] - async fn info_rechecks_a_concurrent_takeover() { - for accepted in [false, true] { - let request = track::Request::new(Arc::new(broadcast::Info::default()), "a"); - let consumer_a = request.consume(); - let mut request = Some(request); - let _track_a = accepted.then(|| request.take().unwrap().accept(track::Info::default())); - let (_track_b, consumer_b) = - track_pair_info("b", track::Info::default().with_timescale(crate::Timescale::MICRO)); - let mut producer = Producer::new(); - let consumer = producer.consume(); - producer.switch(&consumer_a, None).unwrap(); - - // Force the switch between the underlying poll and generation validation. - INFO_POLLED.with(|hook| { - hook.replace(Some(Box::new(move || { - producer.switch(&consumer_b, 1).unwrap(); - }))); - }); - let info = consumer - .info() - .now_or_never() - .expect("takeover must not lose a wakeup") - .unwrap(); - assert_eq!(info.timescale, crate::Timescale::MICRO); - } - } - fn track_pair(name: &str) -> (track::Producer, track::Consumer) { - track_pair_info(name, None) - } - - fn track_pair_info(name: &str, info: impl Into>) -> (track::Producer, track::Consumer) { - let producer = track::Producer::new(Arc::new(broadcast::Info::default()), name, info.into()); + let producer = track::Producer::new(Arc::new(broadcast::Info::default()), name, None); let consumer = producer.consume(); (producer, consumer) } @@ -1530,14 +1472,8 @@ mod test { } #[tokio::test] - async fn info_from_newest_segment() { + async fn info_from_first_segment() { let (_track_a, consumer_a) = track_pair("a"); - let (_track_b, consumer_b) = track_pair_info( - "b", - track::Info::default() - .with_timescale(crate::Timescale::MICRO) - .with_priority(7), - ); let mut producer = Producer::new(); let consumer = producer.consume(); @@ -1548,36 +1484,6 @@ mod test { producer.switch(&consumer_a, None).unwrap(); let info = consumer.info().now_or_never().unwrap().unwrap(); assert_eq!(info.timescale, crate::Timescale::default()); - - // A track republished on the path serves the successor's generation, so its - // info is the answer: the predecessor's timescale would silently rescale - // every frame timestamp the reader is about to receive. - producer.switch(&consumer_b, 1).unwrap(); - let info = consumer.info().now_or_never().unwrap().unwrap(); - assert_eq!(info.timescale, crate::Timescale::MICRO); - assert_eq!(info.priority, 7); - } - - #[tokio::test] - async fn info_waits_for_a_takeover_superseding_an_unaccepted_segment() { - // An unaccepted request: it has a consumer, but no info yet. - let request = track::Request::new(Arc::new(broadcast::Info::default()), "a"); - let consumer_a = request.consume(); - let (_track_b, consumer_b) = - track_pair_info("b", track::Info::default().with_timescale(crate::Timescale::MICRO)); - - let mut producer = Producer::new(); - let consumer = producer.consume(); - producer.switch(&consumer_a, None).unwrap(); - - let info = consumer.info(); - let mut info = std::pin::pin!(info); - assert!(futures::poll!(info.as_mut()).is_pending(), "info should wait"); - - // The takeover supersedes it, so the answer comes from the successor rather - // than staying parked on a generation nobody will read. - producer.switch(&consumer_b, 1).unwrap(); - assert_eq!(info.await.unwrap().timescale, crate::Timescale::MICRO); } #[tokio::test] diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 4a43537297d..4a5549104cb 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1982,8 +1982,7 @@ impl Subscribing { })) } SubscribingKind::Spliced(resume) => { - // Resolved from the newest segment's track: the generation whose - // groups this subscription will carry. The publisher's latency + // Resolved from the first segment's track. The publisher's latency // window is applied to each per-session aggregate, not here. let info = ready!(resume.poll_info(waiter))?;