From 35999b40062c9abadc30aa4ef2d92fe8f59a0cb1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 24 Oct 2026 23:33:46 -0700 Subject: [PATCH 1/2] chore(quest): claim quest/m1/broadcast-lifecycle/lite-close-fin.md Quest-Claim: 5e1917c3aa2458be From aa032c834a0fc3cebcd20a76d7d6055915bd0a62 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 00:12:53 -0700 Subject: [PATCH 2/2] fix(net): read SUBSCRIBE FINs that raced a peer's close On lite-03/04 the subscription FIN is a track's only end, and the publisher's graceful close drains on its transport acknowledgement, so the close can reach the relay before the relay reads the FIN. The subscriber half then aborted the track instead of finishing it. After the closed transport's group streams drain, give each source one last poll so a subscription reads what the peer already delivered. A closed session's grants end with it, which is not a revocation. Co-Authored-By: Claude Opus 5.5 --- quest/m1/broadcast-lifecycle/README.md | 1 - .../m1/broadcast-lifecycle/lite-close-fin.md | 22 --- rs/moq-net/src/lite/subscriber.rs | 29 +++- .../tests/integration/relay_upstream_close.rs | 150 ++++++++++-------- 4 files changed, 102 insertions(+), 100 deletions(-) delete mode 100644 quest/m1/broadcast-lifecycle/lite-close-fin.md diff --git a/quest/m1/broadcast-lifecycle/README.md b/quest/m1/broadcast-lifecycle/README.md index d9ae3b463ca..60a6d1be908 100644 --- a/quest/m1/broadcast-lifecycle/README.md +++ b/quest/m1/broadcast-lifecycle/README.md @@ -47,7 +47,6 @@ timer on main at 1582fdc07d: ## Required - [Closing a binding broadcast ends its media](/quest/m1/broadcast-lifecycle/ffi-close.md) - moq-ffi and moq-c media handles end with a cause on close or drop, never a cause-less drop -- [lite-03/04 close FIN](/quest/m1/broadcast-lifecycle/lite-close-fin.md) - a graceful relay close delivers pending SUBSCRIBE FINs on lite-03 and lite-04 ## Related diff --git a/quest/m1/broadcast-lifecycle/lite-close-fin.md b/quest/m1/broadcast-lifecycle/lite-close-fin.md deleted file mode 100644 index 04e0a2c9e36..00000000000 --- a/quest/m1/broadcast-lifecycle/lite-close-fin.md +++ /dev/null @@ -1,22 +0,0 @@ -# [S] lite-03/04 deliver SUBSCRIBE FIN before a graceful close - -## Goal - -On lite-03 and lite-04, a graceful session close at a relay delivers every -pending SUBSCRIBE FIN before the session closes, so a track that finished -reads as finished downstream, not aborted. - -## Plan - -Decided with the maintainer on 2026-10-24: m1, in this epic, because -broadcasts now end by cause, and a dropped FIN turns a clean end into a -failure for followers on published lite versions. - -- Found by the `relay_upstream_close` test in - [moq-dev/moq#6660](https://github.com/moq-dev/moq/pull/6660): on lite-03/04 - the ACK drain closes first, so a SUBSCRIBE FIN still pending is dropped. - Later lite versions deliver it. -- Fix the close ordering at its source for those versions; no extra wait or - timer. Extend `relay_upstream_close` to leave several SUBSCRIBE FINs - pending at the close and assert every track finishes, on lite-03 and - lite-04 too. diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index ba0a93f6c9b..30f0087b36a 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -631,8 +631,9 @@ impl SubscriberDriver { self.cleanup.close(); } - /// Read only the closed transport's buffered uni streams. Control machines - /// stay owned, so their cancellation guards cannot remove subscription state. + /// Read what the closed transport already delivered: the buffered uni streams, then + /// the subscribe responses. Serve machines wait for the group streams, since ending a + /// subscription removes the state their tails are delivered into. pub fn poll_drain(&mut self, waiter: &kio::Waiter) -> Poll<()> { let closed = self.uni.poll(waiter).is_ready(); let _ = self.uni.children.poll(waiter); @@ -640,11 +641,14 @@ impl SubscriberDriver { // them can deliver it: the peer can open no more. let stuck = self.subscriber.peer_setup.ietf().is_none() && self.uni.children.len() == self.subscriber.setup_waits.load(atomic::Ordering::Relaxed); - if closed && (self.uni.children.is_empty() || stuck) { - Poll::Ready(()) - } else { - Poll::Pending + if !closed || !(self.uni.children.is_empty() || stuck) { + return Poll::Pending; } + // The peer's close can win the race against a SUBSCRIBE FIN it sent first, and on + // Lite03/04 that FIN is the only end: read it, so the track finishes rather than + // aborts. Each source reads its streams once, then retires on the closed session. + let _ = self.sources.poll(waiter); + Poll::Ready(()) } pub fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { @@ -6618,7 +6622,18 @@ impl kio::Task for TrackServeRun { } ready!(self.gate.poll_ready(waiter)); // A finished track only waits for its FIN to be acknowledged: nothing is left to revoke. - if !matches!(self.state, TrackRunState::Finish(_)) && self.gate.poll_denied(waiter).is_ready() { + // Nor on a closed session, whose grants end with it: the session's end decides how + // its tracks end. + if !matches!(self.state, TrackRunState::Finish(_)) + && self.gate.poll_denied(waiter).is_ready() + && self + .serve + .subscriber + .session + .clone() + .poll_closed(&mut waiter.context()) + .is_pending() + { self.revoke(); return Poll::Ready(()); } diff --git a/rs/moq-net/tests/integration/relay_upstream_close.rs b/rs/moq-net/tests/integration/relay_upstream_close.rs index ee3096ca8dd..d38de9c48b2 100644 --- a/rs/moq-net/tests/integration/relay_upstream_close.rs +++ b/rs/moq-net/tests/integration/relay_upstream_close.rs @@ -1,6 +1,6 @@ #![cfg(feature = "async")] -//! A finished track keeps reaching a relay's existing subscribers after the publisher's +//! Finished tracks keep reaching a relay's existing subscribers after the publisher's //! session to the relay ends, cleanly or not. //! //! A broadcast has no finished state: a publisher ends its content with a track FIN and @@ -9,7 +9,7 @@ //! //! Deterministic: simulated time over the mock transport. The relay's group streams to //! the subscriber are held, with a one-message stream buffer, so the relay still holds -//! most of the track when the upstream session ends. +//! most of each track when the upstream session ends. use crate::support; @@ -29,6 +29,8 @@ const TIMEOUT: Duration = Duration::from_secs(10); const MAX_DELAY: Duration = Duration::from_secs(1); const GROUPS: u64 = 4; const FRAMES: u64 = 8; +/// Tracks finished together, so several SUBSCRIBE FINs are pending when the session ends. +const TRACKS: usize = 3; const VERSIONS: &[&str] = &[ "moq-lite-03", @@ -56,13 +58,44 @@ enum End { Abort, } -/// Publish a finished track through a relay, end the publisher's session while the -/// subscriber has read none of it, and return the frames it then reads and how it ended. -async fn round(name: &str, end: End) -> (Vec<(u64, u64)>, Result<(), Error>) { +/// What one subscription read: its `(group, frame)` pairs, then how it ended. +type Outcome = (Vec<(u64, u64)>, Result<(), Error>); + +/// Read a subscription to its end. +async fn read(mut sub: moq_net::track::Subscriber) -> Outcome { + let mut got = Vec::new(); + loop { + let mut group = match sub.recv_group().await { + Ok(Some(group)) => group, + Ok(None) => return (got, Ok(())), + Err(err) => return (got, Err(err)), + }; + loop { + match group.read_frame().await { + Ok(Some(frame)) => { + let index: u64 = std::str::from_utf8(&frame.payload).unwrap().parse().unwrap(); + got.push((group.sequence, index)); + } + Ok(None) => break, + Err(err) => return (got, Err(err)), + } + } + } +} + +/// Publish finished tracks through a relay, end the publisher's session while the +/// subscriber has read none of them, and return what each subscription then reads. +async fn round(name: &str, end: End) -> Vec { let version: Version = name.parse().unwrap(); let publisher = produce_origin(1); let broadcast = publisher.create_broadcast("bcast").unwrap(); - let track = broadcast.create_track("video", timed()).unwrap(); + let tracks: Vec<_> = (0..TRACKS) + .map(|index| { + broadcast + .create_track(format!("track{index}").as_str(), timed()) + .unwrap() + }) + .collect(); broadcast.announce(Default::default()).unwrap(); let relay = produce_origin(2); @@ -88,55 +121,33 @@ async fn round(name: &str, end: End) -> (Vec<(u64, u64)>, Result<(), Error>) { .expect("resolve timeout") .expect("broadcast resolves"); - let (ready, active) = futures::channel::oneshot::channel(); - let reader = moq_clock::sim::spawn(async move { + let mut readers = Vec::new(); + for (index, track) in tracks.iter().enumerate() { let subscription = Subscription::default().with_max_delay(MAX_DELAY).with_range(0..); - let mut sub = remote - .track("video") - .unwrap() - .subscribe(subscription) + let sub = remote.track(&format!("track{index}")).unwrap().subscribe(subscription); + let sub = moq_clock::timeout(TIMEOUT, sub) .await + .expect("the subscription never became active") .expect("subscribe"); - let _ = ready.send(()); - let mut got = Vec::new(); - loop { - let mut group = match sub.recv_group().await { - Ok(Some(group)) => group, - Ok(None) => return (got, Ok(())), - Err(err) => return (got, Err(err)), - }; - loop { - match group.read_frame().await { - Ok(Some(frame)) => { - let index: u64 = std::str::from_utf8(&frame.payload).unwrap().parse().unwrap(); - got.push((group.sequence, index)); - } - Ok(None) => break, - Err(err) => return (got, Err(err)), - } - } - } - }); - - moq_clock::timeout(TIMEOUT, support::harness::subscribed(&track)) - .await - .expect("no subscriber appeared"); - moq_clock::timeout(TIMEOUT, active) - .await - .expect("the subscription never became active") - .expect("reader alive"); + readers.push(moq_clock::sim::spawn(read(sub))); + moq_clock::timeout(TIMEOUT, support::harness::subscribed(track)) + .await + .expect("no subscriber appeared"); + } // The subscriber now reads nothing until the upstream session is gone. downstream.server_transport.hold_unis(); - for sequence in 0..GROUPS { - let mut group = track.append_group().unwrap(); - for index in 0..FRAMES { - let at = Timestamp::from_millis(sequence * FRAMES + index).unwrap(); - group.write_frame(at, index.to_string().into_bytes()).unwrap(); + for track in &tracks { + for sequence in 0..GROUPS { + let mut group = track.append_group().unwrap(); + for index in 0..FRAMES { + let at = Timestamp::from_millis(sequence * FRAMES + index).unwrap(); + group.write_frame(at, index.to_string().into_bytes()).unwrap(); + } + group.finish().unwrap(); } - group.finish().unwrap(); + track.finish().unwrap(); } - track.finish().unwrap(); match end { End::Close => moq_clock::timeout(TIMEOUT, upstream.server.clone().close()) @@ -150,41 +161,40 @@ async fn round(name: &str, end: End) -> (Vec<(u64, u64)>, Result<(), Error>) { upstream.server.abort(Error::Cancel); } } - drop((track, broadcast, publisher, upstream)); + drop((tracks, broadcast, publisher, upstream)); // The relay sees the session and its broadcast go before the subscriber reads on. moq_clock::sleep(Duration::from_millis(100)).await; downstream.server_transport.release_unis(); - let outcome = moq_clock::timeout(TIMEOUT, reader) - .await - .unwrap_or_else(|_| panic!("{name} {end:?}: the track never ended")) - .expect("reader panicked"); + let mut outcomes = Vec::new(); + for reader in readers { + let outcome = moq_clock::timeout(TIMEOUT, reader) + .await + .unwrap_or_else(|_| panic!("{name} {end:?}: a track never ended")) + .expect("reader panicked"); + outcomes.push(outcome); + } drop((downstream, relay, subscriber)); - outcome + outcomes } -/// Every frame, then a clean end. Lite03 and Lite04 declare no end, so the relay is still -/// waiting out its tail grace when the session goes. -/// -/// Their close is left out: it drains on the transport's acknowledgement of the -/// subscription's FIN, which the relay may not have read when the session closes, and -/// without a declared end nothing else says the track is over. +/// Every frame of every track, then a clean end. Lite03 and Lite04 declare no end, so the +/// relay is still waiting out its tail grace when the session goes, and on a close each +/// track's SUBSCRIBE FIN is its only end: the transport acknowledges it, which lets the +/// close drain, before the relay has read it. #[moq_clock::test] -async fn relay_delivers_a_finished_track_after_upstream_closes() { +async fn relay_delivers_finished_tracks_after_upstream_closes() { let want: Vec<(u64, u64)> = (0..GROUPS) .flat_map(|group| (0..FRAMES).map(move |frame| (group, frame))) .collect(); for version in VERSIONS { - let ends: &[End] = match *version { - "moq-lite-03" | "moq-lite-04" => &[End::Abort], - _ => &[End::Close, End::Abort], - }; - for &end in ends { - let (mut got, result) = round(version, end).await; - // Groups arrive in any order; each one's frames arrive in order. - got.sort_by_key(|&(group, _)| group); - assert_eq!(got, want, "{version} {end:?}: result={result:?}"); - assert!(result.is_ok(), "{version} {end:?}: ended with {result:?}"); + for end in [End::Close, End::Abort] { + for (track, (mut got, result)) in round(version, end).await.into_iter().enumerate() { + // Groups arrive in any order; each one's frames arrive in order. + got.sort_by_key(|&(group, _)| group); + assert_eq!(got, want, "{version} {end:?} track{track}: result={result:?}"); + assert!(result.is_ok(), "{version} {end:?} track{track}: ended with {result:?}"); + } } } }