From 7337c2a39545d89eb0355850ceea37c9d8f71ad1 Mon Sep 17 00:00:00 2001 From: David von Wrangel Date: Sat, 31 Oct 2026 15:27:09 -0700 Subject: [PATCH 1/4] feat(net): report when a session carries no subscription or fetch `Session::poll_idle` and `Session::idle` resolve once the session serves and holds no subscription and has no fetch in flight. Each one holds a guard on a count the session's tasks share through its transport. Announcements, track info, track listings, PROBE, and AUTH don't count. A reconnecting client uses it to close a migrated session as soon as nothing flows on it. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/ietf/publisher.rs | 13 +- rs/moq-net/src/ietf/publisher/tracks.rs | 2 + rs/moq-net/src/ietf/subscriber.rs | 2 + rs/moq-net/src/lite/publisher.rs | 9 ++ rs/moq-net/src/lite/subscriber.rs | 41 +++-- rs/moq-net/src/session.rs | 50 ++++++ rs/moq-net/src/transport/boxed.rs | 8 + rs/moq-net/tests/integration/main.rs | 1 + rs/moq-net/tests/integration/session_idle.rs | 156 +++++++++++++++++++ 9 files changed, 270 insertions(+), 12 deletions(-) create mode 100644 rs/moq-net/tests/integration/session_idle.rs diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 6f26ce84ac7..21ad0f2b9c4 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -810,11 +810,15 @@ enum FetchGate { Joining { id: RequestId, auth: crate::auth::Link }, } -struct Serve(Arc); +/// A dispatched SUBSCRIBE or FETCH serve, counted until dropped. +struct Serve { + owed: Arc, + _working: crate::session::Working, +} impl Drop for Serve { fn drop(&mut self) { - self.0.fetch_sub(1, Ordering::Relaxed); + self.owed.fetch_sub(1, Ordering::Relaxed); } } @@ -1024,7 +1028,10 @@ impl Publisher { // Count at dispatch so close cannot pass a request waiting for its first poll. let serve = matches!(id, ietf::Subscribe::ID | ietf::Fetch::ID).then(|| { self.owed.fetch_add(1, Ordering::Relaxed); - Serve(self.owed.clone()) + Serve { + owed: self.owed.clone(), + _working: self.session.work().start(), + } }); let task = match id { ietf::Subscribe::ID => { diff --git a/rs/moq-net/src/ietf/publisher/tracks.rs b/rs/moq-net/src/ietf/publisher/tracks.rs index 0c261c6bff4..33b280085c3 100644 --- a/rs/moq-net/src/ietf/publisher/tracks.rs +++ b/rs/moq-net/src/ietf/publisher/tracks.rs @@ -1169,6 +1169,8 @@ impl Publisher { goaways: &mut super::super::request_stream::Messages, opened: &mut u64, ) -> Result { + // Paused, the PUBLISH is only a listing entry; forwarding, it is a subscription. + let _working = self.session.work().start(); let mut subscription = serving_subscription(params.priority); if let Some(order) = params.order { subscription = subscription.with_order(order.to_order()); diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index afcddcca3f0..7415690a399 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -2933,6 +2933,7 @@ impl Subscriber { group_fetches: &mut TaskSet, gate: &mut crate::auth::Gate, ) -> Option { + let _working = self.session.work().start(); let validity = self.validity.start(); let mut resolved = ResolvedRanges::default(); let subscription = target @@ -5373,6 +5374,7 @@ impl Subscriber { /// hold the earlier groups until it finishes, or be cut short at the Largest Object by a /// publisher following draft-22 section 9.11. async fn serve_fill(self, broadcast: Path, name: String, fill: track::Fill, timescale: Option) { + let _working = self.session.work().start(); let fill = std::sync::Arc::new(fill); let mut last = None; while let Some(run) = fill.next_run() { diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index a5cb9cdde29..8100c379143 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -1537,6 +1537,9 @@ trait Request: Sized { /// Whether the request counts toward the session's subscriptions peak while it is open. /// Its stream's credit, not a count, bounds how many the peer holds. const SUBSCRIPTION: bool = false; + /// Whether the request carries track data (a subscription or a fetch), which keeps + /// the session from reading as idle while it is open. + const DATA: bool = false; /// Whether the request stays open past its reply, keeping what serving it took, until /// the requester closes its side. const HOLDS: bool; @@ -1596,6 +1599,8 @@ struct RequestServe { owing: bool, // A subscription's place in the session's subscriptions peak. _subscription: Option, + // Holds the session busy while a subscription or fetch is open. + _working: Option, // The latest update to arrive before the reply started. Each one replaces the last. update: Option, /// Ends the request, even mid-reply, once the session stops allowing its broadcast. @@ -1635,12 +1640,14 @@ enum RequestState { impl RequestServe { fn new(shared: Arc, stream: Stream) -> Self { shared.owed.fetch_add(1, Ordering::Relaxed); + let _working = R::DATA.then(|| shared.session.work().start()); Self { shared, stream: Some(stream), state: RequestState::Decode, owing: true, _subscription: None, + _working, update: None, gate: None, broadcast: Default::default(), @@ -2024,6 +2031,7 @@ impl Request for SubscribeServe { // Ending a subscription early is still a valid end. const GRACEFUL: bool = true; const SUBSCRIPTION: bool = true; + const DATA: bool = true; const HOLDS: bool = false; type Message = lite::Subscribe<'static>; type Update = lite::SubscribeUpdate; @@ -2268,6 +2276,7 @@ impl Request for FetchServe { const KIND: &'static str = "fetch"; // The response carries no header, so a run cut short would read as whole. const GRACEFUL: bool = false; + const DATA: bool = true; const HOLDS: bool = false; type Message = lite::Fetch<'static>; type Update = NoUpdate; diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index df50ec77974..cad3a101a1f 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -2147,6 +2147,7 @@ mod tests { unanswered: None, bounded: false, updated: false, + _working: crate::session::Work::default().start(), }) } @@ -2322,6 +2323,7 @@ mod tests { unanswered: None, bounded: false, updated: false, + _working: crate::session::Work::default().start(), }); if update { h.serve @@ -2787,6 +2789,7 @@ mod tests { unanswered: None, bounded: false, updated: false, + _working: crate::session::Work::default().start(), }; assert_eq!( sub.owed(None), @@ -6538,6 +6541,8 @@ struct SubStream { /// Whether an update changed the ranges since the SUBSCRIBE, which a publisher may /// FIN before reading. updated: bool, + /// Holds the session busy for as long as the subscription is open. + _working: crate::session::Working, } impl SubStream { @@ -7106,6 +7111,7 @@ impl TrackServe { Establish { validity: self.validity.start(), serve: self.clone(), + _working: session.work().start(), closed: session.clone(), session, id, @@ -7220,6 +7226,8 @@ struct Establish { /// The ranges the SUBSCRIBE carries for `subscription`. ranges: Vec, tail: kio::Producer, + /// Holds the session busy while the SUBSCRIBE opens. + _working: crate::session::Working, state: EstablishState, } @@ -7297,6 +7305,7 @@ impl Establish { wanted: None, bounded: bounded(&self.ranges), updated: false, + _working: self.session.work().start(), } } } @@ -7324,7 +7333,11 @@ enum TrackRunState { }, Serve(ServeLoop), /// Preserve the lite07 completion FIN until the publisher acknowledges it. - Finish(crate::coding::Writer), + Finish { + writer: crate::coding::Writer, + /// The subscription keeps the session busy until then. + _working: crate::session::Working, + }, Done, } @@ -7368,7 +7381,7 @@ impl TrackServeRun { } // Each fetch in flight resets itself as it drops, seeing the same denial. } - TrackRunState::Finish(_) | TrackRunState::Done => {} + TrackRunState::Finish { .. } | TrackRunState::Done => {} } } } @@ -7407,7 +7420,7 @@ impl kio::Task for TrackServeRun { // A finished track only waits for its FIN to be acknowledged: nothing is left to revoke. // Nor on a closed session, whose grants end with it: the session's end decides how // its tracks end. - if !matches!(self.state, TrackRunState::Finish(_)) + if !matches!(self.state, TrackRunState::Finish { .. }) && self.gate.poll_denied(waiter).is_ready() && self.serve.subscriber.session.clone().poll_closed(waiter).is_pending() { @@ -7483,14 +7496,17 @@ impl kio::Task for TrackServeRun { if active.stream.writer.finish().is_ok() && finished && self.serve.subscriber.version.waits_for_subscriber_fin() { - self.state = TrackRunState::Finish(active.stream.writer); + self.state = TrackRunState::Finish { + writer: active.stream.writer, + _working: active._working, + }; continue; } } return Poll::Ready(()); } - TrackRunState::Finish(writer) => { + TrackRunState::Finish { writer, .. } => { let _ = ready!(writer.poll_close(waiter)); self.state = TrackRunState::Done; } @@ -7647,8 +7663,12 @@ struct ServeLoop { /// demand still asks for just them: subscribing again would only resend them. covered: Option>, /// Our side of the covered subscription, finished and held until the publisher reads - /// the FIN, since dropping it would reset it. - closing: Option>, + /// the FIN, since dropping it would reset it. The subscription keeps the session busy + /// until then. + closing: Option<( + crate::coding::Writer, + crate::session::Working, + )>, } #[expect( @@ -7997,7 +8017,7 @@ impl ServeLoop { serve.subscriber.remove_subscribe(active.id); let mut writer = active.stream.writer; if writer.finish().is_ok() { - self.closing = Some(writer); + self.closing = Some((writer, active._working)); } // Unchanged ranges are what the publisher covered, even if a datagram // went missing. Changed ones settled by the grace instead may have @@ -8025,7 +8045,7 @@ impl ServeLoop { return Poll::Pending; } ServeMode::Select => { - if let Some(writer) = &mut self.closing + if let Some((writer, _)) = &mut self.closing && writer.poll_close(waiter).is_ready() { self.closing = None; @@ -8397,6 +8417,8 @@ struct FetchServeRun { info: track::Info, group: u64, state: FetchRunState, + /// Holds the session busy until the fetch ends. + _working: crate::session::Working, } enum FetchRunState { @@ -8445,6 +8467,7 @@ impl FetchServeRun { validity: serve.validity.start(), gate: serve.gate(), serve, + _working: session.work().start(), session, info, group, diff --git a/rs/moq-net/src/session.rs b/rs/moq-net/src/session.rs index b41f20b6147..5637cb78ae4 100644 --- a/rs/moq-net/src/session.rs +++ b/rs/moq-net/src/session.rs @@ -172,6 +172,8 @@ pub struct Session { goaway: Arc, auth: auth::Handle, credential: Option, + /// The subscriptions and fetches the protocol tasks carry. + work: Work, } impl Session { @@ -361,6 +363,21 @@ impl Session { self.goaway.consumer() } + /// Poll until the session carries no subscription or fetch, in either direction. + /// + /// Ready at once while it carries none. Announcements and the other control + /// streams don't count, so a session can be idle while it still serves discovery. + pub fn poll_idle(&self, waiter: &kio::Waiter) -> Poll<()> { + self.work.poll_idle(waiter) + } + + /// Wait until the session carries no subscription or fetch, in either direction. + /// + /// See [`poll_idle`](Self::poll_idle). + pub async fn idle(&self) { + kio::wait(|waiter| self.poll_idle(waiter)).await + } + /// The tokens this side presented and the grant they earned, plus the tokens /// the peer presents. See [`auth`]. /// @@ -445,6 +462,7 @@ impl Session { sample, demanded: false, }); + let work = session.work().clone(); let supervisor = Supervisor { goaway: Some(goaway.deadline(&runtime)), @@ -478,6 +496,7 @@ impl Session { goaway, auth: auth.handle().clone(), credential, + work, }; let driver = crate::Driver::new( runtime.clone(), @@ -887,6 +906,37 @@ impl Drop for Withdrawing { } } +/// The subscriptions and fetches a session carries in either direction, which +/// [`Session::idle`] waits to reach zero. +#[derive(Clone, Default)] +pub(crate) struct Work(kio::Shared); + +impl Work { + /// Count one subscription or fetch until the returned guard drops. + pub(crate) fn start(&self) -> Working { + *self.0.lock() += 1; + Working(self.clone()) + } + + fn poll_idle(&self, waiter: &kio::Waiter) -> Poll<()> { + self.0 + .poll( + waiter, + |count| if **count == 0 { Poll::Ready(()) } else { Poll::Pending }, + ) + .map(|_| ()) + } +} + +/// One subscription or fetch counted in [`Work`], released on drop. +pub(crate) struct Working(Work); + +impl Drop for Working { + fn drop(&mut self) { + *self.0.0.lock() -= 1; + } +} + /// Bounds on what one peer can make a session hold, where its transport does not already. /// /// Each request holds a stream of its own, so the transport's stream credit bounds those. diff --git a/rs/moq-net/src/transport/boxed.rs b/rs/moq-net/src/transport/boxed.rs index 4752b98a2e8..d2bfa6cee35 100644 --- a/rs/moq-net/src/transport/boxed.rs +++ b/rs/moq-net/src/transport/boxed.rs @@ -19,6 +19,8 @@ pub(crate) struct Session { inner: std::sync::Arc, /// Shared by every clone, so whichever task closes the transport first says why. closing: std::sync::Arc, + /// Shared by every clone, so each task counts the subscriptions and fetches it carries. + work: crate::session::Work, } impl Session { @@ -26,6 +28,7 @@ impl Session { Self { inner: std::sync::Arc::new(session), closing: Default::default(), + work: Default::default(), } } @@ -34,6 +37,11 @@ impl Session { &self.closing } + /// The subscriptions and fetches the session's tasks carry over this transport. + pub(crate) fn work(&self) -> &crate::session::Work { + &self.work + } + /// Close with `code` over `cause`, this side's reason for ending the session. pub(crate) fn abort(&self, cause: &crate::Error, code: u32, reason: &str) { self.closing.sent(code, Some(cause)); diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index 691eba77fe1..396dab34d68 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -52,6 +52,7 @@ mod route_change; mod routing_contracts; mod segmented_frame; mod session_close; +mod session_idle; mod session_panic; mod settled_end_survives_abort; mod setup_negotiation; diff --git a/rs/moq-net/tests/integration/session_idle.rs b/rs/moq-net/tests/integration/session_idle.rs new file mode 100644 index 00000000000..53123dec4ea --- /dev/null +++ b/rs/moq-net/tests/integration/session_idle.rs @@ -0,0 +1,156 @@ +#![cfg(feature = "async")] + +//! `Session::idle` reports a session that carries no subscription or fetch in either +//! direction, which is when a migrated session can close without cutting anything off. +//! +//! Deterministic: simulated time over the mock transport. + +use crate::support; + +use std::time::Duration; + +use moq_net::{Hop, Timestamp, Version, kio}; +use support::harness::{MockConnectOptions, MockPair, connect_mock}; + +/// A millisecond track: the frames written here carry timestamps. +fn timed() -> moq_net::track::Info { + moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI) +} + +const TIMEOUT: Duration = Duration::from_secs(10); + +fn produce_origin(hop: u64) -> moq_net::origin::Producer { + let (producer, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::new(Hop::new(hop).unwrap())); + support::harness::spawn(driver); + producer +} + +fn idle(session: &moq_net::Session) -> bool { + session.poll_idle(&kio::Waiter::noop()).is_ready() +} + +/// A client publishing one announced track to a server that subscribes to whatever it hears. +struct Setup { + pair: MockPair, + track: moq_net::track::Producer, + remote: moq_net::broadcast::Consumer, + _broadcast: moq_net::origin::Publication, +} + +async fn setup(version: &str) -> Setup { + let publisher = produce_origin(1); + let broadcast = publisher.create_broadcast("bcast").unwrap(); + let track = broadcast.create_track("video", timed()).unwrap(); + broadcast.announce(Default::default()).unwrap(); + + let subscriber = produce_origin(2); + let mut options = MockConnectOptions::new(version.parse::().unwrap()); + options.client_publish = Some(publisher.consume()); + options.server_subscribe = Some(subscriber.clone()); + let pair = connect_mock(options).await; + + let remote = moq_clock::timeout(TIMEOUT, subscriber.consume().follow("bcast", None).unwrap().available()) + .await + .expect("resolve timeout") + .expect("broadcast resolves"); + + Setup { + pair, + track, + remote, + _broadcast: broadcast, + } +} + +/// An announcement alone leaves both ends idle. A subscription keeps the end serving it +/// and the end holding it busy until the reader leaves. +#[moq_clock::test] +async fn a_subscription_keeps_both_ends_busy() { + for version in [ + "moq-lite-04", + "moq-lite-06", + "moq-lite-07-wip", + "moq-transport-18", + "moq-transport-22", + ] { + let Setup { + pair, + track, + remote, + _broadcast, + } = setup(version).await; + assert!(idle(&pair.client), "{version}: an announcement kept the publisher busy"); + assert!( + idle(&pair.server), + "{version}: an announcement kept the subscriber busy" + ); + + let mut group = track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, b"live".as_ref()).unwrap(); + group.finish().unwrap(); + let mut sub = moq_clock::timeout(TIMEOUT, remote.track("video").unwrap().subscribe(None)) + .await + .expect("subscribe timeout") + .expect("subscribe"); + let received = moq_clock::timeout(TIMEOUT, sub.recv_group()) + .await + .expect("recv timeout") + .expect("recv") + .expect("the track ended"); + assert_eq!(received.sequence, 0, "{version}"); + assert!(!idle(&pair.client), "{version}: idle while serving a subscription"); + assert!(!idle(&pair.server), "{version}: idle while holding a subscription"); + + drop((received, sub)); + moq_clock::timeout(TIMEOUT, pair.server.idle()) + .await + .unwrap_or_else(|_| panic!("{version}: the subscriber stayed busy after the reader left")); + moq_clock::timeout(TIMEOUT, pair.client.idle()) + .await + .unwrap_or_else(|_| panic!("{version}: the publisher stayed busy after the reader left")); + } +} + +/// A fetch keeps both ends busy until it ends: here, until the group it reads finishes. +#[moq_clock::test] +async fn a_fetch_keeps_both_ends_busy() { + for version in ["moq-lite-06", "moq-lite-07-wip", "moq-transport-22"] { + let Setup { + pair, + track, + remote, + _broadcast, + } = setup(version).await; + + let mut writing = track.append_group().unwrap(); + writing.write_frame(Timestamp::ZERO, b"first".as_ref()).unwrap(); + let fetching = remote.track("video").unwrap(); + let fetch = moq_clock::sim::spawn(async move { + let mut fetched = fetching.fetch_group(0, None).await?; + let mut frames = Vec::new(); + while let Some(frame) = fetched.read_frame().await? { + frames.push(frame.payload); + } + Ok::<_, moq_net::Error>(frames) + }); + // Simulated: every task runs until the fetch waits on the unfinished group. + moq_clock::sleep(Duration::from_secs(1)).await; + assert!(!idle(&pair.client), "{version}: idle while serving a fetch"); + assert!(!idle(&pair.server), "{version}: idle while a fetch is in flight"); + + writing.write_frame(Timestamp::ZERO, b"last".as_ref()).unwrap(); + writing.finish().unwrap(); + let frames = moq_clock::timeout(TIMEOUT, fetch) + .await + .expect("fetch timeout") + .expect("fetch task") + .expect("fetch"); + assert_eq!(frames, [&b"first"[..], &b"last"[..]], "{version}"); + moq_clock::timeout(TIMEOUT, pair.server.idle()) + .await + .unwrap_or_else(|_| panic!("{version}: the fetching side stayed busy after the fetch")); + moq_clock::timeout(TIMEOUT, pair.client.idle()) + .await + .unwrap_or_else(|_| panic!("{version}: the serving side stayed busy after the fetch")); + } +} From 2ca5125ad4123492b71bdbd2c4dbe292a5765ed4 Mon Sep 17 00:00:00 2001 From: David von Wrangel Date: Sat, 31 Oct 2026 15:27:09 -0700 Subject: [PATCH 2/4] fix(tokio): close a migrated session once it carries nothing After a GOAWAY, the old session was held until the peer closed it or the handover cap ran out, even with nothing left on it. Once the replacement is connected, the reconnect loop now closes the old session gracefully as soon as it is idle. The cap still bounds one that never goes idle. Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/config.md | 3 +- rs/moq-tokio/src/connection.rs | 40 +++- rs/moq-tokio/tests/integration/main.rs | 1 + rs/moq-tokio/tests/integration/migrate.rs | 237 ++++++++++++++++++++++ 4 files changed, 273 insertions(+), 8 deletions(-) create mode 100644 rs/moq-tokio/tests/integration/migrate.rs diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index 621ea141b1f..d7bd6c2f5d6 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -334,7 +334,8 @@ A draining upstream may name a replacement URI. `same-host` follows it only onto the host already dialed, so a peer can move us between ports and schemes; `follow` also lets it choose the host, which trusts it not to point us into the local network; `ignore` keeps the current address. `handover` caps how long -the old connection keeps serving after a GOAWAY. +the old connection keeps serving after a GOAWAY; it closes sooner, once the +replacement is up and it carries no subscription or fetch. ## \[cache] diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index 1c53e72f53a..2fdf37454ec 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -438,10 +438,11 @@ pub struct Goaway { )] pub redirect: Redirect, - /// How long the old session keeps serving after its replacement connects, e.g. - /// "10s" or "500ms". This is a cap: a GOAWAY naming a shorter deadline wins, - /// since the peer force-closes at its own deadline regardless, but a longer one - /// does not extend it. Defaults to 10 seconds. + /// The longest the old session keeps serving after a GOAWAY, e.g. "10s" or + /// "500ms". It closes sooner, once its replacement is up and it carries no + /// subscription or fetch. A GOAWAY naming a shorter deadline wins, since the peer + /// force-closes at its own deadline regardless, but a longer one does not extend + /// it. Defaults to 10 seconds. /// `None` uses the default; zero disables this duration. #[usage( name = "connect-goaway-handover", @@ -1727,8 +1728,11 @@ enum Ended { /// of leaving an orphan holding the connection open. struct Draining { session: moq_net::Session, + /// Resolves once the session closes, on its own or through [`Self::poll_idle`]. closed: std::pin::Pin + Send>>, deadline: std::pin::Pin>, + /// Whether [`Self::poll_idle`] started the graceful close. + closing: bool, /// Mirrors the session into [`State::predecessor`] for as long as this lives. state: kio::Producer, } @@ -1750,6 +1754,7 @@ impl Draining { session, closed, deadline, + closing: false, state: state.clone(), } } @@ -1759,6 +1764,23 @@ impl Draining { self.session.abort(moq_net::Error::GoawayTimeout); } + /// Close the old session gracefully once it carries no subscription or fetch. + /// + /// Only while a replacement serves: until then everything new still lands here. The + /// handover window still bounds the close. + fn poll_idle(&mut self, waiter: &kio::Waiter) { + if self.closing || self.session.poll_idle(waiter).is_pending() { + return; + } + tracing::debug!("old session carries nothing after GOAWAY; closing"); + self.closing = true; + let session = self.session.clone(); + self.closed = Box::pin(async move { + // A drain cut short by its own deadline still ends the session. + let _ = session.close().await; + }); + } + /// Poll the drain; `true` once it is over and the handle should be released. fn poll(&mut self, waiter: &kio::Waiter) -> bool { if waiter.poll_future(self.closed.as_mut()).is_ready() { @@ -1821,7 +1843,8 @@ async fn sleep_draining(delay: Duration, draining: &mut Option, shared /// Wait for `session` to close or receive a GOAWAY, forwarding its send/recv bandwidth estimates /// into the persistent producers meanwhile so [`Connection`] consumers track the live estimates -/// across the connection, and draining any predecessor left over from an earlier migration. +/// across the connection, and closing any predecessor left over from an earlier migration once +/// it carries nothing. /// /// One `poll_*` step drives it all: [`poll_forward`] mirrors each kio bandwidth estimate, the /// GOAWAY consumer is a kio channel, and the transport's close future (the one non-kio source) is @@ -1846,8 +1869,11 @@ async fn run_session( poll_forward(&mut send, &shared.send_bw, waiter); recv.poll(&shared.recv_bw, waiter, || session.recv_bandwidth()); - // Retire the predecessor once it closes or overstays its handover window. - // The replacement is already serving, so the state stays as it is. + // The replacement serves, so a predecessor carrying nothing closes now. One that + // closes or overstays its handover window retires, and the state stays as it is. + if let Some(old) = draining.as_mut() { + old.poll_idle(waiter); + } poll_draining(draining, waiter); // Checked before the close arm: a GOAWAY means the session is still up, and diff --git a/rs/moq-tokio/tests/integration/main.rs b/rs/moq-tokio/tests/integration/main.rs index f5b5ff9853e..40bb2037002 100644 --- a/rs/moq-tokio/tests/integration/main.rs +++ b/rs/moq-tokio/tests/integration/main.rs @@ -13,6 +13,7 @@ mod broadcast; mod close_code; mod keylog; mod max_age; +mod migrate; mod receive_credit; mod reconnect; mod reset_secret; diff --git a/rs/moq-tokio/tests/integration/migrate.rs b/rs/moq-tokio/tests/integration/migrate.rs new file mode 100644 index 00000000000..05be0a6b238 --- /dev/null +++ b/rs/moq-tokio/tests/integration/migrate.rs @@ -0,0 +1,237 @@ +//! A GOAWAY migration closes the old session once nothing flows on it, rather than holding +//! it to the handover cap. + +#![cfg(feature = "tcp")] + +use std::time::Duration; + +use moq_tokio::moq_net; + +/// The only version whose routes carry the epoch that lets a subscription resume across +/// the move. +const LITE_07: &str = "moq-lite-07-wip"; + +/// The client's handover cap, far past [`TIMEOUT`]: the GOAWAY names no deadline, so only +/// the early close can end the old session within a test. +const HANDOVER: Duration = Duration::from_secs(3600); + +const TIMEOUT: Duration = Duration::from_secs(10); + +/// Groups each viewer reads between two steps. +const GROUPS: usize = 10; + +/// A millisecond track: the frames written here carry timestamps. +fn timed() -> moq_net::track::Info { + moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI) +} + +/// A relay: one origin every accepted session publishes from and subscribes into, +/// speaking `versions`, or the defaults when empty. Returns the port it listens on and +/// each session it accepts. +async fn relay( + origin: &moq_net::origin::Producer, + versions: Vec, +) -> (u16, tokio::sync::mpsc::UnboundedReceiver) { + let mut listen = moq_tokio::listen::Config::default(); + listen.tcp.bind = Some("127.0.0.1:0".parse().unwrap()); + if !versions.is_empty() { + listen.version = versions; + } + let mut server = listen + .init(Default::default()) + .expect("init server") + .listen() + .await + .expect("bind tcp listener"); + let port = server.tcp_local_addr().expect("tcp listener bound").port(); + let (accepted, sessions) = tokio::sync::mpsc::unbounded_channel(); + let origin = origin.clone(); + tokio::spawn(async move { + while let Some(request) = server.accept().await { + let request = request.with_publisher(&origin).with_subscriber(origin.clone()); + if let Ok(session) = request.ok().await { + let _ = accepted.send(session); + } + } + }); + (port, sessions) +} + +/// A client sharing `origin` with the relay on `port`, speaking `versions`, or the +/// defaults when empty. +fn connect(origin: &moq_net::origin::Producer, port: u16, versions: Vec) -> moq_tokio::Connection { + let mut config = moq_tokio::connect::Config::default(); + if !versions.is_empty() { + config.version = versions; + } + config.goaway.handover = Some(HANDOVER.into()); + // Healthy well before the GOAWAY, so the replacement dials without a backoff. + config.backoff.healthy = Some(Duration::from_millis(100).into()); + config + .init(Default::default()) + .expect("init client") + .with_origin(origin.clone()) + .connect(format!("tcp://127.0.0.1:{port}/").parse::().unwrap()) +} + +/// Announce `name` on `origin` as `route`, and publish a group carrying its own sequence +/// number every 20ms for the rest of the test. +fn publish(origin: &moq_net::origin::Producer, name: &str, route: moq_net::origin::Route) { + let broadcast = origin.create_broadcast(name).expect("create broadcast"); + broadcast.announce(route).expect("announce"); + let track = broadcast.create_track("video", timed()).expect("create track"); + tokio::spawn(async move { + let _broadcast = broadcast; + loop { + let mut group = track.append_group().expect("append group"); + let payload = group.sequence.to_string().into_bytes(); + group + .write_frame(moq_net::Timestamp::ZERO, payload) + .expect("write frame"); + group.finish().expect("finish group"); + tokio::time::sleep(Duration::from_millis(20)).await; + } + }); +} + +/// Read `name`'s track through `origin`, sending each group's sequence once its payload +/// checks out. Dropping the receiver ends the subscription. +fn view(origin: &moq_net::origin::Producer, name: &str) -> tokio::sync::mpsc::UnboundedReceiver { + let (seen, sequences) = tokio::sync::mpsc::unbounded_channel(); + let consumer = origin.consume(); + let name = name.to_string(); + tokio::spawn(async move { + let broadcast = consumer + .follow(name.as_str(), None) + .expect("follow") + .available() + .await + .expect("the broadcast resolves"); + // A budget, so a group the move delays is late rather than skipped. + let subscription = moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(5)); + let mut track = broadcast + .track("video") + .expect("track") + .subscribe(subscription) + .await + .expect("subscribe"); + while let Some(mut group) = track.recv_group().await.expect("the subscription failed") { + let frame = group.read_frame().await.expect("read frame").expect("an empty group"); + assert_eq!(frame.payload[..], group.sequence.to_string().into_bytes()[..]); + if seen.send(group.sequence).is_err() { + return; + } + } + panic!("the subscription ended"); + }); + sequences +} + +/// Receive `count` sequences, after any already queued. +async fn read(sequences: &mut tokio::sync::mpsc::UnboundedReceiver, count: usize, into: &mut Vec) { + for _ in 0..count { + let sequence = tokio::time::timeout(TIMEOUT, sequences.recv()) + .await + .expect("no group arrived") + .expect("the viewer stopped"); + into.push(sequence); + } +} + +/// Every group from the first read through the last arrived exactly once. +fn assert_contiguous(who: &str, sequences: &[u64]) { + for pair in sequences.windows(2) { + assert_eq!(pair[1], pair[0] + 1, "{who} skipped or repeated a group: {sequences:?}"); + } +} + +/// Wait for `old` to close, and check it closed cleanly rather than at a cap. +async fn assert_closes(old: &moq_net::Session) { + let closed = tokio::time::timeout(TIMEOUT, old.closed()) + .await + .expect("the old session waited for the handover cap"); + assert!( + matches!(closed, moq_net::Error::Cancel), + "the old session ended with {closed} rather than closing cleanly" + ); +} + +/// A client publishing one track and reading another through a relay migrates on the +/// relay's GOAWAY. With an epoch, both subscriptions resume on the replacement at a group +/// boundary, which leaves the old session carrying nothing, so it closes long before the +/// handover cap. +#[tokio::test] +async fn a_migrated_session_closes_once_nothing_flows() { + let versions = vec![LITE_07.parse().unwrap()]; + let origin = moq_tokio::origin::spawn(); + let (port, mut sessions) = relay(&origin, versions.clone()).await; + let local = moq_tokio::origin::spawn(); + let _connection = connect(&local, port, versions); + let old = tokio::time::timeout(TIMEOUT, sessions.recv()) + .await + .expect("the client never connected") + .expect("the relay stopped"); + + let epoch = || moq_net::origin::Route::default().with_epoch(moq_net::Epoch::mint()); + publish(&local, "up", epoch()); + publish(&origin, "down", epoch()); + let mut upstream = view(&origin, "up"); + let mut downstream = view(&local, "down"); + let (mut up, mut down) = (Vec::new(), Vec::new()); + read(&mut upstream, GROUPS, &mut up).await; + read(&mut downstream, GROUPS, &mut down).await; + + old.drain().send(moq_net::goaway::Goaway::new()).expect("send goaway"); + // Held, since dropping the last handle closes the replacement. + let _replacement = tokio::time::timeout(TIMEOUT, sessions.recv()) + .await + .expect("the client never redialed") + .expect("the relay stopped"); + assert_closes(&old).await; + + // Delivery carries on through the replacement without a hole or a repeat. + read(&mut upstream, GROUPS, &mut up).await; + read(&mut downstream, GROUPS, &mut down).await; + assert_contiguous("the relay", &up); + assert_contiguous("the client", &down); + assert!(sessions.try_recv().is_err(), "the client dialed past its replacement"); +} + +/// Without an epoch, as with default clients, a subscription stays on the instance it +/// started on, so the old session keeps serving it after the replacement is up. It closes +/// once the application lets that subscription go, not at the handover cap. +#[tokio::test] +async fn a_predecessor_serves_until_its_last_subscription_leaves() { + let origin = moq_tokio::origin::spawn(); + let (port, mut sessions) = relay(&origin, Vec::new()).await; + let local = moq_tokio::origin::spawn(); + let _connection = connect(&local, port, Vec::new()); + let old = tokio::time::timeout(TIMEOUT, sessions.recv()) + .await + .expect("the client never connected") + .expect("the relay stopped"); + + publish(&origin, "down", moq_net::origin::Route::default()); + let mut downstream = view(&local, "down"); + let mut down = Vec::new(); + read(&mut downstream, GROUPS, &mut down).await; + + old.drain().send(moq_net::goaway::Goaway::new()).expect("send goaway"); + // Held, since dropping the last handle closes the replacement. + let _replacement = tokio::time::timeout(TIMEOUT, sessions.recv()) + .await + .expect("the client never redialed") + .expect("the relay stopped"); + + // These groups can only come through the old session, which the subscription stays on. + read(&mut downstream, GROUPS, &mut down).await; + assert_contiguous("the client", &down); + assert!( + futures::FutureExt::now_or_never(old.closed()).is_none(), + "the old session closed while it still served a subscription" + ); + + drop(downstream); + assert_closes(&old).await; + assert!(sessions.try_recv().is_err(), "the client dialed past its replacement"); +} From cdee55c681ad292be641a061df78bef5985e782f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 31 Oct 2026 22:47:23 -0700 Subject: [PATCH 3/4] fix(net): count a track request while it sets up A track request waiting on authorization, TRACK_INFO, or TRACK_STATUS holds the session busy, so a draining predecessor cannot read as idle and close while it still owes that request an answer. An IETF fill counts from when it is queued rather than from its first poll. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/ietf/subscriber.rs | 29 ++++++++++++++++++++++++----- rs/moq-net/src/lite/subscriber.rs | 5 ++++- rs/moq-net/src/session.rs | 5 +++-- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 77d168a371b..c9691468bba 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -2881,6 +2881,9 @@ impl Subscriber { route: Option>, request: track::Request, ) { + // Holds the session busy while the request waits on authorization or TRACK_STATUS. + // Past that, only a subscription or a fill counts. + let mut setup = Some(self.session.work().start()); // Data streams wait on the alias bound by SUBSCRIBE_OK, so leave the model request // pending until its immutable track metadata is known. // Subscribe only to what our grant covers (MoQ Auth), and cancel once it no longer @@ -2934,6 +2937,7 @@ impl Subscriber { else { return; }; + setup = None; let Some(next) = self .linger(&broadcast_path, &track_name, idle, &mut group_fetches, &mut gate) .await @@ -2944,6 +2948,8 @@ impl Subscriber { } _ => Target::Request(request), }; + // `subscribe_once` counts the subscription itself. + drop(setup); loop { let Some(idle) = self .subscribe_once(&broadcast_path, &track_name, target, &mut group_fetches, &mut gate) @@ -3858,6 +3864,7 @@ impl Subscriber { } End::Fill(requested) => { let fetch = self.clone().serve_fill( + self.session.work().start(), broadcast_path.clone(), track_name.to_owned(), requested, @@ -4088,9 +4095,13 @@ impl Subscriber { match step { Step::Fill(fill) => { - let fetch = - self.clone() - .serve_fill(broadcast_path.clone(), track_name.to_owned(), fill, idle.timescale); + let fetch = self.clone().serve_fill( + self.session.work().start(), + broadcast_path.clone(), + track_name.to_owned(), + fill, + idle.timescale, + ); group_fetches.push(fetch); } Step::Subscribe => break true, @@ -5437,8 +5448,16 @@ impl Subscriber { /// group always goes alone: a run including the group its source is still writing would /// hold the earlier groups until it finishes, or be cut short at the Largest Object by a /// publisher following draft-22 section 9.11. - async fn serve_fill(self, broadcast: Path, name: String, fill: track::Fill, timescale: Option) { - let _working = self.session.work().start(); + /// + /// `_working` counts the fill from when it is queued, before its first poll. + async fn serve_fill( + self, + _working: crate::session::Working, + broadcast: Path, + name: String, + fill: track::Fill, + timescale: Option, + ) { let fill = std::sync::Arc::new(fill); let mut last = None; while let Some(run) = fill.next_run() { diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index af0e5f06a1d..d57c5bf54c5 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -7378,6 +7378,8 @@ enum TrackRunState { Info { request: Option, info: TrackInfoFetch, + /// Holds the session busy while the request waits on its TRACK_INFO. + _working: crate::session::Working, }, Serve(ServeLoop), /// Preserve the lite07 completion FIN until the publisher acknowledges it. @@ -7395,6 +7397,7 @@ impl TrackServeRun { TrackRunState::Info { request: Some(request), info: TrackInfoFetch::new(&serve), + _working: serve.subscriber.session.work().start(), } } else { // Older wires declare no publisher retention limit, and no timeline: their @@ -7477,7 +7480,7 @@ impl kio::Task for TrackServeRun { } loop { match &mut self.state { - TrackRunState::Info { request, info } => { + TrackRunState::Info { request, info, .. } => { // Abandonment was checked at the top of this poll, arming its wait. let res = ready!(info.poll_fetch(&self.serve, waiter)); let request = request.take().expect("request pending"); diff --git a/rs/moq-net/src/session.rs b/rs/moq-net/src/session.rs index 5637cb78ae4..01a7bdb8172 100644 --- a/rs/moq-net/src/session.rs +++ b/rs/moq-net/src/session.rs @@ -365,8 +365,9 @@ impl Session { /// Poll until the session carries no subscription or fetch, in either direction. /// - /// Ready at once while it carries none. Announcements and the other control - /// streams don't count, so a session can be idle while it still serves discovery. + /// Ready at once while it carries none. A track request still being set up, waiting + /// on authorization or the track's info, counts too. Announcements and the other + /// control streams don't, so a session can be idle while it still serves discovery. pub fn poll_idle(&self, waiter: &kio::Waiter) -> Poll<()> { self.work.poll_idle(waiter) } From 8234038c48d95e0bd4b1e42c548464c619428054 Mon Sep 17 00:00:00 2001 From: quest fixer Date: Sat, 31 Oct 2026 23:06:39 -0700 Subject: [PATCH 4/4] fix(tokio): use ambient timers in migration tests --- rs/moq-tokio/tests/integration/migrate.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rs/moq-tokio/tests/integration/migrate.rs b/rs/moq-tokio/tests/integration/migrate.rs index 05be0a6b238..ed67b7ab37d 100644 --- a/rs/moq-tokio/tests/integration/migrate.rs +++ b/rs/moq-tokio/tests/integration/migrate.rs @@ -89,7 +89,7 @@ fn publish(origin: &moq_net::origin::Producer, name: &str, route: moq_net::origi .write_frame(moq_net::Timestamp::ZERO, payload) .expect("write frame"); group.finish().expect("finish group"); - tokio::time::sleep(Duration::from_millis(20)).await; + moq_clock::sleep(Duration::from_millis(20)).await; } }); } @@ -130,7 +130,7 @@ fn view(origin: &moq_net::origin::Producer, name: &str) -> tokio::sync::mpsc::Un /// Receive `count` sequences, after any already queued. async fn read(sequences: &mut tokio::sync::mpsc::UnboundedReceiver, count: usize, into: &mut Vec) { for _ in 0..count { - let sequence = tokio::time::timeout(TIMEOUT, sequences.recv()) + let sequence = moq_clock::timeout(TIMEOUT, sequences.recv()) .await .expect("no group arrived") .expect("the viewer stopped"); @@ -147,7 +147,7 @@ fn assert_contiguous(who: &str, sequences: &[u64]) { /// Wait for `old` to close, and check it closed cleanly rather than at a cap. async fn assert_closes(old: &moq_net::Session) { - let closed = tokio::time::timeout(TIMEOUT, old.closed()) + let closed = moq_clock::timeout(TIMEOUT, old.closed()) .await .expect("the old session waited for the handover cap"); assert!( @@ -167,7 +167,7 @@ async fn a_migrated_session_closes_once_nothing_flows() { let (port, mut sessions) = relay(&origin, versions.clone()).await; let local = moq_tokio::origin::spawn(); let _connection = connect(&local, port, versions); - let old = tokio::time::timeout(TIMEOUT, sessions.recv()) + let old = moq_clock::timeout(TIMEOUT, sessions.recv()) .await .expect("the client never connected") .expect("the relay stopped"); @@ -183,7 +183,7 @@ async fn a_migrated_session_closes_once_nothing_flows() { old.drain().send(moq_net::goaway::Goaway::new()).expect("send goaway"); // Held, since dropping the last handle closes the replacement. - let _replacement = tokio::time::timeout(TIMEOUT, sessions.recv()) + let _replacement = moq_clock::timeout(TIMEOUT, sessions.recv()) .await .expect("the client never redialed") .expect("the relay stopped"); @@ -206,7 +206,7 @@ async fn a_predecessor_serves_until_its_last_subscription_leaves() { let (port, mut sessions) = relay(&origin, Vec::new()).await; let local = moq_tokio::origin::spawn(); let _connection = connect(&local, port, Vec::new()); - let old = tokio::time::timeout(TIMEOUT, sessions.recv()) + let old = moq_clock::timeout(TIMEOUT, sessions.recv()) .await .expect("the client never connected") .expect("the relay stopped"); @@ -218,7 +218,7 @@ async fn a_predecessor_serves_until_its_last_subscription_leaves() { old.drain().send(moq_net::goaway::Goaway::new()).expect("send goaway"); // Held, since dropping the last handle closes the replacement. - let _replacement = tokio::time::timeout(TIMEOUT, sessions.recv()) + let _replacement = moq_clock::timeout(TIMEOUT, sessions.recv()) .await .expect("the client never redialed") .expect("the relay stopped");