diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index de017e00f63..df987c82f87 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -334,7 +334,8 @@ A draining upstream may name a replacement URI. `same-host` follows it only onto the host already dialed, so a peer can move us between ports and schemes; `follow` also lets it choose the host, which trusts it not to point us into the local network; `ignore` keeps the current address. `handover` caps how long -the old connection keeps serving after a GOAWAY. +the old connection keeps serving after a GOAWAY; it closes sooner, once the +replacement is up and it carries no subscription or fetch. ## \[cache] diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index f1da9ff01ea..3e6345e0d49 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -806,11 +806,15 @@ enum FetchGate { Joining { id: RequestId, auth: crate::auth::Link }, } -struct Serve(Arc); +/// A dispatched SUBSCRIBE or FETCH serve, counted until dropped. +struct Serve { + owed: Arc, + _working: crate::session::Working, +} impl Drop for Serve { fn drop(&mut self) { - self.0.fetch_sub(1, Ordering::Relaxed); + self.owed.fetch_sub(1, Ordering::Relaxed); } } @@ -1027,7 +1031,10 @@ impl Publisher { // Count at dispatch so close cannot pass a request waiting for its first poll. let serve = matches!(id, ietf::Subscribe::ID | ietf::Fetch::ID).then(|| { self.owed.fetch_add(1, Ordering::Relaxed); - Serve(self.owed.clone()) + Serve { + owed: self.owed.clone(), + _working: self.session.work().start(), + } }); let task = match id { ietf::Subscribe::ID => { diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index d8d635e8e8b..c9691468bba 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -2881,6 +2881,9 @@ impl Subscriber { route: Option>, request: track::Request, ) { + // Holds the session busy while the request waits on authorization or TRACK_STATUS. + // Past that, only a subscription or a fill counts. + let mut setup = Some(self.session.work().start()); // Data streams wait on the alias bound by SUBSCRIBE_OK, so leave the model request // pending until its immutable track metadata is known. // Subscribe only to what our grant covers (MoQ Auth), and cancel once it no longer @@ -2934,6 +2937,7 @@ impl Subscriber { else { return; }; + setup = None; let Some(next) = self .linger(&broadcast_path, &track_name, idle, &mut group_fetches, &mut gate) .await @@ -2944,6 +2948,8 @@ impl Subscriber { } _ => Target::Request(request), }; + // `subscribe_once` counts the subscription itself. + drop(setup); loop { let Some(idle) = self .subscribe_once(&broadcast_path, &track_name, target, &mut group_fetches, &mut gate) @@ -2972,6 +2978,7 @@ impl Subscriber { group_fetches: &mut TaskSet, gate: &mut crate::auth::Gate, ) -> Option { + let _working = self.session.work().start(); let validity = self.validity.start(); let mut resolved = ResolvedRanges::default(); let subscription = target @@ -3857,6 +3864,7 @@ impl Subscriber { } End::Fill(requested) => { let fetch = self.clone().serve_fill( + self.session.work().start(), broadcast_path.clone(), track_name.to_owned(), requested, @@ -4087,9 +4095,13 @@ impl Subscriber { match step { Step::Fill(fill) => { - let fetch = - self.clone() - .serve_fill(broadcast_path.clone(), track_name.to_owned(), fill, idle.timescale); + let fetch = self.clone().serve_fill( + self.session.work().start(), + broadcast_path.clone(), + track_name.to_owned(), + fill, + idle.timescale, + ); group_fetches.push(fetch); } Step::Subscribe => break true, @@ -5436,7 +5448,16 @@ impl Subscriber { /// group always goes alone: a run including the group its source is still writing would /// hold the earlier groups until it finishes, or be cut short at the Largest Object by a /// publisher following draft-22 section 9.11. - async fn serve_fill(self, broadcast: Path, name: String, fill: track::Fill, timescale: Option) { + /// + /// `_working` counts the fill from when it is queued, before its first poll. + async fn serve_fill( + self, + _working: crate::session::Working, + broadcast: Path, + name: String, + fill: track::Fill, + timescale: Option, + ) { let fill = std::sync::Arc::new(fill); let mut last = None; while let Some(run) = fill.next_run() { diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index cc7a76e29c4..2522df0b4d5 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -1530,6 +1530,9 @@ trait Request: Sized { /// Whether the request counts toward the session's subscriptions peak while it is open. /// Its stream's credit, not a count, bounds how many the peer holds. const SUBSCRIPTION: bool = false; + /// Whether the request carries track data (a subscription or a fetch), which keeps + /// the session from reading as idle while it is open. + const DATA: bool = false; /// Whether the request stays open past its reply, keeping what serving it took, until /// the requester closes its side. const HOLDS: bool; @@ -1589,6 +1592,8 @@ struct RequestServe { owing: bool, // A subscription's place in the session's subscriptions peak. _subscription: Option, + // Holds the session busy while a subscription or fetch is open. + _working: Option, // The latest update to arrive before the reply started. Each one replaces the last. update: Option, /// Ends the request, even mid-reply, once the session stops allowing its broadcast. @@ -1628,12 +1633,14 @@ enum RequestState { impl RequestServe { fn new(shared: Arc, stream: Stream) -> Self { shared.owed.fetch_add(1, Ordering::Relaxed); + let _working = R::DATA.then(|| shared.session.work().start()); Self { shared, stream: Some(stream), state: RequestState::Decode, owing: true, _subscription: None, + _working, update: None, gate: None, broadcast: Default::default(), @@ -2017,6 +2024,7 @@ impl Request for SubscribeServe { // Ending a subscription early is still a valid end. const GRACEFUL: bool = true; const SUBSCRIPTION: bool = true; + const DATA: bool = true; const HOLDS: bool = false; type Message = lite::Subscribe<'static>; type Update = lite::SubscribeUpdate; @@ -2261,6 +2269,7 @@ impl Request for FetchServe { const KIND: &'static str = "fetch"; // The response carries no header, so a run cut short would read as whole. const GRACEFUL: bool = false; + const DATA: bool = true; const HOLDS: bool = false; type Message = lite::Fetch<'static>; type Update = NoUpdate; diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 0ce4ed44bb7..d57c5bf54c5 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -2150,6 +2150,7 @@ mod tests { unanswered: None, bounded: false, updated: false, + _working: crate::session::Work::default().start(), }) } @@ -2325,6 +2326,7 @@ mod tests { unanswered: None, bounded: false, updated: false, + _working: crate::session::Work::default().start(), }); if update { h.serve @@ -2781,6 +2783,7 @@ mod tests { unanswered: None, bounded: false, updated: false, + _working: crate::session::Work::default().start(), }; assert_eq!( sub.owed(None), @@ -6586,6 +6589,8 @@ struct SubStream { /// Whether an update changed the ranges since the SUBSCRIBE, which a publisher may /// FIN before reading. updated: bool, + /// Holds the session busy for as long as the subscription is open. + _working: crate::session::Working, } impl SubStream { @@ -7154,6 +7159,7 @@ impl TrackServe { Establish { validity: self.validity.start(), serve: self.clone(), + _working: session.work().start(), closed: session.clone(), session, id, @@ -7268,6 +7274,8 @@ struct Establish { /// The ranges the SUBSCRIBE carries for `subscription`. ranges: Vec, tail: kio::Producer, + /// Holds the session busy while the SUBSCRIBE opens. + _working: crate::session::Working, state: EstablishState, } @@ -7345,6 +7353,7 @@ impl Establish { wanted: None, bounded: bounded(&self.ranges), updated: false, + _working: self.session.work().start(), } } } @@ -7369,10 +7378,16 @@ enum TrackRunState { Info { request: Option, info: TrackInfoFetch, + /// Holds the session busy while the request waits on its TRACK_INFO. + _working: crate::session::Working, }, Serve(ServeLoop), /// Preserve the lite07 completion FIN until the publisher acknowledges it. - Finish(crate::coding::Writer), + Finish { + writer: crate::coding::Writer, + /// The subscription keeps the session busy until then. + _working: crate::session::Working, + }, Done, } @@ -7382,6 +7397,7 @@ impl TrackServeRun { TrackRunState::Info { request: Some(request), info: TrackInfoFetch::new(&serve), + _working: serve.subscriber.session.work().start(), } } else { // Older wires declare no publisher retention limit, and no timeline: their @@ -7416,7 +7432,7 @@ impl TrackServeRun { } // Each fetch in flight resets itself as it drops, seeing the same denial. } - TrackRunState::Finish(_) | TrackRunState::Done => {} + TrackRunState::Finish { .. } | TrackRunState::Done => {} } } } @@ -7455,7 +7471,7 @@ impl kio::Task for TrackServeRun { // A finished track only waits for its FIN to be acknowledged: nothing is left to revoke. // Nor on a closed session, whose grants end with it: the session's end decides how // its tracks end. - if !matches!(self.state, TrackRunState::Finish(_)) + if !matches!(self.state, TrackRunState::Finish { .. }) && self.gate.poll_denied(waiter).is_ready() && self.serve.subscriber.session.clone().poll_closed(waiter).is_pending() { @@ -7464,7 +7480,7 @@ impl kio::Task for TrackServeRun { } loop { match &mut self.state { - TrackRunState::Info { request, info } => { + TrackRunState::Info { request, info, .. } => { // Abandonment was checked at the top of this poll, arming its wait. let res = ready!(info.poll_fetch(&self.serve, waiter)); let request = request.take().expect("request pending"); @@ -7531,14 +7547,17 @@ impl kio::Task for TrackServeRun { if active.stream.writer.finish().is_ok() && finished && self.serve.subscriber.version.waits_for_subscriber_fin() { - self.state = TrackRunState::Finish(active.stream.writer); + self.state = TrackRunState::Finish { + writer: active.stream.writer, + _working: active._working, + }; continue; } } return Poll::Ready(()); } - TrackRunState::Finish(writer) => { + TrackRunState::Finish { writer, .. } => { let _ = ready!(writer.poll_close(waiter)); self.state = TrackRunState::Done; } @@ -7695,8 +7714,12 @@ struct ServeLoop { /// demand still asks for just them: subscribing again would only resend them. covered: Option>, /// Our side of the covered subscription, finished and held until the publisher reads - /// the FIN, since dropping it would reset it. - closing: Option>, + /// the FIN, since dropping it would reset it. The subscription keeps the session busy + /// until then. + closing: Option<( + crate::coding::Writer, + crate::session::Working, + )>, } #[expect( @@ -8036,7 +8059,7 @@ impl ServeLoop { serve.subscriber.remove_subscribe(active.id); let mut writer = active.stream.writer; if writer.finish().is_ok() { - self.closing = Some(writer); + self.closing = Some((writer, active._working)); } // Unchanged ranges are what the publisher covered, even if a datagram // went missing. Changed ones settled by the grace instead may have @@ -8064,7 +8087,7 @@ impl ServeLoop { return Poll::Pending; } ServeMode::Select => { - if let Some(writer) = &mut self.closing + if let Some((writer, _)) = &mut self.closing && writer.poll_close(waiter).is_ready() { self.closing = None; @@ -8436,6 +8459,8 @@ struct FetchServeRun { info: track::Info, group: u64, state: FetchRunState, + /// Holds the session busy until the fetch ends. + _working: crate::session::Working, } enum FetchRunState { @@ -8484,6 +8509,7 @@ impl FetchServeRun { validity: serve.validity.start(), gate: serve.gate(), serve, + _working: session.work().start(), session, info, group, diff --git a/rs/moq-net/src/session.rs b/rs/moq-net/src/session.rs index b41f20b6147..01a7bdb8172 100644 --- a/rs/moq-net/src/session.rs +++ b/rs/moq-net/src/session.rs @@ -172,6 +172,8 @@ pub struct Session { goaway: Arc, auth: auth::Handle, credential: Option, + /// The subscriptions and fetches the protocol tasks carry. + work: Work, } impl Session { @@ -361,6 +363,22 @@ impl Session { self.goaway.consumer() } + /// Poll until the session carries no subscription or fetch, in either direction. + /// + /// Ready at once while it carries none. A track request still being set up, waiting + /// on authorization or the track's info, counts too. Announcements and the other + /// control streams don't, so a session can be idle while it still serves discovery. + pub fn poll_idle(&self, waiter: &kio::Waiter) -> Poll<()> { + self.work.poll_idle(waiter) + } + + /// Wait until the session carries no subscription or fetch, in either direction. + /// + /// See [`poll_idle`](Self::poll_idle). + pub async fn idle(&self) { + kio::wait(|waiter| self.poll_idle(waiter)).await + } + /// The tokens this side presented and the grant they earned, plus the tokens /// the peer presents. See [`auth`]. /// @@ -445,6 +463,7 @@ impl Session { sample, demanded: false, }); + let work = session.work().clone(); let supervisor = Supervisor { goaway: Some(goaway.deadline(&runtime)), @@ -478,6 +497,7 @@ impl Session { goaway, auth: auth.handle().clone(), credential, + work, }; let driver = crate::Driver::new( runtime.clone(), @@ -887,6 +907,37 @@ impl Drop for Withdrawing { } } +/// The subscriptions and fetches a session carries in either direction, which +/// [`Session::idle`] waits to reach zero. +#[derive(Clone, Default)] +pub(crate) struct Work(kio::Shared); + +impl Work { + /// Count one subscription or fetch until the returned guard drops. + pub(crate) fn start(&self) -> Working { + *self.0.lock() += 1; + Working(self.clone()) + } + + fn poll_idle(&self, waiter: &kio::Waiter) -> Poll<()> { + self.0 + .poll( + waiter, + |count| if **count == 0 { Poll::Ready(()) } else { Poll::Pending }, + ) + .map(|_| ()) + } +} + +/// One subscription or fetch counted in [`Work`], released on drop. +pub(crate) struct Working(Work); + +impl Drop for Working { + fn drop(&mut self) { + *self.0.0.lock() -= 1; + } +} + /// Bounds on what one peer can make a session hold, where its transport does not already. /// /// Each request holds a stream of its own, so the transport's stream credit bounds those. diff --git a/rs/moq-net/src/transport/boxed.rs b/rs/moq-net/src/transport/boxed.rs index 7c73f6e83f2..459f7abe283 100644 --- a/rs/moq-net/src/transport/boxed.rs +++ b/rs/moq-net/src/transport/boxed.rs @@ -19,6 +19,8 @@ pub(crate) struct Session { inner: std::sync::Arc, /// Shared by every clone, so whichever task closes the transport first says why. closing: std::sync::Arc, + /// Shared by every clone, so each task counts the subscriptions and fetches it carries. + work: crate::session::Work, } impl Session { @@ -26,6 +28,7 @@ impl Session { Self { inner: std::sync::Arc::new(session), closing: Default::default(), + work: Default::default(), } } @@ -34,6 +37,11 @@ impl Session { &self.closing } + /// The subscriptions and fetches the session's tasks carry over this transport. + pub(crate) fn work(&self) -> &crate::session::Work { + &self.work + } + /// Close with `code` over `cause`, this side's reason for ending the session. pub(crate) fn abort(&self, cause: &crate::Error, code: u32, reason: &str) { self.closing.sent(code, Some(cause)); diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index 9f756677753..9f7f1589974 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -52,6 +52,7 @@ mod route_change; mod routing_contracts; mod segmented_frame; mod session_close; +mod session_idle; mod session_panic; mod settled_end_survives_abort; mod setup_negotiation; diff --git a/rs/moq-net/tests/integration/session_idle.rs b/rs/moq-net/tests/integration/session_idle.rs new file mode 100644 index 00000000000..53123dec4ea --- /dev/null +++ b/rs/moq-net/tests/integration/session_idle.rs @@ -0,0 +1,156 @@ +#![cfg(feature = "async")] + +//! `Session::idle` reports a session that carries no subscription or fetch in either +//! direction, which is when a migrated session can close without cutting anything off. +//! +//! Deterministic: simulated time over the mock transport. + +use crate::support; + +use std::time::Duration; + +use moq_net::{Hop, Timestamp, Version, kio}; +use support::harness::{MockConnectOptions, MockPair, connect_mock}; + +/// A millisecond track: the frames written here carry timestamps. +fn timed() -> moq_net::track::Info { + moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI) +} + +const TIMEOUT: Duration = Duration::from_secs(10); + +fn produce_origin(hop: u64) -> moq_net::origin::Producer { + let (producer, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::new(Hop::new(hop).unwrap())); + support::harness::spawn(driver); + producer +} + +fn idle(session: &moq_net::Session) -> bool { + session.poll_idle(&kio::Waiter::noop()).is_ready() +} + +/// A client publishing one announced track to a server that subscribes to whatever it hears. +struct Setup { + pair: MockPair, + track: moq_net::track::Producer, + remote: moq_net::broadcast::Consumer, + _broadcast: moq_net::origin::Publication, +} + +async fn setup(version: &str) -> Setup { + let publisher = produce_origin(1); + let broadcast = publisher.create_broadcast("bcast").unwrap(); + let track = broadcast.create_track("video", timed()).unwrap(); + broadcast.announce(Default::default()).unwrap(); + + let subscriber = produce_origin(2); + let mut options = MockConnectOptions::new(version.parse::().unwrap()); + options.client_publish = Some(publisher.consume()); + options.server_subscribe = Some(subscriber.clone()); + let pair = connect_mock(options).await; + + let remote = moq_clock::timeout(TIMEOUT, subscriber.consume().follow("bcast", None).unwrap().available()) + .await + .expect("resolve timeout") + .expect("broadcast resolves"); + + Setup { + pair, + track, + remote, + _broadcast: broadcast, + } +} + +/// An announcement alone leaves both ends idle. A subscription keeps the end serving it +/// and the end holding it busy until the reader leaves. +#[moq_clock::test] +async fn a_subscription_keeps_both_ends_busy() { + for version in [ + "moq-lite-04", + "moq-lite-06", + "moq-lite-07-wip", + "moq-transport-18", + "moq-transport-22", + ] { + let Setup { + pair, + track, + remote, + _broadcast, + } = setup(version).await; + assert!(idle(&pair.client), "{version}: an announcement kept the publisher busy"); + assert!( + idle(&pair.server), + "{version}: an announcement kept the subscriber busy" + ); + + let mut group = track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, b"live".as_ref()).unwrap(); + group.finish().unwrap(); + let mut sub = moq_clock::timeout(TIMEOUT, remote.track("video").unwrap().subscribe(None)) + .await + .expect("subscribe timeout") + .expect("subscribe"); + let received = moq_clock::timeout(TIMEOUT, sub.recv_group()) + .await + .expect("recv timeout") + .expect("recv") + .expect("the track ended"); + assert_eq!(received.sequence, 0, "{version}"); + assert!(!idle(&pair.client), "{version}: idle while serving a subscription"); + assert!(!idle(&pair.server), "{version}: idle while holding a subscription"); + + drop((received, sub)); + moq_clock::timeout(TIMEOUT, pair.server.idle()) + .await + .unwrap_or_else(|_| panic!("{version}: the subscriber stayed busy after the reader left")); + moq_clock::timeout(TIMEOUT, pair.client.idle()) + .await + .unwrap_or_else(|_| panic!("{version}: the publisher stayed busy after the reader left")); + } +} + +/// A fetch keeps both ends busy until it ends: here, until the group it reads finishes. +#[moq_clock::test] +async fn a_fetch_keeps_both_ends_busy() { + for version in ["moq-lite-06", "moq-lite-07-wip", "moq-transport-22"] { + let Setup { + pair, + track, + remote, + _broadcast, + } = setup(version).await; + + let mut writing = track.append_group().unwrap(); + writing.write_frame(Timestamp::ZERO, b"first".as_ref()).unwrap(); + let fetching = remote.track("video").unwrap(); + let fetch = moq_clock::sim::spawn(async move { + let mut fetched = fetching.fetch_group(0, None).await?; + let mut frames = Vec::new(); + while let Some(frame) = fetched.read_frame().await? { + frames.push(frame.payload); + } + Ok::<_, moq_net::Error>(frames) + }); + // Simulated: every task runs until the fetch waits on the unfinished group. + moq_clock::sleep(Duration::from_secs(1)).await; + assert!(!idle(&pair.client), "{version}: idle while serving a fetch"); + assert!(!idle(&pair.server), "{version}: idle while a fetch is in flight"); + + writing.write_frame(Timestamp::ZERO, b"last".as_ref()).unwrap(); + writing.finish().unwrap(); + let frames = moq_clock::timeout(TIMEOUT, fetch) + .await + .expect("fetch timeout") + .expect("fetch task") + .expect("fetch"); + assert_eq!(frames, [&b"first"[..], &b"last"[..]], "{version}"); + moq_clock::timeout(TIMEOUT, pair.server.idle()) + .await + .unwrap_or_else(|_| panic!("{version}: the fetching side stayed busy after the fetch")); + moq_clock::timeout(TIMEOUT, pair.client.idle()) + .await + .unwrap_or_else(|_| panic!("{version}: the serving side stayed busy after the fetch")); + } +} diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index f348452e25d..fa7791f11af 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -438,10 +438,11 @@ pub struct Goaway { )] pub redirect: Redirect, - /// How long the old session keeps serving after its replacement connects, e.g. - /// "10s" or "500ms". This is a cap: a GOAWAY naming a shorter deadline wins, - /// since the peer force-closes at its own deadline regardless, but a longer one - /// does not extend it. Defaults to 10 seconds. + /// The longest the old session keeps serving after a GOAWAY, e.g. "10s" or + /// "500ms". It closes sooner, once its replacement is up and it carries no + /// subscription or fetch. A GOAWAY naming a shorter deadline wins, since the peer + /// force-closes at its own deadline regardless, but a longer one does not extend + /// it. Defaults to 10 seconds. /// `None` uses the default; zero disables this duration. #[usage( name = "connect-goaway-handover", @@ -1727,8 +1728,11 @@ enum Ended { /// of leaving an orphan holding the connection open. struct Draining { session: moq_net::Session, + /// Resolves once the session closes, on its own or through [`Self::poll_idle`]. closed: std::pin::Pin + Send>>, deadline: std::pin::Pin>, + /// Whether [`Self::poll_idle`] started the graceful close. + closing: bool, /// Mirrors the session into [`State::predecessor`] for as long as this lives. state: kio::Producer, } @@ -1750,6 +1754,7 @@ impl Draining { session, closed, deadline, + closing: false, state: state.clone(), } } @@ -1759,6 +1764,23 @@ impl Draining { self.session.abort(moq_net::Error::GoawayTimeout); } + /// Close the old session gracefully once it carries no subscription or fetch. + /// + /// Only while a replacement serves: until then everything new still lands here. The + /// handover window still bounds the close. + fn poll_idle(&mut self, waiter: &kio::Waiter) { + if self.closing || self.session.poll_idle(waiter).is_pending() { + return; + } + tracing::debug!("old session carries nothing after GOAWAY; closing"); + self.closing = true; + let session = self.session.clone(); + self.closed = Box::pin(async move { + // A drain cut short by its own deadline still ends the session. + let _ = session.close().await; + }); + } + /// Poll the drain; `true` once it is over and the handle should be released. fn poll(&mut self, waiter: &kio::Waiter) -> bool { if waiter.poll_future(self.closed.as_mut()).is_ready() { @@ -1821,7 +1843,8 @@ async fn sleep_draining(delay: Duration, draining: &mut Option, shared /// Wait for `session` to close or receive a GOAWAY, forwarding its send/recv bandwidth estimates /// into the persistent producers meanwhile so [`Connection`] consumers track the live estimates -/// across the connection, and draining any predecessor left over from an earlier migration. +/// across the connection, and closing any predecessor left over from an earlier migration once +/// it carries nothing. /// /// One `poll_*` step drives it all: [`poll_forward`] mirrors each kio bandwidth estimate, the /// GOAWAY consumer is a kio channel, and the transport's close future (the one non-kio source) is @@ -1846,8 +1869,11 @@ async fn run_session( poll_forward(&mut send, &shared.send_bw, waiter); recv.poll(&shared.recv_bw, waiter, || session.recv_bandwidth()); - // Retire the predecessor once it closes or overstays its handover window. - // The replacement is already serving, so the state stays as it is. + // The replacement serves, so a predecessor carrying nothing closes now. One that + // closes or overstays its handover window retires, and the state stays as it is. + if let Some(old) = draining.as_mut() { + old.poll_idle(waiter); + } poll_draining(draining, waiter); // Checked before the close arm: a GOAWAY means the session is still up, and diff --git a/rs/moq-tokio/tests/integration/main.rs b/rs/moq-tokio/tests/integration/main.rs index 7bb7b2e7df6..faeb6ba00bb 100644 --- a/rs/moq-tokio/tests/integration/main.rs +++ b/rs/moq-tokio/tests/integration/main.rs @@ -8,6 +8,7 @@ mod broadcast; mod close_code; mod keylog; mod max_age; +mod migrate; mod receive_credit; mod reconnect; mod reset_secret; diff --git a/rs/moq-tokio/tests/integration/migrate.rs b/rs/moq-tokio/tests/integration/migrate.rs new file mode 100644 index 00000000000..ed67b7ab37d --- /dev/null +++ b/rs/moq-tokio/tests/integration/migrate.rs @@ -0,0 +1,237 @@ +//! A GOAWAY migration closes the old session once nothing flows on it, rather than holding +//! it to the handover cap. + +#![cfg(feature = "tcp")] + +use std::time::Duration; + +use moq_tokio::moq_net; + +/// The only version whose routes carry the epoch that lets a subscription resume across +/// the move. +const LITE_07: &str = "moq-lite-07-wip"; + +/// The client's handover cap, far past [`TIMEOUT`]: the GOAWAY names no deadline, so only +/// the early close can end the old session within a test. +const HANDOVER: Duration = Duration::from_secs(3600); + +const TIMEOUT: Duration = Duration::from_secs(10); + +/// Groups each viewer reads between two steps. +const GROUPS: usize = 10; + +/// A millisecond track: the frames written here carry timestamps. +fn timed() -> moq_net::track::Info { + moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI) +} + +/// A relay: one origin every accepted session publishes from and subscribes into, +/// speaking `versions`, or the defaults when empty. Returns the port it listens on and +/// each session it accepts. +async fn relay( + origin: &moq_net::origin::Producer, + versions: Vec, +) -> (u16, tokio::sync::mpsc::UnboundedReceiver) { + let mut listen = moq_tokio::listen::Config::default(); + listen.tcp.bind = Some("127.0.0.1:0".parse().unwrap()); + if !versions.is_empty() { + listen.version = versions; + } + let mut server = listen + .init(Default::default()) + .expect("init server") + .listen() + .await + .expect("bind tcp listener"); + let port = server.tcp_local_addr().expect("tcp listener bound").port(); + let (accepted, sessions) = tokio::sync::mpsc::unbounded_channel(); + let origin = origin.clone(); + tokio::spawn(async move { + while let Some(request) = server.accept().await { + let request = request.with_publisher(&origin).with_subscriber(origin.clone()); + if let Ok(session) = request.ok().await { + let _ = accepted.send(session); + } + } + }); + (port, sessions) +} + +/// A client sharing `origin` with the relay on `port`, speaking `versions`, or the +/// defaults when empty. +fn connect(origin: &moq_net::origin::Producer, port: u16, versions: Vec) -> moq_tokio::Connection { + let mut config = moq_tokio::connect::Config::default(); + if !versions.is_empty() { + config.version = versions; + } + config.goaway.handover = Some(HANDOVER.into()); + // Healthy well before the GOAWAY, so the replacement dials without a backoff. + config.backoff.healthy = Some(Duration::from_millis(100).into()); + config + .init(Default::default()) + .expect("init client") + .with_origin(origin.clone()) + .connect(format!("tcp://127.0.0.1:{port}/").parse::().unwrap()) +} + +/// Announce `name` on `origin` as `route`, and publish a group carrying its own sequence +/// number every 20ms for the rest of the test. +fn publish(origin: &moq_net::origin::Producer, name: &str, route: moq_net::origin::Route) { + let broadcast = origin.create_broadcast(name).expect("create broadcast"); + broadcast.announce(route).expect("announce"); + let track = broadcast.create_track("video", timed()).expect("create track"); + tokio::spawn(async move { + let _broadcast = broadcast; + loop { + let mut group = track.append_group().expect("append group"); + let payload = group.sequence.to_string().into_bytes(); + group + .write_frame(moq_net::Timestamp::ZERO, payload) + .expect("write frame"); + group.finish().expect("finish group"); + moq_clock::sleep(Duration::from_millis(20)).await; + } + }); +} + +/// Read `name`'s track through `origin`, sending each group's sequence once its payload +/// checks out. Dropping the receiver ends the subscription. +fn view(origin: &moq_net::origin::Producer, name: &str) -> tokio::sync::mpsc::UnboundedReceiver { + let (seen, sequences) = tokio::sync::mpsc::unbounded_channel(); + let consumer = origin.consume(); + let name = name.to_string(); + tokio::spawn(async move { + let broadcast = consumer + .follow(name.as_str(), None) + .expect("follow") + .available() + .await + .expect("the broadcast resolves"); + // A budget, so a group the move delays is late rather than skipped. + let subscription = moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(5)); + let mut track = broadcast + .track("video") + .expect("track") + .subscribe(subscription) + .await + .expect("subscribe"); + while let Some(mut group) = track.recv_group().await.expect("the subscription failed") { + let frame = group.read_frame().await.expect("read frame").expect("an empty group"); + assert_eq!(frame.payload[..], group.sequence.to_string().into_bytes()[..]); + if seen.send(group.sequence).is_err() { + return; + } + } + panic!("the subscription ended"); + }); + sequences +} + +/// Receive `count` sequences, after any already queued. +async fn read(sequences: &mut tokio::sync::mpsc::UnboundedReceiver, count: usize, into: &mut Vec) { + for _ in 0..count { + let sequence = moq_clock::timeout(TIMEOUT, sequences.recv()) + .await + .expect("no group arrived") + .expect("the viewer stopped"); + into.push(sequence); + } +} + +/// Every group from the first read through the last arrived exactly once. +fn assert_contiguous(who: &str, sequences: &[u64]) { + for pair in sequences.windows(2) { + assert_eq!(pair[1], pair[0] + 1, "{who} skipped or repeated a group: {sequences:?}"); + } +} + +/// Wait for `old` to close, and check it closed cleanly rather than at a cap. +async fn assert_closes(old: &moq_net::Session) { + let closed = moq_clock::timeout(TIMEOUT, old.closed()) + .await + .expect("the old session waited for the handover cap"); + assert!( + matches!(closed, moq_net::Error::Cancel), + "the old session ended with {closed} rather than closing cleanly" + ); +} + +/// A client publishing one track and reading another through a relay migrates on the +/// relay's GOAWAY. With an epoch, both subscriptions resume on the replacement at a group +/// boundary, which leaves the old session carrying nothing, so it closes long before the +/// handover cap. +#[tokio::test] +async fn a_migrated_session_closes_once_nothing_flows() { + let versions = vec![LITE_07.parse().unwrap()]; + let origin = moq_tokio::origin::spawn(); + let (port, mut sessions) = relay(&origin, versions.clone()).await; + let local = moq_tokio::origin::spawn(); + let _connection = connect(&local, port, versions); + let old = moq_clock::timeout(TIMEOUT, sessions.recv()) + .await + .expect("the client never connected") + .expect("the relay stopped"); + + let epoch = || moq_net::origin::Route::default().with_epoch(moq_net::Epoch::mint()); + publish(&local, "up", epoch()); + publish(&origin, "down", epoch()); + let mut upstream = view(&origin, "up"); + let mut downstream = view(&local, "down"); + let (mut up, mut down) = (Vec::new(), Vec::new()); + read(&mut upstream, GROUPS, &mut up).await; + read(&mut downstream, GROUPS, &mut down).await; + + old.drain().send(moq_net::goaway::Goaway::new()).expect("send goaway"); + // Held, since dropping the last handle closes the replacement. + let _replacement = moq_clock::timeout(TIMEOUT, sessions.recv()) + .await + .expect("the client never redialed") + .expect("the relay stopped"); + assert_closes(&old).await; + + // Delivery carries on through the replacement without a hole or a repeat. + read(&mut upstream, GROUPS, &mut up).await; + read(&mut downstream, GROUPS, &mut down).await; + assert_contiguous("the relay", &up); + assert_contiguous("the client", &down); + assert!(sessions.try_recv().is_err(), "the client dialed past its replacement"); +} + +/// Without an epoch, as with default clients, a subscription stays on the instance it +/// started on, so the old session keeps serving it after the replacement is up. It closes +/// once the application lets that subscription go, not at the handover cap. +#[tokio::test] +async fn a_predecessor_serves_until_its_last_subscription_leaves() { + let origin = moq_tokio::origin::spawn(); + let (port, mut sessions) = relay(&origin, Vec::new()).await; + let local = moq_tokio::origin::spawn(); + let _connection = connect(&local, port, Vec::new()); + let old = moq_clock::timeout(TIMEOUT, sessions.recv()) + .await + .expect("the client never connected") + .expect("the relay stopped"); + + publish(&origin, "down", moq_net::origin::Route::default()); + let mut downstream = view(&local, "down"); + let mut down = Vec::new(); + read(&mut downstream, GROUPS, &mut down).await; + + old.drain().send(moq_net::goaway::Goaway::new()).expect("send goaway"); + // Held, since dropping the last handle closes the replacement. + let _replacement = moq_clock::timeout(TIMEOUT, sessions.recv()) + .await + .expect("the client never redialed") + .expect("the relay stopped"); + + // These groups can only come through the old session, which the subscription stays on. + read(&mut downstream, GROUPS, &mut down).await; + assert_contiguous("the client", &down); + assert!( + futures::FutureExt::now_or_never(old.closed()).is_none(), + "the old session closed while it still served a subscription" + ); + + drop(downstream); + assert_closes(&old).await; + assert!(sessions.try_recv().is_err(), "the client dialed past its replacement"); +}