From d29e1d7849e0bd2f682327db563c808981239ccd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 05:29:35 -0800 Subject: [PATCH 1/4] chore(quest): claim quest/m1/relay-drain-cuts.md Quest-Claim: 8813a1cff1a3bfa6 From 91023d1454dd9ef0984fe4d4264e4758e548e1e2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 05:45:05 -0800 Subject: [PATCH 2/4] feat(relay)!: report drain cuts and pending end reports Co-Authored-By: GPT-6.1 Sol --- Cargo.lock | 1 + doc/bin/relay/config.md | 6 + doc/bin/relay/http.md | 4 +- quest/m1/README.md | 1 - quest/m1/relay-drain-cuts.md | 94 ----- rs/moq-cli/src/main.rs | 2 +- rs/moq-relay/Cargo.toml | 3 + rs/moq-relay/MIGRATE.md | 14 + rs/moq-relay/benches/session.rs | 39 ++- rs/moq-relay/src/connection.rs | 9 +- rs/moq-relay/src/internal.rs | 2 +- rs/moq-relay/src/relay.rs | 7 +- rs/moq-relay/src/session.rs | 21 +- rs/moq-relay/src/shutdown.rs | 320 +++++++++++++++--- .../tests/integration/auth_lifetime.rs | 119 +++++-- rs/moq-relay/tests/integration/drain.rs | 2 +- 16 files changed, 446 insertions(+), 198 deletions(-) delete mode 100644 quest/m1/relay-drain-cuts.md diff --git a/Cargo.lock b/Cargo.lock index 83555451d05..428c9c4300d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4957,6 +4957,7 @@ dependencies = [ "moq-congestion", "moq-net", "moq-quic", + "moq-relay", "moq-shaper", "moq-sock", "moq-stats", diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index 8457a273ecc..51480f437d8 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -427,6 +427,12 @@ Only moq-lite-04+ and moq-transport clients act on a GOAWAY; older ones are closed when the window ends. An embedder can take over the signals; see [Embed](/bin/relay/#embed). +An embedder's drain alert can page on the `forced` count returned by +[`Observer::drained`](https://docs.rs/moq-relay/latest/moq_relay/shutdown/struct.Observer.html#method.drained). +Its `ending` count distinguishes sessions already closed before the deadline +but still reporting their end. The live tally separates open sessions from +that cleanup, and the summary's elapsed time stays the same for late observers. + ## \[log] ```toml diff --git a/doc/bin/relay/http.md b/doc/bin/relay/http.md index f4b0881c981..5566958fc55 100644 --- a/doc/bin/relay/http.md +++ b/doc/bin/relay/http.md @@ -114,7 +114,9 @@ Live sessions on this node. Query parameters filter on the fields the auth server already saw (`id`, `path` as a pattern, `remote` as an IP or CIDR, `transport`, `tls.name`, ...); every given field must match, and an empty filter is everyone. Each entry is the auth request plus start time, without -`query` or `token`. +`query` or `token`. A session stays listed until its end report finishes; +`closed`, when present, is the Unix seconds timestamp of its observed closure, +so tooling can separate open sessions from those finishing their report. ```bash curl 'http://127.0.0.1:9101/sessions?path=demo/**' diff --git a/quest/m1/README.md b/quest/m1/README.md index 9136e7589c9..d2a80a2fd43 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -178,7 +178,6 @@ their end report, so an embedder's drain alert pages on real cuts. - [Poll-first async](/quest/m1/poll-first/README.md) - library crates compile no coroutines by default: async wrappers are generated from `poll_*` and gated behind an `async` feature - [HLS export lists a same-codec reconfigure's first group under the new init](/quest/m1/hls-reinit-same-codec.md) - the hand-over pick, whichever of the catalog and the records arrives first, after the catalog `hls` removal is planned - [Ordered reads can skip owed holes, and snapshot latest() does](/quest/m1/ordered-owed-holes.md) - moq-net's `Ordered` gains a read that skips past an owed hole to the newest buffered group, while keeping the reader's bounds -- [The relay reports native sessions it force-closes at the drain deadline](/quest/m1/relay-drain-cuts.md) - the drain's public outcome and live tally separate sessions cut while open from sessions only finishing their end report, so an embedder's drain alert pages on real cuts - [A Rust relay never replays from group 0 to a lite-03/04 subscriber](/quest/m1/relay-inflight-replay.md) - A Rust relay or publisher serving a lite-03 or lite-04 SUBSCRIBE from a track copy that is still resolving starts at the newest group once the copy resolves - [The JS resume model passes seed 12345](/quest/m1/js-resume-model-12345.md) - a reader updated to no ranges keeps reading instead of ending cleanly - [Prototype-safe rendition names in JS](/quest/m1/proto-safe-names.md) - `js/msf`'s `ext` and `js/publish`'s rendition maps keep `__proto__` and `constructor` names, like Rust diff --git a/quest/m1/relay-drain-cuts.md b/quest/m1/relay-drain-cuts.md deleted file mode 100644 index 82c4e56622d..00000000000 --- a/quest/m1/relay-drain-cuts.md +++ /dev/null @@ -1,94 +0,0 @@ -# [S] The relay reports native sessions it force-closes at the drain deadline - -## Goal - -An embedder running moq-relay's graceful shutdown (GOAWAY, the drain deadline, -then teardown) can tell, through the public API, which native sessions the -deadline cut while their transport was still open, apart from sessions whose -transport had already closed and were only finishing their end report. A drain -alert can then page on real forced cuts and stay quiet for sessions that were -just cleaning up. Native sessions are the ones `supervise` runs: those the -relay's own QUIC, WebTransport, and WebSocket listeners accept, and those an -embedder hands it. A connection whose SETUP reply was still pending at the -deadline never reaches `supervise`; it stays a refused session, which its -stats already count, and is not a forced cut. - -## Plan - -Decided 2026-11-04 (maintainer): m1, so the public drain API changes before -the cut. - -Today the relay already counts the right thing but keeps it private. -`shutdown::Tally` (`rs/moq-relay/src/shutdown.rs`) is `pub(crate)`: `live` -counts sessions still open or still reporting their end, and `forced` counts -sessions whose `drain_session` was still waiting at the deadline (the -`Draining` guard's drop). `Observer::drained` only logs `forced`, and -`session::View`, the registry entry `GET /sessions` lists, stays registered -until `supervise` has reported the end, with nothing saying the session has -closed. An embedder watching either sees a session finishing its end report -the same as one whose transport is still open, so it can only log forced -closes rather than alert on them. - -- **The live state.** Split `Tally::live` into `open` (served, transport not - yet closed) and `ending` (closed, still reporting its end). `supervise` - moves its `Serving` from open to ending when the session closes, before it - awaits the end report, and the `Serving` records the instant it did. Expose - the split as a public `shutdown::Tally { open, ending }`, read through - `Observer` as a snapshot and as a way to wait on changes (over the existing - `watch` channel), so a drain alert can sample it during the drain. The - bookkeeping counts (`draining`, `forced`) stay crate-private. -- **The drain's outcome.** `Observer::drained` returns a public - `shutdown::Drained { forced, ending, elapsed }` instead of `()`, a separate - type from the live tally: - - `forced`: sessions whose `drain_session` was still waiting at the - deadline, decided by the `Draining` drop as today. A session `supervise` - observed closed strictly before the deadline is never counted as forced, - however long its end report takes; a peer leaving in the instant of the - deadline still counts, the sliver the `Draining` comment already accepts. - - `ending`: sessions that moved to ending strictly before the deadline and - were still reporting their end at it. Decided per session when its - `Serving` drops (closed before the deadline, dropped at or after it), the - same way `Draining` decides `forced`, rather than from a snapshot taken - when `expired()` wins: at the deadline a forced session's task may already - have moved to ending, so a snapshot would race it. - - `elapsed`: from the trigger to the drain's end, the instant its end - conditions became true, not the instant a `drained()` call noticed: the - later of the wait ending (admission closed, or the deadline passed) and the - last counted `Serving` drop. Stamp each where it happens (the drop records - its instant, and the admission close or deadline is recorded as it fires), - so every call, however late, returns the same summary. Test it with a first - `drained()` polled well after the last session left. - - Keep the existing log line, built from the same summary. -- **Per session.** The registry `Entry` (`rs/moq-relay/src/session.rs`) gains - a shared close-time cell that its `Registration` sets; `supervise` sets it - through the registration at the same point it moves its `Serving` to ending. - `Registry::list` copies it into a new `session::View` field, `closed: - Option` (unix seconds in the `GET /sessions` JSON, omitted while open, as - `View`'s `skip_serializing_none` does for its other optional fields), - so the sessions list and an embedder tracking sessions by id can tell an - open session from one finishing its end report, and when it closed. -- Test on simulated time with real sessions through `supervise`, extending - `the_drain_waits_for_a_closed_session_to_report_its_end` - (`rs/moq-relay/tests/integration/auth_lifetime.rs`), which already holds an - end report with `Script::hold_ends`: one session closes before the deadline - and holds its end report past it, another stays open until the deadline - closes it. `Drained` counts one forced and one ending, the live tally showed - the first as ending before the deadline, and the registry view had its - `closed` set. Assert the live tally before the deadline fires, not at it, - since the forced session moves to ending as soon as the deadline closes it. -- Extend the existing unit tests in `shutdown.rs` rather than adding new ones: - `the_deadline_waits_only_for_sessions_served_before_it` (a session served - after the deadline stays uncounted in `open`, `ending`, and `Drained`) and - `a_session_open_at_the_deadline_is_forced` (the instant-of-the-deadline - case). -- Note the downstream consumer generically in the docs and - `rs/moq-relay/MIGRATE.md`: an embedder's drain alert pages on - `Drained::forced`, not on every session still counted at the deadline, - since those include sessions already closed and only finishing their end - report (`Drained::ending`). - -Public API: `Observer::drained` returns `shutdown::Drained`, a public -`shutdown::Tally { open, ending }` with its snapshot and change stream on -`Observer`, and the new `session::View::closed` field (also in the -`GET /sessions` JSON). Wire: none. diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index a2efb6f0a20..b480b47afc9 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -313,7 +313,7 @@ async fn drain(shutdown: Shutdown, signals: &mut Signals) { tracing::info!(window = ?shutdown.observer.drain_timeout, "shutdown requested; draining sessions (signal again to exit immediately)"); tokio::select! { () = signals.recv() => tracing::warn!("second shutdown signal; exiting immediately"), - () = shutdown.observer.drained() => {}, + _ = shutdown.observer.drained() => {}, } } diff --git a/rs/moq-relay/Cargo.toml b/rs/moq-relay/Cargo.toml index d81179214eb..ddb899fe1dc 100644 --- a/rs/moq-relay/Cargo.toml +++ b/rs/moq-relay/Cargo.toml @@ -25,6 +25,8 @@ doc = false [features] # A programmatic relay fixture with generated TLS and ephemeral ports. test-support = [] +# Run shutdown observation on the deterministic in-memory executor. +sim = ["kio/sim", "moq-tokio/sim"] # The optional public STUN Binding listener (`--stun-bind`). stun = ["dep:stun-types"] default = ["cluster-lan", "quic", "websocket", "uds"] @@ -113,6 +115,7 @@ moq-auth = { path = "../moq-auth", features = ["client", "serve"] } moq-congestion = { path = "../moq-congestion" } # The shared session bench fills FETCH runs through the hidden `fuzz` module. moq-net = { path = "../moq-net", features = ["fuzz"] } +moq-relay = { path = ".", default-features = false, features = ["sim"] } moq-shaper = { path = "../moq-shaper", features = ["test"] } qmux = { workspace = true, features = ["tcp"] } rcgen = { workspace = true, features = ["pem", "ring"] } diff --git a/rs/moq-relay/MIGRATE.md b/rs/moq-relay/MIGRATE.md index 9708c6063e3..bb9b976f46a 100644 --- a/rs/moq-relay/MIGRATE.md +++ b/rs/moq-relay/MIGRATE.md @@ -5,6 +5,20 @@ Rolling a cluster across a release also follows the [upgrade guide](https://moq. ## Unreleased +### Shutdown reports deadline cuts separately from end reports + +`shutdown::Observer::drained()` now returns `shutdown::Drained` instead of `()`. +Callers that match its result against `()` should discard it or read the summary. +An embedder's drain alert can page on `forced`: sessions still open at the deadline. +`ending` counts sessions already closed strictly before it but still reporting their end, +and `elapsed` measures the drain itself, even when the first observer arrives later. +Sample `Observer::tally()` or await `Observer::changed()` for the live `Tally { open, ending }`. + +`session::View` adds `closed: Option`; add it to struct literals. +`GET /sessions` omits `closed` while open and includes its Unix seconds timestamp +while the session finishes reporting its end. Refused SETUP replies remain refusals, +not forced cuts. No protocol wire change. + ### Built-in ACME is removed ([#7803](https://github.com/moq-dev/moq/pull/7803)) Remove the `acme` Cargo feature, `[acme]` table, `--acme-*` flags, and `MOQ_ACME_*` variables. diff --git a/rs/moq-relay/benches/session.rs b/rs/moq-relay/benches/session.rs index 730adc32264..3df521361f6 100644 --- a/rs/moq-relay/benches/session.rs +++ b/rs/moq-relay/benches/session.rs @@ -5,12 +5,48 @@ mod session; use std::sync::Arc; -use criterion::{criterion_group, criterion_main}; +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use futures::FutureExt; use moq_auth::{Grant, Pattern, lease}; use moq_net::{cache, origin}; use moq_relay::{Accepted, Connection, auth, cluster, stats}; use session::{Backend, MockConnectOptions, Pair, mock::create_mock_session_pair}; +/// Session lifecycle updates against independent, actively waiting tally readers. +fn tally(c: &mut Criterion) { + let mut group = c.benchmark_group("relay_tally"); + for sessions in [1, 16, 256] { + for readers in [1, 16, 256] { + group.bench_function(BenchmarkId::new(format!("sessions={sessions}"), readers), |b| { + let observer = moq_relay::shutdown::Observer::disabled(); + let mut readers: Vec<_> = (0..readers).map(|_| observer.clone()).collect(); + b.iter(|| { + for _ in 0..sessions { + let mut changes: Vec<_> = readers.iter_mut().map(|reader| Box::pin(reader.changed())).collect(); + for change in &mut changes { + assert!(change.as_mut().now_or_never().is_none()); + } + let serving = observer.serve(); + for change in &mut changes { + assert_eq!(change.as_mut().now_or_never().unwrap().open, 1); + } + drop(changes); + let mut changes: Vec<_> = readers.iter_mut().map(|reader| Box::pin(reader.changed())).collect(); + for change in &mut changes { + assert!(change.as_mut().now_or_never().is_none()); + } + drop(serving); + for change in &mut changes { + assert_eq!(change.as_mut().now_or_never().unwrap(), Default::default()); + } + } + }); + }); + } + } + group.finish(); +} + struct Relay; #[derive(Clone)] @@ -124,6 +160,7 @@ impl Backend for Relay { criterion_group!( benches, + tally, session::session::, session::group_cost::, session::datagram::, diff --git a/rs/moq-relay/src/connection.rs b/rs/moq-relay/src/connection.rs index 38a1d716b74..7217c075c2b 100644 --- a/rs/moq-relay/src/connection.rs +++ b/rs/moq-relay/src/connection.rs @@ -378,13 +378,12 @@ impl Lifecycle { pub async fn supervise( session: moq_net::Session, lease: auth::Lease, - serving: crate::shutdown::Serving, + mut serving: crate::shutdown::Serving, registration: Option, ) -> anyhow::Result<()> { let mut lease = lease.authorizing(&session); let mut shutdown = serving.shutdown.clone(); // Held until the `end` is reported, so a drain past its deadline still waits for it. - let _serving = serving; let reported = lease.reported(); let result = loop { let nudged = async { @@ -412,13 +411,17 @@ pub async fn supervise( // Empty URI: "reconnect to me" (the relay is restarting). The session's // machine runs on its own, so the GOAWAY still reaches the wire while // we wait here, and the drain deadline closes it if the peer stays. - shutdown.drain_session(&session).await; + serving.drain(&session).await; lease.close(moq_auth::lease::Reason::Shutdown, session_bytes(&session)); break Ok(()); } () = nudged => lease.revalidate(), } }; + serving.closed(); + if let Some(registration) = ®istration { + registration.closed(); + } reported.await; result } diff --git a/rs/moq-relay/src/internal.rs b/rs/moq-relay/src/internal.rs index fea46fb2876..9af36742a76 100644 --- a/rs/moq-relay/src/internal.rs +++ b/rs/moq-relay/src/internal.rs @@ -463,7 +463,7 @@ async fn serve_metrics(State(state): State) -> Response { { scrape.demux = state.demux; } - scrape.draining = state.shutdown.map(|shutdown| shutdown.tally().draining); + scrape.draining = state.shutdown.map(|shutdown| shutdown.draining_count()); scrape.refusals = state.refusals.map(|refusals| refusals.snapshot()); ( [( diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index 5fea35f591a..d98d91e4d48 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -849,9 +849,10 @@ async fn drain( tokio::select! { () = signal() => { - tracing::warn!(open = shutdown.tally().live, "second shutdown signal; exiting immediately"); + let tally = shutdown.tally(); + tracing::warn!(open = tally.open, ending = tally.ending, "second shutdown signal; exiting immediately"); } - () = shutdown.drained() => {} + _ = shutdown.drained() => {} } } @@ -1022,7 +1023,7 @@ mod tests { // Arrives just after the count reaches zero: refused at accept. drop(arrival); - assert!(draining.as_mut().now_or_never().is_some()); + draining.await; assert!( activity.enter().is_none(), "admitted a connection after the drain committed to exit" diff --git a/rs/moq-relay/src/session.rs b/rs/moq-relay/src/session.rs index 433a04fc4b8..f3f4878d4e0 100644 --- a/rs/moq-relay/src/session.rs +++ b/rs/moq-relay/src/session.rs @@ -9,7 +9,7 @@ use kio::clock::Wall; use std::collections::HashMap; use std::net::{IpAddr, SocketAddr}; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, Mutex, PoisonError}; +use std::sync::{Arc, Mutex, OnceLock, PoisonError}; use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; @@ -27,6 +27,7 @@ pub struct Registry { struct Entry { request: Request, started: Wall, + closed: Arc>, handle: Handle, } @@ -63,9 +64,14 @@ pub struct Registration { id: String, handle: Handle, seen: AtomicU64, + closed: Arc>, } impl Registration { + pub(crate) fn closed(&self) { + self.closed.get_or_init(kio::clock::wall); + } + /// Wait until a push has asked this session to re-check since the last observation. pub async fn nudged(&self) { let seen = self.seen.load(Ordering::Relaxed); @@ -94,11 +100,13 @@ impl Registry { pub fn register(&self, request: Request) -> Registration { let handle = Handle::default(); let id = request.id.clone(); + let closed = Arc::new(OnceLock::new()); self.inner.lock().unwrap_or_else(PoisonError::into_inner).insert( id.clone(), Entry { request, started: kio::clock::wall(), + closed: closed.clone(), handle: handle.clone(), }, ); @@ -107,6 +115,7 @@ impl Registry { id, handle, seen: AtomicU64::new(0), + closed, } } @@ -117,7 +126,7 @@ impl Registry { .unwrap_or_else(PoisonError::into_inner) .values() .filter(|entry| filter.matches(&entry.request)) - .map(|entry| View::from_session(&entry.request, entry.started)) + .map(|entry| View::from_session(&entry.request, entry.started, entry.closed.get().copied())) .collect() } @@ -413,7 +422,7 @@ impl IntoResponse for Error { } /// One live session as the list route returns it: the request the server saw, -/// minus its credentials (`query` and `token`), plus when it was admitted. +/// minus its credentials (`query` and `token`), plus its admission and close times. #[serde_as] #[serde_with::skip_serializing_none] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] @@ -441,10 +450,13 @@ pub struct View { /// When the session was admitted, as unix seconds. #[serde_as(as = "kio::clock::serde::UnixSeconds")] pub started: Wall, + /// When supervision observed the session closed, as unix seconds, absent while open. + #[serde_as(as = "Option")] + pub closed: Option, } impl View { - fn from_session(request: &Request, started: Wall) -> Self { + fn from_session(request: &Request, started: Wall, closed: Option) -> Self { Self { id: request.id.clone(), node: request.node.clone(), @@ -457,6 +469,7 @@ impl View { role: request.role, tls: request.tls.clone(), started, + closed, } } } diff --git a/rs/moq-relay/src/shutdown.rs b/rs/moq-relay/src/shutdown.rs index bdba461a99b..94def71cfb5 100644 --- a/rs/moq-relay/src/shutdown.rs +++ b/rs/moq-relay/src/shutdown.rs @@ -12,8 +12,9 @@ //! closes every established session left, then waits for those to finish their //! cleanup: the lease's `end`, reported to the decider. -use std::{sync::Arc, time::Duration}; +use std::time::Duration; +use futures::FutureExt; use kio::clock::Instant; use tokio::sync::watch; @@ -21,8 +22,11 @@ use tokio::sync::watch; /// signal path; an embedder clones one to stop the relay from its own task. #[derive(Clone)] pub struct Trigger { - tx: watch::Sender>, + tx: watch::Sender>, drain_timeout: Duration, + activity: moq_net::server::Activity, + state: watch::Sender, + runtime: Option, } impl Trigger { @@ -30,14 +34,48 @@ impl Trigger { /// resolves and sessions begin sending GOAWAY, including any accepted later. /// Calling it again keeps the first deadline. pub fn start(&self) { - let deadline = Instant::now() + self.drain_timeout; - self.tx.send_if_modified(|started| { + // Embedders may fire the trigger from outside the relay's runtime. + let _runtime = self.runtime.as_ref().map(tokio::runtime::Handle::enter); + let now = Instant::now(); + let deadline = now + self.drain_timeout; + let first = self.tx.send_if_modified(|started| { let first = started.is_none(); if first { - *started = Some(deadline); + *started = Some(Window { started: now, deadline }); } first }); + if !first { + return; + } + + // Observe admission independently of `drained`: even its first caller may + // arrive after every session has left. Poll once here so an empty relay + // closes admission at the trigger, without waiting to schedule the task. + let activity = self.activity.clone(); + let mut closing = Box::pin(async move { + activity.close().await; + Instant::now().min(deadline) + }); + if let Some(ended) = closing.as_mut().now_or_never() { + self.state.send_modify(|state| state.wait_ended = Some(ended)); + return; + } + let state = self.state.clone(); + let wait = async move { + let ended = tokio::select! { + ended = closing => ended, + () = kio::clock::sleep_until(deadline) => deadline, + () = state.closed() => return, + }; + state.send_modify(|state| state.wait_ended = Some(ended)); + }; + #[cfg(feature = "sim")] + if kio::clock::sim::is_running() { + drop(kio::clock::sim::spawn(wait)); + return; + } + drop(tokio::spawn(wait)); } } @@ -110,13 +148,43 @@ impl Signals { /// The sessions a drain is waiting on. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub(crate) struct Tally { - /// Established sessions still open or still reporting their end, drained or not. - pub live: usize, +pub struct Tally { + /// Served sessions whose closure has not yet been observed by supervision. + pub open: usize, + /// Closed sessions still reporting their end. + pub ending: usize, +} + +/// How the drain ended, including sessions cut by its deadline. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Drained { + /// Sessions still open at the deadline that were force-closed. + pub forced: usize, + /// Sessions closed strictly before the deadline and still reporting their end at it. + pub ending: usize, + /// Time from the trigger until admission ended and every counted session left. + pub elapsed: Duration, +} + +#[derive(Clone, Copy, Debug, Default)] +struct State { + tally: Tally, /// Sessions sent a GOAWAY that have not left yet. - pub draining: usize, + draining: usize, /// Sessions still open when their drain deadline passed, and so closed by it. - pub forced: usize, + forced: usize, + /// Sessions already closed before the deadline but still reporting at it. + ending: usize, + /// Admission closed, or the deadline passed. + wait_ended: Option, + /// The most recent counted session finished reporting. + last_left: Option, +} + +#[derive(Clone, Copy)] +struct Window { + started: Instant, + deadline: Instant, } /// A per-connection handle observing the relay-wide shutdown broadcast. @@ -126,8 +194,10 @@ pub(crate) struct Tally { #[derive(Clone)] pub struct Observer { pub(crate) activity: moq_net::server::Activity, - rx: watch::Receiver>, - tally: Arc>, + rx: watch::Receiver>, + state: watch::Sender, + changes: watch::Receiver, + seen: Tally, /// How long a drained session may keep running before it is force-closed. pub drain_timeout: Duration, } @@ -136,7 +206,15 @@ impl Observer { /// Create the trigger and its observer half. pub fn new(drain_timeout: Duration) -> (Trigger, Self) { let (tx, rx) = watch::channel(None); - (Trigger { tx, drain_timeout }, Self::from(rx, drain_timeout)) + let observer = Self::from(rx, drain_timeout); + let trigger = Trigger { + tx, + drain_timeout, + activity: observer.activity.clone(), + state: observer.state.clone(), + runtime: tokio::runtime::Handle::try_current().ok(), + }; + (trigger, observer) } /// A handle that never fires, for callers without shutdown coordination @@ -149,10 +227,13 @@ impl Observer { Self::from(rx, crate::DEFAULT_DRAIN_TIMEOUT) } - fn from(rx: watch::Receiver>, drain_timeout: Duration) -> Self { + fn from(rx: watch::Receiver>, drain_timeout: Duration) -> Self { + let (state, changes) = watch::channel(State::default()); Self { rx, - tally: Arc::new(watch::channel(Tally::default()).0), + state, + changes, + seen: Tally::default(), activity: Default::default(), drain_timeout, } @@ -194,7 +275,7 @@ impl Observer { pub(crate) async fn expired(&self) { let mut rx = self.rx.clone(); let deadline = match rx.wait_for(Option::is_some).await { - Ok(deadline) => *deadline, + Ok(window) => window.map(|window| window.deadline), Err(_) => None, }; match deadline { @@ -210,7 +291,7 @@ impl Observer { /// When the drain window ends, if [`Trigger::start`] has fired. pub(crate) fn deadline(&self) -> Option { - *self.rx.borrow() + self.rx.borrow().map(|window| window.deadline) } /// Count a session until the returned guard drops, so the drain waits for it to @@ -227,18 +308,35 @@ impl Observer { pub fn serve(&self) -> Serving { let counted = self.deadline().is_none_or(|deadline| Instant::now() < deadline); if counted { - self.tally.send_modify(|tally| tally.live += 1); + self.state.send_modify(|state| state.tally.open += 1); } Serving { _active: self.activity.enter(), counted, + closed: None, shutdown: self.clone(), } } /// The sessions counted so far. - pub(crate) fn tally(&self) -> Tally { - *self.tally.borrow() + pub fn tally(&self) -> Tally { + self.state.borrow().tally + } + + /// Resolve with the next changed tally since this observer last observed a change. + /// Changes may coalesce; cloning an observer gives it an independent observation point. + pub async fn changed(&mut self) -> Tally { + self.seen = self + .changes + .wait_for(|state| state.tally != self.seen) + .await + .expect("sender held") + .tally; + self.seen + } + + pub(crate) fn draining_count(&self) -> usize { + self.state.borrow().draining } /// Resolve once the drain is over, logging how it ended: every admitted @@ -248,35 +346,44 @@ impl Observer { /// Pending handshakes and admissions are cut at the deadline rather than waited /// for, so they never hold this past it. Waits for [`Trigger::start`] first, and /// never resolves for a [`disabled`](Self::disabled) handle. - pub async fn drained(&self) { + pub async fn drained(&self) -> Drained { self.clone().started().await; - tokio::select! { - biased; - () = self.activity.close() => {} - () = self.expired() => {} - } // Either way, wait for the counted sessions: past the deadline they are still // reporting their end, and before it one served without an admission still // holds the drain. Sessions served after the deadline are not counted, so // this only falls. - let mut tally = self.tally.subscribe(); + let mut state = self.state.subscribe(); // The sender lives in `self`, so this cannot close. - let _ = tally.wait_for(|tally| tally.live == 0).await; - - let deadline = self.deadline().expect("started"); - let elapsed = Instant::now().saturating_duration_since(deadline - self.drain_timeout); - match self.tally().forced { - 0 => tracing::info!(?elapsed, "drain ended with every session gone; exiting"), - forced => tracing::warn!(?elapsed, forced, "drain deadline force-closed sessions; exiting"), + let state = *state + .wait_for(|state| state.wait_ended.is_some() && state.tally == Tally::default()) + .await + .expect("sender held"); + + let window = self.rx.borrow().expect("started"); + let ended = state + .wait_ended + .expect("wait ended") + .max(state.last_left.unwrap_or(window.started)); + let drained = Drained { + forced: state.forced, + ending: state.ending, + elapsed: ended.saturating_duration_since(window.started), + }; + match drained.forced { + 0 => tracing::info!(elapsed = ?drained.elapsed, "drain ended with every session gone; exiting"), + forced => { + tracing::warn!(elapsed = ?drained.elapsed, forced, "drain deadline force-closed sessions; exiting") + } } + drained } /// Count a session as draining until the returned guard drops, and as forced /// if it is still open at `deadline`. fn draining(&self, deadline: Instant) -> Draining { - self.tally.send_modify(|tally| tally.draining += 1); + self.state.send_modify(|state| state.draining += 1); Draining { - tally: self.tally.clone(), + state: self.state.clone(), deadline, } } @@ -296,12 +403,16 @@ impl Observer { /// With no time left (a zero [`drain_timeout`](Self::drain_timeout), or a /// session accepted after the deadline) the session is closed at once. pub async fn drain_session(&self, session: &moq_net::Session) { + self.drain(session, true).await; + } + + async fn drain(&self, session: &moq_net::Session, counted: bool) { let now = Instant::now(); let deadline = self.deadline().unwrap_or(now + self.drain_timeout); let remaining = deadline.saturating_duration_since(now); // Dropped when this returns or is cancelled (a WebSocket driver ending // first), so the count holds whichever way the session goes. - let _draining = self.draining(deadline); + let _draining = counted.then(|| self.draining(deadline)); // No grace left, so there is nothing to drain. Close now rather than send a // GOAWAY: the peer would have no time to act on it, and a zero timeout means @@ -340,20 +451,50 @@ pub struct Serving { _active: Option, /// False for a session served after the deadline, which is not counted. counted: bool, + closed: Option, pub(crate) shutdown: Observer, } +impl Serving { + pub(crate) async fn drain(&self, session: &moq_net::Session) { + self.shutdown.drain(session, self.counted).await; + } + + pub(crate) fn closed(&mut self) { + self.closed = Some(Instant::now()); + if self.counted { + self.shutdown.state.send_modify(|state| { + state.tally.open -= 1; + state.tally.ending += 1; + }); + } + } +} + impl Drop for Serving { fn drop(&mut self) { if self.counted { - self.shutdown.tally.send_modify(|tally| tally.live -= 1); + let now = Instant::now(); + let ending = self + .closed + .zip(self.shutdown.deadline()) + .is_some_and(|(closed, deadline)| closed < deadline && now >= deadline); + self.shutdown.state.send_modify(|state| { + if self.closed.is_some() { + state.tally.ending -= 1; + } else { + state.tally.open -= 1; + } + state.ending += usize::from(ending); + state.last_left = Some(state.last_left.map_or(now, |last| last.max(now))); + }); } } } /// A session sent a GOAWAY; see [`Observer::draining`]. struct Draining { - tally: Arc>, + state: watch::Sender, deadline: Instant, } @@ -362,9 +503,9 @@ impl Drop for Draining { // The force-close fires at `deadline`, so every session it closes is // counted. A peer leaving in that sliver was still there at the deadline too. let forced = Instant::now() >= self.deadline; - self.tally.send_modify(|tally| { - tally.draining -= 1; - tally.forced += usize::from(forced); + self.state.send_modify(|state| { + state.draining -= 1; + state.forced += usize::from(forced); }); } } @@ -578,22 +719,60 @@ mod tests { #[tokio::test(start_paused = true)] async fn drained_waits_for_every_session() { + use futures::FutureExt; let (trigger, shutdown) = Observer::new(Duration::from_secs(10)); + let mut changes = shutdown.clone(); let first = shutdown.serve(); let second = shutdown.serve(); + assert_eq!(changes.changed().await, Tally { open: 2, ending: 0 }); trigger.start(); let drained = tokio::spawn({ let shutdown = shutdown.clone(); async move { shutdown.drained().await } }); + kio::clock::sleep(Duration::from_secs(1)).await; drop(first); tokio::task::yield_now().await; assert!(!drained.is_finished(), "drained with a session still open"); + assert_eq!(changes.changed().await, Tally { open: 1, ending: 0 }); + kio::clock::sleep(Duration::from_secs(1)).await; drop(second); - drained.await.expect("drained task"); + let summary = drained.await.expect("drained task"); + assert_eq!( + summary, + Drained { + forced: 0, + ending: 0, + elapsed: Duration::from_secs(2) + } + ); assert_eq!(shutdown.tally(), Tally::default()); + assert_eq!(changes.changed().await, Tally::default()); + assert!(changes.changed().now_or_never().is_none()); + + // The first caller too can arrive late: admission and session completion + // must be recorded independently of any `drained` future. + let (trigger, mut shutdown) = Observer::new(Duration::from_secs(10)); + let serving = shutdown.serve(); + std::thread::scope(|scope| scope.spawn(|| trigger.start()).join().unwrap()); + kio::clock::sleep(Duration::from_secs(1)).await; + drop(serving); + tokio::task::yield_now().await; + kio::clock::sleep(Duration::from_secs(100)).await; + // An observer's timeout is only a fallback for standalone session drains. + shutdown.drain_timeout = Duration::ZERO; + let summary = shutdown.drained().await; + assert_eq!( + summary, + Drained { + forced: 0, + ending: 0, + elapsed: Duration::from_secs(1) + } + ); + assert_eq!(shutdown.drained().await, summary); } /// A drain whose admissions emptied before the deadline still waits for a session @@ -621,44 +800,77 @@ mod tests { async fn the_deadline_waits_only_for_sessions_served_before_it() { let window = Duration::from_secs(10); let (trigger, shutdown) = Observer::new(window); - let established = shutdown.serve(); + let mut established = shutdown.serve(); + let draining = shutdown.draining(kio::clock::now() + window); trigger.start(); + kio::clock::sleep(window / 2).await; + established.closed(); + drop(draining); let drained = tokio::spawn({ let shutdown = shutdown.clone(); async move { shutdown.drained().await } }); - kio::clock::sleep(window).await; - let late = shutdown.serve(); + kio::clock::sleep(window / 2).await; + let mut late = shutdown.serve(); + assert_eq!(shutdown.tally(), Tally { open: 0, ending: 1 }); + late.closed(); + assert_eq!(shutdown.tally(), Tally { open: 0, ending: 1 }); tokio::task::yield_now().await; assert!(!drained.is_finished(), "drained under a session still cleaning up"); + kio::clock::sleep(Duration::from_secs(1)).await; drop(established); - kio::clock::timeout(Duration::from_secs(1), drained) + let summary = kio::clock::timeout(Duration::from_secs(1), drained) .await .expect("a session served after the deadline held the drain") .unwrap(); + assert_eq!( + summary, + Drained { + forced: 0, + ending: 1, + elapsed: window + Duration::from_secs(1) + } + ); + kio::clock::sleep(Duration::from_secs(10)).await; drop(late); - assert_eq!(shutdown.tally().live, 0); + assert_eq!(shutdown.tally(), Tally::default()); + assert_eq!(shutdown.drained().await, summary); } #[tokio::test(start_paused = true)] async fn a_session_open_at_the_deadline_is_forced() { let window = Duration::from_secs(10); let (trigger, shutdown) = Observer::new(window); + let mut left_serving = shutdown.serve(); + let mut edge_serving = shutdown.serve(); trigger.start(); let deadline = shutdown.deadline().expect("started"); let left = shutdown.draining(deadline); let straggler = shutdown.draining(deadline); - assert_eq!(shutdown.tally().draining, 2); + assert_eq!(shutdown.draining_count(), 2); + kio::clock::sleep(window / 2).await; + left_serving.closed(); drop(left); - kio::clock::sleep(window).await; + kio::clock::sleep(window / 2).await; drop(straggler); - - let tally = shutdown.tally(); - assert_eq!(tally.draining, 0); - assert_eq!(tally.forced, 1, "only the session open at the deadline is forced"); + edge_serving.closed(); + + let state = *shutdown.state.borrow(); + assert_eq!(state.draining, 0); + assert_eq!(state.forced, 1, "only the session open at the deadline is forced"); + drop(left_serving); + drop(edge_serving); + assert_eq!( + shutdown.drained().await, + Drained { + forced: 1, + ending: 1, + elapsed: window + } + ); } } diff --git a/rs/moq-relay/tests/integration/auth_lifetime.rs b/rs/moq-relay/tests/integration/auth_lifetime.rs index fbbbad0c383..1e99cf1a30f 100644 --- a/rs/moq-relay/tests/integration/auth_lifetime.rs +++ b/rs/moq-relay/tests/integration/auth_lifetime.rs @@ -1142,67 +1142,118 @@ async fn the_end_carries_duration_and_bytes() { relay.abort(); } -/// A session the drain deadline force-closes reports its `end` to the auth server, -/// and the drain waits for that report rather than exiting under it. +/// The drain distinguishes a deadline cut from a peer already closed and only +/// reporting its end, and holds both registrations until those reports finish. #[tokio::test] async fn the_drain_waits_for_a_closed_session_to_report_its_end() { let clock = manual(); let script = Script::new(grant(Duration::from_secs(3600))); let gate = script.hold_ends(); let auth = build_auth(script.spawn().await); - let drain_timeout = Duration::from_millis(100); - let (trigger, observer) = moq_relay::shutdown::Observer::new(drain_timeout); - - let lease = auth - .admit(auth.request(moq_auth::Transport::Quic, "/room")) - .await - .expect("admitted before draining"); - let (client, server) = memory_sessions().await; - let supervised = tokio::spawn(moq_relay::supervise(server, lease, observer.serve(), None)); + let drain_timeout = Duration::from_secs(4); + let (trigger, mut observer) = moq_relay::shutdown::Observer::new(drain_timeout); + let registry = moq_relay::session::Registry::new(); + let mut clients = Vec::new(); + let mut supervised = Vec::new(); + for _ in 0..2 { + let request = auth.request(moq_auth::Transport::Quic, "/room"); + let lease = auth.admit(request.clone()).await.expect("admitted before draining"); + let registration = registry.register(request); + let (client, server) = memory_sessions().await; + clients.push(client); + supervised.push(tokio::spawn(moq_relay::supervise( + server, + lease, + observer.serve(), + Some(registration), + ))); + } + assert_eq!( + observer.changed().await, + moq_relay::shutdown::Tally { open: 2, ending: 0 } + ); + for view in registry.list(&Default::default()) { + assert_eq!(view.closed, None); + assert!(serde_json::to_value(view).unwrap().get("closed").is_none()); + } - // Keep the deadline from racing the client receiving GOAWAY under load. - tokio::time::pause(); trigger.start(); - kio::clock::timeout(TIMEOUT, moq_net::kio::wait(|waiter| client.draining().poll(waiter))) + let started = kio::clock::now(); + for client in &clients { + kio::clock::timeout(TIMEOUT, moq_net::kio::wait(|waiter| client.draining().poll(waiter))) + .await + .expect("GOAWAY timeout") + .expect("a GOAWAY"); + } + clock.advance(Duration::from_secs(1)); + let closed = kio::clock::wall(); + clients[0].abort(moq_net::Error::Cancel); + kio::clock::timeout(TIMEOUT, script.end_arrived.notified()) .await - .expect("GOAWAY timeout") - .expect("a GOAWAY"); + .expect("first end"); + assert_eq!( + observer.changed().await, + moq_relay::shutdown::Tally { open: 1, ending: 1 } + ); + let views = registry.list(&Default::default()); + assert_eq!(views.iter().filter(|view| view.closed == Some(closed)).count(), 1); + let view = views.iter().find(|view| view.closed.is_some()).unwrap(); + let json = serde_json::to_value(view).unwrap(); + assert!(json["closed"].is_number(), "{json}"); + assert_eq!(serde_json::from_value::(json).unwrap(), *view); + let drained = tokio::spawn({ let observer = observer.clone(); async move { observer.drained().await } }); - clock.advance(drain_timeout); - tokio::time::advance(drain_timeout).await; - // The end report crosses HTTP, so resume real time after forcing the close. - tokio::time::resume(); + clock.advance(drain_timeout - Duration::from_secs(1)); kio::clock::timeout(TIMEOUT, script.end_arrived.notified()) .await - .expect("the end never reached the server"); - tokio::task::yield_now().await; + .expect("deadline end"); + assert_eq!( + observer.changed().await, + moq_relay::shutdown::Tally { open: 0, ending: 2 } + ); assert!( !drained.is_finished(), - "the drain ended before the server answered the end" + "drain ended before the server answered the ends" ); + let views = registry.list(&Default::default()); + assert_eq!(views.iter().filter(|view| view.closed.is_some()).count(), 2); - gate.add_permits(1); - kio::clock::timeout(TIMEOUT, drained) + clock.advance(Duration::from_secs(2)); + gate.add_permits(2); + let summary = kio::clock::timeout(TIMEOUT, drained) .await - .expect("the drain outlived the end") + .expect("drain outlived the ends") .unwrap(); - supervised.await.expect("supervisor task").expect("drained session"); + for supervised in supervised { + supervised.await.expect("supervisor task").expect("drained session"); + } + assert_eq!( + summary, + moq_relay::shutdown::Drained { + forced: 1, + ending: 1, + elapsed: Duration::from_secs(6), + } + ); + assert_eq!(observer.tally(), Default::default()); + assert!(registry.list(&Default::default()).is_empty()); + clock.advance(Duration::from_secs(100)); + assert!(started.elapsed() > summary.elapsed); + assert_eq!(observer.drained().await, summary, "a late observer changed the outcome"); let ends = script.ends(); - let [end] = ends.as_slice() else { - panic!("one end per session: {ends:?}") - }; + assert_eq!(ends.len(), 2); assert!( - matches!( - &end.event, + ends.iter().all(|end| matches!( + end.event, Event::End { reason: moq_auth::lease::Reason::Shutdown, .. } - ), - "{end:?}" + )), + "{ends:?}" ); } diff --git a/rs/moq-relay/tests/integration/drain.rs b/rs/moq-relay/tests/integration/drain.rs index 9391c0a819e..2e6b918f512 100644 --- a/rs/moq-relay/tests/integration/drain.rs +++ b/rs/moq-relay/tests/integration/drain.rs @@ -89,7 +89,7 @@ async fn admit(observer: &shutdown::Observer, version: moq_net::Version, stalled } /// Start the drain and return its end, which [`shutdown::Observer::drained`] reports. -fn drain(trigger: &shutdown::Trigger, observer: &shutdown::Observer) -> kio::clock::sim::JoinHandle<()> { +fn drain(trigger: &shutdown::Trigger, observer: &shutdown::Observer) -> kio::clock::sim::JoinHandle { trigger.start(); let observer = observer.clone(); kio::clock::sim::spawn(async move { observer.drained().await }) From fd84f41b5db7f6cb55bfa93ac56ae54ae729857f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 05:48:08 -0800 Subject: [PATCH 3/4] refactor(relay): reuse test support for simulated drain tasks Co-Authored-By: GPT-6.1 Sol --- Cargo.lock | 1 - rs/moq-relay/Cargo.toml | 7 ++----- rs/moq-relay/src/shutdown.rs | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 428c9c4300d..83555451d05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4957,7 +4957,6 @@ dependencies = [ "moq-congestion", "moq-net", "moq-quic", - "moq-relay", "moq-shaper", "moq-sock", "moq-stats", diff --git a/rs/moq-relay/Cargo.toml b/rs/moq-relay/Cargo.toml index ddb899fe1dc..55d980dfbe8 100644 --- a/rs/moq-relay/Cargo.toml +++ b/rs/moq-relay/Cargo.toml @@ -23,10 +23,8 @@ path = "src/main.rs" doc = false [features] -# A programmatic relay fixture with generated TLS and ephemeral ports. -test-support = [] -# Run shutdown observation on the deterministic in-memory executor. -sim = ["kio/sim", "moq-tokio/sim"] +# A programmatic relay fixture with generated TLS, ephemeral ports, and simulated tasks. +test-support = ["kio/sim", "moq-tokio/sim"] # The optional public STUN Binding listener (`--stun-bind`). stun = ["dep:stun-types"] default = ["cluster-lan", "quic", "websocket", "uds"] @@ -115,7 +113,6 @@ moq-auth = { path = "../moq-auth", features = ["client", "serve"] } moq-congestion = { path = "../moq-congestion" } # The shared session bench fills FETCH runs through the hidden `fuzz` module. moq-net = { path = "../moq-net", features = ["fuzz"] } -moq-relay = { path = ".", default-features = false, features = ["sim"] } moq-shaper = { path = "../moq-shaper", features = ["test"] } qmux = { workspace = true, features = ["tcp"] } rcgen = { workspace = true, features = ["pem", "ring"] } diff --git a/rs/moq-relay/src/shutdown.rs b/rs/moq-relay/src/shutdown.rs index 94def71cfb5..9c2e27a99b9 100644 --- a/rs/moq-relay/src/shutdown.rs +++ b/rs/moq-relay/src/shutdown.rs @@ -70,7 +70,7 @@ impl Trigger { }; state.send_modify(|state| state.wait_ended = Some(ended)); }; - #[cfg(feature = "sim")] + #[cfg(feature = "test-support")] if kio::clock::sim::is_running() { drop(kio::clock::sim::spawn(wait)); return; From c868e9c89c61f7c647ace6bedc956e73551e6fea Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 06:11:18 -0800 Subject: [PATCH 4/4] refactor(relay): stamp admission completion at its source Co-Authored-By: GPT-6.1 Sol --- rs/moq-net/MIGRATE.md | 9 ++ rs/moq-net/src/server.rs | 94 ++++++++++++++----- rs/moq-relay/Cargo.toml | 4 +- rs/moq-relay/src/connection.rs | 4 +- rs/moq-relay/src/relay.rs | 6 +- rs/moq-relay/src/shutdown.rs | 66 +++++-------- .../tests/integration/auth_lifetime.rs | 47 ++++++++-- 7 files changed, 148 insertions(+), 82 deletions(-) diff --git a/rs/moq-net/MIGRATE.md b/rs/moq-net/MIGRATE.md index 508423d1c49..00cdbc25d3f 100644 --- a/rs/moq-net/MIGRATE.md +++ b/rs/moq-net/MIGRATE.md @@ -4,6 +4,15 @@ How to update your code for each breaking change, newest first. ## Unreleased +### Admission completion records its own time + +`server::Activity::drain()` begins closing admission once its active set empties. +`Activity::close()` and the new poll-native `poll_close()` return that instant, +even for a late observer. Replace matches against `()` with a discarded result +or read the returned `kio::clock::Instant`. Once draining begins, the last `Active` +drop closes admission synchronously, including when a close wait was cancelled. +This is native listener bookkeeping; the protocol wire is unchanged. + ### Authored tracks default to a zero max age ([#7729](https://github.com/moq-dev/moq/pull/7729)) `track::Info::default()` sets `max_age` to `Some(Duration::ZERO)` instead of `None`: a superseded group is stale as soon as its successor arrives, so readers get only the live edge; the newest group never goes stale. diff --git a/rs/moq-net/src/server.rs b/rs/moq-net/src/server.rs index 869b79a2021..05485704d10 100644 --- a/rs/moq-net/src/server.rs +++ b/rs/moq-net/src/server.rs @@ -15,51 +15,68 @@ use crate::{ }; /// Tracks admitted connections until their handshake or session ends. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub struct Activity { - state: kio::Producer<()>, - // Admission goes through the weak handle: its open check shares the lock - // `close` commits under, so a connection is either counted or refused. - admit: kio::ProducerWeak<()>, + state: std::sync::Arc>, } -impl Default for Activity { - fn default() -> Self { - let state = kio::Producer::default(); - let admit = state.weak(); - Self { state, admit } - } +#[derive(Debug, Default)] +struct ActivityState { + active: usize, + draining: bool, + closed: Option, } impl Activity { /// Hold the server active until the returned guard is dropped, or `None` once closed. pub fn enter(&self) -> Option { + let mut state = self.state.write().ok()?; + state.active += 1; Some(Active { - _guard: self.admit.try_consume()?, + state: std::sync::Arc::downgrade(&self.state), }) } - /// Wait until every admitted connection has left, then refuse new ones. - pub async fn close(&self) { - loop { - if self.state.wanted_changed(true).await.is_err() { - return; - } - match self.state.write_unwanted() { - kio::Unwanted::Idle(state) => return state.close(), - // Admitted between the wake and the commit: wait for it too. - kio::Unwanted::Wanted => continue, - kio::Unwanted::Closed => return, - } + /// Refuse new admissions as soon as every active connection has left. + pub fn drain(&self) { + let Ok(mut state) = self.state.write() else { return }; + if state.draining { + return; + } + state.draining = true; + if state.active == 0 { + state.closed = Some(kio::clock::now()); + state.close(); } } + + /// Resolve with the instant admission closed after every active connection left. + #[kio::wrapper] + pub fn poll_close(&self, waiter: &kio::Waiter) -> std::task::Poll { + self.drain(); + std::task::ready!(self.state.poll_closed(waiter)); + std::task::Poll::Ready(self.state.read().closed.expect("admission closed")) + } } /// Keeps an admitted connection in the server's active set. #[derive(Debug)] #[must_use] pub struct Active { - _guard: kio::Consumer<()>, + // A connection observes its source without keeping it open. + state: std::sync::Weak>, +} + +impl Drop for Active { + fn drop(&mut self) { + let Some(source) = self.state.upgrade() else { return }; + let Ok(mut state) = source.write() else { return }; + state.active -= 1; + if state.draining && state.active == 0 { + state.closed = Some(kio::clock::now()); + state.close(); + } + } } /// A MoQ server session builder. @@ -900,6 +917,33 @@ mod tests { use crate::ALPN_LITE_05; use bytes::Bytes; + #[kio::test] + async fn admission_closes_at_its_last_drop() { + use futures::FutureExt; + use std::time::Duration; + + let activity = Activity::default(); + let first = activity.enter().unwrap(); + activity.drain(); + let last = activity.enter().expect("admission stays open while active"); + kio::clock::sleep(Duration::from_secs(1)).await; + drop(first); + assert!(activity.close().now_or_never().is_none()); + kio::clock::sleep(Duration::from_secs(1)).await; + let ended = kio::clock::now(); + drop(last); + assert!(activity.enter().is_none(), "last drop did not close admission"); + kio::clock::sleep(Duration::from_secs(100)).await; + assert_eq!(activity.close().await, ended, "the observer changed the close time"); + + let activity = Activity::default(); + let active = activity.enter().unwrap(); + let source = activity.state.weak(); + drop(activity); + assert!(source.is_closed(), "an active connection kept its source open"); + drop(active); + } + fn occurrences(log: &crate::lite::test_transport::Log, needle: &[u8]) -> usize { let writes = log.writes.lock().unwrap(); writes.windows(needle.len()).filter(|window| *window == needle).count() diff --git a/rs/moq-relay/Cargo.toml b/rs/moq-relay/Cargo.toml index 55d980dfbe8..d81179214eb 100644 --- a/rs/moq-relay/Cargo.toml +++ b/rs/moq-relay/Cargo.toml @@ -23,8 +23,8 @@ path = "src/main.rs" doc = false [features] -# A programmatic relay fixture with generated TLS, ephemeral ports, and simulated tasks. -test-support = ["kio/sim", "moq-tokio/sim"] +# A programmatic relay fixture with generated TLS and ephemeral ports. +test-support = [] # The optional public STUN Binding listener (`--stun-bind`). stun = ["dep:stun-types"] default = ["cluster-lan", "quic", "websocket", "uds"] diff --git a/rs/moq-relay/src/connection.rs b/rs/moq-relay/src/connection.rs index 7217c075c2b..ba13777c00f 100644 --- a/rs/moq-relay/src/connection.rs +++ b/rs/moq-relay/src/connection.rs @@ -383,7 +383,6 @@ pub async fn supervise( ) -> anyhow::Result<()> { let mut lease = lease.authorizing(&session); let mut shutdown = serving.shutdown.clone(); - // Held until the `end` is reported, so a drain past its deadline still waits for it. let reported = lease.reported(); let result = loop { let nudged = async { @@ -418,10 +417,13 @@ pub async fn supervise( () = nudged => lease.revalidate(), } }; + // Abort requests are executed by the driver, so observe the transport close first. + session.closed().await; serving.closed(); if let Some(registration) = ®istration { registration.closed(); } + // Keep `serving` and the registration until the end report finishes. reported.await; result } diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index d98d91e4d48..eb7c56c8497 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -1013,9 +1013,9 @@ mod tests { tokio::pin!(draining); assert!(draining.as_mut().now_or_never().is_none()); - // Arrives before the drain observes the count reach zero: it is counted. - drop(last); + // Arrives before the last connection leaves: it is counted. let arrival = activity.enter().expect("admitted before the drain committed"); + drop(last); assert!( draining.as_mut().now_or_never().is_none(), "drain exited under an admitted arrival" @@ -1023,7 +1023,7 @@ mod tests { // Arrives just after the count reaches zero: refused at accept. drop(arrival); - draining.await; + assert!(draining.as_mut().now_or_never().is_some()); assert!( activity.enter().is_none(), "admitted a connection after the drain committed to exit" diff --git a/rs/moq-relay/src/shutdown.rs b/rs/moq-relay/src/shutdown.rs index 9c2e27a99b9..08551bd6957 100644 --- a/rs/moq-relay/src/shutdown.rs +++ b/rs/moq-relay/src/shutdown.rs @@ -14,7 +14,6 @@ use std::time::Duration; -use futures::FutureExt; use kio::clock::Instant; use tokio::sync::watch; @@ -25,8 +24,6 @@ pub struct Trigger { tx: watch::Sender>, drain_timeout: Duration, activity: moq_net::server::Activity, - state: watch::Sender, - runtime: Option, } impl Trigger { @@ -34,8 +31,6 @@ impl Trigger { /// resolves and sessions begin sending GOAWAY, including any accepted later. /// Calling it again keeps the first deadline. pub fn start(&self) { - // Embedders may fire the trigger from outside the relay's runtime. - let _runtime = self.runtime.as_ref().map(tokio::runtime::Handle::enter); let now = Instant::now(); let deadline = now + self.drain_timeout; let first = self.tx.send_if_modified(|started| { @@ -49,33 +44,8 @@ impl Trigger { return; } - // Observe admission independently of `drained`: even its first caller may - // arrive after every session has left. Poll once here so an empty relay - // closes admission at the trigger, without waiting to schedule the task. - let activity = self.activity.clone(); - let mut closing = Box::pin(async move { - activity.close().await; - Instant::now().min(deadline) - }); - if let Some(ended) = closing.as_mut().now_or_never() { - self.state.send_modify(|state| state.wait_ended = Some(ended)); - return; - } - let state = self.state.clone(); - let wait = async move { - let ended = tokio::select! { - ended = closing => ended, - () = kio::clock::sleep_until(deadline) => deadline, - () = state.closed() => return, - }; - state.send_modify(|state| state.wait_ended = Some(ended)); - }; - #[cfg(feature = "test-support")] - if kio::clock::sim::is_running() { - drop(kio::clock::sim::spawn(wait)); - return; - } - drop(tokio::spawn(wait)); + // Admission records its own completion, even if nobody polls `drained`. + self.activity.drain(); } } @@ -175,8 +145,6 @@ struct State { forced: usize, /// Sessions already closed before the deadline but still reporting at it. ending: usize, - /// Admission closed, or the deadline passed. - wait_ended: Option, /// The most recent counted session finished reporting. last_left: Option, } @@ -211,8 +179,6 @@ impl Observer { tx, drain_timeout, activity: observer.activity.clone(), - state: observer.state.clone(), - runtime: tokio::runtime::Handle::try_current().ok(), }; (trigger, observer) } @@ -348,6 +314,12 @@ impl Observer { /// never resolves for a [`disabled`](Self::disabled) handle. pub async fn drained(&self) -> Drained { self.clone().started().await; + let window = self.rx.borrow().expect("started"); + let wait_ended = tokio::select! { + biased; + closed = self.activity.close() => closed.min(window.deadline), + () = self.expired() => window.deadline, + }; // Either way, wait for the counted sessions: past the deadline they are still // reporting their end, and before it one served without an admission still // holds the drain. Sessions served after the deadline are not counted, so @@ -355,15 +327,11 @@ impl Observer { let mut state = self.state.subscribe(); // The sender lives in `self`, so this cannot close. let state = *state - .wait_for(|state| state.wait_ended.is_some() && state.tally == Tally::default()) + .wait_for(|state| state.tally == Tally::default()) .await .expect("sender held"); - let window = self.rx.borrow().expect("started"); - let ended = state - .wait_ended - .expect("wait ended") - .max(state.last_left.unwrap_or(window.started)); + let ended = wait_ended.max(state.last_left.unwrap_or(window.started)); let drained = Drained { forced: state.forced, ending: state.ending, @@ -756,7 +724,7 @@ mod tests { // must be recorded independently of any `drained` future. let (trigger, mut shutdown) = Observer::new(Duration::from_secs(10)); let serving = shutdown.serve(); - std::thread::scope(|scope| scope.spawn(|| trigger.start()).join().unwrap()); + trigger.start(); kio::clock::sleep(Duration::from_secs(1)).await; drop(serving); tokio::task::yield_now().await; @@ -773,6 +741,18 @@ mod tests { } ); assert_eq!(shutdown.drained().await, summary); + + std::thread::spawn(|| { + let (trigger, shutdown) = Observer::new(Duration::from_secs(10)); + let serving = shutdown.serve(); + trigger.start(); + drop(serving); + let summary = futures::executor::block_on(shutdown.drained()); + assert_eq!(summary.forced, 0); + assert_eq!(summary.ending, 0); + }) + .join() + .unwrap(); } /// A drain whose admissions emptied before the deadline still waits for a session diff --git a/rs/moq-relay/tests/integration/auth_lifetime.rs b/rs/moq-relay/tests/integration/auth_lifetime.rs index 1e99cf1a30f..12950f189ab 100644 --- a/rs/moq-relay/tests/integration/auth_lifetime.rs +++ b/rs/moq-relay/tests/integration/auth_lifetime.rs @@ -1159,7 +1159,7 @@ async fn the_drain_waits_for_a_closed_session_to_report_its_end() { let request = auth.request(moq_auth::Transport::Quic, "/room"); let lease = auth.admit(request.clone()).await.expect("admitted before draining"); let registration = registry.register(request); - let (client, server) = memory_sessions().await; + let (client, server, _driving) = memory_sessions().await; clients.push(client); supervised.push(tokio::spawn(moq_relay::supervise( server, @@ -1516,8 +1516,8 @@ async fn a_fixed_lease_still_expires() { /// A connected `(client, server)` session pair over the in-memory transport, so a /// test on Tokio's paused clock never waits on a socket: the clock only advances -/// once every message in flight has landed. -async fn memory_sessions() -> (moq_net::Session, moq_net::Session) { +/// once every message in flight has landed. The handle can pause the server driver. +async fn memory_sessions() -> (moq_net::Session, moq_net::Session, kio::Producer) { let mut config = moq_net::transport::memory::Config::default(); config.protocol = Some(crate::memory::newest().alpn().to_string()); let (client, server) = moq_net::transport::memory::pair(config); @@ -1531,15 +1531,29 @@ async fn memory_sessions() -> (moq_net::Session, moq_net::Session) { tokio::spawn(moq_net::time::run(driver)); session }; + let driving = kio::Producer::new(true); + let gate = driving.consume(); let server = async { let (session, driver) = moq_net::Server::new() .accept(now, server) .await .expect("server handshake"); - tokio::spawn(moq_net::time::run(driver)); + tokio::spawn(async move { + let mut driver = std::pin::pin!(moq_net::time::run(driver)); + kio::wait(|waiter| { + std::task::ready!(gate.poll(waiter, |running| match **running { + true => std::task::Poll::Ready(()), + false => std::task::Poll::Pending, + })) + .unwrap_or(()); + waiter.poll_future(driver.as_mut()) + }) + .await + }); session }; - tokio::join!(client, server) + let (client, server) = tokio::join!(client, server); + (client, server, driving) } /// An outage keeps the session until `expires`, then closes it as expired and @@ -1578,12 +1592,15 @@ async fn closes_at_expires(clock: &kio::clock::Installed, consumer: moq_auth::le let (lease, ()) = tokio::join!(auth.admit(auth.request(moq_auth::Transport::Tcp, "/room")), async { admissions.next().await.expect("an admission").grant(consumer); }); - let (client, server) = memory_sessions().await; + let (client, server, driving) = memory_sessions().await; + let observer = moq_relay::shutdown::Observer::disabled(); + let registry = moq_relay::session::Registry::new(); + let registration = registry.register(auth.request(moq_auth::Transport::Tcp, "/room")); let supervised = tokio::spawn(moq_relay::supervise( server, lease.expect("admitted"), - moq_relay::shutdown::Observer::disabled().serve(), - None, + observer.serve(), + Some(registration), )); clock.advance(EXPIRES - Duration::from_nanos(1)); @@ -1595,11 +1612,23 @@ async fn closes_at_expires(clock: &kio::clock::Installed, consumer: moq_auth::le )] let early = tokio::time::timeout(Duration::from_secs(1), client.closed()).await; assert!(early.is_err(), "closed before expires"); + *driving.write().unwrap_or_else(|_| panic!("driver gate closed")) = false; clock.advance(Duration::from_nanos(1)); #[expect( clippy::disallowed_methods, reason = "the watchdog runs on paused tokio, beside the installed manual clock" )] + let parked = tokio::time::timeout(Duration::from_secs(1), client.closed()).await; + assert!(parked.is_err(), "the paused driver closed the transport"); + assert_eq!(observer.tally(), moq_relay::shutdown::Tally { open: 1, ending: 0 }); + let views = registry.list(&Default::default()); + assert_eq!(views.len(), 1, "registration left before its transport closed"); + assert_eq!(views[0].closed, None); + *driving.write().unwrap_or_else(|_| panic!("driver gate closed")) = true; + #[expect( + clippy::disallowed_methods, + reason = "the watchdog runs on paused tokio, beside the installed manual clock" + )] tokio::time::timeout(TIMEOUT, client.closed()) .await .expect("closed at expires"); @@ -1607,6 +1636,8 @@ async fn closes_at_expires(clock: &kio::clock::Installed, consumer: moq_auth::le .await .expect("supervisor") .expect("a lease end is not a session error"); + assert_eq!(observer.tally(), Default::default()); + assert!(registry.list(&Default::default()).is_empty()); } /// A relay whose config names no auth source is the embedder's to decide: `run`