From 77b70ad2a7fdce40a8f8096611abd258dc56ac92 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 22:28:08 -0800 Subject: [PATCH 1/8] chore(quest): claim quest/m1/session-policy/accept.md Quest-Claim: c1cc4e46c4a040ab From 2c670584bac21eaf301ed2677de4513945f1ac41 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 23:05:37 -0800 Subject: [PATCH 2/8] feat(net)!: accept policy lives in moq-net moq-net's Server bounds each accept with one deadline across SETUP, the caller's admission and the SETUP reply, closing with the timeout code, and opens a held-back receive window on admit through the new transport::poll::Session::set_limits (default: unsupported). accept_request returns a pollable server::Accept that takes the deadline a transport handshake started (Server::deadline, Accept::with_deadline). Handshake::expired bounds admission; Handshake::with_deadline is gone. moq-tokio drops its Deadline, qmux_limits and window plumbing, and WebRTC's fixed receive window becomes qmux's Config::fixed_receive_window. Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/index.md | 3 +- doc/lib/rs/moq-net.md | 2 +- quest/m1/session-policy/README.md | 5 +- quest/m1/session-policy/accept.md | 40 -- quest/m1/session-policy/tokio-handoff.md | 3 +- quest/m1/session-policy/uring.md | 4 - rs/moq-net/MIGRATE.md | 13 + rs/moq-net/src/server.rs | 476 +++++++++++++++++++---- rs/moq-net/src/transport.rs | 36 ++ rs/moq-net/src/transport/boxed.rs | 11 +- rs/moq-relay/src/connection.rs | 2 +- rs/moq-relay/src/incoming.rs | 13 +- rs/moq-relay/src/uring.rs | 8 +- rs/moq-tokio/MIGRATE.md | 4 + rs/moq-tokio/src/listen.rs | 2 +- rs/moq-tokio/src/quic/backend.rs | 7 +- rs/moq-tokio/src/quic/web/session.rs | 8 + rs/moq-tokio/src/server.rs | 306 ++++++--------- rs/moq-tokio/src/tcp.rs | 5 +- rs/moq-tokio/src/webrtc.rs | 2 + rs/moq-uring/src/quic/web.rs | 6 + rs/moq-uring/src/transport/local.rs | 4 + rs/qmux/src/config.rs | 6 + rs/qmux/src/session.rs | 20 +- rs/qmux/src/session/tests.rs | 25 ++ 25 files changed, 680 insertions(+), 331 deletions(-) delete mode 100644 quest/m1/session-policy/accept.md diff --git a/doc/bin/relay/index.md b/doc/bin/relay/index.md index bd5033433a3..23320cea300 100644 --- a/doc/bin/relay/index.md +++ b/doc/bin/relay/index.md @@ -102,7 +102,8 @@ relay.with_web(web).run().await?; - To serve a session from any other moq-net transport, such as the in-memory one in tests, enable the `test-support` feature, read its SETUP with `moq_net::Server::accept_request` and pass `Accepted::new(handshake, transport)` to `Connection::new`. The client must - send its path in the SETUP. + send its path in the SETUP. The server's accept timeout (`moq_net::Server::with_timeout`) + bounds admission and the SETUP reply too. - To decide admissions yourself, leave `[auth]` empty and answer `relay.admissions()`; see [Authentication](/bin/relay/auth#in-process). - To serve WebRTC media (WHIP, WHEP) on the QUIC port, take `relay.webrtc()`: diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index 90c8bc15e34..25fd9894894 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -26,7 +26,7 @@ What you use it for, beyond what the concept page already describes: - **End content by finishing tracks.** A broadcast has no finished state. When the input ends, finish each track; the broadcast stays announced and keeps serving them until you close it or drop its route. How long to keep it up for late readers is yours to decide: `moq_mux::Linger` holds a hang broadcast for a fixed window. - **Demand says whether anyone is watching.** `demand()` on a track, group, or broadcast says whether a subscriber is attached. First-party capture does not wait for it: video encodes while enabled and audio keeps raw PCM to encode when someone listens, so a late subscription's `delay.max` or a FETCH finds the recent past. Demand gates bandwidth reservations, transcode rungs, and on-demand video where a publisher opts in, which saves the encode but leaves nothing from before the first subscriber. Holding a broadcast or track consumer is not demand: a track consumer requests nothing until it queries, subscribes, or fetches, and those count until dropped. A shared fetch stays up until its last reader leaves. Demand rolls up: a frame being read keeps its group wanted, a group a subscription or fetch handed out keeps its track wanted until read to its end, and any wanted track keeps its broadcast wanted. - **Use the model without a runtime.** By default `moq-net` is its `poll_*` APIs and origin driver with caller-supplied time. The `async` feature adds their async wrappers, `time::run`, and both wire sessions. -- **You drive the session, or `moq-tokio` does.** `connect` and `accept` return a session plus a driver that never reads the clock itself. `moq_net::time::run` polls it on tokio. `moq-tokio` hides that. A custom transport implements `moq_net::transport::poll`, and `moq_net::transport::memory::pair` links two sessions in one process with no sockets, for tests or in-process links. +- **You drive the session, or `moq-tokio` does.** `connect` and `accept` return a session plus a driver that never reads the clock itself. `moq_net::time::run` polls it on tokio. `moq-tokio` hides that. A custom transport implements `moq_net::transport::poll`, and `moq_net::transport::memory::pair` links two sessions in one process with no sockets, for tests or in-process links. `Server` bounds each accept with one deadline (10 s by default, `with_timeout`) across SETUP, your admission decision (`Handshake::expired`), and the SETUP reply, and opens a receive window the transport held back once you admit (`with_receive_window`). - **A session bounds what its peer can hold.** Each request holds a stream, so the transport's stream credit bounds subscriptions and FETCHes. `session::Limits` (via `Client::with_limits` and `Server::with_limits`) bounds the announcements per session. The default suits a relay mesh, so lower it for untrusted peers. Past it, the session closes with `TOO_MANY_REQUESTS`. Peer-declared lengths are capped before they are buffered. - **Authorization is in band.** On moq-lite 07 (`moq-lite-07-wip`, opt-in) and on moq-transport draft-18+ with the [MoQ Auth extension](/draft/moq-auth), `session.auth()` presents tokens without reconnecting and watches the grant the peer sent back. `add(token)` resolves with the token and how the peer took it: `Answer::Granted` on its AUTH stream, or `Answer::Requests` on moq-transport without AUTH, where only a request that names it carries it. A handle passed to `Client::with_auth` outlives each session, so its tokens, and the routes naming them, carry across your own reconnects. A server verifies tokens itself by taking `handshake.requests()` before `ok()`. `authorize(&grant)` narrows or widens a live session on any version: what falls outside resets with `Unauthorized` and the session stays up. A client that publishes outside its grant closes the session, naming the path. - **A graceful close waits.** `session.close().await` withdraws announcements and drains for up to one second. `moq-lite-05` and newer wait for the subscriber to receive finished tails; older Lite versions and IETF wait only for transport ACKs. `abort` ends immediately. diff --git a/quest/m1/session-policy/README.md b/quest/m1/session-policy/README.md index 05865ae6f61..0613dbb72d8 100644 --- a/quest/m1/session-policy/README.md +++ b/quest/m1/session-policy/README.md @@ -30,8 +30,8 @@ in moq-net, and decided: until io_uring dials. - ✅ **Fold** the m2 relay-serve-path quest into [the io_uring path](/quest/m1/session-policy/uring.md) and the - poll-first server-machines quest into - [accept policy](/quest/m1/session-policy/accept.md). + poll-first server-machines quest into accept policy, which landed in + [moq-dev/moq#8003](https://github.com/moq-dev/moq/pull/8003). - ✅ **Break moq-net's public API in m1**, before the cut, with MIGRATE notes. Not chosen: additive APIs beside the old ones. @@ -65,7 +65,6 @@ Research (2026-11-04, `origin/main` at e6be13d5f8): ## Required -- [Accept policy in moq-net](/quest/m1/session-policy/accept.md) - one accept deadline, admission bound and window widening for every transport - [Connect policy in moq-net](/quest/m1/session-policy/connect.md) - the connect deadline and reconnect as a poll machine - [The io_uring relay path uses moq-net's policy](/quest/m1/session-policy/uring.md) - deletes `uring.rs`'s copy - [moq-tokio becomes a thin shell](/quest/m1/session-policy/tokio-handoff.md) - sockets, TLS, accept loops and spawning, plus the transport handoffs WebRTC needs diff --git a/quest/m1/session-policy/accept.md b/quest/m1/session-policy/accept.md deleted file mode 100644 index c32b085d28b..00000000000 --- a/quest/m1/session-policy/accept.md +++ /dev/null @@ -1,40 +0,0 @@ -# [M] Accept policy in moq-net - -## Goal - -moq-net's server accepts a session under one policy for every transport: an -accept deadline that spans the transport handshake, SETUP, admission and the -reply and closes with the Timeout code; the admission bound; and a held-back -receive window widened on admit through a new `set_limits` method on -`transport::poll::Session` (default: unsupported). `Server::accept_request` -(which `Server::accept` wraps, and which moq-tokio and moq-relay's io_uring -path call) and `server::Handshake::ok` are `#[kio::machine]` poll methods with -one-line async wrappers. moq-tokio's `Setup::deadline`, `within`, `Deadline::bound` and its -window widening call this instead of implementing it. - -## Plan - -Part of [Session policy lives in moq-net](/quest/m1/session-policy/README.md); -see it for the decisions and research. - -- Folded in 2026-11-04 (maintainer): the poll-first server-machines quest - ([last version](https://github.com/moq-dev/moq/blob/e6be13d5f8/quest/m1/poll-first/server-machines.md)). - Convert `Server::accept` and `Handshake::ok` to poll machines, following - `kio::machine`'s conventions and [moq-dev/moq#7284](https://github.com/moq-dev/moq/pull/7284); - if the handshake's poll form makes either a loop, hand-write that `poll_*` - and say so. The policy then moves onto those machines rather than the async - bodies they replace. -- moq-net already bounds the reply: `Handshake::with_deadline` refuses a SETUP - reply the peer never takes. Extend that deadline back over SETUP and - admission rather than adding a second one. The transport handshake runs - before moq-net has a session, so the caller bounds it against the same - instant, as moq-tokio's `within` does today. -- WebRTC's receive-window exception (`qmux_limits` in moq-tokio's `server.rs`) - becomes a property the transport reports, not a special case in the accept - path. -- Keep today's defaults (10 s accept) and error codes; tests cover a deadline - hit at each phase and widening on admit for QUIC, qmux and a transport that - can't widen. - -Public API: breaking in moq-net (`Server`, `Handshake`, the transport session -trait), with MIGRATE notes. Wire: none. diff --git a/quest/m1/session-policy/tokio-handoff.md b/quest/m1/session-policy/tokio-handoff.md index 3a9310bb17a..bc024d9ebdd 100644 --- a/quest/m1/session-policy/tokio-handoff.md +++ b/quest/m1/session-policy/tokio-handoff.md @@ -23,7 +23,7 @@ moq-net rather than a copy of the policy. - The handoff's signature (transport plus link metadata in, request or session out) is fixed by this quest and does not change if policy moves further. -- Delete moq-tokio's now-unused `Deadline`, `Backoff` and the like, and its +- Delete moq-tokio's now-unused `Backoff` and the like (the accept `Deadline` already went), and its per-transport `Request` construction that moq-net now owns. Bounding the transport handshake (`within`) stays here, since moq-net has no session yet at that point. @@ -37,5 +37,4 @@ it re-exported, with MIGRATE notes. Wire: none. ## Required -- [Accept policy in moq-net](/quest/m1/session-policy/accept.md) - the accept policy the handoff runs - [Connect policy in moq-net](/quest/m1/session-policy/connect.md) - the connect policy the handoff runs diff --git a/quest/m1/session-policy/uring.md b/quest/m1/session-policy/uring.md index 1a67830d0ff..8fae589be23 100644 --- a/quest/m1/session-policy/uring.md +++ b/quest/m1/session-policy/uring.md @@ -32,10 +32,6 @@ see it for the decisions and research. Public API: none. Wire: none. -## Required - -- [Accept policy in moq-net](/quest/m1/session-policy/accept.md) - the shared policy this path calls - ## Related - [io_uring relay package](/quest/m3/relay-io-uring-package.md) - ships the path this unifies diff --git a/rs/moq-net/MIGRATE.md b/rs/moq-net/MIGRATE.md index 13e37ae6884..4b453cd50e5 100644 --- a/rs/moq-net/MIGRATE.md +++ b/rs/moq-net/MIGRATE.md @@ -4,6 +4,19 @@ How to update your code for each breaking change, newest first. ## Unreleased +### Accepting a session runs under one deadline and opens the receive window + +`Server` now bounds each accept, from SETUP through the caller's admission and the SETUP reply, closing a connection that overruns it with `Error::Timeout`'s code. +The default is `server::DEFAULT_TIMEOUT` (10 s); call `Server::with_timeout(None)` to keep waiting forever. +`Server::accept_request` returns a `server::Accept`, which you `.await` as before, or poll; pass it to `kio::clock::timeout` and the like directly, since it is a `Future`. +A caller that runs a transport handshake first computes `Server::deadline(arrived)`, bounds that handshake by it, and passes it on with `Accept::with_deadline`. +`Handshake::with_deadline` is gone: the deadline set at accept bounds `ok` too. +Race your admission decision against the new `Handshake::expired` (or `poll_expired`), which refuses the session with the timeout once the deadline passes; `ok` past the deadline fails with `Error::Timeout`. + +`Server::with_receive_window` and `Handshake::with_receive_window` name the connection receive window `ok` opens before answering SETUP, through the new `transport::poll::Session::set_limits`. +A transport implementation that can change its window overrides `set_limits`; the default reports `LimitsError::Unsupported`, and `ok` then leaves the window as it is. +A wrapper around another transport forwards it. + ### Socket readers start at the beginning New `track` socket sessions subscribe from group 0, and `tcp` sessions join their diff --git a/rs/moq-net/src/server.rs b/rs/moq-net/src/server.rs index fea9d832b0a..765bbc6fcbf 100644 --- a/rs/moq-net/src/server.rs +++ b/rs/moq-net/src/server.rs @@ -1,7 +1,9 @@ //! Accepting a MoQ session, including the paused handshake that inspects the //! peer's SETUP before granting origins. +use std::num::NonZeroU64; use std::task::{Poll, ready}; +use std::time::Duration; use crate::driver::Transport; use crate::origin; @@ -81,8 +83,12 @@ impl Drop for Active { } } +/// How long [`Server`] gives a connection to be accepted, from its arrival through the SETUP +/// reply, unless [`Server::with_timeout`] says otherwise. +pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); + /// A MoQ server session builder. -#[derive(Default, Clone)] +#[derive(Clone)] pub struct Server { publish: Option, subscribe: Option, @@ -91,6 +97,24 @@ pub struct Server { limits: crate::session::Limits, extensions: setup::Extensions, implementation: Option, + timeout: Option, + receive_window: Option, +} + +impl Default for Server { + fn default() -> Self { + Self { + publish: None, + subscribe: None, + stats: Default::default(), + versions: Default::default(), + limits: Default::default(), + extensions: Default::default(), + implementation: None, + timeout: Some(DEFAULT_TIMEOUT), + receive_window: None, + } + } } impl Server { @@ -153,6 +177,33 @@ impl Server { self } + /// Bound each accept, from the connection's arrival through SETUP, the caller's admission, + /// and the SETUP reply. A connection that overruns it is closed with [`Error::Timeout`]'s + /// code. `None` waits forever. Defaults to [`DEFAULT_TIMEOUT`]. + pub fn with_timeout(mut self, timeout: Option) -> Self { + self.timeout = timeout; + self + } + + /// When a connection that arrived at `arrived` must be accepted by, or `None` to wait + /// forever, including past the clock's range. + /// + /// A caller that runs a transport handshake before [`accept_request`](Self::accept_request) + /// bounds that handshake by this instant, then hands it to [`Accept::with_deadline`]. + pub fn deadline(&self, arrived: Instant) -> Option { + arrived.checked_add(self.timeout?) + } + + /// Open the connection's receive window to `window` once the caller admits the session, + /// before SETUP is answered, so a transport can hold most of it back until then. + /// + /// A transport that cannot change its window keeps its own. Unset, the window stays as the + /// transport set it. + pub fn with_receive_window(mut self, window: NonZeroU64) -> Self { + self.receive_window = Some(window); + self + } + /// The Implementation our SETUP carries, or `None` to omit it. fn implementation(&self) -> Option { setup::implementation(self.implementation.as_deref().unwrap_or(crate::IMPLEMENTATION)) @@ -285,11 +336,11 @@ impl Server { token, peer_implementation, assigned_hop: crate::Hop::random(), + timer: Timer::new(None), auth: crate::auth::Link::new(version.has_auth()), inner: Some(RequestInner { server: self.clone(), runtime, - deadline: None, handshake, }), } @@ -297,7 +348,8 @@ impl Server { /// Perform the MoQ handshake as a server, returning the [`Session`] and its [`Driver`](crate::Driver). /// - /// Poll the returned driver with nondecreasing time, starting at `now`. + /// Poll the returned driver with nondecreasing time, starting at `now`. The accept + /// deadline counts from `now` too. /// /// Convenience wrapper over [`accept_request`](Self::accept_request) that /// completes the handshake immediately. Use `accept_request` when you need to @@ -307,13 +359,12 @@ impl Server { now: Instant, session: impl crate::transport::poll::Boxable, ) -> impl Future> { - // Box before any protocol code runs, so it is compiled once rather than per transport. - let mut accept = Accept::new(Deadlines::new(now), Transport::new(session)); + let mut accept = self.accept_request(now, session); let mut answer = None; kio::wait(move |waiter| { let answer = match &mut answer { Some(answer) => answer, - None => answer.insert(Answer::new(ready!(accept.poll(self, waiter))?)), + None => answer.insert(Answer::new(ready!(accept.poll(waiter))?)), }; answer.poll(waiter) }) @@ -332,14 +383,19 @@ impl Server { /// /// A SETUP that fails to parse or negotiate closes the session with the matching code, /// so the peer learns why instead of seeing a bare disconnect. - pub fn accept_request( - &self, - now: Instant, - session: impl crate::transport::poll::Boxable, - ) -> impl Future> { - // Box before any protocol code runs, so it is compiled once rather than per transport. - let mut accept = Accept::new(Deadlines::new(now), Transport::new(session)); - kio::wait(move |waiter| accept.poll(self, waiter)) + /// + /// The session's clock starts at `now`, and so does the accept deadline + /// ([`with_timeout`](Self::with_timeout)) unless [`Accept::with_deadline`] moves it. The + /// deadline goes on to bound the caller's admission ([`Handshake::expired`]) and the SETUP + /// reply ([`Handshake::ok`]). + pub fn accept_request(&self, now: Instant, session: impl crate::transport::poll::Boxable) -> Accept { + Accept { + server: self.clone(), + timer: Timer::new(self.deadline(now)), + // Box before any protocol code runs, so it is compiled once rather than per transport. + handshake: Handshaking::new(Deadlines::new(now), Transport::new(session)), + park: kio::Park::default(), + } } /// Pause a lite-01/02 handshake once the client's SETUP picked the version; `ok()` sends @@ -359,12 +415,12 @@ impl Server { token: None, peer_implementation: None, assigned_hop: crate::Hop::random(), + timer: Timer::new(None), // Lite 01/02 carry no AUTH. auth: crate::auth::Link::new(false), inner: Some(RequestInner { server: self.clone(), runtime, - deadline: None, handshake: PausedHandshake::Negotiated(PausedNegotiated { session, stream, @@ -394,13 +450,13 @@ impl Server { token: options.token.clone(), peer_implementation: options.implementation.clone(), assigned_hop: crate::Hop::random(), + timer: Timer::new(None), // The client's SETUP already settled whether it offered MoQ Auth, so it is // negotiated exactly when we offer it too. auth: crate::auth::Link::ietf(options.declared.auth && self.extensions.auth), inner: Some(RequestInner { server: self.clone(), runtime, - deadline: None, handshake: PausedHandshake::Ietf(PausedIetf { session, version, @@ -411,13 +467,87 @@ impl Server { } } -/// The server's side of a handshake, up to the pause where [`Handshake`] hands the client's -/// SETUP to the caller. +/// A connection being accepted, up to the pause where [`Handshake`] hands the client's SETUP to +/// the caller. Await it, or [`poll`](Self::poll) it. /// /// A failure closes the transport with the error's code, so the peer learns why instead of -/// seeing a bare disconnect. Dropped while pending, it releases what its step holds and -/// closes nothing. -struct Accept { +/// seeing a bare disconnect; past the deadline, that is [`Error::Timeout`]. Dropped while +/// pending, it releases what it holds and closes nothing. +#[must_use = "an accept does nothing unless awaited or polled"] +pub struct Accept { + server: Server, + timer: Timer, + handshake: Handshaking, + park: kio::Park, +} + +impl Accept { + /// Give up at `at` rather than [`Server::with_timeout`] after the accept's start, or never + /// with `None`: for a connection whose transport handshake started the clock earlier, at + /// the instant [`Server::deadline`] gave. + pub fn with_deadline(mut self, at: Option) -> Self { + self.timer = Timer::new(at); + self + } + + /// The paused handshake, or the error the transport was closed with. + pub fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { + Self::resolve(&self.server, &mut self.timer, &mut self.handshake, waiter) + } + + fn resolve( + server: &Server, + timer: &mut Timer, + handshake: &mut Handshaking, + waiter: &kio::Waiter, + ) -> Poll> { + let res = ready!(handshake.poll(server, timer, waiter)); + Poll::Ready(res.map(|mut handshake| { + // The same deadline bounds admission and the reply. + handshake.timer = std::mem::replace(timer, Timer::new(None)); + handshake + })) + } +} + +impl Future for Accept { + type Output = Result; + + fn poll(self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { + let Self { + server, + timer, + handshake, + park, + } = self.get_mut(); + Self::resolve(server, timer, handshake, park.hold(cx)) + } +} + +/// A deadline's timer, armed on its first poll, that never fires without a deadline. +struct Timer { + at: Option, + sleep: Option, +} + +impl Timer { + fn new(at: Option) -> Self { + Self { at, sleep: None } + } + + /// Ready once the deadline has passed. + fn poll(&mut self, waiter: &kio::Waiter) -> Poll<()> { + let Some(at) = self.at else { + return Poll::Pending; + }; + let sleep = self.sleep.get_or_insert_with(|| kio::clock::sleep_until(at)); + waiter.poll_future(std::pin::Pin::new(sleep)) + } +} + +/// The server's side of a handshake, up to the pause where [`Handshake`] hands the client's +/// SETUP to the caller. +struct Handshaking { /// Closed on failure. Released once resolved, so it never outlives the handshake. refused: Option, /// After a hand-off, the negotiation stream's receive half, kept open until the handshake @@ -426,7 +556,7 @@ struct Accept { step: AcceptStep, } -/// Where [`Accept`] waits. Every step but `Done` holds the session's clock and transport, +/// Where [`Handshaking`] waits. Every step but `Done` holds the session's clock and transport, /// which move into the [`Handshake`]. enum AcceptStep { /// Not polled yet: the transport's ALPN picks the path. @@ -472,7 +602,7 @@ enum AcceptStep { Done, } -impl Accept { +impl Handshaking { fn new(runtime: Deadlines, session: Transport) -> Self { Self { refused: Some(session.clone()), @@ -482,8 +612,15 @@ impl Accept { } /// The paused handshake, or the error the transport was closed with. - fn poll(&mut self, server: &Server, waiter: &kio::Waiter) -> Poll> { - let res = ready!(self.poll_step(server, waiter)); + fn poll(&mut self, server: &Server, timer: &mut Timer, waiter: &kio::Waiter) -> Poll> { + let res = match self.poll_step(server, waiter) { + Poll::Ready(res) => res, + // Polled second, so a SETUP that is ready wins even past the deadline. + Poll::Pending => { + ready!(timer.poll(waiter)); + Err(Error::Timeout) + } + }; // Release everything the handshake held before closing over a failure. self.step = AcceptStep::Done; self.held = None; @@ -703,7 +840,9 @@ pub struct Handshake { /// The session's auth handle, available before [`Handshake::ok`] so the caller can /// take the peer's token requests before any arrive. auth: crate::auth::Link, - // Taken by `ok`/`close`; `Drop` rejects the handshake if neither ran. + /// The accept deadline, which bounds admission and the reply. + timer: Timer, + // Taken by `ok`/`close`, or by the deadline; `Drop` rejects the handshake if none ran. inner: Option, } @@ -712,8 +851,6 @@ struct RequestInner { server: Server, /// Supplies the clock and timers for the accepted session. runtime: Deadlines, - /// When [`Handshake::ok`] gives up on a peer that has not taken the SETUP reply. - deadline: Option, handshake: PausedHandshake, } @@ -1021,15 +1158,14 @@ impl Handshake { /// Publish to the connected client. Overrides any value from the [`Server`] /// builder; typically set after inspecting [`path`](Self::path). - pub fn with_publisher(mut self, publish: impl Consume) -> Self { - self.inner_mut().server.publish = Some(publish.consume()); - self + pub fn with_publisher(self, publish: impl Consume) -> Self { + let publish = publish.consume(); + self.configure(|server| server.publish = Some(publish)) } /// Subscribe to the connected client. Overrides any value from the [`Server`] builder. - pub fn with_subscriber(mut self, subscribe: origin::Producer) -> Self { - self.inner_mut().server.subscribe = Some(subscribe); - self + pub fn with_subscriber(self, subscribe: origin::Producer) -> Self { + self.configure(|server| server.subscribe = Some(subscribe)) } /// Assign the identity this peer's routes are attributed to, overriding the fresh @@ -1051,42 +1187,63 @@ impl Handshake { /// Set the per-connection [`stats::Session`] context. Overrides any value from the /// [`Server`] builder. - pub fn with_stats(mut self, stats: stats::Session) -> Self { - self.inner_mut().server.stats = stats; - self + pub fn with_stats(self, stats: stats::Session) -> Self { + self.configure(|server| server.stats = stats) } /// Set the session's limits after authorizing the peer and before answering SETUP. - pub fn with_limits(mut self, limits: crate::session::Limits) -> Self { - self.inner_mut().server.limits = limits; - self + pub fn with_limits(self, limits: crate::session::Limits) -> Self { + self.configure(|server| server.limits = limits) } - /// Refuse the session with [`Error::Timeout`] if [`ok`](Self::ok) has not answered - /// SETUP by `at`, as when the peer withholds flow-control credit for the reply. - /// - /// Dropping a pending `ok` refuses it the same way, so a caller's own deadline counts too. - pub fn with_deadline(mut self, at: Instant) -> Self { - self.inner_mut().deadline = Some(at); + /// Open the receive window to `window` on [`ok`](Self::ok). Overrides any value from the + /// [`Server`] builder; see [`Server::with_receive_window`]. + pub fn with_receive_window(self, window: NonZeroU64) -> Self { + self.configure(|server| server.receive_window = Some(window)) + } + + /// Change the server config the session starts with, unless the deadline already refused it. + fn configure(mut self, change: impl FnOnce(&mut Server)) -> Self { + if let Some(inner) = &mut self.inner { + change(&mut inner.server); + } self } - fn inner_mut(&mut self) -> &mut RequestInner { - self.inner.as_mut().expect("request already responded") + /// Resolve once the accept deadline passes, having refused the session with + /// [`Error::Timeout`]; never, without a deadline. + /// + /// Race the admission decision against it, so a peer cannot hold the connection open + /// behind a decision that never comes. Once it resolves, [`ok`](Self::ok) fails with the + /// timeout. + #[kio::wrapper] + pub fn poll_expired(&mut self, waiter: &kio::Waiter) -> Poll<()> { + ready!(self.timer.poll(waiter)); + if let Some(inner) = self.inner.take() { + inner.close(Error::Timeout); + } + Poll::Ready(()) } /// Accept the session, returning the [`Session`] and its [`Driver`](crate::Driver). /// + /// Opens the receive window first ([`with_receive_window`](Self::with_receive_window)). + /// Fails with [`Error::Timeout`] once the accept deadline passes, whether admission or a + /// peer that never takes the SETUP reply ran it out. Dropping a pending `ok` refuses the + /// session the same way, so a caller's own deadline counts too. + /// /// Poll or spawn the returned driver to run the session. pub async fn ok(self) -> Result<(Session, crate::Driver), Error> { let mut answer = Answer::new(self); kio::wait(|waiter| answer.poll(waiter)).await } - /// Reject the session, closing the transport with `err`'s wire code. + /// Reject the session, closing the transport with `err`'s wire code. Does nothing once + /// the deadline refused it. pub fn close(mut self, err: Error) { - let inner = self.inner.take().expect("request already responded"); - inner.close(err); + if let Some(inner) = self.inner.take() { + inner.close(err); + } } } @@ -1102,36 +1259,43 @@ impl RequestInner { /// server SETUP. Dropped before it resolves, it refuses the session as the deadline does. // Fields drop in order: the reply before the guard refuses, and the handshake last. struct Answer { - /// What starts the session, taken by the first poll. + /// What starts the session, taken by the first poll; `None` when the deadline already + /// refused it. start: Option<(Server, Deadlines, PausedHandshake)>, /// A negotiated client's server SETUP on its way out. Boxed so the bidi SETUP write of /// these old versions doesn't size every caller's handshake. reply: Option>, - /// The deadline's timer, armed once the reply first waits. - timer: Option, + /// The accept deadline. + timer: Timer, unanswered: Unanswered, - /// When to give up on the peer taking the reply. - deadline: Option, /// The request being answered, emptied of what the start took. handshake: Handshake, } impl Answer { fn new(mut handshake: Handshake) -> Self { - let RequestInner { + let timer = std::mem::replace(&mut handshake.timer, Timer::new(None)); + let Some(RequestInner { server, runtime, - deadline, handshake: paused, - } = handshake.inner.take().expect("request already responded"); + }) = handshake.inner.take() + else { + return Self { + start: None, + reply: None, + timer, + unanswered: Unanswered(None), + handshake, + }; + }; let stats = server.stats.clone(); let (session, version) = paused.transport(); Self { start: Some((server, runtime, paused)), reply: None, - timer: None, - unanswered: Unanswered(Some((session, version, stats.clone()))), - deadline, + timer, + unanswered: Unanswered(Some((session, version, stats))), handshake, } } @@ -1141,7 +1305,7 @@ impl Answer { let res = ready!(self.poll_reply(waiter)); // Release the reply before the guard closes over a deadline. self.reply = None; - self.timer = None; + self.timer = Timer::new(None); let Some(res) = res else { drop(std::mem::replace(&mut self.unanswered, Unanswered(None))); return Poll::Ready(Err(Error::Timeout)); @@ -1156,6 +1320,13 @@ impl Answer { /// The reply's outcome, or `None` once the deadline passed first. fn poll_reply(&mut self, waiter: &kio::Waiter) -> Poll>> { if let Some((server, runtime, paused)) = self.start.take() { + // Admission ran out the deadline: refuse rather than start. + if self.timer.poll(waiter).is_ready() { + return Poll::Ready(None); + } + if let Err(err) = open_window(&server, &paused) { + return Poll::Ready(Some(Err(err))); + } let peer_hop = Some(self.handshake.assigned_hop); let auth = self.handshake.auth.clone(); let started = match paused { @@ -1181,20 +1352,33 @@ impl Answer { }; return Poll::Ready(Some(started)); } - let reply = self.reply.as_mut().expect("answer polled after it resolved"); + // Neither a start nor a reply: the deadline refused the request before `ok`. + let Some(reply) = self.reply.as_mut() else { + return Poll::Ready(None); + }; // The reply is polled first, so one that is ready wins even past the deadline. if let Poll::Ready(res) = reply.poll(waiter) { return Poll::Ready(Some(res)); } - let Some(at) = self.deadline else { - return Poll::Pending; - }; - let timer = self.timer.get_or_insert_with(|| kio::clock::sleep_until(at)); - ready!(waiter.poll_future(std::pin::Pin::new(timer))); + ready!(self.timer.poll(waiter)); Poll::Ready(None) } } +/// Open the receive window the transport held back until admission, before anything answers +/// SETUP. A transport that cannot change it never held it back. +fn open_window(server: &Server, paused: &PausedHandshake) -> Result<(), Error> { + let Some(window) = server.receive_window else { + return Ok(()); + }; + let mut limits = crate::transport::Limits::default(); + limits.receive_window = Some(window); + match paused.transport().0.set_limits(limits) { + Ok(()) | Err(crate::transport::LimitsError::Unsupported) => Ok(()), + Err(err) => Err(Error::Transport(err.to_string())), + } +} + /// A SETUP reply the peer has not taken yet. Dropped unanswered, it refuses the session with /// [`Error::Timeout`]: the reply waits only on the peer, so giving up on it means the peer /// took too long. @@ -1326,6 +1510,9 @@ mod tests { stops: Arc>>, /// The peer grants no credit, so nothing this side writes is ever taken. withheld: bool, + /// The receive window `set_limits` opened, or why it refused, or `None` when the + /// transport cannot change it. + window: Option, String>>>>, } impl FakeSession { @@ -1338,9 +1525,26 @@ mod tests { closed: Default::default(), stops: Default::default(), withheld: false, + window: None, } } + /// A transport that can change its receive window, or refuses with `refusal`. + fn widening(mut self, refusal: Option<&str>) -> Self { + let state = match refusal { + Some(reason) => Err(reason.to_string()), + None => Ok(None), + }; + self.window = Some(Arc::new(Mutex::new(state))); + self + } + + /// The receive window `set_limits` opened. + fn window(&self) -> Option { + let window = self.window.as_ref().expect("a widening transport"); + window.lock().unwrap().clone().expect("limits accepted") + } + fn withholding_credit(mut self) -> Self { self.withheld = true; self @@ -1403,6 +1607,19 @@ mod tests { ) -> std::task::Poll> { std::task::Poll::Pending } + fn set_limits(&self, limits: crate::transport::Limits) -> Result<(), crate::transport::LimitsError> { + let Some(window) = &self.window else { + return Err(crate::transport::LimitsError::Unsupported); + }; + let mut window = window.lock().unwrap(); + match &mut *window { + Ok(opened) => { + *opened = limits.receive_window; + Ok(()) + } + Err(reason) => Err(crate::transport::LimitsError::Refused(reason.clone())), + } + } fn send_group(&self, _: i64) -> Option { None } @@ -1610,7 +1827,7 @@ mod tests { .with_bi(moql_setup()) .withholding_credit(); let start = kio::clock::now(); - let deadline = start + std::time::Duration::from_secs(10); + let deadline = start + DEFAULT_TIMEOUT; let request = Server::new() .accept_request(start, session.clone()) .await @@ -1619,8 +1836,7 @@ mod tests { registry .tier(Default::default()) .session(crate::Path::from_static("room")), - ) - .with_deadline(deadline); + ); let Err(err) = request.ok().await else { panic!("a reply the peer never took completed"); }; @@ -1631,6 +1847,120 @@ mod tests { assert_eq!(failed.count(stats::Failure::Application), 1, "{failed:?}"); } + /// A client that never sends its SETUP is refused at the accept deadline with the timeout + /// code, which its version can't pick yet. The deadline is the one the caller handed over, + /// as after a transport handshake that started the clock. + #[kio::test] + async fn a_setup_that_never_arrives_fails_at_the_deadline() { + for session in [ + FakeSession::new(ALPN_22, []), + FakeSession::new(ALPN_LITE_05, []), + FakeSession::new(ALPN_LITE, []), + ] { + let deadline = kio::clock::now() + std::time::Duration::from_secs(3); + let accept = Server::new() + .accept_request(kio::clock::now(), session.clone()) + .with_deadline(Some(deadline)); + let Err(err) = accept.await else { + panic!("{:?}: accepted without a SETUP", session.protocol); + }; + assert!(matches!(err, Error::Timeout), "{err:?}"); + assert_eq!(kio::clock::now(), deadline); + assert_eq!(session.close_code(), Some(SessionError::Timeout.to_code())); + } + } + + /// An admission decision that never comes is refused at the deadline the accept started, + /// counted in the session's stats, and `ok` after it fails rather than starting. + #[kio::test] + async fn an_admission_past_the_deadline_is_refused() { + let registry = stats::Registry::new(Default::default()); + let session = FakeSession::new(ALPN_LITE_05, [lite05_setup(Some("room"), None, None)]); + let start = kio::clock::now(); + let mut request = Server::new() + .with_timeout(Some(std::time::Duration::from_secs(5))) + .accept_request(start, session.clone()) + .await + .unwrap() + .with_stats( + registry + .tier(Default::default()) + .session(crate::Path::from_static("room")), + ); + request.expired().await; + assert_eq!(kio::clock::now(), start + std::time::Duration::from_secs(5)); + let version = Version::Lite(lite::Version::Lite05); + assert_eq!(session.close_code(), Some(version.session_code(&SessionError::Timeout))); + let failed = registry.snapshot().sessions()[0].1.sessions_failed; + assert_eq!(failed.count(stats::Failure::Application), 1, "{failed:?}"); + let Err(err) = request.with_limits(Default::default()).ok().await else { + panic!("an expired handshake started"); + }; + assert!(matches!(err, Error::Timeout), "{err:?}"); + } + + /// A caller that admits past the deadline without racing it gets the timeout from `ok`. + #[kio::test] + async fn ok_past_the_deadline_is_refused() { + let session = FakeSession::new(ALPN_LITE_05, [lite05_setup(None, None, None)]); + let request = Server::new() + .with_timeout(Some(std::time::Duration::from_secs(1))) + .accept_request(kio::clock::now(), session.clone()) + .await + .unwrap(); + kio::clock::sleep(std::time::Duration::from_secs(1)).await; + let Err(err) = request.ok().await else { + panic!("a late admission started"); + }; + assert!(matches!(err, Error::Timeout), "{err:?}"); + let version = Version::Lite(lite::Version::Lite05); + assert_eq!(session.close_code(), Some(version.session_code(&SessionError::Timeout))); + } + + /// Admitting opens the receive window the server or the request names, before the + /// session starts; a transport that cannot change its window starts as it is. + #[kio::test] + async fn ok_opens_the_receive_window() { + let window = std::num::NonZeroU64::new(1 << 20).unwrap(); + let wider = std::num::NonZeroU64::new(4 << 20).unwrap(); + let server = Server::new().with_receive_window(window); + + let session = FakeSession::new(ALPN_LITE_05, [lite05_setup(None, None, None)]).widening(None); + let request = server.accept_request(kio::clock::now(), session.clone()).await.unwrap(); + let _accepted = request.ok().await.expect("ok"); + assert_eq!(session.window(), Some(window)); + + let session = FakeSession::new(ALPN_LITE_05, [lite05_setup(None, None, None)]).widening(None); + let request = server.accept_request(kio::clock::now(), session.clone()).await.unwrap(); + let _accepted = request.with_receive_window(wider).ok().await.expect("ok"); + assert_eq!(session.window(), Some(wider)); + + let fixed = FakeSession::new(ALPN_LITE_05, [lite05_setup(None, None, None)]); + let request = server.accept_request(kio::clock::now(), fixed.clone()).await.unwrap(); + let _accepted = request.ok().await.expect("a transport that can't widen starts"); + assert_eq!(fixed.close_code(), None); + } + + /// A window the transport refuses fails the session and tells the peer, rather than + /// serving it on the window it was held back to. + #[kio::test] + async fn a_refused_window_closes_the_session() { + let window = std::num::NonZeroU64::new(1 << 20).unwrap(); + let session = FakeSession::new(ALPN_LITE_05, [lite05_setup(None, None, None)]).widening(Some("too small")); + let request = Server::new() + .with_receive_window(window) + .accept_request(kio::clock::now(), session.clone()) + .await + .unwrap(); + let Err(err) = request.ok().await else { + panic!("served on a refused window"); + }; + assert!(matches!(err, Error::Transport(_)), "{err:?}"); + let version = Version::Lite(lite::Version::Lite05); + let code = version.session_code(&SessionError::from(&err)); + assert_eq!(session.close_code(), Some(code)); + } + /// A reply the caller stops waiting for, as a relay does at its drain deadline, is /// refused like one past the handshake's own deadline. #[kio::test] @@ -1962,10 +2292,10 @@ mod tests { peer_implementation: None, assigned_hop: Hop::random(), auth: crate::auth::Link::ietf(false), + timer: Timer::new(None), inner: Some(RequestInner { server: Server::new().with_publisher(&origin), runtime: Deadlines::new(kio::clock::now()), - deadline: None, handshake: PausedHandshake::Ietf(PausedIetf { session: Transport::new(transport), version, diff --git a/rs/moq-net/src/transport.rs b/rs/moq-net/src/transport.rs index 30141b4e1dc..c96efefdc3d 100644 --- a/rs/moq-net/src/transport.rs +++ b/rs/moq-net/src/transport.rs @@ -195,6 +195,27 @@ impl StreamCredit { } } +/// Limits to change on a live session with [`poll::Session::set_limits`]. Each `None` leaves +/// its limit as it is. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[non_exhaustive] +pub struct Limits { + /// The connection-wide receive window, in bytes. + pub receive_window: Option, +} + +/// Why a transport did not apply [`Limits`]. +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum LimitsError { + /// The transport cannot change these limits on a live session. + #[error("the transport cannot change these limits")] + Unsupported, + /// The transport refused the values, with its reason. + #[error("limits refused: {0}")] + Refused(String), +} + /// The smallest connection receive window a transport that leases may offer: one 1 MiB /// storage segment. /// @@ -397,6 +418,17 @@ pub mod poll { None } + /// Apply `limits` to the live session, such as to open the receive window a server held + /// back until it admitted the peer. + /// + /// Errors with [`LimitsError::Unsupported`](super::LimitsError::Unsupported) when the + /// transport cannot change them, which is the default: such a transport holds nothing + /// back to open. + fn set_limits(&self, limits: super::Limits) -> Result<(), super::LimitsError> { + let _ = limits; + Err(super::LimitsError::Unsupported) + } + /// Return the peer's stream credit in `dir`, or `None` when the transport cannot say, /// as a browser's WebTransport cannot. /// @@ -623,6 +655,10 @@ pub mod poll { (**self).receive_window() } + fn set_limits(&self, limits: super::Limits) -> Result<(), super::LimitsError> { + (**self).set_limits(limits) + } + fn stream_credit(&self, dir: super::Dir) -> Option { (**self).stream_credit(dir) } diff --git a/rs/moq-net/src/transport/boxed.rs b/rs/moq-net/src/transport/boxed.rs index 27854c1e2d0..8217d715b18 100644 --- a/rs/moq-net/src/transport/boxed.rs +++ b/rs/moq-net/src/transport/boxed.rs @@ -14,7 +14,7 @@ use bytes::{Buf, Bytes}; use super::open::{self, Open}; use super::poll::{self, SendOptions}; -use super::{Address, CloseKind, Dir, Probe, ReceiveWindow, StreamCredit, Wire}; +use super::{Address, CloseKind, Dir, Limits, LimitsError, Probe, ReceiveWindow, StreamCredit, Wire}; /// A boxed [`poll::Boxable`] session, shared by every task that uses it. #[derive(Clone)] @@ -348,6 +348,7 @@ pub(super) trait ErasedSession: Send + Sync { fn wire(&self) -> Option; fn probe(&self) -> Option; fn receive_window(&self) -> Option; + fn set_limits(&self, limits: Limits) -> Result<(), LimitsError>; fn stream_credit(&self, dir: Dir) -> Option; fn export_keying_material(&self, label: &[u8], context: &[u8]) -> Option<[u8; 32]>; } @@ -486,6 +487,10 @@ impl ErasedSession for S { poll::Session::receive_window(self) } + fn set_limits(&self, limits: Limits) -> Result<(), LimitsError> { + poll::Session::set_limits(self, limits) + } + fn stream_credit(&self, dir: Dir) -> Option { poll::Session::stream_credit(self, dir) } @@ -678,6 +683,10 @@ impl poll::Session for Session { self.inner.session.receive_window() } + fn set_limits(&self, limits: Limits) -> Result<(), LimitsError> { + self.inner.session.set_limits(limits) + } + fn stream_credit(&self, dir: Dir) -> Option { self.inner.session.stream_credit(dir) } diff --git a/rs/moq-relay/src/connection.rs b/rs/moq-relay/src/connection.rs index ba13777c00f..4c824abed5e 100644 --- a/rs/moq-relay/src/connection.rs +++ b/rs/moq-relay/src/connection.rs @@ -86,7 +86,7 @@ impl Connection { lifecycle.run(Self::serve(request, &lifecycle)).await; } - async fn serve(request: R, lifecycle: &Lifecycle) -> anyhow::Result<()> { + async fn serve(mut request: R, lifecycle: &Lifecycle) -> anyhow::Result<()> { // Abandoned at the drain deadline, so the deadline bounds this task and its // socket like it bounds a session, and at the handshake deadline, so a peer // cannot hold the connection open behind a decision that never comes. diff --git a/rs/moq-relay/src/incoming.rs b/rs/moq-relay/src/incoming.rs index 169adda7e8d..9033af43289 100644 --- a/rs/moq-relay/src/incoming.rs +++ b/rs/moq-relay/src/incoming.rs @@ -52,7 +52,7 @@ pub trait Sealed: Sized { fn admission(&self, auth: &auth::Auth) -> moq_auth::Request; /// Finish `work` before the handshake deadline, closing the session if it passes. - async fn within(&self, work: impl std::future::Future) -> anyhow::Result; + async fn within(&mut self, work: impl std::future::Future) -> anyhow::Result; /// Refuse the session, closing it with `code`. async fn reject(self, code: moq_net::SessionError); @@ -100,7 +100,7 @@ impl Sealed for moq_tokio::server::Request { auth::request_for(auth, self) } - async fn within(&self, work: impl std::future::Future) -> anyhow::Result { + async fn within(&mut self, work: impl std::future::Future) -> anyhow::Result { Ok(moq_tokio::server::Request::within(self, work).await?) } @@ -181,9 +181,12 @@ impl Sealed for Accepted { auth::request_for_setup(auth, &self.handshake, self.transport) } - /// No listener deadline: whoever owns the transport bounds the handshake. - async fn within(&self, work: impl std::future::Future) -> anyhow::Result { - Ok(work.await) + async fn within(&mut self, work: impl std::future::Future) -> anyhow::Result { + let expired = self.handshake.expired(); + match futures::future::select(std::pin::pin!(work), std::pin::pin!(expired)).await { + futures::future::Either::Left((done, _)) => Ok(done), + futures::future::Either::Right(_) => Err(moq_net::Error::Timeout.into()), + } } async fn reject(self, code: moq_net::SessionError) { diff --git a/rs/moq-relay/src/uring.rs b/rs/moq-relay/src/uring.rs index f3829e2ddc9..b2020c79249 100644 --- a/rs/moq-relay/src/uring.rs +++ b/rs/moq-relay/src/uring.rs @@ -798,9 +798,10 @@ async fn serve_connection( alpn, url, }; + let at = accepted.deadline.at; let request = accepted .deadline - .bound(server.accept_request(now, session)) + .bound(server.accept_request(now, session).with_deadline(at)) .await .context("moq handshake failed")?; serve_request(accepted, request, lifecycle).await @@ -906,11 +907,6 @@ async fn serve_request( if let Some(publisher) = publisher { request = request.with_subscriber(publisher); } - // moq-net enforces the handshake deadline on the reply, so one the peer never - // took counts in the session's stats. - if let Some(at) = deadline.at { - request = request.with_deadline(at); - } // A grant that no longer holds is refused, so the session records why it ended. let lease = match lease.preparing(request.auth(), request.requests(), request.role()) { Ok(lease) => lease, diff --git a/rs/moq-tokio/MIGRATE.md b/rs/moq-tokio/MIGRATE.md index b3e894652a8..179659c5751 100644 --- a/rs/moq-tokio/MIGRATE.md +++ b/rs/moq-tokio/MIGRATE.md @@ -4,6 +4,10 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `Request::within` takes `&mut self` + +Admission is bounded by moq-net's accept deadline (`moq_net::server::Handshake::expired`), which refuses the session through the request, so hold the `Request` mutably while it waits: `request.within(admit).await`. + ### A listener builds its peers' identities `tls::PeerIdentity::from_chain` is removed. A listener outside this crate that ran its own mTLS handshake, such as the `moq-uring` workers, calls `tls::Serve::peer(chain)` on the `Serve` its TLS config came from. diff --git a/rs/moq-tokio/src/listen.rs b/rs/moq-tokio/src/listen.rs index 6ba5ba4f24d..3073dfa74db 100644 --- a/rs/moq-tokio/src/listen.rs +++ b/rs/moq-tokio/src/listen.rs @@ -75,7 +75,7 @@ impl<'de> serde::Deserialize<'de> for Bind { } /// How long an accepted connection has to finish its handshake, unless overridden by `--listen-timeout`. -pub(crate) const DEFAULT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +pub(crate) const DEFAULT_TIMEOUT: std::time::Duration = moq_net::server::DEFAULT_TIMEOUT; /// The accept side of an endpoint: what to listen on and how to be trusted. /// diff --git a/rs/moq-tokio/src/quic/backend.rs b/rs/moq-tokio/src/quic/backend.rs index f106d174efe..ceff9a0689a 100644 --- a/rs/moq-tokio/src/quic/backend.rs +++ b/rs/moq-tokio/src/quic/backend.rs @@ -621,9 +621,6 @@ fn proto_status(err: &web::proto::ConnectError) -> Option { pub(crate) struct Server { pub quic: driver::endpoint::Endpoint, - /// The configured receive window, opened once a session is admitted: until then a - /// connection holds [`moq_net::transport::SETUP_RECEIVE_WINDOW`]. - pub window: Option, /// Builds each accepted client's identity against the listener's current roots. pub(crate) tls: crate::tls::Serve, /// Reloads the QUIC-LB peers for as long as the server runs. @@ -649,7 +646,8 @@ impl Server { } = options; let quic = quic.resolve(); let mut transport = transport(&quic, "server")?; - let window = transport.limits.receive_window; + // Held back until admission, when moq-net opens the configured window + // (`moq_net::Server::with_receive_window`). transport.limits.receive_window = std::num::NonZeroU64::new(moq_net::transport::SETUP_RECEIVE_WINDOW); let keylog = quic .keylog @@ -765,7 +763,6 @@ impl Server { Ok(Self { quic, - window, tls: served, _peers: peers, }) diff --git a/rs/moq-tokio/src/quic/web/session.rs b/rs/moq-tokio/src/quic/web/session.rs index b81799c783c..180ee992420 100644 --- a/rs/moq-tokio/src/quic/web/session.rs +++ b/rs/moq-tokio/src/quic/web/session.rs @@ -1046,6 +1046,14 @@ impl moq_net::transport::poll::Session for Session { fn receive_window(&self) -> Option { Some(self.conn.receive_window()) } + /// The connection's limits, which every WebTransport session on it shares. + fn set_limits(&self, limits: moq_net::transport::Limits) -> Result<(), moq_net::transport::LimitsError> { + let mut quic = moq_quic::Limits::default(); + quic.receive_window = limits.receive_window; + self.conn + .set_limits(quic) + .map_err(|err| moq_net::transport::LimitsError::Refused(err.to_string())) + } /// The connection's credit, which every WebTransport session on it shares. fn stream_credit(&self, dir: moq_net::transport::Dir) -> Option { Some(self.conn.stream_credit(dir)) diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index d15eb28f5c3..9b2d001678f 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -223,8 +223,6 @@ pub struct Server { activity: moq_net::server::Activity, moq: moq_net::Server, versions: moq_net::Versions, - /// From [`crate::listen::Config::resolved_timeout`]. - timeout: Option, #[cfg(any(feature = "quic", feature = "websocket", feature = "tcp", all(feature = "uds", unix)))] /// Pending handshakes, each isolated: one panicking (`None`) must not end the listener. /// Each is boxed before `isolate` wraps it, so a debug build never holds a @@ -286,6 +284,9 @@ impl Server { // is (the default). A stream-only server (`--listen-unix-bind` with no // `--listen`) doesn't also open UDP/443. quic.validate()?; + // The QUIC endpoint holds each connection's receive window back until admission, and + // a qmux session starts at it already. + let window = std::num::NonZeroU64::new(quic.resolve().receive_window); // Every qmux session this server accepts starts at the same `[quic]` windows. #[cfg(any(feature = "websocket", feature = "tcp", all(feature = "uds", unix)))] let resolved = quic.resolve(); @@ -399,7 +400,6 @@ impl Server { let streams = StreamListeners::new( stream_binds, stream_versions(&versions), - timeout, resolved.clone(), #[cfg(all(feature = "uds", unix))] unix_allow, @@ -409,12 +409,16 @@ impl Server { let tls_stream = tls.map(|tls| TcpTls::new(&config, &versions, tls)).transpose()?; let mut moq = moq_net::Server::new() + .with_timeout(timeout) .with_versions(versions.clone()) .with_extensions(config.extensions) .with_stats(stats); if let Some(implementation) = &config.implementation { moq = moq.with_implementation(implementation); } + if let Some(window) = window { + moq = moq.with_receive_window(window); + } if let Some(publisher) = publisher { moq = moq.with_publisher(publisher); } @@ -428,7 +432,6 @@ impl Server { accept: Default::default(), moq, versions, - timeout, #[cfg(any(feature = "tcp", all(feature = "uds", unix)))] streams, #[cfg(all(feature = "tcp", feature = "_certs"))] @@ -513,7 +516,6 @@ impl Server { tls.serve.clone(), session, server, - self.timeout, )) } @@ -638,9 +640,6 @@ impl Server { // The QUIC endpoint address, reported as a QUIC session's local side. #[cfg(feature = "quic")] let local = self.local_addr().ok(); - // The window a QUIC session opens to once admitted. - #[cfg(feature = "quic")] - let window = self.quic.as_ref().and_then(|quic| quic.window); #[cfg(feature = "quic")] let quic_tls = self.quic.as_ref().map(|quic| quic.tls.clone()); @@ -674,7 +673,6 @@ impl Server { let setup = Setup { activity: self.activity.clone(), server: self.moq.clone(), - timeout: self.timeout, }; #[allow(unused_variables, reason = "a build without a transport feature never reads it")] let versions = self.versions.clone(); @@ -730,10 +728,10 @@ impl Server { let Accepted { session, url, identity, authority, mut link } = within(deadline, crate::quic::accept(_conn, alpns, tls)).await?; link.local = local; let quic = Some(session.connection().clone()); - let (request, deadline) = accept_request(&setup.server, session, deadline).await?; + let request = setup.server.accept_request(moq_net::time::Instant::now(), session).with_deadline(deadline).await?; // Only WebTransport carries a request URL; raw QUIC puts the path in the SETUP. let transport = match url { Some(_) => Transport::WebTransport, None => Transport::Quic }; - Ok(Request { transport, url, identity, authority, link, handshake: Box::new(request), quic, window, quic_limits: Default::default(), #[cfg(any(feature = "websocket", feature = "tcp", feature = "webrtc"))] qmux: None, deadline, active }) + Ok(Request { transport, url, identity, authority, link, handshake: Box::new(request), quic, quic_limits: Default::default(), #[cfg(any(feature = "websocket", feature = "tcp", feature = "webrtc"))] qmux: None, active }) }.boxed()).boxed()); } } @@ -868,9 +866,7 @@ impl Listener { if !server.versions.alpns().contains(&peer.protocol()) { return Err(WebRtcError::Protocol(peer.protocol().to_string()).into()); } - let deadline = server - .timeout - .and_then(|timeout| kio::clock::now().checked_add(timeout)); + let deadline = server.moq.deadline(kio::clock::now()); let handshake = async { let opened = peer.channel().await?; @@ -885,7 +881,11 @@ impl Listener { let (session, protocol) = within(deadline, handshake).await?; let qmux = session.clone(); - let (request, deadline) = accept_request(&server.moq, session, deadline).await?; + let request = server + .moq + .accept_request(moq_net::time::Instant::now(), session) + .with_deadline(deadline) + .await?; Ok(Request { transport: Transport::WebRtc, url: None, @@ -898,13 +898,10 @@ impl Listener { handshake: Box::new(request), #[cfg(feature = "quic")] quic: None, - #[cfg(feature = "quic")] - window: None, #[cfg(feature = "_transport")] quic_limits: Default::default(), #[cfg(any(feature = "websocket", feature = "tcp", feature = "webrtc"))] qmux: Some(qmux), - deadline, active, }) }) @@ -969,59 +966,20 @@ async fn serve_session(request: Request) -> crate::Result<()> { Err(session.closed().await.into()) } -/// The MoQ server an accepted connection handshakes against, and how long it has. +/// The MoQ server an accepted connection handshakes against. #[cfg(any(feature = "quic", feature = "websocket", feature = "tcp", all(feature = "uds", unix)))] #[derive(Clone)] struct Setup { activity: moq_net::server::Activity, server: moq_net::Server, - timeout: Option, } #[cfg(any(feature = "quic", feature = "websocket", feature = "tcp", all(feature = "uds", unix)))] impl Setup { - /// When a connection accepted now must finish its handshake, or `None` to wait - /// forever, including past the clock's range. + /// When a connection accepted now must finish accepting; see [`moq_net::Server::deadline`]. fn deadline(&self) -> Option { - kio::clock::now().checked_add(self.timeout?) - } -} - -#[cfg(any( - feature = "quic", - feature = "websocket", - feature = "tcp", - all(feature = "uds", unix), - feature = "webrtc" -))] -/// Read the peer's MoQ SETUP by `deadline`, closing the session with a timeout -/// code if it hasn't arrived. The deadline comes back too, since answering the -/// SETUP is part of the handshake it bounds. -async fn accept_request( - server: &moq_net::Server, - session: S, - deadline: Option, -) -> crate::Result<(moq_net::server::Handshake, Option)> -where - S: moq_net::transport::poll::Boxable + Send + Sync, -{ - // Shared, so the deadline can close the session the server runs. - let session = std::sync::Arc::new(session); - let deadline = deadline.map(|at| Deadline::new(at, session.clone())); - let accept = async { Ok(server.accept_request(moq_net::time::Instant::now(), session).await?) }; - let request = Deadline::bound(deadline.as_ref(), accept).await?; - Ok((request, deadline)) -} - -/// The limits [`Request::ok`] gives an accepted qmux session on `transport`. A WebRTC data -/// channel keeps its own receive window, which sizes its read queue, so a wider one would let -/// a peer honoring flow control overrun the queue and close the channel. -#[cfg(any(feature = "websocket", feature = "tcp", feature = "webrtc"))] -fn qmux_limits(transport: Transport, mut limits: moq_quic::Limits) -> moq_quic::Limits { - if transport == Transport::WebRtc { - limits.receive_window = None; + self.server.deadline(kio::clock::now()) } - limits } #[cfg(feature = "websocket")] @@ -1031,7 +989,11 @@ pub(crate) async fn accept_websocket( deadline: Option, ) -> crate::Result { let qmux = session.clone(); - let (request, deadline) = accept_request(&options.server, session, deadline).await?; + let request = options + .server + .accept_request(moq_net::time::Instant::now(), session) + .with_deadline(deadline) + .await?; let authority = options.url.host_str().filter(|h| !h.is_empty()).map(str::to_owned); Ok(Request { transport: Transport::WebSocket, @@ -1042,67 +1004,14 @@ pub(crate) async fn accept_websocket( handshake: Box::new(request), #[cfg(feature = "quic")] quic: None, - #[cfg(feature = "quic")] - window: None, #[cfg(feature = "_transport")] quic_limits: Default::default(), #[cfg(any(feature = "websocket", feature = "tcp", feature = "webrtc"))] qmux: Some(qmux), - deadline, active: options.active, }) } -/// When an accepted connection's handshake must be done, and how to tell the peer -/// it wasn't. -struct Deadline { - at: kio::clock::Instant, - /// Closes the session with the timeout code. - close: Box, -} - -impl Deadline { - #[cfg(any( - feature = "quic", - feature = "websocket", - feature = "tcp", - all(feature = "uds", unix), - feature = "webrtc" - ))] - fn new(at: kio::clock::Instant, session: S) -> Self - where - S: moq_net::transport::poll::Session + Send + Sync, - { - let close = move || { - let err = moq_net::Error::Timeout; - moq_net::transport::poll::Session::close( - &session, - moq_net::SessionError::from(&err).to_code(), - &err.to_string(), - ); - }; - Self { - at, - close: Box::new(close), - } - } - - /// Finish `handshake` before the deadline, or close the session with the - /// timeout code. Without a deadline, wait forever. - async fn bound(deadline: Option<&Self>, handshake: impl Future>) -> crate::Result { - let Some(deadline) = deadline else { - return handshake.await; - }; - match kio::clock::timeout_at(deadline.at, handshake).await { - Ok(res) => res, - Err(_) => { - (deadline.close)(); - Err(moq_net::Error::Timeout.into()) - } - } - } -} - /// Finish a transport handshake by `deadline`, or fail with a timeout. /// /// Nothing is established yet that could carry a close code, so an expired @@ -1233,7 +1142,6 @@ struct StreamListeners { /// at startup, long before the first `accept` binds anything. health: Vec, versions: moq_net::Versions, - timeout: Option, /// The `[quic]` section each accepted session starts with. quic: crate::quic::Resolved, #[cfg(all(feature = "uds", unix))] @@ -1253,7 +1161,6 @@ impl StreamListeners { fn new( binds: Vec, versions: moq_net::Versions, - timeout: Option, quic: crate::quic::Resolved, #[cfg(all(feature = "uds", unix))] unix_allow: Option, ) -> Self { @@ -1267,7 +1174,6 @@ impl StreamListeners { binds, health, versions, - timeout, quic, #[cfg(all(feature = "uds", unix))] unix_allow, @@ -1345,7 +1251,6 @@ impl StreamListeners { let setup = Setup { activity: activity.clone(), server: server.clone(), - timeout: self.timeout, }; let (tx, rx) = tokio::sync::mpsc::channel(16); for listener in self.pending.drain(..) { @@ -1577,7 +1482,10 @@ pub(crate) async fn accept_stream( active, } = incoming; let qmux = session.clone(); - let (request, deadline) = accept_request(server, session, deadline).await?; + let request = server + .accept_request(moq_net::time::Instant::now(), session) + .with_deadline(deadline) + .await?; Ok(Request { transport, url: None, @@ -1587,13 +1495,10 @@ pub(crate) async fn accept_stream( handshake: Box::new(request), #[cfg(feature = "quic")] quic: None, - #[cfg(feature = "quic")] - window: None, #[cfg(feature = "_transport")] quic_limits: Default::default(), #[cfg(any(feature = "websocket", feature = "tcp", feature = "webrtc"))] qmux: Some(qmux), - deadline, active, }) } @@ -1654,9 +1559,6 @@ pub struct Request { /// The connection under a QUIC or WebTransport request, for [`Request::quic`]. #[cfg(feature = "quic")] quic: Option, - /// The receive window [`Request::ok`] opens unless [`Request::with_quic_limits`] names one. - #[cfg(feature = "quic")] - window: Option, /// What [`Request::ok`] applies to the QUIC connection or qmux session before answering SETUP. #[cfg(feature = "_transport")] quic_limits: moq_quic::Limits, @@ -1664,8 +1566,6 @@ pub struct Request { /// [`Request::ok`] widens to [`Request::with_quic_limits`]. #[cfg(any(feature = "websocket", feature = "tcp", feature = "webrtc"))] qmux: Option, - /// What remains of the handshake deadline, which also bounds [`Request::ok`]. - deadline: Option, } impl Request { @@ -1718,11 +1618,12 @@ impl Request { /// Set the QUIC limits [`Self::ok`] applies before answering SETUP, replacing any set /// before. /// - /// Until then a QUIC connection holds only [`moq_net::transport::SETUP_RECEIVE_WINDOW`]; - /// `ok` opens `receive_window`, or `[quic] receive_window` when it is unset, after the - /// other limits, so a `receive_rate` set here paces the opened window. A qmux session - /// starts at the `[quic]` defaults and takes the windows and stream count set here; it - /// never paces, so `ok` refuses a rate cap on one. + /// Until then a QUIC connection holds only [`moq_net::transport::SETUP_RECEIVE_WINDOW`]. + /// `ok` applies the other limits first, then has moq-net open `receive_window`, or + /// `[quic] receive_window` when it is unset, so a `receive_rate` set here paces the opened + /// window. A qmux session starts at the `[quic]` windows and takes the windows and stream + /// count set here, except a WebRTC session's receive window, which its data channel fixes; + /// it never paces, so `ok` refuses a rate cap on one. #[cfg(feature = "_transport")] pub fn with_quic_limits(mut self, limits: moq_quic::Limits) -> Result { limits.validate()?; @@ -1730,15 +1631,23 @@ impl Request { Ok(self) } - /// Finish `work` before what remains of the handshake deadline, or close the - /// session with the timeout code and return [`moq_net::Error::Timeout`]. + /// Finish `work` before the accept deadline, or refuse the session with the timeout code + /// and return [`moq_net::Error::Timeout`]; see [`moq_net::server::Handshake::expired`]. /// /// Admitting the session is the caller's part of the handshake, so the same /// `listen.timeout` that bounds the transport and SETUP bounds it too; a peer /// cannot hold a connection open behind a decision that never comes. Without a /// deadline, waits forever. - pub async fn within(&self, work: impl Future) -> crate::Result { - crate::runtime::scope(Deadline::bound(self.deadline.as_ref(), async { Ok(work.await) })).await + pub async fn within(&mut self, work: impl Future) -> crate::Result { + let handshake = &mut self.handshake; + crate::runtime::scope(async move { + tokio::select! { + biased; + done = work => Ok(done), + () = handshake.expired() => Err(moq_net::Error::Timeout.into()), + } + }) + .await } /// The receive window of the qmux session under this request. @@ -1752,35 +1661,33 @@ impl Request { /// /// A driver panic is reported through [`Session::closed`]. /// - /// Bounded by what remains of the handshake deadline: answering SETUP can wait - /// on a peer that withholds flow-control credit, which keep-alives would - /// otherwise hold open. + /// Bounded by the accept deadline: answering SETUP can wait on a peer that withholds + /// flow-control credit, which keep-alives would otherwise hold open. pub async fn ok(self) -> crate::Result { - // Admitted: open the window the handshake held back, before SETUP OK. + // The caller's limits go first; moq-net opens the receive window after them. + #[cfg(feature = "_transport")] + let (limits, window) = { + let mut limits = self.quic_limits; + let window = limits.receive_window.take(); + (limits, window) + }; #[cfg(feature = "quic")] if let Some(quic) = &self.quic { - let mut limits = self.quic_limits; - limits.receive_window = limits.receive_window.or(self.window); - quic.set_limits(limits) - .expect("validated by with_quic_limits and the config"); + quic.set_limits(limits).expect("validated by with_quic_limits"); } #[cfg(any(feature = "websocket", feature = "tcp", feature = "webrtc"))] if let Some(qmux) = &self.qmux - && let Err(err) = qmux.set_limits(qmux_limits(self.transport, self.quic_limits)) + && let Err(err) = qmux.set_limits(limits) { self.handshake.close(moq_net::SessionError::Internal.into()); return Err(crate::Error::QmuxLimits(crate::error::message(err))); } - let Request { - handshake, - deadline, - active, - .. - } = self; - // moq-net enforces it, so a reply the peer never took counts in the session's stats. + let Request { handshake, active, .. } = self; + #[allow(unused_mut, reason = "only a transport feature sets the window")] let mut handshake = *handshake; - if let Some(deadline) = deadline { - handshake = handshake.with_deadline(deadline.at); + #[cfg(feature = "_transport")] + if let Some(window) = window { + handshake = handshake.with_receive_window(window); } let (session, driver) = crate::runtime::scope(handshake.ok()).await?; use tracing::Instrument; @@ -1940,7 +1847,6 @@ mod tests { let mut streams = StreamListeners::new( Vec::new(), Default::default(), - None, Default::default(), #[cfg(all(feature = "uds", unix))] None, @@ -2052,8 +1958,7 @@ mod tests { listener, Setup { activity: activity.clone(), - server: moq_net::Server::new(), - timeout: None, + server: moq_net::Server::new().with_timeout(None), }, tx, ); @@ -2093,8 +1998,7 @@ mod tests { listener, Setup { activity, - server: moq_net::Server::new(), - timeout: None, + server: moq_net::Server::new().with_timeout(None), }, tx, ); @@ -2129,8 +2033,7 @@ mod tests { listener, Setup { activity: Default::default(), - server: moq_net::Server::new(), - timeout: None, + server: moq_net::Server::new().with_timeout(None), }, Some(crate::unix::Allow { uid: vec![if allowed { @@ -2186,8 +2089,7 @@ mod tests { listener, Setup { activity: activity.clone(), - server: moq_net::Server::new(), - timeout: None, + server: moq_net::Server::new().with_timeout(None), }, None, tx, @@ -2230,8 +2132,7 @@ mod tests { listener, Setup { activity: Default::default(), - server: moq_net::Server::new(), - timeout: None, + server: moq_net::Server::new().with_timeout(None), }, tx, ); @@ -2259,8 +2160,7 @@ mod tests { listener, Setup { activity: Default::default(), - server: moq_net::Server::new(), - timeout: None, + server: moq_net::Server::new().with_timeout(None), }, None, tx, @@ -2548,8 +2448,7 @@ mod tests { async fn unreachable_timeout_has_no_deadline() { let setup = Setup { activity: Default::default(), - server: moq_net::Server::new(), - timeout: Some(std::time::Duration::MAX), + server: moq_net::Server::new().with_timeout(Some(std::time::Duration::MAX)), }; assert_eq!(setup.deadline(), None); } @@ -2562,14 +2461,14 @@ mod tests { let timeout = std::time::Duration::from_secs(10); let setup = Setup { activity: Default::default(), - server: moq_net::Server::new(), - timeout: Some(timeout), + server: moq_net::Server::new().with_timeout(Some(timeout)), }; let start = kio::clock::now(); - let Err(err) = accept_request(&setup.server, peer.clone(), setup.deadline()).await else { + let accept = setup.server.accept_request(moq_net::time::Instant::now(), peer.clone()); + let Err(err) = accept.with_deadline(setup.deadline()).await else { panic!("a peer that never sent SETUP was accepted"); }; - assert!(matches!(err, Error::MoqNet(moq_net::Error::Timeout)), "{err}"); + assert!(matches!(err, moq_net::Error::Timeout), "{err}"); assert_eq!(start.elapsed(), timeout); assert_eq!( *peer.closed.lock().unwrap(), @@ -2606,11 +2505,11 @@ mod tests { let timeout = std::time::Duration::from_secs(10); let setup = Setup { activity: Default::default(), - server: moq_net::Server::new(), - timeout: Some(timeout), + server: moq_net::Server::new().with_timeout(Some(timeout)), }; let start = kio::clock::now(); - let Ok((request, deadline)) = accept_request(&setup.server, peer.clone(), setup.deadline()).await else { + let accept = setup.server.accept_request(moq_net::time::Instant::now(), peer.clone()); + let Ok(request) = accept.with_deadline(setup.deadline()).await else { panic!("a readable SETUP was refused"); }; assert_eq!(start.elapsed(), std::time::Duration::ZERO); @@ -2626,13 +2525,10 @@ mod tests { handshake: Box::new(request), #[cfg(feature = "quic")] quic: None, - #[cfg(feature = "quic")] - window: None, #[cfg(feature = "_transport")] quic_limits: Default::default(), #[cfg(any(feature = "websocket", feature = "tcp", feature = "webrtc"))] qmux: None, - deadline, } .with_stats( registry @@ -2663,8 +2559,7 @@ mod tests { let timeout = std::time::Duration::from_secs(10); let setup = Setup { activity: Default::default(), - server: moq_net::Server::new(), - timeout: Some(timeout), + server: moq_net::Server::new().with_timeout(Some(timeout)), }; let deadline = setup.deadline(); @@ -2678,7 +2573,8 @@ mod tests { session }); - let Ok((request, _)) = accept_request(&setup.server, server, deadline).await else { + let accept = setup.server.accept_request(moq_net::time::Instant::now(), server); + let Ok(request) = accept.with_deadline(deadline).await else { panic!("a prompt SETUP was refused"); }; let (_session, driver) = request.ok().await.expect("server handshake"); @@ -3008,6 +2904,52 @@ mod tests { drop((session, connection)); } + /// A QUIC connection holds back its receive window until the request is admitted, and + /// `ok` opens the `[quic]` window, or the one the request names, before SETUP OK. + #[cfg(feature = "quic")] + #[tokio::test] + async fn admission_opens_the_held_back_quic_window() { + let wider = std::num::NonZeroU64::new(moq_net::transport::MIN_RECEIVE_WINDOW * 4).unwrap(); + for named in [None, Some(wider)] { + let config = crate::listen::Config { + bind: Some("[::]:0".parse().unwrap()), + tls: crate::tls::Listen { + generate: vec!["localhost".into()], + ..Default::default() + }, + ..Default::default() + }; + let mut listener = config + .init(small_quic()) + .expect("server") + .listen() + .await + .expect("listen"); + let addr = listener.local_addr().expect("local addr"); + + let mut client = crate::connect::Config::default(); + client.tls.insecure = Some(true); + let client = client.init(Default::default()).expect("client"); + let url: url::Url = format!("moqt://localhost:{}", addr.port()).parse().unwrap(); + let _connection = client.with_reconnect(false).connect(url); + + let mut request = listener.accept().await.expect("no incoming connection"); + let quic = request.quic().expect("a QUIC request").clone(); + assert_eq!( + quic.receive_window().connection, + moq_net::transport::SETUP_RECEIVE_WINDOW + ); + if let Some(window) = named { + let mut limits = moq_quic::Limits::default(); + limits.receive_window = Some(window); + request = request.with_quic_limits(limits).expect("valid limits"); + } + let _session = request.ok().await.expect("server handshake"); + let expected = named.map_or(moq_net::transport::MIN_RECEIVE_WINDOW * 2, |window| window.get()); + assert_eq!(quic.receive_window().connection, expected, "{named:?}"); + } + } + /// A QUIC endpoint that stops under a live listener fails `accept` and the /// `serve_*` loop built on it, instead of reading as a clean stop. #[cfg(feature = "quic")] diff --git a/rs/moq-tokio/src/tcp.rs b/rs/moq-tokio/src/tcp.rs index 59edd751c81..fa1b9e981c6 100644 --- a/rs/moq-tokio/src/tcp.rs +++ b/rs/moq-tokio/src/tcp.rs @@ -251,7 +251,6 @@ pub struct TlsAcceptor { tls: crate::tls::Serve, qmux: qmux::tls::Server, server: moq_net::Server, - timeout: Option, } #[cfg(feature = "_certs")] @@ -261,7 +260,6 @@ impl TlsAcceptor { tls: crate::tls::Serve, session: qmux::Config, server: moq_net::Server, - timeout: Option, ) -> Self { let qmux = qmux::tls::Server::new(config.clone()).with_session_config(session); Self { @@ -269,7 +267,6 @@ impl TlsAcceptor { tls, qmux, server, - timeout, } } @@ -301,7 +298,7 @@ impl TlsAcceptor { T: tokio::io::AsyncRead + tokio::io::AsyncWrite + AsRef + Send + Unpin + 'static, { crate::runtime::scope(async move { - let deadline = self.timeout.and_then(|timeout| kio::clock::now().checked_add(timeout)); + let deadline = self.server.deadline(kio::clock::now()); let tcp: &tokio::net::TcpStream = stream.get_ref().0.as_ref(); let remote = tcp.peer_addr().ok(); let local = tcp.local_addr().ok(); diff --git a/rs/moq-tokio/src/webrtc.rs b/rs/moq-tokio/src/webrtc.rs index 22f38b315b4..528c4956ddf 100644 --- a/rs/moq-tokio/src/webrtc.rs +++ b/rs/moq-tokio/src/webrtc.rs @@ -509,6 +509,8 @@ fn session_config(protocol: &str, max_record_size: u64) -> qmux::Config { config.max_record_size = max_record_size; // Set rather than left to qmux's default, since the read queue is sized from it. config.limits.receive_window = std::num::NonZeroU64::new(RECEIVE_WINDOW); + // The channel's read queue is sized by it, so it never widens. + config.fixed_receive_window = true; // The binding has no datagrams. config.max_datagram_frame_size = None; config diff --git a/rs/moq-uring/src/quic/web.rs b/rs/moq-uring/src/quic/web.rs index fcde2f336b6..0b4494d0988 100644 --- a/rs/moq-uring/src/quic/web.rs +++ b/rs/moq-uring/src/quic/web.rs @@ -830,6 +830,12 @@ impl moq_net::transport::poll::Session for Session { fn receive_window(&self) -> Option { Some(Self::receive_window(self)) } + /// The connection's limits, which every WebTransport session on it shares. + fn set_limits(&self, limits: moq_net::transport::Limits) -> Result<(), moq_net::transport::LimitsError> { + let mut quic = super::Limits::default(); + quic.receive_window = limits.receive_window; + Self::set_limits(self, quic).map_err(|err| moq_net::transport::LimitsError::Refused(err.to_string())) + } /// The connection's credit, which every WebTransport session on it shares. fn stream_credit(&self, dir: moq_net::transport::Dir) -> Option { Some(self.conn.stream_credit(dir)) diff --git a/rs/moq-uring/src/transport/local.rs b/rs/moq-uring/src/transport/local.rs index bdae2cb556a..7e774530156 100644 --- a/rs/moq-uring/src/transport/local.rs +++ b/rs/moq-uring/src/transport/local.rs @@ -114,6 +114,10 @@ impl poll::Session for Local { self.0.receive_window() } + fn set_limits(&self, limits: moq::Limits) -> Result<(), moq::LimitsError> { + self.0.set_limits(limits) + } + fn stream_credit(&self, dir: moq::Dir) -> Option { self.0.stream_credit(dir) } diff --git a/rs/qmux/src/config.rs b/rs/qmux/src/config.rs index 6e2deac6068..43c3ebf5e4f 100644 --- a/rs/qmux/src/config.rs +++ b/rs/qmux/src/config.rs @@ -79,6 +79,11 @@ pub struct Config { /// an application is actively sending. Only the record-framed drafts (QMux01+) carry /// `QX_PING`. Default: 1s. pub rtt_interval: Option, + + /// Keep `limits.receive_window` for the session's life, for a carrier that sizes its own + /// buffers by it, such as a WebRTC data channel: [`Session::set_limits`](crate::Session::set_limits) + /// then refuses to change it. Default: false. + pub fixed_receive_window: bool, } impl Default for Config { @@ -93,6 +98,7 @@ impl Default for Config { max_datagram_frame_size: Some(DEFAULT_MAX_RECORD_SIZE), handshake_timeout: Some(Duration::from_secs(10)), rtt_interval: Some(Duration::from_secs(1)), + fixed_receive_window: false, } } } diff --git a/rs/qmux/src/session.rs b/rs/qmux/src/session.rs index 1681117db8a..e0d504ba35a 100644 --- a/rs/qmux/src/session.rs +++ b/rs/qmux/src/session.rs @@ -25,6 +25,8 @@ pub(crate) struct Inner { pub(crate) wire: net::Wire, socket_stats: Option, address: Address, + /// From [`Config::fixed_receive_window`]. + fixed_receive_window: bool, } pub(crate) struct State { @@ -178,6 +180,7 @@ impl Session { wire: net::Wire::new(), socket_stats: transport.socket_stats(), address: transport.address(), + fixed_receive_window: config.fixed_receive_window, }); let session = Self { _guard: Arc::new(Guard(inner.clone())), @@ -223,9 +226,13 @@ impl Session { /// /// Refuses, changing nothing, a rate cap (QMux never paces, so only /// [`Rate::Unlimited`](moq_quic::Rate::Unlimited) is accepted), a receive window below - /// moq-net's [`MIN_RECEIVE_WINDOW`](net::MIN_RECEIVE_WINDOW), and limits a peer would - /// refuse. A window only grows at once: credit already granted cannot be taken back. + /// moq-net's [`MIN_RECEIVE_WINDOW`](net::MIN_RECEIVE_WINDOW) or on a session whose window + /// is [fixed](Config::fixed_receive_window), and limits a peer would refuse. A window only + /// grows at once: credit already granted cannot be taken back. pub fn set_limits(&self, limits: moq_quic::Limits) -> Result<(), Error> { + if self.inner.fixed_receive_window && limits.receive_window.is_some() { + return Err(Error::InvalidConfig("this session's receive window is fixed")); + } crate::config::check_limits(&limits)?; let capped = |rate: Option| matches!(rate, Some(moq_quic::Rate::BytesPerSecond(_))); if capped(limits.receive_rate) || capped(limits.send_rate) { @@ -393,6 +400,15 @@ impl poll::Session for Session { )) } + fn set_limits(&self, limits: net::Limits) -> Result<(), net::LimitsError> { + if self.inner.fixed_receive_window { + return Err(net::LimitsError::Unsupported); + } + let mut quic = moq_quic::Limits::default(); + quic.receive_window = limits.receive_window; + Session::set_limits(self, quic).map_err(|err| net::LimitsError::Refused(err.to_string())) + } + fn stream_credit(&self, dir: net::Dir) -> Option { let dir = match dir { net::Dir::Uni => Dir::Uni, diff --git a/rs/qmux/src/session/tests.rs b/rs/qmux/src/session/tests.rs index 9e2100fe0bb..14d7859a211 100644 --- a/rs/qmux/src/session/tests.rs +++ b/rs/qmux/src/session/tests.rs @@ -264,6 +264,31 @@ async fn set_limits_widens_and_refuses_a_rate_cap() { )); } +/// moq-net opens a session's receive window through the transport trait, unless the session's +/// window is fixed, which it reports as a limit it cannot change. +#[tokio::test(start_paused = true)] +async fn the_transport_widens_unless_its_window_is_fixed() { + let mut limits = net::Limits::default(); + limits.receive_window = std::num::NonZeroU64::new(16 << 20); + + let (_client, server) = pair(config(Version::QMux02), config(Version::QMux02)).await; + poll::Session::set_limits(&server, limits).expect("widened"); + assert_eq!(poll::Session::receive_window(&server).unwrap().connection, 16 << 20); + + let mut fixed = config(Version::QMux02); + fixed.fixed_receive_window = true; + let (_client, server) = pair(config(Version::QMux02), fixed).await; + let window = poll::Session::receive_window(&server).unwrap().connection; + assert_eq!( + poll::Session::set_limits(&server, limits), + Err(net::LimitsError::Unsupported) + ); + let mut direct = moq_quic::Limits::default(); + direct.receive_window = limits.receive_window; + assert!(matches!(server.set_limits(direct), Err(Error::InvalidConfig(_)))); + assert_eq!(poll::Session::receive_window(&server).unwrap().connection, window); +} + /// A reset seen through a lease frees the stream while the reader still holds it, returning /// its stream credit and the connection credit its partial message withheld. #[tokio::test(start_paused = true)] From e137be8c98bdf5e99c1f5b21d747659cded946f4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 01:07:10 -0800 Subject: [PATCH 3/8] fix(net): build the window limits with a literal Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/server.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/src/server.rs b/rs/moq-net/src/server.rs index 765bbc6fcbf..d5df9ab00ec 100644 --- a/rs/moq-net/src/server.rs +++ b/rs/moq-net/src/server.rs @@ -1371,8 +1371,9 @@ fn open_window(server: &Server, paused: &PausedHandshake) -> Result<(), Error> { let Some(window) = server.receive_window else { return Ok(()); }; - let mut limits = crate::transport::Limits::default(); - limits.receive_window = Some(window); + let limits = crate::transport::Limits { + receive_window: Some(window), + }; match paused.transport().0.set_limits(limits) { Ok(()) | Err(crate::transport::LimitsError::Unsupported) => Ok(()), Err(err) => Err(Error::Transport(err.to_string())), From 83bea6afa9549adc8f1b1016aa09f112c9323a50 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 01:09:18 -0800 Subject: [PATCH 4/8] test(net): name the fake transport's opened window Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/server.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/server.rs b/rs/moq-net/src/server.rs index d5df9ab00ec..b5c61895209 100644 --- a/rs/moq-net/src/server.rs +++ b/rs/moq-net/src/server.rs @@ -1513,9 +1513,12 @@ mod tests { withheld: bool, /// The receive window `set_limits` opened, or why it refused, or `None` when the /// transport cannot change it. - window: Option, String>>>>, + window: Option, } + /// The window a transport opened, or why it refused. + type Opened = Arc, String>>>; + impl FakeSession { fn new(protocol: &'static str, uni: impl IntoIterator>) -> Self { Self { From 232805cb8b5700e697af74e5bd9f9ad17166b462 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 07:56:10 -0800 Subject: [PATCH 5/8] fix(qmux): a fixed window takes limits that leave it alone Co-Authored-By: Claude Opus 5.5 --- rs/qmux/src/session.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/qmux/src/session.rs b/rs/qmux/src/session.rs index e0d504ba35a..2bb15a2f0f1 100644 --- a/rs/qmux/src/session.rs +++ b/rs/qmux/src/session.rs @@ -401,7 +401,7 @@ impl poll::Session for Session { } fn set_limits(&self, limits: net::Limits) -> Result<(), net::LimitsError> { - if self.inner.fixed_receive_window { + if self.inner.fixed_receive_window && limits.receive_window.is_some() { return Err(net::LimitsError::Unsupported); } let mut quic = moq_quic::Limits::default(); From 4f00dd6c9a20ffc299da10cc88974e65edeecb7e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 07:56:10 -0800 Subject: [PATCH 6/8] docs(net): the accept timeout bounds only moq-lite 01/02's SETUP reply Co-Authored-By: Claude Opus 5.5 --- doc/lib/rs/moq-net.md | 2 +- rs/moq-net/MIGRATE.md | 2 +- rs/moq-net/src/server.rs | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index 81b5646939d..e3369cb18cf 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -26,7 +26,7 @@ What you use it for, beyond what the concept page already describes: - **End content by finishing tracks.** A broadcast has no finished state. When the input ends, finish each track; the broadcast stays announced and keeps serving them until you close it or drop its route. How long to keep it up for late readers is yours to decide: `moq_mux::Linger` holds a hang broadcast for a fixed window. - **Demand says whether anyone is watching.** `demand()` on a track, group, or broadcast says whether a subscriber is attached. First-party capture does not wait for it: video encodes while enabled and audio keeps raw PCM to encode when someone listens, so a late subscription's `delay.max` or a FETCH finds the recent past. Demand gates bandwidth reservations, transcode rungs, and on-demand video where a publisher opts in, which saves the encode but leaves nothing from before the first subscriber. Holding a broadcast or track consumer is not demand: a track consumer requests nothing until it queries, subscribes, or fetches, and those count until dropped. A shared fetch stays up until its last reader leaves. Demand rolls up: a frame being read keeps its group wanted, a group a subscription or fetch handed out keeps its track wanted until read to its end, and any wanted track keeps its broadcast wanted. - **Use the model without a runtime.** By default `moq-net` is its `poll_*` APIs and origin driver with caller-supplied time. The `async` feature adds their async wrappers, `time::run`, and both wire sessions. -- **You drive the session, or `moq-tokio` does.** `connect` and `accept` return a session plus a driver that never reads the clock itself. `moq_net::time::run` polls it on tokio. `moq-tokio` hides that. A custom transport implements `moq_net::transport::poll`, and `moq_net::transport::memory::pair` links two sessions in one process with no sockets, for tests or in-process links. `Server` bounds each accept with one deadline (10 s by default, `with_timeout`) across SETUP, your admission decision (`Handshake::expired`), and the SETUP reply, and opens a receive window the transport held back once you admit (`with_receive_window`). +- **You drive the session, or `moq-tokio` does.** `connect` and `accept` return a session plus a driver that never reads the clock itself. `moq_net::time::run` polls it on tokio. `moq-tokio` hides that. A custom transport implements `moq_net::transport::poll`, and `moq_net::transport::memory::pair` links two sessions in one process with no sockets, for tests or in-process links. `Server` bounds each accept with one deadline (10 s by default, `with_timeout`) across SETUP, your admission decision (`Handshake::expired`), and moq-lite 01/02's SETUP reply, and opens a receive window the transport held back once you admit (`with_receive_window`). - **You drive the session, or `moq-tokio` does.** `connect` and `accept` return a session plus a driver that never reads the clock itself. `moq_net::time::run` polls it on tokio. `moq-tokio` hides that. A custom transport implements `moq_net::transport::poll`, and `moq_net::transport::memory::pair` links two sessions in one process with no sockets, for tests or in-process links. - **Reconnect policy comes with it.** With the `async` feature, `moq_net::Reconnect` decides which of a peer's addresses to dial, how long each dial and handshake may take, when to back off and give up, and how to follow a GOAWAY, without doing I/O: you dial each `Attempt` it asks for and report back. `moq-tokio`'s `Connection` runs it, so a client on another runtime gets the same policy rather than a copy. - **A session bounds what its peer can hold.** Each request holds a stream, so the transport's stream credit bounds subscriptions and FETCHes. `session::Limits` (via `Client::with_limits` and `Server::with_limits`) bounds the announcements per session. The default suits a relay mesh, so lower it for untrusted peers. Past it, the session closes with `TOO_MANY_REQUESTS`. Peer-declared lengths are capped before they are buffered. diff --git a/rs/moq-net/MIGRATE.md b/rs/moq-net/MIGRATE.md index 66ed9a7e44e..fc1b8da7676 100644 --- a/rs/moq-net/MIGRATE.md +++ b/rs/moq-net/MIGRATE.md @@ -6,7 +6,7 @@ How to update your code for each breaking change, newest first. ### Accepting a session runs under one deadline and opens the receive window -`Server` now bounds each accept, from SETUP through the caller's admission and the SETUP reply, closing a connection that overruns it with `Error::Timeout`'s code. +`Server` now bounds each accept, from SETUP through the caller's admission and, on moq-lite 01/02, the SETUP reply, closing a connection that overruns it with `Error::Timeout`'s code. The default is `server::DEFAULT_TIMEOUT` (10 s); call `Server::with_timeout(None)` to keep waiting forever. `Server::accept_request` returns a `server::Accept`, which you `.await` as before, or poll; pass it to `kio::clock::timeout` and the like directly, since it is a `Future`. A caller that runs a transport handshake first computes `Server::deadline(arrived)`, bounds that handshake by it, and passes it on with `Accept::with_deadline`. diff --git a/rs/moq-net/src/server.rs b/rs/moq-net/src/server.rs index b5c61895209..d393ca5ab71 100644 --- a/rs/moq-net/src/server.rs +++ b/rs/moq-net/src/server.rs @@ -178,8 +178,9 @@ impl Server { } /// Bound each accept, from the connection's arrival through SETUP, the caller's admission, - /// and the SETUP reply. A connection that overruns it is closed with [`Error::Timeout`]'s - /// code. `None` waits forever. Defaults to [`DEFAULT_TIMEOUT`]. + /// and, on moq-lite 01/02, the server SETUP reply. Other versions send their SETUP from the + /// started session's driver, outside this bound. A connection that overruns it is closed + /// with [`Error::Timeout`]'s code. `None` waits forever. Defaults to [`DEFAULT_TIMEOUT`]. pub fn with_timeout(mut self, timeout: Option) -> Self { self.timeout = timeout; self @@ -1229,7 +1230,7 @@ impl Handshake { /// /// Opens the receive window first ([`with_receive_window`](Self::with_receive_window)). /// Fails with [`Error::Timeout`] once the accept deadline passes, whether admission or a - /// peer that never takes the SETUP reply ran it out. Dropping a pending `ok` refuses the + /// moq-lite 01/02 peer that never takes the SETUP reply ran it out. Dropping a pending `ok` refuses the /// session the same way, so a caller's own deadline counts too. /// /// Poll or spawn the returned driver to run the session. From 34c200273341c2c3353a2a5fbfae6404538bf4e2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 09:00:12 -0800 Subject: [PATCH 7/8] doc: scope the accept deadline to moq-lite 01/02 replies Applies Pullfrog's suggestions, naming Handshake::poll_expired. Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/index.md | 2 +- doc/lib/rs/moq-net.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/bin/relay/index.md b/doc/bin/relay/index.md index 23320cea300..fe2ce4f07e1 100644 --- a/doc/bin/relay/index.md +++ b/doc/bin/relay/index.md @@ -103,7 +103,7 @@ relay.with_web(web).run().await?; one in tests, enable the `test-support` feature, read its SETUP with `moq_net::Server::accept_request` and pass `Accepted::new(handshake, transport)` to `Connection::new`. The client must send its path in the SETUP. The server's accept timeout (`moq_net::Server::with_timeout`) - bounds admission and the SETUP reply too. + bounds admission and, on moq-lite 01/02, the SETUP reply too. - To decide admissions yourself, leave `[auth]` empty and answer `relay.admissions()`; see [Authentication](/bin/relay/auth#in-process). - To serve WebRTC media (WHIP, WHEP) on the QUIC port, take `relay.webrtc()`: diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index e3369cb18cf..df8e96698b5 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -26,7 +26,7 @@ What you use it for, beyond what the concept page already describes: - **End content by finishing tracks.** A broadcast has no finished state. When the input ends, finish each track; the broadcast stays announced and keeps serving them until you close it or drop its route. How long to keep it up for late readers is yours to decide: `moq_mux::Linger` holds a hang broadcast for a fixed window. - **Demand says whether anyone is watching.** `demand()` on a track, group, or broadcast says whether a subscriber is attached. First-party capture does not wait for it: video encodes while enabled and audio keeps raw PCM to encode when someone listens, so a late subscription's `delay.max` or a FETCH finds the recent past. Demand gates bandwidth reservations, transcode rungs, and on-demand video where a publisher opts in, which saves the encode but leaves nothing from before the first subscriber. Holding a broadcast or track consumer is not demand: a track consumer requests nothing until it queries, subscribes, or fetches, and those count until dropped. A shared fetch stays up until its last reader leaves. Demand rolls up: a frame being read keeps its group wanted, a group a subscription or fetch handed out keeps its track wanted until read to its end, and any wanted track keeps its broadcast wanted. - **Use the model without a runtime.** By default `moq-net` is its `poll_*` APIs and origin driver with caller-supplied time. The `async` feature adds their async wrappers, `time::run`, and both wire sessions. -- **You drive the session, or `moq-tokio` does.** `connect` and `accept` return a session plus a driver that never reads the clock itself. `moq_net::time::run` polls it on tokio. `moq-tokio` hides that. A custom transport implements `moq_net::transport::poll`, and `moq_net::transport::memory::pair` links two sessions in one process with no sockets, for tests or in-process links. `Server` bounds each accept with one deadline (10 s by default, `with_timeout`) across SETUP, your admission decision (`Handshake::expired`), and moq-lite 01/02's SETUP reply, and opens a receive window the transport held back once you admit (`with_receive_window`). +- **You drive the session, or `moq-tokio` does.** `connect` and `accept` return a session plus a driver that never reads the clock itself. `moq_net::time::run` polls it on tokio. `moq-tokio` hides that. A custom transport implements `moq_net::transport::poll`, and `moq_net::transport::memory::pair` links two sessions in one process with no sockets, for tests or in-process links. `Server` bounds each accept with one deadline (10 s by default, `with_timeout`) across SETUP, your admission decision (`Handshake::poll_expired`), and moq-lite 01/02's SETUP reply, and opens a receive window the transport held back once you admit (`with_receive_window`). `Server` bounds each accept with one deadline (10 s by default, `with_timeout`) across SETUP, your admission decision (`Handshake::expired`), and moq-lite 01/02's SETUP reply, and opens a receive window the transport held back once you admit (`with_receive_window`). - **You drive the session, or `moq-tokio` does.** `connect` and `accept` return a session plus a driver that never reads the clock itself. `moq_net::time::run` polls it on tokio. `moq-tokio` hides that. A custom transport implements `moq_net::transport::poll`, and `moq_net::transport::memory::pair` links two sessions in one process with no sockets, for tests or in-process links. - **Reconnect policy comes with it.** With the `async` feature, `moq_net::Reconnect` decides which of a peer's addresses to dial, how long each dial and handshake may take, when to back off and give up, and how to follow a GOAWAY, without doing I/O: you dial each `Attempt` it asks for and report back. `moq-tokio`'s `Connection` runs it, so a client on another runtime gets the same policy rather than a copy. - **A session bounds what its peer can hold.** Each request holds a stream, so the transport's stream credit bounds subscriptions and FETCHes. `session::Limits` (via `Client::with_limits` and `Server::with_limits`) bounds the announcements per session. The default suits a relay mesh, so lower it for untrusted peers. Past it, the session closes with `TOO_MANY_REQUESTS`. Peer-declared lengths are capped before they are buffered. From 966e49984d15c572235a64026dc27048437fea19 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 10:28:27 -0800 Subject: [PATCH 8/8] fix(net): address CodeRabbit findings on the accept policy - Pace streams buffered before admission at the window `ok()` opened, by reading a stream's pacing window when ingest first uses it. - Bound upgraded WebSocket accepts by the server's own timeout; drop `websocket::Options::timeout`, which duplicated `Server::with_timeout`. - Build the small-window QUIC admission test under `quic` alone. - Deduplicate the session-driving doc bullet. - Move the MIGRATE notes into the PR body, per #8024. Co-Authored-By: Claude Opus 5.5 --- doc/lib/rs/moq-net.md | 3 +- rs/moq-net/MIGRATE.md | 13 ------- rs/moq-net/src/lite/test_transport.rs | 18 +++++++-- rs/moq-net/src/transport/boxed.rs | 55 ++++++++++++++++++--------- rs/moq-relay/src/websocket.rs | 13 ++++--- rs/moq-tokio/MIGRATE.md | 4 -- rs/moq-tokio/src/server.rs | 2 +- rs/moq-tokio/src/websocket.rs | 23 +++++------ 8 files changed, 68 insertions(+), 63 deletions(-) diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index df8e96698b5..660960d0066 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -26,8 +26,7 @@ What you use it for, beyond what the concept page already describes: - **End content by finishing tracks.** A broadcast has no finished state. When the input ends, finish each track; the broadcast stays announced and keeps serving them until you close it or drop its route. How long to keep it up for late readers is yours to decide: `moq_mux::Linger` holds a hang broadcast for a fixed window. - **Demand says whether anyone is watching.** `demand()` on a track, group, or broadcast says whether a subscriber is attached. First-party capture does not wait for it: video encodes while enabled and audio keeps raw PCM to encode when someone listens, so a late subscription's `delay.max` or a FETCH finds the recent past. Demand gates bandwidth reservations, transcode rungs, and on-demand video where a publisher opts in, which saves the encode but leaves nothing from before the first subscriber. Holding a broadcast or track consumer is not demand: a track consumer requests nothing until it queries, subscribes, or fetches, and those count until dropped. A shared fetch stays up until its last reader leaves. Demand rolls up: a frame being read keeps its group wanted, a group a subscription or fetch handed out keeps its track wanted until read to its end, and any wanted track keeps its broadcast wanted. - **Use the model without a runtime.** By default `moq-net` is its `poll_*` APIs and origin driver with caller-supplied time. The `async` feature adds their async wrappers, `time::run`, and both wire sessions. -- **You drive the session, or `moq-tokio` does.** `connect` and `accept` return a session plus a driver that never reads the clock itself. `moq_net::time::run` polls it on tokio. `moq-tokio` hides that. A custom transport implements `moq_net::transport::poll`, and `moq_net::transport::memory::pair` links two sessions in one process with no sockets, for tests or in-process links. `Server` bounds each accept with one deadline (10 s by default, `with_timeout`) across SETUP, your admission decision (`Handshake::poll_expired`), and moq-lite 01/02's SETUP reply, and opens a receive window the transport held back once you admit (`with_receive_window`). `Server` bounds each accept with one deadline (10 s by default, `with_timeout`) across SETUP, your admission decision (`Handshake::expired`), and moq-lite 01/02's SETUP reply, and opens a receive window the transport held back once you admit (`with_receive_window`). -- **You drive the session, or `moq-tokio` does.** `connect` and `accept` return a session plus a driver that never reads the clock itself. `moq_net::time::run` polls it on tokio. `moq-tokio` hides that. A custom transport implements `moq_net::transport::poll`, and `moq_net::transport::memory::pair` links two sessions in one process with no sockets, for tests or in-process links. +- **You drive the session, or `moq-tokio` does.** `connect` and `accept` return a session plus a driver that never reads the clock itself. `moq_net::time::run` polls it on tokio. `moq-tokio` hides that. A custom transport implements `moq_net::transport::poll`, and `moq_net::transport::memory::pair` links two sessions in one process with no sockets, for tests or in-process links. `Server` bounds each accept with one deadline (10 s by default, `with_timeout`) across SETUP, your admission decision (`Handshake::expired`), and moq-lite 01/02's SETUP reply, and opens a receive window the transport held back once you admit (`with_receive_window`). - **Reconnect policy comes with it.** With the `async` feature, `moq_net::Reconnect` decides which of a peer's addresses to dial, how long each dial and handshake may take, when to back off and give up, and how to follow a GOAWAY, without doing I/O: you dial each `Attempt` it asks for and report back. `moq-tokio`'s `Connection` runs it, so a client on another runtime gets the same policy rather than a copy. - **A session bounds what its peer can hold.** Each request holds a stream, so the transport's stream credit bounds subscriptions and FETCHes. `session::Limits` (via `Client::with_limits` and `Server::with_limits`) bounds the announcements per session. The default suits a relay mesh, so lower it for untrusted peers. Past it, the session closes with `TOO_MANY_REQUESTS`. Peer-declared lengths are capped before they are buffered. - **Authorization is in band.** On moq-lite 07 (`moq-lite-07-wip`, opt-in) and on moq-transport draft-18+ with the [MoQ Auth extension](/draft/moq-auth), `session.auth()` presents tokens without reconnecting and watches the grant the peer sent back. `add(token)` resolves with the token and how the peer took it: `Answer::Granted` on its AUTH stream, or `Answer::Requests` on moq-transport without AUTH, where only a request that names it carries it. A handle passed to `Client::with_auth` outlives each session, so its tokens, and the routes naming them, carry across your own reconnects. A server verifies tokens itself by taking `handshake.requests()` before `ok()`. `authorize(&grant)` narrows or widens a live session on any version: what falls outside resets with `Unauthorized` and the session stays up. A client that publishes outside its grant closes the session, naming the path. diff --git a/rs/moq-net/MIGRATE.md b/rs/moq-net/MIGRATE.md index 4687a1b6295..c9a8ccd87ec 100644 --- a/rs/moq-net/MIGRATE.md +++ b/rs/moq-net/MIGRATE.md @@ -4,19 +4,6 @@ How to update your code for each breaking change, newest first. ## Unreleased -### Accepting a session runs under one deadline and opens the receive window - -`Server` now bounds each accept, from SETUP through the caller's admission and, on moq-lite 01/02, the SETUP reply, closing a connection that overruns it with `Error::Timeout`'s code. -The default is `server::DEFAULT_TIMEOUT` (10 s); call `Server::with_timeout(None)` to keep waiting forever. -`Server::accept_request` returns a `server::Accept`, which you `.await` as before, or poll; pass it to `kio::clock::timeout` and the like directly, since it is a `Future`. -A caller that runs a transport handshake first computes `Server::deadline(arrived)`, bounds that handshake by it, and passes it on with `Accept::with_deadline`. -`Handshake::with_deadline` is gone: the deadline set at accept bounds `ok` too. -Race your admission decision against the new `Handshake::expired` (or `poll_expired`), which refuses the session with the timeout once the deadline passes; `ok` past the deadline fails with `Error::Timeout`. - -`Server::with_receive_window` and `Handshake::with_receive_window` name the connection receive window `ok` opens before answering SETUP, through the new `transport::poll::Session::set_limits`. -A transport implementation that can change its window overrides `set_limits`; the default reports `LimitsError::Unsupported`, and `ok` then leaves the window as it is. -A wrapper around another transport forwards it. - ### Snapshots split connected sessions into idle and active `stats::Snapshot::activity()` returns each tier's `stats::Activity`: connected sessions with diff --git a/rs/moq-net/src/lite/test_transport.rs b/rs/moq-net/src/lite/test_transport.rs index f4977e9e85e..3480c8db5ed 100644 --- a/rs/moq-net/src/lite/test_transport.rs +++ b/rs/moq-net/src/lite/test_transport.rs @@ -618,8 +618,8 @@ pub struct SinkSession { stats: Arc>, /// Set by [`Self::with_probe`]. probe: Option, - /// Set by [`Self::with_receive_window`]. - receive_window: Option, + /// Set by [`Self::with_receive_window`], and widened by `set_limits`. + receive_window: Arc>>, } impl SinkSession { @@ -651,7 +651,7 @@ impl SinkSession { /// Report `window` as the receive windows in force. pub fn with_receive_window(mut self, window: crate::transport::ReceiveWindow) -> Self { - self.receive_window = Some(window); + self.receive_window = Arc::new(Mutex::new(Some(window))); self } @@ -831,7 +831,17 @@ impl poll::Session for SinkSession { } fn receive_window(&self) -> Option { - self.receive_window + *self.receive_window.lock().unwrap() + } + + /// Set the connection window, on a transport that reports one. + fn set_limits(&self, limits: crate::transport::Limits) -> Result<(), crate::transport::LimitsError> { + let mut window = self.receive_window.lock().unwrap(); + let (Some(window), Some(connection)) = (window.as_mut(), limits.receive_window) else { + return Err(crate::transport::LimitsError::Unsupported); + }; + window.connection = connection.get(); + Ok(()) } } diff --git a/rs/moq-net/src/transport/boxed.rs b/rs/moq-net/src/transport/boxed.rs index 8217d715b18..5ed9c2b2170 100644 --- a/rs/moq-net/src/transport/boxed.rs +++ b/rs/moq-net/src/transport/boxed.rs @@ -7,7 +7,7 @@ use std::any::Any; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, Weak}; +use std::sync::{Arc, OnceLock, Weak}; use std::task::Poll; use bytes::{Buf, Bytes}; @@ -211,17 +211,16 @@ impl RecvStream { /// Box a stream outside any session, which therefore never pauses. #[cfg(test)] pub(crate) fn new(stream: impl poll::RecvStream + Send + Sync) -> Self { - Self::budgeted(stream, Weak::>::new(), None) + Self::budgeted(stream, Weak::>::new()) } - /// Box a stream of a session granting `receive`, read when the stream is accepted or - /// opened, which sizes its pacing [`window`](Self::window). - fn budgeted(stream: impl poll::RecvStream + Send + Sync, session: Budget, receive: Option) -> Self { + /// Box a stream of `session`, whose receive windows size its pacing [`window`](Self::window). + fn budgeted(stream: impl poll::RecvStream + Send + Sync, session: Budget) -> Self { Self(Box::new(Budgeted { stream, session, held: 0, - window: crate::model::pace::window(receive), + window: OnceLock::new(), })) } @@ -251,8 +250,9 @@ struct Budgeted { session: Budget, /// The credit reserved while paused, zero while read. held: u64, - /// The pacing window, from the session's receive windows when the stream arrived. - window: u64, + /// The pacing window, read from the session's receive windows when ingest first paces + /// the stream, so one that arrived before admission widened them paces at the wider ones. + window: OnceLock, } impl Budgeted { @@ -403,22 +403,18 @@ fn unbox<'a, G: 'static>(options: SendOptions<'a, Group>) -> SendOptions<'a, G> } } -fn bi(session: &S, (send, recv): poll::BiStreams, budget: Budget) -> (SendStream, RecvStream) { - let receive = poll::Session::receive_window(session); - (SendStream(Box::new(send)), RecvStream::budgeted(recv, budget, receive)) +fn bi((send, recv): poll::BiStreams, budget: Budget) -> (SendStream, RecvStream) { + (SendStream(Box::new(send)), RecvStream::budgeted(recv, budget)) } impl ErasedSession for S { fn poll_accept_uni(&self, waiter: &kio::Waiter, budget: Budget) -> Poll> { - poll::Session::poll_accept_uni(self, waiter).map(|res| { - res.map(|recv| RecvStream::budgeted(recv, budget, poll::Session::receive_window(self))) - .map_err(Error::new) - }) + poll::Session::poll_accept_uni(self, waiter) + .map(|res| res.map(|recv| RecvStream::budgeted(recv, budget)).map_err(Error::new)) } fn poll_accept_bi(&self, waiter: &kio::Waiter, budget: Budget) -> Poll> { - poll::Session::poll_accept_bi(self, waiter) - .map(|res| res.map(|bis| bi::(self, bis, budget)).map_err(Error::new)) + poll::Session::poll_accept_bi(self, waiter).map(|res| res.map(|bis| bi::(bis, budget)).map_err(Error::new)) } fn poll_open_uni(&self, waiter: &kio::Waiter, options: SendOptions<'_, Group>) -> Poll> { @@ -433,7 +429,7 @@ impl ErasedSession for S { budget: Budget, ) -> Poll> { poll::Session::poll_open_bi(self, waiter, unbox(options)) - .map(|res| res.map(|bis| bi::(self, bis, budget)).map_err(Error::new)) + .map(|res| res.map(|bis| bi::(bis, budget)).map_err(Error::new)) } fn send_group(&self, priority: i64) -> Option { @@ -574,7 +570,13 @@ impl ErasedRecv for Budgeted { } fn window(&self) -> u64 { - self.window + *self.window.get_or_init(|| { + let receive = self + .session + .upgrade() + .and_then(|shared| shared.session.receive_window()); + crate::model::pace::window(receive) + }) } fn pause(&mut self) -> bool { @@ -838,6 +840,21 @@ mod test { assert_eq!(accept(&session).window(), AHEAD); } + /// A stream accepted while the connection window was held back, such as one buffered + /// before admission, paces at the window in force once ingest reads it. + #[test] + fn a_stream_paces_at_the_window_widened_after_it_arrived() { + let gate = kio::Producer::new(true); + let session = + Session::new(SinkSession::accepted_bi(gate.consume()).with_receive_window(ReceiveWindow::new(10, 20))); + let early = accept(&session); + let limits = Limits { + receive_window: std::num::NonZeroU64::new(100), + }; + session.set_limits(limits).unwrap(); + assert_eq!(early.window(), 20); + } + /// A transport that reports no window never pauses a stream, since nothing bounds what /// it would hold; nor does a stream that outlived its session. #[test] diff --git a/rs/moq-relay/src/websocket.rs b/rs/moq-relay/src/websocket.rs index 44bbb029085..bb994cc0688 100644 --- a/rs/moq-relay/src/websocket.rs +++ b/rs/moq-relay/src/websocket.rs @@ -60,6 +60,7 @@ pub(crate) async fn serve_ws( )) .map_err(|_| StatusCode::BAD_REQUEST)?; let mut server = moq_net::Server::new() + .with_timeout(state.timeout) .with_versions(versions) .with_extensions(extensions); if let Some(implementation) = implementation { @@ -71,8 +72,6 @@ pub(crate) async fn serve_ws( options.link.server_name = server_name.map(|Extension(ServerName(name))| name); options.identity = mtls.map(|Extension(MtlsPeer(identity))| identity); options.socket_stats = socket_stats.map(|Extension(stats)| stats.0); - // The listener reads `None` as no deadline; `Options` reads it as the default. - options.timeout = Some(state.timeout.unwrap_or_default()); options.quic = state.quic.clone(); let upgrade = hyper::upgrade::on(&mut request); @@ -700,11 +699,12 @@ mod tests { let activity = moq_net::server::Activity::default(); let mut options = moq_tokio::websocket::Options::new( "ws://localhost/".parse().unwrap(), - moq_net::Server::new().with_versions(moq_net::Versions::all()), + moq_net::Server::new() + .with_versions(moq_net::Versions::all()) + .with_timeout(None), activity.enter().unwrap(), ); options.link.alpn = Some(alpn.clone()); - options.timeout = Some(Duration::ZERO); let server = tokio::spawn(moq_tokio::websocket::accept( Pipe::new(server_incoming, server_to_client, frozen.clone()), options, @@ -752,11 +752,12 @@ mod tests { let activity = moq_net::server::Activity::default(); let mut options = moq_tokio::websocket::Options::new( "ws://localhost/".parse().unwrap(), - moq_net::Server::new().with_versions(moq_net::Versions::all()), + moq_net::Server::new() + .with_versions(moq_net::Versions::all()) + .with_timeout(Some(timeout)), activity.enter().unwrap(), ); options.link.alpn = Some(alpn.clone()); - options.timeout = Some(timeout); let start = kio::clock::now(); let server = tokio::spawn(moq_tokio::websocket::accept( Pipe::new(server_incoming, server_to_client, Arc::new(AtomicBool::new(false))), diff --git a/rs/moq-tokio/MIGRATE.md b/rs/moq-tokio/MIGRATE.md index 179659c5751..b3e894652a8 100644 --- a/rs/moq-tokio/MIGRATE.md +++ b/rs/moq-tokio/MIGRATE.md @@ -4,10 +4,6 @@ How to update your code for each breaking change, newest first. ## Unreleased -### `Request::within` takes `&mut self` - -Admission is bounded by moq-net's accept deadline (`moq_net::server::Handshake::expired`), which refuses the session through the request, so hold the `Request` mutably while it waits: `request.within(admit).await`. - ### A listener builds its peers' identities `tls::PeerIdentity::from_chain` is removed. A listener outside this crate that ran its own mTLS handshake, such as the `moq-uring` workers, calls `tls::Serve::peer(chain)` on the `Serve` its TLS config came from. diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index 9b2d001678f..8d2fa5d2d87 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -1884,7 +1884,7 @@ mod tests { } /// The windows `[quic]` configures, smaller than its defaults. - #[cfg(feature = "tcp")] + #[cfg(any(feature = "quic", feature = "tcp"))] fn small_quic() -> crate::quic::Config { crate::quic::Config { receive_window: Some(moq_net::transport::MIN_RECEIVE_WINDOW * 2), diff --git a/rs/moq-tokio/src/websocket.rs b/rs/moq-tokio/src/websocket.rs index f38cf2b2088..7991a9d2cf6 100644 --- a/rs/moq-tokio/src/websocket.rs +++ b/rs/moq-tokio/src/websocket.rs @@ -121,8 +121,6 @@ pub struct Options { pub identity: Option, /// The socket measurements captured before the HTTP server erased its descriptor. pub socket_stats: Option, - /// The SETUP deadline, including its reply; unset uses ten seconds, zero disables it. - pub timeout: Option, /// The `[quic]` windows and stream count the session starts with, until /// [`crate::server::Request::with_quic_limits`] widens it. Defaults to the `[quic]` /// defaults, so a server passes its own resolved section. @@ -139,7 +137,6 @@ impl Options { link: Default::default(), identity: None, socket_stats: None, - timeout: None, quic: Default::default(), } } @@ -162,7 +159,8 @@ fn socket_config() -> tungstenite::protocol::WebSocketConfig { /// Read SETUP on an upgraded WebSocket, leaving admission to the returned request. /// -/// Cap the socket at [`max_message_size`] before the upgrade. +/// The accept deadline runs from here, set by `options`' server +/// ([`moq_net::Server::with_timeout`]). Cap the socket at [`max_message_size`] before the upgrade. pub async fn accept(socket: T, options: Options) -> crate::Result where T: futures::Stream> @@ -172,10 +170,7 @@ where + 'static, { crate::runtime::scope(async move { - let timeout = options.timeout.unwrap_or(crate::listen::DEFAULT_TIMEOUT); - let deadline = (!timeout.is_zero()) - .then(|| kio::clock::now().checked_add(timeout)) - .flatten(); + let deadline = options.server.deadline(kio::clock::now()); // A vanished peer sends no FIN. Ping and timeout bound the same stale-session // lifetime that QUIC's idle timeout bounds on the native path. // The subprotocol names the QMux version; an upgrade without one has no wire format. @@ -1150,17 +1145,17 @@ mod timeout_tests { use moq_net::transport::poll::Session as _; #[tokio::test(start_paused = true)] - async fn unset_timeout_uses_the_default_and_zero_disables_it() { + async fn the_servers_timeout_bounds_setup() { const ALPN: &str = "qmux-01.moq-lite-06"; - for (timeout, expires) in [(None, true), (Some(time::Duration::ZERO), false)] { + let short = time::Duration::from_secs(1); + for (timeout, expires) in [(Some(short), true), (None, false)] { let activity = moq_net::server::Activity::default(); let mut options = Options::new( "ws://localhost/".parse().unwrap(), - Default::default(), + moq_net::Server::new().with_timeout(timeout), activity.enter().unwrap(), ); options.link.alpn = Some(ALPN.into()); - options.timeout = timeout; // Complete QMux over an in-memory WebSocket, then withhold MoQ SETUP. // A silent transport would expire QMux's own parameter deadline first. let (server, peer) = tokio::io::duplex(4096); @@ -1176,7 +1171,7 @@ mod timeout_tests { let _stream = moq_net::kio::wait(|waiter| peer.poll_open_bi(waiter, Default::default())) .await .unwrap(); - tokio::time::advance(crate::listen::DEFAULT_TIMEOUT).await; + tokio::time::advance(timeout.unwrap_or(crate::listen::DEFAULT_TIMEOUT)).await; let result = accept.as_mut().now_or_never(); if expires { assert!( @@ -1185,7 +1180,7 @@ mod timeout_tests { result.map(|outcome| outcome.map(|_| ())) ); } else { - assert!(result.is_none(), "zero must keep waiting for SETUP"); + assert!(result.is_none(), "no timeout must keep waiting for SETUP"); } } }