diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index 3e5e353de93..b5cf12d9869 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -3975,7 +3975,7 @@ class LiveMoqAnnounceEvent extends MoqAnnounceEvent { } } -enum MoqTransport { quic, iroh, webSocket, tcp, unix } +enum MoqTransport { quic, iroh, webSocket, tcp, unix, webTransport } class FfiConverterMoqTransport { static LiftRetVal read(Uint8List buf) { @@ -3991,6 +3991,8 @@ class FfiConverterMoqTransport { return LiftRetVal(MoqTransport.tcp, 4); case 5: return LiftRetVal(MoqTransport.unix, 4); + case 6: + return LiftRetVal(MoqTransport.webTransport, 4); default: throw UniffiInternalError( UniffiInternalError.unexpectedEnumCase, diff --git a/doc/concept/transport.md b/doc/concept/transport.md index 14c74e7005d..bfdf09bd32f 100644 --- a/doc/concept/transport.md +++ b/doc/concept/transport.md @@ -42,7 +42,11 @@ connection (`wss://`) and keep whichever wins. A small multiplexer, [qmux](/draft/qmux-websocket), carries MoQ streams over the socket. It works everywhere but cannot escape TCP head-of-line blocking, so priority and resets only help once bytes leave the TCP queue. The fallback is automatic in every -client; the relay enables it with `[web.https]`. +client; the relay enables it with `[web.https]`. A native Rust client that lands on +WebSocket keeps dialing QUIC and moves the session onto it once the QUIC +session receives the peer's SETUP, staying on WebSocket if the relay refuses it +before SETUP. This crate's relays send SETUP after admission; other servers may +still refuse afterward. ## Raw QUIC (native) diff --git a/doc/lib/rs/index.md b/doc/lib/rs/index.md index 72ccdf15aa7..7334c6b7fd9 100644 --- a/doc/lib/rs/index.md +++ b/doc/lib/rs/index.md @@ -87,7 +87,11 @@ URLs may be `https://` (WebTransport, with raw QUIC preferred for native), `moql://`/`moqt://` (raw QUIC), or `iroh://`. A `?jwt=` query carries the token. `http://` is for a relay on localhost only: it fetches the certificate fingerprint unauthenticated before upgrading, so never send a token over it. Connections race -QUIC against WebSocket and remember which won. +QUIC against WebSocket and remember which won. When WebSocket wins, a `Connection` keeps +dialing QUIC and moves onto it once the relay admits that session, handing live tracks over +at a group boundary and draining the WebSocket session; `Connection::transport()` reports +which is live. A refused or stalled QUIC session leaves the WebSocket session serving, and +moq-lite-03 and -04, which give no sign of admission, never upgrade. The `Default::default()` above is the QUIC transport section, and the same value serves a dial and a listener: diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index 94932fdbead..8c030f915a4 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -52,6 +52,15 @@ let (session, driver) = client.connect(now, transport).await?; tokio::spawn(moq_net::time::run(driver)); ``` +On moq-lite-05+ and moq-transport draft 17+, `Client::connect` returns before +the server's SETUP arrives, so a request made at once may still be +refused along with it. `session.setup().await` waits for the server's SETUP +and returns the close reason if the session is already closed, even after SETUP. A moq-rs +server sends its SETUP only once it admits the client; neither protocol +requires that, so another server may still refuse afterward. Older versions +read that SETUP during the handshake and resolve at once, except moq-lite-03 +and -04, which carry none and return `Error::Unsupported`. + A custom event loop calls `driver.poll(now, waiter)` with a nondecreasing `moq_net::time::Instant`. `Ok(Some(at))` asks to be polled again by `at` or on external activity, `Ok(None)` only on external activity, and `Err` is the diff --git a/doc/setup/upgrade.md b/doc/setup/upgrade.md index dd5adcaa736..52d1b9af381 100644 --- a/doc/setup/upgrade.md +++ b/doc/setup/upgrade.md @@ -87,6 +87,12 @@ These land with the next breaking release, not the 2026-09-23 train. broadcast that ends with media queued behind an undescribed track is an error rather than an empty `Ok(None)`. Restart the export to pick up a new rendition. +- **moq-tokio's `Transport` names WebTransport.** `moq_tokio::server::Transport` + is `moq_tokio::Transport`, with no re-export. A WebTransport session reports + `Transport::WebTransport` (`"webtransport"` in logs) instead of `Quic`, which + now means raw QUIC only, and `Connection::transport()` reports the live + transport. The bindings' `MoqTransport` gains a `WebTransport` case, so an + exhaustive `switch` or `when` needs one more arm. - **moq-net has no `VarInt`.** Varints are plain `u64`s: `VarInt::decode_quic(buf)?.into_inner()` is `moq_net::varint::decode_quic(buf)?`, and `VarInt::try_from(v)?.encode_quic(buf)` is diff --git a/go/wrapper/server.go b/go/wrapper/server.go index 02f668f1161..108e51037d6 100644 --- a/go/wrapper/server.go +++ b/go/wrapper/server.go @@ -22,6 +22,8 @@ const ( TransportTCP = ffi.MoqTransportTcp // TransportUnix is a session that arrived over a Unix domain socket. TransportUnix = ffi.MoqTransportUnix + // TransportWebTransport is a session that arrived over WebTransport (HTTP/3). + TransportWebTransport = ffi.MoqTransportWebTransport ) // Request is an incoming session that can be accepted (Accept) or rejected (Reject). diff --git a/quest/m1/quest-flat-lines.md b/quest/m1/quest-flat-lines.md index 1a6a6bd265f..bfb3648b912 100644 --- a/quest/m1/quest-flat-lines.md +++ b/quest/m1/quest-flat-lines.md @@ -59,7 +59,7 @@ Remaining: #4403 wildcard, #4034 archive, #4039 auth, #4079 cpp, #4519 ffi-shape, #4080 obs-moq-video, #4133 qos (held until [Lag across a splice](/quest/m1/qos/lag-splice.md) is fixed), - #4180 transport-upgrade, #4640 tstd. #4162 (audio-jitter-target), #4437 rs2ts + #4640 tstd. #4162 (audio-jitter-target), #4180 (transport-upgrade), #4437 rs2ts (with #4438 folded in), and #4653 (test-flakes-2) landed. - Fold #4255 (archive/track-timeline) into the archive branch before #4034 lands; as of the 2026-10-05 audit it has not happened and #4034 is a draft. diff --git a/quest/m1/transport-upgrade/README.md b/quest/m1/transport-upgrade/README.md index 91cbd58f60a..49b24267859 100644 --- a/quest/m1/transport-upgrade/README.md +++ b/quest/m1/transport-upgrade/README.md @@ -41,7 +41,13 @@ that. Shared decisions: - The race returns the winner plus the still-pending QUIC dial when WebSocket - wins. The QUIC handshake timeout bounds that dial; no extra deadline. + wins. The attempt's connect deadline bounds that dial; no extra deadline. +- The swap waits for the peer's SETUP on the QUIC session + (`moq_net::Session::setup`) within that same deadline; until + then the WebSocket session gets no GOAWAY. A refused or stalled QUIC session + is dropped and WebSocket keeps serving. moq-lite-03 and -04 carry no server + SETUP, so they never upgrade. This crate's servers send SETUP after admission; + other servers may send it before admission and still refuse afterward. - On a successful upgrade the "WebSocket won" memo (`WEBSOCKET_WON` in `moq-tokio`, `websocketWon` in `js/net`) forgets the URL: QUIC works on this network, so the head start comes back. Otherwise a network where WebSocket @@ -58,5 +64,6 @@ Shared decisions: ## Required -- [Rust](/quest/m1/transport-upgrade/rust.md) - moq-tokio keeps the QUIC dial after WebSocket wins and migrates through the existing Draining path - [JavaScript](/quest/m1/transport-upgrade/js.md) - js/net keeps the WebTransport dial after WebSocket wins and migrates through the client-goaway handover +- [Closed fallback](/quest/m1/transport-upgrade/closed-fallback.md) - a WebSocket session that closes right after connecting falls back to the pending QUIC dial instead of redialing +- [JS qmux finish](/quest/m1/transport-upgrade/js-qmux-finish.md) - `@moq/qmux` reports a cleanly finished send stream as closed without error diff --git a/quest/m1/transport-upgrade/closed-fallback.md b/quest/m1/transport-upgrade/closed-fallback.md new file mode 100644 index 00000000000..8df94079c1c --- /dev/null +++ b/quest/m1/transport-upgrade/closed-fallback.md @@ -0,0 +1,30 @@ +# [S] Fall back to QUIC when the WebSocket session closes at once + +## Goal + +When WebSocket wins the dial race and its session closes right after +connecting, the `Connection` moves onto the QUIC dial that is still pending +instead of dropping it and redialing. On moq-lite and IETF draft 17+ the +client handshake does not wait for the server, so a server refusing the +WebSocket session looks like this rather than like a handshake failure, which +already falls back. + +## Plan + +- Today `run_session` in `moq-tokio`'s `connection.rs` returns when the + WebSocket session closes, and the pending upgrade is dropped. The redial + starts a fresh QUIC dial and has lost QUIC's head start. +- Decide what counts as "at once". The unhealthy-session threshold that + already feeds the backoff may be the right line. Keep the connect deadline + that bounds the pending dial as the only deadline. +- An auth close on the WebSocket session is probably still terminal, the way + it is for a lone session. Check how the race treats a mixed auth pair and + match it. +- Test with the held-QUIC forwarder in the connection tests, as the handshake + fallback test does, with a default moq-lite client. + +Public API: none. Wire: none. + +## Related + +- [#4296](https://github.com/moq-dev/moq/pull/4296) - the handshake-failure fallback this extends diff --git a/quest/m1/transport-upgrade/js-qmux-finish.md b/quest/m1/transport-upgrade/js-qmux-finish.md new file mode 100644 index 00000000000..1ce689a9bd4 --- /dev/null +++ b/quest/m1/transport-upgrade/js-qmux-finish.md @@ -0,0 +1,22 @@ +# [XS] Clean finish in @moq/qmux + +## Goal + +`@moq/qmux` reports a send stream that finished cleanly as closed without +error, so `@moq/net` does not treat a delivered GOAWAY or SETUP as a failure +over the WebSocket fallback. + +## Plan + +The Rust qmux crate reported every finished stream as `connection closed` +(fixed in moq-dev/web-transport#402). Check whether the TypeScript peer in +`moq-dev/web-transport` (`js/qmux`) has the same bug, and whether `@moq/net` +waits on a finished writable anywhere it would notice. If it does, fix it +there with a regression test, release, and bump `@moq/qmux` in `js/net`. If +not, note that in the PR that deletes this quest. + +Public API: none. Wire: none. + +## Related + +- [JavaScript upgrade](/quest/m1/transport-upgrade/js.md) - sends a GOAWAY over the WebSocket session on every upgrade diff --git a/quest/m1/transport-upgrade/js.md b/quest/m1/transport-upgrade/js.md index 16c399515fe..1b4a1df1f59 100644 --- a/quest/m1/transport-upgrade/js.md +++ b/quest/m1/transport-upgrade/js.md @@ -31,6 +31,12 @@ session to close on its own or at the handover cap. See the may not name a redirect URI, and an empty one is legal) and closes at the configured cap. A WebTransport attempt that fails after WebSocket won is logged at debug and the session stays on WebSocket. +- Swap only once the server's SETUP arrives on the WebTransport session: mirror Rust's + `moq_net::Session::setup()` in `js/net` (resolves on the server's SETUP, + rejects with the close reason if already closed, even after SETUP) and await it inside the + connect deadline. A refusal, a stall, or moq-lite-03/-04 (no server SETUP) + keeps WebSocket with no GOAWAY sent. This crate's servers send SETUP after + admission; other servers may still refuse afterward. - A self-sent GOAWAY must gate new requests on the old session too, not only a received one: [JS GOAWAY requests](/quest/m1/js-goaway-requests.md) covers the received case, so check it also covers this path. @@ -40,7 +46,7 @@ session to close on its own or at the handover cap. See the group across the upgrade, the WebSocket session closes within the cap, and the next connect to the same URL gives WebTransport the head start again; with no delay, WebTransport wins and no WebSocket session is ever opened. -- Public API: none beyond what client-goaway adds; `transportOf` already +- Public API: the `setup` mirror, if it is exported; `transportOf` already reports the live transport. Update `doc/lib/js` where the fallback race is described. diff --git a/quest/m1/transport-upgrade/rust.md b/quest/m1/transport-upgrade/rust.md deleted file mode 100644 index d920c9b90f0..00000000000 --- a/quest/m1/transport-upgrade/rust.md +++ /dev/null @@ -1,49 +0,0 @@ -# [M] Rust WebSocket to QUIC upgrade - -## Goal - -A `moq_tokio::Connection` that came up over the WebSocket fallback migrates to -QUIC once the QUIC dial completes: live tracks resume on QUIC from the first -frame they lack, the WebSocket session receives a GOAWAY and drains within the -handover cap, `Status::Migrating` is observable across the swap, and the -"WebSocket won" memo forgets the URL. When QUIC wins the race nothing changes. - -## Plan - -Lands in `rs/moq-tokio`. See the -[questline](/quest/m1/transport-upgrade/README.md) for the shared decisions. - -- `race_moq_connect` (`rs/moq-tokio/src/client.rs`) currently drops the losing - arm. When WebSocket wins, return the handshaken WebSocket session together - with the pending QUIC future; when QUIC wins or the QUIC arm has already - failed, return the session alone. `Client::dial` keeps its one-session shape - for the one-shot `connect()` path; give `Connection` the variant that carries - the pending upgrade. -- `Connection::run` (`rs/moq-tokio/src/connection.rs`) polls the pending QUIC - dial alongside `run_session`, the way it already polls `draining`. When it - completes, run the MoQ handshake on it with the same `moq_net::Client` (the - origins attach and the front reselects to the newest route), call - `shared.migrating()` then `shared.connected(&new)`, send - `old.drain().send(Goaway::same().timeout(handover))` with the configured - `connection::Goaway` cap, and move the old session into `Draining`. A QUIC dial - that fails after WebSocket won is logged at debug, leaves the memo untouched, - and the session carries on over WebSocket. The pending dial is dropped when the WebSocket session ends - first; the reconnect races again. -- On a successful upgrade remove the URL from `WEBSOCKET_WON` - (`rs/moq-tokio/src/websocket.rs`). -- Add `moq_tokio::Transport` (QUIC, WebTransport, WebSocket, TCP, Unix) and - `Connection::transport()` returning it for the live session, so telemetry can - see the swap. It lives in `moq-tokio`, not `moq_net`: the session is generic - over the transport trait and cannot know what it runs on, only the dial - does. Mirrors `js/net`'s `transportOf`. -- Regression test against the in-tree relay: the QUIC path goes through an - in-process UDP forwarder that delays packets past the WebSocket head start so - WebSocket wins deterministically; a subscribed track keeps every group across - the swap, `Status::Migrating` is observed, the WebSocket session closes - within the handover cap, and a second connect to the same URL gives QUIC the - head start again. A second test drops the delay to zero and asserts QUIC wins - and no second session is ever opened. -- Public API: additive (the transport accessor). Wire: none; the client GOAWAY - is already specified for either endpoint on moq-lite 04+ and an empty-URI - GOAWAY is legal for a moq-transport client. Update `doc/lib/rs` where the - fallback race is described. diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index db2ca882454..ec9b924a9e6 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -253,9 +253,9 @@ async fn serve_client( } /// Whether ordinary clients may use this transport on the shared LAN server. -fn is_public_transport(transport: moq_tokio::server::Transport, public_quic: bool) -> bool { +fn is_public_transport(transport: moq_tokio::Transport, public_quic: bool) -> bool { match transport { - moq_tokio::server::Transport::Tcp | moq_tokio::server::Transport::Unix => true, + moq_tokio::Transport::Tcp | moq_tokio::Transport::Unix => true, _ => public_quic, } } @@ -1031,9 +1031,9 @@ mod tests { #[test] fn explicit_stream_listeners_are_public_without_exposing_mesh_quic() { - assert!(is_public_transport(moq_tokio::server::Transport::Tcp, false)); - assert!(is_public_transport(moq_tokio::server::Transport::Unix, false)); - assert!(!is_public_transport(moq_tokio::server::Transport::Quic, false)); - assert!(is_public_transport(moq_tokio::server::Transport::Quic, true)); + assert!(is_public_transport(moq_tokio::Transport::Tcp, false)); + assert!(is_public_transport(moq_tokio::Transport::Unix, false)); + assert!(!is_public_transport(moq_tokio::Transport::Quic, false)); + assert!(is_public_transport(moq_tokio::Transport::Quic, true)); } } diff --git a/rs/moq-ffi/src/server.rs b/rs/moq-ffi/src/server.rs index daae44f026d..c42937fb7a2 100644 --- a/rs/moq-ffi/src/server.rs +++ b/rs/moq-ffi/src/server.rs @@ -202,7 +202,7 @@ struct RequestState { /// The network transport carrying an incoming session. #[derive(Clone, Copy, Debug, Eq, PartialEq, uniffi::Enum)] pub enum MoqTransport { - /// QUIC, either directly or through WebTransport over HTTP/3. + /// Raw QUIC, negotiating a MoQ ALPN directly. Quic, /// An Iroh QUIC connection. Iroh, @@ -212,18 +212,21 @@ pub enum MoqTransport { Tcp, /// A Unix domain socket using qmux framing. Unix, + /// WebTransport over HTTP/3 on QUIC. + WebTransport, } -impl TryFrom for MoqTransport { +impl TryFrom for MoqTransport { type Error = MoqError; - fn try_from(value: moq_tokio::server::Transport) -> Result { + fn try_from(value: moq_tokio::Transport) -> Result { Ok(match value { - moq_tokio::server::Transport::Quic => Self::Quic, - moq_tokio::server::Transport::Iroh => Self::Iroh, - moq_tokio::server::Transport::WebSocket => Self::WebSocket, - moq_tokio::server::Transport::Tcp => Self::Tcp, - moq_tokio::server::Transport::Unix => Self::Unix, + moq_tokio::Transport::Quic => Self::Quic, + moq_tokio::Transport::Iroh => Self::Iroh, + moq_tokio::Transport::WebSocket => Self::WebSocket, + moq_tokio::Transport::Tcp => Self::Tcp, + moq_tokio::Transport::Unix => Self::Unix, + moq_tokio::Transport::WebTransport => Self::WebTransport, _ => return Err(MoqError::Unsupported), }) } @@ -232,7 +235,7 @@ impl TryFrom for MoqTransport { #[cfg(test)] mod transport_tests { use super::MoqTransport; - use moq_tokio::server::Transport; + use moq_tokio::Transport; #[test] fn converts_supported_transports() { @@ -244,6 +247,10 @@ mod transport_tests { ); assert_eq!(MoqTransport::try_from(Transport::Tcp).unwrap(), MoqTransport::Tcp); assert_eq!(MoqTransport::try_from(Transport::Unix).unwrap(), MoqTransport::Unix); + assert_eq!( + MoqTransport::try_from(Transport::WebTransport).unwrap(), + MoqTransport::WebTransport + ); } } diff --git a/rs/moq-net/src/client.rs b/rs/moq-net/src/client.rs index ac95f02a8d0..884de463bf4 100644 --- a/rs/moq-net/src/client.rs +++ b/rs/moq-net/src/client.rs @@ -203,6 +203,7 @@ impl Client { start.recv_bandwidth, crate::driver::Protocol::Lite(Box::new(start.driver)), start.goaway, + start.setup, )) } @@ -259,7 +260,7 @@ impl Client { // Draft-17+: SETUP is exchanged by the connection driver. // We advertise the request path in our SETUP for URL-less transports. - let (protocol, goaway) = ietf::start(ietf::Config { + let (protocol, goaway, setup) = ietf::start(ietf::Config { runtime: runtime.clone(), session: session.clone(), setup: None, @@ -285,6 +286,7 @@ impl Client { None, crate::driver::Protocol::Ietf(protocol), goaway, + setup, )); } Some(ALPN_16) => { @@ -371,7 +373,7 @@ impl Client { .copied() .ok_or(Error::Version)?; - let (recv_bw, protocol, goaway) = match version { + let (recv_bw, protocol, goaway, setup) = match version { Version::Lite(v) => { let stream = stream.with_version(v); let start = lite::start(lite::Config { @@ -392,6 +394,7 @@ impl Client { start.recv_bandwidth, crate::driver::Protocol::Lite(Box::new(start.driver)), start.goaway, + start.setup, ) } Version::Ietf(v) => { @@ -410,7 +413,7 @@ impl Client { let stream = stream.with_version(v); // Draft 14-16: the path rode in the bidi SETUP above, not the uni one. - let (protocol, goaway) = ietf::start(ietf::Config { + let (protocol, goaway, setup) = ietf::start(ietf::Config { runtime: runtime.clone(), session: session.clone(), setup: Some(stream), @@ -427,11 +430,13 @@ impl Client { peer_declared: Some(peer_declared), early_unis: Vec::new(), })?; - (None, crate::driver::Protocol::Ietf(protocol), goaway) + (None, crate::driver::Protocol::Ietf(protocol), goaway, setup) } }; - Ok(Session::new(runtime, session, version, recv_bw, protocol, goaway)) + Ok(Session::new( + runtime, session, version, recv_bw, protocol, goaway, setup, + )) } } diff --git a/rs/moq-net/src/ietf/peer.rs b/rs/moq-net/src/ietf/peer.rs index fad2d163dd0..9d28ba8df91 100644 --- a/rs/moq-net/src/ietf/peer.rs +++ b/rs/moq-net/src/ietf/peer.rs @@ -49,6 +49,16 @@ impl PeerSetup { } } + /// Poll until the peer's SETUP arrives. + pub fn poll_seen(&self, waiter: &kio::Waiter) -> std::task::Poll<()> { + self.0 + .poll(waiter, |peer| match peer.is_some() { + true => std::task::Poll::Ready(()), + false => std::task::Poll::Pending, + }) + .map(|_| ()) + } + /// Await the peer's SETUP. /// /// The peer MUST send exactly one, so this resolves once that stream is read. Waits diff --git a/rs/moq-net/src/ietf/session.rs b/rs/moq-net/src/ietf/session.rs index 74dbcfd2fee..aae9a0ffc3e 100644 --- a/rs/moq-net/src/ietf/session.rs +++ b/rs/moq-net/src/ietf/session.rs @@ -95,7 +95,7 @@ impl std::future::Future for Driver { } } -pub fn start(config: Config) -> Result<(Driver, crate::goaway::Handle), Error> +pub fn start(config: Config) -> Result<(Driver, crate::goaway::Handle, crate::session::Setup), Error> where S: crate::transport::poll::Boxable, { @@ -133,6 +133,24 @@ where let closing = local_close.clone(); let owed = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); let serving = owed.clone(); + + // What the peer declared in its SETUP. Seeded now when that stream was already + // read (the legacy handshake, or a gated server accept), and filled by the uni + // loop otherwise. + let peer_setup = peer::PeerSetup::default(); + let setup_read = peer_declared.is_some(); + match peer_declared { + Some(declared) => peer_setup.set(declared), + // A legacy caller that passed nothing (our own tests, and the lite paths): + // settle the slot rather than leave the announce loops waiting on a value + // that is never coming. + None if !cluster::supported(version) => peer_setup.set(peer::Peer::default()), + None => {} + } + + // Settled already on drafts 14-16, which read the peer's SETUP in the handshake. + let setup_seen = crate::session::Setup::Ietf(peer_setup.clone()); + let driver = async move { // Our own Hop ID, taken from whichever origin the caller actually supplied so // every session out of this process stamps the same one and cross-session loop @@ -147,20 +165,6 @@ where let publish = publish.unwrap_or_else(|| origin::Producer::empty(Hop::random()).consume()); let subscribe = subscribe.unwrap_or_else(|| origin::Producer::empty(Hop::random())); - // What the peer declared in its SETUP. Seeded now when that stream was already - // read (the legacy handshake, or a gated server accept), and filled by the uni - // loop otherwise. - let peer_setup = peer::PeerSetup::default(); - let setup_read = peer_declared.is_some(); - match peer_declared { - Some(declared) => peer_setup.set(declared), - // A legacy caller that passed nothing (our own tests, and the lite paths): - // settle the slot rather than leave the announce loops waiting on a value - // that is never coming. - None if !cluster::supported(version) => peer_setup.set(peer::Peer::default()), - None => {} - } - let res = match version { Version::Draft14 | Version::Draft15 | Version::Draft16 => { let Some(setup) = setup else { @@ -500,6 +504,7 @@ where future: driver, }, goaway_handle, + setup_seen, )) } @@ -1127,7 +1132,7 @@ mod tests { let session = crate::lite::test_transport::ScriptedSession::new(namespace_without_hop_path(VERSION).await); let log = session.log.clone(); - let (driver, _goaway) = start(Config { + let (driver, _goaway, _) = start(Config { runtime: crate::time::Clock::sim(), session, setup: None, @@ -1187,7 +1192,7 @@ mod tests { let session = crate::lite::test_transport::SinkSession::gated_bi(gate.consume()); let log = session.log.clone(); - let (driver, _goaway) = start(Config { + let (driver, _goaway, _) = start(Config { runtime: crate::time::Clock::sim(), session, setup: None, @@ -1240,7 +1245,7 @@ mod tests { let session = crate::lite::test_transport::SinkSession::gated_bi(gate.consume()); let log = session.log.clone(); - let (driver, _goaway) = start(Config { + let (driver, _goaway, _) = start(Config { runtime: crate::time::Clock::sim(), session, setup: None, @@ -1348,7 +1353,7 @@ mod tests { let origin = crate::origin::Config::new(crate::Hop::new(1).unwrap()).produce(); let log = session.log.clone(); - let (driver, _goaway) = start(Config { + let (driver, _goaway, _) = start(Config { runtime: crate::time::Clock::sim(), session, setup: None, @@ -1388,7 +1393,7 @@ mod tests { let session = crate::lite::test_transport::ScriptedSession::new(Vec::new()); let log = session.log.clone(); - let (driver, _goaway) = start(Config { + let (driver, _goaway, _) = start(Config { runtime: crate::time::Clock::sim(), session, setup: None, @@ -1676,7 +1681,7 @@ mod tests { let session = crate::lite::test_transport::ScriptedSession::new(Vec::new()).with_incoming_bidis(vec![payload]); let log = session.log.clone(); - let (driver, _goaway) = start(Config { + let (driver, _goaway, _) = start(Config { runtime: crate::time::Clock::sim(), session, setup: None, @@ -1779,7 +1784,7 @@ mod tests { .await .expect("open the control stream"); - let (driver, _goaway) = start(Config { + let (driver, _goaway, _) = start(Config { runtime: crate::time::Clock::sim(), session, setup: Some(setup), diff --git a/rs/moq-net/src/lite/session.rs b/rs/moq-net/src/lite/session.rs index 2b54fce287c..76f098e22a5 100644 --- a/rs/moq-net/src/lite/session.rs +++ b/rs/moq-net/src/lite/session.rs @@ -18,6 +18,8 @@ pub(crate) struct SessionStart { pub driver: Driver, /// The session-side GOAWAY halves, stored on the public [`crate::Session`]. pub goaway: crate::goaway::Handle, + /// Whether the peer's SETUP arrived, read by [`crate::Session::setup`]. + pub setup: crate::session::Setup, } /// Server: read the peer's single SETUP message off its Setup Stream before starting @@ -242,6 +244,16 @@ where }; let peer_setup = peer_setup_slot; + // Lite-05+ records the peer's SETUP from its Setup Stream. Before that, only the + // legacy bidi handshake carries one, and it was read before the session started. + let setup = if version.has_setup_stream() { + crate::session::Setup::Lite(peer_setup.clone()) + } else if setup_stream.is_some() { + crate::session::Setup::Read + } else { + crate::session::Setup::Never + }; + // GOAWAY wiring: the public Session holds one half (send trigger, received // signal), the protocol tasks below hold the other. moq-lite lets either side // name a redirect URI, unlike moq-transport. @@ -290,6 +302,7 @@ where recv_bandwidth: recv_bw_consumer, driver, goaway: goaway_handle, + setup, }) } diff --git a/rs/moq-net/src/lite/setup.rs b/rs/moq-net/src/lite/setup.rs index 9aba62a5e53..ea5d67328fa 100644 --- a/rs/moq-net/src/lite/setup.rs +++ b/rs/moq-net/src/lite/setup.rs @@ -283,6 +283,11 @@ impl PeerSetup { Ok(()) } + /// Poll until the peer's SETUP arrives. + pub fn poll_seen(&self, waiter: &kio::Waiter) -> std::task::Poll<()> { + self.poll_get(waiter, |_| ()) + } + /// Poll for the peer's advertised probe level, waiting until its SETUP arrives. pub fn poll_probe_level(&self, waiter: &kio::Waiter) -> std::task::Poll { self.poll_get(waiter, |setup| setup.probe) diff --git a/rs/moq-net/src/server.rs b/rs/moq-net/src/server.rs index 78bcfb64b1f..7876fc38c27 100644 --- a/rs/moq-net/src/server.rs +++ b/rs/moq-net/src/server.rs @@ -128,6 +128,7 @@ impl Server { start.recv_bandwidth, crate::driver::Protocol::Lite(Box::new(start.driver)), start.goaway, + start.setup, )) } @@ -519,7 +520,7 @@ where // The client's SETUP was read at the pause; hand the stream back // for GOAWAY. A server never advertises a path, hence `None`. - let (protocol, goaway) = ietf::start(ietf::Config { + let (protocol, goaway, setup) = ietf::start(ietf::Config { runtime: runtime.clone(), session: session.clone(), setup: None, @@ -545,6 +546,7 @@ where None, crate::driver::Protocol::Ietf(protocol), goaway, + setup, )) } .maybe_boxed() @@ -605,7 +607,7 @@ where }; stream.writer.encode(&server_setup).await?; - let (recv_bw, protocol, goaway) = match version { + let (recv_bw, protocol, goaway, setup) = match version { Version::Lite(v) => { let stream = stream.with_version(v); // Pre-lite-05: no Setup Stream, so nothing to advertise or seed. @@ -624,12 +626,13 @@ where start.recv_bandwidth, crate::driver::Protocol::Lite(Box::new(start.driver)), start.goaway, + start.setup, ) } Version::Ietf(v) => { let stream = stream.with_version(v); // Draft 14-16: path came in the bidi SETUP, no uni SETUP to hand back. - let (protocol, goaway) = ietf::start(ietf::Config { + let (protocol, goaway, setup) = ietf::start(ietf::Config { runtime: runtime.clone(), session: session.clone(), setup: Some(stream), @@ -646,11 +649,13 @@ where peer_declared: Some(peer_declared), early_unis: Vec::new(), })?; - (None, crate::driver::Protocol::Ietf(protocol), goaway) + (None, crate::driver::Protocol::Ietf(protocol), goaway, setup) } }; - Ok(Session::new(runtime, session, version, recv_bw, protocol, goaway)) + Ok(Session::new( + runtime, session, version, recv_bw, protocol, goaway, setup, + )) } .maybe_boxed() } diff --git a/rs/moq-net/src/session.rs b/rs/moq-net/src/session.rs index 8c41d4c0bb2..045af26f36b 100644 --- a/rs/moq-net/src/session.rs +++ b/rs/moq-net/src/session.rs @@ -35,6 +35,31 @@ struct StatsState { demanded: bool, } +/// How [`Session::setup`] learns that the peer's SETUP arrived. +#[derive(Clone)] +pub(crate) enum Setup { + /// The handshake read it before the session started. + Read, + /// The lite driver records it from the peer's Setup Stream. + Lite(crate::lite::PeerSetup), + /// The moq-transport driver records it from the peer's SETUP stream. + Ietf(crate::ietf::peer::PeerSetup), + /// The version carries no SETUP from the peer. + Never, +} + +impl Setup { + /// `Ready(true)` once the SETUP arrived, `Ready(false)` if it never will. + fn poll(&self, waiter: &kio::Waiter) -> Poll { + match self { + Self::Read => Poll::Ready(true), + Self::Lite(setup) => setup.poll_seen(waiter).map(|()| true), + Self::Ietf(setup) => setup.poll_seen(waiter).map(|()| true), + Self::Never => Poll::Ready(false), + } + } +} + /// A snapshot of connection statistics for a [`Session`]. /// /// Every field is optional: availability depends on the transport backend (native QUIC @@ -101,6 +126,7 @@ pub struct Session { send_bandwidth: Option, recv_bandwidth: Option, goaway: Arc, + setup: Setup, } impl Session { @@ -216,6 +242,35 @@ impl Session { } } + /// Wait for the peer's SETUP, or return the session's close reason. + /// + /// Resolves when the peer's SETUP arrives. This crate's servers send it only once + /// they admit the client, but neither protocol requires that ordering, so another + /// server may send SETUP and still refuse the session afterward. + /// [`crate::Client::connect`] returns before the SETUP on moq-lite-05+ and + /// moq-transport draft 17+. Older versions read it during the handshake and resolve + /// at once, except moq-lite-03 and -04, which carry none and return + /// [`Error::Unsupported`]. A session that already closed returns its close reason. + pub async fn setup(&self) -> Result<(), Error> { + kio::wait(|waiter| { + match self.closed.poll(waiter, |state| match &**state { + Some(ended) => Poll::Ready(ended.err.clone()), + None => Poll::Pending, + }) { + Poll::Ready(Ok(err)) => return Poll::Ready(Err(err)), + // The driver was dropped before it could observe the close. + Poll::Ready(Err(_)) => return Poll::Ready(Err(Error::Cancel)), + Poll::Pending => {} + } + match self.setup.poll(waiter) { + Poll::Ready(true) => Poll::Ready(Ok(())), + Poll::Ready(false) => Poll::Ready(Err(Error::Unsupported)), + Poll::Pending => Poll::Pending, + } + }) + .await + } + /// Drain the peer gracefully: the handle for sending this session's single /// GOAWAY. /// @@ -255,6 +310,7 @@ impl Session { recv_bandwidth: Option, protocol: crate::driver::Protocol, goaway: goaway::Handle, + setup: Setup, ) -> (Self, crate::Driver) where S: crate::transport::poll::Session, @@ -298,6 +354,7 @@ impl Session { send_bandwidth, recv_bandwidth, goaway: Arc::new(goaway), + setup, }; let driver = crate::Driver::new( runtime.clone(), diff --git a/rs/moq-net/tests/session_setup.rs b/rs/moq-net/tests/session_setup.rs new file mode 100644 index 00000000000..3cfa2f863a0 --- /dev/null +++ b/rs/moq-net/tests/session_setup.rs @@ -0,0 +1,175 @@ +//! `Session::setup` waits for the server's SETUP, and reports the session's close reason. +//! +//! Deterministic: paused time over the mock transport. + +mod support; + +use std::time::Duration; + +use moq_net::{Client, Error, Server, Session, SessionError, Version, server::Handshake}; +use support::{ + harness::{now, spawn}, + mock::{MockSession, create_mock_session_pair}, +}; + +const TIMEOUT: Duration = Duration::from_secs(10); + +/// Versions whose server sends its SETUP after +/// `Client::connect` has returned. +const LATE: [&str; 9] = [ + "moq-lite-05", + "moq-lite-06", + "moq-lite-07-wip", + "moq-transport-17", + "moq-transport-18", + "moq-transport-19", + "moq-transport-20", + "moq-transport-21", + "moq-transport-22", +]; + +/// Versions whose handshake reads the server's SETUP before `Client::connect` returns. +const EARLY: [&str; 5] = [ + "moq-lite-01", + "moq-lite-02", + "moq-transport-14", + "moq-transport-15", + "moq-transport-16", +]; + +/// Versions with no server SETUP at all. +const NONE: [&str; 2] = ["moq-lite-03", "moq-lite-04"]; + +fn version(name: &str) -> Version { + name.parse().unwrap() +} + +/// Dial a `LATE` version, returning the client's session and the server's request, +/// still unanswered. +async fn dial(version: Version) -> (Session, Handshake) { + let (client, server) = create_mock_session_pair(Some(version.alpn())); + let client = async { + let (session, driver) = Client::new() + .with_versions(version.into()) + .connect(now(), client) + .await + .expect("client handshake failed"); + spawn(driver); + session + }; + let server = async { + Server::new() + .with_versions(version.into()) + .accept_request(now(), server) + .await + .expect("server never saw the request") + }; + futures::join!(client, server) +} + +async fn setup(session: &Session) -> Option> { + moq_net_sim::timeout(TIMEOUT, session.setup()).await.ok() +} + +#[moq_net_sim::test] +async fn setup_waits_for_the_server_to_answer() { + for name in LATE { + let (session, request) = dial(version(name)).await; + assert!( + setup(&session).await.is_none(), + "{name}: SETUP arrived before the server answered" + ); + + let (_server, driver) = request.ok().await.expect("server accept failed"); + spawn(driver); + assert!(matches!(setup(&session).await, Some(Ok(()))), "{name}"); + } +} + +/// A session that closed after its SETUP arrived reports the close reason: a caller that +/// checks only now must not take over a dead session. +#[moq_net_sim::test] +async fn setup_reports_a_close_after_the_setup() { + for name in LATE { + let (session, request) = dial(version(name)).await; + let (server, driver) = request.ok().await.expect("server accept failed"); + spawn(driver); + assert!(matches!(setup(&session).await, Some(Ok(()))), "{name}"); + + server.abort(Error::Unauthorized); + moq_net_sim::timeout(TIMEOUT, session.closed()) + .await + .expect("the close never arrived"); + let res = session.setup().await; + assert!(res.is_err(), "{name}: a closed session reported SETUP success"); + } +} + +#[moq_net_sim::test] +async fn setup_reports_a_refusal() { + for name in LATE { + let (session, request) = dial(version(name)).await; + request.close(Error::Unauthorized); + let err = setup(&session).await.expect("the refusal never arrived").unwrap_err(); + assert!( + matches!(err, Error::Session(SessionError::Unauthorized)), + "{name}: {err:?}" + ); + } +} + +#[moq_net_sim::test] +async fn setup_resolves_at_once_when_the_handshake_read_the_setup() { + for name in EARLY { + let version = version(name); + let (client, server) = create_mock_session_pair(Some(version.alpn())); + let client = async { + let (session, driver) = Client::new() + .with_versions(version.into()) + .connect(now(), client) + .await + .expect("client handshake failed"); + spawn(driver); + session + }; + let server = async { + let (session, driver) = Server::new() + .with_versions(version.into()) + .accept(now(), server) + .await + .expect("server handshake failed"); + spawn(driver); + session + }; + let (session, _server) = futures::join!(client, server); + let res = session.setup().await; + assert!(res.is_ok(), "{name}: {res:?}"); + } +} + +#[moq_net_sim::test] +async fn setup_is_unsupported_without_a_server_setup() { + for name in NONE { + let version = version(name); + let (client, _server) = create_mock_session_pair(Some(version.alpn())); + let (session, driver) = Client::new() + .with_versions(version.into()) + .connect(now(), client) + .await + .expect("client handshake failed"); + spawn(driver); + let res = session.setup().await; + assert!(matches!(res, Err(Error::Unsupported)), "{name}: {res:?}"); + } +} + +/// Every version is covered by exactly one of the tests above. +#[test] +fn every_version_is_classified() { + let classified: Vec<&str> = LATE.iter().chain(&EARLY).chain(&NONE).copied().collect(); + let mut all: Vec<&str> = Version::names().collect(); + all.sort_unstable(); + let mut sorted = classified.clone(); + sorted.sort_unstable(); + assert_eq!(sorted, all); +} diff --git a/rs/moq-relay/src/auth.rs b/rs/moq-relay/src/auth.rs index 8e85251ad7a..352f856b0da 100644 --- a/rs/moq-relay/src/auth.rs +++ b/rs/moq-relay/src/auth.rs @@ -721,11 +721,12 @@ impl Admission { /// transport knows, nothing parsed on the server's behalf. pub fn request_for(auth: &Auth, request: &moq_tokio::server::Request) -> Request { let transport = match request.transport() { - moq_tokio::server::Transport::Quic => moq_auth::Transport::Quic, - moq_tokio::server::Transport::Iroh => moq_auth::Transport::Iroh, - moq_tokio::server::Transport::WebSocket => moq_auth::Transport::WebSocket, - moq_tokio::server::Transport::Tcp => moq_auth::Transport::Tcp, - moq_tokio::server::Transport::Unix => moq_auth::Transport::Unix, + // The auth contract names QUIC either way; WebTransport is QUIC underneath. + moq_tokio::Transport::Quic | moq_tokio::Transport::WebTransport => moq_auth::Transport::Quic, + moq_tokio::Transport::Iroh => moq_auth::Transport::Iroh, + moq_tokio::Transport::WebSocket => moq_auth::Transport::WebSocket, + moq_tokio::Transport::Tcp => moq_auth::Transport::Tcp, + moq_tokio::Transport::Unix => moq_auth::Transport::Unix, // A transport this build does not know is still a session on the wire; the // server sees the same facts either way. other => unreachable!("unknown transport {other}"), diff --git a/rs/moq-relay/src/uring.rs b/rs/moq-relay/src/uring.rs index 29374092d03..dd2d5607980 100644 --- a/rs/moq-relay/src/uring.rs +++ b/rs/moq-relay/src/uring.rs @@ -778,7 +778,12 @@ async fn serve_connection( } }); - tracing::info!(id, version = %session.version(), transport = %moq_tokio::server::Transport::Quic, "negotiated"); + // Only a WebTransport session carries a URL. + let transport = match url { + Some(_) => moq_tokio::Transport::WebTransport, + None => moq_tokio::Transport::Quic, + }; + tracing::info!(id, version = %session.version(), %transport, "negotiated"); // The session handle is Send + Sync however its transport is driven, so // its lifecycle (credential expiry, GOAWAY drain) lives with the timers diff --git a/rs/moq-tokio/src/client.rs b/rs/moq-tokio/src/client.rs index d27490e56c6..cddad1b063a 100644 --- a/rs/moq-tokio/src/client.rs +++ b/rs/moq-tokio/src/client.rs @@ -6,8 +6,10 @@ use crate::connection::Goaway; use crate::{Addrs, Backoff, Connection, Error}; +use futures::future::BoxFuture; #[cfg(all(feature = "websocket", feature = "noq"))] use std::future::Future; +use std::task::{Poll, ready}; use url::Url; /// Everything a [`Client`] is built from. @@ -313,7 +315,7 @@ impl Client { feature = "tcp", feature = "uds" )))] - pub(crate) async fn dial(&self, _addr: crate::connect::Addr) -> crate::Result { + pub(crate) async fn dial(&self, _addr: crate::connect::Addr) -> crate::Result { Err(Error::NoBackend( "no backend compiled; enable noq, iroh, websocket, tcp, or uds feature", )) @@ -322,7 +324,8 @@ impl Client { /// Dial the given URL and complete the MoQ handshake. /// /// The scheme picks the transport, and `https://` races QUIC against the - /// WebSocket fallback so a blocked UDP path still connects. The session's + /// WebSocket fallback so a blocked UDP path still connects. When the fallback + /// wins, the QUIC dial keeps going as [`Dialed::upgrade`]. The session's /// protocol driver is spawned on the current tokio runtime; the session /// closes once the last returned handle drops. #[cfg(any( @@ -332,7 +335,7 @@ impl Client { feature = "tcp", feature = "uds" ))] - pub(crate) async fn dial(&self, addr: crate::connect::Addr) -> crate::Result { + pub(crate) async fn dial(&self, addr: crate::connect::Addr) -> crate::Result { // Each compiled backend adds state to this dispatch future. Keep it off the // caller's stack so all-feature builds remain safe on standard 2 MiB threads. let attempt = Box::pin(self.connect_inner(addr)); @@ -340,16 +343,23 @@ impl Client { // The deadline covers the dial AND the handshake, for every transport: it is the // only bound some of them have. Dropping `attempt` on expiry cancels whichever // arm was still pending. - let session = match self.timeout.is_zero() { + let dialed = match self.timeout.is_zero() { true => attempt.await?, - false => match tokio::time::timeout(self.timeout, attempt).await { - Ok(res) => res?, - Err(_) => return Err(Error::ConnectTimeout(self.timeout)), - }, + false => { + let deadline = tokio::time::Instant::now() + self.timeout; + let mut dialed = match tokio::time::timeout_at(deadline, attempt).await { + Ok(res) => res?, + Err(_) => return Err(Error::ConnectTimeout(self.timeout)), + }; + // The QUIC arm that lost to WebSocket is still part of this attempt, so the + // same deadline bounds it. + dialed.upgrade = dialed.upgrade.map(|upgrade| upgrade.until(deadline, self.timeout)); + dialed + } }; - tracing::info!(version = %session.version(), "connected"); - Ok(session) + tracing::info!(version = %dialed.session.version(), transport = %dialed.transport, "connected"); + Ok(dialed) } /// The moq client builder, advertising `path` in the SETUP when there is one. @@ -374,7 +384,7 @@ impl Client { feature = "tcp", feature = "uds" ))] - async fn connect_inner(&self, addr: crate::connect::Addr) -> crate::Result { + async fn connect_inner(&self, addr: crate::connect::Addr) -> crate::Result { let url = addr.url().clone(); // Transports with no request URI of their own advertise the request target in the // SETUP instead; `setup_path` returns `None` for the ones that carry a URI, where @@ -393,7 +403,8 @@ impl Client { if url.scheme() == "tcp" { let session = crate::tcp::connect(url, &self.versions.alpns(), self.failover_delay, self.resolution_delay).await?; - return Ok(connect_session(&moq, crate::transport::Session::new(session)).await?); + let session = connect_session(&moq, crate::transport::Session::new(session)).await?; + return Ok(Dialed::new(session, crate::Transport::Tcp)); } // Qmux over TLS over TCP, for links that need neither QUIC nor a WebSocket. @@ -407,7 +418,8 @@ impl Client { self.resolution_delay, ) .await?; - return Ok(connect_session(&moq, crate::transport::Session::new(session)).await?); + let session = connect_session(&moq, crate::transport::Session::new(session)).await?; + return Ok(Dialed::new(session, crate::Transport::Tcp)); } // Unix domain socket (qmux, no TLS). Same-host only; the server can @@ -415,7 +427,8 @@ impl Client { #[cfg(all(feature = "uds", unix))] if url.scheme() == "unix" { let session = crate::unix::connect(url, &self.versions.alpns()).await?; - return Ok(connect_session(&moq, crate::transport::Session::new(session)).await?); + let session = connect_session(&moq, crate::transport::Session::new(session)).await?; + return Ok(Dialed::new(session, crate::Transport::Unix)); } // A WebSocket URL names its transport. No QUIC backend can dial it, so there is @@ -441,19 +454,23 @@ impl Client { crate::iroh::Binding::H3 => self.moq.clone(), }; - return Ok(connect_session(&moq, crate::transport::Session::new(session)).await?); + let session = connect_session(&moq, crate::transport::Session::new(session)).await?; + return Ok(Dialed::new(session, crate::Transport::Iroh)); } #[cfg(feature = "noq")] - if let Some(noq) = self.noq.as_ref() { + if let Some(noq) = self.noq.clone() { + // Owned rather than borrowed from `self`: when WebSocket wins the race, this dial + // outlives the attempt as the pending upgrade. let tls = self.tls.clone(); + let versions = self.versions.clone(); let quic_addr = addr.clone(); - let quic_handle = async { - noq.connect(&tls, quic_addr, &self.versions) + let quic_handle = Box::pin(async move { + noq.connect(&tls, quic_addr, &versions) .await .map(crate::transport::Session::new) .map_err(Error::from) - }; + }); #[cfg(feature = "websocket")] { @@ -463,7 +480,8 @@ impl Client { #[cfg(not(feature = "websocket"))] { let session = quic_handle.await?; - return Ok(connect_session(&moq, session).await?); + let session = connect_session(&moq, session).await?; + return Ok(Dialed::new(session, quic_transport(&url))); } } @@ -477,15 +495,20 @@ impl Client { /// Connect over WebSocket alone. qmux over WebSocket carries the path in its request /// URI, so the plain builder is used: repeating it in the SETUP is a protocol violation. #[cfg(feature = "websocket")] - async fn connect_websocket(&self, addr: crate::connect::Addr) -> crate::Result { + async fn connect_websocket(&self, addr: crate::connect::Addr) -> crate::Result { let alpns = self.versions.alpns(); let session = crate::websocket::connect(&self.websocket, &self.tls, self.tls_host_name.as_deref(), addr, &alpns).await?; - Ok(connect_session(&self.moq, crate::transport::Session::new(session)).await?) + let session = connect_session(&self.moq, crate::transport::Session::new(session)).await?; + Ok(Dialed::new(session, crate::Transport::WebSocket)) } /// Race the QUIC dial against the WebSocket fallback, handshaking whichever wins. /// + /// When WebSocket wins while QUIC is still dialing, the QUIC dial carries on as + /// [`Dialed::upgrade`] rather than being dropped, and the attempt falls back to it + /// if the MoQ handshake over WebSocket fails. + /// /// `moq` is the QUIC-side builder, which carries the SETUP path for a raw QUIC dial. /// The WebSocket fallback uses the plain builder: qmux over WebSocket carries the /// path in its request URI, so repeating it in the SETUP is a protocol violation. @@ -498,11 +521,13 @@ impl Client { moq: &moq_net::Client, addr: crate::connect::Addr, quic: Q, - ) -> crate::Result + ) -> crate::Result where - Q: Future>, + Q: Future> + Unpin + Send + 'static, S: moq_net::transport::poll::Boxable, { + let url = addr.url().clone(); + let transport = quic_transport(&url); let alpns = self.versions.alpns(); let ws_config = self.websocket.clone(); let ws_tls = self.tls.clone(); @@ -514,9 +539,55 @@ impl Client { }; match race_transport_connect(quic, websocket).await? { - TransportRace::Quic(quic) => Ok(connect_session(moq, quic).await?), - TransportRace::WebSocket(websocket) => { - Ok(connect_session(&self.moq, crate::transport::Session::new(websocket)).await?) + TransportRace::Quic(quic) => Ok(Dialed::new(connect_session(moq, quic).await?, transport)), + TransportRace::WebSocket { session, quic } => { + let session = match connect_session(&self.moq, crate::transport::Session::new(session)).await { + Ok(session) => session, + Err(err) => { + // The fallback got through but its MoQ handshake did not. A QUIC dial still + // pending may yet connect, bounded by the same deadline as the race. + let err = Error::from(err); + let quic = match quic { + Ok(dial) => { + tracing::warn!(%err, "WebSocket handshake failed; waiting on QUIC"); + dial.await + } + Err(quic) => Err(quic), + }; + let quic = match quic { + Ok(quic) => quic, + Err(quic) => return Err(race_error(quic, err)), + }; + // UDP gets through after all, so the next dial gives QUIC its head start. + crate::websocket::forget(&url); + // Both handshakes failing is still a two-arm loss: a mixed auth pair stays retryable. + let session = connect_session(moq, quic) + .await + .map_err(|quic| race_error(quic.into(), err))?; + return Ok(Dialed::new(session, transport)); + } + }; + let mut dialed = Dialed::new(session, crate::Transport::WebSocket); + dialed.upgrade = quic.ok().map(|quic| { + let moq = moq.clone(); + let dial = Box::pin(async move { + let quic = quic.await?; + Ok(Box::pin(async move { + let session = connect_session(&moq, quic).await?; + // Wait for the peer's SETUP before draining WebSocket. This crate's servers + // send SETUP after admission; other servers may still refuse afterward. + // A refusal, or a version with no SETUP to wait on, keeps WebSocket. + session.setup().await?; + // A session already told to leave would hand straight back out of QUIC. + if session.draining().peek().is_some() { + return Err(Error::ConnectFailed); + } + Ok(session) + }) as Handshake) + }); + Upgrade::new(dial, transport) + }); + Ok(dialed) } } } @@ -578,6 +649,117 @@ fn setup_path(url: &Url) -> Option { } } +/// What a noq dial to `url` runs on: WebTransport for `https://` (and the `http://` +/// bootstrap), raw QUIC for `moqt://` and `moql://`. +#[cfg(feature = "noq")] +fn quic_transport(url: &Url) -> crate::Transport { + match url.scheme() { + "moqt" | "moql" => crate::Transport::Quic, + _ => crate::Transport::WebTransport, + } +} + +/// A session [`Client::dial`] brought up. +pub(crate) struct Dialed { + pub session: moq_net::Session, + /// What `session` runs on. + pub transport: crate::Transport, + /// The QUIC dial still in flight when the WebSocket fallback won the race. + pub upgrade: Option, +} + +impl Dialed { + #[cfg_attr( + not(any( + feature = "noq", + feature = "iroh", + feature = "websocket", + feature = "tcp", + feature = "uds" + )), + allow(dead_code) + )] + fn new(session: moq_net::Session, transport: crate::Transport) -> Self { + Self { + session, + transport, + upgrade: None, + } + } +} + +/// The MoQ handshake running on an upgrade's QUIC session. +pub(crate) type Handshake = BoxFuture<'static, crate::Result>; + +/// A QUIC dial that lost the race to the WebSocket fallback but is still going. +/// +/// Polled by [`Connection`] alongside the WebSocket session, which it replaces +/// once both stages complete. Dropping it cancels the dial. +pub(crate) struct Upgrade { + stage: Stage, + /// What the upgraded session runs on. + transport: crate::Transport, + /// The connect deadline of the attempt this dial belongs to, and its length for + /// the error. + deadline: Option<(std::pin::Pin>, std::time::Duration)>, +} + +#[cfg_attr(not(all(feature = "websocket", feature = "noq")), allow(dead_code))] +enum Stage { + /// Waiting on the QUIC transport, and the WebTransport CONNECT for `https://`. + Dialing(BoxFuture<'static, crate::Result>), + /// The QUIC transport is up and the MoQ handshake is running on it. + Handshaking(Handshake), +} + +/// How far an [`Upgrade`] got on one poll. +pub(crate) enum Step { + /// The QUIC transport is up; the MoQ handshake has started on it. + Handshaking, + /// The peer's SETUP arrived over QUIC, which is ready to take over on this transport. + Done(moq_net::Session, crate::Transport), +} + +#[cfg_attr(not(all(feature = "websocket", feature = "noq")), allow(dead_code))] +impl Upgrade { + pub(crate) fn new(dial: BoxFuture<'static, crate::Result>, transport: crate::Transport) -> Self { + Self { + stage: Stage::Dialing(dial), + transport, + deadline: None, + } + } + + /// Fail with [`Error::ConnectTimeout`] if still pending at `deadline`. + fn until(mut self, deadline: tokio::time::Instant, timeout: std::time::Duration) -> Self { + self.deadline = Some((Box::pin(tokio::time::sleep_until(deadline)), timeout)); + self + } + + /// Drive the dial: `Ready(Ok(Step::Handshaking))` once when the QUIC transport + /// comes up, then `Ready(Ok(Step::Done))` once the peer's SETUP arrives. Not + /// polled again after `Done` or an error. + pub(crate) fn poll(&mut self, waiter: &moq_net::kio::Waiter) -> Poll> { + if let Some((sleep, timeout)) = &mut self.deadline + && waiter.poll_future(sleep.as_mut()).is_ready() + { + return Poll::Ready(Err(Error::ConnectTimeout(*timeout))); + } + + match &mut self.stage { + Stage::Dialing(dial) => { + let handshake = ready!(waiter.poll_future(dial.as_mut()))?; + self.stage = Stage::Handshaking(handshake); + Poll::Ready(Ok(Step::Handshaking)) + } + Stage::Handshaking(handshake) => { + let session = ready!(waiter.poll_future(handshake.as_mut()))?; + Poll::Ready(Ok(Step::Done(session, self.transport))) + } + } + } +} + /// The URI authority to advertise in the SETUP, chosen by the dial URL's scheme. /// /// A raw-QUIC `moqt://` client MUST send it (draft-ietf-moq-transport-21, 9.1.1). Built @@ -603,19 +785,22 @@ fn setup_authority(url: &Url) -> Option { } #[cfg(all(feature = "websocket", feature = "noq"))] -#[derive(Debug, PartialEq, Eq)] -enum TransportRace { - Quic(Q), - WebSocket(W), +enum TransportRace { + Quic(QT), + /// The fallback won. `quic` is the QUIC dial if it was still pending, or the error + /// it had already failed with. + WebSocket { + session: WT, + quic: crate::Result, + }, } #[cfg(all(feature = "websocket", feature = "noq"))] -async fn race_transport_connect(quic: Q, websocket: W) -> crate::Result> +async fn race_transport_connect(mut quic: Q, websocket: W) -> crate::Result> where - Q: Future>, + Q: Future> + Unpin, W: Future>>, { - tokio::pin!(quic); tokio::pin!(websocket); let mut quic_err = None; @@ -637,7 +822,10 @@ where } res = &mut websocket, if !websocket_done => { match res { - Some(Ok(session)) => return Ok(TransportRace::WebSocket(session)), + Some(Ok(session)) => { + let quic = quic_err.take().map_or(Ok(quic), Err); + return Ok(TransportRace::WebSocket { session, quic }); + } Some(Err(err)) => { tracing::warn!(%err, "WebSocket connection failed"); websocket_err = Some(err); @@ -656,23 +844,30 @@ where } } - // Auth is terminal only when both arms refused. A WebTransport-only endpoint - // answers the fallback with 403 while QUIC is still in flight, and reconnect - // treats is_auth() as terminal, so a mixed pair reports the retryable error. match (quic_err, websocket_err) { - (Some(quic), Some(websocket)) => Err(match (quic.is_auth(), websocket.is_auth()) { - (false, false) => Error::TransportRace { - quic: std::sync::Arc::new(quic), - websocket: std::sync::Arc::new(websocket), - }, - (true, false) => websocket, - _ => quic, - }), + (Some(quic), Some(websocket)) => Err(race_error(quic, websocket)), (Some(err), None) | (None, Some(err)) => Err(err), (None, None) => Err(Error::ConnectFailed), } } +/// The error for a race both arms lost. +/// +/// Auth is terminal only when both arms refused. A WebTransport-only endpoint +/// answers the fallback with 403 while QUIC is still in flight, and reconnect +/// treats is_auth() as terminal, so a mixed pair reports the retryable error. +#[cfg(all(feature = "websocket", feature = "noq"))] +fn race_error(quic: Error, websocket: Error) -> Error { + match (quic.is_auth(), websocket.is_auth()) { + (false, false) => Error::TransportRace { + quic: std::sync::Arc::new(quic), + websocket: std::sync::Arc::new(websocket), + }, + (true, false) => websocket, + _ => quic, + } +} + #[cfg(any( feature = "noq", feature = "iroh", @@ -696,6 +891,26 @@ async fn connect_session( mod tests { use super::*; + /// A QUIC session whose peer never sends SETUP gives up at the connect deadline, so the + /// WebSocket session keeps serving instead of waiting on the upgrade forever. + #[tokio::test(start_paused = true)] + async fn a_stalled_upgrade_gives_up_at_the_deadline() { + const TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + let waiter = moq_net::kio::Waiter::noop(); + let handshake: Handshake = Box::pin(std::future::pending()); + let mut upgrade = Upgrade::new(Box::pin(async move { Ok(handshake) }), crate::Transport::WebTransport) + .until(tokio::time::Instant::now() + TIMEOUT, TIMEOUT); + + assert!(matches!(upgrade.poll(&waiter), Poll::Ready(Ok(Step::Handshaking)))); + assert!(upgrade.poll(&waiter).is_pending()); + + tokio::time::advance(TIMEOUT).await; + assert!(matches!( + upgrade.poll(&waiter), + Poll::Ready(Err(Error::ConnectTimeout(TIMEOUT))) + )); + } + #[cfg(feature = "noq")] #[tokio::test] async fn fixed_target_preserves_request_and_refuses_redirect() { @@ -1266,8 +1481,43 @@ mod tests { Some(Ok(1usize)) }; - let value = super::race_transport_connect(quic, websocket).await.unwrap(); - assert_eq!(value, super::TransportRace::WebSocket(1)); + let value = super::race_transport_connect(Box::pin(quic), websocket).await.unwrap(); + let super::TransportRace::WebSocket { + session: 1, + quic: Err(err), + } = value + else { + panic!("WebSocket won after QUIC failed, and must carry the QUIC error"); + }; + assert!(err.is_auth(), "unexpected error: {err}"); + } + + /// A QUIC failure that lands before WebSocket connects stays with the race, so a + /// later WebSocket handshake failure combines both arms as `race_error` does. + #[cfg(all(feature = "websocket", feature = "noq"))] + #[tokio::test] + async fn race_transport_connect_keeps_an_earlier_quic_error() { + let quic = async { Err::(Error::ConnectFailed) }; + let websocket = async { + tokio::task::yield_now().await; + Some(Ok(1usize)) + }; + + let value = super::race_transport_connect(Box::pin(quic), websocket).await.unwrap(); + let super::TransportRace::WebSocket { + session: 1, + quic: Err(err), + } = value + else { + panic!("WebSocket won after QUIC failed, and must carry the QUIC error"); + }; + assert!(matches!(err, Error::ConnectFailed), "unexpected error: {err}"); + + let combined = super::race_error(err, crate::ConnectError::Unauthorized.into()); + assert!( + !combined.is_auth(), + "mixed auth/non-auth must stay retryable: {combined}" + ); } #[cfg(all(feature = "websocket", feature = "noq"))] @@ -1279,8 +1529,8 @@ mod tests { }; let websocket = async { Some(Err::(crate::ConnectError::Forbidden.into())) }; - let value = super::race_transport_connect(quic, websocket).await.unwrap(); - assert_eq!(value, super::TransportRace::Quic(3)); + let value = super::race_transport_connect(Box::pin(quic), websocket).await.unwrap(); + assert!(matches!(value, super::TransportRace::Quic(3))); } /// A WebTransport-only endpoint answers the WebSocket fallback with 403 while the @@ -1297,7 +1547,7 @@ mod tests { let listener = tokio::net::TcpListener::bind("[::]:0").await.unwrap(); let ws_port = listener.local_addr().unwrap().port(); - let mut forbid = tokio::spawn(async move { + let forbid = tokio::spawn(async move { let (mut stream, _) = listener.accept().await?; let mut buf = [0; 1024]; let _ = stream.read(&mut buf).await?; @@ -1332,19 +1582,20 @@ mod tests { // The same race `connect_inner` runs, except the fallback dials its own port, // as plain ws:// so the listener above can answer without TLS. - let noq = client.noq.as_ref().unwrap(); + let noq = client.noq.clone().unwrap(); + let (tls, versions) = (client.tls.clone(), client.versions.clone()); let quic_addr: crate::connect::Addr = Url::parse(&format!("https://localhost:{quic_port}")).unwrap().into(); let ws_addr: crate::connect::Addr = Url::parse(&format!("http://localhost:{ws_port}")).unwrap().into(); // Hold QUIC until the fallback has been refused, so the 403 is always exercised. - let quic = async { - (&mut forbid).await.unwrap().expect("fallback listener failed"); - noq.connect(&client.tls, quic_addr, &client.versions) + let quic = Box::pin(async move { + forbid.await.unwrap().expect("fallback listener failed"); + noq.connect(&tls, quic_addr, &versions) .await .map(crate::transport::Session::new) .map_err(Error::from) - }; + }); - let session = tokio::time::timeout( + let dialed = tokio::time::timeout( std::time::Duration::from_secs(10), client.race_moq_connect(&client.moq, ws_addr, quic), ) @@ -1352,7 +1603,8 @@ mod tests { .expect("client connect timed out") .expect("a fallback refused on auth must not end a connect whose QUIC arm succeeds"); - drop(session); + assert_eq!(dialed.transport, crate::Transport::WebTransport); + drop(dialed); accepted.await.unwrap().expect("server handshake failed"); } @@ -1413,7 +1665,7 @@ mod tests { let request = server.accept().await.expect("no incoming connection"); assert_eq!( request.transport(), - crate::server::Transport::Quic, + crate::Transport::WebTransport, "expected the QUIC listener", ); let session = request.with_publisher(&pub_origin).ok().await?; @@ -1425,18 +1677,23 @@ mod tests { }); let client = client.with_subscriber(sub_origin); - let noq = client.noq.as_ref().expect("QUIC backend"); - let quic = async { - let session = noq.connect(&client.tls, quic_addr, &client.versions).await?; + let dialer = client.clone(); + let quic = Box::pin(async move { + let noq = dialer.noq.as_ref().expect("QUIC backend"); + let session = noq.connect(&dialer.tls, quic_addr, &dialer.versions).await?; connected.send(()).expect("fallback listener alive"); Ok::<_, crate::Error>(crate::transport::Session::new(session)) - }; + }); let cc = client .race_moq_connect(&client.moq, ws_addr, quic) .await .expect("client connect failed"); - assert_eq!(cc.version(), expected_version, "client negotiated stale version"); + assert_eq!( + cc.session.version(), + expected_version, + "client negotiated stale version" + ); websocket_handle.abort(); let _ = websocket_handle.await; @@ -1476,7 +1733,9 @@ mod tests { let quic = async { Err::(crate::ConnectError::Unauthorized.into()) }; let websocket = async { Some(Err::(crate::ConnectError::Forbidden.into())) }; - let err = super::race_transport_connect(quic, websocket).await.unwrap_err(); + let Err(err) = super::race_transport_connect(Box::pin(quic), websocket).await else { + panic!("the race must fail"); + }; assert!(err.is_auth(), "unexpected error: {err}"); } @@ -1489,7 +1748,9 @@ mod tests { }; let websocket = async { Some(Err::(crate::ConnectError::Forbidden.into())) }; - let err = super::race_transport_connect(quic, websocket).await.unwrap_err(); + let Err(err) = super::race_transport_connect(Box::pin(quic), websocket).await else { + panic!("the race must fail"); + }; assert!(matches!(err, Error::ConnectFailed), "unexpected error: {err}"); assert!(!err.is_auth(), "mixed auth/non-auth must stay retryable: {err}"); } @@ -1500,8 +1761,34 @@ mod tests { let quic = async { Err::(Error::ConnectFailed) }; let websocket = async { Some(Ok(7usize)) }; - let value = super::race_transport_connect(quic, websocket).await.unwrap(); - assert_eq!(value, super::TransportRace::WebSocket(7)); + let value = super::race_transport_connect(Box::pin(quic), websocket).await.unwrap(); + // `select!` may poll either arm first, so the failed QUIC dial can come back unpolled. + assert!(matches!(value, super::TransportRace::WebSocket { session: 7, .. })); + } + + /// WebSocket winning hands back the QUIC dial it beat, still running, so the + /// session can move onto QUIC once it lands. + #[cfg(all(feature = "websocket", feature = "noq"))] + #[tokio::test] + async fn race_transport_connect_keeps_a_pending_quic_dial() { + let (land, landed) = tokio::sync::oneshot::channel::<()>(); + let quic = async move { + landed.await.unwrap(); + Ok("quic") + }; + let websocket = async { Some(Ok("websocket")) }; + + let race = super::race_transport_connect(Box::pin(quic), websocket).await; + let Ok(super::TransportRace::WebSocket { + session: "websocket", + quic: Ok(quic), + }) = race + else { + panic!("WebSocket won, and the QUIC dial it beat must come back with it"); + }; + + land.send(()).unwrap(); + assert_eq!(quic.await.unwrap(), "quic"); } #[cfg(all(feature = "websocket", feature = "noq"))] @@ -1512,12 +1799,12 @@ mod tests { let value = tokio::time::timeout( std::time::Duration::from_secs(1), - super::race_transport_connect(quic, websocket), + super::race_transport_connect(Box::pin(quic), websocket), ) .await .expect("race waited for WebSocket after QUIC transport connected") .unwrap(); - assert_eq!(value, super::TransportRace::Quic("quic")); + assert!(matches!(value, super::TransportRace::Quic("quic"))); } /// The resolved default has to exist in every build, including ones that compile diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index c66afc1d520..5c5d8f555c8 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -175,8 +175,8 @@ pub enum Status { Connected, /// An established session dropped; a reconnect attempt follows. Disconnected, - /// The peer sent a GOAWAY; the replacement is being dialed while the old - /// session keeps serving. + /// A replacement session is coming up while the old one keeps serving: the + /// peer sent a GOAWAY, or a QUIC dial landed after the WebSocket fallback won. Migrating, } @@ -460,6 +460,8 @@ struct State { presence: moq_net::stats::Presence, /// The negotiated MoQ version of the live session, or `None` when disconnected. version: Option, + /// What the live session runs on, or `None` when disconnected. + transport: Option, /// Set when the reconnect loop permanently gives up (reconnect timeout exceeded). error: Option, /// The currently-connected session, or `None` while reconnecting. Read by @@ -486,7 +488,7 @@ struct Shared { impl Shared { /// A session is live and serving. - fn connected(&self, session: &moq_net::Session) { + fn connected(&self, session: &moq_net::Session, transport: crate::Transport) { // Held across the publish: see [`CloseGuard`]. A close that already ran is // honored here rather than leaving this session parked in the final state. let closed = self.closed.lock().unwrap(); @@ -495,19 +497,35 @@ impl Shared { return; } if let Ok(mut state) = self.state.write() { + // A migration replaces a live session without a disconnect in between, so + // its end is counted here, keeping `started - ended` at 1 while connected. + if state.session.is_some() { + state.presence.sessions_ended += 1; + } state.status = Some(Status::Connected); state.epoch += 1; state.presence.sessions_started += 1; state.version = Some(session.version()); + state.transport = Some(transport); state.session = Some(session.clone()); } } - /// The peer sent a GOAWAY: the session in [`State`] keeps serving while its - /// replacement is dialed, so only the status moves. + /// The session in [`State`] keeps serving while its replacement comes up, so + /// only the status moves. fn migrating(&self) { + self.status(Status::Migrating); + } + + /// The replacement did not come up after all: the session in [`State`] was + /// serving throughout, so only the status moves back. + fn stayed(&self) { + self.status(Status::Connected); + } + + fn status(&self, status: Status) { if let Ok(mut state) = self.state.write() { - state.status = Some(Status::Migrating); + state.status = Some(status); } } @@ -523,6 +541,7 @@ impl Shared { } state.status = Some(Status::Disconnected); state.version = None; + state.transport = None; state.session = None; } let _ = self.send_bw.set(None); @@ -777,9 +796,7 @@ impl Connection { let pacing = Pacing::new(&backoff); let timeout = CliDuration::resolve(backoff.timeout_arg, backoff.timeout); let initial = pacing.initial; - let mut delay = initial; - let mut retry_start = tokio::time::Instant::now(); - let mut last_error: Option = None; + let mut retry = Retry::new(initial); // Sticky across migrations: a redirect is an assignment, not a detour, so a // later drop redials wherever we were last sent. Scoped to this loop, so a // fresh Connection starts from the configured addresses again. @@ -788,27 +805,61 @@ impl Connection { let mut draining: Option = None; loop { - if !timeout.is_zero() && retry_start.elapsed() >= timeout { - return Err(timeout_error(timeout, last_error.as_ref())); + if !timeout.is_zero() && retry.start.elapsed() >= timeout { + return Err(timeout_error(timeout, retry.last_error.as_ref())); } - let budget = retry_budget(client.reconnect, retry_start, timeout); + let budget = retry_budget(client.reconnect, retry.start, timeout); match Self::dial_any(shared, &client, &addrs, &mut draining, budget).await { - Ok((addr, session)) => { + Ok((addr, dialed)) => { let url = addr.url().clone(); - tracing::info!(peer = %Endpoint(&url), "connected"); - shared.connected(&session); + tracing::info!(peer = %Endpoint(&url), transport = %dialed.transport, "connected"); + shared.connected(&dialed.session, dialed.transport); + let mut session = dialed.session; + let mut upgrade = dialed.upgrade; - let connected = tokio::time::Instant::now(); + let mut connected = tokio::time::Instant::now(); // Wait for the session to end, forwarding its bandwidth estimates into the // persistent producers meanwhile so consumers track the live stats across the - // connection, and draining any predecessor left over from a migration. - let ended = run_session(shared, &session, &mut draining).await; + // connection, and draining any predecessor left over from a migration. A QUIC + // dial that lands after WebSocket won takes over here without a redial. + let ended = loop { + match run_session(shared, &session, &mut draining, &mut upgrade).await { + Next::Ended(ended) => break ended, + Next::Upgraded(next, transport) => { + tracing::info!(peer = %Endpoint(&url), %transport, "upgraded from WebSocket"); + // UDP gets through after all, so the next dial gives QUIC its head start. + #[cfg(feature = "websocket")] + crate::websocket::forget(&url); + + // The same handover as a peer's GOAWAY, initiated by us: the new session + // is live, and the old one serves until its routes splice over at a group + // boundary or the cap closes it. + let old = std::mem::replace(&mut session, next); + shared.connected(&session, transport); + // An empty URI is legal from either endpoint on every version; the old + // session's driver closes it at the deadline if the peer lingers. + let msg = moq_net::goaway::Goaway::new().with_timeout(goaway.handover); + if let Err(err) = old.drain().send(msg) { + tracing::debug!(%err, "failed to send GOAWAY on the WebSocket session"); + } + if let Some(mut old) = draining.take() { + old.retire(); + } + draining = Some(Draining::new(old, goaway.handover, &shared.state)); + // Score the WebSocket tenure like any session's, so a QUIC session that + // drops right away retries in a fresh window, not one that opened + // before WebSocket served. + retry.settle(connected, initial); + connected = tokio::time::Instant::now(); + } + } + }; // A session that stayed up past the initial backoff is healthy; one that // ended sooner counts as a failed attempt however it ended. - let healthy = connected.elapsed() >= initial; + let healthy = retry.settle(connected, initial); // The connected target owns the policy, including in one-shot mode. A // refused redirect is terminal: the peer is leaving and named somewhere we @@ -864,11 +915,8 @@ impl Connection { // status finds it and honors its handover deadline. shared.migrating(); + // No backoff sleep: a handover off a healthy session is not a failure. if healthy { - delay = initial; - retry_start = tokio::time::Instant::now(); - last_error = None; - // No backoff sleep: a handover off a healthy session is not a failure. continue; } @@ -876,15 +924,15 @@ impl Connection { // failed attempt so two peers bouncing us between them escalate // through backoff and eventually give up. The old session serves // across the sleep, so the redirect loop costs time, not data. - last_error = Some(Error::Reconnect("peer redirected immediately".to_string())); - let Some(wait) = retry_wait(delay, retry_start, timeout) else { - return Err(timeout_error(timeout, last_error.as_ref())); + retry.last_error = Some(Error::Reconnect("peer redirected immediately".to_string())); + let Some(wait) = retry_wait(retry.delay, retry.start, timeout) else { + return Err(timeout_error(timeout, retry.last_error.as_ref())); }; tracing::warn!(peer = %Endpoint(&url), ?wait, "peer redirected immediately; retrying after backoff"); // Keep the handover bounded across the sleep: nothing else polls the // predecessor while the loop is between connections. sleep_draining(wait, &mut draining, shared).await; - delay = pacing.next(delay); + retry.delay = pacing.next(retry.delay); continue; } @@ -912,11 +960,8 @@ impl Connection { } if healthy { - // Reset the backoff window so a one-off drop reconnects promptly. + // The backoff window was reset, so a one-off drop reconnects promptly. tracing::warn!(peer = %Endpoint(&url), "session closed, reconnecting"); - delay = initial; - retry_start = tokio::time::Instant::now(); - last_error = None; } else { // Connected then dropped almost immediately (e.g. the server accepts then // resets, or redirects us straight back out). Treat it as a failed @@ -940,7 +985,7 @@ impl Connection { match err { Some(err) => { tracing::warn!(peer = %Endpoint(&url), %err, "session severed immediately, retrying"); - last_error = Some(err); + retry.last_error = Some(err); } None => tracing::warn!(peer = %Endpoint(&url), "session severed immediately, retrying"), } @@ -955,12 +1000,12 @@ impl Connection { { return Err(err); } - last_error = Some(err); + retry.last_error = Some(err); } } - let Some(wait) = retry_wait(delay, retry_start, timeout) else { - return Err(timeout_error(timeout, last_error.as_ref())); + let Some(wait) = retry_wait(retry.delay, retry.start, timeout) else { + return Err(timeout_error(timeout, retry.last_error.as_ref())); }; // No URL here: with several candidates there isn't one to name, and each // attempt already logged the address it tried. @@ -971,7 +1016,7 @@ impl Connection { // until some later dial succeeded, holding an upstream that asked to drain // open for the whole retry window, or forever with `--backoff-timeout=0`. sleep_draining(wait, &mut draining, shared).await; - delay = pacing.next(delay); + retry.delay = pacing.next(retry.delay); } } @@ -992,7 +1037,7 @@ impl Connection { addrs: &Addrs, draining: &mut Option, budget: Option, - ) -> crate::Result<(crate::connect::Addr, moq_net::Session)> { + ) -> crate::Result<(crate::connect::Addr, crate::client::Dialed)> { let candidates = addrs.as_slice(); let mut last = None; @@ -1024,7 +1069,7 @@ impl Connection { }; match dialed { - Ok(session) => return Ok((addr.clone(), session)), + Ok(dialed) => return Ok((addr.clone(), dialed)), // A status the peer actually sent is its answer, not this address's, so // unless it invites another attempt it settles the whole walk. Carrying // on would offer the same rejected credentials at the peer's other @@ -1135,6 +1180,14 @@ impl Connection { self.state.read().version } + /// What the live session runs on, or `None` while disconnected. + /// + /// Changes without a disconnect when a session that came up over the WebSocket + /// fallback moves onto QUIC. + pub fn transport(&self) -> Option { + self.state.read().transport + } + /// Observe a GOAWAY from the peer, or `None` while between sessions. /// /// The reconnect loop reads the same GOAWAY to drive migration, reported as @@ -1195,6 +1248,36 @@ impl Connection { } } +/// The reconnect loop's backoff window across a run of failed attempts. +struct Retry { + /// The wait before the next attempt, before jitter. + delay: Duration, + /// When this run of failures began; the give-up timeout counts from here. + start: tokio::time::Instant, + /// The latest failure, so giving up reports a real cause. + last_error: Option, +} + +impl Retry { + fn new(initial: Duration) -> Self { + Self { + delay: initial, + start: tokio::time::Instant::now(), + last_error: None, + } + } + + /// Score a session live since `connected`: one that outlived `initial` was healthy, + /// so the window starts over. Returns whether it was healthy. + fn settle(&mut self, connected: tokio::time::Instant, initial: Duration) -> bool { + let healthy = connected.elapsed() >= initial; + if healthy { + *self = Self::new(initial); + } + healthy + } +} + /// Build the terminal error for an exhausted retry window without discarding the last real cause. fn timeout_error(timeout: Duration, last_error: Option<&Error>) -> Error { let message = match last_error { @@ -1213,6 +1296,15 @@ fn retry_wait(delay: Duration, retry_start: tokio::time::Instant, timeout: Durat Some(wait.min(timeout.checked_sub(retry_start.elapsed())?)) } +/// What [`run_session`] returns on. +enum Next { + /// The session stopped being the live one. + Ended(Ended), + /// The pending QUIC dial finished its handshake; this session, on this + /// transport, replaces the WebSocket one. + Upgraded(moq_net::Session, crate::Transport), +} + /// Why a session stopped being the live one. enum Ended { /// The transport closed. @@ -1326,9 +1418,18 @@ async fn sleep_draining(delay: Duration, draining: &mut Option, shared /// /// One `poll_*` step drives it all: [`poll_forward`] mirrors each kio bandwidth estimate, the /// GOAWAY consumer is a kio channel, and the transport's close future (the one non-kio source) is -/// polled through the waiter's own waker. `migrate` is whether a GOAWAY ends this session's -/// tenure ([`Ended::Goaway`]); when false the arm isn't polled and only a close returns. -async fn run_session(shared: &Shared, session: &moq_net::Session, draining: &mut Option) -> Ended { +/// polled through the waiter's own waker. +/// +/// `upgrade` is a QUIC dial still in flight after the WebSocket fallback won. It reports +/// [`Status::Migrating`] once the QUIC transport is up and returns [`Next::Upgraded`] once +/// the peer's SETUP arrives. A failure leaves this session serving. It is dropped with +/// the session otherwise: the redial races QUIC again. +async fn run_session( + shared: &Shared, + session: &moq_net::Session, + draining: &mut Option, + upgrade: &mut Option, +) -> Next { let mut send = session.send_bandwidth(); let mut recv = session.recv_bandwidth(); let goaway = session.draining(); @@ -1352,14 +1453,43 @@ async fn run_session(shared: &Shared, session: &moq_net::Session, draining: &mut // sides end up waiting on each other forever. The caller ends the connection // instead: asked to leave, with no way to migrate, leaving is the answer. if let Poll::Ready(Ok(msg)) = goaway.poll(waiter) { - return Poll::Ready(Ended::Goaway(msg)); + return Poll::Ready(Next::Ended(Ended::Goaway(msg))); + } + + if let Poll::Ready(err) = waiter.poll_future(closed.as_mut()) { + return Poll::Ready(Next::Ended(Ended::Closed(Err(err)))); } - waiter.poll_future(closed.as_mut()).map(|err| Ended::Closed(Err(err))) + poll_upgrade(shared, upgrade, waiter).map(|(session, transport)| Next::Upgraded(session, transport)) }) .await } +/// Drive a pending upgrade, `Ready` with the QUIC session once the peer's SETUP arrives. +fn poll_upgrade( + shared: &Shared, + upgrade: &mut Option, + waiter: &kio::Waiter, +) -> Poll<(moq_net::Session, crate::Transport)> { + while let Some(pending) = upgrade.as_mut() { + match ready!(pending.poll(waiter)) { + Ok(crate::client::Step::Handshaking) => shared.migrating(), + Ok(crate::client::Step::Done(session, transport)) => { + *upgrade = None; + return Poll::Ready((session, transport)); + } + Err(err) => { + // Only the Handshaking stage moved the status, but restoring it is harmless + // either way: this session was serving throughout. + tracing::debug!(%err, "QUIC upgrade failed; staying on WebSocket"); + shared.stayed(); + *upgrade = None; + } + } + } + Poll::Pending +} + /// Mirror `bw`'s live estimate into `out` for as long as it changes, dropping the source handle once /// the session's producer is gone so we don't keep polling a dead arm. A `poll_*` step: on return, /// `waiter` is registered for the next change (unless the source is gone). Seeding is implicit @@ -1806,6 +1936,25 @@ mod tests { assert_eq!(pacing.next(Duration::from_millis(400)), Duration::from_millis(400)); } + /// A WebSocket session that served for longer than the retry window before QUIC + /// took over leaves no stale window behind: a QUIC session that drops at once is + /// a failed attempt, but one that still gets retried. + #[tokio::test(start_paused = true)] + async fn a_healthy_tenure_before_an_upgrade_starts_a_fresh_window() { + const INITIAL: Duration = Duration::from_secs(1); + const TIMEOUT: Duration = Duration::from_secs(10); + let mut retry = Retry::new(INITIAL); + + let websocket = tokio::time::Instant::now(); + tokio::time::advance(TIMEOUT * 2).await; + assert!(retry.settle(websocket, INITIAL)); + + let quic = tokio::time::Instant::now(); + tokio::time::advance(INITIAL / 2).await; + assert!(!retry.settle(quic, INITIAL)); + assert!(retry_wait(retry.delay, retry.start, TIMEOUT).is_some()); + } + #[test] fn test_backoff_default() { let backoff = Backoff::default(); @@ -2089,6 +2238,489 @@ mod tests { assert_eq!(attempt_timeout(2, 3), None, "the last of three"); } + /// A QUIC server with the WebSocket fallback on the same port number, the QUIC + /// half reached through a [`Forwarder`], publishing `origin`. + /// + /// Yields each accepted session with the transport it arrived on. + #[cfg(all(feature = "websocket", feature = "noq"))] + struct Fallback { + url: Url, + forwarder: Forwarder, + accepted: tokio::sync::mpsc::UnboundedReceiver<(crate::Transport, moq_net::Session)>, + /// QUIC requests held unanswered in [`Quic::Refused`] mode. + held: tokio::sync::mpsc::UnboundedReceiver, + } + + /// When a [`Fallback`] lets QUIC through. + #[cfg(all(feature = "websocket", feature = "noq"))] + #[derive(Clone, Copy, PartialEq)] + enum Quic { + /// From the start. + Open, + /// Once the test calls [`Forwarder::open`]. + Held, + /// Once the WebSocket fallback has closed its first session mid-handshake. No + /// MoQ server sits behind the fallback in this mode. + AfterWebSocketFails, + /// Once the test calls [`Forwarder::open`], like `Held`, and the server leaves + /// each QUIC request for the test to answer, as a session limit would refuse it. + Refused, + } + + #[cfg(all(feature = "websocket", feature = "noq"))] + impl Fallback { + async fn start(origin: &moq_net::origin::Producer, quic: Quic) -> Self { + let mut listen = crate::listen::Config { + bind: Some("127.0.0.1:0".parse().unwrap()), + ..Default::default() + }; + listen.tls.generate = vec!["localhost".into()]; + + // The fallback dials the URL's port over TCP, so the forwarder takes the same + // number over UDP. Nothing reserves the pair, so retry on a collision. + let (websocket, forwarder) = 'bind: { + for _ in 0..20 { + let websocket = crate::websocket::Listener::bind("127.0.0.1:0".parse().unwrap()) + .await + .unwrap(); + let port = websocket.local_addr().unwrap().port(); + if let Ok(front) = tokio::net::UdpSocket::bind(("127.0.0.1", port)).await { + break 'bind (websocket, front); + } + } + panic!("could not bind a matching TCP and UDP port after 20 attempts"); + }; + + // With no MoQ server behind it, the fallback's session gets as far as the MoQ + // handshake and no further. + let (websocket, failing) = match quic { + Quic::AfterWebSocketFails => (None, Some(websocket)), + _ => (Some(websocket), None), + }; + + let config = crate::server::Config { + listen, + websocket, + publisher: Some(origin.consume()), + ..Default::default() + }; + let mut server = config.init().unwrap().listen().await.unwrap(); + let server_addr = server.local_addr().unwrap(); + let port = forwarder.local_addr().unwrap().port(); + let forwarder = Forwarder::start(forwarder, server_addr, quic == Quic::Open).await; + + if let Some(websocket) = failing { + let gate = forwarder.clone(); + tokio::spawn(async move { + use web_transport_trait::Session as _; + let session = websocket.accept().await.unwrap().unwrap(); + // The client's SETUP stream: WebSocket has won the race. + let _setup = session.accept_bi().await.unwrap(); + session.close(1, "no MoQ here"); + gate.open(); + session.closed().await; + }); + } + + let (tx, accepted) = tokio::sync::mpsc::unbounded_channel(); + let (hold, held) = tokio::sync::mpsc::unbounded_channel(); + tokio::spawn(async move { + while let Some(request) = server.accept().await { + let transport = request.transport(); + if quic == Quic::Refused && transport != crate::Transport::WebSocket { + let _ = hold.send(request); + continue; + } + if let Ok(session) = request.ok().await { + let _ = tx.send((transport, session)); + } + } + }); + + Self { + url: url(&format!("http://127.0.0.1:{port}/")), + forwarder, + accepted, + held, + } + } + + /// The next QUIC request held in [`Quic::Refused`] mode. + async fn held(&mut self) -> crate::server::Request { + tokio::time::timeout(UPGRADE_WAIT, self.held.recv()) + .await + .expect("the QUIC request never reached the server") + .expect("the server stopped") + } + + async fn accept(&mut self) -> (crate::Transport, moq_net::Session) { + tokio::time::timeout(UPGRADE_WAIT, self.accepted.recv()) + .await + .expect("the server never accepted a session") + .expect("the server stopped") + } + } + + /// Relays QUIC datagrams to `server`, holding them all until [`Forwarder::open`]. + /// + /// Holding is what makes WebSocket win the race without depending on timing: the + /// QUIC dial cannot finish until the test says so. + #[cfg(all(feature = "websocket", feature = "noq"))] + #[derive(Clone)] + struct Forwarder { + gate: std::sync::Arc>, + } + + #[cfg(all(feature = "websocket", feature = "noq"))] + impl Forwarder { + /// `open` is whether to forward from the start rather than hold. + async fn start(front: tokio::net::UdpSocket, server: std::net::SocketAddr, open: bool) -> Self { + use std::sync::Arc; + + let back = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap(); + back.connect(server).await.unwrap(); + let (front, back) = (Arc::new(front), Arc::new(back)); + let (tx, rx) = tokio::sync::watch::channel(open); + let (client_tx, client_rx) = tokio::sync::watch::channel(None); + + // Each datagram waits for the gate on its own task. + fn relay(gate: &tokio::sync::watch::Receiver, send: impl Future + Send + 'static) { + let mut gate = gate.clone(); + tokio::spawn(async move { + if gate.wait_for(|open| *open).await.is_ok() { + send.await; + } + }); + } + + { + let (front, back, gate) = (front.clone(), back.clone(), rx.clone()); + tokio::spawn(async move { + let mut buf = vec![0; 65536]; + while let Ok((len, from)) = front.recv_from(&mut buf).await { + client_tx.send_replace(Some(from)); + let packet = buf[..len].to_vec(); + let back = back.clone(); + relay(&gate, async move { + let _ = back.send(&packet).await; + }); + } + }); + } + + tokio::spawn(async move { + let mut buf = vec![0; 65536]; + while let Ok(len) = back.recv(&mut buf).await { + let Some(client) = *client_rx.borrow() else { continue }; + let packet = buf[..len].to_vec(); + let front = front.clone(); + relay(&rx, async move { + let _ = front.send_to(&packet, client).await; + }); + } + }); + + Self { gate: tx.into() } + } + + /// Start forwarding, releasing whatever was held. + fn open(&self) { + self.gate.send_replace(true); + } + } + + #[cfg(all(feature = "websocket", feature = "noq"))] + const UPGRADE_WAIT: Duration = Duration::from_secs(10); + + /// The sequence of the next group `sub` receives. + #[cfg(all(feature = "websocket", feature = "noq"))] + async fn next_group(sub: &mut moq_net::track::Subscriber) -> u64 { + let group = tokio::time::timeout(UPGRADE_WAIT, sub.recv_group()) + .await + .expect("no group arrived") + .unwrap() + .expect("the track ended"); + group.sequence + } + + /// The upgrade reports [`Status::Migrating`] while the MoQ handshake runs on the + /// QUIC session, and a failure there puts the status back: the WebSocket session + /// served throughout. + /// + /// Pinned here rather than end to end, since a moq-lite client completes its + /// handshake without waiting on the server, leaving no window to observe. + #[test] + fn a_failed_upgrade_stays_connected() { + let shared = shared(); + shared.status(Status::Connected); + let waiter = kio::Waiter::noop(); + + let (fail, failed) = tokio::sync::oneshot::channel::<()>(); + let handshake: crate::client::Handshake = Box::pin(async move { + let _ = failed.await; + Err(Error::ConnectFailed) + }); + let mut upgrade = Some(crate::client::Upgrade::new( + Box::pin(async move { Ok(handshake) }), + crate::Transport::WebTransport, + )); + + assert!(poll_upgrade(&shared, &mut upgrade, &waiter).is_pending()); + assert_eq!(shared.state.consume().read().status, Some(Status::Migrating)); + assert!(upgrade.is_some()); + + fail.send(()).unwrap(); + assert!(poll_upgrade(&shared, &mut upgrade, &waiter).is_pending()); + assert_eq!(shared.state.consume().read().status, Some(Status::Connected)); + assert!(upgrade.is_none(), "a failed upgrade is not retried"); + } + + /// A session that came up over the WebSocket fallback moves onto QUIC once the + /// QUIC dial lands, without dropping a group, and forgets that WebSocket won. + #[cfg(all(feature = "websocket", feature = "noq"))] + #[tracing_test::traced_test] + #[tokio::test] + async fn websocket_upgrades_to_quic() { + const HANDOVER: Duration = Duration::from_millis(500); + + let origin = crate::origin::spawn(); + let broadcast = origin.create_broadcast("cam").unwrap(); + broadcast.announce(Default::default()).unwrap(); + let track = broadcast.create_track("video", None).unwrap(); + let write = |payload: &'static [u8]| { + let mut group = track.append_group().unwrap(); + group.write_frame(moq_net::Timestamp::ZERO, payload).unwrap(); + group.finish().unwrap(); + }; + + let mut fallback = Fallback::start(&origin, Quic::Held).await; + + let subscriber = crate::origin::spawn(); + let mut config = crate::connect::Config::default(); + config.tls.insecure = Some(true); + config.goaway.handover = HANDOVER; + let client = config + .init(Default::default()) + .unwrap() + .with_subscriber(subscriber.clone()); + let connection = tokio::time::timeout(UPGRADE_WAIT, client.connect(fallback.url.clone()).established()) + .await + .expect("never connected") + .unwrap(); + + // QUIC is held, so the fallback won. + assert_eq!(connection.transport(), Some(crate::Transport::WebSocket)); + assert!( + crate::websocket::won(&fallback.url), + "the fallback's win was not remembered" + ); + let mut monitor = connection.monitor(); + let (transport, websocket) = fallback.accept().await; + assert_eq!(transport, crate::Transport::WebSocket); + + let cam = tokio::time::timeout(UPGRADE_WAIT, subscriber.consume().routed_broadcast("cam")) + .await + .unwrap() + .unwrap(); + let mut sub = cam.track("video").unwrap().subscribe(None).await.unwrap(); + write(b"g0"); + assert_eq!(next_group(&mut sub).await, 0); + + // Let QUIC through, and publish while the replacement comes up. + fallback.forwarder.open(); + write(b"g1"); + + // The swap is a new session on the same handle, with no disconnect between. + let presence = tokio::time::timeout(UPGRADE_WAIT, async { + loop { + let presence = monitor.presence_changed().await.unwrap(); + if presence.sessions_started == 2 { + return presence; + } + } + }) + .await + .expect("never upgraded"); + assert_eq!( + presence.sessions_ended, 1, + "the WebSocket session was not counted as ended" + ); + assert!(connection.connected()); + assert_eq!(connection.transport(), Some(crate::Transport::WebTransport)); + assert_eq!(connection.epoch(), 2); + let (transport, _quic) = fallback.accept().await; + assert_eq!(transport, crate::Transport::WebTransport); + // QUIC works on this network, so the next dial gives it the head start again. + assert!( + !crate::websocket::won(&fallback.url), + "the upgrade kept WebSocket's win" + ); + + // Every group arrives exactly once across the swap: the one written while QUIC + // came up and the one after it. + write(b"g2"); + assert_eq!(next_group(&mut sub).await, 1); + assert_eq!(next_group(&mut sub).await, 2); + + // The WebSocket session is told to leave, and closes within the handover cap + // rather than lingering alongside QUIC. + let goaway = websocket.draining(); + let goaway = tokio::time::timeout(UPGRADE_WAIT, kio::wait(|waiter| goaway.poll(waiter))) + .await + .expect("the WebSocket session never received a GOAWAY") + .unwrap(); + assert_eq!(goaway.uri(), "", "a client may not redirect its server"); + tokio::time::timeout(UPGRADE_WAIT, websocket.closed()) + .await + .expect("the WebSocket session outlived the handover cap"); + assert_eq!(connection.transport(), Some(crate::Transport::WebTransport)); + // The GOAWAY got through, so its sender has nothing to warn about. + assert!(!logs_contain("failed to send goaway")); + } + + /// A QUIC session the server refuses after its transport came up, as a session + /// limit would while the WebSocket session holds the slot, leaves the WebSocket + /// session serving: no GOAWAY, no disconnect, and the next group arrives over it. + #[cfg(all(feature = "websocket", feature = "noq"))] + #[tokio::test] + async fn a_refused_upgrade_keeps_websocket() { + let origin = crate::origin::spawn(); + let broadcast = origin.create_broadcast("cam").unwrap(); + broadcast.announce(Default::default()).unwrap(); + let track = broadcast.create_track("video", None).unwrap(); + let write = |payload: &'static [u8]| { + let mut group = track.append_group().unwrap(); + group.write_frame(moq_net::Timestamp::ZERO, payload).unwrap(); + group.finish().unwrap(); + }; + + let mut fallback = Fallback::start(&origin, Quic::Refused).await; + + let subscriber = crate::origin::spawn(); + let mut config = crate::connect::Config::default(); + config.tls.insecure = Some(true); + let client = config + .init(Default::default()) + .unwrap() + .with_subscriber(subscriber.clone()); + let mut connection = tokio::time::timeout(UPGRADE_WAIT, client.connect(fallback.url.clone()).established()) + .await + .expect("never connected") + .unwrap(); + assert_eq!(connection.transport(), Some(crate::Transport::WebSocket)); + assert_eq!(connection.status().await.unwrap(), Status::Connected); + let (_, websocket) = fallback.accept().await; + + let cam = tokio::time::timeout(UPGRADE_WAIT, subscriber.consume().routed_broadcast("cam")) + .await + .unwrap() + .unwrap(); + let mut sub = cam.track("video").unwrap().subscribe(None).await.unwrap(); + write(b"g0"); + assert_eq!(next_group(&mut sub).await, 0); + + // The QUIC transport comes up and its MoQ handshake reaches the server, which + // holds the request: the WebSocket session must not be drained yet. + fallback.forwarder.open(); + let request = fallback.held().await; + assert_eq!(connection.transport(), Some(crate::Transport::WebSocket)); + assert!(!connection.connected(), "swapped onto QUIC before it was admitted"); + let status = tokio::time::timeout(UPGRADE_WAIT, connection.status()).await.unwrap(); + assert_eq!(status.unwrap(), Status::Migrating); + assert!( + websocket.draining().peek().is_none(), + "drained before QUIC was admitted" + ); + + request.reject(crate::server::Reject::Forbidden).await.unwrap(); + let status = tokio::time::timeout(UPGRADE_WAIT, connection.status()).await.unwrap(); + assert_eq!(status.unwrap(), Status::Connected); + assert_eq!(connection.transport(), Some(crate::Transport::WebSocket)); + assert_eq!(connection.epoch(), 1); + assert!( + websocket.draining().peek().is_none(), + "the refused upgrade drained WebSocket" + ); + + write(b"g1"); + assert_eq!(next_group(&mut sub).await, 1); + } + + /// When WebSocket wins the race but its MoQ handshake fails, the attempt falls back + /// to the QUIC dial still pending instead of failing. + /// + /// One-shot, so a redial cannot stand in for the fallback. Draft 16, since it is the + /// newest version whose client handshake waits on the server's SETUP. + #[cfg(all(feature = "websocket", feature = "noq"))] + #[tokio::test] + async fn failed_websocket_handshake_falls_back_to_quic() { + let origin = crate::origin::spawn(); + let mut fallback = Fallback::start(&origin, Quic::AfterWebSocketFails).await; + + let mut config = crate::connect::Config::default(); + config.tls.insecure = Some(true); + config.version = vec!["moq-transport-16".parse().unwrap()]; + let client = config.init(Default::default()).unwrap().with_reconnect(false); + let connection = tokio::time::timeout(UPGRADE_WAIT, client.connect(fallback.url.clone()).established()) + .await + .expect("never connected") + .expect("a failed WebSocket handshake must fall back to the pending QUIC dial"); + + assert_eq!(connection.transport(), Some(crate::Transport::WebTransport)); + assert_eq!(connection.epoch(), 1); + let (transport, _session) = fallback.accept().await; + assert_eq!(transport, crate::Transport::WebTransport); + // QUIC works on this network, so the next dial gives it the head start again. + assert!( + !crate::websocket::won(&fallback.url), + "the fallback kept WebSocket's win" + ); + } + + /// When QUIC wins the race nothing changes: one session, over QUIC, and no + /// WebSocket session is ever opened. + #[cfg(all(feature = "websocket", feature = "noq"))] + #[tokio::test] + async fn quic_winning_opens_one_session() { + let origin = crate::origin::spawn(); + let broadcast = origin.create_broadcast("cam").unwrap(); + broadcast.announce(Default::default()).unwrap(); + let track = broadcast.create_track("video", None).unwrap(); + + let mut fallback = Fallback::start(&origin, Quic::Open).await; + + let subscriber = crate::origin::spawn(); + let mut config = crate::connect::Config::default(); + config.tls.insecure = Some(true); + let client = config + .init(Default::default()) + .unwrap() + .with_subscriber(subscriber.clone()); + let connection = tokio::time::timeout(UPGRADE_WAIT, client.connect(fallback.url.clone()).established()) + .await + .expect("never connected") + .unwrap(); + assert_eq!(connection.transport(), Some(crate::Transport::WebTransport)); + let (transport, _session) = fallback.accept().await; + assert_eq!(transport, crate::Transport::WebTransport); + + let cam = tokio::time::timeout(UPGRADE_WAIT, subscriber.consume().routed_broadcast("cam")) + .await + .unwrap() + .unwrap(); + let mut sub = cam.track("video").unwrap().subscribe(None).await.unwrap(); + let mut group = track.append_group().unwrap(); + group.write_frame(moq_net::Timestamp::ZERO, b"g0".as_ref()).unwrap(); + group.finish().unwrap(); + assert_eq!(next_group(&mut sub).await, 0); + + // QUIC's win dropped the fallback before it dialed, so there is nothing else to + // accept and nothing to upgrade from. + assert!(fallback.accepted.try_recv().is_err(), "a second session was opened"); + assert_eq!(connection.epoch(), 1); + assert!(!crate::websocket::won(&fallback.url)); + } + /// A stream-only server on a free loopback port, publishing `origin`. /// /// Returns its address, a receiver yielding each accepted session (so the test diff --git a/rs/moq-tokio/src/lib.rs b/rs/moq-tokio/src/lib.rs index e5bea556af3..9bf366fd5a8 100644 --- a/rs/moq-tokio/src/lib.rs +++ b/rs/moq-tokio/src/lib.rs @@ -75,6 +75,7 @@ pub use error::{Error, Result}; pub use log::{Log, RedactedUrl}; #[cfg(feature = "_transport")] pub use server::{Listener, Server}; +pub use transport::Transport; // Re-export these crates. pub use moq_net; diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index 0e4ab43fa2e..96191f91fd5 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -8,9 +8,9 @@ use std::net; #[cfg(any(test, all(feature = "uds", unix)))] use std::path::PathBuf; -use crate::Error; #[cfg(feature = "iroh")] use crate::iroh; +use crate::{Error, Transport}; use moq_net::Session; use url::Url; @@ -650,7 +650,9 @@ impl Server { let Accepted { session, url, identity, authority, mut link } = within(deadline, super::noq::accept(_conn, alpns)).await?; link.local = local; let (request, deadline) = setup.accept(session, deadline).await?; - Ok(Request { transport: Transport::Quic, url, identity, authority, link, kind: RequestKind::Noq(Box::new(request)), deadline }) + // Only WebTransport carries a request URL; raw QUIC puts the path in the SETUP. + let transport = match url { Some(_) => Transport::WebTransport, None => Transport::Quic }; + Ok(Request { transport, url, identity, authority, link, kind: RequestKind::Noq(Box::new(request)), deadline }) }.boxed()); } } @@ -1397,41 +1399,6 @@ pub struct Link { pub alpn: Option, } -/// The network transport carrying an incoming MoQ session. -#[non_exhaustive] -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] -pub enum Transport { - /// QUIC, either directly or through WebTransport over HTTP/3. - Quic, - /// An Iroh QUIC connection. - Iroh, - /// A WebSocket connection using qmux framing. - WebSocket, - /// A TCP connection using qmux framing, plaintext or TLS. - Tcp, - /// A Unix domain socket using qmux framing. - Unix, -} - -impl Transport { - /// Returns the stable lowercase name used in logs and external metadata. - pub const fn as_str(self) -> &'static str { - match self { - Self::Quic => "quic", - Self::Iroh => "iroh", - Self::WebSocket => "websocket", - Self::Tcp => "tcp", - Self::Unix => "unix", - } - } -} - -impl std::fmt::Display for Transport { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.as_str()) - } -} - /// An incoming MoQ session that can be accepted or rejected. /// /// The transport connection and the MoQ SETUP are already complete, so [`path`](Self::path), @@ -2413,6 +2380,7 @@ mod tests { assert_eq!(Transport::WebSocket.as_str(), "websocket"); assert_eq!(Transport::Tcp.as_str(), "tcp"); assert_eq!(Transport::Unix.as_str(), "unix"); + assert_eq!(Transport::WebTransport.as_str(), "webtransport"); } /// Building the endpoint needs a runtime, and `certificates()` must stay diff --git a/rs/moq-tokio/src/tcp.rs b/rs/moq-tokio/src/tcp.rs index 853dc883b8e..60ba4259713 100644 --- a/rs/moq-tokio/src/tcp.rs +++ b/rs/moq-tokio/src/tcp.rs @@ -384,7 +384,7 @@ mod tests { let url: Url = format!("tls://localhost:{port}/room?jwt=credential").parse().unwrap(); let accept = tokio::spawn(async move { let request = server.accept().await.unwrap(); - assert_eq!(request.transport(), crate::server::Transport::Tcp); + assert_eq!(request.transport(), crate::Transport::Tcp); assert_eq!(request.path(), "/room"); assert_eq!(request.query(), Some("jwt=credential")); let session = request.ok().await.unwrap(); diff --git a/rs/moq-tokio/src/transport.rs b/rs/moq-tokio/src/transport.rs index 47eea1a4b60..2409547210c 100644 --- a/rs/moq-tokio/src/transport.rs +++ b/rs/moq-tokio/src/transport.rs @@ -7,6 +7,8 @@ //! closed-watch emulation is weaker than a native implementation (see //! [`Session`]), so a backend that implements the poll interface itself is //! handed to moq-net directly. +//! +//! It also names the [`Transport`] a session runs on, for the accept and dial sides alike. use std::task::{Context, Poll, ready}; @@ -14,6 +16,44 @@ use bytes::Bytes; use futures::FutureExt; use web_transport_trait::poll as wt_poll; +/// The network transport carrying a MoQ session, on either side of it. +#[non_exhaustive] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum Transport { + /// Raw QUIC, negotiating a MoQ ALPN directly. + Quic, + /// An Iroh QUIC connection. + Iroh, + /// A WebSocket connection using qmux framing. + WebSocket, + /// A plaintext TCP connection using qmux framing. + Tcp, + /// A Unix domain socket using qmux framing. + Unix, + /// WebTransport over HTTP/3 on QUIC. + WebTransport, +} + +impl Transport { + /// Returns the stable lowercase name used in logs and external metadata. + pub const fn as_str(self) -> &'static str { + match self { + Self::Quic => "quic", + Self::Iroh => "iroh", + Self::WebSocket => "websocket", + Self::Tcp => "tcp", + Self::Unix => "unix", + Self::WebTransport => "webtransport", + } + } +} + +impl std::fmt::Display for Transport { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + /// A stored in-flight operation future. Native transports are Send, so the /// plain boxed flavor suffices. type OpBox = futures::future::BoxFuture<'static, T>; diff --git a/rs/moq-tokio/src/websocket.rs b/rs/moq-tokio/src/websocket.rs index 950b6289e50..71af8e0fbf9 100644 --- a/rs/moq-tokio/src/websocket.rs +++ b/rs/moq-tokio/src/websocket.rs @@ -3,7 +3,8 @@ //! Used when QUIC is unreachable: UDP blocked by a firewall, a proxy in the way, a //! network that only passes TCP/443. The client races this against QUIC and gives QUIC //! a small head start ([`Config::delay`]), so WebSocket only wins when QUIC can't get -//! through. Servers accept it on a separate TCP port via [`Listener`]. +//! through. A [`crate::Connection`] that lands on WebSocket keeps the QUIC dial going and +//! moves onto it if it completes. Servers accept it on a separate TCP port via [`Listener`]. use qmux::ws::tokio_tungstenite; use qmux::ws::tokio_tungstenite::tungstenite::{self, client::IntoClientRequest, http}; @@ -106,6 +107,32 @@ type Result = std::result::Result; // Track servers (hostname:port) where WebSocket won the race, so we won't give QUIC a headstart next time static WEBSOCKET_WON: LazyLock>> = LazyLock::new(|| Mutex::new(HashSet::new())); +/// The [`WEBSOCKET_WON`] key for a dial URL: the host and port the fallback dials. +fn won_key(url: &Url) -> Result<(String, u16)> { + let host = url.host_str().ok_or(Error::MissingHostname)?.to_string(); + let port = url.port().unwrap_or_else(|| match url.scheme() { + "https" | "wss" | "moql" | "moqt" => 443, + "http" | "ws" => 80, + _ => 443, + }); + Ok((host, port)) +} + +/// Forget that WebSocket won for `url`, giving QUIC its head start again. +/// +/// Called once a QUIC dial to `url` lands after all, which proves UDP gets through. +pub(crate) fn forget(url: &Url) { + if let Ok(key) = won_key(url) { + WEBSOCKET_WON.lock().unwrap().remove(&key); + } +} + +/// Whether the next dial to `url` skips QUIC's head start. +#[cfg(all(test, feature = "noq"))] +pub(crate) fn won(url: &Url) -> bool { + won_key(url).is_ok_and(|key| WEBSOCKET_WON.lock().unwrap().contains(&key)) +} + /// WebSocket configuration for the client. #[derive(Clone, Debug, usage::Args, serde::Serialize, serde::Deserialize)] #[usage(unknown_flags = "error", args_override_self = false)] @@ -278,13 +305,8 @@ pub(crate) async fn connect( return Err(Error::Disabled); } - let host = url.host_str().ok_or(Error::MissingHostname)?.to_string(); - let port = url.port().unwrap_or_else(|| match url.scheme() { - "https" | "wss" | "moql" | "moqt" => 443, - "http" | "ws" => 80, - _ => 443, - }); - let key = (host.clone(), port); + let key = won_key(&url)?; + let (host, port) = key.clone(); // Apply a small penalty to WebSocket to improve odds for QUIC to connect first, // unless we've already had to fall back to WebSockets for this server. diff --git a/rs/moq-tokio/tests/backend.rs b/rs/moq-tokio/tests/backend.rs index 3338fbdaa57..b5d55b5c777 100644 --- a/rs/moq-tokio/tests/backend.rs +++ b/rs/moq-tokio/tests/backend.rs @@ -675,7 +675,7 @@ async fn iroh_connect_test(version: Option<&str>) { assert_eq!(request.role(), Some(moq_tokio::moq_net::Role::Subscriber)); // iroh offers the moq ALPNs ahead of H3, so this lands on raw QUIC: no request // URL, leaving the SETUP as the only place for the request target. - assert_eq!(request.transport(), moq_tokio::server::Transport::Iroh); + assert_eq!(request.transport(), moq_tokio::Transport::Iroh); assert_eq!(request.url(), None); assert_eq!(request.path(), "/room"); assert_eq!(request.query(), Some("jwt=abc")); diff --git a/rs/moq-tokio/tests/broadcast.rs b/rs/moq-tokio/tests/broadcast.rs index 4b286695580..3d5a8d0400b 100644 --- a/rs/moq-tokio/tests/broadcast.rs +++ b/rs/moq-tokio/tests/broadcast.rs @@ -2091,7 +2091,7 @@ async fn broadcast_websocket() { // ── run server and client concurrently ────────────────────────── let server_handle = tokio::spawn(async move { let request = server.accept().await.expect("no incoming connection"); - assert_eq!(request.transport(), moq_tokio::server::Transport::WebSocket); + assert_eq!(request.transport(), moq_tokio::Transport::WebSocket); assert_eq!(request.path(), ""); // The dialed host reaches the server as the authority, like the QUIC transports. assert_eq!(request.authority(), Some("localhost")); @@ -2213,7 +2213,7 @@ async fn broadcast_websocket_fallback() { // ── run server and client concurrently ────────────────────────── let server_handle = tokio::spawn(async move { let request = server.accept().await.expect("no incoming connection"); - assert_eq!(request.transport(), moq_tokio::server::Transport::WebSocket); + assert_eq!(request.transport(), moq_tokio::Transport::WebSocket); assert_eq!(request.path(), "/admin"); assert_eq!(request.query(), Some("jwt=test")); assert_eq!(request.url().and_then(url::Url::query), Some("jwt=test")); @@ -2330,7 +2330,7 @@ async fn broadcast_websocket_uses_newest_version() { let server_handle = tokio::spawn(async move { let request = server.accept().await.expect("no incoming connection"); - assert_eq!(request.transport(), moq_tokio::server::Transport::WebSocket); + assert_eq!(request.transport(), moq_tokio::Transport::WebSocket); let session = request.with_publisher(&pub_origin).ok().await?; assert_eq!(session.version(), expected_version, "server negotiated stale version"); let _broadcast = broadcast;