From dee1710d0c8ea3f47d59f942af74343810df72f3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 05:21:52 -0700 Subject: [PATCH 1/6] chore(quest): claim quest/m2/one-port/tcp-demux.md Quest-Claim: 1b57637f4a6c0ab8 From 424625e5c0e2e4a3ecf215ace3e568f4ba79e80a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 05:39:25 -0700 Subject: [PATCH 2/6] feat(relay): split web connections into HTTP and RTMP by their first bytes Each public web connection is classified on its own task before HTTP: a plaintext C0 (0x03) is RTMP, a TLS connection with no ALPN is sniffed again on its first decrypted byte (RTMPS), and everything else reaches the router. RTMP goes to an embedder through Relay::rtmp / Web::rtmp; untaken, it is logged and closed. The tls:// arm stays in the quest pending its config gate. Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/index.md | 4 + quest/m2/one-port/README.md | 2 +- quest/m2/one-port/tcp-demux.md | 60 ++---- quest/m2/rate-websocket.md | 2 +- rs/moq-relay/src/listener.rs | 13 ++ rs/moq-relay/src/listener/demux.rs | 149 ++++++++++++++ rs/moq-relay/src/relay.rs | 6 + rs/moq-relay/src/web.rs | 317 +++++++++++++++++------------ rs/moq-relay/src/web/rtmp.rs | 80 ++++++++ 9 files changed, 463 insertions(+), 170 deletions(-) create mode 100644 rs/moq-relay/src/listener/demux.rs create mode 100644 rs/moq-relay/src/web/rtmp.rs diff --git a/doc/bin/relay/index.md b/doc/bin/relay/index.md index 79af9ecd559..2551a588057 100644 --- a/doc/bin/relay/index.md +++ b/doc/bin/relay/index.md @@ -99,6 +99,10 @@ relay.with_web(web).run().await?; send its path in the SETUP. - To decide admissions yourself, leave `[auth]` empty and answer `relay.admissions()`; see [Authentication](/bin/relay/auth#in-process). +- To serve RTMP on the web ports, take `relay.rtmp()` and pass each accepted + stream to `moq_rtmp::accept_stream`. The HTTP and HTTPS listeners hand it + plaintext RTMP, and RTMPS (TLS with no ALPN) already decrypted; untaken, the + relay closes them. See [docs.rs/moq-relay](https://docs.rs/moq-relay) and [`rs/moq-relay/examples/embed.rs`](https://github.com/moq-dev/moq/blob/main/rs/moq-relay/examples/embed.rs). diff --git a/quest/m2/one-port/README.md b/quest/m2/one-port/README.md index 862fb1b9c6a..476e82c13a2 100644 --- a/quest/m2/one-port/README.md +++ b/quest/m2/one-port/README.md @@ -96,7 +96,7 @@ pre-accepted streams can stand behind. ## Required - [WebRTC on the shared socket](/quest/m2/one-port/rtc-feed.md) - `moq-rtc` serves WHIP and WHEP media from the WebRTC hook and pins ICE tuples -- [TCP acceptor](/quest/m2/one-port/tcp-demux.md) - one listener carries TLS-terminated HTTP, RTMP, and RTMPS +- [TCP acceptor: tls:// on the web port](/quest/m2/one-port/tcp-demux.md) - the HTTPS listener also carries `tls://` qmux beside HTTP, RTMP, and RTMPS - [SRT on the shared socket](/quest/m2/one-port/srt-demux.md) - moq-srt drives `srt-protocol` on demuxed packets and the flow table pins its 4-tuples ## Related diff --git a/quest/m2/one-port/tcp-demux.md b/quest/m2/one-port/tcp-demux.md index c3f8da62751..d4c8921ea21 100644 --- a/quest/m2/one-port/tcp-demux.md +++ b/quest/m2/one-port/tcp-demux.md @@ -1,41 +1,24 @@ -# [M] TCP acceptor +# [M] TCP acceptor: tls:// on the web port ## Goal -`moq-relay`'s listener accepts TLS-terminated HTTP, RTMP, and RTMPS on one -TCP port and yields classified connections, so an embedder routes RTMP to -`moq-rtmp` and everything else reaches the axum router as today. The relay -binary has no RTMP consumer and gains none here: it refuses a classified -RTMP connection with a log line, and the embedder that carries RTMP today -(moq.pro's edge) is the consumer. +The relay's HTTPS listener also carries `tls://` qmux, so one TCP port serves +TLS-terminated HTTP, RTMP, RTMPS, and `tls://` MoQ. The HTTP, RTMP, and RTMPS +split already landed: `listener::classify` peeks each connection on its own +task, and an embedder takes RTMP from `Relay::rtmp`. ## Plan -`moq-relay/src/listener.rs` grows the acceptor: peek the first byte; 0x16 is -TLS, accepted with the served identity, then the first decrypted byte is -peeked; 0x03 is RTMP, anything else is HTTP. A plaintext 0x03 is RTMP; a -plaintext ASCII method is HTTP. The acceptor yields -`Accepted::{Http(stream), Rtmp(stream)}` where the stream is a boxed -`AsyncRead + AsyncWrite` that may already be TLS. HTTP connections are pushed -into a channel that stands behind `axum_server::Server::from_listener`, so -the router and its ALPN list are untouched. RTMP connections are the -embedder's to hand to `moq_rtmp::Server::accept_stream`; upstream nothing -consumes them, so the relay logs and drops them unless a consumer is wired. - -The accept loop never peeks or handshakes. Each accepted socket is spawned -onto its own task with a bounded read timeout; that task peeks, optionally -terminates TLS, and yields `Accepted`. An idle or slow client stalls only -its own task, so a peer that connects and sends nothing cannot block later -accepts. The peek is non-destructive on `TcpStream` and on the rustls -stream alike. Reuse the handshake bound `listen.timeout` already sets -(`moq_tokio::listen::Config::timeout`) rather than a new knob. - -The TLS arm also carries `tls://` qmux, served today by its own `listen.tcp` -listener with TLS. It negotiates a moq ALPN in the TLS handshake, so split -the decrypted arm on ALPN: a moq ALPN goes to the qmux server and an HTTP -ALPN to the router. A connection with no ALPN keeps the decrypted-byte -sniff, since RTMPS clients negotiate none (`moq-rtmp` documents an empty -ALPN list): `0x03` is RTMP, anything else is HTTP. +`tls://` is served today by its own `listen.tcp` listener with TLS. It +negotiates a moq ALPN in the TLS handshake, so split the decrypted arm on ALPN: +a moq ALPN goes to the qmux server and an HTTP ALPN to the router. A connection +with no ALPN keeps the decrypted-byte sniff, since RTMPS clients negotiate none. +Selecting the server config from the ClientHello (rustls' `LazyConfigAcceptor`) +lets the moq ALPN serve the listen TLS identity while HTTP keeps +`web.https.cert`. + +Open: how an operator turns this on, and what becomes of the separate +`listen.tcp` TLS bind. Asked in the PR that landed the HTTP and RTMP split. Decided 2026-10-08: under one acceptor, the client-certificate request from [mTLS on tls://](/quest/m2/tls-listener-mtls.md) is optional (never required) @@ -43,14 +26,13 @@ and sits on the moq-ALPN TLS config only, so browsers and RTMPS clients on the same port are never asked for one. Whichever lands second keeps `tls://` client-certificate verification and the `MtlsUnsupported` lift working. -Keep the accepted `TcpStream`'s handle reachable after it is boxed: the -relay already captures socket stats at accept for qmux (`SocketStats` in -`web.rs`), and [WebSocket bitrate caps](/quest/m2/rate-websocket.md) set -socket buffer sizes on the same socket. +Keep the accepted `TcpStream` reachable (`listener::Stream::tcp`): qmux +reads socket stats from it, and +[WebSocket bitrate caps](/quest/m2/rate-websocket.md) set socket buffer sizes +on the same socket. -Tests: an HTTP request, a WebSocket upgrade, a raw RTMP C0, an RTMPS C0 with -no ALPN, and a `tls://` qmux client against one listener each land in the -right arm. +Tests: a `tls://` qmux client against the HTTPS listener lands in the qmux +server, beside the HTTP and RTMP cases already covered. ## Related diff --git a/quest/m2/rate-websocket.md b/quest/m2/rate-websocket.md index 1f792155cbb..6a977d0a5f9 100644 --- a/quest/m2/rate-websocket.md +++ b/quest/m2/rate-websocket.md @@ -51,4 +51,4 @@ live session applies. ## Related -- [TCP acceptor](/quest/m2/one-port/tcp-demux.md) - boxes the accepted stream, so it must keep the socket handle these buffer sizes need +- [TCP acceptor: tls:// on the web port](/quest/m2/one-port/tcp-demux.md) - wraps the accepted stream; `listener::Stream::tcp` keeps the socket these buffer sizes need diff --git a/rs/moq-relay/src/listener.rs b/rs/moq-relay/src/listener.rs index 88f4258c31e..1f4d8249b36 100644 --- a/rs/moq-relay/src/listener.rs +++ b/rs/moq-relay/src/listener.rs @@ -14,7 +14,9 @@ //! back. Because the wrapper never yields an error upward, `axum_server`'s flat //! sleep is unreachable. +mod demux; mod idle; +pub(crate) use demux::{Classified, Stream, classify}; pub(crate) use idle::Admission; use std::{ @@ -35,6 +37,17 @@ pub(crate) struct Accepted { pub(crate) admission: Option, } +#[cfg(all(test, unix, feature = "websocket"))] +impl Accepted { + /// A socket accepted outside [`Listener`], holding no admission. + pub(crate) fn new(stream: TcpStream) -> Self { + Self { + stream, + admission: None, + } + } +} + impl AsyncRead for Accepted { fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { Pin::new(&mut self.stream).poll_read(cx, buf) diff --git a/rs/moq-relay/src/listener/demux.rs b/rs/moq-relay/src/listener/demux.rs new file mode 100644 index 00000000000..12c934831b2 --- /dev/null +++ b/rs/moq-relay/src/listener/demux.rs @@ -0,0 +1,149 @@ +//! Splits a public web connection into HTTP and RTMP by its first bytes. +//! +//! `axum_server` runs the acceptor on each connection's own task, so a peer that +//! connects and sends nothing stalls only itself, under the same idle deadline +//! that bounds its TLS handshake and first request. + +use std::{ + io, + pin::Pin, + task::{Context, Poll}, +}; + +use axum_server::{accept::Accept, tls_rustls::RustlsAcceptor}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, ReadBuf}; +use tokio::net::TcpStream; +use tokio_rustls::server::TlsStream; + +use super::Accepted; + +/// The RTMP version byte (C0) an RTMP handshake opens with. +const RTMP_VERSION: u8 = 0x03; + +/// A classified connection: the protocol its first bytes named. +pub(crate) enum Classified { + Http(Stream), + Rtmp(Stream), +} + +/// Classify `accepted`, terminating TLS with `tls` when the listener serves it. +/// +/// A plaintext C0 is RTMP on either listener. On a TLS listener anything else is +/// handed to rustls, which refuses whatever is not a ClientHello (0x16), so +/// plaintext HTTP is still refused there. A negotiated ALPN names HTTP; RTMPS +/// clients offer none, so a connection without one is sniffed again on its first +/// decrypted byte. +pub(crate) async fn classify(accepted: Accepted, tls: Option<&RustlsAcceptor>) -> io::Result { + let mut first = [0u8; 1]; + if accepted.stream.peek(&mut first).await? == 0 { + return Err(io::ErrorKind::UnexpectedEof.into()); + } + let Some(tls) = tls.filter(|_| first[0] != RTMP_VERSION) else { + let stream = Stream { + transport: Transport::Plain(accepted), + peeked: None, + }; + return Ok(match first[0] { + RTMP_VERSION => Classified::Rtmp(stream), + _ => Classified::Http(stream), + }); + }; + + let (mut tls, ()) = tls.accept(accepted, ()).await?; + if tls.get_ref().1.alpn_protocol().is_some() { + return Ok(Classified::Http(Stream { + transport: Transport::Tls(Box::new(tls)), + peeked: None, + })); + } + let first = tls.read_u8().await?; + let stream = Stream { + transport: Transport::Tls(Box::new(tls)), + peeked: Some(first), + }; + Ok(match first { + RTMP_VERSION => Classified::Rtmp(stream), + _ => Classified::Http(stream), + }) +} + +/// An accepted connection after classification, plaintext or with TLS terminated. +pub(crate) struct Stream { + transport: Transport, + /// The decrypted byte read to classify the connection, replayed to the first read. + peeked: Option, +} + +enum Transport { + Plain(Accepted), + Tls(Box>), +} + +impl Stream { + fn accepted(&self) -> &Accepted { + match &self.transport { + Transport::Plain(accepted) => accepted, + Transport::Tls(tls) => tls.get_ref().0, + } + } + + /// The accepted socket, for socket options and kernel statistics. + pub(crate) fn tcp(&self) -> &TcpStream { + &self.accepted().stream + } + + /// The TLS session, when the connection carried one. + pub(crate) fn tls(&self) -> Option<&rustls::ServerConnection> { + match &self.transport { + Transport::Plain(_) => None, + Transport::Tls(tls) => Some(tls.get_ref().1), + } + } + + pub(crate) fn admission(&self) -> Option { + self.accepted().admission.clone() + } +} + +impl AsyncRead for Stream { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + if buf.remaining() > 0 + && let Some(byte) = self.peeked.take() + { + buf.put_slice(&[byte]); + return Poll::Ready(Ok(())); + } + match &mut self.transport { + Transport::Plain(accepted) => Pin::new(accepted).poll_read(cx, buf), + Transport::Tls(tls) => Pin::new(tls.as_mut()).poll_read(cx, buf), + } + } +} + +impl AsyncWrite for Stream { + fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { + match &mut self.transport { + Transport::Plain(accepted) => Pin::new(accepted).poll_write(cx, buf), + Transport::Tls(tls) => Pin::new(tls.as_mut()).poll_write(cx, buf), + } + } + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match &mut self.transport { + Transport::Plain(accepted) => Pin::new(accepted).poll_flush(cx), + Transport::Tls(tls) => Pin::new(tls.as_mut()).poll_flush(cx), + } + } + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match &mut self.transport { + Transport::Plain(accepted) => Pin::new(accepted).poll_shutdown(cx), + Transport::Tls(tls) => Pin::new(tls.as_mut()).poll_shutdown(cx), + } + } +} + +#[cfg(unix)] +impl std::os::fd::AsFd for Stream { + fn as_fd(&self) -> std::os::fd::BorrowedFd<'_> { + std::os::fd::AsFd::as_fd(self.tcp()) + } +} diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index 08e5b3a1b43..6489f975a4c 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -446,6 +446,12 @@ impl Relay { self.admissions.take() } + /// The RTMP and RTMPS connections the public web listeners classify, for the + /// embedder to serve; see [`web::Web::rtmp`]. Untaken, they are logged and closed. + pub fn rtmp(&mut self) -> web::rtmp::Incoming { + self.web.rtmp() + } + /// The shared cluster: the origin every session and peer publishes into. pub fn cluster(&self) -> &cluster::Cluster { &self.cluster diff --git a/rs/moq-relay/src/web.rs b/rs/moq-relay/src/web.rs index b6878ad8c4b..1ccd0d60576 100644 --- a/rs/moq-relay/src/web.rs +++ b/rs/moq-relay/src/web.rs @@ -15,17 +15,17 @@ use axum::{ routing::get, }; use axum_server::{ - accept::{Accept, DefaultAcceptor}, + accept::Accept, tls_rustls::{RustlsAcceptor, RustlsConfig}, }; use bytes::Bytes; use futures::{FutureExt, future::BoxFuture}; -use tokio::io::{AsyncRead, AsyncWrite}; -use tokio_rustls::server::TlsStream; use tower_service::Service; use crate::{auth, cluster}; +pub mod rtmp; + /// Configuration for the HTTP/HTTPS web server. #[derive(usage_rs::Args, Clone, Debug, serde::Deserialize, serde::Serialize)] #[usage(unknown_flags = "error", args_override_self = false)] @@ -209,6 +209,7 @@ pub struct Web { https_tls: Option, /// The ACME certificate resolver HTTPS serves instead of `https.cert`/`https.key`. acme: Option>, + rtmp: rtmp::Handoff, addrs: Addrs, } @@ -241,6 +242,7 @@ impl Web { https_listener: None, https_tls: None, acme: None, + rtmp: rtmp::Handoff::default(), addrs: Addrs::default(), } } @@ -298,6 +300,18 @@ impl Web { self.https_listener.as_ref() } + /// Take the RTMP and RTMPS connections the HTTP and HTTPS listeners classify. + /// + /// RTMP opens with a byte no HTTP request or TLS ClientHello starts with, and an + /// RTMPS client negotiates no ALPN, so both share the web ports. Until this is + /// called, or once the returned consumer is dropped, they are logged and closed. + /// A later call replaces the earlier consumer. + pub fn rtmp(&mut self) -> rtmp::Incoming { + let (handoff, incoming) = rtmp::Handoff::new(); + self.rtmp = handoff; + incoming + } + /// The actual bound addresses after [`bind`](Self::bind) or [`crate::Relay::load`]. pub fn addrs(&self) -> Addrs { self.addrs @@ -440,6 +454,7 @@ impl Web { https_listener, https_tls, acme, + rtmp, .. } = self.bind()?; let app = app @@ -459,7 +474,7 @@ impl Web { timeout: state.timeout, activity: Some(state.shutdown.activity.clone()), }, - WebAcceptor::plain(ws), + WebAcceptor::plain(ws, rtmp.clone()), )?; server.http_builder().http2().max_concurrent_streams(streams); Some(server.serve(app.clone())) @@ -488,7 +503,7 @@ impl Web { timeout: state.timeout, activity: Some(state.shutdown.activity.clone()), }, - WebAcceptor::tls(rustls_config, ws), + WebAcceptor::tls(rustls_config, ws, rtmp), )?; server.http_builder().http2().max_concurrent_streams(streams); Some(server.serve(app)) @@ -669,152 +684,76 @@ pub(crate) struct ServerName(pub(crate) String); /// Accepts a connection on a public web listener. /// -/// Captures the socket, then hands the stream to `inner`: [`DefaultAcceptor`] for -/// `http://`, or [`RustlsAcceptor`] for `https://`, which also gives the connection -/// a peer certificate to report. Both listeners wrap this same acceptor, so the -/// capture cannot reach one and miss the other. +/// Classifies the connection (terminating TLS on `https://`), hands RTMP to +/// [`rtmp::Incoming`], and captures the socket of an HTTP one. Both listeners +/// wrap this same acceptor, so the capture cannot reach one and miss the other. #[derive(Clone)] -struct WebAcceptor { - inner: A, +struct WebAcceptor { + /// Present on `https://`, which also gives the connection a peer certificate to report. + tls: Option, /// Whether a WebSocket route exists to use the handle. The capture holds a /// duplicated descriptor for the life of the connection, so doing it when /// nothing can read it would cost every API, HLS, and health-check connection a /// second descriptor for nothing. ws: bool, + rtmp: rtmp::Handoff, } -impl WebAcceptor { +impl WebAcceptor { /// Accepts plain HTTP, which has no handshake of its own to run. - fn plain(ws: bool) -> Self { - Self { - inner: DefaultAcceptor::new(), - ws, - } + fn plain(ws: bool, rtmp: rtmp::Handoff) -> Self { + Self { tls: None, ws, rtmp } } -} -impl WebAcceptor> { /// Accepts HTTPS, running the TLS handshake and surfacing a verified peer /// certificate as [`MtlsPeer`]. /// /// When no client CA is configured the inner verifier is `NoClientAuth` and /// `peer_certificates()` always returns None, so the marker never appears. - fn tls(config: RustlsConfig, ws: bool) -> Self { + fn tls(config: RustlsConfig, ws: bool, rtmp: rtmp::Handoff) -> Self { Self { - inner: RustlsAcceptor::new(config), + tls: Some(RustlsAcceptor::new(config)), ws, + rtmp, } } } -/// HTTP protocol and peer metadata available after a connection finishes TLS. -/// -/// Which streams can present a client certificate is a property of the stream type, -/// not of the acceptor: only rustls has one to report, and a plain TCP connection -/// never does. -trait HttpStream { - /// The marker for this connection's verified peer certificate. - fn mtls_peer(&self) -> Option; - /// The SNI that selected the certificate, when the connection carried TLS. - fn server_name(&self) -> Option<&str> { - None - } - fn admission(&self) -> Option { - None - } - fn is_http2(&self) -> bool { - false - } - /// Whether this connection is an ACME TLS-ALPN-01 validation, which carries no HTTP. - fn acme_challenge(&self) -> bool { - false - } -} - -impl HttpStream for tokio::net::TcpStream { - fn mtls_peer(&self) -> Option { - None - } -} - -impl HttpStream for crate::listener::Accepted { - fn mtls_peer(&self) -> Option { - None - } - fn admission(&self) -> Option { - self.admission.clone() - } -} - -impl HttpStream for TlsStream { - fn admission(&self) -> Option { - self.get_ref().0.admission() - } - fn is_http2(&self) -> bool { - self.get_ref().1.alpn_protocol() == Some(b"h2") - } - fn acme_challenge(&self) -> bool { - self.get_ref().1.alpn_protocol() == Some(b"acme-tls/1") - } - fn server_name(&self) -> Option<&str> { - self.get_ref().1.server_name() - } - fn mtls_peer(&self) -> Option { - self.get_ref() - .1 - .peer_certificates() - .filter(|certs| !certs.is_empty()) - .map(|certs| MtlsPeer(moq_tokio::tls::PeerIdentity::from_chain(certs.to_vec()))) - } -} - -/// The connection [`WebAcceptor`] is handed: a byte stream, plus a descriptor to -/// read socket statistics from on platforms that expose them. -/// -/// The stream owns the accepted TCP socket, so the extra bound narrows nothing; -/// it exists because a `cfg` can't be written on a `where` clause directly. -#[cfg(any(target_os = "linux", target_os = "macos"))] -pub(crate) trait AcceptStream: AsyncRead + AsyncWrite + Unpin + Send + std::os::fd::AsFd + 'static {} -#[cfg(any(target_os = "linux", target_os = "macos"))] -impl AcceptStream for T {} - -/// As above, where kernel TCP statistics are unavailable. -#[cfg(not(any(target_os = "linux", target_os = "macos")))] -pub(crate) trait AcceptStream: AsyncRead + AsyncWrite + Unpin + Send + 'static {} -#[cfg(not(any(target_os = "linux", target_os = "macos")))] -impl AcceptStream for T {} - -impl Accept for WebAcceptor -where - I: AcceptStream, - S: Send + 'static, - A: Accept, - A::Stream: HttpStream + Send + 'static, - A::Service: Send + 'static, - A::Future: Send + 'static, -{ - type Stream = A::Stream; - type Service = SetConnectionExtensions; +impl Accept for WebAcceptor { + type Stream = crate::listener::Stream; + type Service = SetConnectionExtensions; type Future = BoxFuture<'static, std::io::Result<(Self::Stream, Self::Service)>>; - fn accept(&self, stream: I, service: S) -> Self::Future { - // Read the socket here, while it is still a plain `TcpStream`. Once TLS and - // then hyper's upgrade have wrapped it there is no descriptor left to find. - let socket = self.ws.then(|| socket_stats(&stream)).flatten(); - let inner = self.inner.accept(stream, service); + fn accept(&self, stream: crate::listener::Accepted, service: S) -> Self::Future { + let this = self.clone(); async move { - let (stream, service) = inner.await?; - if stream.acme_challenge() { + let stream = match crate::listener::classify(stream, this.tls.as_ref()).await? { + crate::listener::Classified::Http(stream) => stream, + crate::listener::Classified::Rtmp(stream) => { + this.rtmp.send(stream).await; + return Err(std::io::Error::other("handed off an RTMP connection")); + } + }; + let tls = stream.tls(); + if tls.is_some_and(|tls| tls.alpn_protocol() == Some(b"acme-tls/1")) { // The validator checked the certificate during the handshake; RFC 8737 has the server close. return Err(std::io::Error::other("answered an ACME TLS-ALPN-01 challenge")); } - let peer = stream.mtls_peer(); - let server_name = stream.server_name().map(|name| ServerName(name.to_owned())); + let peer = tls + .and_then(|tls| tls.peer_certificates()) + .filter(|certs| !certs.is_empty()) + .map(|certs| MtlsPeer(moq_tokio::tls::PeerIdentity::from_chain(certs.to_vec()))); + let server_name = tls + .and_then(|tls| tls.server_name()) + .map(|name| ServerName(name.to_owned())); let admission = stream.admission(); - if stream.is_http2() { + if tls.is_some_and(|tls| tls.alpn_protocol() == Some(b"h2")) { // Extended CONNECT is not enabled, so h2 cannot become a WebSocket. drop(admission.as_ref().and_then(crate::listener::Admission::take)); } + // Read the socket now: once hyper's upgrade has wrapped the stream there is + // no descriptor left to find. + let socket = this.ws.then(|| socket_stats(stream.tcp())).flatten(); Ok(( stream, SetConnectionExtensions { @@ -833,14 +772,14 @@ where /// The kernel's view of `stream`, when this platform exposes one and there is a /// qmux session that could use it. #[cfg(all(any(target_os = "linux", target_os = "macos"), feature = "websocket"))] -fn socket_stats(stream: &I) -> Option { +fn socket_stats(stream: &tokio::net::TcpStream) -> Option { qmux::TcpStats::new(stream) .ok() .map(|s| SocketStats(std::sync::Arc::new(s))) } #[cfg(not(all(any(target_os = "linux", target_os = "macos"), feature = "websocket")))] -fn socket_stats(_stream: &I) -> Option { +fn socket_stats(_stream: &tokio::net::TcpStream) -> Option { None } @@ -1216,6 +1155,7 @@ mod tests { use rcgen::{CertificateParams, KeyPair}; use std::io::Write; use tempfile::TempDir; + use tokio::io::{AsyncRead, AsyncWrite}; #[tokio::test(start_paused = true)] async fn http_refusals_count_at_both_endpoints() { @@ -1736,9 +1676,13 @@ mod tests { let addr = listener.local_addr().unwrap(); let connecting = tokio::spawn(async move { tokio::net::TcpStream::connect(addr).await.unwrap() }); let (accepted, _) = listener.accept().await.unwrap(); - let _client = connecting.await.unwrap(); + let mut client = connecting.await.unwrap(); + // The acceptor waits for a first byte to classify the connection. + tokio::io::AsyncWriteExt::write_all(&mut client, b"GET").await.unwrap(); - let (stream, mut service) = Accept::<_, EchoSocket>::accept(&WebAcceptor::plain(true), accepted, EchoSocket) + let acceptor = WebAcceptor::plain(true, rtmp::Handoff::default()); + let accepted = crate::listener::Accepted::new(accepted); + let (stream, mut service) = Accept::<_, EchoSocket>::accept(&acceptor, accepted, EchoSocket) .await .unwrap(); @@ -1781,11 +1725,12 @@ mod tests { let addr = listener.local_addr().unwrap(); let connecting = tokio::spawn(async move { tokio::net::TcpStream::connect(addr).await.unwrap() }); let (accepted, _) = listener.accept().await.unwrap(); - let _client = connecting.await.unwrap(); + let mut client = connecting.await.unwrap(); + tokio::io::AsyncWriteExt::write_all(&mut client, b"GET").await.unwrap(); - let (stream, service) = Accept::<_, ()>::accept(&WebAcceptor::plain(false), accepted, ()) - .await - .unwrap(); + let acceptor = WebAcceptor::plain(false, rtmp::Handoff::default()); + let accepted = crate::listener::Accepted::new(accepted); + let (stream, service) = Accept::<_, ()>::accept(&acceptor, accepted, ()).await.unwrap(); assert!( service.socket.is_none(), @@ -1987,6 +1932,120 @@ mod tests { serving.abort(); } + /// Each web listener splits its connections by their first bytes: HTTP reaches the + /// router, while plaintext RTMP and RTMPS (TLS with no ALPN) reach the RTMP + /// consumer with the classifying byte intact. Untaken, RTMP is closed. + #[tokio::test] + async fn web_listeners_hand_rtmp_to_its_consumer() { + use rustls::pki_types::{CertificateDer, pem::PemObject}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + trait Io: AsyncRead + AsyncWrite + Unpin + Send + 'static {} + impl Io for T {} + + let dir = TempDir::new().unwrap(); + let (ca, cert, key) = make_certs(&dir); + let mut roots = rustls::RootCertStore::empty(); + for cert in CertificateDer::pem_slice_iter(&std::fs::read(&ca).unwrap()) { + roots.add(cert.unwrap()).unwrap(); + } + let connector = |alpn: &[&[u8]]| { + let mut config = rustls::ClientConfig::builder() + .with_root_certificates(roots.clone()) + .with_no_client_auth(); + config.alpn_protocols = alpn.iter().map(|alpn| alpn.to_vec()).collect(); + tokio_rustls::TlsConnector::from(Arc::new(config)) + }; + + let serve = |rtmp: bool| { + let mut config = Config::default(); + config.http.listen = Some("127.0.0.1:0".parse().unwrap()); + config.https.listen = Some("127.0.0.1:0".parse().unwrap()); + config.https.cert = vec![cert.clone()]; + config.https.key = vec![key.clone()]; + let auth = crate::auth::Config { + public_subscribe: vec![moq_auth::Pattern::all()], + ..Default::default() + } + .init("test", &moq_tokio::tls::Connect::default(), false) + .unwrap(); + let cluster = cluster::Cluster::new(crate::cluster::Options::default()).unwrap(); + let certificates = moq_tokio::tls::Certificates::from_pem(&std::fs::read(&cert).unwrap()).unwrap(); + let mut web = Web::new(auth, cluster, certificates, config).bind().unwrap(); + let incoming = rtmp.then(|| web.rtmp()); + let Addrs { + http: Some(http), + https: Some(https), + } = web.addrs() + else { + panic!("both listeners are configured"); + }; + let routes = web.routes(); + (tokio::spawn(web.serve(routes)), http, https, incoming) + }; + let connect = async |addr: net::SocketAddr, alpn: Option<&[&[u8]]>| -> Box { + let tcp = tokio::net::TcpStream::connect(addr).await.unwrap(); + match alpn { + None => Box::new(tcp) as Box, + Some(alpn) => Box::new( + connector(alpn) + .connect("localhost".try_into().unwrap(), tcp) + .await + .expect("TLS handshake"), + ), + } + }; + + let (serving, http, https, incoming) = serve(true); + let mut incoming = incoming.unwrap(); + for (addr, alpn) in [(http, None), (https, None), (https, Some(&[][..]))] { + let mut client = connect(addr, alpn).await; + client.write_all(&[3, 1, 2, 3]).await.unwrap(); + let mut rtmp = tokio::time::timeout(std::time::Duration::from_secs(10), incoming.accept()) + .await + .expect("RTMP reached its consumer") + .unwrap(); + assert_eq!( + rtmp.tcp().local_addr().unwrap(), + addr, + "the accepted socket stays reachable" + ); + let mut c0 = [0u8; 4]; + rtmp.read_exact(&mut c0).await.unwrap(); + assert_eq!(c0, [3, 1, 2, 3], "addr={addr} tls={}", alpn.is_some()); + } + for (addr, alpn) in [ + (http, None), + (https, Some(&[b"http/1.1".as_slice()][..])), + (https, Some(&[][..])), + ] { + let mut client = connect(addr, alpn).await; + client + .write_all(b"GET /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") + .await + .unwrap(); + let mut response = String::new(); + let _ = client.read_to_string(&mut response).await; + assert!( + response.starts_with("HTTP/1.1 200"), + "addr={addr} alpn={alpn:?}: {response}" + ); + } + serving.abort(); + + let (serving, http, https, _) = serve(false); + for (addr, alpn) in [(http, None), (https, Some(&[][..]))] { + let mut client = connect(addr, alpn).await; + client.write_all(&[3]).await.unwrap(); + let mut rest = Vec::new(); + let read = tokio::time::timeout(std::time::Duration::from_secs(10), client.read_to_end(&mut rest)) + .await + .expect("an unserved RTMP connection is closed"); + assert!(rest.is_empty(), "{read:?}"); + } + serving.abort(); + } + /// An embedded route can read the accepted address with Axum's standard /// extractor, regardless of client-supplied proxy headers. #[tokio::test] @@ -2543,7 +2602,7 @@ mod tests { .await .unwrap(); assert!(activity.close().now_or_never().is_none(), "TLS was not counted"); - let acceptor = WebAcceptor::tls(RustlsConfig::from_config(tls), false); + let acceptor = WebAcceptor::tls(RustlsConfig::from_config(tls), false, rtmp::Handoff::default()); let ((stream, _service), _client) = socket_step(async { tokio::try_join!( acceptor.accept(accepted, ()), @@ -2552,7 +2611,7 @@ mod tests { }) .await .unwrap(); - assert!(stream.is_http2()); + assert_eq!(stream.tls().unwrap().alpn_protocol(), Some(&b"h2"[..])); assert!( activity.close().now_or_never().is_some(), "h2 awaited request headers before releasing its admission" diff --git a/rs/moq-relay/src/web/rtmp.rs b/rs/moq-relay/src/web/rtmp.rs new file mode 100644 index 00000000000..b11605e4933 --- /dev/null +++ b/rs/moq-relay/src/web/rtmp.rs @@ -0,0 +1,80 @@ +//! RTMP and RTMPS connections the public web listeners classified, for an embedder to serve. +//! +//! The relay speaks no RTMP itself: without a consumer taken from +//! [`Web::rtmp`](super::Web::rtmp) it logs and closes them. + +use std::{ + io, + pin::Pin, + task::{Context, Poll}, +}; + +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; +use tokio::sync::mpsc; + +/// The RTMP connections accepted on the public web listeners. +pub struct Incoming(mpsc::Receiver); + +impl Incoming { + /// The next RTMP connection, or `None` once the web listeners stop. + pub async fn accept(&mut self) -> Option { + self.0.recv().await + } +} + +/// An RTMP connection, plaintext or with TLS already terminated, for `moq_rtmp::accept_stream`. +pub struct Stream(crate::listener::Stream); + +impl Stream { + /// The accepted TCP socket, for socket options and statistics. + pub fn tcp(&self) -> &tokio::net::TcpStream { + self.0.tcp() + } +} + +impl AsyncRead for Stream { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + Pin::new(&mut self.0).poll_read(cx, buf) + } +} + +impl AsyncWrite for Stream { + fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { + Pin::new(&mut self.0).poll_write(cx, buf) + } + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.0).poll_flush(cx) + } + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.0).poll_shutdown(cx) + } +} + +/// Where the listeners send classified RTMP connections; empty refuses them. +#[derive(Clone, Default)] +pub(crate) struct Handoff(Option>); + +impl Handoff { + /// A handoff and the consumer it feeds. + /// + /// One slot: each connection waits on its own task for the embedder to take it, + /// which the listener's idle deadline bounds like any other stalled handshake. + pub(crate) fn new() -> (Self, Incoming) { + let (tx, rx) = mpsc::channel(1); + (Self(Some(tx)), Incoming(rx)) + } + + /// Give `stream` to the consumer, or close it when there is none. + pub(crate) async fn send(&self, stream: crate::listener::Stream) { + // The embedder owns an RTMP session's lifetime, so the drain stops counting it here. + drop(stream.admission().as_ref().and_then(crate::listener::Admission::take)); + let peer = stream.tcp().peer_addr().ok(); + let refused = match &self.0 { + Some(tx) => tx.send(Stream(stream)).await.is_err(), + None => true, + }; + if refused { + tracing::debug!(?peer, "closed an RTMP connection: nothing serves RTMP here"); + } + } +} From 4e8db4b713afc432c2c9704120907e4abcc2a1e9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 05:53:35 -0700 Subject: [PATCH 3/6] fix(relay): end the RTMP consumer when serving stops Connections still classifying held sender clones, so Incoming::accept never returned None after the listeners stopped with no idle deadline. Co-Authored-By: Claude Opus 5.5 --- rs/moq-relay/src/web.rs | 18 ++++++++++++++++-- rs/moq-relay/src/web/rtmp.rs | 26 +++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/rs/moq-relay/src/web.rs b/rs/moq-relay/src/web.rs index 1ccd0d60576..7313f37f558 100644 --- a/rs/moq-relay/src/web.rs +++ b/rs/moq-relay/src/web.rs @@ -462,6 +462,9 @@ impl Web { .layer(axum::middleware::from_fn(get_cors)) .into_make_service_with_connect_info::(); let ws = config.resolved_ws(); + // Connections still classifying outlive this future, and must not keep the + // RTMP consumer waiting once it returns. + let _rtmp = rtmp.close_on_drop(); // Each request is admitted on its own, so a connection may have as many in flight // as a session may have credentials verifying; the client queues the rest. let streams = u32::try_from(state.auth.verifications()).unwrap_or(u32::MAX); @@ -1934,7 +1937,8 @@ mod tests { /// Each web listener splits its connections by their first bytes: HTTP reaches the /// router, while plaintext RTMP and RTMPS (TLS with no ALPN) reach the RTMP - /// consumer with the classifying byte intact. Untaken, RTMP is closed. + /// consumer with the classifying byte intact, until serving stops. Untaken, RTMP + /// is closed. #[tokio::test] async fn web_listeners_hand_rtmp_to_its_consumer() { use rustls::pki_types::{CertificateDer, pem::PemObject}; @@ -1971,7 +1975,11 @@ mod tests { .unwrap(); let cluster = cluster::Cluster::new(crate::cluster::Options::default()).unwrap(); let certificates = moq_tokio::tls::Certificates::from_pem(&std::fs::read(&cert).unwrap()).unwrap(); - let mut web = Web::new(auth, cluster, certificates, config).bind().unwrap(); + // No idle deadline, so a stalled connection outlives serving below. + let mut web = Web::new(auth, cluster, certificates, config) + .with_timeout(None) + .bind() + .unwrap(); let incoming = rtmp.then(|| web.rtmp()); let Addrs { http: Some(http), @@ -2014,6 +2022,8 @@ mod tests { rtmp.read_exact(&mut c0).await.unwrap(); assert_eq!(c0, [3, 1, 2, 3], "addr={addr} tls={}", alpn.is_some()); } + // Silent, so it stays classifying; the requests below are accepted after it. + let _stalled = tokio::net::TcpStream::connect(http).await.unwrap(); for (addr, alpn) in [ (http, None), (https, Some(&[b"http/1.1".as_slice()][..])), @@ -2031,7 +2041,11 @@ mod tests { "addr={addr} alpn={alpn:?}: {response}" ); } + // The connection still classifying does not keep the consumer open past serving. serving.abort(); + let _ = serving.await; + let ended = tokio::time::timeout(std::time::Duration::from_secs(10), incoming.accept()).await; + assert!(matches!(ended, Ok(None)), "the consumer ends with serving"); let (serving, http, https, _) = serve(false); for (addr, alpn) in [(http, None), (https, Some(&[][..]))] { diff --git a/rs/moq-relay/src/web/rtmp.rs b/rs/moq-relay/src/web/rtmp.rs index b11605e4933..b0a6e5da99d 100644 --- a/rs/moq-relay/src/web/rtmp.rs +++ b/rs/moq-relay/src/web/rtmp.rs @@ -6,6 +6,7 @@ use std::{ io, pin::Pin, + sync::{Arc, Mutex, PoisonError}, task::{Context, Poll}, }; @@ -51,8 +52,12 @@ impl AsyncWrite for Stream { } /// Where the listeners send classified RTMP connections; empty refuses them. +/// +/// Connections still classifying hold clones of this, so the sender sits behind +/// one shared slot that [`Close`] empties when serving stops, ending +/// [`Incoming::accept`] regardless of them. #[derive(Clone, Default)] -pub(crate) struct Handoff(Option>); +pub(crate) struct Handoff(Arc>>>); impl Handoff { /// A handoff and the consumer it feeds. @@ -61,7 +66,12 @@ impl Handoff { /// which the listener's idle deadline bounds like any other stalled handshake. pub(crate) fn new() -> (Self, Incoming) { let (tx, rx) = mpsc::channel(1); - (Self(Some(tx)), Incoming(rx)) + (Self(Arc::new(Mutex::new(Some(tx)))), Incoming(rx)) + } + + /// Refuse every later connection once the returned guard drops. + pub(crate) fn close_on_drop(&self) -> Close { + Close(self.clone()) } /// Give `stream` to the consumer, or close it when there is none. @@ -69,7 +79,8 @@ impl Handoff { // The embedder owns an RTMP session's lifetime, so the drain stops counting it here. drop(stream.admission().as_ref().and_then(crate::listener::Admission::take)); let peer = stream.tcp().peer_addr().ok(); - let refused = match &self.0 { + let tx = self.0.lock().unwrap_or_else(PoisonError::into_inner).clone(); + let refused = match tx { Some(tx) => tx.send(Stream(stream)).await.is_err(), None => true, }; @@ -78,3 +89,12 @@ impl Handoff { } } } + +/// Empties a [`Handoff`] when dropped. +pub(crate) struct Close(Handoff); + +impl Drop for Close { + fn drop(&mut self) { + self.0.0.lock().unwrap_or_else(PoisonError::into_inner).take(); + } +} From a0b038ff27666765c8624cf64c3ace2430f52bcc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 13:04:14 -0700 Subject: [PATCH 4/6] feat(relay): serve tls:// qmux on the HTTPS listener The HTTPS acceptor reads each ClientHello. One that offers a moq ALPN finishes with the listen TLS identity and runs as a MoQ session, through the new moq_tokio::tcp::TlsAcceptor (from Server::tls_acceptor). Every other connection keeps web.https.cert. listen.tcp stays a separate optional listener, and there is no new setting. qmux gains tls::Server::accept_established and a public tcp::adopt, so the HTTPS arm shares both handshake halves with the tls:// listener. The quest is done and deleted. Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/auth.md | 6 +- doc/bin/relay/cluster.md | 5 +- doc/bin/relay/config.md | 2 +- quest/m2/one-port/README.md | 16 +-- quest/m2/one-port/tcp-demux.md | 41 ------ quest/m2/rate-websocket.md | 7 +- quest/m2/tls-listener-mtls.md | 13 +- rs/moq-relay/src/listener.rs | 2 +- rs/moq-relay/src/listener/demux.rs | 59 ++++++-- rs/moq-relay/src/relay.rs | 3 +- rs/moq-relay/src/web.rs | 217 +++++++++++++++++++++++++---- rs/moq-tokio/src/server.rs | 66 +++++++-- rs/moq-tokio/src/tcp.rs | 77 ++++++++++ rs/qmux/src/tcp.rs | 5 +- rs/qmux/src/tls.rs | 24 +++- 15 files changed, 415 insertions(+), 128 deletions(-) delete mode 100644 quest/m2/one-port/tcp-demux.md diff --git a/doc/bin/relay/auth.md b/doc/bin/relay/auth.md index de40300e735..0bd259f14e8 100644 --- a/doc/bin/relay/auth.md +++ b/doc/bin/relay/auth.md @@ -36,7 +36,7 @@ The fields are documented on `moq_auth::Request` and `moq_auth::Grant` ([Rust](/lib/rs/moq-auth), [TypeScript](/lib/js/auth)). `server_name` is the SNI that selected the relay's certificate. It is absent -over plaintext, on the `tls://` TCP listener, and for an IP-literal dial, and +over plaintext, on the `listen.tcp` `tls://` listener, and for an IP-literal dial, and never filled from a `Host` header or a WebTransport authority. The client still picks it among the names the relay serves, so a hostname-scoped server binds the tenant to the credential, not to `server_name` alone. @@ -422,3 +422,7 @@ parent directory or an allowlist. `listen.tcp.tls = true` serves the TCP listener over TLS instead, for `tls://` dials such as [cluster links](/bin/relay/cluster#tls-links). It asks for no client certificate, so a peer on it presents a token, not mTLS. +The HTTPS listener (`web.https`) accepts `tls://` too, with no setting: a +ClientHello that offers a moq ALPN is served the listen certificate, the same +way and with no client certificate either, while every other connection keeps +`web.https.cert`. diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index c16547f2a0c..a5ce58a4dc0 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -134,8 +134,9 @@ topology check; whatever generates the peer list applies the layout's rules. A link inside a region is rarely congested, so it can skip QUIC: a `tls://` peer URL dials qmux over TLS on TCP, verified with the same `connect.tls` settings as any other dial. The accepting relay serves it from its TCP -listener with TLS on. That listener asks for no client certificate, so a peer -on it authenticates with a token (`cluster.token`, `token`, or `?jwt=`), not +listener with TLS on, or from its HTTPS listener, which serves `tls://` with +the listen certificate. Neither asks for a client certificate, so a peer on +them authenticates with a token (`cluster.token`, `token`, or `?jwt=`), not mTLS. ```toml diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index 407ca90be6c..5646364cabf 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -193,7 +193,7 @@ start rather than being ignored. If io\_uring workers fail to start naming listen = "[::]:4443" # HTTP: fingerprint, announced, fetch, health. [web.https] -listen = "[::]:443" # HTTPS plus the WebSocket fallback. +listen = "[::]:443" # HTTPS, the WebSocket fallback, and tls:// with the listen certificate. cert = "cert.pem" # cert, key, and root need listen. Omit cert and key with [acme]. key = "key.pem" diff --git a/quest/m2/one-port/README.md b/quest/m2/one-port/README.md index 476e82c13a2..3fe9ba70501 100644 --- a/quest/m2/one-port/README.md +++ b/quest/m2/one-port/README.md @@ -6,7 +6,8 @@ A relay, or a binary embedding the gateway crates beside one, speaks everything it serves on one UDP port and one TCP port. UDP carries QUIC (WebTransport and raw moq), the WebRTC media path (STUN, DTLS, SRTP) for WHIP and WHEP, and SRT. TCP carries TLS-terminated -HTTP (WebSocket qmux, WHIP and WHEP signaling, HLS, ops), RTMP, and RTMPS. +HTTP (WebSocket qmux, WHIP and WHEP signaling, HLS, ops), `tls://` qmux, +RTMP, and RTMPS. Upstream delivers the demux and stacks that accept a fed socket or stream; `moq-relay` itself serves QUIC and HTTP on them, and an embedder such as moq.pro's edge wires WebRTC, RTMP, and SRT, which the @@ -81,22 +82,9 @@ and goes to WebRTC; any other is dropped and counted. Decided 2026-10-13: the responder binds its own port ([relay STUN](/doc/bin/relay/config.md#stun)), so the shared port never answers public Binding requests. -### TCP - -The 1935 listener downstream already peeks one byte to split RTMPS from RTMP. -The unified acceptor generalizes it: 0x16 is TLS, terminated here, then the -first decrypted byte is peeked again; 0x03 is an RTMP handshake and anything -else is HTTP, which goes to the axum router. A plaintext 0x03 is RTMP and a -plaintext ASCII method is HTTP. RTMPS clients send no ALPN, so ALPN cannot do -this. The acceptor yields classified connections; `moq-rtmp`'s -`accept_stream` already takes any `AsyncRead + AsyncWrite`, and -`axum_server::Server::from_listener` takes a listener that a channel of -pre-accepted streams can stand behind. - ## Required - [WebRTC on the shared socket](/quest/m2/one-port/rtc-feed.md) - `moq-rtc` serves WHIP and WHEP media from the WebRTC hook and pins ICE tuples -- [TCP acceptor: tls:// on the web port](/quest/m2/one-port/tcp-demux.md) - the HTTPS listener also carries `tls://` qmux beside HTTP, RTMP, and RTMPS - [SRT on the shared socket](/quest/m2/one-port/srt-demux.md) - moq-srt drives `srt-protocol` on demuxed packets and the flow table pins its 4-tuples ## Related diff --git a/quest/m2/one-port/tcp-demux.md b/quest/m2/one-port/tcp-demux.md deleted file mode 100644 index d4c8921ea21..00000000000 --- a/quest/m2/one-port/tcp-demux.md +++ /dev/null @@ -1,41 +0,0 @@ -# [M] TCP acceptor: tls:// on the web port - -## Goal - -The relay's HTTPS listener also carries `tls://` qmux, so one TCP port serves -TLS-terminated HTTP, RTMP, RTMPS, and `tls://` MoQ. The HTTP, RTMP, and RTMPS -split already landed: `listener::classify` peeks each connection on its own -task, and an embedder takes RTMP from `Relay::rtmp`. - -## Plan - -`tls://` is served today by its own `listen.tcp` listener with TLS. It -negotiates a moq ALPN in the TLS handshake, so split the decrypted arm on ALPN: -a moq ALPN goes to the qmux server and an HTTP ALPN to the router. A connection -with no ALPN keeps the decrypted-byte sniff, since RTMPS clients negotiate none. -Selecting the server config from the ClientHello (rustls' `LazyConfigAcceptor`) -lets the moq ALPN serve the listen TLS identity while HTTP keeps -`web.https.cert`. - -Open: how an operator turns this on, and what becomes of the separate -`listen.tcp` TLS bind. Asked in the PR that landed the HTTP and RTMP split. - -Decided 2026-10-08: under one acceptor, the client-certificate request from -[mTLS on tls://](/quest/m2/tls-listener-mtls.md) is optional (never required) -and sits on the moq-ALPN TLS config only, so browsers and RTMPS clients on -the same port are never asked for one. Whichever lands second keeps `tls://` -client-certificate verification and the `MtlsUnsupported` lift working. - -Keep the accepted `TcpStream` reachable (`listener::Stream::tcp`): qmux -reads socket stats from it, and -[WebSocket bitrate caps](/quest/m2/rate-websocket.md) set socket buffer sizes -on the same socket. - -Tests: a `tls://` qmux client against the HTTPS listener lands in the qmux -server, beside the HTTP and RTMP cases already covered. - -## Related - -- [UDP demux](/quest/m2/p2p/udp-demux.md) - the UDP half -- [mTLS on tls://](/quest/m2/tls-listener-mtls.md) - client-certificate verification on the `tls://` arm this acceptor carries -- [WebSocket bitrate caps](/quest/m2/rate-websocket.md) - sizes the kernel buffers of the socket this acceptor boxes diff --git a/quest/m2/rate-websocket.md b/quest/m2/rate-websocket.md index 6a977d0a5f9..30a139fa53c 100644 --- a/quest/m2/rate-websocket.md +++ b/quest/m2/rate-websocket.md @@ -33,7 +33,8 @@ excess: `SO_SNDBUF` to about the burst, since paced reads cannot stop bytes the kernel already accepted and paced writes can queue a send buffer that drains faster than the cap. Fixing the size disables autotuning, which is the - point. The buffers bound the burst only; the sustained cap comes from the + point. The web acceptor keeps the accepted socket reachable through + `listener::Stream::tcp` (`rs/moq-relay/src/listener/demux.rs`) for this. The buffers bound the burst only; the sustained cap comes from the paced IO. Behind a TCP-terminating proxy, the buffers bound only the relay's side. - Reads stop at the bucket, so the client's TCP window closes; writes stop @@ -48,7 +49,3 @@ live session applies. ## Required - [QUIC caps](/quest/m2/rate-quic.md) - introduces the moq-auth cap type this reuses - -## Related - -- [TCP acceptor: tls:// on the web port](/quest/m2/one-port/tcp-demux.md) - wraps the accepted stream; `listener::Stream::tcp` keeps the socket these buffer sizes need diff --git a/quest/m2/tls-listener-mtls.md b/quest/m2/tls-listener-mtls.md index bf89741cb99..ea999456f92 100644 --- a/quest/m2/tls-listener-mtls.md +++ b/quest/m2/tls-listener-mtls.md @@ -33,14 +33,17 @@ that refusal for a server with a `tls://` listener and updates the refusal's test. Decided 2026-10-08: request the certificate optionally, never require it, -and only on the moq-ALPN TLS config, so once -[the TCP acceptor](/quest/m2/one-port/tcp-demux.md) shares the port with -HTTP and RTMPS, those clients are never asked for one. Whichever lands second -keeps both working. +and only on the moq-ALPN TLS config, so HTTP and RTMPS clients on the shared +HTTPS port are never asked for one. That port already serves `tls://`: the +relay's web acceptor picks `moq_tokio::tcp::TlsAcceptor::config` (built like +the `listen.tcp` TLS config, by `TcpTls::new` in `rs/moq-tokio/src/server.rs`) +from the ClientHello, so the optional request goes on that config too. There +the relay holds the `rustls::ServerConnection` (`listener::Stream::tls`), so it +reads the peer certificate itself and `TlsAcceptor::accept` needs a way to +carry it into the `Request`, as `websocket::Options::identity` does. Public API: relay config may gain a listener option. Wire: none. ## Related -- [TCP acceptor](/quest/m2/one-port/tcp-demux.md) - carries `tls://` on the shared TCP port, where the certificate request stays on the moq ALPN - [qmux drives its own socket](/quest/m2/perf/stream/driver.md) - replaces qmux's reader and writer tasks and its constructors' return type diff --git a/rs/moq-relay/src/listener.rs b/rs/moq-relay/src/listener.rs index 1f4d8249b36..514c587a060 100644 --- a/rs/moq-relay/src/listener.rs +++ b/rs/moq-relay/src/listener.rs @@ -16,7 +16,7 @@ mod demux; mod idle; -pub(crate) use demux::{Classified, Stream, classify}; +pub(crate) use demux::{Classified, Stream, Tls, classify}; pub(crate) use idle::Admission; use std::{ diff --git a/rs/moq-relay/src/listener/demux.rs b/rs/moq-relay/src/listener/demux.rs index 12c934831b2..64e89b1fa11 100644 --- a/rs/moq-relay/src/listener/demux.rs +++ b/rs/moq-relay/src/listener/demux.rs @@ -1,4 +1,4 @@ -//! Splits a public web connection into HTTP and RTMP by its first bytes. +//! Splits a public web connection into HTTP, RTMP, and `tls://` qmux by its first bytes. //! //! `axum_server` runs the acceptor on each connection's own task, so a peer that //! connects and sends nothing stalls only itself, under the same idle deadline @@ -8,32 +8,46 @@ use std::{ io, pin::Pin, task::{Context, Poll}, + time::Duration, }; -use axum_server::{accept::Accept, tls_rustls::RustlsAcceptor}; +use axum_server::tls_rustls::RustlsConfig; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, ReadBuf}; use tokio::net::TcpStream; -use tokio_rustls::server::TlsStream; +use tokio_rustls::{LazyConfigAcceptor, server::TlsStream}; use super::Accepted; /// The RTMP version byte (C0) an RTMP handshake opens with. const RTMP_VERSION: u8 = 0x03; +/// The bound on a TLS handshake, as `axum_server`'s own rustls acceptor sets. +const TLS_HANDSHAKE: Duration = Duration::from_secs(10); + /// A classified connection: the protocol its first bytes named. pub(crate) enum Classified { Http(Stream), Rtmp(Stream), + /// `tls://`: its ClientHello offered a moq ALPN, and the handshake negotiated one. + Qmux(Stream), +} + +/// The TLS a listener terminates: HTTP's config, and `tls://`'s when it is served. +#[derive(Clone)] +pub(crate) struct Tls { + pub(crate) http: RustlsConfig, + pub(crate) qmux: Option, } /// Classify `accepted`, terminating TLS with `tls` when the listener serves it. /// /// A plaintext C0 is RTMP on either listener. On a TLS listener anything else is /// handed to rustls, which refuses whatever is not a ClientHello (0x16), so -/// plaintext HTTP is still refused there. A negotiated ALPN names HTTP; RTMPS -/// clients offer none, so a connection without one is sniffed again on its first -/// decrypted byte. -pub(crate) async fn classify(accepted: Accepted, tls: Option<&RustlsAcceptor>) -> io::Result { +/// plaintext HTTP is still refused there. A ClientHello that offers a moq ALPN is +/// `tls://`, finished with the listen identity. Otherwise a negotiated ALPN names +/// HTTP; RTMPS clients offer none, so a connection without one is sniffed again +/// on its first decrypted byte. +pub(crate) async fn classify(accepted: Accepted, tls: Option<&Tls>) -> io::Result { let mut first = [0u8; 1]; if accepted.stream.peek(&mut first).await? == 0 { return Err(io::ErrorKind::UnexpectedEof.into()); @@ -49,12 +63,31 @@ pub(crate) async fn classify(accepted: Accepted, tls: Option<&RustlsAcceptor>) - }); }; - let (mut tls, ()) = tls.accept(accepted, ()).await?; + let handshake = async { + let start = LazyConfigAcceptor::new(rustls::server::Acceptor::default(), accepted).await?; + let alpns = start.client_hello().alpn(); + let qmux = tls + .qmux + .as_ref() + .filter(|qmux| alpns.is_some_and(|alpns| qmux.offered(alpns))); + let config = match qmux { + Some(qmux) => qmux.config(), + None => tls.http.get_inner(), + }; + Ok::<_, io::Error>((start.into_stream(config).await?, qmux.is_some())) + }; + let (mut tls, qmux) = moq_clock::timeout(TLS_HANDSHAKE, handshake) + .await + .map_err(|_| io::Error::from(io::ErrorKind::TimedOut))??; if tls.get_ref().1.alpn_protocol().is_some() { - return Ok(Classified::Http(Stream { + let stream = Stream { transport: Transport::Tls(Box::new(tls)), peeked: None, - })); + }; + return Ok(match qmux { + true => Classified::Qmux(stream), + false => Classified::Http(stream), + }); } let first = tls.read_u8().await?; let stream = Stream { @@ -141,6 +174,12 @@ impl AsyncWrite for Stream { } } +impl AsRef for Stream { + fn as_ref(&self) -> &TcpStream { + self.tcp() + } +} + #[cfg(unix)] impl std::os::fd::AsFd for Stream { fn as_fd(&self) -> std::os::fd::BorrowedFd<'_> { diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index 6489f975a4c..2423058d1a7 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -329,7 +329,8 @@ impl Relay { .with_extensions(server_extensions) .with_implementation(implementation) .with_timeout(handshake_timeout) - .with_sessions(sessions.clone()); + .with_sessions(sessions.clone()) + .with_tls_acceptor(server.tls_acceptor()); #[cfg(feature = "acme")] let web = web.with_acme(acme.as_ref().map(crate::acme::Acme::resolver)); let web = web.bind()?; diff --git a/rs/moq-relay/src/web.rs b/rs/moq-relay/src/web.rs index 7313f37f558..89f93abde0d 100644 --- a/rs/moq-relay/src/web.rs +++ b/rs/moq-relay/src/web.rs @@ -14,10 +14,7 @@ use axum::{ response::{Html, IntoResponse, Response}, routing::get, }; -use axum_server::{ - accept::Accept, - tls_rustls::{RustlsAcceptor, RustlsConfig}, -}; +use axum_server::{accept::Accept, tls_rustls::RustlsConfig}; use bytes::Bytes; use futures::{FutureExt, future::BoxFuture}; use tower_service::Service; @@ -161,19 +158,11 @@ pub(crate) struct WebState { pub(crate) cluster: cluster::Cluster, /// TLS certificate information served at `/certificate.sha256`. pub(crate) certificates: moq_tokio::tls::Certificates, - /// Relay-wide shutdown broadcast; WebSocket sessions drain with a GOAWAY - /// when it fires. Defaults to a handle that never fires. - #[cfg_attr( - not(feature = "websocket"), - allow(dead_code, reason = "only the `websocket` feature uses it") - )] + /// Relay-wide shutdown broadcast; WebSocket and `tls://` sessions drain with a + /// GOAWAY when it fires. Defaults to a handle that never fires. pub(crate) shutdown: crate::shutdown::Observer, - /// Live sessions on this node, so a WebSocket session is listed and nudged - /// like a QUIC one. - #[cfg_attr( - not(feature = "websocket"), - allow(dead_code, reason = "only the `websocket` feature uses it") - )] + /// Live sessions on this node, so a WebSocket or `tls://` session is listed and + /// nudged like a QUIC one. pub(crate) sessions: crate::session::Registry, /// How long a connection may sit with no complete request in flight (TLS /// handshake and partial headers included), and a WebSocket wait for its MoQ @@ -210,6 +199,8 @@ pub struct Web { /// The ACME certificate resolver HTTPS serves instead of `https.cert`/`https.key`. acme: Option>, rtmp: rtmp::Handoff, + /// Serves `tls://` on the HTTPS listener, when the relay has a listen identity. + tls_stream: Option, addrs: Addrs, } @@ -243,6 +234,7 @@ impl Web { https_tls: None, acme: None, rtmp: rtmp::Handoff::default(), + tls_stream: None, addrs: Addrs::default(), } } @@ -312,6 +304,17 @@ impl Web { incoming } + /// Serve `tls://` on the HTTPS listener with `acceptor`, usually + /// [`moq_tokio::Server::tls_acceptor`]. + /// + /// A ClientHello that offers one of its moq ALPNs finishes with the listen TLS + /// identity and becomes a MoQ session. Every other connection keeps + /// `https.cert`, so browsers and RTMPS clients are unaffected. + pub fn with_tls_acceptor(mut self, acceptor: Option) -> Self { + self.tls_stream = acceptor; + self + } + /// The actual bound addresses after [`bind`](Self::bind) or [`crate::Relay::load`]. pub fn addrs(&self) -> Addrs { self.addrs @@ -455,6 +458,7 @@ impl Web { https_tls, acme, rtmp, + tls_stream, .. } = self.bind()?; let app = app @@ -477,7 +481,7 @@ impl Web { timeout: state.timeout, activity: Some(state.shutdown.activity.clone()), }, - WebAcceptor::plain(ws, rtmp.clone()), + WebAcceptor::plain(ws, rtmp.clone(), state.clone()), )?; server.http_builder().http2().max_concurrent_streams(streams); Some(server.serve(app.clone())) @@ -506,7 +510,15 @@ impl Web { timeout: state.timeout, activity: Some(state.shutdown.activity.clone()), }, - WebAcceptor::tls(rustls_config, ws, rtmp), + WebAcceptor::tls( + crate::listener::Tls { + http: rustls_config, + qmux: tls_stream, + }, + ws, + rtmp, + state.clone(), + ), )?; server.http_builder().http2().max_concurrent_streams(streams); Some(server.serve(app)) @@ -688,24 +700,32 @@ pub(crate) struct ServerName(pub(crate) String); /// Accepts a connection on a public web listener. /// /// Classifies the connection (terminating TLS on `https://`), hands RTMP to -/// [`rtmp::Incoming`], and captures the socket of an HTTP one. Both listeners -/// wrap this same acceptor, so the capture cannot reach one and miss the other. +/// [`rtmp::Incoming`], serves `tls://`, and captures the socket of an HTTP one. +/// Both listeners wrap this same acceptor, so the capture cannot reach one and +/// miss the other. #[derive(Clone)] struct WebAcceptor { /// Present on `https://`, which also gives the connection a peer certificate to report. - tls: Option, + tls: Option, /// Whether a WebSocket route exists to use the handle. The capture holds a /// duplicated descriptor for the life of the connection, so doing it when /// nothing can read it would cost every API, HLS, and health-check connection a /// second descriptor for nothing. ws: bool, rtmp: rtmp::Handoff, + /// What a `tls://` session runs against. + state: Arc, } impl WebAcceptor { /// Accepts plain HTTP, which has no handshake of its own to run. - fn plain(ws: bool, rtmp: rtmp::Handoff) -> Self { - Self { tls: None, ws, rtmp } + fn plain(ws: bool, rtmp: rtmp::Handoff, state: Arc) -> Self { + Self { + tls: None, + ws, + rtmp, + state, + } } /// Accepts HTTPS, running the TLS handshake and surfacing a verified peer @@ -713,13 +733,60 @@ impl WebAcceptor { /// /// When no client CA is configured the inner verifier is `NoClientAuth` and /// `peer_certificates()` always returns None, so the marker never appears. - fn tls(config: RustlsConfig, ws: bool, rtmp: rtmp::Handoff) -> Self { + fn tls(tls: crate::listener::Tls, ws: bool, rtmp: rtmp::Handoff, state: Arc) -> Self { Self { - tls: Some(RustlsAcceptor::new(config)), + tls: Some(tls), ws, rtmp, + state, } } + + /// Run a `tls://` connection as a MoQ session on its own task, past the idle + /// deadline that bounds this acceptor. + fn serve_qmux(&self, stream: crate::listener::Stream) { + let Some(acceptor) = self.tls.as_ref().and_then(|tls| tls.qmux.clone()) else { + return; + }; + let state = self.state.clone(); + // The admission carries into the session, as a WebSocket upgrade's does. + let Some(active) = stream + .admission() + .and_then(|admission| admission.take()) + .or_else(|| state.shutdown.activity.enter()) + else { + return; + }; + let tls = stream.tls().expect("tls:// is only classified under TLS"); + let mut link = moq_tokio::server::Link::default(); + link.server_name = tls.server_name().map(str::to_owned); + link.alpn = tls + .alpn_protocol() + .and_then(|alpn| std::str::from_utf8(alpn).ok()) + .map(str::to_owned); + tokio::spawn(async move { + let Some(request) = state + .shutdown + .until_deadline(acceptor.accept(stream, link, active)) + .await + else { + return; + }; + let request = match request { + Ok(request) => request, + Err(err) => { + tracing::debug!(%err, "failed to accept tls:// on the HTTPS listener"); + return; + } + }; + crate::connection::Connection::new(request, state.cluster.clone(), state.auth.clone()) + .with_id(state.cluster.next_connection_id()) + .with_shutdown(state.shutdown.clone()) + .with_sessions(state.sessions.clone()) + .run() + .await; + }); + } } impl Accept for WebAcceptor { @@ -736,6 +803,10 @@ impl Accept for WebAcceptor { this.rtmp.send(stream).await; return Err(std::io::Error::other("handed off an RTMP connection")); } + crate::listener::Classified::Qmux(stream) => { + this.serve_qmux(stream); + return Err(std::io::Error::other("handed off a tls:// connection")); + } }; let tls = stream.tls(); if tls.is_some_and(|tls| tls.alpn_protocol() == Some(b"acme-tls/1")) { @@ -1155,6 +1226,20 @@ impl IntoResponse for ServeGroup { )] mod tests { use super::*; + + /// The state an acceptor built outside [`Web::serve`] hands a `tls://` session. + fn acceptor_state() -> Arc { + let cluster = cluster::Cluster::new(crate::cluster::Options::default()).unwrap(); + let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let certificates = moq_tokio::tls::Certificates::from_pem(cert.cert.pem().as_bytes()).unwrap(); + Web::new( + crate::auth::Auth::refuse("test"), + cluster, + certificates, + Config::default(), + ) + .state + } use rcgen::{CertificateParams, KeyPair}; use std::io::Write; use tempfile::TempDir; @@ -1683,7 +1768,7 @@ mod tests { // The acceptor waits for a first byte to classify the connection. tokio::io::AsyncWriteExt::write_all(&mut client, b"GET").await.unwrap(); - let acceptor = WebAcceptor::plain(true, rtmp::Handoff::default()); + let acceptor = WebAcceptor::plain(true, rtmp::Handoff::default(), acceptor_state()); let accepted = crate::listener::Accepted::new(accepted); let (stream, mut service) = Accept::<_, EchoSocket>::accept(&acceptor, accepted, EchoSocket) .await @@ -1731,7 +1816,7 @@ mod tests { let mut client = connecting.await.unwrap(); tokio::io::AsyncWriteExt::write_all(&mut client, b"GET").await.unwrap(); - let acceptor = WebAcceptor::plain(false, rtmp::Handoff::default()); + let acceptor = WebAcceptor::plain(false, rtmp::Handoff::default(), acceptor_state()); let accepted = crate::listener::Accepted::new(accepted); let (stream, service) = Accept::<_, ()>::accept(&acceptor, accepted, ()).await.unwrap(); @@ -1935,6 +2020,78 @@ mod tests { serving.abort(); } + /// A `tls://` dial to the HTTPS port lands a MoQ session served with the listen + /// identity rather than `https.cert`, while the port keeps serving HTTPS. + #[cfg(feature = "quic")] + #[tokio::test] + async fn https_listener_serves_tls_qmux() { + use rustls::pki_types::{CertificateDer, pem::PemObject}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let dir = TempDir::new().unwrap(); + let (ca, cert, key) = make_certs(&dir); + let mut config = crate::Config::default(); + config.listen.bind = Some("127.0.0.1:0".parse().unwrap()); + config.listen.tls.generate = vec!["localhost".into()]; + config.web.https.listen = Some("127.0.0.1:0".parse().unwrap()); + config.web.https.cert = vec![cert]; + config.web.https.key = vec![key]; + config.auth.public = vec![moq_auth::Pattern::all()]; + let relay = crate::Relay::load(config).await.unwrap().with_signals(false); + let https = relay.web_addrs().https.unwrap(); + let ready = relay.ready(); + let trigger = relay.shutdown_trigger().clone(); + let running = tokio::spawn(relay.run()); + ready.wait().await.unwrap(); + + let url: url::Url = format!("tls://localhost:{}/room", https.port()).parse().unwrap(); + let dial = |tls: moq_tokio::tls::Connect| { + let mut config = moq_tokio::connect::Config::default(); + config.tls = tls; + let connection = config + .init(Default::default()) + .unwrap() + .with_reconnect(false) + .connect(url.clone()); + tokio::time::timeout(std::time::Duration::from_secs(10), connection.established()) + }; + let mut https_ca = moq_tokio::tls::Connect::default(); + https_ca.root = vec![ca.clone()]; + https_ca.system_roots = Some(false); + let refused = dial(https_ca).await.expect("the dial with the HTTPS CA timed out"); + assert!(refused.is_err(), "tls:// was served with https.cert"); + let mut insecure = moq_tokio::tls::Connect::default(); + insecure.insecure = Some(true); + let _session = dial(insecure) + .await + .expect("the tls:// dial timed out") + .expect("tls:// did not land a MoQ session"); + + let mut roots = rustls::RootCertStore::empty(); + for cert in CertificateDer::pem_slice_iter(&std::fs::read(&ca).unwrap()) { + roots.add(cert.unwrap()).unwrap(); + } + let mut client = rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + client.alpn_protocols = vec![b"http/1.1".to_vec()]; + let tcp = tokio::net::TcpStream::connect(https).await.unwrap(); + let mut client = tokio_rustls::TlsConnector::from(Arc::new(client)) + .connect("localhost".try_into().unwrap(), tcp) + .await + .expect("HTTPS still serves https.cert"); + client + .write_all(b"GET /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") + .await + .unwrap(); + let mut response = String::new(); + let _ = client.read_to_string(&mut response).await; + assert!(response.starts_with("HTTP/1.1 200"), "{response}"); + + trigger.start(); + running.await.unwrap().unwrap(); + } + /// Each web listener splits its connections by their first bytes: HTTP reaches the /// router, while plaintext RTMP and RTMPS (TLS with no ALPN) reach the RTMP /// consumer with the classifying byte intact, until serving stops. Untaken, RTMP @@ -2616,7 +2773,11 @@ mod tests { .await .unwrap(); assert!(activity.close().now_or_never().is_none(), "TLS was not counted"); - let acceptor = WebAcceptor::tls(RustlsConfig::from_config(tls), false, rtmp::Handoff::default()); + let tls = crate::listener::Tls { + http: RustlsConfig::from_config(tls), + qmux: None, + }; + let acceptor = WebAcceptor::tls(tls, false, rtmp::Handoff::default(), acceptor_state()); let ((stream, _service), _client) = socket_step(async { tokio::try_join!( acceptor.accept(accepted, ()), diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index be5b7c3a840..8afe377b70f 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -247,6 +247,9 @@ pub struct Server { accept: FuturesUnordered>>, #[cfg(any(feature = "tcp", all(feature = "uds", unix)))] streams: StreamListeners, + /// The `tls://` config another listener can serve, when a listen identity is loaded. + #[cfg(all(feature = "tcp", feature = "_certs"))] + tls_stream: Option>, #[cfg(feature = "iroh")] iroh: Option, #[cfg(feature = "quic")] @@ -412,6 +415,12 @@ impl Server { unix_allow, ); + #[cfg(all(feature = "tcp", feature = "_certs"))] + let tls_stream = tls + .map(|tls| TcpTls::new(&config, &versions, tls)) + .transpose()? + .map(|tls| tls.config); + // The endpoint was bound before this server's versions were known, so it // accepts nothing until they name its ALPNs. #[cfg(feature = "iroh")] @@ -448,6 +457,8 @@ impl Server { timeout, #[cfg(any(feature = "tcp", all(feature = "uds", unix)))] streams, + #[cfg(all(feature = "tcp", feature = "_certs"))] + tls_stream, #[cfg(feature = "iroh")] iroh, #[cfg(feature = "quic")] @@ -514,6 +525,18 @@ impl Server { Ok(()) } + /// Serve `tls://` on connections another listener accepts, such as an HTTPS + /// server that shares its port with it (see [`crate::tcp::TlsAcceptor`]). + /// + /// `None` when this server loaded no listen TLS identity, as a stream-only + /// server without `--listen-tcp-tls` does not. + #[cfg(all(feature = "tcp", feature = "_certs"))] + pub fn tls_acceptor(&self) -> Option { + let config = self.tls_stream.clone()?; + let server = self.moq.clone().with_versions(self.streams.versions.clone()); + Some(crate::tcp::TlsAcceptor::new(config, server, self.timeout)) + } + /// A live handle to the certificates this server is serving. /// /// Use it to publish the SHA-256 fingerprints of a generated certificate at @@ -1160,7 +1183,7 @@ impl Deadline { all(feature = "uds", unix), feature = "webrtc" ))] -async fn within( +pub(crate) async fn within( deadline: Option, handshake: impl Future>, ) -> crate::Result @@ -1566,26 +1589,39 @@ async fn stream_request( deadline, active, } = incoming; - match accept_request(&setup.server, session, deadline).await { - Ok((request, deadline)) => { - let request = Request { - transport, - url: None, - authority: None, - identity: None, - link, - handshake: Box::new(request), - #[cfg(feature = "quic")] - quic: None, - deadline, - active, - }; + match accept_stream(&setup.server, session, transport, link, deadline, active).await { + Ok(request) => { let _ = tx.send(request).await; } Err(err) => tracing::debug!(%err, "stream SETUP handshake failed"), } } +/// Read SETUP on a stream session, which carries its path in SETUP rather than a URL. +#[cfg(any(feature = "tcp", all(feature = "uds", unix)))] +pub(crate) async fn accept_stream( + server: &moq_net::Server, + session: qmux::Session, + transport: Transport, + link: Link, + deadline: Option, + active: moq_net::server::Active, +) -> crate::Result { + let (request, deadline) = accept_request(server, session, deadline).await?; + Ok(Request { + transport, + url: None, + authority: None, + identity: None, + link, + handshake: Box::new(request), + #[cfg(feature = "quic")] + quic: None, + deadline, + active, + }) +} + /// The transport-level facts a backend captures while accepting a connection, before the /// MoQ SETUP. Grouped so the shared accept loop builds a [`Request`] from named fields /// rather than a wide tuple. diff --git a/rs/moq-tokio/src/tcp.rs b/rs/moq-tokio/src/tcp.rs index 821421ff95e..720c9fd3d29 100644 --- a/rs/moq-tokio/src/tcp.rs +++ b/rs/moq-tokio/src/tcp.rs @@ -231,6 +231,83 @@ async fn connect_addrs( .await } +/// Serves `tls://` on TCP connections another listener accepted, such as a web +/// server that reads each ClientHello so HTTPS and `tls://` share its port. +/// +/// From [`crate::Server::tls_acceptor`]: the listen TLS identity, offering the +/// same `qmux-01.` ALPNs as a [`Listener`] serving `tls://`. +#[cfg(feature = "_certs")] +#[derive(Clone)] +pub struct TlsAcceptor { + config: std::sync::Arc, + qmux: qmux::tls::Server, + server: moq_net::Server, + timeout: Option, +} + +#[cfg(feature = "_certs")] +impl TlsAcceptor { + pub(crate) fn new( + config: std::sync::Arc, + server: moq_net::Server, + timeout: Option, + ) -> Self { + let qmux = qmux::tls::Server::new(config.clone()).with_session_config(crate::qmux_config()); + Self { + config, + qmux, + server, + timeout, + } + } + + /// The config to finish the TLS handshake with: the listen identity and its moq ALPNs. + pub fn config(&self) -> std::sync::Arc { + self.config.clone() + } + + /// Whether a ClientHello offering `alpns` asks for `tls://`: it names one of + /// [`config`](Self::config)'s ALPNs. + pub fn offered<'a>(&self, mut alpns: impl Iterator) -> bool { + alpns.any(|offered| self.config.alpn_protocols.iter().any(|alpn| alpn == offered)) + } + + /// Run the qmux handshake and read SETUP on a connection whose TLS handshake + /// finished with [`config`](Self::config). + /// + /// `link.alpn` names the ALPN that handshake negotiated, which picks the qmux + /// wire format. An unset `link.remote` or `link.local` is read from the socket. + /// The listen timeout bounds the handshake from this call, and `active` keeps + /// the connection counted toward the drain. + pub async fn accept( + &self, + stream: T, + mut link: crate::server::Link, + active: moq_net::server::Active, + ) -> crate::Result + where + T: tokio::io::AsyncRead + tokio::io::AsyncWrite + AsRef + Send + Unpin + 'static, + { + crate::runtime::scope(async move { + let deadline = self.timeout.and_then(|timeout| moq_clock::now().checked_add(timeout)); + let tcp: &tokio::net::TcpStream = stream.as_ref(); + link.remote = link.remote.or_else(|| tcp.peer_addr().ok()); + link.local = link.local.or_else(|| tcp.local_addr().ok()); + let stats = qmux::tcp::adopt(tcp).map_err(Error::Io)?; + let alpn = link.alpn.clone(); + let handshake = async { + self.qmux + .accept_established(stream, alpn.as_deref(), stats) + .await + .map_err(|err| Error::Accept(crate::error::message(err))) + }; + let session = crate::server::within(deadline, handshake).await?; + crate::server::accept_stream(&self.server, session, crate::Transport::Tcp, link, deadline, active).await + }) + .await + } +} + /// Listens for incoming qmux connections on a TCP port, plaintext or TLS. pub struct Listener { listener: tokio::net::TcpListener, diff --git a/rs/qmux/src/tcp.rs b/rs/qmux/src/tcp.rs index 66fb5e41f9f..da82d4b551d 100644 --- a/rs/qmux/src/tcp.rs +++ b/rs/qmux/src/tcp.rs @@ -30,7 +30,10 @@ async fn finish(stream: TcpStream, config: crate::Config, is_server: bool) -> Re /// Disables Nagle: the session already batches records and flushes once per batch, /// so Nagle has nothing left to coalesce and would only hold a batch's tail until /// the peer's delayed ACK. -pub(crate) fn adopt(stream: &TcpStream) -> std::io::Result> { +/// +/// For a socket whose session starts elsewhere, such as +/// `tls::Server::accept_established`. +pub fn adopt(stream: &TcpStream) -> std::io::Result> { stream.set_nodelay(true)?; // The kernel already tracks this socket's RTT (and, on Linux, its delivery diff --git a/rs/qmux/src/tls.rs b/rs/qmux/src/tls.rs index 473b9b19424..a09c5e9ded0 100644 --- a/rs/qmux/src/tls.rs +++ b/rs/qmux/src/tls.rs @@ -152,14 +152,32 @@ impl Server { let tls_stream = acceptor.accept(stream).await?; let negotiated = tls_stream.get_ref().1.alpn_protocol(); - let negotiated_str = negotiated.and_then(|a| std::str::from_utf8(a).ok()); + let negotiated_str = negotiated.and_then(|a| std::str::from_utf8(a).ok()).map(str::to_owned); tracing::debug!(?negotiated_str, "TLS accepted, negotiated ALPN"); + self.accept_established(tls_stream, negotiated_str.as_deref(), socket_stats) + .await + } - let (version, protocol) = alpn::parse(negotiated_str)?; + /// Accept a session over a TLS stream whose handshake the caller already ran, + /// such as a listener that reads the ClientHello to pick a config for its port. + /// + /// `alpn` is the ALPN that handshake negotiated, which names the wire-format + /// version. `socket_stats` reports on the TCP connection underneath; take it + /// before TLS wraps the socket, as [`accept`](Self::accept) does. + pub async fn accept_established( + &self, + stream: T, + alpn: Option<&str>, + socket_stats: Option, + ) -> Result + where + T: tokio::io::AsyncRead + tokio::io::AsyncWrite + Send + 'static, + { + let (version, protocol) = alpn::parse(alpn)?; tracing::debug!(?version, ?protocol, "parsed ALPN"); let session_config = self.session.clone().with_negotiated(version, protocol); - let mut transport = Stream::new(tls_stream); + let mut transport = Stream::new(stream); if let Some(stats) = socket_stats { transport = transport.with_socket_stats(stats); } From b6cbee2c4edce9f34ec11b378a7dfa75e725896a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 13:11:33 -0700 Subject: [PATCH 5/6] fix(relay): gate the acceptor test helper with its callers Co-Authored-By: Claude Opus 5.5 --- rs/moq-relay/src/web.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rs/moq-relay/src/web.rs b/rs/moq-relay/src/web.rs index 89f93abde0d..1075aed7662 100644 --- a/rs/moq-relay/src/web.rs +++ b/rs/moq-relay/src/web.rs @@ -1228,6 +1228,7 @@ mod tests { use super::*; /// The state an acceptor built outside [`Web::serve`] hands a `tls://` session. + #[cfg(all(feature = "websocket", any(unix, windows)))] fn acceptor_state() -> Arc { let cluster = cluster::Cluster::new(crate::cluster::Options::default()).unwrap(); let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); From b201d4220f36b8fc62647b105cd997d1c4fa6148 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 01:30:51 -0800 Subject: [PATCH 6/6] test(relay): time the web listener tests on kio's clock Main's lint disallows tokio::time::timeout workspace-wide. Co-Authored-By: Claude Opus 5.5 --- rs/moq-relay/src/web.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rs/moq-relay/src/web.rs b/rs/moq-relay/src/web.rs index a0349ae9500..f3763c9130a 100644 --- a/rs/moq-relay/src/web.rs +++ b/rs/moq-relay/src/web.rs @@ -2148,7 +2148,7 @@ mod tests { .unwrap() .with_reconnect(false) .connect(url.clone()); - tokio::time::timeout(std::time::Duration::from_secs(10), connection.established()) + kio::clock::timeout(std::time::Duration::from_secs(10), connection.established()) }; let mut https_ca = moq_tokio::tls::Connect::default(); https_ca.root = vec![ca.clone()]; @@ -2221,7 +2221,7 @@ mod tests { .unwrap() .with_reconnect(false) .connect(url); - let refused = tokio::time::timeout(std::time::Duration::from_secs(10), connection.established()) + let refused = kio::clock::timeout(std::time::Duration::from_secs(10), connection.established()) .await .expect("the tls:// dial timed out"); assert!(refused.is_err(), "tls:// accepted a dial without a pinned certificate"); @@ -2325,7 +2325,7 @@ mod tests { for (addr, alpn) in [(http, None), (https, None), (https, Some(&[][..]))] { let mut client = connect(addr, alpn).await; client.write_all(&[3, 1, 2, 3]).await.unwrap(); - let mut rtmp = tokio::time::timeout(std::time::Duration::from_secs(10), incoming.accept()) + let mut rtmp = kio::clock::timeout(std::time::Duration::from_secs(10), incoming.accept()) .await .expect("RTMP reached its consumer") .unwrap(); @@ -2360,7 +2360,7 @@ mod tests { // The connection still classifying does not keep the consumer open past serving. serving.abort(); let _ = serving.await; - let ended = tokio::time::timeout(std::time::Duration::from_secs(10), incoming.accept()).await; + let ended = kio::clock::timeout(std::time::Duration::from_secs(10), incoming.accept()).await; assert!(matches!(ended, Ok(None)), "the consumer ends with serving"); let (serving, http, https, _) = serve(false); @@ -2368,7 +2368,7 @@ mod tests { let mut client = connect(addr, alpn).await; client.write_all(&[3]).await.unwrap(); let mut rest = Vec::new(); - let read = tokio::time::timeout(std::time::Duration::from_secs(10), client.read_to_end(&mut rest)) + let read = kio::clock::timeout(std::time::Duration::from_secs(10), client.read_to_end(&mut rest)) .await .expect("an unserved RTMP connection is closed"); assert!(rest.is_empty(), "{read:?}");