From 16c8a6659e08ddf81f7783a26ff21d0079381e02 Mon Sep 17 00:00:00 2001 From: Zac Shenker Date: Sun, 18 Oct 2026 13:12:14 -0600 Subject: [PATCH 1/9] feat(quic): derive the stateless reset key from listen.reset_key Both drivers built their endpoint from EndpointConfig::default(), which draws a random reset key per process, so a restarted server could not reset its predecessor's connections and peers waited out the idle timeout (#5861). The moq-quic default also keys its hashed connection ID generator per process, so the restarted server dropped those packets as invalid IDs before it could reset them. EndpointConfig::reset_secret derives both keys from an operator secret with HKDF-SHA256. listen.reset_key (--listen-quic-reset-key, MOQ_LISTEN_QUIC_RESET_KEY) names a file holding at least 32 bytes of it; moq-tokio's backend and the relay's io_uring workers apply it. Unset keeps the random key. Closes #5861 Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/config.md | 11 +- rs/moq-quic/README.md | 1 + rs/moq-quic/src/config/mod.rs | 37 ++++++ rs/moq-quic/src/tests/mod.rs | 34 ++++++ rs/moq-relay/src/uring.rs | 12 ++ rs/moq-tokio/src/listen.rs | 68 ++++++++++- rs/moq-tokio/src/quic/backend.rs | 12 +- rs/moq-tokio/src/settings.rs | 3 + rs/moq-tokio/tests/integration/main.rs | 1 + rs/moq-tokio/tests/integration/reset_key.rs | 129 ++++++++++++++++++++ rs/moq-uring/src/quic/driver/endpoint.rs | 6 +- rs/moq-uring/src/quic/driver/mod.rs | 13 +- rs/moq-uring/src/quic/endpoint.rs | 9 ++ 13 files changed, 328 insertions(+), 8 deletions(-) create mode 100644 rs/moq-tokio/tests/integration/reset_key.rs diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index a17ddd07ded..926f601a61d 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -19,6 +19,7 @@ Precedence is CLI, then env, then file, then defaults. bind = "[::]:443" # QUIC (UDP), as --listen. Omit for a stream-only relay. version = ["moq-lite-05"] # Restrict accepted versions. Omit for all. timeout = "10s" # Handshake deadline. "0" waits forever. +reset_key = "/etc/moq/reset.key" # Secret file, 32+ bytes, keyed into stateless resets. Omit for a random key. [listen.tls] cert = "cert.pem" # Certificate chain and key. Reloaded on change. @@ -43,9 +44,15 @@ epoch = false # Decline MoQ Epoch, so routes over moq-tra `timeout` bounds the whole handshake, transport through MoQ SETUP, so a peer that connects and never speaks is closed instead of held open by keep-alives. +`reset_key` lets a relay restarted on the same address reset its +predecessor's QUIC connections, so clients reconnect at once instead of +waiting out the idle timeout. Keep the file secret, and give it to every relay +that should be able to reset the others' connections. Create one with +`openssl rand -hex 32 > reset.key`. + A setting no configured listener reads stops startup rather than being ignored. -A stream-only relay refuses `preferred_v4`, `preferred_v6`, and `lb_id` (or `load_balancer`), which -only QUIC reads, and a `[listen.tls]` `cert`, `key`, or `generate` unless +A stream-only relay refuses `preferred_v4`, `preferred_v6`, `lb_id` (or `load_balancer`), and +`reset_key`, which only QUIC reads, and a `[listen.tls]` `cert`, `key`, or `generate` unless `tcp.tls` serves it. `lb_nonce` needs `lb_id`. `lb_id` and `lb_nonce` cannot be combined with `load_balancer`. `unix.allow` needs `unix.bind`. diff --git a/rs/moq-quic/README.md b/rs/moq-quic/README.md index 7ea6dad5580..c74baa2ed0c 100644 --- a/rs/moq-quic/README.md +++ b/rs/moq-quic/README.md @@ -80,6 +80,7 @@ Changes on top of the upstream commit, besides the renames: - **Draining endpoint event**: `EndpointEvent::is_draining` fires once per connection when it stops being active, before the drained event, so the driver's `Endpoint::wait_all_draining` can end a shutdown without sitting out every closing period. - **Late stream limits**: peer transport parameters that raise a stream limit after an open was refused emit `StreamEvent::Available`, as `MAX_STREAMS` does. A server accepted before the rest of a multi-packet ClientHello (a post-quantum key share is enough) otherwise leaves a 0.5-RTT `open_uni` parked until the idle timeout. Not offered upstream yet. - **Stream credit on stop**: `RecvStream::stop` on a stream whose final size is already known (finished or reset) frees it and queues `MAX_STREAMS` at once, as reading the reset does, instead of leaving the credit until the next packet arrives. Not offered upstream yet; quinn `main` has the same gap. +- **Reset secret** (`EndpointConfig::reset_secret`): derives the stateless reset key from an operator's secret with HKDF-SHA256, and keys a `HashedConnectionIdGenerator` from it too, since a restarted endpoint drops packets for connection IDs it can't validate before it would reset them. Both drivers take one, so a restarted server resets its predecessor's connections. Not offered upstream: it only combines quinn's `reset_key` and `HashedConnectionIdGenerator::from_key`. - **Connection limits** (`Limits`, `Connection::set_limits`, and the drivers' `set_limits`): one validated call changes a live connection's stream limit and windows, refusing values a peer would reject and changing nothing then. moq-tokio and moq-uring re-export the type, so both runtimes check limits the same way. Not offered upstream: it only bundles quinn's existing setters. Recheck the BBR3 changes when cherry-picking quinn#2481 updates, or if it merges. diff --git a/rs/moq-quic/src/config/mod.rs b/rs/moq-quic/src/config/mod.rs index 826d4d02eed..1055769bcd8 100644 --- a/rs/moq-quic/src/config/mod.rs +++ b/rs/moq-quic/src/config/mod.rs @@ -89,6 +89,43 @@ impl EndpointConfig { self } + /// Derive the reset key from `secret` with HKDF-SHA256, so endpoints given the same secret + /// reset each other's connections, including a predecessor's after a restart + /// + /// Also keys a [`HashedConnectionIdGenerator`] from it, since an endpoint drops packets for + /// connection IDs it can't validate instead of resetting them. Set + /// [`cid_generator`](Self::cid_generator) afterwards to use another generator; it must accept + /// the predecessor's IDs for the reset to reach them. + #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] + pub fn reset_secret(&mut self, secret: &[u8]) -> &mut Self { + #[cfg(all(feature = "aws-lc-rs", not(feature = "ring")))] + use aws_lc_rs::{hkdf, hmac}; + #[cfg(feature = "ring")] + use ring::{hkdf, hmac}; + + struct Len(usize); + impl hkdf::KeyType for Len { + fn len(&self) -> usize { + self.0 + } + } + + let prk = hkdf::Salt::new(hkdf::HKDF_SHA256, &[]).extract(secret); + let reset_key = prk + .expand(&[b"moq-quic reset key"], hmac::HMAC_SHA256) + .expect("an HMAC key fits HKDF-SHA256"); + self.reset_key = Arc::new(hmac::Key::from(reset_key)); + + let mut cid_key = [0; 8]; + prk.expand(&[b"moq-quic cid key"], Len(cid_key.len())) + .and_then(|okm| okm.fill(&mut cid_key)) + .expect("8 bytes fit HKDF-SHA256"); + let cid_key = u64::from_le_bytes(cid_key); + self.connection_id_generator_factory = + Arc::new(move || Box::new(HashedConnectionIdGenerator::from_key(cid_key))); + self + } + /// Maximum UDP payload size accepted from peers (excluding UDP and IP overhead). /// /// Must be greater or equal than 1200. diff --git a/rs/moq-quic/src/tests/mod.rs b/rs/moq-quic/src/tests/mod.rs index 74e8509e48a..be414ff3f7e 100644 --- a/rs/moq-quic/src/tests/mod.rs +++ b/rs/moq-quic/src/tests/mod.rs @@ -577,6 +577,40 @@ fn client_stateless_reset() { assert_matches!(conn.poll(), None); } +/// Restart the server from a fresh config built from `restart`, and report whether the client's +/// next packet for the old connection drew a stateless reset +fn restarted_server_resets(restart: &[u8]) -> bool { + let _guard = subscribe(); + let config = |secret: &[u8]| { + let mut config = EndpointConfig::default(); + config.reset_secret(secret); + Arc::new(config) + }; + + let mut pair = Pair::new(config(b"first secret"), server_config()); + let (client_ch, _) = pair.connect(); + pair.drive(); // Flush any post-handshake frames + pair.server.endpoint = Endpoint::new(config(restart), Some(Arc::new(server_config())), true); + pair.client.connections.get_mut(&client_ch).unwrap().ping(); + pair.drive(); + matches!( + pair.client_conn_mut(client_ch).poll(), + Some(Event::ConnectionLost { + reason: ConnectionError::Reset + }) + ) +} + +#[test] +fn reset_secret_survives_a_restart() { + assert!(restarted_server_resets(b"first secret")); +} + +#[test] +fn reset_secret_must_match() { + assert!(!restarted_server_resets(b"other secret")); +} + /// The client reads CONNECTION_CLOSE after the server forgets the connection. /// The client's reply triggers a reset before the application polls. #[test] diff --git a/rs/moq-relay/src/uring.rs b/rs/moq-relay/src/uring.rs index 72973b321a4..2cf8e8b0280 100644 --- a/rs/moq-relay/src/uring.rs +++ b/rs/moq-relay/src/uring.rs @@ -77,6 +77,9 @@ struct Serve { /// `listen.timeout`, which each worker's endpoint starts at a connection's /// first packet. timeout: Option, + /// `listen.reset_key`'s secret, shared so any worker resets any worker's + /// connections, a restarted group's included. + reset_secret: Option>, } /// A bound group of io_uring QUIC workers sharing one port. @@ -102,6 +105,7 @@ pub struct Workers { alpns: Arc>, versions: moq_net::Versions, timeout: Option, + reset_secret: Option>, threads: Vec<(u16, std::thread::JoinHandle<()>)>, stops: Vec>, @@ -214,6 +218,11 @@ impl Workers { let mut udp = moq_uring::udp::Config::default(); udp.gso = quic.gso.unwrap_or(true); + let reset_secret = listen + .reset_secret() + .context("failed to load the stateless reset key")? + .map(Arc::from); + let (failures_tx, failures) = tokio::sync::mpsc::unbounded_channel(); tracing::info!(workers = count, %addr, "bound io_uring QUIC workers"); @@ -228,6 +237,7 @@ impl Workers { alpns: Arc::new(alpns), versions, timeout: listen.resolved_timeout(), + reset_secret, threads: Vec::new(), stops: Vec::new(), failures, @@ -280,6 +290,7 @@ impl Workers { versions: self.versions.clone(), local: self.addr, timeout: self.timeout, + reset_secret: self.reset_secret.clone(), }; let cores = match self.pin { @@ -517,6 +528,7 @@ fn serve_worker(spawn: Spawn) -> bool { let mut endpoint_config = moq_uring::quic::endpoint::Config::default().with_server(server); endpoint_config.activity = serve.shutdown.activity.clone(); endpoint_config.handshake.timeout = serve.timeout; + endpoint_config.reset_secret = serve.reset_secret.clone(); let endpoint = moq_uring::quic::Endpoint::new(socket, endpoint_config).context("failed to build the QUIC endpoint")?; Ok((worker, endpoint)) diff --git a/rs/moq-tokio/src/listen.rs b/rs/moq-tokio/src/listen.rs index 0acf2667450..ef5255f4e8f 100644 --- a/rs/moq-tokio/src/listen.rs +++ b/rs/moq-tokio/src/listen.rs @@ -77,6 +77,9 @@ impl<'de> serde::Deserialize<'de> for Bind { /// How long an accepted connection has to finish its handshake, unless overridden by `--listen-timeout`. pub(crate) const DEFAULT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +/// The shortest reset key secret accepted: 256 bits, if it is random bytes. +const MIN_RESET_SECRET: usize = 32; + /// The accept side of an endpoint: what to listen on and how to be trusted. /// /// Derives [`usage::Args`], so flatten it into a binary's own parser with @@ -249,6 +252,22 @@ pub struct Config { #[serde(default, skip_serializing_if = "Option::is_none")] pub load_balancer: Option, + /// File holding the secret the QUIC stateless reset key is derived from. + /// + /// A server restarted with the same file resets its predecessor's connections, + /// so peers reconnect at once instead of waiting out their idle timeout. Servers + /// sharing it can reset each other's. Unset draws a random key per process. + /// Needs at least 32 bytes, ignoring surrounding whitespace. + #[usage( + name = "listen-quic-reset-key", + long = "listen-quic-reset-key", + env = "MOQ_LISTEN_QUIC_RESET_KEY", + setting = "listen.reset_key" + )] + #[usage(value_hint = usage::ValueHint::FilePath)] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reset_key: Option, + /// The released `--server-*` spellings and their env vars, kept parsing but /// hidden. Never read as settings: [`Config::deprecated`] names what replaced /// each one so a process can say so and stop. @@ -286,6 +305,7 @@ impl Default for Config { lb_id: None, lb_nonce: None, load_balancer: None, + reset_key: None, legacy: Default::default(), quic: None, } @@ -507,6 +527,10 @@ impl Config { self.lb_id.is_some() || self.load_balancer.is_some(), "--listen-quic-lb-id (load_balancer) needs a QUIC listener (--listen)", ), + ( + self.reset_key.is_some(), + "--listen-quic-reset-key needs a QUIC listener (--listen)", + ), ]; match refused.into_iter().find(|(set, _)| *set) { Some((_, reason)) => Err(crate::Error::NoBackend(reason)), @@ -525,6 +549,29 @@ impl Config { (!timeout.is_zero() && std::time::Instant::now().checked_add(timeout).is_some()).then_some(timeout) } + /// Read the secret in the [`reset_key`](Self::reset_key) file, if one is configured. + /// + /// Errors when the file can't be read, or holds fewer than 32 bytes once trimmed. + pub fn reset_secret(&self) -> std::io::Result>> { + let Some(path) = &self.reset_key else { + return Ok(None); + }; + let secret = + std::fs::read(path).map_err(|err| std::io::Error::new(err.kind(), format!("{}: {err}", path.display())))?; + let secret = secret.trim_ascii(); + if secret.len() < MIN_RESET_SECRET { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "{} holds {} bytes, but a reset key needs at least {MIN_RESET_SECRET}", + path.display(), + secret.len() + ), + )); + } + Ok(Some(secret.to_vec())) + } + #[cfg(feature = "quic")] /// Return the effective QUIC-LB connection-ID encoding. pub fn load_balancer(&self) -> Option { @@ -723,6 +770,24 @@ load_balancer = { id = "cd", nonce = 4 } ); } + /// A reset key file is read trimmed, and refused when too short to be a secret. + #[test] + fn reset_key_needs_32_bytes() { + let dir = tempfile::tempdir().unwrap(); + let config = |contents: &[u8]| { + let path = dir.path().join("reset.key"); + std::fs::write(&path, contents).unwrap(); + let toml = format!("reset_key = {:?}", path.display().to_string()); + toml::from_str::(&toml).unwrap().reset_secret() + }; + + assert_eq!(Config::default().reset_secret().unwrap(), None); + let secret = config(format!("{}\n", "a".repeat(32)).as_bytes()).unwrap(); + assert_eq!(secret.as_deref(), Some("a".repeat(32).as_bytes())); + let err = config(format!(" {} \n", "a".repeat(31)).as_bytes()).unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + } + /// Programmatic configuration keeps the QUIC-LB id and nonce paired. #[cfg(feature = "quic")] #[test] @@ -770,7 +835,7 @@ load_balancer = { id = "ab", nonce = 8 } ..Default::default() }; type Set = fn(&mut Config); - let cases: [(&str, Set); 7] = [ + let cases: [(&str, Set); 8] = [ ("--listen-tls-cert", |c| c.tls.cert = vec!["cert.pem".into()]), ("--listen-tls-key", |c| c.tls.key = vec!["key.pem".into()]), ("--listen-tls-generate", |c| c.tls.generate = vec!["localhost".into()]), @@ -787,6 +852,7 @@ load_balancer = { id = "ab", nonce = 8 } nonce: 8, }) }), + ("--listen-quic-reset-key", |c| c.reset_key = Some("reset.key".into())), ]; for (name, set) in cases { let mut config = stream_only(); diff --git a/rs/moq-tokio/src/quic/backend.rs b/rs/moq-tokio/src/quic/backend.rs index f08733e1a9e..cbf767dc7c9 100644 --- a/rs/moq-tokio/src/quic/backend.rs +++ b/rs/moq-tokio/src/quic/backend.rs @@ -209,6 +209,10 @@ pub enum Error { #[error("QUIC-LB connection IDs cannot be combined with per-core workers")] ShardWithQuicLb, + /// The stateless reset key file could not be read, or holds too short a secret. + #[error("failed to load the stateless reset key")] + ResetKey(#[source] std::io::Error), + /// The mTLS client verifier could not be built from the configured roots. #[error("failed to build client certificate verifier")] ClientVerifier(#[source] rustls::server::VerifierBuilderError), @@ -798,8 +802,14 @@ impl Server { .unwrap_or(crate::server::DEFAULT_BIND); let load_balancer = config.load_balancer(); - // Configure connection ID generator with server ID if provided let mut endpoint_config = driver::EndpointConfig::default(); + // First, since it also keys the default connection ID generator. The ones + // below validate a predecessor's IDs without it. + if let Some(secret) = config.reset_secret().map_err(Error::ResetKey)? { + endpoint_config.reset_secret(&secret); + } + + // Configure connection ID generator with server ID if provided if let Some(shard) = member.as_ref().map(listen::Socket::shard) { if load_balancer.is_some() { return Err(Error::ShardWithQuicLb); diff --git a/rs/moq-tokio/src/settings.rs b/rs/moq-tokio/src/settings.rs index 8a13be4c4db..5cc03d01ccc 100644 --- a/rs/moq-tokio/src/settings.rs +++ b/rs/moq-tokio/src/settings.rs @@ -32,6 +32,9 @@ pub struct Listen { #[usage(env = "MOQ_LISTEN_QUIC_LB_NONCE", cli("--listen-quic-lb-nonce"))] lb_nonce: Option, + #[usage(env = "MOQ_LISTEN_QUIC_RESET_KEY", cli("--listen-quic-reset-key"))] + reset_key: Option, + #[usage(flatten)] tls: ListenTls, diff --git a/rs/moq-tokio/tests/integration/main.rs b/rs/moq-tokio/tests/integration/main.rs index 4e926c31778..9c9a5794cf9 100644 --- a/rs/moq-tokio/tests/integration/main.rs +++ b/rs/moq-tokio/tests/integration/main.rs @@ -10,6 +10,7 @@ mod close_code; mod keylog; mod max_age; mod reconnect; +mod reset_key; mod stream_code; mod subscription_end_integrity; mod unannounce; diff --git a/rs/moq-tokio/tests/integration/reset_key.rs b/rs/moq-tokio/tests/integration/reset_key.rs new file mode 100644 index 00000000000..d842f27f18d --- /dev/null +++ b/rs/moq-tokio/tests/integration/reset_key.rs @@ -0,0 +1,129 @@ +//! A server restarted with the same `listen.reset_key` resets its predecessor's +//! connections, so a peer learns at once that they are gone instead of waiting out +//! its idle timeout. +//! +//! The predecessor "crashes" by dropping its runtime, which stops every task before +//! it can send a CONNECTION_CLOSE, and the successor binds the same port. + +#![cfg(feature = "quic")] + +use std::{path::PathBuf, sync::Arc, time::Duration}; + +use moq_tokio::quic::driver; + +struct Fixture { + cert: rustls::pki_types::CertificateDer<'static>, + cert_path: PathBuf, + key_path: PathBuf, + reset_key: PathBuf, + _dir: tempfile::TempDir, +} + +impl Fixture { + fn new() -> Self { + let dir = tempfile::tempdir().expect("tempdir"); + let key = rcgen::KeyPair::generate().expect("key"); + let cert = rcgen::CertificateParams::new(vec!["localhost".to_string()]) + .expect("params") + .self_signed(&key) + .expect("cert"); + let write = |name: &str, contents: &[u8]| { + let path = dir.path().join(name); + std::fs::write(&path, contents).expect("write fixture file"); + path + }; + Self { + cert: cert.der().clone(), + cert_path: write("server.pem", cert.pem().as_bytes()), + key_path: write("server.key", key.serialize_pem().as_bytes()), + reset_key: write("reset.key", &[0x5a; 32]), + _dir: dir, + } + } + + /// Serve on `port` (0 for any) from a runtime of its own, returned with the bound port. + fn serve(&self, port: u16, reset_key: bool) -> (tokio::runtime::Runtime, u16) { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build() + .expect("server runtime"); + + let mut config = moq_tokio::server::Config::default(); + config.listen.bind = Some(format!("127.0.0.1:{port}").parse().unwrap()); + config.listen.tls.cert = vec![self.cert_path.clone()]; + config.listen.tls.key = vec![self.key_path.clone()]; + config.listen.reset_key = reset_key.then(|| self.reset_key.clone()); + + let port = runtime.block_on(async move { + let mut listener = config + .init() + .expect("failed to init server") + .listen() + .await + .expect("failed to listen"); + let port = listener.local_addr().expect("no quic addr").port(); + tokio::spawn(async move { + while let Some(request) = listener.accept().await { + tokio::spawn(async move { + let _session = request.ok().await; + std::future::pending::<()>().await; + }); + } + }); + port + }); + (runtime, port) + } +} + +/// Connect, crash the server and restart it on the same port, then report how the +/// client's connection ended once it next sent something. +fn restart(reset_key: bool) -> Result { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let fixture = Fixture::new(); + let client = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("client runtime"); + + let (server, port) = fixture.serve(0, reset_key); + let (_endpoint, conn) = client.block_on(async { + let mut roots = rustls::RootCertStore::empty(); + roots.add(fixture.cert.clone()).expect("root"); + let mut tls = rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + tls.alpn_protocols = vec![b"moq-lite-06".to_vec()]; + let quic = driver::crypto::rustls::QuicClientConfig::try_from(tls).expect("quic config"); + + let endpoint = driver::Endpoint::client("127.0.0.1:0".parse().unwrap()).expect("endpoint"); + let conn = endpoint + .connect_with( + driver::ClientConfig::new(Arc::new(quic)), + ([127, 0, 0, 1], port).into(), + "localhost", + ) + .expect("connect") + .await + .expect("handshake"); + (endpoint, conn) + }); + + drop(server); + let (_server, _) = fixture.serve(port, reset_key); + + client.block_on(async { + let mut send = conn.open_uni(Default::default()).await.expect("open"); + send.write_all(&[0; 64]).await.expect("write"); + send.finish().expect("finish"); + // Far below the client's 30s idle timeout, and ample for a loopback round trip. + tokio::time::timeout(Duration::from_secs(5), conn.closed()).await + }) +} + +#[test] +fn a_restarted_server_resets_its_predecessors_connections() { + let closed = restart(true); + assert!(matches!(closed, Ok(driver::ConnectionError::Reset)), "{closed:?}"); +} diff --git a/rs/moq-uring/src/quic/driver/endpoint.rs b/rs/moq-uring/src/quic/driver/endpoint.rs index d580274a951..022d62e5c32 100644 --- a/rs/moq-uring/src/quic/driver/endpoint.rs +++ b/rs/moq-uring/src/quic/driver/endpoint.rs @@ -105,7 +105,11 @@ impl Endpoint { let accepting = server.is_some().then(|| Accepting { queue: VecDeque::new() }); // MTU discovery is off (the GSO pool sends fixed SEGMENT datagrams), // so the endpoint has no reason to allow it either. - let endpoint = moq_quic::Endpoint::new(super::endpoint_config(socket.shard())?, server, false); + let endpoint = moq_quic::Endpoint::new( + super::endpoint_config(socket.shard(), config.reset_secret.as_deref())?, + server, + false, + ); let inner = Rc::new(Inner { activity: config.activity, diff --git a/rs/moq-uring/src/quic/driver/mod.rs b/rs/moq-uring/src/quic/driver/mod.rs index 87f5bccbe8b..708497f289a 100644 --- a/rs/moq-uring/src/quic/driver/mod.rs +++ b/rs/moq-uring/src/quic/driver/mod.rs @@ -100,10 +100,17 @@ fn provider() -> Arc { .clone() } -/// The endpoint-wide configuration: how connection ids are minted, and the -/// largest datagram we tell peers we can receive. -pub(crate) fn endpoint_config(shard: Option) -> Result, Error> { +/// The endpoint-wide configuration: how connection ids are minted, the reset +/// key, and the largest datagram we tell peers we can receive. +pub(crate) fn endpoint_config( + shard: Option, + reset_secret: Option<&[u8]>, +) -> Result, Error> { let mut config = moq_quic::EndpointConfig::default(); + // The random ids below validate a predecessor's too, so the reset reaches them. + if let Some(secret) = reset_secret { + config.reset_secret(secret); + } // The steering filter recognizes short headers by their fixed bit. config.grease_quic_bit(shard.is_none()); // moq-quic asks its generator for every id it issues, dials and rotations diff --git a/rs/moq-uring/src/quic/endpoint.rs b/rs/moq-uring/src/quic/endpoint.rs index 6842a87b276..1283cc94691 100644 --- a/rs/moq-uring/src/quic/endpoint.rs +++ b/rs/moq-uring/src/quic/endpoint.rs @@ -47,6 +47,14 @@ pub struct Config { /// How incoming handshakes are bounded. pub handshake: super::handshake::Config, + + /// The secret the stateless reset key is derived from, or `None` for a random + /// key per endpoint. + /// + /// Endpoints given the same secret reset each other's connections, so a + /// restarted one resets its predecessor's instead of leaving peers to wait out + /// their idle timeout. + pub reset_secret: Option>, } impl Default for Config { @@ -56,6 +64,7 @@ impl Default for Config { activity: Default::default(), backlog: 1024, handshake: Default::default(), + reset_secret: None, } } } From 8ad9d4fe73c2b35d75b6e33e9a91cf40e15faed1 Mon Sep 17 00:00:00 2001 From: Zac Shenker Date: Mon, 19 Oct 2026 16:02:31 -0600 Subject: [PATCH 2/9] docs(quic): scope a shared reset key to one relay's restarts Every holder of the reset secret derives the same reset and connection ID keys, so it resets every other holder's connections. RFC 9000 section 21.11 requires that endpoints sharing a static key always deliver a live connection's packets to the instance holding its state. Live relays behind one address that does not route by connection ID would reset each other's clients, and an off-path attacker could trigger it by spoofing. Relays on different addresses gain nothing, since a client only accepts a reset from the address its token came from. Scope the docs to restarts of the same relay, plus instances behind one address that routes by connection ID. Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/config.md | 9 ++++++--- rs/moq-quic/src/config/mod.rs | 8 ++++++-- rs/moq-tokio/src/listen.rs | 8 +++++--- rs/moq-uring/src/quic/endpoint.rs | 7 ++++--- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index 926f601a61d..3a6f6cc39e2 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -46,9 +46,12 @@ that connects and never speaks is closed instead of held open by keep-alives. `reset_key` lets a relay restarted on the same address reset its predecessor's QUIC connections, so clients reconnect at once instead of -waiting out the idle timeout. Keep the file secret, and give it to every relay -that should be able to reset the others' connections. Create one with -`openssl rand -hex 32 > reset.key`. +waiting out the idle timeout. Keep the file secret and to one relay: any holder +resets every other holder's connections, so live relays behind one address +would reset each other's clients unless the load balancer routes by connection +ID (`lb_id`, QUIC-LB). Relays on different addresses gain nothing from sharing +it, since a client accepts a reset only from its relay's address. +Create one with `openssl rand -hex 32 > reset.key`. A setting no configured listener reads stops startup rather than being ignored. A stream-only relay refuses `preferred_v4`, `preferred_v6`, `lb_id` (or `load_balancer`), and diff --git a/rs/moq-quic/src/config/mod.rs b/rs/moq-quic/src/config/mod.rs index 1055769bcd8..96b88f47791 100644 --- a/rs/moq-quic/src/config/mod.rs +++ b/rs/moq-quic/src/config/mod.rs @@ -89,8 +89,12 @@ impl EndpointConfig { self } - /// Derive the reset key from `secret` with HKDF-SHA256, so endpoints given the same secret - /// reset each other's connections, including a predecessor's after a restart + /// Derive the reset key from `secret` with HKDF-SHA256, so an endpoint restarted with the same + /// secret resets its predecessor's connections + /// + /// Every endpoint holding the secret resets every other holder's connections, so RFC 9000 + /// section 21.11 requires that packets for a live connection only ever reach its owner: share + /// it across live endpoints only behind one address that routes by connection ID. /// /// Also keys a [`HashedConnectionIdGenerator`] from it, since an endpoint drops packets for /// connection IDs it can't validate instead of resetting them. Set diff --git a/rs/moq-tokio/src/listen.rs b/rs/moq-tokio/src/listen.rs index ef5255f4e8f..c8b57923b0f 100644 --- a/rs/moq-tokio/src/listen.rs +++ b/rs/moq-tokio/src/listen.rs @@ -255,9 +255,11 @@ pub struct Config { /// File holding the secret the QUIC stateless reset key is derived from. /// /// A server restarted with the same file resets its predecessor's connections, - /// so peers reconnect at once instead of waiting out their idle timeout. Servers - /// sharing it can reset each other's. Unset draws a random key per process. - /// Needs at least 32 bytes, ignoring surrounding whitespace. + /// so peers reconnect at once instead of waiting out their idle timeout. Every + /// holder can reset every other holder's connections, so share it between live + /// servers only behind one address that routes by connection ID (QUIC-LB). + /// Unset draws a random key per process. Needs at least 32 bytes, ignoring + /// surrounding whitespace. #[usage( name = "listen-quic-reset-key", long = "listen-quic-reset-key", diff --git a/rs/moq-uring/src/quic/endpoint.rs b/rs/moq-uring/src/quic/endpoint.rs index 1283cc94691..1a8aa7d9de8 100644 --- a/rs/moq-uring/src/quic/endpoint.rs +++ b/rs/moq-uring/src/quic/endpoint.rs @@ -51,9 +51,10 @@ pub struct Config { /// The secret the stateless reset key is derived from, or `None` for a random /// key per endpoint. /// - /// Endpoints given the same secret reset each other's connections, so a - /// restarted one resets its predecessor's instead of leaving peers to wait out - /// their idle timeout. + /// A restarted endpoint given the same secret resets its predecessor's + /// connections instead of leaving peers to wait out their idle timeout. Live + /// endpoints sharing it reset each other's, so only share it behind one + /// address that routes by connection ID. pub reset_secret: Option>, } From d66dc3085c54029216cd4d2b59dbf1ec5958641b Mon Sep 17 00:00:00 2001 From: Zac Shenker Date: Mon, 19 Oct 2026 16:02:31 -0600 Subject: [PATCH 3/9] feat(quic): reset 1000 connections a second with a reset secret The stateless reset limit is endpoint-wide and defaults to 20ms, 50 resets a second, so a restarted relay with 2,000 connections needs 40s to reset them and most clients still wait out the idle timeout. reset_secret now lowers it to 1ms, resetting those in 2s. A reset is always smaller than the packet that drew it (RFC 9000 section 10.3), so the lower interval never amplifies, and keeping a limit still caps what a flood of spoofed packets can reflect. Co-Authored-By: Claude Opus 5.5 --- rs/moq-quic/README.md | 2 +- rs/moq-quic/src/config/mod.rs | 8 +++++++- rs/moq-quic/src/tests/mod.rs | 37 +++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/rs/moq-quic/README.md b/rs/moq-quic/README.md index c74baa2ed0c..d783ac1bca2 100644 --- a/rs/moq-quic/README.md +++ b/rs/moq-quic/README.md @@ -80,7 +80,7 @@ Changes on top of the upstream commit, besides the renames: - **Draining endpoint event**: `EndpointEvent::is_draining` fires once per connection when it stops being active, before the drained event, so the driver's `Endpoint::wait_all_draining` can end a shutdown without sitting out every closing period. - **Late stream limits**: peer transport parameters that raise a stream limit after an open was refused emit `StreamEvent::Available`, as `MAX_STREAMS` does. A server accepted before the rest of a multi-packet ClientHello (a post-quantum key share is enough) otherwise leaves a 0.5-RTT `open_uni` parked until the idle timeout. Not offered upstream yet. - **Stream credit on stop**: `RecvStream::stop` on a stream whose final size is already known (finished or reset) frees it and queues `MAX_STREAMS` at once, as reading the reset does, instead of leaving the credit until the next packet arrives. Not offered upstream yet; quinn `main` has the same gap. -- **Reset secret** (`EndpointConfig::reset_secret`): derives the stateless reset key from an operator's secret with HKDF-SHA256, and keys a `HashedConnectionIdGenerator` from it too, since a restarted endpoint drops packets for connection IDs it can't validate before it would reset them. Both drivers take one, so a restarted server resets its predecessor's connections. Not offered upstream: it only combines quinn's `reset_key` and `HashedConnectionIdGenerator::from_key`. +- **Reset secret** (`EndpointConfig::reset_secret`): derives the stateless reset key from an operator's secret with HKDF-SHA256, and keys a `HashedConnectionIdGenerator` from it too, since a restarted endpoint drops packets for connection IDs it can't validate before it would reset them. It also lowers `min_reset_interval` to 1ms, so a restart resets thousands of connections in seconds. Both drivers take one, so a restarted server resets its predecessor's connections. Not offered upstream: it only combines quinn's `reset_key` and `HashedConnectionIdGenerator::from_key`. - **Connection limits** (`Limits`, `Connection::set_limits`, and the drivers' `set_limits`): one validated call changes a live connection's stream limit and windows, refusing values a peer would reject and changing nothing then. moq-tokio and moq-uring re-export the type, so both runtimes check limits the same way. Not offered upstream: it only bundles quinn's existing setters. Recheck the BBR3 changes when cherry-picking quinn#2481 updates, or if it merges. diff --git a/rs/moq-quic/src/config/mod.rs b/rs/moq-quic/src/config/mod.rs index 96b88f47791..a7a6cecf0df 100644 --- a/rs/moq-quic/src/config/mod.rs +++ b/rs/moq-quic/src/config/mod.rs @@ -100,6 +100,11 @@ impl EndpointConfig { /// connection IDs it can't validate instead of resetting them. Set /// [`cid_generator`](Self::cid_generator) afterwards to use another generator; it must accept /// the predecessor's IDs for the reset to reach them. + /// + /// Lowers [`min_reset_interval`](Self::min_reset_interval) to 1ms too, since a restart leaves + /// every peer waiting on a reset at once: 1000 a second resets 2,000 connections in 2s, where + /// 20ms takes 40s. A reset is always smaller than the packet that drew it, so this never + /// amplifies, and the limit still caps what a flood of spoofed packets can reflect. #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] pub fn reset_secret(&mut self, secret: &[u8]) -> &mut Self { #[cfg(all(feature = "aws-lc-rs", not(feature = "ring")))] @@ -127,6 +132,7 @@ impl EndpointConfig { let cid_key = u64::from_le_bytes(cid_key); self.connection_id_generator_factory = Arc::new(move || Box::new(HashedConnectionIdGenerator::from_key(cid_key))); + self.min_reset_interval = Duration::from_millis(1); self } @@ -177,7 +183,7 @@ impl EndpointConfig { /// Minimum interval between outgoing stateless reset packets /// - /// Defaults to 20ms. Limits the impact of attacks which flood an endpoint with garbage packets, + /// Defaults to 20ms, or 1ms after [`reset_secret`](Self::reset_secret). Limits the impact of attacks which flood an endpoint with garbage packets, /// e.g. [ISAKMP/IKE amplification]. Larger values provide a stronger defense, but may delay /// detection of some error conditions by clients. Using a [`ConnectionIdGenerator`] with a low /// rate of false positives in [`validate`](ConnectionIdGenerator::validate) reduces the risk diff --git a/rs/moq-quic/src/tests/mod.rs b/rs/moq-quic/src/tests/mod.rs index be414ff3f7e..a4d54e8413f 100644 --- a/rs/moq-quic/src/tests/mod.rs +++ b/rs/moq-quic/src/tests/mod.rs @@ -611,6 +611,43 @@ fn reset_secret_must_match() { assert!(!restarted_server_resets(b"other secret")); } +/// After a restart every peer's next packet draws a reset at once, so a reset secret answers +/// 1000 connections a second, still dropping a packet that arrives within 1ms of the last reset +#[test] +fn reset_secret_resets_connections_each_millisecond() { + let _guard = subscribe(); + let mut config = EndpointConfig::default(); + config.reset_secret(b"secret"); + let mut cids = (config.connection_id_generator_factory)(); + let mut endpoint = Endpoint::new(Arc::new(config), Some(Arc::new(server_config())), true); + + let interval = Duration::from_millis(1); + let start = Instant::now(); + let mut buf = Vec::new(); + for i in 0..2_000u16 { + let remote = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 1024 + i); + let mut packet = vec![0x40]; + packet.extend_from_slice(&cids.generate_cid()); + packet.resize(100, 0); + + let now = start + interval * u32::from(i); + let event = endpoint.handle(now, remote, None, None, packet[..].into(), &mut buf); + assert!( + matches!(event, Some(DatagramEvent::Response(_))), + "connection {i} was not reset" + ); + let event = endpoint.handle( + now + interval / 2, + remote, + None, + None, + packet[..].into(), + &mut buf, + ); + assert!(event.is_none(), "connection {i} was reset twice within 1ms"); + } +} + /// The client reads CONNECTION_CLOSE after the server forgets the connection. /// The client's reply triggers a reset before the application polls. #[test] From 601df6ab8a46658f64c316f5764c7ac368925661 Mon Sep 17 00:00:00 2001 From: Zac Shenker Date: Mon, 19 Oct 2026 16:02:32 -0600 Subject: [PATCH 4/9] test(tokio): show a restart without the reset key leaves peers waiting The integration test only exercised the keyed arm. Add the keyless control, which must time out rather than draw a reset, and tighten the bound to 1s: a reset takes one loopback round trip, and the control waits out the whole bound. Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/tests/integration/reset_key.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/rs/moq-tokio/tests/integration/reset_key.rs b/rs/moq-tokio/tests/integration/reset_key.rs index d842f27f18d..b63495b0c67 100644 --- a/rs/moq-tokio/tests/integration/reset_key.rs +++ b/rs/moq-tokio/tests/integration/reset_key.rs @@ -117,8 +117,9 @@ fn restart(reset_key: bool) -> Result Date: Mon, 19 Oct 2026 16:03:45 -0600 Subject: [PATCH 5/9] docs(quic): tidy the reset key and interval docs Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/config.md | 8 ++++---- rs/moq-quic/src/config/mod.rs | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index 3a6f6cc39e2..cf664e75bcd 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -46,10 +46,10 @@ that connects and never speaks is closed instead of held open by keep-alives. `reset_key` lets a relay restarted on the same address reset its predecessor's QUIC connections, so clients reconnect at once instead of -waiting out the idle timeout. Keep the file secret and to one relay: any holder -resets every other holder's connections, so live relays behind one address -would reset each other's clients unless the load balancer routes by connection -ID (`lb_id`, QUIC-LB). Relays on different addresses gain nothing from sharing +waiting out the idle timeout. Keep the file secret, and give it to one relay +only: any holder resets every other holder's connections, so live relays behind +one address would reset each other's clients unless the load balancer routes by +connection ID (`lb_id`, QUIC-LB). Relays on different addresses gain nothing from sharing it, since a client accepts a reset only from its relay's address. Create one with `openssl rand -hex 32 > reset.key`. diff --git a/rs/moq-quic/src/config/mod.rs b/rs/moq-quic/src/config/mod.rs index a7a6cecf0df..1924de37af6 100644 --- a/rs/moq-quic/src/config/mod.rs +++ b/rs/moq-quic/src/config/mod.rs @@ -183,8 +183,8 @@ impl EndpointConfig { /// Minimum interval between outgoing stateless reset packets /// - /// Defaults to 20ms, or 1ms after [`reset_secret`](Self::reset_secret). Limits the impact of attacks which flood an endpoint with garbage packets, - /// e.g. [ISAKMP/IKE amplification]. Larger values provide a stronger defense, but may delay + /// Defaults to 20ms, or 1ms after [`reset_secret`](Self::reset_secret). Limits the impact of + /// attacks which flood an endpoint with garbage packets, e.g. [ISAKMP/IKE amplification]. Larger values provide a stronger defense, but may delay /// detection of some error conditions by clients. Using a [`ConnectionIdGenerator`] with a low /// rate of false positives in [`validate`](ConnectionIdGenerator::validate) reduces the risk /// incurred by a small minimum reset interval. From 57a2c883d142e8bd468591747086d60b45945bd3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 20 Oct 2026 13:05:26 -0700 Subject: [PATCH 6/9] feat(quic): take listen.reset_secret as 64 hex chars or a file holding them Rename `listen.reset_key` to `listen.reset_secret` (`--listen-quic-reset-secret`, `MOQ_LISTEN_QUIC_RESET_SECRET`) and parse it like `cluster.lan.secret`: exactly 32 bytes, inline hex or a file holding the hex. The secret type moves from `mdns::Secret` to a shared `secret::Secret`. moq-quic's `reset_secret` and moq-uring's `Config::reset_secret` take `[u8; 32]`, so the minimum-length check and the whitespace trim are gone, and moq-uring's `Config` redacts the secret from `Debug`. Deletes the finished quest. Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/config.md | 8 +- quest/m1/README.md | 1 - quest/m1/stateless-reset-key.md | 35 ------ quest/m2/quic-outage-probe.md | 1 - rs/moq-quic/src/config/mod.rs | 6 +- rs/moq-quic/src/tests/mod.rs | 12 +- rs/moq-relay/src/cluster.rs | 2 +- rs/moq-relay/src/uring.rs | 19 +-- rs/moq-tokio/MIGRATE.md | 5 + rs/moq-tokio/src/lib.rs | 1 + rs/moq-tokio/src/listen.rs | 74 +++--------- rs/moq-tokio/src/mdns.rs | 75 +----------- rs/moq-tokio/src/quic/backend.rs | 11 +- rs/moq-tokio/src/secret.rs | 108 ++++++++++++++++++ rs/moq-tokio/src/settings.rs | 4 +- rs/moq-tokio/tests/integration/main.rs | 2 +- .../{reset_key.rs => reset_secret.rs} | 22 ++-- rs/moq-uring/src/quic/driver/endpoint.rs | 2 +- rs/moq-uring/src/quic/driver/mod.rs | 2 +- rs/moq-uring/src/quic/endpoint.rs | 21 +++- 20 files changed, 200 insertions(+), 211 deletions(-) delete mode 100644 quest/m1/stateless-reset-key.md create mode 100644 rs/moq-tokio/src/secret.rs rename rs/moq-tokio/tests/integration/{reset_key.rs => reset_secret.rs} (82%) diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index da5b11b7af7..c7928a9b9ed 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -19,7 +19,7 @@ Precedence is CLI, then env, then file, then defaults. bind = "[::]:443" # QUIC (UDP), as --listen. Omit for a stream-only relay. version = ["moq-lite-05"] # Restrict accepted versions. Omit for all. timeout = "10s" # Handshake deadline. "0" waits forever. -reset_key = "/etc/moq/reset.key" # Secret file, 32+ bytes, keyed into stateless resets. Omit for a random key. +reset_secret = "/etc/moq/reset.key" # 64 hex chars, or a file holding them. Omit for a random key. [listen.tls] cert = "cert.pem" # Certificate chain and key. Reloaded on change. @@ -44,9 +44,9 @@ epoch = false # Decline MoQ Epoch, so routes over moq-tra `timeout` bounds the whole handshake, transport through MoQ SETUP, so a peer that connects and never speaks is closed instead of held open by keep-alives. -`reset_key` lets a relay restarted on the same address reset its +`reset_secret` lets a relay restarted on the same address reset its predecessor's QUIC connections, so clients reconnect at once instead of -waiting out the idle timeout. Keep the file secret, and give it to one relay +waiting out the idle timeout. Keep it secret, and give it to one relay only: any holder resets every other holder's connections, so live relays behind one address would reset each other's clients unless the load balancer routes by connection ID (`lb_id`, QUIC-LB). Relays on different addresses gain nothing from sharing @@ -55,7 +55,7 @@ Create one with `openssl rand -hex 32 > reset.key`. A setting no configured listener reads stops startup rather than being ignored. A stream-only relay refuses `preferred_v4`, `preferred_v6`, `lb_id` (or `load_balancer`), and -`reset_key`, which only QUIC reads, and a `[listen.tls]` `cert`, `key`, or `generate` unless +`reset_secret`, which only QUIC reads, and a `[listen.tls]` `cert`, `key`, or `generate` unless `tcp.tls` serves it. `lb_nonce` needs `lb_id`. `lb_id` and `lb_nonce` cannot be combined with `load_balancer`. `unix.allow` needs `unix.bind`. diff --git a/quest/m1/README.md b/quest/m1/README.md index 0520ca02390..64cf2ad163d 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -89,7 +89,6 @@ lands before the cut. - [A/V clock](/quest/m1/av-clock.md) - the audio playhead drives Sync.reference while audio plays, through per-track sync handles - [Publish channel count](/quest/m1/publish-audio-channel-count.md) - forcing a channel count on an Audio.Capture stops costing the subscriber gaps of silence, and the media fixture's cold-start audio-tone flake is fixed at its cause - [Handshake deadline covers admission](/quest/m1/handshake-admission-deadline.md) - `listen.timeout` bounds a connection through auth admission on both listeners, replacing the hardcoded admission timeout -- [Restart reset](/quest/m1/stateless-reset-key.md) - a relay restarted with the same `listen.reset_key` stateless-resets its predecessor's connections, so clients reconnect within an RTT instead of the idle timeout - [Drain cleanup](/quest/m1/drain-cleanup.md) - sessions force-closed at the drain deadline record their lease end and bytes before the relay or CLI exits, while unfinished handshakes stay bounded - [GOAWAY coverage](/quest/m1/goaway-coverage.md) - every IETF request stream handles GOAWAY, a closed control stream is a violation, and the draft-18 cutoff is honored - [Container reads drop backfill](/quest/m1/read-backfill.md) - the JS container consumer drops groups below its read cursor like Rust diff --git a/quest/m1/stateless-reset-key.md b/quest/m1/stateless-reset-key.md deleted file mode 100644 index f0c9aacb8b9..00000000000 --- a/quest/m1/stateless-reset-key.md +++ /dev/null @@ -1,35 +0,0 @@ -# [S] A restarted relay resets its predecessor's connections - -## Goal - -A relay restarted on the same address with the same configured secret -stateless-resets the connections its predecessor held, so a publisher learns -within one RTT of the relay coming back and a subscriber within one -keep-alive interval, instead of waiting out the QUIC idle timeout. A relay -without the secret keeps today's random per-process key. - -## Plan - -Decided 2026-10-19 by the maintainer (#5861): land -[#5910](https://github.com/moq-dev/moq/pull/5910), which already builds it. - -- The secret is a file named by `[listen] reset_key` - (`--listen-quic-reset-key`, `MOQ_LISTEN_QUIC_RESET_KEY`), not derived from - the TLS key: ACME and certificate rotation would change a derived key - silently. -- HKDF derives both the reset key and the hashed connection ID key from it. - An unsharded, non-LB tokio server hashes connection IDs with a per-process - key too, so a stable reset key alone would drop the old IDs as invalid - before resetting them. -- Cover tokio and io_uring, with the crash-and-restart test #5910 carries. -- #5910 rebases onto [#5878](https://github.com/moq-dev/moq/pull/5878) - (receive leases). -- Don't reuse `cluster.token` or `cluster.lan.secret`: one is a credential, - the other keys the LAN mesh. - -Public API: a new relay config field and `moq_quic::EndpointConfig::reset_secret`. -Docs: `doc/bin/relay/`. - -## Closes - -- [#5861](https://github.com/moq-dev/moq/issues/5861) - a restarted relay can't stateless-reset its predecessor's connections diff --git a/quest/m2/quic-outage-probe.md b/quest/m2/quic-outage-probe.md index 0c2d0c082f1..7fe096ee012 100644 --- a/quest/m2/quic-outage-probe.md +++ b/quest/m2/quic-outage-probe.md @@ -35,4 +35,3 @@ Public API: none. - [Keep-alive from the idle timeout](/quest/m2/quic-keep-alive.md) - the other outage-time knob - [Per-stream deadlines](/quest/m2/quic-deadline.md) - an early probe ahead of a deadline -- [Restart reset](/quest/m1/stateless-reset-key.md) - the restart half of #5953's item 4 diff --git a/rs/moq-quic/src/config/mod.rs b/rs/moq-quic/src/config/mod.rs index 1924de37af6..69f06de8a07 100644 --- a/rs/moq-quic/src/config/mod.rs +++ b/rs/moq-quic/src/config/mod.rs @@ -89,8 +89,8 @@ impl EndpointConfig { self } - /// Derive the reset key from `secret` with HKDF-SHA256, so an endpoint restarted with the same - /// secret resets its predecessor's connections + /// Derive the reset key from a random 32-byte `secret` with HKDF-SHA256, so an endpoint + /// restarted with the same secret resets its predecessor's connections /// /// Every endpoint holding the secret resets every other holder's connections, so RFC 9000 /// section 21.11 requires that packets for a live connection only ever reach its owner: share @@ -106,7 +106,7 @@ impl EndpointConfig { /// 20ms takes 40s. A reset is always smaller than the packet that drew it, so this never /// amplifies, and the limit still caps what a flood of spoofed packets can reflect. #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] - pub fn reset_secret(&mut self, secret: &[u8]) -> &mut Self { + pub fn reset_secret(&mut self, secret: &[u8; 32]) -> &mut Self { #[cfg(all(feature = "aws-lc-rs", not(feature = "ring")))] use aws_lc_rs::{hkdf, hmac}; #[cfg(feature = "ring")] diff --git a/rs/moq-quic/src/tests/mod.rs b/rs/moq-quic/src/tests/mod.rs index a4d54e8413f..768204b67de 100644 --- a/rs/moq-quic/src/tests/mod.rs +++ b/rs/moq-quic/src/tests/mod.rs @@ -579,15 +579,15 @@ fn client_stateless_reset() { /// Restart the server from a fresh config built from `restart`, and report whether the client's /// next packet for the old connection drew a stateless reset -fn restarted_server_resets(restart: &[u8]) -> bool { +fn restarted_server_resets(restart: &[u8; 32]) -> bool { let _guard = subscribe(); - let config = |secret: &[u8]| { + let config = |secret: &[u8; 32]| { let mut config = EndpointConfig::default(); config.reset_secret(secret); Arc::new(config) }; - let mut pair = Pair::new(config(b"first secret"), server_config()); + let mut pair = Pair::new(config(&[1; 32]), server_config()); let (client_ch, _) = pair.connect(); pair.drive(); // Flush any post-handshake frames pair.server.endpoint = Endpoint::new(config(restart), Some(Arc::new(server_config())), true); @@ -603,12 +603,12 @@ fn restarted_server_resets(restart: &[u8]) -> bool { #[test] fn reset_secret_survives_a_restart() { - assert!(restarted_server_resets(b"first secret")); + assert!(restarted_server_resets(&[1; 32])); } #[test] fn reset_secret_must_match() { - assert!(!restarted_server_resets(b"other secret")); + assert!(!restarted_server_resets(&[2; 32])); } /// After a restart every peer's next packet draws a reset at once, so a reset secret answers @@ -617,7 +617,7 @@ fn reset_secret_must_match() { fn reset_secret_resets_connections_each_millisecond() { let _guard = subscribe(); let mut config = EndpointConfig::default(); - config.reset_secret(b"secret"); + config.reset_secret(&[1; 32]); let mut cids = (config.connection_id_generator_factory)(); let mut endpoint = Endpoint::new(Arc::new(config), Some(Arc::new(server_config())), true); diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index c28c8304f59..50d2a4cda0f 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -2157,7 +2157,7 @@ async fn lan_discovery( config = config.with_node(url); } if let Some(secret) = secret { - let secret = moq_tokio::mdns::Secret::load(secret).context("invalid --cluster-lan-secret")?; + let secret = moq_tokio::secret::Secret::load(secret).context("invalid --cluster-lan-secret")?; config = config.with_secret(secret); } Ok(config.advertise().await?) diff --git a/rs/moq-relay/src/uring.rs b/rs/moq-relay/src/uring.rs index f7164f8dc76..0428b17fc0c 100644 --- a/rs/moq-relay/src/uring.rs +++ b/rs/moq-relay/src/uring.rs @@ -77,9 +77,9 @@ struct Serve { /// `listen.timeout`, which each worker's endpoint starts at a connection's /// first packet. timeout: Option, - /// `listen.reset_key`'s secret, shared so any worker resets any worker's + /// `listen.reset_secret`, shared so any worker resets any worker's /// connections, a restarted group's included. - reset_secret: Option>, + reset_secret: Option<[u8; 32]>, } /// A bound group of io_uring QUIC workers sharing one port. @@ -105,7 +105,7 @@ pub struct Workers { alpns: Arc>, versions: moq_net::Versions, timeout: Option, - reset_secret: Option>, + reset_secret: Option<[u8; 32]>, threads: Vec<(u16, std::thread::JoinHandle<()>)>, stops: Vec>, @@ -219,9 +219,12 @@ impl Workers { udp.gso = quic.gso.unwrap_or(true); let reset_secret = listen - .reset_secret() - .context("failed to load the stateless reset key")? - .map(Arc::from); + .reset_secret + .as_deref() + .map(moq_tokio::secret::Secret::load) + .transpose() + .context("invalid --listen-quic-reset-secret")? + .map(|secret| *secret.as_bytes()); let (failures_tx, failures) = tokio::sync::mpsc::unbounded_channel(); tracing::info!(workers = count, %addr, "bound io_uring QUIC workers"); @@ -290,7 +293,7 @@ impl Workers { versions: self.versions.clone(), local: self.addr, timeout: self.timeout, - reset_secret: self.reset_secret.clone(), + reset_secret: self.reset_secret, }; let cores = match self.pin { @@ -533,7 +536,7 @@ fn serve_worker(spawn: Spawn) -> bool { let mut endpoint_config = moq_uring::quic::endpoint::Config::default().with_server(server); endpoint_config.activity = serve.shutdown.activity.clone(); endpoint_config.handshake.timeout = serve.timeout; - endpoint_config.reset_secret = serve.reset_secret.clone(); + endpoint_config.reset_secret = serve.reset_secret; let endpoint = moq_uring::quic::Endpoint::new(socket, endpoint_config).context("failed to build the QUIC endpoint")?; Ok((worker, endpoint)) diff --git a/rs/moq-tokio/MIGRATE.md b/rs/moq-tokio/MIGRATE.md index e76df241415..7928ecb6f6e 100644 --- a/rs/moq-tokio/MIGRATE.md +++ b/rs/moq-tokio/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `mdns::Secret` is `secret::Secret` (#5910) + +The 32-byte secret type moved out of `mdns` so `listen.reset_secret` shares it. +Name it `moq_tokio::secret::Secret`; `new` and `load` now return `secret::Error` instead of `mdns::Error`. + ### `jemalloc::tikv_jemallocator` is 0.7 (#6053) The re-exported allocator crate moved from tikv-jemallocator 0.6 to 0.7. diff --git a/rs/moq-tokio/src/lib.rs b/rs/moq-tokio/src/lib.rs index 82c6a029d1e..2910e76dc83 100644 --- a/rs/moq-tokio/src/lib.rs +++ b/rs/moq-tokio/src/lib.rs @@ -46,6 +46,7 @@ pub mod panic; pub mod quic; #[cfg(any(feature = "quic", feature = "tcp", feature = "websocket"))] mod resolve; +pub mod secret; #[cfg(feature = "_transport")] pub mod server; #[doc(hidden)] diff --git a/rs/moq-tokio/src/listen.rs b/rs/moq-tokio/src/listen.rs index 96d311a8fbf..8863959ce10 100644 --- a/rs/moq-tokio/src/listen.rs +++ b/rs/moq-tokio/src/listen.rs @@ -77,9 +77,6 @@ impl<'de> serde::Deserialize<'de> for Bind { /// How long an accepted connection has to finish its handshake, unless overridden by `--listen-timeout`. pub(crate) const DEFAULT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); -/// The shortest reset key secret accepted: 256 bits, if it is random bytes. -const MIN_RESET_SECRET: usize = 32; - /// The accept side of an endpoint: what to listen on and how to be trusted. /// /// Derives [`usage_rs::Args`], so flatten it into a binary's own parser with @@ -252,23 +249,23 @@ pub struct Config { #[serde(default, skip_serializing_if = "Option::is_none")] pub load_balancer: Option, - /// File holding the secret the QUIC stateless reset key is derived from. + /// The secret the QUIC stateless reset key is derived from, as 64 hexadecimal + /// characters or a path to a file containing them. /// - /// A server restarted with the same file resets its predecessor's connections, + /// A server restarted with the same secret resets its predecessor's connections, /// so peers reconnect at once instead of waiting out their idle timeout. Every /// holder can reset every other holder's connections, so share it between live /// servers only behind one address that routes by connection ID (QUIC-LB). - /// Unset draws a random key per process. Needs at least 32 bytes, ignoring - /// surrounding whitespace. + /// Unset draws a random key per process. #[usage( - name = "listen-quic-reset-key", - long = "listen-quic-reset-key", - env = "MOQ_LISTEN_QUIC_RESET_KEY", - setting = "listen.reset_key" + name = "listen-quic-reset-secret", + long = "listen-quic-reset-secret", + env = "MOQ_LISTEN_QUIC_RESET_SECRET", + value_name = "HEX_OR_PATH", + setting = "listen.reset_secret" )] - #[usage(value_hint = usage::ValueHint::FilePath)] #[serde(default, skip_serializing_if = "Option::is_none")] - pub reset_key: Option, + pub reset_secret: Option, /// The released `--server-*` spellings and their env vars, kept parsing but /// hidden. Never read as settings: [`Config::deprecated`] names what replaced @@ -307,7 +304,7 @@ impl Default for Config { lb_id: None, lb_nonce: None, load_balancer: None, - reset_key: None, + reset_secret: None, legacy: Default::default(), quic: None, } @@ -530,8 +527,8 @@ impl Config { "--listen-quic-lb-id (load_balancer) needs a QUIC listener (--listen)", ), ( - self.reset_key.is_some(), - "--listen-quic-reset-key needs a QUIC listener (--listen)", + self.reset_secret.is_some(), + "--listen-quic-reset-secret needs a QUIC listener (--listen)", ), ]; match refused.into_iter().find(|(set, _)| *set) { @@ -551,29 +548,6 @@ impl Config { (!timeout.is_zero() && std::time::Instant::now().checked_add(timeout).is_some()).then_some(timeout) } - /// Read the secret in the [`reset_key`](Self::reset_key) file, if one is configured. - /// - /// Errors when the file can't be read, or holds fewer than 32 bytes once trimmed. - pub fn reset_secret(&self) -> std::io::Result>> { - let Some(path) = &self.reset_key else { - return Ok(None); - }; - let secret = - std::fs::read(path).map_err(|err| std::io::Error::new(err.kind(), format!("{}: {err}", path.display())))?; - let secret = secret.trim_ascii(); - if secret.len() < MIN_RESET_SECRET { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!( - "{} holds {} bytes, but a reset key needs at least {MIN_RESET_SECRET}", - path.display(), - secret.len() - ), - )); - } - Ok(Some(secret.to_vec())) - } - #[cfg(feature = "quic")] /// Return the effective QUIC-LB connection-ID encoding. pub fn load_balancer(&self) -> Option { @@ -772,24 +746,6 @@ load_balancer = { id = "cd", nonce = 4 } ); } - /// A reset key file is read trimmed, and refused when too short to be a secret. - #[test] - fn reset_key_needs_32_bytes() { - let dir = tempfile::tempdir().unwrap(); - let config = |contents: &[u8]| { - let path = dir.path().join("reset.key"); - std::fs::write(&path, contents).unwrap(); - let toml = format!("reset_key = {:?}", path.display().to_string()); - toml::from_str::(&toml).unwrap().reset_secret() - }; - - assert_eq!(Config::default().reset_secret().unwrap(), None); - let secret = config(format!("{}\n", "a".repeat(32)).as_bytes()).unwrap(); - assert_eq!(secret.as_deref(), Some("a".repeat(32).as_bytes())); - let err = config(format!(" {} \n", "a".repeat(31)).as_bytes()).unwrap_err(); - assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); - } - /// Programmatic configuration keeps the QUIC-LB id and nonce paired. #[cfg(feature = "quic")] #[test] @@ -854,7 +810,9 @@ load_balancer = { id = "ab", nonce = 8 } nonce: 8, }) }), - ("--listen-quic-reset-key", |c| c.reset_key = Some("reset.key".into())), + ("--listen-quic-reset-secret", |c| { + c.reset_secret = Some("reset.key".into()) + }), ]; for (name, set) in cases { let mut config = stream_only(); diff --git a/rs/moq-tokio/src/mdns.rs b/rs/moq-tokio/src/mdns.rs index 18a31deba37..4cd0b0e45f1 100644 --- a/rs/moq-tokio/src/mdns.rs +++ b/rs/moq-tokio/src/mdns.rs @@ -36,6 +36,8 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer}; use subtle::ConstantTimeEq; use url::Url; +use crate::secret::Secret; + /// The DNS-SD service MoQ processes advertise under. const SERVICE_TYPE: &str = "_moq._udp.local."; @@ -74,14 +76,6 @@ enum ErrorKind { #[error(transparent)] Mdns(#[from] mdns_sd::Error), - #[error( - "secret must be exactly 32 bytes encoded as 64 hexadecimal characters, or a path to a file containing them" - )] - InvalidSecret, - - #[error("secret must be 64 hexadecimal characters or a path to a file containing them, and {0} is neither")] - SecretFile(String, #[source] std::io::Error), - #[error("app must be 1 to 62 lowercase ASCII letters, digits, or hyphens")] InvalidApp, @@ -176,58 +170,6 @@ impl<'de> Deserialize<'de> for App { } } -/// A 32-byte pre-shared key proving membership of a discovery group. -/// -/// Peers holding different secrets (or none) never see each other. The proofs -/// are HMACs over a public nonce, so the secret never travels the network but a -/// weak one can be brute-forced offline by anyone watching it; generate a random -/// 32 bytes rather than choosing one. -#[derive(Clone, PartialEq, Eq)] -pub struct Secret([u8; 32]); - -impl Secret { - /// Decode a key from exactly 64 hexadecimal characters. - pub fn new(secret: impl AsRef) -> Result { - let secret = secret.as_ref(); - let mut key = [0; 32]; - if secret.len() != key.len() * 2 || hex::decode_to_slice(secret, &mut key).is_err() { - return Err(Error(ErrorKind::InvalidSecret)); - } - Ok(Self(key)) - } - - /// Decode a key given inline, or read it from the file `value` names. - /// - /// What a `--*-secret ` flag accepts. A path that exists but - /// holds something other than a key is an error rather than a silent fallback, - /// and a path that doesn't exist is never created. - pub fn load(value: impl AsRef) -> Result { - let value = value.as_ref(); - if let Ok(secret) = Self::new(value) { - return Ok(secret); - } - // Not a key, so it can only have been meant as a path. A read failure here - // is the useful error: it names the file the operator actually typed. - let contents = - std::fs::read_to_string(value).map_err(|err| Error(ErrorKind::SecretFile(value.to_string(), err)))?; - Self::new(contents.trim()) - } -} - -impl FromStr for Secret { - type Err = Error; - - fn from_str(secret: &str) -> Result { - Self::new(secret) - } -} - -impl fmt::Debug for Secret { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("Secret([redacted])") - } -} - /// Everything a proof is bound to, so one captured off the network /// authenticates nowhere else. /// @@ -249,7 +191,7 @@ struct Identity<'a> { /// An HMAC-SHA256 proof of knowing `secret`, bound to one listener's /// [`Identity`], plus a `context` separating the advertise and dial roles. fn proof(secret: &Secret, context: &str, id: &Identity<'_>) -> String { - let mut mac = hmac::Hmac::::new_from_slice(&secret.0).expect("HMAC accepts any key length"); + let mut mac = hmac::Hmac::::new_from_slice(secret.as_bytes()).expect("HMAC accepts any key length"); let port = id.port.to_string(); for field in [ context, @@ -850,17 +792,6 @@ mod tests { assert_eq!(instance_id("not-a-moq-service._http._tcp.local."), None); } - #[test] - fn secret_requires_exactly_32_hex_bytes_and_redacts_debug() { - for invalid in ["", "00", &"x".repeat(64), &"00".repeat(33)] { - assert!(Secret::new(invalid).is_err(), "{invalid:?} must not parse"); - } - assert_eq!( - format!("{:?}", Secret::new(KEY_A).expect("valid")), - "Secret([redacted])" - ); - } - /// 62, not 63: the subtype label is `_`, and a DNS label holds 63 octets. #[test] fn app_is_a_dns_sd_subtype_label() { diff --git a/rs/moq-tokio/src/quic/backend.rs b/rs/moq-tokio/src/quic/backend.rs index 311cab06e33..67fbcdb91d7 100644 --- a/rs/moq-tokio/src/quic/backend.rs +++ b/rs/moq-tokio/src/quic/backend.rs @@ -210,9 +210,9 @@ pub enum Error { #[error("QUIC-LB connection IDs cannot be combined with per-core workers")] ShardWithQuicLb, - /// The stateless reset key file could not be read, or holds too short a secret. - #[error("failed to load the stateless reset key")] - ResetKey(#[source] std::io::Error), + /// The stateless reset secret is neither 64 hexadecimal characters nor a file holding them. + #[error("invalid stateless reset secret")] + ResetSecret(#[source] crate::secret::Error), /// The mTLS client verifier could not be built from the configured roots. #[error("failed to build client certificate verifier")] @@ -806,8 +806,9 @@ impl Server { let mut endpoint_config = driver::EndpointConfig::default(); // First, since it also keys the default connection ID generator. The ones // below validate a predecessor's IDs without it. - if let Some(secret) = config.reset_secret().map_err(Error::ResetKey)? { - endpoint_config.reset_secret(&secret); + if let Some(secret) = &config.reset_secret { + let secret = crate::secret::Secret::load(secret).map_err(Error::ResetSecret)?; + endpoint_config.reset_secret(secret.as_bytes()); } // Configure connection ID generator with server ID if provided diff --git a/rs/moq-tokio/src/secret.rs b/rs/moq-tokio/src/secret.rs new file mode 100644 index 00000000000..3d31b0fc35c --- /dev/null +++ b/rs/moq-tokio/src/secret.rs @@ -0,0 +1,108 @@ +//! A 32-byte secret an operator configures, such as `cluster.lan.secret` or +//! `listen.reset_secret`. + +use std::fmt; +use std::str::FromStr; + +/// A secret could not be decoded or read. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + /// The value is not exactly 32 bytes encoded as 64 hexadecimal characters. + #[error( + "secret must be exactly 32 bytes encoded as 64 hexadecimal characters, or a path to a file containing them" + )] + Invalid, + + /// The value is not a secret, and not a file that could be read either. + #[error("secret must be 64 hexadecimal characters or a path to a file containing them, and {0} is neither")] + File(String, #[source] std::io::Error), +} + +/// A random 32-byte secret, given as 64 hexadecimal characters. +/// +/// Anyone watching what it keys can brute-force a weak one offline, so generate +/// it (`openssl rand -hex 32`) rather than choosing one. +#[derive(Clone, PartialEq, Eq)] +pub struct Secret([u8; 32]); + +impl Secret { + /// Decode a secret from exactly 64 hexadecimal characters. + pub fn new(secret: impl AsRef) -> Result { + let secret = secret.as_ref(); + let mut key = [0; 32]; + if secret.len() != key.len() * 2 || hex::decode_to_slice(secret, &mut key).is_err() { + return Err(Error::Invalid); + } + Ok(Self(key)) + } + + /// Decode a secret given inline, or read it from the file `value` names. + /// + /// What a `--*-secret ` flag accepts. A path that exists but + /// holds something other than a secret is an error rather than a silent + /// fallback, and a path that doesn't exist is never created. + pub fn load(value: impl AsRef) -> Result { + let value = value.as_ref(); + if let Ok(secret) = Self::new(value) { + return Ok(secret); + } + // Not a secret, so it can only have been meant as a path. A read failure + // here is the useful error: it names the file the operator actually typed. + let contents = std::fs::read_to_string(value).map_err(|err| Error::File(value.to_string(), err))?; + Self::new(contents.trim()) + } + + /// The secret's 32 bytes. + pub fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } +} + +impl FromStr for Secret { + type Err = Error; + + fn from_str(secret: &str) -> Result { + Self::new(secret) + } +} + +impl fmt::Debug for Secret { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("Secret([redacted])") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const KEY: &str = "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f"; + + #[test] + fn new_requires_exactly_32_hex_bytes_and_redacts_debug() { + for invalid in ["", "00", &"x".repeat(64), &"00".repeat(33)] { + assert!(Secret::new(invalid).is_err(), "{invalid:?} must not parse"); + } + assert_eq!(format!("{:?}", Secret::new(KEY).expect("valid")), "Secret([redacted])"); + } + + /// A file holding the hex, as `openssl rand -hex 32` writes it, decodes to the + /// same secret as the hex given inline. + #[test] + fn load_reads_inline_hex_or_a_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("secret.key"); + std::fs::write(&path, format!("{KEY}\n")).unwrap(); + + let inline = Secret::load(KEY).unwrap(); + assert_eq!(Secret::load(path.to_str().unwrap()).unwrap(), inline); + assert_eq!(inline.as_bytes()[31], 0x1f); + + // Raw bytes are refused rather than taken as a secret. + std::fs::write(&path, [0x5a; 32]).unwrap(); + assert!(matches!(Secret::load(path.to_str().unwrap()), Err(Error::Invalid))); + let missing = dir.path().join("missing.key"); + assert!(matches!(Secret::load(missing.to_str().unwrap()), Err(Error::File(..)))); + } +} diff --git a/rs/moq-tokio/src/settings.rs b/rs/moq-tokio/src/settings.rs index 397db205eca..9e7f9610531 100644 --- a/rs/moq-tokio/src/settings.rs +++ b/rs/moq-tokio/src/settings.rs @@ -32,8 +32,8 @@ pub struct Listen { #[usage(env = "MOQ_LISTEN_QUIC_LB_NONCE", cli("--listen-quic-lb-nonce"))] lb_nonce: Option, - #[usage(env = "MOQ_LISTEN_QUIC_RESET_KEY", cli("--listen-quic-reset-key"))] - reset_key: Option, + #[usage(env = "MOQ_LISTEN_QUIC_RESET_SECRET", cli("--listen-quic-reset-secret"))] + reset_secret: Option, #[usage(flatten)] tls: ListenTls, diff --git a/rs/moq-tokio/tests/integration/main.rs b/rs/moq-tokio/tests/integration/main.rs index 3719d22f60b..e95dab914d1 100644 --- a/rs/moq-tokio/tests/integration/main.rs +++ b/rs/moq-tokio/tests/integration/main.rs @@ -10,7 +10,7 @@ mod close_code; mod keylog; mod max_age; mod reconnect; -mod reset_key; +mod reset_secret; mod stream_code; mod subscription_end_integrity; mod unannounce; diff --git a/rs/moq-tokio/tests/integration/reset_key.rs b/rs/moq-tokio/tests/integration/reset_secret.rs similarity index 82% rename from rs/moq-tokio/tests/integration/reset_key.rs rename to rs/moq-tokio/tests/integration/reset_secret.rs index b63495b0c67..c399871f00f 100644 --- a/rs/moq-tokio/tests/integration/reset_key.rs +++ b/rs/moq-tokio/tests/integration/reset_secret.rs @@ -1,4 +1,4 @@ -//! A server restarted with the same `listen.reset_key` resets its predecessor's +//! A server restarted with the same `listen.reset_secret` resets its predecessor's //! connections, so a peer learns at once that they are gone instead of waiting out //! its idle timeout. //! @@ -11,11 +11,13 @@ use std::{path::PathBuf, sync::Arc, time::Duration}; use moq_tokio::quic::driver; +const RESET_SECRET: &str = "5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a"; + struct Fixture { cert: rustls::pki_types::CertificateDer<'static>, cert_path: PathBuf, key_path: PathBuf, - reset_key: PathBuf, + reset_secret: PathBuf, _dir: tempfile::TempDir, } @@ -36,13 +38,13 @@ impl Fixture { cert: cert.der().clone(), cert_path: write("server.pem", cert.pem().as_bytes()), key_path: write("server.key", key.serialize_pem().as_bytes()), - reset_key: write("reset.key", &[0x5a; 32]), + reset_secret: write("reset.key", format!("{RESET_SECRET}\n").as_bytes()), _dir: dir, } } /// Serve on `port` (0 for any) from a runtime of its own, returned with the bound port. - fn serve(&self, port: u16, reset_key: bool) -> (tokio::runtime::Runtime, u16) { + fn serve(&self, port: u16, reset_secret: Option) -> (tokio::runtime::Runtime, u16) { let runtime = tokio::runtime::Builder::new_multi_thread() .worker_threads(1) .enable_all() @@ -53,7 +55,7 @@ impl Fixture { config.listen.bind = Some(format!("127.0.0.1:{port}").parse().unwrap()); config.listen.tls.cert = vec![self.cert_path.clone()]; config.listen.tls.key = vec![self.key_path.clone()]; - config.listen.reset_key = reset_key.then(|| self.reset_key.clone()); + config.listen.reset_secret = reset_secret; let port = runtime.block_on(async move { let mut listener = config @@ -79,7 +81,10 @@ impl Fixture { /// Connect, crash the server and restart it on the same port, then report how the /// client's connection ended once it next sent something. -fn restart(reset_key: bool) -> Result { +/// +/// With a secret, the predecessor reads it from a file and the successor takes the +/// same hex inline, so both forms must derive the same key. +fn restart(reset_secret: bool) -> Result { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); let fixture = Fixture::new(); let client = tokio::runtime::Builder::new_current_thread() @@ -87,7 +92,8 @@ fn restart(reset_key: bool) -> Result Result Arc { /// key, and the largest datagram we tell peers we can receive. pub(crate) fn endpoint_config( shard: Option, - reset_secret: Option<&[u8]>, + reset_secret: Option<&[u8; 32]>, ) -> Result, Error> { let mut config = moq_quic::EndpointConfig::default(); // The random ids below validate a predecessor's too, so the reset reaches them. diff --git a/rs/moq-uring/src/quic/endpoint.rs b/rs/moq-uring/src/quic/endpoint.rs index 1a8aa7d9de8..b6a95752e5f 100644 --- a/rs/moq-uring/src/quic/endpoint.rs +++ b/rs/moq-uring/src/quic/endpoint.rs @@ -28,7 +28,7 @@ use super::server; pub use super::driver::Endpoint; /// What an endpoint serves, beyond dialing. -#[derive(Clone, Debug)] +#[derive(Clone)] #[non_exhaustive] pub struct Config { /// Tracks accepted connections from admission until transport closure. @@ -48,14 +48,27 @@ pub struct Config { /// How incoming handshakes are bounded. pub handshake: super::handshake::Config, - /// The secret the stateless reset key is derived from, or `None` for a random - /// key per endpoint. + /// The random 32-byte secret the stateless reset key is derived from, or + /// `None` for a random key per endpoint. /// /// A restarted endpoint given the same secret resets its predecessor's /// connections instead of leaving peers to wait out their idle timeout. Live /// endpoints sharing it reset each other's, so only share it behind one /// address that routes by connection ID. - pub reset_secret: Option>, + pub reset_secret: Option<[u8; 32]>, +} + +// By hand, so the reset secret never reaches a log. +impl std::fmt::Debug for Config { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Config") + .field("activity", &self.activity) + .field("server", &self.server) + .field("backlog", &self.backlog) + .field("handshake", &self.handshake) + .field("reset_secret", &self.reset_secret.map(|_| "[redacted]")) + .finish() + } } impl Default for Config { From 5fe9b442290ac878af2999e004b8a1de6761bea1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 20 Oct 2026 15:05:16 -0700 Subject: [PATCH 7/9] feat(quic): read secrets from files as listen.reset.secret Both the stateless reset secret and cluster.lan.secret are now paths to a file of 64 hex characters, like listen.tls.key, so an inline secret never lands in shell history or a traced config. The reset secret moves to [listen.reset] secret (--listen-reset-secret, MOQ_LISTEN_RESET_SECRET). Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/cluster.md | 2 +- doc/bin/relay/config.md | 10 +- rs/moq-cli/MIGRATE.md | 5 + rs/moq-cli/src/args.rs | 2 +- rs/moq-relay/MIGRATE.md | 5 + rs/moq-relay/src/cluster.rs | 16 +-- rs/moq-relay/src/uring.rs | 9 +- rs/moq-tokio/MIGRATE.md | 7 +- rs/moq-tokio/src/listen.rs | 56 +++++---- rs/moq-tokio/src/mdns.rs | 18 +-- rs/moq-tokio/src/quic/backend.rs | 6 +- rs/moq-tokio/src/secret.rs | 107 ++++++++---------- rs/moq-tokio/src/settings.rs | 12 +- .../tests/integration/reset_secret.rs | 15 +-- 14 files changed, 144 insertions(+), 126 deletions(-) diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index 762ea095e52..fcbc81cd29b 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -182,7 +182,7 @@ node = "us-west.local:4443" [cluster.lan] enabled = true -# secret = "/etc/moq/cluster.key" # Optional. 64 hex chars, or a file holding them. +# secret = "/etc/moq/cluster.key" # Optional. A file holding 64 hex chars (openssl rand -hex 32). # app = "default" # DNS-SD subtype; moq-cli shares this name. ``` diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index dd30b024319..04d8d1b2b94 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -19,7 +19,9 @@ Precedence is CLI, then env, then file, then defaults. bind = "[::]:443" # QUIC (UDP), as --listen. Omit for a stream-only relay. version = ["moq-lite-05"] # Restrict accepted versions. Omit for all. timeout = "10s" # Handshake deadline. "0" waits forever. -reset_secret = "/etc/moq/reset.key" # 64 hex chars, or a file holding them. Omit for a random key. + +[listen.reset] +secret = "/etc/moq/reset.key" # File holding 64 hex chars. Omit for a random key. [listen.tls] cert = "cert.pem" # Certificate chain and key. Reloaded on change. @@ -45,7 +47,7 @@ epoch = false # Decline MoQ Epoch, so routes over moq-tra `timeout` bounds the whole handshake, transport through MoQ SETUP, so a peer that connects and never speaks is closed instead of held open by keep-alives. -`reset_secret` lets a relay restarted on the same address reset its +`reset.secret` lets a relay restarted on the same address reset its predecessor's QUIC connections, so clients reconnect at once instead of waiting out the idle timeout. Keep it secret, and give it to one relay only: any holder resets every other holder's connections, so live relays behind @@ -56,7 +58,7 @@ Create one with `openssl rand -hex 32 > reset.key`. A setting no configured listener reads stops startup rather than being ignored. A stream-only relay refuses `preferred_v4`, `preferred_v6`, `lb_id` (or `load_balancer`), and -`reset_secret`, which only QUIC reads, and a `[listen.tls]` `cert`, `key`, or `generate` unless +`reset.secret`, which only QUIC reads, and a `[listen.tls]` `cert`, `key`, or `generate` unless `tcp.tls` serves it. `lb_nonce` needs `lb_id`. `lb_id` and `lb_nonce` cannot be combined with `load_balancer`. `unix.allow` needs `unix.bind`. @@ -275,7 +277,7 @@ id = 12345 # Stable Hop ID across res [cluster.lan] # Find peers on the LAN over mDNS. enabled = true -# secret = "/etc/moq/cluster.key" # Optional: 64 hex chars, or a file holding them. +# secret = "/etc/moq/cluster.key" # Optional: a file holding 64 hex chars. # app = "default" # DNS-SD subtype; moq-cli shares this name. ``` diff --git a/rs/moq-cli/MIGRATE.md b/rs/moq-cli/MIGRATE.md index 3e697d3d0fe..26cda4269e9 100644 --- a/rs/moq-cli/MIGRATE.md +++ b/rs/moq-cli/MIGRATE.md @@ -4,6 +4,11 @@ How to update your invocation and config for each breaking change, newest first. ## Unreleased +### `--cluster-lan-secret` is a file path only (#5910) + +`--cluster-lan-secret`, `MOQ_CLUSTER_LAN_SECRET`, and `[cluster.lan] secret` no longer accept 64 hex characters inline. +Write the hex to a file (`openssl rand -hex 32 > cluster.key`) and pass its path. + ### `--epoch` is removed (#5967) Remove `--epoch` and `MOQ_EPOCH`: `--epoch` is now an unknown flag, and `MOQ_EPOCH` is ignored. diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index 72f2c212112..752042cdd53 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -2384,7 +2384,7 @@ mod tests { ]) .expect("parse"); assert!(cli.moq.validate().is_ok()); - assert_eq!(cli.moq.cluster.lan.secret.as_deref(), Some("cluster.key")); + assert_eq!(cli.moq.cluster.lan.secret.as_deref(), Some(std::path::Path::new("cluster.key"))); } /// The app is only read by the mesh, so configuring one without it is an diff --git a/rs/moq-relay/MIGRATE.md b/rs/moq-relay/MIGRATE.md index 6f6e7637526..f8d958063dc 100644 --- a/rs/moq-relay/MIGRATE.md +++ b/rs/moq-relay/MIGRATE.md @@ -5,6 +5,11 @@ Rolling a cluster across a release also follows the [upgrade guide](https://moq. ## Unreleased +### `cluster.lan.secret` is a file path only (#5910) + +`--cluster-lan-secret`, `MOQ_CLUSTER_LAN_SECRET`, and `[cluster.lan] secret` no longer accept 64 hex characters inline. +Write the hex to a file (`openssl rand -hex 32 > cluster.key`) and pass its path; `LanConfig::secret` is now a `PathBuf`. + ### Usage types come from usage-rs 7 (#6060) The relay's `Args` and `Config` derives now implement usage-rs 7 traits. diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 50d2a4cda0f..f7bca714152 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -876,8 +876,8 @@ pub struct LanConfig { )] pub enabled: bool, - /// The shared key admitting a peer to the LAN mesh, as 64 hexadecimal - /// characters or a path to a file containing them. + /// A file holding the shared key admitting a peer to the LAN mesh, as 64 + /// hexadecimal characters (`openssl rand -hex 32`). /// /// Optional. Without it, anyone who can reach the listener joins, so leave /// it unset only on networks you trust. With it, only peers that prove they @@ -886,10 +886,10 @@ pub struct LanConfig { name = "cluster-lan-secret", long = "cluster-lan-secret", env = "MOQ_CLUSTER_LAN_SECRET", - value_name = "HEX_OR_PATH", setting = "cluster.lan.secret" )] - pub secret: Option, + #[usage(value_hint = usage_rs::ValueHint::FilePath)] + pub secret: Option, /// DNS-SD application this relay advertises under. Peers using a different /// name never discover this one. Defaults to `default`, which moq-cli @@ -2126,7 +2126,7 @@ impl Cluster { async fn lan_discovery( advertise: &LanAdvertise, node: Option<&str>, - secret: Option<&str>, + secret: Option<&std::path::Path>, app: moq_tokio::mdns::App, ) -> anyhow::Result { let mut config = moq_tokio::mdns::Config::new(app, advertise.port); @@ -2157,7 +2157,7 @@ async fn lan_discovery( config = config.with_node(url); } if let Some(secret) = secret { - let secret = moq_tokio::secret::Secret::load(secret).context("invalid --cluster-lan-secret")?; + let secret = moq_tokio::secret::Secret::read(secret).context("invalid --cluster-lan-secret")?; config = config.with_secret(secret); } Ok(config.advertise().await?) @@ -3264,7 +3264,7 @@ mod tests { let args = vec![std::ffi::OsString::from("moq-relay"), std::ffi::OsString::from(&path)]; let config = RelayConfig::parse_and_merge(args).expect("config load"); assert!(config.cluster.lan.enabled); - assert_eq!(config.cluster.lan.secret.as_deref(), Some("cluster.key")); + assert_eq!(config.cluster.lan.secret.as_deref(), Some(std::path::Path::new("cluster.key"))); assert_eq!( config.cluster.lan.app.as_ref().map(ToString::to_string).as_deref(), Some("custom") @@ -3295,7 +3295,7 @@ mod tests { std::ffi::OsString::from("from-cli"), ]; let config = RelayConfig::parse_and_merge(args).expect("config load"); - assert_eq!(config.cluster.lan.secret.as_deref(), Some("from-cli.key")); + assert_eq!(config.cluster.lan.secret.as_deref(), Some(std::path::Path::new("from-cli.key"))); assert_eq!( config.cluster.lan.app.as_ref().map(ToString::to_string).as_deref(), Some("from-cli") diff --git a/rs/moq-relay/src/uring.rs b/rs/moq-relay/src/uring.rs index 0428b17fc0c..f663de00529 100644 --- a/rs/moq-relay/src/uring.rs +++ b/rs/moq-relay/src/uring.rs @@ -77,7 +77,7 @@ struct Serve { /// `listen.timeout`, which each worker's endpoint starts at a connection's /// first packet. timeout: Option, - /// `listen.reset_secret`, shared so any worker resets any worker's + /// `listen.reset.secret`, shared so any worker resets any worker's /// connections, a restarted group's included. reset_secret: Option<[u8; 32]>, } @@ -219,11 +219,12 @@ impl Workers { udp.gso = quic.gso.unwrap_or(true); let reset_secret = listen - .reset_secret + .reset + .secret .as_deref() - .map(moq_tokio::secret::Secret::load) + .map(moq_tokio::secret::Secret::read) .transpose() - .context("invalid --listen-quic-reset-secret")? + .context("invalid --listen-reset-secret")? .map(|secret| *secret.as_bytes()); let (failures_tx, failures) = tokio::sync::mpsc::unbounded_channel(); diff --git a/rs/moq-tokio/MIGRATE.md b/rs/moq-tokio/MIGRATE.md index 7928ecb6f6e..d1819ed8f14 100644 --- a/rs/moq-tokio/MIGRATE.md +++ b/rs/moq-tokio/MIGRATE.md @@ -4,10 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased -### `mdns::Secret` is `secret::Secret` (#5910) +### `mdns::Secret` is `secret::Secret`, read from a file (#5910) -The 32-byte secret type moved out of `mdns` so `listen.reset_secret` shares it. -Name it `moq_tokio::secret::Secret`; `new` and `load` now return `secret::Error` instead of `mdns::Error`. +The 32-byte secret type moved out of `mdns` so `listen.reset.secret` shares it, and it no longer parses inline hex. +Name it `moq_tokio::secret::Secret`. +Replace `Secret::load(value)` with `Secret::read(path)`, which reads a file of 64 hexadecimal characters and returns `secret::Error`; replace `Secret::new(hex)` and `str::parse` with `Secret::from([u8; 32])`, decoding the hex yourself. ### `jemalloc::tikv_jemallocator` is 0.7 (#6053) diff --git a/rs/moq-tokio/src/listen.rs b/rs/moq-tokio/src/listen.rs index ecf0ce12e12..7142e51787e 100644 --- a/rs/moq-tokio/src/listen.rs +++ b/rs/moq-tokio/src/listen.rs @@ -249,23 +249,10 @@ pub struct Config { #[serde(default, skip_serializing_if = "Option::is_none")] pub load_balancer: Option, - /// The secret the QUIC stateless reset key is derived from, as 64 hexadecimal - /// characters or a path to a file containing them. - /// - /// A server restarted with the same secret resets its predecessor's connections, - /// so peers reconnect at once instead of waiting out their idle timeout. Every - /// holder can reset every other holder's connections, so share it between live - /// servers only behind one address that routes by connection ID (QUIC-LB). - /// Unset draws a random key per process. - #[usage( - name = "listen-quic-reset-secret", - long = "listen-quic-reset-secret", - env = "MOQ_LISTEN_QUIC_RESET_SECRET", - value_name = "HEX_OR_PATH", - setting = "listen.reset_secret" - )] - #[serde(default, skip_serializing_if = "Option::is_none")] - pub reset_secret: Option, + /// QUIC stateless resets (`--listen-reset-*`). + #[usage(flatten)] + #[serde(default)] + pub reset: Reset, /// The released `--server-*` spellings and their env vars, kept parsing but /// hidden. Never read as settings: [`Config::deprecated`] names what replaced @@ -304,13 +291,38 @@ impl Default for Config { lb_id: None, lb_nonce: None, load_balancer: None, - reset_secret: None, + reset: Default::default(), legacy: Default::default(), quic: None, } } } +/// QUIC stateless reset settings (`[listen.reset]`, `--listen-reset-*`). +#[derive(usage_rs::Args, Clone, Default, Debug, serde::Serialize, serde::Deserialize)] +#[usage(unknown_flags = "error", args_override_self = false)] +#[serde(deny_unknown_fields)] +#[non_exhaustive] +pub struct Reset { + /// A file holding the secret the stateless reset key is derived from, as 64 + /// hexadecimal characters (`openssl rand -hex 32`). + /// + /// A server restarted with the same secret resets its predecessor's connections, + /// so peers reconnect at once instead of waiting out their idle timeout. Every + /// holder can reset every other holder's connections, so share it between live + /// servers only behind one address that routes by connection ID (QUIC-LB). + /// Unset draws a random key per process. + #[usage( + name = "listen-reset-secret", + long = "listen-reset-secret", + env = "MOQ_LISTEN_RESET_SECRET", + setting = "listen.reset.secret" + )] + #[usage(value_hint = usage_rs::ValueHint::FilePath)] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub secret: Option, +} + /// One server's socket in a complete `SO_REUSEPORT` group. /// /// Crate-private on purpose: [`crate::worker::Workers`] is the only thing that @@ -527,8 +539,8 @@ impl Config { "--listen-quic-lb-id (load_balancer) needs a QUIC listener (--listen)", ), ( - self.reset_secret.is_some(), - "--listen-quic-reset-secret needs a QUIC listener (--listen)", + self.reset.secret.is_some(), + "--listen-reset-secret needs a QUIC listener (--listen)", ), ]; match refused.into_iter().find(|(set, _)| *set) { @@ -810,8 +822,8 @@ load_balancer = { id = "ab", nonce = 8 } nonce: 8, }) }), - ("--listen-quic-reset-secret", |c| { - c.reset_secret = Some("reset.key".into()) + ("--listen-reset-secret", |c| { + c.reset.secret = Some("reset.key".into()) }), ]; for (name, set) in cases { diff --git a/rs/moq-tokio/src/mdns.rs b/rs/moq-tokio/src/mdns.rs index 4cd0b0e45f1..b02280bf8f7 100644 --- a/rs/moq-tokio/src/mdns.rs +++ b/rs/moq-tokio/src/mdns.rs @@ -770,8 +770,8 @@ fn instance_id(fullname: &str) -> Option<&str> { mod tests { use super::*; - const KEY_A: &str = "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f"; - const KEY_B: &str = "202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f"; + const KEY_A: [u8; 32] = [0xa; 32]; + const KEY_B: [u8; 32] = [0xb; 32]; /// Exactly one side of every pair dials: the lower id. Symmetric ids never /// dial themselves. @@ -822,8 +822,8 @@ mod tests { /// the listener, so one captured anywhere stands in nowhere else. #[test] fn proofs_are_bound_to_one_listener() { - let a = Secret::new(KEY_A).expect("valid secret"); - let b = Secret::new(KEY_B).expect("valid secret"); + let a = Secret::from(KEY_A); + let b = Secret::from(KEY_B); let base = proof( &a, CONTEXT_DIAL, @@ -1010,8 +1010,8 @@ mod tests { /// never receives a credential. #[test] fn verify_admits_only_the_secret_holder() { - let ours = Secret::new(KEY_A).expect("valid secret"); - let theirs = Secret::new(KEY_B).expect("valid secret"); + let ours = Secret::from(KEY_A); + let theirs = Secret::from(KEY_B); // The holder is admitted, and gets the proof that peer will check. let app = App::default(); @@ -1063,7 +1063,7 @@ mod tests { /// service instance and port are what make the proof non-transferable. #[test] fn a_stolen_proof_does_not_transfer() { - let ours = Secret::new(KEY_A).expect("valid secret"); + let ours = Secret::from(KEY_A); let app = App::default(); let other: App = "other".parse().expect("valid app"); @@ -1152,10 +1152,10 @@ mod tests { } /// Advertise on a port nothing listens on, purely to exercise the record. - async fn discovery(app: App, secret: Option<&str>, fingerprint: &str) -> Result { + async fn discovery(app: App, secret: Option<[u8; 32]>, fingerprint: &str) -> Result { let mut config = Config::new(app, 4443).with_fingerprint(fingerprint); if let Some(secret) = secret { - config = config.with_secret(Secret::new(secret).expect("valid secret")); + config = config.with_secret(Secret::from(secret)); } config.advertise().await } diff --git a/rs/moq-tokio/src/quic/backend.rs b/rs/moq-tokio/src/quic/backend.rs index 67fbcdb91d7..e7f5b64cb9b 100644 --- a/rs/moq-tokio/src/quic/backend.rs +++ b/rs/moq-tokio/src/quic/backend.rs @@ -210,7 +210,7 @@ pub enum Error { #[error("QUIC-LB connection IDs cannot be combined with per-core workers")] ShardWithQuicLb, - /// The stateless reset secret is neither 64 hexadecimal characters nor a file holding them. + /// The stateless reset secret file could not be read or does not hold 64 hexadecimal characters. #[error("invalid stateless reset secret")] ResetSecret(#[source] crate::secret::Error), @@ -806,8 +806,8 @@ impl Server { let mut endpoint_config = driver::EndpointConfig::default(); // First, since it also keys the default connection ID generator. The ones // below validate a predecessor's IDs without it. - if let Some(secret) = &config.reset_secret { - let secret = crate::secret::Secret::load(secret).map_err(Error::ResetSecret)?; + if let Some(secret) = &config.reset.secret { + let secret = crate::secret::Secret::read(secret).map_err(Error::ResetSecret)?; endpoint_config.reset_secret(secret.as_bytes()); } diff --git a/rs/moq-tokio/src/secret.rs b/rs/moq-tokio/src/secret.rs index 3d31b0fc35c..a40cef7bcbd 100644 --- a/rs/moq-tokio/src/secret.rs +++ b/rs/moq-tokio/src/secret.rs @@ -1,56 +1,39 @@ -//! A 32-byte secret an operator configures, such as `cluster.lan.secret` or -//! `listen.reset_secret`. +//! A 32-byte secret an operator configures as a file, such as `cluster.lan.secret` +//! or `listen.reset.secret`. use std::fmt; -use std::str::FromStr; +use std::path::{Path, PathBuf}; -/// A secret could not be decoded or read. +/// A secret file could not be read or decoded. #[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum Error { - /// The value is not exactly 32 bytes encoded as 64 hexadecimal characters. - #[error( - "secret must be exactly 32 bytes encoded as 64 hexadecimal characters, or a path to a file containing them" - )] - Invalid, + /// The file does not hold exactly 64 hexadecimal characters. + #[error("secret file {0} must hold exactly 64 hexadecimal characters (`openssl rand -hex 32`)")] + Invalid(PathBuf), - /// The value is not a secret, and not a file that could be read either. - #[error("secret must be 64 hexadecimal characters or a path to a file containing them, and {0} is neither")] - File(String, #[source] std::io::Error), + /// The file could not be read. + #[error("failed to read secret file {0}")] + Read(PathBuf, #[source] std::io::Error), } -/// A random 32-byte secret, given as 64 hexadecimal characters. +/// A random 32-byte secret. /// /// Anyone watching what it keys can brute-force a weak one offline, so generate -/// it (`openssl rand -hex 32`) rather than choosing one. +/// it (`openssl rand -hex 32`) rather than choosing one. `Debug` never prints it. #[derive(Clone, PartialEq, Eq)] pub struct Secret([u8; 32]); impl Secret { - /// Decode a secret from exactly 64 hexadecimal characters. - pub fn new(secret: impl AsRef) -> Result { - let secret = secret.as_ref(); - let mut key = [0; 32]; - if secret.len() != key.len() * 2 || hex::decode_to_slice(secret, &mut key).is_err() { - return Err(Error::Invalid); - } - Ok(Self(key)) - } - - /// Decode a secret given inline, or read it from the file `value` names. - /// - /// What a `--*-secret ` flag accepts. A path that exists but - /// holds something other than a secret is an error rather than a silent - /// fallback, and a path that doesn't exist is never created. - pub fn load(value: impl AsRef) -> Result { - let value = value.as_ref(); - if let Ok(secret) = Self::new(value) { - return Ok(secret); - } - // Not a secret, so it can only have been meant as a path. A read failure - // here is the useful error: it names the file the operator actually typed. - let contents = std::fs::read_to_string(value).map_err(|err| Error::File(value.to_string(), err))?; - Self::new(contents.trim()) + /// Read a secret from a file holding 64 hexadecimal characters, optionally + /// followed by one newline, as `openssl rand -hex 32` writes it. + pub fn read(path: impl AsRef) -> Result { + let path = path.as_ref(); + let contents = std::fs::read(path).map_err(|err| Error::Read(path.to_path_buf(), err))?; + let hex = contents.strip_suffix(b"\n").unwrap_or(&contents); + let mut secret = [0; 32]; + hex::decode_to_slice(hex, &mut secret).map_err(|_| Error::Invalid(path.to_path_buf()))?; + Ok(Self(secret)) } /// The secret's 32 bytes. @@ -59,11 +42,9 @@ impl Secret { } } -impl FromStr for Secret { - type Err = Error; - - fn from_str(secret: &str) -> Result { - Self::new(secret) +impl From<[u8; 32]> for Secret { + fn from(secret: [u8; 32]) -> Self { + Self(secret) } } @@ -80,29 +61,35 @@ mod tests { const KEY: &str = "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f"; #[test] - fn new_requires_exactly_32_hex_bytes_and_redacts_debug() { - for invalid in ["", "00", &"x".repeat(64), &"00".repeat(33)] { - assert!(Secret::new(invalid).is_err(), "{invalid:?} must not parse"); - } - assert_eq!(format!("{:?}", Secret::new(KEY).expect("valid")), "Secret([redacted])"); + fn debug_never_prints_the_bytes() { + let secret = Secret::from([0xab; 32]); + assert_eq!(format!("{secret:?}"), "Secret([redacted])"); } - /// A file holding the hex, as `openssl rand -hex 32` writes it, decodes to the - /// same secret as the hex given inline. + /// The file `openssl rand -hex 32 > secret.key` writes decodes with or + /// without its newline, and anything else is refused. #[test] - fn load_reads_inline_hex_or_a_file() { + fn read_requires_exactly_64_hex_characters() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("secret.key"); - std::fs::write(&path, format!("{KEY}\n")).unwrap(); - let inline = Secret::load(KEY).unwrap(); - assert_eq!(Secret::load(path.to_str().unwrap()).unwrap(), inline); - assert_eq!(inline.as_bytes()[31], 0x1f); + for valid in [format!("{KEY}\n"), KEY.to_string()] { + std::fs::write(&path, valid).unwrap(); + let secret = Secret::read(&path).unwrap(); + assert_eq!(secret.as_bytes()[31], 0x1f); + } + + // Empty, short, raw bytes, non-hex, and a byte too few or too many. + let invalid: [&[u8]; 6] = [b"", b"\n", &[0x5a; 32], &[b'x'; 64], &[b'0'; 62], &[b'0'; 66]]; + for contents in invalid { + std::fs::write(&path, contents).unwrap(); + assert!(matches!(Secret::read(&path), Err(Error::Invalid(_))), "{contents:?}"); + } + for contents in [format!("{KEY}\n\n"), format!(" {KEY}"), format!("{KEY}\r\n")] { + std::fs::write(&path, &contents).unwrap(); + assert!(matches!(Secret::read(&path), Err(Error::Invalid(_))), "{contents:?}"); + } - // Raw bytes are refused rather than taken as a secret. - std::fs::write(&path, [0x5a; 32]).unwrap(); - assert!(matches!(Secret::load(path.to_str().unwrap()), Err(Error::Invalid))); - let missing = dir.path().join("missing.key"); - assert!(matches!(Secret::load(missing.to_str().unwrap()), Err(Error::File(..)))); + assert!(matches!(Secret::read(dir.path().join("missing.key")), Err(Error::Read(..)))); } } diff --git a/rs/moq-tokio/src/settings.rs b/rs/moq-tokio/src/settings.rs index 9e7f9610531..017b6c0b1b9 100644 --- a/rs/moq-tokio/src/settings.rs +++ b/rs/moq-tokio/src/settings.rs @@ -32,8 +32,8 @@ pub struct Listen { #[usage(env = "MOQ_LISTEN_QUIC_LB_NONCE", cli("--listen-quic-lb-nonce"))] lb_nonce: Option, - #[usage(env = "MOQ_LISTEN_QUIC_RESET_SECRET", cli("--listen-quic-reset-secret"))] - reset_secret: Option, + #[usage(flatten)] + reset: ListenReset, #[usage(flatten)] tls: ListenTls, @@ -47,6 +47,14 @@ pub struct Listen { unix: ListenUnix, } +/// `[listen.reset]` / `--listen-reset-*`. +#[derive(usage_rs::Config)] +#[usage(prefix = "listen.reset")] +struct ListenReset { + #[usage(env = "MOQ_LISTEN_RESET_SECRET", cli("--listen-reset-secret"))] + secret: Option, +} + /// `[listen.tls]` / `--listen-tls-*`. #[derive(usage_rs::Config)] #[usage(prefix = "listen.tls")] diff --git a/rs/moq-tokio/tests/integration/reset_secret.rs b/rs/moq-tokio/tests/integration/reset_secret.rs index c399871f00f..1073dd50200 100644 --- a/rs/moq-tokio/tests/integration/reset_secret.rs +++ b/rs/moq-tokio/tests/integration/reset_secret.rs @@ -1,4 +1,4 @@ -//! A server restarted with the same `listen.reset_secret` resets its predecessor's +//! A server restarted with the same `listen.reset.secret` resets its predecessor's //! connections, so a peer learns at once that they are gone instead of waiting out //! its idle timeout. //! @@ -44,7 +44,7 @@ impl Fixture { } /// Serve on `port` (0 for any) from a runtime of its own, returned with the bound port. - fn serve(&self, port: u16, reset_secret: Option) -> (tokio::runtime::Runtime, u16) { + fn serve(&self, port: u16, reset_secret: Option) -> (tokio::runtime::Runtime, u16) { let runtime = tokio::runtime::Builder::new_multi_thread() .worker_threads(1) .enable_all() @@ -55,7 +55,7 @@ impl Fixture { config.listen.bind = Some(format!("127.0.0.1:{port}").parse().unwrap()); config.listen.tls.cert = vec![self.cert_path.clone()]; config.listen.tls.key = vec![self.key_path.clone()]; - config.listen.reset_secret = reset_secret; + config.listen.reset.secret = reset_secret; let port = runtime.block_on(async move { let mut listener = config @@ -81,9 +81,6 @@ impl Fixture { /// Connect, crash the server and restart it on the same port, then report how the /// client's connection ended once it next sent something. -/// -/// With a secret, the predecessor reads it from a file and the successor takes the -/// same hex inline, so both forms must derive the same key. fn restart(reset_secret: bool) -> Result { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); let fixture = Fixture::new(); @@ -92,8 +89,8 @@ fn restart(reset_secret: bool) -> Result Result Date: Tue, 20 Oct 2026 15:30:43 -0700 Subject: [PATCH 8/9] style: rustfmt Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/args.rs | 5 ++++- rs/moq-relay/src/cluster.rs | 10 ++++++++-- rs/moq-tokio/src/listen.rs | 4 +--- rs/moq-tokio/src/secret.rs | 5 ++++- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index 752042cdd53..5e91455458f 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -2384,7 +2384,10 @@ mod tests { ]) .expect("parse"); assert!(cli.moq.validate().is_ok()); - assert_eq!(cli.moq.cluster.lan.secret.as_deref(), Some(std::path::Path::new("cluster.key"))); + assert_eq!( + cli.moq.cluster.lan.secret.as_deref(), + Some(std::path::Path::new("cluster.key")) + ); } /// The app is only read by the mesh, so configuring one without it is an diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index f7bca714152..93f0f07dedf 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -3264,7 +3264,10 @@ mod tests { let args = vec![std::ffi::OsString::from("moq-relay"), std::ffi::OsString::from(&path)]; let config = RelayConfig::parse_and_merge(args).expect("config load"); assert!(config.cluster.lan.enabled); - assert_eq!(config.cluster.lan.secret.as_deref(), Some(std::path::Path::new("cluster.key"))); + assert_eq!( + config.cluster.lan.secret.as_deref(), + Some(std::path::Path::new("cluster.key")) + ); assert_eq!( config.cluster.lan.app.as_ref().map(ToString::to_string).as_deref(), Some("custom") @@ -3295,7 +3298,10 @@ mod tests { std::ffi::OsString::from("from-cli"), ]; let config = RelayConfig::parse_and_merge(args).expect("config load"); - assert_eq!(config.cluster.lan.secret.as_deref(), Some(std::path::Path::new("from-cli.key"))); + assert_eq!( + config.cluster.lan.secret.as_deref(), + Some(std::path::Path::new("from-cli.key")) + ); assert_eq!( config.cluster.lan.app.as_ref().map(ToString::to_string).as_deref(), Some("from-cli") diff --git a/rs/moq-tokio/src/listen.rs b/rs/moq-tokio/src/listen.rs index 7142e51787e..730d37748e2 100644 --- a/rs/moq-tokio/src/listen.rs +++ b/rs/moq-tokio/src/listen.rs @@ -822,9 +822,7 @@ load_balancer = { id = "ab", nonce = 8 } nonce: 8, }) }), - ("--listen-reset-secret", |c| { - c.reset.secret = Some("reset.key".into()) - }), + ("--listen-reset-secret", |c| c.reset.secret = Some("reset.key".into())), ]; for (name, set) in cases { let mut config = stream_only(); diff --git a/rs/moq-tokio/src/secret.rs b/rs/moq-tokio/src/secret.rs index a40cef7bcbd..e27f0c1d377 100644 --- a/rs/moq-tokio/src/secret.rs +++ b/rs/moq-tokio/src/secret.rs @@ -90,6 +90,9 @@ mod tests { assert!(matches!(Secret::read(&path), Err(Error::Invalid(_))), "{contents:?}"); } - assert!(matches!(Secret::read(dir.path().join("missing.key")), Err(Error::Read(..)))); + assert!(matches!( + Secret::read(dir.path().join("missing.key")), + Err(Error::Read(..)) + )); } } From 6af71d11a6ab03738190f39d5ef6aa5b50c19f10 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 20 Oct 2026 15:48:36 -0700 Subject: [PATCH 9/9] fix(tokio): refuse a reset secret without the quic feature Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/server.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index b84e07ee320..b7586e9d720 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -317,6 +317,10 @@ impl Server { if config.bind.is_some() && parts.quic() { return Err(Error::NoBackend("--listen requires the quic feature")); } + #[cfg(not(feature = "quic"))] + if config.reset.secret.is_some() && parts.quic() { + return Err(Error::NoBackend("--listen-reset-secret requires the quic feature")); + } // Only the QUIC backend verifies client certificates; the qmux listeners // (tcp/unix/websocket) never ask for one, even over `tls://`, so a @@ -2866,6 +2870,24 @@ mod tests { )); } + /// A reset secret on the default QUIC listener would never be read without a + /// QUIC backend. + #[cfg(not(feature = "quic"))] + #[test] + fn reset_secret_without_a_quic_backend_is_rejected() { + let mut config = crate::listen::Config::default(); + config.reset.secret = Some("reset.key".into()); + + assert!(matches!( + Config { + listen: config, + ..Default::default() + } + .init(), + Err(Error::NoBackend(reason)) if reason.starts_with("--listen-reset-secret") + )); + } + /// A client CA on the default QUIC listener names the missing backend, not a /// stream-only server the caller never configured. #[cfg(not(feature = "quic"))]