From 26e82e29a2383f8ef158efc363e32a20d4d58d8e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 05:26:47 -0800 Subject: [PATCH 1/2] chore(quest): claim quest/m2/relay-connect-keys.md Quest-Claim: 9d79bcca7baf2432 From 941e841c0e9fa5ca99ecc43ea7b4ea668b93ef9e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 05:45:37 -0800 Subject: [PATCH 2/2] fix(relay)!: refuse connect.url and connect.tokens, and keep template tokens off cluster dials The relay flattened the whole client fragment as its cluster dial template, so connect.tokens rode SETUP and AUTH on every cluster dial, LAN peers included, and connect.url was read by nothing. Config::resolve now refuses both by every spelling, naming cluster.connect and cluster.token, and peer_client clears the template's tokens so an embedder's Cluster gets the same guarantee. --help stops listing the two flags. Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 3 +- doc/bin/relay/config.md | 5 +- quest/m2/README.md | 3 +- quest/m2/relay-config-names.md | 1 - quest/m2/relay-connect-keys.md | 72 ------------------ rs/moq-relay/src/cluster.rs | 83 +++++++++++++++++++++ rs/moq-relay/src/config.rs | 132 ++++++++++++++++++++++++++++++++- 7 files changed, 220 insertions(+), 79 deletions(-) delete mode 100644 quest/m2/relay-connect-keys.md diff --git a/doc/bin/cli.md b/doc/bin/cli.md index ffd93532706..c6af11792b2 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -431,7 +431,8 @@ moq --cluster-lan --cluster-lan-secret /etc/moq/cluster.key import capture MoQ process and `moq-relay` on the LAN, with no other configuration. Without `--cluster-lan-secret`, anyone who can reach the listener joins, so leave it unset only on networks you trust. `--cluster-connect` is a MoQ side on its own, -so it needs no `--connect`. WAN cluster dials (`--cluster-connect`, +so it needs no `--connect`. `--connect-token` goes only to `--connect`, never to +a cluster peer: give those `--cluster-token`. WAN cluster dials (`--cluster-connect`, `--cluster-api-url`) bind `--connect-bind`'s address. On port 0, each cluster session takes its own ephemeral port, so a redial leaves from a fresh one. A fixed port pins every WAN cluster dial to it, while LAN dials still take diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index 3cef62e66aa..c772acd2b20 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -294,7 +294,10 @@ See [Clustering](/bin/relay/cluster). ## \[connect] -Settings for outbound dials (cluster peers, auth API). +Settings for outbound dials (cluster peers, auth API). Where and with what +credential to dial come from `[cluster]`, so the relay refuses to start with +`connect.url` (`--connect`): use `cluster.connect`; or `connect.tokens` +(`--connect-token`): use `cluster.token` or a peer's `token`. ```toml [connect] diff --git a/quest/m2/README.md b/quest/m2/README.md index 42df4d4cfa6..35ae6d8b9e3 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -44,8 +44,7 @@ removal, a rename, or a wire, catalog, or format change that is free until item re-checked for a downstream caller first. - **Relay and transports.** [Memory](/quest/m2/relay-memory.md), which the cut waits on by name; [congestion names](/quest/m2/congestion-names.md); - [config names](/quest/m2/relay-config-names.md), - [the two dial keys](/quest/m2/relay-connect-keys.md), and + [config names](/quest/m2/relay-config-names.md) and [the 0.14 leftovers](/quest/m2/relay-014-leftovers.md); [transport names](/quest/m2/transport-names.md); and [stats names](/quest/m2/stats-old-names.md). diff --git a/quest/m2/relay-config-names.md b/quest/m2/relay-config-names.md index 0aa217860cd..be3a2722cb8 100644 --- a/quest/m2/relay-config-names.md +++ b/quest/m2/relay-config-names.md @@ -92,5 +92,4 @@ key renamed, one nix option renamed. Wire: none. ## Related -- [The relay refuses dial keys it never uses](/quest/m2/relay-connect-keys.md) - the other config change at this freeze - [The switch](/quest/m2/multicast/switch.md) - retires what else reads a hop bound diff --git a/quest/m2/relay-connect-keys.md b/quest/m2/relay-connect-keys.md deleted file mode 100644 index 398fddb76d6..00000000000 --- a/quest/m2/relay-connect-keys.md +++ /dev/null @@ -1,72 +0,0 @@ -# [S] The relay refuses the two dial keys it has no use for - -## Goal - -`moq-relay` stops at startup when given `connect.url` (`--connect`, -`MOQ_CONNECT`) or `connect.tokens` (`--connect-token`, -`MOQ_CONNECT_TOKEN`), with a message naming `cluster.connect` and -`cluster.token`. A relay's cluster dials carry only the credentials the -cluster settings name, so no token reaches a peer the operator did not -choose it for, a LAN peer included. The other `[connect]` keys (bind, TLS, -versions, timeouts) keep working as the template for cluster dials. - -## Plan - -Decided 2026-11-16 (maintainer, /quest-plan of the pre-cut audit): - -- ✅ Refuse both keys by name (rejected: splitting the `[connect]` fragment - so the relay flattens only the dial settings, a larger change to a type - the clients share; documenting them as they behave). -- ✅ Reproduce the token reaching a cluster dial before changing anything: - the audit found it by reading, ran nothing, and no test covers it. - -Facts found by the audit: - -- The relay flattens the whole client fragment - (`rs/moq-relay/src/config.rs:42`, `pub connect: - moq_tokio::connect::Config`). -- `connect.url` (`rs/moq-tokio/src/connect.rs:513-514`) is read only by - `Client::publish` and `Client::consume` (`rs/moq-tokio/src/client.rs:164`, - `:346`, `:356`). Nothing in `rs/moq-relay/src` reads it, so - `moq-relay --connect https://peer` parses and does nothing, where the - operator meant `--cluster-connect`. The relay's rule elsewhere is that a - setting nothing reads stops startup. -- `connect.tokens` (`rs/moq-tokio/src/connect.rs:524-530`, new since - release) is documented as "credentials to present on every dial". - `peer_client` (`rs/moq-relay/src/cluster.rs:2316-2345`) clones the whole - template for each cluster dial, `rs/moq-tokio/src/client.rs:165` keeps its - tokens, and `rs/moq-tokio/src/connection.rs:952` presents them. The LAN - branch (`cluster.rs:2330`) narrows versions and pins a fingerprint but - leaves the tokens, while `doc/bin/relay/cluster.md:250-251` says a LAN - peer "is never handed `cluster.token`". So `connect.tokens` is an - undocumented second cluster credential that goes to every peer. - -For the PR: - -1. Write the failing test first: a relay configured with `connect.tokens` - and a static peer, then a LAN peer, and assert what each peer's SETUP and - AUTH streams receive. If no token arrives, the leak is not real: say so - in the PR, and the refusal still lands for the unread key. -2. Refuse both in `Config::resolve` (`rs/moq-relay/src/config.rs:322`), - beside the released `[client]` table it already names `[connect]` for. - The message names the flag, the environment variable, and the TOML key in - use, and what replaces each: `cluster.connect` for the URL, and - `cluster.token` or a peer's `token` for the credential. -3. Clear the template's tokens in `peer_client` as well, so an embedder that - builds a `Cluster` with `with_connect` gets the same guarantee as the - binary. -4. `moq-relay --help` and `doc/bin/relay/config.md` stop listing the two - keys under `[connect]`; `doc/bin/relay/cluster.md` keeps its sentence - about LAN peers, now true by construction. - -Tests: the refusal names each spelling; the reproduction from step 1 passes -with no token on any cluster dial; a relay with the remaining `[connect]` -keys still dials its peers. - -Public API: none (`Cluster::with_connect` keeps its type; its tokens are -ignored). Config and CLI: breaking for moq-relay only, `connect.url` and -`connect.tokens` refused. Wire: none. - -## Related - -- [The relay's config names](/quest/m2/relay-config-names.md) - the renames at this freeze diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 9ddc6d776f5..dcf85285dce 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -1647,6 +1647,9 @@ impl Cluster { /// A fixed bind port is kept instead: static and API peers share one socket /// on it, while LAN dials still take ephemeral ports. /// + /// The template's [`tokens`](moq_tokio::connect::Config::tokens) are ignored: a + /// peer is handed only `cluster.token` or its own `token`. + /// /// Required when any peer is configured; [`start`](Self::start) returns an /// error otherwise. pub fn with_connect(mut self, connect: moq_tokio::connect::Config, quic: moq_tokio::quic::Config) -> Self { @@ -2319,6 +2322,9 @@ impl Cluster { .as_deref() .cloned() .context("cluster dial without a dial template (call Cluster::with_connect)")?; + // A peer gets only the credential the cluster settings chose for it, so the + // template's would reach every peer, LAN ones included. + connect.tokens.clear(); if ephemeral { let mut bind = connect.resolve().bind; bind.set_port(0); @@ -4397,6 +4403,83 @@ mod tests { } } + /// Cluster dials carry only the credentials the cluster settings name: a dial + /// template's own tokens reach neither a static peer nor a LAN peer, in SETUP + /// or the URL. lite-07 puts the template's first token in SETUP and the rest on + /// AUTH streams, so an empty SETUP token means the template sent none. The + /// template's other settings (version, TLS) still dial both. + #[tokio::test] + async fn cluster_dials_present_no_template_tokens() { + const TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + moq_tokio::crypto::install_default().unwrap(); + + let mut server = moq_tokio::server::Config::default(); + server.listen.bind = Some("127.0.0.1:0".parse().unwrap()); + server.listen.tls.generate = vec!["localhost".into()]; + server.listen.version = vec!["moq-lite-07-wip".parse().unwrap()]; + let server = server.init().expect("server"); + let fingerprint = server + .certificates() + .fingerprints() + .into_iter() + .next() + .expect("generated fingerprint"); + let mut listener = server.listen().await.expect("listen"); + let port = listener.local_addr().expect("quic addr").port(); + let (sender, mut seen) = tokio::sync::mpsc::unbounded_channel(); + tokio::spawn(async move { + while let Ok(request) = listener.accept().await { + let token = request.token().map(|token| token.value.clone()); + sender + .send((request.path().to_string(), request.query().map(str::to_string), token)) + .ok(); + tokio::spawn(async move { + let Ok(_session) = request.ok().await else { return }; + std::future::pending::<()>().await; + }); + } + }); + let mut next = async || { + kio::clock::timeout(TIMEOUT, seen.recv()) + .await + .expect("no dial arrived") + .expect("server stopped") + }; + + let mut connect = moq_tokio::connect::Config::default(); + connect.tls.insecure = Some(true); + connect.version = vec!["moq-lite-07-wip".parse().unwrap()]; + connect.tokens = vec!["template-setup".parse().unwrap(), "template-auth".parse().unwrap()]; + + let config = Config { + connect: vec![Peer::new(format!("moqt://localhost:{port}/static"))], + ..Default::default() + }; + let cluster = new_cluster(config) + .expect("cluster") + .with_connect(connect, Default::default()); + let started = cluster.clone().start().await.expect("start"); + let _running = tokio::spawn(started.run()); + assert_eq!(next().await, ("/static".to_string(), None, None), "static peer"); + + cluster.set_lan_credential("lan-proof"); + let url: Url = format!("moqt://127.0.0.1:{port}{CLUSTER_PATH}/lan-proof") + .parse() + .expect("url"); + let target = DialTarget { + key: PeerKey::canonicalize(&format!("moqt://127.0.0.1:{port}/")), + url: url.clone(), + urls: vec![url], + cost: None, + fingerprint: Some(fingerprint), + lan: true, + upstream: false, + }; + let _lan = cluster.dial_lan_target(&target).expect("dial"); + let (path, query, token) = next().await; + assert_eq!((query, token), (None, None), "LAN peer at {path}"); + } + /// A second close awaits the shutdown the first started, and still finishes it /// when the first caller gives up, so no caller returns while links are open. #[tokio::test] diff --git a/rs/moq-relay/src/config.rs b/rs/moq-relay/src/config.rs index 93c85a79222..6741eecf3c3 100644 --- a/rs/moq-relay/src/config.rs +++ b/rs/moq-relay/src/config.rs @@ -157,9 +157,35 @@ struct Cli { /// flattens it has its own spec, so `Config::spec()` would hand back the wrong /// one and read as though it were theirs. pub fn spec() -> &'static usage_rs::spec::Spec<'static> { - Cli::spec() + static SPEC: std::sync::OnceLock> = std::sync::OnceLock::new(); + SPEC.get_or_init(|| { + let base = Cli::spec(); + // `[connect]` is the shared client fragment, flattened whole; its `url` and `tokens` + // still parse so [`Config::resolve`] can refuse them, but help no longer offers them. + let flags = base + .root + .flags + .iter() + .map(|meta| usage_rs::spec::FlagMeta { + hide: meta.hide || meta.flag.longs.iter().any(|long| REFUSED_CONNECT.contains(long)), + ..*meta + }) + .collect::>(); + let root = usage_rs::spec::CommandMeta { + flags: flags.leak(), + ..*base.root + }; + usage_rs::spec::Spec { + root: Box::leak(Box::new(root)), + ..*base + } + }) } +/// The `[connect]` flags a relay refuses: cluster dials take their peers and +/// credentials from `[cluster]`. +const REFUSED_CONNECT: [&str; 2] = ["connect", "connect-token"]; + /// Every setting the relay merges, as its flags declare them. pub(crate) const fn registry() -> usage_rs::config::Registry { Cli::SETTINGS_REGISTRY @@ -219,7 +245,7 @@ impl Config { let (cli, cli_layer) = match Cli::parse_from_with_settings(&argv) { Ok(parsed) => parsed, Err(err) => { - let answer = moq_tokio::cli::answer(Cli::spec(), Cli::command(), &argv, err); + let answer = moq_tokio::cli::answer(spec(), Cli::command(), &argv, err); if answer.is_question() { answer.exit(); } @@ -322,6 +348,7 @@ impl Config { pub(crate) fn resolve(mut self) -> anyhow::Result { let deprecated = self.deprecated(); anyhow::ensure!(deprecated.is_empty(), "{deprecated}"); + self.check_connect()?; // Here rather than at an endpoint, since io_uring workers and lazily built // dial clients would otherwise check `[quic]` late or never. `[limits.peer.quic]` // is checked by `Cluster::new`, which embedders call directly. @@ -349,6 +376,33 @@ impl Config { Ok(Resolved(self)) } + /// Refuse the `[connect]` keys a cluster dial has no use for. + /// + /// The relay flattens the whole client fragment as the template for cluster dials, + /// but a peer's address and credential come from `[cluster]`: a URL here would be + /// read by nothing, and tokens here would reach every peer, LAN ones included. + fn check_connect(&self) -> anyhow::Result<()> { + let mut refused = Vec::new(); + if self.connect.url.is_some() { + refused.push( + "--connect / MOQ_CONNECT / connect.url: name the peers to dial with \ + --cluster-connect / MOQ_CLUSTER_CONNECT / cluster.connect", + ); + } + if !self.connect.tokens.is_empty() { + refused.push( + "--connect-token / MOQ_CONNECT_TOKEN / connect.tokens: give peers a credential with \ + --cluster-token / MOQ_CLUSTER_TOKEN / cluster.token, or a `token` on a cluster.connect peer", + ); + } + anyhow::ensure!( + refused.is_empty(), + "moq-relay does not read these settings; remove them and try again:\n {}", + refused.join("\n ") + ); + Ok(()) + } + /// Refuse worker options that need a worker group, or a build, this relay lacks. fn check_runtime(&self) -> anyhow::Result<()> { let workers = self.runtime.workers().is_some(); @@ -670,6 +724,80 @@ mod tests { } } + /// What the refusal says for `[connect]`'s URL and its tokens. + const REFUSED_URL: &str = "--connect / MOQ_CONNECT / connect.url: name the peers to dial with \ + --cluster-connect / MOQ_CLUSTER_CONNECT / cluster.connect"; + const REFUSED_TOKENS: &str = "--connect-token / MOQ_CONNECT_TOKEN / connect.tokens: give peers a credential \ + with --cluster-token / MOQ_CLUSTER_TOKEN / cluster.token"; + + /// `[connect]`'s URL and tokens stop the relay from a flag or the file, naming + /// what replaces each, while the rest of `[connect]` still resolves. + #[test] + fn connect_url_and_tokens_refuse_to_boot() { + if !test_env::Env::without(&["MOQ_CONNECT", "MOQ_CONNECT_TOKEN"]).isolate() { + return; + } + let refusal = |config: Config| config.resolve().expect_err("must refuse").to_string(); + + let flags = Config::parse_and_merge(["moq-relay", "--connect", "https://peer.example/"]).unwrap(); + let err = refusal(flags); + assert!(err.contains(REFUSED_URL) && !err.contains("connect.tokens"), "{err}"); + let flags = Config::parse_and_merge(["moq-relay", "--connect-token", "secret"]).unwrap(); + let err = refusal(flags); + assert!(err.contains(REFUSED_TOKENS) && !err.contains("connect.url"), "{err}"); + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("relay.toml"); + std::fs::write(&path, "[connect]\nurl = 'https://peer.example/'\ntokens = ['secret']\n").unwrap(); + let file = Config::parse_and_merge([std::ffi::OsString::from("moq-relay"), path.into()]).unwrap(); + let err = refusal(file); + assert!(err.contains(REFUSED_URL) && err.contains(REFUSED_TOKENS), "{err}"); + + let rest = Config::parse_and_merge([ + "moq-relay", + "--connect-bind", + "[::]:0", + "--connect-timeout", + "5s", + "--connect-version", + "moq-lite-05", + "--connect-tls-root", + "ca.pem", + ]) + .unwrap(); + rest.resolve().expect("the dial template's other keys still resolve"); + } + + /// The same refusal from the environment, the half a deployment most often uses. + #[test] + fn connect_env_refuses_to_boot() { + if !test_env::Env::default() + .set("MOQ_CONNECT", "https://peer.example/") + .set("MOQ_CONNECT_TOKEN", "secret") + .isolate() + { + return; + } + let config = Config::parse_and_merge(["moq-relay"]).unwrap(); + let err = config.resolve().expect_err("must refuse").to_string(); + assert!(err.contains(REFUSED_URL) && err.contains(REFUSED_TOKENS), "{err}"); + } + + /// `--help` no longer offers the two refused `[connect]` flags, and still lists the rest. + #[test] + fn help_omits_the_refused_connect_flags() { + let argv = [std::ffi::OsStr::new("--help")]; + let err = Cli::parse_from(&argv).unwrap_err(); + let help = moq_tokio::cli::answer(spec(), Cli::command(), &argv, err) + .message() + .to_string(); + assert!(help.contains("--connect-bind"), "{help}"); + assert!( + !help.contains("--connect <") && !help.contains("--connect-token"), + "{help}" + ); + } + /// A released config file is parsed where the tables used to live, so the relay /// can name `[quic]` instead of failing with `unknown field quic`, which is what /// `deny_unknown_fields` would say on its own.