From c6dea421a3fee875ef1cf0b1d5d4ec52df204059 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 18:02:03 -0800 Subject: [PATCH 1/5] chore(quest): claim quest/m1/precut-fixes/relay-config-loud.md Quest-Claim: 02ca91897d8a2b85 From 475ba62bcd68a734f006284b9a412c088e6d40b3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 18:12:38 -0800 Subject: [PATCH 2/5] fix(relay): cluster config fails loud, secrets stay out of logs, and the upgrade notes name real flags - moq-tokio: a key given where its secret file belongs is refused without echoing it. - relay: setup holds the refreshed grant to the root, mounts, link, role, and cap checks. - relay: the redial backoff resets after an attempt that connected. - relay: cluster.api.root without cluster.api.url, and cluster.token with no peer to dial, refuse startup. - moq-auth: null on a key's kid, scope, and d, and a grant's root, revalidate, tier, and label, is refused as @moq/auth does. - docs: MIGRATE.md names --mtls-link and the client limits; the auth doc names the 422. Co-Authored-By: Claude Sonnet 5.5 --- doc/bin/relay/auth.md | 7 +- .../m1/precut-fixes/relay-backoff-backport.md | 27 ++++++ quest/m1/precut-fixes/relay-config-loud.md | 40 --------- rs/moq-auth/src/key.rs | 19 ++-- rs/moq-auth/src/wire.rs | 23 ++++- rs/moq-auth/tests/corpus.json | 11 +++ rs/moq-relay/MIGRATE.md | 3 +- rs/moq-relay/src/auth/refresh.rs | 24 ++++- rs/moq-relay/src/cluster.rs | 87 +++++++++++++------ rs/moq-relay/src/config.rs | 18 ++++ rs/moq-tokio/src/secret.rs | 25 +++++- 11 files changed, 200 insertions(+), 84 deletions(-) create mode 100644 quest/m1/precut-fixes/relay-backoff-backport.md delete mode 100644 quest/m1/precut-fixes/relay-config-loud.md diff --git a/doc/bin/relay/auth.md b/doc/bin/relay/auth.md index bc2eff36eed..63ae6e6ecdb 100644 --- a/doc/bin/relay/auth.md +++ b/doc/bin/relay/auth.md @@ -425,9 +425,10 @@ subscribe = [] Upgrade auth servers before relays when adopting the `webtransport` or `tls` transport value. WebTransport sessions report `webtransport`; native QUIC sessions report `quic`. `tls://` sessions report `tls`; plaintext `tcp://` sessions report `tcp`. -An auth server built before `tls` existed does not fail to parse it: the Rust and JavaScript -parsers read an unrecognized transport name as `unknown`, so admission then depends on how your -policy treats `unknown`. Update any policy keyed on `tcp` for stream links to handle `tls` too. +A `moq auth serve` from a `release` build answers 422 to a request naming a transport it predates, +such as `webtransport` or `tls`, which the relay refuses. Current parsers read an unrecognized +transport name as `unknown`, so admission then depends on how your policy treats `unknown`. +Update any policy keyed on `tcp` for stream links to handle `tls` too. `moq auth serve` is the reference auth server, answering the contract above. diff --git a/quest/m1/precut-fixes/relay-backoff-backport.md b/quest/m1/precut-fixes/relay-backoff-backport.md new file mode 100644 index 00000000000..6f350bae28f --- /dev/null +++ b/quest/m1/precut-fixes/relay-backoff-backport.md @@ -0,0 +1,27 @@ +# [XS] Backport the relay redial backoff reset to the next release + +## Goal + +A relay on the next `release` patch redials a cluster peer from the first +backoff ceiling once an attempt connected, so a flap after a long outage no +longer waits out 30 s. + +## Plan + +Port the `main` fix from the PR that finished `relay-config-loud` as a separate +PR onto `release`, per RELEASE.md, keeping the published API. +`rs/moq-relay/src/cluster.rs` `run_remote` on `release` has the same dead +`Ok(()) if elapsed >= stable_threshold` branch, because `run_remote_once` only +returns an error. Report whether the attempt connected (any observation with +`connects > 0`) and pass it to `Redial::next`. + +The same PR on `main` also refuses `cluster.api.root` without `cluster.api.url` +and `cluster.token` with no peer to dial. Those are startup failures for +configs that ran before, so a patch release should not carry them. + +## Waiting + +- Maintainer: whether the backport is wanted at all, and whether it carries the + two config refusals. Recommend backporting the backoff reset only: a flapping + peer is a degraded mesh, while a patch that stops a working relay from + booting is a break. Check by this quest's Plan naming the choice. diff --git a/quest/m1/precut-fixes/relay-config-loud.md b/quest/m1/precut-fixes/relay-config-loud.md deleted file mode 100644 index b5f3e24621d..00000000000 --- a/quest/m1/precut-fixes/relay-config-loud.md +++ /dev/null @@ -1,40 +0,0 @@ -# [S] Relay cluster config fails loud, keeps secrets out of logs, and its upgrade notes are right - -## Goal - -The relay refuses cluster settings it would ignore, never echoes a secret in -an error, re-checks a refreshed grant like every other credential, resets -its redial backoff after a stable session, and its upgrade notes name real -flags. - -## Plan - -Found by the 2026-11-17 pre-cut correctness audit; each item confirmed from -source by its reviewer, reproduce first. - -- `rs/moq-relay/src/cluster.rs`: `release` accepted the 64-hex LAN secret - inline; `main` reads it as a file path and fails with "failed to read - secret file ", printing the mesh key into logs on upgrade. Refuse - an inline-looking value without echoing it. -- `rs/moq-relay/src/auth/refresh.rs`: the grant re-read at protocol setup - skips the root, mounts, link, and role checks the other credential paths - run (`valid()`); run them. -- `rs/moq-relay/src/cluster.rs`: `run_remote_once` only returns an error, so - the "reset after a stable session" backoff branch is unreachable and long - outages leave every later redial at 30 s. Reset on whether the attempt - connected. On `release` too. -- `rs/moq-relay/src/config.rs`: `cluster.api.root` without `cluster.api.url`, - and `cluster.token` on a relay with no peers to dial, are accepted and - never read; refuse both at startup. On `release` too. A config key that - starts failing is a break: note it in the PR's Impact for the migration - audit. -- `rs/moq-auth/src/wire.rs` and `key.rs`: Rust accepts `null` on grant and key - fields `@moq/auth` refuses (`"scope": null` loads as an unrestricted key). - Use the existing reject-`null` deserializer and add corpus cases. -- Docs: `rs/moq-relay/MIGRATE.md` names `--mtls-peer`, which does not exist - (`--mtls-link peer`), and never mentions that certificate-only peers now get - client limits and are cut off past 1000 broadcasts; `doc/bin/relay/auth.md` - says an older auth server reads new transport names as `unknown`, but a - `release` `moq auth serve` answers 422. - -Public API: none. Wire: none. Config: two ignored keys start failing. diff --git a/rs/moq-auth/src/key.rs b/rs/moq-auth/src/key.rs index aa8c1db0eca..3108bfbd0a6 100644 --- a/rs/moq-auth/src/key.rs +++ b/rs/moq-auth/src/key.rs @@ -252,11 +252,19 @@ pub struct Jwk { pub material: KeyMaterial, /// The key ID, useful for rotating keys. - #[serde(skip_serializing_if = "Option::is_none")] + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::wire::present" + )] pub kid: Option, /// Optional authorization limits for tokens signed by this key. - #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::wire::present" + )] pub scope: Option, } @@ -736,10 +744,9 @@ fn deserialize_base64url_optional<'de, D>(deserializer: D) -> Result, { - let s: Option = Option::deserialize(deserializer)?; - s.map(|s| decode_base64url(&s)) - .transpose() - .map_err(serde::de::Error::custom) + // A present field: `null` is refused, as `@moq/auth` does, rather than read as absent. + let s = String::deserialize(deserializer)?; + decode_base64url(&s).map(Some).map_err(serde::de::Error::custom) } /// A key field: unpadded base64url, or padded for backwards compatibility. Empty is refused, diff --git a/rs/moq-auth/src/wire.rs b/rs/moq-auth/src/wire.rs index 154c57f0f16..ccbdde0731e 100644 --- a/rs/moq-auth/src/wire.rs +++ b/rs/moq-auth/src/wire.rs @@ -212,10 +212,26 @@ impl TryFrom> for crate::Grant { /// A present field, refusing an explicit `null` rather than reading it as absent, /// so a `null` can't hide one encoding from the mixed-encoding check. -fn present<'de, D: Deserializer<'de>, T: Deserialize<'de>>(deserializer: D) -> Result, D::Error> { +pub(crate) fn present<'de, D: Deserializer<'de>, T: Deserialize<'de>>(deserializer: D) -> Result, D::Error> { T::deserialize(deserializer).map(Some) } +/// A present `revalidate` in whole seconds, refusing `null` as [`present`] does. +mod revalidate { + use super::*; + + pub(super) fn serialize(value: &Option, serializer: S) -> Result { + match value { + Some(value) => serializer.serialize_u64(value.as_secs()), + None => serializer.serialize_none(), + } + } + + pub(super) fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + as serde_with::DeserializeAs<'de, Duration>>::deserialize_as(deserializer).map(Some) + } +} + /// A present time in whole unix seconds, from the epoch to the clock's last second, which /// `@moq/auth` holds exactly too. Anything else is refused rather than saturated, and an /// explicit `null` rather than read as absent, which for `exp` would mean never. @@ -300,14 +316,17 @@ pub(crate) struct Grant { publish: Access, #[serde(skip_serializing_if = "Access::is_unset")] subscribe: Access, + #[serde(deserialize_with = "present")] root: Option, #[serde(skip_serializing_if = "BTreeMap::is_empty")] mounts: BTreeMap, #[serde(with = "seconds", skip_serializing_if = "Option::is_none")] expires: Option, - #[serde_as(as = "Option>")] + #[serde(with = "revalidate", skip_serializing_if = "Option::is_none")] revalidate: Option, + #[serde(deserialize_with = "present")] tier: Option, + #[serde(deserialize_with = "present")] label: Option, #[serde(deserialize_with = "present")] link: Option, diff --git a/rs/moq-auth/tests/corpus.json b/rs/moq-auth/tests/corpus.json index b3e08ff565e..5efb9225879 100644 --- a/rs/moq-auth/tests/corpus.json +++ b/rs/moq-auth/tests/corpus.json @@ -302,6 +302,11 @@ }, "valid": false }, + { "name": "a_null_scope", "jwk": { "kty": "oct", "alg": "HS256", "k": "Fp8kipWUJeUFqeSqWym_tRC_tyI8z-QpqopIGrbrD68", "scope": null }, "valid": false }, + { "name": "a_null_kid", "jwk": { "kty": "oct", "alg": "HS256", "k": "Fp8kipWUJeUFqeSqWym_tRC_tyI8z-QpqopIGrbrD68", "kid": null }, "valid": false }, + { "name": "a_null_key_ops", "jwk": { "kty": "oct", "alg": "HS256", "k": "Fp8kipWUJeUFqeSqWym_tRC_tyI8z-QpqopIGrbrD68", "key_ops": null }, "valid": false }, + { "name": "a_null_scope_root", "jwk": { "kty": "oct", "alg": "HS256", "k": "Fp8kipWUJeUFqeSqWym_tRC_tyI8z-QpqopIGrbrD68", "scope": { "root": null, "publish": ["**"] } }, "valid": false }, + { "name": "a_null_okp_private_key", "jwk": { "kty": "OKP", "alg": "EdDSA", "crv": "Ed25519", "x": "UiU9fT_SdBBpkFtJPRCY0gX1jK_Dr9syYLFuEz4QUM4", "d": null }, "valid": false }, { "name": "short_secret", "jwk": { "kty": "oct", "alg": "HS256", "k": "c2hvcnQ" }, @@ -617,6 +622,12 @@ }, { "name": "unknown_link", "grant": { "publish": ["**"], "link": "downstream" }, "valid": false }, { "name": "null_link", "grant": { "publish": ["**"], "link": null }, "valid": false }, + { "name": "null_root", "grant": { "publish": ["**"], "root": null }, "valid": false }, + { "name": "null_revalidate", "grant": { "publish": ["**"], "revalidate": null }, "valid": false }, + { "name": "null_tier", "grant": { "publish": ["**"], "tier": null }, "valid": false }, + { "name": "null_label", "grant": { "publish": ["**"], "label": null }, "valid": false }, + { "name": "null_mounts", "grant": { "publish": ["**"], "mounts": null }, "valid": false }, + { "name": "null_peer", "grant": { "publish": ["**"], "peer": null }, "valid": false }, { "name": "positional", "grant": [["**"]], "valid": false }, { "name": "null_grant", "grant": null, "valid": false } ] diff --git a/rs/moq-relay/MIGRATE.md b/rs/moq-relay/MIGRATE.md index 6cff81e44f8..655d5923c9e 100644 --- a/rs/moq-relay/MIGRATE.md +++ b/rs/moq-relay/MIGRATE.md @@ -125,7 +125,8 @@ An auth server that scopes tenants by hostname must see them over TLS, and still ### A client certificate alone lists no hidden routes (#6509) Only a grant with `peer: true` lists `.`-named routes to a session that did not ask for them. -Have the auth server mark cluster peers as `peer` (`moq auth serve --mtls-peer`) before upgrading, or a peer below moq-lite-07-wip loses those paths. +Have the auth server mark cluster peers as `peer` (`moq auth serve --mtls-link peer`) before upgrading, or a peer below moq-lite-07-wip loses those paths. +A certificate-only peer also gets `[limits.client]` instead of `[limits.peer]`, so it is closed with `TOO_MANY_REQUESTS` past 1000 announced broadcasts (`announces`) instead of 100000. ### `stats.interval` is a duration (#6472) diff --git a/rs/moq-relay/src/auth/refresh.rs b/rs/moq-relay/src/auth/refresh.rs index b9b486d1916..e3f3cd8bcf0 100644 --- a/rs/moq-relay/src/auth/refresh.rs +++ b/rs/moq-relay/src/auth/refresh.rs @@ -109,9 +109,6 @@ impl Refresh { Err(lease::Reason::Expired) => return Err(moq_net::Error::Expired), Err(_) => return Err(moq_net::Error::Unauthorized), } - if lease.pacer.is_none() && Caps::of(&initial.grant).is_capped() { - return Err(moq_net::Error::Unauthorized); - } initial.initial = true; // The connection applies the first caps itself, with the window it opens. let caps = Caps::of(&initial.grant); @@ -133,6 +130,10 @@ impl Refresh { pacer: lease.pacer.clone(), caps, }; + // The same root, mounts, link, role, and cap checks a refreshed grant passes. + if !this.valid(&this.credentials[&0].grant) { + return Err(moq_net::Error::Unauthorized); + } this.authorize(); Ok(this) } @@ -656,6 +657,23 @@ mod tests { .await; } + /// Setup holds the grant it reads to the root, mounts, and link admission bound the session to. + #[tokio::test(start_paused = true)] + async fn a_moved_root_mount_or_link_before_protocol_setup_is_refused() { + let _clock = manual(); + type Change = fn(&mut moq_auth::Grant); + let changes: [Change; 3] = [ + |grant| grant.root = Some("elsewhere".into()), + |grant| { + grant.mounts.insert("m".into(), "other".into()); + }, + |grant| grant.link = Some(moq_auth::Link::Peer), + ]; + for change in changes { + refused_before_setup(change, SessionError::Unauthorized).await; + } + } + #[tokio::test(start_paused = true)] async fn a_grant_expiring_before_protocol_setup_is_refused_as_expired() { let clock = manual(); diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 06fcaccf8d0..0ba85ab451a 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -856,6 +856,19 @@ impl Config { || self.api.url.is_some() } + /// Refuse a setting that only a dialed peer or API reads, set with none to read it. + pub fn check_dials(&self) -> anyhow::Result<()> { + anyhow::ensure!( + self.api.root.is_empty() || self.api.url.is_some(), + "--cluster-api-root requires --cluster-api-url" + ); + anyhow::ensure!( + self.token.is_none() || self.dials_statics(), + "--cluster-token requires a peer to dial: --cluster-connect naming another relay, or --cluster-api-url (LAN peers never receive it)" + ); + Ok(()) + } + /// Released spellings this config was parsed from, each paired with what replaced it. pub fn deprecated(&self) -> moq_tokio::cli::Deprecated { let mut found = moq_tokio::cli::Deprecated::default(); @@ -1459,6 +1472,7 @@ impl Cluster { crate::configured_tier(config.tier.clone()).context("invalid --cluster-tier")?; peer.quic.validate()?; config.lan.validate()?; + config.check_dials()?; stats.check(&config)?; let mut work = kio::Tasks::new(); let mut origin_config = origin::Config::default(); @@ -2136,38 +2150,31 @@ impl Cluster { token: ClusterToken, dial: &crate::nodes::Dial, ) -> anyhow::Result<()> { - // Sessions shorter than this are treated as churn: we keep backing off - // instead of resetting, otherwise a peer that rejects us instantly would - // turn into a tight reconnect loop. - let stable_threshold = std::time::Duration::from_secs(10); - let mut redial = Redial::default(); loop { - let started = kio::clock::now(); + let mut connected = false; let (result, presented) = match Self::dial_addrs(target, &token) { - Ok((addrs, presented)) => (self.run_remote_once(&addrs, target, dial).await, presented), + Ok((addrs, presented)) => ( + self.run_remote_once(&addrs, target, dial, &mut connected).await, + presented, + ), Err(err) => (Err(err), None), }; dial.disconnected(); - let elapsed = started.elapsed(); - match result { - Ok(()) if elapsed >= stable_threshold => redial = Redial::default(), - Ok(()) => tracing::warn!(?elapsed, "cluster peer session closed cleanly but quickly; backing off"), - Err(err) => { - tracing::warn!(%err, "cluster peer error; will retry"); - // The session's own reconnects reuse the token it started with, so a - // failure after the file rotated is most likely that token expiring: - // present the new one at once instead of after a backoff. - if presented.is_some() && token.read().ok().flatten() != presented { - tracing::info!("cluster token rotated; redialing with it"); - continue; - } - } + // An attempt only ends in an error: its own reconnects retry until they give up. + let Err(err) = result; + tracing::warn!(%err, "cluster peer error; will retry"); + // The session's own reconnects reuse the token it started with, so a + // failure after the file rotated is most likely that token expiring: + // present the new one at once instead of after a backoff. + if presented.is_some() && token.read().ok().flatten() != presented { + tracing::info!("cluster token rotated; redialing with it"); + continue; } - let wait = redial.next(&mut kio::clock::rng()); + let wait = redial.next(connected, &mut kio::clock::rng()); kio::clock::sleep(wait).await; } } @@ -2196,18 +2203,20 @@ impl Cluster { Ok((addrs, presented)) } - /// One session to `target` over `addrs`, which carry any credential. + /// One session to `target` over `addrs`, which carry any credential. It runs until the + /// session gives up, and sets `connected` once the peer has accepted it. async fn run_remote_once( &self, addrs: &moq_tokio::Addrs, target: &DialTarget, dial: &crate::nodes::Dial, - ) -> anyhow::Result<()> { + connected: &mut bool, + ) -> anyhow::Result { // Each attempt is its own session, so it gets its own id. Matches the span an // accepted connection runs under, so both directions log the same way. let id = self.next_connection_id(); // A panic fails this attempt like any other error, so the peer is redialed. - moq_tokio::panic::isolate(self.run_remote_session(id, addrs, target, dial)) + moq_tokio::panic::isolate(self.run_remote_session(id, addrs, target, dial, connected)) .instrument(tracing::info_span!("conn", id)) .await .context("cluster peer session panicked")? @@ -2219,7 +2228,8 @@ impl Cluster { addrs: &moq_tokio::Addrs, target: &DialTarget, dial: &crate::nodes::Dial, - ) -> anyhow::Result<()> { + connected: &mut bool, + ) -> anyhow::Result { // The peer URL may carry the cluster JWT in its query, so neither the log // line nor the node label below may show the raw URL. let first = addrs.as_slice().first().expect("Addrs is non-empty").url(); @@ -2293,6 +2303,7 @@ impl Cluster { let mut observation = monitor.state(); loop { dial.observe(id, &observation); + *connected |= observation.connects > 0; observation = monitor.state_changed(observation.revision).await?; } } @@ -2535,7 +2546,12 @@ impl Redial { const CAP: std::time::Duration = std::time::Duration::from_secs(30); /// A wait drawn uniformly below the ceiling, which then doubles up to [`Self::CAP`]. - fn next(&mut self, rng: &mut impl rand::RngExt) -> std::time::Duration { + /// An attempt that `connected` starts over at [`Self::FIRST`]: the peer was reachable, + /// so what ended it is new, not the outage the ceiling grew through. + fn next(&mut self, connected: bool, rng: &mut impl rand::RngExt) -> std::time::Duration { + if connected { + *self = Self::default(); + } let wait = self.ceiling.mul_f64(rng.random()); self.ceiling = (self.ceiling * 2).min(Self::CAP); wait @@ -2884,7 +2900,7 @@ mod tests { let ceilings = [1, 2, 4, 8, 16, 30, 30].map(std::time::Duration::from_secs); let mut redials: Vec = (0..1000).map(|_| Redial::default()).collect(); for ceiling in ceilings { - let waits: Vec<_> = redials.iter_mut().map(|redial| redial.next(&mut rng)).collect(); + let waits: Vec<_> = redials.iter_mut().map(|redial| redial.next(false, &mut rng)).collect(); assert!(waits.iter().all(|wait| *wait <= ceiling), "a wait passed {ceiling:?}"); // Uniform over the range: every tenth of it is hit, so none is a lockstep. for tenth in 0..10 { @@ -2897,6 +2913,21 @@ mod tests { } } + /// A peer that was reachable redials from the first ceiling, so a flap after a + /// long outage does not wait out the 30 s the outage grew it to. + #[test] + fn redial_resets_after_an_attempt_that_connected() { + use rand::SeedableRng; + let mut rng = rand::rngs::StdRng::seed_from_u64(1); + let mut redial = Redial::default(); + for _ in 0..8 { + redial.next(false, &mut rng); + } + assert_eq!(redial.ceiling, Redial::CAP); + assert!(redial.next(true, &mut rng) <= Redial::FIRST); + assert_eq!(redial.ceiling, Redial::FIRST * 2); + } + /// `?cost=` is read and stripped (it rides SETUP, not the URL), other /// query params survive, and a garbage value is an error rather than a /// silent default. diff --git a/rs/moq-relay/src/config.rs b/rs/moq-relay/src/config.rs index 5a9b10fe877..4eaf16a974c 100644 --- a/rs/moq-relay/src/config.rs +++ b/rs/moq-relay/src/config.rs @@ -355,6 +355,7 @@ impl Config { self.check_runtime()?; self.check_handoff()?; self.stats.check(&self.cluster)?; + self.cluster.check_dials()?; self.check_lan()?; self.quic @@ -996,6 +997,23 @@ max_streams = 64 } resolve("[stats]\nenabled = true\n"); + // A setting only a dialed peer or API reads is refused with none to read it. + refuse("[cluster.api]\nroot = [\"api.pem\"]\n", "--cluster-api-root requires --cluster-api-url"); + resolve("[cluster.api]\nurl = \"https://api.example/peers\"\nroot = [\"api.pem\"]\n"); + for alone in [ + "[cluster]\ntoken = \"cluster.jwt\"\n", + "[cluster]\nnode = \"https://a.example\"\nconnect = [\"https://a.example\"]\ntoken = \"cluster.jwt\"\n", + "[cluster]\ntoken = \"cluster.jwt\"\n[cluster.lan]\nenabled = true\n", + ] { + refuse(alone, "--cluster-token requires a peer to dial"); + } + for peers in [ + "connect = [\"https://b.example\"]\n", + "api.url = \"https://api.example/peers\"\n", + ] { + resolve(&format!("[cluster]\ntoken = \"cluster.jwt\"\n{peers}")); + } + // Embedders build a cluster without resolving, so it refuses the same. let cluster: cluster::Config = toml::from_str("node = \"https://a.example\"\n").expect("parse"); let stats: crate::stats::Config = toml::from_str("enabled = true\n").expect("parse"); diff --git a/rs/moq-tokio/src/secret.rs b/rs/moq-tokio/src/secret.rs index e27f0c1d377..9198e849e50 100644 --- a/rs/moq-tokio/src/secret.rs +++ b/rs/moq-tokio/src/secret.rs @@ -15,6 +15,10 @@ pub enum Error { /// The file could not be read. #[error("failed to read secret file {0}")] Read(PathBuf, #[source] std::io::Error), + + /// The key itself was given where its file belongs. The error never repeats it. + #[error("the secret must be the path of a file holding it, not the key itself (`openssl rand -hex 32 > secret.key`)")] + Inline, } /// A random 32-byte secret. @@ -29,7 +33,10 @@ impl Secret { /// followed by one newline, as `openssl rand -hex 32` writes it. pub fn read(path: impl AsRef) -> Result { let path = path.as_ref(); - let contents = std::fs::read(path).map_err(|err| Error::Read(path.to_path_buf(), err))?; + let contents = std::fs::read(path).map_err(|err| match is_key(path) { + true => Error::Inline, + false => Error::Read(path.to_path_buf(), err), + })?; let hex = contents.strip_suffix(b"\n").unwrap_or(&contents); let mut secret = [0; 32]; hex::decode_to_slice(hex, &mut secret).map_err(|_| Error::Invalid(path.to_path_buf()))?; @@ -42,6 +49,13 @@ impl Secret { } } +/// Whether `path` spells a key, so an error never prints it into a log. +fn is_key(path: &Path) -> bool { + let mut key = [0; 32]; + path.to_str() + .is_some_and(|path| hex::decode_to_slice(path, &mut key).is_ok()) +} + impl From<[u8; 32]> for Secret { fn from(secret: [u8; 32]) -> Self { Self(secret) @@ -95,4 +109,13 @@ mod tests { Err(Error::Read(..)) )); } + + /// A key given where its file belongs is refused without echoing it, as an + /// upgrade from the inline form would otherwise print it into the log. + #[test] + fn an_inline_key_is_refused_without_echoing_it() { + let err = Secret::read(KEY).unwrap_err(); + assert!(matches!(err, Error::Inline), "{err:?}"); + assert!(!err.to_string().contains(KEY) && !format!("{err:?}").contains(KEY)); + } } From 3e48d5a1d5775e5c326e611cb9e76898284ea773 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 18:15:48 -0800 Subject: [PATCH 3/5] fix(auth): a subsecond revalidate still writes rounded, never zero Co-Authored-By: Claude Sonnet 5.5 --- rs/moq-auth/src/grant.rs | 10 ++++++++++ rs/moq-auth/src/wire.rs | 4 +++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/rs/moq-auth/src/grant.rs b/rs/moq-auth/src/grant.rs index f19d0d0eb60..99b24d6559b 100644 --- a/rs/moq-auth/src/grant.rs +++ b/rs/moq-auth/src/grant.rs @@ -181,6 +181,16 @@ mod tests { } } + /// A subsecond cadence writes as the nearest whole second, never the zero a relay refuses. + #[test] + fn a_subsecond_revalidate_writes_rounded() { + let mut grant = full(); + for (cadence, seconds) in [(750, 1), (1400, 1), (1600, 2)] { + grant.revalidate = Some(Duration::from_millis(cadence)); + assert_eq!(serde_json::to_value(&grant).unwrap()["revalidate"], seconds); + } + } + #[test] fn round_trips_in_seconds() { let grant = full(); diff --git a/rs/moq-auth/src/wire.rs b/rs/moq-auth/src/wire.rs index ccbdde0731e..09222a449e0 100644 --- a/rs/moq-auth/src/wire.rs +++ b/rs/moq-auth/src/wire.rs @@ -222,7 +222,9 @@ mod revalidate { pub(super) fn serialize(value: &Option, serializer: S) -> Result { match value { - Some(value) => serializer.serialize_u64(value.as_secs()), + // Rounded to the nearest second, as `DurationSeconds` writes it: a `750ms` interval + // must not truncate to a zero the relay refuses. + Some(value) => as serde_with::SerializeAs>::serialize_as(value, serializer), None => serializer.serialize_none(), } } From 0a9bf74fac352f39dca12f9886268cb0d09e64d0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 18:46:30 -0800 Subject: [PATCH 4/5] fix(relay): check fixes: format, retarget the redial drill mutation, leave MIGRATE.md to the cut Co-Authored-By: Claude Sonnet 5.5 --- rs/moq-auth/tests/corpus.json | 46 +++++++++++++++++-- rs/moq-relay/MIGRATE.md | 3 +- rs/moq-relay/src/config.rs | 5 +- rs/moq-tokio/src/secret.rs | 4 +- .../mutations/peer-link-never-redialed.patch | 6 +-- 5 files changed, 52 insertions(+), 12 deletions(-) diff --git a/rs/moq-auth/tests/corpus.json b/rs/moq-auth/tests/corpus.json index 5efb9225879..1246fbc718f 100644 --- a/rs/moq-auth/tests/corpus.json +++ b/rs/moq-auth/tests/corpus.json @@ -302,11 +302,47 @@ }, "valid": false }, - { "name": "a_null_scope", "jwk": { "kty": "oct", "alg": "HS256", "k": "Fp8kipWUJeUFqeSqWym_tRC_tyI8z-QpqopIGrbrD68", "scope": null }, "valid": false }, - { "name": "a_null_kid", "jwk": { "kty": "oct", "alg": "HS256", "k": "Fp8kipWUJeUFqeSqWym_tRC_tyI8z-QpqopIGrbrD68", "kid": null }, "valid": false }, - { "name": "a_null_key_ops", "jwk": { "kty": "oct", "alg": "HS256", "k": "Fp8kipWUJeUFqeSqWym_tRC_tyI8z-QpqopIGrbrD68", "key_ops": null }, "valid": false }, - { "name": "a_null_scope_root", "jwk": { "kty": "oct", "alg": "HS256", "k": "Fp8kipWUJeUFqeSqWym_tRC_tyI8z-QpqopIGrbrD68", "scope": { "root": null, "publish": ["**"] } }, "valid": false }, - { "name": "a_null_okp_private_key", "jwk": { "kty": "OKP", "alg": "EdDSA", "crv": "Ed25519", "x": "UiU9fT_SdBBpkFtJPRCY0gX1jK_Dr9syYLFuEz4QUM4", "d": null }, "valid": false }, + { + "name": "a_null_scope", + "jwk": { "kty": "oct", "alg": "HS256", "k": "Fp8kipWUJeUFqeSqWym_tRC_tyI8z-QpqopIGrbrD68", "scope": null }, + "valid": false + }, + { + "name": "a_null_kid", + "jwk": { "kty": "oct", "alg": "HS256", "k": "Fp8kipWUJeUFqeSqWym_tRC_tyI8z-QpqopIGrbrD68", "kid": null }, + "valid": false + }, + { + "name": "a_null_key_ops", + "jwk": { + "kty": "oct", + "alg": "HS256", + "k": "Fp8kipWUJeUFqeSqWym_tRC_tyI8z-QpqopIGrbrD68", + "key_ops": null + }, + "valid": false + }, + { + "name": "a_null_scope_root", + "jwk": { + "kty": "oct", + "alg": "HS256", + "k": "Fp8kipWUJeUFqeSqWym_tRC_tyI8z-QpqopIGrbrD68", + "scope": { "root": null, "publish": ["**"] } + }, + "valid": false + }, + { + "name": "a_null_okp_private_key", + "jwk": { + "kty": "OKP", + "alg": "EdDSA", + "crv": "Ed25519", + "x": "UiU9fT_SdBBpkFtJPRCY0gX1jK_Dr9syYLFuEz4QUM4", + "d": null + }, + "valid": false + }, { "name": "short_secret", "jwk": { "kty": "oct", "alg": "HS256", "k": "c2hvcnQ" }, diff --git a/rs/moq-relay/MIGRATE.md b/rs/moq-relay/MIGRATE.md index 655d5923c9e..6cff81e44f8 100644 --- a/rs/moq-relay/MIGRATE.md +++ b/rs/moq-relay/MIGRATE.md @@ -125,8 +125,7 @@ An auth server that scopes tenants by hostname must see them over TLS, and still ### A client certificate alone lists no hidden routes (#6509) Only a grant with `peer: true` lists `.`-named routes to a session that did not ask for them. -Have the auth server mark cluster peers as `peer` (`moq auth serve --mtls-link peer`) before upgrading, or a peer below moq-lite-07-wip loses those paths. -A certificate-only peer also gets `[limits.client]` instead of `[limits.peer]`, so it is closed with `TOO_MANY_REQUESTS` past 1000 announced broadcasts (`announces`) instead of 100000. +Have the auth server mark cluster peers as `peer` (`moq auth serve --mtls-peer`) before upgrading, or a peer below moq-lite-07-wip loses those paths. ### `stats.interval` is a duration (#6472) diff --git a/rs/moq-relay/src/config.rs b/rs/moq-relay/src/config.rs index 4eaf16a974c..36cab4c58ca 100644 --- a/rs/moq-relay/src/config.rs +++ b/rs/moq-relay/src/config.rs @@ -998,7 +998,10 @@ max_streams = 64 resolve("[stats]\nenabled = true\n"); // A setting only a dialed peer or API reads is refused with none to read it. - refuse("[cluster.api]\nroot = [\"api.pem\"]\n", "--cluster-api-root requires --cluster-api-url"); + refuse( + "[cluster.api]\nroot = [\"api.pem\"]\n", + "--cluster-api-root requires --cluster-api-url", + ); resolve("[cluster.api]\nurl = \"https://api.example/peers\"\nroot = [\"api.pem\"]\n"); for alone in [ "[cluster]\ntoken = \"cluster.jwt\"\n", diff --git a/rs/moq-tokio/src/secret.rs b/rs/moq-tokio/src/secret.rs index 9198e849e50..d734513ba0b 100644 --- a/rs/moq-tokio/src/secret.rs +++ b/rs/moq-tokio/src/secret.rs @@ -17,7 +17,9 @@ pub enum Error { Read(PathBuf, #[source] std::io::Error), /// The key itself was given where its file belongs. The error never repeats it. - #[error("the secret must be the path of a file holding it, not the key itself (`openssl rand -hex 32 > secret.key`)")] + #[error( + "the secret must be the path of a file holding it, not the key itself (`openssl rand -hex 32 > secret.key`)" + )] Inline, } diff --git a/test/drill/mutations/peer-link-never-redialed.patch b/test/drill/mutations/peer-link-never-redialed.patch index 363a8442164..51f60b6bdda 100644 --- a/test/drill/mutations/peer-link-never-redialed.patch +++ b/test/drill/mutations/peer-link-never-redialed.patch @@ -6,13 +6,13 @@ # the route for good: the groups caught in the flap and every burst after it # never arrive. A steady link never drops, so only the flap drill depends on it. diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs -index 844ee1e1ba..73ee5dde72 100644 +index 0ba85ab451..fe18bdbfc0 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs -@@ -2303,6 +2303,11 @@ impl Cluster { - let mut observation = monitor.state(); +@@ -2304,6 +2304,11 @@ impl Cluster { loop { dial.observe(id, &observation); + *connected |= observation.connects > 0; + // MUTATION: a lost peer link is never redialed. + if matches!(observation.phase, moq_tokio::connection::Phase::Disconnected { .. }) { + drop(reconnect); From 07274a44a78c8a148094f238ad5225057f04cbe5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 19:20:07 -0800 Subject: [PATCH 5/5] fix(relay): keep an inline key out of the config trace, and record the backport pick An inline secret in cluster.lan.secret, listen.reset.secret, or the forward key no longer reaches the trace of the resolved config; a trailing newline and a decode failure on a key-shaped path stay redacted too. A redial after a token rotation also starts the backoff over when the attempt connected. The relay-backoff-backport quest records the maintainer's pick: backport only the backoff reset. Co-Authored-By: Claude Sonnet 5.5 --- .../m1/precut-fixes/relay-backoff-backport.md | 26 ++++++------- rs/moq-relay/src/cluster.rs | 19 +++++++++- rs/moq-tokio/src/listen.rs | 38 ++++++++++++++++++- rs/moq-tokio/src/secret.rs | 34 ++++++++++++++++- 4 files changed, 98 insertions(+), 19 deletions(-) diff --git a/quest/m1/precut-fixes/relay-backoff-backport.md b/quest/m1/precut-fixes/relay-backoff-backport.md index 6f350bae28f..53dde7f40e8 100644 --- a/quest/m1/precut-fixes/relay-backoff-backport.md +++ b/quest/m1/precut-fixes/relay-backoff-backport.md @@ -8,20 +8,20 @@ longer waits out 30 s. ## Plan -Port the `main` fix from the PR that finished `relay-config-loud` as a separate -PR onto `release`, per RELEASE.md, keeping the published API. +Decided by the maintainer when the PR that finished `relay-config-loud` was +picked: ✅ backport only the redial backoff reset to `release`, and leave the +two startup refusals on `main` (rejected: backporting all of it, which would +let a patch stop a working relay from booting on a config that ran before; no +backport, which leaves a flapping peer waiting out 30 s on `release`). + +Port the backoff reset from that PR as a separate PR onto `release`, per +RELEASE.md, keeping the published API. `rs/moq-relay/src/cluster.rs` `run_remote` on `release` has the same dead `Ok(()) if elapsed >= stable_threshold` branch, because `run_remote_once` only returns an error. Report whether the attempt connected (any observation with -`connects > 0`) and pass it to `Redial::next`. - -The same PR on `main` also refuses `cluster.api.root` without `cluster.api.url` -and `cluster.token` with no peer to dial. Those are startup failures for -configs that ran before, so a patch release should not carry them. - -## Waiting +`connects > 0`) and pass it to `Redial::next`, which starts over at the first +ceiling when it did, including on the immediate retry after the cluster token +rotated. -- Maintainer: whether the backport is wanted at all, and whether it carries the - two config refusals. Recommend backporting the backoff reset only: a flapping - peer is a degraded mesh, while a patch that stops a working relay from - booting is a break. Check by this quest's Plan naming the choice. +Do not port `cluster.api.root` without `cluster.api.url` or `cluster.token` +with no peer to dial as startup errors: those stay on `main`. diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 0ba85ab451a..3fa6a9660cc 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -857,7 +857,7 @@ impl Config { } /// Refuse a setting that only a dialed peer or API reads, set with none to read it. - pub fn check_dials(&self) -> anyhow::Result<()> { + pub(crate) fn check_dials(&self) -> anyhow::Result<()> { anyhow::ensure!( self.api.root.is_empty() || self.api.url.is_some(), "--cluster-api-root requires --cluster-api-url" @@ -918,7 +918,7 @@ impl Config { /// so a rack or a home lab meshes with no seed list and no shared rendezvous. /// A LAN peer authenticates with its mDNS credential on `/.cluster/` /// and is never handed [`Config::token`]. -#[derive(usage_rs::Args, Clone, Debug, serde::Serialize, serde::Deserialize, Default)] +#[derive(usage_rs::Args, Clone, serde::Serialize, serde::Deserialize, Default)] #[usage(unknown_flags = "error", args_override_self = false)] #[serde_with::skip_serializing_none] #[serde(default, deny_unknown_fields)] @@ -966,6 +966,17 @@ pub struct LanConfig { pub app: Option, } +impl std::fmt::Debug for LanConfig { + /// Redact an inline key: [`crate::Config::load`] traces the whole resolved config. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LanConfig") + .field("enabled", &self.enabled) + .field("secret", &self.secret.as_deref().map(moq_tokio::secret::Redacted)) + .field("app", &self.app) + .finish() + } +} + impl LanConfig { /// Reject a secret or app configured without the mesh that would read it. pub fn validate(&self) -> anyhow::Result<()> { @@ -2171,6 +2182,10 @@ impl Cluster { // present the new one at once instead of after a backoff. if presented.is_some() && token.read().ok().flatten() != presented { tracing::info!("cluster token rotated; redialing with it"); + // An attempt that connected starts the backoff over, even when it skips the wait. + if connected { + redial = Redial::default(); + } continue; } diff --git a/rs/moq-tokio/src/listen.rs b/rs/moq-tokio/src/listen.rs index e0310f495c3..bf054aae12d 100644 --- a/rs/moq-tokio/src/listen.rs +++ b/rs/moq-tokio/src/listen.rs @@ -239,7 +239,7 @@ pub struct Config { } /// QUIC stateless reset settings (`[listen.reset]`, `--listen-reset-*`). -#[derive(usage_rs::Args, Clone, Default, Debug, serde::Serialize, serde::Deserialize)] +#[derive(usage_rs::Args, Clone, Default, serde::Serialize, serde::Deserialize)] #[usage(unknown_flags = "error", args_override_self = false)] #[serde(deny_unknown_fields)] #[non_exhaustive] @@ -264,6 +264,15 @@ pub struct Reset { pub secret: Option, } +impl std::fmt::Debug for Reset { + /// Redact an inline key: the relay traces its whole resolved config. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Reset") + .field("secret", &self.secret.as_deref().map(crate::secret::Redacted)) + .finish() + } +} + /// The QUIC listener settings with defaults applied, produced by [`Config::resolve`]. /// /// Exhaustive on purpose, unlike [`crate::connect::Resolved`]: a QUIC runtime outside @@ -623,6 +632,20 @@ impl Config { #[cfg(test)] mod tests { + /// The relay traces its whole config, so an inline key never reaches a `Debug`. + #[test] + fn debug_redacts_an_inline_secret() { + let key = "00".repeat(32); + let reset = Reset { + secret: Some(key.clone().into()), + }; + let forward = Forward { + key: Some(key.clone().into()), + ..Default::default() + }; + assert!(!format!("{reset:?} {forward:?}").contains(&key)); + } + use super::*; /// The SETUP extensions are no setting: a file naming them is refused. @@ -1008,7 +1031,7 @@ pub struct LoadBalancer { /// packet whose connection ID names another server to the owner, which processes it as if the /// client had sent it there and keeps replying from the shared address. Only packets after the /// handshake are forwarded, so the front end must still keep a handshake on one server. -#[derive(Clone, Debug, Default, usage_rs::Args, serde::Serialize, serde::Deserialize)] +#[derive(Clone, Default, usage_rs::Args, serde::Serialize, serde::Deserialize)] #[serde(deny_unknown_fields)] #[non_exhaustive] pub struct Forward { @@ -1051,6 +1074,17 @@ pub struct Forward { pub peers: Option, } +impl std::fmt::Debug for Forward { + /// Redact an inline key: the relay traces its whole resolved config. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Forward") + .field("bind", &self.bind) + .field("key", &self.key.as_deref().map(crate::secret::Redacted)) + .field("peers", &self.peers) + .finish() + } +} + impl Forward { /// Whether any forwarding setting is present. pub(crate) fn configured(&self) -> bool { diff --git a/rs/moq-tokio/src/secret.rs b/rs/moq-tokio/src/secret.rs index d734513ba0b..56eab81e025 100644 --- a/rs/moq-tokio/src/secret.rs +++ b/rs/moq-tokio/src/secret.rs @@ -41,7 +41,10 @@ impl Secret { })?; let hex = contents.strip_suffix(b"\n").unwrap_or(&contents); let mut secret = [0; 32]; - hex::decode_to_slice(hex, &mut secret).map_err(|_| Error::Invalid(path.to_path_buf()))?; + hex::decode_to_slice(hex, &mut secret).map_err(|_| match is_key(path) { + true => Error::Inline, + false => Error::Invalid(path.to_path_buf()), + })?; Ok(Self(secret)) } @@ -52,10 +55,25 @@ impl Secret { } /// Whether `path` spells a key, so an error never prints it into a log. +/// A trailing newline counts: a key read from a file into an environment variable has one. fn is_key(path: &Path) -> bool { let mut key = [0; 32]; path.to_str() - .is_some_and(|path| hex::decode_to_slice(path, &mut key).is_ok()) + .is_some_and(|path| hex::decode_to_slice(path.trim_end(), &mut key).is_ok()) +} + +/// A secret file's path for `Debug`, which prints a key given where its file belongs +/// as `[redacted]`. `moq-relay` traces the whole resolved config, so a +/// derived `Debug` over the path would write the key into the log. +pub struct Redacted<'a>(pub &'a Path); + +impl fmt::Debug for Redacted<'_> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match is_key(self.0) { + true => formatter.write_str("[redacted]"), + false => self.0.fmt(formatter), + } + } } impl From<[u8; 32]> for Secret { @@ -119,5 +137,17 @@ mod tests { let err = Secret::read(KEY).unwrap_err(); assert!(matches!(err, Error::Inline), "{err:?}"); assert!(!err.to_string().contains(KEY) && !format!("{err:?}").contains(KEY)); + + // With the trailing newline a file's contents carry. + let with_newline = format!("{KEY}\n"); + let err = Secret::read(&with_newline).unwrap_err(); + assert!(matches!(err, Error::Inline), "{err:?}"); + } + + /// A config's `Debug` never prints an inline key, nor hides a real path. + #[test] + fn debug_redacts_an_inline_key_only() { + assert_eq!(format!("{:?}", Redacted(Path::new(KEY))), "[redacted]"); + assert_eq!(format!("{:?}", Redacted(Path::new("secret.key"))), "\"secret.key\""); } }