diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index a08f00c2984..9ef4ba083ec 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -26,7 +26,9 @@ versions translates: a moq-lite-06 peer's routes enter the route layer as the relay's own claims, and lite-07 routes reach a moq-lite-06 peer as a hop list that starts at their origin node. A relay drops a moq-lite-06 route whose hop list names its own routing identity, since that is one of its claims coming -back. +back. It holds back one naming an identity it routes to over moq-lite-07, a +copy of that node's claims that would otherwise outlive their withdrawal, and +announces it afresh once that route is lost, so the moq-lite-06 path takes over. A track from a moq-transport publisher keeps its Object IDs, object properties, and END\_OF\_GROUP status objects across a moq-lite-07 link, since relays diff --git a/quest/m2/cluster-routing/routes/README.md b/quest/m2/cluster-routing/routes/README.md index 1816813739e..05743abd669 100644 --- a/quest/m2/cluster-routing/routes/README.md +++ b/quest/m2/cluster-routing/routes/README.md @@ -542,7 +542,6 @@ benchmarks. - [Route layer acceptance](/quest/m2/cluster-routing/routes/acceptance.md) - forwarding fixtures ported and the model deleted, incident fixtures, end-to-end checks, and benchmarks - [Firefox tabs prove their route key](/quest/m2/cluster-routing/routes/browser-proof.md) - a current Firefox tab opens a lite-07 session with a route-key proof the relay verifies, once neqo's exporter follows the draft - [Playwright bundles Chromium's spec exporter](/quest/m2/cluster-routing/routes/playwright-pin.md) - the toolchain's Playwright bundles Chromium 158+, so the route key lane proves the Chromium tab -- [A declined lite-06 route is reconsidered when the lite-07 path fails](/quest/m2/cluster-routing/routes/lite06-failover.md) - a mixed-version relay fails over to the lite-06 route it declined - [A covering predecessor's takeover shows under a mount](/quest/m2/cluster-routing/routes/mount-takeover.md) - lite-07 presents a takeover beneath a mount as lite-06 does ## Related diff --git a/quest/m2/cluster-routing/routes/lite06-failover.md b/quest/m2/cluster-routing/routes/lite06-failover.md deleted file mode 100644 index 01e0ac1055e..00000000000 --- a/quest/m2/cluster-routing/routes/lite06-failover.md +++ /dev/null @@ -1,23 +0,0 @@ -# [M] A declined lite-06 route is reconsidered when the lite-07 path fails - -## Goal - -In a mixed-version cluster, a relay that declined an identity's lite-06 -route because it already knew that identity over lite-07 accepts the lite-06 -route again once the lite-07 path is gone, so the lite-06 path takes over -instead of leaving the broadcast unreachable. Today the relay declines it -once and never revisits it. - -## Plan - -Decided 2026-11-06 (maintainer, /quest-complete, a follow-up of -[moq-dev/moq#8023](https://github.com/moq-dev/moq/pull/8023)): ✅ m2, in this -epic, since the cut ships mixed-version clusters and this is a failover gap. -Not chosen: m3; m4. - -- Recognize known identities on lite-06 ingress, and keep the declined route - so it can be accepted again when the lite-07 route for that identity ends. -- Cover it with a mixed-version test: a relay with both paths to one - publisher loses the lite-07 one and keeps serving over lite-06. - -Starts once #8023 lands. Public API: none expected. Wire: none. diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 58f24509804..9ae5666b2b1 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -114,6 +114,9 @@ pub(super) struct Subscriber { /// The instances minted for the paths the peer advertises without one, shared by the /// announce streams. minted: Minted, + /// The identities the origin routes to over lite-07, whose chains are held back where + /// those routes reach. + known: kio::Shared, } /// The instance a session minted for each path a peer advertises without one (every @@ -474,6 +477,7 @@ impl Subscriber { // every session sharing that origin, required for cross-session // loop detection. let self_origin = crate::Hop::from(config.origin.hop()); + let known = config.origin.known(); Self { session: config.session, runtime: config.runtime, @@ -500,6 +504,7 @@ impl Subscriber { announces: Default::default(), viewers: Default::default(), minted: Default::default(), + known, } } @@ -743,6 +748,15 @@ impl Subscriber { route } + /// Whether the route the peer advertised at `path` names, in its chain, an identity + /// the origin routes to over lite-07 through a route that carries that path. + fn shadowed(&self, path: &Path, route: &crate::origin::RouteInfo) -> bool { + let Ok(path) = self.origin.root().join(path) else { + return false; + }; + self.known.read().names(&route.hops, &path) + } + /// The announcing session's declared or assigned identity, for split-horizon. /// /// A non-zero AnnounceOk origin is the declared id. Otherwise the caller-assigned @@ -787,8 +801,11 @@ impl Subscriber { let metadata = self.announced_route(&path, hops, cost, link_cost, responder_origin, announced); // A re-price keeps the route's prefix (and its - // served paths) and re-prices in place. In-flight tracks keep flowing. - if let Some(entry) = announced.attached(&path) { + // served paths) and re-prices in place. In-flight tracks keep flowing. A chain + // that now names an identity routed over lite-07 is offered, to be held back. + if !self.shadowed(&path, &metadata) + && let Some(entry) = announced.attached(&path) + { entry.update(metadata); return Ok(true); } @@ -2133,6 +2150,8 @@ struct PrefixRun { decoder: lite::AnnounceDecoder, /// The auth epoch last applied, so a new limit holds back or brings back routes. auth_epoch: u64, + /// The [`crate::route::node::Known`] epoch last applied. + known_epoch: u64, /// Lite05+: the ANNOUNCE_STARTs of the initial set still to read. Anything else /// before them, or the stream ending, cuts the counted set short. initial: u64, @@ -2231,6 +2250,7 @@ impl AnnouncePrefix { announced: Announced::new(self.subscriber.announces.clone(), self.subscriber.minted.clone()), decoder: lite::AnnounceDecoder::default(), auth_epoch: 0, + known_epoch: 0, initial, routed: match version.has_routes() { true => { @@ -2299,6 +2319,20 @@ impl AnnouncePrefix { routed.limit(&self.subscriber, &permit)?; } } + // A hop chain naming an identity this node gains or loses a lite-07 route to + // is held back or taken up. Lite-07 streams carry no chains. + while run.routed.is_none() { + let seen = run.known_epoch; + let Poll::Ready(known) = self.subscriber.known.poll(waiter, |known| match known.epoch() != seen { + true => Poll::Ready(()), + false => Poll::Pending, + }) else { + break; + }; + run.known_epoch = known.epoch(); + drop(known); + run.announced.reconsider(&self.subscriber); + } // Each announce routes into the origin, so a peer streaming them must not hold // the session's other tasks. A yield has woken the task, and this arm always // ends Pending, so the routes below are still served this turn. @@ -9621,6 +9655,11 @@ enum Held { /// Outside the session's limit: the route as advertised, attached once a /// limit allows it. Withheld(crate::origin::RouteInfo), + /// Its chain names an identity the origin routes to over lite-07 through a route + /// that carries this path: the route as advertised, attached once none is left. Filed now, it would be + /// another copy of that identity's claim under this node's own, outliving its + /// withdrawal until this chain retracts too. + Shadowed(crate::origin::RouteInfo), /// In the origin, serving requests. Attached(Box), } @@ -9671,6 +9710,11 @@ impl Announced { /// session on demand, or hold it back when the session's limit does not cover it. /// Returns whether it attached. fn offer(&mut self, subscriber: &Subscriber, path: Path, route: crate::origin::RouteInfo) -> bool { + if subscriber.shadowed(&path, &route) { + tracing::debug!(route = %subscriber.log_path(&path), "holding back announce of an identity routed over lite-07"); + self.routes.insert(path, Held::Shadowed(route)); + return false; + } if !subscriber .auth .within_limit(crate::auth::Direction::Subscribe, path.as_str()) @@ -9786,7 +9830,7 @@ impl Announced { .filter(|(path, held)| match held { Held::Attached(_) => !permit.within_limit(path.as_str()), Held::Withheld(_) => permit.within_limit(path.as_str()), - Held::Declined => false, + Held::Declined | Held::Shadowed(_) => false, }) .map(|(path, _)| path.clone()) .collect(); @@ -9808,6 +9852,31 @@ impl Announced { } } + /// Apply a change in what the origin routes to over lite-07: hold back every attached + /// route whose chain now names such an identity, closing its sources, and offer every + /// held-back one whose chain no longer does, so the lite-06 path takes over. + fn reconsider(&mut self, subscriber: &Subscriber) { + let changed: Vec = (self.routes.iter()) + .filter(|(path, held)| match held { + Held::Attached(entry) => subscriber.shadowed(path, &entry.route), + Held::Shadowed(route) => !subscriber.shadowed(path, route), + Held::Declined | Held::Withheld(_) => false, + }) + .map(|(path, _)| path.clone()) + .collect(); + for path in changed { + let mut route = match self.routes.remove(&path) { + Some(Held::Attached(entry)) => entry.route.clone(), + Some(Held::Shadowed(route)) => route, + _ => unreachable!("only attached and held-back routes change"), + }; + if subscriber.going_away.is_set() { + route.cost = crate::origin::Cost::DRAIN; + } + self.offer(subscriber, path, route); + } + } + /// Re-price every attached route to a draining cost (the peer sent a GOAWAY). fn drain(&mut self) { for held in self.routes.values_mut() { diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index c4491adeda1..e94702ee55f 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -2404,6 +2404,15 @@ impl Producer { Some(crate::route::node::hop(node.declared())) } + /// The identities this node routes to over moq-lite-07, as the lite-06 hops they + /// travel as. A lite-06 route naming one is held back while that route lasts, so + /// the broadcast is not filed again as this node's own claim, and taken up once it + /// goes. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn known(&self) -> kio::Shared { + self.shared.lock().known.clone() + } + /// This origin's construction config. pub fn config(&self) -> Config { let shared = self.shared.lock(); @@ -6004,6 +6013,9 @@ struct OriginState { // The route layer node this origin's lite-07 sessions share; see // [`Consumer::route_node`]. route_node: Option, + // What that node routes to, published before it exists so older-version sessions + // can watch it from the start. + known: kio::Shared, } impl OriginState { @@ -6023,6 +6035,7 @@ impl OriginState { next_listener: 0, hidden_withheld: HiddenWithheld::default(), route_node: None, + known: Default::default(), } } @@ -7222,7 +7235,7 @@ impl Consumer { if let Some(node) = &shared.route_node { return node.clone(); } - let node = crate::route::Node::random(); + let node = crate::route::Node::random(shared.known.clone()); shared.route_node = Some(node.clone()); drop(shared); // The whole table, whatever this handle's scope. diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index 89efac0beb1..c9b1fc265a0 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -281,6 +281,20 @@ impl Peer { } } +/// Which claims one unscoped route can carry: those within the prefix this node asked +/// its neighbor for, or every claim when unset. See [`Router::carried`]. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Carried { + asked: Option, +} + +impl Carried { + /// Whether a claim at `claim` goes through this route. + pub fn carries(&self, claim: &Path) -> bool { + self.asked.as_ref().is_none_or(|asked| within(claim, asked)) + } +} + /// An identity this node serves itself. struct Origin { seqno: Seqno, @@ -870,6 +884,24 @@ impl Router { .collect() } + /// What each usable feasible route to `id` through an unscoped neighbor can carry, + /// as [`Self::routes`] filters it per claim. A scoped neighbor is left out: the + /// session serves only what its permit's limit admits, which its regions only + /// bound. Empty for an identity this node serves itself or has no such route to. + pub fn carried(&self, id: Id) -> Vec { + if self.origins.contains_key(&id) { + return Vec::new(); + } + (self.rx.get(&id).into_iter().flatten()) + .filter(|(_, entry)| !entry.held && self.feasible(id, entry.advert)) + .map(|(next, entry)| (&self.neighbors[next], entry)) + .filter(|(peer, entry)| peer.link.scope.is_none() && add(entry.advert.metric, peer.link.cost).is_some()) + .map(|(peer, _)| Carried { + asked: peer.asked.clone(), + }) + .collect() + } + #[cfg(test)] /// The route this node would offer an upstream link for `id`. pub fn export(&self, id: Id) -> Option> { diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 5cb25080859..090f2176ed4 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -24,7 +24,7 @@ use crate::{Instance, Path}; use rand::RngExt; -use super::{Error, Id, Link, Metric, Out, QUALIFIED, Reach, Rev, Router, Seqno}; +use super::{Carried, Error, Id, Link, Metric, Out, QUALIFIED, Reach, Rev, Router, Seqno}; /// One announce stream in one direction. pub(crate) type Key = u64; @@ -89,6 +89,29 @@ pub(crate) enum Output { Unserve { id: Id, claim: Path }, } +/// The identities a node routes to over lite-07, by the lite-06 hop each travels +/// as, and what their usable routes can carry, so its older-version sessions +/// recognize a hop chain that names one. +#[derive(Debug, Default)] +pub(crate) struct Known { + hops: HashMap>, + /// Bumped on every change, for a reader to tell when to look again. + epoch: u64, +} + +impl Known { + /// Whether `hops` names an identity with a usable lite-07 route that carries a + /// claim at the absolute `path`. + pub fn names(&self, hops: &crate::Hops, path: &Path) -> bool { + (hops.iter().filter_map(|hop| self.hops.get(hop))) + .any(|carried| carried.iter().any(|route| route.carries(path))) + } + + pub fn epoch(&self) -> u64 { + self.epoch + } +} + /// The node every lite-07 session of one origin shares. #[derive(Clone)] pub(crate) struct Node { @@ -301,6 +324,9 @@ struct State { queues: Queues, /// Each shown claim of another identity, and the streams serving it. homes: BTreeMap<(Id, Path), Home>, + /// The other identities with a usable route, and what those routes carry. + routed: BTreeMap>, + known: kio::Shared, } /// Where one shown claim is served: each stream, what it was last sent, and @@ -343,7 +369,13 @@ impl Home { impl Node { /// A node declaring a fresh identity, `seed` keying its rotation. + #[cfg(test)] pub fn new(id: Id, seed: u64) -> Self { + Self::publishing(id, seed, Default::default()) + } + + /// A node declaring a fresh identity that keeps `known` current. + fn publishing(id: Id, seed: u64, known: kio::Shared) -> Self { debug_assert!( id != 0 && id & QUALIFIED == 0, "a declared identity is non-zero and unqualified" @@ -356,15 +388,17 @@ impl Node { next_key: 1, queues: HashMap::new(), homes: BTreeMap::new(), + routed: BTreeMap::new(), + known, })), } } - /// A node with a random identity. - pub fn random() -> Self { + /// A node with a random identity, keeping `known` current. + pub fn random(known: kio::Shared) -> Self { let mut rng = kio::clock::rng(); let id = rng.random_range(1..QUALIFIED); - Self::new(id, rng.random()) + Self::publishing(id, rng.random(), known) } fn lock(&self) -> MutexGuard<'_, State> { @@ -608,10 +642,29 @@ impl State { self.router.events(); let changed = self.router.changed(); let declared = self.router.declared(); + let mut known = false; for id in changed { - if Some(id) != declared { - self.home(id); + if Some(id) == declared { + continue; + } + let carried = self.router.carried(id); + known |= match carried.is_empty() { + true => self.routed.remove(&id).is_some(), + false => self.routed.insert(id, carried.clone()).as_ref() != Some(&carried), + }; + self.home(id); + } + if known { + let mut published = self.known.lock(); + published.hops.clear(); + for (id, carried) in &self.routed { + published + .hops + .entry(hop(*id)) + .or_default() + .extend(carried.iter().cloned()); } + published.epoch += 1; } } @@ -762,6 +815,61 @@ mod tests { .collect() } + /// What a node publishes for its older-version sessions: the route it holds to + /// another identity, by that identity's lite-06 hop, only where that route carries + /// a claim. A withdrawn claim leaves the route, and losing the route clears it. + #[test] + fn known_names_a_routed_identity_where_its_route_carries() { + let known = kio::Shared::::default(); + let (origin, relay) = (Node::new(1, 1), Node::publishing(2, 2, known.clone())); + let answer = origin.answer(Link::default(), p("live")).unwrap(); + let receive = relay.receive(Link::default(), p("live")).unwrap(); + origin.own(p("live/x"), Some(claim(&Instance::random(), 5)), None); + pump(&answer, &receive); + let mut chain = crate::Hops::new(); + chain.push(hop(1)).unwrap(); + let names = |path| known.read().names(&chain, &p(path)); + assert!(names("live/x")); + assert!(names("live/y"), "a route reaches the node, not each of its claims"); + assert!(!names("other/x"), "nothing asked of the peer there"); + + origin.own(p("live/x"), None, None); + pump(&answer, &receive); + assert!(names("live/x"), "a withdrawal keeps the route"); + + let epoch = known.read().epoch(); + drop(receive); + assert!(!names("live/x")); + assert!(known.read().epoch() > epoch); + } + + /// A scoped neighbor serves only what its permit admits, which its regions only + /// bound, so its routes name nothing, even by the qualified identity's own hop. + #[test] + fn known_leaves_out_a_scoped_route() { + let known = kio::Shared::::default(); + let (origin, relay) = (Node::new(1, 1), Node::publishing(2, 2, known.clone())); + let qualify = super::super::qualifier([7; 32]); + let scoped = Link { + scope: Some(super::super::Scope { + regions: vec![p("live")], + qualify: qualify.clone(), + }), + ..Link::default() + }; + let answer = origin.answer(Link::default(), p("live")).unwrap(); + let receive = relay.receive(scoped, p("live")).unwrap(); + origin.own(p("live/x"), Some(claim(&Instance::random(), 5)), None); + pump(&answer, &receive); + assert!( + !served(outputs(&receive)).is_empty(), + "the scoped route serves the claim" + ); + let mut chain = crate::Hops::new(); + chain.push(hop(qualify(1) | QUALIFIED)).unwrap(); + assert!(!known.read().names(&chain, &p("live/x"))); + } + #[test] fn a_claim_is_served_through_the_stream_its_route_arrived_on() { let instance = Instance::random(); diff --git a/rs/moq-net/tests/integration/mesh_withdraw.rs b/rs/moq-net/tests/integration/mesh_withdraw.rs index ffdbb0cb14b..6ce7520ffb7 100644 --- a/rs/moq-net/tests/integration/mesh_withdraw.rs +++ b/rs/moq-net/tests/integration/mesh_withdraw.rs @@ -751,3 +751,72 @@ async fn a_node_that_does_not_proxy_keeps_what_it_learned_over_lite06() { assert_eq!(seen.get("room/local"), Some(&vec![Kind::Start]), "{seen:?}"); assert!(!seen.contains_key("room/learned"), "{seen:?}"); } + +/// `d` reaches `p` both directly over moq-lite-07 and through `b`, which translates +/// `p`'s claims onto its moq-lite-06 link to `d`. +async fn mixed_version_triangle() -> (origin::Producer, origin::Producer, origin::Producer, Vec) { + let lite07: Version = "moq-lite-07-wip".parse().unwrap(); + let lite06: Version = "moq-lite-06".parse().unwrap(); + let p = produce_origin(1); + let b = produce_origin(2); + let d = produce_origin(3); + let pairs = vec![ + peer(lite07, &p, &b).await, + peer(lite06, &b, &d).await, + peer(lite07, &p, &d).await, + ]; + (p, b, d, pairs) +} + +/// A relay hearing a claim over moq-lite-07 holds back the moq-lite-06 copy of it, so +/// the copy never becomes one of its own claims: once the publisher withdraws, no node +/// restarts onto that copy before it retracts. +#[kio::test] +async fn mixed_version_triangle_withdraw_retracts_once() { + let (p, b, d, _pairs) = mixed_version_triangle().await; + let mut watched: Vec<_> = [&p, &b, &d].iter().map(|node| watch(node.consume().routes())).collect(); + let broadcast = p.route("room/x", origin::RouteOptions::hidden()).unwrap(); + broadcast.announce().unwrap(); + kio::clock::sim::idle().await; + for (relay, watched) in ["p", "b", "d"].into_iter().zip(&mut watched) { + assert_eq!(drain(watched).get("room/x"), Some(&vec![Kind::Start]), "{relay}"); + } + + drop(broadcast); + kio::clock::sim::idle().await; + for (relay, watched) in ["p", "b", "d"].into_iter().zip(&mut watched) { + assert_eq!(drain(watched).get("room/x"), Some(&vec![Kind::End]), "{relay}"); + } +} + +/// The moq-lite-06 copy a relay held back takes over once its moq-lite-07 route to the +/// publisher is gone, and is held back again once that route returns. +#[kio::test] +async fn mixed_version_failover_takes_up_the_lite06_route() { + let (p, _b, d, mut pairs) = mixed_version_triangle().await; + let mut at_d = watch(d.consume().routes()); + let broadcast = p.route("room/x", origin::RouteOptions::hidden()).unwrap(); + broadcast.announce().unwrap(); + kio::clock::sim::idle().await; + assert_eq!(drain(&mut at_d).get("room/x"), Some(&vec![Kind::Start])); + + let direct = pairs.pop().unwrap(); + direct.server.abort(moq_net::Error::Cancel); + direct.client.abort(moq_net::Error::Cancel); + kio::clock::sim::idle().await; + // The lost session's routes leave with it, before the route layer hears the route + // to `p` is gone, so the copy starts afresh: lite-06 carries no instance, so it is + // another stream either way. + assert_eq!(drain(&mut at_d).get("room/x"), Some(&vec![Kind::End, Kind::Start])); + kio::clock::timeout(Duration::from_secs(10), d.consume().request("room/x", None)) + .await + .expect("request hung") + .expect("the lite-06 route serves the broadcast"); + + let _direct = peer("moq-lite-07-wip".parse().unwrap(), &p, &d).await; + kio::clock::sim::idle().await; + assert_eq!(drain(&mut at_d).get("room/x"), Some(&vec![Kind::Restart])); + drop(broadcast); + kio::clock::sim::idle().await; + assert_eq!(drain(&mut at_d).get("room/x"), Some(&vec![Kind::End])); +} diff --git a/test/drill/mutations/subscriber-leaks-broadcasts.patch b/test/drill/mutations/subscriber-leaks-broadcasts.patch index 8738695e886..12e3adeff5d 100644 --- a/test/drill/mutations/subscriber-leaks-broadcasts.patch +++ b/test/drill/mutations/subscriber-leaks-broadcasts.patch @@ -21,7 +21,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. // Lite06+: announce ids. Each received `active` implicitly assigns the next // per-stream ordinal; `ended`/`restart` reference it instead of repeating the // path, and lite-07 bases name it too. Tracked even for announces we drop -@@ -2145,7 +2146,10 @@ +@@ -2145,8 +2146,11 @@ let mut run: PrefixRun = PrefixRun { responder_origin, link_cost, @@ -32,6 +32,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. + )), decoder: lite::AnnounceDecoder::default(), auth_epoch: 0, + known_epoch: 0, initial, @@ -2301,7 +2305,8 @@ path: Path,