From cc1f1f0bb9a3527468812c68c8b8ed575acaeccb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 22:44:29 -0800 Subject: [PATCH 1/8] chore(quest): claim quest/m1/cluster-routing/routes.md Quest-Claim: 3aa8deafc7a01404 From 211673c9e46684ca4198030b24a05df32ffbb998 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 23:01:03 -0800 Subject: [PATCH 2/8] feat(net): add the per-origin route layer as a sans-IO router A crate-private Router implements the routing contract from quest/m1/cluster-routing/routes.md: Babel feasibility keyed by routing identity, checked strictly positive metrics, the down-only bit, GOAWAY as lowest preference, hop-by-hop seqno requests with no timers, ROUTE_END echoes draining the feasibility distance, qualified identities for scoped neighbors, and merged per-region floors for announcements. The contract's handoff fixtures and route-layer fixtures run against it in a simulated network. Nothing drives it on the wire yet. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lib.rs | 1 + rs/moq-net/src/route/mod.rs | 1257 +++++++++++++++++++++++++ rs/moq-net/src/route/tests/handoff.rs | 1246 ++++++++++++++++++++++++ rs/moq-net/src/route/tests/mod.rs | 735 +++++++++++++++ rs/moq-net/src/route/tests/routing.rs | 752 +++++++++++++++ rs/moq-net/src/route/view.rs | 220 +++++ 6 files changed, 4211 insertions(+) create mode 100644 rs/moq-net/src/route/mod.rs create mode 100644 rs/moq-net/src/route/tests/handoff.rs create mode 100644 rs/moq-net/src/route/tests/mod.rs create mode 100644 rs/moq-net/src/route/tests/routing.rs create mode 100644 rs/moq-net/src/route/view.rs diff --git a/rs/moq-net/src/lib.rs b/rs/moq-net/src/lib.rs index 3d594cad42c..99e7de3f1b1 100644 --- a/rs/moq-net/src/lib.rs +++ b/rs/moq-net/src/lib.rs @@ -120,6 +120,7 @@ pub mod path; mod priority; #[cfg(feature = "async")] mod recv; +mod route; #[cfg(feature = "async")] pub mod setup; #[cfg(not(feature = "async"))] diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs new file mode 100644 index 00000000000..00d713bcbb5 --- /dev/null +++ b/rs/moq-net/src/route/mod.rs @@ -0,0 +1,1257 @@ +//! The per-origin route layer, as one node's state machine. +//! +//! Routing is Babel (RFC 8966) keyed by routing identity: a node selects one +//! feasible route per identity, advertises it to every neighbor whose stream +//! asks, and only the identity's own node advances its seqno. Each origin +//! versions its own claims with a revision, and a node merges every neighbor's +//! certified view claim by claim, so no stale copy revives a withdrawal it +//! already knows. See `quest/m1/cluster-routing/routes.md` for the contract. +//! +//! The router does no I/O: the caller feeds it what each neighbor sent and +//! drains [`Out`] messages and reader [`Event`]s. Neighbor keys `N` and claim +//! attributes `A` are opaque to it. + +#![cfg_attr(not(test), allow(dead_code, reason = "the lite-07 session does not drive it yet"))] + +mod view; + +#[cfg(test)] +mod tests; + +use std::{ + cmp::Reverse, + collections::{BTreeMap, BTreeSet}, + fmt, + sync::Arc, +}; + +use crate::Path; + +pub(crate) use view::{View, overlaps, within}; + +/// An opaque routing identity, declared by an origin or derived by [`Scope`]. +pub(crate) type Id = u64; + +/// Set on every identity a [`Scope`] qualifies, so any node recognizes a +/// qualified key and refuses to qualify it again. +pub(crate) const QUALIFIED: Id = 1 << 63; + +/// A route metric: the sum of configured link costs. +pub(crate) type Metric = u64; + +/// An origin's route seqno. +pub(crate) type Seqno = u64; + +/// An origin's claim revision, advancing on every change to its own claims. +pub(crate) type Rev = u64; + +/// The largest lite-07 varint. As a metric it means held; no origin ever +/// advertises it as a seqno or revision, rotating its identity instead, so every +/// advertised seqno leaves room for the successor a request names. +pub(crate) const MAX: u64 = u64::MAX; + +/// One side's configuration of a link to a neighbor. +#[derive(Clone)] +pub(crate) struct Link { + /// Strictly positive: a zero-cost link would leave a route equal to its own + /// feasibility distance. + pub cost: Metric, + /// The application's local priority for this connection; higher wins among + /// feasible routes, before cost. + pub priority: i8, + /// The neighbor is upstream: routes learned here are down-only, and no + /// down-only route is sent back. + pub upstream: bool, + /// Whether this neighbor is sent routes learned from other neighbors. + pub proxy: bool, + /// Set when the neighbor publishes under a scoped grant. + pub scope: Option, +} + +impl Default for Link { + fn default() -> Self { + Self { + cost: 1, + priority: 0, + upstream: false, + proxy: true, + scope: None, + } + } +} + +/// A scoped neighbor's grant: every identity it sends is filed under a +/// qualified key, and its claims count only within the grant's regions. +#[derive(Clone)] +pub(crate) struct Scope { + /// The prefixes the grant covers. + pub regions: Vec, + /// Derives the qualified key for an identity the neighbor names. It must + /// set [`QUALIFIED`]. + pub qualify: Arc Id + Send + Sync>, +} + +impl fmt::Debug for Scope { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Scope") + .field("regions", &self.regions) + .finish_non_exhaustive() + } +} + +/// What a node advertises for one identity. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Advert { + pub seqno: Seqno, + pub metric: Metric, + pub down_only: bool, +} + +/// A ROUTE's state on a stream. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Reach { + /// A usable route. + Finite(Advert), + /// An infinite metric: unusable, but the sender still holds the identity's + /// announcements while it recovers. Only follows a finite route. + Held, +} + +/// A message for one neighbor. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum Out { + /// ROUTE_START or ROUTE_UPDATE, or ROUTE_END for `None`. + Route { to: N, id: Id, reach: Option }, + /// A batch of announcement changes for `id`, closed by its floor list, which + /// replaces the last one sent for that route. + Batch { + to: N, + id: Id, + changes: Vec<(Path, Option)>, + floors: BTreeMap, + }, + /// ROUTE_REQUEST, toward the neighbor whose route this node would use. + Request { to: N, id: Id, seqno: Seqno }, + /// The echo of a ROUTE_END this node applied. + Echo { to: N, id: Id }, +} + +/// The route a node selected for an identity. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Selected { + /// The neighbor it goes through, or `None` when this node is the identity. + pub next: Option, + pub seqno: Seqno, + /// This node's metric. + pub metric: Metric, + /// Whether an upstream link is on the path. + pub down_only: bool, +} + +/// How a node's reader sees one announced claim. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Shown { + pub item: A, + /// Whether the node has a usable route to the claim's origin through a + /// neighbor certifying it. + pub available: bool, +} + +/// A change a node's announcement reader sees. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Kind { + Start, + /// The claim's attributes changed. + Update, + /// The origin withdrew the claim, as far as this node knows. + End, + /// The node let go of the claim: no usable or recovering route certifies it. + Offline, + /// The node stopped watching the claim. + Unwatched, + Available, + Unavailable, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Event { + pub id: Id, + pub claim: Path, + pub kind: Kind, + pub item: A, + /// How many times this node forgot this claim, by collecting or retiring + /// it, before this event: fixtures excuse a stale replay only after one. + #[cfg(test)] + pub forgot: u32, +} + +/// A route a node holds from one neighbor. +#[derive(Clone, Copy, Debug)] +struct Entry { + /// The last finite advertisement. + advert: Advert, + held: bool, +} + +/// A seqno request a node keeps until a route at least that new is selected or +/// no route is left. +#[derive(Clone, Debug)] +struct Pending { + seqno: Seqno, + /// Neighbors it forwards the request for; empty when only its own. + from: BTreeSet, + /// The neighbor it last sent the request to. + asked: Option, +} + +/// The last view a neighbor sent for one identity. +#[derive(Clone, Debug)] +struct Heard { + /// As the neighbor sent it, to apply the next batch to. + raw: View, + /// Within the neighbor's scope: what it certifies here. + view: View, + /// The interest generation it answers. + interest: u32, +} + +struct Neighbor { + link: Link, + /// The neighbor sent GOAWAY: rank it below every other eligible route. + draining: bool, + /// The prefix the neighbor asked this node for, once it opened a stream. + watch: Option, +} + +/// An identity this node serves itself. +struct Origin { + seqno: Seqno, + /// Whether the node declared it, rather than forging it with no claims. + declared: bool, + view: View, +} + +/// Retained state, summed over the node, for budgets and leak checks. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct Retained { + pub items: usize, + pub floors: usize, + pub heard: usize, + pub sent: usize, + pub echoes: usize, + pub pending: usize, + pub distances: usize, + pub certified: usize, +} + +pub(crate) struct Router { + neighbors: BTreeMap, + origins: BTreeMap>, + rx: BTreeMap>, + /// For a key this node qualified, the identity the neighbor named. + filed: BTreeMap<(N, Id), Id>, + /// Babel's feasibility distance per identity. + fd: BTreeMap, + sent: BTreeMap<(N, Id), Reach>, + /// ROUTE_ENDs sent to each neighbor and not yet echoed back. Until they all + /// return, a neighbor may still hold this node's advertisement, so the + /// feasibility distance stays. + unacked: BTreeMap<(N, Id), u32>, + pending: BTreeMap>, + /// The prefix this node asks its neighbors for announcements under. + watch: Path, + /// Counts this node's interest changes. + interest: u32, + /// Each origin's claims as this node knows them. + views: BTreeMap>, + /// After the interest widened, the regions each view is known current in: + /// the old interest, plus what each batch answering the new one certified. + /// The rest is hidden until certified. + current: BTreeMap>, + heard: BTreeMap<(N, Id), Heard>, + sent_views: BTreeMap<(N, Id), View>, + shown: BTreeMap<(Id, Path), Shown>, + /// Mints a fresh identity on rotation. + rng: u64, + /// Per claim this node has shown, how many times it forgot that claim. + #[cfg(test)] + forgot: BTreeMap<(Id, Path), u32>, + out: Vec>, + events: Vec>, +} + +/// A refused input. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub(crate) enum Error { + #[error("zero-cost link")] + ZeroCost, + #[error("unknown neighbor")] + UnknownNeighbor, + #[error("no declared identity")] + NoIdentity, +} + +impl Router { + /// A node with no sessions, minting rotated identities from `seed`. + pub fn new(seed: u64) -> Self { + Self { + neighbors: BTreeMap::new(), + origins: BTreeMap::new(), + rx: BTreeMap::new(), + filed: BTreeMap::new(), + fd: BTreeMap::new(), + sent: BTreeMap::new(), + unacked: BTreeMap::new(), + pending: BTreeMap::new(), + watch: Path::empty(), + interest: 0, + views: BTreeMap::new(), + current: BTreeMap::new(), + heard: BTreeMap::new(), + sent_views: BTreeMap::new(), + shown: BTreeMap::new(), + rng: seed | 1, + #[cfg(test)] + forgot: BTreeMap::new(), + out: Vec::new(), + events: Vec::new(), + } + } + + /// Take the messages queued for neighbors, in order. + pub fn drain(&mut self) -> Vec> { + std::mem::take(&mut self.out) + } + + /// Take the reader events, in order. + pub fn events(&mut self) -> Vec> { + std::mem::take(&mut self.events) + } + + // --- Sessions --- + + /// A session to `neighbor` opened. + pub fn connect(&mut self, neighbor: N, link: Link) -> Result<(), Error> { + if link.cost == 0 { + return Err(Error::ZeroCost); + } + self.neighbors.insert( + neighbor, + Neighbor { + link, + draining: false, + watch: None, + }, + ); + self.advertise(); + Ok(()) + } + + /// The session to `neighbor` closed: everything learned on it goes, and it + /// counts as the echo of every ROUTE_END sent on it. + pub fn disconnect(&mut self, neighbor: N) { + if self.neighbors.remove(&neighbor).is_none() { + return; + } + for routes in self.rx.values_mut() { + routes.remove(&neighbor); + } + self.rx.retain(|_, routes| !routes.is_empty()); + self.filed.retain(|(from, _), _| *from != neighbor); + self.heard.retain(|(from, _), _| *from != neighbor); + for pending in self.pending.values_mut() { + pending.from.remove(&neighbor); + } + self.unacked.retain(|(to, _), _| *to != neighbor); + self.sent.retain(|(to, _), _| *to != neighbor); + self.sent_views.retain(|(to, _), _| *to != neighbor); + self.advertise(); + } + + /// `neighbor` opened a stream asking for announcements under `prefix`: it + /// gets a complete snapshot. + pub fn peer_interest(&mut self, neighbor: N, prefix: Path) -> Result<(), Error> { + let peer = self.neighbors.get_mut(&neighbor).ok_or(Error::UnknownNeighbor)?; + peer.watch = Some(prefix); + self.sent_views.retain(|(to, _), _| *to != neighbor); + self.advertise(); + Ok(()) + } + + /// `neighbor` sent GOAWAY: it ranks below every other eligible route while + /// its session stays up. + pub fn goaway(&mut self, neighbor: N) { + if let Some(peer) = self.neighbors.get_mut(&neighbor) { + peer.draining = true; + } + self.advertise(); + } + + /// Ask every neighbor for announcements under `prefix` instead, returning + /// the new interest generation. Each neighbor answers on a new stream, and a + /// batch from an older one is discarded. + pub fn watch(&mut self, prefix: Path) -> u32 { + if !within(&prefix, &self.watch) { + let old = self.watch.clone(); + for id in self.views.keys().filter(|id| !self.origins.contains_key(id)) { + self.current.entry(*id).or_insert_with(|| vec![old.clone()]); + } + } + self.current + .retain(|_, regions| !regions.iter().any(|region| within(&prefix, region))); + self.watch = prefix; + self.interest += 1; + self.advertise(); + self.interest + } + + /// The current interest generation. + pub fn interest(&self) -> u32 { + self.interest + } + + // --- The origin's own identity --- + + /// Start originating `id` at `seqno`, with no claims. + pub fn originate(&mut self, id: Id, seqno: Seqno) { + debug_assert_eq!(id & QUALIFIED, 0, "a declared identity is never qualified"); + let mut view = View::default(); + view.floors.insert(Path::empty(), 0); + self.origins.insert( + id, + Origin { + seqno, + declared: true, + view, + }, + ); + self.advertise(); + } + + /// Advertise `id` as if serving it, with no claims: what a forwarder that + /// lies about an identity does. + #[cfg(test)] + pub fn forge(&mut self, id: Id, seqno: Seqno) { + self.origins.insert( + id, + Origin { + seqno, + declared: false, + view: View::default(), + }, + ); + self.advertise(); + } + + /// The identity this node declares. + pub fn declared(&self) -> Option { + self.origins + .iter() + .find(|(_, origin)| origin.declared) + .map(|(id, _)| *id) + } + + /// The declared identity's seqno. + pub fn seqno(&self) -> Option { + self.origins.values().find(|origin| origin.declared).map(|o| o.seqno) + } + + /// The declared identity's claim revision. + pub fn revision(&self) -> Option { + let origin = self.origins.values().find(|origin| origin.declared)?; + origin.view.floors.get(&Path::empty()).copied() + } + + /// Advance the declared identity's seqno, rotating it rather than reach [`MAX`]. + pub fn bump(&mut self) -> Result<(), Error> { + let id = self.declared().ok_or(Error::NoIdentity)?; + let seqno = self.origins[&id].seqno; + if seqno + 1 == MAX { + self.rotate(id); + } else { + self.origins.get_mut(&id).expect("declared").seqno = seqno + 1; + self.advertise(); + } + Ok(()) + } + + /// Start or update a claim of the declared identity. + pub fn announce(&mut self, claim: Path, item: A) -> Result<(), Error> { + self.change(claim, Some(item)) + } + + /// Withdraw a claim of the declared identity. + pub fn unannounce(&mut self, claim: Path) -> Result<(), Error> { + self.change(claim, None) + } + + fn change(&mut self, claim: Path, item: Option) -> Result<(), Error> { + let id = self.declared().ok_or(Error::NoIdentity)?; + let rev = self.origins[&id].view.floors[&Path::empty()] + 1; + if rev == MAX { + self.rotate(id); + return self.change(claim, item); + } + let view = &mut self.origins.get_mut(&id).expect("declared").view; + view.floors.insert(Path::empty(), rev); + match item { + Some(item) => view.items.insert(claim, item), + None => view.items.remove(&claim), + }; + self.advertise(); + Ok(()) + } + + /// Jump the declared identity's revision to `rev`, as a long-lived origin's + /// claim changes reach it. + #[cfg(test)] + pub fn age(&mut self, rev: Rev) { + let id = self.declared().expect("declared"); + let view = &mut self.origins.get_mut(&id).expect("declared").view; + view.floors.insert(Path::empty(), rev); + } + + /// Retire the declared identity for a fresh one, announcing its claims again + /// there. Every neighbor lets the old one go. + fn rotate(&mut self, old: Id) { + let origin = self.origins.remove(&old).expect("declared"); + let id = loop { + // xorshift64: a fresh, unqualified identity. + self.rng ^= self.rng << 13; + self.rng ^= self.rng >> 7; + self.rng ^= self.rng << 17; + let id = self.rng & !QUALIFIED; + if id != old && !self.origins.contains_key(&id) { + break id; + } + }; + let rev = origin.view.items.len() as Rev; + let mut view = origin.view; + view.floors = BTreeMap::from([(Path::empty(), rev)]); + self.origins.insert( + id, + Origin { + seqno: 0, + declared: true, + view, + }, + ); + self.advertise(); + } + + // --- Received messages --- + + /// ROUTE_START or ROUTE_UPDATE from `from`, or ROUTE_END for `None`. + pub fn route(&mut self, from: N, named: Id, reach: Option) { + if !self.neighbors.contains_key(&from) { + return; + } + // Echo every ROUTE_END once applied, filed or refused. + if reach.is_none() { + self.out.push(Out::Echo { to: from, id: named }); + } + let Some(id) = self.key(from, named) else { + return; + }; + if id != named { + match reach { + Some(_) => self.filed.insert((from, id), named), + None => self.filed.remove(&(from, id)), + }; + } + let routes = self.rx.entry(id).or_default(); + match reach { + Some(Reach::Finite(advert)) => { + routes.insert(from, Entry { advert, held: false }); + } + Some(Reach::Held) => { + if let Some(entry) = routes.get_mut(&from) { + entry.held = true; + } + } + None => { + routes.remove(&from); + } + } + if routes.is_empty() { + self.rx.remove(&id); + } + self.advertise(); + } + + /// A batch of announcement changes for `named` from `from`, closed by its + /// floor list, on the stream answering interest generation `interest`. + pub fn batch( + &mut self, + from: N, + named: Id, + changes: Vec<(Path, Option)>, + floors: BTreeMap, + interest: u32, + ) { + let Some(peer) = self.neighbors.get(&from) else { + return; + }; + // A batch from a stream the node has since replaced. + if interest != self.interest { + return; + } + let Some(id) = self.key(from, named) else { + return; + }; + let scope = peer.link.scope.as_ref().map(|scope| scope.regions.clone()); + let heard = self.heard.entry((from, id)).or_insert_with(|| Heard { + raw: View::default(), + view: View::default(), + interest, + }); + // A new stream starts from an empty view. + if heard.interest != interest { + heard.raw = View::default(); + heard.interest = interest; + } + heard.raw.apply(changes, floors); + heard.view = match &scope { + Some(regions) => heard.raw.restrict_all(regions), + None => heard.raw.clone(), + }; + let view = heard.view.clone(); + + // The batch certifies the regions its floors name. + for region in view.floors.keys() { + let Some(regions) = self.current.get_mut(&id) else { + break; + }; + if *region == self.watch { + self.current.remove(&id); + } else if !regions.iter().any(|known| within(region, known)) { + regions.retain(|known| !within(known, region)); + regions.push(region.clone()); + } + } + if !self.origins.contains_key(&id) { + let merged = self.views.entry(id).or_default(); + merged.merge(&view); + // The batch certifies the interest, so what lies outside can go. + let kept = merged.retire(&self.watch); + if kept != *merged { + *merged = kept; + let watch = [self.watch.clone()]; + self.forgot_outside(id, &watch); + } + } + self.advertise(); + } + + /// ROUTE_REQUEST from `from` for `id`, as this node advertised it. + pub fn request(&mut self, from: N, id: Id, seqno: Seqno) { + if !self.neighbors.contains_key(&from) { + return; + } + if let Some(origin) = self.origins.get_mut(&id) { + // Only the origin advances its seqno to answer. + if seqno == MAX { + if origin.declared { + self.rotate(id); + } + } else { + origin.seqno = origin.seqno.max(seqno); + } + self.advertise(); + return; + } + if self.selected(id).is_some_and(|route| route.seqno >= seqno) { + // Already advertised to the requester. + return; + } + let pending = self.pending.entry(id).or_insert(Pending { + seqno, + from: BTreeSet::new(), + asked: None, + }); + if seqno > pending.seqno { + pending.seqno = seqno; + pending.asked = None; + } + pending.from.insert(from); + self.advertise(); + } + + /// The echo of a ROUTE_END this node sent `from` for `id`. + pub fn echo(&mut self, from: N, id: Id) { + let Some(left) = self.unacked.get_mut(&(from, id)) else { + // Unsolicited: nothing of ours to drain. + return; + }; + *left -= 1; + if *left == 0 { + self.unacked.remove(&(from, id)); + } + self.advertise(); + } + + /// Forget `id`'s routes and announcements. The feasibility distance goes on + /// its own once drained. + pub fn collect(&mut self, id: Id) { + self.rx.remove(&id); + self.filed.retain(|(_, key), _| *key != id); + self.views.remove(&id); + self.current.remove(&id); + self.heard.retain(|(_, key), _| *key != id); + self.pending.remove(&id); + self.forgot_outside(id, &[]); + self.advertise(); + } + + // --- Queries --- + + /// The route this node selected for `id`. + pub fn selected(&self, id: Id) -> Option> { + if let Some(origin) = self.origins.get(&id) { + return Some(Selected { + next: None, + seqno: origin.seqno, + metric: 0, + down_only: false, + }); + } + type Rank = (bool, Reverse, Metric, N); + let mut best: Option<(Rank, Selected)> = None; + for (&next, entry) in self.rx.get(&id)? { + if entry.held || !self.feasible(id, entry.advert) { + continue; + } + let peer = &self.neighbors[&next]; + let Some(metric) = add(entry.advert.metric, peer.link.cost) else { + continue; + }; + let rank = (peer.draining, Reverse(peer.link.priority), metric, next); + if best.as_ref().is_some_and(|(best, _)| *best <= rank) { + continue; + } + let selected = Selected { + next: Some(next), + seqno: entry.advert.seqno, + metric, + down_only: peer.link.upstream || entry.advert.down_only, + }; + best = Some((rank, selected)); + } + best.map(|(_, selected)| selected) + } + + /// The route this node would offer an upstream link for `id`. + pub fn export(&self, id: Id) -> Option> { + self.selected(id).filter(|route| !route.down_only) + } + + /// Whether this node holds a reachable route to `id` but none is feasible. + pub fn starved(&self, id: Id) -> bool { + self.selected(id).is_none() && self.infeasible(id).next().is_some() + } + + /// Whether this node, with no usable route to `id`, holds a held route it + /// could recover through. + pub fn grounded(&self, id: Id) -> bool { + self.selected(id).is_none() && self.recoverable(id) + } + + /// Whether this node holds any route state for `id`. + pub fn holds(&self, id: Id) -> bool { + self.rx.contains_key(&id) + } + + /// Whether this node keeps a feasibility distance for `id`. + pub fn remembers(&self, id: Id) -> bool { + self.fd.contains_key(&id) + } + + /// How many identities this node holds routes to. + pub fn identities(&self) -> usize { + self.rx.len() + } + + /// How many qualified keys this node remembers the forwarded name of. + pub fn aliases(&self) -> usize { + self.filed.len() + } + + /// How many region floors this node keeps for `id`. + pub fn floors(&self, id: Id) -> usize { + self.views.get(&id).map_or(0, |view| view.floors.len()) + } + + /// The view this node keeps for `id`, if any. + pub fn view(&self, id: Id) -> Option<&View> { + self.origins + .get(&id) + .map(|origin| &origin.view) + .or_else(|| self.views.get(&id)) + } + + /// Every other origin's view this node keeps. + #[cfg(test)] + pub fn views(&self) -> impl Iterator)> { + self.views + .iter() + .filter(|(id, _)| !self.origins.contains_key(id)) + .map(|(id, view)| (*id, view)) + } + + /// What this node's reader sees. + pub fn shown(&self) -> &BTreeMap<(Id, Path), Shown> { + &self.shown + } + + /// Whether this node shows `claim` from `id`, and whether it is available. + pub fn shows(&self, id: Id, claim: &Path) -> Option { + self.shown.get(&(id, claim.clone())).map(|shown| shown.available) + } + + pub fn retained(&self) -> Retained { + let mut out = Retained::default(); + for (id, view) in &self.views { + if !self.origins.contains_key(id) { + out.items += view.items.len(); + out.floors += view.floors.len(); + } + } + out.heard = self.heard.values().map(|heard| heard.raw.items.len()).sum(); + out.sent = self.sent_views.values().map(|view| view.items.len()).sum(); + out.echoes = self.unacked.values().map(|&count| count as usize).sum(); + out.pending = self.pending.len(); + out.distances = self.fd.keys().filter(|id| !self.origins.contains_key(id)).count(); + out.certified = self.current.values().map(Vec::len).sum(); + out + } + + // --- Internals --- + + /// The key this node files `named` under when `from` sends it, or `None` + /// when refused: a scoped neighbor's identities are all qualified, and one + /// already qualified is refused rather than qualified again. + fn key(&self, from: N, named: Id) -> Option { + match &self.neighbors.get(&from)?.link.scope { + Some(scope) => { + if named & QUALIFIED != 0 { + return None; + } + let key = (scope.qualify)(named); + debug_assert_ne!(key & QUALIFIED, 0, "a qualified key sets QUALIFIED"); + Some(key | QUALIFIED) + } + None => Some(named), + } + } + + /// The identity this node names for `id` in a request to `next`: the one + /// `next` advertised, if this node qualified it. + fn name(&self, next: N, id: Id) -> Id { + self.filed.get(&(next, id)).copied().unwrap_or(id) + } + + /// Babel's feasibility condition against this node's feasibility distance. + fn feasible(&self, id: Id, advert: Advert) -> bool { + match self.fd.get(&id) { + None => true, + Some(&(seqno, metric)) => advert.seqno > seqno || (advert.seqno == seqno && advert.metric < metric), + } + } + + /// The usable but infeasible routes for `id`, with their metrics. + fn infeasible(&self, id: Id) -> impl Iterator + '_ { + self.rx + .get(&id) + .into_iter() + .flatten() + .filter(move |(_, entry)| !entry.held && !self.feasible(id, entry.advert)) + .filter_map(move |(&next, entry)| Some((add(entry.advert.metric, self.neighbors[&next].link.cost)?, next))) + } + + /// Where this node sends a request for `id`: its feasible route's next hop, + /// else its best infeasible route's, even when that neighbor asked. + fn target(&self, id: Id) -> Option { + self.selected(id) + .and_then(|route| route.next) + .or_else(|| self.infeasible(id).min().map(|(_, next)| next)) + } + + /// Whether this node holds a held route to `id` it could recover through. + /// One counts only if its last finite advertisement is feasible here, so + /// holds follow a strictly decreasing metric toward a starved node and + /// cannot hold each other up. + fn recoverable(&self, id: Id) -> bool { + self.rx.get(&id).is_some_and(|routes| { + routes + .values() + .any(|entry| entry.held && self.feasible(id, entry.advert)) + }) + } + + /// Whether this node, having selected `route` for `id`, keeps showing its + /// announcements: routed, starved, or holding a recoverable route. + fn live(&self, id: Id, route: Option>) -> bool { + route.is_some() || self.infeasible(id).next().is_some() || self.recoverable(id) + } + + /// The neighbors whose views can certify `id`: those whose last finite + /// advertisement is feasible here, held or not. Feasible edges strictly + /// decrease the metric toward the origin, so certification never circulates. + fn certifiers(&self, id: Id) -> Vec { + self.rx.get(&id).map_or_else(Vec::new, |routes| { + routes + .iter() + .filter(|(_, entry)| self.feasible(id, entry.advert)) + .map(|(&next, _)| next) + .collect() + }) + } + + /// The regions of `id` that this node's certifiers certify, those whose + /// route is held only when `held`, or `None` while it holds no route or a + /// certifier has not yet answered the current interest. + fn certified(&self, id: Id, held: bool) -> Option> { + let routes = self.rx.get(&id)?; + let mut regions = Vec::new(); + for next in self.certifiers(id) { + let heard = self + .heard + .get(&(next, id)) + .filter(|heard| heard.interest == self.interest)?; + if held || !routes[&next].held { + regions.extend(heard.view.floors.keys().cloned()); + } + } + Some(regions) + } + + /// Let go of every region of `id`'s merged view that no certifier certifies + /// any more, once each has answered the current interest. + fn prune(&mut self, id: Id) { + // A starved or held node keeps presence until it recovers or lets go. + if self.origins.contains_key(&id) || self.selected(id).is_none() { + return; + } + let Some(regions) = self.certified(id, true) else { + return; + }; + let Some(merged) = self.views.get(&id) else { + return; + }; + let lost = merged + .floors + .iter() + .filter(|(floor, _)| !regions.iter().any(|region| within(floor, region))) + .map(|(_, &rev)| rev) + .max(); + let Some(lost) = lost else { + return; + }; + // Keep what the view knows within the regions still certified. That is + // sound only if no lost floor is newer than every kept one: otherwise a + // claim above both was last learned through the lost floor, and is + // unknown at the revision the kept floors certify. + let kept = merged.restrict_all(®ions); + let (rebuilt, remembered) = if kept.floors.values().all(|&floor| floor >= lost) { + (kept, regions) + } else { + let mut rebuilt = View::default(); + for next in self.certifiers(id) { + rebuilt.merge(&self.heard[&(next, id)].view); + } + (rebuilt, Vec::new()) + }; + self.views.insert(id, rebuilt); + self.forgot_outside(id, &remembered); + } + + /// Count a forget against each shown claim of `id` overlapping none of `kept`. + #[cfg_attr(not(test), allow(unused_variables, reason = "only fixtures count forgets"))] + fn forgot_outside(&mut self, id: Id, kept: &[Path]) { + #[cfg(test)] + for ((key, claim), count) in self.forgot.iter_mut() { + if *key == id && !kept.iter().any(|region| overlaps(claim, region)) { + *count += 1; + } + } + } + + /// The view this node shows and forwards for `id`, having selected `route`. + fn adopted(&self, id: Id, route: Option>) -> Option<&View> { + if let Some(origin) = self.origins.get(&id) { + return Some(&origin.view); + } + self.live(id, route).then(|| self.views.get(&id)).flatten() + } + + /// Whether this node sends `id` to `neighbor` at all: its own identities to + /// everyone, learned ones only to a proxied neighbor. + fn serves(&self, neighbor: &Neighbor, id: Id) -> bool { + neighbor.link.proxy || self.origins.contains_key(&id) + } + + /// Recompute every selection, record feasibility distances for what this + /// node advertises, and queue any change for each neighbor. + fn advertise(&mut self) { + let ids: Vec = self.views.keys().copied().collect(); + for id in ids { + self.prune(id); + } + let mut ids: BTreeSet = self.rx.keys().copied().collect(); + ids.extend(self.origins.keys()); + ids.extend(self.sent.keys().map(|(_, id)| *id)); + let peers: Vec = self + .neighbors + .iter() + .filter(|(_, peer)| peer.watch.is_some()) + .map(|(n, _)| *n) + .collect(); + + for id in ids { + let selected = self.selected(id); + // The feasibility distance tracks finite updates actually sent (RFC 8966 s3.7.3). + if let Some(selected) = selected + && peers.iter().any(|peer| { + let peer = &self.neighbors[peer]; + self.serves(peer, id) && !(peer.link.upstream && selected.down_only) + }) { + self.record(id, selected); + } + // Recording may have tightened the feasibility distance: select again. + let route = self.selected(id); + let live = self.live(id, route); + let adopted = self.adopted(id, route).cloned(); + // A sender that certifies anything answers every stream, even one + // whose interest none of its regions meets, so the receiver knows it + // certifies nothing there. + let answers = adopted.as_ref().is_some_and(|view| !view.floors.is_empty()); + let mut restricted: Vec<(Path, View)> = Vec::new(); + + for &to in &peers { + let peer = &self.neighbors[&to]; + let serves = self.serves(peer, id); + let was = self.sent.get(&(to, id)).copied(); + if !serves && was.is_none() { + continue; + } + let reach = match selected { + Some(route) if serves && !(peer.link.upstream && route.down_only) => Some(Reach::Finite(Advert { + seqno: route.seqno, + metric: route.metric, + down_only: route.down_only, + })), + None if serves && live && was.is_some() => Some(Reach::Held), + _ => None, + }; + let watch = peer.watch.clone().expect("a watching peer"); + if was != reach { + match reach { + Some(reach) => self.sent.insert((to, id), reach), + None => { + *self.unacked.entry((to, id)).or_default() += 1; + self.sent.remove(&(to, id)) + } + }; + self.out.push(Out::Route { to, id, reach }); + } + // Announcements ride on the route and end with it. + if reach.is_none() { + self.sent_views.remove(&(to, id)); + continue; + } + // Peers mostly share an interest, so restrict once per interest. + let index = match restricted.iter().position(|(interest, _)| *interest == watch) { + Some(index) => index, + None => { + let view = adopted + .as_ref() + .map_or_else(View::default, |view| view.restrict(&watch)); + restricted.push((watch, view)); + restricted.len() - 1 + } + }; + let view = &restricted[index].1; + let changes = match self.sent_views.get(&(to, id)) { + Some(sent) if sent == view => continue, + Some(sent) => view.diff(sent), + None if !answers => continue, + None => view.diff(&View::default()), + }; + self.sent_views.insert((to, id), view.clone()); + self.out.push(Out::Batch { + to, + id, + changes, + floors: view.floors.clone(), + }); + } + } + // Drop a feasibility distance once nothing here routes or advertises the + // identity and every neighbor echoed its ROUTE_END: no node then holds an + // advertisement from this one, so no older one can return as a loop. + let (rx, origins, sent, unacked) = (&self.rx, &self.origins, &self.sent, &self.unacked); + self.fd.retain(|id, _| { + rx.contains_key(id) + || origins.contains_key(id) + || sent.keys().any(|(_, sent)| sent == id) + || unacked.keys().any(|(_, unacked)| unacked == id) + }); + self.requests(); + self.show(); + } + + /// Start, resend, or clear this node's seqno requests. + fn requests(&mut self) { + let mut ids: BTreeSet = self.pending.keys().copied().collect(); + ids.extend(self.rx.keys()); + for id in ids { + if self.origins.contains_key(&id) { + self.pending.remove(&id); + continue; + } + let route = self.selected(id); + let starved = route.is_none() && self.infeasible(id).next().is_some(); + if starved { + let want = self.fd[&id].0 + 1; + let pending = self.pending.entry(id).or_insert(Pending { + seqno: want, + from: BTreeSet::new(), + asked: None, + }); + if pending.seqno < want { + pending.seqno = want; + pending.asked = None; + } + } + let Some(pending) = self.pending.get(&id) else { + continue; + }; + let answered = route.is_some_and(|route| route.seqno >= pending.seqno); + if answered || (pending.from.is_empty() && !starved) { + self.pending.remove(&id); + continue; + } + // A request follows the next hop even back through a requester: next + // hops form no loop, one pending request per hop stops a bounce, and + // no requester ever cancels, so excluding one can strand a request + // while every route still runs through it. + let Some(target) = self.target(id) else { + self.pending.remove(&id); + continue; + }; + let pending = self.pending.get_mut(&id).expect("pending"); + if pending.asked != Some(target) { + pending.asked = Some(target); + let seqno = pending.seqno; + let named = self.name(target, id); + self.out.push(Out::Request { + to: target, + id: named, + seqno, + }); + } + } + } + + /// Recompute what this node's reader sees and log each change. + fn show(&mut self) { + let mut ids: BTreeSet = self.views.keys().copied().collect(); + ids.extend(self.origins.keys()); + ids.extend(self.heard.keys().map(|(_, id)| *id)); + let mut shown = BTreeMap::new(); + for id in ids { + let route = self.selected(id); + let Some(view) = self.adopted(id, route) else { + continue; + }; + let usable = route.is_some(); + let own = self.origins.contains_key(&id); + // A claim is available only through a certifier that is not held: + // another scope's usable route cannot serve it. + let serving = match usable && !view.items.is_empty() && !own { + true => self.certified(id, false), + false => None, + }; + let current = self.current.get(&id); + for (claim, item) in &view.items { + let certified = current.is_none_or(|regions| regions.iter().any(|region| overlaps(claim, region))); + let available = usable + && serving + .as_ref() + .is_none_or(|regions| regions.iter().any(|region| overlaps(claim, region))); + if overlaps(claim, &self.watch) && certified { + shown.insert( + (id, claim.clone()), + Shown { + item: item.clone(), + available, + }, + ); + } + } + } + let old = std::mem::take(&mut self.shown); + for ((id, claim), now) in &shown { + match old.get(&(*id, claim.clone())) { + None => self.log(*id, claim, Kind::Start, &now.item), + Some(was) => { + if was.item != now.item { + self.log(*id, claim, Kind::Update, &now.item); + } + if was.available != now.available { + let kind = match now.available { + true => Kind::Available, + false => Kind::Unavailable, + }; + self.log(*id, claim, kind, &now.item); + } + } + } + } + for ((id, claim), was) in &old { + if shown.contains_key(&(*id, claim.clone())) { + continue; + } + // A claim no routed neighbor's region covers was let go, not ended. + let uncertified = self + .certified(*id, true) + .is_some_and(|regions| !regions.iter().any(|region| overlaps(claim, region))); + let kind = if !overlaps(claim, &self.watch) { + Kind::Unwatched + } else if self.live(*id, self.selected(*id)) && !uncertified { + Kind::End + } else { + Kind::Offline + }; + self.log(*id, claim, kind, &was.item); + } + self.shown = shown; + } + + fn log(&mut self, id: Id, claim: &Path, kind: Kind, item: &A) { + #[cfg(test)] + let forgot = *self.forgot.entry((id, claim.clone())).or_insert(0); + self.events.push(Event { + id, + claim: claim.clone(), + kind, + item: item.clone(), + #[cfg(test)] + forgot, + }); + } + + /// Babel's feasibility distance: reset by a newer seqno, lowered by a better metric. + fn record(&mut self, id: Id, selected: Selected) { + let fd = self.fd.entry(id).or_insert((selected.seqno, selected.metric)); + if selected.seqno > fd.0 || (selected.seqno == fd.0 && selected.metric < fd.1) { + *fd = (selected.seqno, selected.metric); + } + } +} + +/// `metric` plus a strictly positive link `cost`, or `None` when it reaches the +/// infinite metric: overflow is unreachable, never saturated. +fn add(metric: Metric, cost: Metric) -> Option { + metric.checked_add(cost).filter(|&metric| metric < MAX) +} diff --git a/rs/moq-net/src/route/tests/handoff.rs b/rs/moq-net/src/route/tests/handoff.rs new file mode 100644 index 00000000000..a150071f4a5 --- /dev/null +++ b/rs/moq-net/src/route/tests/handoff.rs @@ -0,0 +1,1246 @@ +//! Next-hop handoff: complete and fresh announcements across route changes. +//! +//! Each origin versions its own claims with a revision. A node sends each +//! neighbor its view of an origin's claims within that neighbor's interest, +//! closed by floors certifying the view complete as of a revision for the +//! region they cover. Receivers merge every neighbor's view claim by claim, +//! keeping whichever knowledge is newer, and keep showing an origin's claims +//! while any route to it is usable or recovering. + +use std::collections::BTreeSet; + +use super::*; + +/// An origin reaching a node through two relays, the first preferred and the +/// second a feasible successor. +fn two_relays() -> Sim { + let (origin, a, b, node) = (0, 1, 2, 3); + let mut sim = Sim::new(4); + sim.leaf(origin); + sim.join(origin, a, 1); + sim.join(origin, b, 1); + sim.join(node, a, 1); + sim.join(node, b, 2); + sim.originate(origin, 7, 0); + assert!(sim.run()); + sim +} + +/// A withdrawal reaches the node through its next hop while the alternative's +/// copy is still in flight. When the next hop goes, the alternative's complete +/// snapshot is older than what the node already knows, and is refused. +#[test] +fn an_older_complete_snapshot_revives_no_known_withdrawal() { + let (origin, a, b, node) = (0, 1, 2, 3); + let mut sim = two_relays(); + sim.announce(origin, "live/x", 0, None); + assert!(sim.run()); + assert_eq!(sim.shows(node, 7, "live/x"), Some(true)); + + sim.hold(origin, b); + sim.unannounce(origin, "live/x"); + assert!(sim.run()); + assert_eq!(sim.shows(node, 7, "live/x"), None); + // Forgetting another origin excuses no replay of this one. + sim.collect(node, 99); + + sim.disconnect(node, a); + assert!(sim.run()); + assert_eq!(sim.route(node, 7).and_then(|route| route.next), Some(b)); + assert_eq!(sim.shows(node, 7, "live/x"), None); + + sim.release(origin, b); + assert!(sim.run()); + assert_eq!(sim.shows(node, 7, "live/x"), None); + let events = at(&sim, node); + assert_eq!(revivals(&events), 0); + assert_eq!(kinds(&events), [Kind::Start, Kind::End]); + sim.check().unwrap(); +} + +/// A new next hop's ROUTE arrives before its announcements. Merged views keep +/// what the node knows, and the reader sees nothing change. +#[test] +fn trailing_announcements_report_no_false_absence() { + let (origin, a, b, node) = (0, 1, 2, 3); + let mut sim = two_relays(); + sim.disconnect(node, b); + sim.announce(origin, "live/x", 0, None); + assert!(sim.run()); + let before = sim.events.len(); + + sim.hold(b, node); + sim.connect(node, b); + assert!(sim.run()); + assert!(sim.queued(b, node) >= 2, "a ROUTE then its announcements"); + assert!(sim.deliver(b, node), "only the ROUTE"); + sim.disconnect(node, a); + assert!(sim.run()); + assert_eq!(sim.route(node, 7).and_then(|route| route.next), Some(b)); + assert_eq!(sim.shows(node, 7, "live/x"), Some(true)); + + sim.release(b, node); + assert!(sim.run()); + assert_eq!(sim.shows(node, 7, "live/x"), Some(true)); + let events: Vec<&SimEvent> = sim.events[before..].iter().filter(|e| e.node == node).collect(); + assert!(events.is_empty(), "{events:?}"); +} + +/// The partitioned local mesh, with a client behind the starved relay. The +/// CDN detour is infeasible until the origin advances its seqno, and through +/// that window the relay and its client keep the claim unavailable rather +/// than ending it. The relay's seqno request recovers it through the CDN. +#[test] +fn a_starved_relay_keeps_presence_unavailable_until_its_request_recovers_it() { + let (c1, c2, l1, l2, origin, client) = (0, 1, 2, 3, 4, 5); + let mut sim = Sim::new(6); + sim.join(c1, c2, 10); + sim.join(l1, l2, 1); + sim.link(l1, c1, SimLink::cost(1).upstream(), SimLink::cost(1)); + sim.link(l2, c2, SimLink::cost(1).upstream(), SimLink::cost(1)); + sim.leaf(origin); + sim.leaf(client); + sim.join(origin, l2, 1); + sim.join(client, l1, 1); + sim.originate(origin, 7, 0); + sim.announce(origin, "local/x", 0, None); + assert!(sim.run()); + assert_eq!(sim.shows(client, 7, "local/x"), Some(true)); + + // Hold the request so the starved window is observable. + let before = sim.events.len(); + sim.hold(l1, c1); + sim.disconnect(l1, l2); + assert!(sim.run()); + assert!(sim.starved(l1, 7)); + assert!(sim.grounded(client, 7)); + for node in [l1, client] { + assert_eq!(sim.shows(node, 7, "local/x"), Some(false), "node {node}"); + } + assert_eq!(false_ends(&sim.events), 0); + + sim.release(l1, c1); + assert!(sim.run()); + assert_eq!(sim.seqno(origin), 1, "the request reached the origin"); + assert_eq!(sim.route(l1, 7).and_then(|route| route.next), Some(c1)); + let events: Vec = sim.events[before..] + .iter() + .filter(|e| e.node == client) + .cloned() + .collect(); + assert_eq!(kinds(&events), [Kind::Unavailable, Kind::Available]); +} + +/// A client of two relays, one lagging behind its origin. +fn lagging(watch: &str) -> Sim { + let (origin, r1, r2, client) = (0, 1, 2, 3); + let mut sim = Sim::new(4); + sim.leaf(origin); + sim.leaf(client); + sim.join(origin, r1, 1); + sim.join(origin, r2, 1); + sim.join(client, r1, 1); + sim.join(client, r2, 1); + sim.watch(client, watch); + sim.originate(origin, 7, 0); + sim +} + +/// A client watching `live/a` knows `live/a/x` ended, then widens to +/// everything. The lagging relay's wider snapshot must not revive `live/a/x`, +/// yet is the first to report `live/b/y`: floors scoped to the region they +/// cover do both. +#[test] +fn widening_interest_needs_floors_scoped_to_what_they_cover() { + let (origin, r1, r2, client) = (0, 1, 2, 3); + let mut sim = lagging("live/a/"); + sim.announce(origin, "live/a/x", 0, None); + sim.announce(origin, "live/b/y", 0, None); + assert!(sim.run()); + sim.hold(origin, r2); + sim.unannounce(origin, "live/a/x"); + assert!(sim.run()); + assert!(sim.shown(client).is_empty()); + + sim.hold(r1, client); + sim.watch(client, ""); + assert!(sim.run()); + sim.release(r1, client); + assert!(sim.run()); + assert_eq!(sim.shows(client, 7, "live/a/x"), None); + assert_eq!(sim.shows(client, 7, "live/b/y"), Some(true)); + assert_eq!(revivals(&sim.events), 0); + sim.check().unwrap(); +} + +/// Narrowing retires what a client knew outside the new interest. On rejoin a +/// lagging relay's snapshot replays a withdrawal the client saw before it +/// narrowed, as after a collection, and only until the newer snapshot lands. +#[test] +fn cancelling_interest_retires_its_region() { + let (origin, r1, r2, client) = (0, 1, 2, 3); + let mut sim = lagging(""); + sim.announce(origin, "live/a/x", 0, None); + assert!(sim.run()); + sim.hold(origin, r2); + sim.unannounce(origin, "live/a/x"); + assert!(sim.run()); + assert_eq!(sim.shows(client, 7, "live/a/x"), None); + + sim.watch(client, "live/b/"); + assert!(sim.run()); + assert_eq!(sim.floors(client, 7), 1, "only the watched region"); + sim.hold(r1, client); + sim.watch(client, ""); + assert!(sim.run()); + assert_eq!( + sim.shows(client, 7, "live/a/x"), + Some(true), + "the lagging relay replays" + ); + sim.release(r1, client); + assert!(sim.run()); + assert_eq!(sim.shows(client, 7, "live/a/x"), None); + sim.release(origin, r2); + assert!(sim.run()); + assert_eq!( + life(&sim, client, "live/a/x"), + [Kind::Start, Kind::End, Kind::Start, Kind::End] + ); + assert_eq!(revivals(&sim.events), 0, "a retired region, like a collection"); + assert_eq!(sim.floors(client, 7), 1); + sim.check().unwrap(); +} + +/// A claim outside a narrowed interest ends unseen. On widening, what was +/// kept there stays hidden until a floor covering it arrives, rather than +/// showing a stale Start the next snapshot ends. +#[test] +fn widening_hides_what_was_frozen_until_a_covering_floor_arrives() { + let (origin, r1, r2, client) = (0, 1, 2, 3); + let mut sim = lagging(""); + sim.announce(origin, "live/a/x", 0, None); + sim.announce(origin, "live/b/y", 0, None); + assert!(sim.run()); + sim.watch(client, "live/b/"); + assert!(sim.run()); + sim.unannounce(origin, "live/a/x"); + assert!(sim.run()); + + sim.hold(r1, client); + sim.hold(r2, client); + sim.watch(client, ""); + assert_eq!(sim.shows(client, 7, "live/a/x"), None, "frozen while unwatched"); + assert_eq!(sim.shows(client, 7, "live/b/y"), Some(true), "still current"); + + sim.release(r1, client); + sim.release(r2, client); + assert!(sim.run()); + assert_eq!(life(&sim, client, "live/a/x"), [Kind::Start, Kind::Unwatched]); + sim.check().unwrap(); +} + +/// A snapshot sent before the client narrowed still waits when it widens +/// again. It answers the old interest, so it must not lift the guard on what +/// stayed frozen. +#[test] +fn a_snapshot_queued_before_widening_keeps_the_frozen_rest_hidden() { + let (origin, r1, r2, client) = (0, 1, 2, 3); + let mut sim = lagging(""); + sim.announce(origin, "live/a/x", 0, None); + sim.announce(origin, "live/b/y", 0, None); + assert!(sim.run()); + sim.hold(r2, client); + sim.announce(origin, "live/b/z", 0, None); + assert!(sim.run()); + assert!(sim.queued(r2, client) > 0, "a root snapshot waits"); + sim.watch(client, "live/b/"); + assert!(sim.run()); + sim.unannounce(origin, "live/a/x"); + assert!(sim.run()); + + sim.hold(r1, client); + sim.watch(client, ""); + assert!(sim.deliver(r2, client), "the root snapshot from before narrowing"); + assert_eq!(sim.shows(client, 7, "live/a/x"), None, "it answers the old interest"); + + sim.release(r1, client); + sim.release(r2, client); + assert!(sim.run()); + assert_eq!(life(&sim, client, "live/a/x"), [Kind::Start, Kind::Unwatched]); + sim.check().unwrap(); +} + +/// A client behind a scoped grant widens its interest. Every batch it gets is +/// cut to the grant's scope, so no floor covers the widened interest, yet the +/// batch answering it shows the newly covered claims, and claim churn keeps +/// the certified regions bounded. +#[test] +fn widening_behind_a_scoped_grant_shows_the_rest_of_the_scope() { + let (origin, relay, client) = (0, 1, 2); + let mut sim = Sim::new(3); + sim.leaf(origin); + sim.leaf(client); + sim.join(origin, relay, 1); + let grant = Grant { + subject: "local", + scope: "live/a/", + }; + sim.link(client, relay, SimLink::cost(1).grant(grant), SimLink::cost(1)); + sim.watch(client, "live/a/x/"); + sim.originate(origin, 7, 0); + sim.announce(origin, "live/a/x/1", 0, None); + sim.announce(origin, "live/a/y", 0, None); + assert!(sim.run()); + let id = qualified("local", 7); + assert_eq!(sim.shows(client, id, "live/a/x/1"), Some(true)); + assert_eq!(sim.shows(client, id, "live/a/y"), None); + + sim.watch(client, ""); + assert!(sim.run()); + assert_eq!(sim.shows(client, id, "live/a/y"), Some(true)); + let certified = sim.retained().certified; + for _ in 0..8 { + sim.announce(origin, "live/a/z", 0, None); + sim.unannounce(origin, "live/a/z"); + assert!(sim.run()); + } + assert_eq!(sim.retained().certified, certified, "claim churn adds no state"); + assert!( + sim.check().unwrap().contains(&(client, id)), + "the qualified view is checked" + ); +} + +/// Two scoped grants qualify one origin under one subject. After widening, a +/// batch through the first grant certifies only its own scope: a claim in the +/// second scope that ended while unwatched stays hidden until the second +/// grant answers too. +#[test] +fn widening_behind_two_grants_waits_for_each_scope() { + let (origin, r1, r2, client) = (0, 1, 2, 3); + let mut sim = scoped_pair(["live/a/", "live/b/"]); + sim.announce(origin, "live/a/x/1", 0, None); + sim.announce(origin, "live/b/z", 0, None); + assert!(sim.run()); + let id = qualified("local", 7); + assert_eq!(sim.shows(client, id, "live/b/z"), Some(true)); + + sim.hold(r1, client); + sim.hold(r2, client); + sim.watch(client, "live/a/x/"); + sim.unannounce(origin, "live/b/z"); + assert!(sim.run()); + sim.watch(client, ""); + while sim.deliver(r1, client) {} + assert_eq!(sim.shows(client, id, "live/a/x/1"), Some(true)); + assert_eq!(sim.shows(client, id, "live/b/z"), None, "only the first scope answered"); + + sim.release(r1, client); + sim.release(r2, client); + assert!(sim.run()); + assert_eq!(life(&sim, client, "live/b/z"), [Kind::Start, Kind::Unwatched]); + sim.check().unwrap(); +} + +/// An origin reaching a client through two relays, each admitting the client +/// under a grant scoped to one of `scopes` with one subject. +fn scoped_pair(scopes: [&'static str; 2]) -> Sim { + let (origin, r1, r2, client) = (0, 1, 2, 3); + let mut sim = Sim::new(4); + sim.leaf(origin); + sim.leaf(client); + sim.join(origin, r1, 1); + sim.join(origin, r2, 1); + for (relay, scope) in [(r1, scopes[0]), (r2, scopes[1])] { + let grant = Grant { + subject: "local", + scope, + }; + sim.link(client, relay, SimLink::cost(1).grant(grant), SimLink::cost(1)); + } + sim.originate(origin, 7, 0); + sim +} + +/// One key arrives through two grants. When one grant's session closes, +/// nothing certifies its scope, so its claims go offline rather than staying +/// shown through the other grant, which could never deliver their withdrawal. +#[test] +fn a_closed_grant_lets_its_scope_go() { + let (origin, r1, client) = (0, 1, 3); + let mut sim = scoped_pair(["live/a/", "live/b/"]); + sim.announce(origin, "live/a/x", 0, None); + sim.announce(origin, "live/b/z", 0, None); + assert!(sim.run()); + let id = qualified("local", 7); + assert_eq!(sim.shows(client, id, "live/a/x"), Some(true)); + + sim.disconnect(client, r1); + assert!(sim.run()); + assert_eq!(sim.shows(client, id, "live/a/x"), None); + assert_eq!(sim.shows(client, id, "live/b/z"), Some(true)); + sim.unannounce(origin, "live/a/x"); + sim.announce(origin, "live/a/y", 0, None); + assert!(sim.run()); + sim.connect(client, r1); + assert!(sim.run()); + assert_eq!(sim.shows(client, id, "live/a/x"), None); + assert_eq!(sim.shows(client, id, "live/a/y"), Some(true)); + assert_eq!(life(&sim, client, "live/a/x"), [Kind::Start, Kind::Offline]); + assert_eq!(false_ends(&sim.events), 0); + sim.check().unwrap(); +} + +/// Two relays each admit an uplink scoped to a different prefix and forward +/// the key to a third, whose view carries one floor per scope. +fn two_scoped_uplinks() -> Sim { + let (origin, local, x, y, z) = (0, 1, 2, 3, 4); + let mut sim = Sim::new(5); + sim.leaf(origin); + sim.join(origin, local, 1); + for (relay, scope) in [(x, "live/a/"), (y, "live/b/")] { + let grant = Grant { + subject: "local", + scope, + }; + sim.link(relay, local, SimLink::cost(1).grant(grant), SimLink::cost(1)); + sim.join(relay, z, 1); + } + sim.originate(origin, 7, 0); + sim +} + +/// A relay forwards a key whose floors are all grant scopes, and its peer +/// moves to an interest no scope meets. The relay still answers the new +/// stream, certifying nothing, so the peer lets go of a claim it kept from the +/// scoped view instead of waiting on an answer that never comes. +#[test] +fn a_disjoint_interest_is_answered_with_nothing() { + let (origin, z) = (0, 4); + let mut sim = two_scoped_uplinks(); + sim.announce(origin, "live/", 0, None); + assert!(sim.run()); + let id = qualified("local", 7); + assert_eq!(sim.shows(z, id, "live/"), Some(true)); + + sim.watch(z, "live/c"); + assert!(sim.run()); + assert_eq!(sim.shows(z, id, "live/"), None); + assert_eq!(life(&sim, z, "live/"), [Kind::Start, Kind::Offline]); + sim.check().unwrap(); +} + +/// An interest matches whole path segments: watching `live/a` sees what lies +/// below it, never a sibling `live/ab` that merely shares its bytes. +#[test] +fn an_interest_matches_whole_segments() { + let (origin, client) = (0, 1); + let mut sim = Sim::new(2); + sim.leaf(origin); + sim.join(origin, client, 1); + sim.originate(origin, 7, 0); + sim.announce(origin, "live/a/x", 0, None); + sim.announce(origin, "live/ab", 0, None); + sim.watch(client, "live/a"); + assert!(sim.run()); + assert_eq!(sim.shows(client, 7, "live/a/x"), Some(true)); + assert_eq!(sim.shows(client, 7, "live/ab"), None); + sim.check().unwrap(); +} + +/// One key arrives through a wide grant and a narrow one. The narrow session +/// lags behind a withdrawal the wide one delivered; when the wide session +/// closes, only what lies outside the narrow scope goes offline, and the newer +/// knowledge inside it stays rather than reviving the withdrawn claim. +#[test] +fn a_lost_scope_keeps_newer_knowledge_where_still_certified() { + let (origin, r1, r2, client) = (0, 1, 2, 3); + let mut sim = scoped_pair(["live/", "live/b/"]); + sim.announce(origin, "live/a/x", 0, None); + sim.announce(origin, "live/b/z", 0, None); + assert!(sim.run()); + let id = qualified("local", 7); + assert_eq!(sim.shows(client, id, "live/b/z"), Some(true)); + + sim.hold(r2, client); + sim.unannounce(origin, "live/b/z"); + assert!(sim.run()); + assert_eq!(sim.shows(client, id, "live/b/z"), None); + sim.disconnect(client, r1); + assert!(sim.run()); + assert_eq!(sim.shows(client, id, "live/a/x"), None, "outside the scope left"); + assert_eq!( + sim.shows(client, id, "live/b/z"), + None, + "the lagging scope revives nothing" + ); + sim.check().unwrap(); + + sim.release(r2, client); + assert!(sim.run()); + assert_eq!(sim.shows(client, id, "live/b/z"), None); + assert_eq!(revivals(&at(&sim, client)), 0); + sim.check().unwrap(); +} + +/// When one scoped uplink closes, its relay is starved and holds, so the third +/// relay shows that scope unavailable; once a seqno request recovers a route, +/// nothing certifies the lost scope and the third lets it go. Certifiers are +/// feasible neighbors only, so the two never keep the scope alive through each +/// other. +#[test] +fn a_relay_lets_go_of_a_scope_its_uplinks_lost() { + let (origin, local, x, z) = (0, 1, 2, 4); + for request in [false, true] { + let mut sim = two_scoped_uplinks(); + sim.announce(origin, "live/a/x", 0, None); + sim.announce(origin, "live/b/z", 0, None); + assert!(sim.run()); + let id = qualified("local", 7); + assert_eq!(sim.shows(z, id, "live/a/x"), Some(true)); + assert_eq!(sim.shows(z, id, "live/b/z"), Some(true)); + + if !request { + // Hold every request at the starved relay. + for peer in [z, local] { + sim.hold(x, peer); + } + } + sim.disconnect(x, local); + sim.unannounce(origin, "live/a/x"); + assert!(sim.run()); + let lost = if request { None } else { Some(false) }; + if !request { + // Only the ROUTE update reaches z, not the requests. + while sim.deliver(x, z) {} + assert!(sim.run()); + } + assert_eq!(sim.shows(z, id, "live/a/x"), lost, "request {request}"); + assert_eq!(sim.shows(z, id, "live/b/z"), Some(true), "request {request}"); + assert_eq!(false_ends(&sim.events), 0, "request {request}"); + sim.check().unwrap(); + } +} + +/// Every lite-07 claim covers the paths below it, so a claim above an +/// interest is inside it. +#[test] +fn a_claim_above_an_interest_overlaps_it() { + let (origin, client) = (0, 3); + let mut sim = lagging("live/a/x"); + sim.announce(origin, "live/a", 0, None); + sim.announce(origin, "live/b", 0, None); + assert!(sim.run()); + assert_eq!(sim.shows(client, 7, "live/a"), Some(true)); + assert_eq!(sim.shows(client, 7, "live/b"), None); + sim.check().unwrap(); +} + +/// Two origins hold the same wildcard capability for one epoch, and a client +/// watches a region below it. Withdrawing one origin's wildcard ends only that +/// origin's claim, and a lagging relay's older copy of it is refused after the +/// client's next hop moves there. +#[test] +fn wildcard_claims_end_per_origin_and_are_certified_below() { + let (o1, o2, r1, r2, client) = (0, 1, 2, 3, 4); + let mut sim = Sim::new(5); + for origin in [o1, o2] { + sim.leaf(origin); + sim.join(origin, r1, 1); + sim.join(origin, r2, 1); + } + sim.leaf(client); + sim.join(client, r1, 1); + sim.join(client, r2, 1); + sim.watch(client, "live/a/"); + sim.originate(o1, 10, 0); + sim.originate(o2, 11, 0); + sim.announce(o1, "live/", 0, Some(1)); + sim.announce(o2, "live/", 0, Some(1)); + assert!(sim.run()); + assert_eq!(sim.shown(client).len(), 2); + + sim.hold(o1, r2); + sim.unannounce(o1, "live/"); + assert!(sim.run()); + assert_eq!(sim.shows(client, 10, "live/"), None); + assert_eq!( + sim.shows(client, 11, "live/"), + Some(true), + "the other origin's claim stays" + ); + + sim.disconnect(client, r1); + assert!(sim.run()); + assert_eq!(sim.shows(client, 10, "live/"), None); + assert_eq!(sim.shows(client, 11, "live/"), Some(true)); +} + +/// An origin process restarts with a fresh identity: its claims start a new +/// revision history and the old identity's go offline once nothing can +/// recover them. Reusing the identity would restart its revision below the +/// floors other nodes hold: they keep the dead process's claims and refuse +/// the new ones. +#[test] +fn origin_restart_takes_a_fresh_identity_for_its_claims() { + let (origin, a, b) = (0, 1, 2); + for fresh in [false, true] { + let mut sim = Sim::new(3); + sim.leaf(origin); + sim.join(origin, a, 1); + sim.join(a, b, 1); + sim.originate(origin, 7, 0); + sim.announce(origin, "live/x", 0, None); + sim.unannounce(origin, "live/x"); + sim.announce(origin, "live/y", 0, None); + assert!(sim.run()); + + sim.kill(origin); + assert!(sim.run()); + for node in [a, b] { + assert!(sim.shown(node).is_empty()); + assert!(!sim.holds(node, 7), "nothing can recover the dead identity"); + } + let offline = sim.events.iter().filter(|e| e.kind == Kind::Offline).count(); + assert_eq!(offline, 2, "offline, not ended by the origin"); + assert_eq!(false_ends(&sim.events), 0); + + let id = if fresh { 8 } else { 7 }; + sim.originate(origin, id, 10); + sim.reconnect(origin); + sim.announce(origin, "live/x", 0, None); + assert!(sim.run()); + for node in [a, b] { + if fresh { + assert_eq!(sim.shows(node, 8, "live/x"), Some(true)); + assert_eq!(sim.shown(node).len(), 1); + } else { + assert_eq!(sim.shows(node, 7, "live/y"), Some(true), "the dead process's claim"); + assert_eq!(sim.shows(node, 7, "live/x"), None); + } + } + } +} + +/// Node b learned that `live/x` ended, then lost every route and collected the +/// origin. A session still delivering older messages brings the claim back, +/// but only until the withdrawal queued behind it lands. Without collection, +/// b's retained floor refuses the replay outright. +#[test] +fn stale_replay_after_collection_lasts_only_until_the_withdrawal_behind_it() { + let (origin, a, b, c) = (0, 1, 2, 3); + for collect in [false, true] { + let mut sim = Sim::new(4); + sim.leaf(origin); + sim.join(origin, a, 1); + sim.join(a, b, 1); + sim.join(a, c, 1); + sim.join(c, b, 1); + sim.hold(a, b); + sim.originate(origin, 7, 0); + sim.announce(origin, "live/x", 0, None); + assert!(sim.run()); + sim.unannounce(origin, "live/x"); + assert!(sim.run()); + assert_eq!(kinds(&at(&sim, b)), [Kind::Start, Kind::End]); + + sim.kill(origin); + assert!(sim.run()); + assert!(!sim.holds(b, 7)); + if collect { + sim.collect(b, 7); + } + // The held ROUTE, an empty snapshot, then the one carrying `live/x`. + for _ in 0..3 { + assert!(sim.deliver(a, b)); + } + assert_eq!(sim.shows(b, 7, "live/x").is_some(), collect); + assert!(sim.deliver(a, b), "the withdrawal"); + assert_eq!(sim.shows(b, 7, "live/x"), None); + sim.release(a, b); + assert!(sim.run()); + let replayed: &[Kind] = if collect { + &[Kind::Start, Kind::End, Kind::Start, Kind::End] + } else { + &[Kind::Start, Kind::End] + }; + assert_eq!(kinds(&at(&sim, b)), replayed); + assert!(!sim.holds(b, 7)); + } +} + +/// Holds follow feasible advertisements only, so they cannot prop each other +/// up. When the origin dies, every node in a ring goes offline and keeps no +/// route state, with no timer, and none reports an END the origin did not send. +#[test] +fn holds_retire_once_nothing_can_recover() { + let origin = 4; + let mut sim = Sim::new(5); + for relay in 0..4 { + sim.join(relay, (relay + 1) % 4, 1); + } + sim.leaf(origin); + sim.join(origin, 0, 1); + sim.originate(origin, 7, 0); + sim.announce(origin, "live/x", 0, None); + assert!(sim.run()); + + sim.kill(origin); + assert!(sim.run()); + for relay in 0..4 { + assert!(sim.shown(relay).is_empty()); + assert!(!sim.holds(relay, 7), "relay {relay}"); + assert!(!sim.remembers(relay, 7), "relay {relay} drained its distance"); + } + assert_eq!(false_ends(&sim.events), 0); + assert_eq!(sim.retained().pending, 0); +} + +/// Seqnos and revisions are varints. An origin never advertises the last +/// value: a bump, a claim change, or a request that would reach it rotates the +/// identity instead, so every node lets the old one go and sees the claims +/// again under the new one. Nothing wraps. +#[test] +fn exhausted_counters_rotate_the_identity() { + let (origin, a, b, c, client) = (0, 1, 2, 3, 4); + let rotated = |sim: &Sim| { + let fresh = sim.proven(origin); + assert_ne!(fresh, 7, "a fresh identity"); + assert_eq!(sim.shows(client, 7, "live/x"), None); + assert_eq!(sim.shows(client, fresh, "live/x"), Some(true)); + assert!(sim.route(a, 7).is_none()); + sim.check().unwrap(); + }; + let build = |seqno| { + let mut sim = Sim::new(5); + sim.leaf(origin); + sim.leaf(client); + sim.join(origin, a, 1); + sim.join(a, b, 1); + sim.join(b, c, 1); + sim.join(origin, c, 1); + sim.join(client, a, 1); + sim.disconnect(origin, c); + sim.originate(origin, 7, seqno); + sim.announce(origin, "live/x", 0, None); + assert!(sim.run()); + sim + }; + + let mut sim = build(MAX - 2); + sim.bump(origin); + assert!(sim.run()); + assert_eq!(sim.proven(origin), 7); + sim.bump(origin); + assert!(sim.run()); + rotated(&sim); + + let mut sim = build(0); + sim.age(origin, MAX - 2); + sim.announce(origin, "live/y", 0, None); + assert!(sim.run()); + assert_eq!(sim.proven(origin), 7); + sim.unannounce(origin, "live/y"); + assert!(sim.run()); + rotated(&sim); + + // The origin moves to a farther attachment, and the starved node asks for + // the last seqno. + let mut sim = build(MAX - 1); + sim.disconnect(origin, a); + sim.connect(origin, c); + assert!(sim.run()); + rotated(&sim); + assert!(sim.route(a, sim.proven(origin)).is_some()); +} + +/// Node b forwards a request along its route through x, whose own route just +/// failed but whose retraction to b is still in flight, so x drops it. When +/// the retraction lands, b moves to an equal-metric route through y and +/// advertises nothing new. Keeping the request at every hop that forwarded +/// it, and resending it when that hop's next hop changes, reaches the origin +/// with no timer. +#[test] +fn a_quiet_origin_is_reached_by_a_kept_request() { + let (origin, w, x, y2, y, b, n, z, client) = (0, 1, 2, 3, 4, 5, 6, 7, 8); + let mut sim = Sim::new(9); + sim.leaf(origin); + sim.leaf(client); + sim.join(origin, w, 1); + sim.join(w, x, 1); + sim.join(origin, y2, 1); + sim.join(y2, y, 1); + sim.link(b, x, SimLink::cost(1).upstream(), SimLink::cost(1)); + sim.link(b, y, SimLink::cost(1).upstream(), SimLink::cost(1)); + sim.join(b, n, 1); + sim.join(n, z, 1); + sim.join(z, origin, 1); + sim.join(n, client, 1); + sim.originate(origin, 7, 0); + sim.announce(origin, "live/x", 0, None); + assert!(sim.run()); + assert_eq!(sim.route(n, 7).and_then(|route| route.next), Some(z)); + assert_eq!(sim.route(b, 7).and_then(|route| route.next), Some(x)); + + sim.disconnect(n, z); + sim.hold(x, b); + sim.disconnect(w, x); + assert!(sim.run()); + assert!(sim.starved(n, 7)); + sim.release(x, b); + assert!(sim.run()); + assert_eq!(sim.route(b, 7).and_then(|route| route.next), Some(y)); + assert_eq!(sim.shows(client, 7, "live/x"), Some(true)); + assert_eq!(sim.seqno(origin), 1); + assert!(sim.route(n, 7).is_some()); +} + +/// Random meshes churn sessions and hold and deliver messages out of step, +/// then release every hold. Every relay that can still reach a quiet origin +/// ends with a route to it, which only a seqno request can bring. +#[test] +fn sweep_requests_reach_a_quiet_origin() { + const RELAYS: usize = 5; + let mut stranded = 0; + for seed in 0..300u64 { + let mut rng = XorShift(0x9e37_79b9 ^ (seed * 0x1234_5678_9abc)); + let nodes = RELAYS + 2; + let mut sim = Sim::new(nodes); + let mut links = Vec::new(); + for a in 0..RELAYS { + for b in a + 1..RELAYS { + if b == a + 1 || rng.chance(40) { + sim.join(a, b, 1 + rng.next(3)); + links.push((a, b)); + } + } + } + for origin in RELAYS..nodes { + sim.leaf(origin); + for _ in 0..2 { + let relay = rng.next(RELAYS as u64) as usize; + if !links.contains(&(relay, origin)) { + sim.join(origin, relay, 1 + rng.next(3)); + links.push((relay, origin)); + } + } + sim.originate(origin, origin as u64, 0); + } + assert!(sim.run()); + for _ in 0..30 { + match rng.next(6) { + 0 | 1 => { + let (a, b) = links[rng.next(links.len() as u64) as usize]; + if sim.connected(a, b) { + sim.disconnect(a, b); + } else { + sim.connect(a, b); + } + } + 2 | 3 => { + let (a, b) = links[rng.next(links.len() as u64) as usize]; + let (from, to) = if rng.chance(50) { (a, b) } else { (b, a) }; + if rng.chance(60) { + sim.hold(from, to); + } else { + sim.release(from, to); + } + } + _ => { + let busy = sim.busy(); + if !busy.is_empty() { + let (from, to) = busy[rng.next(busy.len() as u64) as usize]; + sim.deliver(from, to); + } + } + } + sim.settle(1); + } + for &(a, b) in &links { + sim.release(a, b); + sim.release(b, a); + } + assert!(sim.run()); + for origin in RELAYS..nodes { + // The relays an up session path still joins to the origin. + let mut reach = BTreeSet::from([origin]); + let mut frontier = vec![origin]; + while let Some(at) = frontier.pop() { + for &(a, b) in &links { + let next = match at { + _ if at == a => b, + _ if at == b => a, + _ => continue, + }; + if next < RELAYS && sim.connected(a, b) && reach.insert(next) { + frontier.push(next); + } + } + } + stranded += reach + .iter() + .filter(|&&node| node < RELAYS && sim.route(node, origin as Id).is_none()) + .count(); + } + } + assert_eq!(stranded, 0, "relays left without a route to a reachable origin"); +} + +/// One change to a claim costs one batch per session carrying its origin's +/// route and one event per node watching it; retained state follows sessions, +/// live claims, and current interest, and neither withdrawal history nor +/// interest history leaves anything behind. +#[test] +fn announcement_cost_follows_sessions_and_live_claims() { + const PREFIXES: [&str; 4] = ["live/a", "live/b", "live/c", "live/d"]; + for (relays, claims) in [(4, 2), (8, 2), (4, 4)] { + let mut sim = Sim::new(relays + 2); + for relay in 0..relays { + sim.join(relay, (relay + 1) % relays, 1); + } + let origin = relays; + sim.leaf(origin); + sim.join(origin, 0, 1); + // A client watching a region with no claims yet. + let client = relays + 1; + sim.leaf(client); + sim.join(client, relays / 2, 1); + sim.watch(client, "none/"); + sim.originate(origin, 7, 0); + for prefix in &PREFIXES[..claims] { + sim.announce(origin, prefix, 0, None); + } + assert!(sim.run()); + + // Each relay sends to both ring neighbors, and the origin and its relay to + // each other: without split horizon the relay echoes the origin's claims. + let sessions = 2 * relays + 2; + let retained = sim.retained(); + assert_eq!(retained.items, relays * claims); + assert_eq!(retained.floors, relays + 1); + assert_eq!(retained.heard, sessions * claims); + assert_eq!(retained.sent, retained.heard, "both ends cache the same view"); + + let (stats, events) = (sim.stats, sim.events.len()); + sim.unannounce(origin, PREFIXES[0]); + assert!(sim.run()); + // The client's session carries only the advanced floor. + assert_eq!(sim.stats.claim_msgs - stats.claim_msgs, sessions + 1); + assert_eq!(sim.stats.claim_items - stats.claim_items, sessions); + assert_eq!(sim.events.len() - events, relays + 1, "once per node"); + + for _ in 0..50 { + sim.announce(origin, PREFIXES[0], 0, None); + sim.unannounce(origin, PREFIXES[0]); + } + assert!(sim.run()); + let after = sim.retained(); + assert_eq!(after.items, relays * (claims - 1)); + assert_eq!(after.floors, retained.floors, "no tombstones"); + assert_eq!(after.echoes, 0, "every ROUTE_END echo drained"); + sim.check().unwrap(); + + // The client cycles through disjoint regions, one with a live claim. + for cycles in [1, 16] { + for room in 0..cycles { + sim.watch(client, &format!("room/{room}")); + assert!(sim.run()); + sim.watch(client, PREFIXES[1]); + assert!(sim.run()); + assert_eq!(sim.shows(client, 7, PREFIXES[1]), Some(true)); + } + sim.watch(client, "none/"); + assert!(sim.run()); + assert_eq!(sim.retained(), after, "interest history after {cycles} cycles"); + } + sim.check().unwrap(); + } +} + +/// A client reaches one origin through two relays, each session scoped by a +/// grant under one subject, so both feed one qualified view. Under churn every +/// view stays some revision of the origin's truth, the widening guard stays +/// bounded, no claim ends while the origin still has it, and once churn stops +/// the client shows exactly the claims its interest and the scopes admit. +#[test] +fn sweep_scoped_handoff() { + const CLAIMS: [&str; 5] = ["live/a/x", "live/a/y/", "live/b/z", "live/c", "live/"]; + const WATCHES: [&str; 5] = ["", "live/", "live/a/", "live/b/z", "live/c"]; + const SCOPES: [&str; 2] = ["live/a/", "live/b/"]; + let (origin, r1, r2, client) = (0, 1, 2, 3); + let mut rng = XorShift(0x5c0b_ed00); + let mut live_starts = 0; + for scenario in 0..150 { + let mut sim = Sim::new(4); + sim.leaf(origin); + sim.leaf(client); + sim.join(origin, r1, 1); + sim.join(origin, r2, 1); + sim.join(r1, r2, 1); + for (relay, scope) in [(r1, SCOPES[0]), (r2, SCOPES[1])] { + let grant = Grant { + subject: "local", + scope, + }; + sim.link(client, relay, SimLink::cost(1).grant(grant), SimLink::cost(1)); + } + let links = [(origin, r1), (origin, r2), (r1, r2), (client, r1), (client, r2)]; + sim.originate(origin, 7, 0); + assert!(sim.run()); + for _ in 0..40 { + match rng.next(8) { + 0..=2 => { + let claim = CLAIMS[rng.next(CLAIMS.len() as u64) as usize]; + if rng.chance(60) { + sim.announce(origin, claim, rng.next(3), None); + } else { + sim.unannounce(origin, claim); + } + } + 3 => sim.watch(client, WATCHES[rng.next(WATCHES.len() as u64) as usize]), + 4 => { + let (a, b) = links[rng.next(links.len() as u64) as usize]; + if sim.connected(a, b) { + sim.disconnect(a, b); + } else { + sim.connect(a, b); + } + } + 5 => { + let (a, b) = links[rng.next(links.len() as u64) as usize]; + let (from, to) = if rng.chance(50) { (a, b) } else { (b, a) }; + if rng.chance(60) { + sim.hold(from, to); + } else { + sim.release(from, to); + } + } + _ => { + let busy = sim.busy(); + if !busy.is_empty() { + let (from, to) = busy[rng.next(busy.len() as u64) as usize]; + sim.deliver(from, to); + } + } + } + sim.settle(1); + sim.check().unwrap_or_else(|err| panic!("scenario {scenario}: {err}")); + assert!( + sim.retained().certified <= SCOPES.len() + 1, + "scenario {scenario}: the widening guard grows" + ); + } + for (a, b) in links { + sim.release(a, b); + sim.release(b, a); + if !sim.connected(a, b) { + sim.connect(a, b); + } + } + assert!(sim.run()); + sim.check().unwrap_or_else(|err| panic!("scenario {scenario}: {err}")); + let watch = sim.watching(client).clone(); + let rev = sim.revision(7); + let expected: BTreeSet = CLAIMS + .into_iter() + .filter(|claim| sim.truth(7, claim, rev).is_some()) + .map(p) + .filter(|claim| { + SCOPES.iter().map(|scope| p(scope)).any(|scope| { + let region = if within(&scope, &watch) { + scope + } else if within(&watch, &scope) { + watch.clone() + } else { + return false; + }; + overlaps(claim, ®ion) + }) + }) + .collect(); + let id = qualified("local", 7); + let shown: BTreeSet = sim + .shown(client) + .iter() + .filter(|((shown, _), view)| *shown == id && view.available) + .map(|((_, claim), _)| claim.clone()) + .collect(); + assert_eq!(shown, expected, "scenario {scenario} watching {watch:?}"); + assert_eq!(false_ends(&sim.events), 0, "scenario {scenario}"); + live_starts += sim + .events + .iter() + .filter(|event| event.id == id && event.kind == Kind::Start && !event.withdrawn) + .count(); + } + assert!(live_starts > 0); +} + +const RELAYS: usize = 5; +const CLAIMS: [&str; 4] = ["live/", "live/a/", "live/a/x", "live/b"]; + +/// Random meshes with two origins and a client under churn: claims change, +/// sessions drop and return, messages are held and replayed out of step, the +/// client changes interest, nodes collect route-less identities, and an +/// origin restarts with a fresh identity. Selected routes never loop, every +/// view is always some revision of the origin's truth, nothing revives and +/// nothing ends falsely, and once churn stops every node shows every live +/// origin's current claims and holds nothing for a dead one. +#[test] +fn sweep_handoff() { + let mut rng = XorShift(0x4a4d_0ff5); + for scenario in 0..150 { + let mut sim = churn(&mut rng); + let label = format!("scenario {scenario}"); + assert_eq!(revivals(&sim.events), 0, "{label}"); + assert_eq!(false_ends(&sim.events), 0, "{label}"); + settled(&mut sim, &label); + } +} + +/// Origins sit at `RELAYS` and `RELAYS + 1`, the client at `RELAYS + 2`. +fn churn(rng: &mut XorShift) -> Sim { + let (first, client) = (RELAYS, RELAYS + 2); + let mut sim = Sim::new(RELAYS + 3); + let mut links = Vec::new(); + for relay in 0..RELAYS { + links.push((relay, (relay + 1) % RELAYS)); + } + let (a, b) = (rng.next(RELAYS as u64) as usize, rng.next(RELAYS as u64) as usize); + if a.abs_diff(b) > 1 && a.abs_diff(b) != RELAYS - 1 { + links.push((a, b)); + } + for leaf in [first, first + 1, client] { + sim.leaf(leaf); + let a = rng.next(RELAYS as u64) as usize; + links.push((leaf, a)); + let b = (a + 1 + rng.next(RELAYS as u64 - 1) as usize) % RELAYS; + links.push((leaf, b)); + } + for &(a, b) in &links { + sim.join(a, b, 1 + rng.next(3)); + } + let mut ids = [100, 200]; + sim.originate(first, ids[0], 0); + sim.originate(first + 1, ids[1], 0); + assert!(sim.run()); + + for _ in 0..40 { + match rng.next(10) { + 0..=3 => { + let origin = rng.next(2) as usize; + let claim = CLAIMS[rng.next(CLAIMS.len() as u64) as usize]; + if rng.chance(50) { + sim.announce(first + origin, claim, rng.next(3), None); + } else { + sim.unannounce(first + origin, claim); + } + } + 4 => { + let (a, b) = links[rng.next(links.len() as u64) as usize]; + if sim.connected(a, b) { + sim.disconnect(a, b); + } else { + sim.connect(a, b); + } + } + 5 => { + let (a, b) = links[rng.next(links.len() as u64) as usize]; + let (from, to) = if rng.chance(50) { (a, b) } else { (b, a) }; + if rng.chance(70) { + sim.hold(from, to); + } else { + sim.release(from, to); + } + } + 6 => { + sim.watch(client, ["", "live/a/", "live/b"][rng.next(3) as usize]); + } + 7 if rng.chance(50) => { + // Collection is allowed once a node holds no route to an identity. + let node = rng.next(RELAYS as u64 + 3) as usize; + let id = ids[rng.next(2) as usize]; + if !sim.holds(node, id) && (RELAYS..RELAYS + 2).all(|origin| node != origin) { + sim.collect(node, id); + } + } + 7 if rng.chance(40) => { + let origin = rng.next(2) as usize; + sim.kill(first + origin); + ids[origin] += 1; + sim.originate(first + origin, ids[origin], 0); + sim.reconnect(first + origin); + } + _ => { + // Echoed ROUTE_ENDs interleave freely with the echoing node's routes. + for _ in 0..rng.next(6) { + let busy: Vec<(bool, usize, usize)> = (sim.busy().into_iter().map(|(from, to)| (false, from, to))) + .chain(sim.busy_acks().into_iter().map(|(from, to)| (true, from, to))) + .collect(); + if busy.is_empty() { + break; + } + match busy[rng.next(busy.len() as u64) as usize] { + (false, from, to) => sim.deliver(from, to), + (true, from, to) => sim.deliver_ack(from, to), + }; + } + } + } + sim.check().unwrap(); + for id in (100..=ids[0]).chain(200..=ids[1]) { + assert_loop_free(&sim, RELAYS + 3, id); + } + } + + for &(a, b) in &links { + sim.release(a, b); + sim.release(b, a); + if !sim.connected(a, b) { + sim.connect(a, b); + } + } + assert!(sim.run()); + sim +} + +/// After churn stops with every session up, each node shows exactly the live +/// origins' current claims within its interest, available, and keeps nothing +/// for an identity whose process is gone. +fn settled(sim: &mut Sim, label: &str) { + sim.check().unwrap(); + let live: Vec = (RELAYS..RELAYS + 2).map(|origin| sim.proven(origin)).collect(); + for node in 0..RELAYS + 3 { + let mut expected = BTreeSet::new(); + for &id in &live { + assert!( + sim.route(node, id).is_some(), + "{label}: node {node} has no route to {id}" + ); + for claim in CLAIMS { + if overlaps(&p(claim), sim.watching(node)) && sim.truth(id, claim, sim.revision(id)).is_some() { + expected.insert((id, p(claim))); + } + } + } + let shown: BTreeSet<(Id, Path)> = sim.shown(node).keys().cloned().collect(); + assert_eq!(shown, expected, "{label}: node {node}"); + for ((id, claim), shown) in sim.shown(node) { + assert!(shown.available, "{label}: node {node} {id} {claim}"); + assert_eq!( + sim.truth(*id, claim.as_str(), sim.revision(*id)), + Some(shown.item), + "{label}: node {node} {id} {claim}" + ); + } + for id in (100..110).chain(200..210).filter(|id| !live.contains(id)) { + assert!(!sim.holds(node, id), "{label}: node {node} holds dead {id}"); + assert!( + !sim.remembers(node, id), + "{label}: node {node} keeps dead {id}'s distance" + ); + } + } + assert_eq!(sim.retained().pending, 0, "{label}"); +} diff --git a/rs/moq-net/src/route/tests/mod.rs b/rs/moq-net/src/route/tests/mod.rs new file mode 100644 index 00000000000..a49ae20ef71 --- /dev/null +++ b/rs/moq-net/src/route/tests/mod.rs @@ -0,0 +1,735 @@ +//! The routing contract's fixtures, run against [`Router`] in a simulated network. +//! +//! [`Sim`] wires one router per node through ordered, holdable message queues, +//! so a fixture can deliver messages out of step and replay them late. Claims +//! are written with a trailing `/` where the contract means a wildcard; every +//! lite-07 claim already covers the paths below it, so the slash is dropped. + +mod handoff; +mod routing; + +use std::{ + collections::{BTreeMap, BTreeSet, VecDeque}, + sync::Arc, +}; + +use super::*; + +pub(super) type Node = usize; + +/// A claim's attributes, plus the revision its life began at so fixtures can +/// tell a revival from a fresh announcement. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Item { + pub seed: Metric, + pub epoch: Option, + pub since: Rev, +} + +/// A contract path: a trailing `/` (a wildcard in the contract) is dropped. +pub(super) fn p(path: &str) -> Path { + Path::new(path.strip_suffix('/').unwrap_or(path)).expect("a valid path") +} + +/// A scoped publish grant: the credential subject that qualifies identities, +/// and the prefix standing in for the grant's pattern union. +#[derive(Clone, Copy, Debug)] +pub(super) struct Grant { + pub subject: &'static str, + pub scope: &'static str, +} + +/// A qualified key, as a stand-in derivation keyed by the grant's subject. +pub(super) fn qualified(subject: &str, id: Id) -> Id { + fnv(subject, &[id]) | QUALIFIED +} + +fn fnv(label: &str, ids: &[u64]) -> u64 { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for byte in label.bytes().chain(ids.iter().flat_map(|id| id.to_le_bytes())) { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x0100_0000_01b3); + } + hash +} + +/// One side's configuration of a link. +#[derive(Clone, Copy, Debug)] +pub(super) struct SimLink { + pub cost: Metric, + pub priority: i8, + pub upstream: bool, + pub grant: Option, +} + +impl SimLink { + pub fn cost(cost: Metric) -> Self { + Self { + cost, + priority: 0, + upstream: false, + grant: None, + } + } + + pub fn priority(mut self, priority: i8) -> Self { + self.priority = priority; + self + } + + pub fn upstream(mut self) -> Self { + self.upstream = true; + self + } + + pub fn grant(mut self, grant: Grant) -> Self { + self.grant = Some(grant); + self + } +} + +#[derive(Clone, Debug)] +enum Msg { + Route { + id: Id, + reach: Option, + }, + Batch { + id: Id, + changes: Vec<(Path, Option)>, + floors: BTreeMap, + interest: u32, + }, + Request { + id: Id, + seqno: Seqno, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct SimEvent { + pub node: Node, + pub id: Id, + pub claim: String, + pub kind: Kind, + pub since: Rev, + /// Restarts of the node, then how many times it forgot this claim. + pub forgot: (u32, u32), + /// Whether the origin had withdrawn or replaced this life of the claim by then. + pub withdrawn: bool, +} + +/// Messages sent, as protocol counts. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) struct Stats { + pub route_msgs: usize, + pub claim_msgs: usize, + pub claim_items: usize, + pub requests: usize, + /// Requests a node sent the same neighbor again for the same seqno. + pub repeated_requests: usize, +} + +struct Change { + rev: Rev, + claim: Path, + item: Option, +} + +pub(super) struct Sim { + nodes: Vec>, + leaf: Vec, + watch: Vec, + restarts: Vec, + /// The identity each origin node declared last, to spot a rotation. + declared: Vec>, + links: Vec>>, + grants: BTreeSet<&'static str>, + up: BTreeSet<(Node, Node)>, + queues: BTreeMap<(Node, Node), VecDeque>, + /// Echoed ROUTE_ENDs: unordered against the echoing node's own routes. + acks: BTreeMap<(Node, Node), VecDeque>, + held: BTreeSet<(Node, Node)>, + history: BTreeMap>, + /// Every request sent, to spot a repeat. + requested: BTreeSet<(Node, Node, Id, Seqno)>, + pub stats: Stats, + pub events: Vec, +} + +impl Sim { + pub fn new(nodes: usize) -> Self { + Self { + nodes: (0..nodes).map(|node| Router::new(node as u64 + 1)).collect(), + leaf: vec![false; nodes], + watch: vec![Path::empty(); nodes], + restarts: vec![0; nodes], + declared: vec![None; nodes], + links: vec![vec![None; nodes]; nodes], + grants: BTreeSet::new(), + up: BTreeSet::new(), + queues: BTreeMap::new(), + acks: BTreeMap::new(), + held: BTreeSet::new(), + history: BTreeMap::new(), + requested: BTreeSet::new(), + stats: Stats::default(), + events: Vec::new(), + } + } + + /// A node that sends only identities it serves, like a browser. + pub fn leaf(&mut self, node: Node) { + self.leaf[node] = true; + } + + /// Configure both sides of a link and open its session. + pub fn link(&mut self, a: Node, b: Node, ab: SimLink, ba: SimLink) { + self.links[a][b] = Some(ab); + self.links[b][a] = Some(ba); + for grant in [ab.grant, ba.grant].into_iter().flatten() { + self.grants.insert(grant.subject); + } + self.connect(a, b); + } + + /// A symmetric link of `cost`. + pub fn join(&mut self, a: Node, b: Node, cost: Metric) { + self.link(a, b, SimLink::cost(cost), SimLink::cost(cost)); + } + + fn pair(a: Node, b: Node) -> (Node, Node) { + (a.min(b), a.max(b)) + } + + fn config(&self, node: Node, peer: Node) -> Link { + let side = self.links[node][peer].expect("a configured link"); + Link { + cost: side.cost, + priority: side.priority, + upstream: side.upstream, + proxy: !self.leaf[node], + scope: side.grant.map(|grant| Scope { + regions: vec![p(grant.scope)], + qualify: Arc::new(move |id| qualified(grant.subject, id)), + }), + } + } + + /// Open a session: each side opens a stream asking for its interest. + pub fn connect(&mut self, a: Node, b: Node) { + assert!(!self.connected(a, b), "{a}-{b} already up"); + self.up.insert(Self::pair(a, b)); + for (x, y) in [(a, b), (b, a)] { + let link = self.config(x, y); + self.nodes[x].connect(y, link).expect("a positive cost"); + } + for (x, y) in [(a, b), (b, a)] { + let watch = self.watch[y].clone(); + self.nodes[x].peer_interest(y, watch).unwrap(); + } + self.flush(); + } + + pub fn connected(&self, a: Node, b: Node) -> bool { + self.up.contains(&Self::pair(a, b)) + } + + /// Close a session: in-flight messages and everything learned on it go. + pub fn disconnect(&mut self, a: Node, b: Node) { + self.up.remove(&Self::pair(a, b)); + for (x, y) in [(a, b), (b, a)] { + self.queues.remove(&(x, y)); + self.acks.remove(&(x, y)); + self.held.remove(&(x, y)); + self.nodes[y].disconnect(x); + } + self.flush(); + } + + /// The process at `node` exits: its sessions close and its state is lost. + pub fn kill(&mut self, node: Node) { + let peers: Vec = self.peers(node).collect(); + for peer in peers { + self.disconnect(node, peer); + } + self.restarts[node] += 1; + self.declared[node] = None; + self.nodes[node] = Router::new(fnv("restart", &[node as u64, u64::from(self.restarts[node])])); + let watch = self.watch[node].clone(); + self.nodes[node].watch(watch); + self.flush(); + } + + /// Re-open every configured link of `node`. + pub fn reconnect(&mut self, node: Node) { + let peers: Vec = (0..self.nodes.len()) + .filter(|&peer| self.links[node][peer].is_some() && !self.connected(node, peer)) + .collect(); + for peer in peers { + self.connect(node, peer); + } + } + + fn peers(&self, node: Node) -> impl Iterator + '_ { + (0..self.nodes.len()).filter(move |&peer| self.links[node][peer].is_some() && self.connected(node, peer)) + } + + pub fn originate(&mut self, node: Node, id: Id, seqno: Seqno) { + self.nodes[node].originate(id, seqno); + self.declared[node] = Some(id); + self.history.entry(id).or_default(); + self.flush(); + } + + pub fn forge(&mut self, node: Node, id: Id, seqno: Seqno) { + self.nodes[node].forge(id, seqno); + self.flush(); + } + + pub fn bump(&mut self, node: Node) { + self.nodes[node].bump().unwrap(); + self.flush(); + } + + pub fn age(&mut self, node: Node, rev: Rev) { + self.nodes[node].age(rev); + self.flush(); + } + + /// The identity `node` declares. + pub fn proven(&self, node: Node) -> Id { + self.nodes[node].declared().expect("not an origin") + } + + pub fn seqno(&self, node: Node) -> Seqno { + self.nodes[node].seqno().expect("not an origin") + } + + pub fn announce(&mut self, node: Node, claim: &str, seed: Metric, epoch: Option) { + let claim = p(claim); + let router = &self.nodes[node]; + let id = router.declared().expect("not an origin"); + let rev = router.revision().unwrap() + 1; + let since = router + .view(id) + .and_then(|view| view.items.get(&claim)) + .map_or(rev, |item| item.since); + let item = Item { seed, epoch, since }; + self.nodes[node].announce(claim.clone(), item).unwrap(); + self.record(node, claim, Some(item)); + } + + pub fn unannounce(&mut self, node: Node, claim: &str) { + let claim = p(claim); + self.nodes[node].unannounce(claim.clone()).unwrap(); + self.record(node, claim, None); + } + + fn record(&mut self, node: Node, claim: Path, item: Option) { + let router = &self.nodes[node]; + let id = router.declared().unwrap(); + let rev = router.revision().unwrap(); + if self.declared[node] == Some(id) { + self.history.entry(id).or_default().push(Change { rev, claim, item }); + } + self.flush(); + } + + /// The prefix `node` asks its neighbors for announcements under; each + /// neighbor answers on a new stream with a complete snapshot. + pub fn watch(&mut self, node: Node, prefix: &str) { + let prefix = p(prefix); + self.watch[node] = prefix.clone(); + self.nodes[node].watch(prefix.clone()); + let peers: Vec = self.peers(node).collect(); + for peer in peers { + self.nodes[peer].peer_interest(node, prefix.clone()).unwrap(); + } + self.flush(); + } + + pub fn watching(&self, node: Node) -> &Path { + &self.watch[node] + } + + pub fn goaway(&mut self, node: Node) { + let peers: Vec = self.peers(node).collect(); + for peer in peers { + self.nodes[peer].goaway(node); + } + self.flush(); + } + + pub fn collect(&mut self, node: Node, id: Id) { + self.nodes[node].collect(id); + self.flush(); + } + + pub fn hold(&mut self, from: Node, to: Node) { + self.held.insert((from, to)); + } + + pub fn release(&mut self, from: Node, to: Node) { + self.held.remove(&(from, to)); + } + + /// Deliver one in-order message from `from` to `to`, even if held. + pub fn deliver(&mut self, from: Node, to: Node) -> bool { + let Some(msg) = self.queues.get_mut(&(from, to)).and_then(VecDeque::pop_front) else { + return false; + }; + let router = &mut self.nodes[to]; + match msg { + Msg::Route { id, reach } => router.route(from, id, reach), + Msg::Batch { + id, + changes, + floors, + interest, + } => router.batch(from, id, changes, floors, interest), + Msg::Request { id, seqno } => router.request(from, id, seqno), + } + self.flush(); + true + } + + /// Deliver one echoed ROUTE_END from `from` to `to`, even if held. + pub fn deliver_ack(&mut self, from: Node, to: Node) -> bool { + let Some(id) = self.acks.get_mut(&(from, to)).and_then(VecDeque::pop_front) else { + return false; + }; + self.nodes[to].echo(from, id); + self.flush(); + true + } + + pub fn queued(&self, from: Node, to: Node) -> usize { + self.queues.get(&(from, to)).map_or(0, VecDeque::len) + } + + pub fn queued_acks(&self, from: Node, to: Node) -> usize { + self.acks.get(&(from, to)).map_or(0, VecDeque::len) + } + + pub fn busy(&self) -> Vec<(Node, Node)> { + Self::nonempty(&self.queues) + } + + pub fn busy_acks(&self) -> Vec<(Node, Node)> { + Self::nonempty(&self.acks) + } + + fn nonempty(queues: &BTreeMap<(Node, Node), VecDeque>) -> Vec<(Node, Node)> { + queues + .iter() + .filter(|(_, queue)| !queue.is_empty()) + .map(|(key, _)| *key) + .collect() + } + + /// Deliver round-robin until every unheld queue is empty. False if it never + /// settles within a generous budget, which is how oscillation shows up. + pub fn run(&mut self) -> bool { + self.settle(10_000) + } + + pub fn settle(&mut self, rounds: usize) -> bool { + for _ in 0..rounds { + let ready: Vec<(Node, Node)> = self.busy().into_iter().filter(|key| !self.held.contains(key)).collect(); + let acks: Vec<(Node, Node)> = self + .busy_acks() + .into_iter() + .filter(|key| !self.held.contains(key)) + .collect(); + if ready.is_empty() && acks.is_empty() { + return true; + } + for (from, to) in ready { + self.deliver(from, to); + } + for (from, to) in acks { + self.deliver_ack(from, to); + } + } + false + } + + /// Move every router's output onto the queues and log its reader events. + fn flush(&mut self) { + for node in 0..self.nodes.len() { + for out in self.nodes[node].drain() { + let (to, msg) = match out { + Out::Route { to, id, reach } => { + self.stats.route_msgs += 1; + (to, Msg::Route { id, reach }) + } + Out::Batch { + to, + id, + changes, + floors, + } => { + self.stats.claim_msgs += 1; + self.stats.claim_items += changes.len(); + let interest = self.nodes[to].interest(); + ( + to, + Msg::Batch { + id, + changes, + floors, + interest, + }, + ) + } + Out::Request { to, id, seqno } => { + self.stats.requests += 1; + if !self.requested.insert((node, to, id, seqno)) { + self.stats.repeated_requests += 1; + } + (to, Msg::Request { id, seqno }) + } + Out::Echo { to, id } => { + self.stats.route_msgs += 1; + self.acks.entry((node, to)).or_default().push_back(id); + continue; + } + }; + self.queues.entry((node, to)).or_default().push_back(msg); + } + // A rotated origin starts a fresh history from the claims it carried. + let declared = self.nodes[node].declared(); + if self.declared[node].is_some() && declared != self.declared[node] { + let id = declared.expect("rotated"); + let router = &self.nodes[node]; + let rev = router.revision().unwrap(); + let changes = router + .view(id) + .unwrap() + .items + .iter() + .map(|(claim, item)| Change { + rev, + claim: claim.clone(), + item: Some(*item), + }) + .collect(); + self.history.insert(id, changes); + self.declared[node] = declared; + } + for event in self.nodes[node].events() { + let origin = self.source(event.id).unwrap_or(event.id); + let since = event.item.since; + let withdrawn = self + .truth(origin, event.claim.as_str(), self.revision(origin)) + .is_none_or(|item| item.since != since); + self.events.push(SimEvent { + node, + id: event.id, + claim: event.claim.as_str().to_string(), + kind: event.kind, + since, + forgot: (self.restarts[node], event.forgot), + withdrawn, + }); + } + } + } + + pub fn route(&self, node: Node, id: Id) -> Option> { + self.nodes[node].selected(id) + } + + pub fn export_route(&self, node: Node, id: Id) -> Option> { + self.nodes[node].export(id) + } + + pub fn starved(&self, node: Node, id: Id) -> bool { + self.nodes[node].starved(id) + } + + pub fn grounded(&self, node: Node, id: Id) -> bool { + self.nodes[node].grounded(id) + } + + pub fn holds(&self, node: Node, id: Id) -> bool { + self.nodes[node].holds(id) + } + + pub fn remembers(&self, node: Node, id: Id) -> bool { + self.nodes[node].remembers(id) + } + + pub fn floors(&self, node: Node, id: Id) -> usize { + self.nodes[node].floors(id) + } + + pub fn identities(&self, node: Node) -> usize { + self.nodes[node].identities() + } + + pub fn aliases(&self, node: Node) -> usize { + self.nodes[node].aliases() + } + + pub fn shown(&self, node: Node) -> &BTreeMap<(Id, Path), Shown> { + self.nodes[node].shown() + } + + pub fn shows(&self, node: Node, id: Id, claim: &str) -> Option { + self.nodes[node].shows(id, &p(claim)) + } + + /// Retained state summed over every node. + pub fn retained(&self) -> Retained { + let mut out = Retained::default(); + for router in &self.nodes { + let r = router.retained(); + out.items += r.items; + out.floors += r.floors; + out.heard += r.heard; + out.sent += r.sent; + out.echoes += r.echoes; + out.pending += r.pending; + out.distances += r.distances; + out.certified += r.certified; + } + out + } + + /// The origin's state of `claim` as of revision `rev`. + pub fn truth(&self, id: Id, claim: &str, rev: Rev) -> Option { + let claim = p(claim); + self.history + .get(&id)? + .iter() + .rev() + .find(|change| change.rev <= rev && change.claim == claim) + .and_then(|change| change.item) + } + + pub fn revision(&self, id: Id) -> Rev { + self.history + .get(&id) + .and_then(|history| history.last()) + .map_or(0, |change| change.rev) + } + + /// Every claim each node knows must be the origin's state at the revision it + /// knows that claim at. Returns the views checked. + pub fn check(&self) -> Result, String> { + let mut checked = BTreeSet::new(); + for (node, router) in self.nodes.iter().enumerate() { + for (key, view) in router.views() { + let Some(id) = self.source(key) else { + continue; + }; + checked.insert((node, key)); + let claims: BTreeSet<&Path> = self.history[&id].iter().map(|change| &change.claim).collect(); + for claim in claims { + let Some(rev) = view.as_of(claim) else { + if view.items.contains_key(claim) { + return Err(format!("node {node} holds {id}'s {claim} with no floor")); + } + continue; + }; + let known = view.items.get(claim).copied(); + let truth = self.truth(id, claim.as_str(), rev); + if known != truth { + return Err(format!( + "node {node} holds {key}'s {claim} as {known:?} at revision {rev}, where the origin had {truth:?}" + )); + } + } + } + } + Ok(checked) + } + + /// The origin identity whose claims `key` carries. + fn source(&self, key: Id) -> Option { + if self.history.contains_key(&key) { + return Some(key); + } + if key & QUALIFIED == 0 { + return None; + } + self.history + .keys() + .copied() + .find(|&inner| self.grants.iter().any(|subject| qualified(subject, inner) == key)) + } +} + +pub(super) struct XorShift(pub u64); + +impl XorShift { + pub fn next(&mut self, below: u64) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 % below + } + + pub fn chance(&mut self, percent: u64) -> bool { + self.next(100) < percent + } +} + +pub(super) fn kinds(events: &[SimEvent]) -> Vec { + events.iter().map(|event| event.kind).collect() +} + +pub(super) fn at(sim: &Sim, node: Node) -> Vec { + sim.events.iter().filter(|event| event.node == node).cloned().collect() +} + +/// The kinds `node` saw for `claim`. +pub(super) fn life(sim: &Sim, node: Node, claim: &str) -> Vec { + let claim = p(claim); + sim.events + .iter() + .filter(|event| event.node == node && event.claim == claim.as_str()) + .map(|event| event.kind) + .collect() +} + +/// Starts of a claim's life that the same node had already shown ending. +pub(super) fn revivals(events: &[SimEvent]) -> usize { + let mut ended = BTreeSet::new(); + let mut count = 0; + for event in events { + let life = (event.node, event.forgot, event.id, event.claim.clone(), event.since); + match event.kind { + Kind::End => { + ended.insert(life); + } + Kind::Start if ended.contains(&life) => count += 1, + _ => {} + } + } + count +} + +/// Ends shown while the origin still had that life of the claim. +pub(super) fn false_ends(events: &[SimEvent]) -> usize { + events + .iter() + .filter(|event| event.kind == Kind::End && !event.withdrawn) + .count() +} + +/// Every node's next-hop chain toward `id` ends without revisiting a node. +pub(super) fn assert_loop_free(sim: &Sim, nodes: usize, id: Id) { + for node in 0..nodes { + let mut at = node; + for hop in 0.. { + let Some(next) = sim.route(at, id).and_then(|selected| selected.next) else { + break; + }; + assert!(hop <= nodes, "{id} loops from node {node}"); + at = next; + } + } +} diff --git a/rs/moq-net/src/route/tests/routing.rs b/rs/moq-net/src/route/tests/routing.rs new file mode 100644 index 00000000000..ac25fbd64f0 --- /dev/null +++ b/rs/moq-net/src/route/tests/routing.rs @@ -0,0 +1,752 @@ +//! Route selection: identity, feasibility, seqno requests, GOAWAY, the +//! down-only bit, and qualified identities. + +use super::*; + +const LOCAL: Grant = Grant { + subject: "local", + scope: "local/", +}; + +fn next(sim: &Sim, node: Node, id: Id) -> Option { + sim.route(node, id).and_then(|route| route.next) +} + +/// A restart that reuses its routing identity restarts its seqno below the +/// feasibility distance every other node still holds, so its routes are +/// refused until a seqno request lifts it past the old process's seqno. A +/// fresh identity has no history to lose to. +#[test] +fn restart_reusing_identity_starves_until_requested() { + let (origin, a, b) = (0, 1, 2); + let mut sim = Sim::new(3); + sim.join(origin, a, 1); + sim.join(a, b, 1); + sim.originate(origin, 7, 0); + for _ in 0..3 { + sim.bump(origin); + } + assert!(sim.run()); + assert!(sim.route(b, 7).is_some()); + + sim.kill(origin); + sim.originate(origin, 7, 0); + sim.reconnect(origin); + sim.hold(a, origin); + assert!(sim.run()); + assert!(sim.starved(a, 7), "seqno 0 is older than the 3 a remembers"); + assert!(sim.route(b, 7).is_none()); + sim.release(a, origin); + assert!(sim.run()); + assert_eq!(sim.seqno(origin), 4, "the request names the seqno after a's distance"); + assert!(sim.route(b, 7).is_some()); + + sim.kill(origin); + sim.originate(origin, 8, 0); + sim.reconnect(origin); + let requests = sim.stats.requests; + assert!(sim.run()); + assert!(sim.route(b, 8).is_some()); + assert_eq!(sim.stats.requests, requests, "a fresh identity needs no request"); +} + +/// The same process moving to a farther attachment keeps its identity and +/// seqno, so the node next to its old attachment holds a feasibility distance +/// shorter than any route it now has and starves until its seqno request +/// reaches the origin. +#[test] +fn reconnect_without_restart_needs_a_seqno_advance() { + let (origin, a, b, c) = (0, 1, 2, 3); + let mut sim = Sim::new(4); + sim.join(a, b, 1); + sim.join(b, c, 1); + sim.join(origin, a, 1); + sim.join(origin, c, 1); + sim.disconnect(origin, c); + sim.originate(origin, 7, 0); + assert!(sim.run()); + + sim.hold(a, b); + sim.disconnect(origin, a); + sim.connect(origin, c); + assert!(sim.run()); + assert!(sim.starved(a, 7), "a advertised one hop; every new route is longer"); + + sim.release(a, b); + assert!(sim.run()); + assert_eq!(sim.seqno(origin), 1); + assert_eq!(next(&sim, a, 7), Some(b)); +} + +/// A dead incarnation's route can return after a node collects it, but only +/// from a message already in flight on a live session, and that session then +/// delivers the retraction behind it. +#[test] +fn collected_incarnation_returns_only_while_its_retraction_is_in_flight() { + let (origin, a, b, c) = (0, 1, 2, 3); + let mut sim = Sim::new(4); + sim.join(origin, a, 1); + sim.join(a, b, 1); + sim.join(a, c, 1); + sim.join(c, b, 1); + // Everything a sends b waits: first the route, then its retraction. + sim.hold(a, b); + sim.originate(origin, 7, 0); + assert!(sim.run()); + assert_eq!(next(&sim, b, 7), Some(c)); + + sim.kill(origin); + assert!(sim.run()); + assert!(sim.route(b, 7).is_none()); + + sim.collect(b, 7); + assert!(sim.deliver(a, b), "the held update"); + assert!(sim.route(b, 7).is_some(), "the dead incarnation is back at b"); + assert!(sim.run()); + assert!(sim.route(a, 7).is_none()); + assert_loop_free(&sim, 4, 7); + + sim.release(a, b); + assert!(sim.run()); + for node in 0..4 { + assert!(sim.route(node, 7).is_none(), "node {node} routes to a dead incarnation"); + } +} + +/// No route entry is not proof that no advertisement derived from this node's +/// own is still in flight. b keeps its feasibility distance until every +/// neighbor it advertised to echoes the ROUTE_END back, so its own echo from c +/// is refused rather than looping. A leaf never advertises the identity back, +/// so only its explicit echo ends the drain. +#[test] +fn collection_keeps_the_feasibility_distance_against_a_queued_echo() { + let (origin, b, c, leaf) = (0, 1, 2, 3); + let mut sim = Sim::new(4); + sim.leaf(leaf); + sim.join(origin, b, 1); + sim.join(b, c, 1); + sim.join(b, leaf, 1); + sim.hold(c, b); + sim.originate(origin, 7, 0); + assert!(sim.run()); + assert_eq!(next(&sim, c, 7), Some(b)); + assert!(sim.queued(c, b) > 0, "c's echo waits"); + + sim.hold(b, c); + sim.hold(leaf, b); + sim.disconnect(origin, b); + assert!(!sim.holds(b, 7)); + sim.collect(b, 7); + assert!(sim.remembers(b, 7), "draining: the ROUTE_ENDs are not echoed yet"); + assert!(sim.deliver(c, b), "the echo of b's own advertisement"); + assert!(sim.route(b, 7).is_none(), "b refuses its own echo"); + assert!(sim.starved(b, 7)); + + sim.release(b, c); + sim.release(c, b); + assert!(sim.run()); + assert!(sim.queued_acks(leaf, b) > 0, "the leaf's echo waits"); + assert!(sim.remembers(b, 7), "the leaf may still hold b's advertisement"); + sim.release(leaf, b); + assert!(sim.run()); + for node in [b, c, leaf] { + assert!( + sim.route(node, 7).is_none(), + "node {node} routes to an unreachable origin" + ); + assert!(!sim.holds(node, 7), "node {node}"); + assert!(!sim.remembers(node, 7), "node {node} keeps a drained distance"); + } +} + +/// A restarted relay loses every feasibility distance, yet stays loop-free: +/// its sessions close with the process, and each neighbor drops what it +/// learned on them. A stale echo still reaches it through a third node's +/// session that outlived it, and is safe, because every node on that echo's +/// path kept its own distance. +#[test] +fn restart_drops_the_feasibility_distance_with_every_session() { + let (origin, b, c, d) = (0, 1, 2, 3); + let mut sim = Sim::new(4); + sim.join(origin, b, 1); + sim.join(b, c, 1); + sim.join(c, d, 1); + sim.join(d, b, 10); + sim.originate(origin, 7, 0); + assert!(sim.run()); + assert_eq!(next(&sim, d, 7), Some(c)); + + // The update c sends d after losing b waits on a session that outlives b. + sim.hold(c, d); + sim.kill(b); + assert!(sim.run()); + assert_loop_free(&sim, 4, 7); + assert!(sim.route(c, 7).is_none()); + assert_eq!(next(&sim, d, 7), Some(c), "stale"); + + sim.connect(b, d); + assert!(sim.run()); + assert_eq!( + next(&sim, b, 7), + Some(d), + "the old advertisement reaches the new process" + ); + assert_loop_free(&sim, 4, 7); + sim.connect(b, c); + assert!(sim.run()); + assert!(sim.route(c, 7).is_none(), "the distance at c refuses the longer copy"); + assert_loop_free(&sim, 4, 7); + + sim.release(c, d); + assert!(sim.run()); + assert_loop_free(&sim, 4, 7); + sim.connect(origin, b); + assert!(sim.run()); + assert_loop_free(&sim, 4, 7); + for node in [b, c, d] { + assert!(sim.route(node, 7).is_some(), "node {node} recovers"); + } +} + +/// Random meshes churn links, hold and deliver messages out of step, and +/// collect route-less identities. Each node drops a feasibility distance only +/// once every neighbor echoed its ROUTE_END, so no selection ever loops. +#[test] +fn sweep_forgetting_distances() { + const RELAYS: usize = 5; + let mut rng = XorShift(0x5eed_f0d5); + let mut collected = 0; + for _ in 0..200 { + let nodes = RELAYS + 2; + let mut sim = Sim::new(nodes); + let mut links = Vec::new(); + for a in 0..RELAYS { + for b in a + 1..RELAYS { + if b == a + 1 || rng.chance(40) { + sim.join(a, b, 1 + rng.next(3)); + links.push((a, b)); + } + } + } + let ids = [RELAYS as u64, RELAYS as u64 + 1]; + for origin in RELAYS..nodes { + let relay = rng.next(RELAYS as u64) as usize; + sim.join(origin, relay, 1); + links.push((relay, origin)); + sim.originate(origin, origin as u64, 0); + } + assert!(sim.run()); + + for _ in 0..60 { + match rng.next(8) { + 0 => { + let (a, b) = links[rng.next(links.len() as u64) as usize]; + if sim.connected(a, b) { + sim.disconnect(a, b); + } else { + sim.connect(a, b); + } + } + 1 | 2 => { + let (a, b) = links[rng.next(links.len() as u64) as usize]; + let (from, to) = if rng.chance(50) { (a, b) } else { (b, a) }; + if rng.chance(60) { + sim.hold(from, to); + } else { + sim.release(from, to); + } + } + 3 => { + let busy = sim.busy(); + if !busy.is_empty() { + let (from, to) = busy[rng.next(busy.len() as u64) as usize]; + sim.deliver(from, to); + } + } + 4 => sim.bump(RELAYS + rng.next(2) as usize), + _ => { + let node = rng.next(RELAYS as u64) as usize; + let id = ids[rng.next(2) as usize]; + if !sim.holds(node, id) { + sim.collect(node, id); + collected += 1; + } + } + } + sim.settle(1); + for &id in &ids { + assert_loop_free(&sim, nodes, id); + } + } + } + assert!(collected > 0); +} + +/// Babel's feasibility needs every link to increase the metric, so a +/// zero-cost link is refused when it is configured. +#[test] +fn zero_cost_links_are_refused() { + let mut router: Router = Router::new(1); + let link = Link { + cost: 0, + ..Link::default() + }; + assert_eq!(router.connect(1, link), Err(Error::ZeroCost)); +} + +/// A total reaching the infinite metric is unreachable, never saturated, and +/// not waiting on a seqno. +#[test] +fn overflow_is_unreachable() { + let (origin, a, b) = (0, 1, 2); + let mut sim = Sim::new(3); + sim.join(origin, a, MAX - 1); + sim.join(a, b, 5); + sim.originate(origin, 7, 0); + assert!(sim.run()); + assert_eq!(sim.route(a, 7).map(|route| route.metric), Some(MAX - 1)); + assert!(sim.route(b, 7).is_none()); + assert!(!sim.starved(b, 7), "unreachable, not waiting on a seqno"); +} + +/// Redundant cores are feasible successors: losing one fails over at once, +/// with no seqno request. +#[test] +fn redundant_cores_fail_over_without_a_seqno_request() { + let (ingest, core_a, core_b, edge) = (0, 1, 2, 3); + let mut sim = Sim::new(4); + sim.join(ingest, core_a, 1); + sim.join(ingest, core_b, 1); + sim.join(core_a, core_b, 1); + sim.link(edge, core_a, SimLink::cost(1).upstream(), SimLink::cost(1)); + sim.link(edge, core_b, SimLink::cost(1).upstream(), SimLink::cost(1)); + sim.originate(ingest, 7, 0); + assert!(sim.run()); + let primary = next(&sim, edge, 7).unwrap(); + + let requests = sim.stats.requests; + sim.disconnect(edge, primary); + assert!( + sim.route(edge, 7).is_some(), + "the other core advertises no more than the lost one" + ); + assert!(sim.run()); + assert_eq!(sim.stats.requests, requests); +} + +/// A publisher reached directly over P2P and through the CDN. A browser +/// advertises nothing it learned, so losing the P2P session falls back at +/// once. A routing node advertised the one-hop route, so the CDN detour is +/// infeasible until its seqno request reaches the publisher. +#[test] +fn p2p_fallback_is_immediate_for_leaves_and_needs_a_seqno_for_transit() { + for transit in [false, true] { + let (publisher, edge, subscriber) = (0, 1, 2); + let mut sim = Sim::new(3); + sim.leaf(publisher); + if !transit { + sim.leaf(subscriber); + } + sim.join(publisher, edge, 1); + sim.link(subscriber, edge, SimLink::cost(1).upstream(), SimLink::cost(1)); + sim.link(subscriber, publisher, SimLink::cost(1).priority(1), SimLink::cost(1)); + sim.originate(publisher, 9, 0); + assert!(sim.run()); + assert_eq!(next(&sim, subscriber, 9), Some(publisher)); + + sim.disconnect(subscriber, publisher); + assert_eq!(sim.starved(subscriber, 9), transit); + assert!(sim.run()); + assert_eq!(next(&sim, subscriber, 9), Some(edge)); + assert_eq!(sim.seqno(publisher), u64::from(transit)); + } +} + +/// A relay whose only peers are upstream sends no update for its down-only +/// route, so it holds no feasibility distance and takes a longer route at the +/// same seqno once its best uplink goes. +#[test] +fn feasibility_tracks_only_updates_actually_sent() { + let (relay, c1, c2, origin) = (0, 1, 2, 3); + let mut sim = Sim::new(4); + sim.link(relay, c1, SimLink::cost(1).upstream(), SimLink::cost(1)); + sim.link(relay, c2, SimLink::cost(1).upstream(), SimLink::cost(1)); + sim.leaf(origin); + sim.join(origin, c1, 1); + sim.join(origin, c2, 5); + sim.originate(origin, 7, 0); + assert!(sim.run()); + assert_eq!(next(&sim, relay, 7), Some(c1)); + assert!(!sim.remembers(relay, 7)); + + sim.disconnect(relay, c1); + assert_eq!(next(&sim, relay, 7), Some(c2)); + assert!(sim.run()); + assert_eq!(sim.seqno(origin), 0); +} + +/// A GOAWAY ranks the peer below every eligible peer, priority included, and +/// the edge fails over at once to the other core. The draining core stays a +/// last resort while its session is up. +#[test] +fn goaway_fails_over_and_keeps_the_draining_peer_last() { + let (ingest, core_a, core_b, edge) = (0, 1, 2, 3); + let mut sim = Sim::new(4); + sim.join(ingest, core_a, 1); + sim.join(ingest, core_b, 1); + sim.link(edge, core_a, SimLink::cost(1).upstream().priority(1), SimLink::cost(1)); + sim.link(edge, core_b, SimLink::cost(1).upstream(), SimLink::cost(1)); + sim.originate(ingest, 7, 0); + assert!(sim.run()); + assert_eq!(next(&sim, edge, 7), Some(core_a)); + + sim.goaway(core_a); + assert!(sim.run()); + assert_eq!(next(&sim, edge, 7), Some(core_b)); + + sim.disconnect(edge, core_b); + assert!(sim.run()); + assert_eq!( + next(&sim, edge, 7), + Some(core_a), + "the draining core is the last resort" + ); +} + +/// An edge serving a client through two uplinks, where the second uplink's +/// route is longer than the one the edge advertised. A GOAWAY keeps the +/// draining uplink serving while the alternative is infeasible. The refresh +/// reaches the edge through both uplinks: if the alternative's arrives first +/// the edge moves; if the draining uplink's does, the edge keeps it until its +/// session closes and then waits for one more refresh, which its request asks +/// for. +#[test] +fn goaway_keeps_serving_until_a_refresh_makes_the_alternative_feasible() { + let (origin, a, b, edge, client) = (0, 1, 2, 3, 4); + let build = || { + let mut sim = Sim::new(5); + sim.leaf(origin); + sim.leaf(client); + sim.join(origin, a, 1); + sim.join(origin, b, 5); + sim.link(edge, a, SimLink::cost(1).upstream(), SimLink::cost(1)); + sim.link(edge, b, SimLink::cost(1).upstream(), SimLink::cost(1)); + sim.join(edge, client, 1); + sim.originate(origin, 7, 0); + assert!(sim.run()); + sim + }; + + for draining_first in [false, true] { + let mut sim = build(); + sim.goaway(a); + assert!(sim.run()); + assert_eq!( + next(&sim, edge, 7), + Some(a), + "b advertised 5, above the edge's distance of 2" + ); + assert!(sim.route(client, 7).is_some()); + + let late = if draining_first { b } else { a }; + sim.hold(late, edge); + sim.bump(origin); + assert!(sim.run()); + sim.release(late, edge); + assert!(sim.run()); + assert!(sim.route(client, 7).is_some()); + if draining_first { + assert_eq!( + next(&sim, edge, 7), + Some(a), + "the refresh through a lowered the distance first" + ); + sim.disconnect(edge, a); + assert!(sim.starved(edge, 7), "only another refresh moves the edge to b"); + assert!(sim.run()); + assert_eq!(sim.seqno(origin), 2, "the edge's request asked for it"); + } + assert_eq!(next(&sim, edge, 7), Some(b)); + } +} + +/// Two CDN relays, each the uplink of one relay in a local mesh, reaching a +/// CDN origin. The CDN admits each local relay under `grant`. +fn two_uplinks(l2_marks_upstream: bool, grant: Option) -> Sim { + let (c1, c2, l1, l2, origin) = (0, 1, 2, 3, 4); + let mut sim = Sim::new(5); + sim.join(c1, c2, 10); + sim.join(l1, l2, 1); + let admit = SimLink { + grant, + ..SimLink::cost(1) + }; + let up = if l2_marks_upstream { + SimLink::cost(1).upstream() + } else { + SimLink::cost(1) + }; + sim.link(l1, c1, SimLink::cost(1).upstream(), admit); + sim.link(l2, c2, up, admit); + sim.leaf(origin); + sim.join(origin, c1, 1); + sim.originate(origin, 7, 0); + sim +} + +/// The down-only bit keeps the CDN on its own link while the local relays +/// still reach the CDN origin. +#[test] +fn two_uplinks_never_carry_cdn_transit() { + let (c1, c2, l2) = (0, 1, 3); + let mut sim = two_uplinks(true, None); + assert!(sim.run()); + assert_eq!(next(&sim, c2, 7), Some(c1)); + assert!(sim.route(l2, 7).is_some()); + assert!(sim.route(l2, 7).unwrap().down_only); +} + +/// A local relay whose preferred route is down-only exports nothing, so the +/// CDN hauls the origin across its expensive internal link while a LAN path +/// sits unadvertised. Ranking local peers above a backup uplink avoids it. +/// Babel selects only feasible routes, so a transit relay's priority for a +/// longer route holds only if that route arrived before the shorter one was +/// advertised. +#[test] +fn preferred_down_only_route_hides_the_exportable_alternative() { + let (c1, c2, l1, l2, origin) = (0, 1, 2, 3, 4); + let build = |lan_first: bool, prefer_uplink: bool| { + let mut sim = Sim::new(5); + sim.join(c1, c2, 10); + sim.leaf(origin); + sim.join(origin, l1, 1); + sim.originate(origin, 7, 0); + let (uplink, lan) = if prefer_uplink { (1, 0) } else { (0, 1) }; + let join_lan = |sim: &mut Sim| sim.link(l2, l1, SimLink::cost(1).priority(lan), SimLink::cost(1)); + if lan_first { + join_lan(&mut sim); + } + sim.link(l1, c1, SimLink::cost(1).upstream(), SimLink::cost(1)); + sim.link(l2, c2, SimLink::cost(1).upstream().priority(uplink), SimLink::cost(1)); + assert!(sim.run()); + if !lan_first { + join_lan(&mut sim); + } + assert!(sim.run()); + sim + }; + + let sim = build(false, true); + assert_eq!(next(&sim, l2, 7), Some(c2)); + assert!(sim.export_route(l2, 7).is_none()); + assert_eq!(next(&sim, c2, 7), Some(c1)); + assert_eq!(sim.route(c2, 7).map(|route| route.metric), Some(12)); + + let sim = build(false, false); + assert_eq!(next(&sim, l2, 7), Some(l1)); + assert_eq!(next(&sim, c2, 7), Some(l2)); + + // Arrival order decides the transit priority's outcome. + assert_eq!(next(&build(true, true), l2, 7), Some(l1)); +} + +/// A scoped peer forwards a victim's identity with a newer seqno. Unscoped, +/// the forgery captures the victim's route. Filed under a qualified key, it is +/// a separate identity, and the victim's route is untouched. A scoped peer's +/// own identity is qualified too, since nothing proves it. +#[test] +fn forged_identity_is_filed_apart_under_a_scoped_grant() { + for scoped in [false, true] { + let (victim, core, edge, forger) = (0, 1, 2, 3); + let mut sim = Sim::new(4); + sim.leaf(victim); + sim.leaf(forger); + sim.join(victim, core, 5); + sim.join(core, edge, 1); + let grant = Grant { + subject: "forger", + scope: "forger/", + }; + let admission = if scoped { + SimLink::cost(1).grant(grant) + } else { + SimLink::cost(1) + }; + sim.link(edge, forger, admission, SimLink::cost(1)); + sim.originate(victim, 7, 0); + sim.originate(forger, 3, 0); + assert!(sim.run()); + + sim.forge(forger, 7, 1000); + assert!(sim.run()); + let own = if scoped { qualified("forger", 3) } else { 3 }; + assert!(sim.route(core, own).is_some()); + assert_eq!( + sim.route(core, 3).is_some(), + !scoped, + "an unproven own identity is qualified" + ); + if scoped { + assert_eq!(next(&sim, core, 7), Some(victim)); + assert_eq!(next(&sim, core, qualified("forger", 7)), Some(edge)); + } else { + assert_eq!( + next(&sim, core, 7), + Some(edge), + "the forger captures the victim's route" + ); + } + + sim.kill(forger); + assert!(sim.run()); + assert_eq!(next(&sim, core, 7), Some(victim)); + } +} + +/// A local mesh publishing through two uplinks, each admitted under the same +/// subject. Both CDN relays file the local origin under one key, so it +/// survives losing an uplink; a retracted identity leaves no alias behind. +#[test] +fn scoped_delivery_through_two_uplinks_reaches_one_key() { + let (c1, c2, l1, l2, origin) = (0, 1, 2, 3, 4); + let mut sim = Sim::new(5); + sim.join(c1, c2, 10); + sim.join(l1, l2, 1); + let admit = SimLink::cost(1).grant(LOCAL); + sim.link(l1, c1, SimLink::cost(1).upstream(), admit); + sim.link(l2, c2, SimLink::cost(1).upstream(), admit); + sim.leaf(origin); + sim.join(origin, l1, 1); + sim.originate(origin, 9, 0); + sim.originate(l1, 20, 0); + assert!(sim.run()); + + let key = qualified(LOCAL.subject, 9); + assert!(sim.route(c1, 9).is_none()); + assert_eq!(next(&sim, c1, key), Some(l1)); + assert_eq!(next(&sim, c2, key), Some(l2)); + assert_eq!(next(&sim, c1, qualified(LOCAL.subject, 20)), Some(l1)); + + sim.disconnect(l2, c2); + assert!(sim.run()); + assert_eq!(next(&sim, c2, key), Some(c1)); + + assert_eq!(sim.aliases(c1), 2, "the origin's and the local relay's own"); + sim.kill(origin); + assert!(sim.run()); + assert_eq!(sim.aliases(c1), 1); +} + +/// A local relay missing its `upstream` mark leaks CDN routes back to the +/// CDN. Admitted under a scoped grant, the leaked identity is filed apart, so +/// the CDN keeps its own route. +#[test] +fn misconfigured_uplink_is_contained_by_qualified_identities() { + let (c1, c2, l2) = (0, 1, 3); + let mut sim = two_uplinks(false, None); + assert!(sim.run()); + assert_eq!(next(&sim, c2, 7), Some(l2)); + + let mut sim = two_uplinks(false, Some(LOCAL)); + assert!(sim.run()); + assert_eq!(next(&sim, c2, 7), Some(c1)); + assert_eq!( + next(&sim, c2, qualified(LOCAL.subject, 7)), + Some(l2), + "the leak is filed apart" + ); +} + +/// A qualified key the CDN sent down comes back up through the misconfigured +/// uplink. A scoped connection forwarding a qualified key has that route +/// refused, so the leak is filed once and never again. +#[test] +fn scoped_connections_cannot_requalify_a_qualified_identity() { + let c2 = 1; + let mut sim = two_uplinks(true, Some(LOCAL)); + assert!(sim.run()); + let marked = sim.identities(c2); + + let mut sim = two_uplinks(false, Some(LOCAL)); + assert!(sim.run()); + assert_eq!(sim.identities(c2), marked + 1, "the leak is filed once, never again"); +} + +/// A partitioned local mesh reaches its other half through the CDN once a +/// seqno request reaches the origin, and heals back onto the LAN. +#[test] +fn partitioned_local_mesh_reaches_itself_through_the_cdn() { + let (c1, c2, l1, l2, origin) = (0, 1, 2, 3, 4); + let mut sim = Sim::new(5); + sim.join(c1, c2, 10); + sim.join(l1, l2, 1); + sim.link(l1, c1, SimLink::cost(1).upstream(), SimLink::cost(1)); + sim.link(l2, c2, SimLink::cost(1).upstream(), SimLink::cost(1)); + sim.leaf(origin); + sim.join(origin, l2, 1); + sim.originate(origin, 7, 0); + assert!(sim.run()); + assert_eq!(next(&sim, l1, 7), Some(l2)); + + sim.disconnect(l1, l2); + assert!(sim.starved(l1, 7), "the CDN detour is longer than the LAN route"); + assert!(sim.run()); + assert_eq!(sim.seqno(origin), 1); + assert_eq!(next(&sim, l1, 7), Some(c1)); + + sim.connect(l1, l2); + assert!(sim.run()); + assert_eq!(next(&sim, l1, 7), Some(l2), "heals"); +} + +/// A five-node ring loses its preferred origin link while a longer backup +/// remains. The backup's route is infeasible, so the starved node's request, +/// with duplicates suppressed at every hop, reaches the origin while the +/// backup never loses its session. +#[test] +fn a_ring_recovers_its_longer_backup_through_one_request() { + let origin = 5; + let mut sim = Sim::new(6); + for relay in 0..5 { + sim.join(relay, (relay + 1) % 5, 1); + } + sim.leaf(origin); + sim.join(origin, 0, 1); + sim.join(origin, 2, 10); + sim.originate(origin, 7, 0); + assert!(sim.run()); + assert_eq!(next(&sim, 2, 7), Some(1)); + + let requests = sim.stats.requests; + sim.disconnect(origin, 0); + assert!(sim.run()); + for relay in 0..5 { + assert!(sim.route(relay, 7).is_some(), "relay {relay}"); + } + assert_eq!(sim.seqno(origin), 1); + assert!(sim.stats.requests > requests); + assert_eq!(sim.stats.repeated_requests, 0, "duplicates suppressed"); + assert_eq!(sim.retained().pending, 0); +} + +/// A request for an origin no usable route reaches stops once no usable +/// route is left, leaving nothing pending. +#[test] +fn an_unanswerable_request_stops() { + let origin = 4; + let mut sim = Sim::new(5); + for relay in 0..4 { + sim.join(relay, (relay + 1) % 4, 1); + } + sim.leaf(origin); + sim.join(origin, 0, 1); + sim.originate(origin, 7, 0); + assert!(sim.run()); + sim.disconnect(origin, 0); + assert!(sim.run()); + for relay in 0..4 { + assert!(sim.route(relay, 7).is_none()); + assert!(!sim.holds(relay, 7)); + } + assert_eq!(sim.retained().pending, 0); +} diff --git a/rs/moq-net/src/route/view.rs b/rs/moq-net/src/route/view.rs new file mode 100644 index 00000000000..e11b44a6d88 --- /dev/null +++ b/rs/moq-net/src/route/view.rs @@ -0,0 +1,220 @@ +//! An origin's claims as one node knows them, certified per region by floors. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::Path; + +use super::Rev; + +/// Whether `path` is `prefix` or lies below it, matching whole segments. +pub(crate) fn within(path: &Path, prefix: &Path) -> bool { + path.has_prefix(prefix) +} + +/// Whether a claim on `claim` can concern a path under `region`. +/// +/// Every claim covers its own path and everything below it, so a claim meets a +/// region when either lies within the other. +pub(crate) fn overlaps(claim: &Path, region: &Path) -> bool { + within(claim, region) || within(region, claim) +} + +/// What a node knows of one origin's claims. +/// +/// Each region's floor certifies that the claims overlapping it are exactly the +/// origin's claims there as of that revision. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct View { + pub floors: BTreeMap, + pub items: BTreeMap, +} + +impl Default for View { + fn default() -> Self { + Self { + floors: BTreeMap::new(), + items: BTreeMap::new(), + } + } +} + +impl View { + /// The view as a stream to a peer interested in `prefix` carries it. + /// + /// A region meeting the interest is narrowed to whichever of the two lies + /// within the other, so every region a stream carries lies within its + /// interest. + pub fn restrict(&self, prefix: &Path) -> Self { + let mut floors: BTreeMap = BTreeMap::new(); + for (region, &floor) in &self.floors { + let narrowed = if within(region, prefix) { + region + } else if within(prefix, region) { + prefix + } else { + continue; + }; + let entry = floors.entry(narrowed.clone()).or_insert(floor); + *entry = (*entry).max(floor); + } + // A claim stays only while a kept floor still certifies it. + let items = self + .items + .iter() + .filter(|(claim, _)| floors.keys().any(|region| overlaps(claim, region))) + .map(|(claim, item)| (claim.clone(), item.clone())) + .collect(); + Self { floors, items } + } + + /// The view restricted to the union of `regions`. + pub fn restrict_all(&self, regions: &[Path]) -> Self { + let mut out = Self::default(); + for region in regions { + out.merge(&self.restrict(region)); + } + out + } + + /// The newest revision this view knows `claim` at, if any region covers it. + pub fn as_of(&self, claim: &Path) -> Option { + self.floors + .iter() + .filter(|(region, _)| overlaps(claim, region)) + .map(|(_, floor)| *floor) + .max() + } + + /// Take `other`'s state for every claim it knows at a newer revision, then + /// raise the floors and drop any region a wider one at least as new covers. + pub fn merge(&mut self, other: &Self) { + let claims: BTreeSet = self.items.keys().chain(other.items.keys()).cloned().collect(); + for claim in claims { + if other.as_of(&claim) > self.as_of(&claim) { + match other.items.get(&claim) { + Some(item) => self.items.insert(claim, item.clone()), + None => self.items.remove(&claim), + }; + } + } + for (region, &floor) in &other.floors { + let entry = self.floors.entry(region.clone()).or_insert(floor); + *entry = (*entry).max(floor); + } + let floors = self.floors.clone(); + self.floors.retain(|region, floor| { + !floors + .iter() + .any(|(wider, newer)| wider != region && within(region, wider) && newer >= floor) + }); + } + + /// What a node keeps of its view once its interest is `watch`. + /// + /// A floor outside the interest may certify a claim above both at a newer + /// revision than any floor inside, so it stays, with what it certifies, + /// until a floor inside catches up; dropping it would leave that claim + /// known only at an older revision than the item it holds. Once no + /// neighbor certifies it, pruning lets it go. + pub fn retire(&self, watch: &Path) -> Self { + let mut kept = self.restrict(watch); + let newest = kept.floors.values().max().copied(); + for (region, &floor) in &self.floors { + if Some(floor) > newest && !kept.floors.contains_key(region) { + kept.floors.insert(region.clone(), floor); + for (claim, item) in &self.items { + if overlaps(claim, region) { + kept.items.insert(claim.clone(), item.clone()); + } + } + } + } + kept + } + + /// The changes that move a receiver from `old` to `self`: every changed or + /// added claim with its attributes, and every removed claim with `None`. + pub fn diff(&self, old: &Self) -> Vec<(Path, Option)> { + let mut out: Vec<(Path, Option)> = self + .items + .iter() + .filter(|(claim, item)| old.items.get(*claim) != Some(item)) + .map(|(claim, item)| (claim.clone(), Some(item.clone()))) + .collect(); + out.extend( + old.items + .keys() + .filter(|claim| !self.items.contains_key(*claim)) + .map(|claim| (claim.clone(), None)), + ); + out + } + + /// Apply a batch: its changes, then its floor list, which replaces the + /// last one. + pub fn apply(&mut self, changes: Vec<(Path, Option)>, floors: BTreeMap) { + for (claim, item) in changes { + match item { + Some(item) => self.items.insert(claim, item), + None => self.items.remove(&claim), + }; + } + self.floors = floors; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn p(s: &'static str) -> Path { + Path::from_static(s) + } + + fn view(floors: &[(&'static str, Rev)], items: &[(&'static str, u32)]) -> View { + View { + floors: floors.iter().map(|(r, f)| (p(r), *f)).collect(), + items: items.iter().map(|(c, i)| (p(c), *i)).collect(), + } + } + + #[test] + fn restrict_narrows_regions_to_the_interest() { + let v = view(&[("", 5)], &[("live/a", 1), ("live", 2), ("other/x", 3)]); + let r = v.restrict(&p("live/a")); + assert_eq!(r, view(&[("live/a", 5)], &[("live/a", 1), ("live", 2)])); + } + + #[test] + fn merge_keeps_the_newer_knowledge_per_claim() { + let mut a = view(&[("live", 3)], &[("live/x", 1)]); + let b = view(&[("live", 4)], &[]); + a.merge(&b); + assert_eq!(a, view(&[("live", 4)], &[])); + + // An older view revives nothing. + let c = view(&[("live", 2)], &[("live/x", 1)]); + a.merge(&c); + assert_eq!(a, view(&[("live", 4)], &[])); + } + + #[test] + fn merge_drops_a_region_a_wider_newer_floor_covers() { + let mut a = view(&[("live/a", 3)], &[]); + a.merge(&view(&[("", 4)], &[])); + assert_eq!(a.floors, view(&[("", 4)], &[]).floors); + } + + #[test] + fn diff_then_apply_round_trips() { + let old = view(&[("", 1)], &[("a", 1), ("b", 2)]); + let new = view(&[("", 2)], &[("a", 1), ("b", 3), ("c", 4)]); + let mut got = old.clone(); + got.apply(new.diff(&old), new.floors.clone()); + assert_eq!(got, new); + + let gone = view(&[("", 3)], &[("c", 4)]); + got.apply(gone.diff(&new), gone.floors.clone()); + assert_eq!(got, gone); + } +} From 740e4709ece32797ef051a0541dda8859bb5781d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 23:08:39 -0800 Subject: [PATCH 3/8] perf(net): recompute only the identity a route message names Per-identity state is keyed identity first, and each message updates only the identity it carries: its selection, feasibility distance, advertisements, seqno request, and reader view. A session event that touches every identity (a new interest, a closed session) still walks them all. A fixture pins one recomputed identity per message, and another shows a link flap behind a hundred broadcasts re-sends no claim. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/mod.rs | 521 +++++++++++++++----------- rs/moq-net/src/route/tests/handoff.rs | 50 +++ rs/moq-net/src/route/tests/mod.rs | 9 + rs/moq-net/src/route/tests/routing.rs | 30 ++ 4 files changed, 390 insertions(+), 220 deletions(-) diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index 00d713bcbb5..41d1bfe8fe4 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -50,6 +50,22 @@ pub(crate) type Rev = u64; /// advertised seqno leaves room for the successor a request names. pub(crate) const MAX: u64 = u64::MAX; +/// A neighbor key, bounded so per-identity state can be read as one range. +pub(crate) trait Key: Copy + Ord + fmt::Debug { + const MIN: Self; + const MAX: Self; +} + +impl Key for usize { + const MIN: Self = usize::MIN; + const MAX: Self = usize::MAX; +} + +impl Key for u64 { + const MIN: Self = u64::MIN; + const MAX: Self = u64::MAX; +} + /// One side's configuration of a link to a neighbor. #[derive(Clone)] pub(crate) struct Link { @@ -215,7 +231,7 @@ struct Heard { interest: u32, } -struct Neighbor { +struct Peer { link: Link, /// The neighbor sent GOAWAY: rank it below every other eligible route. draining: bool, @@ -245,18 +261,20 @@ pub(crate) struct Retained { } pub(crate) struct Router { - neighbors: BTreeMap, + neighbors: BTreeMap, origins: BTreeMap>, rx: BTreeMap>, + // Per-identity state is keyed identity first, so one identity's state is + // one range and a message costs only the identity it names. /// For a key this node qualified, the identity the neighbor named. - filed: BTreeMap<(N, Id), Id>, + filed: BTreeMap<(Id, N), Id>, /// Babel's feasibility distance per identity. fd: BTreeMap, - sent: BTreeMap<(N, Id), Reach>, + sent: BTreeMap<(Id, N), Reach>, /// ROUTE_ENDs sent to each neighbor and not yet echoed back. Until they all /// return, a neighbor may still hold this node's advertisement, so the /// feasibility distance stays. - unacked: BTreeMap<(N, Id), u32>, + unacked: BTreeMap<(Id, N), u32>, pending: BTreeMap>, /// The prefix this node asks its neighbors for announcements under. watch: Path, @@ -268,14 +286,17 @@ pub(crate) struct Router { /// the old interest, plus what each batch answering the new one certified. /// The rest is hidden until certified. current: BTreeMap>, - heard: BTreeMap<(N, Id), Heard>, - sent_views: BTreeMap<(N, Id), View>, + heard: BTreeMap<(Id, N), Heard>, + sent_views: BTreeMap<(Id, N), View>, shown: BTreeMap<(Id, Path), Shown>, /// Mints a fresh identity on rotation. rng: u64, /// Per claim this node has shown, how many times it forgot that claim. #[cfg(test)] forgot: BTreeMap<(Id, Path), u32>, + /// Identities recomputed, so fixtures can bound the work per message. + #[cfg(test)] + touched: usize, out: Vec>, events: Vec>, } @@ -291,7 +312,7 @@ pub(crate) enum Error { NoIdentity, } -impl Router { +impl Router { /// A node with no sessions, minting rotated identities from `seed`. pub fn new(seed: u64) -> Self { Self { @@ -313,6 +334,8 @@ impl Router { rng: seed | 1, #[cfg(test)] forgot: BTreeMap::new(), + #[cfg(test)] + touched: 0, out: Vec::new(), events: Vec::new(), } @@ -337,13 +360,13 @@ impl Router { } self.neighbors.insert( neighbor, - Neighbor { + Peer { link, draining: false, watch: None, }, ); - self.advertise(); + // Nothing is sent or learned until a stream opens. Ok(()) } @@ -353,19 +376,31 @@ impl Router { if self.neighbors.remove(&neighbor).is_none() { return; } - for routes in self.rx.values_mut() { - routes.remove(&neighbor); + let mut ids = BTreeSet::new(); + for (id, routes) in self.rx.iter_mut() { + if routes.remove(&neighbor).is_some() { + ids.insert(*id); + } } self.rx.retain(|_, routes| !routes.is_empty()); - self.filed.retain(|(from, _), _| *from != neighbor); - self.heard.retain(|(from, _), _| *from != neighbor); - for pending in self.pending.values_mut() { - pending.from.remove(&neighbor); + for (id, pending) in self.pending.iter_mut() { + if pending.from.remove(&neighbor) { + ids.insert(*id); + } } - self.unacked.retain(|(to, _), _| *to != neighbor); - self.sent.retain(|(to, _), _| *to != neighbor); - self.sent_views.retain(|(to, _), _| *to != neighbor); - self.advertise(); + let mut on = |(id, n): &(Id, N)| { + let on = *n == neighbor; + if on { + ids.insert(*id); + } + !on + }; + self.filed.retain(|key, _| on(key)); + self.heard.retain(|key, _| on(key)); + self.unacked.retain(|key, _| on(key)); + self.sent.retain(|key, _| on(key)); + self.sent_views.retain(|key, _| on(key)); + self.update(ids); } /// `neighbor` opened a stream asking for announcements under `prefix`: it @@ -373,8 +408,8 @@ impl Router { pub fn peer_interest(&mut self, neighbor: N, prefix: Path) -> Result<(), Error> { let peer = self.neighbors.get_mut(&neighbor).ok_or(Error::UnknownNeighbor)?; peer.watch = Some(prefix); - self.sent_views.retain(|(to, _), _| *to != neighbor); - self.advertise(); + self.sent_views.retain(|(_, to), _| *to != neighbor); + self.update_all(); Ok(()) } @@ -384,7 +419,13 @@ impl Router { if let Some(peer) = self.neighbors.get_mut(&neighbor) { peer.draining = true; } - self.advertise(); + let ids: Vec = self + .rx + .iter() + .filter(|(_, routes)| routes.contains_key(&neighbor)) + .map(|(id, _)| *id) + .collect(); + self.update(ids); } /// Ask every neighbor for announcements under `prefix` instead, returning @@ -401,7 +442,7 @@ impl Router { .retain(|_, regions| !regions.iter().any(|region| within(&prefix, region))); self.watch = prefix; self.interest += 1; - self.advertise(); + self.update_all(); self.interest } @@ -425,7 +466,7 @@ impl Router { view, }, ); - self.advertise(); + self.update([id]); } /// Advertise `id` as if serving it, with no claims: what a forwarder that @@ -440,7 +481,7 @@ impl Router { view: View::default(), }, ); - self.advertise(); + self.update([id]); } /// The identity this node declares. @@ -470,7 +511,7 @@ impl Router { self.rotate(id); } else { self.origins.get_mut(&id).expect("declared").seqno = seqno + 1; - self.advertise(); + self.update([id]); } Ok(()) } @@ -498,7 +539,7 @@ impl Router { Some(item) => view.items.insert(claim, item), None => view.items.remove(&claim), }; - self.advertise(); + self.update([id]); Ok(()) } @@ -536,7 +577,7 @@ impl Router { view, }, ); - self.advertise(); + self.update([old, id]); } // --- Received messages --- @@ -555,8 +596,8 @@ impl Router { }; if id != named { match reach { - Some(_) => self.filed.insert((from, id), named), - None => self.filed.remove(&(from, id)), + Some(_) => self.filed.insert((id, from), named), + None => self.filed.remove(&(id, from)), }; } let routes = self.rx.entry(id).or_default(); @@ -576,7 +617,7 @@ impl Router { if routes.is_empty() { self.rx.remove(&id); } - self.advertise(); + self.update([id]); } /// A batch of announcement changes for `named` from `from`, closed by its @@ -600,7 +641,7 @@ impl Router { return; }; let scope = peer.link.scope.as_ref().map(|scope| scope.regions.clone()); - let heard = self.heard.entry((from, id)).or_insert_with(|| Heard { + let heard = self.heard.entry((id, from)).or_insert_with(|| Heard { raw: View::default(), view: View::default(), interest, @@ -640,7 +681,7 @@ impl Router { self.forgot_outside(id, &watch); } } - self.advertise(); + self.update([id]); } /// ROUTE_REQUEST from `from` for `id`, as this node advertised it. @@ -656,8 +697,8 @@ impl Router { } } else { origin.seqno = origin.seqno.max(seqno); + self.update([id]); } - self.advertise(); return; } if self.selected(id).is_some_and(|route| route.seqno >= seqno) { @@ -674,33 +715,39 @@ impl Router { pending.asked = None; } pending.from.insert(from); - self.advertise(); + self.update([id]); } /// The echo of a ROUTE_END this node sent `from` for `id`. pub fn echo(&mut self, from: N, id: Id) { - let Some(left) = self.unacked.get_mut(&(from, id)) else { + let Some(left) = self.unacked.get_mut(&(id, from)) else { // Unsolicited: nothing of ours to drain. return; }; *left -= 1; if *left == 0 { - self.unacked.remove(&(from, id)); + self.unacked.remove(&(id, from)); } - self.advertise(); + self.update([id]); } /// Forget `id`'s routes and announcements. The feasibility distance goes on /// its own once drained. pub fn collect(&mut self, id: Id) { self.rx.remove(&id); - self.filed.retain(|(_, key), _| *key != id); + let filed: Vec<_> = Self::of(&self.filed, id).map(|(key, _)| *key).collect(); + for key in filed { + self.filed.remove(&key); + } self.views.remove(&id); self.current.remove(&id); - self.heard.retain(|(_, key), _| *key != id); + let heard: Vec<_> = Self::of(&self.heard, id).map(|(key, _)| *key).collect(); + for key in heard { + self.heard.remove(&key); + } self.pending.remove(&id); self.forgot_outside(id, &[]); - self.advertise(); + self.update([id]); } // --- Queries --- @@ -798,6 +845,12 @@ impl Router { .map(|(id, view)| (*id, view)) } + /// How many identities this node has recomputed. + #[cfg(test)] + pub fn touched(&self) -> usize { + self.touched + } + /// What this node's reader sees. pub fn shown(&self) -> &BTreeMap<(Id, Path), Shown> { &self.shown @@ -847,7 +900,7 @@ impl Router { /// The identity this node names for `id` in a request to `next`: the one /// `next` advertised, if this node qualified it. fn name(&self, next: N, id: Id) -> Id { - self.filed.get(&(next, id)).copied().unwrap_or(id) + self.filed.get(&(id, next)).copied().unwrap_or(id) } /// Babel's feasibility condition against this node's feasibility distance. @@ -916,7 +969,7 @@ impl Router { for next in self.certifiers(id) { let heard = self .heard - .get(&(next, id)) + .get(&(id, next)) .filter(|heard| heard.interest == self.interest)?; if held || !routes[&next].held { regions.extend(heard.view.floors.keys().cloned()); @@ -957,7 +1010,7 @@ impl Router { } else { let mut rebuilt = View::default(); for next in self.certifiers(id) { - rebuilt.merge(&self.heard[&(next, id)].view); + rebuilt.merge(&self.heard[&(id, next)].view); } (rebuilt, Vec::new()) }; @@ -986,184 +1039,205 @@ impl Router { /// Whether this node sends `id` to `neighbor` at all: its own identities to /// everyone, learned ones only to a proxied neighbor. - fn serves(&self, neighbor: &Neighbor, id: Id) -> bool { + fn serves(&self, neighbor: &Peer, id: Id) -> bool { neighbor.link.proxy || self.origins.contains_key(&id) } - /// Recompute every selection, record feasibility distances for what this - /// node advertises, and queue any change for each neighbor. - fn advertise(&mut self) { - let ids: Vec = self.views.keys().copied().collect(); - for id in ids { - self.prune(id); - } + /// One identity's entries in a map keyed identity first. + fn of(map: &BTreeMap<(Id, N), V>, id: Id) -> impl Iterator { + map.range((id, N::MIN)..=(id, N::MAX)) + } + + /// Every identity this node keeps any state for. + fn ids(&self) -> BTreeSet { let mut ids: BTreeSet = self.rx.keys().copied().collect(); ids.extend(self.origins.keys()); - ids.extend(self.sent.keys().map(|(_, id)| *id)); + ids.extend(self.fd.keys()); + ids.extend(self.views.keys()); + ids.extend(self.pending.keys()); + ids.extend(self.sent.keys().map(|(id, _)| *id)); + ids.extend(self.unacked.keys().map(|(id, _)| *id)); + ids.extend(self.heard.keys().map(|(id, _)| *id)); + ids.extend(self.shown.keys().map(|(id, _)| *id)); + ids + } + + /// Recompute every identity: for a change that touches them all, such as a + /// new interest. + fn update_all(&mut self) { + let ids = self.ids(); + self.update(ids); + } + + /// Recompute each of `ids`: its selection, the feasibility distance for what + /// this node advertises, what each neighbor is sent, its seqno request, and + /// what the reader sees. + fn update(&mut self, ids: impl IntoIterator) { let peers: Vec = self .neighbors .iter() .filter(|(_, peer)| peer.watch.is_some()) .map(|(n, _)| *n) .collect(); - + let ids: BTreeSet = ids.into_iter().collect(); + #[cfg(test)] + { + self.touched += ids.len(); + } for id in ids { - let selected = self.selected(id); - // The feasibility distance tracks finite updates actually sent (RFC 8966 s3.7.3). - if let Some(selected) = selected - && peers.iter().any(|peer| { - let peer = &self.neighbors[peer]; - self.serves(peer, id) && !(peer.link.upstream && selected.down_only) - }) { - self.record(id, selected); - } - // Recording may have tightened the feasibility distance: select again. - let route = self.selected(id); - let live = self.live(id, route); - let adopted = self.adopted(id, route).cloned(); - // A sender that certifies anything answers every stream, even one - // whose interest none of its regions meets, so the receiver knows it - // certifies nothing there. - let answers = adopted.as_ref().is_some_and(|view| !view.floors.is_empty()); - let mut restricted: Vec<(Path, View)> = Vec::new(); - - for &to in &peers { - let peer = &self.neighbors[&to]; - let serves = self.serves(peer, id); - let was = self.sent.get(&(to, id)).copied(); - if !serves && was.is_none() { - continue; - } - let reach = match selected { - Some(route) if serves && !(peer.link.upstream && route.down_only) => Some(Reach::Finite(Advert { - seqno: route.seqno, - metric: route.metric, - down_only: route.down_only, - })), - None if serves && live && was.is_some() => Some(Reach::Held), - _ => None, - }; - let watch = peer.watch.clone().expect("a watching peer"); - if was != reach { - match reach { - Some(reach) => self.sent.insert((to, id), reach), - None => { - *self.unacked.entry((to, id)).or_default() += 1; - self.sent.remove(&(to, id)) - } - }; - self.out.push(Out::Route { to, id, reach }); - } - // Announcements ride on the route and end with it. - if reach.is_none() { - self.sent_views.remove(&(to, id)); - continue; - } - // Peers mostly share an interest, so restrict once per interest. - let index = match restricted.iter().position(|(interest, _)| *interest == watch) { - Some(index) => index, - None => { - let view = adopted - .as_ref() - .map_or_else(View::default, |view| view.restrict(&watch)); - restricted.push((watch, view)); - restricted.len() - 1 - } - }; - let view = &restricted[index].1; - let changes = match self.sent_views.get(&(to, id)) { - Some(sent) if sent == view => continue, - Some(sent) => view.diff(sent), - None if !answers => continue, - None => view.diff(&View::default()), - }; - self.sent_views.insert((to, id), view.clone()); - self.out.push(Out::Batch { - to, - id, - changes, - floors: view.floors.clone(), - }); + self.prune(id); + self.advertise(id, &peers); + // Drop a feasibility distance once nothing here routes or advertises + // the identity and every neighbor echoed its ROUTE_END: no node then + // holds an advertisement from this one, so no older one can return as + // a loop. + if !self.rx.contains_key(&id) + && !self.origins.contains_key(&id) + && Self::of(&self.sent, id).next().is_none() + && Self::of(&self.unacked, id).next().is_none() + { + self.fd.remove(&id); } + self.requests(id); + self.show(id); } - // Drop a feasibility distance once nothing here routes or advertises the - // identity and every neighbor echoed its ROUTE_END: no node then holds an - // advertisement from this one, so no older one can return as a loop. - let (rx, origins, sent, unacked) = (&self.rx, &self.origins, &self.sent, &self.unacked); - self.fd.retain(|id, _| { - rx.contains_key(id) - || origins.contains_key(id) - || sent.keys().any(|(_, sent)| sent == id) - || unacked.keys().any(|(_, unacked)| unacked == id) - }); - self.requests(); - self.show(); } - /// Start, resend, or clear this node's seqno requests. - fn requests(&mut self) { - let mut ids: BTreeSet = self.pending.keys().copied().collect(); - ids.extend(self.rx.keys()); - for id in ids { - if self.origins.contains_key(&id) { - self.pending.remove(&id); + /// Queue any change to what each watching neighbor is sent for `id`. + fn advertise(&mut self, id: Id, peers: &[N]) { + let selected = self.selected(id); + // The feasibility distance tracks finite updates actually sent (RFC 8966 s3.7.3). + if let Some(selected) = selected + && peers.iter().any(|peer| { + let peer = &self.neighbors[peer]; + self.serves(peer, id) && !(peer.link.upstream && selected.down_only) + }) { + self.record(id, selected); + } + // Recording may have tightened the feasibility distance: select again. + let route = self.selected(id); + let live = self.live(id, route); + let adopted = self.adopted(id, route).cloned(); + // A sender that certifies anything answers every stream, even one whose + // interest none of its regions meets, so the receiver knows it certifies + // nothing there. + let answers = adopted.as_ref().is_some_and(|view| !view.floors.is_empty()); + let mut restricted: Vec<(Path, View)> = Vec::new(); + + for &to in peers { + let peer = &self.neighbors[&to]; + let serves = self.serves(peer, id); + let was = self.sent.get(&(id, to)).copied(); + if !serves && was.is_none() { continue; } - let route = self.selected(id); - let starved = route.is_none() && self.infeasible(id).next().is_some(); - if starved { - let want = self.fd[&id].0 + 1; - let pending = self.pending.entry(id).or_insert(Pending { - seqno: want, - from: BTreeSet::new(), - asked: None, - }); - if pending.seqno < want { - pending.seqno = want; - pending.asked = None; - } - } - let Some(pending) = self.pending.get(&id) else { - continue; + let reach = match selected { + Some(route) if serves && !(peer.link.upstream && route.down_only) => Some(Reach::Finite(Advert { + seqno: route.seqno, + metric: route.metric, + down_only: route.down_only, + })), + None if serves && live && was.is_some() => Some(Reach::Held), + _ => None, }; - let answered = route.is_some_and(|route| route.seqno >= pending.seqno); - if answered || (pending.from.is_empty() && !starved) { - self.pending.remove(&id); - continue; + let watch = peer.watch.clone().expect("a watching peer"); + if was != reach { + match reach { + Some(reach) => self.sent.insert((id, to), reach), + None => { + *self.unacked.entry((id, to)).or_default() += 1; + self.sent.remove(&(id, to)) + } + }; + self.out.push(Out::Route { to, id, reach }); } - // A request follows the next hop even back through a requester: next - // hops form no loop, one pending request per hop stops a bounce, and - // no requester ever cancels, so excluding one can strand a request - // while every route still runs through it. - let Some(target) = self.target(id) else { - self.pending.remove(&id); + // Announcements ride on the route and end with it. + if reach.is_none() { + self.sent_views.remove(&(id, to)); continue; + } + // Peers mostly share an interest, so restrict once per interest. + let index = match restricted.iter().position(|(interest, _)| *interest == watch) { + Some(index) => index, + None => { + let view = adopted + .as_ref() + .map_or_else(View::default, |view| view.restrict(&watch)); + restricted.push((watch, view)); + restricted.len() - 1 + } }; - let pending = self.pending.get_mut(&id).expect("pending"); - if pending.asked != Some(target) { - pending.asked = Some(target); - let seqno = pending.seqno; - let named = self.name(target, id); - self.out.push(Out::Request { - to: target, - id: named, - seqno, - }); + let view = &restricted[index].1; + let changes = match self.sent_views.get(&(id, to)) { + Some(sent) if sent == view => continue, + Some(sent) => view.diff(sent), + None if !answers => continue, + None => view.diff(&View::default()), + }; + self.sent_views.insert((id, to), view.clone()); + self.out.push(Out::Batch { + to, + id, + changes, + floors: view.floors.clone(), + }); + } + } + + /// Start, resend, or clear this node's seqno request for `id`. + fn requests(&mut self, id: Id) { + if self.origins.contains_key(&id) { + self.pending.remove(&id); + return; + } + let route = self.selected(id); + let starved = route.is_none() && self.infeasible(id).next().is_some(); + if starved { + let want = self.fd[&id].0 + 1; + let pending = self.pending.entry(id).or_insert(Pending { + seqno: want, + from: BTreeSet::new(), + asked: None, + }); + if pending.seqno < want { + pending.seqno = want; + pending.asked = None; } } + let Some(pending) = self.pending.get(&id) else { + return; + }; + let answered = route.is_some_and(|route| route.seqno >= pending.seqno); + if answered || (pending.from.is_empty() && !starved) { + self.pending.remove(&id); + return; + } + // A request follows the next hop even back through a requester: next hops + // form no loop, one pending request per hop stops a bounce, and no + // requester ever cancels, so excluding one can strand a request while every + // route still runs through it. + let Some(target) = self.target(id) else { + self.pending.remove(&id); + return; + }; + let pending = self.pending.get_mut(&id).expect("pending"); + if pending.asked != Some(target) { + pending.asked = Some(target); + let seqno = pending.seqno; + let named = self.name(target, id); + self.out.push(Out::Request { + to: target, + id: named, + seqno, + }); + } } - /// Recompute what this node's reader sees and log each change. - fn show(&mut self) { - let mut ids: BTreeSet = self.views.keys().copied().collect(); - ids.extend(self.origins.keys()); - ids.extend(self.heard.keys().map(|(_, id)| *id)); + /// Recompute what this node's reader sees of `id` and log each change. + fn show(&mut self, id: Id) { let mut shown = BTreeMap::new(); - for id in ids { - let route = self.selected(id); - let Some(view) = self.adopted(id, route) else { - continue; - }; + let route = self.selected(id); + if let Some(view) = self.adopted(id, route) { let usable = route.is_some(); let own = self.origins.contains_key(&id); // A claim is available only through a certifier that is not held: @@ -1181,7 +1255,7 @@ impl Router { .is_none_or(|regions| regions.iter().any(|region| overlaps(claim, region))); if overlaps(claim, &self.watch) && certified { shown.insert( - (id, claim.clone()), + claim.clone(), Shown { item: item.clone(), available, @@ -1190,42 +1264,49 @@ impl Router { } } } - let old = std::mem::take(&mut self.shown); - for ((id, claim), now) in &shown { - match old.get(&(*id, claim.clone())) { - None => self.log(*id, claim, Kind::Start, &now.item), - Some(was) => { + let old: Vec<(Path, Shown)> = self + .shown + .range((id, Path::empty())..) + .take_while(|((key, _), _)| *key == id) + .map(|((_, claim), shown)| (claim.clone(), shown.clone())) + .collect(); + for (claim, was) in &old { + match shown.get(claim) { + Some(now) => { if was.item != now.item { - self.log(*id, claim, Kind::Update, &now.item); + self.log(id, claim, Kind::Update, &now.item); } if was.available != now.available { let kind = match now.available { true => Kind::Available, false => Kind::Unavailable, }; - self.log(*id, claim, kind, &now.item); + self.log(id, claim, kind, &now.item); } } + None => { + // A claim no routed neighbor's region covers was let go, not ended. + let uncertified = self + .certified(id, true) + .is_some_and(|regions| !regions.iter().any(|region| overlaps(claim, region))); + let kind = if !overlaps(claim, &self.watch) { + Kind::Unwatched + } else if self.live(id, route) && !uncertified { + Kind::End + } else { + Kind::Offline + }; + self.log(id, claim, kind, &was.item); + self.shown.remove(&(id, claim.clone())); + } } } - for ((id, claim), was) in &old { - if shown.contains_key(&(*id, claim.clone())) { - continue; + for (claim, now) in shown { + if !old.iter().any(|(known, _)| *known == claim) { + self.log(id, &claim, Kind::Start, &now.item); } - // A claim no routed neighbor's region covers was let go, not ended. - let uncertified = self - .certified(*id, true) - .is_some_and(|regions| !regions.iter().any(|region| overlaps(claim, region))); - let kind = if !overlaps(claim, &self.watch) { - Kind::Unwatched - } else if self.live(*id, self.selected(*id)) && !uncertified { - Kind::End - } else { - Kind::Offline - }; - self.log(*id, claim, kind, &was.item); + self.shown.insert((id, claim), now); } - self.shown = shown; } fn log(&mut self, id: Id, claim: &Path, kind: Kind, item: &A) { diff --git a/rs/moq-net/src/route/tests/handoff.rs b/rs/moq-net/src/route/tests/handoff.rs index a150071f4a5..4ba49e688dc 100644 --- a/rs/moq-net/src/route/tests/handoff.rs +++ b/rs/moq-net/src/route/tests/handoff.rs @@ -1244,3 +1244,53 @@ fn settled(sim: &mut Sim, label: &str) { } assert_eq!(sim.retained().pending, 0, "{label}"); } + +/// A link flap with many broadcasts behind one origin costs ROUTE changes +/// only: while the link is down no session re-sends a claim and the client +/// sees no claim end or restart, and reopening the link costs one snapshot per +/// direction of that session. +#[test] +fn a_link_flap_costs_routes_not_broadcasts() { + const CLAIMS: usize = 100; + let (origin, a, b, c, client) = (0, 1, 2, 3, 4); + let mut sim = Sim::new(5); + sim.leaf(origin); + sim.leaf(client); + sim.join(origin, a, 1); + sim.join(a, b, 1); + sim.join(a, c, 1); + sim.join(b, c, 1); + sim.join(client, c, 1); + sim.originate(origin, 7, 0); + for claim in 0..CLAIMS { + sim.announce(origin, &format!("live/{claim}"), 0, None); + } + assert!(sim.run()); + assert_eq!(sim.shown(client).len(), CLAIMS); + + let (stats, events) = (sim.stats, sim.events.len()); + sim.disconnect(a, c); + assert!(sim.run()); + assert_eq!(next_hop(&sim, c), Some(b)); + assert!(sim.stats.route_msgs > stats.route_msgs); + assert_eq!(sim.stats.claim_items, stats.claim_items, "no claim travels again"); + let churn = sim.events[events..] + .iter() + .filter(|event| event.node == client && !matches!(event.kind, Kind::Available | Kind::Unavailable)) + .count(); + assert_eq!(churn, 0, "the client keeps every claim"); + assert!(sim.shown(client).values().all(|shown| shown.available)); + + let stats = sim.stats; + sim.connect(a, c); + assert!(sim.run()); + assert_eq!( + sim.stats.claim_items - stats.claim_items, + 2 * CLAIMS, + "one snapshot each way" + ); +} + +fn next_hop(sim: &Sim, node: Node) -> Option { + sim.route(node, 7).and_then(|route| route.next) +} diff --git a/rs/moq-net/src/route/tests/mod.rs b/rs/moq-net/src/route/tests/mod.rs index a49ae20ef71..cb58f130467 100644 --- a/rs/moq-net/src/route/tests/mod.rs +++ b/rs/moq-net/src/route/tests/mod.rs @@ -128,6 +128,8 @@ pub(super) struct Stats { pub requests: usize, /// Requests a node sent the same neighbor again for the same seqno. pub repeated_requests: usize, + /// Messages and echoes delivered. + pub delivered: usize, } struct Change { @@ -390,6 +392,7 @@ impl Sim { } => router.batch(from, id, changes, floors, interest), Msg::Request { id, seqno } => router.request(from, id, seqno), } + self.stats.delivered += 1; self.flush(); true } @@ -400,6 +403,7 @@ impl Sim { return false; }; self.nodes[to].echo(from, id); + self.stats.delivered += 1; self.flush(); true } @@ -537,6 +541,11 @@ impl Sim { } } + /// Identities recomputed across every node. + pub fn touched(&self) -> usize { + self.nodes.iter().map(Router::touched).sum() + } + pub fn route(&self, node: Node, id: Id) -> Option> { self.nodes[node].selected(id) } diff --git a/rs/moq-net/src/route/tests/routing.rs b/rs/moq-net/src/route/tests/routing.rs index ac25fbd64f0..f31c49811f4 100644 --- a/rs/moq-net/src/route/tests/routing.rs +++ b/rs/moq-net/src/route/tests/routing.rs @@ -750,3 +750,33 @@ fn an_unanswerable_request_stops() { } assert_eq!(sim.retained().pending, 0); } + +/// Each message recomputes only the identity it names, so the work a message +/// costs does not grow with how many other origins a node routes to. +#[test] +fn a_message_recomputes_only_its_identity() { + for origins in [1, 50] { + let (a, b, client) = (0, 1, 2); + let mut sim = Sim::new(3 + origins); + sim.join(a, b, 1); + sim.leaf(client); + sim.join(client, b, 1); + for origin in 3..3 + origins { + sim.leaf(origin); + sim.join(origin, a, 1); + sim.originate(origin, origin as Id, 0); + sim.announce(origin, &format!("live/{origin}"), 0, None); + } + assert!(sim.run()); + assert_eq!(sim.shown(client).len(), origins); + + let (touched, delivered) = (sim.touched(), sim.stats.delivered); + sim.announce(3, "live/extra", 0, None); + assert!(sim.run()); + assert_eq!( + sim.touched() - touched, + sim.stats.delivered - delivered + 1, + "one identity per message, plus the origin's own change" + ); + } +} From 16ab2002857853b2f73467de677cc42392a78e51 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 23:09:23 -0800 Subject: [PATCH 4/8] quest: record the route layer's progress and what remains Co-Authored-By: Claude Opus 5.5 --- quest/m1/cluster-routing/routes.md | 56 ++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/quest/m1/cluster-routing/routes.md b/quest/m1/cluster-routing/routes.md index b7192ab4c77..1422373aa49 100644 --- a/quest/m1/cluster-routing/routes.md +++ b/quest/m1/cluster-routing/routes.md @@ -13,6 +13,62 @@ claim whose withdrawal is already known. ## Plan +Progress in [moq-dev/moq#7676](https://github.com/moq-dev/moq/pull/7676): +the route layer exists as a crate-private, sans-IO `Router` in +`rs/moq-net/src/route/`, generic over the neighbor key and the claim +attributes, with every holding rule of the contract below (checked metric, +down-only bit, GOAWAY rank, hop-by-hop seqno requests, ROUTE_END echoes +draining the feasibility distance, qualified keys with no own-identity +exemption, merged per-region floors, held and let-go). Each message +recomputes only the identity it names. The handoff fixtures, both sweeps, +and the route-layer fixtures of the contract run against it in +`route/tests/`; the forwarding and first-hop selection fixtures +(`Forward`, `Aggregate`, pool ties) still run only on the reference model, +which stays until they are ported. Nothing drives the router on the wire +yet. Remaining, in order: + +1. lite-07 wire: ROUTE_START/UPDATE/END, ANNOUNCE_FLOOR, and on the request + side ROUTE_REQUEST and the ROUTE_END echo; ANNOUNCE_START/UPDATE carry a + Route ID and the origin's seed instead of hops; draft text; rerun the + [moq-dev/moq#4196](https://github.com/moq-dev/moq/pull/4196) announce + bench on path-less ANNOUNCE before keeping `Path Base`/`Path Keep`. +2. Session integration: one router per origin node, fed by every lite-07 + session, its shown claims and selected routes driving the origin's + routes, and its outbox driving each announce stream; the first hop pins + requests to an identity; a lite-06 peer translated at the relay. Per- + session bounds on the routes and claims a peer makes the router hold. + Config: zero-cost links refused, `cluster.id` removed, per-connection + `proxy`. +3. Reader API: availability and the offline event in Rust and JS. +4. `js/net` encodes, decodes, and resolves the route layer, including the + detached-request fix from [moq-dev/moq#6947](https://github.com/moq-dev/moq/pull/6947). +5. Acceptance: port the forwarding fixtures, then delete the reference + model; the `residual_*` and toggle fixtures in `mesh_withdraw.rs`; the + tiered-cluster and partition end-to-end checks; benchmarks. + +Implementation choices so far, for review: + +- Every lite-07 claim covers its own path and everything below it, so the + model's exact-versus-wildcard distinction is gone: a claim above an + interest overlaps it (`a_claim_above_an_interest_overlaps_it`). +- The region list is ordered by path, as a sorted map yields it, and a lost + region keeps the proof's rule: keep every still-certified region, or + rebuild them all if a lost floor is newer than a kept one. +- An interest change keys the router by announce stream: the old stream + stays open, its routes counting and its batches discarded, until the new + stream has answered, so no route drops in between. Not yet built; the + router already supports it, since a stream is just another neighbor key. +- The qualified key's derivation stays open (below): the router takes it + from the caller per scoped neighbor (`Scope::qualify`), and the fixtures + use a stand-in keyed by a label. +- A rotated identity ends the old one with ROUTE_END on every stream rather + than closing sessions, since the own identity is no longer proven per + session. +- A batch carries changes against the last batch on that stream; the + receiver keeps each stream's last view to apply it, as the contract says. + Restricting a view to an interest is still linear in that origin's + claims per change; make it incremental with the session integration. + Keep the 2026-10-01 two-layer direction, reconciled with the maintainer's 2026-10-13 [decisions](/quest/m1/cluster-routing/README.md) and the routing contract below. The routing contract's fixtures live in From c5b9e17f963d6746104426257043afd4b29eaeb8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 09:46:47 -0800 Subject: [PATCH 5/8] quest: split routes and announces into an epic The route layer's core lands here, so the quest becomes an epic holding the routing contract, with five children in order: the lite-07 wire, session integration, announcement availability, js/net, and acceptance. Scoped identity keying waits on the maintainer in the session child. Co-Authored-By: Claude Opus 5.5 --- quest/m1/announce-takeover.md | 6 +- quest/m1/api-audit/route/README.md | 4 +- quest/m1/api-audit/route/handle.md | 2 +- quest/m1/cluster-routing/README.md | 10 +- quest/m1/cluster-routing/federation.md | 7 +- .../{routes.md => routes/README.md} | 311 ++---------------- quest/m1/cluster-routing/routes/acceptance.md | 81 +++++ .../m1/cluster-routing/routes/availability.md | 30 ++ quest/m1/cluster-routing/routes/js.md | 29 ++ quest/m1/cluster-routing/routes/session.md | 160 +++++++++ quest/m1/cluster-routing/routes/wire.md | 91 +++++ quest/m1/cut-audit/README.md | 2 +- quest/m1/lite07-finalize.md | 4 +- quest/m2/link-quality.md | 2 +- quest/m2/mesh/README.md | 2 +- quest/m2/p2p/cost-scopes.md | 2 +- quest/m2/p2p/signal.md | 2 +- .../m2/perf/kio-channel-contention/README.md | 2 +- quest/m2/route-health/selection.md | 2 +- quest/m2/route-wakes.md | 2 +- rs/moq-net/src/route/mod.rs | 2 +- rs/moq-net/tests/integration/mesh_withdraw.rs | 2 +- .../integration/routing_contracts/handoff.rs | 2 +- .../integration/routing_contracts/mod.rs | 2 +- .../integration/routing_contracts/model.rs | 2 +- 25 files changed, 455 insertions(+), 306 deletions(-) rename quest/m1/cluster-routing/{routes.md => routes/README.md} (63%) create mode 100644 quest/m1/cluster-routing/routes/acceptance.md create mode 100644 quest/m1/cluster-routing/routes/availability.md create mode 100644 quest/m1/cluster-routing/routes/js.md create mode 100644 quest/m1/cluster-routing/routes/session.md create mode 100644 quest/m1/cluster-routing/routes/wire.md diff --git a/quest/m1/announce-takeover.md b/quest/m1/announce-takeover.md index 0a55f9850f3..68be18d9358 100644 --- a/quest/m1/announce-takeover.md +++ b/quest/m1/announce-takeover.md @@ -126,8 +126,8 @@ separate from the identity rewrite of this Plan, which landed with - Reword "takeover" as an event or operation, in terms of identity continuity, in [moq-bench pins the instances it measures](/quest/m2/bench-epoch-pin.md) (Goal and Plan), [the HLS server follows each path once](/quest/m2/hls-follow-once.md) - (Goal and Tests), and [routes and announces](/quest/m1/cluster-routing/routes.md) - (around line 37, "the separately proven Takeover operation"). + (Goal and Tests), and [lite-07 route wire](/quest/m1/cluster-routing/routes/wire.md) + (the wire sketch, "the separately proven Takeover operation"). - Rewrite the EPOCH references in `drafts/draft-lcurley-moq-cluster.md` and `drafts/draft-lcurley-moq-epoch.md` in terms of IDENTITY. - Add a regression test where a publisher restart's new announcement @@ -151,4 +151,4 @@ beyond [Instance and epoch](https://github.com/moq-dev/moq/pull/7489)'s fields. - [Listing and follow](/quest/m1/api-audit/route/events.md) - continuity shows up inside `follow`, not as an event - [Claim answer epochs](https://github.com/moq-dev/moq/pull/5678) - why a restarted job must never be stitched to its predecessor - [Retired requests](https://github.com/moq-dev/moq/pull/5294) - replacements never substitute content into held handles -- [Routes and announces](/quest/m1/cluster-routing/routes.md) - the route layer whose ANNOUNCE_START layout carries the instance +- [Routes and announces](/quest/m1/cluster-routing/routes/README.md) - the route layer whose ANNOUNCE_START layout carries the instance diff --git a/quest/m1/api-audit/route/README.md b/quest/m1/api-audit/route/README.md index 03feacbbf08..0bd42496385 100644 --- a/quest/m1/api-audit/route/README.md +++ b/quest/m1/api-audit/route/README.md @@ -17,7 +17,7 @@ kinds. Rust, JS, moq-ffi, moq-c and the wrappers mirror it. Non-goals: the route layer's wire messages and federation -([Routes and announces](/quest/m1/cluster-routing/routes.md)), lite-07 +([Routes and announces](/quest/m1/cluster-routing/routes/README.md)), lite-07 takeover ([announce takeover](/quest/m1/announce-takeover.md)), and any moq-transport instance ([moq-transport carries no epoch](https://github.com/moq-dev/moq/pull/7239)). @@ -174,4 +174,4 @@ stale, including `doc/concept/identity.md`. ## Related - [A job's exact route continues its claim's readers](/quest/m1/announce-takeover.md) - a job's route inherits its claim's instance, so lite-07 readers keep the first subscribe -- [Routes and announces](/quest/m1/cluster-routing/routes.md) - the wire route layer; instances take over its epochless deletion +- [Routes and announces](/quest/m1/cluster-routing/routes/README.md) - the wire route layer; instances take over its epochless deletion diff --git a/quest/m1/api-audit/route/handle.md b/quest/m1/api-audit/route/handle.md index f666d58cf61..512074f242f 100644 --- a/quest/m1/api-audit/route/handle.md +++ b/quest/m1/api-audit/route/handle.md @@ -42,7 +42,7 @@ read-only facts on a received announcement, and `Hop`, `Hops` and Non-goals: the instance model ([Instance and epoch](https://github.com/moq-dev/moq/pull/7489)), announce events ([Listing and follow](/quest/m1/api-audit/route/events.md)), the read side, which instance owns, and the route layer's wire metric -([Routes and announces](/quest/m1/cluster-routing/routes.md)). +([lite-07 route wire](/quest/m1/cluster-routing/routes/wire.md)). ## Plan diff --git a/quest/m1/cluster-routing/README.md b/quest/m1/cluster-routing/README.md index a45d387a821..a8c03177191 100644 --- a/quest/m1/cluster-routing/README.md +++ b/quest/m1/cluster-routing/README.md @@ -51,8 +51,8 @@ Decided with the maintainer, 2026-10-13: qualified when its publish grant is scoped, and stays bare when it is unscoped, so a forger only damages routes inside its own scope. (Decided 2026-10-14; own identities included 2026-10-26, since they are no longer - proven.) How a scoped peer's identities are keyed is open in - [Routes and announces](/quest/m1/cluster-routing/routes.md): the + proven.) How a scoped peer's identities are keyed waits on the maintainer in + [Route session integration](/quest/m1/cluster-routing/routes/session.md): the issuer-qualified subject key was dropped 2026-11-02 (maintainer). - Matching explicit epochs authorize resume. Epochless changes retain Restart and fresh resolution; neither node identity nor a healthy @@ -86,7 +86,7 @@ Decided with the maintainer, 2026-10-13: The handoff proof showed a complete neighbor snapshot does not prove its claims are current; receivers merge origin-versioned views by floor instead. -The routing contract is recorded in [Routes and announces](/quest/m1/cluster-routing/routes.md) +The routing contract is recorded in [Routes and announces](/quest/m1/cluster-routing/routes/README.md) with deterministic counterexamples: path-only subscriptions that reselect origins at every hop loop and deadlock, so forwarded requests name their identity; feasibility makes transit priorities depend on arrival order; @@ -109,7 +109,7 @@ The 300 ms update hold alone is no longer the baseline. Keep regression coverage for announcement churn, idle work, active continuity, and retained state separately; fewer control bytes does not prove cheaper runtime work. -The earlier simulator estimates remain in [Routes and announces](/quest/m1/cluster-routing/routes.md). +The earlier simulator estimates remain in [Routes and announces](/quest/m1/cluster-routing/routes/README.md). They do not prove the handoff correct; its fixtures do. Preserve public synthetic fixtures and controlled-time regressions, including residual stale revival, not only results from deployment tooling. Do not claim per-origin aggregation @@ -126,7 +126,7 @@ setup after the capabilities land. ## Required -- [Routes and announces](/quest/m1/cluster-routing/routes.md) - implement the proven route layer with Rust/JS parity and tiered-cluster acceptance +- [Routes and announces](/quest/m1/cluster-routing/routes/README.md) - implement the proven route layer with Rust/JS parity and tiered-cluster acceptance - [Outbound federation](/quest/m1/cluster-routing/federation.md) - an outbound-only local relay exchanges authorized broadcasts with the CDN and survives partition/healing ## Related diff --git a/quest/m1/cluster-routing/federation.md b/quest/m1/cluster-routing/federation.md index 67cb5eac7a2..00742dcfb68 100644 --- a/quest/m1/cluster-routing/federation.md +++ b/quest/m1/cluster-routing/federation.md @@ -18,8 +18,9 @@ Use authenticated, namespace-scoped admission: the CDN scopes the local relay's connection to a path prefix, and identities it sends, its own and its local publishers', are qualified, so a forger or a misconfigured uplink only affects its own scope (decided 2026-10-14; see -[Routes and announces](/quest/m1/cluster-routing/routes.md)). Open: how -those identities are keyed, owned by routes. Without a key, a scoped forger +[Routes and announces](/quest/m1/cluster-routing/routes/README.md)). Open: how +those identities are keyed, owned by +[Route session integration](/quest/m1/cluster-routing/routes/session.md). Without a key, a scoped forger declaring a victim's id captures claims outside its scope (`forged_identity_hijacks_claims_outside_the_forgers_scope`). The scope itself (`hash(scope, id)`) was rejected 2026-10-15, and the grant's @@ -62,7 +63,7 @@ Rust/JS parity. Run `just check`, `just test interop --all`, and `just drafts ch ## Required -- [Routes and announces](/quest/m1/cluster-routing/routes.md) - identity, scoped claims, reachability, and export policy +- [Routes and announces](/quest/m1/cluster-routing/routes/README.md) - identity, scoped claims, reachability, and export policy ## Related diff --git a/quest/m1/cluster-routing/routes.md b/quest/m1/cluster-routing/routes/README.md similarity index 63% rename from quest/m1/cluster-routing/routes.md rename to quest/m1/cluster-routing/routes/README.md index 1422373aa49..a2404f228bc 100644 --- a/quest/m1/cluster-routing/routes.md +++ b/quest/m1/cluster-routing/routes/README.md @@ -1,4 +1,4 @@ -# [XL] Routes and announces +# Routes and announces ## Goal @@ -13,135 +13,46 @@ claim whose withdrawal is already known. ## Plan -Progress in [moq-dev/moq#7676](https://github.com/moq-dev/moq/pull/7676): -the route layer exists as a crate-private, sans-IO `Router` in -`rs/moq-net/src/route/`, generic over the neighbor key and the claim -attributes, with every holding rule of the contract below (checked metric, -down-only bit, GOAWAY rank, hop-by-hop seqno requests, ROUTE_END echoes -draining the feasibility distance, qualified keys with no own-identity -exemption, merged per-region floors, held and let-go). Each message -recomputes only the identity it names. The handoff fixtures, both sweeps, -and the route-layer fixtures of the contract run against it in -`route/tests/`; the forwarding and first-hop selection fixtures -(`Forward`, `Aggregate`, pool ties) still run only on the reference model, -which stays until they are ported. Nothing drives the router on the wire -yet. Remaining, in order: - -1. lite-07 wire: ROUTE_START/UPDATE/END, ANNOUNCE_FLOOR, and on the request - side ROUTE_REQUEST and the ROUTE_END echo; ANNOUNCE_START/UPDATE carry a - Route ID and the origin's seed instead of hops; draft text; rerun the - [moq-dev/moq#4196](https://github.com/moq-dev/moq/pull/4196) announce - bench on path-less ANNOUNCE before keeping `Path Base`/`Path Keep`. -2. Session integration: one router per origin node, fed by every lite-07 - session, its shown claims and selected routes driving the origin's - routes, and its outbox driving each announce stream; the first hop pins - requests to an identity; a lite-06 peer translated at the relay. Per- - session bounds on the routes and claims a peer makes the router hold. - Config: zero-cost links refused, `cluster.id` removed, per-connection - `proxy`. -3. Reader API: availability and the offline event in Rust and JS. -4. `js/net` encodes, decodes, and resolves the route layer, including the - detached-request fix from [moq-dev/moq#6947](https://github.com/moq-dev/moq/pull/6947). -5. Acceptance: port the forwarding fixtures, then delete the reference - model; the `residual_*` and toggle fixtures in `mesh_withdraw.rs`; the - tiered-cluster and partition end-to-end checks; benchmarks. - -Implementation choices so far, for review: +The route layer's core landed in +[moq-dev/moq#7676](https://github.com/moq-dev/moq/pull/7676): a +crate-private, sans-IO `Router` in `rs/moq-net/src/route/`, generic over the +neighbor key and the claim attributes, with every holding rule of the +contract below. Each message recomputes only the identity it names. The +handoff fixtures, both sweeps, and the route-layer fixtures of the contract +run against it in `route/tests/`; the forwarding and first-hop selection +fixtures still run only on the reference model in +`rs/moq-net/tests/integration/routing_contracts/`. Nothing drives the router +on the wire yet. The children below carry the rest, in order, and land in +one release: lite-07 carries the route layer as a whole +([lite-07 finalize](/quest/m1/lite07-finalize.md) waits on this epic). This +README holds the routing contract every child implements. + +Decided 2026-11-03 (maintainer, on +[moq-dev/moq#7676](https://github.com/moq-dev/moq/pull/7676)): land the core +and split the quest into this epic. Options were: ✅ land the core and split; +one PR; the core plus the Rust wire there, with JS as a child. + +Settled by the core, for review: - Every lite-07 claim covers its own path and everything below it, so the model's exact-versus-wildcard distinction is gone: a claim above an interest overlaps it (`a_claim_above_an_interest_overlaps_it`). -- The region list is ordered by path, as a sorted map yields it, and a lost - region keeps the proof's rule: keep every still-certified region, or - rebuild them all if a lost floor is newer than a kept one. -- An interest change keys the router by announce stream: the old stream - stays open, its routes counting and its batches discarded, until the new - stream has answered, so no route drops in between. Not yet built; the - router already supports it, since a stream is just another neighbor key. -- The qualified key's derivation stays open (below): the router takes it - from the caller per scoped neighbor (`Scope::qualify`), and the fixtures - use a stand-in keyed by a label. +- Letting go of a region keeps the proof's rule: keep every still-certified + region, or rebuild them all if a lost floor is newer than a kept one, a + collection whose replay is bounded like any other. - A rotated identity ends the old one with ROUTE_END on every stream rather than closing sessions, since the own identity is no longer proven per session. -- A batch carries changes against the last batch on that stream; the - receiver keeps each stream's last view to apply it, as the contract says. - Restricting a view to an interest is still linear in that origin's - claims per change; make it incremental with the session integration. Keep the 2026-10-01 two-layer direction, reconciled with the maintainer's 2026-10-13 [decisions](/quest/m1/cluster-routing/README.md) and the routing -contract below. The routing contract's fixtures live in +contract below. The routing contract's remaining model fixtures live in `rs/moq-net/tests/integration/routing_contracts/` and run against a small reference model; port each topology to the implementation as acceptance, then delete the model. The handoff contract below, decided 2026-10-16, settles claim freshness: -- Wire sketch, flexible in detail. ROUTE_START carries the node id, a seqno, - the metric, and the down-only bit (below), and implicitly takes the next - stream-local Route ID, as ANNOUNCE_START takes the next Announce ID today. - ROUTE_UPDATE (Route ID, seqno, metric, down-only) and ROUTE_END (Route - ID) reference it; the bit rides UPDATE so a route moving between a - mesh path and a CDN fallback changes it without retracting its announces. - ANNOUNCE_START carries the prefix (keeping today's Path Base/Keep - compression), a Route ID, the origin's cost for that prefix, and the epoch, - and lite-07's restart message from - Restart keeps its meaning. On a - route without an epoch, the source is the origin node and its ANNOUNCE: a - new one at the same path is a restart. This is routing provenance, not - permission to stitch. A concrete serving-route change without an epoch - remains Restart even when the origin node is unchanged; only a matching - explicit epoch or the separately proven Takeover operation preserves an - instance (superseded 2026-10-30: a shared route `instance` preserves it, per - [Instance and epoch](https://github.com/moq-dev/moq/pull/7489)). Test a same-origin next-hop change with no epoch. - ANNOUNCE_UPDATE re-prices it and ANNOUNCE_END withdraws this origin's claim - while the route stays up. A wildcard claim is capability, not an inventory - of broadcasts; another origin may still serve the same epoch. An announce - never changes its Route ID: another origin - serving the same path is another ANNOUNCE. An ANNOUNCE naming an unknown - Route ID is a protocol violation. A ROUTE_UPDATE with an infinite metric - (2^64-1, the largest lite-07 varint; a sum reaching it is unreachable) - means held: the sender has no usable route but still holds the origin's - announcements while it recovers. ROUTE_END means the sender let go of the - origin on this stream; neither is proof that the origin withdrew its - claims (see the handoff contract). Each batch of ANNOUNCEs for a route - closes with one ANNOUNCE_FLOOR listing every region the sender certifies - for that route: a count, then per region a suffix of the stream's interest - prefix (empty for the whole interest) and the origin revision as of which - the stream's set for that route in that region is complete. The list - replaces the previous one, so a region left out is no longer certified, - and a count of zero answers a stream whose interest none of the sender's - regions meets. - ROUTE_REQUEST (node id, seqno) travels - the other way on the same stream, toward the neighbour that sent the route, - and so does the echo of every ROUTE_END: the receiver sends the same - ROUTE_END (Route ID) back once it has dropped the route, which tells the - sender no neighbour still holds its advertisement (see the handoff - contract). Every receiver echoes, clients included; a closed session - counts as the echo. - The hop list leaves lite-07, and with it the - `Hop Base`/`Hop Keep` compression; `cluster.max_hops` then stops applying - to lite-07. Most of the 62% saving measured in - [moq-dev/moq#4196](https://github.com/moq-dev/moq/pull/4196) came from hop - chains, so rerun its announce bench on path-less ANNOUNCE and drop - `Path Base`/`Path Keep` too unless it still pays for its rules. +The wire sketch is in [wire](/quest/m1/cluster-routing/routes/wire.md); the bullets below are the contract it carries. - ```text - ROUTE_START node=0x7a3f seqno=41 metric=12 -> route 0 - ANNOUNCE_START prefix=transcode/ route=0 cost=10 -> announce 0 - ANNOUNCE_START prefix=transcode/foobar route=0 cost=1 -> announce 1 - ANNOUNCE_FLOOR route=0 count=1 ""@7 # complete as of revision 7 - ROUTE_UPDATE route=0 seqno=42 metric=15 - ANNOUNCE_END announce=1 # this origin withdraws a claim - ANNOUNCE_FLOOR route=0 count=1 ""@8 - ANNOUNCE_FLOOR route=1 count=2 "live/a/"@3 "live/b/"@5 # a scoped key - ANNOUNCE_FLOOR route=1 count=1 "live/b/"@6 # live/a/ is no longer certified - ROUTE_UPDATE route=0 seqno=42 metric=inf # held: unusable, still recovering - ROUTE_END route=0 # let go: forget it on this stream - ROUTE_END route=0 # <- the receiver echoes it back - ``` -- Scope route state to interested origins, not the CDN's whole node table. - Send a route before any announcement referencing it. The handoff contract - defines held routes, letting go, and collection. - The announce cost is the origin's per-prefix seed (`transcode/**` at 10 and `transcode/foobar` at 1 from one node, decided 2026-10-01) and nobody changes it in transit; a path's cost through a route is that seed plus the @@ -218,30 +129,8 @@ The handoff contract below, decided 2026-10-16, settles claim freshness: claim no longer decides transit trust. Legitimate scoped delivery survives losing one of two uplinks under one key (`scoped_delivery_through_two_uplinks_reaches_one_key`). -- Open: how a scoped peer's routing identities are keyed. The key must - survive token refresh, be shared by one operator's two uplinks, and leave - a forger unable to capture claims outside its scope; each claim is still - checked against the current pattern union. Rejected 2026-10-15: the - connection's publish scope (`hash(scope, id)`), since a publish grant is - a union of patterns that refresh and withdrawal change, so neither the - whole union nor one matching pattern is a key that survives refresh and - that two uplinks share; and the connection root, too coarse since two - scoped peers under one root would share keys. Dropped 2026-11-02 - (maintainer): the grant's issuer-qualified credential subject - (`hash(subject, id)`, decided 2026-10-15), along with refusing scoped - tokens without a subject and tokens whose subject differs from the - session's; the implementation is in - [moq-dev/moq@453e1ec](https://github.com/moq-dev/moq/commit/453e1ece7c5435ee19541cfc2e9bbe0b6fbc430e). - The answer also covers sessions allowed to publish part of the tree with - no token (public rules, mTLS certificates, or a third-party auth server). - Whatever the key, add multi-pattern, overlapping-pattern, and refresh - fixtures. "Scope" means only a grant's pattern union. -- ✅ Decided 2026-10-15 (maintainer): retract and re-file when a session's - grant union crosses the scoped boundary in either direction (a `**` token - added, or its expiry). Every forwarded route the session delivered is - retracted, and its current advertisements are re-filed under the new key, - so no route, feasibility distance, or alias stays under the old one. - Expiry cannot be refused, so refusing the transition was not an option. +- Open: how a scoped peer's routing identities are keyed, owned by + [session integration](/quest/m1/cluster-routing/routes/session.md). The router takes the key from the caller (`Scope::qualify`). - Decided 2026-10-14: qualified keys carry a marker every relay recognizes, and a scoped connection forwarding an already qualified identity has that route refused, with no wire change. Otherwise a key sent down through a @@ -298,32 +187,6 @@ The handoff contract below, decided 2026-10-16, settles claim freshness: exchange, though not its latency. - Claim freshness follows the handoff contract below, independently of the requested path's forwarding choice. -- Only the first hop selects the origin. It keeps `route_order`'s shape over - the origin nodes announcing the path: longest prefix, newest epoch (within - one instance only, per [Instance and epoch](https://github.com/moq-dev/moq/pull/7489), - decided 2026-10-30), the link's priority (`origin::Producer::with_preference`), seed plus route - metric, then a rendezvous hash of the path and the identity, and the newest - connection only for an otherwise identical route. The hash drops the hop - chain, which differs per node, so every node agrees on a pool member - (`pool_ties_agree_only_on_a_path_and_identity_hash`). The request then names - that identity, and every later hop follows its own route to it, so - forwarding needs a route but not the claim. -- Reselecting the origin at every hop loops although each origin's routes are - loop-free (RFC 8966 s2.7): a ring of same-epoch origins whose relays - prioritize different links, and nested wildcards where a relay has not - heard the specific claim, both cycle past requester exclusion; joining - another relay's subscription by path or epoch deadlocks; and without an - epoch a later hop splices another publisher under the requester's - announcement with no Restart (`local_priorities_loop_path_only_requests`, - `overlapping_wildcards_loop_where_specific_claims_are_unheard`, - `aggregation_cycles_starve_without_a_forwarding_constraint`, - `epochless_reselection_splices_another_publisher`, - `sweep_forwarding_progress`). A relay therefore joins a forwarded request - only to a subscription following its route to the same identity. When that - identity becomes unreachable the forwarded request ends, and the first hop - re-resolves, resuming only on a matching epoch. A request carrying a cost - bound instead also never loops, but stalls where a transit hop has not - heard the claim and still splices epochless sources. - Decided 2026-10-13: preserve application priority among feasible transit routes, then compare additive cost. Feasibility always takes precedence: a preferred longer route can remain ineligible after a shorter route was @@ -361,8 +224,6 @@ The handoff contract below, decided 2026-10-16, settles claim freshness: backbone addresses, but are not themselves authorization. A client advertises nothing it receives, so it holds no feasibility distance and falls back from P2P to the CDN at once. It still echoes every ROUTE_END. -- Mixed versions: a lite-06 peer keeps today's path vector, translated at the - relay that speaks both, for the rollout window only. Handoff contract, proven 2026-10-15 by the fixtures in `handoff.rs` against next-hop authority and a snapshot barrier alone. A next-hop change adopts the @@ -593,13 +454,6 @@ Decided with the maintainer 2026-10-16, from the handoff proof: meaning and costs a floor per dropped region; and an end-of-batch marker, a message for what one floor frames. -Open, for this quest's implementation: any further wire detail of the region -list, such as its order, is settled there with Rust and JS interop, not in -the proof. So is whether letting go of a region keeps each still-certified -region's knowledge independently: the proof keeps them all or, if any lost -floor is newer than a kept one, rebuilds them all, a collection whose replay -is bounded like any other. - Why not a hold-down: [moq-dev/moq#4644](https://github.com/moq-dev/moq/pull/4644) retracted at once and re-announced a replacement after 1 s, which ended a downstream front whose only path ran through the relay, dropping subscribers for about a second while the relay kept serving; @@ -638,112 +492,15 @@ benchmarks. subscriptions through the radio mesh and the split refuses, and after a partition each island reaches the other through its uplink. -Open, for the implementer to settle and record: - -- The earlier simulator persisted seqnos across restart. Re-run with the - settled fresh routing incarnation and the stale-state rules above; do not - carry the old assumption into acceptance. -- How much of today's route trie, fronts, and `route_order` survives intact. - -The counterexamples from #4349 (closed; this quest still owns the fix) are the `residual_*` tests in -`rs/moq-net/tests/integration/mesh_withdraw.rs`. They pin today's hunting on lite-06: -a stale path outlasting the 300 ms hold, a revival the quarantine misses, -and 179 hunting relays (3 reviving) over 200 seeded meshes. Turn each into a -single retraction everywhere, and keep `cross_node_close_retracts_once` -passing. On `main` an epochless switch to a stale path is a Restart, which a -lite-06 client sees as a retraction and an announcement; the release sent an -in-place update. - -Publishing toggles too, decided 2026-10-13: add a time-mocked fixture where, -on some rings, a longer path arrives first and the better route then replaces -it as a Restart, so a lite-06 client sees `Start, End, Start` while the -broadcast stays live. Pin today's sequence beside the `residual_*` tests; -the epochless Restart rule above decides what it becomes. - -Tests, time mocked: the handoff fixtures and sweep in `handoff.rs`; -[moq-dev/moq#4644](https://github.com/moq-dev/moq/pull/4644)'s live-graph withdrawal and failover tests (the PR closed and -its branch is gone; fetch `refs/pull/4644/head`) as acceptance, run on the wip version; a -link flap with many broadcasts behind one origin costs one ROUTE change per -link; a ring of three where a stale neighbour cannot revive an ended -broadcast; a two-uplink mesh that never carries CDN routes back to the CDN; -an island reached through the CDN after a partition; an unknown Route ID -refused; a five-node ring whose preferred origin link fails recovers its -longer backup through a seqno request, duplicates suppressed, while that -backup never loses its session; a ring where a node's own preferred route -points back at the requester forwards the request to it, which carries it -on along its own route, and an unanswerable request stops once no usable -route is left. Failover is measured -on broadcast liveness while the next hop changes, including a new next hop -whose ROUTE arrives before its ANNOUNCE set, and reports per-path -retractions and re-announcements beside the ROUTE budget; eventual route -recovery alone does not show it is seamless. +## Required -Tiered-cluster and partition/healing end-to-end checks belong in this quest, -not after lite-07 can be finalized: two regions with redundant cores, publish, -claim withdrawal, core loss, and matching-epoch failover reaching a far edge; -a two-uplink mesh reached through CDN during partition. Include overlapping -wildcards, a hidden exportable alternative under one selection, epochless Restart, and a -grant union crossing the scoped boundary each way (a `**` token added, then expiring). Fixtures are -public and time-mocked, wired into CI alongside Rust/JS interop. - -Benchmark origins, broadcasts per origin, neighbors, active tracks/fronts, -and announce cursors independently. A route update must not expand back into -per-broadcast table rewrites or poll unrelated fronts. Measure messages, bytes, -wakes/select calls, CPU, replay work, and retained state. One broadcast per -origin is an explicit case. Coordinate with Route wakes and Announce replay; -benchmark smoke alone does not enforce complexity, so add operation-count -regressions. Bound pending requests and freshness state under same-clock churn. - -Wire: `drafts/draft-lcurley-moq-lite.md` in the same PR, and `js/net` -encodes, decodes, and resolves it. Requests forwarded between routing nodes -name the serving identity (decided 2026-10-13). ROUTE carries one down-only -bit and no export metric. Browsers remain non-transit; P2P reuses this model -rather than adding a browser forwarding engine. Public API: the route-change -surface on `broadcast::Route` and its bindings will likely change, `Hops` -and saturating `Cost` go, zero-cost links are refused, a per-connection -`proxy` preference is added, and `cluster.id`/`--cluster-id` is removed -rather than kept as an unused label (decided 2026-10-13). Announcements -expose availability, and a let-go is reported as offline rather than END -(decided 2026-10-16). `Preference` -continues to rank feasible transit routes; report it. This may -split at start (Rust and draft, then JS), as long as both land in one -release. - -Decided 2026-10-27 (maintainer): this quest deletes the epochless instance -machinery in `rs/moq-net/src/model/origin.rs`: `Instance::Route(generation)` -and its renewals, `restarted_since`, the held-restart Release for epochless -routes, and Selection retirement, leaving Follow's same-epoch -Update/Restart split as the only takeover rule. Once each origin is its own -ANNOUNCE and the first hop pins the request to its identity, the first hop's -route is the instance, so the per-hop renewal has nothing left to protect. -An epochless change of serving route stays a Restart, including the -same-origin next-hop change in the wire sketch above, and the routing -contract's epochless fixtures keep passing. Minting an epoch for every route -instead was tried and rejected -([moq-dev/moq#6948](https://github.com/moq-dev/moq/pull/6948)): each relay -mints at its own time, so mint order outranks cost, preference, and the -local-broadcast tie and relays disagree on a pool path's winner, while -ranking a minted epoch last keeps the renewal it was meant to delete. - -Moved 2026-10-30 (maintainer, the [route epic](/quest/m1/api-audit/route/README.md)): -[Instance and epoch](https://github.com/moq-dev/moq/pull/7489) now deletes that -epochless machinery, so this quest no longer does. A session mints -an `instance` (`~` plus random bits) for any route that -arrives without one, and newest-wins compares epochs only within one -instance, so mint order no longer outranks cost or preference. The rest of -the 2026-10-27 decision stands. The broadcast epoch fields here become that -publisher instance and its per-instance epoch, in place of the UUIDv7 -epoch. The route layer's own origin-node identity, with its seqno and -revision, stays separate ("keep administrative identity and broadcast epoch -separate", above). - -Decided 2026-10-27 (maintainer), from -[moq-dev/moq#6947](https://github.com/moq-dev/moq/pull/6947): in `js/net`'s -`origin.ts` a detached request stays on its own route instead of following -the cheapest one; fix it with the route-selection rework here. +- [lite-07 route wire](/quest/m1/cluster-routing/routes/wire.md) - lite-07 and the draft carry ROUTE, ANNOUNCE_FLOOR, ROUTE_REQUEST, and path-less ANNOUNCE +- [Route session integration](/quest/m1/cluster-routing/routes/session.md) - one router per origin node drives the origin and every lite-07 announce stream +- [Announcement availability](/quest/m1/cluster-routing/routes/availability.md) - announcements report availability, and a let-go is offline rather than END, in Rust and JS +- [js/net route layer](/quest/m1/cluster-routing/routes/js.md) - `js/net` encodes, decodes, and resolves the route layer +- [Route layer acceptance](/quest/m1/cluster-routing/routes/acceptance.md) - forwarding fixtures ported and the model deleted, incident fixtures, end-to-end checks, and benchmarks ## Related - [Route wakes](/quest/m2/route-wakes.md) - index work follows the settled source-selection model - [Route health](/quest/m2/route-health/README.md) - epoch-only progress observation and local delivery failover - diff --git a/quest/m1/cluster-routing/routes/acceptance.md b/quest/m1/cluster-routing/routes/acceptance.md new file mode 100644 index 00000000000..0aab45bbdf0 --- /dev/null +++ b/quest/m1/cluster-routing/routes/acceptance.md @@ -0,0 +1,81 @@ +# [L] Route layer acceptance + +## Goal + +The route layer is proven on the implementation, not the model: the +forwarding and first-hop selection fixtures run against the router and the +session, and the reference model in +`rs/moq-net/tests/integration/routing_contracts/` is deleted. The incident +fixtures, tiered-cluster and partition end-to-end checks, failover +measurements, and benchmarks below pass in CI, and the mitigations they +replace are retired. + +## Plan + +- Port the forwarding fixtures (`Forward`, `Aggregate`, pool ties, and the + sweeps over them) to the implementation, then delete the model. +- The earlier simulator persisted seqnos across restart. Re-run with the + settled fresh routing incarnation and the stale-state rules; do not carry + the old assumption into acceptance. +- Retire [moq-dev/moq#5375](https://github.com/moq-dev/moq/pull/5375)'s + quarantine, [moq-dev/moq#5391](https://github.com/moq-dev/moq/pull/5391)'s + dedup, and the 300 ms hold only once the fixtures below demonstrate their + replacement (see the epic's hold-down note). + +The counterexamples from #4349 (closed; this epic still owns the fix) are the `residual_*` tests in +`rs/moq-net/tests/integration/mesh_withdraw.rs`. They pin today's hunting on lite-06: +a stale path outlasting the 300 ms hold, a revival the quarantine misses, +and 179 hunting relays (3 reviving) over 200 seeded meshes. Turn each into a +single retraction everywhere, and keep `cross_node_close_retracts_once` +passing. On `main` an epochless switch to a stale path is a Restart, which a +lite-06 client sees as a retraction and an announcement; the release sent an +in-place update. + +Publishing toggles too, decided 2026-10-13: add a time-mocked fixture where, +on some rings, a longer path arrives first and the better route then replaces +it as a Restart, so a lite-06 client sees `Start, End, Start` while the +broadcast stays live. Pin today's sequence beside the `residual_*` tests; +the epic's epochless Restart rule decides what it becomes. + +Tests, time mocked: the handoff fixtures and sweeps, already on the router; +[moq-dev/moq#4644](https://github.com/moq-dev/moq/pull/4644)'s live-graph withdrawal and failover tests (the PR closed and +its branch is gone; fetch `refs/pull/4644/head`) as acceptance, run on the wip version; a +link flap with many broadcasts behind one origin costs one ROUTE change per +link; a ring of three where a stale neighbour cannot revive an ended +broadcast; a two-uplink mesh that never carries CDN routes back to the CDN; +an island reached through the CDN after a partition; an unknown Route ID +refused; a five-node ring whose preferred origin link fails recovers its +longer backup through a seqno request, duplicates suppressed, while that +backup never loses its session; a ring where a node's own preferred route +points back at the requester forwards the request to it, which carries it +on along its own route, and an unanswerable request stops once no usable +route is left. Failover is measured +on broadcast liveness while the next hop changes, including a new next hop +whose ROUTE arrives before its ANNOUNCE set, and reports per-path +retractions and re-announcements beside the ROUTE budget; eventual route +recovery alone does not show it is seamless. + +Tiered-cluster and partition/healing end-to-end checks belong here, +not after lite-07 can be finalized: two regions with redundant cores, publish, +claim withdrawal, core loss, and matching-epoch failover reaching a far edge; +a two-uplink mesh reached through CDN during partition. Include overlapping +wildcards, a hidden exportable alternative under one selection, epochless Restart, and a +grant union crossing the scoped boundary each way (a `**` token added, then expiring). Fixtures are +public and time-mocked, wired into CI alongside Rust/JS interop. + +Benchmark origins, broadcasts per origin, neighbors, active tracks/fronts, +and announce cursors independently. A route update must not expand back into +per-broadcast table rewrites or poll unrelated fronts. Measure messages, bytes, +wakes/select calls, CPU, replay work, and retained state. One broadcast per +origin is an explicit case. Coordinate with Route wakes and Announce replay; +benchmark smoke alone does not enforce complexity, so add operation-count +regressions. Bound pending requests and freshness state under same-clock churn. + +## Required + +- [Route session integration](/quest/m1/cluster-routing/routes/session.md) - the implementation under test +- [js/net route layer](/quest/m1/cluster-routing/routes/js.md) - the JS side of the interop checks + +## Related + +- [Routes and announces](/quest/m1/cluster-routing/routes/README.md) - the contract and its fixtures diff --git a/quest/m1/cluster-routing/routes/availability.md b/quest/m1/cluster-routing/routes/availability.md new file mode 100644 index 00000000000..eeeb9dbf560 --- /dev/null +++ b/quest/m1/cluster-routing/routes/availability.md @@ -0,0 +1,30 @@ +# [M] Announcement availability + +## Goal + +An announcement carries availability, and letting go is a distinct offline +event, separate from the origin's END, in Rust and in `js/net`'s +announcement API (decision 3 of the epic's handoff proof). A held route +shows its claims unavailable; a let-go shows them offline; only the origin +ends them. Bindings mirror Rust. + +## Plan + +- Decided 2026-10-16: an announcement carries availability, and letting go + is a distinct offline event, separate from the origin's END. Rejected: + presence alone, which gives a reader nothing to re-request on when a held + route recovers; let-go reported as END, as today; and availability with + let-go kept as END, which hides whether the origin ended or the network + lost it. +- Build on [Listing and follow](/quest/m1/api-audit/route/events.md)'s + listing and its offline report rather than adding event kinds beside it. +- JS gets the API shape here, fed by what `js/net` knows today; the + [js/net route layer](/quest/m1/cluster-routing/routes/js.md) feeds it from the wire. + +## Required + +- [Route session integration](/quest/m1/cluster-routing/routes/session.md) - the router whose held and let-go states this reports + +## Related + +- [Routes and announces](/quest/m1/cluster-routing/routes/README.md) - the handoff contract's held and let-go states diff --git a/quest/m1/cluster-routing/routes/js.md b/quest/m1/cluster-routing/routes/js.md new file mode 100644 index 00000000000..8a97d15c8f4 --- /dev/null +++ b/quest/m1/cluster-routing/routes/js.md @@ -0,0 +1,29 @@ +# [L] js/net route layer + +## Goal + +`js/net` encodes, decodes, and resolves lite-07's route layer: it reads +ROUTE and ANNOUNCE_FLOOR, merges announcements by floor as the contract +says, echoes every ROUTE_END, and pins each request to the identity it +selected. Browsers stay non-transit: a tab advertises its own identity and +publications only, with `proxy` off, until the +[Rust-to-TypeScript](/quest/m2/rs2ts-js.md) port; P2P reuses this model +rather than adding a browser forwarding engine. + +## Plan + +Decided 2026-10-27 (maintainer), from +[moq-dev/moq#6947](https://github.com/moq-dev/moq/pull/6947): in `js/net`'s +`origin.ts` a detached request stays on its own route instead of following +the cheapest one; fix it with the route-selection rework here. +- Feed the availability API from [Announcement availability](/quest/m1/cluster-routing/routes/availability.md). +- Rust/JS interop covers the route messages, the floor list, and the echo. + +## Required + +- [lite-07 route wire](/quest/m1/cluster-routing/routes/wire.md) - the messages +- [Announcement availability](/quest/m1/cluster-routing/routes/availability.md) - the API this feeds + +## Related + +- [Routes and announces](/quest/m1/cluster-routing/routes/README.md) - the contract diff --git a/quest/m1/cluster-routing/routes/session.md b/quest/m1/cluster-routing/routes/session.md new file mode 100644 index 00000000000..ec2395ad365 --- /dev/null +++ b/quest/m1/cluster-routing/routes/session.md @@ -0,0 +1,160 @@ +# [XL] Route session integration + +## Goal + +One `Router` per origin node, fed by every lite-07 session, drives that +node's origin and every announce stream it answers. Its shown claims and +selected routes become the origin's routes, its outbox becomes each +stream's ROUTE and ANNOUNCE messages, and the first hop pins each request +to the identity it selected. A lite-06 peer is translated at the relay. A +peer can make the router hold only bounded state. The configuration +changes the contract decided ship with it. + +## Plan + +- Scope route state to interested origins, not the CDN's whole node table. + Send a route before any announcement referencing it. The epic's handoff contract + defines held routes, letting go, and collection. +- ✅ Decided 2026-10-15 (maintainer): retract and re-file when a session's + grant union crosses the scoped boundary in either direction (a `**` token + added, or its expiry). Every forwarded route the session delivered is + retracted, and its current advertisements are re-filed under the new key, + so no route, feasibility distance, or alias stays under the old one. + Expiry cannot be refused, so refusing the transition was not an option. +- Only the first hop selects the origin. It keeps `route_order`'s shape over + the origin nodes announcing the path: longest prefix, newest epoch (within + one instance only, per [Instance and epoch](https://github.com/moq-dev/moq/pull/7489), + decided 2026-10-30), the link's priority (`origin::Producer::with_preference`), seed plus route + metric, then a rendezvous hash of the path and the identity, and the newest + connection only for an otherwise identical route. The hash drops the hop + chain, which differs per node, so every node agrees on a pool member + (`pool_ties_agree_only_on_a_path_and_identity_hash`). The request then names + that identity, and every later hop follows its own route to it, so + forwarding needs a route but not the claim. +- Reselecting the origin at every hop loops although each origin's routes are + loop-free (RFC 8966 s2.7): a ring of same-epoch origins whose relays + prioritize different links, and nested wildcards where a relay has not + heard the specific claim, both cycle past requester exclusion; joining + another relay's subscription by path or epoch deadlocks; and without an + epoch a later hop splices another publisher under the requester's + announcement with no Restart (`local_priorities_loop_path_only_requests`, + `overlapping_wildcards_loop_where_specific_claims_are_unheard`, + `aggregation_cycles_starve_without_a_forwarding_constraint`, + `epochless_reselection_splices_another_publisher`, + `sweep_forwarding_progress`). A relay therefore joins a forwarded request + only to a subscription following its route to the same identity. When that + identity becomes unreachable the forwarded request ends, and the first hop + re-resolves, resuming only on a matching epoch. A request carrying a cost + bound instead also never loops, but stalls where a transit hop has not + heard the claim and still splices epochless sources. +- Mixed versions: a lite-06 peer keeps today's path vector, translated at the + relay that speaks both, for the rollout window only. +- Interest changes: key the router by announce stream, not session. The old + stream stays open, its routes counting and its batches discarded, until + the new stream has answered, so no route drops in between. A stream is + just another neighbor key to the router. +- A batch carries changes against the last batch on its stream. Restricting + a view to an interest is still linear in that origin's claims per change; + make it incremental here. +- Bound what a peer makes the router hold (routes, claims, views per + session) by design, as the session caps announcements today. +- Public API: the route-change surface on `broadcast::Route` and its + bindings will likely change, `Hops` and saturating `Cost` go, zero-cost + links are refused (update `?cost=0` examples and report the breaking + configuration change), a per-connection `proxy` preference is added, and + `cluster.id`/`--cluster-id` is removed rather than kept as an unused label + (decided 2026-10-13). `Preference` continues to rank feasible transit + routes; report it. +- Open, for the implementer to settle and record: how much of today's route + trie, fronts, and `route_order` survives intact. + +Decided 2026-10-27 (maintainer): the route layer deletes the epochless instance +machinery in `rs/moq-net/src/model/origin.rs`: `Instance::Route(generation)` +and its renewals, `restarted_since`, the held-restart Release for epochless +routes, and Selection retirement, leaving Follow's same-epoch +Update/Restart split as the only takeover rule. Once each origin is its own +ANNOUNCE and the first hop pins the request to its identity, the first hop's +route is the instance, so the per-hop renewal has nothing left to protect. +An epochless change of serving route stays a Restart, including the +same-origin next-hop change in the [wire sketch](/quest/m1/cluster-routing/routes/wire.md), and the routing +contract's epochless fixtures keep passing. Minting an epoch for every route +instead was tried and rejected +([moq-dev/moq#6948](https://github.com/moq-dev/moq/pull/6948)): each relay +mints at its own time, so mint order outranks cost, preference, and the +local-broadcast tie and relays disagree on a pool path's winner, while +ranking a minted epoch last keeps the renewal it was meant to delete. + +Moved 2026-10-30 (maintainer, the [route epic](/quest/m1/api-audit/route/README.md)): +[Instance and epoch](https://github.com/moq-dev/moq/pull/7489) now deletes that +epochless machinery, so the route layer no longer does. A session mints +an `instance` (`~` plus random bits) for any route that +arrives without one, and newest-wins compares epochs only within one +instance, so mint order no longer outranks cost or preference. The rest of +the 2026-10-27 decision stands. The broadcast epoch fields here become that +publisher instance and its per-instance epoch, in place of the UUIDv7 +epoch. The route layer's own origin-node identity, with its seqno and +revision, stays separate ("keep administrative identity and broadcast epoch +separate", in the epic). + +## Waiting + +- Maintainer: how a scoped peer's routing identities are keyed. The router + takes the key from the caller (`Scope::qualify`), so nothing else waits on + it. Recommendation: (1), a route label carried in the grant. + + What the key is for. Every origin declares a routing identity: an opaque + number nobody verifies. A peer admitted under a scoped grant (say it may + publish only `acme/**`) could declare a victim's identity. One route + serves every claim of an identity, so the forgery would capture the + victim's claims outside `acme/`, and its seqno would outlive it + (`forged_identity_hijacks_claims_outside_the_forgers_scope`). So a relay + files every identity a scoped peer sends under a qualified key, derived + from something about that peer and the identity, and counts claims under + the key only inside the grant. The question is what that something is. + It must stay the same across token refresh, be shared by one operator's + sessions so losing one uplink fails over instead of restarting readers + (`scoped_delivery_through_two_uplinks_reaches_one_key`), differ between + operators, and work for sessions admitted with no token (public rules, + mTLS certificates, or a third-party auth server). + + Why not the credential subject: `hash(subject, id)` was decided + 2026-10-15 and dropped by the maintainer 2026-11-02, together with + refusing scoped tokens that carry no subject; the implementation is in + [moq-dev/moq@453e1ec](https://github.com/moq-dev/moq/commit/453e1ece7c5435ee19541cfc2e9bbe0b6fbc430e). + It existed only for tokens, so token-less admission had no key. Rejected + earlier: the grant's pattern union, which refresh and withdrawal change + and two uplinks need not share, and the connection root, which two scoped + operators under one root would share. + + Example: operator Acme runs a local relay L with uplinks to CDN relays C1 + and C2. Each admits L with a token scoped to `acme/**`, refreshed hourly. + + 1. A route label carried in the grant. Whoever issues it names it: a + token claim such as `route: "acme"`, a public rule's entry, or an mTLS + mapping. The key hashes the label with the identity in its own domain. + Both of Acme's tokens say `acme`, so C1 and C2 file L's origins under + one key, refresh keeps it, and losing C1 fails over through C2. + Another operator's label differs. A grant with no label gets a + per-session key, as in (2). Costs a grant field in `moq-auth`, + `js/auth`, and the docs, and issuers must not hand two operators one + label. + 2. A per-session key, with nothing to configure. L's sessions to C1 and + C2 get two keys, so losing C1 restarts the readers it served onto C2's + key instead of failing over, and so does a refresh that reconnects. + Simplest; gives up the two-uplink fixture. + 3. A key the relay operator configures per scoped remote, such as C1 and + C2 both mapping admissions under `acme/**` to `acme`. No grant change, + but every CDN relay's configuration must agree, and it names patterns + rather than who connected, so two operators admitted under one pattern + share a key. + +## Required + +- [lite-07 route wire](/quest/m1/cluster-routing/routes/wire.md) - the messages this drives +- [One Route handle](/quest/m1/api-audit/route/handle.md) - the origin's route surface this rewrites +- [Listing and follow](/quest/m1/api-audit/route/events.md) - the announcement events this feeds ([moq-dev/moq#7678](https://github.com/moq-dev/moq/pull/7678)) + +## Related + +- [Routes and announces](/quest/m1/cluster-routing/routes/README.md) - the contract +- [Route wakes](/quest/m2/route-wakes.md) - index work follows the settled source-selection model diff --git a/quest/m1/cluster-routing/routes/wire.md b/quest/m1/cluster-routing/routes/wire.md new file mode 100644 index 00000000000..581fd5b6857 --- /dev/null +++ b/quest/m1/cluster-routing/routes/wire.md @@ -0,0 +1,91 @@ +# [L] lite-07 route wire + +## Goal + +lite-07 carries the route layer of [Routes and announces](/quest/m1/cluster-routing/routes/README.md) on +the announce stream: ROUTE_START, ROUTE_UPDATE, ROUTE_END, and +ANNOUNCE_FLOOR toward the subscriber, and ROUTE_REQUEST and the ROUTE_END +echo back toward the publisher. ANNOUNCE_START and ANNOUNCE_UPDATE carry a +Route ID and the origin's seed in place of the hop list. Rust encodes and +decodes it, and `drafts/draft-lcurley-moq-lite.md` specifies it. Driving the +router with it is [session integration](/quest/m1/cluster-routing/routes/session.md). + +## Plan + +- Wire sketch, flexible in detail. ROUTE_START carries the node id, a seqno, + the metric, and the down-only bit (the epic's one-selection decision), and implicitly takes the next + stream-local Route ID, as ANNOUNCE_START takes the next Announce ID today. + ROUTE_UPDATE (Route ID, seqno, metric, down-only) and ROUTE_END (Route + ID) reference it; the bit rides UPDATE so a route moving between a + mesh path and a CDN fallback changes it without retracting its announces. + ANNOUNCE_START carries the prefix (keeping today's Path Base/Keep + compression), a Route ID, the origin's cost for that prefix, and the epoch, + and lite-07's restart message from + Restart keeps its meaning. On a + route without an epoch, the source is the origin node and its ANNOUNCE: a + new one at the same path is a restart. This is routing provenance, not + permission to stitch. A concrete serving-route change without an epoch + remains Restart even when the origin node is unchanged; only a matching + explicit epoch or the separately proven Takeover operation preserves an + instance (superseded 2026-10-30: a shared route `instance` preserves it, per + [Instance and epoch](https://github.com/moq-dev/moq/pull/7489)). Test a same-origin next-hop change with no epoch. + ANNOUNCE_UPDATE re-prices it and ANNOUNCE_END withdraws this origin's claim + while the route stays up. A wildcard claim is capability, not an inventory + of broadcasts; another origin may still serve the same epoch. An announce + never changes its Route ID: another origin + serving the same path is another ANNOUNCE. An ANNOUNCE naming an unknown + Route ID is a protocol violation. A ROUTE_UPDATE with an infinite metric + (2^64-1, the largest lite-07 varint; a sum reaching it is unreachable) + means held: the sender has no usable route but still holds the origin's + announcements while it recovers. ROUTE_END means the sender let go of the + origin on this stream; neither is proof that the origin withdrew its + claims (see the epic's handoff contract). Each batch of ANNOUNCEs for a route + closes with one ANNOUNCE_FLOOR listing every region the sender certifies + for that route: a count, then per region a suffix of the stream's interest + prefix (empty for the whole interest) and the origin revision as of which + the stream's set for that route in that region is complete. The list + replaces the previous one, so a region left out is no longer certified, + and a count of zero answers a stream whose interest none of the sender's + regions meets. + ROUTE_REQUEST (node id, seqno) travels + the other way on the same stream, toward the neighbour that sent the route, + and so does the echo of every ROUTE_END: the receiver sends the same + ROUTE_END (Route ID) back once it has dropped the route, which tells the + sender no neighbour still holds its advertisement (see the epic's + handoff contract). Every receiver echoes, clients included; a closed session + counts as the echo. + The hop list leaves lite-07, and with it the + `Hop Base`/`Hop Keep` compression; `cluster.max_hops` then stops applying + to lite-07. Most of the 62% saving measured in + [moq-dev/moq#4196](https://github.com/moq-dev/moq/pull/4196) came from hop + chains, so rerun its announce bench on path-less ANNOUNCE and drop + `Path Base`/`Path Keep` too unless it still pays for its rules. + + ```text + ROUTE_START node=0x7a3f seqno=41 metric=12 -> route 0 + ANNOUNCE_START prefix=transcode/ route=0 cost=10 -> announce 0 + ANNOUNCE_START prefix=transcode/foobar route=0 cost=1 -> announce 1 + ANNOUNCE_FLOOR route=0 count=1 ""@7 # complete as of revision 7 + ROUTE_UPDATE route=0 seqno=42 metric=15 + ANNOUNCE_END announce=1 # this origin withdraws a claim + ANNOUNCE_FLOOR route=0 count=1 ""@8 + ANNOUNCE_FLOOR route=1 count=2 "live/a/"@3 "live/b/"@5 # a scoped key + ANNOUNCE_FLOOR route=1 count=1 "live/b/"@6 # live/a/ is no longer certified + ROUTE_UPDATE route=0 seqno=42 metric=inf # held: unusable, still recovering + ROUTE_END route=0 # let go: forget it on this stream + ROUTE_END route=0 # <- the receiver echoes it back + ``` +- Requests forwarded between routing nodes name the serving identity + (decided 2026-10-13). ROUTE carries one down-only bit and no export metric. +- The region list's order and any other wire detail of it are settled here + with Rust and JS interop, not in the proof. The core keeps regions sorted + by path. +- A hop-less ANNOUNCE_START cannot run before the router drives sessions, so + the new messages may land first and the ANNOUNCE layout switch with + [session integration](/quest/m1/cluster-routing/routes/session.md), as long as every lite-07 build + between them still interoperates with itself. + +## Related + +- [Routes and announces](/quest/m1/cluster-routing/routes/README.md) - the contract the wire carries +- [A job's exact route continues its claim's readers](/quest/m1/announce-takeover.md) - also edits lite-07's ANNOUNCE messages diff --git a/quest/m1/cut-audit/README.md b/quest/m1/cut-audit/README.md index 7441dd98a4e..64eebcb78ab 100644 --- a/quest/m1/cut-audit/README.md +++ b/quest/m1/cut-audit/README.md @@ -46,7 +46,7 @@ moqsrc pad actors and publish rendition identity (both since landed). Other redesign evidence went into the quests that already own the mechanism: [subscribe ranges](/quest/m1/subscribe-ranges/README.md), [drop set](/quest/m1/drop-set.md), -[per-origin routes](/quest/m1/cluster-routing/routes.md), +[per-origin routes](/quest/m1/cluster-routing/routes/README.md), drafts 14-17 removal (since landed), and [buffered playback](/quest/m1/buffered-ascending.md). The audit recommends leaving JS-only redesigns of `js/net` to [rs2ts](/quest/m2/rs2ts-js.md). diff --git a/quest/m1/lite07-finalize.md b/quest/m1/lite07-finalize.md index ed639960eb8..249dbc5d0b3 100644 --- a/quest/m1/lite07-finalize.md +++ b/quest/m1/lite07-finalize.md @@ -29,7 +29,7 @@ The route layer of [Cluster routing](/quest/m1/cluster-routing/README.md) lite-07, decided 2026-10-05. That removes lite-07's `Hop Base`/`Hop Keep` announce compression along with the hop list, and moq.pro's lite-07 rollout has to plan for a lite-07 without hop lists. Only -[Routes and announces](/quest/m1/cluster-routing/routes.md) gates the cut, +[Routes and announces](/quest/m1/cluster-routing/routes/README.md) gates the cut, not the later federation/P2P integrations. Decided 2026-10-13: routes now owns the tier/partition correctness tests, so they run before the cut. Peer grants and metadata-only subscriptions do not gate it: neither is clustering @@ -106,7 +106,7 @@ timestamp bound in [#6168](https://github.com/moq-dev/moq/pull/6168). ## Required -- [Routes and announces](/quest/m1/cluster-routing/routes.md) - lite-07 carries the route layer: ROUTE per origin node and path-less ANNOUNCE, with the hop list gone +- [Routes and announces](/quest/m1/cluster-routing/routes/README.md) - lite-07 carries the route layer: ROUTE per origin node and path-less ANNOUNCE, with the hop list gone - [Drop set](/quest/m1/drop-set.md) - lite-07's SUBSCRIBE_DROP carries `Max Age`, so a widening re-offers groups reported dropped - [Lite-07 ranges](/quest/m1/subscribe-ranges/lite.md) - a relay fills a lite-07 upstream through the subscription - [JS ranges](/quest/m1/subscribe-ranges/js.md) - JS speaks lite-07 ranges diff --git a/quest/m2/link-quality.md b/quest/m2/link-quality.md index 073f3ad2b63..014e99c80ce 100644 --- a/quest/m2/link-quality.md +++ b/quest/m2/link-quality.md @@ -37,4 +37,4 @@ Wire: none. ## Required -- [Routes and announces](/quest/m1/cluster-routing/routes.md) - a cost change costs one ROUTE per origin there, instead of a re-announce per broadcast +- [Routes and announces](/quest/m1/cluster-routing/routes/README.md) - a cost change costs one ROUTE per origin there, instead of a re-announce per broadcast diff --git a/quest/m2/mesh/README.md b/quest/m2/mesh/README.md index 604ba6da015..3c13dfa5ad5 100644 --- a/quest/m2/mesh/README.md +++ b/quest/m2/mesh/README.md @@ -46,7 +46,7 @@ Decided 2026-10-25 with the maintainer (planning interview), still in force: tracks. No ICE, STUN, or hole punching; the [P2P](/quest/m2/p2p/README.md) line is separate. - An endpoint ID is a persistent Ed25519 device key, iroh-style. It is not - the [routing identity](/quest/m1/cluster-routing/routes.md), which each + the [routing identity](/quest/m1/cluster-routing/routes/README.md), which each process declares fresh. - Addressing is announcement: each node publishes under `mesh//...`, and reaching an ID is subscribing there. There is no separate directory. diff --git a/quest/m2/p2p/cost-scopes.md b/quest/m2/p2p/cost-scopes.md index 04a75e30bd8..d6d1abfd6d3 100644 --- a/quest/m2/p2p/cost-scopes.md +++ b/quest/m2/p2p/cost-scopes.md @@ -47,7 +47,7 @@ in Peers and native configuration. Wire: none beyond shared routing and existing ## Required -- [Routes and announces](/quest/m1/cluster-routing/routes.md) - route identities and route ordering +- [Routes and announces](/quest/m1/cluster-routing/routes/README.md) - route identities and route ordering - [Signaling and policy](/quest/m2/p2p/signal.md) - explicit application peer selection - [moq-cli joins](/quest/m2/p2p/cli.md) - matching native controls diff --git a/quest/m2/p2p/signal.md b/quest/m2/p2p/signal.md index 713bdb4d49c..b94cf7bd43d 100644 --- a/quest/m2/p2p/signal.md +++ b/quest/m2/p2p/signal.md @@ -25,7 +25,7 @@ roster `` (also the perfect-negotiation tiebreak) is its public key, like an iroh endpoint id. Nothing proves the key yet: peers are trusted (see Trust). Which roster peer serves an announce is the application's explicit selection; moq maps nothing automatically. The route-layer node id stays declared and -unverified ([routes](/quest/m1/cluster-routing/routes.md)); if a use case +unverified ([routes](/quest/m1/cluster-routing/routes/README.md)); if a use case needs provable identity or signing, moq-lite-07's route identity becomes a public key, independent of P2P. P2P identity planning and peer grants were dropped until a real use case asks for them. Switching between a CDN and a diff --git a/quest/m2/perf/kio-channel-contention/README.md b/quest/m2/perf/kio-channel-contention/README.md index 0a004871759..7f543d7938b 100644 --- a/quest/m2/perf/kio-channel-contention/README.md +++ b/quest/m2/perf/kio-channel-contention/README.md @@ -454,7 +454,7 @@ Decided (maintainer, 2026-11-01, cluster spike): route it learned to every other peer, and hop chains prevent loops. How routes from outside the relay (other relays' peer links) reach every worker without re-announce is for the planning to settle. - [Routes and announces](/quest/m1/cluster-routing/routes.md) plans a + [Routes and announces](/quest/m1/cluster-routing/routes/README.md) plans a per-connection `proxy` preference (decided 2026-10-14, relays default on) that decides whether a connection is sent routes learned from other connections; a worker link with it off is this condition. diff --git a/quest/m2/route-health/selection.md b/quest/m2/route-health/selection.md index 6c50f7c2087..974379a1a89 100644 --- a/quest/m2/route-health/selection.md +++ b/quest/m2/route-health/selection.md @@ -42,4 +42,4 @@ report any required surface change. No independent wire extension planned. ## Required -- [Routes and announces](/quest/m1/cluster-routing/routes.md) - shared candidate and identity model +- [Routes and announces](/quest/m1/cluster-routing/routes/README.md) - shared candidate and identity model diff --git a/quest/m2/route-wakes.md b/quest/m2/route-wakes.md index fb146fe5a89..3d4e23abeb1 100644 --- a/quest/m2/route-wakes.md +++ b/quest/m2/route-wakes.md @@ -82,5 +82,5 @@ Public API: none. Wire: none. ## Related -- [Routes and announces](/quest/m1/cluster-routing/routes.md) - the identity and selection views the index serves +- [Routes and announces](/quest/m1/cluster-routing/routes/README.md) - the identity and selection views the index serves - [Front parking](/quest/m2/origin-front-parks.md) - replaces the `routed_broadcast` watch loop this wakes diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index 41d1bfe8fe4..0fe42e9c01b 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -5,7 +5,7 @@ //! asks, and only the identity's own node advances its seqno. Each origin //! versions its own claims with a revision, and a node merges every neighbor's //! certified view claim by claim, so no stale copy revives a withdrawal it -//! already knows. See `quest/m1/cluster-routing/routes.md` for the contract. +//! already knows. See `quest/m1/cluster-routing/routes/README.md` for the contract. //! //! The router does no I/O: the caller feeds it what each neighbor sent and //! drains [`Out`] messages and reader [`Event`]s. Neighbor keys `N` and claim diff --git a/rs/moq-net/tests/integration/mesh_withdraw.rs b/rs/moq-net/tests/integration/mesh_withdraw.rs index e80b8654bdf..3f3fd6338e3 100644 --- a/rs/moq-net/tests/integration/mesh_withdraw.rs +++ b/rs/moq-net/tests/integration/mesh_withdraw.rs @@ -443,7 +443,7 @@ async fn cross_node_close_without_hold_reannounces() { // Residual counterexamples on `main`, with the hold and the retraction // quarantine. Each pins what the cluster does today, so a change either way -// fails here. Routes and announces (quest/m1/cluster-routing/routes.md) must +// fails here. Routes and announces (quest/m1/cluster-routing/routes/README.md) must // bring each to a single retraction everywhere. /// A ring of six relays with links under 150 ms. Relay 3 hears the withdrawal diff --git a/rs/moq-net/tests/integration/routing_contracts/handoff.rs b/rs/moq-net/tests/integration/routing_contracts/handoff.rs index a84af6c6283..2f11a6e8949 100644 --- a/rs/moq-net/tests/integration/routing_contracts/handoff.rs +++ b/rs/moq-net/tests/integration/routing_contracts/handoff.rs @@ -6,7 +6,7 @@ //! view complete as of a revision for the region they cover. Receivers merge //! every neighbor's view claim by claim, keeping whichever knowledge is newer, //! and keep showing an origin's claims while any route to it is usable or -//! recovering. See `quest/m1/cluster-routing/routes.md`. +//! recovering. See `quest/m1/cluster-routing/routes/README.md`. use std::collections::BTreeSet; diff --git a/rs/moq-net/tests/integration/routing_contracts/mod.rs b/rs/moq-net/tests/integration/routing_contracts/mod.rs index 23e5259419f..b968dcda11a 100644 --- a/rs/moq-net/tests/integration/routing_contracts/mod.rs +++ b/rs/moq-net/tests/integration/routing_contracts/mod.rs @@ -1,6 +1,6 @@ //! Deterministic counterexamples for the per-origin routing contract. //! -//! Each fixture runs one scenario from `quest/m1/cluster-routing/routes.md` +//! Each fixture runs one scenario from `quest/m1/cluster-routing/routes/README.md` //! against [`model`], under the rule that fails and the rule that holds. The //! model is a reference for the contract, not the relay: when lite-07's route //! layer lands, these topologies become its acceptance tests. Failover fixtures diff --git a/rs/moq-net/tests/integration/routing_contracts/model.rs b/rs/moq-net/tests/integration/routing_contracts/model.rs index 12ce354d60a..4a1cca63da1 100644 --- a/rs/moq-net/tests/integration/routing_contracts/model.rs +++ b/rs/moq-net/tests/integration/routing_contracts/model.rs @@ -1,7 +1,7 @@ //! A reference model of the proposed per-origin route layer, small enough to read. //! //! It is not the implementation. It exists so the routing contract in -//! `quest/m1/cluster-routing/routes.md` has deterministic counterexamples before +//! `quest/m1/cluster-routing/routes/README.md` has deterministic counterexamples before //! the lite-07 wire is fixed: every rule the contract depends on is a switch //! here, and each fixture runs a scenario under the rule that fails and the rule //! that holds. From b816ef5e8f1863c4d7e8e8c69220a365e2444c25 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 09:53:40 -0800 Subject: [PATCH 6/8] fix(net): start a restarted route's announcements from an empty view A sender forgets what it sent for a route it ended, so its next snapshot on that stream carries no deletions. The receiver now applies that snapshot to an empty view rather than the one it held before the end, which revived a claim withdrawn in between. Reconciling the shown claims is also keyed, not quadratic in an origin's claims. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/mod.rs | 12 +++++++++-- rs/moq-net/src/route/tests/handoff.rs | 29 +++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index 0fe42e9c01b..51a9d596798 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -617,6 +617,14 @@ impl Router { if routes.is_empty() { self.rx.remove(&id); } + // The sender forgets what it sent for a route it ended, so a restart + // starts from an empty view: apply the next batch to that, keeping the + // last view only for what it certified. + if reach.is_none() + && let Some(heard) = self.heard.get_mut(&(id, from)) + { + heard.raw = View::default(); + } self.update([id]); } @@ -1264,7 +1272,7 @@ impl Router { } } } - let old: Vec<(Path, Shown)> = self + let old: BTreeMap> = self .shown .range((id, Path::empty())..) .take_while(|((key, _), _)| *key == id) @@ -1302,7 +1310,7 @@ impl Router { } } for (claim, now) in shown { - if !old.iter().any(|(known, _)| *known == claim) { + if !old.contains_key(&claim) { self.log(id, &claim, Kind::Start, &now.item); } self.shown.insert((id, claim), now); diff --git a/rs/moq-net/src/route/tests/handoff.rs b/rs/moq-net/src/route/tests/handoff.rs index 4ba49e688dc..d0ea5576588 100644 --- a/rs/moq-net/src/route/tests/handoff.rs +++ b/rs/moq-net/src/route/tests/handoff.rs @@ -1294,3 +1294,32 @@ fn a_link_flap_costs_routes_not_broadcasts() { fn next_hop(sim: &Sim, node: Node) -> Option { sim.route(node, 7).and_then(|route| route.next) } + +/// A neighbor ends a route, then advertises it again on the same stream. Its +/// new snapshot is complete, so a claim withdrawn in between must not survive +/// from the view it sent before the end. +#[test] +fn a_restarted_route_sends_a_complete_snapshot() { + let (origin, relay, node) = (0, 1, 2); + let mut sim = Sim::new(3); + sim.leaf(origin); + sim.join(origin, relay, 1); + sim.join(relay, node, 1); + sim.originate(origin, 7, 0); + sim.announce(origin, "live/x", 0, None); + assert!(sim.run()); + assert_eq!(sim.shows(node, 7, "live/x"), Some(true)); + + sim.disconnect(origin, relay); + assert!(sim.run()); + assert!(!sim.holds(node, 7), "the relay let go"); + // The relay forgets the origin, so its next snapshot starts empty. + sim.collect(relay, 7); + sim.unannounce(origin, "live/x"); + sim.connect(origin, relay); + assert!(sim.run()); + assert!(sim.route(node, 7).is_some()); + assert_eq!(sim.shows(node, 7, "live/x"), None); + assert_eq!(revivals(&sim.events), 0); + sim.check().unwrap(); +} From a3975e821dfb6aefd61bdfb04e090ccf224da11e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 10:58:37 -0800 Subject: [PATCH 7/8] quest: add an interest-scoped route transit fixture to routes acceptance Co-Authored-By: Claude Opus 5.5 --- quest/m1/cluster-routing/routes/acceptance.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/quest/m1/cluster-routing/routes/acceptance.md b/quest/m1/cluster-routing/routes/acceptance.md index 0aab45bbdf0..366dc1d941d 100644 --- a/quest/m1/cluster-routing/routes/acceptance.md +++ b/quest/m1/cluster-routing/routes/acceptance.md @@ -21,6 +21,16 @@ replace are retired. quarantine, [moq-dev/moq#5391](https://github.com/moq-dev/moq/pull/5391)'s dedup, and the 300 ms hold only once the fixtures below demonstrate their replacement (see the epic's hold-down note). +- Add a transit fixture for routes scoped to interested origins (the + session child's "scope route state to interested origins"). The router + sends every route to every watching neighbor today, so a client is sent + the CDN's whole node table. The fixture decides when a route must still + go out to a stream whose interest meets none of the origin's claims: + - a transit node forwarding requests for paths outside its own interest; + - a node holding or forwarding a seqno request for that identity; + - a sender whose certified regions miss the interest, which must still + answer with an empty floor list + (`a_disjoint_interest_is_answered_with_nothing`). The counterexamples from #4349 (closed; this epic still owns the fix) are the `residual_*` tests in `rs/moq-net/tests/integration/mesh_withdraw.rs`. They pin today's hunting on lite-06: From 6062a939084b547eac7e129e9553996842dd6511 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 11:49:04 -0800 Subject: [PATCH 8/8] fix(net): an unreachable route neither certifies nor holds an identity A route whose metric plus the link cost reaches the infinite metric was already refused by selection, but still counted as a certifier and as a recoverable held route, so its regions showed claims available and a held copy kept the identity from being let go. Both now apply the same reachability test. The fixtures' freshness check also refuses a claim the origin never made, and a fixture pins a covering claim across a disjoint interest change. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/mod.rs | 17 ++++++--- rs/moq-net/src/route/tests/handoff.rs | 36 +++++++++++++++++++ rs/moq-net/src/route/tests/mod.rs | 3 ++ rs/moq-net/src/route/tests/routing.rs | 50 +++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 5 deletions(-) diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index 51a9d596798..b14b9ffa373 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -938,14 +938,15 @@ impl Router { } /// Whether this node holds a held route to `id` it could recover through. - /// One counts only if its last finite advertisement is feasible here, so + /// One counts only if its last finite advertisement is feasible and + /// reachable here, so /// holds follow a strictly decreasing metric toward a starved node and /// cannot hold each other up. fn recoverable(&self, id: Id) -> bool { self.rx.get(&id).is_some_and(|routes| { routes - .values() - .any(|entry| entry.held && self.feasible(id, entry.advert)) + .iter() + .any(|(&next, entry)| entry.held && self.reachable(id, next, entry)) }) } @@ -956,18 +957,24 @@ impl Router { } /// The neighbors whose views can certify `id`: those whose last finite - /// advertisement is feasible here, held or not. Feasible edges strictly + /// advertisement is feasible and reachable here, held or not. Feasible edges strictly /// decrease the metric toward the origin, so certification never circulates. fn certifiers(&self, id: Id) -> Vec { self.rx.get(&id).map_or_else(Vec::new, |routes| { routes .iter() - .filter(|(_, entry)| self.feasible(id, entry.advert)) + .filter(|&(&next, entry)| self.reachable(id, next, entry)) .map(|(&next, _)| next) .collect() }) } + /// Whether `entry`'s last finite advertisement from `next` is feasible here + /// and reachable across the link, as selection requires of a usable route. + fn reachable(&self, id: Id, next: N, entry: &Entry) -> bool { + self.feasible(id, entry.advert) && add(entry.advert.metric, self.neighbors[&next].link.cost).is_some() + } + /// The regions of `id` that this node's certifiers certify, those whose /// route is held only when `held`, or `None` while it holds no route or a /// certifier has not yet answered the current interest. diff --git a/rs/moq-net/src/route/tests/handoff.rs b/rs/moq-net/src/route/tests/handoff.rs index d0ea5576588..4c72ca7a1b6 100644 --- a/rs/moq-net/src/route/tests/handoff.rs +++ b/rs/moq-net/src/route/tests/handoff.rs @@ -1323,3 +1323,39 @@ fn a_restarted_route_sends_a_complete_snapshot() { assert_eq!(revivals(&sim.events), 0); sim.check().unwrap(); } + +/// A claim covering both interests stays shown when a client moves to a +/// disjoint interest: what it knows of that claim is current as of the old +/// interest, which carried every change to it until the switch. A withdrawal +/// still queued on the old stream is discarded with it and arrives on the +/// new stream's snapshot instead: the claim ends once, never revives, and no +/// END is false. +#[test] +fn a_covering_claim_survives_a_disjoint_interest_until_the_new_snapshot() { + let (origin, relay, client) = (0, 1, 2); + let mut sim = Sim::new(3); + sim.leaf(origin); + sim.leaf(client); + sim.join(origin, relay, 1); + sim.join(relay, client, 1); + sim.watch(client, "live/a"); + sim.originate(origin, 7, 0); + sim.announce(origin, "live", 0, None); + assert!(sim.run()); + assert_eq!(sim.shows(client, 7, "live"), Some(true)); + + sim.hold(relay, client); + sim.unannounce(origin, "live"); + assert!(sim.run()); + sim.watch(client, "live/b"); + assert!(sim.run()); + assert_eq!(sim.shows(client, 7, "live"), Some(true), "lags like any snapshot"); + + sim.release(relay, client); + assert!(sim.run()); + assert_eq!(sim.shows(client, 7, "live"), None); + assert_eq!(life(&sim, client, "live"), [Kind::Start, Kind::End]); + assert_eq!(revivals(&sim.events), 0); + assert_eq!(false_ends(&sim.events), 0); + sim.check().unwrap(); +} diff --git a/rs/moq-net/src/route/tests/mod.rs b/rs/moq-net/src/route/tests/mod.rs index cb58f130467..7c46f61cfd5 100644 --- a/rs/moq-net/src/route/tests/mod.rs +++ b/rs/moq-net/src/route/tests/mod.rs @@ -636,6 +636,9 @@ impl Sim { }; checked.insert((node, key)); let claims: BTreeSet<&Path> = self.history[&id].iter().map(|change| &change.claim).collect(); + if let Some(claim) = view.items.keys().find(|claim| !claims.contains(claim)) { + return Err(format!("node {node} holds {key}'s {claim} with no origin history")); + } for claim in claims { let Some(rev) = view.as_of(claim) else { if view.items.contains_key(claim) { diff --git a/rs/moq-net/src/route/tests/routing.rs b/rs/moq-net/src/route/tests/routing.rs index f31c49811f4..2397704c8c7 100644 --- a/rs/moq-net/src/route/tests/routing.rs +++ b/rs/moq-net/src/route/tests/routing.rs @@ -780,3 +780,53 @@ fn a_message_recomputes_only_its_identity() { ); } } + +/// A route whose metric plus the link cost reaches the infinite metric is +/// unreachable, so it neither certifies regions nor keeps an identity held: +/// only the reachable neighbor's region shows, and a held overflowing route +/// is let go. +#[test] +fn an_overflowing_route_neither_certifies_nor_holds() { + let finite = |metric| { + Some(Reach::Finite(Advert { + seqno: 0, + metric, + down_only: false, + })) + }; + let floors = |region: &str| BTreeMap::from([(p(region), 1)]); + let item = Item { + seed: 0, + epoch: None, + since: 1, + }; + let (near, far) = (1, 2); + let far_link = Link { + cost: MAX - 1, + ..Link::default() + }; + + let mut router: Router = Router::new(1); + router.connect(near, Link::default()).unwrap(); + router.connect(far, far_link.clone()).unwrap(); + router.route(near, 7, finite(1)); + router.batch(near, 7, vec![(p("live/a/x"), Some(item))], floors("live/a"), 0); + router.route(far, 7, finite(1)); + router.batch(far, 7, vec![(p("live/b/y"), Some(item))], floors(""), 0); + assert_eq!(router.selected(7).and_then(|route| route.next), Some(near)); + assert_eq!(router.shows(7, &p("live/a/x")), Some(true)); + assert_eq!( + router.shows(7, &p("live/b/y")), + None, + "only the unreachable route certifies it" + ); + + let mut router: Router = Router::new(1); + router.connect(far, far_link).unwrap(); + router.route(far, 7, finite(1)); + router.batch(far, 7, vec![(p("live/x"), Some(item))], floors(""), 0); + router.route(far, 7, Some(Reach::Held)); + assert!(router.selected(7).is_none()); + assert!(!router.grounded(7), "an unreachable held route recovers nothing"); + assert_eq!(router.shows(7, &p("live/x")), None); +}