From 898f3ec56659c33f6d58cbb94a72f42f2534cc27 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 02:48:18 -0800 Subject: [PATCH 01/17] chore(quest): claim quest/m2/multicast/router.md Quest-Claim: 393a0f228b5f0ff0 From 8eca12a096fa071dbb4f54b2a7e70bef55dbee12 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 03:20:43 -0800 Subject: [PATCH 02/17] feat(moq-net): the multicast route rule as route::relay::Router The bake-off's pick (moq-dev/moq#8880) as one node's sans-IO state machine beside path::Router: routes to forwarding nodes only, dead hops named once per neighbour, session floors per pair of relays ordered by start then session, drains named once, one advertisement per burst, claims per forwarding node ordered by revision and certified by floors, chains across scoped grants, and one mark. The reader is keyed by Instance and debounces unavailable to offline on a mocked clock. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/mod.rs | 20 +- rs/moq-net/src/route/reader.rs | 55 +- rs/moq-net/src/route/relay.rs | 2123 ++++++++++++++++++++++++++++++++ 3 files changed, 2169 insertions(+), 29 deletions(-) create mode 100644 rs/moq-net/src/route/relay.rs diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index c7290a0fa1a..28b16768c4b 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -20,6 +20,8 @@ pub(crate) mod node; pub(crate) mod path; mod proof; mod reader; +#[cfg_attr(not(test), allow(dead_code, reason = "driven only by tests until the switch"))] +pub(crate) mod relay; mod view; #[cfg(test)] @@ -208,9 +210,11 @@ pub(crate) enum Kind { Unavailable, } +/// A change the reader sees: `id` is the publisher a path router keys claims by, or +/// the Instance under [`relay::Router`]. #[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct Event { - pub id: Id, +pub(crate) struct Event { + pub id: K, pub claim: Path, pub kind: Kind, pub item: A, @@ -233,4 +237,16 @@ pub(crate) enum Error { Nested, #[error("no such interest prefix")] UnknownPrefix, + #[error("a neighbor connected twice")] + Duplicate, + #[error("a routing neighbor asked for part of the tree")] + Partial, + #[error("a claim outside what was asked, or anything before the answer began")] + Unasked, + #[error("claims or news about no route the neighbor offered")] + Unrouted, + #[error("a route or chain the rule never builds")] + Malformed, + #[error("a message the neighbor's role never sends")] + Role, } diff --git a/rs/moq-net/src/route/reader.rs b/rs/moq-net/src/route/reader.rs index 89d6f3c199d..d08b1ac15cb 100644 --- a/rs/moq-net/src/route/reader.rs +++ b/rs/moq-net/src/route/reader.rs @@ -1,5 +1,6 @@ //! What a node's announcement reader sees, and the events it logs as that changes: -//! one implementation for every route rule. +//! one implementation for every route rule, keyed by whatever names a claim's owner +//! there (a publisher under the path rule, an Instance under the relay rule). use std::collections::{BTreeMap, BTreeSet}; @@ -10,18 +11,18 @@ use super::{Event, Id, Kind, Shown}; /// How the reader sees each claim, and the events not yet taken. #[derive(Clone, Debug)] -pub(crate) struct Reader { - shown: BTreeMap<(Id, Path), Shown>, +pub(crate) struct Reader { + shown: BTreeMap<(K, Path), Shown>, /// The identities recomputed since [`Self::changed`] last took them, each with /// the claims whose shown state changed. - changed: BTreeMap>, - events: Vec>, + changed: BTreeMap>, + events: Vec>, /// Per claim this node has shown, how many times it forgot that claim. #[cfg(test)] - forgot: BTreeMap<(Id, Path), u32>, + forgot: BTreeMap<(K, Path), u32>, } -impl Default for Reader { +impl Default for Reader { fn default() -> Self { Self { shown: BTreeMap::new(), @@ -33,38 +34,38 @@ impl Default for Reader { } } -impl Reader { - pub fn shown(&self) -> &BTreeMap<(Id, Path), Shown> { +impl Reader { + pub fn shown(&self) -> &BTreeMap<(K, Path), Shown> { &self.shown } /// How the reader sees `claim` of `id`. - pub fn get(&self, id: Id, claim: &Path) -> Option<&Shown> { + pub fn get(&self, id: K, claim: &Path) -> Option<&Shown> { self.shown.get(&(id, claim.clone())) } /// The claims of `id` the reader sees. - pub fn claims(&self, id: Id) -> impl Iterator { + pub fn claims(&self, id: K) -> impl Iterator { self.shown - .range((id, Path::empty())..) + .range((id.clone(), Path::empty())..) .take_while(move |((key, _), _)| *key == id) .map(|((_, claim), _)| claim) } /// Take the identities recomputed since the last call, each with the claims whose /// shown state changed. - pub fn changed(&mut self) -> BTreeMap> { + pub fn changed(&mut self) -> BTreeMap> { std::mem::take(&mut self.changed) } /// Take the events, in order. - pub fn events(&mut self) -> Vec> { + pub fn events(&mut self) -> Vec> { std::mem::take(&mut self.events) } /// Count a forget against each claim of `id` this node has shown that `kept` does not keep. #[cfg_attr(not(test), allow(unused_variables, reason = "only fixtures count forgets"))] - pub fn forget(&mut self, id: Id, kept: impl Fn(&Path) -> bool) { + pub fn forget(&mut self, id: K, kept: impl Fn(&Path) -> bool) { #[cfg(test)] for ((key, claim), count) in self.forgot.iter_mut() { if *key == id && !kept(claim) { @@ -76,13 +77,13 @@ impl Reader { /// Assert that `now`, recomputed for some claims of `id`, agrees with `full`, /// recomputed for all of them. #[cfg(test)] - pub fn check(&self, id: Id, now: &BTreeMap>>, full: BTreeMap>>) { + pub fn check(&self, id: K, now: &BTreeMap>>, full: BTreeMap>>) { for (claim, full) in full { let got = match now.get(&claim) { Some(got) => got.as_ref(), - None => self.get(id, &claim), + None => self.get(id.clone(), &claim), }; - assert_eq!(got, full.as_ref(), "incremental reader view of {id} {claim:?}"); + assert_eq!(got, full.as_ref(), "incremental reader view of {id:?} {claim:?}"); } } @@ -91,18 +92,18 @@ impl Reader { /// as far as this node knows, and went offline if the node let go of it instead. pub fn apply( &mut self, - id: Id, + id: K, watched: impl Fn(&Path) -> bool, now: BTreeMap>>, ended: impl Fn(&Path) -> bool, ) { let shown = |claim: &Path| match now.get(claim) { Some(now) => now.as_ref(), - None => self.get(id, claim), + None => self.get(id.clone(), claim), }; let mut events = Vec::new(); for (claim, now) in &now { - let Some(was) = self.get(id, claim) else { + let Some(was) = self.get(id.clone(), claim) else { continue; }; match now { @@ -138,7 +139,7 @@ impl Reader { } for (claim, now) in &now { if let Some(now) = now - && self.get(id, claim).is_none() + && self.get(id.clone(), claim).is_none() { events.push((claim, Kind::Start, &now.item)); } @@ -147,13 +148,13 @@ impl Reader { .map(|(claim, kind, item)| (claim.clone(), kind, item.clone())) .collect(); for (claim, kind, item) in events { - self.log(id, claim, kind, item); + self.log(id.clone(), claim, kind, item); } // Named as recomputed whether or not anything it shows changed, so a driver // serves its claims through the routes it has now. - let changed = self.changed.entry(id).or_default(); + let changed = self.changed.entry(id.clone()).or_default(); for (claim, now) in now { - let key = (id, claim); + let key = (id.clone(), claim); if self.shown.get(&key) == now.as_ref() { continue; } @@ -169,9 +170,9 @@ impl Reader { } } - fn log(&mut self, id: Id, claim: Path, kind: Kind, item: A) { + fn log(&mut self, id: K, claim: Path, kind: Kind, item: A) { #[cfg(test)] - let forgot = *self.forgot.entry((id, claim.clone())).or_insert(0); + let forgot = *self.forgot.entry((id.clone(), claim.clone())).or_insert(0); self.events.push(Event { id, claim, diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs new file mode 100644 index 00000000000..cf21fa979f1 --- /dev/null +++ b/rs/moq-net/src/route/relay.rs @@ -0,0 +1,2123 @@ +//! The multicast epic's route rule as one node's state machine, beside +//! [`super::path::Router`] until the switch puts lite-07 sessions on it +//! (`quest/m2/multicast/switch.md`). +//! +//! A route leads to a forwarding node that announces claims, never to a publisher: an +//! endpoint's claims belong to the relay it connects to, which announces them under its +//! own [`NodeId`] and revision. A node offers a route to itself once it has announced a +//! claim, and from then on until it restarts. A route carries its path, drain, priority, +//! and cost. A node never uses one whose path names its relay, sends none back over the +//! session it learned it on or to a neighbour already on its path, and never sends one +//! learned over a marked link to another marked link. It ranks routes by drain, then the +//! lowest priority on the path, then the summed cost, each link adding at least 1, so +//! extending a route never improves it; among equal ranks it stays on its next hop. +//! +//! A closed session is named once per neighbour in a fixed-size [`Msg::Dead`], which ends +//! every route its sender offered through it until the sender offers it again or ends +//! it. Per pair of relays a node keeps, for good, the highest session it knows closed +//! (the session floor), and refuses any path naming that pair at or below it: a session +//! orders by its node's start and then its number, so a restart replaces the pair's +//! entry. A node that sent GOAWAY is named once per neighbour in a [`Msg::Drain`], and +//! every route through it ranks as draining. A node advertises once per burst of input, +//! when the caller drains it. There is no hold, no doubt, and no routing timer: the only +//! clock is the reader's debounce from unavailable to offline, which never reaches the +//! wire. +//! +//! A forwarding node's claims are ordered by its revision and certified by floors: a node +//! keeps each neighbour's copy and the newest floor wins, so no stale copy revives a +//! withdrawal. A claims message carries its floor even with nothing changed, and a +//! withdrawal says whether the publisher ended the claim or its announcer lost it. Across +//! a scoped grant only claims travel: the receiver announces each again under its own +//! identity with the chain of relays that did so, and refuses one whose chain names it. +//! +//! Each neighbour asks for a set of prefixes edited in place, as under the path rule: a +//! routing neighbour forwards for the whole domain, so it asks for the whole tree, and +//! an endpoint or scoped neighbour is told the closest claim per broadcast within what it +//! asks, a broader claim presented at the prefix it asks for. +//! +//! The rule is the bake-off's pick (moq-dev/moq#8880): `tests/multi.rs` with `floor`, +//! `epoch`, `batch`, `drain`, and `claimed`, which `tests/relay.rs` compares it against. + +use std::{ + cmp::Reverse, + collections::{BTreeMap, BTreeSet}, + fmt, + sync::Arc, + time::Duration, +}; + +use kio::clock::Instant; + +use crate::Path; + +use super::reader::Reader; +use super::view::{ancestors, below, within_any, within_set}; +use super::{Error, Event, Id, Item, Key, MAX_HOPS, Metric, Rev, Shown}; + +/// How a forwarding node names itself on a path: its relay, and which start of that +/// relay's process. A start only grows; one that repeats or goes back gets the relay's +/// new sessions refused. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(crate) struct NodeId { + pub relay: Id, + pub start: u64, +} + +/// One node on a route's path, and the session it learned the route on. The forwarding +/// node a route leads to names itself with session 0. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(crate) struct Hop { + pub node: NodeId, + pub session: u64, +} + +/// A route's rank, lower first: draining, then the lowest priority, then the cost. +type Rank = (bool, Reverse, Metric); + +/// What a route says. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Offer { + /// Every node it crossed, the forwarding node it leads to first and the sender last. + pub path: Arc<[Hop]>, + /// A node on its path sent GOAWAY: it ranks below every route that avoids one. + pub draining: bool, + /// The lowest link priority on its path. + pub priority: i8, + /// The summed link costs, each at least 1. + pub cost: Metric, + /// A marked link is on its path, so it is never sent over another. + pub marked: bool, +} + +impl Offer { + fn rank(&self) -> Rank { + (self.draining, Reverse(self.priority), self.cost) + } +} + +/// Whether `path` crosses any start of `relay`. +fn names(path: &[Hop], relay: Id) -> bool { + path.iter().any(|hop| hop.node.relay == relay) +} + +/// The pairs of relays a path joins, each with where its session stands in the pair's +/// order: the node that learned the route over a session, the node at its far end, and +/// that session as the first node's start and number. +fn pairs(path: &[Hop]) -> impl Iterator + '_ { + path.windows(2).map(|pair| floor_of(pair[1], pair[0].node)) +} + +/// The pair of relays `hop`'s session joins, as the floor files it, and where the session +/// stands in that pair's order. +fn floor_of(hop: Hop, far: NodeId) -> ((Id, Id), (u64, u64)) { + ((hop.node.relay, far.relay), (hop.node.start, hop.session)) +} + +/// Every hop of `node`, whatever its session. +fn hops_of(node: NodeId) -> std::ops::RangeInclusive { + Hop { node, session: 0 }..=Hop { + node, + session: u64::MAX, + } +} + +/// A claim as a forwarding node announces it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Claim { + /// What the publisher says of it, such as its Epoch. + pub item: A, + /// From the announcing node to the publisher: 0 for its own endpoint. + pub cost: Metric, + /// The lowest link priority between the two. + pub priority: i8, + /// The way to the publisher crosses a node that sent GOAWAY. + pub draining: bool, + /// The relays that announced it again across a scoped grant, the first first. + pub chain: Vec, +} + +impl Claim { + fn rank(&self) -> Rank { + (self.draining, Reverse(self.priority), self.cost) + } + + fn lowered(&self, by: u64) -> Self { + Self { + item: self.item.lowered(by), + ..self.clone() + } + } +} + +/// A broadcast: a path, and the Instance a claim there serves. +pub(crate) type Broadcast = (Path, I); + +/// Claims by broadcast. +pub(crate) type Claims = BTreeMap, Claim>; + +/// A change to one claim. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum Change { + Live(Claim), + /// The publisher withdrew it. + Ended, + /// Its announcer lost it without the publisher saying so, as when a session closes. + Lost, +} + +/// An edit of the prefixes a node asks a neighbour for: the live ones it removes, then the +/// ones it adds. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct Edit { + pub remove: Vec, + pub add: Vec, +} + +/// A message on the announce stream, either way. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum Msg { + /// A route to forwarding node `origin`, or its end. + Route { origin: NodeId, offer: Option }, + /// Changes to `origin`'s claims, complete as of its revision `floor`. + Claims { + origin: NodeId, + changes: Vec<(Broadcast, Change)>, + floor: Rev, + }, + /// The session `hop` names closed, which ends every route its sender offered through + /// it. `far` is the node at its other end. + Dead { hop: Hop, far: NodeId }, + /// `node` sent GOAWAY. + Drain { node: NodeId }, + /// Claims with no route: an endpoint's own, or the closest claims across a scoped grant. + Announce { changes: Vec<(Broadcast, Change)> }, + /// ANNOUNCE_REQUEST: an edit of what the sender asks for. + Ask(Edit), + /// The sender's oldest unanswered edit takes effect here, for both sides. + Edited, + /// The sender sent the initial set of `prefix`, which the receiver added. + Synced(Path), +} + +/// What a neighbour is to this node. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum Role { + /// A forwarding node trusted with the whole tree, named so: routes are exchanged. + Routing(NodeId), + /// A forwarding node across a scoped grant covering these prefixes: claims are + /// exchanged, with a chain, and it may give claims only within them. + Scoped(Vec), + /// A node that never forwards: its claims are this node's own. + Endpoint, +} + +/// One side's configuration of a link. +#[derive(Clone, Debug)] +pub(crate) struct Port { + /// Strictly positive, so extending a route never leaves it as cheap. + pub cost: Metric, + /// The application's local priority for the link: a path ranks by the lowest on it, + /// higher first, before cost. + pub priority: i8, + /// The one mark: a route learned here is never sent to another marked neighbour. + pub upstream: bool, + pub role: Role, +} + +/// Where a request for a broadcast goes next. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Next { + /// Toward the forwarding node the request names. + Via(N), + /// To the endpoint or scoped neighbour whose claim this node announces. + Local(N), +} + +/// The hop delay the bake-off simulated: one round of deliveries. +pub(crate) const HOP: Duration = Duration::from_millis(25); + +/// How long a reader shows a claim unavailable before offline: the longest path a route +/// may take, times a hop's delay, since a reroute settles within that. +pub(crate) fn debounce(hop: Duration) -> Duration { + hop * MAX_HOPS as u32 +} + +#[derive(Clone, Debug)] +struct Peer { + port: Port, + /// How this node numbers the session in the hops of the routes it learns there. + session: u64, + /// It sent GOAWAY. + draining: bool, + /// Its marker arrived: what it sends now answers what this node asked. + answering: bool, + /// What this node asked it for, each with whether its initial set arrived. + asked: BTreeMap, + /// What it asks this node for, none inside another. + asks: BTreeSet, + /// The forwarding nodes it offers a route to. + offers: BTreeSet, + /// What it was offered of each forwarding node: `None` once a closed session this node + /// named ended the route, until its next offer or explicit end. + sent: BTreeMap>, + /// Claims it gave this node, as an endpoint or across a grant. + given: Claims, + /// Claims this node announced to it, by path, as an endpoint or across a grant. + told: BTreeMap>>, +} + +impl Peer { + fn routing(&self) -> Option { + match self.port.role { + Role::Routing(id) => Some(id), + Role::Scoped(_) | Role::Endpoint => None, + } + } +} + +/// What one neighbour offers for one forwarding node, and its copy of that node's claims. +#[derive(Clone, Debug)] +struct Heard { + offer: Offer, + /// The neighbour named a closed session on this path, which ends the route until it + /// offers another. + ended: bool, + floor: Rev, + items: Claims, +} + +/// A forwarding node's claims as this node knows them, complete as of `floor`. +#[derive(Clone, Debug)] +struct Known { + floor: Rev, + items: Claims, +} + +impl Default for Known { + fn default() -> Self { + Self { + floor: 0, + items: BTreeMap::new(), + } + } +} + +/// The route this node uses, as it stands here: the heard offer with the link added. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Best { + via: N, + offer: Offer, +} + +/// How candidate routes order, lower first: rank, then the next hop in use, the shorter +/// path, and the neighbour. +type Order = (bool, Reverse, Metric, bool, usize, N); + +/// What a router retains, for budgets and leak checks. +#[cfg(test)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct Kept { + /// Routes heard, one per neighbour offering a forwarding node. + pub routes: usize, + /// Routes selected, one per forwarding node. + pub used: usize, + /// Hops on the heard routes' paths. + pub hops: usize, + /// Routes sent, one per neighbour and forwarding node. + pub sent: usize, + /// Claims in the merged view of each forwarding node. + pub items: usize, + /// Claims in each neighbour's copy. + pub heard: usize, + /// Claims told to endpoints and scoped neighbours. + pub told: usize, + /// Dead hops and draining nodes. + pub pending: usize, + /// Pairs of relays with a session known closed, kept for good. + pub closed: usize, + /// Claims shown unavailable while the reader waits to show them offline. + pub waiting: usize, +} + +/// One node's router: `N` keys its neighbours, `I` is the Instance a claim serves, and +/// `A` what else a publisher says of a claim. +#[derive(Clone)] +pub(crate) struct Router { + me: NodeId, + /// How long the reader shows a lost claim unavailable before offline. + debounce: Duration, + now: Instant, + sessions: u64, + peers: BTreeMap>, + /// The routing neighbours asking for the whole tree, which routes go to. + routing: BTreeSet, + /// Routing neighbours that newly asked, owed every route as it stands. + fresh: BTreeSet, + heard: BTreeMap>>, + /// The heard routes whose path holds each hop, ended or not. + rx_hops: BTreeMap>>, + /// The heard routes whose path joins each pair of relays, as the floor files them. + rx_pairs: BTreeMap<(Id, Id), BTreeSet<(NodeId, N)>>, + /// How many heard routes that are not ended hold each hop. + live: BTreeMap, + /// Hops and nodes no live heard route may name any more, for [`Self::forget`]. + unnamed: (BTreeSet, BTreeSet), + /// Closed sessions a live heard route still names, each with its far end. + dead: BTreeMap, + /// Per pair of relays, the highest session known closed, as the near relay's start + /// and its session number. + closed: BTreeMap<(Id, Id), (u64, u64)>, + /// Nodes named as draining that a live heard route still crosses. + draining: BTreeSet, + /// Forwarding nodes whose route changed since the neighbours were last told. + stale: BTreeSet, + best: BTreeMap>, + /// The sent routes, live, whose path holds each hop. + tx_hops: BTreeMap>>, + /// The neighbours sent a route to each forwarding node, ended or not. + sent_to: BTreeMap>, + known: BTreeMap>, + /// This node's revision of its own claims. + rev: Rev, + /// It has announced a claim, so it offers a route to itself. + announced: bool, + /// Per broadcast, the claims each endpoint or scoped neighbour gave this node. + givers: BTreeMap, BTreeMap>>, + /// The neighbour behind each of this node's own claims. + sources: BTreeMap, N>, + /// The forwarding nodes claiming each broadcast, by path. + claimants: BTreeMap>>, + /// Per forwarding node, how many of its broadcasts another node claims too. + shared: BTreeMap, + /// The endpoints and scoped neighbours asking for each prefix. + askers: BTreeMap>, + /// Broadcasts whose last removal was the publisher's own withdrawal. + ended: BTreeSet>, + dirty: BTreeSet>, + /// Claims shown unavailable since they lost every claimant, by when they go offline. + waiting: BTreeMap, Instant>, + deadlines: BTreeSet<(Instant, Broadcast)>, + reader: Reader, + out: Vec<(N, Msg)>, + /// Routes, claims, hops, and neighbours visited, for per-change work checks. + #[cfg(test)] + work: std::cell::Cell, +} + +impl Router { + /// A node with no sessions, naming itself `me`, whose reader waits `debounce` before + /// showing a lost claim offline. + pub fn new(me: NodeId, debounce: Duration) -> Self { + Self { + me, + debounce, + now: Instant::ZERO, + sessions: 0, + peers: BTreeMap::new(), + routing: BTreeSet::new(), + fresh: BTreeSet::new(), + heard: BTreeMap::new(), + rx_hops: BTreeMap::new(), + rx_pairs: BTreeMap::new(), + live: BTreeMap::new(), + unnamed: (BTreeSet::new(), BTreeSet::new()), + dead: BTreeMap::new(), + closed: BTreeMap::new(), + draining: BTreeSet::new(), + stale: BTreeSet::new(), + best: BTreeMap::new(), + tx_hops: BTreeMap::new(), + sent_to: BTreeMap::new(), + known: BTreeMap::new(), + rev: 0, + announced: false, + givers: BTreeMap::new(), + sources: BTreeMap::new(), + claimants: BTreeMap::new(), + shared: BTreeMap::new(), + askers: BTreeMap::new(), + ended: BTreeSet::new(), + dirty: BTreeSet::new(), + waiting: BTreeMap::new(), + deadlines: BTreeSet::new(), + reader: Reader::default(), + out: Vec::new(), + #[cfg(test)] + work: std::cell::Cell::new(0), + } + } + + /// How this node names itself on a path. + pub fn me(&self) -> NodeId { + self.me + } + + /// What to send, with every route as it stands now: a burst of input is read first. + pub fn drain(&mut self) -> Vec<(N, Msg)> { + self.advertise(); + std::mem::take(&mut self.out) + } + + /// Take the reader events, in order. + pub fn events(&mut self) -> Vec> { + self.reader.events() + } + + /// How the reader sees each broadcast. + pub fn shown(&self) -> &BTreeMap<(I, Path), Shown> { + self.reader.shown() + } + + /// The time is `now`: a claim lost from here waits its debounce from it. + pub fn clock(&mut self, now: Instant) { + self.now = self.now.max(now); + } + + /// When the reader next shows a waiting claim offline. + pub fn deadline(&self) -> Option { + self.deadlines.first().map(|(at, _)| *at) + } + + /// It is `now`: show offline each claim that waited out its debounce. + pub fn timeout(&mut self, now: Instant) { + self.clock(now); + let mut due: BTreeMap>>> = BTreeMap::new(); + while let Some((at, _)) = self.deadlines.first() + && *at <= self.now + { + let (_, (path, instance)) = self.deadlines.pop_first().expect("a deadline"); + self.visit(); + let broadcast = (path, instance); + self.waiting.remove(&broadcast); + let (path, instance) = broadcast; + due.entry(instance).or_default().insert(path, None); + } + for (instance, claims) in due { + self.reader.apply(instance, |_| true, claims, |_| false); + } + } + + // --- Sessions --- + + /// A session to `peer` opened. This node asks it at once, and answers nothing until it + /// asks in turn. + pub fn connect(&mut self, peer: N, port: Port) -> Result<(), Error> { + if port.cost == 0 { + return Err(Error::ZeroCost); + } + if self.peers.contains_key(&peer) { + return Err(Error::Duplicate); + } + self.sessions += 1; + let ask: BTreeSet = match &port.role { + // Of a scoped neighbour, only what its grant covers, the wider of two nested. + Role::Scoped(regions) => (regions.iter()) + .filter(|region| !ancestors(region).any(|above| above != **region && regions.contains(&above))) + .cloned() + .collect(), + Role::Routing(_) | Role::Endpoint => BTreeSet::from([Path::empty()]), + }; + self.peers.insert( + peer, + Peer { + port, + session: self.sessions, + draining: false, + answering: false, + asked: ask.iter().map(|prefix| (prefix.clone(), false)).collect(), + asks: BTreeSet::new(), + offers: BTreeSet::new(), + sent: BTreeMap::new(), + given: BTreeMap::new(), + told: BTreeMap::new(), + }, + ); + let edit = Edit { + remove: Vec::new(), + add: ask.into_iter().collect(), + }; + self.out.push((peer, Msg::Ask(edit))); + Ok(()) + } + + /// The session to `peer` closed: everything learned or sent on it goes, and a routing + /// neighbour's session is named to the neighbours this node sent a route through it. + pub fn disconnect(&mut self, peer: N) { + let Some(gone) = self.peers.remove(&peer) else { + return; + }; + self.routing.remove(&peer); + self.fresh.remove(&peer); + // Nothing still unsent goes to a session that closed. + self.out.retain(|(to, _)| *to != peer); + let mut touched = BTreeSet::new(); + for &origin in &gone.offers { + self.unhear(origin, peer); + self.unheard(origin); + touched.insert(origin); + } + for (origin, offer) in &gone.sent { + self.unsend(peer, *origin, offer.as_ref()); + if let Some(peers) = self.sent_to.get_mut(origin) { + peers.remove(&peer); + if peers.is_empty() { + self.sent_to.remove(origin); + } + } + } + if gone.routing().is_none() { + for prefix in &gone.asks { + self.unask(prefix, peer); + } + } + for broadcast in gone.given.into_keys() { + self.ungive(&broadcast, peer); + self.restate(broadcast, Change::Lost); + } + if let Role::Routing(far) = gone.port.role { + let hop = Hop { + node: self.me, + session: gone.session, + }; + self.closed(hop, far, None, &mut touched); + } + for origin in touched { + self.reselect(origin); + } + self.done(); + } + + /// `peer` sent GOAWAY: every route through it ranks as draining, and so does every + /// claim it gave. + pub fn goaway(&mut self, peer: N) { + let Some(state) = self.peers.get_mut(&peer) else { + return; + }; + state.draining = true; + let Some(node) = state.routing() else { + // The claims it gave this node now lead through a draining neighbour. + let broadcasts: Vec> = state.given.keys().cloned().collect(); + for broadcast in broadcasts { + self.restate(broadcast, Change::Lost); + } + self.done(); + return; + }; + let mut touched = BTreeSet::new(); + self.drains(node, None, &mut touched); + for origin in touched { + self.reselect(origin); + } + self.done(); + } + + /// `peer` edited what it asks this node for. The edit takes effect behind the marker + /// this node sends first: what both sets cover stays, and what the added prefixes newly + /// cover follows, closed by SYNCED for each. A routing neighbour asks for the whole + /// tree or nothing, since it forwards for every node behind it. + pub fn edit(&mut self, peer: N, edit: Edit) -> Result<(), Error> { + let state = self.peers.get(&peer).ok_or(Error::UnknownNeighbor)?; + let old = state.asks.clone(); + let mut new = old.clone(); + for prefix in &edit.remove { + if !new.remove(prefix) { + return Err(Error::UnknownPrefix); + } + } + for prefix in &edit.add { + if within_any(prefix, &new) || within_set(&new, prefix).next().is_some() { + return Err(Error::Nested); + } + new.insert(prefix.clone()); + } + let routing = state.routing().is_some(); + if routing && !new.is_empty() && new != BTreeSet::from([Path::empty()]) { + return Err(Error::Partial); + } + self.out.push((peer, Msg::Edited)); + let state = self.peers.get_mut(&peer).expect("a neighbour"); + state.asks = new.clone(); + if routing { + match (old.is_empty(), new.is_empty()) { + (true, false) => { + self.routing.insert(peer); + self.fresh.insert(peer); + } + (false, true) => { + // Both sides drop the routes at the marker. + self.routing.remove(&peer); + self.fresh.remove(&peer); + let sent = std::mem::take(&mut state.sent); + for (origin, offer) in sent { + self.unsend(peer, origin, offer.as_ref()); + if let Some(peers) = self.sent_to.get_mut(&origin) { + peers.remove(&peer); + if peers.is_empty() { + self.sent_to.remove(&origin); + } + } + } + } + _ => {} + } + self.done(); + return Ok(()); + } + for prefix in old.difference(&new) { + // Both sides drop what it covered at the marker. + let state = self.peers.get_mut(&peer).expect("a neighbour"); + let inside: Vec = (state.told.get_key_value(prefix).map(|(path, _)| path).into_iter()) + .chain(below(&state.told, prefix).map(|(path, _)| path)) + .cloned() + .collect(); + for path in inside { + state.told.remove(&path); + } + self.unask(prefix, peer); + } + let added: Vec = new.difference(&old).cloned().collect(); + for prefix in &added { + self.askers.entry(prefix.clone()).or_default().insert(peer); + // Every broadcast within it, at its own path. + let inside: Vec<(Path, I)> = (self.claimants.get_key_value(prefix).into_iter()) + .chain(below(&self.claimants, prefix)) + .flat_map(|(path, claims)| claims.keys().map(move |instance| (path.clone(), instance.clone()))) + .collect(); + // Every Instance claimed above it, presented at the prefix itself. + let above: BTreeSet = (ancestors(prefix).filter(|above| above != prefix)) + .filter_map(|above| self.claimants.get(&above)) + .flat_map(|claims| claims.keys().cloned()) + .collect(); + for (path, instance) in inside { + self.export(peer, &path, &instance); + } + for instance in above { + self.export(peer, prefix, &instance); + } + } + for prefix in added { + self.out.push((peer, Msg::Synced(prefix))); + } + self.done(); + Ok(()) + } + + /// Whether `peer` sent the initial set of `prefix`, which this node asks it for, or + /// `None` when it does not ask for it. + pub fn is_synced(&self, peer: N, prefix: &Path) -> Option { + self.peers.get(&peer)?.asked.get(prefix).copied() + } + + /// Take in a message `from` sent. An error is a message the rule never sends, which + /// closes the session; nothing of it is applied. + pub fn recv(&mut self, from: N, msg: Msg) -> Result<(), Error> { + let peer = self.peers.get_mut(&from).ok_or(Error::UnknownNeighbor)?; + let routing = peer.routing().is_some(); + match msg { + Msg::Ask(edit) => return self.edit(from, edit), + Msg::Edited => { + peer.answering = true; + return Ok(()); + } + Msg::Synced(prefix) => { + let synced = peer.asked.get_mut(&prefix).filter(|_| peer.answering).ok_or(Error::Unasked)?; + *synced = true; + return Ok(()); + } + _ if !peer.answering => return Err(Error::Unasked), + Msg::Route { origin, offer } if routing => self.route(from, origin, offer)?, + Msg::Claims { origin, changes, floor } if routing => self.claims(from, origin, changes, floor)?, + Msg::Dead { hop, far } if routing => { + self.check_dead(from, hop, far)?; + let mut touched = BTreeSet::new(); + self.closed(hop, far, Some(from), &mut touched); + for origin in touched { + self.reselect(origin); + } + } + Msg::Drain { node } if routing => { + // It names a node only on a route it offered. + let named = (self.rx_hops.range(hops_of(node))) + .any(|(hop, by)| by.contains_key(&from) && self.live.contains_key(hop)); + if !named { + return Err(Error::Unrouted); + } + let mut touched = BTreeSet::new(); + self.drains(node, Some(from), &mut touched); + for origin in touched { + self.reselect(origin); + } + } + Msg::Announce { changes } if !routing => self.adopt(from, changes)?, + _ => return Err(Error::Role), + } + self.done(); + Ok(()) + } + + /// An input is done: show readers, and announce to endpoints and scoped neighbours. + /// Routes wait for [`Self::drain`]. + fn done(&mut self) { + self.refresh(); + #[cfg(test)] + self.check(); + } + + // --- Routes --- + + fn route(&mut self, from: N, origin: NodeId, offer: Option) -> Result<(), Error> { + self.visit(); + let sender = self.peers[&from].routing().expect("a routing neighbour"); + match offer { + Some(offer) => { + // The sender extended a route that led to `origin` and did not name this + // relay, by its own hop, and never past the longest path. + let path = &offer.path; + let built = path.len() >= 2 || offer.cost == 0; + if origin.relay == self.me.relay + || path.first() != Some(&Hop { node: origin, session: 0 }) + || path.last().map(|hop| hop.node) != Some(sender) + || path.len() > MAX_HOPS + || names(path, self.me.relay) + || !built + { + return Err(Error::Malformed); + } + self.hear(origin, from, offer); + } + None => { + if !self.heard.get(&origin).is_some_and(|offers| offers.contains_key(&from)) { + return Err(Error::Unrouted); + } + self.unhear(origin, from); + self.unheard(origin); + } + } + self.forget(); + self.reselect(origin); + Ok(()) + } + + /// The neighbour heard from names a closed session only on a live route it offered, + /// and the node it names at the far end is the one before the session's hop there. + fn check_dead(&self, from: N, hop: Hop, far: NodeId) -> Result<(), Error> { + let origins = self.rx_hops.get(&hop).and_then(|by| by.get(&from)); + let mut live = (origins.into_iter().flatten()) + .map(|origin| &self.heard[origin][&from]) + .filter(|heard| !heard.ended) + .peekable(); + if live.peek().is_none() { + return Err(Error::Unrouted); + } + for heard in live { + let at = heard.offer.path.iter().position(|named| *named == hop); + if at.is_none_or(|at| at == 0 || heard.offer.path[at - 1].node != far) { + return Err(Error::Malformed); + } + } + Ok(()) + } + + /// File `offer` as what `from` says of `origin`, keeping its copy of the claims. + fn hear(&mut self, origin: NodeId, from: N, offer: Offer) { + let offers = self.heard.entry(origin).or_default(); + let old = match offers.get_mut(&from) { + Some(heard) => { + let ended = std::mem::replace(&mut heard.ended, false); + Some((std::mem::replace(&mut heard.offer, offer.clone()), ended)) + } + None => { + offers.insert( + from, + Heard { + offer: offer.clone(), + ended: false, + floor: 0, + items: BTreeMap::new(), + }, + ); + self.peers.get_mut(&from).expect("a neighbour").offers.insert(origin); + None + } + }; + if let Some((old, ended)) = old { + self.untrack(origin, from, &old.path, !ended); + } + self.track(origin, from, &offer.path); + } + + /// `from` no longer offers `origin`: its copy of the claims goes with the route. + fn unhear(&mut self, origin: NodeId, from: N) { + let Some(offers) = self.heard.get_mut(&origin) else { + return; + }; + let Some(heard) = offers.remove(&from) else { + return; + }; + if let Some(peer) = self.peers.get_mut(&from) { + peer.offers.remove(&origin); + } + self.untrack(origin, from, &heard.offer.path, !heard.ended); + } + + /// The neighbour's word ends `from`'s route to `origin`. + fn end_heard(&mut self, origin: NodeId, from: N) { + let heard = self.heard.get_mut(&origin).and_then(|offers| offers.get_mut(&from)); + let Some(heard) = heard.filter(|heard| !heard.ended) else { + return; + }; + heard.ended = true; + let path = heard.offer.path.clone(); + for hop in path.iter() { + self.unlive(*hop); + } + } + + /// No neighbour offers `origin` any more: its claims go. + fn unheard(&mut self, origin: NodeId) { + if !self.heard.get(&origin).is_some_and(BTreeMap::is_empty) { + return; + } + self.heard.remove(&origin); + if let Some(known) = self.known.remove(&origin) { + self.count(known.items.len()); + for broadcast in known.items.into_keys() { + self.unclaim(&broadcast, origin); + self.dirty.insert(broadcast); + } + } + } + + fn track(&mut self, origin: NodeId, from: N, path: &[Hop]) { + for hop in path { + self.visit(); + self.rx_hops.entry(*hop).or_default().entry(from).or_default().insert(origin); + *self.live.entry(*hop).or_default() += 1; + } + for (pair, _) in pairs(path) { + self.rx_pairs.entry(pair).or_default().insert((origin, from)); + } + } + + fn untrack(&mut self, origin: NodeId, from: N, path: &[Hop], live: bool) { + for hop in path { + self.visit(); + if let Some(by) = self.rx_hops.get_mut(hop) { + if let Some(origins) = by.get_mut(&from) { + origins.remove(&origin); + if origins.is_empty() { + by.remove(&from); + } + } + if by.is_empty() { + self.rx_hops.remove(hop); + } + } + if live { + self.unlive(*hop); + } + } + for (pair, _) in pairs(path) { + if let Some(routes) = self.rx_pairs.get_mut(&pair) { + routes.remove(&(origin, from)); + if routes.is_empty() { + self.rx_pairs.remove(&pair); + } + } + } + } + + fn unlive(&mut self, hop: Hop) { + let count = self.live.get_mut(&hop).expect("a live hop"); + *count -= 1; + if *count == 0 { + self.live.remove(&hop); + self.unnamed.0.insert(hop); + self.unnamed.1.insert(hop.node); + } + } + + /// Forget each dead hop and draining node no live heard route names. + fn forget(&mut self) { + for hop in std::mem::take(&mut self.unnamed.0) { + self.visit(); + if !self.live.contains_key(&hop) { + self.dead.remove(&hop); + } + } + for node in std::mem::take(&mut self.unnamed.1) { + self.visit(); + if self.live.range(hops_of(node)).next().is_none() { + self.draining.remove(&node); + } + } + } + + /// A session closed: this node's own when `from` is `None`. The origins whose routes + /// it may change go in `touched`. + fn closed(&mut self, hop: Hop, far: NodeId, from: Option, touched: &mut BTreeSet) { + self.visit(); + // The neighbour's word ends every route it offered through the session. + if let Some(from) = from { + let origins: Vec = (self.rx_hops.get(&hop).and_then(|by| by.get(&from))) + .into_iter() + .flatten() + .copied() + .collect(); + for origin in origins { + self.end_heard(origin, from); + touched.insert(origin); + } + } + // A path naming this relay is refused anyway, so its own sessions need no floor. + if hop.node.relay != self.me.relay { + let (pair, at) = floor_of(hop, far); + let highest = self.closed.entry(pair).or_default(); + if at > *highest { + *highest = at; + let routes = self.rx_pairs.get(&pair).into_iter().flatten(); + touched.extend(routes.map(|(origin, _)| *origin)); + } + } + let named = self.live.contains_key(&hop); + let tell: Vec = (self.tx_hops.get(&hop).into_iter()) + .flat_map(BTreeMap::keys) + .filter(|peer| Some(**peer) != from) + .copied() + .collect(); + if !self.dead.contains_key(&hop) && (named || !tell.is_empty() || from.is_none()) { + if named { + self.dead.insert(hop, far); + let routes = self.rx_hops.get(&hop).into_iter().flat_map(BTreeMap::values).flatten(); + touched.extend(routes.copied()); + } + for peer in tell { + self.out.push((peer, Msg::Dead { hop, far })); + let origins = self.tx_hops[&hop][&peer].clone(); + for origin in origins { + let sent = (self.peers.get_mut(&peer)) + .and_then(|state| state.sent.get_mut(&origin)) + .and_then(Option::take); + self.unsend(peer, origin, sent.as_ref()); + // Owed its next offer, or its explicit end. + self.stale.insert(origin); + } + } + } + self.forget(); + } + + /// `node` is draining: `from` said so, or its own GOAWAY when `None`. + fn drains(&mut self, node: NodeId, from: Option, touched: &mut BTreeSet) { + self.visit(); + if !self.draining.insert(node) { + return; + } + let tell: BTreeSet = (self.tx_hops.range(hops_of(node))) + .flat_map(|(_, by)| by.keys()) + .filter(|peer| Some(**peer) != from) + .copied() + .collect(); + for peer in tell { + self.out.push((peer, Msg::Drain { node })); + } + self.unnamed.1.insert(node); + self.forget(); + let routes = (self.rx_hops.range(hops_of(node))).flat_map(|(_, by)| by.values().flatten()); + touched.extend(routes.copied()); + } + + /// Whether a session on `path` is at or below its pair's floor. + fn floored(&self, path: &[Hop]) -> bool { + pairs(path).any(|(pair, at)| self.closed.get(&pair).is_some_and(|highest| at <= *highest)) + } + + /// Pick the route to `origin`, and mark it stale for the neighbours if it changed. + fn reselect(&mut self, origin: NodeId) { + let current = self.best.get(&origin).map(|best| best.via); + let mut pick: Option<(Best, Order)> = None; + for (&via, heard) in self.heard.get(&origin).into_iter().flatten() { + self.visit(); + let peer = &self.peers[&via]; + let offer = &heard.offer; + if heard.ended + || offer.path.len() >= MAX_HOPS + || self.floored(&offer.path) + || offer.path.iter().any(|hop| self.dead.contains_key(hop)) + { + continue; + } + let draining = + offer.draining || peer.draining || offer.path.iter().any(|hop| self.draining.contains(&hop.node)); + let mine = Best { + via, + offer: Offer { + path: offer.path.clone(), + draining, + priority: offer.priority.min(peer.port.priority), + cost: offer.cost + peer.port.cost, + marked: offer.marked || peer.port.upstream, + }, + }; + let (draining, priority, cost) = mine.offer.rank(); + // Among equal ranks a node stays on its next hop. + let key = (draining, priority, cost, current != Some(via), offer.path.len(), via); + if pick.as_ref().is_none_or(|(_, best)| key < *best) { + pick = Some((mine, key)); + } + } + let pick = pick.map(|(best, _)| best); + let was = self.best.get(&origin); + if was == pick.as_ref() { + return; + } + let flipped = was.is_some() != pick.is_some(); + let moved = was.map(|best| best.offer.rank()) != pick.as_ref().map(|best| best.offer.rank()); + match pick { + Some(best) => self.best.insert(origin, best), + None => self.best.remove(&origin), + }; + // Whether its claims are reachable follows the route, and so does their cost, + // which an endpoint or scoped neighbour is told and which may move the closest + // claim of a broadcast another node claims too. + let exported = !self.askers.is_empty() || self.shared.get(&origin).is_some_and(|count| *count > 0); + if (flipped || (moved && exported)) + && let Some(known) = self.known.get(&origin) + { + self.count(known.items.len()); + self.dirty.extend(known.items.keys().cloned()); + } + self.stale.insert(origin); + } + + /// What `peer` should hear of `origin` now. + fn offer_for(&self, origin: NodeId, peer: N) -> Option { + let to = self.peers.get(&peer)?; + let to_id = to.routing()?; + if origin == self.me { + // Nothing routes to a node that announces nothing. + return self.announced.then(|| Offer { + path: Arc::new([Hop { + node: self.me, + session: 0, + }]), + draining: false, + priority: i8::MAX, + cost: 0, + marked: false, + }); + } + let best = self.best.get(&origin)?; + // Never back over the session it came on, to a node already on its path, or from + // one marked link to another. + if best.via == peer || names(&best.offer.path, to_id.relay) || (best.offer.marked && to.port.upstream) { + return None; + } + let mut path = best.offer.path.to_vec(); + path.push(Hop { + node: self.me, + session: self.peers[&best.via].session, + }); + Some(Offer { + path: path.into(), + ..best.offer.clone() + }) + } + + /// Tell every routing neighbour of each route that changed, and a neighbour that newly + /// asked of every route. + fn advertise(&mut self) { + for peer in std::mem::take(&mut self.fresh) { + let origins: BTreeSet = (self.best.keys().copied()) + .chain(self.announced.then_some(self.me)) + .collect(); + for origin in origins { + self.advertise_to(origin, peer); + } + self.out.push((peer, Msg::Synced(Path::empty()))); + } + let peers: Vec = self.routing.iter().copied().collect(); + while let Some(origin) = self.stale.pop_first() { + for &peer in &peers { + self.advertise_to(origin, peer); + } + } + } + + fn advertise_to(&mut self, origin: NodeId, peer: N) { + self.visit(); + let offer = self.offer_for(origin, peer); + let state = self.peers.get_mut(&peer).expect("a routing neighbour"); + let was = state.sent.get(&origin); + // Unchanged, unless a closed session ended what it was sent, which it is owed. + if was.is_none_or(Option::is_some) && was.and_then(Option::as_ref) == offer.as_ref() { + return; + } + let fresh = was.is_none(); + let old = match &offer { + Some(offer) => state.sent.insert(origin, Some(offer.clone())), + None => state.sent.remove(&origin), + }; + self.unsend(peer, origin, old.flatten().as_ref()); + match &offer { + Some(offer) => { + for hop in offer.path.iter() { + let by = self.tx_hops.entry(*hop).or_default(); + by.entry(peer).or_default().insert(origin); + } + self.sent_to.entry(origin).or_default().insert(peer); + } + None => { + if let Some(peers) = self.sent_to.get_mut(&origin) { + peers.remove(&peer); + if peers.is_empty() { + self.sent_to.remove(&origin); + } + } + } + } + let started = fresh && offer.is_some(); + self.out.push((peer, Msg::Route { origin, offer })); + // A route that starts brings the claims as they stand, even none: the floor alone + // outranks an older copy of what was withdrawn. + if started + && let Some(known) = self.known.get(&origin) + && known.floor > 0 + { + self.count(known.items.len()); + let changes = (known.items.iter()) + .map(|(broadcast, claim)| (broadcast.clone(), Change::Live(claim.clone()))) + .collect(); + let floor = known.floor; + self.out.push((peer, Msg::Claims { origin, changes, floor })); + } + } + + /// Unindex a live route sent to `peer`. + fn unsend(&mut self, peer: N, origin: NodeId, offer: Option<&Offer>) { + let Some(offer) = offer else { + return; + }; + for hop in offer.path.iter() { + self.visit(); + if let Some(by) = self.tx_hops.get_mut(hop) { + if let Some(origins) = by.get_mut(&peer) { + origins.remove(&origin); + if origins.is_empty() { + by.remove(&peer); + } + } + if by.is_empty() { + self.tx_hops.remove(hop); + } + } + } + } + + // --- Claims --- + + fn claims( + &mut self, + from: N, + origin: NodeId, + changes: Vec<(Broadcast, Change)>, + floor: Rev, + ) -> Result<(), Error> { + self.count(changes.len()); + let heard = (self.heard.get_mut(&origin)) + .and_then(|offers| offers.get_mut(&from)) + .ok_or(Error::Unrouted)?; + // Each message moves the sender's copy to a newer revision. + if floor <= heard.floor || changes.iter().any(|(_, change)| !well_built(change)) { + return Err(Error::Malformed); + } + let was = heard.floor; + for (broadcast, change) in &changes { + match change { + Change::Live(claim) => heard.items.insert(broadcast.clone(), claim.clone()), + Change::Ended | Change::Lost => heard.items.remove(broadcast), + }; + } + heard.floor = floor; + let known = self.known.entry(origin).or_default(); + // The newest floor wins, so an older copy never revives what a newer one withdrew. + if floor <= known.floor { + return Ok(()); + } + let mut visited = 0; + let changes = match was == known.floor { + // The copy matched this node's at the same revision, so the message's changes + // bring it to the new one. + true => changes, + // A copy that lagged overtook it: what differs is the difference. + false => { + let heard = &self.heard[&origin][&from]; + let mut diff: Vec<(Broadcast, Change)> = (heard.items.iter()) + .filter(|(broadcast, claim)| known.items.get(*broadcast) != Some(claim)) + .map(|(broadcast, claim)| (broadcast.clone(), Change::Live(claim.clone()))) + .collect(); + // What the newer copy lacks is lost, unless this message says its + // publisher ended it. + let ended = |broadcast: &Broadcast| { + (changes.iter()).any(|(named, change)| named == broadcast && *change == Change::Ended) + }; + diff.extend( + (known.items.keys()) + .filter(|broadcast| !heard.items.contains_key(*broadcast)) + .map(|broadcast| match ended(broadcast) { + true => (broadcast.clone(), Change::Ended), + false => (broadcast.clone(), Change::Lost), + }), + ); + visited = heard.items.len() + known.items.len(); + diff + } + }; + self.count(visited); + self.known.get_mut(&origin).expect("known").floor = floor; + self.publish(origin, changes); + Ok(()) + } + + /// Apply `changes` to this node's view of `origin` and pass them, with its floor, to + /// every neighbour it offers `origin` to. + fn publish(&mut self, origin: NodeId, changes: Vec<(Broadcast, Change)>) { + let known = self.known.entry(origin).or_default(); + let floor = known.floor; + let mut claimed = Vec::new(); + let mut unclaimed = Vec::new(); + for (broadcast, change) in &changes { + match change { + Change::Live(claim) => { + if known.items.insert(broadcast.clone(), claim.clone()).is_none() { + claimed.push(broadcast.clone()); + } + self.ended.remove(broadcast); + } + Change::Ended | Change::Lost => { + if known.items.remove(broadcast).is_some() { + unclaimed.push(broadcast.clone()); + } + if *change == Change::Ended { + self.ended.insert(broadcast.clone()); + } + } + } + self.dirty.insert(broadcast.clone()); + } + for broadcast in &claimed { + self.claim(broadcast, origin); + } + for broadcast in &unclaimed { + self.unclaim(broadcast, origin); + } + // Even with nothing changed: the floor alone outranks an older copy elsewhere. + for &peer in self.sent_to.get(&origin).into_iter().flatten() { + self.visit(); + let changes = changes.clone(); + self.out.push((peer, Msg::Claims { origin, changes, floor })); + } + } + + fn claim(&mut self, (path, instance): &Broadcast, origin: NodeId) { + let origins = (self.claimants.entry(path.clone()).or_default()) + .entry(instance.clone()) + .or_default(); + if !origins.insert(origin) { + return; + } + if origins.len() == 2 { + let other = *origins.iter().find(|other| **other != origin).expect("another"); + *self.shared.entry(other).or_default() += 1; + } + if origins.len() >= 2 { + *self.shared.entry(origin).or_default() += 1; + } + } + + fn unclaim(&mut self, (path, instance): &Broadcast, origin: NodeId) { + let Some(claims) = self.claimants.get_mut(path) else { + return; + }; + let Some(origins) = claims.get_mut(instance) else { + return; + }; + let shared = origins.len() >= 2; + if !origins.remove(&origin) { + return; + } + let mut unshare = |node: NodeId| { + let count = self.shared.get_mut(&node).expect("a shared count"); + *count -= 1; + if *count == 0 { + self.shared.remove(&node); + } + }; + if shared { + unshare(origin); + } + if origins.len() == 1 { + unshare(*origins.first().expect("one")); + } + if origins.is_empty() { + claims.remove(instance); + if claims.is_empty() { + self.claimants.remove(path); + } + } + } + + /// Claims from an endpoint or across a scoped grant become this node's own. + fn adopt(&mut self, from: N, changes: Vec<(Broadcast, Change)>) -> Result<(), Error> { + let peer = &self.peers[&from]; + // Only within what this node asked it for. + let asked: BTreeSet = peer.asked.keys().cloned().collect(); + if changes.iter().any(|((path, _), _)| !within_any(path, &asked)) { + return Err(Error::Unasked); + } + if changes.iter().any(|(_, change)| !well_built(change)) { + return Err(Error::Malformed); + } + let (cost, priority) = (peer.port.cost, peer.port.priority); + let scoped = matches!(peer.port.role, Role::Scoped(_)); + for (broadcast, change) in changes { + self.visit(); + let change = match change { + // A claim that already crossed this relay came round a ring of grants. + Change::Live(claim) if claim.chain.contains(&self.me.relay) => Change::Lost, + Change::Live(mut claim) => { + claim.cost += cost; + claim.priority = claim.priority.min(priority); + if scoped { + claim.chain.push(self.me.relay); + } + Change::Live(claim) + } + change => change, + }; + let peer = self.peers.get_mut(&from).expect("a neighbour"); + match change { + Change::Live(claim) => { + peer.given.insert(broadcast.clone(), claim.clone()); + self.givers.entry(broadcast.clone()).or_default().insert(from, claim); + self.restate(broadcast, Change::Lost); + } + cause => { + if peer.given.remove(&broadcast).is_some() { + self.ungive(&broadcast, from); + self.restate(broadcast, cause); + } + } + } + } + Ok(()) + } + + fn ungive(&mut self, broadcast: &Broadcast, peer: N) { + if let Some(givers) = self.givers.get_mut(broadcast) { + givers.remove(&peer); + if givers.is_empty() { + self.givers.remove(broadcast); + } + } + } + + /// Announce the cheapest claim any neighbour gave this node for `broadcast`, or + /// withdraw it for `cause` when none is left. + fn restate(&mut self, broadcast: Broadcast, cause: Change) { + let cheapest = (self.givers.get(&broadcast).into_iter().flatten()) + .map(|(&peer, claim)| { + self.visit(); + // A claim from a neighbour that sent GOAWAY is draining from here on. + let mut claim = claim.clone(); + claim.draining |= self.peers.get(&peer).is_some_and(|peer| peer.draining); + (claim, peer) + }) + .min_by_key(|(claim, peer)| { + ( + claim.draining, + Reverse(claim.priority), + claim.cost, + claim.chain.len(), + *peer, + ) + }); + let me = self.me; + let was = self.known.get(&me).and_then(|known| known.items.get(&broadcast)); + let change = match cheapest { + Some((claim, peer)) => { + self.sources.insert(broadcast.clone(), peer); + if was == Some(&claim) { + // The source alone moved, which a scoped neighbour must hear of. + self.dirty.insert(broadcast); + return; + } + Change::Live(claim) + } + None => { + self.sources.remove(&broadcast); + if was.is_none() { + return; + } + cause + } + }; + self.rev += 1; + self.known.entry(me).or_default().floor = self.rev; + if !self.announced { + // Its first claim: the route starts, and brings the claim with it. + self.announced = true; + self.stale.insert(me); + } + self.publish(me, vec![(broadcast, change)]); + } + + // --- Requests --- + + /// The forwarding node a request for `broadcast` names from here: the closest claim. + pub fn pick(&self, broadcast: &Broadcast) -> Option { + self.closest(broadcast).map(|(origin, _)| origin) + } + + /// Where a request for `broadcast` naming `origin` goes from here. `None` fails it. + pub fn step(&self, origin: NodeId, broadcast: &Broadcast) -> Option> { + match origin == self.me { + true => self.sources.get(broadcast).copied().map(Next::Local), + false => self.best.get(&origin).map(|best| Next::Via(best.via)), + } + } + + fn closest(&self, (path, instance): &Broadcast) -> Option<(NodeId, &Claim)> { + let origins = self.claimants.get(path)?.get(instance)?; + let broadcast = (path.clone(), instance.clone()); + (origins.iter()) + .filter_map(|&origin| { + self.visit(); + let claim = self.known.get(&origin)?.items.get(&broadcast)?; + let rank = match origin == self.me { + true => claim.rank(), + false => { + let (draining, Reverse(priority), cost) = self.best.get(&origin)?.offer.rank(); + ( + draining || claim.draining, + Reverse(priority.min(claim.priority)), + cost + claim.cost, + ) + } + }; + Some(((rank, claim.chain.len(), origin), claim)) + }) + .min_by(|(a, _), (b, _)| a.cmp(b)) + .map(|((_, _, origin), claim)| (origin, claim)) + } + + // --- Readers, endpoints, and scoped neighbours --- + + /// Bring what the reader sees, and what endpoints and scoped neighbours are told, up to + /// date for every broadcast that may have changed. + fn refresh(&mut self) { + let dirty = std::mem::take(&mut self.dirty); + let mut now: BTreeMap>>> = BTreeMap::new(); + for broadcast in &dirty { + self.visit(); + let (path, instance) = broadcast; + let closest = self.closest(broadcast).map(|(_, claim)| claim.item.clone()); + let claimed = (self.claimants.get(path).and_then(|claims| claims.get(instance))) + .map(|origins| origins.first().copied()); + let shown = match claimed { + Some(first) => { + self.unwait(broadcast); + let available = closest.is_some(); + // Unreachable, it still shows what a claimant says. + let item = closest.or_else(|| { + let origin = first?; + Some(self.known.get(&origin)?.items.get(broadcast)?.item.clone()) + }); + Some(item.map(|item| Shown { + item, + available, + presented: false, + })) + } + None if self.ended.contains(broadcast) => { + self.unwait(broadcast); + Some(None) + } + // Lost: shown unavailable for the debounce, in case a claimant returns. + None => match self.reader.get(instance.clone(), path) { + Some(_) if self.waiting.contains_key(broadcast) => None, + Some(_) if self.debounce.is_zero() => Some(None), + Some(was) => { + let item = was.item.clone(); + let at = self.now.saturating_add(self.debounce); + self.waiting.insert(broadcast.clone(), at); + self.deadlines.insert((at, broadcast.clone())); + Some(Some(Shown { + item, + available: false, + presented: false, + })) + } + None => None, + }, + }; + if let Some(shown) = shown { + now.entry(instance.clone()).or_default().insert(path.clone(), shown); + } + } + let Self { reader, ended, .. } = self; + for (instance, claims) in now { + let ended = |claim: &Path| ended.contains(&(claim.clone(), instance.clone())); + reader.apply(instance.clone(), |_| true, claims, ended); + } + for broadcast in &dirty { + self.exports(broadcast); + } + for broadcast in dirty { + self.ended.remove(&broadcast); + } + } + + fn unwait(&mut self, broadcast: &Broadcast) { + if let Some(at) = self.waiting.remove(broadcast) { + self.deadlines.remove(&(at, broadcast.clone())); + } + } + + /// Tell each endpoint and scoped neighbour asking for `broadcast` what it shows there. + fn exports(&mut self, (path, instance): &Broadcast) { + if self.askers.is_empty() { + return; + } + // A neighbour asking at or above the claim is shown it at its own path. + let at: Vec = (ancestors(path).filter_map(|above| self.askers.get(&above))) + .flatten() + .copied() + .collect(); + for peer in at { + self.export(peer, path, instance); + } + // A neighbour asking below it is shown what covers its prefix. + let under: Vec<(Path, N)> = (below(&self.askers, path)) + .flat_map(|(prefix, peers)| peers.iter().map(move |peer| (prefix.clone(), *peer))) + .collect(); + for (prefix, peer) in under { + self.export(peer, &prefix, instance); + } + } + + /// Tell `peer` what this node shows it of `instance` at `path` now. + fn export(&mut self, peer: N, path: &Path, instance: &I) { + self.visit(); + let claim = self.shown_to(peer, path, instance); + let state = self.peers.get_mut(&peer).expect("a neighbour"); + let told = state.told.get(path).and_then(|told| told.get(instance)); + if told == claim.as_ref() { + return; + } + let change = match claim { + Some(claim) => { + let told = state.told.entry(path.clone()).or_default(); + told.insert(instance.clone(), claim.clone()); + Change::Live(claim) + } + None => { + let told = state.told.get_mut(path).expect("told"); + told.remove(instance); + if told.is_empty() { + state.told.remove(path); + } + // It ended if the publisher of the claim it stood for ended it. + let ended = ancestors(path).any(|above| self.ended.contains(&(above, instance.clone()))); + match ended { + true => Change::Ended, + false => Change::Lost, + } + } + }; + let changes = vec![((path.clone(), instance.clone()), change)]; + self.out.push((peer, Msg::Announce { changes })); + } + + /// What `peer` is shown of `instance` at `path`: the closest claim there, or at a + /// prefix it asks for, the most specific claim above, lowered to it. + fn shown_to(&self, peer: N, path: &Path, instance: &I) -> Option> { + let asks = &self.peers[&peer].asks; + if !within_any(path, asks) { + return None; + } + if let Some(claim) = self.exported(peer, &(path.clone(), instance.clone())) { + return Some(claim); + } + if !asks.contains(path) { + return None; + } + (ancestors(path).rev().skip(1).zip(1..)) + .find_map(|(above, by)| Some(self.exported(peer, &(above, instance.clone()))?.lowered(by))) + } + + /// The closest claim for `broadcast` as a neighbour of this node sees it, unless it came + /// from `peer`. + fn exported(&self, peer: N, broadcast: &Broadcast) -> Option> { + let (origin, claim) = self.closest(broadcast)?; + if origin == self.me { + return match self.sources.get(broadcast) == Some(&peer) { + true => None, + false => Some(claim.clone()), + }; + } + let route = &self.best[&origin].offer; + Some(Claim { + item: claim.item.clone(), + cost: claim.cost + route.cost, + priority: claim.priority.min(route.priority), + draining: claim.draining || route.draining, + chain: claim.chain.clone(), + }) + } + + fn unask(&mut self, prefix: &Path, peer: N) { + if let Some(peers) = self.askers.get_mut(prefix) { + peers.remove(&peer); + if peers.is_empty() { + self.askers.remove(prefix); + } + } + } + + // --- Accounting --- + + /// Count one visit. + fn visit(&self) { + #[cfg(test)] + self.work.set(self.work.get() + 1); + } + + /// Count `count` visits. + #[cfg_attr(not(test), allow(unused_variables, reason = "only tests count work"))] + fn count(&self, count: usize) { + #[cfg(test)] + self.work.set(self.work.get() + count); + } +} + +/// A claim the rule never builds: a chain past the longest path. +fn well_built(change: &Change) -> bool { + match change { + Change::Live(claim) => claim.chain.len() < MAX_HOPS, + Change::Ended | Change::Lost => true, + } +} + +#[cfg(test)] +impl Router { + /// How many routes, claims, hops, and neighbours this node has visited. + pub fn work_done(&self) -> usize { + self.work.get() + } + + pub fn kept(&self) -> Kept { + let heard = || self.heard.values().flat_map(BTreeMap::values); + Kept { + routes: heard().count(), + used: self.best.len(), + hops: heard().map(|heard| heard.offer.path.len()).sum(), + sent: self.peers.values().map(|peer| peer.sent.len()).sum(), + items: self.known.values().map(|known| known.items.len()).sum(), + heard: heard().map(|heard| heard.items.len()).sum(), + told: (self.peers.values().flat_map(|peer| peer.told.values())) + .map(BTreeMap::len) + .sum(), + pending: self.dead.len() + self.draining.len(), + closed: self.closed.len(), + waiting: self.waiting.len(), + } + } + + /// The pairs of relays with a session known closed, and where each floor stands. + pub fn floors(&self) -> &BTreeMap<(Id, Id), (u64, u64)> { + &self.closed + } + + /// This node's revision of its own claims, and each of its claims as it stands. + pub fn own(&self) -> (Rev, Claims) { + let items = self.known.get(&self.me).map(|known| known.items.clone()); + (self.rev, items.unwrap_or_default()) + } + + /// Every copy of a forwarding node's claims this node holds, with its floor: each + /// neighbour's, and the merged view. + pub fn copies(&self) -> Vec<(NodeId, Rev, &Claims)> { + let heard = (self.heard.iter()) + .flat_map(|(origin, offers)| offers.values().map(move |heard| (*origin, heard.floor, &heard.items))); + let known = (self.known.iter()) + .filter(|(origin, _)| **origin != self.me) + .map(|(origin, known)| (*origin, known.floor, &known.items)); + heard.chain(known).collect() + } + + /// Hash what decides this node's future as of its clock, leaving out the indexes, + /// which follow from the rest, and the work counter. Its output and reader events + /// must already be taken. + pub fn fingerprint(&self, hasher: &mut impl std::hash::Hasher) { + let Self { + me, + debounce, + now, + sessions, + peers, + routing, + fresh, + heard, + rx_hops: _, + rx_pairs: _, + live: _, + unnamed, + dead, + closed, + draining, + stale, + best, + tx_hops: _, + sent_to: _, + known, + rev, + announced, + givers: _, + sources, + claimants: _, + shared: _, + askers: _, + ended, + dirty, + waiting, + deadlines: _, + reader, + out, + work: _, + } = self; + assert!(out.is_empty(), "drain before fingerprinting"); + // A deadline counts by how far off it is; one due counts as zero. + let waiting: BTreeMap<&Broadcast, Duration> = (waiting.iter()) + .map(|(broadcast, at)| (broadcast, at.saturating_duration_since(*now))) + .collect(); + let state = ( + (me, debounce, sessions, peers, routing, fresh, heard, unnamed), + (dead, closed, draining, stale, best, known, rev, announced), + (sources, ended, dirty, waiting, reader.shown()), + ); + super::digest(hasher, &state); + } + + /// Rebuild every index from what it indexes, and compare. + fn check(&self) { + let mut rx_hops: BTreeMap>> = BTreeMap::new(); + let mut rx_pairs: BTreeMap<(Id, Id), BTreeSet<(NodeId, N)>> = BTreeMap::new(); + let mut live: BTreeMap = BTreeMap::new(); + for (origin, offers) in &self.heard { + assert!(!offers.is_empty(), "an empty route entry for {origin:?}"); + for (peer, heard) in offers { + assert!(self.peers[peer].offers.contains(origin), "offers index"); + for hop in heard.offer.path.iter() { + rx_hops.entry(*hop).or_default().entry(*peer).or_default().insert(*origin); + if !heard.ended { + *live.entry(*hop).or_default() += 1; + } + } + for (pair, _) in pairs(&heard.offer.path) { + rx_pairs.entry(pair).or_default().insert((*origin, *peer)); + } + } + } + assert_eq!(self.rx_hops, rx_hops, "heard hops"); + assert_eq!(self.rx_pairs, rx_pairs, "heard pairs"); + assert_eq!(self.live, live, "live hops"); + let mut tx_hops: BTreeMap>> = BTreeMap::new(); + let mut sent_to: BTreeMap> = BTreeMap::new(); + let mut givers: BTreeMap, BTreeMap>> = BTreeMap::new(); + let mut askers: BTreeMap> = BTreeMap::new(); + for (peer, state) in &self.peers { + for (origin, offer) in &state.sent { + sent_to.entry(*origin).or_default().insert(*peer); + for hop in offer.iter().flat_map(|offer| offer.path.iter()) { + tx_hops.entry(*hop).or_default().entry(*peer).or_default().insert(*origin); + } + } + for (broadcast, claim) in &state.given { + givers.entry(broadcast.clone()).or_default().insert(*peer, claim.clone()); + } + if state.routing().is_none() { + for prefix in &state.asks { + askers.entry(prefix.clone()).or_default().insert(*peer); + } + } + assert_eq!( + self.routing.contains(peer), + state.routing().is_some() && !state.asks.is_empty(), + "routing index" + ); + } + assert_eq!(self.tx_hops, tx_hops, "sent hops"); + assert_eq!(self.sent_to, sent_to, "sent routes"); + assert_eq!(self.givers, givers, "given claims"); + assert_eq!(self.askers, askers, "askers"); + let mut claimants: BTreeMap>> = BTreeMap::new(); + for (origin, known) in &self.known { + for (path, instance) in known.items.keys() { + let claims = claimants.entry(path.clone()).or_default(); + claims.entry(instance.clone()).or_default().insert(*origin); + } + } + let mut shared: BTreeMap = BTreeMap::new(); + for origins in claimants.values().flat_map(BTreeMap::values) { + if origins.len() >= 2 { + for origin in origins { + *shared.entry(*origin).or_default() += 1; + } + } + } + assert_eq!(self.claimants, claimants, "claimants"); + assert_eq!(self.shared, shared, "shared claims"); + let deadlines: BTreeSet<(Instant, Broadcast)> = + (self.waiting.iter()).map(|(broadcast, at)| (*at, broadcast.clone())).collect(); + assert_eq!(self.deadlines, deadlines, "deadlines"); + assert!(self.dirty.is_empty(), "refreshed"); + } +} + +impl fmt::Debug for Router { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Router") + .field("me", &self.me) + .field("best", &self.best) + .field("closed", &self.closed) + .finish_non_exhaustive() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const ROOT: &str = ""; + + fn p(path: &str) -> Path { + Path::new(path).expect("a path") + } + + fn node(relay: Id) -> NodeId { + NodeId { relay, start: 1 } + } + + fn port(role: Role) -> Port { + Port { + cost: 1, + priority: 0, + upstream: false, + role, + } + } + + type Relay = Router; + + fn relay(me: Id) -> Relay { + Router::new(node(me), debounce(HOP)) + } + + /// Connect `peer` and exchange asks, as the session's first messages do. + fn open(router: &mut Relay, peer: usize, role: Role) { + router.connect(peer, port(role.clone())).unwrap(); + router.recv(peer, Msg::Edited).unwrap(); + let ask = match role { + Role::Routing(_) => vec![p(ROOT)], + Role::Scoped(_) | Role::Endpoint => Vec::new(), + }; + if !ask.is_empty() { + router.edit(peer, Edit { remove: Vec::new(), add: ask }).unwrap(); + } + } + + fn live(cost: Metric) -> Change { + Change::Live(Claim { + item: 0, + cost, + priority: i8::MAX, + draining: false, + chain: Vec::new(), + }) + } + + /// A claim outside what this node asked a scoped neighbour for is refused whole. + #[test] + fn a_claim_outside_the_grant_is_refused() { + let mut router = relay(1); + open(&mut router, 2, Role::Scoped(vec![p("a")])); + let inside = ((p("a/x"), 7), live(0)); + let outside = ((p("b/x"), 7), live(0)); + let err = router.recv(2, Msg::Announce { + changes: vec![inside.clone(), outside], + }); + assert_eq!(err, Err(Error::Unasked)); + assert!(router.shown().is_empty(), "nothing of it applied"); + router.recv(2, Msg::Announce { changes: vec![inside] }).unwrap(); + assert_eq!(router.shown().len(), 1); + } + + /// A routing neighbour forwards for the whole domain, so it asks for all of it. + #[test] + fn a_routing_neighbour_asks_for_the_whole_tree() { + let mut router = relay(1); + router.connect(2, port(Role::Routing(node(2)))).unwrap(); + let edit = Edit { + remove: Vec::new(), + add: vec![p("a")], + }; + assert_eq!(router.edit(2, edit), Err(Error::Partial)); + } + + /// Nothing a neighbour sends counts until its answer begins at the marker. + #[test] + fn nothing_counts_before_the_marker() { + let mut router = relay(1); + router.connect(2, port(Role::Endpoint)).unwrap(); + let changes = vec![((p("a"), 7), live(0))]; + assert_eq!(router.recv(2, Msg::Announce { changes }), Err(Error::Unasked)); + } + + /// Claims and news must name a route the neighbour offered, and a route must lead to + /// its origin through its sender. + #[test] + fn news_about_no_route_is_refused() { + let mut router = relay(1); + open(&mut router, 2, Role::Routing(node(2))); + let hop = Hop { + node: node(3), + session: 4, + }; + assert_eq!(router.recv(2, Msg::Dead { hop, far: node(5) }), Err(Error::Unrouted)); + assert_eq!(router.recv(2, Msg::Drain { node: node(5) }), Err(Error::Unrouted)); + let claims = Msg::Claims { + origin: node(5), + changes: Vec::new(), + floor: 1, + }; + assert_eq!(router.recv(2, claims), Err(Error::Unrouted)); + let offer = |path: Vec| Offer { + path: path.into(), + draining: false, + priority: i8::MAX, + cost: 1, + marked: false, + }; + let origin = Hop { + node: node(5), + session: 0, + }; + let sender = Hop { + node: node(2), + session: 1, + }; + // Not through its sender. + let route = Msg::Route { + origin: node(5), + offer: Some(offer(vec![origin, hop])), + }; + assert_eq!(router.recv(2, route), Err(Error::Malformed)); + // Through this relay. + let me = Hop { + node: NodeId { relay: 1, start: 0 }, + session: 3, + }; + let route = Msg::Route { + origin: node(5), + offer: Some(offer(vec![origin, me, sender])), + }; + assert_eq!(router.recv(2, route), Err(Error::Malformed)); + let route = Msg::Route { + origin: node(5), + offer: Some(offer(vec![origin, sender])), + }; + router.recv(2, route).unwrap(); + // A dead hop the route crosses, with the wrong node at its far end. + assert_eq!(router.recv(2, Msg::Dead { hop: sender, far: node(9) }), Err(Error::Malformed)); + router.recv(2, Msg::Dead { hop: sender, far: node(5) }).unwrap(); + } + + /// A claim lost with nothing to replace it shows unavailable for the debounce, then + /// offline; one that returns meanwhile shows available again, and never went away. + #[test] + fn a_lost_claim_waits_out_the_debounce() { + let mut router = relay(1); + open(&mut router, 2, Role::Endpoint); + let claim = |change| Msg::Announce { + changes: vec![((p("live"), 7), change)], + }; + let kinds = |router: &mut Relay| router.events().into_iter().map(|event| event.kind).collect::>(); + router.recv(2, claim(live(0))).unwrap(); + assert_eq!(kinds(&mut router), [super::super::Kind::Start]); + router.recv(2, claim(Change::Lost)).unwrap(); + assert_eq!(kinds(&mut router), [super::super::Kind::Unavailable]); + let half = Instant::ZERO.saturating_add(debounce(HOP) / 2); + router.timeout(half); + router.recv(2, claim(live(0))).unwrap(); + assert_eq!(kinds(&mut router), [super::super::Kind::Available]); + assert_eq!(router.deadline(), None); + router.recv(2, claim(Change::Lost)).unwrap(); + let due = router.deadline().expect("a deadline"); + assert_eq!(due, half.saturating_add(debounce(HOP))); + router.timeout(due.saturating_sub(Duration::from_millis(1))); + assert_eq!(kinds(&mut router), [super::super::Kind::Unavailable]); + router.timeout(due); + assert_eq!(kinds(&mut router), [super::super::Kind::Offline]); + // A publisher ending its claim shows at once. + router.recv(2, claim(live(0))).unwrap(); + router.recv(2, claim(Change::Ended)).unwrap(); + let kinds = kinds(&mut router); + assert_eq!(kinds, [super::super::Kind::Start, super::super::Kind::End]); + assert_eq!(router.deadline(), None); + } + + /// An endpoint asking for a prefix under a broader claim is shown that claim at the + /// prefix, and an edit swaps what it is shown in place. + #[test] + fn an_endpoint_is_shown_what_covers_its_prefix() { + let mut router = relay(1); + open(&mut router, 2, Role::Endpoint); + router.recv(2, Msg::Announce { + changes: vec![((p("live"), 7), live(0)), ((p("live/a/x"), 8), live(0))], + }).unwrap(); + open(&mut router, 3, Role::Endpoint); + router.drain(); + router.edit(3, Edit { + remove: Vec::new(), + add: vec![p("live/a")], + }).unwrap(); + let out = router.drain(); + let told: Vec<&Msg> = (out.iter()).filter(|(to, _)| *to == 3).map(|(_, msg)| msg).collect(); + let shown = |path: &str, instance: u64| { + told.iter().any(|msg| { + matches!(msg, Msg::Announce { changes } if changes.iter().any(|((at, id), change)| *at == p(path) && *id == instance && matches!(change, Change::Live(_)))) + }) + }; + assert!(shown("live/a", 7), "the broader claim at the prefix: {told:?}"); + assert!(shown("live/a/x", 8), "the claim inside: {told:?}"); + assert_eq!(told.first(), Some(&&Msg::Edited)); + assert_eq!(told.last(), Some(&&Msg::Synced(p("live/a")))); + // Withdrawing the broader claim withdraws what stood for it. + router.recv(2, Msg::Announce { + changes: vec![((p("live"), 7), Change::Ended)], + }).unwrap(); + let out = router.drain(); + assert!(out.iter().any(|(to, msg)| *to == 3 + && matches!(msg, Msg::Announce { changes } if changes == &vec![((p("live/a"), 7), Change::Ended)]))); + // Swapping its prefix in place drops what the old one covered, silently. + router.edit(3, Edit { + remove: vec![p("live/a")], + add: vec![p("other")], + }).unwrap(); + let out = router.drain(); + let told: Vec<&Msg> = (out.iter()).filter(|(to, _)| *to == 3).map(|(_, msg)| msg).collect(); + assert_eq!(told, [&Msg::Edited, &Msg::Synced(p("other"))]); + assert_eq!(router.kept().told, 0); + } +} From ca6cbc914e8e4a5777590e3911d37b39c2099e4a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 03:49:54 -0800 Subject: [PATCH 03/17] test(moq-net): run the bake-off on route::relay::Router and compare it with the prototype RelaySim wires the production router through the bake-off's queues as Arm::Relay, so Lab, the shapes, and the failure schedule measure it unchanged. It converges on every shape, settles after churn, and carries every failure at 9 relays (and at 32 in the nightly sweep). With its readers letting go at once, it matches the prototype's recommended arm on every shape and failure: the same next hops, route messages, rounds, and reader events, and fewer claims where the prototype re-sends a copy at a floor its neighbours already hold. Two fixes the comparison found: a node names a closed session before it withdraws the routes the session carried, so neighbours learn its floor, and withdraws its routes to a forwarding node at once when it loses every copy of that node's claims, so no session sees that node's floor go back. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/relay.rs | 48 +- rs/moq-net/src/route/tests/bakeoff.rs | 58 ++- rs/moq-net/src/route/tests/compare.rs | 15 +- rs/moq-net/src/route/tests/mod.rs | 1 + rs/moq-net/src/route/tests/multicast.rs | 662 ++++++++++++++++++++++++ 5 files changed, 748 insertions(+), 36 deletions(-) create mode 100644 rs/moq-net/src/route/tests/multicast.rs diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index cf21fa979f1..575bfe53a58 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -238,8 +238,8 @@ pub(crate) const HOP: Duration = Duration::from_millis(25); /// How long a reader shows a claim unavailable before offline: the longest path a route /// may take, times a hop's delay, since a reroute settles within that. -pub(crate) fn debounce(hop: Duration) -> Duration { - hop * MAX_HOPS as u32 +pub(crate) const fn debounce(hop: Duration) -> Duration { + hop.saturating_mul(MAX_HOPS as u32) } #[derive(Clone, Debug)] @@ -550,12 +550,6 @@ impl Router { self.fresh.remove(&peer); // Nothing still unsent goes to a session that closed. self.out.retain(|(to, _)| *to != peer); - let mut touched = BTreeSet::new(); - for &origin in &gone.offers { - self.unhear(origin, peer); - self.unheard(origin); - touched.insert(origin); - } for (origin, offer) in &gone.sent { self.unsend(peer, *origin, offer.as_ref()); if let Some(peers) = self.sent_to.get_mut(origin) { @@ -565,6 +559,11 @@ impl Router { } } } + let mut touched = BTreeSet::new(); + for &origin in &gone.offers { + self.unhear(origin, peer); + touched.insert(origin); + } if gone.routing().is_none() { for prefix in &gone.asks { self.unask(prefix, peer); @@ -574,6 +573,8 @@ impl Router { self.ungive(&broadcast, peer); self.restate(broadcast, Change::Lost); } + // The session is named before any route it carried is withdrawn: a neighbour that + // hears only the withdrawal still takes the routes others offer through it. if let Role::Routing(far) = gone.port.role { let hop = Hop { node: self.me, @@ -581,6 +582,9 @@ impl Router { }; self.closed(hop, far, None, &mut touched); } + for &origin in &gone.offers { + self.unheard(origin); + } for origin in touched { self.reselect(origin); } @@ -736,9 +740,11 @@ impl Router { } } Msg::Drain { node } if routing => { - // It names a node only on a route it offered. + // It names a node only on a live route it offered. let named = (self.rx_hops.range(hops_of(node))) - .any(|(hop, by)| by.contains_key(&from) && self.live.contains_key(hop)); + .filter_map(|(_, by)| by.get(&from)) + .flatten() + .any(|origin| !self.heard[origin][&from].ended); if !named { return Err(Error::Unrouted); } @@ -873,7 +879,9 @@ impl Router { } } - /// No neighbour offers `origin` any more: its claims go. + /// No neighbour offers `origin` any more: its claims go, and so does every route this + /// node sent to it, at once. A copy of its claims learned later may be older than the + /// one this node passed on, so it travels only on a route that starts afresh. fn unheard(&mut self, origin: NodeId) { if !self.heard.get(&origin).is_some_and(BTreeMap::is_empty) { return; @@ -886,6 +894,12 @@ impl Router { self.dirty.insert(broadcast); } } + for peer in self.sent_to.remove(&origin).into_iter().flatten() { + self.visit(); + let sent = (self.peers.get_mut(&peer)).and_then(|state| state.sent.remove(&origin)); + self.unsend(peer, origin, sent.flatten().as_ref()); + self.out.push((peer, Msg::Route { origin, offer: None })); + } } fn track(&mut self, origin: NodeId, from: N, path: &[Hop]) { @@ -1734,6 +1748,18 @@ impl Router { } } + /// The route this node uses to `origin`: the neighbour, and the offer as it stands here. + pub fn best(&self, origin: NodeId) -> Option<(N, &Offer)> { + self.best.get(&origin).map(|best| (best.via, &best.offer)) + } + + /// What each neighbour offers for `origin`, and whether a closed session ended it. + pub fn heard(&self, origin: NodeId) -> Vec<(N, &Offer, bool)> { + (self.heard.get(&origin).into_iter().flatten()) + .map(|(peer, heard)| (*peer, &heard.offer, heard.ended)) + .collect() + } + /// The pairs of relays with a session known closed, and where each floor stands. pub fn floors(&self) -> &BTreeMap<(Id, Id), (u64, u64)> { &self.closed diff --git a/rs/moq-net/src/route/tests/bakeoff.rs b/rs/moq-net/src/route/tests/bakeoff.rs index f32f4c37d58..4856539694c 100644 --- a/rs/moq-net/src/route/tests/bakeoff.rs +++ b/rs/moq-net/src/route/tests/bakeoff.rs @@ -47,19 +47,19 @@ fn arms() -> Vec { } /// The prototype's arms. -const MULTI: [Arm; 4] = [Arm::PLAIN, Arm::TUNED, Arm::LEAN, Arm::KEPT]; +pub(super) const MULTI: [Arm; 4] = [Arm::PLAIN, Arm::TUNED, Arm::LEAN, Arm::KEPT]; /// How long a silent death takes its neighbours to notice, in hops of delay: an idle /// timeout of 3 s. -const IDLE: usize = 120; +pub(super) const IDLE: usize = 120; /// A mesh to measure, and whether today's rule can run it. -struct Shape { - name: &'static str, - mesh: Mesh, +pub(super) struct Shape { + pub name: &'static str, + pub mesh: Mesh, /// Today's rule qualifies an identity across one scoped link and refuses to /// again, so a publisher two grants away is never seen. - today: bool, + pub today: bool, } fn pair(a: Node, b: Node) -> (Node, Node) { @@ -75,7 +75,7 @@ fn spread(relays: &[Node], publishers: usize) -> Vec { /// to both cores of the regions beside it, and every edge on both of its region's /// cores, which it marks as upstream. Edges of one region peer when `peered`. /// Publishers join edges. -fn tiers(regions: usize, edges: usize, peered: bool, publishers: usize) -> Mesh { +pub(super) fn tiers(regions: usize, edges: usize, peered: bool, publishers: usize) -> Mesh { let core = |region: usize, side: usize| 2 * (region % regions) + side; let edge = |region: usize, index: usize| 2 * regions + region * edges + index; let mut links = BTreeSet::new(); @@ -111,7 +111,7 @@ fn tiers(regions: usize, edges: usize, peered: bool, publishers: usize) -> Mesh } /// A ring with a chord from every relay at each power of two: log N of them. -fn chords(relays: usize, publishers: usize) -> Mesh { +pub(super) fn chords(relays: usize, publishers: usize) -> Mesh { let mut links = BTreeSet::new(); let mut step = 1; while step <= relays / 2 { @@ -141,7 +141,7 @@ fn sparse(relays: usize, publishers: usize, rng: &mut XorShift) -> Mesh { } } -fn clique(relays: usize, publishers: usize) -> Mesh { +pub(super) fn clique(relays: usize, publishers: usize) -> Mesh { Mesh { homes: spread(&(0..relays).collect::>(), publishers), claims: 1, @@ -206,7 +206,7 @@ fn joined(count: usize, size: usize, joins: usize, ring: bool, publishers: usize } /// Every shape, at about `relays` relays each. -fn shapes(relays: usize, publishers: usize) -> Vec { +pub(super) fn shapes(relays: usize, publishers: usize) -> Vec { let mut rng = XorShift(0x6d6f_7100 + relays as u64); let open = |name, mesh| Shape { name, @@ -253,7 +253,7 @@ fn shapes(relays: usize, publishers: usize) -> Vec { /// The failures measured, each from a settled mesh. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] -enum Failure { +pub(super) enum Failure { /// A link relays route over closes cleanly, and every relay still has a path. Close, /// It returns. @@ -291,7 +291,7 @@ enum Failure { } impl Failure { - fn as_str(self) -> &'static str { + pub fn as_str(self) -> &'static str { match self { Self::Close => "link_closes", Self::Reopen => "link_reopens", @@ -316,19 +316,25 @@ impl Failure { /// What one failure cost and showed. #[derive(Clone, Debug)] -struct Row { - cost: Cost, - faults: Faults, +pub(super) struct Row { + pub cost: Cost, + pub faults: Faults, /// Dead hops, doubts, and suspected nodes left once it settled. - pending: usize, + pub pending: usize, /// Pairs of nodes with a session known closed, kept for good. - lasting: usize, + pub lasting: usize, /// The hops every relay's request crosses once it settled, summed. - hops: usize, + pub hops: usize, /// Whether every relay with a path routed to its publisher once it settled. - right: Result<(), String>, + pub right: Result<(), String>, + /// Per publisher and relay, the nodes its request visits once it settled, and + /// whether it reached the publisher. + pub answers: Vec>>, } +/// The nodes a request visits, and whether it reached its publisher. +pub(super) type Chased = (Vec, bool); + fn observe(lab: &mut Lab, change: &dyn Fn(&mut Lab)) -> Row { let before = lab.net.faults(); let cost = lab.measure(change); @@ -346,11 +352,17 @@ fn observe(lab: &mut Lab, change: &dyn Fn(&mut Lab)) -> Row { true => lab.check(), false => Err("never settled".into()), }, + answers: (0..lab.mesh.homes.len()) + .map(|index| { + let (_, id) = lab.mesh.publisher(index); + (0..lab.mesh.relays).map(|relay| lab.net.chase(relay, id)).collect() + }) + .collect(), } } /// The relay the most others route through toward the first publisher, its own aside. -fn transit(mesh: &Mesh, tree: &[Option]) -> Option { +pub(super) fn transit(mesh: &Mesh, tree: &[Option]) -> Option { let mut through = vec![0; mesh.relays]; for relay in 0..mesh.relays { let mut at = relay; @@ -372,7 +384,7 @@ fn depths(tree: &[Option]) -> Vec { } /// Run every failure on `mesh` under `arm`, each from a settled mesh. -fn fail(arm: Arm, mesh: &Mesh, coarse: bool) -> Vec<(Failure, Row)> { +pub(super) fn fail(arm: Arm, mesh: &Mesh, coarse: bool) -> Vec<(Failure, Row)> { let tree = mesh.tree(); let depth = depths(&tree); let fresh = || { @@ -519,7 +531,7 @@ fn the_prototype_settles_after_churn() { /// routes to its publisher, and no reader sees an end the publisher never made or a /// life it had seen end. On three meshes in a ring that includes the chain stopping /// a claim circling: once the publisher leaves, every relay lets go. -fn carries(arm: Arm, relays: usize) { +pub(super) fn carries(arm: Arm, relays: usize) { for shape in shapes(relays, 2) { for (failure, row) in fail(arm, &shape.mesh, relays > 16) { let case = format!("{}: {} on {}", arm.name(), failure.as_str(), shape.name); @@ -668,7 +680,7 @@ fn quiesce(net: &mut MultiSim) { fn tune(arm: Arm) -> multi::Tune { match arm { Arm::Multi(tune) => tune, - Arm::Path(_) => panic!("today's rule"), + Arm::Path(_) | Arm::Relay(_) => panic!("not the prototype"), } } diff --git a/rs/moq-net/src/route/tests/compare.rs b/rs/moq-net/src/route/tests/compare.rs index fef0d4bfbd4..72d7f26094c 100644 --- a/rs/moq-net/src/route/tests/compare.rs +++ b/rs/moq-net/src/route/tests/compare.rs @@ -12,13 +12,18 @@ use std::collections::BTreeSet; use super::multi::{Broadcast, MultiSim, Suspect, Tie, Tune}; +use super::multicast::RelaySim; use super::*; -/// A router under comparison: today's rule, or the multicast epic's prototype. +/// A router under comparison: today's rule, the multicast epic's prototype, or its +/// production router. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum Arm { Path(Rule), Multi(Tune), + /// [`relay::Router`]: the rule [`Self::KEPT`] measures, as it ships, its readers + /// showing a lost claim unavailable for this long before offline. + Relay(Duration), } impl Arm { @@ -69,6 +74,9 @@ impl Arm { epoch: true, }); + /// The production router as it ships. + pub const RELAY: Self = Self::Relay(relay::debounce(HOP)); + /// The epic's rule with `tune` changed from how it is written. pub fn with(tune: impl FnOnce(&mut Tune)) -> Self { let mut out = Tune::default(); @@ -80,6 +88,8 @@ impl Arm { let tune = match self { Self::Path(rule) => return rule.as_str().into(), Self::Multi(tune) => tune, + Self::Relay(debounce) if debounce.is_zero() => return "relay-undebounced".into(), + Self::Relay(_) => return "relay".into(), }; let parts = [ (tune.suspect == Suspect::Flagged, "+suspect"), @@ -99,6 +109,7 @@ impl Arm { match self { Self::Path(rule) => Box::new(Sim::with(rule, nodes)), Self::Multi(tune) => Box::new(MultiSim::new(tune, nodes)), + Self::Relay(debounce) => Box::new(RelaySim::new(nodes, debounce)), } } } @@ -221,7 +232,7 @@ pub(super) trait Net { fn work(&self) -> usize; } -fn tally(kinds: impl Iterator) -> (usize, usize, usize) { +pub(super) fn tally(kinds: impl Iterator) -> (usize, usize, usize) { let (mut offline, mut unavailable, mut starts) = (0, 0, 0); for kind in kinds { match kind { diff --git a/rs/moq-net/src/route/tests/mod.rs b/rs/moq-net/src/route/tests/mod.rs index 74a6b2326b3..29d0f4dbda6 100644 --- a/rs/moq-net/src/route/tests/mod.rs +++ b/rs/moq-net/src/route/tests/mod.rs @@ -17,6 +17,7 @@ mod forward; mod handoff; mod interest; mod multi; +mod multicast; mod residual; mod routing; mod scope; diff --git a/rs/moq-net/src/route/tests/multicast.rs b/rs/moq-net/src/route/tests/multicast.rs new file mode 100644 index 00000000000..c0269fc75ec --- /dev/null +++ b/rs/moq-net/src/route/tests/multicast.rs @@ -0,0 +1,662 @@ +//! [`relay::Router`], the multicast epic's rule as it ships, on the bake-off's network. +//! +//! [`RelaySim`] wires one router per relay through [`MultiSim`]'s ordered, holdable +//! queues, so the bake-off's [`Lab`] measures it beside the prototype. The interest +//! exchange that opens a session (ANNOUNCE_REQUEST, its marker, and SYNCED) crosses at +//! once and is not counted, as it travels with the session's setup; everything after +//! it waits its turn. Each round of deliveries takes [`HOP`] on a mocked clock, which +//! only the readers' debounce reads. + +use std::collections::{BTreeMap, BTreeSet, VecDeque}; + +use kio::clock::Instant; + +use crate::route::relay::{self, Claim, Msg, Next, NodeId, Port, Role}; + +use super::bakeoff::{Row, carries, fail, shapes}; +use super::compare::{Arm, Chan, Faults, Held, Mesh, Net, Sent, churn, run, tally}; +use super::multi::{self, Seen, Tie}; +use super::*; + +pub(super) type Router = relay::Router; +type Message = Msg; + +/// What sits at a simulated node. +#[derive(Clone, Debug)] +enum Spot { + Relay(Box), + /// A publisher: an endpoint that never forwards. + Endpoint { + id: Option, + claims: BTreeSet, + /// What each relay asked it for. + asked: BTreeMap>, + }, +} + +/// A network of [`relay::Router`]s and endpoints. +#[derive(Clone)] +pub(super) struct RelaySim { + spots: Vec, + restarts: Vec, + /// Each side's cost and mark, keyed from that side. + ports: BTreeMap<(Node, Node), (Metric, bool)>, + ties: BTreeMap<(Node, Node), Tie>, + up: BTreeSet<(Node, Node)>, + queues: BTreeMap<(Node, Node), VecDeque>, + held: BTreeSet<(Node, Node)>, + lives: BTreeMap, + pub stats: Stats, + pub events: Vec, + spent: usize, + now: Instant, + debounce: Duration, +} + +fn pair(a: Node, b: Node) -> (Node, Node) { + (a.min(b), a.max(b)) +} + +/// A claim as its publisher makes it. +fn own() -> Claim<()> { + Claim { + item: (), + cost: 0, + priority: i8::MAX, + draining: false, + chain: Vec::new(), + } +} + +impl RelaySim { + /// `nodes` relays, whose readers show a lost claim unavailable for `debounce`. + pub fn new(nodes: usize, debounce: Duration) -> Self { + let mut sim = Self { + spots: Vec::new(), + restarts: vec![0; nodes], + ports: BTreeMap::new(), + ties: BTreeMap::new(), + up: BTreeSet::new(), + queues: BTreeMap::new(), + held: BTreeSet::new(), + lives: BTreeMap::new(), + stats: Stats::default(), + events: Vec::new(), + spent: 0, + now: Instant::ZERO, + debounce, + }; + sim.spots = (0..nodes).map(|node| Spot::Relay(Box::new(sim.fresh(node)))).collect(); + sim + } + + /// How `node` names itself on a path as it runs now. + pub fn id(&self, node: Node) -> NodeId { + NodeId { + relay: node as Id, + start: u64::from(self.restarts[node]) + 1, + } + } + + fn fresh(&self, node: Node) -> Router { + let mut router = Router::new(self.id(node), self.debounce); + router.clock(self.now); + router + } + + pub fn router(&self, node: Node) -> Option<&Router> { + match &self.spots[node] { + Spot::Relay(router) => Some(router), + Spot::Endpoint { .. } => None, + } + } + + fn relay(&mut self, node: Node) -> Option<&mut Router> { + match &mut self.spots[node] { + Spot::Relay(router) => Some(router), + Spot::Endpoint { .. } => None, + } + } + + fn peers(&self, node: Node) -> Vec { + (self.up.iter()) + .filter_map(|&(a, b)| match node { + _ if node == a => Some(b), + _ if node == b => Some(a), + _ => None, + }) + .collect() + } + + pub fn link(&mut self, a: Node, b: Node, cost: Metric, tie: Tie) { + // Each side's port describes the node at the other end. + self.ports.insert((a, b), (cost, tie == Tie::Up)); + self.ports.insert((b, a), (cost, false)); + self.ties.insert(pair(a, b), tie); + self.open(a, b); + } + + /// How `x` sees `y` on the session opening now. + fn port(&self, x: Node, y: Node) -> Port { + let (cost, upstream) = self.ports[&(x, y)]; + let role = match (&self.spots[y], self.ties[&pair(x, y)]) { + (Spot::Endpoint { .. }, _) => Role::Endpoint, + (_, Tie::Scoped) => Role::Scoped(vec![Path::empty()]), + _ => Role::Routing(self.id(y)), + }; + Port { + cost, + priority: 0, + upstream, + role, + } + } + + pub fn open(&mut self, a: Node, b: Node) { + assert!(self.up.insert(pair(a, b)), "{a}-{b} already up"); + for (x, y) in [(a, b), (b, a)] { + let port = self.port(x, y); + if let Some(router) = self.relay(x) { + router.connect(y, port).expect("a new session"); + } + } + self.flush(); + } + + /// `peer` lets go of its session to `node`. + fn drop_session(&mut self, node: Node, peer: Node) { + match &mut self.spots[peer] { + Spot::Relay(router) => router.disconnect(node), + Spot::Endpoint { asked, .. } => { + asked.remove(&node); + } + } + } + + pub fn close(&mut self, a: Node, b: Node) { + self.up.remove(&pair(a, b)); + for (x, y) in [(a, b), (b, a)] { + self.queues.remove(&(x, y)); + self.held.remove(&(x, y)); + self.drop_session(x, y); + } + self.flush(); + } + + /// `peer` notices its session to `node` closed, which `node` has yet to. + pub fn sever(&mut self, node: Node, peer: Node) { + self.up.remove(&pair(node, peer)); + for key in [(node, peer), (peer, node)] { + self.queues.remove(&key); + self.held.remove(&key); + } + self.drop_session(node, peer); + self.flush(); + } + + /// The process at `node` exits: its sessions close and its state is lost. + pub fn reset(&mut self, node: Node) { + for peer in self.peers(node) { + self.close(node, peer); + } + self.restarts[node] += 1; + if let Spot::Relay(router) = &self.spots[node] { + self.spent += router.work_done(); + self.spots[node] = Spot::Relay(Box::new(self.fresh(node))); + } + } + + pub fn goaway(&mut self, node: Node) { + for peer in self.peers(node) { + if let Some(router) = self.relay(peer) { + router.goaway(node); + } + } + self.flush(); + } + + pub fn client(&mut self, node: Node) { + self.spots[node] = Spot::Endpoint { + id: None, + claims: BTreeSet::new(), + asked: BTreeMap::new(), + }; + } + + pub fn publish(&mut self, node: Node, named: Id) { + let Spot::Endpoint { id, .. } = &mut self.spots[node] else { + panic!("node {node} is a relay"); + }; + *id = Some(named); + } + + pub fn claim(&mut self, node: Node, claim: &str, live: bool) { + let claim = p(claim); + let Spot::Endpoint { id, claims, asked } = &mut self.spots[node] else { + panic!("node {node} is a relay"); + }; + let id = id.expect("a publisher"); + let change = match live { + true if claims.insert(claim.clone()) => { + *self.lives.entry((id, claim.clone())).or_default() += 1; + relay::Change::Live(own()) + } + false if claims.remove(&claim) => relay::Change::Ended, + _ => return, + }; + let tell: Vec = (asked.iter()) + .filter(|(_, prefixes)| prefixes.iter().any(|prefix| within(&claim, prefix))) + .map(|(peer, _)| *peer) + .collect(); + for peer in tell { + let changes = vec![((claim.clone(), id), change.clone())]; + self.send(node, peer, Msg::Announce { changes }); + } + } + + fn send(&mut self, from: Node, to: Node, msg: Message) { + // What a dead process still had to say goes nowhere once its link is down, and + // never onto the session that replaces it. + if !self.up.contains(&pair(from, to)) { + return; + } + match &msg { + Msg::Route { .. } | Msg::Dead { .. } | Msg::Drain { .. } => { + self.stats.route_msgs += 1; + self.stats.route_items += 1; + } + Msg::Claims { changes, .. } | Msg::Announce { changes } => { + self.stats.claim_msgs += 1; + self.stats.claim_items += changes.len(); + } + Msg::Ask(_) | Msg::Edited | Msg::Synced(_) => unreachable!("the interest exchange crosses at once"), + } + self.queues.entry((from, to)).or_default().push_back(msg); + } + + /// Hand `to` a message of the interest exchange, and answer it at once if `to` is a + /// publisher. + fn exchange(&mut self, from: Node, to: Node, msg: Message) { + let Spot::Endpoint { id, claims, asked } = &mut self.spots[to] else { + let router = self.relay(to).expect("a relay"); + if let Err(err) = router.recv(from, msg.clone()) { + panic!("{to} refused {msg:?} from {from}: {err}"); + } + return; + }; + // A publisher asks nothing, and answers what it is asked. + let Msg::Ask(edit) = msg else { + return; + }; + let asks = asked.entry(from).or_default(); + for prefix in &edit.remove { + asks.remove(prefix); + } + asks.extend(edit.add.iter().cloned()); + let changes: Vec<_> = (claims.iter()) + .filter(|claim| edit.add.iter().any(|prefix| within(claim, prefix))) + .map(|claim| ((claim.clone(), id.expect("a publisher")), relay::Change::Live(own()))) + .collect(); + self.exchange(to, from, Msg::Edited); + if !changes.is_empty() { + self.send(to, from, Msg::Announce { changes }); + } + for prefix in edit.add { + self.exchange(to, from, Msg::Synced(prefix)); + } + } + + pub fn hold(&mut self, from: Node, to: Node) { + self.held.insert((from, to)); + } + + pub fn ready(&self) -> Vec<((Node, Node), usize)> { + (self.queues.iter()) + .filter(|(key, queue)| !queue.is_empty() && !self.held.contains(key)) + .map(|(key, queue)| (*key, queue.len())) + .collect() + } + + pub fn deliver(&mut self, from: Node, to: Node) -> bool { + let Some(msg) = self.queues.get_mut(&(from, to)).and_then(VecDeque::pop_front) else { + return false; + }; + + if let Some(router) = self.relay(to) + && let Err(err) = router.recv(from, msg.clone()) + { + panic!("{to} refused {msg:?} from {from}: {err}"); + } + self.stats.delivered += 1; + // A relay reads a burst first and answers when the round ends. + self.show(); + true + } + + fn live(&self, broadcast: &multi::Broadcast) -> bool { + self.spots.iter().enumerate().any(|(node, spot)| match spot { + Spot::Endpoint { id, claims, .. } => { + *id == Some(broadcast.0) && claims.contains(&broadcast.1) && !self.peers(node).is_empty() + } + Spot::Relay(_) => false, + }) + } + + /// Move every router's output onto the queues, the interest exchange across at once, + /// and log the reader events. + pub fn flush(&mut self) { + loop { + let mut exchanged = Vec::new(); + for node in 0..self.spots.len() { + let Some(router) = self.relay(node) else { + continue; + }; + for (to, msg) in router.drain() { + match msg { + Msg::Ask(_) | Msg::Edited | Msg::Synced(_) => exchanged.push((node, to, msg)), + msg => self.send(node, to, msg), + } + } + } + if exchanged.is_empty() { + break; + } + for (from, to, msg) in exchanged { + self.exchange(from, to, msg); + } + } + self.show(); + } + + /// One hop of delay passes: what the round left goes out, and a reader whose debounce + /// ran out shows its claim offline. Whether a debounce still runs. + pub fn tick(&mut self) -> bool { + self.flush(); + self.now = self.now.saturating_add(HOP); + let now = self.now; + let mut waiting = false; + for node in 0..self.spots.len() { + if let Some(router) = self.relay(node) { + router.timeout(now); + waiting |= router.deadline().is_some(); + } + } + self.show(); + waiting + } + + fn show(&mut self) { + for node in 0..self.spots.len() { + let Some(router) = self.relay(node) else { + continue; + }; + for event in router.events() { + let broadcast = (event.id, event.claim); + let seen = Seen { + node, + live: self.live(&broadcast), + life: self.lives.get(&broadcast).copied().unwrap_or(0), + broadcast, + kind: event.kind, + }; + self.events.push(seen); + } + } + } + + /// The nodes a request from `from` for `claim` of `id` visits, and whether it ended at + /// a publisher. `None` when `from` knows no claim it can reach. A chain longer than + /// the network loops. + pub fn chase(&self, from: Node, id: Id, claim: &Path) -> Option<(Vec, bool)> { + let broadcast = (claim.clone(), id); + let (mut at, mut named, mut hops) = (from, None, Vec::new()); + while hops.len() <= self.spots.len() { + let router = match &self.spots[at] { + Spot::Endpoint { id: owner, claims, .. } => { + return Some((hops, *owner == Some(id) && claims.contains(claim))); + } + Spot::Relay(router) => router, + }; + let Some(origin) = named.or_else(|| router.pick(&broadcast)) else { + return (!hops.is_empty()).then_some((hops, false)); + }; + match router.step(origin, &broadcast) { + // A request names the forwarding node its first hop chose. + Some(Next::Via(next)) => { + named = Some(origin); + hops.push(next); + at = next; + } + // Past a claim's announcer the next node chooses again. + Some(Next::Local(next)) => { + named = None; + hops.push(next); + at = next; + } + None => return Some((hops, false)), + } + } + Some((hops, false)) + } + + pub fn work(&self) -> usize { + self.spent + + (0..self.spots.len()) + .filter_map(|node| self.router(node)) + .map(Router::work_done) + .sum::() + } + + pub fn kept(&self) -> relay::Kept { + let mut out = relay::Kept::default(); + for router in (0..self.spots.len()).filter_map(|node| self.router(node)) { + let kept = router.kept(); + out.routes += kept.routes; + out.used += kept.used; + out.hops += kept.hops; + out.sent += kept.sent; + out.items += kept.items; + out.heard += kept.heard; + out.told += kept.told; + out.pending += kept.pending; + out.closed += kept.closed; + out.waiting += kept.waiting; + } + out + } +} + +impl Net for RelaySim { + fn client(&mut self, node: Node) { + RelaySim::client(self, node); + } + + fn link(&mut self, a: Node, b: Node, cost: Metric, tie: Tie) { + RelaySim::link(self, a, b, cost, tie); + } + + fn goaway(&mut self, node: Node) { + RelaySim::goaway(self, node); + } + + fn open(&mut self, a: Node, b: Node) { + RelaySim::open(self, a, b); + } + + fn close(&mut self, a: Node, b: Node) { + RelaySim::close(self, a, b); + } + + fn mute(&mut self, node: Node, peer: Node) { + self.hold(node, peer); + self.hold(peer, node); + } + + fn sever(&mut self, node: Node, peer: Node) { + RelaySim::sever(self, node, peer); + } + + fn reset(&mut self, node: Node) { + RelaySim::reset(self, node); + } + + fn publish(&mut self, node: Node, id: Id) { + RelaySim::publish(self, node, id); + } + + fn claim(&mut self, node: Node, claim: &str, live: bool) { + RelaySim::claim(self, node, claim, live); + } + + fn ready(&self) -> Vec<(Chan, usize)> { + RelaySim::ready(self) + } + + fn step(&mut self, (from, to): Chan) { + self.deliver(from, to); + } + + fn tick(&mut self) -> bool { + RelaySim::tick(self) + } + + fn chase(&self, from: Node, id: Id) -> Option<(Vec, bool)> { + RelaySim::chase(self, from, id, &p(&Mesh::claim(0))) + } + + fn showing(&self, node: Node, id: Id, claim: &str) -> Option { + let shown = self.router(node)?.shown().get(&(id, p(claim)))?; + Some(shown.available) + } + + fn sent(&self) -> Sent { + let (offline, unavailable, starts) = tally(self.events.iter().map(|event| event.kind)); + Sent { + msgs: self.stats.route_msgs, + items: self.stats.route_items, + claims: self.stats.claim_items, + offline, + unavailable, + starts, + } + } + + fn held(&self) -> Held { + let kept = self.kept(); + Held { + routes: kept.used, + offers: kept.routes, + claims: kept.items + kept.heard + kept.told, + pending: kept.pending, + lasting: kept.closed, + } + } + + fn faults(&self) -> Faults { + Faults { + false_ends: multi::false_ends(&self.events), + revivals: multi::revivals(&self.events), + } + } + + fn work(&self) -> usize { + RelaySim::work(self) + } +} + +/// It carries every change of the path rule's comparison, and is right whenever it +/// settles. +#[test] +fn the_router_converges_on_every_shape() { + let mut rng = XorShift(0x6d6f_7100); + let mut meshes = vec![Mesh::ring(6), Mesh::clique(6, 1), Mesh::clique(4, 3)]; + meshes.extend((0..8).map(|_| Mesh::sparse(6, &mut rng))); + for mesh in &meshes { + run(Arm::RELAY, mesh); + } +} + +/// Under churn it settles and is right once it does. +#[test] +fn the_router_settles_after_churn() { + for two_homes in [false, true] { + churn(Arm::RELAY, 0..60, two_homes); + } +} + +/// Every failure on every shape of the bake-off: it settles, every relay with a path +/// routes to its publisher, no reader sees an end the publisher never made or a life it +/// had seen end, and nothing waits on another node once it settles. +#[test] +fn the_router_carries_every_failure_on_every_shape() { + carries(Arm::RELAY, 9); +} + +/// The same at the quest's largest size. +#[test] +#[ignore = "a sweep; Nightly runs it from test/route-sweep.sh"] +fn sweep_the_router_carries_every_failure_at_32_relays() { + carries(Arm::RELAY, 32); +} + +/// What one failure's row says, by field. +fn routing(row: &Row) -> Vec<(&'static str, String)> { + let cost = &row.cost; + vec![ + ("route_msgs", format!("{}", cost.sent.msgs)), + ("claim_items", format!("{}", cost.sent.claims)), + ("settle_rounds", format!("{}", cost.rounds)), + ("starts", format!("{}", cost.sent.starts)), + ("offline", format!("{}", cost.sent.offline)), + ("fed_rounds", format!("{}", cost.right)), + ("stale", format!("{}", cost.stale)), + ("dark", format!("{}", cost.dark)), + ("loops", format!("{}", cost.loops)), + ("settled", format!("{}", cost.settled)), + ("faults", format!("{:?}", row.faults)), + ("pending", format!("{}", row.pending)), + ("hops", format!("{}", row.hops)), + ("right", format!("{:?}", row.right)), + ("answers", format!("{:?}", row.answers)), + ] +} + +/// The fields of two rows that differ, each as the prototype's and this router's. +fn differ(kept: &Row, relay: &Row) -> Vec { + (routing(kept).into_iter().zip(routing(relay))) + .filter(|((_, a), (_, b))| a != b) + .map(|((field, a), (_, b))| match field { + "answers" => "answers".to_string(), + _ => format!("{field} {a}->{b}"), + }) + .collect() +} + +/// The production router, its readers letting go at once as the prototype's do, gives +/// the prototype's answers on every shape and failure of the bake-off at the same cost: +/// the same next hops for every request once it settles, the same route messages, the +/// same rounds, and the same reader events. +/// +/// It sends fewer claims in one case. A relay that loses every route to a forwarding node +/// and then hears a lagging neighbour's copy of its claims within one burst passes them +/// on in the prototype, at a floor its neighbours already hold, on routes it withdraws +/// in the same burst (`home_dies` on `tiers`); this router withdraws those routes first. +#[test] +fn the_router_gives_the_prototypes_answers() { + let mut out = Vec::new(); + for shape in shapes(9, 2) { + let kept = fail(Arm::KEPT, &shape.mesh, false); + let relay = fail(Arm::Relay(Duration::ZERO), &shape.mesh, false); + assert_eq!(kept.len(), relay.len(), "{}", shape.name); + for ((failure, kept), (_, relay)) in kept.iter().zip(&relay) { + let fields = differ(kept, relay); + // Fewer claims is the one field allowed to differ. + let fewer = kept.cost.sent.claims > relay.cost.sent.claims; + if fields.len() > usize::from(fewer) { + out.push(format!("{} on {}: {}", failure.as_str(), shape.name, fields.join(", "))); + } + } + } + assert!(out.is_empty(), "{}", out.join("\n")); +} + From edb24a09696961fd60a46b85e8eff003ecd595ce Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 04:21:03 -0800 Subject: [PATCH 04/17] test(moq-net): search every delivery order on route::relay::Router The triangle from its publisher's first claim and a diamond whose shared link closes, in the gate; the triangle from its first flush, nightly. Each property has a planted defect it alone is shown to catch. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/relay.rs | 102 ++- rs/moq-net/src/route/tests/mod.rs | 1 + rs/moq-net/src/route/tests/multi.rs | 4 + rs/moq-net/src/route/tests/multicast.rs | 151 ++++- .../src/route/tests/multicast_search.rs | 621 ++++++++++++++++++ 5 files changed, 854 insertions(+), 25 deletions(-) create mode 100644 rs/moq-net/src/route/tests/multicast_search.rs diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index 575bfe53a58..7fca866822c 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -402,6 +402,42 @@ pub(crate) struct Router { /// Routes, claims, hops, and neighbours visited, for per-change work checks. #[cfg(test)] work: std::cell::Cell, + /// This node's own claims at each revision, for the search's truth. + #[cfg(test)] + history: Vec>, + /// How many copies of a forwarding node's claims this node dropped. + #[cfg(test)] + forgets: u32, + /// A defect a fixture plants, to show a check catches it. + #[cfg(test)] + mutant: Option, +} + +/// Whether a fixture planted the defect `$mutant` in `$router`: never outside tests. +macro_rules! mutated { + ($router:expr, $mutant:ident) => {{ + #[cfg(test)] + let planted = $router.mutant == Some(Mutant::$mutant); + #[cfg(not(test))] + let planted = false; + planted + }}; +} + +/// A defect a fixture plants in a router, to show the property it breaks is checked. +#[cfg(test)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Mutant { + /// A claim its announcer lost shows as ended (NoFalseEnd). + LostEnds, + /// An older copy of a node's claims is merged into a newer one (NoRevival). + OldFloor, + /// A copy that lagged is applied as if it were current (Truthful). + Lagging, + /// A closed session is never forgotten (QuietIsSettled). + KeepDead, + /// Every input re-advertises every route, changed or not (Budget). + Chatter, } impl Router { @@ -444,6 +480,12 @@ impl Router { out: Vec::new(), #[cfg(test)] work: std::cell::Cell::new(0), + #[cfg(test)] + history: vec![BTreeMap::new()], + #[cfg(test)] + forgets: 0, + #[cfg(test)] + mutant: None, } } @@ -773,6 +815,9 @@ impl Router { fn route(&mut self, from: N, origin: NodeId, offer: Option) -> Result<(), Error> { self.visit(); + if mutated!(self, Chatter) { + self.stale.extend(self.best.keys().copied().chain(self.announced.then_some(self.me))); + } let sender = self.peers[&from].routing().expect("a routing neighbour"); match offer { Some(offer) => { @@ -893,6 +938,12 @@ impl Router { self.unclaim(&broadcast, origin); self.dirty.insert(broadcast); } + // Without any copy, this node no longer knows how new a claim of `origin` is: + // a stale one may start it again, until the newer floor behind it arrives. + #[cfg(test)] + { + self.forgets += u32::from(known.floor > 0); + } } for peer in self.sent_to.remove(&origin).into_iter().flatten() { self.visit(); @@ -955,7 +1006,7 @@ impl Router { fn forget(&mut self) { for hop in std::mem::take(&mut self.unnamed.0) { self.visit(); - if !self.live.contains_key(&hop) { + if !self.live.contains_key(&hop) && !mutated!(self, KeepDead) { self.dead.remove(&hop); } } @@ -1164,7 +1215,8 @@ impl Router { let state = self.peers.get_mut(&peer).expect("a routing neighbour"); let was = state.sent.get(&origin); // Unchanged, unless a closed session ended what it was sent, which it is owed. - if was.is_none_or(Option::is_some) && was.and_then(Option::as_ref) == offer.as_ref() { + let unchanged = was.is_none_or(Option::is_some) && was.and_then(Option::as_ref) == offer.as_ref(); + if unchanged && !(mutated!(self, Chatter) && offer.is_some()) { return; } let fresh = was.is_none(); @@ -1255,11 +1307,11 @@ impl Router { heard.floor = floor; let known = self.known.entry(origin).or_default(); // The newest floor wins, so an older copy never revives what a newer one withdrew. - if floor <= known.floor { + if floor <= known.floor && !mutated!(self, OldFloor) { return Ok(()); } let mut visited = 0; - let changes = match was == known.floor { + let changes = match was == known.floor || mutated!(self, Lagging) { // The copy matched this node's at the same revision, so the message's changes // bring it to the new one. true => changes, @@ -1288,7 +1340,8 @@ impl Router { } }; self.count(visited); - self.known.get_mut(&origin).expect("known").floor = floor; + let known = self.known.get_mut(&origin).expect("known"); + known.floor = known.floor.max(floor); self.publish(origin, changes); Ok(()) } @@ -1484,6 +1537,8 @@ impl Router { self.stale.insert(me); } self.publish(me, vec![(broadcast, change)]); + #[cfg(test)] + self.history.push(self.known[&me].items.clone()); } // --- Requests --- @@ -1553,7 +1608,7 @@ impl Router { presented: false, })) } - None if self.ended.contains(broadcast) => { + None if self.ended.contains(broadcast) || mutated!(self, LostEnds) => { self.unwait(broadcast); Some(None) } @@ -1579,9 +1634,10 @@ impl Router { now.entry(instance.clone()).or_default().insert(path.clone(), shown); } } + let lost_ends = mutated!(self, LostEnds); let Self { reader, ended, .. } = self; for (instance, claims) in now { - let ended = |claim: &Path| ended.contains(&(claim.clone(), instance.clone())); + let ended = |claim: &Path| lost_ends || ended.contains(&(claim.clone(), instance.clone())); reader.apply(instance.clone(), |_| true, claims, ended); } for broadcast in &dirty { @@ -1748,11 +1804,37 @@ impl Router { } } + /// Plant `mutant`. + pub fn mutate(&mut self, mutant: Mutant) { + self.mutant = Some(mutant); + } + + /// This node's own claims at each revision, the first empty. + pub fn history(&self) -> &[Claims] { + &self.history + } + + /// How many copies of a forwarding node's claims this node dropped, after each of which + /// a stale copy may start a claim it saw end. + pub fn forgets(&self) -> u32 { + self.forgets + } + + /// Whether anything waits for [`Self::drain`]. + pub fn pending(&self) -> bool { + !self.out.is_empty() || !self.stale.is_empty() || !self.fresh.is_empty() + } + /// The route this node uses to `origin`: the neighbour, and the offer as it stands here. pub fn best(&self, origin: NodeId) -> Option<(N, &Offer)> { self.best.get(&origin).map(|best| (best.via, &best.offer)) } + /// The forwarding nodes this node routes to. + pub fn routes(&self) -> Vec { + self.best.keys().copied().collect() + } + /// What each neighbour offers for `origin`, and whether a closed session ended it. pub fn heard(&self, origin: NodeId) -> Vec<(N, &Offer, bool)> { (self.heard.get(&origin).into_iter().flatten()) @@ -1821,8 +1903,10 @@ impl Router { reader, out, work: _, + history: _, + forgets: _, + mutant: _, } = self; - assert!(out.is_empty(), "drain before fingerprinting"); // A deadline counts by how far off it is; one due counts as zero. let waiting: BTreeMap<&Broadcast, Duration> = (waiting.iter()) .map(|(broadcast, at)| (broadcast, at.saturating_duration_since(*now))) @@ -1830,7 +1914,7 @@ impl Router { let state = ( (me, debounce, sessions, peers, routing, fresh, heard, unnamed), (dead, closed, draining, stale, best, known, rev, announced), - (sources, ended, dirty, waiting, reader.shown()), + (sources, ended, dirty, waiting, reader.shown(), out), ); super::digest(hasher, &state); } diff --git a/rs/moq-net/src/route/tests/mod.rs b/rs/moq-net/src/route/tests/mod.rs index 29d0f4dbda6..09096a7fa6b 100644 --- a/rs/moq-net/src/route/tests/mod.rs +++ b/rs/moq-net/src/route/tests/mod.rs @@ -18,6 +18,7 @@ mod handoff; mod interest; mod multi; mod multicast; +mod multicast_search; mod residual; mod routing; mod scope; diff --git a/rs/moq-net/src/route/tests/multi.rs b/rs/moq-net/src/route/tests/multi.rs index be57a98471b..a85e7f0a84f 100644 --- a/rs/moq-net/src/route/tests/multi.rs +++ b/rs/moq-net/src/route/tests/multi.rs @@ -1224,6 +1224,9 @@ pub(super) struct Seen { pub life: u64, /// Whether a publisher with an open session claimed it. pub live: bool, + /// How many times the node had let go of a forwarding node's claims before, after + /// which a stale copy may start a claim it saw end. The prototype does not count. + pub forgot: u32, } /// A network of [`Router`]s and endpoints, with [`Sim`]'s queues. @@ -1501,6 +1504,7 @@ impl MultiSim { life: self.lives.get(&event.broadcast).copied().unwrap_or(0), broadcast: event.broadcast, kind: event.kind, + forgot: 0, }; self.events.push(seen); } diff --git a/rs/moq-net/src/route/tests/multicast.rs b/rs/moq-net/src/route/tests/multicast.rs index c0269fc75ec..fb92085f480 100644 --- a/rs/moq-net/src/route/tests/multicast.rs +++ b/rs/moq-net/src/route/tests/multicast.rs @@ -27,8 +27,10 @@ enum Spot { Relay(Box), /// A publisher: an endpoint that never forwards. Endpoint { + /// The Instance its next claim serves. id: Option, - claims: BTreeSet, + /// Its claims, each with the Instance it serves. + claims: BTreeSet<(Path, Id)>, /// What each relay asked it for. asked: BTreeMap>, }, @@ -51,6 +53,12 @@ pub(super) struct RelaySim { spent: usize, now: Instant, debounce: Duration, + /// Each forwarding node's claims at every revision, kept past its restart. + histories: BTreeMap>>, + mutant: Option, + /// Every router sends what it has after each change to the network, as the bake-off + /// runs it; the search flushes each router as a step of its own instead. + pub auto: bool, } fn pair(a: Node, b: Node) -> (Node, Node) { @@ -85,6 +93,9 @@ impl RelaySim { spent: 0, now: Instant::ZERO, debounce, + histories: BTreeMap::new(), + mutant: None, + auto: true, }; sim.spots = (0..nodes).map(|node| Spot::Relay(Box::new(sim.fresh(node)))).collect(); sim @@ -101,9 +112,98 @@ impl RelaySim { fn fresh(&self, node: Node) -> Router { let mut router = Router::new(self.id(node), self.debounce); router.clock(self.now); + if let Some(mutant) = self.mutant { + router.mutate(mutant); + } router } + /// Plant `mutant` in every router, now and after a restart. + pub fn mutate(&mut self, mutant: relay::Mutant) { + self.mutant = Some(mutant); + for node in 0..self.spots.len() { + if let Some(router) = self.relay(node) { + router.mutate(mutant); + } + } + } + + /// `origin`'s claims at revision `floor`, as it announced them. + pub fn truth(&self, origin: NodeId, floor: Rev) -> Option<&relay::Claims> { + let current = (0..self.spots.len()) + .filter_map(|node| self.router(node)) + .find(|router| router.me() == origin) + .map(Router::history); + let history = current.or_else(|| self.histories.get(&origin).map(Vec::as_slice))?; + history.get(usize::try_from(floor).ok()?) + } + + /// Whether `node` has output waiting for its next flush. + pub fn pending(&self, node: Node) -> bool { + self.router(node).is_some_and(Router::pending) + } + + /// `node` sends what it has: [`Self::flush`] for one router. + pub fn flush_node(&mut self, node: Node) { + let Some(router) = self.relay(node) else { + return; + }; + let out = router.drain(); + let mut exchanged = Vec::new(); + for (to, msg) in out { + match msg { + Msg::Ask(_) | Msg::Edited | Msg::Synced(_) => exchanged.push((node, to, msg)), + msg => self.send(node, to, msg), + } + } + for (from, to, msg) in exchanged { + self.exchange(from, to, msg); + } + self.show(); + } + + /// `node`'s next reader deadline passes. + pub fn fire(&mut self, node: Node) { + let Some(router) = self.relay(node) else { + return; + }; + if let Some(at) = router.deadline() { + router.timeout(at); + } + self.show(); + } + + /// The nodes with a reader deadline set. + pub fn timers(&self) -> Vec { + (0..self.spots.len()) + .filter(|&node| self.router(node).is_some_and(|router| router.deadline().is_some())) + .collect() + } + + /// The open links. + pub fn up(&self) -> &BTreeSet<(Node, Node)> { + &self.up + } + + /// The claims the publisher at `node` makes now, each with its Instance. + pub fn claims(&self, node: Node) -> &BTreeSet<(Path, Id)> { + match &self.spots[node] { + Spot::Endpoint { claims, .. } => claims, + Spot::Relay(_) => panic!("node {node} is a relay"), + } + } + + /// Hash what decides the network's future, leaving out what only records its past. + pub fn fingerprint(&self, hasher: &mut impl std::hash::Hasher) { + for spot in &self.spots { + match spot { + Spot::Relay(router) => router.fingerprint(hasher), + Spot::Endpoint { id, claims, asked } => digest(hasher, &(id, claims, asked)), + } + } + digest(hasher, &(&self.restarts, &self.up, &self.queues, &self.held)); + } + pub fn router(&self, node: Node) -> Option<&Router> { match &self.spots[node] { Spot::Relay(router) => Some(router), @@ -160,7 +260,9 @@ impl RelaySim { router.connect(y, port).expect("a new session"); } } - self.flush(); + if self.auto { + self.flush(); + } } /// `peer` lets go of its session to `node`. @@ -180,7 +282,9 @@ impl RelaySim { self.held.remove(&(x, y)); self.drop_session(x, y); } - self.flush(); + if self.auto { + self.flush(); + } } /// `peer` notices its session to `node` closed, which `node` has yet to. @@ -191,7 +295,9 @@ impl RelaySim { self.held.remove(&key); } self.drop_session(node, peer); - self.flush(); + if self.auto { + self.flush(); + } } /// The process at `node` exits: its sessions close and its state is lost. @@ -202,6 +308,7 @@ impl RelaySim { self.restarts[node] += 1; if let Spot::Relay(router) = &self.spots[node] { self.spent += router.work_done(); + self.histories.insert(router.me(), router.history().to_vec()); self.spots[node] = Spot::Relay(Box::new(self.fresh(node))); } } @@ -212,7 +319,9 @@ impl RelaySim { router.goaway(node); } } - self.flush(); + if self.auto { + self.flush(); + } } pub fn client(&mut self, node: Node) { @@ -235,15 +344,23 @@ impl RelaySim { let Spot::Endpoint { id, claims, asked } = &mut self.spots[node] else { panic!("node {node} is a relay"); }; - let id = id.expect("a publisher"); let change = match live { - true if claims.insert(claim.clone()) => { + true if !claims.iter().any(|(path, _)| *path == claim) => { + let id = id.expect("a publisher"); + claims.insert((claim.clone(), id)); *self.lives.entry((id, claim.clone())).or_default() += 1; - relay::Change::Live(own()) + (id, relay::Change::Live(own())) } - false if claims.remove(&claim) => relay::Change::Ended, + false => match claims.iter().find(|(path, _)| *path == claim).cloned() { + Some(gone) => { + claims.remove(&gone); + (gone.1, relay::Change::Ended) + } + None => return, + }, _ => return, }; + let (id, change) = change; let tell: Vec = (asked.iter()) .filter(|(_, prefixes)| prefixes.iter().any(|prefix| within(&claim, prefix))) .map(|(peer, _)| *peer) @@ -277,7 +394,7 @@ impl RelaySim { /// Hand `to` a message of the interest exchange, and answer it at once if `to` is a /// publisher. fn exchange(&mut self, from: Node, to: Node, msg: Message) { - let Spot::Endpoint { id, claims, asked } = &mut self.spots[to] else { + let Spot::Endpoint { claims, asked, .. } = &mut self.spots[to] else { let router = self.relay(to).expect("a relay"); if let Err(err) = router.recv(from, msg.clone()) { panic!("{to} refused {msg:?} from {from}: {err}"); @@ -294,8 +411,8 @@ impl RelaySim { } asks.extend(edit.add.iter().cloned()); let changes: Vec<_> = (claims.iter()) - .filter(|claim| edit.add.iter().any(|prefix| within(claim, prefix))) - .map(|claim| ((claim.clone(), id.expect("a publisher")), relay::Change::Live(own()))) + .filter(|(claim, _)| edit.add.iter().any(|prefix| within(claim, prefix))) + .map(|claim| (claim.clone(), relay::Change::Live(own()))) .collect(); self.exchange(to, from, Msg::Edited); if !changes.is_empty() { @@ -335,8 +452,8 @@ impl RelaySim { fn live(&self, broadcast: &multi::Broadcast) -> bool { self.spots.iter().enumerate().any(|(node, spot)| match spot { - Spot::Endpoint { id, claims, .. } => { - *id == Some(broadcast.0) && claims.contains(&broadcast.1) && !self.peers(node).is_empty() + Spot::Endpoint { claims, .. } => { + claims.contains(&(broadcast.1.clone(), broadcast.0)) && !self.peers(node).is_empty() } Spot::Relay(_) => false, }) @@ -390,6 +507,7 @@ impl RelaySim { let Some(router) = self.relay(node) else { continue; }; + let forgot = router.forgets(); for event in router.events() { let broadcast = (event.id, event.claim); let seen = Seen { @@ -398,6 +516,7 @@ impl RelaySim { life: self.lives.get(&broadcast).copied().unwrap_or(0), broadcast, kind: event.kind, + forgot, }; self.events.push(seen); } @@ -412,8 +531,8 @@ impl RelaySim { let (mut at, mut named, mut hops) = (from, None, Vec::new()); while hops.len() <= self.spots.len() { let router = match &self.spots[at] { - Spot::Endpoint { id: owner, claims, .. } => { - return Some((hops, *owner == Some(id) && claims.contains(claim))); + Spot::Endpoint { claims, .. } => { + return Some((hops, claims.contains(&(claim.clone(), id)))); } Spot::Relay(router) => router, }; diff --git a/rs/moq-net/src/route/tests/multicast_search.rs b/rs/moq-net/src/route/tests/multicast_search.rs new file mode 100644 index 00000000000..807817bd5ec --- /dev/null +++ b/rs/moq-net/src/route/tests/multicast_search.rs @@ -0,0 +1,621 @@ +//! Every order in which [`RelaySim`] can deliver, flush a router, let a reader's debounce +//! run out, change a claim, and open or close a link on a small mesh, searched depth-first +//! on [`relay::Router`], with the multicast rule's properties checked in every state it +//! reaches: `search.rs` for the rule the switch puts lite-07 sessions on. +//! +//! A router reads any burst before it answers, so flushing one is a step of its own and +//! every batching is an order. The interest exchange that opens a session crosses as the +//! router flushes, as [`RelaySim`] carries it. A state is the whole [`RelaySim`], +//! fingerprinted without what only records its past, with each reader deadline by how far +//! off it is. A broken property prints the path to it as [`RelaySim`] calls. +//! +//! The properties: no reader sees an end its publisher never made (NoFalseEnd) or a life +//! it saw end start again (NoRevival); every copy of a forwarding node's claims at a floor +//! is that node's claims at that revision (Truthful); messages per change stay within a +//! budget per link and forwarding node (Budget); and once nothing is in flight, nothing +//! waits to flush, and no debounce runs, each relay shows exactly the claims a path of +//! open sessions carries, every request reaches its publisher, every route leads to its +//! forwarding node, and nothing about a closed session is left (QuietIsSettled). Requests +//! may loop while news is in flight, which the bake-off measures, so loops are checked +//! once quiet. Each property has a [`Mutant`] it is shown to catch. + +use std::{ + collections::{BTreeSet, HashSet, VecDeque, hash_map::DefaultHasher}, + fmt, + hash::Hasher, + panic::{AssertUnwindSafe, catch_unwind}, +}; + +use crate::route::relay::{self, Mutant}; + +use super::multi::Tie; +use super::multicast::RelaySim; +use super::*; + +/// How far the search may go. +#[derive(Clone, Debug)] +struct Bounds { + /// The paths the publisher may claim at, each life a new Instance. + claims: &'static [&'static str], + /// Claims announced or withdrawn. + changes: u32, + /// Links opened or closed. + flaps: u32, +} + +/// One step of a path, printed as the [`RelaySim`] call that replays it. +#[derive(Clone, Debug, PartialEq, Eq)] +enum Step { + Deliver(Node, Node), + Flush(Node), + Fire(Node), + Announce(&'static str, Id), + Withdraw(&'static str), + Open(Node, Node), + Close(Node, Node), +} + +impl fmt::Display for Step { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Deliver(from, to) => write!(f, "sim.deliver({from}, {to});"), + Self::Flush(node) => write!(f, "sim.flush_node({node});"), + Self::Fire(node) => write!(f, "sim.fire({node});"), + Self::Announce(claim, id) => write!(f, "sim.publish(P, {id}); sim.claim(P, {claim:?}, true);"), + Self::Withdraw(claim) => write!(f, "sim.claim(P, {claim:?}, false);"), + Self::Open(a, b) => write!(f, "sim.open({a}, {b});"), + Self::Close(a, b) => write!(f, "sim.close({a}, {b});"), + } + } +} + +/// A property the search checks. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Rule { + NoFalseEnd, + NoRevival, + Truthful, + Budget, + QuietIsSettled, +} + +/// What broke in a step: a property, or an assertion in a router or the sim. +#[derive(Debug)] +enum Broken { + Rule(Rule, String), + Panic(String), +} + +impl fmt::Display for Broken { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Rule(rule, why) => write!(f, "{rule:?}: {why}"), + Self::Panic(message) => write!(f, "a panic: {message}"), + } + } +} + +/// What the search covered. +#[derive(Clone, Copy, Debug)] +struct Report { + states: usize, + steps: usize, + depth: usize, + /// The most messages one change cost. + burst: usize, +} + +/// A broken property, and the path the search took to it. +#[derive(Debug)] +struct Failure { + broken: Broken, + path: Vec, +} + +impl fmt::Display for Failure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "{} after {} steps:", self.broken, self.path.len())?; + for step in &self.path { + writeln!(f, "\t{step}")?; + } + Ok(()) + } +} + +/// The mesh a search starts from: relays, one publisher, and the links that may flap. +struct Mesh { + sim: RelaySim, + relays: usize, + publisher: Node, + links: Vec<(Node, Node)>, + /// The Instance the publisher's next claim serves. + next: Id, +} + +/// A searched state: the [`RelaySim`], and what the properties and bounds remember. +#[derive(Clone)] +struct World { + sim: RelaySim, + changes: u32, + flaps: u32, + next: Id, + /// The broadcasts each reader saw end, and how many times it had forgotten each then. + ended: BTreeSet<(Node, Path, Id, u32)>, + /// Messages sent since the network was last quiet. + burst: usize, + /// External changes since the network was last quiet. + churn: usize, +} + +impl World { + fn fingerprint(&self) -> u64 { + let mut hasher = DefaultHasher::new(); + self.sim.fingerprint(&mut hasher); + digest( + &mut hasher, + &(self.changes, self.flaps, self.next, &self.ended, self.burst, self.churn), + ); + hasher.finish() + } + + fn queued(&self) -> usize { + self.sim.ready().iter().map(|(_, count)| count).sum() + } +} + +struct Search<'a> { + bounds: &'a Bounds, + relays: usize, + publisher: Node, + links: Vec<(Node, Node)>, + /// Messages one change may cost: four per direction of each link for every + /// forwarding node and every claim. + budget: usize, + /// The one property checked, if not every one: where a planted defect breaks another + /// first, or an assertion, the search goes on past it, so it shows this one is checked. + only: Option, +} + +fn search(mesh: Mesh, bounds: &Bounds) -> Result { + search_for(mesh, bounds, None) +} + +fn search_for(mesh: Mesh, bounds: &Bounds, only: Option) -> Result { + let Mesh { + mut sim, + relays, + publisher, + links, + next, + } = mesh; + sim.events.clear(); + sim.auto = false; + let budget = 4 * (2 * links.len()) * (relays + bounds.claims.len()); + let search = Search { + bounds, + relays, + publisher, + links, + budget, + only, + }; + let root = World { + sim, + changes: 0, + flaps: 0, + next, + ended: BTreeSet::new(), + burst: 0, + churn: 1, + }; + search.check(&root).map_err(|broken| Failure { + broken, + path: Vec::new(), + })?; + let mut seen = HashSet::from([root.fingerprint()]); + let next = search.steps(&root); + let mut stack = vec![(root, None::, next)]; + let (mut steps, mut depth, mut burst) = (0, 0, 0); + while let Some(at) = stack.len().checked_sub(1) { + depth = depth.max(at); + let Some(step) = stack[at].2.pop() else { + stack.pop(); + continue; + }; + steps += 1; + let mut world = stack[at].0.clone(); + match search.apply(&mut world, &step) { + Ok(()) => {} + // What broke it is not what this search looks for, and a state an assertion + // stopped halfway is no state to go on from. + Err(Broken::Panic(_)) if only.is_some() => continue, + Err(broken) => { + let mut path: Vec = stack.iter().filter_map(|(_, by, _)| by.clone()).collect(); + path.push(step); + return Err(Failure { broken, path }); + } + } + burst = burst.max(world.burst / world.churn.max(1)); + if !seen.insert(world.fingerprint()) { + continue; + } + let next = search.steps(&world); + stack.push((world, Some(step), next)); + } + Ok(Report { + states: seen.len(), + steps, + depth, + burst, + }) +} + +impl Search<'_> { + fn checks(&self, rule: Rule) -> bool { + self.only.is_none_or(|only| only == rule) + } + + /// Nothing in flight, nothing waiting to flush, and no debounce running. + fn quiet(&self, sim: &RelaySim) -> bool { + sim.ready().is_empty() && (0..self.relays).all(|node| !sim.pending(node)) && sim.timers().is_empty() + } + + /// Every step enabled in `world`. + fn steps(&self, world: &World) -> Vec { + let sim = &world.sim; + let mut out: Vec = (sim.ready().into_iter()) + .map(|((from, to), _)| Step::Deliver(from, to)) + .collect(); + out.extend((0..self.relays).filter(|&node| sim.pending(node)).map(Step::Flush)); + out.extend(sim.timers().into_iter().map(Step::Fire)); + if world.changes < self.bounds.changes { + for &claim in self.bounds.claims { + match sim.claims(self.publisher).iter().any(|(path, _)| *path == p(claim)) { + true => out.push(Step::Withdraw(claim)), + false => out.push(Step::Announce(claim, world.next)), + } + } + } + if world.flaps < self.bounds.flaps { + for &(a, b) in &self.links { + match sim.up().contains(&(a.min(b), a.max(b))) { + true => out.push(Step::Close(a, b)), + false => out.push(Step::Open(a, b)), + } + } + } + out + } + + /// Take `step` and check every property, catching a router's or the sim's assertion. + fn apply(&self, world: &mut World, step: &Step) -> Result<(), Broken> { + let quiet = self.quiet(&world.sim); + let queued = world.queued(); + let taken = catch_unwind(AssertUnwindSafe(|| self.take(world, step))); + if let Err(panic) = taken { + let message = (panic.downcast_ref::().cloned()) + .or_else(|| panic.downcast_ref::<&str>().map(|s| s.to_string())) + .unwrap_or_default(); + return Err(Broken::Panic(message)); + } + // What this step sent: every queue now, less what was there and stayed. + let consumed = usize::from(matches!(step, Step::Deliver(..))); + let sent = (world.queued() + consumed).saturating_sub(queued); + let external = matches!( + step, + Step::Announce(..) | Step::Withdraw(_) | Step::Open(..) | Step::Close(..) + ); + world.burst = if external && quiet { 0 } else { world.burst } + sent; + if external { + world.churn = if quiet { 0 } else { world.churn } + 1; + } + for event in std::mem::take(&mut world.sim.events) { + let (id, path) = event.broadcast; + match event.kind { + Kind::End if event.live && self.checks(Rule::NoFalseEnd) => { + let why = format!("node {} saw {path} of {id} end", event.node); + return Err(Broken::Rule(Rule::NoFalseEnd, why)); + } + Kind::End => { + world.ended.insert((event.node, path, id, event.forgot)); + } + // Excused once the node forgot it, as `search.rs` excuses the path rule. + Kind::Start + if self.checks(Rule::NoRevival) + && world.ended.contains(&(event.node, path.clone(), id, event.forgot)) => + { + let why = format!("node {} saw {path} of {id} start again", event.node); + return Err(Broken::Rule(Rule::NoRevival, why)); + } + _ => {} + } + } + self.check(world) + } + + fn take(&self, world: &mut World, step: &Step) { + let sim = &mut world.sim; + match *step { + Step::Deliver(from, to) => assert!(sim.deliver(from, to)), + Step::Flush(node) => sim.flush_node(node), + Step::Fire(node) => sim.fire(node), + Step::Announce(claim, id) => { + sim.publish(self.publisher, id); + sim.claim(self.publisher, claim, true); + world.next += 1; + world.changes += 1; + } + Step::Withdraw(claim) => { + sim.claim(self.publisher, claim, false); + world.changes += 1; + } + Step::Open(a, b) => { + sim.open(a, b); + world.flaps += 1; + } + Step::Close(a, b) => { + sim.close(a, b); + world.flaps += 1; + } + } + } + + fn check(&self, world: &World) -> Result<(), Broken> { + let sim = &world.sim; + if self.checks(Rule::Truthful) { + self.truthful(sim).map_err(|why| Broken::Rule(Rule::Truthful, why))?; + } + if self.checks(Rule::Budget) && world.burst > self.budget * world.churn { + let why = format!( + "{} messages for {} changes, over {} each", + world.burst, world.churn, self.budget + ); + return Err(Broken::Rule(Rule::Budget, why)); + } + if self.checks(Rule::QuietIsSettled) && self.quiet(sim) { + self.settled(sim).map_err(|why| Broken::Rule(Rule::QuietIsSettled, why))?; + } + Ok(()) + } + + /// Every copy a relay holds of a forwarding node's claims is that node's claims at the + /// copy's floor. + fn truthful(&self, sim: &RelaySim) -> Result<(), String> { + for node in 0..self.relays { + let Some(router) = sim.router(node) else { continue }; + for (origin, floor, items) in router.copies() { + let truth = sim.truth(origin, floor); + if truth != Some(items) { + return Err(format!( + "node {node} holds {origin:?}'s claims at {floor} as {items:?}, where it announced {truth:?}" + )); + } + } + } + Ok(()) + } + + /// The relays a path of open sessions joins to `from`, `from` included. + fn reach(&self, sim: &RelaySim, from: &[Node]) -> BTreeSet { + let mut seen: BTreeSet = from.iter().copied().collect(); + let mut queue: VecDeque = from.iter().copied().collect(); + while let Some(at) = queue.pop_front() { + for &(a, b) in sim.up() { + let next = match at { + _ if at == a => b, + _ if at == b => a, + _ => continue, + }; + if next < self.relays && seen.insert(next) { + queue.push_back(next); + } + } + } + seen + } + + /// Each relay shows exactly the claims a path of open sessions carries to it, available, + /// and a request for each reaches the publisher; every route a relay holds leads over + /// open sessions to its forwarding node, and it holds one to every forwarding node + /// such a path joins it to; and no relay remembers a closed session or a drain. + fn settled(&self, sim: &RelaySim) -> Result<(), String> { + let homes: Vec = (sim.up().iter()) + .filter_map(|&(a, b)| match self.publisher { + _ if a == self.publisher => Some(b), + _ if b == self.publisher => Some(a), + _ => None, + }) + .collect(); + let carried = match homes.is_empty() { + true => BTreeSet::new(), + false => self.reach(sim, &homes), + }; + let claims = sim.claims(self.publisher); + for node in 0..self.relays { + let router = sim.router(node).expect("a relay"); + for (claim, id) in claims { + let shown = router.shown().get(&(*id, claim.clone())); + match carried.contains(&node) { + true if shown.is_none_or(|shown| !shown.available) => { + return Err(format!("node {node} shows {claim} of {id} as {shown:?}")); + } + true if sim.chase(node, *id, claim).is_none_or(|(_, reached)| !reached) => { + return Err(format!("a request for {claim} of {id} from node {node} stops short")); + } + _ => {} + } + } + for (id, claim) in router.shown().keys() { + if !carried.contains(&node) || !claims.contains(&(claim.clone(), *id)) { + return Err(format!("node {node} still shows {claim} of {id}")); + } + } + // Routes: to every forwarding node an open path joins, and over open sessions. + let joined = self.reach(sim, &[node]); + for other in (0..self.relays).filter(|&other| other != node) { + let origin = sim.router(other).expect("a relay").me(); + let announced = sim.router(other).expect("a relay").own().0 > 0; + let want = announced && joined.contains(&other); + if router.best(origin).is_some() != want { + return Err(format!( + "node {node} {} a route to {origin:?}", + if want { "lacks" } else { "holds" } + )); + } + if want { + let (mut at, mut hops) = (node, 0); + while at != other { + let Some((via, _)) = sim.router(at).and_then(|router| router.best(origin)) else { + return Err(format!("node {node}'s route to {origin:?} stops at {at}")); + }; + if !sim.up().contains(&(at.min(via), at.max(via))) || hops > self.relays { + return Err(format!("node {node}'s route to {origin:?} breaks at {at}")); + } + (at, hops) = (via, hops + 1); + } + } + } + for origin in router.routes() { + let current = (0..self.relays).any(|other| sim.router(other).expect("a relay").me() == origin); + if !current { + return Err(format!("node {node} holds a route to {origin:?}, which is gone")); + } + } + let kept = router.kept(); + if kept.pending > 0 { + return Err(format!("node {node} still remembers {kept:?}")); + } + } + Ok(()) + } +} + +/// Three relays in a triangle with a publisher on relay 0, every session open and its +/// interest exchanged, the publisher's first claim in flight. Every link may flap. +fn triangle() -> Mesh { + let mut mesh = unanswered(); + settle(&mut mesh.sim); + mesh.sim.claim(mesh.publisher, "x", true); + mesh +} + +/// The triangle with every session just opened and nothing flushed. +fn unanswered() -> Mesh { + let mut sim = RelaySim::new(4, relay::debounce(HOP)); + sim.client(3); + sim.publish(3, 100); + sim.auto = false; + for (a, b) in [(0, 1), (0, 2), (1, 2)] { + sim.link(a, b, 1, Tie::Peer); + } + sim.link(3, 0, 1, Tie::Peer); + Mesh { + sim, + relays: 3, + publisher: 3, + links: vec![(0, 1), (0, 2), (1, 2), (3, 0)], + next: 101, + } +} + +/// Relay 1 behind relay 0 and its publisher, relays 2 and 3 behind relay 1, and relay 4 +/// behind both, everything settled: relay 4 hears two routes through the link from 0 to 1, +/// so whichever neighbour names it closed first, the other's route still names it. That +/// link may close. +fn diamond() -> Mesh { + let mut sim = RelaySim::new(6, relay::debounce(HOP)); + sim.client(5); + sim.publish(5, 100); + sim.auto = false; + for (a, b) in [(0, 1), (1, 2), (1, 3), (2, 4), (3, 4)] { + sim.link(a, b, 1, Tie::Peer); + } + sim.link(5, 0, 1, Tie::Peer); + sim.claim(5, "x", true); + settle(&mut sim); + Mesh { + sim, + relays: 5, + publisher: 5, + links: vec![(0, 1)], + next: 101, + } +} + +/// Flush and deliver until nothing is in flight. +fn settle(sim: &mut RelaySim) { + loop { + sim.flush(); + let ready = sim.ready(); + if ready.is_empty() { + return; + } + for ((from, to), _) in ready { + while sim.deliver(from, to) {} + } + } +} + +/// One claim change and one flap. +const ONE: Bounds = Bounds { + claims: &["x"], + changes: 1, + flaps: 1, +}; + +fn run(mesh: Mesh, bounds: &Bounds) -> Report { + let report = search(mesh, bounds).unwrap_or_else(|failure| panic!("{failure}")); + eprintln!("{report:?}"); + report +} + +/// The triangle from its publisher's first claim, in the merge gate. +#[test] +#[cfg_attr(miri, ignore = "searches thousands of states")] +fn every_order_on_a_triangle_keeps_the_multicast_rules() { + run(triangle(), &ONE); +} + +/// The diamond's shared link closing, with one claim change: the dead hop a relay keeps +/// for a route that still names it, until that route goes. +#[test] +#[cfg_attr(miri, ignore = "searches thousands of states")] +fn every_order_past_a_shared_hop_keeps_the_multicast_rules() { + run(diamond(), &ONE); +} + +// What the gate leaves to Nightly (`test/route-sweep.sh`). + +/// The triangle from its sessions' first flush: every order of the interest exchange, +/// the first claim, and the route starts it brings. +#[test] +#[ignore = "Nightly"] +fn every_order_from_the_first_flush_keeps_the_multicast_rules() { + let mut mesh = unanswered(); + mesh.sim.claim(mesh.publisher, "x", true); + run(mesh, &ONE); +} + +/// Each property fails when a defect it guards against is planted, so the search checks +/// what it claims to. +#[test] +#[cfg_attr(miri, ignore = "searches thousands of states")] +fn every_property_catches_its_mutant() { + for (mutant, rule) in [ + (Mutant::LostEnds, Rule::NoFalseEnd), + (Mutant::OldFloor, Rule::NoRevival), + (Mutant::Lagging, Rule::Truthful), + (Mutant::KeepDead, Rule::QuietIsSettled), + (Mutant::Chatter, Rule::Budget), + ] { + // Only the diamond keeps a dead hop. + let mut mesh = match mutant { + Mutant::KeepDead => diamond(), + _ => triangle(), + }; + mesh.sim.mutate(mutant); + let failure = search_for(mesh, &ONE, Some(rule)).expect_err(&format!("{mutant:?} went unnoticed")); + eprintln!("{mutant:?}: {failure}"); + assert!(matches!(failure.broken, Broken::Rule(broke, _) if broke == rule)); + } +} From 1289fb078fa96809ce88d8a435d3260f6eaacee3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 05:44:56 -0800 Subject: [PATCH 05/17] test(moq-net): the bake-off's fixtures, the debounce, and per-event work on route::relay::Router The fixtures the prototype's rule earned run on the router too. The readers' debounce outlasts every failure that brings nothing back, at 9 relays and at 32. A sweep over forwarding nodes, publishers, broadcasts, and sessions shows an event costs the router what it changed. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/relay.rs | 151 +++++++-- rs/moq-net/src/route/tests/bakeoff.rs | 71 ++-- rs/moq-net/src/route/tests/mod.rs | 1 + rs/moq-net/src/route/tests/multicast.rs | 193 ++++++++++- rs/moq-net/src/route/tests/multicast_cost.rs | 315 ++++++++++++++++++ .../src/route/tests/multicast_search.rs | 3 +- 6 files changed, 672 insertions(+), 62 deletions(-) create mode 100644 rs/moq-net/src/route/tests/multicast_cost.rs diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index 7fca866822c..c628e0fe89c 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -411,6 +411,10 @@ pub(crate) struct Router { /// A defect a fixture plants, to show a check catches it. #[cfg(test)] mutant: Option, + /// Whether each input ends by rebuilding every index to compare: on but where a + /// benchmark's mesh is too large for it. + #[cfg(test)] + checked: bool, } /// Whether a fixture planted the defect `$mutant` in `$router`: never outside tests. @@ -486,6 +490,8 @@ impl Router { forgets: 0, #[cfg(test)] mutant: None, + #[cfg(test)] + checked: true, } } @@ -766,7 +772,11 @@ impl Router { return Ok(()); } Msg::Synced(prefix) => { - let synced = peer.asked.get_mut(&prefix).filter(|_| peer.answering).ok_or(Error::Unasked)?; + let synced = peer + .asked + .get_mut(&prefix) + .filter(|_| peer.answering) + .ok_or(Error::Unasked)?; *synced = true; return Ok(()); } @@ -808,7 +818,9 @@ impl Router { fn done(&mut self) { self.refresh(); #[cfg(test)] - self.check(); + if self.checked { + self.check(); + } } // --- Routes --- @@ -816,7 +828,8 @@ impl Router { fn route(&mut self, from: N, origin: NodeId, offer: Option) -> Result<(), Error> { self.visit(); if mutated!(self, Chatter) { - self.stale.extend(self.best.keys().copied().chain(self.announced.then_some(self.me))); + self.stale + .extend(self.best.keys().copied().chain(self.announced.then_some(self.me))); } let sender = self.peers[&from].routing().expect("a routing neighbour"); match offer { @@ -826,8 +839,11 @@ impl Router { let path = &offer.path; let built = path.len() >= 2 || offer.cost == 0; if origin.relay == self.me.relay - || path.first() != Some(&Hop { node: origin, session: 0 }) - || path.last().map(|hop| hop.node) != Some(sender) + || path.first() + != Some(&Hop { + node: origin, + session: 0, + }) || path.last().map(|hop| hop.node) != Some(sender) || path.len() > MAX_HOPS || names(path, self.me.relay) || !built @@ -956,7 +972,12 @@ impl Router { fn track(&mut self, origin: NodeId, from: N, path: &[Hop]) { for hop in path { self.visit(); - self.rx_hops.entry(*hop).or_default().entry(from).or_default().insert(origin); + self.rx_hops + .entry(*hop) + .or_default() + .entry(from) + .or_default() + .insert(origin); *self.live.entry(*hop).or_default() += 1; } for (pair, _) in pairs(path) { @@ -1809,6 +1830,11 @@ impl Router { self.mutant = Some(mutant); } + /// Stop rebuilding every index after each input, for a benchmark's large mesh. + pub fn unchecked(&mut self) { + self.checked = false; + } + /// This node's own claims at each revision, the first empty. pub fn history(&self) -> &[Claims] { &self.history @@ -1906,6 +1932,7 @@ impl Router { history: _, forgets: _, mutant: _, + checked: _, } = self; // A deadline counts by how far off it is; one due counts as zero. let waiting: BTreeMap<&Broadcast, Duration> = (waiting.iter()) @@ -1929,7 +1956,12 @@ impl Router { for (peer, heard) in offers { assert!(self.peers[peer].offers.contains(origin), "offers index"); for hop in heard.offer.path.iter() { - rx_hops.entry(*hop).or_default().entry(*peer).or_default().insert(*origin); + rx_hops + .entry(*hop) + .or_default() + .entry(*peer) + .or_default() + .insert(*origin); if !heard.ended { *live.entry(*hop).or_default() += 1; } @@ -1950,11 +1982,19 @@ impl Router { for (origin, offer) in &state.sent { sent_to.entry(*origin).or_default().insert(*peer); for hop in offer.iter().flat_map(|offer| offer.path.iter()) { - tx_hops.entry(*hop).or_default().entry(*peer).or_default().insert(*origin); + tx_hops + .entry(*hop) + .or_default() + .entry(*peer) + .or_default() + .insert(*origin); } } for (broadcast, claim) in &state.given { - givers.entry(broadcast.clone()).or_default().insert(*peer, claim.clone()); + givers + .entry(broadcast.clone()) + .or_default() + .insert(*peer, claim.clone()); } if state.routing().is_none() { for prefix in &state.asks { @@ -1988,8 +2028,9 @@ impl Router { } assert_eq!(self.claimants, claimants, "claimants"); assert_eq!(self.shared, shared, "shared claims"); - let deadlines: BTreeSet<(Instant, Broadcast)> = - (self.waiting.iter()).map(|(broadcast, at)| (*at, broadcast.clone())).collect(); + let deadlines: BTreeSet<(Instant, Broadcast)> = (self.waiting.iter()) + .map(|(broadcast, at)| (*at, broadcast.clone())) + .collect(); assert_eq!(self.deadlines, deadlines, "deadlines"); assert!(self.dirty.is_empty(), "refreshed"); } @@ -2043,7 +2084,15 @@ mod tests { Role::Scoped(_) | Role::Endpoint => Vec::new(), }; if !ask.is_empty() { - router.edit(peer, Edit { remove: Vec::new(), add: ask }).unwrap(); + router + .edit( + peer, + Edit { + remove: Vec::new(), + add: ask, + }, + ) + .unwrap(); } } @@ -2064,9 +2113,12 @@ mod tests { open(&mut router, 2, Role::Scoped(vec![p("a")])); let inside = ((p("a/x"), 7), live(0)); let outside = ((p("b/x"), 7), live(0)); - let err = router.recv(2, Msg::Announce { - changes: vec![inside.clone(), outside], - }); + let err = router.recv( + 2, + Msg::Announce { + changes: vec![inside.clone(), outside], + }, + ); assert_eq!(err, Err(Error::Unasked)); assert!(router.shown().is_empty(), "nothing of it applied"); router.recv(2, Msg::Announce { changes: vec![inside] }).unwrap(); @@ -2149,8 +2201,25 @@ mod tests { }; router.recv(2, route).unwrap(); // A dead hop the route crosses, with the wrong node at its far end. - assert_eq!(router.recv(2, Msg::Dead { hop: sender, far: node(9) }), Err(Error::Malformed)); - router.recv(2, Msg::Dead { hop: sender, far: node(5) }).unwrap(); + assert_eq!( + router.recv( + 2, + Msg::Dead { + hop: sender, + far: node(9) + } + ), + Err(Error::Malformed) + ); + router + .recv( + 2, + Msg::Dead { + hop: sender, + far: node(5), + }, + ) + .unwrap(); } /// A claim lost with nothing to replace it shows unavailable for the debounce, then @@ -2193,15 +2262,25 @@ mod tests { fn an_endpoint_is_shown_what_covers_its_prefix() { let mut router = relay(1); open(&mut router, 2, Role::Endpoint); - router.recv(2, Msg::Announce { - changes: vec![((p("live"), 7), live(0)), ((p("live/a/x"), 8), live(0))], - }).unwrap(); + router + .recv( + 2, + Msg::Announce { + changes: vec![((p("live"), 7), live(0)), ((p("live/a/x"), 8), live(0))], + }, + ) + .unwrap(); open(&mut router, 3, Role::Endpoint); router.drain(); - router.edit(3, Edit { - remove: Vec::new(), - add: vec![p("live/a")], - }).unwrap(); + router + .edit( + 3, + Edit { + remove: Vec::new(), + add: vec![p("live/a")], + }, + ) + .unwrap(); let out = router.drain(); let told: Vec<&Msg> = (out.iter()).filter(|(to, _)| *to == 3).map(|(_, msg)| msg).collect(); let shown = |path: &str, instance: u64| { @@ -2214,17 +2293,27 @@ mod tests { assert_eq!(told.first(), Some(&&Msg::Edited)); assert_eq!(told.last(), Some(&&Msg::Synced(p("live/a")))); // Withdrawing the broader claim withdraws what stood for it. - router.recv(2, Msg::Announce { - changes: vec![((p("live"), 7), Change::Ended)], - }).unwrap(); + router + .recv( + 2, + Msg::Announce { + changes: vec![((p("live"), 7), Change::Ended)], + }, + ) + .unwrap(); let out = router.drain(); assert!(out.iter().any(|(to, msg)| *to == 3 && matches!(msg, Msg::Announce { changes } if changes == &vec![((p("live/a"), 7), Change::Ended)]))); // Swapping its prefix in place drops what the old one covered, silently. - router.edit(3, Edit { - remove: vec![p("live/a")], - add: vec![p("other")], - }).unwrap(); + router + .edit( + 3, + Edit { + remove: vec![p("live/a")], + add: vec![p("other")], + }, + ) + .unwrap(); let out = router.drain(); let told: Vec<&Msg> = (out.iter()).filter(|(to, _)| *to == 3).map(|(_, msg)| msg).collect(); assert_eq!(told, [&Msg::Edited, &Msg::Synced(p("other"))]); diff --git a/rs/moq-net/src/route/tests/bakeoff.rs b/rs/moq-net/src/route/tests/bakeoff.rs index 4856539694c..6ae950995a7 100644 --- a/rs/moq-net/src/route/tests/bakeoff.rs +++ b/rs/moq-net/src/route/tests/bakeoff.rs @@ -584,22 +584,26 @@ fn an_endpoint_changes_no_route() { /// for a moment announces it again, its first claim. #[test] fn a_first_claim_starts_a_route() { - let (mut echoed, mut started) = (0, 0); - for shape in shapes(9, 2) { - let scoped = shape.mesh.ties.values().any(|tie| *tie == Tie::Scoped); - for (failure, row) in fail(Arm::LEAN, &shape.mesh, false) { - let case = format!("{} on {}", failure.as_str(), shape.name); - match failure { - Failure::Leave | Failure::LoseOne | Failure::LoseBoth if scoped => echoed += row.cost.sent.msgs, - Failure::Leave | Failure::LoseOne | Failure::LoseBoth => assert_eq!(row.cost.sent.msgs, 0, "{case}"), - // Unless the relay it moves to has announced before. - Failure::Move => started += row.cost.sent.msgs, - _ => {} + for arm in [Arm::LEAN, Arm::RELAY] { + let (mut echoed, mut started) = (0, 0); + for shape in shapes(9, 2) { + let scoped = shape.mesh.ties.values().any(|tie| *tie == Tie::Scoped); + for (failure, row) in fail(arm, &shape.mesh, false) { + let case = format!("{} on {} under {}", failure.as_str(), shape.name, arm.name()); + match failure { + Failure::Leave | Failure::LoseOne | Failure::LoseBoth if scoped => echoed += row.cost.sent.msgs, + Failure::Leave | Failure::LoseOne | Failure::LoseBoth => { + assert_eq!(row.cost.sent.msgs, 0, "{case}") + } + // Unless the relay it moves to has announced before. + Failure::Move => started += row.cost.sent.msgs, + _ => {} + } } } + assert!(started > 0, "no move started a route under {}", arm.name()); + assert!(echoed > 0, "no gateway announced an echo under {}", arm.name()); } - assert!(started > 0, "no move started a route"); - assert!(echoed > 0, "no gateway announced an echo"); } /// A relay with nothing to announce costs no route anywhere: the routes a mesh @@ -611,26 +615,30 @@ fn a_relay_with_no_claim_offers_no_route() { lab.net.held() }; for publishers in [1, 2, 9] { - let (all, lean, today) = ( - held(Arm::TUNED, publishers), - held(Arm::LEAN, publishers), - held(Arm::TODAY, publishers), - ); - println!("{publishers} publishers: {all:?} {lean:?} {today:?}"); - assert!( - lean.offers * 15 <= all.offers * publishers.min(9) + all.offers, - "{publishers}" - ); - assert!(lean.routes <= today.routes, "{publishers}"); + let (all, today) = (held(Arm::TUNED, publishers), held(Arm::TODAY, publishers)); + for arm in [Arm::LEAN, Arm::RELAY] { + let lean = held(arm, publishers); + println!( + "{publishers} publishers: {all:?} {lean:?} {today:?} under {}", + arm.name() + ); + assert!( + lean.offers * 15 <= all.offers * publishers.min(9) + all.offers, + "{publishers} under {}", + arm.name() + ); + assert!(lean.routes <= today.routes, "{publishers} under {}", arm.name()); + } } } /// A failure costs route messages per forwarding node, never per publisher behind it. +/// Every relay is one under the prototype as tuned; under the router, every relay a +/// publisher joins is, so it is measured with the same relays behind more publishers. #[test] fn a_failure_costs_the_same_with_more_publishers() { - let cost = |publishers: usize| { - let mesh = tiers(3, 3, true, publishers); - let rows = fail(Arm::TUNED, &mesh, false); + let cost = |arm: Arm, mesh: &Mesh| { + let rows = fail(arm, mesh, false); let of = |failure: Failure| { rows.iter() .find(|(found, _)| *found == failure) @@ -638,7 +646,14 @@ fn a_failure_costs_the_same_with_more_publishers() { }; (of(Failure::Close), of(Failure::Partition)) }; - assert_eq!(cost(1), cost(12)); + let mesh = |publishers: usize| tiers(3, 3, true, publishers); + assert_eq!(cost(Arm::TUNED, &mesh(1)), cost(Arm::TUNED, &mesh(12))); + let few = mesh(3); + let many = Mesh { + homes: few.homes.iter().cycle().take(12).copied().collect(), + ..few.clone() + }; + assert_eq!(cost(Arm::RELAY, &few), cost(Arm::RELAY, &many)); } /// Remembering the highest session closed between a pair of nodes stops a stale diff --git a/rs/moq-net/src/route/tests/mod.rs b/rs/moq-net/src/route/tests/mod.rs index 09096a7fa6b..f0951f5be78 100644 --- a/rs/moq-net/src/route/tests/mod.rs +++ b/rs/moq-net/src/route/tests/mod.rs @@ -18,6 +18,7 @@ mod handoff; mod interest; mod multi; mod multicast; +mod multicast_cost; mod multicast_search; mod residual; mod routing; diff --git a/rs/moq-net/src/route/tests/multicast.rs b/rs/moq-net/src/route/tests/multicast.rs index fb92085f480..5fcfc814ec9 100644 --- a/rs/moq-net/src/route/tests/multicast.rs +++ b/rs/moq-net/src/route/tests/multicast.rs @@ -13,7 +13,7 @@ use kio::clock::Instant; use crate::route::relay::{self, Claim, Msg, Next, NodeId, Port, Role}; -use super::bakeoff::{Row, carries, fail, shapes}; +use super::bakeoff::{Failure, Row, carries, fail, shapes}; use super::compare::{Arm, Chan, Faults, Held, Mesh, Net, Sent, churn, run, tally}; use super::multi::{self, Seen, Tie}; use super::*; @@ -49,6 +49,8 @@ pub(super) struct RelaySim { held: BTreeSet<(Node, Node)>, lives: BTreeMap, pub stats: Stats, + /// Messages each node sent, the interest exchange left out. + pub sent: BTreeMap, pub events: Vec, spent: usize, now: Instant, @@ -56,6 +58,8 @@ pub(super) struct RelaySim { /// Each forwarding node's claims at every revision, kept past its restart. histories: BTreeMap>>, mutant: Option, + /// Whether each router rebuilds its indexes after every input to compare. + checked: bool, /// Every router sends what it has after each change to the network, as the bake-off /// runs it; the search flushes each router as a step of its own instead. pub auto: bool, @@ -89,12 +93,14 @@ impl RelaySim { held: BTreeSet::new(), lives: BTreeMap::new(), stats: Stats::default(), + sent: BTreeMap::new(), events: Vec::new(), spent: 0, now: Instant::ZERO, debounce, histories: BTreeMap::new(), mutant: None, + checked: true, auto: true, }; sim.spots = (0..nodes).map(|node| Spot::Relay(Box::new(sim.fresh(node)))).collect(); @@ -115,9 +121,23 @@ impl RelaySim { if let Some(mutant) = self.mutant { router.mutate(mutant); } + if !self.checked { + router.unchecked(); + } router } + /// Stop every router rebuilding its indexes after each input, now and after a restart: + /// for a benchmark's mesh, where it would cost more than the routers. + pub fn unchecked(&mut self) { + self.checked = false; + for node in 0..self.spots.len() { + if let Some(router) = self.relay(node) { + router.unchecked(); + } + } + } + /// Plant `mutant` in every router, now and after a restart. pub fn mutate(&mut self, mutant: relay::Mutant) { self.mutant = Some(mutant); @@ -388,6 +408,7 @@ impl RelaySim { } Msg::Ask(_) | Msg::Edited | Msg::Synced(_) => unreachable!("the interest exchange crosses at once"), } + *self.sent.entry(from).or_default() += 1; self.queues.entry((from, to)).or_default().push_back(msg); } @@ -423,6 +444,18 @@ impl RelaySim { } } + /// The endpoint at `node` asks `relay` for `prefix`. + pub fn ask(&mut self, node: Node, relay: Node, prefix: &str) { + let edit = relay::Edit { + remove: Vec::new(), + add: vec![p(prefix)], + }; + self.exchange(node, relay, Msg::Ask(edit)); + if self.auto { + self.flush(); + } + } + pub fn hold(&mut self, from: Node, to: Node) { self.held.insert((from, to)); } @@ -711,11 +744,41 @@ fn the_router_carries_every_failure_on_every_shape() { carries(Arm::RELAY, 9); } -/// The same at the quest's largest size. +/// The same at the quest's largest size, where the debounce still outlasts every failure. #[test] #[ignore = "a sweep; Nightly runs it from test/route-sweep.sh"] fn sweep_the_router_carries_every_failure_at_32_relays() { carries(Arm::RELAY, 32); + assert_eq!(flaps(Arm::RELAY, 32), Vec::::new()); +} + +/// Each failure on each shape at `relays` that showed a reader a claim start again, though +/// it brings nothing back, with how many times. +fn flaps(arm: Arm, relays: usize) -> Vec { + let mut out = Vec::new(); + for shape in shapes(relays, 2) { + for (failure, row) in fail(arm, &shape.mesh, relays > 16) { + let returns = matches!(failure, Failure::Heal | Failure::Restart); + if !returns && row.cost.sent.starts > 0 { + out.push(format!("{} on {}: {}", failure.as_str(), shape.name, row.cost.sent.starts)); + } + } + } + out +} + +/// The readers' debounce, a hop for each node a route may cross, outlasts the hunting a +/// failure causes: a failure that brings nothing back never shows a reader a claim start +/// again, only unavailable for a while. Without it a reader lets go and takes the claim +/// back on most failures, and on every relay when a publisher moves. +#[test] +fn the_debounce_outlasts_a_failure() { + assert_eq!(flaps(Arm::RELAY, 9), Vec::::new()); + let undebounced = flaps(Arm::Relay(Duration::ZERO), 9); + assert!( + undebounced.iter().any(|flap| flap.starts_with(Failure::Move.as_str())), + "{undebounced:?}" + ); } /// What one failure's row says, by field. @@ -779,3 +842,129 @@ fn the_router_gives_the_prototypes_answers() { assert!(out.is_empty(), "{}", out.join("\n")); } +/// Deliver and flush round by round, the clock advancing a hop each, until nothing is in +/// flight and no debounce runs. +pub(super) fn quiesce(sim: &mut RelaySim) { + for _ in 0..200 { + let ready = sim.ready(); + for &((from, to), count) in &ready { + for _ in 0..count { + sim.deliver(from, to); + } + } + let waiting = sim.tick(); + if ready.is_empty() && !waiting && sim.ready().is_empty() { + return; + } + } + panic!("never settles"); +} + +/// Two relays that share a draining neighbour and a scoped link each hold the other's +/// copy of its claim. The copy says it is draining too, so neither prefers it to its own +/// route and the two never trade places. +#[test] +fn a_claim_across_a_grant_keeps_its_drain() { + let mut sim = RelaySim::new(4, relay::debounce(HOP)); + sim.client(3); + sim.publish(3, 100); + sim.link(0, 1, 1, Tie::Peer); + sim.link(0, 2, 1, Tie::Peer); + sim.link(1, 2, 1, Tie::Scoped); + sim.link(3, 0, 1, Tie::Peer); + sim.claim(3, "live/0", true); + quiesce(&mut sim); + // Both read the GOAWAY before either hears from the other. + sim.goaway(0); + quiesce(&mut sim); + let broadcast = (p("live/0"), 100); + for relay in [1, 2] { + let router = sim.router(relay).expect("a relay"); + assert_eq!(router.pick(&broadcast), Some(sim.id(0)), "relay {relay}"); + } +} + +/// A scoped neighbour that sends GOAWAY is draining for the claims it gave, so a node +/// holding the same claim from another neighbour announces that one instead. +#[test] +fn a_scoped_neighbour_draining_drains_its_claims() { + // A publisher on 0 and 2, each of which grants 1 the claim, 0 the cheaper. + let mut sim = RelaySim::new(4, relay::debounce(HOP)); + sim.client(3); + sim.publish(3, 100); + sim.link(3, 0, 1, Tie::Peer); + sim.link(3, 2, 1, Tie::Peer); + sim.link(0, 1, 1, Tie::Scoped); + sim.link(2, 1, 2, Tie::Scoped); + sim.claim(3, "live/0", true); + quiesce(&mut sim); + let broadcast = (p("live/0"), 100); + let source = |sim: &RelaySim| sim.router(1).expect("a relay").step(sim.id(1), &broadcast); + assert_eq!(source(&sim), Some(Next::Local(0))); + sim.goaway(0); + quiesce(&mut sim); + assert_eq!(source(&sim), Some(Next::Local(2))); +} + +/// A relay that takes a newer copy of a node's claims from another neighbour tells its +/// own neighbours the floor even when nothing in the copy differs from what it had, so +/// one that heard a claim in between, by a route since gone, lets go of it. +#[test] +fn a_newer_floor_travels_with_nothing_changed() { + // 0 announces. 3 hears it through 1, and later 2; 4 hears it through 3 and 5. + let mut sim = RelaySim::new(7, relay::debounce(HOP)); + sim.client(6); + sim.publish(6, 100); + for (a, b) in [(0, 1), (0, 2), (1, 3), (3, 4), (0, 5), (5, 4), (6, 0)] { + sim.link(a, b, 1, Tie::Peer); + } + sim.claim(6, "live/a", true); + quiesce(&mut sim); + let shown = |sim: &RelaySim, relay: Node| sim.showing(relay, 100, "live/b"); + + // 3 misses the claim starting, which reaches 4 through 5. + sim.hold(1, 3); + sim.hold(4, 3); + sim.claim(6, "live/b", true); + quiesce(&mut sim); + assert_eq!((shown(&sim, 3), shown(&sim, 4)), (None, Some(true))); + + // 5 misses it ending, and then 4 loses its route through 5. + sim.hold(0, 5); + sim.claim(6, "live/b", false); + quiesce(&mut sim); + sim.close(5, 4); + quiesce(&mut sim); + assert_eq!(shown(&sim, 4), Some(true), "nothing has told it yet"); + + // 3 hears a newer copy from 2 that matches what it holds. + sim.link(2, 3, 1, Tie::Peer); + quiesce(&mut sim); + assert_eq!(shown(&sim, 4), None); +} + +/// A relay that restarts is a new node, but its neighbours file the session floor per +/// pair of relays with the restart in the session's order, so a restart replaces the +/// entry instead of adding one. +#[test] +fn a_session_floor_keeps_an_entry_per_pair_of_relays() { + let mut sim = RelaySim::new(3, relay::debounce(HOP)); + for (a, b) in [(0, 1), (1, 2), (0, 2)] { + sim.link(a, b, 1, Tie::Peer); + } + quiesce(&mut sim); + let mut kept = Vec::new(); + for _ in 0..8 { + sim.reset(0); + quiesce(&mut sim); + for peer in [1, 2] { + sim.open(0, peer); + } + quiesce(&mut sim); + kept.push(sim.kept().closed); + } + // Two ways round each of the three links at most. + assert!(kept.iter().all(|&pairs| pairs <= 6), "{kept:?}"); + assert_eq!(kept[0], kept[7], "{kept:?}"); +} + diff --git a/rs/moq-net/src/route/tests/multicast_cost.rs b/rs/moq-net/src/route/tests/multicast_cost.rs new file mode 100644 index 00000000000..4ded9a1d765 --- /dev/null +++ b/rs/moq-net/src/route/tests/multicast_cost.rs @@ -0,0 +1,315 @@ +//! What one event costs a relay under [`relay::Router`], swept along each axis it could +//! fan out over: the forwarding nodes it routes to, the publishers behind each, the +//! broadcasts each publisher claims, and the sessions it holds: upstream, each neighbour +//! offering every route; downstream, each sent every route; and endpoints watching, each +//! asking for the whole tree or for a prefix no claim falls under. A sweep holds the other +//! axes fixed and counts the measured relay's work and messages, so an event that visits +//! what it did not change fails here rather than only timing slower: `cost.rs` for the +//! rule the switch puts lite-07 sessions on. + +use crate::route::relay; + +use super::multi::Tie; +use super::multicast::{RelaySim, quiesce}; +use super::*; + +/// The measured relay. +const ME: Node = 0; + +#[derive(Clone, Copy, Debug)] +struct Shape { + /// Forwarding nodes, each a relay its publishers join. + origins: usize, + /// Publishers on each forwarding node. + publishers: usize, + /// Broadcasts each publisher claims. + broadcasts: usize, + /// Relays between the measured one and every forwarding node. + upstream: usize, + /// Relays the measured one alone reaches the mesh through. + downstream: usize, + /// Endpoints on the measured relay that ask it for something. + watchers: usize, + /// Whether each watcher asks for the whole tree, or a prefix no claim falls under. + watched: bool, +} + +const BASE: Shape = Shape { + origins: 4, + publishers: 2, + broadcasts: 2, + upstream: 2, + downstream: 2, + watchers: 2, + watched: true, +}; + +/// What one event cost the measured relay. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct Cost { + /// What it visited. + work: usize, + /// Messages it sent. + sent: usize, +} + +/// A mesh shaped as a [`Shape`], settled. +struct Bed { + sim: RelaySim, + shape: Shape, + /// The next broadcast the first publisher claims. + next: usize, +} + +impl Bed { + fn up(&self, index: usize) -> Node { + 1 + index + } + + fn origin(&self, index: usize) -> Node { + 1 + self.shape.upstream + index + } + + fn down(&self, index: usize) -> Node { + 1 + self.shape.upstream + self.shape.origins + index + } + + fn publisher(&self, origin: usize, index: usize) -> Node { + 1 + self.shape.upstream + self.shape.origins + self.shape.downstream + origin * self.shape.publishers + index + } + + fn watcher(&self, index: usize) -> Node { + self.publisher(self.shape.origins, 0) + index + } + + /// Built with every router rebuilding its indexes after each input, if `checked`. + fn new(shape: Shape, checked: bool) -> Self { + let nodes = 1 + + shape.upstream + + shape.origins + + shape.downstream + + shape.origins * shape.publishers + + shape.watchers; + let mut bed = Self { + sim: RelaySim::new(nodes, relay::debounce(HOP)), + shape, + next: 0, + }; + if !checked { + bed.sim.unchecked(); + } + for origin in 0..shape.origins { + for index in 0..shape.publishers { + let publisher = bed.publisher(origin, index); + bed.sim.client(publisher); + bed.sim.publish(publisher, publisher as Id); + } + } + for up in 0..shape.upstream { + bed.sim.link(ME, bed.up(up), 1, Tie::Peer); + for origin in 0..shape.origins { + bed.sim.link(bed.up(up), bed.origin(origin), 1, Tie::Peer); + } + } + for down in 0..shape.downstream { + bed.sim.link(ME, bed.down(down), 1, Tie::Peer); + } + for origin in 0..shape.origins { + for index in 0..shape.publishers { + let publisher = bed.publisher(origin, index); + bed.sim.link(publisher, bed.origin(origin), 1, Tie::Peer); + for broadcast in 0..shape.broadcasts { + bed.sim.claim(publisher, &format!("o{origin}/p{index}/b{broadcast}"), true); + } + } + } + for index in 0..shape.watchers { + let watcher = bed.watcher(index); + bed.sim.client(watcher); + bed.sim.link(watcher, ME, 1, Tie::Peer); + bed.sim.ask(watcher, ME, if shape.watched { "" } else { "none" }); + } + quiesce(&mut bed.sim); + bed + } + + fn measure(&mut self, change: impl FnOnce(&mut Self)) -> Cost { + let router = |bed: &Self| bed.sim.router(ME).expect("the measured relay").work_done(); + let sent = |bed: &Self| bed.sim.sent.get(&ME).copied().unwrap_or(0); + let (work, was) = (router(self), sent(self)); + change(self); + quiesce(&mut self.sim); + Cost { + work: router(self) - work, + sent: sent(self) - was, + } + } + + /// The first publisher claims one more broadcast. + fn announce(&mut self) -> Cost { + let (publisher, claim) = (self.publisher(0, 0), format!("o0/p0/new{}", self.next)); + self.next += 1; + self.measure(|bed| bed.sim.claim(publisher, &claim, true)) + } + + /// The first upstream relay's session to the first forwarding node closes, then + /// reopens unmeasured. + fn cut(&mut self) -> Cost { + let (up, origin) = (self.up(0), self.origin(0)); + let cost = self.measure(|bed| bed.sim.close(up, origin)); + self.sim.open(up, origin); + quiesce(&mut self.sim); + cost + } + + /// The first downstream relay's session closes, then reopens unmeasured. + fn leave(&mut self) -> Cost { + let down = self.down(0); + let cost = self.measure(|bed| bed.sim.close(ME, down)); + self.sim.open(ME, down); + quiesce(&mut self.sim); + cost + } +} + +/// The cost of `event` on meshes shaped as `BASE` with one axis set to each size, +/// measured after one warm-up event so first-time effects are left out. +fn sweep(axis: fn(&mut Shape, usize), sizes: [usize; 3], event: fn(&mut Bed) -> Cost) -> [Cost; 3] { + sweep_at(axis, sizes, event, true) +} + +fn sweep_at( + axis: fn(&mut Shape, usize), + sizes: [usize; N], + event: fn(&mut Bed) -> Cost, + checked: bool, +) -> [Cost; N] { + sizes.map(|size| { + let mut shape = BASE; + axis(&mut shape, size); + let mut bed = Bed::new(shape, checked); + event(&mut bed); + event(&mut bed) + }) +} + +fn origins(shape: &mut Shape, n: usize) { + shape.origins = n; +} + +fn publishers(shape: &mut Shape, n: usize) { + shape.publishers = n; +} + +fn broadcasts(shape: &mut Shape, n: usize) { + shape.broadcasts = n; +} + +fn upstream(shape: &mut Shape, n: usize) { + shape.upstream = n; +} + +fn downstream(shape: &mut Shape, n: usize) { + shape.downstream = n; +} + +fn watched(shape: &mut Shape, n: usize) { + shape.watchers = n; +} + +fn unwatched(shape: &mut Shape, n: usize) { + shape.watchers = n; + shape.watched = false; +} + +/// One of each, a few, and many. +const FLAT: [usize; 3] = [1, 4, 16]; +/// Doublings, for axes that may grow linearly. Two upstream at least, so a cut leaves the +/// measured relay a route. +const LINEAR: [usize; 3] = [4, 8, 16]; + +fn assert_flat(name: &str, costs: [Cost; 3]) { + assert!( + costs.iter().all(|cost| *cost == costs[0]), + "{name} must not grow: {costs:?}" + ); +} + +fn assert_linear(name: &str, costs: [Cost; 3]) { + for pair in costs.windows(2) { + // Quadratic work would quadruple. + assert!( + pair[1].work <= 5 * pair[0].work / 2, + "{name} grows faster than linear: {costs:?}" + ); + } +} + +/// A claim announced behind a forwarding node costs the relay the sessions it hears it on +/// and the sessions it tells, and nothing per other forwarding node, publisher, broadcast, +/// or watcher whose prefix it misses: the change travels as itself, never as a resend of +/// the claims beside it. +#[test] +fn an_announce_costs_the_sessions_it_reaches() { + assert_flat("forwarding nodes", sweep(origins, FLAT, Bed::announce)); + assert_flat("publishers", sweep(publishers, FLAT, Bed::announce)); + assert_flat("broadcasts", sweep(broadcasts, FLAT, Bed::announce)); + assert_flat("watchers it misses", sweep(unwatched, FLAT, Bed::announce)); + assert_linear("upstream", sweep(upstream, LINEAR, Bed::announce)); + assert_linear("downstream", sweep(downstream, LINEAR, Bed::announce)); + assert_linear("watchers it reaches", sweep(watched, LINEAR, Bed::announce)); +} + +/// A neighbour's route that breaks costs the relay the neighbours offering the same +/// route, which it ranks, and nothing per other forwarding node, publisher, broadcast, or +/// session it sends to. +#[test] +fn a_cut_costs_the_neighbours_offering_the_route() { + assert_flat("forwarding nodes", sweep(origins, FLAT, Bed::cut)); + assert_flat("publishers", sweep(publishers, FLAT, Bed::cut)); + assert_flat("broadcasts", sweep(broadcasts, FLAT, Bed::cut)); + assert_flat("downstream", sweep(downstream, FLAT, Bed::cut)); + assert_flat("watchers", sweep(watched, FLAT, Bed::cut)); + assert_linear("upstream", sweep(upstream, LINEAR, Bed::cut)); +} + +/// A downstream session that closes costs the relay the routes it was sent, one per +/// forwarding node, and nothing per publisher, broadcast, or other session. +#[test] +fn a_leave_costs_the_routes_it_was_sent() { + assert_flat("publishers", sweep(publishers, FLAT, Bed::leave)); + assert_flat("broadcasts", sweep(broadcasts, FLAT, Bed::leave)); + assert_flat("upstream", sweep(upstream, FLAT, Bed::leave)); + assert_flat("downstream", sweep(downstream, FLAT, Bed::leave)); + assert_flat("watchers", sweep(watched, FLAT, Bed::leave)); + assert_linear("forwarding nodes", sweep(origins, LINEAR, Bed::leave)); +} + +/// Each event's cost along each axis, at sizes past the gate's. +#[test] +#[ignore = "a sweep; Nightly runs it from test/route-sweep.sh"] +fn benchmark_relay_event_work() { + type Axis = (&'static str, fn(&mut Shape, usize)); + let axes: [Axis; 7] = [ + ("forwarding_nodes", origins), + ("publishers", publishers), + ("broadcasts", broadcasts), + ("upstream", upstream), + ("downstream", downstream), + ("watched", watched), + ("unwatched", unwatched), + ]; + type Event = (&'static str, fn(&mut Bed) -> Cost); + let events: [Event; 3] = [("announce", Bed::announce), ("cut", Bed::cut), ("leave", Bed::leave)]; + println!("event,axis,size,work,sent"); + for (event, run) in events { + for (axis, set) in axes { + let sizes = [2, 16, 128]; + // A mesh this large checks itself in the gate's sweeps instead. + for (size, cost) in sizes.into_iter().zip(sweep_at(set, sizes, run, false)) { + println!("{event},{axis},{size},{},{}", cost.work, cost.sent); + } + } + } +} + diff --git a/rs/moq-net/src/route/tests/multicast_search.rs b/rs/moq-net/src/route/tests/multicast_search.rs index 807817bd5ec..31d3536e92d 100644 --- a/rs/moq-net/src/route/tests/multicast_search.rs +++ b/rs/moq-net/src/route/tests/multicast_search.rs @@ -373,7 +373,8 @@ impl Search<'_> { return Err(Broken::Rule(Rule::Budget, why)); } if self.checks(Rule::QuietIsSettled) && self.quiet(sim) { - self.settled(sim).map_err(|why| Broken::Rule(Rule::QuietIsSettled, why))?; + self.settled(sim) + .map_err(|why| Broken::Rule(Rule::QuietIsSettled, why))?; } Ok(()) } From f7333e22fbc4fbc7691dccfc7ffd94a0f26ff0d1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 06:01:39 -0800 Subject: [PATCH 06/17] test(moq-net): measure session floor growth on route::relay::Router A relay replaced under a new identity leaves every relay that heard a route across one of its sessions a floor entry for as long as that relay runs; a restart under its own identity replaces the entry instead. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/tests/compare.rs | 12 ++- rs/moq-net/src/route/tests/multicast.rs | 120 +++++++++++++++++++++++- 2 files changed, 126 insertions(+), 6 deletions(-) diff --git a/rs/moq-net/src/route/tests/compare.rs b/rs/moq-net/src/route/tests/compare.rs index 72d7f26094c..f99e35e2790 100644 --- a/rs/moq-net/src/route/tests/compare.rs +++ b/rs/moq-net/src/route/tests/compare.rs @@ -227,6 +227,10 @@ pub(super) trait Net { fn showing(&self, node: Node, id: Id, claim: &str) -> Option; fn sent(&self) -> Sent; fn held(&self) -> Held; + /// The session floors `nodes` keep, where the rule keeps any. + fn floors(&self, _nodes: &[Node]) -> usize { + 0 + } fn faults(&self) -> Faults; /// Routes, claims, and streams visited. fn work(&self) -> usize; @@ -642,9 +646,13 @@ fn pair(a: Node, b: Node) -> (Node, Node) { impl Lab { /// Build `mesh` on `rule`'s router, with every publisher announcing, and let it settle. pub fn new(arm: impl Into, mesh: &Mesh) -> Self { - let nodes = mesh.relays + mesh.homes.len(); + Self::with(arm.into().net(mesh.relays + mesh.homes.len()), mesh) + } + + /// Build `mesh` on `net`, with a node for every relay and publisher. + pub fn with(net: Box, mesh: &Mesh) -> Self { let mut lab = Self { - net: arm.into().net(nodes), + net, mesh: mesh.clone(), up: BTreeSet::new(), linked: BTreeSet::new(), diff --git a/rs/moq-net/src/route/tests/multicast.rs b/rs/moq-net/src/route/tests/multicast.rs index 5fcfc814ec9..9061d1ece54 100644 --- a/rs/moq-net/src/route/tests/multicast.rs +++ b/rs/moq-net/src/route/tests/multicast.rs @@ -14,7 +14,7 @@ use kio::clock::Instant; use crate::route::relay::{self, Claim, Msg, Next, NodeId, Port, Role}; use super::bakeoff::{Failure, Row, carries, fail, shapes}; -use super::compare::{Arm, Chan, Faults, Held, Mesh, Net, Sent, churn, run, tally}; +use super::compare::{Arm, Chan, Faults, Held, Lab, Mesh, Net, Sent, churn, run, tally}; use super::multi::{self, Seen, Tie}; use super::*; @@ -60,6 +60,9 @@ pub(super) struct RelaySim { mutant: Option, /// Whether each router rebuilds its indexes after every input to compare. checked: bool, + /// A relay that resets comes back under a new identity, as a device replaced by + /// another, rather than as a new start of the same relay. + pub mint: bool, /// Every router sends what it has after each change to the network, as the bake-off /// runs it; the search flushes each router as a step of its own instead. pub auto: bool, @@ -101,6 +104,7 @@ impl RelaySim { histories: BTreeMap::new(), mutant: None, checked: true, + mint: false, auto: true, }; sim.spots = (0..nodes).map(|node| Spot::Relay(Box::new(sim.fresh(node)))).collect(); @@ -109,9 +113,16 @@ impl RelaySim { /// How `node` names itself on a path as it runs now. pub fn id(&self, node: Node) -> NodeId { - NodeId { - relay: node as Id, - start: u64::from(self.restarts[node]) + 1, + let generation = u64::from(self.restarts[node]); + match self.mint { + true => NodeId { + relay: node as Id | generation << 32, + start: 1, + }, + false => NodeId { + relay: node as Id, + start: generation + 1, + }, } } @@ -704,6 +715,13 @@ impl Net for RelaySim { } } + fn floors(&self, nodes: &[Node]) -> usize { + (nodes.iter()) + .filter_map(|&node| self.router(node)) + .map(|router| router.kept().closed) + .sum() + } + fn faults(&self) -> Faults { Faults { false_ends: multi::false_ends(&self.events), @@ -968,3 +986,97 @@ fn a_session_floor_keeps_an_entry_per_pair_of_relays() { assert_eq!(kept[0], kept[7], "{kept:?}"); } + +/// The floor's growth with identities that never return: each round a relay under a new +/// identity joins a ring as a chord with a publisher of its own, announces, and leaves for +/// good. Every relay that heard a route across one of its sessions keeps that pair's floor +/// for as long as it runs, at most one entry per direction of each departed session: the +/// epoch bounds only restarts of one relay, so nothing here replaces or removes an entry. +/// The quest's PR brings eviction as a decision. +#[test] +fn a_relay_that_leaves_for_good_leaves_its_floors() { + let (stable, rounds) = (8, 12); + let mut sim = RelaySim::new(stable + 2 * rounds + 1, relay::debounce(HOP)); + let home = stable + 2 * rounds; + sim.client(home); + sim.publish(home, 1); + for a in 0..stable { + sim.link(a, (a + 1) % stable, 1, Tie::Peer); + } + sim.link(home, 0, 1, Tie::Peer); + sim.claim(home, "live/0", true); + quiesce(&mut sim); + let floors = |sim: &RelaySim| -> Vec { + (0..stable) + .map(|relay| sim.router(relay).expect("a relay").kept().closed) + .collect() + }; + assert_eq!(floors(&sim), vec![0; stable]); + let mut sums = Vec::new(); + for round in 0..rounds { + let (relay, client) = (stable + 2 * round, stable + 2 * round + 1); + let (a, b) = (round % stable, (round + stable / 2) % stable); + sim.client(client); + sim.publish(client, 100 + round as Id); + sim.link(relay, a, 1, Tie::Peer); + sim.link(relay, b, 1, Tie::Peer); + sim.link(client, relay, 1, Tie::Peer); + sim.claim(client, &format!("live/r{round}"), true); + quiesce(&mut sim); + for peer in [a, b, client] { + sim.close(relay, peer); + } + quiesce(&mut sim); + let kept = floors(&sim); + println!("after {} departures: {kept:?}", round + 1); + // Two sessions each, a pair each way. + assert!(kept.iter().all(|&entries| entries <= 4 * (round + 1)), "{kept:?}"); + sums.push(kept.iter().sum::()); + } + assert!(sums.windows(2).all(|pair| pair[1] > pair[0]), "{sums:?}"); +} + +/// What the floors cost on the bake-off's shapes as half the relays are replaced, one at +/// a time and over and over, each restarting under its own identity or coming back under +/// a new one: the floor entries a relay of the other half, which never restarts, holds on +/// average after each round of replacements. A restart clears a relay's own table, so a +/// replaced relay's says nothing about growth. +#[test] +#[ignore = "a sweep; Nightly runs it from test/route-sweep.sh"] +fn benchmark_relay_floor_growth() { + println!("shape,relays,links,identity,replaced,floors_per_lasting_relay"); + for relays in [12, 32] { + for shape in shapes(relays, 2) { + let mesh = &shape.mesh; + // The odd relays come and go; the even ones, every publisher's first home + // among them, stay. + let replaced: Vec = (1..mesh.relays).step_by(2).collect(); + let lasting: Vec = (0..mesh.relays).step_by(2).collect(); + for mint in [false, true] { + let mut sim = RelaySim::new(mesh.relays + mesh.homes.len(), relay::debounce(HOP)); + sim.mint = mint; + sim.unchecked(); + let mut lab = Lab::with(Box::new(sim), mesh); + let mut count = 0; + for _ in 0..4 { + for &relay in &replaced { + lab.crash(relay); + lab.settle(); + lab.revive(relay); + lab.settle(); + count += 1; + } + let held = lab.net.floors(&lasting); + println!( + "{},{},{},{},{count},{:.1}", + shape.name, + mesh.relays, + mesh.links.len(), + if mint { "new" } else { "same" }, + held as f64 / lasting.len() as f64, + ); + } + } + } + } +} From af0b3d0f013faa37c79a768d821c7883b72ad81b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 06:19:32 -0800 Subject: [PATCH 07/17] fix(moq-net): at equal rank, a route with no marked link beats a marked one Decided by the coordinator on 2026-11-16 while the maintainer is away, pending their confirmation, as the proof quest found it. The order is now draining, priority, cost, marked, next hop in use, shorter path, neighbour, in route::relay::Router and the prototype alike. Before, a relay holding two routes of equal rank, one learned over an upstream link, kept whichever arrived first, so its own upstream neighbour got a route or none by arrival order. Bake-off failures at 12 and 32 relays, the kept arm and the router: only home_dies on peered tiers moved (15 relays: route messages 220 to 225, claim items 4 to 5; 32 relays: 954 to 924, 10 to 6). No settled result moved. The router joins the bake-off's tables. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/relay.rs | 27 +++++++--- rs/moq-net/src/route/tests/bakeoff.rs | 5 +- rs/moq-net/src/route/tests/multi.rs | 10 ++-- rs/moq-net/src/route/tests/multicast.rs | 67 +++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 12 deletions(-) diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index c628e0fe89c..53be7bbc37c 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -10,7 +10,8 @@ //! session it learned it on or to a neighbour already on its path, and never sends one //! learned over a marked link to another marked link. It ranks routes by drain, then the //! lowest priority on the path, then the summed cost, each link adding at least 1, so -//! extending a route never improves it; among equal ranks it stays on its next hop. +//! extending a route never improves it. Among equal ranks it takes one with no marked link +//! on its path, which it may send to more neighbours, and then stays on its next hop. //! //! A closed session is named once per neighbour in a fixed-size [`Msg::Dead`], which ends //! every route its sender offered through it until the sender offers it again or ends @@ -309,9 +310,9 @@ struct Best { offer: Offer, } -/// How candidate routes order, lower first: rank, then the next hop in use, the shorter -/// path, and the neighbour. -type Order = (bool, Reverse, Metric, bool, usize, N); +/// How candidate routes order, lower first: rank, then one with no marked link on its +/// path, the next hop in use, the shorter path, and the neighbour. +type Order = (bool, Reverse, Metric, bool, bool, usize, N); /// What a router retains, for budgets and leak checks. #[cfg(test)] @@ -1146,8 +1147,17 @@ impl Router { }, }; let (draining, priority, cost) = mine.offer.rank(); - // Among equal ranks a node stays on its next hop. - let key = (draining, priority, cost, current != Some(via), offer.path.len(), via); + // Among equal ranks one with no marked link, which more neighbours may be sent, + // and then the next hop in use. + let key = ( + draining, + priority, + cost, + mine.offer.marked, + current != Some(via), + offer.path.len(), + via, + ); if pick.as_ref().is_none_or(|(_, best)| key < *best) { pick = Some((mine, key)); } @@ -1874,6 +1884,11 @@ impl Router { } /// This node's revision of its own claims, and each of its claims as it stands. + /// Whether it offers a route to itself: it has announced a claim. + pub fn announced(&self) -> bool { + self.announced + } + pub fn own(&self) -> (Rev, Claims) { let items = self.known.get(&self.me).map(|known| known.items.clone()); (self.rev, items.unwrap_or_default()) diff --git a/rs/moq-net/src/route/tests/bakeoff.rs b/rs/moq-net/src/route/tests/bakeoff.rs index 6ae950995a7..09a09511ca1 100644 --- a/rs/moq-net/src/route/tests/bakeoff.rs +++ b/rs/moq-net/src/route/tests/bakeoff.rs @@ -36,6 +36,7 @@ fn arms() -> Vec { Arm::TUNED, Arm::LEAN, Arm::KEPT, + Arm::RELAY, Arm::with(|tune| tune.claimed = true), Arm::with(|tune| { tune.suspect = Suspect::Flagged; @@ -917,7 +918,7 @@ fn benchmark_bakeoff_growth() { cases.extend([1, 8, 64].map(|claims| ("broadcasts", tiers_of(32, 8, claims)))); cases.extend([8, 16, 32].map(|relays| ("links", clique(relays, 8)))); for (grows, mesh) in cases { - for arm in [Arm::TODAY, Arm::TUNED, Arm::LEAN, Arm::KEPT] { + for arm in [Arm::TODAY, Arm::TUNED, Arm::LEAN, Arm::KEPT, Arm::RELAY] { let tree = mesh.tree(); let mut lab = Lab::new(arm, &mesh); lab.coarse = true; @@ -957,7 +958,7 @@ fn benchmark_bakeoff_growth() { fn benchmark_bakeoff_baseline() { println!("case,rule,route_messages,claim_items,settle_ms,fed_ms,loops,stale,dark"); let ms = |rounds: f64| rounds * HOP.as_secs_f64() * 1000.0; - for arm in [Arm::TODAY, Arm::PLAIN, Arm::TUNED, Arm::LEAN, Arm::KEPT] { + for arm in [Arm::TODAY, Arm::PLAIN, Arm::TUNED, Arm::LEAN, Arm::KEPT, Arm::RELAY] { let mut rng = XorShift(0x6d6f_7100 + 24); let sparse: Vec = (0..16).map(|_| Mesh::sparse(24, &mut rng)).collect(); let groups = [ diff --git a/rs/moq-net/src/route/tests/multi.rs b/rs/moq-net/src/route/tests/multi.rs index a85e7f0a84f..eaf1b1103e2 100644 --- a/rs/moq-net/src/route/tests/multi.rs +++ b/rs/moq-net/src/route/tests/multi.rs @@ -234,9 +234,9 @@ impl Default for Known { } } -/// How candidate routes order, lower first: rank, then the next hop in use, the -/// shorter path, and the neighbour. -type Order = (bool, Reverse, Metric, bool, usize, N); +/// How candidate routes order, lower first: rank, then unmarked, the next hop in use, +/// the shorter path, and the neighbour. +type Order = (bool, Reverse, Metric, bool, bool, usize, N); /// Where a request for a broadcast goes next. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -929,11 +929,13 @@ impl Router { suspect, }; let (draining, priority, cost) = mine.rank(); - // Among equal ranks a node stays on its next hop. + // Among equal ranks one with no marked link, which more neighbours may be + // sent, and then the next hop in use. let key = ( draining, priority, cost, + mine.offer.marked, current != Some(via), mine.offer.path.len(), via, diff --git a/rs/moq-net/src/route/tests/multicast.rs b/rs/moq-net/src/route/tests/multicast.rs index 9061d1ece54..e684693a6a7 100644 --- a/rs/moq-net/src/route/tests/multicast.rs +++ b/rs/moq-net/src/route/tests/multicast.rs @@ -471,6 +471,10 @@ impl RelaySim { self.held.insert((from, to)); } + pub fn release(&mut self, from: Node, to: Node) { + self.held.remove(&(from, to)); + } + pub fn ready(&self) -> Vec<((Node, Node), usize)> { (self.queues.iter()) .filter(|(key, queue)| !queue.is_empty() && !self.held.contains(key)) @@ -987,6 +991,69 @@ fn a_session_floor_keeps_an_entry_per_pair_of_relays() { } +/// A route learned over an upstream link is marked, and a marked route is never sent over +/// another upstream link. Relay 3 hears two routes of equal rank to relay 0: through 2, +/// over a link it marked upstream, and through 1, over a peer link. Were the mark below the +/// next hop in use, 3 would keep whichever came first, and with 2's first 4 would be +/// offered nothing. Unmarked first, 3 moves to 1's route whichever came first, and offers +/// it to 4, its own upstream. +#[test] +fn an_equal_route_in_use_never_leaves_a_neighbour_without_one() { + let settled = |first: Node, second: Node| { + let mut sim = RelaySim::new(6, relay::debounce(HOP)); + sim.client(5); + sim.publish(5, 100); + sim.link(5, 0, 1, Tie::Peer); + sim.link(0, 1, 1, Tie::Peer); + sim.link(0, 2, 1, Tie::Peer); + sim.link(3, 2, 1, Tie::Up); + sim.link(3, 1, 1, Tie::Peer); + sim.link(3, 4, 1, Tie::Up); + let origin = sim.id(0); + let via = |sim: &RelaySim, relay: Node| sim.router(relay)?.best(origin).map(|(via, _)| via); + sim.hold(second, 3); + sim.claim(5, "live/0", true); + quiesce(&mut sim); + assert_eq!(via(&sim, 3), Some(first)); + sim.release(second, 3); + quiesce(&mut sim); + (via(&sim, 3), via(&sim, 4).is_some(), sim.showing(4, 100, "live/0")) + }; + assert_eq!(settled(1, 2), (Some(1), true, Some(true)), "the unmarked route first"); + assert_eq!(settled(2, 1), (Some(1), true, Some(true)), "the marked route first"); +} + +/// Under `claimed` a relay offers a route to itself from its first claim on, for good. +/// Relay 2 has no endpoint: it can only announce a claim relay 1 grants it. Relay 1 grants +/// its closest claim, unless that is the one relay 2 granted it. 2's grant, over 2's cheap +/// route to 0, beats 1's own dear route. If 2's grant reaches 1 first, 1 never grants 2 +/// anything and 2 never announces. If 1 hears 0's claim first, it grants it to 2, which +/// announces; 1 then takes 2's cheaper grant and withdraws its own, and 2 keeps a route +/// to itself with no claim behind it. A known limit, as in the prototype. +#[test] +fn whether_a_relay_announces_hangs_on_arrival_order() { + let settled = |late: (Node, Node)| { + let mut sim = RelaySim::new(4, relay::debounce(HOP)); + sim.client(3); + sim.publish(3, 100); + sim.link(3, 0, 1, Tie::Peer); + sim.link(0, 2, 1, Tie::Peer); + sim.link(0, 1, 3, Tie::Peer); + sim.link(1, 2, 1, Tie::Scoped); + sim.hold(late.0, late.1); + sim.claim(3, "live/0", true); + quiesce(&mut sim); + sim.release(late.0, late.1); + quiesce(&mut sim); + let relay = sim.router(2).expect("a relay"); + let claimed = relay.step(sim.id(2), &(p("live/0"), 100)).is_some(); + let routed = sim.router(0).expect("a relay").best(sim.id(2)).is_some(); + (relay.announced(), claimed, routed) + }; + assert_eq!(settled((0, 1)), (false, false, false), "2's grant first"); + assert_eq!(settled((2, 1)), (true, false, true), "0's claim first at 1"); +} + /// The floor's growth with identities that never return: each round a relay under a new /// identity joins a ring as a chord with a publisher of its own, announces, and leaves for /// good. Every relay that heard a route across one of its sessions keeps that pair's floor From c2957b5b986a57ad006f81f2d5ae440465c881cc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 07:01:25 -0800 Subject: [PATCH 08/17] chore(quest): finish quest/m2/multicast/router.md The router's sweeps, nightly searches, and benchmarks join test/route-sweep.sh; two more nightly searches cover every diamond link and a link that reopens. The switch quest carries the new route order, and the finished quest's links point at the router. Co-Authored-By: Claude Opus 5.5 --- quest/m2/cluster-routes/route-fanout.md | 3 +- quest/m2/multicast/router.md | 117 ------------------ quest/m2/multicast/switch.md | 9 +- quest/m4/resume-trace-validation.md | 2 +- rs/moq-net/src/route/relay.rs | 46 ++++--- rs/moq-net/src/route/tests/multicast.rs | 8 +- rs/moq-net/src/route/tests/multicast_cost.rs | 12 +- .../src/route/tests/multicast_search.rs | 18 +++ test/route-sweep.sh | 4 +- 9 files changed, 73 insertions(+), 146 deletions(-) delete mode 100644 quest/m2/multicast/router.md diff --git a/quest/m2/cluster-routes/route-fanout.md b/quest/m2/cluster-routes/route-fanout.md index f0a172115c8..19af7b79c3d 100644 --- a/quest/m2/cluster-routes/route-fanout.md +++ b/quest/m2/cluster-routes/route-fanout.md @@ -13,7 +13,8 @@ multicast](/quest/m2/multicast/README.md)): this waits on the [bake-off](https://github.com/moq-dev/moq/pull/8880). Its first and third findings are in `path::Router`'s indexes and its doubts and dead hops, which the new router replaces; the second, per-broadcast rewrites in `route/node.rs`, the -[router](/quest/m2/multicast/router.md) takes as a requirement. If the new +router (`route::relay::Router`, from +[moq-dev/moq#8901](https://github.com/moq-dev/moq/pull/8901)) takes as a requirement. If the new rule goes ahead, [the switch](/quest/m2/multicast/switch.md) deletes this quest with the fixtures that pin it; if it does not, this starts as written. diff --git a/quest/m2/multicast/router.md b/quest/m2/multicast/router.md deleted file mode 100644 index a34bd67bcc7..00000000000 --- a/quest/m2/multicast/router.md +++ /dev/null @@ -1,117 +0,0 @@ -# [XL] The new route rule runs as a sans-IO router - -## Goal - -`rs/moq-net/src/route/` holds a production router for the -[epic](/quest/m2/multicast/README.md)'s rule beside `path::Router`, sans-IO -and driven only by tests until the [switch](/quest/m2/multicast/switch.md). -The [bake-off](https://github.com/moq-dev/moq/pull/8880)'s fixtures and sweeps run on -it, the exhaustive delivery-order search checks it, and a benchmark swept -over forwarding nodes, publishers, broadcasts, and sessions shows that an -event costs work in what it changed. - -## Plan - -Decided 2026-11-15 (maintainer, /quest-plan): the router first with no wire -change, then the switch, the order -[Routes and announces](/quest/m2/cluster-routes/README.md) Plan item 28 used -for the path rule. - -- Build the rule as the bake-off's pick leaves it: routes to forwarding nodes - only, the dead-hop withdrawal with no moved list, no hold, no doubt, no - timer, claims ordered by revision and certified by floors, the chain - across a scoped grant, and the one mark. -- Keep what is independent of the rule: the interest set edited in place - ([moq-dev/moq#8728](https://github.com/moq-dev/moq/pull/8728)), and - `Reader` and `View`. -- The reader debounces unavailable to offline locally, with mocked time, for - as long as the bake-off measured routes take to settle. It replaces what a - hold protected and is never on the wire. -- A relay stays on its next hop among routes of equal rank, and never sends - a route back over the session it learned it on. Both were quests of their - own until 2026-11-15: a change of hops alone was 3.5% to 11% of route - messages, one link coming up on a 32-relay full mesh cost 592 route - messages against 251, and advertising a route back to its source doubled - route traffic between every pair of relays. -- Per-event work follows what changed, never every answering stream or every - claim behind a route. [Route fan-out](/quest/m2/cluster-routes/route-fanout.md) - pins where the path router fails this. -- Bound what a peer can make the router hold by its own requests, not by a - constant. -- The exhaustive search over every delivery order - ([moq-dev/moq#8729](https://github.com/moq-dev/moq/pull/8729)) moves to - this router, and each property it checks is shown to fail on a mutation - that breaks it. The maintainer chose on 2026-11-14 to keep that search over - the compiled code and retire the TLA+ route model. - -Measured 2026-11-15 by the [bake-off](https://github.com/moq-dev/moq/pull/8880), -whose prototype is `rs/moq-net/src/route/tests/multi.rs`, and decided -2026-11-16 -([maintainer](https://github.com/moq-dev/moq/pull/8880#issuecomment-6911055514), -the [epic](/quest/m2/multicast/README.md)'s items 13 and 14): - -- The rule as written hunts: 6.6 million route messages for a partition of a - 32-relay CDN, against 579 today. Build these five with it, each an arm of - the bake-off: the highest session known closed per pair of nodes, kept for - good and refusing any path at or below it (`floor`, a departure from the - epic's "no dead hop remembered", and `epoch`, below, which bounds it); - advertising once per burst of input read - rather than after every message (`batch`); and a node that sent GOAWAY named - once per neighbour as a closed session is (`drain`); and a node offering a - route to itself only once it has a claim to announce, its endpoints' or one - it announces again across a scoped grant, and from then on until it restarts - (`claimed`). With all five that partition cost 690. -- `claimed` makes the route per announcing relay, not per forwarding node: - held routes fell from 6,576 to 207 on 32 relays with one publisher, under - today's 221. A publisher's first claim on a relay then starts a route at - every node (201 route messages on that CDN), and across scoped grants a - leaving claim's echo can make a gateway announce for the first time - (`a_first_claim_starts_a_route`). Keep the route standing after the last - claim leaves: withdrawing it races the claim's own end, which would show a - reader offline where the publisher ended cleanly. -- A closed session's message ends, on that stream, every route its sender - offered through the session; the sender owes each one its next offer or an - explicit end. A neighbour's copy of the claims stays through that, so a - reroute sends no claim again. -- A claim's withdrawal says whether its publisher ended it or its announcer - lost it, so a publisher's session closing shows readers offline and never a - clean end. A claim carries the lowest link priority to its publisher and - whether the way there is draining, or a node prefers its own echo of a claim - that crossed two grants and never settles: two relays sharing a draining - neighbour and a scoped link traded places for good without the second - (`a_claim_across_a_grant_keeps_its_drain`). A scoped neighbour's - GOAWAY drains the claims it gave the same way - (`a_scoped_neighbour_draining_drains_its_claims`), and a session that - closes takes what was still unsent to it. -- A node that takes a newer copy of a forwarding node's claims passes the floor - on even when nothing in the copy differs, or a neighbour that heard a claim - in between keeps it after its publisher ended it - (`a_newer_floor_travels_with_nothing_changed`). -- Bound the session floor: alone it keeps an entry per restart, since a - restarted relay is a new node. Put the relay and a start that only grows in - the node identity, file the floor per pair of relays, and order a session - by its node's start and then its number (`epoch`): a restart then replaces - the entry, and no message or time changed on any row - (`a_session_floor_keeps_an_entry_per_restart_unless_filed_by_relay`). A - start that repeats or goes back gets the relay's new sessions refused. - The bound is then pairs of relay identities ever heard closed: fixed where - relays are configured, as on a CDN, and growing by a departed device's - pairs on a mesh whose devices mint identities. Measure that growth on the - mesh shapes, and list any eviction it needs as a decision in this PR; one - that waits on a clock is the maintainer's. -- Not picked, with their costs on the PR: holding adverts until neighbours - agree (`quiet`, which can wait for good on a neighbour that learned a stale - route late), ranking the far end of a closed session as draining - (`suspect`, which shortens a silent death but doubles the messages of a - flap and has no word that always lifts it), and refusing a route a - neighbour's own offer contradicts (`agree`, a small gain). -- Promote `bakeoff.rs`'s fixtures and `sweep_the_prototype_carries_every_failure_at_32_relays`. - Size the reader's debounce from the longest path a route may take times a - hop's delay, not from a sample: the PR's `fed_ms` column is what was seen, - 775 ms at most for a failure that left a path on 32 relays (the - publisher's relay dying on the full mesh), a silent death aside, and a - longer path settles later. -- Selected routes looped for a moment in partitions and deaths (up to 3 rounds - at 32 relays, where today's reached 4); requests must fail closed first. - -Public API: none. Wire: none. diff --git a/quest/m2/multicast/switch.md b/quest/m2/multicast/switch.md index 8984b7e6f4b..b3a55477fe7 100644 --- a/quest/m2/multicast/switch.md +++ b/quest/m2/multicast/switch.md @@ -3,7 +3,8 @@ ## Goal Every lite-07 session in Rust drives the -[new router](/quest/m2/multicast/router.md) on one announce stream per side, +new router (`route::relay::Router`, from +[moq-dev/moq#8901](https://github.com/moq-dev/moq/pull/8901)) on one announce stream per side, the draft specifies it, and the old rule is gone: ROUTE_HOLD, the doubt, DEAD's moved list, the Route Key Parameter with its proof and symbolic model, qualified Node IDs, routes per prefix with their Route Suffix and Route @@ -98,6 +99,11 @@ for the wire: - A claim's end carries one bit, ended or lost, and a claim carries the lowest link priority to its publisher and a draining bit beside its cost and chain. - A claims message with no change still carries its floor. +- Among routes of equal rank a node takes one with no upstream-marked link on + its path before staying on its next hop, or a neighbour behind its own + upstream link is reachable by arrival order (decided by the coordinator on + 2026-11-16, pending the maintainer, in + [moq-dev/moq#8901](https://github.com/moq-dev/moq/pull/8901)). - A node offers a route to itself only once it announces a claim, so a relay that has never announced a claim and only forwards or only serves viewers has no route on the wire. @@ -212,5 +218,4 @@ extension on moq-transport is no longer sent or read. ## Required -- [The router](/quest/m2/multicast/router.md) - the rule these sessions drive - [The route stream](/quest/m2/cluster-routes/route-stream.md) - one announce stream per side, which this builds on diff --git a/quest/m4/resume-trace-validation.md b/quest/m4/resume-trace-validation.md index e9d50cfe85d..f7442f78f58 100644 --- a/quest/m4/resume-trace-validation.md +++ b/quest/m4/resume-trace-validation.md @@ -17,7 +17,7 @@ can keep passing on code that has moved on. Milestone m4. [reconnect](/quest/m4/reconnect-trace-validation.md) and [subscribe](/quest/m4/subscribe-trace-validation.md) reuse: route's trace validation, which was to build them, gave way to the - exhaustive route search, which [the router](/quest/m2/multicast/router.md) now carries (decided + exhaustive route search, which `route::tests::multicast_search` now runs on the new router (decided 2026-11-14, maintainer, on [moq-dev/moq#8764](https://github.com/moq-dev/moq/pull/8764)). The sweep records each step as the model action it corresponds to (the `verify-resume` skill's Coverage table is the mapping) with the variables diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index 53be7bbc37c..1f5f94edc21 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -37,7 +37,9 @@ //! asks, a broader claim presented at the prefix it asks for. //! //! The rule is the bake-off's pick (moq-dev/moq#8880): `tests/multi.rs` with `floor`, -//! `epoch`, `batch`, `drain`, and `claimed`, which `tests/relay.rs` compares it against. +//! `epoch`, `batch`, `drain`, and `claimed`, which `tests/multicast.rs` compares it +//! against. `tests/multicast_search.rs` searches every delivery order on small meshes, +//! and `tests/multicast_cost.rs` sweeps what one event costs. use std::{ cmp::Reverse, @@ -1871,24 +1873,12 @@ impl Router { self.best.keys().copied().collect() } - /// What each neighbour offers for `origin`, and whether a closed session ended it. - pub fn heard(&self, origin: NodeId) -> Vec<(N, &Offer, bool)> { - (self.heard.get(&origin).into_iter().flatten()) - .map(|(peer, heard)| (*peer, &heard.offer, heard.ended)) - .collect() - } - - /// The pairs of relays with a session known closed, and where each floor stands. - pub fn floors(&self) -> &BTreeMap<(Id, Id), (u64, u64)> { - &self.closed - } - - /// This node's revision of its own claims, and each of its claims as it stands. /// Whether it offers a route to itself: it has announced a claim. pub fn announced(&self) -> bool { self.announced } + /// This node's revision of its own claims, and each of its claims as it stands. pub fn own(&self) -> (Rev, Claims) { let items = self.known.get(&self.me).map(|known| known.items.clone()); (self.rev, items.unwrap_or_default()) @@ -2152,6 +2142,34 @@ mod tests { assert_eq!(router.edit(2, edit), Err(Error::Partial)); } + /// A neighbour's answer to a prefix this node asks for is synced once it says so, and + /// a prefix not asked for has no answer to wait on. + #[test] + fn a_prefix_is_synced_once_its_answer_says_so() { + let mut router = relay(1); + router + .connect(2, port(Role::Scoped(vec![p("a"), p("a/b"), p("c")]))) + .unwrap(); + let asked: Vec> = router.drain().into_iter().map(|(_, msg)| msg).collect(); + let edit = Edit { + remove: Vec::new(), + add: vec![p("a"), p("c")], + }; + assert_eq!(asked, vec![Msg::Ask(edit)], "the wider of two nested regions"); + assert_eq!(router.is_synced(2, &p("a")), Some(false)); + assert_eq!( + router.recv(2, Msg::Synced(p("a"))), + Err(Error::Unasked), + "before the marker" + ); + router.recv(2, Msg::Edited).unwrap(); + router.recv(2, Msg::Synced(p("a"))).unwrap(); + assert_eq!(router.is_synced(2, &p("a")), Some(true)); + assert_eq!(router.is_synced(2, &p("c")), Some(false)); + assert_eq!(router.is_synced(2, &p("a/b")), None); + assert_eq!(router.recv(2, Msg::Synced(p("d"))), Err(Error::Unasked)); + } + /// Nothing a neighbour sends counts until its answer begins at the marker. #[test] fn nothing_counts_before_the_marker() { diff --git a/rs/moq-net/src/route/tests/multicast.rs b/rs/moq-net/src/route/tests/multicast.rs index e684693a6a7..786467cd201 100644 --- a/rs/moq-net/src/route/tests/multicast.rs +++ b/rs/moq-net/src/route/tests/multicast.rs @@ -782,7 +782,12 @@ fn flaps(arm: Arm, relays: usize) -> Vec { for (failure, row) in fail(arm, &shape.mesh, relays > 16) { let returns = matches!(failure, Failure::Heal | Failure::Restart); if !returns && row.cost.sent.starts > 0 { - out.push(format!("{} on {}: {}", failure.as_str(), shape.name, row.cost.sent.starts)); + out.push(format!( + "{} on {}: {}", + failure.as_str(), + shape.name, + row.cost.sent.starts + )); } } } @@ -990,7 +995,6 @@ fn a_session_floor_keeps_an_entry_per_pair_of_relays() { assert_eq!(kept[0], kept[7], "{kept:?}"); } - /// A route learned over an upstream link is marked, and a marked route is never sent over /// another upstream link. Relay 3 hears two routes of equal rank to relay 0: through 2, /// over a link it marked upstream, and through 1, over a peer link. Were the mark below the diff --git a/rs/moq-net/src/route/tests/multicast_cost.rs b/rs/moq-net/src/route/tests/multicast_cost.rs index 4ded9a1d765..c89f0a7ce53 100644 --- a/rs/moq-net/src/route/tests/multicast_cost.rs +++ b/rs/moq-net/src/route/tests/multicast_cost.rs @@ -84,12 +84,8 @@ impl Bed { /// Built with every router rebuilding its indexes after each input, if `checked`. fn new(shape: Shape, checked: bool) -> Self { - let nodes = 1 - + shape.upstream - + shape.origins - + shape.downstream - + shape.origins * shape.publishers - + shape.watchers; + let nodes = + 1 + shape.upstream + shape.origins + shape.downstream + shape.origins * shape.publishers + shape.watchers; let mut bed = Self { sim: RelaySim::new(nodes, relay::debounce(HOP)), shape, @@ -119,7 +115,8 @@ impl Bed { let publisher = bed.publisher(origin, index); bed.sim.link(publisher, bed.origin(origin), 1, Tie::Peer); for broadcast in 0..shape.broadcasts { - bed.sim.claim(publisher, &format!("o{origin}/p{index}/b{broadcast}"), true); + bed.sim + .claim(publisher, &format!("o{origin}/p{index}/b{broadcast}"), true); } } } @@ -312,4 +309,3 @@ fn benchmark_relay_event_work() { } } } - diff --git a/rs/moq-net/src/route/tests/multicast_search.rs b/rs/moq-net/src/route/tests/multicast_search.rs index 31d3536e92d..1b6e283c159 100644 --- a/rs/moq-net/src/route/tests/multicast_search.rs +++ b/rs/moq-net/src/route/tests/multicast_search.rs @@ -96,6 +96,7 @@ impl fmt::Display for Broken { } /// What the search covered. +#[expect(dead_code, reason = "search statistics are reported through derived Debug")] #[derive(Clone, Copy, Debug)] struct Report { states: usize, @@ -620,3 +621,20 @@ fn every_property_catches_its_mutant() { assert!(matches!(failure.broken, Broken::Rule(broke, _) if broke == rule)); } } + +/// The diamond with any of its links, the publisher's included, closing or opening. +#[test] +#[ignore = "Nightly"] +fn every_order_past_any_diamond_link_keeps_the_multicast_rules() { + let mut mesh = diamond(); + mesh.links = vec![(0, 1), (1, 2), (1, 3), (2, 4), (3, 4), (5, 0)]; + run(mesh, &ONE); +} + +/// The triangle with a link that may close and open again, so a later session of one +/// pair of relays races what is left of the earlier one. +#[test] +#[ignore = "Nightly"] +fn every_order_through_a_reopened_link_keeps_the_multicast_rules() { + run(triangle(), &Bounds { flaps: 2, ..ONE }); +} diff --git a/test/route-sweep.sh b/test/route-sweep.sh index 3d82af799b4..991217f2d1c 100755 --- a/test/route-sweep.sh +++ b/test/route-sweep.sh @@ -16,5 +16,7 @@ MOQ_ROUTE_SCENARIOS=${MOQ_ROUTE_SCENARIOS:-20000} just rs test -P sweep -p moq-n just rs test --profile=sweep -p moq-net --release --run-ignored only -E 'test(/^route::tests::search::/)' # Cache-aware routing's long sweep, from the same seed, settles every churned mesh of copies: 3000 scenarios each with and without held links, about 3 minutes. MOQ_ROUTE_CASES=3000 just rs test -P sweep -p moq-net --release --run-ignored only -E 'test(=route::tests::cache::sweep_cache_routing_long)' -# The multicast bake-off: its prototype carries every failure on every shape at 32 relays, and its tables against the path rule at 12. +# The multicast bake-off: its prototype carries every failure on every shape at 32 relays, and its tables against the path rule and the router at 12. MOQ_BAKEOFF_RELAYS=12 just rs test --profile=sweep -p moq-net --release --run-ignored only -E 'test(/^route::tests::bakeoff::(sweep|benchmark)_/)' +# The multicast router: every failure on every shape at 32 relays, every delivery order the gate's searches leave out, what one event costs it, and how its floors grow. +just rs test --profile=sweep -p moq-net --release --run-ignored only -E 'test(/^route::tests::multicast(_search|_cost)?::/)' From be7dc806fa61c5a740dedd04a45039042fa6d6af Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 08:22:57 -0800 Subject: [PATCH 09/17] fix(moq-net): route::relay::Router answers requests under a claim, takes the newest epoch, and dirties only what a rank reaches - A request for a path under a claim follows the most specific claim covering it, at the first relay (`pick`) and at the claimant (`step`). - Among claims of one Instance at one path, the newest epoch wins whatever it costs, among endpoints and among forwarding nodes, as `model::origin` ranks them. `Item` gains `epoch()`. - A route or claim whose cost a link would carry past the metric is never used, instead of overflowing. - A rank-only move dirties only the broadcasts the rank reaches: shared with another claimant, or shown to an endpoint or scoped neighbour. Indexed per forwarding node. - An overtaking copy's diff looks up ended claims in a set instead of scanning the message per claim. - The reader's recomputed list is discarded, since nothing reads it. - `multicast_cost.rs` sweeps a same-hop detour from a base no watcher sees, and an overtaking copy. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/mod.rs | 9 + rs/moq-net/src/route/node.rs | 4 + rs/moq-net/src/route/relay.rs | 340 +++++++++++++++---- rs/moq-net/src/route/tests/mod.rs | 4 + rs/moq-net/src/route/tests/multicast_cost.rs | 127 ++++++- 5 files changed, 412 insertions(+), 72 deletions(-) diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index 28b16768c4b..73b3116d339 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -47,6 +47,11 @@ pub(crate) trait Item: Clone + Eq + fmt::Debug { /// much broader it is, so the claim keeps that distance: a claim exactly at a prefix /// outranks every broader one there. `lowered(0)` is the claim itself. fn lowered(&self, by: u64) -> Self; + + /// Which run of its Instance the publisher is on: a claim at a higher epoch supersedes + /// one of the same Instance at the same path whatever either's route costs, as + /// `model::origin` ranks them. 0 for attributes that carry none. + fn epoch(&self) -> u64; } /// Opaque attributes that cover nothing broader than their own path. @@ -57,6 +62,10 @@ macro_rules! opaque { fn lowered(&self, _: u64) -> Self { self.clone() } + + fn epoch(&self) -> u64 { + 0 + } } )*}; } diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 6e8818da7e7..ebcd5fe17c5 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -75,6 +75,10 @@ impl super::Item for Claim { ..self.clone() } } + + fn epoch(&self) -> u64 { + self.epoch + } } /// A shown claim as the origin serves it through one live route. diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index 1f5f94edc21..85f21a6a46e 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -34,7 +34,10 @@ //! Each neighbour asks for a set of prefixes edited in place, as under the path rule: a //! routing neighbour forwards for the whole domain, so it asks for the whole tree, and //! an endpoint or scoped neighbour is told the closest claim per broadcast within what it -//! asks, a broader claim presented at the prefix it asks for. +//! asks, a broader claim presented at the prefix it asks for. A request follows the most +//! specific claim covering its path that a route reaches. Among the claims of one +//! Instance at one path, the newest epoch wins whatever it costs, then the best rank, and +//! a route or claim whose cost a link would carry past the metric is never used. //! //! The rule is the bake-off's pick (moq-dev/moq#8880): `tests/multi.rs` with `floor`, //! `epoch`, `batch`, `drain`, and `claimed`, which `tests/multicast.rs` compares it @@ -390,8 +393,10 @@ pub(crate) struct Router { sources: BTreeMap, N>, /// The forwarding nodes claiming each broadcast, by path. claimants: BTreeMap>>, - /// Per forwarding node, how many of its broadcasts another node claims too. - shared: BTreeMap, + /// Per forwarding node, the broadcasts its route's rank reaches past whether a reader + /// can reach them: one another node claims too, whose closest claim the rank decides, + /// and one an endpoint or scoped neighbour is shown, which is told the claim's cost. + ranked: BTreeMap>>, /// The endpoints and scoped neighbours asking for each prefix. askers: BTreeMap>, /// Broadcasts whose last removal was the publisher's own withdrawal. @@ -477,7 +482,7 @@ impl Router { givers: BTreeMap::new(), sources: BTreeMap::new(), claimants: BTreeMap::new(), - shared: BTreeMap::new(), + ranked: BTreeMap::new(), askers: BTreeMap::new(), ended: BTreeSet::new(), dirty: BTreeSet::new(), @@ -546,6 +551,8 @@ impl Router { for (instance, claims) in due { self.reader.apply(instance, |_| true, claims, |_| false); } + // As in `refresh`: nothing reads which claims the reader recomputed. + self.reader.changed(); } // --- Sessions --- @@ -732,7 +739,11 @@ impl Router { } let added: Vec = new.difference(&old).cloned().collect(); for prefix in &added { - self.askers.entry(prefix.clone()).or_default().insert(peer); + let askers = self.askers.entry(prefix.clone()).or_default(); + askers.insert(peer); + if askers.len() == 1 { + self.rewatch(prefix); + } // Every broadcast within it, at its own path. let inside: Vec<(Path, I)> = (self.claimants.get_key_value(prefix).into_iter()) .chain(below(&self.claimants, prefix)) @@ -1136,6 +1147,10 @@ impl Router { { continue; } + // A route no link could extend is past every other, so it is never used. + let Some(cost) = offer.cost.checked_add(peer.port.cost) else { + continue; + }; let draining = offer.draining || peer.draining || offer.path.iter().any(|hop| self.draining.contains(&hop.node)); let mine = Best { @@ -1144,7 +1159,7 @@ impl Router { path: offer.path.clone(), draining, priority: offer.priority.min(peer.port.priority), - cost: offer.cost + peer.port.cost, + cost, marked: offer.marked || peer.port.upstream, }, }; @@ -1175,15 +1190,14 @@ impl Router { Some(best) => self.best.insert(origin, best), None => self.best.remove(&origin), }; - // Whether its claims are reachable follows the route, and so does their cost, - // which an endpoint or scoped neighbour is told and which may move the closest - // claim of a broadcast another node claims too. - let exported = !self.askers.is_empty() || self.shared.get(&origin).is_some_and(|count| *count > 0); - if (flipped || (moved && exported)) - && let Some(known) = self.known.get(&origin) - { + // Whether its claims are reachable follows the route, and so does the rank of each + // that the rank reaches. + if flipped && let Some(known) = self.known.get(&origin) { self.count(known.items.len()); self.dirty.extend(known.items.keys().cloned()); + } else if moved && let Some(ranked) = self.ranked.get(&origin) { + self.count(ranked.len()); + self.dirty.extend(ranked.iter().cloned()); } self.stale.insert(origin); } @@ -1357,13 +1371,14 @@ impl Router { .collect(); // What the newer copy lacks is lost, unless this message says its // publisher ended it. - let ended = |broadcast: &Broadcast| { - (changes.iter()).any(|(named, change)| named == broadcast && *change == Change::Ended) - }; + let ended: BTreeSet<&Broadcast> = (changes.iter()) + .filter(|(_, change)| *change == Change::Ended) + .map(|(broadcast, _)| broadcast) + .collect(); diff.extend( (known.items.keys()) .filter(|broadcast| !heard.items.contains_key(*broadcast)) - .map(|broadcast| match ended(broadcast) { + .map(|broadcast| match ended.contains(broadcast) { true => (broadcast.clone(), Change::Ended), false => (broadcast.clone(), Change::Lost), }), @@ -1419,52 +1434,90 @@ impl Router { } } - fn claim(&mut self, (path, instance): &Broadcast, origin: NodeId) { + fn claim(&mut self, broadcast: &Broadcast, origin: NodeId) { + let (path, instance) = broadcast; let origins = (self.claimants.entry(path.clone()).or_default()) .entry(instance.clone()) .or_default(); if !origins.insert(origin) { return; } - if origins.len() == 2 { - let other = *origins.iter().find(|other| **other != origin).expect("another"); - *self.shared.entry(other).or_default() += 1; + let shared = origins.len() >= 2; + // The claimant it now shares the broadcast with ranks it too. + let other = (origins.len() == 2).then(|| *origins.iter().find(|other| **other != origin).expect("another")); + if shared || self.watched(path) { + self.rank(origin, broadcast, true); } - if origins.len() >= 2 { - *self.shared.entry(origin).or_default() += 1; + if let Some(other) = other { + self.rank(other, broadcast, true); } } - fn unclaim(&mut self, (path, instance): &Broadcast, origin: NodeId) { + fn unclaim(&mut self, broadcast: &Broadcast, origin: NodeId) { + let (path, instance) = broadcast; let Some(claims) = self.claimants.get_mut(path) else { return; }; let Some(origins) = claims.get_mut(instance) else { return; }; - let shared = origins.len() >= 2; if !origins.remove(&origin) { return; } - let mut unshare = |node: NodeId| { - let count = self.shared.get_mut(&node).expect("a shared count"); - *count -= 1; - if *count == 0 { - self.shared.remove(&node); - } - }; - if shared { - unshare(origin); - } - if origins.len() == 1 { - unshare(*origins.first().expect("one")); - } + let left = (origins.len() == 1).then(|| *origins.first().expect("one")); if origins.is_empty() { claims.remove(instance); if claims.is_empty() { self.claimants.remove(path); } } + self.rank(origin, broadcast, false); + // The one claimant left ranks it only if it is shown. + if let Some(left) = left { + let watched = self.watched(path); + self.rank(left, broadcast, watched); + } + } + + /// File `broadcast` among the ones `origin`'s rank reaches, or not. + fn rank(&mut self, origin: NodeId, broadcast: &Broadcast, ranked: bool) { + self.visit(); + match ranked { + true => { + self.ranked.entry(origin).or_default().insert(broadcast.clone()); + } + false => { + if let Some(set) = self.ranked.get_mut(&origin) { + set.remove(broadcast); + if set.is_empty() { + self.ranked.remove(&origin); + } + } + } + } + } + + /// Whether an endpoint or scoped neighbour is shown the claims at `path`: one asks at + /// or above it, or below it, where the most specific claim above is presented. + fn watched(&self, path: &Path) -> bool { + ancestors(path).any(|above| self.askers.contains_key(&above)) || below(&self.askers, path).next().is_some() + } + + /// `prefix` gained its first asker or lost its last: file again every broadcast whose + /// claims it is shown, at, below, or above it. + fn rewatch(&mut self, prefix: &Path) { + let near: Vec> = (ancestors(prefix).filter_map(|above| self.claimants.get_key_value(&above))) + .chain(below(&self.claimants, prefix)) + .flat_map(|(path, claims)| claims.keys().map(move |instance| (path.clone(), instance.clone()))) + .collect(); + for broadcast in near { + let (path, instance) = &broadcast; + let origins: Vec = self.claimants[path][instance].iter().copied().collect(); + let ranked = origins.len() >= 2 || self.watched(path); + for origin in origins { + self.rank(origin, &broadcast, ranked); + } + } } /// Claims from an endpoint or across a scoped grant become this node's own. @@ -1483,8 +1536,13 @@ impl Router { for (broadcast, change) in changes { self.visit(); let change = match change { - // A claim that already crossed this relay came round a ring of grants. - Change::Live(claim) if claim.chain.contains(&self.me.relay) => Change::Lost, + // A claim that already crossed this relay came round a ring of grants, and + // one the link would make dearer than the metric holds is never used. + Change::Live(claim) + if claim.chain.contains(&self.me.relay) || claim.cost.checked_add(cost).is_none() => + { + Change::Lost + } Change::Live(mut claim) => { claim.cost += cost; claim.priority = claim.priority.min(priority); @@ -1533,8 +1591,10 @@ impl Router { claim.draining |= self.peers.get(&peer).is_some_and(|peer| peer.draining); (claim, peer) }) + // The newest epoch whatever it costs, as `closest` takes it. .min_by_key(|(claim, peer)| { ( + Reverse(claim.item.epoch()), claim.draining, Reverse(claim.priority), claim.cost, @@ -1576,19 +1636,29 @@ impl Router { // --- Requests --- - /// The forwarding node a request for `broadcast` names from here: the closest claim. - pub fn pick(&self, broadcast: &Broadcast) -> Option { - self.closest(broadcast).map(|(origin, _)| origin) + /// The forwarding node a request for `broadcast` names from here: the closest of the + /// most specific claims covering it that one can reach, as a neighbour asking for its + /// path is shown. + pub fn pick(&self, (path, instance): &Broadcast) -> Option { + (ancestors(path).rev()) + .find_map(|above| self.closest(&(above, instance.clone()))) + .map(|(origin, _)| origin) } /// Where a request for `broadcast` naming `origin` goes from here. `None` fails it. - pub fn step(&self, origin: NodeId, broadcast: &Broadcast) -> Option> { + pub fn step(&self, origin: NodeId, (path, instance): &Broadcast) -> Option> { match origin == self.me { - true => self.sources.get(broadcast).copied().map(Next::Local), + // The neighbour behind the most specific of this node's claims covering it. + true => (ancestors(path).rev()) + .find_map(|above| self.sources.get(&(above, instance.clone())).copied()) + .map(Next::Local), false => self.best.get(&origin).map(|best| Next::Via(best.via)), } } + /// The claim of `broadcast` a request takes from here, with its claimant: the newest + /// epoch, then the best rank of the route and the claim together, a claim no link would + /// make dearer than the metric holds. fn closest(&self, (path, instance): &Broadcast) -> Option<(NodeId, &Claim)> { let origins = self.claimants.get(path)?.get(instance)?; let broadcast = (path.clone(), instance.clone()); @@ -1603,14 +1673,14 @@ impl Router { ( draining || claim.draining, Reverse(priority.min(claim.priority)), - cost + claim.cost, + cost.checked_add(claim.cost)?, ) } }; - Some(((rank, claim.chain.len(), origin), claim)) + Some(((Reverse(claim.item.epoch()), rank, claim.chain.len(), origin), claim)) }) .min_by(|(a, _), (b, _)| a.cmp(b)) - .map(|((_, _, origin), claim)| (origin, claim)) + .map(|((_, _, _, origin), claim)| (origin, claim)) } // --- Readers, endpoints, and scoped neighbours --- @@ -1673,6 +1743,9 @@ impl Router { let ended = |claim: &Path| lost_ends || ended.contains(&(claim.clone(), instance.clone())); reader.apply(instance.clone(), |_| true, claims, ended); } + // A request finds its way by `pick` and `step`, never by which claims the reader + // recomputed, so nothing keeps that list. + reader.changed(); for broadcast in &dirty { self.exports(broadcast); } @@ -1772,7 +1845,7 @@ impl Router { let route = &self.best[&origin].offer; Some(Claim { item: claim.item.clone(), - cost: claim.cost + route.cost, + cost: claim.cost.checked_add(route.cost)?, priority: claim.priority.min(route.priority), draining: claim.draining || route.draining, chain: claim.chain.clone(), @@ -1784,6 +1857,7 @@ impl Router { peers.remove(&peer); if peers.is_empty() { self.askers.remove(prefix); + self.rewatch(prefix); } } } @@ -1925,7 +1999,7 @@ impl Router { givers: _, sources, claimants: _, - shared: _, + ranked: _, askers: _, ended, dirty, @@ -2023,16 +2097,21 @@ impl Router { claims.entry(instance.clone()).or_default().insert(*origin); } } - let mut shared: BTreeMap = BTreeMap::new(); - for origins in claimants.values().flat_map(BTreeMap::values) { - if origins.len() >= 2 { - for origin in origins { - *shared.entry(*origin).or_default() += 1; + let mut ranked: BTreeMap>> = BTreeMap::new(); + for (path, claims) in &claimants { + for (instance, origins) in claims { + if origins.len() >= 2 || self.watched(path) { + for origin in origins { + ranked + .entry(*origin) + .or_default() + .insert((path.clone(), instance.clone())); + } } } } assert_eq!(self.claimants, claimants, "claimants"); - assert_eq!(self.shared, shared, "shared claims"); + assert_eq!(self.ranked, ranked, "ranked claims"); let deadlines: BTreeSet<(Instant, Broadcast)> = (self.waiting.iter()) .map(|(broadcast, at)| (*at, broadcast.clone())) .collect(); @@ -2081,7 +2160,7 @@ mod tests { } /// Connect `peer` and exchange asks, as the session's first messages do. - fn open(router: &mut Relay, peer: usize, role: Role) { + fn open(router: &mut Router, peer: usize, role: Role) { router.connect(peer, port(role.clone())).unwrap(); router.recv(peer, Msg::Edited).unwrap(); let ask = match role { @@ -2111,6 +2190,153 @@ mod tests { }) } + /// A route from routing neighbour `via`, as `node(via)`, to forwarding node `origin`. + fn route(origin: Id, via: Id, cost: Metric) -> Msg { + let path = vec![ + Hop { + node: node(origin), + session: 0, + }, + Hop { + node: node(via), + session: 1, + }, + ]; + Msg::Route { + origin: node(origin), + offer: Some(Offer { + path: path.into(), + draining: false, + priority: i8::MAX, + cost, + marked: false, + }), + } + } + + /// Attributes with an epoch, as a restarted publisher's claims carry. + #[derive(Clone, Debug, PartialEq, Eq)] + struct Epoch(u64); + + impl Item for Epoch { + fn lowered(&self, _: u64) -> Self { + self.clone() + } + + fn epoch(&self) -> u64 { + self.0 + } + } + + fn claim(epoch: u64, cost: Metric) -> Change { + Change::Live(Claim { + item: Epoch(epoch), + cost, + priority: i8::MAX, + draining: false, + chain: Vec::new(), + }) + } + + /// A request for a path under a claim follows the most specific claim covering it, as + /// a neighbour asking for that path is shown. + #[test] + fn a_request_follows_the_claim_covering_it() { + let mut router = relay(1); + open(&mut router, 2, Role::Endpoint); + open(&mut router, 3, Role::Endpoint); + let announce = |path: &str| Msg::Announce { + changes: vec![((p(path), 7), live(0))], + }; + router.recv(2, announce("live")).unwrap(); + router.recv(3, announce("live/a")).unwrap(); + let me = router.me(); + for (path, endpoint) in [("live", 2), ("live/b/x", 2), ("live/a", 3), ("live/a/x", 3)] { + let broadcast = (p(path), 7); + assert_eq!(router.pick(&broadcast), Some(me), "{path}"); + assert_eq!(router.step(me, &broadcast), Some(Next::Local(endpoint)), "{path}"); + } + for broadcast in [(p("other"), 7), (p("live/a"), 8)] { + assert_eq!(router.pick(&broadcast), None, "{broadcast:?}"); + assert_eq!(router.step(me, &broadcast), None, "{broadcast:?}"); + } + } + + /// A claim at a newer epoch wins over an older one of the same Instance however much + /// cheaper the older is, among a node's endpoints and among forwarding nodes alike. + #[test] + fn a_newer_epoch_wins_whatever_it_costs() { + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Endpoint); + open(&mut router, 3, Role::Endpoint); + let broadcast = (p("live"), 7); + let announce = |change| Msg::Announce { + changes: vec![(broadcast.clone(), change)], + }; + router.recv(2, announce(claim(1, 0))).unwrap(); + router.recv(3, announce(claim(2, 9))).unwrap(); + let me = router.me(); + assert_eq!(router.step(me, &broadcast), Some(Next::Local(3))); + assert_eq!(router.own().1[&broadcast].item, Epoch(2)); + assert_eq!(router.shown()[&(7, p("live"))].item, Epoch(2)); + open(&mut router, 4, Role::Routing(node(4))); + router.recv(4, route(5, 4, 1)).unwrap(); + let claims = |change, floor| Msg::Claims { + origin: node(5), + changes: vec![(broadcast.clone(), change)], + floor, + }; + router.recv(4, claims(claim(1, 0), 1)).unwrap(); + assert_eq!(router.pick(&broadcast), Some(me), "an older claim, however close"); + router.recv(4, claims(claim(3, 100), 2)).unwrap(); + assert_eq!(router.pick(&broadcast), Some(node(5)), "a newer claim, however far"); + assert_eq!(router.shown()[&(7, p("live"))].item, Epoch(3)); + } + + /// A route or claim whose cost a link would carry past the metric is never used, and + /// never wraps round to a cheap one. + #[test] + fn a_cost_past_the_metric_is_never_used() { + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Routing(node(2))); + router.recv(2, route(5, 2, Metric::MAX)).unwrap(); + assert_eq!(router.best(node(5)), None); + router.recv(2, route(5, 2, 1)).unwrap(); + assert!(router.best(node(5)).is_some()); + let claims = Msg::Claims { + origin: node(5), + changes: vec![((p("far"), 7), claim(0, Metric::MAX - 1))], + floor: 1, + }; + router.recv(2, claims).unwrap(); + assert!(!router.shown()[&(7, p("far"))].available, "shown, never reachable"); + assert_eq!(router.pick(&(p("far"), 7)), None); + open(&mut router, 3, Role::Endpoint); + let announce = Msg::Announce { + changes: vec![((p("near"), 7), claim(0, Metric::MAX))], + }; + router.recv(3, announce).unwrap(); + assert!(router.own().1.is_empty(), "an endpoint's claim past what its link adds"); + } + + /// The reader keeps nothing of claims that came and went, ended or lost. + #[test] + fn a_claim_that_went_leaves_nothing_in_the_reader() { + let mut router = relay(1); + open(&mut router, 2, Role::Endpoint); + let announce = |path: &str, change| Msg::Announce { + changes: vec![((p(path), 7), change)], + }; + router.recv(2, announce("ended", live(0))).unwrap(); + router.recv(2, announce("ended", Change::Ended)).unwrap(); + router.recv(2, announce("lost", live(0))).unwrap(); + router.recv(2, announce("lost", Change::Lost)).unwrap(); + router.timeout(router.deadline().expect("a debounce")); + router.events(); + assert!(router.shown().is_empty()); + assert!(router.reader.changed().is_empty(), "nothing reads which claims changed"); + } + /// A claim outside what this node asked a scoped neighbour for is refused whole. #[test] fn a_claim_outside_the_grant_is_refused() { diff --git a/rs/moq-net/src/route/tests/mod.rs b/rs/moq-net/src/route/tests/mod.rs index f0951f5be78..adf85919409 100644 --- a/rs/moq-net/src/route/tests/mod.rs +++ b/rs/moq-net/src/route/tests/mod.rs @@ -119,6 +119,10 @@ impl crate::route::Item for Item { fn lowered(&self, _: u64) -> Self { *self } + + fn epoch(&self) -> u64 { + self.epoch.map_or(0, u64::from) + } } /// `peer` replaces everything it asks `router` for with `prefixes`, in one edit. diff --git a/rs/moq-net/src/route/tests/multicast_cost.rs b/rs/moq-net/src/route/tests/multicast_cost.rs index c89f0a7ce53..393124f64b1 100644 --- a/rs/moq-net/src/route/tests/multicast_cost.rs +++ b/rs/moq-net/src/route/tests/multicast_cost.rs @@ -5,7 +5,9 @@ //! asking for the whole tree or for a prefix no claim falls under. A sweep holds the other //! axes fixed and counts the measured relay's work and messages, so an event that visits //! what it did not change fails here rather than only timing slower: `cost.rs` for the -//! rule the switch puts lite-07 sessions on. +//! rule the switch puts lite-07 sessions on. The events: a claim announced, a route the +//! relay does not use breaking, the one it uses going round dearer, a downstream session +//! leaving, and a copy of a forwarding node's claims overtaking the relay's. use crate::route::relay; @@ -44,6 +46,9 @@ const BASE: Shape = Shape { watched: true, }; +/// `BASE` with no watcher asking for any claim, so a route's cost reaches nobody's view. +const QUIET: Shape = Shape { watched: false, ..BASE }; + /// What one event cost the measured relay. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] struct Cost { @@ -102,7 +107,10 @@ impl Bed { } } for up in 0..shape.upstream { - bed.sim.link(ME, bed.up(up), 1, Tie::Peer); + // The first is so much cheaper that the measured relay keeps to it even when + // its route goes round. + let cost = if up == 0 { 1 } else { 8 }; + bed.sim.link(ME, bed.up(up), cost, Tie::Peer); for origin in 0..shape.origins { bed.sim.link(bed.up(up), bed.origin(origin), 1, Tie::Peer); } @@ -149,10 +157,21 @@ impl Bed { self.measure(|bed| bed.sim.claim(publisher, &claim, true)) } - /// The first upstream relay's session to the first forwarding node closes, then - /// reopens unmeasured. + /// The last upstream relay's session to the first forwarding node closes, then reopens + /// unmeasured: a route the measured relay does not use breaks. fn cut(&mut self) -> Cost { - let (up, origin) = (self.up(0), self.origin(0)); + self.sever(self.shape.upstream - 1) + } + + /// The first upstream relay's session to the first forwarding node closes, then + /// reopens unmeasured: the route the measured relay uses goes round through another + /// forwarding node, dearer, on the same next hop. + fn detour(&mut self) -> Cost { + self.sever(0) + } + + fn sever(&mut self, up: usize) -> Cost { + let (up, origin) = (self.up(up), self.origin(0)); let cost = self.measure(|bed| bed.sim.close(up, origin)); self.sim.open(up, origin); quiesce(&mut self.sim); @@ -172,17 +191,23 @@ impl Bed { /// The cost of `event` on meshes shaped as `BASE` with one axis set to each size, /// measured after one warm-up event so first-time effects are left out. fn sweep(axis: fn(&mut Shape, usize), sizes: [usize; 3], event: fn(&mut Bed) -> Cost) -> [Cost; 3] { - sweep_at(axis, sizes, event, true) + sweep_at(BASE, axis, sizes, event, true) +} + +/// [`sweep`] from `QUIET`. +fn sweep_quiet(axis: fn(&mut Shape, usize), sizes: [usize; 3], event: fn(&mut Bed) -> Cost) -> [Cost; 3] { + sweep_at(QUIET, axis, sizes, event, true) } fn sweep_at( + base: Shape, axis: fn(&mut Shape, usize), sizes: [usize; N], event: fn(&mut Bed) -> Cost, checked: bool, ) -> [Cost; N] { sizes.map(|size| { - let mut shape = BASE; + let mut shape = base; axis(&mut shape, size); let mut bed = Bed::new(shape, checked); event(&mut bed); @@ -221,6 +246,8 @@ fn unwatched(shape: &mut Shape, n: usize) { /// One of each, a few, and many. const FLAT: [usize; 3] = [1, 4, 16]; +/// From two forwarding nodes, so a route that breaks has another to go round through. +const AROUND: [usize; 3] = [2, 4, 16]; /// Doublings, for axes that may grow linearly. Two upstream at least, so a cut leaves the /// measured relay a route. const LINEAR: [usize; 3] = [4, 8, 16]; @@ -257,12 +284,12 @@ fn an_announce_costs_the_sessions_it_reaches() { assert_linear("watchers it reaches", sweep(watched, LINEAR, Bed::announce)); } -/// A neighbour's route that breaks costs the relay the neighbours offering the same -/// route, which it ranks, and nothing per other forwarding node, publisher, broadcast, or -/// session it sends to. +/// A neighbour's route that breaks, one the relay does not use, costs it the neighbours +/// offering the same route, which it ranks, and nothing per other forwarding node, +/// publisher, broadcast, or session it sends to. #[test] fn a_cut_costs_the_neighbours_offering_the_route() { - assert_flat("forwarding nodes", sweep(origins, FLAT, Bed::cut)); + assert_flat("forwarding nodes", sweep(origins, AROUND, Bed::cut)); assert_flat("publishers", sweep(publishers, FLAT, Bed::cut)); assert_flat("broadcasts", sweep(broadcasts, FLAT, Bed::cut)); assert_flat("downstream", sweep(downstream, FLAT, Bed::cut)); @@ -270,6 +297,70 @@ fn a_cut_costs_the_neighbours_offering_the_route() { assert_linear("upstream", sweep(upstream, LINEAR, Bed::cut)); } +/// A route the relay uses that grows dearer costs it the neighbours offering the route +/// and the ones it tells, and the claims behind it only where the cost reaches a view: +/// nothing per publisher or broadcast while no watcher is shown them, and each broadcast +/// once per watcher shown its new cost. +#[test] +fn a_detour_costs_the_claims_whose_cost_is_shown() { + assert_flat("forwarding nodes", sweep_quiet(origins, AROUND, Bed::detour)); + assert_flat("publishers", sweep_quiet(publishers, FLAT, Bed::detour)); + assert_flat("broadcasts", sweep_quiet(broadcasts, FLAT, Bed::detour)); + assert_flat("watchers it misses", sweep_quiet(unwatched, FLAT, Bed::detour)); + assert_linear("upstream", sweep_quiet(upstream, LINEAR, Bed::detour)); + assert_linear("downstream", sweep_quiet(downstream, LINEAR, Bed::detour)); + assert_flat("forwarding nodes, watched", sweep(origins, AROUND, Bed::detour)); + assert_linear("publishers, watched", sweep(publishers, LINEAR, Bed::detour)); + assert_linear("broadcasts, watched", sweep(broadcasts, LINEAR, Bed::detour)); + assert_linear("watchers it reaches", sweep(watched, LINEAR, Bed::detour)); +} + +/// A neighbour's copy of a forwarding node's claims that overtakes the relay's own, as +/// when the neighbour it heard them from lags, costs what the two copies hold, never +/// their product: every claim replaced, read back from the snapshot a new neighbour +/// sends. +#[test] +fn an_overtaking_copy_costs_the_claims_it_holds() { + assert_linear("claims", [8, 16, 32].map(overtake)); +} + +fn overtake(claims: usize) -> Cost { + const R: Node = 0; + const LAGGING: Node = 1; + const ORIGIN: Node = 2; + const FRESH: Node = 3; + const PUBLISHER: Node = 4; + let mut sim = RelaySim::new(5, relay::debounce(HOP)); + sim.client(PUBLISHER); + sim.publish(PUBLISHER, PUBLISHER as Id); + sim.link(R, LAGGING, 1, Tie::Peer); + sim.link(LAGGING, ORIGIN, 1, Tie::Peer); + sim.link(FRESH, ORIGIN, 1, Tie::Peer); + sim.link(PUBLISHER, ORIGIN, 1, Tie::Peer); + for index in 0..claims { + sim.claim(PUBLISHER, &format!("old{index}"), true); + } + quiesce(&mut sim); + sim.hold(LAGGING, R); + for index in 0..claims { + sim.claim(PUBLISHER, &format!("old{index}"), false); + sim.claim(PUBLISHER, &format!("new{index}"), true); + } + quiesce(&mut sim); + let router = |sim: &RelaySim| sim.router(R).expect("the measured relay").work_done(); + let sent = |sim: &RelaySim| sim.sent.get(&R).copied().unwrap_or(0); + let (work, was) = (router(&sim), sent(&sim)); + sim.link(R, FRESH, 1, Tie::Peer); + quiesce(&mut sim); + let new = format!("new{}", claims - 1); + let shown = sim.router(R).expect("the measured relay").shown(); + assert!(shown.contains_key(&(PUBLISHER as Id, p(&new))), "the newer copy won"); + Cost { + work: router(&sim) - work, + sent: sent(&sim) - was, + } +} + /// A downstream session that closes costs the relay the routes it was sent, one per /// forwarding node, and nothing per publisher, broadcast, or other session. #[test] @@ -296,14 +387,20 @@ fn benchmark_relay_event_work() { ("watched", watched), ("unwatched", unwatched), ]; - type Event = (&'static str, fn(&mut Bed) -> Cost); - let events: [Event; 3] = [("announce", Bed::announce), ("cut", Bed::cut), ("leave", Bed::leave)]; + type Event = (&'static str, Shape, fn(&mut Bed) -> Cost); + let events: [Event; 5] = [ + ("announce", BASE, Bed::announce), + ("cut", BASE, Bed::cut), + ("detour", BASE, Bed::detour), + ("detour_quiet", QUIET, Bed::detour), + ("leave", BASE, Bed::leave), + ]; println!("event,axis,size,work,sent"); - for (event, run) in events { + for (event, base, run) in events { for (axis, set) in axes { let sizes = [2, 16, 128]; // A mesh this large checks itself in the gate's sweeps instead. - for (size, cost) in sizes.into_iter().zip(sweep_at(set, sizes, run, false)) { + for (size, cost) in sizes.into_iter().zip(sweep_at(base, set, sizes, run, false)) { println!("{event},{axis},{size},{},{}", cost.work, cost.sent); } } From 3f1b9d59aabe8fd68d9050d87a05c91baa674cad Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 09:06:58 -0800 Subject: [PATCH 10/17] fix(moq-net): route::relay::Router serves only the newest run of the closest claim, and saturates claim costs - Which claims of one Instance at one path may serve is decided over every claimant, reachable or not: the one exactly at the path before a broader one shown there (`Item::above`), then the newest epoch among those, as `model::origin` ranks them. A route's rank chooses only among those, so an older run never stands in for a newer one that lost its route, and a broader claim never displaces an exact one. - A claim's cost saturates instead of making the claim unusable, as `model::origin`'s claim cost does, so a route's cost never flips whether a claim can serve. A route past the metric is still never used, as under the path rule. - The switch quest takes the proof's router items, which the finished router quest held, with the precedence the claim instance needs first. Co-Authored-By: Claude Opus 5.5 --- quest/m2/multicast/switch.md | 24 +++ rs/moq-net/src/route/mod.rs | 16 +- rs/moq-net/src/route/node.rs | 4 + rs/moq-net/src/route/relay.rs | 246 ++++++++++++++++++++---------- rs/moq-net/src/route/tests/mod.rs | 4 + 5 files changed, 211 insertions(+), 83 deletions(-) diff --git a/quest/m2/multicast/switch.md b/quest/m2/multicast/switch.md index b3a55477fe7..2c66475880b 100644 --- a/quest/m2/multicast/switch.md +++ b/quest/m2/multicast/switch.md @@ -113,6 +113,30 @@ for the wire: joined by two grants; the chain cost none. Keep that shape in the acceptance fixtures. +Found 2026-11-16 by [the proof](https://github.com/moq-dev/moq/pull/8904) +against the prototype, and moved here when +[moq-dev/moq#8901](https://github.com/moq-dev/moq/pull/8901) finished the +router quest that listed them: + +- Order equal routes by the neighbour's Node ID, or give the proof's binding + the key the router orders them by: the proof orders them by the neighbour, + the prototype by its simulation's node index, and `route::relay::Router` by + its neighbour key. +- Record the chain as the claim instance's path: the announcing relay first, + then each grant's exporter and adopter, so a node refuses a claim naming it + in the role it would take. The prototype and the router record only the + adopters, so they refuse fewer echoes than the proof assumes. +- Move the binding to the router before this PR changes it: point + `route/tests/algebra.rs` and `route/tests/sticky.rs` at the router's own + functions for a node's own route, sending a route on, taking one in, + ordering candidates, and listing them, regenerate `Vectors.agda`, and run + `just test route-proof`. The claim instance first needs the router's + precedence as its leading keys: a claim exactly at its path before a + broader one shown there, then the newest epoch, decided over every claim + of the Instance at that path whether or not a route reaches it, and only + then the route joined with the claim's drain, priority, and cost. Neither + the mechanized instance nor its Rust vectors cover precedence yet. + Delete with the old rule: - `rs/moq-net/src/route/proof.rs`, `js/net/src/lite/route_key.ts`, diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index 73b3116d339..46de54efa05 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -48,9 +48,15 @@ pub(crate) trait Item: Clone + Eq + fmt::Debug { /// outranks every broader one there. `lowered(0)` is the claim itself. fn lowered(&self, by: u64) -> Self; - /// Which run of its Instance the publisher is on: a claim at a higher epoch supersedes - /// one of the same Instance at the same path whatever either's route costs, as - /// `model::origin` ranks them. 0 for attributes that carry none. + /// How many segments the claim's own prefix lacks of the path it is shown at, as + /// [`Self::lowered`] counts them: 0 for a claim exactly there, which outranks every + /// broader one whatever their epochs or routes. + fn above(&self) -> u64; + + /// Which run of its Instance the publisher is on: among claims of one Instance at one + /// path and distance, a higher epoch supersedes a lower one whatever either's route + /// costs, or whether a route reaches it, as `model::origin` ranks them. 0 for + /// attributes that carry none. fn epoch(&self) -> u64; } @@ -63,6 +69,10 @@ macro_rules! opaque { self.clone() } + fn above(&self) -> u64 { + 0 + } + fn epoch(&self) -> u64 { 0 } diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index ebcd5fe17c5..3f6090c16bb 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -76,6 +76,10 @@ impl super::Item for Claim { } } + fn above(&self) -> u64 { + self.above + } + fn epoch(&self) -> u64 { self.epoch } diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index 85f21a6a46e..246e33ebfb3 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -36,8 +36,10 @@ //! an endpoint or scoped neighbour is told the closest claim per broadcast within what it //! asks, a broader claim presented at the prefix it asks for. A request follows the most //! specific claim covering its path that a route reaches. Among the claims of one -//! Instance at one path, the newest epoch wins whatever it costs, then the best rank, and -//! a route or claim whose cost a link would carry past the metric is never used. +//! Instance at one path, only the one shown closest to its own path, at the newest epoch +//! among those, may serve, reachable or not, as `model::origin` ranks them; a route's rank +//! chooses among those alone. A route whose cost a link would carry past the metric is +//! never used, and a claim's cost saturates there. //! //! The rule is the bake-off's pick (moq-dev/moq#8880): `tests/multi.rs` with `floor`, //! `epoch`, `batch`, `drain`, and `claimed`, which `tests/multicast.rs` compares it @@ -80,6 +82,10 @@ pub(crate) struct Hop { /// A route's rank, lower first: draining, then the lowest priority, then the cost. type Rank = (bool, Reverse, Metric); +/// How claims of one precedence order, lower first: the rank of the route and the claim +/// together, then the shorter chain, and the claimant. +type ClaimOrder = (Rank, usize, NodeId); + /// What a route says. #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct Offer { @@ -155,6 +161,13 @@ impl Claim { } } +/// Which claims of one Instance at one path may serve it, lower first: the one closest to +/// its own path, then the newest epoch. A route's rank chooses only among claims that tie +/// here, so an older run never stands in for a newer one that lost its route. +fn precedence(item: &A) -> (u64, Reverse) { + (item.above(), Reverse(item.epoch())) +} + /// A broadcast: a path, and the Instance a claim there serves. pub(crate) type Broadcast = (Path, I); @@ -1536,15 +1549,12 @@ impl Router { for (broadcast, change) in changes { self.visit(); let change = match change { - // A claim that already crossed this relay came round a ring of grants, and - // one the link would make dearer than the metric holds is never used. - Change::Live(claim) - if claim.chain.contains(&self.me.relay) || claim.cost.checked_add(cost).is_none() => - { - Change::Lost - } + // A claim that already crossed this relay came round a ring of grants. + Change::Live(claim) if claim.chain.contains(&self.me.relay) => Change::Lost, Change::Live(mut claim) => { - claim.cost += cost; + // Saturating, as `model::origin`'s claim cost does, so a claim's cost + // never decides whether it can serve. + claim.cost = claim.cost.saturating_add(cost); claim.priority = claim.priority.min(priority); if scoped { claim.chain.push(self.me.relay); @@ -1591,10 +1601,10 @@ impl Router { claim.draining |= self.peers.get(&peer).is_some_and(|peer| peer.draining); (claim, peer) }) - // The newest epoch whatever it costs, as `closest` takes it. + // Every giver is reachable, so precedence first and then rank, as `closest`. .min_by_key(|(claim, peer)| { ( - Reverse(claim.item.epoch()), + precedence(&claim.item), claim.draining, Reverse(claim.priority), claim.cost, @@ -1656,31 +1666,57 @@ impl Router { } } - /// The claim of `broadcast` a request takes from here, with its claimant: the newest - /// epoch, then the best rank of the route and the claim together, a claim no link would - /// make dearer than the metric holds. - fn closest(&self, (path, instance): &Broadcast) -> Option<(NodeId, &Claim)> { - let origins = self.claimants.get(path)?.get(instance)?; - let broadcast = (path.clone(), instance.clone()); - (origins.iter()) - .filter_map(|&origin| { - self.visit(); - let claim = self.known.get(&origin)?.items.get(&broadcast)?; - let rank = match origin == self.me { - true => claim.rank(), - false => { - let (draining, Reverse(priority), cost) = self.best.get(&origin)?.offer.rank(); - ( - draining || claim.draining, - Reverse(priority.min(claim.priority)), - cost.checked_add(claim.cost)?, - ) - } - }; - Some(((Reverse(claim.item.epoch()), rank, claim.chain.len(), origin), claim)) - }) - .min_by(|(a, _), (b, _)| a.cmp(b)) - .map(|((_, _, _, origin), claim)| (origin, claim)) + /// Every claim of `broadcast` this node knows, with its claimant, reachable or not. + fn claims_of<'a>( + &'a self, + broadcast: &Broadcast, + ) -> impl Iterator)> + use<'a, N, I, A> { + let (path, instance) = broadcast; + let origins = self.claimants.get(path).and_then(|claims| claims.get(instance)); + let broadcast = broadcast.clone(); + (origins.into_iter().flatten()).filter_map(move |&origin| { + self.visit(); + let claim = self.known.get(&origin)?.items.get(&broadcast)?; + Some((origin, claim)) + }) + } + + /// The claim of `broadcast` a request takes from here, with its claimant: among the + /// claims of the best precedence, reachable or not, the best rank of the route and the + /// claim together that a route reaches. + fn closest(&self, broadcast: &Broadcast) -> Option<(NodeId, &Claim)> { + let mut top = None; + let mut pick: Option<(ClaimOrder, &Claim)> = None; + for (origin, claim) in self.claims_of(broadcast) { + let precedence = precedence(&claim.item); + if top.is_some_and(|top| precedence > top) { + continue; + } + // A better precedence outranks whatever was picked, reachable or not. + if top != Some(precedence) { + top = Some(precedence); + pick = None; + } + let rank = match origin == self.me { + true => claim.rank(), + false => { + let Some(best) = self.best.get(&origin) else { + continue; + }; + let (draining, Reverse(priority), cost) = best.offer.rank(); + ( + draining || claim.draining, + Reverse(priority.min(claim.priority)), + cost.saturating_add(claim.cost), + ) + } + }; + let key = (rank, claim.chain.len(), origin); + if pick.as_ref().is_none_or(|(best, _)| key < *best) { + pick = Some((key, claim)); + } + } + pick.map(|((_, _, origin), claim)| (origin, claim)) } // --- Readers, endpoints, and scoped neighbours --- @@ -1694,16 +1730,16 @@ impl Router { self.visit(); let (path, instance) = broadcast; let closest = self.closest(broadcast).map(|(_, claim)| claim.item.clone()); - let claimed = (self.claimants.get(path).and_then(|claims| claims.get(instance))) - .map(|origins| origins.first().copied()); + let claimed = (self.claimants.get(path)).is_some_and(|claims| claims.contains_key(instance)); let shown = match claimed { - Some(first) => { + true => { self.unwait(broadcast); let available = closest.is_some(); - // Unreachable, it still shows what a claimant says. + // Unreachable, it still shows what the claim of the best precedence says. let item = closest.or_else(|| { - let origin = first?; - Some(self.known.get(&origin)?.items.get(broadcast)?.item.clone()) + (self.claims_of(broadcast)) + .min_by_key(|(origin, claim)| (precedence(&claim.item), *origin)) + .map(|(_, claim)| claim.item.clone()) }); Some(item.map(|item| Shown { item, @@ -1711,12 +1747,12 @@ impl Router { presented: false, })) } - None if self.ended.contains(broadcast) || mutated!(self, LostEnds) => { + false if self.ended.contains(broadcast) || mutated!(self, LostEnds) => { self.unwait(broadcast); Some(None) } // Lost: shown unavailable for the debounce, in case a claimant returns. - None => match self.reader.get(instance.clone(), path) { + false => match self.reader.get(instance.clone(), path) { Some(_) if self.waiting.contains_key(broadcast) => None, Some(_) if self.debounce.is_zero() => Some(None), Some(was) => { @@ -1845,7 +1881,7 @@ impl Router { let route = &self.best[&origin].offer; Some(Claim { item: claim.item.clone(), - cost: claim.cost.checked_add(route.cost)?, + cost: claim.cost.saturating_add(route.cost), priority: claim.priority.min(route.priority), draining: claim.draining || route.draining, chain: claim.chain.clone(), @@ -2191,7 +2227,7 @@ mod tests { } /// A route from routing neighbour `via`, as `node(via)`, to forwarding node `origin`. - fn route(origin: Id, via: Id, cost: Metric) -> Msg { + fn route(origin: Id, via: Id, cost: Metric) -> Msg { let path = vec![ Hop { node: node(origin), @@ -2214,23 +2250,39 @@ mod tests { } } - /// Attributes with an epoch, as a restarted publisher's claims carry. - #[derive(Clone, Debug, PartialEq, Eq)] - struct Epoch(u64); + /// Attributes with an epoch and a distance, as a restarted publisher's claims, or a + /// broader claim presented at a narrower path, carry. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + struct Run { + epoch: u64, + above: u64, + } + + impl Item for Run { + fn lowered(&self, by: u64) -> Self { + Self { + above: self.above + by, + ..*self + } + } - impl Item for Epoch { - fn lowered(&self, _: u64) -> Self { - self.clone() + fn above(&self) -> u64 { + self.above } fn epoch(&self) -> u64 { - self.0 + self.epoch } } - fn claim(epoch: u64, cost: Metric) -> Change { + /// A claim exactly at its path, at `epoch`. + fn run(epoch: u64) -> Run { + Run { epoch, above: 0 } + } + + fn claim(item: Run, cost: Metric) -> Change { Change::Live(Claim { - item: Epoch(epoch), + item, cost, priority: i8::MAX, draining: false, @@ -2262,61 +2314,95 @@ mod tests { } } - /// A claim at a newer epoch wins over an older one of the same Instance however much - /// cheaper the older is, among a node's endpoints and among forwarding nodes alike. + /// Among claims of one Instance at one path, one exactly there outranks a broader one + /// shown there, and among those a newer epoch outranks an older one however much + /// cheaper the older is, or whether a route reaches the newer: among a node's + /// endpoints and among forwarding nodes alike. #[test] - fn a_newer_epoch_wins_whatever_it_costs() { - let mut router = Router::::new(node(1), debounce(HOP)); + fn the_newest_run_of_the_closest_claim_serves() { + let mut router = Router::::new(node(1), debounce(HOP)); open(&mut router, 2, Role::Endpoint); open(&mut router, 3, Role::Endpoint); let broadcast = (p("live"), 7); let announce = |change| Msg::Announce { changes: vec![(broadcast.clone(), change)], }; - router.recv(2, announce(claim(1, 0))).unwrap(); - router.recv(3, announce(claim(2, 9))).unwrap(); + router.recv(2, announce(claim(run(1), 0))).unwrap(); + router.recv(3, announce(claim(run(2), 9))).unwrap(); let me = router.me(); - assert_eq!(router.step(me, &broadcast), Some(Next::Local(3))); - assert_eq!(router.own().1[&broadcast].item, Epoch(2)); - assert_eq!(router.shown()[&(7, p("live"))].item, Epoch(2)); + assert_eq!( + router.step(me, &broadcast), + Some(Next::Local(3)), + "a newer epoch, however dear" + ); + assert_eq!(router.own().1[&broadcast].item, run(2)); + router.recv(2, announce(claim(Run { epoch: 3, above: 1 }, 0))).unwrap(); + assert_eq!( + router.step(me, &broadcast), + Some(Next::Local(3)), + "a broader claim, however new" + ); open(&mut router, 4, Role::Routing(node(4))); router.recv(4, route(5, 4, 1)).unwrap(); - let claims = |change, floor| Msg::Claims { + let claims = |item, floor| Msg::Claims { origin: node(5), - changes: vec![(broadcast.clone(), change)], + changes: vec![(broadcast.clone(), claim(item, 0))], floor, }; - router.recv(4, claims(claim(1, 0), 1)).unwrap(); + router.recv(4, claims(run(1), 1)).unwrap(); assert_eq!(router.pick(&broadcast), Some(me), "an older claim, however close"); - router.recv(4, claims(claim(3, 100), 2)).unwrap(); - assert_eq!(router.pick(&broadcast), Some(node(5)), "a newer claim, however far"); - assert_eq!(router.shown()[&(7, p("live"))].item, Epoch(3)); + router.recv(4, claims(Run { epoch: 4, above: 1 }, 2)).unwrap(); + assert_eq!(router.pick(&broadcast), Some(me), "a broader claim, however new"); + router.recv(4, claims(run(3), 3)).unwrap(); + assert_eq!(router.pick(&broadcast), Some(node(5)), "a newer claim"); + // The newer claim's route goes while its claim stays known: the older run never + // stands in for it. + router.recv(4, route(5, 4, Metric::MAX)).unwrap(); + assert_eq!(router.best(node(5)), None); + assert_eq!(router.pick(&broadcast), None); + let shown = &router.shown()[&(7, p("live"))]; + assert_eq!((shown.item, shown.available), (run(3), false)); } - /// A route or claim whose cost a link would carry past the metric is never used, and - /// never wraps round to a cheap one. + /// A route whose cost a link would carry past the metric is never used, and never wraps + /// round to a cheap one. A claim's cost saturates there instead, so it never decides + /// whether the claim can serve. #[test] - fn a_cost_past_the_metric_is_never_used() { - let mut router = Router::::new(node(1), debounce(HOP)); + fn a_route_past_the_metric_is_never_used() { + let mut router = Router::::new(node(1), debounce(HOP)); open(&mut router, 2, Role::Routing(node(2))); router.recv(2, route(5, 2, Metric::MAX)).unwrap(); assert_eq!(router.best(node(5)), None); router.recv(2, route(5, 2, 1)).unwrap(); assert!(router.best(node(5)).is_some()); + let far = (p("far"), 7); let claims = Msg::Claims { origin: node(5), - changes: vec![((p("far"), 7), claim(0, Metric::MAX - 1))], + changes: vec![(far.clone(), claim(run(0), Metric::MAX - 1))], floor: 1, }; router.recv(2, claims).unwrap(); - assert!(!router.shown()[&(7, p("far"))].available, "shown, never reachable"); - assert_eq!(router.pick(&(p("far"), 7)), None); + assert!(router.shown()[&(7, p("far"))].available); + assert_eq!(router.pick(&far), Some(node(5))); open(&mut router, 3, Role::Endpoint); + router.drain(); + let edit = Edit { + remove: Vec::new(), + add: vec![p(ROOT)], + }; + router.edit(3, edit).unwrap(); + let told = router.drain().into_iter().any(|(to, msg)| { + to == 3 + && matches!(&msg, Msg::Announce { changes } if changes.iter().any(|(broadcast, change)| { + *broadcast == far && matches!(change, Change::Live(claim) if claim.cost == Metric::MAX) + })) + }); + assert!(told, "told at the metric's ceiling"); let announce = Msg::Announce { - changes: vec![((p("near"), 7), claim(0, Metric::MAX))], + changes: vec![((p("near"), 7), claim(run(0), Metric::MAX))], }; router.recv(3, announce).unwrap(); - assert!(router.own().1.is_empty(), "an endpoint's claim past what its link adds"); + assert_eq!(router.own().1[&(p("near"), 7)].cost, Metric::MAX, "an endpoint's claim"); } /// The reader keeps nothing of claims that came and went, ended or lost. diff --git a/rs/moq-net/src/route/tests/mod.rs b/rs/moq-net/src/route/tests/mod.rs index adf85919409..f40a247e708 100644 --- a/rs/moq-net/src/route/tests/mod.rs +++ b/rs/moq-net/src/route/tests/mod.rs @@ -120,6 +120,10 @@ impl crate::route::Item for Item { *self } + fn above(&self) -> u64 { + 0 + } + fn epoch(&self) -> u64 { self.epoch.map_or(0, u64::from) } From 3239cb86b7bd3da68983316bd9c93c6cbf82ff03 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 10:47:41 -0800 Subject: [PATCH 11/17] fix(moq-net): route::relay::Router tells an endpoint or scoped neighbour what its reader shows An endpoint or scoped neighbour now hears a claim unavailable, never withdrawn, while the claim is known but unreachable and while a lost claim waits out the debounce. It hears the claim lost only when the reader goes offline. A claim gains an unavailable bit, which a scoped neighbour's adopter carries into its own domain. The route exchange between routing nodes still has no timer. - The debounce runs from when the reader first showed a claim unavailable, so the domain past a grant lets a lost claim go about when the near one does, not a debounce later. - A request fails closed at the most specific path the reader shows while that claim is unavailable, and at a claimant whose own claim is unavailable. - A withdrawal carries the cause of the claim it withdraws, so a viewer shown an exact claim is never told ended when only a broader claim ended. - A scoped claim whose chain would reach the longest path is not adopted, and a message naming one claim twice is refused. - RelaySim sends what a timeout makes, and the floor fixture routes across the relay that restarts. Co-Authored-By: Claude Opus 5.5 --- quest/m2/multicast/README.md | 7 +- quest/m2/multicast/switch.md | 16 + rs/moq-net/src/route/relay.rs | 476 ++++++++++++++++++++---- rs/moq-net/src/route/tests/multicast.rs | 263 ++++++++++++- 4 files changed, 666 insertions(+), 96 deletions(-) diff --git a/quest/m2/multicast/README.md b/quest/m2/multicast/README.md index 0de6d4eb75f..c7f1d7a60a8 100644 --- a/quest/m2/multicast/README.md +++ b/quest/m2/multicast/README.md @@ -102,7 +102,12 @@ Settled: on evidence ([requests](https://github.com/moq-dev/moq/pull/8881), [route health](/quest/m3/route-health/README.md)). What a hold protected, a claim flickering while routes settle, becomes a reader's local debounce - from unavailable to offline, never on the wire. This reverses items 19 and + from unavailable to offline, never on the wire (narrowed 2026-11-16 by the + coordinator, pending the maintainer, in + [moq-dev/moq#8901](https://github.com/moq-dev/moq/pull/8901): the route + exchange between routing nodes keeps no timer, and an endpoint or scoped + neighbour is told what the reader shows, so it hears a lost claim + unavailable until the debounce runs out). This reverses items 19 and 27's doubt (rejected: keeping holds and doubt with the doubt tied to the idle timeout; a doubt lifted by an epoch the surviving node floods). 5. ✅ **Three ways to find a warm copy are measured before one is picked** diff --git a/quest/m2/multicast/switch.md b/quest/m2/multicast/switch.md index 2c66475880b..6c7f1874529 100644 --- a/quest/m2/multicast/switch.md +++ b/quest/m2/multicast/switch.md @@ -104,6 +104,22 @@ for the wire: upstream link is reachable by arrival order (decided by the coordinator on 2026-11-16, pending the maintainer, in [moq-dev/moq#8901](https://github.com/moq-dev/moq/pull/8901)). +- A claim carries an unavailable bit beside its draining bit. An endpoint or + scoped neighbour is told what the node's own reader shows: the closest claim + while a route reaches one, and while none does, the claim of the best + precedence marked unavailable, withdrawn only when the reader's debounce + runs out. A scoped neighbour's adopter announces the claim again with the + bit, so its domain keeps the claim and fails requests closed, a request + reaching that adopter included, instead of withdrawing and starting it + again. The debounce runs from when a reader first showed the claim + unavailable, so the domain past a grant lets the claim go about when the + near one does, not a debounce later. The route exchange between routing + nodes keeps no timer (decided by the coordinator on 2026-11-16, pending the + maintainer, in [moq-dev/moq#8901](https://github.com/moq-dev/moq/pull/8901)). +- A message names each claim once, and a receiver refuses one that names a + claim twice, so no order within a message decides what it says (decided by + the implementer on 2026-11-16, pending the maintainer, in + [moq-dev/moq#8901](https://github.com/moq-dev/moq/pull/8901)). - A node offers a route to itself only once it announces a claim, so a relay that has never announced a claim and only forwards or only serves viewers has no route on the wire. diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index 246e33ebfb3..7ca539c7d51 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -21,24 +21,33 @@ //! entry. A node that sent GOAWAY is named once per neighbour in a [`Msg::Drain`], and //! every route through it ranks as draining. A node advertises once per burst of input, //! when the caller drains it. There is no hold, no doubt, and no routing timer: the only -//! clock is the reader's debounce from unavailable to offline, which never reaches the -//! wire. +//! clock is the reader's debounce from unavailable to offline, which runs from when the +//! reader first showed the claim unavailable. The route exchange between routing nodes +//! never waits on it. An endpoint or scoped neighbour is told what the reader shows, so +//! it hears a lost claim withdrawn only when the debounce runs out, and a domain past a +//! scoped grant, which showed the claim unavailable from the start, lets it go then too +//! rather than a debounce later. //! //! A forwarding node's claims are ordered by its revision and certified by floors: a node //! keeps each neighbour's copy and the newest floor wins, so no stale copy revives a //! withdrawal. A claims message carries its floor even with nothing changed, and a //! withdrawal says whether the publisher ended the claim or its announcer lost it. Across //! a scoped grant only claims travel: the receiver announces each again under its own -//! identity with the chain of relays that did so, and refuses one whose chain names it. +//! identity with the chain of relays that did so, and refuses one whose chain names it or +//! would grow past the longest path. //! //! Each neighbour asks for a set of prefixes edited in place, as under the path rule: a //! routing neighbour forwards for the whole domain, so it asks for the whole tree, and -//! an endpoint or scoped neighbour is told the closest claim per broadcast within what it -//! asks, a broader claim presented at the prefix it asks for. A request follows the most -//! specific claim covering its path that a route reaches. Among the claims of one -//! Instance at one path, only the one shown closest to its own path, at the newest epoch -//! among those, may serve, reachable or not, as `model::origin` ranks them; a route's rank -//! chooses among those alone. A route whose cost a link would carry past the metric is +//! an endpoint or scoped neighbour is told what this node's reader shows per broadcast +//! within what it asks, a broader claim presented at the prefix it asks for. That is the +//! closest claim while one is reachable, and the claim marked unavailable, never +//! withdrawn, while none is and while a lost claim waits out the debounce. A scoped +//! neighbour announces it again unavailable, so its domain sees the claim stay. A request +//! follows the claim at the most specific path covering it that the reader shows, and +//! fails closed while that one is unavailable, as it does at a claimant whose own claim +//! is. Among the claims of one Instance at one path, only the one shown closest to its +//! own path, at the newest epoch among those, may serve, reachable or not, as +//! `model::origin` ranks them; a route's rank chooses among those alone. A route whose cost a link would carry past the metric is //! never used, and a claim's cost saturates there. //! //! The rule is the bake-off's pick (moq-dev/moq#8880): `tests/multi.rs` with `floor`, @@ -144,6 +153,10 @@ pub(crate) struct Claim { pub priority: i8, /// The way to the publisher crosses a node that sent GOAWAY. pub draining: bool, + /// No way to the publisher reaches it from the node that told it, which keeps it, as + /// its reader shows it, while the claim may return. Set only on a claim told to an + /// endpoint or scoped neighbour, and on one a scoped neighbour's adopter announces. + pub unavailable: bool, /// The relays that announced it again across a scoped grant, the first first. pub chain: Vec, } @@ -282,7 +295,16 @@ struct Peer { /// Claims it gave this node, as an endpoint or across a grant. given: Claims, /// Claims this node announced to it, by path, as an endpoint or across a grant. - told: BTreeMap>>, + told: BTreeMap>>, +} + +/// What a neighbour was told at one path. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Told { + claim: Claim, + /// The path of the claim it stands for: the one told at, or a broader one presented + /// there. + from: Path, } impl Peer { @@ -417,6 +439,8 @@ pub(crate) struct Router { dirty: BTreeSet>, /// Claims shown unavailable since they lost every claimant, by when they go offline. waiting: BTreeMap, Instant>, + /// When the reader began showing each claim unavailable, which the debounce runs from. + since: BTreeMap, Instant>, deadlines: BTreeSet<(Instant, Broadcast)>, reader: Reader, out: Vec<(N, Msg)>, @@ -500,6 +524,7 @@ impl Router { ended: BTreeSet::new(), dirty: BTreeSet::new(), waiting: BTreeMap::new(), + since: BTreeMap::new(), deadlines: BTreeSet::new(), reader: Reader::default(), out: Vec::new(), @@ -558,14 +583,21 @@ impl Router { self.visit(); let broadcast = (path, instance); self.waiting.remove(&broadcast); + self.since.remove(&broadcast); let (path, instance) = broadcast; due.entry(instance).or_default().insert(path, None); } + let mut gone = Vec::new(); for (instance, claims) in due { + gone.extend(claims.keys().map(|path| (path.clone(), instance.clone()))); self.reader.apply(instance, |_| true, claims, |_| false); } // As in `refresh`: nothing reads which claims the reader recomputed. self.reader.changed(); + // An endpoint or scoped neighbour hears it lost only now. + for broadcast in &gone { + self.exports(broadcast); + } } // --- Sessions --- @@ -1353,8 +1385,8 @@ impl Router { let heard = (self.heard.get_mut(&origin)) .and_then(|offers| offers.get_mut(&from)) .ok_or(Error::Unrouted)?; - // Each message moves the sender's copy to a newer revision. - if floor <= heard.floor || changes.iter().any(|(_, change)| !well_built(change)) { + // Each message moves the sender's copy to a newer revision, once per claim. + if floor <= heard.floor || !well_built(&changes) { return Err(Error::Malformed); } let was = heard.floor; @@ -1541,7 +1573,7 @@ impl Router { if changes.iter().any(|((path, _), _)| !within_any(path, &asked)) { return Err(Error::Unasked); } - if changes.iter().any(|(_, change)| !well_built(change)) { + if !well_built(&changes) { return Err(Error::Malformed); } let (cost, priority) = (peer.port.cost, peer.port.priority); @@ -1549,8 +1581,14 @@ impl Router { for (broadcast, change) in changes { self.visit(); let change = match change { - // A claim that already crossed this relay came round a ring of grants. - Change::Live(claim) if claim.chain.contains(&self.me.relay) => Change::Lost, + // A claim that already crossed this relay came round a ring of grants, and + // one whose chain this relay would carry past the longest path is never + // taken, as a route past the metric is never used. + Change::Live(claim) + if claim.chain.contains(&self.me.relay) || (scoped && claim.chain.len() + 1 >= MAX_HOPS) => + { + Change::Lost + } Change::Live(mut claim) => { // Saturating, as `model::origin`'s claim cost does, so a claim's cost // never decides whether it can serve. @@ -1601,10 +1639,11 @@ impl Router { claim.draining |= self.peers.get(&peer).is_some_and(|peer| peer.draining); (claim, peer) }) - // Every giver is reachable, so precedence first and then rank, as `closest`. + // Precedence first and then rank, as `closest`, an unavailable claim last. .min_by_key(|(claim, peer)| { ( precedence(&claim.item), + claim.unavailable, claim.draining, Reverse(claim.priority), claim.cost, @@ -1646,22 +1685,30 @@ impl Router { // --- Requests --- - /// The forwarding node a request for `broadcast` names from here: the closest of the - /// most specific claims covering it that one can reach, as a neighbour asking for its - /// path is shown. + /// The forwarding node a request for `broadcast` names from here: the closest claim + /// at the most specific path covering it that the reader shows, reachable or not, as a + /// neighbour asking for its path is shown. `None` fails it closed while that claim is + /// unavailable. pub fn pick(&self, (path, instance): &Broadcast) -> Option { - (ancestors(path).rev()) - .find_map(|above| self.closest(&(above, instance.clone()))) - .map(|(origin, _)| origin) + let from = (ancestors(path).rev()).find(|above| self.reader.get(instance.clone(), above).is_some())?; + self.closest(&(from, instance.clone())).map(|(origin, _)| origin) } - /// Where a request for `broadcast` naming `origin` goes from here. `None` fails it. + /// Where a request for `broadcast` naming `origin` goes from here. `None` fails it, + /// as it does here while the claim covering it is unavailable. pub fn step(&self, origin: NodeId, (path, instance): &Broadcast) -> Option> { match origin == self.me { // The neighbour behind the most specific of this node's claims covering it. - true => (ancestors(path).rev()) - .find_map(|above| self.sources.get(&(above, instance.clone())).copied()) - .map(Next::Local), + true => { + let own = self.known.get(&self.me); + (ancestors(path).rev()) + .map(|above| (above, instance.clone())) + .find_map(|claim| self.sources.get(&claim).map(|&source| (source, claim))) + .filter(|(_, claim)| { + !(own.and_then(|own| own.items.get(claim))).is_some_and(|claim| claim.unavailable) + }) + .map(|(source, _)| Next::Local(source)) + } false => self.best.get(&origin).map(|best| Next::Via(best.via)), } } @@ -1697,6 +1744,10 @@ impl Router { top = Some(precedence); pick = None; } + // Its announcer has no way to the publisher, whatever route reaches it. + if claim.unavailable { + continue; + } let rank = match origin == self.me { true => claim.rank(), false => { @@ -1735,6 +1786,12 @@ impl Router { true => { self.unwait(broadcast); let available = closest.is_some(); + if available { + self.since.remove(broadcast); + } else { + // Unavailable since the reader first showed it so, lost or not. + self.since.entry(broadcast.clone()).or_insert(self.now); + } // Unreachable, it still shows what the claim of the best precedence says. let item = closest.or_else(|| { (self.claims_of(broadcast)) @@ -1749,24 +1806,37 @@ impl Router { } false if self.ended.contains(broadcast) || mutated!(self, LostEnds) => { self.unwait(broadcast); + self.since.remove(broadcast); Some(None) } - // Lost: shown unavailable for the debounce, in case a claimant returns. + // Lost: shown unavailable for the debounce, in case a claimant returns. The + // debounce runs from when the reader first showed it unavailable, so a claim + // a scoped neighbour told unavailable and then lost goes offline about when + // that neighbour's reader did. false => match self.reader.get(instance.clone(), path) { Some(_) if self.waiting.contains_key(broadcast) => None, - Some(_) if self.debounce.is_zero() => Some(None), Some(was) => { - let item = was.item.clone(); - let at = self.now.saturating_add(self.debounce); - self.waiting.insert(broadcast.clone(), at); - self.deadlines.insert((at, broadcast.clone())); - Some(Some(Shown { - item, - available: false, - presented: false, - })) + let since = self.since.get(broadcast).copied().unwrap_or(self.now); + let at = since.saturating_add(self.debounce); + if at <= self.now { + self.since.remove(broadcast); + Some(None) + } else { + self.since.insert(broadcast.clone(), since); + let item = was.item.clone(); + self.waiting.insert(broadcast.clone(), at); + self.deadlines.insert((at, broadcast.clone())); + Some(Some(Shown { + item, + available: false, + presented: false, + })) + } + } + None => { + self.since.remove(broadcast); + None } - None => None, }, }; if let Some(shown) = shown { @@ -1821,27 +1891,35 @@ impl Router { /// Tell `peer` what this node shows it of `instance` at `path` now. fn export(&mut self, peer: N, path: &Path, instance: &I) { self.visit(); - let claim = self.shown_to(peer, path, instance); + let now = self.shown_to(peer, path, instance); let state = self.peers.get_mut(&peer).expect("a neighbour"); let told = state.told.get(path).and_then(|told| told.get(instance)); - if told == claim.as_ref() { + if told.map(|told| &told.claim) == now.as_ref().map(|now| &now.claim) { + // The same claim, perhaps standing for another: nothing to say. + if let Some(now) = now { + let told = state.told.get_mut(path).and_then(|told| told.get_mut(instance)); + *told.expect("told") = now; + } return; } - let change = match claim { - Some(claim) => { - let told = state.told.entry(path.clone()).or_default(); - told.insert(instance.clone(), claim.clone()); + let change = match now { + Some(now) => { + let claim = now.claim.clone(); + state + .told + .entry(path.clone()) + .or_default() + .insert(instance.clone(), now); Change::Live(claim) } None => { let told = state.told.get_mut(path).expect("told"); - told.remove(instance); + let gone = told.remove(instance).expect("told"); if told.is_empty() { state.told.remove(path); } - // It ended if the publisher of the claim it stood for ended it. - let ended = ancestors(path).any(|above| self.ended.contains(&(above, instance.clone()))); - match ended { + // It ended only if the publisher of the claim it stood for ended it. + match self.ended.contains(&(gone.from, instance.clone())) { true => Change::Ended, false => Change::Lost, } @@ -1851,40 +1929,76 @@ impl Router { self.out.push((peer, Msg::Announce { changes })); } - /// What `peer` is shown of `instance` at `path`: the closest claim there, or at a - /// prefix it asks for, the most specific claim above, lowered to it. - fn shown_to(&self, peer: N, path: &Path, instance: &I) -> Option> { + /// What `peer` is told of `instance` at `path`: what this node's reader shows of the + /// claim there, or at a prefix it asks for, of the most specific claim above that the + /// reader shows, lowered to it. + fn shown_to(&self, peer: N, path: &Path, instance: &I) -> Option> { let asks = &self.peers[&peer].asks; if !within_any(path, asks) { return None; } - if let Some(claim) = self.exported(peer, &(path.clone(), instance.clone())) { - return Some(claim); + if let Some(told) = self.stands(peer, path, path, instance) { + return Some(told); } if !asks.contains(path) { return None; } - (ancestors(path).rev().skip(1).zip(1..)) - .find_map(|(above, by)| Some(self.exported(peer, &(above, instance.clone()))?.lowered(by))) - } - - /// The closest claim for `broadcast` as a neighbour of this node sees it, unless it came - /// from `peer`. - fn exported(&self, peer: N, broadcast: &Broadcast) -> Option> { - let (origin, claim) = self.closest(broadcast)?; - if origin == self.me { - return match self.sources.get(broadcast) == Some(&peer) { - true => None, - false => Some(claim.clone()), - }; - } - let route = &self.best[&origin].offer; - Some(Claim { - item: claim.item.clone(), - cost: claim.cost.saturating_add(route.cost), - priority: claim.priority.min(route.priority), - draining: claim.draining || route.draining, - chain: claim.chain.clone(), + (ancestors(path).rev().skip(1)).find_map(|above| self.stands(peer, path, &above, instance)) + } + + /// What `peer`, told at `path`, is shown of the claim of `instance` at `from`, if this + /// node's reader shows one there and it did not come from `peer`. It is the closest + /// claim while one is reachable; else the claim of the best precedence, unavailable; + /// else, while the reader waits to show it offline, what `peer` was told of it, + /// unavailable. Never a withdrawal while the reader still shows the claim. + fn stands(&self, peer: N, path: &Path, from: &Path, instance: &I) -> Option> { + self.reader.get(instance.clone(), from)?; + let broadcast = (from.clone(), instance.clone()); + let source = |origin: NodeId| origin == self.me && self.sources.get(&broadcast) == Some(&peer); + let claim = match self.closest(&broadcast) { + Some((origin, _)) if source(origin) => return None, + Some((origin, claim)) if origin == self.me => claim.clone(), + Some((origin, claim)) => { + let route = &self.best[&origin].offer; + Claim { + item: claim.item.clone(), + cost: claim.cost.saturating_add(route.cost), + priority: claim.priority.min(route.priority), + draining: claim.draining || route.draining, + unavailable: false, + chain: claim.chain.clone(), + } + } + None => match self + .claims_of(&broadcast) + .min_by_key(|(origin, claim)| (precedence(&claim.item), *origin)) + { + Some((origin, _)) if source(origin) => return None, + Some((_, claim)) => Claim { + unavailable: true, + ..claim.clone() + }, + // Lost, and waiting out the debounce. + None => { + let told = self.peers[&peer].told.get(path)?.get(instance)?; + if told.from != *from { + return None; + } + let claim = Claim { + unavailable: true, + ..told.claim.clone() + }; + return Some(Told { + claim, + from: from.clone(), + }); + } + }, + }; + let by = (path.parts().count() - from.parts().count()) as u64; + Some(Told { + claim: claim.lowered(by), + from: from.clone(), }) } @@ -1914,12 +2028,17 @@ impl Router { } } -/// A claim the rule never builds: a chain past the longest path. -fn well_built(change: &Change) -> bool { - match change { - Change::Live(claim) => claim.chain.len() < MAX_HOPS, - Change::Ended | Change::Lost => true, - } +/// Whether the rule could have built `changes`: no chain past the longest path, and no +/// claim changed twice. +fn well_built(changes: &[(Broadcast, Change)]) -> bool { + let mut seen = BTreeSet::new(); + changes.iter().all(|(broadcast, change)| { + let built = match change { + Change::Live(claim) => claim.chain.len() < MAX_HOPS, + Change::Ended | Change::Lost => true, + }; + built && seen.insert(broadcast) + }) } #[cfg(test)] @@ -2040,6 +2159,7 @@ impl Router { ended, dirty, waiting, + since, deadlines: _, reader, out, @@ -2053,10 +2173,14 @@ impl Router { let waiting: BTreeMap<&Broadcast, Duration> = (waiting.iter()) .map(|(broadcast, at)| (broadcast, at.saturating_duration_since(*now))) .collect(); + // How long a claim has shown unavailable matters only up to the debounce. + let since: BTreeMap<&Broadcast, Duration> = (since.iter()) + .map(|(broadcast, at)| (broadcast, now.saturating_duration_since(*at).min(*debounce))) + .collect(); let state = ( (me, debounce, sessions, peers, routing, fresh, heard, unnamed), (dead, closed, draining, stale, best, known, rev, announced), - (sources, ended, dirty, waiting, reader.shown(), out), + (sources, ended, dirty, waiting, since, reader.shown(), out), ); super::digest(hasher, &state); } @@ -2152,6 +2276,10 @@ impl Router { .map(|(broadcast, at)| (*at, broadcast.clone())) .collect(); assert_eq!(self.deadlines, deadlines, "deadlines"); + for (path, instance) in self.since.keys() { + let shown = self.reader.get(instance.clone(), path); + assert!(shown.is_some_and(|shown| !shown.available), "unavailable since"); + } assert!(self.dirty.is_empty(), "refreshed"); } } @@ -2222,6 +2350,7 @@ mod tests { cost, priority: i8::MAX, draining: false, + unavailable: false, chain: Vec::new(), }) } @@ -2286,6 +2415,7 @@ mod tests { cost, priority: i8::MAX, draining: false, + unavailable: false, chain: Vec::new(), }) } @@ -2405,6 +2535,146 @@ mod tests { assert_eq!(router.own().1[&(p("near"), 7)].cost, Metric::MAX, "an endpoint's claim"); } + /// A viewer is withdrawn with the cause of the claim it was shown, never with a broader + /// claim's: one shown the exact claim at its prefix hears it lost even as a broader + /// claim above ends in the same batch, and one shown only the broader claim hears it + /// ended. With a debounce, the lost claim is first shown unavailable, and lost when the + /// relay's reader goes offline. + #[test] + fn a_withdrawal_carries_the_cause_of_the_claim_shown() { + let viewer = |router: &mut Relay, peer: usize, prefix: &str| { + open(router, peer, Role::Endpoint); + let edit = Edit { + remove: Vec::new(), + add: vec![p(prefix)], + }; + router.edit(peer, edit).unwrap(); + }; + let told = |out: &[(usize, Msg)], peer: usize| -> Vec<(Path, Change)> { + (out.iter()) + .filter(|(to, _)| *to == peer) + .filter_map(|(_, msg)| match msg { + Msg::Announce { changes } => Some(changes.clone()), + _ => None, + }) + .flatten() + .map(|((path, _), change)| (path, change)) + .collect() + }; + let batch = Msg::Announce { + changes: vec![((p("a"), 7), Change::Ended), ((p("a/b"), 7), Change::Lost)], + }; + for debounce in [Duration::ZERO, super::debounce(HOP)] { + let mut router: Relay = Router::new(node(1), debounce); + open(&mut router, 2, Role::Endpoint); + let announce = Msg::Announce { + changes: vec![((p("a"), 7), live(0)), ((p("a/b"), 7), live(0))], + }; + router.recv(2, announce).unwrap(); + viewer(&mut router, 3, "a/b"); + viewer(&mut router, 4, "a/c"); + router.drain(); + router.recv(2, batch.clone()).unwrap(); + let out = router.drain(); + let (exact, broader) = (told(&out, 3), told(&out, 4)); + assert_eq!(broader, [(p("a/c"), Change::Ended)], "{debounce:?}"); + let lost = (p("a/b"), Change::Lost); + if debounce.is_zero() { + assert_eq!(exact, [lost], "{debounce:?}"); + continue; + } + assert!( + matches!(exact.as_slice(), [(path, Change::Live(claim))] if *path == p("a/b") && claim.unavailable), + "{exact:?}" + ); + router.timeout(router.deadline().expect("a debounce")); + assert_eq!(told(&router.drain(), 3), [lost]); + } + } + + /// A scoped neighbour's claim whose chain this relay would carry to the longest path + /// is not taken, and the message carrying it stands: this relay never sends a chain a + /// neighbour refuses. + #[test] + fn a_chain_is_never_carried_past_the_longest_path() { + let mut router = relay(1); + open(&mut router, 2, Role::Scoped(vec![p(ROOT)])); + open(&mut router, 3, Role::Routing(node(3))); + router.drain(); + let chained = |len: usize| { + Change::Live(Claim { + chain: (100..100 + len as Id).collect(), + ..match live(0) { + Change::Live(claim) => claim, + _ => unreachable!(), + } + }) + }; + let announce = Msg::Announce { + changes: vec![ + ((p("short"), 7), chained(MAX_HOPS - 2)), + ((p("long"), 7), chained(MAX_HOPS - 1)), + ], + }; + router.recv(2, announce).unwrap(); + let own = router.own().1; + assert_eq!(own[&(p("short"), 7)].chain.len(), MAX_HOPS - 1); + assert!(!own.contains_key(&(p("long"), 7))); + for (_, msg) in router.drain() { + if let Msg::Claims { changes, .. } | Msg::Announce { changes } = &msg { + assert!(well_built(changes), "{msg:?}"); + } + } + } + + /// A message that changes one claim twice is one the rule never sends, so it is + /// refused whole, from a routing neighbour or an endpoint. + #[test] + fn a_claim_changed_twice_in_one_message_is_refused() { + let mut router = relay(1); + open(&mut router, 2, Role::Routing(node(2))); + open(&mut router, 3, Role::Endpoint); + let offer = Offer { + path: vec![ + Hop { + node: node(5), + session: 0, + }, + Hop { + node: node(2), + session: 1, + }, + ] + .into(), + draining: false, + priority: i8::MAX, + cost: 1, + marked: false, + }; + let route = Msg::Route { + origin: node(5), + offer: Some(offer), + }; + router.recv(2, route).unwrap(); + let broadcast = (p("live"), 7); + let claims = |changes, floor| Msg::Claims { + origin: node(5), + changes, + floor, + }; + router.recv(2, claims(vec![(broadcast.clone(), live(0))], 1)).unwrap(); + let twice = vec![(broadcast.clone(), Change::Ended), (broadcast.clone(), live(0))]; + assert_eq!(router.recv(2, claims(twice.clone(), 2)), Err(Error::Malformed)); + assert_eq!(router.pick(&broadcast), Some(node(5)), "nothing of it applied"); + router + .recv(2, claims(vec![(broadcast.clone(), Change::Ended)], 2)) + .unwrap(); + assert_eq!(router.pick(&broadcast), None); + let announce = Msg::Announce { changes: twice }; + assert_eq!(router.recv(3, announce), Err(Error::Malformed)); + assert!(router.own().1.is_empty()); + } + /// The reader keeps nothing of claims that came and went, ended or lost. #[test] fn a_claim_that_went_leaves_nothing_in_the_reader() { @@ -2601,6 +2871,50 @@ mod tests { assert_eq!(router.deadline(), None); } + /// A claim a scoped neighbour tells unavailable stays, and a request naming this node + /// fails closed meanwhile. Lost, it waits only what is left of the debounce from when + /// the reader first showed it unavailable, so a domain past a grant lets it go about + /// when the neighbour's reader did, not a debounce later. + #[test] + fn a_claim_told_unavailable_waits_from_then() { + use super::super::Kind; + let mut router = relay(1); + open(&mut router, 2, Role::Scoped(vec![p(ROOT)])); + let claim = |change| Msg::Announce { + changes: vec![((p("live"), 7), change)], + }; + let unavailable = match live(0) { + Change::Live(claim) => Change::Live(Claim { + unavailable: true, + ..claim + }), + _ => unreachable!(), + }; + let kinds = |router: &mut Relay| router.events().into_iter().map(|event| event.kind).collect::>(); + let request = (p("live/x"), 7); + router.recv(2, claim(live(0))).unwrap(); + assert_eq!(router.step(node(1), &request), Some(Next::Local(2))); + let since = Instant::ZERO; + router.recv(2, claim(unavailable.clone())).unwrap(); + assert_eq!(kinds(&mut router), [Kind::Start, Kind::Unavailable]); + assert_eq!(router.step(node(1), &request), None, "fails closed"); + assert_eq!(router.pick(&request), None, "fails closed"); + assert_eq!(router.deadline(), None, "a claim kept waits for nothing"); + router.timeout(since.saturating_add(debounce(HOP) / 2)); + router.recv(2, claim(Change::Lost)).unwrap(); + assert_eq!(router.deadline(), Some(since.saturating_add(debounce(HOP)))); + router.timeout(since.saturating_add(debounce(HOP))); + assert_eq!(kinds(&mut router), [Kind::Offline]); + // Unavailable for the whole debounce before it is lost, it goes at once. + router.recv(2, claim(live(0))).unwrap(); + router.recv(2, claim(unavailable)).unwrap(); + let since = since.saturating_add(debounce(HOP)); + router.timeout(since.saturating_add(debounce(HOP))); + router.recv(2, claim(Change::Lost)).unwrap(); + assert_eq!(kinds(&mut router), [Kind::Start, Kind::Unavailable, Kind::Offline]); + assert_eq!(router.deadline(), None); + } + /// An endpoint asking for a prefix under a broader claim is shown that claim at the /// prefix, and an edit swaps what it is shown in place. #[test] diff --git a/rs/moq-net/src/route/tests/multicast.rs b/rs/moq-net/src/route/tests/multicast.rs index 786467cd201..9adda844dfa 100644 --- a/rs/moq-net/src/route/tests/multicast.rs +++ b/rs/moq-net/src/route/tests/multicast.rs @@ -36,6 +36,16 @@ enum Spot { }, } +/// A claim change one node told another, and when it arrived. +#[derive(Clone, Debug)] +pub(super) struct Told { + pub at: Instant, + pub from: Node, + pub to: Node, + pub claim: Path, + pub change: relay::Change<()>, +} + /// A network of [`relay::Router`]s and endpoints. #[derive(Clone)] pub(super) struct RelaySim { @@ -52,6 +62,8 @@ pub(super) struct RelaySim { /// Messages each node sent, the interest exchange left out. pub sent: BTreeMap, pub events: Vec, + /// Every claim change delivered to an endpoint or across a grant, in order. + pub told: Vec, spent: usize, now: Instant, debounce: Duration, @@ -79,6 +91,7 @@ fn own() -> Claim<()> { cost: 0, priority: i8::MAX, draining: false, + unavailable: false, chain: Vec::new(), } } @@ -98,6 +111,7 @@ impl RelaySim { stats: Stats::default(), sent: BTreeMap::new(), events: Vec::new(), + told: Vec::new(), spent: 0, now: Instant::ZERO, debounce, @@ -111,6 +125,11 @@ impl RelaySim { sim } + /// The simulated time. + pub fn now(&self) -> Instant { + self.now + } + /// How `node` names itself on a path as it runs now. pub fn id(&self, node: Node) -> NodeId { let generation = u64::from(self.restarts[node]); @@ -486,6 +505,18 @@ impl RelaySim { let Some(msg) = self.queues.get_mut(&(from, to)).and_then(VecDeque::pop_front) else { return false; }; + if let Msg::Announce { changes } = &msg { + for ((claim, _), change) in changes { + let (at, claim, change) = (self.now, claim.clone(), change.clone()); + self.told.push(Told { + at, + from, + to, + claim, + change, + }); + } + } if let Some(router) = self.relay(to) && let Err(err) = router.recv(from, msg.clone()) @@ -546,7 +577,8 @@ impl RelaySim { waiting |= router.deadline().is_some(); } } - self.show(); + // What a reader going offline tells endpoints and scoped neighbours goes out now. + self.flush(); waiting } @@ -830,17 +862,28 @@ fn routing(row: &Row) -> Vec<(&'static str, String)> { ] } -/// The fields of two rows that differ, each as the prototype's and this router's. -fn differ(kept: &Row, relay: &Row) -> Vec { +/// The fields of two rows that differ, each named and shown as the prototype's and this +/// router's. +fn differ(kept: &Row, relay: &Row) -> Vec<(&'static str, String)> { (routing(kept).into_iter().zip(routing(relay))) .filter(|((_, a), (_, b))| a != b) .map(|((field, a), (_, b))| match field { - "answers" => "answers".to_string(), - _ => format!("{field} {a}->{b}"), + "answers" => (field, "answers".to_string()), + _ => (field, format!("{field} {a}->{b}")), }) .collect() } +/// Where the router tells a scoped neighbour a claim unavailable that the prototype +/// withdraws, by failure and shape. +const UNAVAILABLE: [(&str, &str); 5] = [ + ("home_dies", "two_meshes_one_grant"), + ("home_dies", "two_meshes_two_grants"), + ("home_dies", "three_meshes_chain"), + ("hub_dies_slowly", "three_meshes_chain"), + ("home_dies", "three_meshes_ring"), +]; + /// The production router, its readers letting go at once as the prototype's do, gives /// the prototype's answers on every shape and failure of the bake-off at the same cost: /// the same next hops for every request once it settles, the same route messages, the @@ -850,6 +893,17 @@ fn differ(kept: &Row, relay: &Row) -> Vec { /// and then hears a lagging neighbour's copy of its claims within one burst passes them /// on in the prototype, at a floor its neighbours already hold, on routes it withdraws /// in the same burst (`home_dies` on `tiers`); this router withdraws those routes first. +/// +/// It differs in more, by design (`UNAVAILABLE`). While a claim is known but no route +/// reaches it, the router tells a scoped neighbour what its reader shows, the claim +/// unavailable, where the prototype withdraws it. A relay that loses its last route to +/// the publisher's home before it hears the home's claims withdrawn therefore tells the +/// next mesh the claim unavailable and then lost, and the adopter there passes on both. +/// That costs more claims, and on the two-mesh shapes one more round, and fewer far +/// readers let go and start the claim again. On the ring of grants, both rules route the +/// two ends of one grant to each other for a moment, each on the copy of the claim that +/// came the other way round (`home_dies`); the adopter's unavailable is one more message +/// delivered while that lasts, and it clears in the same round. #[test] fn the_router_gives_the_prototypes_answers() { let mut out = Vec::new(); @@ -858,10 +912,22 @@ fn the_router_gives_the_prototypes_answers() { let relay = fail(Arm::Relay(Duration::ZERO), &shape.mesh, false); assert_eq!(kept.len(), relay.len(), "{}", shape.name); for ((failure, kept), (_, relay)) in kept.iter().zip(&relay) { - let fields = differ(kept, relay); - // Fewer claims is the one field allowed to differ. - let fewer = kept.cost.sent.claims > relay.cost.sent.claims; - if fields.len() > usize::from(fewer) { + let (a, b) = (&kept.cost, &relay.cost); + let unavailable = UNAVAILABLE.contains(&(failure.as_str(), shape.name)); + let excused = |field: &str| match field { + "claim_items" if a.sent.claims > b.sent.claims => true, + "claim_items" => unavailable, + "settle_rounds" => unavailable && b.rounds == a.rounds + 1, + "starts" => unavailable && b.sent.starts < a.sent.starts, + "offline" => unavailable && b.sent.offline < a.sent.offline, + "loops" => unavailable && shape.name == "three_meshes_ring" && b.loops == a.loops + 1, + _ => false, + }; + let fields: Vec = (differ(kept, relay).into_iter()) + .filter(|(field, _)| !excused(field)) + .map(|(_, shown)| shown) + .collect(); + if !fields.is_empty() { out.push(format!("{} on {}: {}", failure.as_str(), shape.name, fields.join(", "))); } } @@ -970,28 +1036,197 @@ fn a_newer_floor_travels_with_nothing_changed() { assert_eq!(shown(&sim, 4), None); } +/// What a relay tells an endpoint or scoped neighbour of a claim, as it reads. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Said { + Live, + Unavailable, + Lost, + Ended, +} + +/// A mesh where relay 1 serves a viewer endpoint asking for the whole tree, and a scoped +/// neighbour, relay 2, which relay 3 reads beyond. The publisher's home is relay 0, and +/// relay 5 is the one it moves to. +mod watch { + use super::*; + + pub const HOME: Node = 0; + pub const RELAY: Node = 1; + pub const SCOPED: Node = 2; + pub const FAR: Node = 3; + pub const PUBLISHER: Node = 4; + pub const NEXT: Node = 5; + pub const VIEWER: Node = 6; + pub const CLAIM: &str = "live"; + + pub fn mesh() -> RelaySim { + let mut sim = RelaySim::new(7, relay::debounce(HOP)); + sim.client(PUBLISHER); + sim.publish(PUBLISHER, 100); + sim.client(VIEWER); + sim.link(HOME, RELAY, 1, Tie::Peer); + sim.link(NEXT, RELAY, 1, Tie::Peer); + sim.link(RELAY, SCOPED, 1, Tie::Scoped); + sim.link(SCOPED, FAR, 1, Tie::Peer); + sim.link(PUBLISHER, HOME, 1, Tie::Peer); + sim.link(VIEWER, RELAY, 1, Tie::Peer); + sim.ask(VIEWER, RELAY, ""); + sim.claim(PUBLISHER, CLAIM, true); + quiesce(&mut sim); + sim + } + + /// What relay 1 told `to` of the claim from the `since`th change on, each with when it + /// arrived, a run of the same kind said once. + pub fn said(sim: &RelaySim, to: Node, since: usize) -> Vec<(Instant, Said)> { + let mut out: Vec<(Instant, Said)> = Vec::new(); + for told in &sim.told[since..] { + if (told.from, told.to, &told.claim) != (RELAY, to, &p(CLAIM)) { + continue; + } + let said = match &told.change { + relay::Change::Live(claim) if claim.unavailable => Said::Unavailable, + relay::Change::Live(_) => Said::Live, + relay::Change::Lost => Said::Lost, + relay::Change::Ended => Said::Ended, + }; + if out.last().is_none_or(|(_, last)| *last != said) { + out.push((told.at, said)); + } + } + out + } + + /// What `node`'s reader showed from the `since`th event on. + pub fn read(sim: &RelaySim, node: Node, since: usize) -> Vec { + (sim.events[since..].iter()) + .filter(|seen| seen.node == node) + .map(|seen| seen.kind) + .collect() + } +} + +/// An endpoint or scoped neighbour is told what the relay's own reader shows, never more: +/// unavailable at once and never withdrawn while the claim is known or may return, and +/// lost only when the relay's reader goes offline. A publisher that moves leaves a viewer +/// and a scoped neighbour unavailable for a hop and then live again, and the mesh beyond +/// the grant sees the same, with no withdrawal and no new start. A publisher that leaves +/// is withdrawn one debounce later, and the mesh beyond the grant lets it go then too; +/// one that ends its claim is withdrawn at once. +#[test] +fn a_viewer_is_told_what_the_relay_reads() { + use watch::*; + let kinds = |said: Vec<(Instant, Said)>| said.into_iter().map(|(_, said)| said).collect::>(); + let mut sim = mesh(); + for to in [VIEWER, SCOPED] { + assert_eq!(kinds(said(&sim, to, 0)), [Said::Live], "{to} before"); + } + + // The move the bake-off's `Failure::Move` makes. + let (told, events) = (sim.told.len(), sim.events.len()); + sim.close(PUBLISHER, HOME); + sim.link(PUBLISHER, NEXT, 1, Tie::Peer); + quiesce(&mut sim); + for to in [VIEWER, SCOPED] { + assert_eq!( + kinds(said(&sim, to, told)), + [Said::Unavailable, Said::Live], + "{to} on a move" + ); + } + assert_eq!(read(&sim, RELAY, events), [Kind::Unavailable, Kind::Available]); + assert_eq!( + read(&sim, FAR, events), + [Kind::Unavailable, Kind::Available], + "beyond the grant" + ); + + // A failure that brings nothing back. + let (told, events, left) = (sim.told.len(), sim.events.len(), sim.now()); + sim.close(PUBLISHER, NEXT); + // As `quiesce`, noting when the mesh beyond the grant lets the claim go. + let mut far = None; + for _ in 0..200 { + let ready = sim.ready(); + for &((from, to), count) in &ready { + for _ in 0..count { + sim.deliver(from, to); + } + } + let waiting = sim.tick(); + if far.is_none() && read(&sim, FAR, events).contains(&Kind::Offline) { + far = Some(sim.now()); + } + if ready.is_empty() && !waiting && sim.ready().is_empty() { + break; + } + } + for to in [VIEWER, SCOPED] { + let said = said(&sim, to, told); + assert_eq!(kinds(said.clone()), [Said::Unavailable, Said::Lost], "{to} on a leave"); + let (unavailable, lost) = (said[0].0, said[1].0); + assert!(unavailable <= left.saturating_add(2 * HOP), "{to}: told at once"); + assert!( + lost >= left.saturating_add(relay::debounce(HOP)), + "{to}: lost only after the debounce" + ); + } + assert_eq!(read(&sim, RELAY, events), [Kind::Unavailable, Kind::Offline]); + assert_eq!( + read(&sim, FAR, events), + [Kind::Unavailable, Kind::Offline], + "beyond the grant" + ); + // It showed the claim unavailable from the start, so it waits no second debounce. + let lost = said(&sim, SCOPED, told)[1].0; + let far = far.expect("offline beyond the grant"); + assert!( + far <= lost.saturating_add(2 * HOP), + "beyond the grant: {far:?}, told lost {lost:?}" + ); + + // A publisher that ends its claim is withdrawn at once, as ended. + let mut sim = mesh(); + let (told, events) = (sim.told.len(), sim.events.len()); + sim.claim(PUBLISHER, CLAIM, false); + quiesce(&mut sim); + for to in [VIEWER, SCOPED] { + assert_eq!(kinds(said(&sim, to, told)), [Said::Ended], "{to} on an end"); + } + assert_eq!(read(&sim, FAR, events), [Kind::End], "beyond the grant"); +} + /// A relay that restarts is a new node, but its neighbours file the session floor per /// pair of relays with the restart in the session's order, so a restart replaces the /// entry instead of adding one. #[test] fn a_session_floor_keeps_an_entry_per_pair_of_relays() { - let mut sim = RelaySim::new(3, relay::debounce(HOP)); - for (a, b) in [(0, 1), (1, 2), (0, 2)] { + let mut sim = RelaySim::new(5, relay::debounce(HOP)); + let links = [(0, 1), (1, 2), (0, 2), (0, 3)]; + for (a, b) in links { sim.link(a, b, 1, Tie::Peer); } + // A publisher behind the relay that restarts, so 1 and 2 offer each other routes + // through their sessions to it, and name those sessions closed when it goes. + sim.client(4); + sim.publish(4, 100); + sim.link(4, 3, 1, Tie::Peer); + sim.claim(4, "live", true); quiesce(&mut sim); let mut kept = Vec::new(); for _ in 0..8 { sim.reset(0); quiesce(&mut sim); - for peer in [1, 2] { + for peer in [1, 2, 3] { sim.open(0, peer); } quiesce(&mut sim); kept.push(sim.kept().closed); } - // Two ways round each of the three links at most. - assert!(kept.iter().all(|&pairs| pairs <= 6), "{kept:?}"); + // Two ways round each link at most. + assert!(kept[0] > 0, "the restart closed sessions routes crossed: {kept:?}"); + assert!(kept.iter().all(|&pairs| pairs <= 2 * links.len()), "{kept:?}"); assert_eq!(kept[0], kept[7], "{kept:?}"); } From bd0a8f3735ea365f65463a3c17e3c41f2dea4343 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 11:09:31 -0800 Subject: [PATCH 12/17] fix(moq-net): route::relay::Router tells a late asker what a waiting claim showed A neighbour that asked while a lost claim waited out the debounce was shown the broader claim above it as available, while the reader still showed the lost claim unavailable and an earlier asker was told so. The router now keeps what each waiting claim showed, once for every asker, and never tells it back to the neighbour that gave it. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/relay.rs | 150 +++++++++++++++++++++++++++++----- 1 file changed, 131 insertions(+), 19 deletions(-) diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index 7ca539c7d51..935ef368232 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -307,6 +307,16 @@ struct Told { from: Path, } +/// The claim a lost broadcast showed, kept while it waits out the debounce so a +/// neighbour that asks meanwhile is told it as one that asked before is. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Gone { + claimant: NodeId, + claim: Claim, + /// The neighbour that gave this node's own claim, which is never told it back. + source: Option, +} + impl Peer { fn routing(&self) -> Option { match self.port.role { @@ -441,6 +451,11 @@ pub(crate) struct Router { waiting: BTreeMap, Instant>, /// When the reader began showing each claim unavailable, which the debounce runs from. since: BTreeMap, Instant>, + /// What each waiting claim showed. + gone: BTreeMap, Gone>, + /// The claims lost in this input, with their claimants, until `refresh` keeps what + /// a waiting claim showed. + dropped: BTreeMap, Vec>>, deadlines: BTreeSet<(Instant, Broadcast)>, reader: Reader, out: Vec<(N, Msg)>, @@ -525,6 +540,8 @@ impl Router { dirty: BTreeSet::new(), waiting: BTreeMap::new(), since: BTreeMap::new(), + gone: BTreeMap::new(), + dropped: BTreeMap::new(), deadlines: BTreeSet::new(), reader: Reader::default(), out: Vec::new(), @@ -584,6 +601,7 @@ impl Router { let broadcast = (path, instance); self.waiting.remove(&broadcast); self.since.remove(&broadcast); + self.gone.remove(&broadcast); let (path, instance) = broadcast; due.entry(instance).or_default().insert(path, None); } @@ -1009,9 +1027,10 @@ impl Router { self.heard.remove(&origin); if let Some(known) = self.known.remove(&origin) { self.count(known.items.len()); - for broadcast in known.items.into_keys() { + for (broadcast, claim) in known.items { self.unclaim(&broadcast, origin); - self.dirty.insert(broadcast); + self.dirty.insert(broadcast.clone()); + self.drop_claim(broadcast, origin, claim, None); } // Without any copy, this node no longer knows how new a claim of `origin` is: // a stale one may start it again, until the newer floor behind it arrives. @@ -1446,6 +1465,7 @@ impl Router { let floor = known.floor; let mut claimed = Vec::new(); let mut unclaimed = Vec::new(); + let mut lost = Vec::new(); for (broadcast, change) in &changes { match change { Change::Live(claim) => { @@ -1455,8 +1475,12 @@ impl Router { self.ended.remove(broadcast); } Change::Ended | Change::Lost => { - if known.items.remove(broadcast).is_some() { + if let Some(claim) = known.items.remove(broadcast) { unclaimed.push(broadcast.clone()); + // `restate` keeps its own claim's source. + if *change == Change::Lost && origin != self.me { + lost.push((broadcast.clone(), claim)); + } } if *change == Change::Ended { self.ended.insert(broadcast.clone()); @@ -1465,6 +1489,9 @@ impl Router { } self.dirty.insert(broadcast.clone()); } + for (broadcast, claim) in lost { + self.drop_claim(broadcast, origin, claim, None); + } for broadcast in &claimed { self.claim(broadcast, origin); } @@ -1498,6 +1525,20 @@ impl Router { } } + /// `claimant` lost its claim of `broadcast`, which `source` gave this node if its own. + fn drop_claim(&mut self, broadcast: Broadcast, claimant: NodeId, claim: Claim, source: Option) { + let claim = Claim { + unavailable: true, + ..claim + }; + let gone = Gone { + claimant, + claim, + source, + }; + self.dropped.entry(broadcast).or_default().push(gone); + } + fn unclaim(&mut self, broadcast: &Broadcast, origin: NodeId) { let (path, instance) = broadcast; let Some(claims) = self.claimants.get_mut(path) else { @@ -1664,9 +1705,12 @@ impl Router { Change::Live(claim) } None => { - self.sources.remove(&broadcast); - if was.is_none() { + let source = self.sources.remove(&broadcast); + let Some(was) = was.cloned() else { return; + }; + if cause == Change::Lost { + self.drop_claim(broadcast.clone(), me, was, source); } cause } @@ -1824,6 +1868,16 @@ impl Router { } else { self.since.insert(broadcast.clone(), since); let item = was.item.clone(); + // The claim it showed: of those just lost, one with its item, by + // precedence and claimant as an unreachable one is shown. + let gone = (self.dropped.get(broadcast).into_iter().flatten()) + .min_by_key(|gone| { + (gone.claim.item != item, precedence(&gone.claim.item), gone.claimant) + }) + .cloned(); + if let Some(gone) = gone { + self.gone.insert(broadcast.clone(), gone); + } self.waiting.insert(broadcast.clone(), at); self.deadlines.insert((at, broadcast.clone())); Some(Some(Shown { @@ -1858,12 +1912,14 @@ impl Router { for broadcast in dirty { self.ended.remove(&broadcast); } + self.dropped.clear(); } fn unwait(&mut self, broadcast: &Broadcast) { if let Some(at) = self.waiting.remove(broadcast) { self.deadlines.remove(&(at, broadcast.clone())); } + self.gone.remove(broadcast); } /// Tell each endpoint and scoped neighbour asking for `broadcast` what it shows there. @@ -1949,8 +2005,8 @@ impl Router { /// What `peer`, told at `path`, is shown of the claim of `instance` at `from`, if this /// node's reader shows one there and it did not come from `peer`. It is the closest /// claim while one is reachable; else the claim of the best precedence, unavailable; - /// else, while the reader waits to show it offline, what `peer` was told of it, - /// unavailable. Never a withdrawal while the reader still shows the claim. + /// else, while the reader waits to show it offline, the claim it showed, unavailable. + /// Never a withdrawal while the reader still shows the claim. fn stands(&self, peer: N, path: &Path, from: &Path, instance: &I) -> Option> { self.reader.get(instance.clone(), from)?; let broadcast = (from.clone(), instance.clone()); @@ -1978,20 +2034,13 @@ impl Router { unavailable: true, ..claim.clone() }, - // Lost, and waiting out the debounce. + // Lost, and waiting out the debounce: what it showed, to every asker alike. None => { - let told = self.peers[&peer].told.get(path)?.get(instance)?; - if told.from != *from { + let gone = self.gone.get(&broadcast)?; + if gone.source == Some(peer) { return None; } - let claim = Claim { - unavailable: true, - ..told.claim.clone() - }; - return Some(Told { - claim, - from: from.clone(), - }); + gone.claim.clone() } }, }; @@ -2160,6 +2209,8 @@ impl Router { dirty, waiting, since, + gone, + dropped: _, deadlines: _, reader, out, @@ -2180,7 +2231,7 @@ impl Router { let state = ( (me, debounce, sessions, peers, routing, fresh, heard, unnamed), (dead, closed, draining, stale, best, known, rev, announced), - (sources, ended, dirty, waiting, since, reader.shown(), out), + (sources, ended, dirty, waiting, since, gone, reader.shown(), out), ); super::digest(hasher, &state); } @@ -2276,6 +2327,11 @@ impl Router { .map(|(broadcast, at)| (*at, broadcast.clone())) .collect(); assert_eq!(self.deadlines, deadlines, "deadlines"); + assert!( + self.gone.keys().all(|broadcast| self.waiting.contains_key(broadcast)), + "gone" + ); + assert!(self.dropped.is_empty(), "dropped"); for (path, instance) in self.since.keys() { let shown = self.reader.get(instance.clone(), path); assert!(shown.is_some_and(|shown| !shown.available), "unavailable since"); @@ -2592,6 +2648,62 @@ mod tests { } } + /// A neighbour that asks while a lost claim waits out the debounce is told what one + /// that asked before is: the lost claim, unavailable, not the broader claim above it, + /// which the reader does not show there yet. Both are shown the broader one once the + /// lost claim goes offline. + #[test] + fn an_ask_during_the_debounce_is_told_what_an_earlier_one_is() { + let ask = |router: &mut Relay, peer: usize| { + open(router, peer, Role::Endpoint); + let edit = Edit { + remove: Vec::new(), + add: vec![p("a/b")], + }; + router.edit(peer, edit).unwrap(); + }; + let told = |out: &[(usize, Msg)], peer: usize| -> Vec> { + (out.iter()) + .filter(|(to, _)| *to == peer) + .filter_map(|(_, msg)| match msg { + Msg::Announce { changes } => Some(changes.clone()), + _ => None, + }) + .flatten() + .inspect(|((path, _), _)| assert_eq!(*path, p("a/b"))) + .map(|(_, change)| change) + .collect() + }; + let mut router = relay(1); + open(&mut router, 2, Role::Endpoint); + let announce = Msg::Announce { + changes: vec![((p("a"), 7), live(0)), ((p("a/b"), 7), live(0))], + }; + router.recv(2, announce).unwrap(); + ask(&mut router, 3); + router.drain(); + let lost = Msg::Announce { + changes: vec![((p("a/b"), 7), Change::Lost)], + }; + router.recv(2, lost).unwrap(); + ask(&mut router, 4); + let out = router.drain(); + let (before, after) = (told(&out, 3), told(&out, 4)); + assert!( + matches!(before.as_slice(), [Change::Live(claim)] if claim.unavailable), + "{before:?}" + ); + assert_eq!(after, before, "the same unavailable claim"); + router.timeout(router.deadline().expect("a debounce")); + let out = router.drain(); + let (before, after) = (told(&out, 3), told(&out, 4)); + assert!( + matches!(before.as_slice(), [Change::Live(claim)] if !claim.unavailable), + "{before:?}" + ); + assert_eq!(after, before, "the broader claim"); + } + /// A scoped neighbour's claim whose chain this relay would carry to the longest path /// is not taken, and the message carrying it stands: this relay never sends a chain a /// neighbour refuses. From 2a98e2986c6a86eec30ae9607be039fb7d311404 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 11:29:25 -0800 Subject: [PATCH 13/17] fix(moq-net): route::relay::Router shows a waiting claim to a new asker, and fails a request under it at the claimant A neighbour asking a prefix over a lost claim that waits out the debounce, with no live claim left there, was sent nothing for it. Its snapshot now covers the waiting claims within and above the prefix. A request already naming this node took the most specific of its own claims that still had a source. Under a lost narrower claim it went on to a broader one. It now takes the claim at the most specific path the reader shows, as `pick` does, and fails closed unless that claim is this node's own and available. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/relay.rs | 140 +++++++++++++++++++++++++--------- 1 file changed, 105 insertions(+), 35 deletions(-) diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index 935ef368232..d596d3b36ec 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -43,12 +43,12 @@ //! closest claim while one is reachable, and the claim marked unavailable, never //! withdrawn, while none is and while a lost claim waits out the debounce. A scoped //! neighbour announces it again unavailable, so its domain sees the claim stay. A request -//! follows the claim at the most specific path covering it that the reader shows, and -//! fails closed while that one is unavailable, as it does at a claimant whose own claim -//! is. Among the claims of one Instance at one path, only the one shown closest to its -//! own path, at the newest epoch among those, may serve, reachable or not, as -//! `model::origin` ranks them; a route's rank chooses among those alone. A route whose cost a link would carry past the metric is -//! never used, and a claim's cost saturates there. +//! follows the claim at the most specific path covering it that the reader shows, at its +//! claimant too, and fails closed while that one is unavailable or lost. Among the claims +//! of one Instance at one path, only the one shown closest to its own path, at the newest +//! epoch among those, may serve, reachable or not, as `model::origin` ranks them; a +//! route's rank chooses among those alone. A route whose cost a link would carry past the +//! metric is never used, and a claim's cost saturates there. //! //! The rule is the bake-off's pick (moq-dev/moq#8880): `tests/multi.rs` with `floor`, //! `epoch`, `batch`, `drain`, and `claimed`, which `tests/multicast.rs` compares it @@ -452,7 +452,7 @@ pub(crate) struct Router { /// When the reader began showing each claim unavailable, which the debounce runs from. since: BTreeMap, Instant>, /// What each waiting claim showed. - gone: BTreeMap, Gone>, + gone: BTreeMap>>, /// The claims lost in this input, with their claimants, until `refresh` keeps what /// a waiting claim showed. dropped: BTreeMap, Vec>>, @@ -601,7 +601,7 @@ impl Router { let broadcast = (path, instance); self.waiting.remove(&broadcast); self.since.remove(&broadcast); - self.gone.remove(&broadcast); + self.ungone(&broadcast); let (path, instance) = broadcast; due.entry(instance).or_default().insert(path, None); } @@ -807,15 +807,22 @@ impl Router { if askers.len() == 1 { self.rewatch(prefix); } - // Every broadcast within it, at its own path. - let inside: Vec<(Path, I)> = (self.claimants.get_key_value(prefix).into_iter()) + // Every broadcast within it, at its own path, claimed or a lost one waiting out + // the debounce. + let claimed = (self.claimants.get_key_value(prefix).into_iter()) .chain(below(&self.claimants, prefix)) - .flat_map(|(path, claims)| claims.keys().map(move |instance| (path.clone(), instance.clone()))) - .collect(); - // Every Instance claimed above it, presented at the prefix itself. + .flat_map(|(path, claims)| claims.keys().map(move |instance| (path.clone(), instance.clone()))); + let waiting = (self.gone.get_key_value(prefix).into_iter()) + .chain(below(&self.gone, prefix)) + .flat_map(|(path, gone)| gone.keys().map(move |instance| (path.clone(), instance.clone()))); + let inside: BTreeSet<(Path, I)> = claimed.chain(waiting).collect(); + // Every Instance claimed or waiting above it, presented at the prefix itself. let above: BTreeSet = (ancestors(prefix).filter(|above| above != prefix)) - .filter_map(|above| self.claimants.get(&above)) - .flat_map(|claims| claims.keys().cloned()) + .flat_map(|above| { + let claimed = self.claimants.get(&above).into_iter().flat_map(BTreeMap::keys); + claimed.chain(self.gone.get(&above).into_iter().flat_map(BTreeMap::keys)) + }) + .cloned() .collect(); for (path, instance) in inside { self.export(peer, &path, &instance); @@ -1738,20 +1745,18 @@ impl Router { self.closest(&(from, instance.clone())).map(|(origin, _)| origin) } - /// Where a request for `broadcast` naming `origin` goes from here. `None` fails it, - /// as it does here while the claim covering it is unavailable. + /// Where a request for `broadcast` naming `origin` goes from here. `None` fails it. pub fn step(&self, origin: NodeId, (path, instance): &Broadcast) -> Option> { match origin == self.me { - // The neighbour behind the most specific of this node's claims covering it. + // As `pick`, the claim at the most specific path covering it that the reader + // shows: the neighbour behind it if it is this node's own and available, and + // closed while it is unavailable, lost, or another node's. true => { - let own = self.known.get(&self.me); - (ancestors(path).rev()) - .map(|above| (above, instance.clone())) - .find_map(|claim| self.sources.get(&claim).map(|&source| (source, claim))) - .filter(|(_, claim)| { - !(own.and_then(|own| own.items.get(claim))).is_some_and(|claim| claim.unavailable) - }) - .map(|(source, _)| Next::Local(source)) + let from = (ancestors(path).rev()).find(|above| self.reader.get(instance.clone(), above).is_some())?; + let claim = (from, instance.clone()); + let source = *self.sources.get(&claim)?; + let own = self.known.get(&self.me).and_then(|own| own.items.get(&claim)); + (!own.is_some_and(|claim| claim.unavailable)).then_some(Next::Local(source)) } false => self.best.get(&origin).map(|best| Next::Via(best.via)), } @@ -1876,7 +1881,7 @@ impl Router { }) .cloned(); if let Some(gone) = gone { - self.gone.insert(broadcast.clone(), gone); + (self.gone.entry(path.clone()).or_default()).insert(instance.clone(), gone); } self.waiting.insert(broadcast.clone(), at); self.deadlines.insert((at, broadcast.clone())); @@ -1919,7 +1924,16 @@ impl Router { if let Some(at) = self.waiting.remove(broadcast) { self.deadlines.remove(&(at, broadcast.clone())); } - self.gone.remove(broadcast); + self.ungone(broadcast); + } + + fn ungone(&mut self, (path, instance): &Broadcast) { + if let Some(gone) = self.gone.get_mut(path) { + gone.remove(instance); + if gone.is_empty() { + self.gone.remove(path); + } + } } /// Tell each endpoint and scoped neighbour asking for `broadcast` what it shows there. @@ -2036,7 +2050,7 @@ impl Router { }, // Lost, and waiting out the debounce: what it showed, to every asker alike. None => { - let gone = self.gone.get(&broadcast)?; + let gone = self.gone.get(from)?.get(instance)?; if gone.source == Some(peer) { return None; } @@ -2327,10 +2341,12 @@ impl Router { .map(|(broadcast, at)| (*at, broadcast.clone())) .collect(); assert_eq!(self.deadlines, deadlines, "deadlines"); - assert!( - self.gone.keys().all(|broadcast| self.waiting.contains_key(broadcast)), - "gone" - ); + for (path, gone) in &self.gone { + assert!(!gone.is_empty(), "gone"); + for instance in gone.keys() { + assert!(self.waiting.contains_key(&(path.clone(), instance.clone())), "gone"); + } + } assert!(self.dropped.is_empty(), "dropped"); for (path, instance) in self.since.keys() { let shown = self.reader.get(instance.clone(), path); @@ -2498,6 +2514,22 @@ mod tests { assert_eq!(router.pick(&broadcast), None, "{broadcast:?}"); assert_eq!(router.step(me, &broadcast), None, "{broadcast:?}"); } + // The narrower claim lost, a request under it fails closed, even one already + // naming this node, until the reader lets it go. + router + .recv( + 3, + Msg::Announce { + changes: vec![((p("live/a"), 7), Change::Lost)], + }, + ) + .unwrap(); + let under = (p("live/a/x"), 7); + assert_eq!(router.pick(&under), None); + assert_eq!(router.step(me, &under), None); + router.timeout(router.deadline().expect("a debounce")); + assert_eq!(router.pick(&under), Some(me)); + assert_eq!(router.step(me, &under), Some(Next::Local(2))); } /// Among claims of one Instance at one path, one exactly there outranks a broader one @@ -2650,8 +2682,8 @@ mod tests { /// A neighbour that asks while a lost claim waits out the debounce is told what one /// that asked before is: the lost claim, unavailable, not the broader claim above it, - /// which the reader does not show there yet. Both are shown the broader one once the - /// lost claim goes offline. + /// which the reader does not show there yet, nor nothing when no claim is left. Both + /// are shown the broader one, or the loss, once the lost claim goes offline. #[test] fn an_ask_during_the_debounce_is_told_what_an_earlier_one_is() { let ask = |router: &mut Relay, peer: usize| { @@ -2702,6 +2734,44 @@ mod tests { "{before:?}" ); assert_eq!(after, before, "the broader claim"); + + // With nothing above it, a neighbour asking a prefix over the lost claim is told it + // too, and lost with the earlier asker when it goes. + let mut router = relay(1); + open(&mut router, 2, Role::Endpoint); + router + .recv( + 2, + Msg::Announce { + changes: vec![((p("a/b"), 7), live(0))], + }, + ) + .unwrap(); + ask(&mut router, 3); + router + .recv( + 2, + Msg::Announce { + changes: vec![((p("a/b"), 7), Change::Lost)], + }, + ) + .unwrap(); + router.drain(); + open(&mut router, 4, Role::Endpoint); + let edit = Edit { + remove: Vec::new(), + add: vec![p("a")], + }; + router.edit(4, edit).unwrap(); + let after = told(&router.drain(), 4); + assert!( + matches!(after.as_slice(), [Change::Live(claim)] if claim.unavailable), + "{after:?}" + ); + router.timeout(router.deadline().expect("a debounce")); + let out = router.drain(); + assert_eq!(told(&out, 3), [Change::Lost]); + assert_eq!(told(&out, 4), [Change::Lost]); } /// A scoped neighbour's claim whose chain this relay would carry to the longest path From b47d3278db4c2d70d35feb9a1461111ed6dbec04 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 11:58:40 -0800 Subject: [PATCH 14/17] fix(moq-net): route::relay::Router never takes a claim first told unavailable, and serves its own claim only at the best precedence A scoped neighbour's or endpoint's claim that arrives unavailable from a giver that never gave it is not there: nothing can serve it, so the adopter starts no announcement and no route until it is told available. The bit stays a change to a claim already taken from that giver. On two meshes joined by two grants, a publisher leaving costs 0 route messages again (19 at the previous head), and claim items fall on every grant shape with an echo. step(me) now also requires its own claim to be of the best precedence at the path, as closest does, so a request naming this node never follows an older run when a newer one is known. Co-Authored-By: Claude Opus 5.5 --- quest/m2/multicast/switch.md | 9 ++ rs/moq-net/src/route/relay.rs | 97 +++++++++++++++++-- rs/moq-net/src/route/tests/multicast.rs | 118 +++++++++++++++++++++--- 3 files changed, 206 insertions(+), 18 deletions(-) diff --git a/quest/m2/multicast/switch.md b/quest/m2/multicast/switch.md index 6c7f1874529..fc37dedbf5e 100644 --- a/quest/m2/multicast/switch.md +++ b/quest/m2/multicast/switch.md @@ -116,6 +116,15 @@ for the wire: near one does, not a debounce later. The route exchange between routing nodes keeps no timer (decided by the coordinator on 2026-11-16, pending the maintainer, in [moq-dev/moq#8901](https://github.com/moq-dev/moq/pull/8901)). +- The unavailable bit counts only as a change to a claim the receiver already + holds from that sender. A sender still tells a new asker what its reader + shows, the bit included, but a receiver never takes a claim it is first told + unavailable, which nothing can serve: that claim is not there until it is + told available, and then is taken as usual. A gateway whose session opens + while a claim is unavailable therefore announces nothing and starts no + route, and a claim that comes back unavailable across a second grant starts + none either (decided by the coordinator on 2026-11-16, pending the + maintainer, in [moq-dev/moq#8901](https://github.com/moq-dev/moq/pull/8901)). - A message names each claim once, and a receiver refuses one that names a claim twice, so no order within a message decides what it says (decided by the implementer on 2026-11-16, pending the maintainer, in diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index d596d3b36ec..adfca87df29 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -42,7 +42,9 @@ //! within what it asks, a broader claim presented at the prefix it asks for. That is the //! closest claim while one is reachable, and the claim marked unavailable, never //! withdrawn, while none is and while a lost claim waits out the debounce. A scoped -//! neighbour announces it again unavailable, so its domain sees the claim stay. A request +//! neighbour that took the claim announces it again unavailable, so its domain sees the +//! claim stay; one first told it unavailable, which nothing can serve, takes nothing until +//! it is told available. A request //! follows the claim at the most specific path covering it that the reader shows, at its //! claimant too, and fails closed while that one is unavailable or lost. Among the claims //! of one Instance at one path, only the one shown closest to its own path, at the newest @@ -1637,6 +1639,12 @@ impl Router { { Change::Lost } + // Unavailable is only ever news of a claim this neighbour gave before: one + // first seen unavailable, which nothing can serve, is not there until it is + // available. + Change::Live(claim) if claim.unavailable && !self.peers[&from].given.contains_key(&broadcast) => { + continue; + } Change::Live(mut claim) => { // Saturating, as `model::origin`'s claim cost does, so a claim's cost // never decides whether it can serve. @@ -1749,14 +1757,18 @@ impl Router { pub fn step(&self, origin: NodeId, (path, instance): &Broadcast) -> Option> { match origin == self.me { // As `pick`, the claim at the most specific path covering it that the reader - // shows: the neighbour behind it if it is this node's own and available, and - // closed while it is unavailable, lost, or another node's. + // shows: the neighbour behind it if it is this node's own, available, and of the + // best precedence there, and closed while it is unavailable, lost, outranked, + // or another node's. true => { let from = (ancestors(path).rev()).find(|above| self.reader.get(instance.clone(), above).is_some())?; let claim = (from, instance.clone()); let source = *self.sources.get(&claim)?; - let own = self.known.get(&self.me).and_then(|own| own.items.get(&claim)); - (!own.is_some_and(|claim| claim.unavailable)).then_some(Next::Local(source)) + let own = self.known.get(&self.me)?.items.get(&claim)?; + // Only a claim of the best precedence there may serve, reachable or not, as + // in `closest`; among those, this node's own serves whatever its rank. + let best = (self.claims_of(&claim)).map(|(_, claim)| precedence(&claim.item)).min(); + (!own.unavailable && best == Some(precedence(&own.item))).then_some(Next::Local(source)) } false => self.best.get(&origin).map(|best| Next::Via(best.via)), } @@ -2569,10 +2581,18 @@ mod tests { }; router.recv(4, claims(run(1), 1)).unwrap(); assert_eq!(router.pick(&broadcast), Some(me), "an older claim, however close"); - router.recv(4, claims(Run { epoch: 4, above: 1 }, 2)).unwrap(); + router.recv(4, claims(run(2), 2)).unwrap(); + assert_eq!(router.pick(&broadcast), Some(node(5)), "the same run, closer"); + assert_eq!( + router.step(me, &broadcast), + Some(Next::Local(3)), + "named, its own run serves" + ); + router.recv(4, claims(Run { epoch: 4, above: 1 }, 3)).unwrap(); assert_eq!(router.pick(&broadcast), Some(me), "a broader claim, however new"); - router.recv(4, claims(run(3), 3)).unwrap(); + router.recv(4, claims(run(3), 4)).unwrap(); assert_eq!(router.pick(&broadcast), Some(node(5)), "a newer claim"); + assert_eq!(router.step(me, &broadcast), None, "named, its older run never serves"); // The newer claim's route goes while its claim stays known: the older run never // stands in for it. router.recv(4, route(5, 4, Metric::MAX)).unwrap(); @@ -3097,6 +3117,69 @@ mod tests { assert_eq!(router.deadline(), None); } + /// A scoped neighbour's claim first told unavailable is not there: nothing can serve + /// it, so it is not taken until it is told available. Of the claims neighbours gave at + /// one precedence, an available one is announced and serves, and with every one + /// unavailable a request naming this node fails closed. A newer run first told + /// unavailable leaves an older one standing, as one never told would; taken, it + /// outranks the older run, unavailable or not. + #[test] + fn a_claim_first_told_unavailable_is_not_there() { + use super::super::Kind; + let mut router = Router::::new(node(1), debounce(HOP)); + for peer in [2, 3, 4] { + open(&mut router, peer, Role::Scoped(vec![p(ROOT)])); + } + let broadcast = (p("live"), 7); + let told = |epoch, unavailable| { + let Change::Live(claim) = claim(run(epoch), 0) else { + unreachable!() + }; + Msg::Announce { + changes: vec![(broadcast.clone(), Change::Live(Claim { unavailable, ..claim }))], + } + }; + let me = router.me(); + let own = |router: &Router| { + (router.own().1.get(&broadcast)).map(|claim| (claim.item.epoch, claim.unavailable)) + }; + + router.recv(2, told(1, true)).unwrap(); + assert_eq!(own(&router), None, "first told unavailable"); + assert_eq!(router.pick(&broadcast), None); + assert!(router.events().is_empty()); + router.recv(2, told(1, false)).unwrap(); + assert_eq!(own(&router), Some((1, false)), "then available"); + assert_eq!(router.step(me, &broadcast), Some(Next::Local(2))); + + // Two neighbours give the run; the one still available serves. + router.recv(3, told(1, false)).unwrap(); + router.recv(2, told(1, true)).unwrap(); + assert_eq!(own(&router), Some((1, false))); + assert_eq!(router.step(me, &broadcast), Some(Next::Local(3))); + router.recv(3, told(1, true)).unwrap(); + assert_eq!(own(&router), Some((1, true)), "every one unavailable"); + assert_eq!(router.step(me, &broadcast), None, "fails closed"); + assert_eq!(router.pick(&broadcast), None, "fails closed"); + router.recv(2, told(1, false)).unwrap(); + assert_eq!(router.step(me, &broadcast), Some(Next::Local(2))); + assert_eq!( + (router.events().into_iter().map(|event| event.kind)).collect::>(), + [Kind::Start, Kind::Unavailable, Kind::Available] + ); + + // A newer run first told unavailable is not there either. + router.recv(4, told(2, true)).unwrap(); + assert_eq!(own(&router), Some((1, false)), "the older run stands"); + assert_eq!(router.step(me, &broadcast), Some(Next::Local(2))); + router.recv(4, told(2, false)).unwrap(); + assert_eq!(own(&router), Some((2, false)), "taken, the newer run outranks"); + assert_eq!(router.step(me, &broadcast), Some(Next::Local(4))); + router.recv(4, told(2, true)).unwrap(); + assert_eq!(own(&router), Some((2, true)), "unavailable or not"); + assert_eq!(router.step(me, &broadcast), None); + } + /// An endpoint asking for a prefix under a broader claim is shown that claim at the /// prefix, and an edit swaps what it is shown in place. #[test] diff --git a/rs/moq-net/src/route/tests/multicast.rs b/rs/moq-net/src/route/tests/multicast.rs index 9adda844dfa..7153518379c 100644 --- a/rs/moq-net/src/route/tests/multicast.rs +++ b/rs/moq-net/src/route/tests/multicast.rs @@ -1060,14 +1060,17 @@ mod watch { pub const VIEWER: Node = 6; pub const CLAIM: &str = "live"; - pub fn mesh() -> RelaySim { + /// The mesh, settled, with relay 2's grant open or still to open. + pub fn mesh(granted: bool) -> RelaySim { let mut sim = RelaySim::new(7, relay::debounce(HOP)); sim.client(PUBLISHER); sim.publish(PUBLISHER, 100); sim.client(VIEWER); sim.link(HOME, RELAY, 1, Tie::Peer); sim.link(NEXT, RELAY, 1, Tie::Peer); - sim.link(RELAY, SCOPED, 1, Tie::Scoped); + if granted { + sim.link(RELAY, SCOPED, 1, Tie::Scoped); + } sim.link(SCOPED, FAR, 1, Tie::Peer); sim.link(PUBLISHER, HOME, 1, Tie::Peer); sim.link(VIEWER, RELAY, 1, Tie::Peer); @@ -1105,6 +1108,17 @@ mod watch { .map(|seen| seen.kind) .collect() } + + /// Deliver every message ready now, the first half of a round. Whether any was. + pub fn deliver(sim: &mut RelaySim) -> bool { + let ready = sim.ready(); + for &((from, to), count) in &ready { + for _ in 0..count { + sim.deliver(from, to); + } + } + !ready.is_empty() + } } /// An endpoint or scoped neighbour is told what the relay's own reader shows, never more: @@ -1118,7 +1132,7 @@ mod watch { fn a_viewer_is_told_what_the_relay_reads() { use watch::*; let kinds = |said: Vec<(Instant, Said)>| said.into_iter().map(|(_, said)| said).collect::>(); - let mut sim = mesh(); + let mut sim = mesh(true); for to in [VIEWER, SCOPED] { assert_eq!(kinds(said(&sim, to, 0)), [Said::Live], "{to} before"); } @@ -1148,17 +1162,12 @@ fn a_viewer_is_told_what_the_relay_reads() { // As `quiesce`, noting when the mesh beyond the grant lets the claim go. let mut far = None; for _ in 0..200 { - let ready = sim.ready(); - for &((from, to), count) in &ready { - for _ in 0..count { - sim.deliver(from, to); - } - } + let delivered = deliver(&mut sim); let waiting = sim.tick(); if far.is_none() && read(&sim, FAR, events).contains(&Kind::Offline) { far = Some(sim.now()); } - if ready.is_empty() && !waiting && sim.ready().is_empty() { + if !delivered && !waiting && sim.ready().is_empty() { break; } } @@ -1187,7 +1196,7 @@ fn a_viewer_is_told_what_the_relay_reads() { ); // A publisher that ends its claim is withdrawn at once, as ended. - let mut sim = mesh(); + let mut sim = mesh(true); let (told, events) = (sim.told.len(), sim.events.len()); sim.claim(PUBLISHER, CLAIM, false); quiesce(&mut sim); @@ -1197,6 +1206,93 @@ fn a_viewer_is_told_what_the_relay_reads() { assert_eq!(read(&sim, FAR, events), [Kind::End], "beyond the grant"); } +/// A gateway whose first session to the mesh opens while the claim it is told is +/// unavailable takes none: nothing can serve it, so the gateway's reader shows nothing, +/// it announces nothing, and no route starts toward it. Its mesh hears of the claim one +/// message after the gateway is told it is live again, and never if it is lost. +#[test] +fn a_gateway_takes_no_claim_it_first_hears_unavailable() { + use watch::*; + let kinds = |said: Vec<(Instant, Said)>| said.into_iter().map(|(_, said)| said).collect::>(); + for returns in [true, false] { + let mut sim = mesh(false); + let (told, events, left) = (sim.told.len(), sim.events.len(), sim.now()); + sim.close(PUBLISHER, HOME); + while read(&sim, RELAY, events).is_empty() { + deliver(&mut sim); + sim.tick(); + } + assert_eq!(read(&sim, RELAY, events), [Kind::Unavailable]); + let route_msgs = sim.stats.route_msgs; + sim.link(RELAY, SCOPED, 1, Tie::Scoped); + for _ in 0..3 { + deliver(&mut sim); + sim.tick(); + } + assert!( + sim.now() < left.saturating_add(relay::debounce(HOP)), + "under the debounce" + ); + assert_eq!( + kinds(said(&sim, SCOPED, told)), + [Said::Unavailable], + "returns: {returns}" + ); + let gateway = sim.id(SCOPED); + let unseen = |sim: &RelaySim| { + assert!( + sim.router(SCOPED).expect("a relay").own().1.is_empty(), + "the gateway takes nothing" + ); + assert!(read(sim, SCOPED, events).is_empty(), "the gateway's reader"); + assert!(read(sim, FAR, events).is_empty(), "its mesh"); + assert!( + !sim.router(FAR).expect("a relay").routes().contains(&gateway), + "no route to it" + ); + assert_eq!(sim.stats.route_msgs, route_msgs, "no route message"); + assert_eq!(sim.chase(FAR, 100, &p(CLAIM)), None); + }; + unseen(&sim); + + if !returns { + quiesce(&mut sim); + assert_eq!(kinds(said(&sim, SCOPED, told)), [Said::Unavailable, Said::Lost]); + unseen(&sim); + continue; + } + sim.link(PUBLISHER, NEXT, 1, Tie::Peer); + let mut heard = None; + for _ in 0..200 { + let delivered = deliver(&mut sim); + if heard.is_none() && !read(&sim, FAR, events).is_empty() { + heard = Some(sim.now()); + } + let waiting = sim.tick(); + if !delivered && !waiting && sim.ready().is_empty() { + break; + } + } + let said = said(&sim, SCOPED, told); + assert_eq!(kinds(said.clone()), [Said::Unavailable, Said::Live]); + assert_eq!(read(&sim, SCOPED, events), [Kind::Start]); + assert_eq!(read(&sim, FAR, events), [Kind::Start], "its mesh"); + assert_eq!( + heard, + Some(said[1].0.saturating_add(HOP)), + "one message after the gateway" + ); + assert_eq!( + sim.chase(FAR, 100, &p(CLAIM)), + Some((vec![SCOPED, RELAY, NEXT, PUBLISHER], true)) + ); + assert!( + sim.router(FAR).expect("a relay").routes().contains(&gateway), + "a route to it" + ); + } +} + /// A relay that restarts is a new node, but its neighbours file the session floor per /// pair of relays with the restart in the session's order, so a restart replaces the /// entry instead of adding one. From 5099172937527749aad6600c0f5e30f0cbc3b402 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 11:59:23 -0800 Subject: [PATCH 15/17] test(moq-net): a request naming a relay stays closed once a newer run's route fails Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/relay.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index adfca87df29..c1158475ed7 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -2598,6 +2598,7 @@ mod tests { router.recv(4, route(5, 4, Metric::MAX)).unwrap(); assert_eq!(router.best(node(5)), None); assert_eq!(router.pick(&broadcast), None); + assert_eq!(router.step(me, &broadcast), None, "named, still closed"); let shown = &router.shown()[&(7, p("live"))]; assert_eq!((shown.item, shown.available), (run(3), false)); } From 7a84a688f9a361c97180986c69b6d20176774f86 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 12:15:05 -0800 Subject: [PATCH 16/17] fix(moq-net): route::relay::Router judges a claim first told unavailable per run A neighbour that gave one run of a claim and then tells a newer run unavailable no longer gives the older run, and the newer one is first seen unavailable: the older contribution is dropped as lost and the newer run is not taken. Taking it, as the per-claim rule did, let an unreachable newer run fence an older one another neighbour still serves. Co-Authored-By: Claude Opus 5.5 --- quest/m2/multicast/switch.md | 22 +++++++++------- rs/moq-net/src/route/relay.rs | 48 +++++++++++++++++++++++++++-------- 2 files changed, 50 insertions(+), 20 deletions(-) diff --git a/quest/m2/multicast/switch.md b/quest/m2/multicast/switch.md index fc37dedbf5e..1dc958315a5 100644 --- a/quest/m2/multicast/switch.md +++ b/quest/m2/multicast/switch.md @@ -116,15 +116,19 @@ for the wire: near one does, not a debounce later. The route exchange between routing nodes keeps no timer (decided by the coordinator on 2026-11-16, pending the maintainer, in [moq-dev/moq#8901](https://github.com/moq-dev/moq/pull/8901)). -- The unavailable bit counts only as a change to a claim the receiver already - holds from that sender. A sender still tells a new asker what its reader - shows, the bit included, but a receiver never takes a claim it is first told - unavailable, which nothing can serve: that claim is not there until it is - told available, and then is taken as usual. A gateway whose session opens - while a claim is unavailable therefore announces nothing and starts no - route, and a claim that comes back unavailable across a second grant starts - none either (decided by the coordinator on 2026-11-16, pending the - maintainer, in [moq-dev/moq#8901](https://github.com/moq-dev/moq/pull/8901)). +- The unavailable bit counts only as a change to a run the receiver already + holds from that sender, a run being the claim's distance and epoch. A sender + still tells a new asker what its reader shows, the bit included, but a + receiver never takes a run it is first told unavailable, which nothing can + serve: that run is not there until it is told available, and then is taken + as usual. A sender that moves to such a run no longer gives the one it gave, + which the receiver drops as lost. A gateway whose session opens while a + claim is unavailable therefore announces nothing and starts no route, a + claim that comes back unavailable across a second grant starts none either, + and an unreachable newer run never fences an older one another neighbour + still serves (decided by the coordinator on 2026-11-16, pending the + maintainer, in [moq-dev/moq#8901](https://github.com/moq-dev/moq/pull/8901); + per run rather than per claim by the implementer, on review). - A message names each claim once, and a receiver refuses one that names a claim twice, so no order within a message decides what it says (decided by the implementer on 2026-11-16, pending the maintainer, in diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index c1158475ed7..126f3db8a7d 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -43,8 +43,8 @@ //! closest claim while one is reachable, and the claim marked unavailable, never //! withdrawn, while none is and while a lost claim waits out the debounce. A scoped //! neighbour that took the claim announces it again unavailable, so its domain sees the -//! claim stay; one first told it unavailable, which nothing can serve, takes nothing until -//! it is told available. A request +//! claim stay; a run it is first told unavailable, which nothing can serve, it takes only +//! once told it available. A request //! follows the claim at the most specific path covering it that the reader shows, at its //! claimant too, and fails closed while that one is unavailable or lost. Among the claims //! of one Instance at one path, only the one shown closest to its own path, at the newest @@ -1639,11 +1639,18 @@ impl Router { { Change::Lost } - // Unavailable is only ever news of a claim this neighbour gave before: one - // first seen unavailable, which nothing can serve, is not there until it is - // available. - Change::Live(claim) if claim.unavailable && !self.peers[&from].given.contains_key(&broadcast) => { - continue; + // Unavailable is only ever news of a run this neighbour gave before: one first + // seen unavailable, which nothing can serve, is not there until it is + // available. A neighbour that moved to it no longer gives the run it gave. + Change::Live(claim) + if claim.unavailable + && !(self.peers[&from].given.get(&broadcast)) + .is_some_and(|had| precedence(&had.item) == precedence(&claim.item)) => + { + match self.peers[&from].given.contains_key(&broadcast) { + true => Change::Lost, + false => continue, + } } Change::Live(mut claim) => { // Saturating, as `model::origin`'s claim cost does, so a claim's cost @@ -3122,8 +3129,9 @@ mod tests { /// it, so it is not taken until it is told available. Of the claims neighbours gave at /// one precedence, an available one is announced and serves, and with every one /// unavailable a request naming this node fails closed. A newer run first told - /// unavailable leaves an older one standing, as one never told would; taken, it - /// outranks the older run, unavailable or not. + /// unavailable leaves an older one standing, as one never told would, whether a new + /// neighbour tells it or one that gave the older run, which no longer gives that run. + /// Taken, the newer run outranks the older one, unavailable or not. #[test] fn a_claim_first_told_unavailable_is_not_there() { use super::super::Kind; @@ -3169,16 +3177,34 @@ mod tests { [Kind::Start, Kind::Unavailable, Kind::Available] ); - // A newer run first told unavailable is not there either. + // A newer run first told unavailable is not there either, from a neighbour that + // gave the older run, which it no longer gives, or from a new one. + router.recv(3, told(1, false)).unwrap(); + router.recv(2, told(2, true)).unwrap(); + assert_eq!(own(&router), Some((1, false)), "the older run stands"); + assert_eq!( + router.step(me, &broadcast), + Some(Next::Local(3)), + "where it is still given" + ); router.recv(4, told(2, true)).unwrap(); assert_eq!(own(&router), Some((1, false)), "the older run stands"); - assert_eq!(router.step(me, &broadcast), Some(Next::Local(2))); + assert_eq!(router.step(me, &broadcast), Some(Next::Local(3))); router.recv(4, told(2, false)).unwrap(); assert_eq!(own(&router), Some((2, false)), "taken, the newer run outranks"); assert_eq!(router.step(me, &broadcast), Some(Next::Local(4))); router.recv(4, told(2, true)).unwrap(); assert_eq!(own(&router), Some((2, true)), "unavailable or not"); assert_eq!(router.step(me, &broadcast), None); + // The last neighbour giving the older run moves to the newer one unavailable: the + // older run is lost, and the newer one, which neighbour 4 gave, stays. + router.recv(3, told(2, true)).unwrap(); + assert_eq!(own(&router), Some((2, true))); + let lost = Msg::Announce { + changes: vec![(broadcast.clone(), Change::Lost)], + }; + router.recv(4, lost).unwrap(); + assert_eq!(own(&router), None, "no neighbour gives either run"); } /// An endpoint asking for a prefix under a broader claim is shown that claim at the From ecca03b08caf693dbf58f9fb222cd4ae8ab20723 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 12:19:32 -0800 Subject: [PATCH 17/17] test(moq-net): bound the gateway fixture's wait for the relay to read the claim unavailable Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/tests/multicast.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/route/tests/multicast.rs b/rs/moq-net/src/route/tests/multicast.rs index 7153518379c..63cfc527054 100644 --- a/rs/moq-net/src/route/tests/multicast.rs +++ b/rs/moq-net/src/route/tests/multicast.rs @@ -1218,7 +1218,11 @@ fn a_gateway_takes_no_claim_it_first_hears_unavailable() { let mut sim = mesh(false); let (told, events, left) = (sim.told.len(), sim.events.len(), sim.now()); sim.close(PUBLISHER, HOME); - while read(&sim, RELAY, events).is_empty() { + // Bounded as `quiesce` is, so a relay that never reads the claim unavailable fails. + for _ in 0..200 { + if !read(&sim, RELAY, events).is_empty() { + break; + } deliver(&mut sim); sim.tick(); }