From 8f58042b63e08fe80ae2b6326f74f5ccbd2264f6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 14:11:49 -0800 Subject: [PATCH 01/13] chore(quest): claim quest/m2/multicast/bakeoff.md Quest-Claim: 7589b20c110a423c From cab96b4c42bdc364c1263ae8562438e44df89dfa Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 14:27:34 -0800 Subject: [PATCH 02/13] test(moq-net): prototype the multicast route rule beside today's router Co-Authored-By: Claude Fable 5.2 --- rs/moq-net/src/route/tests/bakeoff.rs | 35 + rs/moq-net/src/route/tests/compare.rs | 315 +++++- rs/moq-net/src/route/tests/mod.rs | 2 + rs/moq-net/src/route/tests/multi.rs | 1315 +++++++++++++++++++++++++ 4 files changed, 1635 insertions(+), 32 deletions(-) create mode 100644 rs/moq-net/src/route/tests/bakeoff.rs create mode 100644 rs/moq-net/src/route/tests/multi.rs diff --git a/rs/moq-net/src/route/tests/bakeoff.rs b/rs/moq-net/src/route/tests/bakeoff.rs new file mode 100644 index 00000000000..e94afffdfdb --- /dev/null +++ b/rs/moq-net/src/route/tests/bakeoff.rs @@ -0,0 +1,35 @@ +//! The bake-off: the multicast epic's rule ([`multi::Router`]) beside today's +//! [`path::Router`] on the same meshes and failure schedules. + +use super::compare::*; +use super::*; + +/// The arms measured, in the report's order. +const ARMS: [Arm; 3] = [Arm::TODAY, Arm::PLAIN, Arm::SUSPECT]; + +/// The prototype's arms. +const MULTI: [Arm; 2] = [Arm::PLAIN, Arm::SUSPECT]; + +/// The prototype carries every change of the path rule's comparison on its shapes, +/// and is right whenever it settles. +#[test] +fn the_prototype_converges_on_every_shape() { + let mut rng = XorShift(0x6d6f_7100); + let mut meshes = vec![Mesh::ring(6), Mesh::clique(6, 1), Mesh::clique(4, 3)]; + meshes.extend((0..8).map(|_| Mesh::sparse(6, &mut rng))); + for mesh in &meshes { + for arm in MULTI { + run(arm, mesh); + } + } +} + +/// Under churn the prototype settles and is right once it does. +#[test] +fn the_prototype_settles_after_churn() { + for arm in MULTI { + for two_homes in [false, true] { + churn(arm, 0..60, two_homes); + } + } +} diff --git a/rs/moq-net/src/route/tests/compare.rs b/rs/moq-net/src/route/tests/compare.rs index 85daa23d486..20d5a77be7a 100644 --- a/rs/moq-net/src/route/tests/compare.rs +++ b/rs/moq-net/src/route/tests/compare.rs @@ -11,8 +11,72 @@ use std::collections::BTreeSet; +use super::multi::{Broadcast, MultiSim, Suspect, Tie}; use super::*; +/// A router under comparison: today's rule, or the multicast epic's prototype. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Arm { + Path(Rule), + Multi(Suspect), +} + +impl Arm { + /// Today's rule as it ships. + pub const TODAY: Self = Self::Path(Rule::PATH); + /// The epic's rule as written. + pub const PLAIN: Self = Self::Multi(Suspect::Never); + /// The epic's rule ranking routes through the far end of a closed session as draining. + pub const SUSPECT: Self = Self::Multi(Suspect::FarEnd); + + pub fn as_str(self) -> &'static str { + match self { + Self::Path(rule) => rule.as_str(), + Self::Multi(Suspect::Never) => "multi", + Self::Multi(Suspect::FarEnd) => "multi+suspect", + } + } + + fn net(self, nodes: usize) -> Box { + match self { + Self::Path(rule) => Box::new(Sim::with(rule, nodes)), + Self::Multi(suspect) => Box::new(MultiSim::new(suspect, nodes)), + } + } +} + +impl From for Arm { + fn from(rule: Rule) -> Self { + Self::Path(rule) + } +} + +/// What the routers retain, summed over every node. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) struct Held { + /// Routes heard and used. + pub routes: usize, + /// Claims, in every copy a node keeps. + pub claims: usize, + /// What waits on another node: dead hops, doubts, and suspected nodes. + pub pending: usize, +} + +/// What readers were shown that the publishers never did. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) struct Faults { + /// Ends shown while the publisher still claimed. + pub false_ends: usize, + /// Starts of a life the same reader had seen end. + pub revivals: usize, +} + +/// The scope every scoped link in a comparison grants. +const GRANT: Grant = Grant { + subject: "mesh", + scope: "", +}; + /// The production rule, checked in ordinary CI. const RULES: [Rule; 1] = [Rule::PATH]; @@ -58,7 +122,13 @@ impl std::ops::Sub for Sent { pub(super) trait Net { /// `node` passes on no route it learned, like a client. fn client(&mut self, node: Node); - fn join(&mut self, a: Node, b: Node, cost: Metric); + fn join(&mut self, a: Node, b: Node, cost: Metric) { + self.link(a, b, cost, Tie::Peer); + } + /// Configure a link and open it. + fn link(&mut self, a: Node, b: Node, cost: Metric, tie: Tie); + /// `node` sends GOAWAY on every session. + fn goaway(&mut self, node: Node); fn open(&mut self, a: Node, b: Node); fn close(&mut self, a: Node, b: Node); /// The process at `node` is gone: nothing more passes between it and `peer`, @@ -77,10 +147,16 @@ pub(super) trait Net { fn tick(&mut self) -> bool { false } - /// The neighbour `node` routes `id` through: `Some(None)` at the publisher. - fn next(&self, node: Node, id: Id) -> Option>; + /// The nodes a request from `from` for publisher `id` visits, and whether it + /// ended at the publisher: `None` with no route. A chain longer than the network + /// loops. + fn chase(&self, from: Node, id: Id) -> Option<(Vec, bool)>; fn showing(&self, node: Node, id: Id, claim: &str) -> Option; fn sent(&self) -> Sent; + fn held(&self) -> Held; + fn faults(&self) -> Faults; + /// Routes, claims, and streams visited. + fn work(&self) -> usize; } fn tally(kinds: impl Iterator) -> (usize, usize, usize) { @@ -101,8 +177,17 @@ impl Net for Sim { self.leaf(node); } - fn join(&mut self, a: Node, b: Node, cost: Metric) { - Sim::join(self, a, b, cost); + fn link(&mut self, a: Node, b: Node, cost: Metric, tie: Tie) { + let side = SimLink::cost(cost); + match tie { + Tie::Peer => Sim::link(self, a, b, side, side), + Tie::Up => Sim::link(self, a, b, side.upstream(), side), + Tie::Scoped => Sim::link(self, a, b, side.grant(GRANT), side.grant(GRANT)), + } + } + + fn goaway(&mut self, node: Node) { + Sim::goaway(self, node); } fn open(&mut self, a: Node, b: Node) { @@ -153,12 +238,65 @@ impl Net for Sim { self.deadline().is_some() } - fn next(&self, node: Node, id: Id) -> Option> { - self.toward(node, id) + fn chase(&self, from: Node, id: Id) -> Option<(Vec, bool)> { + let mut hops = Vec::new(); + if self.grants.is_empty() { + let mut at = from; + self.toward(at, id)?; + while hops.len() <= self.nodes.len() { + match self.toward(at, id) { + None => return Some((hops, false)), + Some(None) => return Some((hops, true)), + Some(Some(next)) => { + hops.push(next); + at = next; + } + } + } + return Some((hops, false)); + } + // Past a scoped link the publisher goes by its qualified identity. + let claim = p(&Mesh::claim(0)); + let key = [id, qualified(GRANT.subject, id)] + .into_iter() + .find(|key| self.path(from).forward(*key, &claim, None).is_some())?; + let (mut at, mut key, mut prev) = (from, key, None); + while hops.len() <= self.nodes.len() { + match self.path(at).forward(key, &claim, prev.take()) { + None => return Some((hops, false)), + Some(path::Forward::Here) => return Some((hops, true)), + Some(path::Forward::Next { next, named, prefix }) => { + prev = Some((at, prefix)); + hops.push(next); + (at, key) = (next, named); + } + } + } + Some((hops, false)) } fn showing(&self, node: Node, id: Id, claim: &str) -> Option { - self.shows(node, id, claim) + (self.shows(node, id, claim)).or_else(|| self.shows(node, qualified(GRANT.subject, id), claim)) + } + + fn held(&self) -> Held { + let kept = self.kept(); + Held { + routes: kept.routes, + claims: kept.items + kept.heard + kept.sent, + pending: kept.dead + kept.doubt + kept.draining, + } + } + + fn faults(&self) -> Faults { + Faults { + false_ends: false_ends(&self.events), + revivals: revivals(&self.events), + } + } + + fn work(&self) -> usize { + Sim::work(self) } fn sent(&self) -> Sent { @@ -174,11 +312,105 @@ impl Net for Sim { } } +impl Net for MultiSim { + fn client(&mut self, node: Node) { + MultiSim::client(self, node); + } + + fn link(&mut self, a: Node, b: Node, cost: Metric, tie: Tie) { + MultiSim::link(self, a, b, cost, tie); + } + + fn goaway(&mut self, node: Node) { + MultiSim::goaway(self, node); + } + + fn open(&mut self, a: Node, b: Node) { + MultiSim::open(self, a, b); + } + + fn close(&mut self, a: Node, b: Node) { + MultiSim::close(self, a, b); + } + + fn mute(&mut self, node: Node, peer: Node) { + self.hold(node, peer); + self.hold(peer, node); + } + + fn sever(&mut self, node: Node, peer: Node) { + MultiSim::sever(self, node, peer); + } + + fn reset(&mut self, node: Node) { + MultiSim::reset(self, node); + } + + fn publish(&mut self, node: Node, id: Id) { + MultiSim::publish(self, node, id); + } + + fn claim(&mut self, node: Node, claim: &str, live: bool) { + MultiSim::claim(self, node, claim, live); + } + + fn ready(&self) -> Vec<(Chan, usize)> { + MultiSim::ready(self) + } + + fn step(&mut self, (from, to): Chan) { + self.deliver(from, to); + } + + fn chase(&self, from: Node, id: Id) -> Option<(Vec, bool)> { + MultiSim::chase(self, from, &(id, p(&Mesh::claim(0)))) + } + + fn showing(&self, node: Node, id: Id, claim: &str) -> Option { + let broadcast: Broadcast = (id, p(claim)); + self.router(node)?.shown(&broadcast) + } + + fn sent(&self) -> Sent { + let (offline, unavailable, starts) = tally(self.events.iter().map(|event| event.kind)); + Sent { + msgs: self.stats.route_msgs, + items: self.stats.route_items, + claims: self.stats.claim_items, + offline, + unavailable, + starts, + } + } + + fn held(&self) -> Held { + let kept = self.kept(); + Held { + routes: kept.routes, + claims: kept.items + kept.heard + kept.foreign, + pending: kept.pending, + } + } + + fn faults(&self) -> Faults { + Faults { + false_ends: multi::false_ends(&self.events), + revivals: multi::revivals(&self.events), + } + } + + fn work(&self) -> usize { + MultiSim::work(self) + } +} + /// A mesh of relays, each publisher attached to one of them. #[derive(Clone, Debug)] pub(super) struct Mesh { pub relays: usize, pub links: Vec<(Node, Node, Metric)>, + /// The links that are not between peers, keyed as `links` orders their ends. + pub ties: BTreeMap<(Node, Node), Tie>, /// The relay each publisher attaches to. pub homes: Vec, /// Claims each publisher announces. @@ -188,7 +420,7 @@ pub(super) struct Mesh { impl Mesh { /// A deterministic cheapest-path tree, shared by every rule when choosing /// which link or transit relay to fail. Ties use the next relay's number. - fn tree(&self) -> Vec> { + pub fn tree(&self) -> Vec> { let home = self.homes[0]; let mut best = vec![None; self.relays]; best[home] = Some((0, home)); @@ -222,6 +454,7 @@ impl Mesh { Self { relays, links, + ties: BTreeMap::new(), homes: vec![0], claims: 4, } @@ -235,6 +468,7 @@ impl Mesh { Self { relays, links, + ties: BTreeMap::new(), homes: vec![0; publishers], claims: 4, } @@ -258,6 +492,7 @@ impl Mesh { Self { relays, links, + ties: BTreeMap::new(), homes, claims: 4, } @@ -267,7 +502,7 @@ impl Mesh { (self.relays + index, 100 + index as Id) } - fn claim(index: usize) -> String { + pub fn claim(index: usize) -> String { format!("live/{index}") } } @@ -318,10 +553,10 @@ fn pair(a: Node, b: Node) -> (Node, Node) { impl Lab { /// Build `mesh` on `rule`'s router, with every publisher announcing, and let it settle. - pub fn new(rule: Rule, mesh: &Mesh) -> Self { + pub fn new(arm: impl Into, mesh: &Mesh) -> Self { let nodes = mesh.relays + mesh.homes.len(); let mut lab = Self { - net: Box::new(Sim::with(rule, nodes)), + net: arm.into().net(nodes), mesh: mesh.clone(), up: BTreeSet::new(), linked: BTreeSet::new(), @@ -330,7 +565,8 @@ impl Lab { cost: Cost::default(), }; for &(a, b, cost) in &mesh.links { - lab.net.join(a, b, cost); + let tie = mesh.ties.get(&(a, b)).copied().unwrap_or(Tie::Peer); + lab.net.link(a, b, cost, tie); lab.up.insert(pair(a, b)); } lab.settle(); @@ -341,7 +577,7 @@ impl Lab { } /// Publisher `index` joins its relay for the first time and announces its claims. - fn arrive(&mut self, index: usize) { + pub fn arrive(&mut self, index: usize) { let (node, id) = self.mesh.publisher(index); let home = self.mesh.homes[index]; self.net.client(node); @@ -375,41 +611,53 @@ impl Lab { fn reach_over(&self, up: &BTreeSet<(Node, Node)>, index: usize) -> BTreeSet { let (node, _) = self.mesh.publisher(index); - let mut seen = BTreeSet::from([node]); - let mut queue = vec![node]; - while let Some(at) = queue.pop() { + // A route that crossed a marked link never crosses another, so a relay is + // reached with or without one behind it. + let mut seen = BTreeSet::from([(node, false)]); + let mut queue = vec![(node, false)]; + while let Some((at, marked)) = queue.pop() { for &(a, b) in up { let peer = match at { _ if at == a => b, _ if at == b => a, _ => continue, }; + // `peer` learns it from `at`, and marks it if `at` is its upstream. A + // scoped link starts the claim afresh. + let tie = |from: Node, to: Node| self.mesh.ties.get(&(from, to)).copied(); + if marked && tie(at, peer) == Some(Tie::Up) { + continue; + } + let scoped = tie(at, peer).or(tie(peer, at)) == Some(Tie::Scoped); + let marked = !scoped && (marked || tie(peer, at) == Some(Tie::Up)); // A publisher forwards nothing, and neither does a relay that died, // whether or not its neighbours have noticed. - if peer < self.mesh.relays && !self.down.contains(&peer) && seen.insert(peer) { - queue.push(peer); + if peer < self.mesh.relays && !self.down.contains(&peer) && seen.insert((peer, marked)) { + queue.push((peer, marked)); } } } - seen.remove(&node); - seen + (seen.into_iter().map(|(relay, _)| relay)) + .filter(|&relay| relay != node) + .collect() } fn walk(&self, from: Node, index: usize) -> Option { let (origin, id) = self.mesh.publisher(index); let nodes = self.mesh.relays + self.mesh.homes.len(); - self.net.next(from, id)?; + let (hops, here) = self.net.chase(from, id)?; let mut at = from; - for _ in 0..=nodes { - match self.net.next(at, id) { - None => return Some(Walk::Broken), - Some(None) if at == origin => return Some(Walk::Reached), - Some(None) => return Some(Walk::Broken), - Some(Some(next)) if !self.is_up(at, next) || self.down.contains(&next) => return Some(Walk::Broken), - Some(Some(next)) => at = next, + for &next in &hops { + if !self.is_up(at, next) || self.down.contains(&next) { + return Some(Walk::Broken); } + at = next; } - Some(Walk::Loop) + Some(match () { + _ if hops.len() > nodes => Walk::Loop, + _ if here && at == origin => Walk::Reached, + _ => Walk::Broken, + }) } /// Whether any relay's next hops lead round a loop. @@ -677,7 +925,8 @@ impl Change { /// Run every change on `mesh` under `rule`, each from a settled mesh, checking the /// mesh is right whenever it settles. -pub(super) fn run(rule: Rule, mesh: &Mesh) -> Vec<(Change, Cost)> { +pub(super) fn run(arm: impl Into, mesh: &Mesh) -> Vec<(Change, Cost)> { + let rule: Arm = arm.into(); let mut out = Vec::new(); let mut lab = Lab::new(rule, mesh); let tree = mesh.tree(); @@ -774,7 +1023,8 @@ pub(super) struct Churn { /// sees the same meshes and the same changes at the same steps. Once the churn /// stops the mesh must settle and be right. With `two_homes` the publisher also /// joins a second relay, and its flaps pick either link. -pub(super) fn churn(rule: Rule, cases: std::ops::Range, two_homes: bool) -> Churn { +pub(super) fn churn(arm: impl Into, cases: std::ops::Range, two_homes: bool) -> Churn { + let rule: Arm = arm.into(); let mut total = Churn::default(); for case in cases { let mut script = XorShift(0x0bad_c0de + case as u64 * 0x9e37_79b9); @@ -791,6 +1041,7 @@ pub(super) fn churn(rule: Rule, cases: std::ops::Range, two_homes: bool) let mesh = Mesh { relays, links, + ties: BTreeMap::new(), homes: vec![script.next(relays as u64) as usize], claims: 2, }; diff --git a/rs/moq-net/src/route/tests/mod.rs b/rs/moq-net/src/route/tests/mod.rs index d1258d6016a..35d04d63a20 100644 --- a/rs/moq-net/src/route/tests/mod.rs +++ b/rs/moq-net/src/route/tests/mod.rs @@ -8,6 +8,7 @@ //! means a wildcard; every lite-07 claim already covers the paths below it, so //! the slash is dropped. +mod bakeoff; mod cache; mod compare; mod cost; @@ -15,6 +16,7 @@ mod flap; mod forward; mod handoff; mod interest; +mod multi; mod residual; mod routing; mod scope; diff --git a/rs/moq-net/src/route/tests/multi.rs b/rs/moq-net/src/route/tests/multi.rs new file mode 100644 index 00000000000..c2f60d4f842 --- /dev/null +++ b/rs/moq-net/src/route/tests/multi.rs @@ -0,0 +1,1315 @@ +//! The multicast epic's route rule as a test-only prototype, measured against +//! [`path::Router`] by `bakeoff.rs`. Nothing here ships. +//! +//! A route leads to a forwarding node, never to a publisher: an endpoint's claims +//! belong to the relay it connects to, which announces them under its own Node ID +//! and revision. A route carries its path, drain, priority, and cost, and a node +//! never uses one whose path names it. A closed session is named once per +//! neighbour in a fixed-size [`Msg::Dead`], and a route through that hop is unusable +//! until the neighbour that offered it says what became of it. There is no hold, no +//! doubt, and no timer: [`Router`] has no clock. +//! +//! Across a scoped grant only claims travel: the receiver announces each again +//! under its own Node ID with the chain of nodes that did so, and refuses one whose +//! chain names it. + +use std::{ + cmp::Reverse, + collections::{BTreeMap, BTreeSet}, +}; + +use super::*; + +/// The Instance a claim serves: the publisher's identity in the fixtures. +pub(super) type Instance = Id; + +/// What a reader asks for: an Instance at a path. +pub(super) type Broadcast = (Instance, Path); + +/// What a closed session says about the node at its far end. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Suspect { + /// Nothing: the epic's plain rule. + Never, + /// Every route through the far end ranks as draining until that node's own word + /// about the session arrives, or a newer session to it shows on a path. + FarEnd, +} + +/// One node on a route's path, and the session it learned the route on. The +/// forwarding node a route leads to names itself with session 0. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(super) struct Hop { + pub node: Id, + pub session: u64, +} + +/// What a route says. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct Offer { + /// Every node it crossed, the forwarding node it leads to first and the sender last. + pub path: Vec, + pub draining: bool, + /// The lowest link priority on the path. + pub priority: i8, + /// The summed link costs, each at least 1. + pub cost: Metric, + /// A marked link is on its path, so it is never sent over another. + pub marked: bool, +} + +impl Offer { + fn names(&self, node: Id) -> bool { + self.path.iter().any(|hop| hop.node == node) + } + + /// Lower first. Crossing a link never improves drain or priority and strictly + /// raises the cost. + fn rank(&self) -> (bool, Reverse, Metric) { + (self.draining, Reverse(self.priority), self.cost) + } +} + +/// A claim as a forwarding node announces it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct Claim { + /// From the announcing node to the publisher: 0 for its own endpoint. + pub cost: Metric, + /// The nodes that announced it again across a scoped grant, the first first. + pub chain: Vec, +} + +/// A change to one claim. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) enum Change { + Live(Claim), + /// The publisher withdrew it. + Ended, + /// Its announcer lost it without the publisher saying so, as when a session closes. + Lost, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) enum Msg { + /// A route to forwarding node `origin`, or its end. + Route { origin: Id, offer: Option }, + /// Changes to `origin`'s claims, complete as of its revision `floor`. + Claims { + origin: Id, + changes: Vec<(Broadcast, Change)>, + floor: Rev, + }, + /// The session `hop` names closed. `far` is the node at its other end. + Dead { hop: Hop, far: Id }, + /// Claims from an endpoint, or across a scoped grant: no route comes with them. + Foreign { changes: Vec<(Broadcast, Change)> }, +} + +/// What a neighbour is to this node. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Role { + /// A forwarding node trusted with the whole tree: routes are exchanged. + Routing, + /// A forwarding node across a scoped grant: claims are exchanged, with a chain. + Scoped, + /// A node that never forwards: its claims are this node's own. + Endpoint, +} + +/// One side's configuration of a link. +#[derive(Clone, Copy, Debug)] +pub(super) struct Port { + pub cost: Metric, + pub priority: i8, + /// The one mark: a route learned here is never sent to another marked neighbour. + pub upstream: bool, + pub role: Role, +} + +#[derive(Clone, Debug)] +struct Peer { + id: Id, + port: Port, + session: u64, + /// It sent GOAWAY. + draining: bool, +} + +/// What one neighbour offers for one forwarding node. +#[derive(Clone, Debug)] +struct Heard { + offer: Offer, + floor: Rev, + items: BTreeMap, +} + +/// The route this node uses, as it stands here: the heard offer with the link added. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Best { + via: N, + offer: Offer, +} + +/// A forwarding node's claims as this node knows them, complete as of `floor`. +#[derive(Clone, Debug)] +struct Known { + floor: Rev, + items: BTreeMap, + /// The neighbour whose copy this mirrors, so its next changes apply as they are. + source: Option, +} + +impl Default for Known { + fn default() -> Self { + Self { + floor: 0, + items: BTreeMap::new(), + source: None, + } + } +} + +/// How candidate routes order, lower first: rank, then the next hop in use, the +/// shorter path, and the neighbour. +type Order = (bool, Reverse, Metric, bool, usize, N); + +/// Where a request for a broadcast goes next. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Next { + /// Toward the forwarding node the request names. + Via(N), + /// To the endpoint or scoped neighbour whose claim this node announces. + Local(N), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct Event { + pub broadcast: Broadcast, + pub kind: Kind, +} + +/// What a router retains. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) struct Kept { + /// Routes heard, one per neighbour offering a forwarding node. + pub routes: usize, + /// Hops on those routes' paths. + pub hops: usize, + /// Routes sent, one per neighbour and forwarding node. + pub sent: usize, + /// Claims in the merged view of each forwarding node. + pub items: usize, + /// Claims in each neighbour's copy. + pub heard: usize, + /// Claims sent across scoped grants. + pub foreign: usize, + /// Dead hops and suspected nodes. + pub pending: usize, +} + +#[derive(Clone, Debug)] +pub(super) struct Router { + me: Id, + suspect: Suspect, + sessions: u64, + peers: BTreeMap, + heard: BTreeMap>, + /// Closed sessions some heard route still names, each with its far end. + dead: BTreeMap, + /// Nodes at the far end of a closed session, by the near ends that said so. + suspects: BTreeMap>, + best: BTreeMap>, + sent: BTreeMap>, + known: BTreeMap>, + /// This node's revision of its own claims. + rev: Rev, + /// Claims heard from endpoints and scoped neighbours, by neighbour. + local: BTreeMap>, + /// The neighbour behind each of this node's own claims. + sources: BTreeMap, + /// Claims sent to each scoped neighbour. + foreign: BTreeMap>, + /// The forwarding nodes claiming each broadcast, as far as this node knows. + claimants: BTreeMap>, + shown: BTreeMap, + dirty: BTreeSet, + /// Broadcasts whose last removal was the publisher's own withdrawal. + ended: BTreeSet, + out: Vec<(N, Msg)>, + events: Vec, + /// Routes and claims visited, to compare work. + work: usize, +} + +impl Router { + pub fn new(me: Id, suspect: Suspect) -> Self { + let mut router = Self { + me, + suspect, + sessions: 0, + peers: BTreeMap::new(), + heard: BTreeMap::new(), + dead: BTreeMap::new(), + suspects: BTreeMap::new(), + best: BTreeMap::new(), + sent: BTreeMap::new(), + known: BTreeMap::new(), + rev: 0, + local: BTreeMap::new(), + sources: BTreeMap::new(), + foreign: BTreeMap::new(), + claimants: BTreeMap::new(), + shown: BTreeMap::new(), + dirty: BTreeSet::new(), + ended: BTreeSet::new(), + out: Vec::new(), + events: Vec::new(), + work: 0, + }; + router.known.insert(me, Known::default()); + router + } + + pub fn drain(&mut self) -> Vec<(N, Msg)> { + std::mem::take(&mut self.out) + } + + pub fn events(&mut self) -> Vec { + std::mem::take(&mut self.events) + } + + pub fn work(&self) -> usize { + self.work + } + + pub fn shown(&self, broadcast: &Broadcast) -> Option { + self.shown.get(broadcast).copied() + } + + pub fn connect(&mut self, peer: N, id: Id, port: Port) { + assert!(port.cost > 0, "a zero-cost link"); + self.sessions += 1; + let session = self.sessions; + self.peers.insert( + peer, + Peer { + id, + port, + session, + draining: false, + }, + ); + // A session to a suspected node is its own word that it lives. + if let Some(nears) = self.suspects.get_mut(&id) { + let me = self.me; + nears.retain(|near| near.node != me); + if nears.is_empty() { + self.suspects.remove(&id); + } + self.reselect_all(); + } + match port.role { + Role::Routing => { + let origins: Vec = self.best.keys().copied().chain([self.me]).collect(); + for origin in origins { + self.advertise_to(origin, peer); + } + } + Role::Scoped => { + let broadcasts: Vec = self.shown.keys().cloned().collect(); + for broadcast in broadcasts { + self.export(&broadcast, peer); + } + } + Role::Endpoint => {} + } + self.refresh(); + } + + pub fn disconnect(&mut self, peer: N) { + let Some(gone) = self.peers.remove(&peer) else { + return; + }; + let mut lost = Vec::new(); + for (&origin, offers) in &mut self.heard { + if offers.remove(&peer).is_some() { + lost.push(origin); + } + } + for origin in lost { + self.unheard(origin, peer); + } + self.sent.remove(&peer); + self.foreign.remove(&peer); + if let Some(claims) = self.local.remove(&peer) { + for broadcast in claims.into_keys() { + self.restate(broadcast, Change::Lost); + } + } + match gone.port.role { + Role::Routing => self.closed( + Hop { + node: self.me, + session: gone.session, + }, + gone.id, + None, + ), + Role::Scoped | Role::Endpoint => {} + } + self.refresh(); + } + + /// `peer` sent GOAWAY: every route through it ranks as draining. + pub fn goaway(&mut self, peer: N) { + if let Some(peer) = self.peers.get_mut(&peer) { + peer.draining = true; + } + self.reselect_all(); + self.refresh(); + } + + pub fn recv(&mut self, from: N, msg: Msg) { + let Some(peer) = self.peers.get(&from) else { + return; + }; + let role = peer.port.role; + match msg { + Msg::Route { origin, offer } if role == Role::Routing => self.route(from, origin, offer), + Msg::Claims { origin, changes, floor } if role == Role::Routing => self.claims(from, origin, changes, floor), + Msg::Dead { hop, far } if role == Role::Routing => self.closed(hop, far, Some(from)), + Msg::Foreign { changes } if role != Role::Routing => self.adopt(from, changes), + // A neighbour sends only what its role allows; anything else would close the session. + other => panic!("{other:?} from a {role:?} neighbour"), + } + self.refresh(); + } + + fn route(&mut self, from: N, origin: Id, offer: Option) { + self.work += 1; + if origin == self.me { + return; + } + match offer { + Some(offer) => { + self.vouch(&offer); + let offers = self.heard.entry(origin).or_default(); + match offers.get_mut(&from) { + Some(heard) => heard.offer = offer, + None => { + offers.insert( + from, + Heard { + offer, + floor: 0, + items: BTreeMap::new(), + }, + ); + } + } + } + None => { + if let Some(offers) = self.heard.get_mut(&origin) { + offers.remove(&from); + } + self.unheard(origin, from); + } + } + self.forget(); + self.reselect(origin); + } + + /// A path that crosses a suspected node and then a newer session of the near end + /// that cast the suspicion shows the two met again. + fn vouch(&mut self, offer: &Offer) { + if self.suspects.is_empty() { + return; + } + let mut lifted = false; + for pair in offer.path.windows(2) { + let (far, near) = (pair[0].node, pair[1]); + if let Some(nears) = self.suspects.get_mut(&far) { + let before = nears.len(); + nears.retain(|cast| cast.node != near.node || cast.session >= near.session); + lifted |= nears.len() != before; + if nears.is_empty() { + self.suspects.remove(&far); + } + } + } + if lifted { + self.reselect_all(); + } + } + + /// `from` no longer offers `origin`: its copy of the claims went with the route. + fn unheard(&mut self, origin: Id, from: N) { + if let Some(known) = self.known.get_mut(&origin) + && known.source == Some(from) + { + known.source = None; + } + if self.heard.get(&origin).is_some_and(BTreeMap::is_empty) { + self.heard.remove(&origin); + if let Some(known) = self.known.remove(&origin) { + for broadcast in known.items.into_keys() { + self.unclaim(origin, broadcast); + } + } + } + } + + fn claims(&mut self, from: N, origin: Id, changes: Vec<(Broadcast, Change)>, floor: Rev) { + let Some(heard) = self.heard.get_mut(&origin).and_then(|offers| offers.get_mut(&from)) else { + return; + }; + self.work += changes.len(); + for (broadcast, change) in &changes { + match change { + Change::Live(claim) => heard.items.insert(broadcast.clone(), claim.clone()), + Change::Ended | Change::Lost => heard.items.remove(broadcast), + }; + } + heard.floor = floor; + let known = self.known.entry(origin).or_default(); + // The newest floor wins, so an older copy never revives what a newer one withdrew. + if floor <= known.floor { + return; + } + let changes = match known.source == Some(from) { + true => changes, + false => { + let mut diff: Vec<(Broadcast, Change)> = (heard.items.iter()) + .filter(|(broadcast, claim)| known.items.get(*broadcast) != Some(claim)) + .map(|(broadcast, claim)| (broadcast.clone(), Change::Live(claim.clone()))) + .collect(); + diff.extend( + (known.items.keys()) + .filter(|broadcast| !heard.items.contains_key(*broadcast)) + .map(|broadcast| (broadcast.clone(), Change::Lost)), + ); + self.work += heard.items.len(); + diff + } + }; + known.source = Some(from); + known.floor = floor; + self.publish(origin, changes); + } + + /// Apply `changes` to this node's view of `origin` and pass them to every + /// neighbour it offers `origin` to. + fn publish(&mut self, origin: Id, changes: Vec<(Broadcast, Change)>) { + let known = self.known.entry(origin).or_default(); + let floor = known.floor; + for (broadcast, change) in &changes { + match change { + Change::Live(claim) => { + known.items.insert(broadcast.clone(), claim.clone()); + self.claimants.entry(broadcast.clone()).or_default().insert(origin); + self.ended.remove(broadcast); + self.dirty.insert(broadcast.clone()); + } + Change::Ended | Change::Lost => { + known.items.remove(broadcast); + if *change == Change::Ended { + self.ended.insert(broadcast.clone()); + } + if let Some(origins) = self.claimants.get_mut(broadcast) { + origins.remove(&origin); + if origins.is_empty() { + self.claimants.remove(broadcast); + } + } + self.dirty.insert(broadcast.clone()); + } + } + } + if changes.is_empty() { + return; + } + for (&peer, sent) in &self.sent { + if sent.contains_key(&origin) { + self.out.push(( + peer, + Msg::Claims { + origin, + changes: changes.clone(), + floor, + }, + )); + } + } + } + + fn unclaim(&mut self, origin: Id, broadcast: Broadcast) { + if let Some(origins) = self.claimants.get_mut(&broadcast) { + origins.remove(&origin); + if origins.is_empty() { + self.claimants.remove(&broadcast); + } + } + self.dirty.insert(broadcast); + } + + /// Claims from an endpoint or across a scoped grant become this node's own. + fn adopt(&mut self, from: N, changes: Vec<(Broadcast, Change)>) { + let peer = &self.peers[&from]; + let (cost, scoped) = (peer.port.cost, peer.port.role == Role::Scoped); + for (broadcast, change) in changes { + self.work += 1; + let change = match change { + // A claim that already crossed this node came round a ring of grants. + Change::Live(claim) if claim.chain.contains(&self.me) => Change::Lost, + Change::Live(mut claim) => { + claim.cost += cost; + if scoped { + claim.chain.push(self.me); + } + Change::Live(claim) + } + change => change, + }; + let claims = self.local.entry(from).or_default(); + match change { + Change::Live(claim) => { + claims.insert(broadcast.clone(), claim); + self.restate(broadcast, Change::Lost); + } + cause => { + if claims.remove(&broadcast).is_some() { + self.restate(broadcast, cause); + } + } + } + } + } + + /// Announce the cheapest claim any neighbour gave this node for `broadcast`, or + /// withdraw it for `cause` when none is left. + fn restate(&mut self, broadcast: Broadcast, cause: Change) { + let cheapest = (self.local.iter()) + .filter_map(|(&peer, claims)| Some((claims.get(&broadcast)?, peer))) + .min_by_key(|(claim, peer)| (claim.cost, claim.chain.len(), *peer)) + .map(|(claim, peer)| (claim.clone(), peer)); + let me = self.me; + let was = self.known.entry(me).or_default().items.get(&broadcast).cloned(); + let change = match cheapest { + Some((claim, peer)) => { + self.sources.insert(broadcast.clone(), peer); + if was.as_ref() == Some(&claim) { + // The source alone moved, which a scoped neighbour must hear of. + self.dirty.insert(broadcast); + return; + } + Change::Live(claim) + } + None => { + self.sources.remove(&broadcast); + if was.is_none() { + return; + } + cause + } + }; + self.rev += 1; + self.known.entry(me).or_default().floor = self.rev; + self.publish(me, vec![(broadcast, change)]); + } + + /// A session closed: this node's own when `from` is `None`. + fn closed(&mut self, hop: Hop, far: Id, from: Option) { + self.work += 1; + let named = (self.heard.values()) + .flat_map(BTreeMap::values) + .any(|heard| heard.offer.path.contains(&hop)); + let tell: Vec = (self.sent.iter()) + .filter(|(peer, sent)| Some(**peer) != from && sent.values().any(|offer| offer.path.contains(&hop))) + .map(|(&peer, _)| peer) + .collect(); + // The far end's own word about a session lifts what its near end cast on it. + if let Some(nears) = self.suspects.get_mut(&hop.node) { + nears.retain(|near| near.node != far); + if nears.is_empty() { + self.suspects.remove(&hop.node); + } + } + let news = !self.dead.contains_key(&hop) && (named || !tell.is_empty() || from.is_none()); + if news { + if named { + self.dead.insert(hop, far); + } + for peer in tell { + self.out.push((peer, Msg::Dead { hop, far })); + } + let answered = self.dead.iter().any(|(word, about)| word.node == far && *about == hop.node); + if self.suspect == Suspect::FarEnd && !answered { + self.suspects.entry(far).or_default().insert(hop); + } + } + self.forget(); + self.reselect_all(); + } + + /// Forget each dead hop and suspected node no heard route names. + fn forget(&mut self) { + if self.dead.is_empty() && self.suspects.is_empty() { + return; + } + let mut hops = BTreeSet::new(); + let mut nodes = BTreeSet::new(); + for heard in self.heard.values().flat_map(BTreeMap::values) { + for hop in &heard.offer.path { + hops.insert(*hop); + nodes.insert(hop.node); + } + } + self.work += hops.len(); + self.dead.retain(|hop, _| hops.contains(hop)); + self.suspects.retain(|node, _| nodes.contains(node)); + } + + fn reselect_all(&mut self) { + let origins: BTreeSet = self.heard.keys().chain(self.best.keys()).copied().collect(); + for origin in origins { + self.reselect(origin); + } + } + + /// Pick the route to `origin` and tell every neighbour what changed. + fn reselect(&mut self, origin: Id) { + let current = self.best.get(&origin).map(|best| best.via); + let mut pick: Option<(Best, Order)> = None; + for (&via, heard) in self.heard.get(&origin).into_iter().flatten() { + self.work += 1; + let Some(peer) = self.peers.get(&via) else { + continue; + }; + let offer = &heard.offer; + if offer.names(self.me) || offer.path.len() >= MAX_HOPS || offer.path.iter().any(|hop| self.dead.contains_key(hop)) { + continue; + } + let suspected = offer.path.iter().any(|hop| self.suspects.contains_key(&hop.node)); + let mine = Offer { + path: offer.path.clone(), + draining: offer.draining || peer.draining || suspected, + priority: offer.priority.min(peer.port.priority), + cost: offer.cost + peer.port.cost, + marked: offer.marked || peer.port.upstream, + }; + let (draining, priority, cost) = mine.rank(); + // Among equal ranks a node stays on its next hop. + let key = (draining, priority, cost, current != Some(via), mine.path.len(), via); + if pick.as_ref().is_none_or(|(_, best)| key < *best) { + pick = Some((Best { via, offer: mine }, key)); + } + } + let pick = pick.map(|(best, _)| best); + if self.best.get(&origin) == pick.as_ref() { + return; + } + let (was, now) = (self.best.contains_key(&origin), pick.is_some()); + let moved = self.best.get(&origin).map(|best| best.offer.rank()) != pick.as_ref().map(|best| best.offer.rank()); + match pick { + Some(best) => self.best.insert(origin, best), + None => self.best.remove(&origin), + }; + // Whether its claims are reachable, and at what cost, follows the route. + if (was != now || moved) + && let Some(known) = self.known.get(&origin) + { + self.work += known.items.len(); + self.dirty.extend(known.items.keys().cloned()); + } + let peers: Vec = self.peers.keys().copied().collect(); + for peer in peers { + self.advertise_to(origin, peer); + } + } + + /// What `peer` should hear of `origin` now. + fn offer_for(&self, origin: Id, peer: N) -> Option { + let to = self.peers.get(&peer).filter(|to| to.port.role == Role::Routing)?; + if origin == self.me { + return Some(Offer { + path: vec![Hop { + node: self.me, + session: 0, + }], + draining: false, + priority: i8::MAX, + cost: 0, + marked: false, + }); + } + let best = self.best.get(&origin)?; + // Never back over the session it came on, to a node already on its path, or + // from one marked link to another. + if best.via == peer || best.offer.names(to.id) || (best.offer.marked && to.port.upstream) { + return None; + } + let mut offer = best.offer.clone(); + offer.path.push(Hop { + node: self.me, + session: self.peers[&best.via].session, + }); + Some(offer) + } + + fn advertise_to(&mut self, origin: Id, peer: N) { + if !self.peers.get(&peer).is_some_and(|to| to.port.role == Role::Routing) { + return; + } + let offer = self.offer_for(origin, peer); + let sent = self.sent.entry(peer).or_default(); + if sent.get(&origin) == offer.as_ref() { + return; + } + let fresh = !sent.contains_key(&origin); + match &offer { + Some(offer) => sent.insert(origin, offer.clone()), + None => sent.remove(&origin), + }; + let started = fresh && offer.is_some(); + self.out.push((peer, Msg::Route { origin, offer })); + // A route that starts brings the claims as they stand. + // Even an empty one: its floor outranks an older copy of what was withdrawn. + if started + && let Some(known) = self.known.get(&origin) + && known.floor > 0 + { + self.work += known.items.len(); + let changes = (known.items.iter()) + .map(|(broadcast, claim)| (broadcast.clone(), Change::Live(claim.clone()))) + .collect(); + self.out.push(( + peer, + Msg::Claims { + origin, + changes, + floor: known.floor, + }, + )); + } + } + + /// The forwarding node a request for `broadcast` names from here: the closest claim. + pub fn pick(&self, broadcast: &Broadcast) -> Option { + self.closest(broadcast).map(|(origin, _)| origin) + } + + fn closest(&self, broadcast: &Broadcast) -> Option<(Id, &Claim)> { + (self.claimants.get(broadcast)?.iter()) + .filter_map(|&origin| { + let claim = self.known.get(&origin)?.items.get(broadcast)?; + let rank = match origin == self.me { + true => (false, Reverse(i8::MAX), claim.cost), + false => { + let (draining, priority, cost) = self.best.get(&origin)?.offer.rank(); + (draining, priority, cost + claim.cost) + } + }; + Some(((rank, claim.chain.len(), origin), claim)) + }) + .min_by(|(a, _), (b, _)| a.cmp(b)) + .map(|((_, _, origin), claim)| (origin, claim)) + } + + /// Where a request for `broadcast` naming `origin` goes from here. `None` fails it. + pub fn step(&self, origin: Id, broadcast: &Broadcast) -> Option> { + match origin == self.me { + true => self.sources.get(broadcast).copied().map(Next::Local), + false => self.best.get(&origin).map(|best| Next::Via(best.via)), + } + } + + /// The neighbour this node routes `origin` through. + pub fn via(&self, origin: Id) -> Option { + self.best.get(&origin).map(|best| best.via) + } + + /// The cost of the route this node uses to `origin`. + pub fn cost(&self, origin: Id) -> Option { + self.best.get(&origin).map(|best| best.offer.cost) + } + + /// Bring what a reader sees, and what scoped neighbours are told, up to date. + fn refresh(&mut self) { + while let Some(broadcast) = self.dirty.pop_first() { + self.work += 1; + let known = self.claimants.contains_key(&broadcast); + let available = self.closest(&broadcast).is_some(); + let mut emit = |kind| { + self.events.push(Event { + broadcast: broadcast.clone(), + kind, + }); + }; + match (self.shown.get(&broadcast).copied(), known) { + (None, true) => { + emit(Kind::Start); + if !available { + emit(Kind::Unavailable); + } + } + (Some(_), false) => emit(match self.ended.contains(&broadcast) { + true => Kind::End, + false => Kind::Offline, + }), + (Some(was), true) if was != available => emit(match available { + true => Kind::Available, + false => Kind::Unavailable, + }), + _ => {} + } + match known { + true => self.shown.insert(broadcast.clone(), available), + false => self.shown.remove(&broadcast), + }; + let scoped: Vec = (self.peers.iter()) + .filter(|(_, peer)| peer.port.role == Role::Scoped) + .map(|(&peer, _)| peer) + .collect(); + for peer in scoped { + self.export(&broadcast, peer); + } + self.ended.remove(&broadcast); + } + } + + /// Tell scoped `peer` this node's closest claim for `broadcast`, unless it came + /// from that neighbour. + fn export(&mut self, broadcast: &Broadcast, peer: N) { + let claim = self.closest(broadcast).and_then(|(origin, claim)| { + let cost = match origin == self.me { + true if self.sources.get(broadcast) == Some(&peer) => return None, + true => claim.cost, + false => claim.cost + self.best[&origin].offer.cost, + }; + Some(Claim { + cost, + chain: claim.chain.clone(), + }) + }); + let sent = self.foreign.entry(peer).or_default(); + if sent.get(broadcast) == claim.as_ref() { + return; + } + let change = match claim { + Some(claim) => { + sent.insert(broadcast.clone(), claim.clone()); + Change::Live(claim) + } + None => { + sent.remove(broadcast); + match self.ended.contains(broadcast) { + true => Change::Ended, + false => Change::Lost, + } + } + }; + self.out.push(( + peer, + Msg::Foreign { + changes: vec![(broadcast.clone(), change)], + }, + )); + } + + pub fn kept(&self) -> Kept { + let heard = || self.heard.values().flat_map(BTreeMap::values); + Kept { + routes: heard().count(), + hops: heard().map(|heard| heard.offer.path.len()).sum(), + sent: self.sent.values().map(BTreeMap::len).sum(), + items: self.known.values().map(|known| known.items.len()).sum(), + heard: heard().map(|heard| heard.items.len()).sum(), + foreign: self.foreign.values().map(BTreeMap::len).sum(), + pending: self.dead.len() + self.suspects.len(), + } + } +} + +/// What sits at a simulated node. +#[derive(Clone, Debug)] +enum Spot { + Relay(Box>), + /// A publisher: an endpoint that never forwards. + Endpoint { id: Option, claims: BTreeSet }, +} + +/// How one side of a link relates to the other. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Tie { + Peer, + /// The first node marks the second as its upstream. + Up, + /// A scoped grant each way: a trust boundary. + Scoped, +} + +/// A reader event with what was true when it showed. +#[derive(Clone, Debug)] +pub(super) struct Seen { + pub node: Node, + pub broadcast: Broadcast, + pub kind: Kind, + /// How many times the publisher had started this broadcast. + pub life: u64, + /// Whether a publisher with an open session claimed it. + pub live: bool, +} + +/// A network of [`Router`]s and endpoints, with [`Sim`]'s queues. +#[derive(Clone)] +pub(super) struct MultiSim { + suspect: Suspect, + spots: Vec, + restarts: Vec, + ports: BTreeMap<(Node, Node), Port>, + up: BTreeSet<(Node, Node)>, + queues: BTreeMap<(Node, Node), VecDeque>, + held: BTreeSet<(Node, Node)>, + lives: BTreeMap, + pub stats: Stats, + pub events: Vec, + spent: usize, +} + +impl MultiSim { + pub fn new(suspect: Suspect, nodes: usize) -> Self { + Self { + suspect, + spots: (0..nodes) + .map(|node| Spot::Relay(Box::new(Router::new(hop(node, 0), suspect)))) + .collect(), + restarts: vec![0; nodes], + ports: BTreeMap::new(), + up: BTreeSet::new(), + queues: BTreeMap::new(), + held: BTreeSet::new(), + lives: BTreeMap::new(), + stats: Stats::default(), + events: Vec::new(), + spent: 0, + } + } + + fn pair(a: Node, b: Node) -> (Node, Node) { + (a.min(b), a.max(b)) + } + + pub fn router(&self, node: Node) -> Option<&Router> { + match &self.spots[node] { + Spot::Relay(router) => Some(router), + Spot::Endpoint { .. } => None, + } + } + + fn relay(&mut self, node: Node) -> Option<&mut Router> { + match &mut self.spots[node] { + Spot::Relay(router) => Some(router), + Spot::Endpoint { .. } => None, + } + } + + fn peers(&self, node: Node) -> Vec { + (self.up.iter()) + .filter_map(|&(a, b)| match node { + _ if node == a => Some(b), + _ if node == b => Some(a), + _ => None, + }) + .collect() + } + + pub fn link(&mut self, a: Node, b: Node, cost: Metric, tie: Tie) { + let role = |node: Node, spots: &[Spot]| match (&spots[node], tie) { + (Spot::Endpoint { .. }, _) => Role::Endpoint, + (_, Tie::Scoped) => Role::Scoped, + _ => Role::Routing, + }; + let port = |upstream, role| Port { + cost, + priority: 0, + upstream, + role, + }; + // Each side's port describes the node at the other end. + self.ports.insert((a, b), port(tie == Tie::Up, role(b, &self.spots))); + self.ports.insert((b, a), port(false, role(a, &self.spots))); + self.open(a, b); + } + + pub fn open(&mut self, a: Node, b: Node) { + assert!(self.up.insert(Self::pair(a, b)), "{a}-{b} already up"); + for (x, y) in [(a, b), (b, a)] { + let port = self.ports[&(x, y)]; + let named = hop(y, self.restarts[y]); + let changes: Vec<(Broadcast, Change)> = match &mut self.spots[x] { + Spot::Relay(router) => { + router.connect(y, named, port); + continue; + } + Spot::Endpoint { id, claims } => (claims.iter()) + .map(|path| { + let claim = Claim { + cost: 0, + chain: Vec::new(), + }; + ((id.expect("a publisher"), path.clone()), Change::Live(claim)) + }) + .collect(), + }; + if !changes.is_empty() { + self.send(x, y, Msg::Foreign { changes }); + } + } + self.flush(); + } + + pub fn close(&mut self, a: Node, b: Node) { + self.up.remove(&Self::pair(a, b)); + for (x, y) in [(a, b), (b, a)] { + self.queues.remove(&(x, y)); + self.held.remove(&(x, y)); + if let Some(router) = self.relay(y) { + router.disconnect(x); + } + } + self.flush(); + } + + /// `peer` notices its session to `node` closed, which `node` has yet to. + pub fn sever(&mut self, node: Node, peer: Node) { + self.up.remove(&Self::pair(node, peer)); + for key in [(node, peer), (peer, node)] { + self.queues.remove(&key); + self.held.remove(&key); + } + if let Some(router) = self.relay(peer) { + router.disconnect(node); + } + self.flush(); + } + + /// The process at `node` exits: its sessions close and its state is lost. + pub fn reset(&mut self, node: Node) { + for peer in self.peers(node) { + self.close(node, peer); + } + self.restarts[node] += 1; + if let Spot::Relay(router) = &self.spots[node] { + self.spent += router.work(); + self.spots[node] = Spot::Relay(Box::new(Router::new(hop(node, self.restarts[node]), self.suspect))); + } + } + + pub fn goaway(&mut self, node: Node) { + for peer in self.peers(node) { + if let Some(router) = self.relay(peer) { + router.goaway(node); + } + } + self.flush(); + } + + pub fn client(&mut self, node: Node) { + self.spots[node] = Spot::Endpoint { + id: None, + claims: BTreeSet::new(), + }; + } + + pub fn publish(&mut self, node: Node, named: Id) { + let Spot::Endpoint { id, .. } = &mut self.spots[node] else { + panic!("node {node} is a relay"); + }; + *id = Some(named); + } + + pub fn claim(&mut self, node: Node, claim: &str, live: bool) { + let claim = p(claim); + let Spot::Endpoint { id, claims } = &mut self.spots[node] else { + panic!("node {node} is a relay"); + }; + let broadcast = (id.expect("a publisher"), claim.clone()); + let change = match live { + true if claims.insert(claim.clone()) => { + *self.lives.entry(broadcast.clone()).or_default() += 1; + Change::Live(Claim { + cost: 0, + chain: Vec::new(), + }) + } + false if claims.remove(&claim) => Change::Ended, + _ => return, + }; + for peer in self.peers(node) { + self.send( + node, + peer, + Msg::Foreign { + changes: vec![(broadcast.clone(), change.clone())], + }, + ); + } + } + + fn send(&mut self, from: Node, to: Node, msg: Msg) { + match &msg { + Msg::Route { .. } | Msg::Dead { .. } => { + self.stats.route_msgs += 1; + self.stats.route_items += 1; + } + Msg::Claims { changes, .. } | Msg::Foreign { changes } => { + self.stats.claim_msgs += 1; + self.stats.claim_items += changes.len(); + } + } + self.queues.entry((from, to)).or_default().push_back(msg); + } + + pub fn hold(&mut self, from: Node, to: Node) { + self.held.insert((from, to)); + } + + pub fn ready(&self) -> Vec<((Node, Node), usize)> { + (self.queues.iter()) + .filter(|(key, queue)| !queue.is_empty() && !self.held.contains(key)) + .map(|(key, queue)| (*key, queue.len())) + .collect() + } + + pub fn deliver(&mut self, from: Node, to: Node) -> bool { + let Some(msg) = self.queues.get_mut(&(from, to)).and_then(VecDeque::pop_front) else { + return false; + }; + if let Some(router) = self.relay(to) { + router.recv(from, msg); + } + self.stats.delivered += 1; + self.flush(); + true + } + + /// Deliver in rounds until every unheld queue is empty. False if that never happens. + pub fn run(&mut self) -> bool { + for _ in 0..10_000 { + let ready = self.ready(); + if ready.is_empty() { + return true; + } + for ((from, to), _) in ready { + self.deliver(from, to); + } + } + false + } + + fn live(&self, broadcast: &Broadcast) -> bool { + self.spots.iter().enumerate().any(|(node, spot)| match spot { + Spot::Endpoint { id, claims } => { + *id == Some(broadcast.0) && claims.contains(&broadcast.1) && !self.peers(node).is_empty() + } + Spot::Relay(_) => false, + }) + } + + fn flush(&mut self) { + for node in 0..self.spots.len() { + let Some(router) = self.relay(node) else { + continue; + }; + let (outs, events) = (router.drain(), router.events()); + for (to, msg) in outs { + self.send(node, to, msg); + } + for event in events { + let seen = Seen { + node, + live: self.live(&event.broadcast), + life: self.lives.get(&event.broadcast).copied().unwrap_or(0), + broadcast: event.broadcast, + kind: event.kind, + }; + self.events.push(seen); + } + } + } + + /// The nodes a request from `from` for `broadcast` visits, and whether it ended + /// at a publisher. `None` when `from` knows no claim it can reach. A chain longer + /// than the network loops. + pub fn chase(&self, from: Node, broadcast: &Broadcast) -> Option<(Vec, bool)> { + let (mut at, mut named, mut hops) = (from, None, Vec::new()); + while hops.len() <= self.spots.len() { + let router = match &self.spots[at] { + Spot::Endpoint { id, claims } => { + return Some((hops, *id == Some(broadcast.0) && claims.contains(&broadcast.1))); + } + Spot::Relay(router) => router, + }; + let Some(origin) = named.or_else(|| router.pick(broadcast)) else { + return (!hops.is_empty()).then_some((hops, false)); + }; + match router.step(origin, broadcast) { + // A request names the forwarding node its first hop chose. + Some(Next::Via(next)) => { + named = Some(origin); + hops.push(next); + at = next; + } + // Past a claim's announcer the next node chooses again. + Some(Next::Local(next)) => { + named = None; + hops.push(next); + at = next; + } + None => return Some((hops, false)), + } + } + Some((hops, false)) + } + + pub fn work(&self) -> usize { + self.spent + (0..self.spots.len()).filter_map(|node| self.router(node)).map(Router::work).sum::() + } + + pub fn kept(&self) -> Kept { + let mut out = Kept::default(); + for router in (0..self.spots.len()).filter_map(|node| self.router(node)) { + let kept = router.kept(); + out.routes += kept.routes; + out.hops += kept.hops; + out.sent += kept.sent; + out.items += kept.items; + out.heard += kept.heard; + out.foreign += kept.foreign; + out.pending += kept.pending; + } + out + } +} + +/// Starts of a broadcast's life that the same node had already shown ending. +pub(super) fn revivals(events: &[Seen]) -> usize { + let mut ended = BTreeSet::new(); + let mut count = 0; + for event in events { + let life = (event.node, event.broadcast.clone(), event.life); + match event.kind { + Kind::End => { + ended.insert(life); + } + Kind::Start if ended.contains(&life) => count += 1, + _ => {} + } + } + count +} + +/// Ends shown while a connected publisher still claimed the broadcast. +pub(super) fn false_ends(events: &[Seen]) -> usize { + events.iter().filter(|event| event.kind == Kind::End && event.live).count() +} From 8c1b3b13d729d3cb00f993212b33fdef649706fb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 14:51:36 -0800 Subject: [PATCH 03/13] test(moq-net): measure every failure on every shape, with each addition as its own arm Co-Authored-By: Claude Fable 5.2 --- rs/moq-net/src/route/tests/bakeoff.rs | 508 +++++++++++++++++++++++++- rs/moq-net/src/route/tests/compare.rs | 130 +++++-- rs/moq-net/src/route/tests/multi.rs | 281 ++++++++++---- 3 files changed, 833 insertions(+), 86 deletions(-) diff --git a/rs/moq-net/src/route/tests/bakeoff.rs b/rs/moq-net/src/route/tests/bakeoff.rs index e94afffdfdb..4e0afff7817 100644 --- a/rs/moq-net/src/route/tests/bakeoff.rs +++ b/rs/moq-net/src/route/tests/bakeoff.rs @@ -1,14 +1,457 @@ //! The bake-off: the multicast epic's rule ([`multi::Router`]) beside today's //! [`path::Router`] on the same meshes and failure schedules. +//! +//! Every arm builds the same [`Mesh`], sees the same failure at the same instant, and +//! is measured by the same [`Lab`]: messages, hops of delay until every relay's +//! request reaches its publisher for good, the relay-rounds a reader went unfed +//! meanwhile, loops among selected routes, and what readers were shown. The ignored +//! benchmarks print the tables; the tests pin what they show. + +use std::collections::BTreeSet; use super::compare::*; +use super::multi::{Suspect, Tie}; use super::*; /// The arms measured, in the report's order. -const ARMS: [Arm; 3] = [Arm::TODAY, Arm::PLAIN, Arm::SUSPECT]; +fn arms() -> Vec { + vec![ + Arm::TODAY, + Arm::PLAIN, + Arm::with(|tune| tune.suspect = Suspect::Flagged), + Arm::with(|tune| tune.suspect = Suspect::Local), + Arm::with(|tune| tune.quiet = true), + Arm::with(|tune| tune.batch = true), + Arm::with(|tune| tune.agree = true), + Arm::with(|tune| tune.drain = true), + Arm::with(|tune| { + tune.quiet = true; + tune.batch = true; + tune.drain = true; + }), + Arm::with(|tune| { + tune.suspect = Suspect::Local; + tune.quiet = true; + tune.batch = true; + tune.drain = true; + }), + Arm::TUNED, + ] +} /// The prototype's arms. -const MULTI: [Arm; 2] = [Arm::PLAIN, Arm::SUSPECT]; +const MULTI: [Arm; 2] = [Arm::PLAIN, Arm::TUNED]; + +/// How long a silent death takes its neighbours to notice, in hops of delay: an idle +/// timeout of 3 s. +const IDLE: usize = 120; + +/// A mesh to measure, and whether today's rule can run it. +struct Shape { + name: &'static str, + mesh: Mesh, + /// Today's rule qualifies an identity across one scoped link and refuses to + /// again, so a publisher two grants away is never seen. + today: bool, +} + +fn pair(a: Node, b: Node) -> (Node, Node) { + (a.min(b), a.max(b)) +} + +/// Publishers spread over `relays`, the first on the first. +fn spread(relays: &[Node], publishers: usize) -> Vec { + (0..publishers).map(|index| relays[index * 7 % relays.len()]).collect() +} + +/// CDN tiers: two cores per region, each region's cores linked to each other and +/// to both cores of the regions beside it, and every edge on both of its region's +/// cores, which it marks as upstream. Edges of one region peer when `peered`. +/// Publishers join edges. +fn tiers(regions: usize, edges: usize, peered: bool, publishers: usize) -> Mesh { + let core = |region: usize, side: usize| 2 * (region % regions) + side; + let edge = |region: usize, index: usize| 2 * regions + region * edges + index; + let mut links = BTreeSet::new(); + let mut ties = BTreeMap::new(); + for region in 0..regions { + links.insert(pair(core(region, 0), core(region, 1))); + if regions > 1 { + for (a, b) in [(0, 0), (0, 1), (1, 0), (1, 1)] { + links.insert(pair(core(region, a), core(region + 1, b))); + } + } + for index in 0..edges { + for side in 0..2 { + // Cores come first, so the pair already orders the edge second. + links.insert((core(region, side), edge(region, index))); + ties.insert((edge(region, index), core(region, side)), Tie::Up); + } + for other in index + 1..edges { + if peered { + links.insert((edge(region, index), edge(region, other))); + } + } + } + } + let all: Vec = (0..regions * edges).map(|index| 2 * regions + index).collect(); + Mesh { + relays: 2 * regions + regions * edges, + links: links.into_iter().map(|(a, b)| (a, b, 1)).collect(), + ties, + homes: spread(&all, publishers), + claims: 1, + } +} + +/// A ring with a chord from every relay at each power of two: log N of them. +fn chords(relays: usize, publishers: usize) -> Mesh { + let mut links = BTreeSet::new(); + let mut step = 1; + while step <= relays / 2 { + for a in 0..relays { + links.insert(pair(a, (a + step) % relays)); + } + step *= 2; + } + Mesh { + relays, + links: links.into_iter().filter(|(a, b)| a != b).map(|(a, b)| (a, b, 1)).collect(), + ties: BTreeMap::new(), + homes: spread(&(0..relays).collect::>(), publishers), + claims: 1, + } +} + +fn sparse(relays: usize, publishers: usize, rng: &mut XorShift) -> Mesh { + Mesh { + homes: spread(&(0..relays).collect::>(), publishers), + claims: 1, + ..Mesh::sparse(relays, rng) + } +} + +fn clique(relays: usize, publishers: usize) -> Mesh { + Mesh { + homes: spread(&(0..relays).collect::>(), publishers), + claims: 1, + ..Mesh::clique(relays, 1) + } +} + +/// A chorded ring of `core` relays with `leaves` more, each on two of them and +/// marking both links, like a relay on a camera. Publishers join leaves. +fn leafy(core: usize, leaves: usize, publishers: usize) -> Mesh { + let mut mesh = chords(core, 1); + for leaf in 0..leaves { + for at in [leaf * 3 % core, (leaf * 3 + core / 2) % core] { + mesh.links.push((at, core + leaf, 1)); + mesh.ties.insert((core + leaf, at), Tie::Up); + } + } + mesh.relays = core + leaves; + mesh.homes = spread(&(core..core + leaves).collect::>(), publishers); + mesh +} + +/// `count` chorded rings of `size` relays, each joined to the next by `joins` +/// scoped links, and the last to the first when `ring`. Publishers join the first. +fn joined(count: usize, size: usize, joins: usize, ring: bool, publishers: usize) -> Mesh { + let one = chords(size, 1); + let mut mesh = Mesh { + relays: count * size, + links: Vec::new(), + ties: BTreeMap::new(), + homes: spread(&(0..size).collect::>(), publishers), + claims: 1, + }; + for at in 0..count { + (mesh.links).extend(one.links.iter().map(|&(a, b, cost)| (at * size + a, at * size + b, cost))); + if at + 1 == count && !(ring && count > 2) { + continue; + } + let next = (at + 1) % count; + for join in 0..joins { + // Each boundary leaves a mesh at another relay than the one it entered by. + let (a, b) = (at * size + size / 2 + join, next * size + 1 + join); + mesh.links.push((a, b, 1)); + mesh.ties.insert((a, b), Tie::Scoped); + } + } + mesh +} + +/// Every shape, at about `relays` relays each. +fn shapes(relays: usize, publishers: usize) -> Vec { + let mut rng = XorShift(0x6d6f_7100 + relays as u64); + let open = |name, mesh| Shape { + name, + mesh, + today: true, + }; + let (regions, edges) = match relays { + ..12 => (2, 2), + 12..24 => (3, 3), + 24..48 => (4, 6), + _ => (6, 8), + }; + vec![ + open("tiers", tiers(regions, edges, true, publishers)), + open("tiers_unpeered", tiers(regions, edges, false, publishers)), + open("chords", chords(relays, publishers)), + open("sparse", sparse(relays, publishers, &mut rng)), + open("clique", clique(relays, publishers)), + open("leaves", leafy(relays / 2, relays - relays / 2, publishers)), + open("two_meshes_one_grant", joined(2, relays / 2, 1, false, publishers)), + open("two_meshes_two_grants", joined(2, relays / 2, 2, false, publishers)), + Shape { + name: "three_meshes_chain", + mesh: joined(3, relays / 3, 1, false, publishers), + today: false, + }, + Shape { + name: "three_meshes_ring", + mesh: joined(3, relays / 3, 1, true, publishers), + today: false, + }, + ] +} + +/// The failures measured, each from a settled mesh. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +enum Failure { + /// A link relays route over closes cleanly, and every relay still has a path. + Close, + /// It returns. + Reopen, + /// It closes and reopens eight times, a hop of delay apart: faster than any timer. + Flap, + /// It and a link sharing no relay with it close at the same instant. + Twin, + /// Every link between the far half of the mesh and the publisher's half closes. + Partition, + /// They return. + Heal, + /// The relay the most others route through dies, noticed a hop of delay apart. + Death, + /// It restarts. + Restart, + /// The same death, noticed over an idle timeout. + SlowDeath, + /// The relay with the most neighbours dies, noticed over an idle timeout. + HubDeath, + /// A publisher on two relays loses the session to one. + LoseOne, + /// Then to the other. + LoseBoth, + /// A publisher's session closes and it joins a neighbouring relay at once. + Move, + /// The relay the most others route through sends GOAWAY. + Goaway, + /// Then closes every session. + Drained, + /// The publisher's own relay dies. + Home, + /// The publisher's session closes. + Leave, +} + +impl Failure { + fn as_str(self) -> &'static str { + match self { + Self::Close => "link_closes", + Self::Reopen => "link_reopens", + Self::Flap => "link_flaps", + Self::Twin => "two_links_close_at_once", + Self::Partition => "partition", + Self::Heal => "partition_heals", + Self::Death => "transit_dies", + Self::Restart => "transit_restarts", + Self::SlowDeath => "transit_dies_slowly", + Self::HubDeath => "hub_dies_slowly", + Self::LoseOne => "two_homes_lose_one", + Self::LoseBoth => "two_homes_lose_both", + Self::Move => "publisher_moves", + Self::Goaway => "transit_goaway", + Self::Drained => "transit_drained", + Self::Home => "home_dies", + Self::Leave => "publisher_leaves", + } + } +} + +/// What one failure cost and showed. +#[derive(Clone, Debug)] +struct Row { + cost: Cost, + faults: Faults, + /// Dead hops, doubts, and suspected nodes left once it settled. + pending: usize, + /// The hops every relay's request crosses once it settled, summed. + hops: usize, + /// Whether every relay with a path routed to its publisher once it settled. + right: Result<(), String>, +} + +fn observe(lab: &mut Lab, change: &dyn Fn(&mut Lab)) -> Row { + let before = lab.net.faults(); + let cost = lab.measure(change); + let after = lab.net.faults(); + Row { + cost, + faults: Faults { + false_ends: after.false_ends - before.false_ends, + revivals: after.revivals - before.revivals, + }, + pending: lab.net.held().pending, + hops: lab.hops(), + right: match cost.settled { + true => lab.check(), + false => Err("never settled".into()), + }, + } +} + +/// The relay the most others route through toward the first publisher, its own aside. +fn transit(mesh: &Mesh, tree: &[Option]) -> Option { + let mut through = vec![0; mesh.relays]; + for relay in 0..mesh.relays { + let mut at = relay; + while let Some(next) = tree[at] { + through[next] += 1; + at = next; + } + } + (0..mesh.relays) + .filter(|relay| !mesh.homes.contains(relay) && through[*relay] > 0) + .max_by_key(|&relay| (through[relay], std::cmp::Reverse(relay))) +} + +/// How far each relay is from the first publisher's along the cheapest-path tree. +fn depths(tree: &[Option]) -> Vec { + (0..tree.len()) + .map(|relay| std::iter::successors(tree[relay], |&at| tree[at]).count()) + .collect() +} + +/// Run every failure on `mesh` under `arm`, each from a settled mesh. +fn fail(arm: Arm, mesh: &Mesh, coarse: bool) -> Vec<(Failure, Row)> { + let tree = mesh.tree(); + let depth = depths(&tree); + let fresh = || { + let mut lab = Lab::new(arm, mesh); + lab.coarse = coarse; + lab + }; + let mut out = Vec::new(); + let (node, _) = mesh.publisher(0); + let home = mesh.homes[0]; + let degree = |relay: Node| mesh.links.iter().filter(|&&(a, b, _)| a == relay || b == relay).count(); + + let mut lab = fresh(); + if let Some((a, b)) = lab.tree_link(0, true, &tree) { + out.push((Failure::Close, observe(&mut lab, &|lab| lab.cut(a, b)))); + out.push((Failure::Reopen, observe(&mut lab, &|lab| lab.mend(a, b)))); + let flap = observe(&mut lab, &|lab| { + for _ in 0..8 { + lab.cut(a, b); + lab.pass(1); + lab.mend(a, b); + lab.pass(1); + } + }); + out.push((Failure::Flap, flap)); + // The link farthest from the publisher that shares no relay with the first. + let other = (mesh.links.iter()) + .filter(|&&(c, d, _)| ![c, d].contains(&a) && ![c, d].contains(&b) && ![c, d].contains(&home)) + .max_by_key(|&&(c, d, _)| (depth[c] + depth[d], c, d)); + if let Some(&(c, d, _)) = other { + let twin = observe(&mut lab, &|lab| { + lab.cut(a, b); + lab.cut(c, d); + }); + out.push((Failure::Twin, twin)); + } + } + + // The half of the relays farthest from the publisher lose every link to the rest. + let mut far: Vec = (0..mesh.relays).filter(|&relay| relay != home).collect(); + far.sort_by_key(|&relay| (std::cmp::Reverse(depth[relay]), relay)); + let far: BTreeSet = far.into_iter().take(mesh.relays / 2).collect(); + let crossing: Vec<(Node, Node)> = (mesh.links.iter()) + .filter(|&&(a, b, _)| far.contains(&a) != far.contains(&b)) + .map(|&(a, b, _)| (a, b)) + .collect(); + let mut lab = fresh(); + let partition = observe(&mut lab, &|lab| { + for &(a, b) in &crossing { + lab.cut(a, b); + } + }); + out.push((Failure::Partition, partition)); + let heal = observe(&mut lab, &|lab| { + for &(a, b) in &crossing { + lab.mend(a, b); + } + }); + out.push((Failure::Heal, heal)); + + let transit = transit(mesh, &tree); + if let Some(relay) = transit { + let mut lab = fresh(); + out.push((Failure::Death, observe(&mut lab, &|lab| lab.crash(relay)))); + out.push((Failure::Restart, observe(&mut lab, &|lab| lab.revive(relay)))); + let mut lab = fresh(); + let slow = observe(&mut lab, &|lab| lab.crash_over(relay, IDLE)); + out.push((Failure::SlowDeath, slow)); + let mut lab = fresh(); + out.push((Failure::Goaway, observe(&mut lab, &|lab| lab.net.goaway(relay)))); + let drained = observe(&mut lab, &|lab| { + for peer in lab.links_of(relay) { + lab.cut(relay, peer); + } + }); + out.push((Failure::Drained, drained)); + } + let hub = (0..mesh.relays) + .filter(|relay| !mesh.homes.contains(relay) && Some(*relay) != transit) + .max_by_key(|&relay| (degree(relay), std::cmp::Reverse(relay))); + if let Some(relay) = hub { + let mut lab = fresh(); + let slow = observe(&mut lab, &|lab| lab.crash_over(relay, IDLE)); + out.push((Failure::HubDeath, slow)); + } + + // A second home as far from the first as the mesh allows. + let second = (0..mesh.relays) + .filter(|&relay| relay != home) + .max_by_key(|&relay| (depth[relay], std::cmp::Reverse(relay))) + .expect("two relays"); + let mut lab = fresh(); + lab.attach(node, second); + assert!(lab.settle()); + out.push((Failure::LoseOne, observe(&mut lab, &|lab| lab.cut(node, home)))); + out.push((Failure::LoseBoth, observe(&mut lab, &|lab| lab.cut(node, second)))); + + let next = (mesh.links.iter()).find_map(|&(a, b, _)| match home { + _ if a == home => Some(b), + _ if b == home => Some(a), + _ => None, + }); + if let Some(next) = next { + let mut lab = fresh(); + let moved = observe(&mut lab, &|lab| { + lab.cut(node, home); + lab.attach(node, next); + }); + out.push((Failure::Move, moved)); + } + + let mut lab = fresh(); + out.push((Failure::Leave, observe(&mut lab, &|lab| lab.cut(node, home)))); + let mut lab = fresh(); + out.push((Failure::Home, observe(&mut lab, &|lab| lab.crash(home)))); + out +} /// The prototype carries every change of the path rule's comparison on its shapes, /// and is right whenever it settles. @@ -33,3 +476,64 @@ fn the_prototype_settles_after_churn() { } } } + +/// The prototype carries every failure on every shape: it settles, every relay with a +/// path routes to its publisher, and no reader sees an end the publisher never made +/// or a life it had seen end. +#[test] +fn the_prototype_carries_every_failure_on_every_shape() { + for shape in shapes(12, 2) { + for arm in MULTI { + for (failure, row) in fail(arm, &shape.mesh, false) { + let case = format!("{}: {} on {}", arm.name(), failure.as_str(), shape.name); + assert_eq!(row.right, Ok(()), "{case}"); + assert_eq!(row.faults, Faults::default(), "{case}"); + } + } + } +} + +/// Every failure on every shape, per arm: what it cost and what readers saw. +#[test] +#[ignore = "a benchmark; Nightly reports it"] +fn benchmark_bakeoff_failures() { + println!( + "shape,relays,links,failure,rule,route_messages,claim_messages,settle_ms,fed_ms,unfed,loops,offline,unavailable,false_ends,revivals,pending,hops,right" + ); + let sizes: Vec = std::env::var("MOQ_BAKEOFF_RELAYS").map_or(vec![12, 32], |sizes| { + sizes.split(',').map(|size| size.parse().expect("MOQ_BAKEOFF_RELAYS")).collect() + }); + for relays in sizes { + for shape in shapes(relays, 2) { + for arm in arms() { + if arm == Arm::TODAY && !shape.today { + continue; + } + for (failure, row) in fail(arm, &shape.mesh, relays > 16) { + let ms = |rounds: usize| rounds as f64 * HOP.as_secs_f64() * 1000.0; + println!( + "{},{},{},{},{},{},{},{:.0},{:.0},{},{},{},{},{},{},{},{},{}", + shape.name, + shape.mesh.relays, + shape.mesh.links.len(), + failure.as_str(), + arm.name(), + row.cost.sent.msgs, + row.cost.sent.claims, + ms(row.cost.rounds), + ms(row.cost.right), + row.cost.stale + row.cost.dark, + row.cost.loops, + row.cost.sent.offline, + row.cost.sent.unavailable, + row.faults.false_ends, + row.faults.revivals, + row.pending, + row.hops, + row.right.as_ref().map_or_else(|err| err.replace(',', ";"), |()| "yes".into()), + ); + } + } + } + } +} diff --git a/rs/moq-net/src/route/tests/compare.rs b/rs/moq-net/src/route/tests/compare.rs index 20d5a77be7a..88f0f8dd128 100644 --- a/rs/moq-net/src/route/tests/compare.rs +++ b/rs/moq-net/src/route/tests/compare.rs @@ -11,36 +11,63 @@ use std::collections::BTreeSet; -use super::multi::{Broadcast, MultiSim, Suspect, Tie}; +use super::multi::{Broadcast, MultiSim, Suspect, Tie, Tune}; use super::*; /// A router under comparison: today's rule, or the multicast epic's prototype. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum Arm { Path(Rule), - Multi(Suspect), + Multi(Tune), } impl Arm { /// Today's rule as it ships. pub const TODAY: Self = Self::Path(Rule::PATH); /// The epic's rule as written. - pub const PLAIN: Self = Self::Multi(Suspect::Never); - /// The epic's rule ranking routes through the far end of a closed session as draining. - pub const SUSPECT: Self = Self::Multi(Suspect::FarEnd); + pub const PLAIN: Self = Self::Multi(Tune { + suspect: Suspect::Never, + quiet: false, + batch: false, + agree: false, + drain: false, + }); + /// The epic's rule with what the bake-off found worth adding. + pub const TUNED: Self = Self::Multi(Tune { + suspect: Suspect::Flagged, + quiet: true, + batch: true, + agree: false, + drain: true, + }); - pub fn as_str(self) -> &'static str { - match self { - Self::Path(rule) => rule.as_str(), - Self::Multi(Suspect::Never) => "multi", - Self::Multi(Suspect::FarEnd) => "multi+suspect", - } + /// The epic's rule with `tune` changed from how it is written. + pub fn with(tune: impl FnOnce(&mut Tune)) -> Self { + let mut out = Tune::default(); + tune(&mut out); + Self::Multi(out) + } + + pub fn name(self) -> String { + let tune = match self { + Self::Path(rule) => return rule.as_str().into(), + Self::Multi(tune) => tune, + }; + let parts = [ + (tune.suspect == Suspect::Flagged, "+suspect"), + (tune.suspect == Suspect::Local, "+local"), + (tune.quiet, "+quiet"), + (tune.batch, "+batch"), + (tune.agree, "+agree"), + (tune.drain, "+drain"), + ]; + (parts.into_iter().filter(|(on, _)| *on)).fold("multi".to_string(), |name, (_, part)| name + part) } fn net(self, nodes: usize) -> Box { match self { Self::Path(rule) => Box::new(Sim::with(rule, nodes)), - Self::Multi(suspect) => Box::new(MultiSim::new(suspect, nodes)), + Self::Multi(tune) => Box::new(MultiSim::new(tune, nodes)), } } } @@ -362,6 +389,11 @@ impl Net for MultiSim { self.deliver(from, to); } + fn tick(&mut self) -> bool { + self.flush(); + false + } + fn chase(&self, from: Node, id: Id) -> Option<(Vec, bool)> { MultiSim::chase(self, from, &(id, p(&Mesh::claim(0)))) } @@ -545,6 +577,9 @@ pub(super) struct Lab { down: BTreeSet, order: XorShift, cost: Cost, + /// Look for loops once a round rather than after every delivery, on meshes too + /// large to walk that often. + pub coarse: bool, } fn pair(a: Node, b: Node) -> (Node, Node) { @@ -563,10 +598,15 @@ impl Lab { down: BTreeSet::new(), order: XorShift(0x5eed_0001), cost: Cost::default(), + coarse: false, }; for &(a, b, cost) in &mesh.links { - let tie = mesh.ties.get(&(a, b)).copied().unwrap_or(Tie::Peer); - lab.net.link(a, b, cost, tie); + // A tie is keyed from the relay that marks the link. + match (mesh.ties.get(&(a, b)), mesh.ties.get(&(b, a))) { + (Some(&tie), _) => lab.net.link(a, b, cost, tie), + (None, Some(&tie)) => lab.net.link(b, a, cost, tie), + (None, None) => lab.net.link(a, b, cost, Tie::Peer), + } lab.up.insert(pair(a, b)); } lab.settle(); @@ -702,7 +742,7 @@ impl Lab { for (chan, count) in ready { for _ in 0..count { self.net.step(chan); - self.cost.loops += usize::from(self.looping()); + self.cost.loops += usize::from(!self.coarse && self.looping()); } } if !idle { @@ -715,7 +755,8 @@ impl Lab { /// Count the relays left wrong as the latest round ends. fn note(&mut self) { - let (_, stale, dark) = self.wrong(); + let (loops, stale, dark) = self.wrong(); + self.cost.loops += usize::from(self.coarse && loops > 0); self.cost.stale += stale; self.cost.dark += dark; if stale + dark > 0 { @@ -782,6 +823,53 @@ impl Lab { self.net.reset(relay); } + /// Let `rounds` hops of delay pass, delivering whatever is queued in each. + pub fn pass(&mut self, rounds: usize) { + for _ in 0..rounds { + if !self.round() { + self.cost.rounds += 1; + self.note(); + } + } + } + + /// The process at `relay` exits, and its neighbours notice spread evenly over + /// `rounds` hops of delay, as an idle timeout leaves them. + pub fn crash_over(&mut self, relay: Node, rounds: usize) { + let mut peers = self.peers(relay); + for &peer in &peers { + self.net.mute(relay, peer); + } + for i in (1..peers.len()).rev() { + peers.swap(i, self.order.next(i as u64 + 1) as usize); + } + self.down.insert(relay); + let gap = rounds / peers.len().max(2).saturating_sub(1); + for (i, peer) in peers.into_iter().enumerate() { + if i > 0 { + self.pass(gap); + } + self.net.sever(relay, peer); + self.up.remove(&pair(relay, peer)); + } + self.net.reset(relay); + } + + /// Every node `node` has an open link to. + pub fn links_of(&self, node: Node) -> Vec { + self.peers(node) + } + + /// The hops every relay's request crosses to each publisher, summed: what the + /// settled routes cost to use. + pub fn hops(&self) -> usize { + (0..self.mesh.homes.len()) + .flat_map(|index| (0..self.mesh.relays).map(move |relay| (relay, index))) + .filter_map(|(relay, index)| self.net.chase(relay, self.mesh.publisher(index).1)) + .map(|(hops, _)| hops.len()) + .sum() + } + /// `relay` starts again with no state, and its links and publishers reconnect. pub fn revive(&mut self, relay: Node) { self.down.remove(&relay); @@ -855,7 +943,7 @@ impl Lab { /// The link of a publisher's cheapest-path tree that the most relays route /// over, among those whose loss leaves every relay a path (`detour`) or cuts /// some off. - fn tree_link(&self, index: usize, detour: bool, tree: &[Option]) -> Option<(Node, Node)> { + pub fn tree_link(&self, index: usize, detour: bool, tree: &[Option]) -> Option<(Node, Node)> { let mut uses: BTreeMap<(Node, Node), usize> = BTreeMap::new(); for relay in 0..self.mesh.relays { let mut at = relay; @@ -932,9 +1020,9 @@ pub(super) fn run(arm: impl Into, mesh: &Mesh) -> Vec<(Change, Cost)> { let tree = mesh.tree(); let mut apply = |lab: &mut Lab, change: Change, apply: &dyn Fn(&mut Lab)| { let cost = lab.measure(apply); - assert!(cost.settled, "{}: {change:?} never settles on {mesh:?}", rule.as_str()); + assert!(cost.settled, "{}: {change:?} never settles on {mesh:?}", rule.name()); if let Err(err) = lab.check() { - panic!("{}: after {change:?} on {mesh:?}: {err}", rule.as_str()); + panic!("{}: after {change:?} on {mesh:?}: {err}", rule.name()); } out.push((change, cost)); }; @@ -1089,9 +1177,9 @@ pub(super) fn churn(arm: impl Into, cases: std::ops::Range, two_home total.stale += stale; total.dark += dark; } - assert!(lab.settle(), "{}: case {case} never settles on {mesh:?}", rule.as_str()); + assert!(lab.settle(), "{}: case {case} never settles on {mesh:?}", rule.name()); if let Err(err) = lab.check() { - panic!("{}: case {case} on {mesh:?}: {err}", rule.as_str()); + panic!("{}: case {case} on {mesh:?}: {err}", rule.name()); } let sent = lab.net.sent() - before; total.sent = Sent { diff --git a/rs/moq-net/src/route/tests/multi.rs b/rs/moq-net/src/route/tests/multi.rs index c2f60d4f842..dbbbffacb49 100644 --- a/rs/moq-net/src/route/tests/multi.rs +++ b/rs/moq-net/src/route/tests/multi.rs @@ -26,14 +26,37 @@ pub(super) type Instance = Id; /// What a reader asks for: an Instance at a path. pub(super) type Broadcast = (Instance, Path); -/// What a closed session says about the node at its far end. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +/// What a closed session says about the node at its far end: every route through +/// it ranks as draining until that node's own word about the session arrives, or a +/// newer session to it shows on a path. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub(super) enum Suspect { - /// Nothing: the epic's plain rule. + /// Nothing: the epic's rule as written. + #[default] Never, - /// Every route through the far end ranks as draining until that node's own word - /// about the session arrives, or a newer session to it shows on a path. - FarEnd, + /// The rank travels in the offer, as a drain does. + Flagged, + /// Each node ranks by what it heard itself, and offers say nothing of it. + Local, +} + +/// What the prototype adds to the epic's rule as written, each measured on its own. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) struct Tune { + pub suspect: Suspect, + /// A node that knows a closed session sends no route until every neighbour that + /// offered one through it has said so too: it uses its new routes at once and + /// advertises them once the news has settled around it. + pub quiet: bool, + /// A node advertises once it has read everything delivered to it, not after + /// every message. + pub batch: bool, + /// A route through a neighbour is refused while that neighbour's own route to + /// the same node says otherwise. + pub agree: bool, + /// A node that sent GOAWAY is named once per neighbour, as a closed session is, + /// so every route through it ranks as draining at once. + pub drain: bool, } /// One node on a route's path, and the session it learned the route on. The @@ -75,6 +98,8 @@ impl Offer { pub(super) struct Claim { /// From the announcing node to the publisher: 0 for its own endpoint. pub cost: Metric, + /// The lowest link priority between the two. + pub priority: i8, /// The nodes that announced it again across a scoped grant, the first first. pub chain: Vec, } @@ -101,6 +126,8 @@ pub(super) enum Msg { }, /// The session `hop` names closed. `far` is the node at its other end. Dead { hop: Hop, far: Id }, + /// `node` sent GOAWAY. + Drain { node: Id }, /// Claims from an endpoint, or across a scoped grant: no route comes with them. Foreign { changes: Vec<(Broadcast, Change)> }, } @@ -139,6 +166,9 @@ struct Peer { #[derive(Clone, Debug)] struct Heard { offer: Offer, + /// The neighbour named a closed session on this path, which ends the route until + /// it offers another. + ended: bool, floor: Rev, items: BTreeMap, } @@ -148,6 +178,15 @@ struct Heard { struct Best { via: N, offer: Offer, + /// It crosses a node this node suspects, which no offer says. + suspect: bool, +} + +impl Best { + fn rank(&self) -> (bool, Reverse, Metric) { + let (draining, priority, cost) = self.offer.rank(); + (draining || self.suspect, priority, cost) + } } /// A forwarding node's claims as this node knows them, complete as of `floor`. @@ -210,7 +249,7 @@ pub(super) struct Kept { #[derive(Clone, Debug)] pub(super) struct Router { me: Id, - suspect: Suspect, + tune: Tune, sessions: u64, peers: BTreeMap, heard: BTreeMap>, @@ -218,8 +257,14 @@ pub(super) struct Router { dead: BTreeMap, /// Nodes at the far end of a closed session, by the near ends that said so. suspects: BTreeMap>, + /// Nodes named as draining that some heard route still crosses. + draining: BTreeSet, + /// Forwarding nodes whose route changed since the neighbours were last told. + stale: BTreeSet, best: BTreeMap>, - sent: BTreeMap>, + /// What each neighbour was offered: `None` once a closed session this node named + /// to it ended the route, until the next offer or its explicit end. + sent: BTreeMap>>, known: BTreeMap>, /// This node's revision of its own claims. rev: Rev, @@ -242,15 +287,17 @@ pub(super) struct Router { } impl Router { - pub fn new(me: Id, suspect: Suspect) -> Self { + pub fn new(me: Id, tune: Tune) -> Self { let mut router = Self { me, - suspect, + tune, sessions: 0, peers: BTreeMap::new(), heard: BTreeMap::new(), dead: BTreeMap::new(), suspects: BTreeMap::new(), + draining: BTreeSet::new(), + stale: BTreeSet::new(), best: BTreeMap::new(), sent: BTreeMap::new(), known: BTreeMap::new(), @@ -270,10 +317,33 @@ impl Router { router } + /// What to send, with every route as it stands now. pub fn drain(&mut self) -> Vec<(N, Msg)> { + self.advertise(); std::mem::take(&mut self.out) } + /// Tell every neighbour of each route that changed. + fn advertise(&mut self) { + if self.tune.quiet && !self.dead.is_empty() { + return; + } + let peers: Vec = self.peers.keys().copied().collect(); + while let Some(origin) = self.stale.pop_first() { + for &peer in &peers { + self.advertise_to(origin, peer); + } + } + } + + /// An input is done: advertise unless a burst is read first, and show readers. + fn done(&mut self) { + if !self.tune.batch { + self.advertise(); + } + self.refresh(); + } + pub fn events(&mut self) -> Vec { std::mem::take(&mut self.events) } @@ -310,10 +380,8 @@ impl Router { } match port.role { Role::Routing => { - let origins: Vec = self.best.keys().copied().chain([self.me]).collect(); - for origin in origins { - self.advertise_to(origin, peer); - } + let me = self.me; + self.stale.extend(self.best.keys().copied().chain([me])); } Role::Scoped => { let broadcasts: Vec = self.shown.keys().cloned().collect(); @@ -323,7 +391,7 @@ impl Router { } Role::Endpoint => {} } - self.refresh(); + self.done(); } pub fn disconnect(&mut self, peer: N) { @@ -357,16 +425,38 @@ impl Router { ), Role::Scoped | Role::Endpoint => {} } - self.refresh(); + self.done(); } /// `peer` sent GOAWAY: every route through it ranks as draining. pub fn goaway(&mut self, peer: N) { - if let Some(peer) = self.peers.get_mut(&peer) { - peer.draining = true; + let Some(peer) = self.peers.get_mut(&peer) else { + return; + }; + peer.draining = true; + let node = peer.id; + match self.tune.drain { + true => self.drains(node, None), + false => self.reselect_all(), + } + self.done(); + } + + /// `node` is draining: `from` said so, or its own GOAWAY when `None`. + fn drains(&mut self, node: Id, from: Option) { + self.work += 1; + if !self.draining.insert(node) { + return; + } + let tell: Vec = (self.sent.iter()) + .filter(|(peer, sent)| Some(**peer) != from && sent.values().flatten().any(|offer| offer.names(node))) + .map(|(&peer, _)| peer) + .collect(); + for peer in tell { + self.out.push((peer, Msg::Drain { node })); } + self.forget(); self.reselect_all(); - self.refresh(); } pub fn recv(&mut self, from: N, msg: Msg) { @@ -378,11 +468,12 @@ impl Router { Msg::Route { origin, offer } if role == Role::Routing => self.route(from, origin, offer), Msg::Claims { origin, changes, floor } if role == Role::Routing => self.claims(from, origin, changes, floor), Msg::Dead { hop, far } if role == Role::Routing => self.closed(hop, far, Some(from)), + Msg::Drain { node } if role == Role::Routing => self.drains(node, Some(from)), Msg::Foreign { changes } if role != Role::Routing => self.adopt(from, changes), // A neighbour sends only what its role allows; anything else would close the session. other => panic!("{other:?} from a {role:?} neighbour"), } - self.refresh(); + self.done(); } fn route(&mut self, from: N, origin: Id, offer: Option) { @@ -395,12 +486,16 @@ impl Router { self.vouch(&offer); let offers = self.heard.entry(origin).or_default(); match offers.get_mut(&from) { - Some(heard) => heard.offer = offer, + Some(heard) => { + heard.offer = offer; + heard.ended = false; + } None => { offers.insert( from, Heard { offer, + ended: false, floor: 0, items: BTreeMap::new(), }, @@ -555,7 +650,7 @@ impl Router { /// Claims from an endpoint or across a scoped grant become this node's own. fn adopt(&mut self, from: N, changes: Vec<(Broadcast, Change)>) { let peer = &self.peers[&from]; - let (cost, scoped) = (peer.port.cost, peer.port.role == Role::Scoped); + let (cost, priority, scoped) = (peer.port.cost, peer.port.priority, peer.port.role == Role::Scoped); for (broadcast, change) in changes { self.work += 1; let change = match change { @@ -563,6 +658,7 @@ impl Router { Change::Live(claim) if claim.chain.contains(&self.me) => Change::Lost, Change::Live(mut claim) => { claim.cost += cost; + claim.priority = claim.priority.min(priority); if scoped { claim.chain.push(self.me); } @@ -590,7 +686,7 @@ impl Router { fn restate(&mut self, broadcast: Broadcast, cause: Change) { let cheapest = (self.local.iter()) .filter_map(|(&peer, claims)| Some((claims.get(&broadcast)?, peer))) - .min_by_key(|(claim, peer)| (claim.cost, claim.chain.len(), *peer)) + .min_by_key(|(claim, peer)| (Reverse(claim.priority), claim.cost, claim.chain.len(), *peer)) .map(|(claim, peer)| (claim.clone(), peer)); let me = self.me; let was = self.known.entry(me).or_default().items.get(&broadcast).cloned(); @@ -620,11 +716,19 @@ impl Router { /// A session closed: this node's own when `from` is `None`. fn closed(&mut self, hop: Hop, far: Id, from: Option) { self.work += 1; + // The neighbour's word ends every route it offered through the session. + if let Some(from) = from { + for heard in self.heard.values_mut().filter_map(|offers| offers.get_mut(&from)) { + heard.ended |= heard.offer.path.contains(&hop); + } + } let named = (self.heard.values()) .flat_map(BTreeMap::values) - .any(|heard| heard.offer.path.contains(&hop)); + .any(|heard| !heard.ended && heard.offer.path.contains(&hop)); let tell: Vec = (self.sent.iter()) - .filter(|(peer, sent)| Some(**peer) != from && sent.values().any(|offer| offer.path.contains(&hop))) + .filter(|(peer, sent)| { + Some(**peer) != from && sent.values().flatten().any(|offer| offer.path.contains(&hop)) + }) .map(|(&peer, _)| peer) .collect(); // The far end's own word about a session lifts what its near end cast on it. @@ -641,9 +745,14 @@ impl Router { } for peer in tell { self.out.push((peer, Msg::Dead { hop, far })); + for offer in self.sent.get_mut(&peer).into_iter().flat_map(BTreeMap::values_mut) { + if offer.as_ref().is_some_and(|offer| offer.path.contains(&hop)) { + *offer = None; + } + } } let answered = self.dead.iter().any(|(word, about)| word.node == far && *about == hop.node); - if self.suspect == Suspect::FarEnd && !answered { + if self.tune.suspect != Suspect::Never && !answered { self.suspects.entry(far).or_default().insert(hop); } } @@ -653,12 +762,12 @@ impl Router { /// Forget each dead hop and suspected node no heard route names. fn forget(&mut self) { - if self.dead.is_empty() && self.suspects.is_empty() { + if self.dead.is_empty() && self.suspects.is_empty() && self.draining.is_empty() { return; } let mut hops = BTreeSet::new(); let mut nodes = BTreeSet::new(); - for heard in self.heard.values().flat_map(BTreeMap::values) { + for heard in self.heard.values().flat_map(BTreeMap::values).filter(|heard| !heard.ended) { for hop in &heard.offer.path { hops.insert(*hop); nodes.insert(hop.node); @@ -667,6 +776,7 @@ impl Router { self.work += hops.len(); self.dead.retain(|hop, _| hops.contains(hop)); self.suspects.retain(|node, _| nodes.contains(node)); + self.draining.retain(|node| nodes.contains(node)); } fn reselect_all(&mut self) { @@ -686,22 +796,34 @@ impl Router { continue; }; let offer = &heard.offer; + if heard.ended { + continue; + } if offer.names(self.me) || offer.path.len() >= MAX_HOPS || offer.path.iter().any(|hop| self.dead.contains_key(hop)) { continue; } - let suspected = offer.path.iter().any(|hop| self.suspects.contains_key(&hop.node)); - let mine = Offer { - path: offer.path.clone(), - draining: offer.draining || peer.draining || suspected, - priority: offer.priority.min(peer.port.priority), - cost: offer.cost + peer.port.cost, - marked: offer.marked || peer.port.upstream, + if self.tune.agree && !self.agrees(origin, via, offer) { + continue; + } + let named = |nodes: &dyn Fn(&Id) -> bool| offer.path.iter().any(|hop| nodes(&hop.node)); + let suspect = named(&|node| self.suspects.contains_key(node)); + let flagged = suspect && self.tune.suspect == Suspect::Flagged; + let mine = Best { + via, + offer: Offer { + path: offer.path.clone(), + draining: offer.draining || peer.draining || flagged || named(&|node| self.draining.contains(node)), + priority: offer.priority.min(peer.port.priority), + cost: offer.cost + peer.port.cost, + marked: offer.marked || peer.port.upstream, + }, + suspect, }; let (draining, priority, cost) = mine.rank(); // Among equal ranks a node stays on its next hop. - let key = (draining, priority, cost, current != Some(via), mine.path.len(), via); + let key = (draining, priority, cost, current != Some(via), mine.offer.path.len(), via); if pick.as_ref().is_none_or(|(_, best)| key < *best) { - pick = Some((Best { via, offer: mine }, key)); + pick = Some((mine, key)); } } let pick = pick.map(|(best, _)| best); @@ -709,7 +831,7 @@ impl Router { return; } let (was, now) = (self.best.contains_key(&origin), pick.is_some()); - let moved = self.best.get(&origin).map(|best| best.offer.rank()) != pick.as_ref().map(|best| best.offer.rank()); + let moved = self.best.get(&origin).map(Best::rank) != pick.as_ref().map(Best::rank); match pick { Some(best) => self.best.insert(origin, best), None => self.best.remove(&origin), @@ -721,10 +843,23 @@ impl Router { self.work += known.items.len(); self.dirty.extend(known.items.keys().cloned()); } - let peers: Vec = self.peers.keys().copied().collect(); - for peer in peers { - self.advertise_to(origin, peer); - } + self.stale.insert(origin); + } + + /// Whether `offer`, heard from `via`, crosses no other neighbour whose own route + /// to `origin` differs: a neighbour's word about its route is newer than anyone's + /// copy of it. + fn agrees(&self, origin: Id, via: N, offer: &Offer) -> bool { + let Some(offers) = self.heard.get(&origin) else { + return true; + }; + (offers.iter()).filter(|(peer, own)| **peer != via && !own.ended).all(|(peer, own)| { + let id = self.peers[peer].id; + match offer.path.iter().position(|hop| hop.node == id) { + Some(at) => offer.path[..=at] == own.offer.path[..], + None => true, + } + }) } /// What `peer` should hear of `origin` now. @@ -762,12 +897,13 @@ impl Router { } let offer = self.offer_for(origin, peer); let sent = self.sent.entry(peer).or_default(); - if sent.get(&origin) == offer.as_ref() { + let was = sent.get(&origin); + if was.is_none_or(|was| was.is_some()) && was.and_then(Option::as_ref) == offer.as_ref() { return; } - let fresh = !sent.contains_key(&origin); + let fresh = was.is_none(); match &offer { - Some(offer) => sent.insert(origin, offer.clone()), + Some(offer) => sent.insert(origin, Some(offer.clone())), None => sent.remove(&origin), }; let started = fresh && offer.is_some(); @@ -803,10 +939,10 @@ impl Router { .filter_map(|&origin| { let claim = self.known.get(&origin)?.items.get(broadcast)?; let rank = match origin == self.me { - true => (false, Reverse(i8::MAX), claim.cost), + true => (false, Reverse(claim.priority), claim.cost), false => { - let (draining, priority, cost) = self.best.get(&origin)?.offer.rank(); - (draining, priority, cost + claim.cost) + let (draining, Reverse(priority), cost) = self.best.get(&origin)?.rank(); + (draining, Reverse(priority.min(claim.priority)), cost + claim.cost) } }; Some(((rank, claim.chain.len(), origin), claim)) @@ -881,13 +1017,17 @@ impl Router { /// from that neighbour. fn export(&mut self, broadcast: &Broadcast, peer: N) { let claim = self.closest(broadcast).and_then(|(origin, claim)| { - let cost = match origin == self.me { + let (cost, priority) = match origin == self.me { true if self.sources.get(broadcast) == Some(&peer) => return None, - true => claim.cost, - false => claim.cost + self.best[&origin].offer.cost, + true => (claim.cost, claim.priority), + false => { + let offer = &self.best[&origin].offer; + (claim.cost + offer.cost, claim.priority.min(offer.priority)) + } }; Some(Claim { cost, + priority, chain: claim.chain.clone(), }) }); @@ -925,7 +1065,7 @@ impl Router { items: self.known.values().map(|known| known.items.len()).sum(), heard: heard().map(|heard| heard.items.len()).sum(), foreign: self.foreign.values().map(BTreeMap::len).sum(), - pending: self.dead.len() + self.suspects.len(), + pending: self.dead.len() + self.suspects.len() + self.draining.len(), } } } @@ -963,7 +1103,7 @@ pub(super) struct Seen { /// A network of [`Router`]s and endpoints, with [`Sim`]'s queues. #[derive(Clone)] pub(super) struct MultiSim { - suspect: Suspect, + tune: Tune, spots: Vec, restarts: Vec, ports: BTreeMap<(Node, Node), Port>, @@ -977,11 +1117,11 @@ pub(super) struct MultiSim { } impl MultiSim { - pub fn new(suspect: Suspect, nodes: usize) -> Self { + pub fn new(tune: Tune, nodes: usize) -> Self { Self { - suspect, + tune, spots: (0..nodes) - .map(|node| Spot::Relay(Box::new(Router::new(hop(node, 0), suspect)))) + .map(|node| Spot::Relay(Box::new(Router::new(hop(node, 0), tune)))) .collect(), restarts: vec![0; nodes], ports: BTreeMap::new(), @@ -1055,6 +1195,7 @@ impl MultiSim { .map(|path| { let claim = Claim { cost: 0, + priority: i8::MAX, chain: Vec::new(), }; ((id.expect("a publisher"), path.clone()), Change::Live(claim)) @@ -1101,7 +1242,7 @@ impl MultiSim { self.restarts[node] += 1; if let Spot::Relay(router) = &self.spots[node] { self.spent += router.work(); - self.spots[node] = Spot::Relay(Box::new(Router::new(hop(node, self.restarts[node]), self.suspect))); + self.spots[node] = Spot::Relay(Box::new(Router::new(hop(node, self.restarts[node]), self.tune))); } } @@ -1139,6 +1280,7 @@ impl MultiSim { *self.lives.entry(broadcast.clone()).or_default() += 1; Change::Live(Claim { cost: 0, + priority: i8::MAX, chain: Vec::new(), }) } @@ -1158,7 +1300,7 @@ impl MultiSim { fn send(&mut self, from: Node, to: Node, msg: Msg) { match &msg { - Msg::Route { .. } | Msg::Dead { .. } => { + Msg::Route { .. } | Msg::Dead { .. } | Msg::Drain { .. } => { self.stats.route_msgs += 1; self.stats.route_items += 1; } @@ -1189,7 +1331,11 @@ impl MultiSim { router.recv(from, msg); } self.stats.delivered += 1; - self.flush(); + // A relay that reads a burst first answers when the round ends. + match self.tune.batch { + true => self.show(), + false => self.flush(), + } true } @@ -1216,16 +1362,25 @@ impl MultiSim { }) } - fn flush(&mut self) { + /// Move every router's output onto the queues and log its reader events. + pub fn flush(&mut self) { for node in 0..self.spots.len() { let Some(router) = self.relay(node) else { continue; }; - let (outs, events) = (router.drain(), router.events()); - for (to, msg) in outs { + for (to, msg) in router.drain() { self.send(node, to, msg); } - for event in events { + } + self.show(); + } + + fn show(&mut self) { + for node in 0..self.spots.len() { + let Some(router) = self.relay(node) else { + continue; + }; + for event in router.events() { let seen = Seen { node, live: self.live(&event.broadcast), From adf73d631b95e2e5f292d2c9caa2728b21c8bc01 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 16:12:50 -0800 Subject: [PATCH 04/13] test(moq-net): the bake-off's tables, fixtures, and nightly sweep; amend the router, switch, and proof quests Co-Authored-By: Claude Fable 5.2 --- quest/m2/cluster-routes/README.md | 2 +- quest/m2/cluster-routes/doubt-holds.md | 5 +- quest/m2/cluster-routes/doubt-keepalive.md | 5 +- quest/m2/cluster-routes/held-after-close.md | 5 +- quest/m2/cluster-routes/route-fanout.md | 5 +- quest/m2/multicast/README.md | 2 +- quest/m2/multicast/bakeoff.md | 90 ------- quest/m2/multicast/proof.md | 21 +- quest/m2/multicast/router.md | 36 ++- quest/m2/multicast/switch.md | 17 ++ quest/m2/multicast/warmth.md | 6 +- rs/moq-net/src/route/tests/bakeoff.rs | 262 ++++++++++++++++++-- rs/moq-net/src/route/tests/compare.rs | 11 +- rs/moq-net/src/route/tests/multi.rs | 121 +++++---- test/route-sweep.sh | 2 + 15 files changed, 410 insertions(+), 180 deletions(-) delete mode 100644 quest/m2/multicast/bakeoff.md diff --git a/quest/m2/cluster-routes/README.md b/quest/m2/cluster-routes/README.md index f8b0b7ea5a3..120d0325e49 100644 --- a/quest/m2/cluster-routes/README.md +++ b/quest/m2/cluster-routes/README.md @@ -82,7 +82,7 @@ The items below were decided 2026-11-10 (maintainer, /quest-plan on lite-07 rout 30. ✅ Decided 2026-11-12 (maintainer, /quest-audit before the cut), after a survey of BGP, EIGRP, and their successors against item 27: most of the path rule already matches that prior art (AS_PATH loop detection, root-cause withdrawal, graceful restart, End-of-RIB, outbound route filtering, and item 29's Only-To-Customer flag). Three gaps are taken, all in [the switch](https://github.com/moq-dev/moq/pull/8731) and [acceptance](/quest/m2/cluster-routes/acceptance.md): **a dead-hop message on a scoped session applies only to the routes heard on that session and casts no doubt**, since in BGP a neighbour withdraws only what it sent, and a scoped peer (a federated relay, later a browser) could otherwise name a core hop it only read in a path and drop healthy routes outside its scope; **holds are bounded in time**, a neighbour's missing answer counting as nothing after a deadline, and a session whose announce stream has stopped draining is closed, which avoids EIGRP's stuck-in-active and BGP's zombie routes (RFC 9687), and is safe because a hold protects presence, not loop freedom; and **three tests before any design change**: neighbours noticing a dead relay up to the 3 s keep-alive apart (to size the doubt timer, measured in hops of delay today), a three-relay priority dispute wheel (RFC 4264's wedgies, since priority ranks above cost), and a primary that fails and returns. No wire bytes change; the first defines what the dead-hop message means. Rejected: declaring the link relation in SETUP and refusing leaked routes on receipt (RFC 9234), since the accepting side already knows the relation from its grant and it adds wire for a misconfiguration; MRAI and route-flap damping, which exist because BGP delays announcements; link state; per-hop signatures. -31. Reopened 2026-11-15 (maintainer, /quest-plan on the whole route layer): ✅ **this rule is the baseline, and [cache-aware multicast](/quest/m2/multicast/README.md) replaces it on lite-07-wip before the cut.** An audit of what is built found requirements the path rule does not meet: its exhaustive checking stops at 3 relays and rests on a timer that outlasts every stale offer, route state follows publishers, nothing prefers a warm copy, and a stalled route is never left. Superseded there: item 18 (every publisher an origin node), item 19 (ROUTE_HOLD), item 26's route per scope, item 27's doubt and remembered dead hops, the Route Key and qualified Node IDs of the contract below, and item 14. Standing: the path with its self-naming rule and the withdrawal that names a dead hop (item 27), revisions and floors, one announce stream per side with its interest set (items 21 to 24), the sticky down-only flag (item 29), and drain, then priority, then cost. This epic's children still land: they are what the [bake-off](/quest/m2/multicast/bakeoff.md) measures against. Rejected: repairing this rule in place; freezing lite-07 with it and redesigning in lite-08. +31. Reopened 2026-11-15 (maintainer, /quest-plan on the whole route layer): ✅ **this rule is the baseline, and [cache-aware multicast](/quest/m2/multicast/README.md) replaces it on lite-07-wip before the cut.** An audit of what is built found requirements the path rule does not meet: its exhaustive checking stops at 3 relays and rests on a timer that outlasts every stale offer, route state follows publishers, nothing prefers a warm copy, and a stalled route is never left. Superseded there: item 18 (every publisher an origin node), item 19 (ROUTE_HOLD), item 26's route per scope, item 27's doubt and remembered dead hops, the Route Key and qualified Node IDs of the contract below, and item 14. Standing: the path with its self-naming rule and the withdrawal that names a dead hop (item 27), revisions and floors, one announce stream per side with its interest set (items 21 to 24), the sticky down-only flag (item 29), and drain, then priority, then cost. This epic's children still land: they are what the [bake-off](https://github.com/moq-dev/moq/pull/8880) measures against. Rejected: repairing this rule in place; freezing lite-07 with it and redesigning in lite-08. The route layer's core landed in [moq-dev/moq#7676](https://github.com/moq-dev/moq/pull/7676): a diff --git a/quest/m2/cluster-routes/doubt-holds.md b/quest/m2/cluster-routes/doubt-holds.md index bb494ab31df..3957ee5207e 100644 --- a/quest/m2/cluster-routes/doubt-holds.md +++ b/quest/m2/cluster-routes/doubt-holds.md @@ -10,7 +10,7 @@ go while its neighbours answer that they have nothing. Reordered 2026-11-15 (maintainer, /quest-plan on [cache-aware multicast](/quest/m2/multicast/README.md)): this waits on the -[bake-off](/quest/m2/multicast/bakeoff.md). The new rule has neither holds nor +[bake-off](https://github.com/moq-dev/moq/pull/8880). The new rule has neither holds nor doubt, so nothing would be left for this to fix. If the new rule goes ahead, [the switch](/quest/m2/multicast/switch.md) deletes this quest with the fixtures that pin it; if it does not, this starts as written. @@ -43,7 +43,8 @@ before changing `rs/moq-net/src/route/path.rs`. ## Required -- [The bake-off](/quest/m2/multicast/bakeoff.md) - decides whether the hold and the doubt this fixes remain +- [The switch](/quest/m2/multicast/switch.md) - deletes this with the old rule, unless the maintainer keeps today's rule on [moq-dev/moq#8880](https://github.com/moq-dev/moq/pull/8880) + ## Related diff --git a/quest/m2/cluster-routes/doubt-keepalive.md b/quest/m2/cluster-routes/doubt-keepalive.md index be6a172a659..8f2305d7ef6 100644 --- a/quest/m2/cluster-routes/doubt-keepalive.md +++ b/quest/m2/cluster-routes/doubt-keepalive.md @@ -9,7 +9,7 @@ apart its neighbours notice the death within the keep-alive. Reordered 2026-11-15 (maintainer, /quest-plan on [cache-aware multicast](/quest/m2/multicast/README.md)): this waits on the -[bake-off](/quest/m2/multicast/bakeoff.md). The maintainer deferred how the +[bake-off](https://github.com/moq-dev/moq/pull/8880). The maintainer deferred how the doubt covers a silent death on 2026-11-15, "not yet sure about the doubt design", and decided the same day that evidence replaces the doubt. If the new rule goes ahead, [the switch](/quest/m2/multicast/switch.md) deletes this @@ -37,7 +37,8 @@ that window goes to a dead relay. A 1 s doubt does not cover a 3 s spread. ## Required -- [The bake-off](/quest/m2/multicast/bakeoff.md) - decides whether the doubt this sizes remains +- [The switch](/quest/m2/multicast/switch.md) - deletes this with the old rule, unless the maintainer keeps today's rule on [moq-dev/moq#8880](https://github.com/moq-dev/moq/pull/8880) + ## Related diff --git a/quest/m2/cluster-routes/held-after-close.md b/quest/m2/cluster-routes/held-after-close.md index 561e0d9edf0..bc00d135ca9 100644 --- a/quest/m2/cluster-routes/held-after-close.md +++ b/quest/m2/cluster-routes/held-after-close.md @@ -11,7 +11,7 @@ route recovers. Reordered 2026-11-15 (maintainer, /quest-plan on [cache-aware multicast](/quest/m2/multicast/README.md)): this waits on the -[bake-off](/quest/m2/multicast/bakeoff.md). The new rule has no hold: a reader +[bake-off](https://github.com/moq-dev/moq/pull/8880). The new rule has no hold: a reader debounces unavailable to offline locally, which answers the same symptom. If the new rule goes ahead, [the switch](/quest/m2/multicast/switch.md) deletes this quest with the fixtures that pin it; if it does not, this starts as @@ -42,7 +42,8 @@ three pinned fixtures into the contract's held-then-available sequence. ## Required -- [The bake-off](/quest/m2/multicast/bakeoff.md) - decides whether the hold this shows remains +- [The switch](/quest/m2/multicast/switch.md) - deletes this with the old rule, unless the maintainer keeps today's rule on [moq-dev/moq#8880](https://github.com/moq-dev/moq/pull/8880) + ## Related diff --git a/quest/m2/cluster-routes/route-fanout.md b/quest/m2/cluster-routes/route-fanout.md index 3b474bce219..f0a172115c8 100644 --- a/quest/m2/cluster-routes/route-fanout.md +++ b/quest/m2/cluster-routes/route-fanout.md @@ -10,7 +10,7 @@ remembered for sessions nothing was learned on. Reordered 2026-11-15 (maintainer, /quest-plan on [cache-aware multicast](/quest/m2/multicast/README.md)): this waits on the -[bake-off](/quest/m2/multicast/bakeoff.md). Its first and third findings are +[bake-off](https://github.com/moq-dev/moq/pull/8880). Its first and third findings are in `path::Router`'s indexes and its doubts and dead hops, which the new router replaces; the second, per-broadcast rewrites in `route/node.rs`, the [router](/quest/m2/multicast/router.md) takes as a requirement. If the new @@ -54,7 +54,8 @@ which also edits `path.rs`. ## Required -- [The bake-off](/quest/m2/multicast/bakeoff.md) - decides whether the path router these findings are in remains +- [The switch](/quest/m2/multicast/switch.md) - deletes this with the old rule, unless the maintainer keeps today's rule on [moq-dev/moq#8880](https://github.com/moq-dev/moq/pull/8880) + ## Related diff --git a/quest/m2/multicast/README.md b/quest/m2/multicast/README.md index a9c81669e2e..b572bf18204 100644 --- a/quest/m2/multicast/README.md +++ b/quest/m2/multicast/README.md @@ -73,7 +73,7 @@ Settled: interoperate for a release; lite-07 without a route layer, a large strip that leaves lite-07 clusters with lite-06's hunting). The route stream and the acceptance fixtures still land: they are the baseline the - [bake-off](/quest/m2/multicast/bakeoff.md) measures against, and one + [bake-off](https://github.com/moq-dev/moq/pull/8880) measures against, and one announce stream per side is wanted by any design. The quests that only polish holds, doubt, or the path router's indexes wait on the bake-off. 3. ✅ **A routing node is a node that forwards and is trusted to; claims, diff --git a/quest/m2/multicast/bakeoff.md b/quest/m2/multicast/bakeoff.md deleted file mode 100644 index c6350036dd9..00000000000 --- a/quest/m2/multicast/bakeoff.md +++ /dev/null @@ -1,90 +0,0 @@ -# [XL] The new route rule is measured against today's - -## Goal - -A test-only prototype of the [epic](/quest/m2/multicast/README.md)'s route -rule runs beside `route::path::Router` on the same meshes and failure -schedules, and the PR reports for both what each requirement costs: messages -and retained state as forwarding nodes, publishers, broadcasts, and links -grow; the time to a settled route and the time a reader goes unfed after each -kind of failure; and every loop, stale route, false END, and revival seen. -The maintainer picks from those numbers whether the rule goes ahead as -decided and with what changes, and the PR amends the -[router](/quest/m2/multicast/router.md), -[switch](/quest/m2/multicast/switch.md), and -[proof](/quest/m2/multicast/proof.md) quests with what it learned. - -## Plan - -Decided 2026-11-15 (maintainer, /quest-plan): the rule's direction is settled -in the epic (items 3, 4, and 6); this measures it before anything is built on -it, as [moq-dev/moq#8533](https://github.com/moq-dev/moq/pull/8533) measured -paths against Babel. - -Build on `rs/moq-net/src/route/tests/`: `Sim` drives a router over a mesh -with messages held and delivered out of step, `compare.rs` ran two routers -over one schedule, and `search.rs` walks every delivery order on a small -mesh. The prototype stays under `tests/`; nothing ships from this PR. - -The rule to prototype: - -- A route per forwarding node, ranked not draining, then the lowest priority - on its path, then cost. A node never uses a route whose path names it. -- A closed session is named once per neighbour, and every node drops every - route through that hop. No moved list, no hold, no doubt, and a dead hop - is forgotten once no heard route names it. -- An endpoint's claims belong to the relay it connects to. A claim crossing - a scoped grant is announced again with the chain of the epic's item 3. -- One mark per link, as `upstream` is today. -- A request is confirmed only once its upstream is, and one fed around a - loop reports no success. Count what that costs readers: this is the - accepted price of having no timer. - -Shapes, each at more than one size: - -- CDN tiers: edges on two cores per region with the cores meshed, up to at - least 32 relays and 123 links at diameter 4. -- Meshes: a ring with log N chords, a sparse random mesh, a clique, and a - mesh with leaves that mark every link. -- Two meshes joined by one scoped link and by two, and three meshes in a - chain and in a ring, where the chain must stop a claim circling. -- One broadcast per publisher, the case where routes per publisher helped - least. - -Failures: a clean close; a silent death its neighbours notice up to the idle -timeout apart; a relay with many neighbours dying; a link that flaps faster -than any timer; a publisher on two relays losing one, then both; a publisher -moving; a GOAWAY drain; a partition that heals; and unrelated failures at the -same instant. - -Today's numbers, to beat or to explain: - -| Case | Today | -| --- | --- | -| A relay with several neighbours dies, 24 relays | 124 route messages, no loop | -| A link closes in a clique, one publisher behind it, then eight | 44, then 198 messages | -| A detour, settled | 12 hops of delay, 6.2 avoiding the far end | -| A publisher moves to another relay, clique of 24 | 1,100 ms dark | -| Churn, 120,000 steps | 32 steps with a route loop | -| A silent death noticed over 3 s | three relays each move onto a dead route twice | -| A link comes up on a 32-relay full mesh | 592 route messages | -| One route change beside 1,000 unrelated streams | 1.09 ms, against 9.8 µs beside 10 | - -The known risk is the reason doubt exists. Paths with no dead-hop message -hunted for 65,219 messages when a publisher left a clique of 12, and with the -dead hop but no doubt, selected routes looped when a relay with several -neighbours died. The epic accepts a loop among selected routes while -requests fail closed; show what it costs in messages and unfed readers. If -that is worse than today, try evidence before any timer: for one, rank every -route through the far end of a closed session as draining until that node's -own word about the session arrives, which reuses the drain and needs no -clock. A timer in the rule goes back to the maintainer. - -Keep the gate fast: the fixtures that pin a property run in the merge gate, -and the long sweeps run nightly from `test/route-sweep.sh`. - -Public API: none. Wire: none. - -## Related - -- [Routes and announces](/quest/m2/cluster-routes/README.md) - where today's numbers were measured diff --git a/quest/m2/multicast/proof.md b/quest/m2/multicast/proof.md index 0343dc184ac..fe4f00aeecb 100644 --- a/quest/m2/multicast/proof.md +++ b/quest/m2/multicast/proof.md @@ -39,6 +39,23 @@ What to show, each a lemma about one message handler: the iteration would remove, so it only arrives sooner. - The chain is the same algebra at the grain of trust boundaries. +Measured 2026-11-15 by the [bake-off](https://github.com/moq-dev/moq/pull/8880), +pending the maintainer's pick there. The rank is the epic's: not draining, +the lowest priority on the path, the summed cost, each link adding at least +1, then the next hop in use, the shorter path, and the neighbour. Three more +lemmas if its recommendations stand: + +- The session floor is a filter: a path naming a pair of nodes at or below the + highest session known closed between them maps to invalid, and the set only + grows, so it removes only routes the iteration would remove. +- A named drain sets a route's drain from its path and the nodes known + draining, and the drain still travels in the offer, so the order stays one + order. +- A claim ranks by the route to its announcer joined with its own priority and + cost; show an echo across two grants never beats the claim it echoes. + +A node advertising once per burst changes no fixed point. + State plainly what the theorem does not give, and where each is covered: - No loop at every instant: [requests](/quest/m2/multicast/requests.md) fail @@ -56,7 +73,3 @@ goes with it in the [switch](/quest/m2/multicast/switch.md). Report how long the Agda check takes. Public API: none. Wire: none. - -## Required - -- [The bake-off](/quest/m2/multicast/bakeoff.md) - fixes the rank and the transformations to prove diff --git a/quest/m2/multicast/router.md b/quest/m2/multicast/router.md index 4edfba9453b..df5f861c1d2 100644 --- a/quest/m2/multicast/router.md +++ b/quest/m2/multicast/router.md @@ -5,7 +5,7 @@ `rs/moq-net/src/route/` holds a production router for the [epic](/quest/m2/multicast/README.md)'s rule beside `path::Router`, sans-IO and driven only by tests until the [switch](/quest/m2/multicast/switch.md). -The [bake-off](/quest/m2/multicast/bakeoff.md)'s fixtures and sweeps run on +The [bake-off](https://github.com/moq-dev/moq/pull/8880)'s fixtures and sweeps run on it, the exhaustive delivery-order search checks it, and a benchmark swept over forwarding nodes, publishers, broadcasts, and sessions shows that an event costs work in what it changed. @@ -44,8 +44,36 @@ for the path rule. that breaks it. The maintainer chose on 2026-11-14 to keep that search over the compiled code and retire the TLA+ route model. -Public API: none. Wire: none. +Measured 2026-11-15 by the [bake-off](https://github.com/moq-dev/moq/pull/8880), +whose prototype is `rs/moq-net/src/route/tests/multi.rs`; its recommendations +stand until the maintainer picks otherwise on that PR: -## Required +- The rule as written hunts: 6.6 million route messages for a partition of a + 32-relay CDN, against 579 today. Build these three with it, each an arm of + the bake-off: the highest session known closed per pair of nodes, kept for + good and refusing any path at or below it (`floor`, a departure from the + epic's "no dead hop remembered"); advertising once per burst of input read + rather than after every message (`batch`); and a node that sent GOAWAY named + once per neighbour as a closed session is (`drain`). +- A closed session's message ends, on that stream, every route its sender + offered through the session; the sender owes each one its next offer or an + explicit end. A neighbour's copy of the claims stays through that, so a + reroute sends no claim again. +- A claim's withdrawal says whether its publisher ended it or its announcer + lost it, so a publisher's session closing shows readers offline and never a + clean end. A claim carries the lowest link priority to its publisher, or a + node prefers its own echo of a claim that crossed two grants and never + settles. +- Not picked, with their costs on the PR: holding adverts until neighbours + agree (`quiet`, which can wait for good on a neighbour that learned a stale + route late), ranking the far end of a closed session as draining + (`suspect`, which shortens a silent death but doubles the messages of a + flap and has no word that always lifts it), and refusing a route a + neighbour's own offer contradicts (`agree`, a small gain). +- Promote `bakeoff.rs`'s fixtures and `sweep_the_prototype_carries_every_failure_at_32_relays`. + Size the reader's debounce from the PR's `fed_ms` column: 375 ms was the + longest for a failure that left a path on 32 relays, a silent death aside. +- Selected routes looped for a moment in partitions and deaths (up to 3 rounds + at 32 relays), which today's never do; requests must fail closed first. -- [The bake-off](/quest/m2/multicast/bakeoff.md) - the pick, and the fixtures this promotes +Public API: none. Wire: none. diff --git a/quest/m2/multicast/switch.md b/quest/m2/multicast/switch.md index 21ab958223a..c3ad87d38bc 100644 --- a/quest/m2/multicast/switch.md +++ b/quest/m2/multicast/switch.md @@ -36,6 +36,23 @@ is a decision to list in the PR: the relay hard-wires on, goes if nothing else needs it; that is a public API change to report. +Measured 2026-11-15 by the [bake-off](https://github.com/moq-dev/moq/pull/8880), +for the wire, pending the maintainer's pick there: + +- The closed-session message names the session and the node at its far end, + and ends every route its sender offered through it, so no end follows per + route. +- A draining node gets a fixed-size message of its own, sent once per + neighbour: flagging each route cost 158,462 route messages for one GOAWAY on + 32 relays, against 1,454 with it and 116 today. +- A claim's end carries one bit, ended or lost, and a claim carries the lowest + link priority to its publisher beside its cost and chain. +- A scoped neighbour is sent the node's closest claim per broadcast, and never + one that came from that neighbour. +- Today's rule cost 300,100 route messages when a publisher left two meshes + joined by two grants; the chain cost none. Keep that shape in the acceptance + fixtures. + Delete with the old rule: - `rs/moq-net/src/route/proof.rs`, `js/net/src/lite/route_key.ts`, diff --git a/quest/m2/multicast/warmth.md b/quest/m2/multicast/warmth.md index ff48f165c53..085c6ca7447 100644 --- a/quest/m2/multicast/warmth.md +++ b/quest/m2/multicast/warmth.md @@ -2,7 +2,7 @@ ## Goal -The [bake-off](/quest/m2/multicast/bakeoff.md)'s harness runs three ways for +The [bake-off](https://github.com/moq-dev/moq/pull/8880)'s harness runs three ways for a relay to learn that a neighbour already has demand for a broadcast, over the prototype route rule, and the PR reports for each: whether a CDN join skips every pop without demand and whether drones with an uplink each settle @@ -108,7 +108,3 @@ lands in whichever wip is current with its implementation, as decided 2026-11-14; the PR says whether the pick changes bytes lite-07 would freeze. Public API: none. Wire: none. - -## Required - -- [The bake-off](/quest/m2/multicast/bakeoff.md) - the harness and the route rule the three run over diff --git a/rs/moq-net/src/route/tests/bakeoff.rs b/rs/moq-net/src/route/tests/bakeoff.rs index 4e0afff7817..babaa069c71 100644 --- a/rs/moq-net/src/route/tests/bakeoff.rs +++ b/rs/moq-net/src/route/tests/bakeoff.rs @@ -21,21 +21,25 @@ fn arms() -> Vec { Arm::with(|tune| tune.suspect = Suspect::Flagged), Arm::with(|tune| tune.suspect = Suspect::Local), Arm::with(|tune| tune.quiet = true), + Arm::with(|tune| tune.floor = true), Arm::with(|tune| tune.batch = true), Arm::with(|tune| tune.agree = true), Arm::with(|tune| tune.drain = true), Arm::with(|tune| { tune.quiet = true; tune.batch = true; - tune.drain = true; }), Arm::with(|tune| { - tune.suspect = Suspect::Local; - tune.quiet = true; + tune.floor = true; tune.batch = true; - tune.drain = true; }), Arm::TUNED, + Arm::with(|tune| { + tune.suspect = Suspect::Flagged; + tune.floor = true; + tune.batch = true; + tune.drain = true; + }), ] } @@ -115,7 +119,11 @@ fn chords(relays: usize, publishers: usize) -> Mesh { } Mesh { relays, - links: links.into_iter().filter(|(a, b)| a != b).map(|(a, b)| (a, b, 1)).collect(), + links: links + .into_iter() + .filter(|(a, b)| a != b) + .map(|(a, b)| (a, b, 1)) + .collect(), ties: BTreeMap::new(), homes: spread(&(0..relays).collect::>(), publishers), claims: 1, @@ -165,7 +173,11 @@ fn joined(count: usize, size: usize, joins: usize, ring: bool, publishers: usize claims: 1, }; for at in 0..count { - (mesh.links).extend(one.links.iter().map(|&(a, b, cost)| (at * size + a, at * size + b, cost))); + (mesh.links).extend( + one.links + .iter() + .map(|&(a, b, cost)| (at * size + a, at * size + b, cost)), + ); if at + 1 == count && !(ring && count > 2) { continue; } @@ -286,6 +298,8 @@ struct Row { faults: Faults, /// Dead hops, doubts, and suspected nodes left once it settled. pending: usize, + /// Pairs of nodes with a session known closed, kept for good. + lasting: usize, /// The hops every relay's request crosses once it settled, summed. hops: usize, /// Whether every relay with a path routed to its publisher once it settled. @@ -303,6 +317,7 @@ fn observe(lab: &mut Lab, change: &dyn Fn(&mut Lab)) -> Row { revivals: after.revivals - before.revivals, }, pending: lab.net.held().pending, + lasting: lab.net.held().lasting, hops: lab.hops(), right: match cost.settled { true => lab.check(), @@ -477,42 +492,118 @@ fn the_prototype_settles_after_churn() { } } -/// The prototype carries every failure on every shape: it settles, every relay with a -/// path routes to its publisher, and no reader sees an end the publisher never made -/// or a life it had seen end. +/// Every failure on every shape under `arm`: it settles, every relay with a path +/// routes to its publisher, and no reader sees an end the publisher never made or a +/// life it had seen end. On three meshes in a ring that includes the chain stopping +/// a claim circling: once the publisher leaves, every relay lets go. +fn carries(arm: Arm, relays: usize) { + for shape in shapes(relays, 2) { + for (failure, row) in fail(arm, &shape.mesh, relays > 16) { + let case = format!("{}: {} on {}", arm.name(), failure.as_str(), shape.name); + assert_eq!(row.right, Ok(()), "{case}"); + assert_eq!(row.faults, Faults::default(), "{case}"); + // A relay that sent GOAWAY stays named while routes cross it. + if failure != Failure::Goaway { + assert_eq!(row.pending, 0, "{case}"); + } + } + } +} + #[test] fn the_prototype_carries_every_failure_on_every_shape() { - for shape in shapes(12, 2) { - for arm in MULTI { - for (failure, row) in fail(arm, &shape.mesh, false) { - let case = format!("{}: {} on {}", arm.name(), failure.as_str(), shape.name); - assert_eq!(row.right, Ok(()), "{case}"); - assert_eq!(row.faults, Faults::default(), "{case}"); + for arm in MULTI { + carries(arm, 9); + } +} + +/// The same at the quest's largest size, for the rule the bake-off recommends. +#[test] +#[ignore = "a sweep; Nightly runs it from test/route-sweep.sh"] +fn sweep_the_prototype_carries_every_failure_at_32_relays() { + carries(Arm::TUNED, 32); +} + +/// A publisher is an endpoint of its relay, so one leaving, moving, or losing a home +/// changes claims and no route, however the mesh is shaped. +#[test] +fn an_endpoint_changes_no_route() { + for shape in shapes(9, 2) { + for (failure, row) in fail(Arm::TUNED, &shape.mesh, false) { + if matches!( + failure, + Failure::Leave | Failure::Move | Failure::LoseOne | Failure::LoseBoth + ) { + assert_eq!(row.cost.sent.msgs, 0, "{} on {}", failure.as_str(), shape.name); } } } } +/// A failure costs route messages per forwarding node, never per publisher behind it. +#[test] +fn a_failure_costs_the_same_with_more_publishers() { + let cost = |publishers: usize| { + let mesh = tiers(3, 3, true, publishers); + let rows = fail(Arm::TUNED, &mesh, false); + let of = |failure: Failure| { + rows.iter() + .find(|(found, _)| *found == failure) + .map(|(_, row)| row.cost.sent.msgs) + }; + (of(Failure::Close), of(Failure::Partition)) + }; + assert_eq!(cost(1), cost(12)); +} + +/// Remembering the highest session closed between a pair of nodes stops a stale +/// offer reviving a path through it, which is most of what the rule as written +/// hunts through when a relay drains and closes. +#[test] +fn a_session_floor_stops_most_of_the_hunting() { + let mesh = chords(12, 2); + let cost = |arm: Arm| { + let rows = fail(arm, &mesh, false); + let drained = rows.iter().find(|(failure, _)| *failure == Failure::Drained); + drained.expect("a transit relay").1.cost.sent.msgs + }; + let (plain, floored) = (cost(Arm::PLAIN), cost(Arm::with(|tune| tune.floor = true))); + assert!(floored * 10 < plain, "{floored} against {plain}"); +} + /// Every failure on every shape, per arm: what it cost and what readers saw. #[test] -#[ignore = "a benchmark; Nightly reports it"] +#[ignore = "a sweep; Nightly runs it from test/route-sweep.sh"] fn benchmark_bakeoff_failures() { println!( - "shape,relays,links,failure,rule,route_messages,claim_messages,settle_ms,fed_ms,unfed,loops,offline,unavailable,false_ends,revivals,pending,hops,right" + "shape,relays,links,failure,rule,route_messages,claim_messages,settle_ms,fed_ms,unfed,loops,offline,unavailable,false_ends,revivals,pending,lasting,hops,right" ); let sizes: Vec = std::env::var("MOQ_BAKEOFF_RELAYS").map_or(vec![12, 32], |sizes| { - sizes.split(',').map(|size| size.parse().expect("MOQ_BAKEOFF_RELAYS")).collect() + sizes + .split(',') + .map(|size| size.parse().expect("MOQ_BAKEOFF_RELAYS")) + .collect() }); for relays in sizes { for shape in shapes(relays, 2) { for arm in arms() { - if arm == Arm::TODAY && !shape.today { + let wanted = |var: &str, name: &str| { + std::env::var(var).is_ok_and(|list| !list.split(',').any(|want| want == name)) + }; + // Without the session floor and a burst read first, a 32-relay mesh + // hunts through millions of messages: the 12-relay rows show it. + let hunts = matches!(arm, Arm::Multi(tune) if !(tune.floor && tune.batch)); + if (arm == Arm::TODAY && !shape.today) + || (relays > 16 && hunts && std::env::var("MOQ_BAKEOFF_ARMS").is_err()) + || wanted("MOQ_BAKEOFF_ARMS", &arm.name()) + || wanted("MOQ_BAKEOFF_SHAPES", shape.name) + { continue; } for (failure, row) in fail(arm, &shape.mesh, relays > 16) { let ms = |rounds: usize| rounds as f64 * HOP.as_secs_f64() * 1000.0; println!( - "{},{},{},{},{},{},{},{:.0},{:.0},{},{},{},{},{},{},{},{},{}", + "{},{},{},{},{},{},{},{:.0},{:.0},{},{},{},{},{},{},{},{},{},{}", shape.name, shape.mesh.relays, shape.mesh.links.len(), @@ -529,11 +620,140 @@ fn benchmark_bakeoff_failures() { row.faults.false_ends, row.faults.revivals, row.pending, + row.lasting, row.hops, - row.right.as_ref().map_or_else(|err| err.replace(',', ";"), |()| "yes".into()), + row.right + .as_ref() + .map_or_else(|err| err.replace(',', ";"), |()| "yes".into()), ); } } } } } + +/// What a mesh costs to build and hold, and what one link closing and one relay +/// dying cost on it, as relays, publishers, and broadcasts grow. +#[test] +#[ignore = "a sweep; Nightly runs it from test/route-sweep.sh"] +fn benchmark_bakeoff_growth() { + println!( + "grows,relays,links,publishers,broadcasts,rule,build_route_messages,build_claim_messages,held_routes,held_claims,held_lasting,close_route_messages,death_route_messages,death_claim_messages,work" + ); + let tiers_of = |relays: usize, publishers: usize, claims: usize| { + let (regions, edges) = match relays { + ..24 => (3, 3), + 24..48 => (4, 6), + _ => (6, 8), + }; + Mesh { + claims, + ..tiers(regions, edges, true, publishers) + } + }; + let mut cases = Vec::new(); + cases.extend([15, 32, 60].map(|relays| ("relays", tiers_of(relays, 8, 1)))); + cases.extend([1, 8, 64, 256].map(|publishers| ("publishers", tiers_of(32, publishers, 1)))); + cases.extend([1, 8, 64].map(|claims| ("broadcasts", tiers_of(32, 8, claims)))); + cases.extend([8, 16, 32].map(|relays| ("links", clique(relays, 8)))); + for (grows, mesh) in cases { + for arm in [Arm::TODAY, Arm::TUNED] { + let tree = mesh.tree(); + let mut lab = Lab::new(arm, &mesh); + lab.coarse = true; + let (built, held) = (lab.net.sent(), lab.net.held()); + let close = lab.tree_link(0, true, &tree).map(|(a, b)| { + let cost = lab.measure(&|lab| lab.cut(a, b)); + lab.mend(a, b); + assert!(lab.settle()); + cost.sent.msgs + }); + let death = transit(&mesh, &tree).map(|relay| lab.measure(&|lab| lab.crash(relay))); + println!( + "{grows},{},{},{},{},{},{},{},{},{},{},{},{},{},{}", + mesh.relays, + mesh.links.len(), + mesh.homes.len(), + mesh.homes.len() * mesh.claims, + arm.name(), + built.msgs, + built.claims, + held.routes, + held.claims, + lab.net.held().lasting, + close.unwrap_or(0), + death.map_or(0, |cost| cost.sent.msgs), + death.map_or(0, |cost| cost.sent.claims), + lab.net.work(), + ); + } + } +} + +/// The cases the routes epic measured today's rule on, for both. +#[test] +#[ignore = "a sweep; Nightly runs it from test/route-sweep.sh"] +fn benchmark_bakeoff_baseline() { + println!("case,rule,route_messages,claim_messages,settle_ms,fed_ms,loops,stale,dark"); + let ms = |rounds: f64| rounds * HOP.as_secs_f64() * 1000.0; + for arm in [Arm::TODAY, Arm::PLAIN, Arm::TUNED] { + let mut rng = XorShift(0x6d6f_7100 + 24); + let sparse: Vec = (0..16).map(|_| Mesh::sparse(24, &mut rng)).collect(); + let groups = [ + ("sparse_24", sparse), + ("clique_24", vec![Mesh::clique(24, 1)]), + ("clique_24_8_publishers", vec![Mesh::clique(24, 8)]), + ]; + for (shape, meshes) in &groups { + let mut all: BTreeMap> = BTreeMap::new(); + for mesh in meshes { + for (change, cost) in run(arm, mesh) { + all.entry(change).or_default().push(cost); + } + } + for (change, costs) in all { + let mean = + |field: fn(&Cost) -> usize| costs.iter().map(field).sum::() as f64 / costs.len() as f64; + println!( + "{shape}:{},{},{:.1},{:.1},{:.0},{:.0},{:.1},{:.1},{:.1}", + change.as_str(), + arm.name(), + mean(|cost| cost.sent.msgs), + mean(|cost| cost.sent.claims), + ms(mean(|cost| cost.rounds)), + ms(mean(|cost| cost.right)), + mean(|cost| cost.loops), + mean(|cost| cost.stale), + mean(|cost| cost.dark), + ); + } + } + // One link comes up on a full mesh of 32. + let mut mesh = Mesh::clique(32, 1); + mesh.links.retain(|&(a, b, _)| (a, b) != (30, 31)); + let mut lab = Lab::new(arm, &mesh); + let cost = lab.measure(&|lab| lab.attach(30, 31)); + println!( + "clique_32:link_comes_up,{},{},{},{:.0},{:.0},{},{},{}", + arm.name(), + cost.sent.msgs, + cost.sent.claims, + ms(cost.rounds as f64), + ms(cost.right as f64), + cost.loops, + cost.stale, + cost.dark, + ); + let churn = churn(arm, 0..400, false); + println!( + "churn_{}_steps,{},{},{},0,0,{},{},{}", + churn.steps, + arm.name(), + churn.sent.msgs, + churn.sent.claims, + churn.loops, + churn.stale, + churn.dark, + ); + } +} diff --git a/rs/moq-net/src/route/tests/compare.rs b/rs/moq-net/src/route/tests/compare.rs index 88f0f8dd128..473498bd366 100644 --- a/rs/moq-net/src/route/tests/compare.rs +++ b/rs/moq-net/src/route/tests/compare.rs @@ -28,14 +28,16 @@ impl Arm { pub const PLAIN: Self = Self::Multi(Tune { suspect: Suspect::Never, quiet: false, + floor: false, batch: false, agree: false, drain: false, }); /// The epic's rule with what the bake-off found worth adding. pub const TUNED: Self = Self::Multi(Tune { - suspect: Suspect::Flagged, - quiet: true, + suspect: Suspect::Never, + quiet: false, + floor: true, batch: true, agree: false, drain: true, @@ -57,6 +59,7 @@ impl Arm { (tune.suspect == Suspect::Flagged, "+suspect"), (tune.suspect == Suspect::Local, "+local"), (tune.quiet, "+quiet"), + (tune.floor, "+floor"), (tune.batch, "+batch"), (tune.agree, "+agree"), (tune.drain, "+drain"), @@ -87,6 +90,8 @@ pub(super) struct Held { pub claims: usize, /// What waits on another node: dead hops, doubts, and suspected nodes. pub pending: usize, + /// What is kept for good: pairs of nodes with a session known closed. + pub lasting: usize, } /// What readers were shown that the publishers never did. @@ -312,6 +317,7 @@ impl Net for Sim { routes: kept.routes, claims: kept.items + kept.heard + kept.sent, pending: kept.dead + kept.doubt + kept.draining, + lasting: 0, } } @@ -421,6 +427,7 @@ impl Net for MultiSim { routes: kept.routes, claims: kept.items + kept.heard + kept.foreign, pending: kept.pending, + lasting: kept.closed, } } diff --git a/rs/moq-net/src/route/tests/multi.rs b/rs/moq-net/src/route/tests/multi.rs index dbbbffacb49..5899b1cbec9 100644 --- a/rs/moq-net/src/route/tests/multi.rs +++ b/rs/moq-net/src/route/tests/multi.rs @@ -5,9 +5,13 @@ //! belong to the relay it connects to, which announces them under its own Node ID //! and revision. A route carries its path, drain, priority, and cost, and a node //! never uses one whose path names it. A closed session is named once per -//! neighbour in a fixed-size [`Msg::Dead`], and a route through that hop is unusable -//! until the neighbour that offered it says what became of it. There is no hold, no -//! doubt, and no timer: [`Router`] has no clock. +//! neighbour in a fixed-size [`Msg::Dead`], which ends every route its sender offered +//! through that hop; one another neighbour offered is unusable until that neighbour +//! says what became of it. There is no hold, no doubt, and no timer: [`Router`] has +//! no clock. +//! +//! [`Tune`] holds what the bake-off tried on top of the rule as written, each +//! switched on its own so its cost and gain are measured apart. None is a timer. //! //! Across a scoped grant only claims travel: the receiver announces each again //! under its own Node ID with the chain of nodes that did so, and refuses one whose @@ -48,6 +52,11 @@ pub(super) struct Tune { /// offered one through it has said so too: it uses its new routes at once and /// advertises them once the news has settled around it. pub quiet: bool, + /// A node keeps, per pair of nodes it heard a closed session for, the highest + /// session known closed, and refuses any path naming that pair at or below it, + /// for good: sessions between a pair are numbered upward, so the fact carries + /// its own order and no stale offer revives the path. + pub floor: bool, /// A node advertises once it has read everything delivered to it, not after /// every message. pub batch: bool, @@ -244,6 +253,8 @@ pub(super) struct Kept { pub foreign: usize, /// Dead hops and suspected nodes. pub pending: usize, + /// Pairs of nodes with a session known closed, kept for good. + pub closed: usize, } #[derive(Clone, Debug)] @@ -257,6 +268,8 @@ pub(super) struct Router { dead: BTreeMap, /// Nodes at the far end of a closed session, by the near ends that said so. suspects: BTreeMap>, + /// The highest session known closed that the first node held to the second. + closed: BTreeMap<(Id, Id), u64>, /// Nodes named as draining that some heard route still crosses. draining: BTreeSet, /// Forwarding nodes whose route changed since the neighbours were last told. @@ -296,6 +309,7 @@ impl Router { heard: BTreeMap::new(), dead: BTreeMap::new(), suspects: BTreeMap::new(), + closed: BTreeMap::new(), draining: BTreeSet::new(), stale: BTreeSet::new(), best: BTreeMap::new(), @@ -466,7 +480,9 @@ impl Router { let role = peer.port.role; match msg { Msg::Route { origin, offer } if role == Role::Routing => self.route(from, origin, offer), - Msg::Claims { origin, changes, floor } if role == Role::Routing => self.claims(from, origin, changes, floor), + Msg::Claims { origin, changes, floor } if role == Role::Routing => { + self.claims(from, origin, changes, floor) + } Msg::Dead { hop, far } if role == Role::Routing => self.closed(hop, far, Some(from)), Msg::Drain { node } if role == Role::Routing => self.drains(node, Some(from)), Msg::Foreign { changes } if role != Role::Routing => self.adopt(from, changes), @@ -722,6 +738,10 @@ impl Router { heard.ended |= heard.offer.path.contains(&hop); } } + if self.tune.floor { + let highest = self.closed.entry((hop.node, far)).or_default(); + *highest = (*highest).max(hop.session); + } let named = (self.heard.values()) .flat_map(BTreeMap::values) .any(|heard| !heard.ended && heard.offer.path.contains(&hop)); @@ -745,13 +765,18 @@ impl Router { } for peer in tell { self.out.push((peer, Msg::Dead { hop, far })); - for offer in self.sent.get_mut(&peer).into_iter().flat_map(BTreeMap::values_mut) { + for (origin, offer) in self.sent.get_mut(&peer).into_iter().flatten() { if offer.as_ref().is_some_and(|offer| offer.path.contains(&hop)) { *offer = None; + // Owed its next offer, or its explicit end. + self.stale.insert(*origin); } } } - let answered = self.dead.iter().any(|(word, about)| word.node == far && *about == hop.node); + let answered = self + .dead + .iter() + .any(|(word, about)| word.node == far && *about == hop.node); if self.tune.suspect != Suspect::Never && !answered { self.suspects.entry(far).or_default().insert(hop); } @@ -767,7 +792,12 @@ impl Router { } let mut hops = BTreeSet::new(); let mut nodes = BTreeSet::new(); - for heard in self.heard.values().flat_map(BTreeMap::values).filter(|heard| !heard.ended) { + for heard in self + .heard + .values() + .flat_map(BTreeMap::values) + .filter(|heard| !heard.ended) + { for hop in &heard.offer.path { hops.insert(*hop); nodes.insert(hop.node); @@ -796,10 +826,16 @@ impl Router { continue; }; let offer = &heard.offer; - if heard.ended { + let floored = offer.path.windows(2).any(|pair| { + (self.closed.get(&(pair[1].node, pair[0].node))).is_some_and(|highest| pair[1].session <= *highest) + }); + if heard.ended || floored { continue; } - if offer.names(self.me) || offer.path.len() >= MAX_HOPS || offer.path.iter().any(|hop| self.dead.contains_key(hop)) { + if offer.names(self.me) + || offer.path.len() >= MAX_HOPS + || offer.path.iter().any(|hop| self.dead.contains_key(hop)) + { continue; } if self.tune.agree && !self.agrees(origin, via, offer) { @@ -821,7 +857,14 @@ impl Router { }; let (draining, priority, cost) = mine.rank(); // Among equal ranks a node stays on its next hop. - let key = (draining, priority, cost, current != Some(via), mine.offer.path.len(), via); + let key = ( + draining, + priority, + cost, + current != Some(via), + mine.offer.path.len(), + via, + ); if pick.as_ref().is_none_or(|(_, best)| key < *best) { pick = Some((mine, key)); } @@ -853,13 +896,15 @@ impl Router { let Some(offers) = self.heard.get(&origin) else { return true; }; - (offers.iter()).filter(|(peer, own)| **peer != via && !own.ended).all(|(peer, own)| { - let id = self.peers[peer].id; - match offer.path.iter().position(|hop| hop.node == id) { - Some(at) => offer.path[..=at] == own.offer.path[..], - None => true, - } - }) + (offers.iter()) + .filter(|(peer, own)| **peer != via && !own.ended) + .all(|(peer, own)| { + let id = self.peers[peer].id; + match offer.path.iter().position(|hop| hop.node == id) { + Some(at) => offer.path[..=at] == own.offer.path[..], + None => true, + } + }) } /// What `peer` should hear of `origin` now. @@ -959,16 +1004,6 @@ impl Router { } } - /// The neighbour this node routes `origin` through. - pub fn via(&self, origin: Id) -> Option { - self.best.get(&origin).map(|best| best.via) - } - - /// The cost of the route this node uses to `origin`. - pub fn cost(&self, origin: Id) -> Option { - self.best.get(&origin).map(|best| best.offer.cost) - } - /// Bring what a reader sees, and what scoped neighbours are told, up to date. fn refresh(&mut self) { while let Some(broadcast) = self.dirty.pop_first() { @@ -1066,6 +1101,7 @@ impl Router { heard: heard().map(|heard| heard.items.len()).sum(), foreign: self.foreign.values().map(BTreeMap::len).sum(), pending: self.dead.len() + self.suspects.len() + self.draining.len(), + closed: self.closed.len(), } } } @@ -1075,7 +1111,10 @@ impl Router { enum Spot { Relay(Box>), /// A publisher: an endpoint that never forwards. - Endpoint { id: Option, claims: BTreeSet }, + Endpoint { + id: Option, + claims: BTreeSet, + }, } /// How one side of a link relates to the other. @@ -1339,20 +1378,6 @@ impl MultiSim { true } - /// Deliver in rounds until every unheld queue is empty. False if that never happens. - pub fn run(&mut self) -> bool { - for _ in 0..10_000 { - let ready = self.ready(); - if ready.is_empty() { - return true; - } - for ((from, to), _) in ready { - self.deliver(from, to); - } - } - false - } - fn live(&self, broadcast: &Broadcast) -> bool { self.spots.iter().enumerate().any(|(node, spot)| match spot { Spot::Endpoint { id, claims } => { @@ -1428,7 +1453,11 @@ impl MultiSim { } pub fn work(&self) -> usize { - self.spent + (0..self.spots.len()).filter_map(|node| self.router(node)).map(Router::work).sum::() + self.spent + + (0..self.spots.len()) + .filter_map(|node| self.router(node)) + .map(Router::work) + .sum::() } pub fn kept(&self) -> Kept { @@ -1442,6 +1471,7 @@ impl MultiSim { out.heard += kept.heard; out.foreign += kept.foreign; out.pending += kept.pending; + out.closed += kept.closed; } out } @@ -1466,5 +1496,8 @@ pub(super) fn revivals(events: &[Seen]) -> usize { /// Ends shown while a connected publisher still claimed the broadcast. pub(super) fn false_ends(events: &[Seen]) -> usize { - events.iter().filter(|event| event.kind == Kind::End && event.live).count() + events + .iter() + .filter(|event| event.kind == Kind::End && event.live) + .count() } diff --git a/test/route-sweep.sh b/test/route-sweep.sh index 14ccfe6adaf..3d82af799b4 100755 --- a/test/route-sweep.sh +++ b/test/route-sweep.sh @@ -16,3 +16,5 @@ MOQ_ROUTE_SCENARIOS=${MOQ_ROUTE_SCENARIOS:-20000} just rs test -P sweep -p moq-n just rs test --profile=sweep -p moq-net --release --run-ignored only -E 'test(/^route::tests::search::/)' # Cache-aware routing's long sweep, from the same seed, settles every churned mesh of copies: 3000 scenarios each with and without held links, about 3 minutes. MOQ_ROUTE_CASES=3000 just rs test -P sweep -p moq-net --release --run-ignored only -E 'test(=route::tests::cache::sweep_cache_routing_long)' +# The multicast bake-off: its prototype carries every failure on every shape at 32 relays, and its tables against the path rule at 12. +MOQ_BAKEOFF_RELAYS=12 just rs test --profile=sweep -p moq-net --release --run-ignored only -E 'test(/^route::tests::bakeoff::(sweep|benchmark)_/)' From 1b9b823ddcd0cb524277b0703d45a3fa1e536a01 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 17:20:18 -0800 Subject: [PATCH 05/13] test(moq-net): fix what the review found, and measure a route per announcing relay A claim across a grant keeps its drain, a newer floor travels with nothing changed, a dying relay's neighbours notice in the same order under every arm, and both routers report selected and heard routes. The claimed arm offers a node's route only once it announces a claim. Co-Authored-By: Claude Fable 5.2 --- quest/m2/multicast/proof.md | 5 +- quest/m2/multicast/router.md | 32 ++++- quest/m2/multicast/switch.md | 5 +- rs/moq-net/src/route/path.rs | 3 + rs/moq-net/src/route/tests/bakeoff.rs | 186 +++++++++++++++++++++++++- rs/moq-net/src/route/tests/compare.rs | 29 +++- rs/moq-net/src/route/tests/mod.rs | 1 + rs/moq-net/src/route/tests/multi.rs | 59 ++++++-- 8 files changed, 292 insertions(+), 28 deletions(-) diff --git a/quest/m2/multicast/proof.md b/quest/m2/multicast/proof.md index fe4f00aeecb..201a63e8337 100644 --- a/quest/m2/multicast/proof.md +++ b/quest/m2/multicast/proof.md @@ -51,8 +51,9 @@ lemmas if its recommendations stand: - A named drain sets a route's drain from its path and the nodes known draining, and the drain still travels in the offer, so the order stays one order. -- A claim ranks by the route to its announcer joined with its own priority and - cost; show an echo across two grants never beats the claim it echoes. +- A claim ranks by the route to its announcer joined with its own drain, + priority, and cost; show an echo across two grants never beats the claim it + echoes. A node advertising once per burst changes no fixed point. diff --git a/quest/m2/multicast/router.md b/quest/m2/multicast/router.md index df5f861c1d2..1c0bfd35dfa 100644 --- a/quest/m2/multicast/router.md +++ b/quest/m2/multicast/router.md @@ -49,21 +49,43 @@ whose prototype is `rs/moq-net/src/route/tests/multi.rs`; its recommendations stand until the maintainer picks otherwise on that PR: - The rule as written hunts: 6.6 million route messages for a partition of a - 32-relay CDN, against 579 today. Build these three with it, each an arm of + 32-relay CDN, against 579 today. Build these four with it, each an arm of the bake-off: the highest session known closed per pair of nodes, kept for good and refusing any path at or below it (`floor`, a departure from the epic's "no dead hop remembered"); advertising once per burst of input read rather than after every message (`batch`); and a node that sent GOAWAY named - once per neighbour as a closed session is (`drain`). + once per neighbour as a closed session is (`drain`); and a node offering a + route to itself only once it has a claim to announce, its endpoints' or one + it announces again across a scoped grant, and from then on until it restarts + (`claimed`). With all four that partition cost 690. +- `claimed` makes the route per announcing relay, not per forwarding node: + held routes fell from 6,576 to 207 on 32 relays with one publisher, under + today's 221. A publisher's first claim on a relay then starts a route at + every node (201 route messages on that CDN), and across scoped grants a + leaving claim's echo can make a gateway announce for the first time + (`a_first_claim_starts_a_route`). Keep the route standing after the last + claim leaves: withdrawing it races the claim's own end, which would show a + reader offline where the publisher ended cleanly. - A closed session's message ends, on that stream, every route its sender offered through the session; the sender owes each one its next offer or an explicit end. A neighbour's copy of the claims stays through that, so a reroute sends no claim again. - A claim's withdrawal says whether its publisher ended it or its announcer lost it, so a publisher's session closing shows readers offline and never a - clean end. A claim carries the lowest link priority to its publisher, or a - node prefers its own echo of a claim that crossed two grants and never - settles. + clean end. A claim carries the lowest link priority to its publisher and + whether the way there is draining, or a node prefers its own echo of a claim + that crossed two grants and never settles: two relays sharing a draining + neighbour and a scoped link traded places for good without the second + (`a_claim_across_a_grant_keeps_its_drain`). +- A node that takes a newer copy of a forwarding node's claims passes the floor + on even when nothing in the copy differs, or a neighbour that heard a claim + in between keeps it after its publisher ended it + (`a_newer_floor_travels_with_nothing_changed`). +- Bound the session floor before it ships: a restarted relay is a new node, so + each restart leaves every node that heard of it entries nothing removes + (`a_session_floor_keeps_an_entry_per_restart`: 4 more per restart of one + relay in three). A node identity that survives a restart, with sessions and + claim revisions that never repeat, bounds it by pairs of nodes. - Not picked, with their costs on the PR: holding adverts until neighbours agree (`quiet`, which can wait for good on a neighbour that learned a stale route late), ranking the far end of a closed session as draining diff --git a/quest/m2/multicast/switch.md b/quest/m2/multicast/switch.md index c3ad87d38bc..fda6a6b9cf8 100644 --- a/quest/m2/multicast/switch.md +++ b/quest/m2/multicast/switch.md @@ -46,7 +46,10 @@ for the wire, pending the maintainer's pick there: neighbour: flagging each route cost 158,462 route messages for one GOAWAY on 32 relays, against 1,454 with it and 116 today. - A claim's end carries one bit, ended or lost, and a claim carries the lowest - link priority to its publisher beside its cost and chain. + link priority to its publisher and a draining bit beside its cost and chain. +- A claims message with no change still carries its floor. +- A node offers a route to itself only once it announces a claim, so a relay + that only forwards or only serves viewers has no route on the wire. - A scoped neighbour is sent the node's closest claim per broadcast, and never one that came from that neighbour. - Today's rule cost 300,100 route messages when a publisher left two meshes diff --git a/rs/moq-net/src/route/path.rs b/rs/moq-net/src/route/path.rs index 8febddc1dd0..dea2a48092f 100644 --- a/rs/moq-net/src/route/path.rs +++ b/rs/moq-net/src/route/path.rs @@ -570,6 +570,8 @@ pub(crate) struct Retained { /// Claims on routes sent. pub sent: usize, pub routes: usize, + /// Routes heard, one per neighbour offering one. + pub offers: usize, pub dead: usize, pub doubt: usize, pub draining: usize, @@ -1897,6 +1899,7 @@ impl Router { .keys() .filter(|(id, _)| !self.origins.contains_key(id)) .count(), + offers: self.sessions.values().map(|session| session.heard.len()).sum(), dead: self.dead.len(), doubt: self.doubted.len(), draining: self.draining.len(), diff --git a/rs/moq-net/src/route/tests/bakeoff.rs b/rs/moq-net/src/route/tests/bakeoff.rs index babaa069c71..7393651f284 100644 --- a/rs/moq-net/src/route/tests/bakeoff.rs +++ b/rs/moq-net/src/route/tests/bakeoff.rs @@ -10,7 +10,7 @@ use std::collections::BTreeSet; use super::compare::*; -use super::multi::{Suspect, Tie}; +use super::multi::{MultiSim, Suspect, Tie}; use super::*; /// The arms measured, in the report's order. @@ -34,6 +34,8 @@ fn arms() -> Vec { tune.batch = true; }), Arm::TUNED, + Arm::LEAN, + Arm::with(|tune| tune.claimed = true), Arm::with(|tune| { tune.suspect = Suspect::Flagged; tune.floor = true; @@ -44,7 +46,7 @@ fn arms() -> Vec { } /// The prototype's arms. -const MULTI: [Arm; 2] = [Arm::PLAIN, Arm::TUNED]; +const MULTI: [Arm; 3] = [Arm::PLAIN, Arm::TUNED, Arm::LEAN]; /// How long a silent death takes its neighbours to notice, in hops of delay: an idle /// timeout of 3 s. @@ -522,6 +524,7 @@ fn the_prototype_carries_every_failure_on_every_shape() { #[ignore = "a sweep; Nightly runs it from test/route-sweep.sh"] fn sweep_the_prototype_carries_every_failure_at_32_relays() { carries(Arm::TUNED, 32); + carries(Arm::LEAN, 32); } /// A publisher is an endpoint of its relay, so one leaving, moving, or losing a home @@ -540,6 +543,54 @@ fn an_endpoint_changes_no_route() { } } +/// With routes only to relays that announce, a publisher leaving still changes no +/// route, since its relay's route stands, but one arriving at a relay that never +/// announced starts that relay's route at every node. Across a scoped grant a +/// leaving claim can do the same: a gateway that hears the other mesh's echo of it +/// for a moment announces it again, its first claim. +#[test] +fn a_first_claim_starts_a_route() { + let (mut echoed, mut started) = (0, 0); + for shape in shapes(9, 2) { + let scoped = shape.mesh.ties.values().any(|tie| *tie == Tie::Scoped); + for (failure, row) in fail(Arm::LEAN, &shape.mesh, false) { + let case = format!("{} on {}", failure.as_str(), shape.name); + match failure { + Failure::Leave | Failure::LoseOne | Failure::LoseBoth if scoped => echoed += row.cost.sent.msgs, + Failure::Leave | Failure::LoseOne | Failure::LoseBoth => assert_eq!(row.cost.sent.msgs, 0, "{case}"), + // Unless the relay it moves to has announced before. + Failure::Move => started += row.cost.sent.msgs, + _ => {} + } + } + } + assert!(started > 0, "no move started a route"); + assert!(echoed > 0, "no gateway announced an echo"); +} + +/// A relay with nothing to announce costs no route anywhere: the routes a mesh +/// holds follow the relays that announce, not the relays. +#[test] +fn a_relay_with_no_claim_offers_no_route() { + let held = |arm: Arm, publishers: usize| { + let lab = Lab::new(arm, &tiers(3, 3, true, publishers)); + lab.net.held() + }; + for publishers in [1, 2, 9] { + let (all, lean, today) = ( + held(Arm::TUNED, publishers), + held(Arm::LEAN, publishers), + held(Arm::TODAY, publishers), + ); + println!("{publishers} publishers: {all:?} {lean:?} {today:?}"); + assert!( + lean.offers * 15 <= all.offers * publishers.min(9) + all.offers, + "{publishers}" + ); + assert!(lean.routes <= today.routes, "{publishers}"); + } +} + /// A failure costs route messages per forwarding node, never per publisher behind it. #[test] fn a_failure_costs_the_same_with_more_publishers() { @@ -571,6 +622,128 @@ fn a_session_floor_stops_most_of_the_hunting() { assert!(floored * 10 < plain, "{floored} against {plain}"); } +/// Deliver everything queued, as one hop of delay. False once nothing is left. +fn deliver(net: &mut MultiSim) -> bool { + let ready = net.ready(); + for &((from, to), count) in &ready { + for _ in 0..count { + net.deliver(from, to); + } + } + net.flush(); + !ready.is_empty() +} + +fn quiesce(net: &mut MultiSim) { + for _ in 0..200 { + if !deliver(net) { + return; + } + } + panic!("never settles"); +} + +fn tune(arm: Arm) -> multi::Tune { + match arm { + Arm::Multi(tune) => tune, + Arm::Path(_) => panic!("today's rule"), + } +} + +/// Two relays that share a draining neighbour and a scoped link each hold the +/// other's copy of its claim. The copy says it is draining too, so neither prefers +/// it to its own route and the two never trade places. +#[test] +fn a_claim_across_a_grant_keeps_its_drain() { + for arm in MULTI { + let mut net = MultiSim::new(tune(arm), 4); + net.client(3); + net.publish(3, 100); + net.link(0, 1, 1, Tie::Peer); + net.link(0, 2, 1, Tie::Peer); + net.link(1, 2, 1, Tie::Scoped); + net.link(3, 0, 1, Tie::Peer); + net.claim(3, "live/0", true); + quiesce(&mut net); + // Both read the GOAWAY before either hears from the other. + net.goaway(0); + quiesce(&mut net); + let broadcast = (100, p("live/0")); + for relay in [1, 2] { + let router = net.router(relay).expect("a relay"); + assert_eq!( + router.pick(&broadcast), + Some(hop(0, 0)), + "relay {relay} under {}", + arm.name() + ); + } + } +} + +/// A relay that takes a newer copy of a node's claims from another neighbour tells +/// its own neighbours the floor even when nothing in the copy differs from what it +/// had, so one that heard a claim in between, by a route since gone, lets go of it. +#[test] +fn a_newer_floor_travels_with_nothing_changed() { + for arm in MULTI { + // 0 announces. 3 hears it through 1, and later 2; 4 hears it through 3 and 5. + let mut net = MultiSim::new(tune(arm), 7); + net.client(6); + net.publish(6, 100); + for (a, b) in [(0, 1), (0, 2), (1, 3), (3, 4), (0, 5), (5, 4), (6, 0)] { + net.link(a, b, 1, Tie::Peer); + } + net.claim(6, "live/a", true); + quiesce(&mut net); + let late = (100, p("live/b")); + let shown = |net: &MultiSim, relay: Node| net.router(relay).expect("a relay").shown(&late); + + // 3 misses the claim starting, which reaches 4 through 5. + net.hold(1, 3); + net.hold(4, 3); + net.claim(6, "live/b", true); + quiesce(&mut net); + assert_eq!((shown(&net, 3), shown(&net, 4)), (None, Some(true))); + + // 5 misses it ending, and then 4 loses its route through 5. + net.hold(0, 5); + net.claim(6, "live/b", false); + quiesce(&mut net); + net.close(5, 4); + quiesce(&mut net); + assert_eq!(shown(&net, 4), Some(true), "nothing has told it yet"); + + // 3 hears a newer copy from 2 that matches what it holds. + net.link(2, 3, 1, Tie::Peer); + quiesce(&mut net); + assert_eq!(shown(&net, 4), None, "under {}", arm.name()); + } +} + +/// The floor's cost: a relay that restarts is a new node, so each restart leaves +/// its neighbours an entry nothing removes. +#[test] +fn a_session_floor_keeps_an_entry_per_restart() { + let mut net = MultiSim::new(tune(Arm::TUNED), 3); + for (a, b) in [(0, 1), (1, 2), (0, 2)] { + net.link(a, b, 1, Tie::Peer); + } + quiesce(&mut net); + let mut kept = Vec::new(); + for _ in 0..8 { + net.reset(0); + quiesce(&mut net); + for peer in [1, 2] { + net.open(0, peer); + } + quiesce(&mut net); + kept.push(net.kept().closed); + } + assert!(kept.windows(2).all(|pair| pair[1] > pair[0]), "{kept:?}"); + println!("pairs kept after each of 8 restarts of one relay in three: {kept:?}"); +} + /// Every failure on every shape, per arm: what it cost and what readers saw. #[test] #[ignore = "a sweep; Nightly runs it from test/route-sweep.sh"] @@ -638,7 +811,7 @@ fn benchmark_bakeoff_failures() { #[ignore = "a sweep; Nightly runs it from test/route-sweep.sh"] fn benchmark_bakeoff_growth() { println!( - "grows,relays,links,publishers,broadcasts,rule,build_route_messages,build_claim_messages,held_routes,held_claims,held_lasting,close_route_messages,death_route_messages,death_claim_messages,work" + "grows,relays,links,publishers,broadcasts,rule,build_route_messages,build_claim_messages,held_routes,heard_routes,held_claims,held_lasting,close_route_messages,death_route_messages,death_claim_messages,work" ); let tiers_of = |relays: usize, publishers: usize, claims: usize| { let (regions, edges) = match relays { @@ -657,7 +830,7 @@ fn benchmark_bakeoff_growth() { cases.extend([1, 8, 64].map(|claims| ("broadcasts", tiers_of(32, 8, claims)))); cases.extend([8, 16, 32].map(|relays| ("links", clique(relays, 8)))); for (grows, mesh) in cases { - for arm in [Arm::TODAY, Arm::TUNED] { + for arm in [Arm::TODAY, Arm::TUNED, Arm::LEAN] { let tree = mesh.tree(); let mut lab = Lab::new(arm, &mesh); lab.coarse = true; @@ -670,7 +843,7 @@ fn benchmark_bakeoff_growth() { }); let death = transit(&mesh, &tree).map(|relay| lab.measure(&|lab| lab.crash(relay))); println!( - "{grows},{},{},{},{},{},{},{},{},{},{},{},{},{},{}", + "{grows},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}", mesh.relays, mesh.links.len(), mesh.homes.len(), @@ -679,6 +852,7 @@ fn benchmark_bakeoff_growth() { built.msgs, built.claims, held.routes, + held.offers, held.claims, lab.net.held().lasting, close.unwrap_or(0), @@ -696,7 +870,7 @@ fn benchmark_bakeoff_growth() { fn benchmark_bakeoff_baseline() { println!("case,rule,route_messages,claim_messages,settle_ms,fed_ms,loops,stale,dark"); let ms = |rounds: f64| rounds * HOP.as_secs_f64() * 1000.0; - for arm in [Arm::TODAY, Arm::PLAIN, Arm::TUNED] { + for arm in [Arm::TODAY, Arm::PLAIN, Arm::TUNED, Arm::LEAN] { let mut rng = XorShift(0x6d6f_7100 + 24); let sparse: Vec = (0..16).map(|_| Mesh::sparse(24, &mut rng)).collect(); let groups = [ diff --git a/rs/moq-net/src/route/tests/compare.rs b/rs/moq-net/src/route/tests/compare.rs index 473498bd366..68c5407f633 100644 --- a/rs/moq-net/src/route/tests/compare.rs +++ b/rs/moq-net/src/route/tests/compare.rs @@ -32,6 +32,7 @@ impl Arm { batch: false, agree: false, drain: false, + claimed: false, }); /// The epic's rule with what the bake-off found worth adding. pub const TUNED: Self = Self::Multi(Tune { @@ -41,6 +42,17 @@ impl Arm { batch: true, agree: false, drain: true, + claimed: false, + }); + /// [`Self::TUNED`], and a node with nothing to announce offers no route to itself. + pub const LEAN: Self = Self::Multi(Tune { + suspect: Suspect::Never, + quiet: false, + floor: true, + batch: true, + agree: false, + drain: true, + claimed: true, }); /// The epic's rule with `tune` changed from how it is written. @@ -63,6 +75,7 @@ impl Arm { (tune.batch, "+batch"), (tune.agree, "+agree"), (tune.drain, "+drain"), + (tune.claimed, "+claimed"), ]; (parts.into_iter().filter(|(on, _)| *on)).fold("multi".to_string(), |name, (_, part)| name + part) } @@ -84,8 +97,10 @@ impl From for Arm { /// What the routers retain, summed over every node. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub(super) struct Held { - /// Routes heard and used. + /// Routes selected: one per publisher today, one per forwarding node in the prototype. pub routes: usize, + /// Routes heard, one per neighbour offering one, selected or not. + pub offers: usize, /// Claims, in every copy a node keeps. pub claims: usize, /// What waits on another node: dead hops, doubts, and suspected nodes. @@ -315,6 +330,7 @@ impl Net for Sim { let kept = self.kept(); Held { routes: kept.routes, + offers: kept.offers, claims: kept.items + kept.heard + kept.sent, pending: kept.dead + kept.doubt + kept.draining, lasting: 0, @@ -424,7 +440,8 @@ impl Net for MultiSim { fn held(&self) -> Held { let kept = self.kept(); Held { - routes: kept.routes, + routes: kept.used, + offers: kept.routes, claims: kept.items + kept.heard + kept.foreign, pending: kept.pending, lasting: kept.closed, @@ -583,6 +600,9 @@ pub(super) struct Lab { linked: BTreeSet<(Node, Node)>, down: BTreeSet, order: XorShift, + /// The order a dying relay's neighbours notice in: its own, so every rule sees the + /// same failure whatever it sent before. + notice: XorShift, cost: Cost, /// Look for loops once a round rather than after every delivery, on meshes too /// large to walk that often. @@ -604,6 +624,7 @@ impl Lab { linked: BTreeSet::new(), down: BTreeSet::new(), order: XorShift(0x5eed_0001), + notice: XorShift(0x5eed_0002), cost: Cost::default(), coarse: false, }; @@ -819,7 +840,7 @@ impl Lab { self.net.mute(relay, peer); } for i in (1..peers.len()).rev() { - peers.swap(i, self.order.next(i as u64 + 1) as usize); + peers.swap(i, self.notice.next(i as u64 + 1) as usize); } self.down.insert(relay); for peer in peers { @@ -848,7 +869,7 @@ impl Lab { self.net.mute(relay, peer); } for i in (1..peers.len()).rev() { - peers.swap(i, self.order.next(i as u64 + 1) as usize); + peers.swap(i, self.notice.next(i as u64 + 1) as usize); } self.down.insert(relay); let gap = rounds / peers.len().max(2).saturating_sub(1); diff --git a/rs/moq-net/src/route/tests/mod.rs b/rs/moq-net/src/route/tests/mod.rs index 35d04d63a20..759a5cf1e9c 100644 --- a/rs/moq-net/src/route/tests/mod.rs +++ b/rs/moq-net/src/route/tests/mod.rs @@ -889,6 +889,7 @@ impl Sim { out.heard += r.heard; out.sent += r.sent; out.routes += r.routes; + out.offers += r.offers; out.dead += r.dead; out.doubt += r.doubt; out.draining += r.draining; diff --git a/rs/moq-net/src/route/tests/multi.rs b/rs/moq-net/src/route/tests/multi.rs index 5899b1cbec9..c4284c8dd8f 100644 --- a/rs/moq-net/src/route/tests/multi.rs +++ b/rs/moq-net/src/route/tests/multi.rs @@ -66,6 +66,10 @@ pub(super) struct Tune { /// A node that sent GOAWAY is named once per neighbour, as a closed session is, /// so every route through it ranks as draining at once. pub drain: bool, + /// A node offers a route to itself only once it has a claim to announce, its + /// endpoints' or one it announces again across a scoped grant, and from then on: + /// nothing routes to a node that announces nothing. + pub claimed: bool, } /// One node on a route's path, and the session it learned the route on. The @@ -109,6 +113,8 @@ pub(super) struct Claim { pub cost: Metric, /// The lowest link priority between the two. pub priority: i8, + /// The way to the publisher crosses a node that sent GOAWAY. + pub draining: bool, /// The nodes that announced it again across a scoped grant, the first first. pub chain: Vec, } @@ -241,6 +247,8 @@ pub(super) struct Event { pub(super) struct Kept { /// Routes heard, one per neighbour offering a forwarding node. pub routes: usize, + /// Routes selected, one per forwarding node. + pub used: usize, /// Hops on those routes' paths. pub hops: usize, /// Routes sent, one per neighbour and forwarding node. @@ -281,6 +289,8 @@ pub(super) struct Router { known: BTreeMap>, /// This node's revision of its own claims. rev: Rev, + /// It has announced a claim, so it offers a route to itself. + announced: bool, /// Claims heard from endpoints and scoped neighbours, by neighbour. local: BTreeMap>, /// The neighbour behind each of this node's own claims. @@ -316,6 +326,7 @@ impl Router { sent: BTreeMap::new(), known: BTreeMap::new(), rev: 0, + announced: false, local: BTreeMap::new(), sources: BTreeMap::new(), foreign: BTreeMap::new(), @@ -636,9 +647,7 @@ impl Router { } } } - if changes.is_empty() { - return; - } + // Even with nothing changed: the floor alone outranks an older copy elsewhere. for (&peer, sent) in &self.sent { if sent.contains_key(&origin) { self.out.push(( @@ -702,7 +711,15 @@ impl Router { fn restate(&mut self, broadcast: Broadcast, cause: Change) { let cheapest = (self.local.iter()) .filter_map(|(&peer, claims)| Some((claims.get(&broadcast)?, peer))) - .min_by_key(|(claim, peer)| (Reverse(claim.priority), claim.cost, claim.chain.len(), *peer)) + .min_by_key(|(claim, peer)| { + ( + claim.draining, + Reverse(claim.priority), + claim.cost, + claim.chain.len(), + *peer, + ) + }) .map(|(claim, peer)| (claim.clone(), peer)); let me = self.me; let was = self.known.entry(me).or_default().items.get(&broadcast).cloned(); @@ -726,6 +743,11 @@ impl Router { }; self.rev += 1; self.known.entry(me).or_default().floor = self.rev; + if !self.announced { + // Its first claim: the route starts, and brings the claim with it. + self.announced = true; + self.stale.insert(me); + } self.publish(me, vec![(broadcast, change)]); } @@ -911,6 +933,9 @@ impl Router { fn offer_for(&self, origin: Id, peer: N) -> Option { let to = self.peers.get(&peer).filter(|to| to.port.role == Role::Routing)?; if origin == self.me { + if self.tune.claimed && !self.announced { + return None; + } return Some(Offer { path: vec![Hop { node: self.me, @@ -984,10 +1009,14 @@ impl Router { .filter_map(|&origin| { let claim = self.known.get(&origin)?.items.get(broadcast)?; let rank = match origin == self.me { - true => (false, Reverse(claim.priority), claim.cost), + true => (claim.draining, Reverse(claim.priority), claim.cost), false => { let (draining, Reverse(priority), cost) = self.best.get(&origin)?.rank(); - (draining, Reverse(priority.min(claim.priority)), cost + claim.cost) + ( + draining || claim.draining, + Reverse(priority.min(claim.priority)), + cost + claim.cost, + ) } }; Some(((rank, claim.chain.len(), origin), claim)) @@ -1052,17 +1081,23 @@ impl Router { /// from that neighbour. fn export(&mut self, broadcast: &Broadcast, peer: N) { let claim = self.closest(broadcast).and_then(|(origin, claim)| { - let (cost, priority) = match origin == self.me { + let (cost, priority, draining) = match origin == self.me { true if self.sources.get(broadcast) == Some(&peer) => return None, - true => (claim.cost, claim.priority), + true => (claim.cost, claim.priority, claim.draining), false => { - let offer = &self.best[&origin].offer; - (claim.cost + offer.cost, claim.priority.min(offer.priority)) + let best = &self.best[&origin]; + let (draining, ..) = best.rank(); + ( + claim.cost + best.offer.cost, + claim.priority.min(best.offer.priority), + claim.draining || draining, + ) } }; Some(Claim { cost, priority, + draining, chain: claim.chain.clone(), }) }); @@ -1095,6 +1130,7 @@ impl Router { let heard = || self.heard.values().flat_map(BTreeMap::values); Kept { routes: heard().count(), + used: self.best.len(), hops: heard().map(|heard| heard.offer.path.len()).sum(), sent: self.sent.values().map(BTreeMap::len).sum(), items: self.known.values().map(|known| known.items.len()).sum(), @@ -1235,6 +1271,7 @@ impl MultiSim { let claim = Claim { cost: 0, priority: i8::MAX, + draining: false, chain: Vec::new(), }; ((id.expect("a publisher"), path.clone()), Change::Live(claim)) @@ -1320,6 +1357,7 @@ impl MultiSim { Change::Live(Claim { cost: 0, priority: i8::MAX, + draining: false, chain: Vec::new(), }) } @@ -1465,6 +1503,7 @@ impl MultiSim { for router in (0..self.spots.len()).filter_map(|node| self.router(node)) { let kept = router.kept(); out.routes += kept.routes; + out.used += kept.used; out.hops += kept.hops; out.sent += kept.sent; out.items += kept.items; From 3faba64997da857f360f8c258f8018304572a6a4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 17:56:26 -0800 Subject: [PATCH 06/13] test(moq-net): bound the session floor by filing it per relay The epoch arm puts a start in the node identity and orders a session by it, so a restart replaces a pair's floor where it added one. Co-Authored-By: Claude Fable 5.2 --- quest/m2/multicast/router.md | 17 ++++---- rs/moq-net/src/route/tests/bakeoff.rs | 56 +++++++++++++++++---------- rs/moq-net/src/route/tests/compare.rs | 15 +++++++ rs/moq-net/src/route/tests/multi.rs | 36 ++++++++++++++--- 4 files changed, 92 insertions(+), 32 deletions(-) diff --git a/quest/m2/multicast/router.md b/quest/m2/multicast/router.md index 1c0bfd35dfa..47ba39dddbe 100644 --- a/quest/m2/multicast/router.md +++ b/quest/m2/multicast/router.md @@ -49,10 +49,11 @@ whose prototype is `rs/moq-net/src/route/tests/multi.rs`; its recommendations stand until the maintainer picks otherwise on that PR: - The rule as written hunts: 6.6 million route messages for a partition of a - 32-relay CDN, against 579 today. Build these four with it, each an arm of + 32-relay CDN, against 579 today. Build these five with it, each an arm of the bake-off: the highest session known closed per pair of nodes, kept for good and refusing any path at or below it (`floor`, a departure from the - epic's "no dead hop remembered"); advertising once per burst of input read + epic's "no dead hop remembered", and `epoch`, below, which bounds it); + advertising once per burst of input read rather than after every message (`batch`); and a node that sent GOAWAY named once per neighbour as a closed session is (`drain`); and a node offering a route to itself only once it has a claim to announce, its endpoints' or one @@ -81,11 +82,13 @@ stand until the maintainer picks otherwise on that PR: on even when nothing in the copy differs, or a neighbour that heard a claim in between keeps it after its publisher ended it (`a_newer_floor_travels_with_nothing_changed`). -- Bound the session floor before it ships: a restarted relay is a new node, so - each restart leaves every node that heard of it entries nothing removes - (`a_session_floor_keeps_an_entry_per_restart`: 4 more per restart of one - relay in three). A node identity that survives a restart, with sessions and - claim revisions that never repeat, bounds it by pairs of nodes. +- Bound the session floor: alone it keeps an entry per restart, since a + restarted relay is a new node. Put the relay and a start that only grows in + the node identity, file the floor per pair of relays, and order a session + by its node's start and then its number (`epoch`): a restart then replaces + the entry, and no message or time changed on any row + (`a_session_floor_keeps_an_entry_per_restart_unless_filed_by_relay`). A + start that repeats or goes back gets the relay's new sessions refused. - Not picked, with their costs on the PR: holding adverts until neighbours agree (`quiet`, which can wait for good on a neighbour that learned a stale route late), ranking the far end of a closed session as draining diff --git a/rs/moq-net/src/route/tests/bakeoff.rs b/rs/moq-net/src/route/tests/bakeoff.rs index 7393651f284..4234e0fb90d 100644 --- a/rs/moq-net/src/route/tests/bakeoff.rs +++ b/rs/moq-net/src/route/tests/bakeoff.rs @@ -35,6 +35,7 @@ fn arms() -> Vec { }), Arm::TUNED, Arm::LEAN, + Arm::KEPT, Arm::with(|tune| tune.claimed = true), Arm::with(|tune| { tune.suspect = Suspect::Flagged; @@ -46,7 +47,7 @@ fn arms() -> Vec { } /// The prototype's arms. -const MULTI: [Arm; 3] = [Arm::PLAIN, Arm::TUNED, Arm::LEAN]; +const MULTI: [Arm; 4] = [Arm::PLAIN, Arm::TUNED, Arm::LEAN, Arm::KEPT]; /// How long a silent death takes its neighbours to notice, in hops of delay: an idle /// timeout of 3 s. @@ -525,6 +526,7 @@ fn the_prototype_carries_every_failure_on_every_shape() { fn sweep_the_prototype_carries_every_failure_at_32_relays() { carries(Arm::TUNED, 32); carries(Arm::LEAN, 32); + carries(Arm::KEPT, 32); } /// A publisher is an endpoint of its relay, so one leaving, moving, or losing a home @@ -722,26 +724,40 @@ fn a_newer_floor_travels_with_nothing_changed() { } /// The floor's cost: a relay that restarts is a new node, so each restart leaves -/// its neighbours an entry nothing removes. +/// its neighbours an entry nothing removes. Filed per pair of relays, with the +/// restart in the session's order, a restart replaces the entry instead. #[test] -fn a_session_floor_keeps_an_entry_per_restart() { - let mut net = MultiSim::new(tune(Arm::TUNED), 3); - for (a, b) in [(0, 1), (1, 2), (0, 2)] { - net.link(a, b, 1, Tie::Peer); - } - quiesce(&mut net); - let mut kept = Vec::new(); - for _ in 0..8 { - net.reset(0); - quiesce(&mut net); - for peer in [1, 2] { - net.open(0, peer); +fn a_session_floor_keeps_an_entry_per_restart_unless_filed_by_relay() { + let kept = |arm: Arm| { + let mut net = MultiSim::new(tune(arm), 3); + for (a, b) in [(0, 1), (1, 2), (0, 2)] { + net.link(a, b, 1, Tie::Peer); } quiesce(&mut net); - kept.push(net.kept().closed); - } - assert!(kept.windows(2).all(|pair| pair[1] > pair[0]), "{kept:?}"); - println!("pairs kept after each of 8 restarts of one relay in three: {kept:?}"); + let mut kept = Vec::new(); + for _ in 0..8 { + net.reset(0); + quiesce(&mut net); + for peer in [1, 2] { + net.open(0, peer); + } + quiesce(&mut net); + kept.push(net.kept().closed); + } + kept + }; + let (grows, bounded) = ( + kept(Arm::TUNED), + kept(Arm::with(|tune| { + tune.floor = true; + tune.epoch = true; + })), + ); + println!("pairs kept after each of 8 restarts of one relay in three: {grows:?}, by relay {bounded:?}"); + assert!(grows.windows(2).all(|pair| pair[1] > pair[0]), "{grows:?}"); + // Two ways round each of the three links at most. + assert!(bounded.iter().all(|&pairs| pairs <= 6), "{bounded:?}"); + assert_eq!(bounded[0], bounded[7], "{bounded:?}"); } /// Every failure on every shape, per arm: what it cost and what readers saw. @@ -830,7 +846,7 @@ fn benchmark_bakeoff_growth() { cases.extend([1, 8, 64].map(|claims| ("broadcasts", tiers_of(32, 8, claims)))); cases.extend([8, 16, 32].map(|relays| ("links", clique(relays, 8)))); for (grows, mesh) in cases { - for arm in [Arm::TODAY, Arm::TUNED, Arm::LEAN] { + for arm in [Arm::TODAY, Arm::TUNED, Arm::LEAN, Arm::KEPT] { let tree = mesh.tree(); let mut lab = Lab::new(arm, &mesh); lab.coarse = true; @@ -870,7 +886,7 @@ fn benchmark_bakeoff_growth() { fn benchmark_bakeoff_baseline() { println!("case,rule,route_messages,claim_messages,settle_ms,fed_ms,loops,stale,dark"); let ms = |rounds: f64| rounds * HOP.as_secs_f64() * 1000.0; - for arm in [Arm::TODAY, Arm::PLAIN, Arm::TUNED, Arm::LEAN] { + for arm in [Arm::TODAY, Arm::PLAIN, Arm::TUNED, Arm::LEAN, Arm::KEPT] { let mut rng = XorShift(0x6d6f_7100 + 24); let sparse: Vec = (0..16).map(|_| Mesh::sparse(24, &mut rng)).collect(); let groups = [ diff --git a/rs/moq-net/src/route/tests/compare.rs b/rs/moq-net/src/route/tests/compare.rs index 68c5407f633..489878f43f7 100644 --- a/rs/moq-net/src/route/tests/compare.rs +++ b/rs/moq-net/src/route/tests/compare.rs @@ -33,6 +33,7 @@ impl Arm { agree: false, drain: false, claimed: false, + epoch: false, }); /// The epic's rule with what the bake-off found worth adding. pub const TUNED: Self = Self::Multi(Tune { @@ -43,6 +44,7 @@ impl Arm { agree: false, drain: true, claimed: false, + epoch: false, }); /// [`Self::TUNED`], and a node with nothing to announce offers no route to itself. pub const LEAN: Self = Self::Multi(Tune { @@ -53,6 +55,18 @@ impl Arm { agree: false, drain: true, claimed: true, + epoch: false, + }); + /// [`Self::LEAN`], with the floor kept per pair of relays: what the bake-off recommends. + pub const KEPT: Self = Self::Multi(Tune { + suspect: Suspect::Never, + quiet: false, + floor: true, + batch: true, + agree: false, + drain: true, + claimed: true, + epoch: true, }); /// The epic's rule with `tune` changed from how it is written. @@ -76,6 +90,7 @@ impl Arm { (tune.agree, "+agree"), (tune.drain, "+drain"), (tune.claimed, "+claimed"), + (tune.epoch, "+epoch"), ]; (parts.into_iter().filter(|(on, _)| *on)).fold("multi".to_string(), |name, (_, part)| name + part) } diff --git a/rs/moq-net/src/route/tests/multi.rs b/rs/moq-net/src/route/tests/multi.rs index c4284c8dd8f..ac73df9ecdb 100644 --- a/rs/moq-net/src/route/tests/multi.rs +++ b/rs/moq-net/src/route/tests/multi.rs @@ -70,6 +70,17 @@ pub(super) struct Tune { /// endpoints' or one it announces again across a scoped grant, and from then on: /// nothing routes to a node that announces nothing. pub claimed: bool, + /// With `floor`: a node's identity carries the relay and how many times it has + /// started, the floor is kept per pair of relays, and a session of a later start + /// is above every session of an earlier one. A restart then replaces a pair's + /// entry where it would add one, so the floor is bounded by pairs of relays. + pub epoch: bool, +} + +/// The relay a simulated node identity names and how many times it had started, +/// as [`hop`] numbers them. A real identity would carry a start that only grows. +fn started(id: Id) -> (Id, u64) { + (id % 1_000, id / 1_000) } /// One node on a route's path, and the session it learned the route on. The @@ -276,8 +287,9 @@ pub(super) struct Router { dead: BTreeMap, /// Nodes at the far end of a closed session, by the near ends that said so. suspects: BTreeMap>, - /// The highest session known closed that the first node held to the second. - closed: BTreeMap<(Id, Id), u64>, + /// The highest session known closed that the first node held to the second, as + /// the start of the first and its session. + closed: BTreeMap<(Id, Id), (u64, u64)>, /// Nodes named as draining that some heard route still crosses. draining: BTreeSet, /// Forwarding nodes whose route changed since the neighbours were last told. @@ -751,6 +763,18 @@ impl Router { self.publish(me, vec![(broadcast, change)]); } + /// The pair of nodes `hop`'s session joins, as the floor files it, and where the + /// session stands in that pair's order. + fn floor_of(&self, hop: Hop, far: Id) -> ((Id, Id), (u64, u64)) { + match self.tune.epoch { + true => { + let ((near, start), (far, _)) = (started(hop.node), started(far)); + ((near, far), (start, hop.session)) + } + false => ((hop.node, far), (0, hop.session)), + } + } + /// A session closed: this node's own when `from` is `None`. fn closed(&mut self, hop: Hop, far: Id, from: Option) { self.work += 1; @@ -761,8 +785,9 @@ impl Router { } } if self.tune.floor { - let highest = self.closed.entry((hop.node, far)).or_default(); - *highest = (*highest).max(hop.session); + let (pair, at) = self.floor_of(hop, far); + let highest = self.closed.entry(pair).or_default(); + *highest = (*highest).max(at); } let named = (self.heard.values()) .flat_map(BTreeMap::values) @@ -849,7 +874,8 @@ impl Router { }; let offer = &heard.offer; let floored = offer.path.windows(2).any(|pair| { - (self.closed.get(&(pair[1].node, pair[0].node))).is_some_and(|highest| pair[1].session <= *highest) + let (key, at) = self.floor_of(pair[1], pair[0].node); + self.closed.get(&key).is_some_and(|highest| at <= *highest) }); if heard.ended || floored { continue; From 039d6db60421aa091a7e8600e3ff114f3fb87a95 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 00:30:40 -0800 Subject: [PATCH 07/13] quest: the multicast route rule goes ahead with five additions The maintainer's pick on the bake-off: floor, epoch, batch, drain, and claimed, with a silent death left to keep-alives and the metadata standby. Co-Authored-By: Claude Fable 5.2 --- quest/m2/multicast/README.md | 80 +++++++++++++++++++++++++++++++----- quest/m2/multicast/proof.md | 18 ++++---- quest/m2/multicast/router.md | 6 ++- quest/m2/multicast/switch.md | 4 +- 4 files changed, 87 insertions(+), 21 deletions(-) diff --git a/quest/m2/multicast/README.md b/quest/m2/multicast/README.md index baa5accabb9..7c51ed28ab5 100644 --- a/quest/m2/multicast/README.md +++ b/quest/m2/multicast/README.md @@ -95,8 +95,9 @@ Settled: the prototype). 4. ✅ **Evidence, not timers.** The route layer is a plain path vector with the dead-hop withdrawal as an accelerator: no ROUTE_HOLD, no doubt, and no - dead hop remembered past the routes that name it. Routing state is a - hint. A request is confirmed only once its upstream is, data is identified + dead hop remembered past the routes that name it (that last clause + reversed 2026-11-16 by item 13: a session floor is kept for good). + Routing state is a hint. A request is confirmed only once its upstream is, data is identified by Instance, track, group, and frame, and a stale or stalled route is left on evidence ([requests](/quest/m2/multicast/requests.md), [route health](/quest/m3/route-health/README.md)). What a hold protected, @@ -168,20 +169,77 @@ flagged as costly, on standby wherever a second route exists, which about doubles a track's subscription state across the cluster). -The shape those decisions give, for the children to confirm or improve; each -departure is a decision to list in its PR: +Decided 2026-11-16 (maintainer, "go for it" on all three, on +[the bake-off](https://github.com/moq-dev/moq/pull/8880#issuecomment-6911055514), +whose tables hold every number here; 32 relays on CDN tiers with 2 +publishers, in route messages, unless noted): + +13. ✅ **The new rule replaces today's, with five additions, none a timer** + (rejected: the rule as written, which hunts through 6,625,456 for a + partition against today's 579 and 1,368,094 for a relay that drains and + closes; keeping today's rule, which leaves readers unfed about 1 s when + a link closes beside a detour, grows with publishers to 73,472 selected + routes at 256, and cannot cross two scoped grants). With all five a + partition costs 690, a link closing feeds every reader at once, and a + publisher leaving changes no route. No arm showed a false END or a + revival. + - `floor`: per pair of relays, a node keeps the highest session it knows + closed and refuses any path naming that pair at or below it, for good. + This reverses item 4's "no dead hop remembered past the routes that + name it": that clause is what the rule hunted through, a stale offer + arriving after the hop was forgotten (rejected: `quiet`, holding + adverts until neighbours agree, which can wait for good and cost 4,641 + against 2,037 as written for a relay dying on a 12-relay chorded ring; + `agree`, refusing a route a neighbour's own offer contradicts, a small + gain `floor` covers). + - `epoch`: a relay's identity carries a start that only grows, and a + session orders by its relay's start and then its number, so a restart + replaces a pair's floor where it added one: 4 pairs after each of 8 + restarts against 4 more per restart, with no row of any table changed. + - `batch`: a node advertises once it has read what was delivered, not + after every message. `floor` alone still spent 28.8 million on one + GOAWAY of a 32-relay chorded ring; with `batch`, 5,065. + - `drain`: a node that sent GOAWAY is named once per neighbour in a + message of its own, as a closed session is: 5,065 to 723. + - `claimed`: a node offers a route to itself only once it announces a + claim, and from then on until it restarts, so a route is per announcing + relay. Building the mesh for one publisher fell from 6,576 to 207, + under today's 221, and the partition from 7,549 to 690. A publisher's + first claim on a relay then costs a route at every node (201 for a + move). The route stands after the last claim leaves (rejected: + withdrawing it, which races the claim's own end and would show a + reader offline where the publisher ended cleanly). +14. ✅ **A silent death is left to keep-alives and the opt-in metadata + standby** ([route health](/quest/m3/route-health/README.md)). No arm + fixed it: readers behind a relay that dies unnoticed for 3 s stay unfed + until the last session notices, as today (rejected: `suspect`, ranking + the far end of a closed session as draining until its own word arrives, + which shortened some silent deaths, nearly doubled a flap from 656 to + 1,133, and left 20 to 42 suspicions that never lifted). + +The shape those decisions give. The bake-off confirmed or changed each line; +a further departure is a decision to list in its PR: - **Identity.** A path and an Instance name content, and a request names the Instance. A relay holds one upstream subscription per Instance and track, whichever origin node its downstream requests named, so a join stops at the first node already carrying it. -- **Reachability.** One route per forwarding node: its path, drain, priority, - and cost. A node never uses a route whose path names it. Rank is not - draining, then the lowest priority on the path, then the summed cost, each - link adding at least 1, so extending a route never improves it. -- **Claims.** A prefix, Instance, Epoch, and cost, announced by a forwarding - node for itself and its local endpoints, ordered by the node's revision and - certified by floors as today, which measured no revival and no false END. +- **Reachability.** One route per announcing relay, offered once it has a + claim: its path, drain, priority, and cost. A node never uses a route + whose path names it, sends none to a neighbour already on the path, and + refuses one at or below a session floor. Rank is not draining, then the + lowest priority on the path, then the summed cost, each link adding at + least 1, so extending a route never improves it. A closed session and a + draining node are each named once per neighbour, and a closed session's + message ends every route its sender offered through it. A node advertises + once per burst of input. +- **Claims.** A prefix, Instance, Epoch, cost, the lowest link priority to + its publisher, and whether the way there is draining, announced by a + forwarding node for itself and its local endpoints, ordered by the node's + revision and certified by floors as today, which measured no revival and + no false END. A claims message carries its floor even with nothing + changed, and a withdrawal says whether the publisher ended the claim or + its announcer lost it. - **Boundaries.** The chain of item 3. - **Requests.** A request names the origin node its first hop chose, so relays that rank differently never loop it, and fails closed. diff --git a/quest/m2/multicast/proof.md b/quest/m2/multicast/proof.md index 201a63e8337..2f94ab1cb0b 100644 --- a/quest/m2/multicast/proof.md +++ b/quest/m2/multicast/proof.md @@ -40,14 +40,18 @@ What to show, each a lemma about one message handler: - The chain is the same algebra at the grain of trust boundaries. Measured 2026-11-15 by the [bake-off](https://github.com/moq-dev/moq/pull/8880), -pending the maintainer's pick there. The rank is the epic's: not draining, -the lowest priority on the path, the summed cost, each link adding at least -1, then the next hop in use, the shorter path, and the neighbour. Three more -lemmas if its recommendations stand: +and decided 2026-11-16 +([maintainer](https://github.com/moq-dev/moq/pull/8880#issuecomment-6911055514)). +The rank is the epic's: not draining, the lowest priority on the path, the +summed cost, each link adding at least 1, then the next hop in use, the +shorter path, and the neighbour. Four more lemmas for what it added: -- The session floor is a filter: a path naming a pair of nodes at or below the - highest session known closed between them maps to invalid, and the set only - grows, so it removes only routes the iteration would remove. +- The session floor is a filter: a path naming a pair of relays at or below + the highest session known closed between them maps to invalid, a session + ordering by its relay's start and then its number, and the set only grows, + so it removes only routes the iteration would remove. +- A node that announces no claim offers no route to itself, which removes + only routes no request names. - A named drain sets a route's drain from its path and the nodes known draining, and the drain still travels in the offer, so the order stays one order. diff --git a/quest/m2/multicast/router.md b/quest/m2/multicast/router.md index 47ba39dddbe..012b7f42064 100644 --- a/quest/m2/multicast/router.md +++ b/quest/m2/multicast/router.md @@ -45,8 +45,10 @@ for the path rule. the compiled code and retire the TLA+ route model. Measured 2026-11-15 by the [bake-off](https://github.com/moq-dev/moq/pull/8880), -whose prototype is `rs/moq-net/src/route/tests/multi.rs`; its recommendations -stand until the maintainer picks otherwise on that PR: +whose prototype is `rs/moq-net/src/route/tests/multi.rs`, and decided +2026-11-16 +([maintainer](https://github.com/moq-dev/moq/pull/8880#issuecomment-6911055514), +the [epic](/quest/m2/multicast/README.md)'s items 13 and 14): - The rule as written hunts: 6.6 million route messages for a partition of a 32-relay CDN, against 579 today. Build these five with it, each an arm of diff --git a/quest/m2/multicast/switch.md b/quest/m2/multicast/switch.md index fda6a6b9cf8..67a709d2815 100644 --- a/quest/m2/multicast/switch.md +++ b/quest/m2/multicast/switch.md @@ -37,7 +37,9 @@ is a decision to list in the PR: API change to report. Measured 2026-11-15 by the [bake-off](https://github.com/moq-dev/moq/pull/8880), -for the wire, pending the maintainer's pick there: +and decided 2026-11-16 +([maintainer](https://github.com/moq-dev/moq/pull/8880#issuecomment-6911055514)), +for the wire: - The closed-session message names the session and the node at its far end, and ends every route its sender offered through it, so no end follows per From 29adfb26bc5e9d7ddfd344b21f4128d911a44c3f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 00:53:12 -0800 Subject: [PATCH 08/13] test(moq-net): distinct gateways in a mesh of three, and a filter named for what it skips Pullfrog's findings on the bake-off: a three-relay middle mesh entered and left by one relay, so the gate never routed a claim between two grants. Co-Authored-By: Claude Fable 5.2 --- quest/m2/multicast/router.md | 2 +- rs/moq-net/src/route/tests/bakeoff.rs | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/quest/m2/multicast/router.md b/quest/m2/multicast/router.md index 012b7f42064..eebecdddaf9 100644 --- a/quest/m2/multicast/router.md +++ b/quest/m2/multicast/router.md @@ -60,7 +60,7 @@ the [epic](/quest/m2/multicast/README.md)'s items 13 and 14): once per neighbour as a closed session is (`drain`); and a node offering a route to itself only once it has a claim to announce, its endpoints' or one it announces again across a scoped grant, and from then on until it restarts - (`claimed`). With all four that partition cost 690. + (`claimed`). With all five that partition cost 690. - `claimed` makes the route per announcing relay, not per forwarding node: held routes fell from 6,576 to 207 on 32 relays with one publisher, under today's 221. A publisher's first claim on a relay then starts a route at diff --git a/rs/moq-net/src/route/tests/bakeoff.rs b/rs/moq-net/src/route/tests/bakeoff.rs index 4234e0fb90d..097267d04ec 100644 --- a/rs/moq-net/src/route/tests/bakeoff.rs +++ b/rs/moq-net/src/route/tests/bakeoff.rs @@ -186,8 +186,9 @@ fn joined(count: usize, size: usize, joins: usize, ring: bool, publishers: usize } let next = (at + 1) % count; for join in 0..joins { - // Each boundary leaves a mesh at another relay than the one it entered by. - let (a, b) = (at * size + size / 2 + join, next * size + 1 + join); + // Each boundary leaves a mesh at another relay than the one it entered by, + // even in a mesh of three, so a claim crosses the mesh between two grants. + let (a, b) = (at * size + (size / 2).max(2) + join, next * size + 1 + join); mesh.links.push((a, b, 1)); mesh.ties.insert((a, b), Tie::Scoped); } @@ -776,7 +777,7 @@ fn benchmark_bakeoff_failures() { for relays in sizes { for shape in shapes(relays, 2) { for arm in arms() { - let wanted = |var: &str, name: &str| { + let unwanted = |var: &str, name: &str| { std::env::var(var).is_ok_and(|list| !list.split(',').any(|want| want == name)) }; // Without the session floor and a burst read first, a 32-relay mesh @@ -784,8 +785,8 @@ fn benchmark_bakeoff_failures() { let hunts = matches!(arm, Arm::Multi(tune) if !(tune.floor && tune.batch)); if (arm == Arm::TODAY && !shape.today) || (relays > 16 && hunts && std::env::var("MOQ_BAKEOFF_ARMS").is_err()) - || wanted("MOQ_BAKEOFF_ARMS", &arm.name()) - || wanted("MOQ_BAKEOFF_SHAPES", shape.name) + || unwanted("MOQ_BAKEOFF_ARMS", &arm.name()) + || unwanted("MOQ_BAKEOFF_SHAPES", shape.name) { continue; } From 7fb0d7dfa4722b5467ba950583a7926b8a9d706f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 01:02:59 -0800 Subject: [PATCH 09/13] test(moq-net): refuse a joined mesh too small for its gateways Co-Authored-By: Claude Fable 5.2 --- rs/moq-net/src/route/tests/bakeoff.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/route/tests/bakeoff.rs b/rs/moq-net/src/route/tests/bakeoff.rs index 097267d04ec..90d4d08d998 100644 --- a/rs/moq-net/src/route/tests/bakeoff.rs +++ b/rs/moq-net/src/route/tests/bakeoff.rs @@ -167,6 +167,12 @@ fn leafy(core: usize, leaves: usize, publishers: usize) -> Mesh { /// `count` chorded rings of `size` relays, each joined to the next by `joins` /// scoped links, and the last to the first when `ring`. Publishers join the first. fn joined(count: usize, size: usize, joins: usize, ring: bool, publishers: usize) -> Mesh { + // Each mesh leaves by relays past the ones it is entered by. + let exit = (size / 2).max(2); + assert!( + exit + joins <= size, + "a mesh of {size} relays has no room for {joins} gateways each way" + ); let one = chords(size, 1); let mut mesh = Mesh { relays: count * size, @@ -188,7 +194,7 @@ fn joined(count: usize, size: usize, joins: usize, ring: bool, publishers: usize for join in 0..joins { // Each boundary leaves a mesh at another relay than the one it entered by, // even in a mesh of three, so a claim crosses the mesh between two grants. - let (a, b) = (at * size + (size / 2).max(2) + join, next * size + 1 + join); + let (a, b) = (at * size + exit + join, next * size + 1 + join); mesh.links.push((a, b, 1)); mesh.ties.insert((a, b), Tie::Scoped); } From 48e8d88fd306b630aec4f5be103cd9432120ef94 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 01:12:23 -0800 Subject: [PATCH 10/13] quest: what bounds the session floor, and a debounce sized from the path Co-Authored-By: Claude Fable 5.2 --- quest/m2/multicast/router.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/quest/m2/multicast/router.md b/quest/m2/multicast/router.md index eebecdddaf9..a30030cfc02 100644 --- a/quest/m2/multicast/router.md +++ b/quest/m2/multicast/router.md @@ -91,6 +91,11 @@ the [epic](/quest/m2/multicast/README.md)'s items 13 and 14): the entry, and no message or time changed on any row (`a_session_floor_keeps_an_entry_per_restart_unless_filed_by_relay`). A start that repeats or goes back gets the relay's new sessions refused. + The bound is then pairs of relay identities ever heard closed: fixed where + relays are configured, as on a CDN, and growing by a departed device's + pairs on a mesh whose devices mint identities. Measure that growth on the + mesh shapes, and list any eviction it needs as a decision in this PR; one + that waits on a clock is the maintainer's. - Not picked, with their costs on the PR: holding adverts until neighbours agree (`quiet`, which can wait for good on a neighbour that learned a stale route late), ranking the far end of a closed session as draining @@ -98,9 +103,12 @@ the [epic](/quest/m2/multicast/README.md)'s items 13 and 14): flap and has no word that always lifts it), and refusing a route a neighbour's own offer contradicts (`agree`, a small gain). - Promote `bakeoff.rs`'s fixtures and `sweep_the_prototype_carries_every_failure_at_32_relays`. - Size the reader's debounce from the PR's `fed_ms` column: 375 ms was the - longest for a failure that left a path on 32 relays, a silent death aside. + Size the reader's debounce from the longest path a route may take times a + hop's delay, not from a sample: the PR's `fed_ms` column is what was seen, + 775 ms at most for a failure that left a path on 32 relays (the + publisher's relay dying on the full mesh), a silent death aside, and a + longer path settles later. - Selected routes looped for a moment in partitions and deaths (up to 3 rounds - at 32 relays), which today's never do; requests must fail closed first. + at 32 relays, where today's reached 4); requests must fail closed first. Public API: none. Wire: none. From a0613335916cb233d90c8f772ca373f8a0a7ef45 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 01:36:56 -0800 Subject: [PATCH 11/13] test(moq-net): answer the online review of the bake-off A scoped neighbour's GOAWAY drains the claims it gave, an explicit end survives a change of source, a closed session takes its unsent output, and the tables name what their columns count. Co-Authored-By: Claude Fable 5.2 --- quest/m2/multicast/router.md | 5 +- rs/moq-net/src/route/tests/bakeoff.rs | 91 +++++++++++++++++++-------- rs/moq-net/src/route/tests/compare.rs | 8 ++- rs/moq-net/src/route/tests/multi.rs | 35 +++++++++-- 4 files changed, 103 insertions(+), 36 deletions(-) diff --git a/quest/m2/multicast/router.md b/quest/m2/multicast/router.md index a30030cfc02..a34bd67bcc7 100644 --- a/quest/m2/multicast/router.md +++ b/quest/m2/multicast/router.md @@ -79,7 +79,10 @@ the [epic](/quest/m2/multicast/README.md)'s items 13 and 14): whether the way there is draining, or a node prefers its own echo of a claim that crossed two grants and never settles: two relays sharing a draining neighbour and a scoped link traded places for good without the second - (`a_claim_across_a_grant_keeps_its_drain`). + (`a_claim_across_a_grant_keeps_its_drain`). A scoped neighbour's + GOAWAY drains the claims it gave the same way + (`a_scoped_neighbour_draining_drains_its_claims`), and a session that + closes takes what was still unsent to it. - A node that takes a newer copy of a forwarding node's claims passes the floor on even when nothing in the copy differs, or a neighbour that heard a claim in between keeps it after its publisher ended it diff --git a/rs/moq-net/src/route/tests/bakeoff.rs b/rs/moq-net/src/route/tests/bakeoff.rs index 90d4d08d998..f32f4c37d58 100644 --- a/rs/moq-net/src/route/tests/bakeoff.rs +++ b/rs/moq-net/src/route/tests/bakeoff.rs @@ -166,13 +166,18 @@ fn leafy(core: usize, leaves: usize, publishers: usize) -> Mesh { /// `count` chorded rings of `size` relays, each joined to the next by `joins` /// scoped links, and the last to the first when `ring`. Publishers join the first. -fn joined(count: usize, size: usize, joins: usize, ring: bool, publishers: usize) -> Mesh { - // Each mesh leaves by relays past the ones it is entered by. - let exit = (size / 2).max(2); - assert!( - exit + joins <= size, - "a mesh of {size} relays has no room for {joins} gateways each way" - ); +/// `None` when a mesh of `size` has no room for its gateways. +fn joined(count: usize, size: usize, joins: usize, ring: bool, publishers: usize) -> Option { + // A mesh that is both entered and left, any but the ends of a chain, leaves by + // relays past the ones it is entered by, so a claim crosses it between grants. + let through = ring || count > 2; + let exit = match through { + true => (size / 2).max(1 + joins), + false => size / 2, + }; + if size < 2 || exit + joins > size { + return None; + } let one = chords(size, 1); let mut mesh = Mesh { relays: count * size, @@ -192,14 +197,12 @@ fn joined(count: usize, size: usize, joins: usize, ring: bool, publishers: usize } let next = (at + 1) % count; for join in 0..joins { - // Each boundary leaves a mesh at another relay than the one it entered by, - // even in a mesh of three, so a claim crosses the mesh between two grants. let (a, b) = (at * size + exit + join, next * size + 1 + join); mesh.links.push((a, b, 1)); mesh.ties.insert((a, b), Tie::Scoped); } } - mesh + Some(mesh) } /// Every shape, at about `relays` relays each. @@ -210,32 +213,42 @@ fn shapes(relays: usize, publishers: usize) -> Vec { mesh, today: true, }; + // Today's rule cannot run three meshes: a qualified identity crosses one grant. + let grants = |name, today, mesh: Option| mesh.map(|mesh| Shape { name, mesh, today }); let (regions, edges) = match relays { ..12 => (2, 2), 12..24 => (3, 3), 24..48 => (4, 6), _ => (6, 8), }; - vec![ + let mut shapes = vec![ open("tiers", tiers(regions, edges, true, publishers)), open("tiers_unpeered", tiers(regions, edges, false, publishers)), open("chords", chords(relays, publishers)), open("sparse", sparse(relays, publishers, &mut rng)), open("clique", clique(relays, publishers)), open("leaves", leafy(relays / 2, relays - relays / 2, publishers)), - open("two_meshes_one_grant", joined(2, relays / 2, 1, false, publishers)), - open("two_meshes_two_grants", joined(2, relays / 2, 2, false, publishers)), - Shape { - name: "three_meshes_chain", - mesh: joined(3, relays / 3, 1, false, publishers), - today: false, - }, - Shape { - name: "three_meshes_ring", - mesh: joined(3, relays / 3, 1, true, publishers), - today: false, - }, - ] + ]; + // A mesh too small for its gateways is left out. + shapes.extend( + [ + grants( + "two_meshes_one_grant", + true, + joined(2, relays / 2, 1, false, publishers), + ), + grants( + "two_meshes_two_grants", + true, + joined(2, relays / 2, 2, false, publishers), + ), + grants("three_meshes_chain", false, joined(3, relays / 3, 1, false, publishers)), + grants("three_meshes_ring", false, joined(3, relays / 3, 1, true, publishers)), + ] + .into_iter() + .flatten(), + ); + shapes } /// The failures measured, each from a settled mesh. @@ -690,6 +703,30 @@ fn a_claim_across_a_grant_keeps_its_drain() { } } +/// A scoped neighbour that sends GOAWAY is draining for the claims it gave, so a +/// node holding the same claim from another neighbour announces that one instead. +#[test] +fn a_scoped_neighbour_draining_drains_its_claims() { + for arm in MULTI { + // A publisher on 0 and 2, each of which grants 1 the claim, 0 the cheaper. + let mut net = MultiSim::new(tune(arm), 4); + net.client(3); + net.publish(3, 100); + net.link(3, 0, 1, Tie::Peer); + net.link(3, 2, 1, Tie::Peer); + net.link(0, 1, 1, Tie::Scoped); + net.link(2, 1, 2, Tie::Scoped); + net.claim(3, "live/0", true); + quiesce(&mut net); + let broadcast = (100, p("live/0")); + let source = |net: &MultiSim| net.router(1).expect("a relay").step(hop(1, 0), &broadcast); + assert_eq!(source(&net), Some(multi::Next::Local(0)), "under {}", arm.name()); + net.goaway(0); + quiesce(&mut net); + assert_eq!(source(&net), Some(multi::Next::Local(2)), "under {}", arm.name()); + } +} + /// A relay that takes a newer copy of a node's claims from another neighbour tells /// its own neighbours the floor even when nothing in the copy differs from what it /// had, so one that heard a claim in between, by a route since gone, lets go of it. @@ -772,7 +809,7 @@ fn a_session_floor_keeps_an_entry_per_restart_unless_filed_by_relay() { #[ignore = "a sweep; Nightly runs it from test/route-sweep.sh"] fn benchmark_bakeoff_failures() { println!( - "shape,relays,links,failure,rule,route_messages,claim_messages,settle_ms,fed_ms,unfed,loops,offline,unavailable,false_ends,revivals,pending,lasting,hops,right" + "shape,relays,links,failure,rule,route_messages,claim_items,settle_ms,fed_ms,unfed,loops,offline,unavailable,false_ends,revivals,pending,lasting,hops,right" ); let sizes: Vec = std::env::var("MOQ_BAKEOFF_RELAYS").map_or(vec![12, 32], |sizes| { sizes @@ -834,7 +871,7 @@ fn benchmark_bakeoff_failures() { #[ignore = "a sweep; Nightly runs it from test/route-sweep.sh"] fn benchmark_bakeoff_growth() { println!( - "grows,relays,links,publishers,broadcasts,rule,build_route_messages,build_claim_messages,held_routes,heard_routes,held_claims,held_lasting,close_route_messages,death_route_messages,death_claim_messages,work" + "grows,relays,links,publishers,broadcasts,rule,build_route_messages,build_claim_items,held_routes,heard_routes,held_claims,lasting_after_failures,close_route_messages,death_route_messages,death_claim_items,work" ); let tiers_of = |relays: usize, publishers: usize, claims: usize| { let (regions, edges) = match relays { @@ -891,7 +928,7 @@ fn benchmark_bakeoff_growth() { #[test] #[ignore = "a sweep; Nightly runs it from test/route-sweep.sh"] fn benchmark_bakeoff_baseline() { - println!("case,rule,route_messages,claim_messages,settle_ms,fed_ms,loops,stale,dark"); + println!("case,rule,route_messages,claim_items,settle_ms,fed_ms,loops,stale,dark"); let ms = |rounds: f64| rounds * HOP.as_secs_f64() * 1000.0; for arm in [Arm::TODAY, Arm::PLAIN, Arm::TUNED, Arm::LEAN, Arm::KEPT] { let mut rng = XorShift(0x6d6f_7100 + 24); diff --git a/rs/moq-net/src/route/tests/compare.rs b/rs/moq-net/src/route/tests/compare.rs index 489878f43f7..b26b46d4d61 100644 --- a/rs/moq-net/src/route/tests/compare.rs +++ b/rs/moq-net/src/route/tests/compare.rs @@ -908,6 +908,8 @@ impl Lab { pub fn hops(&self) -> usize { (0..self.mesh.homes.len()) .flat_map(|index| (0..self.mesh.relays).map(move |relay| (relay, index))) + // Only a walk that reaches its publisher over open links is in use. + .filter(|&(relay, index)| self.walk(relay, index) == Some(Walk::Reached)) .filter_map(|(relay, index)| self.net.chase(relay, self.mesh.publisher(index).1)) .map(|(hops, _)| hops.len()) .sum() @@ -1254,7 +1256,7 @@ fn benchmark_route_churn() { // Nightly's report reads stdout and keeps the columns it has a unit for, so the // message counts go there and the whole table to stderr. A claim sent is one // message on the wire. - println!("rule,route_messages,claim_messages"); + println!("rule,route_messages,claim_items"); eprintln!("rule,steps,loops,stale,dark,msgs,items,claims,offline,unavailable"); for rule in COMPARED { let churn = churn(rule, 0..400, false); @@ -1272,7 +1274,7 @@ fn benchmark_route_churn() { #[test] #[ignore = "a benchmark; Nightly reports it"] fn benchmark_route_churn_two_homes() { - println!("rule,homes,route_messages,claim_messages"); + println!("rule,homes,route_messages,claim_items"); eprintln!("rule,steps,loops,stale,dark,msgs,items,claims,offline,unavailable"); for rule in COMPARED { let churn = churn(rule, 0..400, true); @@ -1324,7 +1326,7 @@ fn path_changes_keep_claims_and_withdrawals_keep_routes() { fn benchmark_route_rules() { // As above: message counts to stdout for the report, which keeps the largest mesh // alone, and the whole table to stderr. - println!("shape,relays,change,rule,route_messages,claim_messages,loop_messages,reroute_ms"); + println!("shape,relays,change,rule,route_messages,claim_items,loop_messages,reroute_ms"); eprintln!("shape,relays,change,rule,msgs,items,claims,rounds,right,stale,dark,loops,offline,unavailable"); for relays in [3, 6, 12, 24] { let mut rng = XorShift(0x6d6f_7100 + relays as u64); diff --git a/rs/moq-net/src/route/tests/multi.rs b/rs/moq-net/src/route/tests/multi.rs index ac73df9ecdb..be57a98471b 100644 --- a/rs/moq-net/src/route/tests/multi.rs +++ b/rs/moq-net/src/route/tests/multi.rs @@ -435,6 +435,8 @@ impl Router { let Some(gone) = self.peers.remove(&peer) else { return; }; + // Nothing still unsent goes to a session that closed. + self.out.retain(|(to, _)| *to != peer); let mut lost = Vec::new(); for (&origin, offers) in &mut self.heard { if offers.remove(&peer).is_some() { @@ -467,11 +469,22 @@ impl Router { /// `peer` sent GOAWAY: every route through it ranks as draining. pub fn goaway(&mut self, peer: N) { + let peer_key = peer; let Some(peer) = self.peers.get_mut(&peer) else { return; }; peer.draining = true; - let node = peer.id; + let (node, role) = (peer.id, peer.port.role); + if role != Role::Routing { + // The claims it gave this node now lead through a draining neighbour. + let claims = self.local.get(&peer_key).into_iter().flatten(); + let broadcasts: Vec = claims.map(|(broadcast, _)| broadcast.clone()).collect(); + for broadcast in broadcasts { + self.restate(broadcast, Change::Lost); + } + self.done(); + return; + } match self.tune.drain { true => self.drains(node, None), false => self.reselect_all(), @@ -617,10 +630,18 @@ impl Router { .filter(|(broadcast, claim)| known.items.get(*broadcast) != Some(claim)) .map(|(broadcast, claim)| (broadcast.clone(), Change::Live(claim.clone()))) .collect(); + // What the newer copy lacks is lost, unless this message says its + // publisher ended it. + let ended = |broadcast: &Broadcast| { + (changes.iter()).any(|(named, change)| named == broadcast && *change == Change::Ended) + }; diff.extend( (known.items.keys()) .filter(|broadcast| !heard.items.contains_key(*broadcast)) - .map(|broadcast| (broadcast.clone(), Change::Lost)), + .map(|broadcast| match ended(broadcast) { + true => (broadcast.clone(), Change::Ended), + false => (broadcast.clone(), Change::Lost), + }), ); self.work += heard.items.len(); diff @@ -722,7 +743,12 @@ impl Router { /// withdraw it for `cause` when none is left. fn restate(&mut self, broadcast: Broadcast, cause: Change) { let cheapest = (self.local.iter()) - .filter_map(|(&peer, claims)| Some((claims.get(&broadcast)?, peer))) + .filter_map(|(&peer, claims)| { + // A claim from a neighbour that sent GOAWAY is draining from here on. + let mut claim = claims.get(&broadcast)?.clone(); + claim.draining |= self.peers.get(&peer).is_some_and(|peer| peer.draining); + Some((claim, peer)) + }) .min_by_key(|(claim, peer)| { ( claim.draining, @@ -731,8 +757,7 @@ impl Router { claim.chain.len(), *peer, ) - }) - .map(|(claim, peer)| (claim.clone(), peer)); + }); let me = self.me; let was = self.known.entry(me).or_default().items.get(&broadcast).cloned(); let change = match cheapest { From 4c5cc3ed33597b67bb03e492adaf9d5e5901759a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 01:45:47 -0800 Subject: [PATCH 12/13] test(moq-net): keep today's benchmark headers and report claim items as a unit Co-Authored-By: Claude Fable 5.2 --- rs/moq-net/src/route/tests/compare.rs | 6 +++--- sh/gh/bench-report.py | 4 ++++ sh/gh/bench-report.test.py | 2 ++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/rs/moq-net/src/route/tests/compare.rs b/rs/moq-net/src/route/tests/compare.rs index b26b46d4d61..fef0d4bfbd4 100644 --- a/rs/moq-net/src/route/tests/compare.rs +++ b/rs/moq-net/src/route/tests/compare.rs @@ -1256,7 +1256,7 @@ fn benchmark_route_churn() { // Nightly's report reads stdout and keeps the columns it has a unit for, so the // message counts go there and the whole table to stderr. A claim sent is one // message on the wire. - println!("rule,route_messages,claim_items"); + println!("rule,route_messages,claim_messages"); eprintln!("rule,steps,loops,stale,dark,msgs,items,claims,offline,unavailable"); for rule in COMPARED { let churn = churn(rule, 0..400, false); @@ -1274,7 +1274,7 @@ fn benchmark_route_churn() { #[test] #[ignore = "a benchmark; Nightly reports it"] fn benchmark_route_churn_two_homes() { - println!("rule,homes,route_messages,claim_items"); + println!("rule,homes,route_messages,claim_messages"); eprintln!("rule,steps,loops,stale,dark,msgs,items,claims,offline,unavailable"); for rule in COMPARED { let churn = churn(rule, 0..400, true); @@ -1326,7 +1326,7 @@ fn path_changes_keep_claims_and_withdrawals_keep_routes() { fn benchmark_route_rules() { // As above: message counts to stdout for the report, which keeps the largest mesh // alone, and the whole table to stderr. - println!("shape,relays,change,rule,route_messages,claim_items,loop_messages,reroute_ms"); + println!("shape,relays,change,rule,route_messages,claim_messages,loop_messages,reroute_ms"); eprintln!("shape,relays,change,rule,msgs,items,claims,rounds,right,stale,dark,loops,offline,unavailable"); for relays in [3, 6, 12, 24] { let mut rng = XorShift(0x6d6f_7100 + relays as u64); diff --git a/sh/gh/bench-report.py b/sh/gh/bench-report.py index dfa6cba34ef..5d8db906803 100644 --- a/sh/gh/bench-report.py +++ b/sh/gh/bench-report.py @@ -31,6 +31,7 @@ "streams", "instructions", "messages", + "items", } ROOT = Path(__file__).resolve().parents[2] @@ -167,6 +168,9 @@ def measurement_unit(column): return "packets" if column.endswith("_messages"): return "messages" + # Changes carried in messages, counted apart from the messages. + if column.endswith("_items"): + return "items" for unit in ("ns", "us", "ms"): if re.search(f"(^|_){unit}($|_per_)", column): return unit diff --git a/sh/gh/bench-report.test.py b/sh/gh/bench-report.test.py index ea782a35785..8267c575f8a 100644 --- a/sh/gh/bench-report.test.py +++ b/sh/gh/bench-report.test.py @@ -64,6 +64,7 @@ def test_measured_columns(self): "mib_per_s", "overflowed_packets", "max_messages", + "claim_items", "payload_bytes", "bytes", "frames_per_chunk", @@ -83,6 +84,7 @@ def test_measured_columns(self): "mib_per_s": "MiB/s", "overflowed_packets": "packets", "max_messages": "messages", + "claim_items": "items", # Workload dimensions, as the JS benchmarks name them. "payload_bytes": None, "bytes": None, From 5b56cf76240476620d4779a9196c48d566289148 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 02:28:37 -0800 Subject: [PATCH 13/13] quest: only a relay that never announced a claim has no route on the wire Applies CodeRabbit's suggested patch: a relay whose last claim left keeps its route. Co-Authored-By: Claude Fable 5.2 --- quest/m2/multicast/switch.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/quest/m2/multicast/switch.md b/quest/m2/multicast/switch.md index e0b2a85838c..8984b7e6f4b 100644 --- a/quest/m2/multicast/switch.md +++ b/quest/m2/multicast/switch.md @@ -99,7 +99,8 @@ for the wire: link priority to its publisher and a draining bit beside its cost and chain. - A claims message with no change still carries its floor. - A node offers a route to itself only once it announces a claim, so a relay - that only forwards or only serves viewers has no route on the wire. + that has never announced a claim and only forwards or only serves viewers + has no route on the wire. - A scoped neighbour is sent the node's closest claim per broadcast, and never one that came from that neighbour. - Today's rule cost 300,100 route messages when a publisher left two meshes