From cd98ec13016c3dc32607df86f8f8d82609a21bac Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 03:11:53 -0800 Subject: [PATCH 01/11] chore(quest): claim quest/m2/cluster-routes/interest-set.md Quest-Claim: b8e85daf8b7441f0 From 831ee1e658bd973e3330bcf97620da2b26c4a84b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 03:28:47 -0800 Subject: [PATCH 02/11] wip(route): the path router takes an interest as an edited set of prefixes Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/mod.rs | 6 +- rs/moq-net/src/route/path.rs | 580 ++++++++++++++++++-------- rs/moq-net/src/route/reader.rs | 8 +- rs/moq-net/src/route/tests/handoff.rs | 11 +- rs/moq-net/src/route/tests/mod.rs | 137 +++--- rs/moq-net/src/route/tests/scope.rs | 103 +++-- 6 files changed, 554 insertions(+), 291 deletions(-) diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index 8e17adef175..106b414b4cb 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -477,6 +477,10 @@ pub(crate) enum Error { UnknownNeighbor, #[error("no declared identity")] NoIdentity, + #[error("one interest prefix inside another")] + Nested, + #[error("no such interest prefix")] + UnknownPrefix, } impl Router { @@ -1944,7 +1948,7 @@ impl Router { || !ctx.own && views.get(&id).is_none_or(|view| view.as_of(claim).is_none()) }; self.reader - .apply(id, &self.watch, now, |claim| ctx.live && !uncertified(claim)); + .apply(id, |claim| within(claim, &self.watch), now, |claim| ctx.live && !uncertified(claim)); } /// How the reader now sees each of `claims` of `id`, in `ctx`. diff --git a/rs/moq-net/src/route/path.rs b/rs/moq-net/src/route/path.rs index 0c318bfa13c..5caf0f15e1c 100644 --- a/rs/moq-net/src/route/path.rs +++ b/rs/moq-net/src/route/path.rs @@ -31,6 +31,16 @@ //! shows each claim from the narrowest view above it, available while a live //! route covers it. //! +//! What a node asks a neighbour for is a set of prefixes, none inside another, +//! edited in place: its own interest met with the neighbour's scope. The +//! neighbour answers each edit with a marker on its stream, where the edit takes +//! effect for both sides, then sends what the added prefixes newly cover and +//! SYNCED for each. At the marker both sides rekey the stream's routes alike +//! ([`rekey`]): a route inside a prefix both sets cover stays, a route at a +//! removed prefix that an added one widens moves to the wider prefix with its +//! claims, and a wider route carries its claims and floor to each added prefix +//! inside it. So a swap re-sends nothing both sets cover. +//! //! `rs/moq-net/verify/route/RouteHandoff.tla` model-checks the rule. Nothing drives //! this router on the wire yet: sessions run [`super::Router`] until the switch //! (`quest/m2/cluster-routes/switch.md`). @@ -39,7 +49,7 @@ use std::{ cmp::Reverse, - collections::{BTreeMap, BTreeSet}, + collections::{BTreeMap, BTreeSet, VecDeque}, sync::Arc, time::Duration, }; @@ -49,7 +59,7 @@ use kio::clock::Instant; use crate::Path; use super::reader::Reader; -use super::view::{ancestors, below, covering, within_set}; +use super::view::{ancestors, below, covering, within_any, within_set}; use super::{Error, Event, GENERATED, Id, Key, Link, MAX, MAX_HOPS, Metric, QUALIFIED, Rev, Shown, View, within}; /// One node on a route's path, and the session it learned the route on. A @@ -157,12 +167,31 @@ pub(crate) enum Out { to: N, dead: Dead, }, - /// The stream answering the neighbour's interest sent its initial set. + /// ANNOUNCE_REQUEST: an edit of the prefixes this node asks the neighbour for. + Ask { + to: N, + edit: Edit, + }, + /// The marker where the neighbour's oldest unanswered edit takes effect on this + /// node's stream to it, for both sides. + Edited { + to: N, + }, + /// The stream sent the initial set of `prefix`, which the neighbour added. Synced { to: N, + prefix: Path, }, } +/// An edit of the prefixes a node asks a neighbour for: the live ones it removes, +/// then the ones it adds. A prefix both removed and added stays as it was. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct Edit { + pub remove: Vec, + pub add: Vec, +} + /// When a doubt about a node lifts. #[cfg(test)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -221,8 +250,6 @@ enum State { struct Heard { offer: Offer, state: State, - /// The interest generation of the stream it came on. - interest: u32, /// Its claims as presented at its prefix, with one floor or none yet. view: View, } @@ -286,8 +313,9 @@ enum SentState { #[derive(Clone, Debug)] struct SentOffer { offer: Offer, - /// The prefix of the view its claims were presented from. - from: Path, + /// The prefix of the view its claims were presented from, or `None` once an edit + /// rekeyed it: the next batch is diffed against all it holds. + from: Option, view: View, } @@ -328,7 +356,7 @@ impl Sent { } SentState::Ended => SentOffer { offer, - from: Path::empty(), + from: None, view: View::default(), }, }; @@ -401,16 +429,37 @@ struct Session { link: Link, rtt: Option, draining: bool, - /// The newest receive stream whose initial set has arrived (or was cancelled). - /// Older streams cannot recreate a route after that boundary. - synced: u32, + /// What this node asks the neighbour for as of the last marker, each prefix with + /// whether its initial set arrived (SYNCED). + asked: BTreeMap, + /// The set each edit sent and not yet marked leaves, oldest first. + edits: VecDeque>, /// The routes it offered, so closing it costs what it held. heard: BTreeSet, } +impl Session { + /// The prefixes this node asks the neighbour for once every edit in flight lands. + fn requested(&self) -> BTreeSet { + match self.edits.back() { + Some(set) => set.clone(), + None => self.asked.keys().cloned().collect(), + } + } + + /// Whether this node asked the neighbour for `region` at the last marker and in + /// every edit still in flight: what the neighbour says there answers what this + /// node asks now, and no edit in flight may yet drop it. + fn asking(&self, region: &Path) -> bool { + ancestors(region).any(|prefix| self.asked.contains_key(&prefix)) + && self.edits.iter().all(|set| within_any(region, set)) + } +} + /// Presence in `Router::answering` is the answering phase; absence is idle. struct Answer { - watch: Path, + /// The prefixes the neighbour asks this node for, none inside another. + prefixes: BTreeSet, tx: BTreeMap>, /// The dead hops it was told of, while a route sent to it names one. told: BTreeSet, @@ -513,8 +562,8 @@ pub(crate) struct Router { doubted: BTreeMap, /// Nodes that sent GOAWAY, kept while a heard route names one or it is in doubt. draining: BTreeSet, - watch: Path, - interest: u32, + /// This node's interest: the prefixes its reader sees, none inside another. + watch: BTreeSet, reader: Reader, /// Claim paths indexed by publisher, only for this rule's scoped reader updates. shown_claims: BTreeMap>, @@ -554,6 +603,104 @@ pub(super) fn meet(a: &Path, b: &Path) -> Option { } } +/// Whether a prefix of `set` lies inside another. +fn nested(set: &BTreeSet) -> bool { + set.iter() + .any(|prefix| ancestors(prefix).any(|above| above != *prefix && set.contains(&above))) +} + +/// What a stream carries for a route, as [`rekey`] moves it. +trait Carried: Clone { + /// Its claims, unless it was ended. + fn view(&mut self) -> Option<&mut View>; +} + +impl Carried for Heard { + fn view(&mut self) -> Option<&mut View> { + Some(&mut self.view) + } +} + +impl Carried for Sent { + fn view(&mut self) -> Option<&mut View> { + match &mut self.state { + SentState::Live(sent) | SentState::Held(sent) => { + // The receiver keeps what was rekeyed, so the next batch is diffed against it. + sent.from = None; + Some(&mut sent.view) + } + SentState::Ended => None, + } + } +} + +/// The routes one stream carries once an edit of its prefixes from `old` to `new` +/// takes effect at its marker, and the keys it no longer carries. Both sides apply +/// it alike to what was sent and what was heard, so they stay in step and nothing +/// both sets cover is sent again. +/// +/// A route inside a prefix the new set covers stays. A route at a removed prefix +/// that an added prefix widens moves to the wider prefix: the first by key keeps +/// its offer and state, the rest give only their claims, and the claim presented +/// at the removed prefix goes with every floor, since it may stand for a broader +/// claim and the floors certified only the narrower prefix. A route at a removed +/// prefix is carried, as presented there, to each added prefix inside it that no +/// route keys, so its floor carries too. The rest go. +fn rekey>( + old: &BTreeSet, + new: &BTreeSet, + routes: BTreeMap, +) -> (BTreeMap, Vec) { + let mut kept: BTreeMap = BTreeMap::new(); + let mut gone = Vec::new(); + for (key, mut route) in routes.iter().map(|(key, route)| (key.clone(), route.clone())) { + let (id, prefix) = &key; + let Some(region) = ancestors(prefix).find(|above| old.contains(above)) else { + gone.push(key); + continue; + }; + if within_any(prefix, new) && (new.contains(®ion) || *prefix != region) { + kept.insert(key, route); + continue; + } + if let Some(wider) = ancestors(®ion).find(|above| new.contains(above)) { + // At the removed prefix, an added one widens it. + gone.push(key.clone()); + let Some(view) = route.view() else { + continue; + }; + view.items.remove(prefix); + view.floors.clear(); + let items = std::mem::take(&mut view.items); + match kept.get_mut(&(*id, wider.clone())) { + Some(first) => first.view().expect("a carried route").items.extend(items), + None => { + route.view().expect("a carried route").items = items; + kept.insert((*id, wider), route); + } + } + continue; + } + gone.push(key.clone()); + if *prefix != region { + continue; + } + let Some(view) = route.view().map(|view| view.clone()) else { + continue; + }; + for inside in within_set(new, ®ion) { + let key = (*id, inside.clone()); + if routes.contains_key(&key) { + continue; + } + let mut copy = route.clone(); + *copy.view().expect("a carried route") = view.present(inside); + kept.insert(key, copy); + } + } + (kept, gone) +} + /// The entries of `map` under identity `id`. fn of(map: &BTreeMap, id: Id) -> impl Iterator { map.range((id, Path::empty())..) @@ -585,8 +732,7 @@ impl Router { dead: BTreeMap::new(), doubted: BTreeMap::new(), draining: BTreeSet::new(), - watch: Path::empty(), - interest: 0, + watch: BTreeSet::from([Path::empty()]), reader: Reader::default(), shown_claims: BTreeMap::new(), rng: seed | 1, @@ -628,11 +774,13 @@ impl Router { link, rtt: None, draining: false, - synced: 0, + asked: BTreeMap::new(), + edits: VecDeque::new(), heard: BTreeSet::new(), }; self.sessions.insert(neighbor, session); - // Nothing is sent until the neighbour asks. + // It asks at once; nothing is answered until the neighbour asks. + self.ask(neighbor); Ok(()) } @@ -669,47 +817,109 @@ impl Router { self.update(ids, hops, None); } - /// `neighbor` asked this node for announcements under `prefix`, or for none: its - /// stream starts over, a complete snapshot closed by SYNCED. Each neighbour has - /// one interest prefix; another call replaces it rather than adding a region. - pub fn peer_interest(&mut self, neighbor: N, prefix: Option) -> Result<(), Error> { + /// `neighbor` edited the prefixes it asks this node for. The edit takes effect at + /// once, behind the marker this node sends first: what both sets cover stays, and + /// the stream then sends what the added prefixes newly cover, closed by SYNCED for + /// each. An edit removing a prefix that is not live, or leaving one live prefix + /// inside another, is refused whole. + pub fn edit(&mut self, neighbor: N, edit: Edit) -> Result<(), Error> { if !self.sessions.contains_key(&neighbor) { return Err(Error::UnknownNeighbor); } - if let Some(old) = self.answering.remove(&neighbor) { - for (key, sent) in old.tx { + let old = (self.answering.get(&neighbor)).map_or_else(BTreeSet::new, |answer| answer.prefixes.clone()); + let mut new = old.clone(); + for prefix in &edit.remove { + if !new.remove(prefix) { + return Err(Error::UnknownPrefix); + } + } + for prefix in edit.add { + if within_any(&prefix, &new) || within_set(&new, &prefix).next().is_some() { + return Err(Error::Nested); + } + new.insert(prefix); + } + self.out.push(Out::Edited { to: neighbor }); + if let Some(answer) = self.answering.get_mut(&neighbor) { + let tx = std::mem::take(&mut answer.tx); + for (key, sent) in &tx { if let Some(offer) = sent.offer() { - self.unindex_sent(neighbor, &key, &offer.path); + self.unindex_sent(neighbor, key, &offer.path); } + self.unreflect(neighbor, key, &sent.src); + } + let (rekeyed, gone) = rekey(&old, &new, tx.clone()); + // A child that learned a route this stream no longer carries under its key + // may still owe an answer to the route it reflected. + for key in gone { if self.child_pending(key.0, neighbor, &key.1) { - self.waiting - .entry((key.0, sent.src.clone())) - .or_default() - .insert((neighbor, key.1.clone())); + let src = (key.0, tx[&key].src.clone()); + self.waiting.entry(src).or_default().insert((neighbor, key.1)); + } + } + let tx = rekeyed; + for (key, sent) in &tx { + if let Some(offer) = sent.offer() { + self.index_sent(neighbor, key, &offer.path); } - self.unreflect(neighbor, &key, &sent.src); + self.reflect(neighbor, key, &sent.src); } + self.answering.get_mut(&neighbor).expect("answering").tx = tx; } - let has = prefix.is_some(); - if let Some(watch) = prefix { - self.answering.insert( - neighbor, - Answer { - watch, + let added: Vec = new.difference(&old).cloned().collect(); + match new.is_empty() { + true => { + self.answering.remove(&neighbor); + } + false => { + let answer = self.answering.entry(neighbor).or_insert_with(|| Answer { + prefixes: BTreeSet::new(), tx: BTreeMap::new(), told: BTreeSet::new(), told_draining: BTreeSet::new(), - }, - ); + }); + answer.prefixes = new; + } } let ids = self.ids(); self.update(ids, Vec::new(), None); - if has { - self.out.push(Out::Synced { to: neighbor }); + for prefix in added { + self.out.push(Out::Synced { to: neighbor, prefix }); } Ok(()) } + /// The prefixes this node asks `neighbor` for: its interest met with the regions + /// of the neighbour's scope, keeping the wider of two nested ones. + fn asks(&self, neighbor: N) -> BTreeSet { + let Some(scope) = &self.sessions[&neighbor].link.scope else { + return self.watch.clone(); + }; + let met: BTreeSet = (self.watch.iter()) + .flat_map(|watch| scope.regions.iter().filter_map(|region| meet(watch, region))) + .collect(); + met.iter() + .filter(|prefix| !ancestors(prefix).any(|above| above != **prefix && met.contains(&above))) + .cloned() + .collect() + } + + /// Send `neighbor` an edit bringing what this node asks it for to [`Self::asks`]. + fn ask(&mut self, neighbor: N) { + let want = self.asks(neighbor); + let session = self.sessions.get_mut(&neighbor).expect("a session"); + let requested = session.requested(); + if want == requested { + return; + } + let edit = Edit { + remove: requested.difference(&want).cloned().collect(), + add: want.difference(&requested).cloned().collect(), + }; + session.edits.push_back(want); + self.out.push(Out::Ask { to: neighbor, edit }); + } + /// `neighbor` sent GOAWAY at `now`. Every path through it ranks below every path /// that avoids it, and stays usable while its session stays up. This node moves at /// once only onto a path that avoids it, and tells the neighbours it sent a route @@ -745,41 +955,44 @@ impl Router { .collect() } - /// Change the local interest to `prefix`, returning its new generation. - /// The driver replaces each neighbour's stream with the intersection of - /// `prefix` and its single scope region (the whole prefix if unscoped), then - /// calls `synced` after the replacement's initial set arrives. This interface - /// supports one prefix per neighbour: disjoint regions on the same neighbour - /// need the interest-set extension, including its per-prefix SYNCED. - /// Until then, old routes stay but only new claims are - /// believed. With no intersection, the driver cancels the old streams and - /// calls `synced` locally for this generation: no replacement can answer. - /// What this node knows outside `prefix` is forgotten. - pub fn watch(&mut self, prefix: Path) -> u32 { - self.interest += 1; - self.watch = prefix.clone(); + /// Change this node's interest to `prefixes`, none inside another, and edit what + /// it asks each neighbour for to match. What this node knows outside them is + /// forgotten. A route a neighbour sends stays, but is believed for a region only + /// once every edit in flight that could drop it has taken effect. + pub fn watch(&mut self, prefixes: BTreeSet) -> Result<(), Error> { + if nested(&prefixes) { + return Err(Error::Nested); + } + self.watch = prefixes; // The narrowest covering view carries the knowledge the reader already used. let keys: Vec = self.views.keys().rev().cloned().collect(); for key in keys { - if within(&key.1, &prefix) { + if within_any(&key.1, &self.watch) { continue; } let view = self.views.remove(&key).expect("a view"); - self.forget(key.0, &key.1, Some(&prefix)); - if within(&prefix, &key.1) { - let narrow = (key.0, prefix.clone()); + let inside: Vec = within_set(&self.watch, &key.1).cloned().collect(); + self.forget(key.0, &key.1, &inside); + for prefix in inside { self.scopes.entry(key.0).or_default().insert(prefix.clone()); - self.views.entry(narrow).or_insert_with(|| view.present(&prefix)); + self.views + .entry((key.0, prefix.clone())) + .or_insert_with(|| view.present(&prefix)); } } + let peers: Vec = self.sessions.keys().copied().collect(); + for peer in peers { + self.ask(peer); + } let ids = self.ids(); self.refresh(ids); - self.interest + Ok(()) } - /// The current interest generation. - pub fn interest(&self) -> u32 { - self.interest + /// Whether `neighbor` sent the initial set of `prefix`, which this node asks it + /// for, or `None` when it does not ask for it. + pub fn is_synced(&self, neighbor: N, prefix: &Path) -> Option { + self.sessions.get(&neighbor)?.asked.get(prefix).copied() } // --- The origin's own identity --- @@ -888,10 +1101,14 @@ impl Router { // --- Received messages --- - /// A route `from` sent for `named` under `prefix`, on the stream answering interest - /// generation `interest`: ROUTE_START or ROUTE_UPDATE, ROUTE_HOLD, or ROUTE_END for `None`. - pub fn route(&mut self, from: N, named: Id, prefix: Path, reach: Option, interest: u32) { - if self.sessions.get(&from).is_none_or(|session| interest < session.synced) { + /// A route `from` sent for `named` under `prefix`: ROUTE_START or ROUTE_UPDATE, + /// ROUTE_HOLD, or ROUTE_END for `None`. One outside what this node asked `from` + /// for, as of the last marker, is refused. + pub fn route(&mut self, from: N, named: Id, prefix: Path, reach: Option) { + let Some(session) = self.sessions.get(&from) else { + return; + }; + if !ancestors(&prefix).any(|above| session.asked.contains_key(&above)) { return; } let Some(id) = self.admit(from, named, &prefix) else { @@ -899,18 +1116,10 @@ impl Router { }; let key = (id, prefix); let entry = self.rx.get(&key).and_then(|routes| routes.get(&from)); - // A message from a stream older than the route it names. - if entry.is_some_and(|entry| entry.interest > interest) { - return; - } - let same = entry.is_some_and(|entry| entry.interest == interest); match reach { // A path a node could not extend is refused, as if ended. Some(Reach::Finite(offer)) if offer.extendable() => { - let view = match same { - true => self.rx[&key][&from].view.clone(), - false => View::default(), - }; + let view = entry.map(|entry| entry.view.clone()).unwrap_or_default(); if entry.is_some() { self.forget_heard(&key, from); } @@ -918,26 +1127,20 @@ impl Router { true => State::Dead, false => State::Live, }; - let heard = Heard { - offer, - state, - interest, - view, - }; - self.hear(key.clone(), from, heard); + self.hear(key.clone(), from, Heard { offer, state, view }); if id != named { self.filed.insert((id, from), named); } } - Some(Reach::Held) if same => { + Some(Reach::Held) => { let heard = self.rx.get_mut(&key).and_then(|routes| routes.get_mut(&from)); - if let Some(heard) = heard { - heard.state = State::Held; - } + let Some(heard) = heard else { + return; + }; + heard.state = State::Held; } - Some(Reach::Held) => return, Some(Reach::Finite(_)) | None => { - if entry.is_none() || !same && reach.is_none() { + if entry.is_none() { return; } self.forget_heard(&key, from); @@ -947,16 +1150,8 @@ impl Router { } /// A batch of claim changes for the route `from` sent for `named` under `prefix`, - /// closed by its floor, on the stream answering interest generation `interest`. - pub fn batch( - &mut self, - from: N, - named: Id, - prefix: Path, - changes: Vec<(Path, Option)>, - floor: Option, - interest: u32, - ) { + /// closed by its floor. + pub fn batch(&mut self, from: N, named: Id, prefix: Path, changes: Vec<(Path, Option)>, floor: Option) { let Some(id) = self.admit(from, named, &prefix) else { return; }; @@ -964,9 +1159,6 @@ impl Router { let Some(heard) = self.rx.get_mut(&key).and_then(|routes| routes.get_mut(&from)) else { return; }; - if heard.interest != interest { - return; - } let was = heard.view.floors.get(&prefix).copied(); let claims: Vec = changes.iter().map(|(claim, _)| claim.clone()).collect(); let floors = floor.map(|rev| BTreeMap::from([(prefix, rev)])).unwrap_or_default(); @@ -1029,25 +1221,55 @@ impl Router { self.update(ids, vec![Report::Closed(hop)], None); } - /// `from`'s stream answering interest generation `interest` sent its initial set: - /// what an older stream left is gone. The driver also calls this after cancelling - /// every stream to a scoped peer whose scope no longer intersects the interest. - pub fn synced(&mut self, from: N, interest: u32) { + /// `from` sent the marker for this node's oldest edit in flight: the edit takes + /// effect, and the routes `from` sent are rekeyed as it rekeyed them ([`rekey`]). + /// A marker with no edit in flight is ignored. + pub fn edited(&mut self, from: N) { let Some(session) = self.sessions.get_mut(&from) else { return; }; - session.synced = session.synced.max(interest); - let stale: Vec = (session.heard.iter()) - .filter(|key| self.rx[*key][&from].interest < interest) - .cloned() + let Some(new) = session.edits.pop_front() else { + return; + }; + let old: BTreeSet = session.asked.keys().cloned().collect(); + let asked = new + .iter() + .map(|prefix| (prefix.clone(), session.asked.get(prefix).copied().unwrap_or(false))) + .collect(); + session.asked = asked; + let keys: Vec = session.heard.iter().cloned().collect(); + let ids: BTreeSet = keys.iter().map(|(id, _)| *id).collect(); + let filed: Vec<(Id, Id)> = ids + .iter() + .filter_map(|id| Some((*id, *self.filed.get(&(*id, from))?))) + .collect(); + let heard: BTreeMap> = keys + .iter() + .map(|key| (key.clone(), self.rx[key][&from].clone())) .collect(); - let ids: BTreeSet = stale.iter().map(|(id, _)| *id).collect(); - for key in &stale { + for key in &keys { self.forget_heard(key, from); } + let (heard, _) = rekey(&old, &new, heard); + for (key, heard) in heard { + self.hear(key, from, heard); + } + for (id, named) in filed { + if of(&self.rx, id).any(|(_, routes)| routes.contains_key(&from)) { + self.filed.insert((id, from), named); + } + } self.update(ids, Vec::new(), None); } + /// `from` sent the initial set of `prefix`, which this node asks it for. + pub fn synced(&mut self, from: N, prefix: Path) { + let synced = (self.sessions.get_mut(&from)).and_then(|session| session.asked.get_mut(&prefix)); + if let Some(synced) = synced { + *synced = true; + } + } + /// Fire every doubt and dead-hop timer due by `now`. pub fn timeout(&mut self, now: Instant) { let due: Vec = self @@ -1092,7 +1314,7 @@ impl Router { let views: Vec = of(&self.views, id).map(|(key, _)| key.clone()).collect(); for key in views { self.views.remove(&key); - self.forget(id, &key.1, None); + self.forget(id, &key.1, &[]); } self.refresh([id].into()); } @@ -1113,13 +1335,16 @@ impl Router { self.dead, self.doubted, self.filed ); for (peer, answer) in &self.answering { - out += &format!(" to {peer:?} watch {:?} sent {:?}\n", answer.watch, answer.tx); + out += &format!(" to {peer:?} prefixes {:?} sent {:?}\n", answer.prefixes, answer.tx); + } + for (peer, session) in &self.sessions { + out += &format!(" asked {peer:?} {:?} edits {:?}\n", session.asked, session.edits); } for (key, routes) in &self.rx { for (peer, heard) in routes { out += &format!( - " heard {key:?} from {peer:?}: {:?} {:?} gen {} view {:?}\n", - heard.state, heard.offer.path, heard.interest, heard.view + " heard {key:?} from {peer:?}: {:?} {:?} view {:?}\n", + heard.state, heard.offer.path, heard.view ); } } @@ -1129,12 +1354,7 @@ impl Router { for (key, view) in &self.views { out += &format!(" view {key:?}: {view:?}\n"); } - out += &format!( - " interest {} watch {:?} shown {:?}", - self.interest, - self.watch, - self.reader.shown() - ); + out += &format!(" watch {:?} shown {:?}", self.watch, self.reader.shown()); out } @@ -1169,8 +1389,8 @@ impl Router { (prefix, None) } Some((prev, prefix)) => { - let watch = self.answering.get(&prev)?.watch.clone(); - (self.src(id, &prefix, prev, &watch), Some(prev)) + let answer = self.answering.get(&prev)?; + (self.src(id, &prefix, prev, &answer.prefixes), Some(prev)) } }; let route = self.routes.get(&(id, prefix.clone()))?; @@ -1635,13 +1855,12 @@ impl Router { } /// The prefix of this node's own route that what it offers a stream answering - /// `watch` under `prefix` reflects: the route itself inside the interest, and at the - /// interest the best-ranked live route covering it that does not name the - /// neighbour. Prefer the narrower scope among equal ranks. Without a usable + /// `prefixes` under `prefix` reflects: the route itself inside one of them, and at + /// one the best-ranked live route covering it that does not name the neighbour. Prefer the narrower scope among equal ranks. Without a usable /// path, reflect a live or held route so a child can answer its loss. Preferring /// a narrower route regardless of cost can make covering paths oscillate. - fn src(&self, id: Id, prefix: &Path, peer: N, watch: &Path) -> Path { - if prefix != watch { + fn src(&self, id: Id, prefix: &Path, peer: N, prefixes: &BTreeSet) -> Path { + if !prefixes.contains(prefix) { return prefix.clone(); } let covering = || ancestors(prefix).rev().map(|above| (id, above)); @@ -1668,6 +1887,15 @@ impl Router { // --- Internals: views --- + /// The views this node keeps of a route under `prefix`: the prefix itself within + /// its interest, else each prefix of its interest inside it. + fn watched(&self, prefix: &Path) -> Vec { + match within_any(prefix, &self.watch) { + true => vec![prefix.clone()], + false => within_set(&self.watch, prefix).cloned().collect(), + } + } + /// Whether this node holds a route of `id` covering `prefix`. fn presence(&self, id: Id, prefix: &Path) -> bool { ancestors(prefix).any(|above| self.routes.contains_key(&(id, above))) @@ -1696,10 +1924,11 @@ impl Router { .map(|((_, prefix), view)| (prefix.clone(), view)) } - /// Whether a route this node heard is believed for its claims: live, its path does - /// not name this node, and its stream answers the current interest. - fn believed(&self, heard: &Heard) -> bool { - heard.state == State::Live && !names(&heard.offer.path, self.hop) && heard.interest == self.interest + /// Whether a route `peer` sent is believed for its claims within `region`: live, + /// its path does not name this node, and this node asks `peer` for the region with + /// no edit in flight that could drop it. + fn believed(&self, peer: N, heard: &Heard, region: &Path) -> bool { + heard.state == State::Live && !names(&heard.offer.path, self.hop) && self.sessions[&peer].asking(region) } /// Bring every view of `id` up to date: each prefix within the interest that a route @@ -1721,7 +1950,7 @@ impl Router { let key = (id, prefix.clone()); if !self.presence(id, prefix) { if self.views.remove(&key).is_some() { - self.forget(id, prefix, None); + self.forget(id, prefix, &[]); dirty.regions.insert(prefix.clone()); } continue; @@ -1747,7 +1976,7 @@ impl Router { let Some(&rev) = heard.view.floors.get(&route.1) else { continue; }; - if !self.believed(heard) || Some(rev) <= floor { + if !self.believed(peer, heard, prefix) || Some(rev) <= floor { continue; } if best.as_ref().is_none_or(|(best, ..)| rev > *best) { @@ -1760,7 +1989,10 @@ impl Router { continue; }; let heard = &self.rx[&route][&peer].view; - let follows = delta.filter(|delta| delta.route == route && delta.from == peer && delta.was == floor); + // A route with no floor yet may hold claims an edit carried to it, which the + // view never followed. + let follows = delta + .filter(|delta| delta.route == route && delta.from == peer && delta.was.is_some() && delta.was == floor); let view = match follows { Some(delta) => { let mut view = base; @@ -1820,24 +2052,24 @@ impl Router { /// Count a forget against each claim of `id` this node has shown within `prefix`, /// outside `kept`. - fn forget(&mut self, id: Id, prefix: &Path, kept: Option<&Path>) { + fn forget(&mut self, id: Id, prefix: &Path, kept: &[Path]) { let views = &self.views; (self.reader).forget(id, |claim| { !within(claim, prefix) - || kept.is_some_and(|kept| within(claim, kept)) + || kept.iter().any(|kept| within(claim, kept)) || of(views, id).any(|((_, prefix), view)| within(claim, prefix) && view.floors.contains_key(prefix)) }); } // --- Internals: sending --- - /// What this node sends `peer`, whose stream answers `watch`, for `id` under `prefix`. - fn want(&self, id: Id, prefix: &Path, peer: N, watch: &Path) -> Want { + /// What this node sends `peer`, whose stream answers `prefixes`, for `id` under `prefix`. + fn want(&self, id: Id, prefix: &Path, peer: N, prefixes: &BTreeSet) -> Want { let session = &self.sessions[&peer]; if !session.link.proxy && !self.origins.contains_key(&id) { return Want::End; } - let src = self.src(id, prefix, peer, watch); + let src = self.src(id, prefix, peer, prefixes); let Some(route) = self.routes.get(&(id, src.clone())) else { return Want::End; }; @@ -1857,6 +2089,8 @@ impl Router { // when down-only: neither would use it. Some(offer) if session.link.upstream && offer.down_only => Want::End, Some(offer) if offer.path[0].node == session.hop => Want::End, + // The neighbour could not extend it, so it would refuse it. + Some(offer) if !offer.extendable() => Want::End, Some(offer) => Want::Path(offer.clone(), src.clone()), // The next hop it holds under hears nothing until those behind this node answered. None if route.next() == Some(peer) => { @@ -1894,9 +2128,9 @@ impl Router { || covering(&view.floors, prefix, |claim| view.items.get(claim)).is_some() } - /// The prefixes a stream answering `watch` carries for `id` to `peer`: the interest - /// itself, each route inside it, and each it was sent and may need to end. - fn stream(&self, id: Id, peer: N, watch: &Path, region: Option<&Path>) -> BTreeSet { + /// The prefixes a stream answering `prefixes` carries for `id` to `peer`: each of + /// them, each route inside one, and each it was sent and may need to end. + fn stream(&self, id: Id, peer: N, prefixes: &BTreeSet, region: Option<&Path>) -> BTreeSet { let session = &self.sessions[&peer]; let answer = &self.answering[&peer]; let sent_only = !session.link.proxy && !self.origins.contains_key(&id); @@ -1904,14 +2138,16 @@ impl Router { // A non-proxy peer is sent only what it was already sent, so skip scanning every route. return of(&answer.tx, id).map(|((_, prefix), _)| prefix.clone()).collect(); } - let mut out = BTreeSet::new(); - if region.is_none_or(|region| meet(watch, region).is_some()) { - out.insert(watch.clone()); - } + let mut out: BTreeSet = match region { + Some(region) => (ancestors(region).filter(|above| prefixes.contains(above))) + .chain(within_set(prefixes, region).cloned()) + .collect(), + None => prefixes.clone(), + }; if let Some(region) = region { for prefix in self.prefixes(id, Some(region)) { let key = (id, prefix.clone()); - if self.routes.contains_key(&key) && within(&prefix, watch) || answer.tx.contains_key(&key) { + if self.routes.contains_key(&key) && within_any(&prefix, prefixes) || answer.tx.contains_key(&key) { out.insert(prefix); } } @@ -1919,7 +2155,7 @@ impl Router { out.extend( of(&self.routes, id) .map(|((_, prefix), _)| prefix) - .filter(|prefix| within(prefix, watch)) + .filter(|prefix| within_any(prefix, prefixes)) .cloned(), ); out.extend(of(&answer.tx, id).map(|((_, prefix), _)| prefix.clone())); @@ -1947,13 +2183,13 @@ impl Router { let Some(answer) = self.answering.get(&peer) else { continue; }; - let watch = answer.watch.clone(); + let prefixes = answer.prefixes.clone(); let first = !self.answering[&peer].told.contains(&hop); let (mut moved, mut holds, mut parents, mut ends, mut rest) = (Vec::new(), Vec::new(), Vec::new(), Vec::new(), 0); for key in &through { self.step(); - let want = self.want(key.0, &key.1, peer, &watch); + let want = self.want(key.0, &key.1, peer, &prefixes); let sent = &self.answering[&peer].tx[key]; match want { Want::Path(offer, src) => { @@ -2130,19 +2366,19 @@ impl Router { /// Bring what each watching neighbour was sent for `id` up to date, with the `dirty` /// claims changed since. fn advertise(&mut self, id: Id, dirty: &Dirty, region: Option<&Path>) { - let peers: Vec<(N, Path)> = self + let peers: Vec<(N, BTreeSet)> = self .answering .iter() - .map(|(peer, answer)| (*peer, answer.watch.clone())) + .map(|(peer, answer)| (*peer, answer.prefixes.clone())) .collect(); // Plan every stream against the state as it stands, then send. let mut plans = Vec::new(); let mut planned: BTreeMap> = BTreeMap::new(); - for (peer, watch) in &peers { - for prefix in self.stream(id, *peer, watch, region) { + for (peer, prefixes) in &peers { + for prefix in self.stream(id, *peer, prefixes, region) { planned.entry(*peer).or_default().insert(prefix.clone()); self.step(); - let want = self.want(id, &prefix, *peer, watch); + let want = self.want(id, &prefix, *peer, prefixes); let on = (self.answering[peer].tx.get(&(id, prefix.clone()))).is_some_and(|sent| sent.offer().is_some()); let batch = match &want { @@ -2217,7 +2453,7 @@ impl Router { }); sent.src = src; let sent = sent.reoffer(offer); - sent.from = from; + sent.from = Some(from); // Even unchanged claims need their newer floor forwarded: a neighbour // may still hold an intervening revision learned on another path. let advanced = floor.is_some() && sent.view.floors.get(&prefix).copied() != floor; @@ -2290,7 +2526,7 @@ impl Router { let (from, view) = self.covering_view(id, prefix).expect("a route that meets"); let floor = view.floors.get(&from).copied(); let sent = self.answering[&peer].tx.get(&(id, prefix.clone())); - let fresh = sent.is_none_or(|sent| sent.active().is_none_or(|sent| sent.from != from)); + let fresh = sent.is_none_or(|sent| sent.active().is_none_or(|sent| sent.from.as_ref() != Some(&from))); let changes: Vec<(Path, Option)> = match (sent, dirty.all || fresh) { (Some(sent), false) => { let old = &sent.active().expect("offered").view; @@ -2445,13 +2681,13 @@ impl Router { // Reflected sources must move before old holds ask whether a child answered. let mut sources = Vec::new(); for (&peer, answer) in self.answering.iter().filter(|_| route_changed || !held.is_empty()) { - let watch = &answer.watch; + let prefixes = &answer.prefixes; for &id in &ids { let region = scope.as_ref().filter(|(key, _)| *key == id).map(|(_, prefix)| prefix); - for prefix in self.stream(id, peer, watch, region) { + for prefix in self.stream(id, peer, prefixes, region) { let key = (id, prefix); if let Some(sent) = answer.tx.get(&key).filter(|sent| sent.offer().is_some()) { - sources.push((peer, key.clone(), sent.src.clone(), self.src(id, &key.1, peer, watch))); + sources.push((peer, key.clone(), sent.src.clone(), self.src(id, &key.1, peer, prefixes))); } } } @@ -2502,7 +2738,8 @@ impl Router { .chain( self.routes .contains_key(&key) - .then(|| meet(prefix, &self.watch)) + .then(|| self.watched(prefix)) + .into_iter() .flatten(), ) }) @@ -2647,6 +2884,20 @@ impl Router { self.answering.keys().all(|peer| self.sessions.contains_key(peer)), "answering sessions" ); + for (peer, answer) in &self.answering { + assert!(!answer.prefixes.is_empty() && !nested(&answer.prefixes), "{peer:?}'s prefixes"); + for (_, prefix) in answer.tx.keys() { + assert!(within_any(prefix, &answer.prefixes), "sent {prefix:?} outside {peer:?}'s prefixes"); + } + } + for (peer, session) in &self.sessions { + for (_, prefix) in &session.heard { + assert!( + ancestors(prefix).any(|above| session.asked.contains_key(&above)), + "heard {prefix:?} outside what {peer:?} was asked" + ); + } + } assert!( self.routes .values() @@ -2718,7 +2969,7 @@ impl Router { /// How the reader sees `claim` of `id` now, from its [`Self::source`], available /// while a live route covers it. fn shown_now(&self, id: Id, claim: &Path) -> Option> { - if !within(claim, &self.watch) { + if !within_any(claim, &self.watch) { return None; } let available = || ancestors(claim).any(|above| self.routes.get(&(id, above)).is_some_and(Route::live)); @@ -2780,7 +3031,8 @@ impl Router { } } } - self.reader.apply(id, &self.watch, now, ended); + let watch = &self.watch; + self.reader.apply(id, |claim| within_any(claim, watch), now, ended); #[cfg(test)] assert_eq!( self.shown_claims diff --git a/rs/moq-net/src/route/reader.rs b/rs/moq-net/src/route/reader.rs index 41453589082..f4b4b682a66 100644 --- a/rs/moq-net/src/route/reader.rs +++ b/rs/moq-net/src/route/reader.rs @@ -6,7 +6,7 @@ use std::collections::{BTreeMap, BTreeSet}; use crate::Path; use super::view::ancestors; -use super::{Event, Id, Kind, Shown, within}; +use super::{Event, Id, Kind, Shown}; /// How the reader sees each claim, and the events not yet taken. pub(crate) struct Reader { @@ -90,13 +90,13 @@ impl Reader { } } - /// The reader now sees each claim of `id` in `now` as it says, inside `watch`. Log + /// The reader now sees each claim of `id` in `now` as it says, where `watched`. Log /// each change. A claim no longer shown ended if `ended` says the origin withdrew it /// as far as this node knows, and went offline if the node let go of it instead. pub fn apply( &mut self, id: Id, - watch: &Path, + watched: impl Fn(&Path) -> bool, now: BTreeMap>>, ended: impl Fn(&Path) -> bool, ) { @@ -129,7 +129,7 @@ impl Reader { let moved = ancestors(claim) .filter(|above| above != claim) .any(|above| shown(&above).is_some_and(|now| was.presented || now.item == was.item)); - let kind = if !within(claim, watch) { + let kind = if !watched(claim) { Kind::Unwatched } else if !moved && ended(claim) { Kind::End diff --git a/rs/moq-net/src/route/tests/handoff.rs b/rs/moq-net/src/route/tests/handoff.rs index 1d1ddcebebf..f75b5068a11 100644 --- a/rs/moq-net/src/route/tests/handoff.rs +++ b/rs/moq-net/src/route/tests/handoff.rs @@ -110,7 +110,10 @@ fn trailing_announcements_report_no_false_absence(rule: Rule) { sim.connect(node, b); assert!(sim.run()); assert!(sim.queued(b, node) >= 2, "a ROUTE then its announcements"); - assert!(sim.deliver(b, node), "only the ROUTE"); + // Under the path rule the stream opens with b's own ask and the marker answering node's. + for _ in 0..1 + 2 * usize::from(sim.paths()) { + assert!(sim.deliver(b, node), "only the ROUTE"); + } sim.disconnect(node, a); assert!(sim.run()); assert_eq!(sim.next_hop(node, 7), Some(b)); @@ -821,9 +824,9 @@ fn stale_replay_after_collection_lasts_only_until_the_withdrawal_behind_it(rule: if collect { sim.collect(b, 7); } - // The held ROUTE, then the snapshot carrying `live/x`: after SYNCED under the - // path rule, whose stream opens with it. - for _ in 0..2 + usize::from(sim.paths()) { + // The held ROUTE, then the snapshot carrying `live/x`: under the path rule after + // a's own ask, the marker answering b's, and SYNCED, which open its stream. + for _ in 0..2 + 3 * usize::from(sim.paths()) { assert!(sim.deliver(a, b)); } let forgot = collect || sim.paths(); diff --git a/rs/moq-net/src/route/tests/mod.rs b/rs/moq-net/src/route/tests/mod.rs index 85f7f5d9667..c666741a0af 100644 --- a/rs/moq-net/src/route/tests/mod.rs +++ b/rs/moq-net/src/route/tests/mod.rs @@ -91,10 +91,11 @@ impl Relay { } } + /// The distance rule's interest generation. fn interest(&self) -> u32 { match self { Self::Distance(router) => router.interest(), - Self::Path(router) => router.interest(), + Self::Path(_) => unreachable!("the path rule edits its interest in place"), } } @@ -265,12 +266,11 @@ enum Msg { id: Id, seqno: Seqno, }, - /// A [`path::Router`] route, on the stream answering interest generation `interest`. + /// A [`path::Router`] route. Path { id: Id, prefix: Path, reach: Option, - interest: u32, }, /// A [`path::Router`] route's claims. Claims { @@ -278,11 +278,14 @@ enum Msg { prefix: Path, changes: Vec<(Path, Option)>, floor: Option, - interest: u32, }, Dead(path::Dead), + /// An edit of what the sender asks the receiver for. + Ask(path::Edit), + /// The marker where the receiver's oldest edit in flight takes effect. + Edited, Synced { - interest: u32, + prefix: Path, }, } @@ -343,7 +346,7 @@ pub(super) struct Sim { nodes: Vec, now: Instant, leaf: Vec, - watch: Vec, + watch: Vec>, restarts: Vec, /// The identity each origin node declared last, to spot a rotation. declared: Vec>, @@ -382,7 +385,7 @@ impl Sim { .collect(), now: Instant::ZERO, leaf: vec![false; nodes], - watch: vec![Path::empty(); nodes], + watch: vec![BTreeSet::from([Path::empty()]); nodes], restarts: vec![0; nodes], declared: vec![None; nodes], links: vec![vec![None; nodes]; nodes], @@ -471,7 +474,8 @@ impl Sim { } } - /// Open a session: each side opens a stream asking for its interest. + /// Open a session: each side asks for its interest, the path rule by an edit it + /// sends itself. pub fn connect(&mut self, a: Node, b: Node) { assert!(!self.connected(a, b), "{a}-{b} already up"); self.up.insert(Self::pair(a, b)); @@ -484,37 +488,19 @@ impl Sim { } .expect("a positive cost"); } - for (x, y) in [(a, b), (b, a)] { - self.ask(y, x); + if !self.paths() { + for (x, y) in [(a, b), (b, a)] { + self.ask(y, x); + } } self.flush(); } - /// `node` asks `peer` for announcements under its interest. Under the path rule a - /// node asks a neighbour publishing under a scoped grant for only what the scope - /// covers of it, and for nothing where they are disjoint. + /// Under the distance rule, `node` asks `peer` for announcements under its + /// interest on a new stream. fn ask(&mut self, node: Node, peer: Node) { - let watch = self.watch[node].clone(); - let scope = self.links[node][peer] - .and_then(|side| side.grant) - .map(|grant| p(grant.scope)); - let ask = match scope { - None => Some(watch.clone()), - Some(scope) if within(&scope, &watch) => Some(scope), - Some(scope) if within(&watch, &scope) => Some(watch.clone()), - Some(_) => None, - }; - let absent = ask.is_none(); - match &mut self.nodes[peer] { - Relay::Distance(router) => router.peer_interest(node, watch), - Relay::Path(router) => router.peer_interest(node, ask), - } - .unwrap(); - // No replacement stream will send SYNCED when the grant is disjoint. - // The receiver already knows it cancelled this part of its interest. - if absent && let Relay::Path(router) = &mut self.nodes[node] { - router.synced(peer, router.interest()); - } + let watch = self.watching(node).clone(); + self.babel_mut(peer).peer_interest(node, watch).unwrap(); } pub fn connected(&self, a: Node, b: Node) -> bool { @@ -574,8 +560,10 @@ impl Sim { self.nodes[node] = Relay::new(self.rule, hop(node, restarts), seed); let watch = self.watch[node].clone(); match &mut self.nodes[node] { - Relay::Distance(router) => router.watch(watch), - Relay::Path(router) => router.watch(watch), + Relay::Distance(router) => { + router.watch(watch.into_iter().next().expect("one prefix")); + } + Relay::Path(router) => router.watch(watch).expect("no prefix inside another"), }; self.flush(); } @@ -676,23 +664,42 @@ impl Sim { self.flush(); } - /// The prefix `node` asks its neighbors for announcements under; each - /// neighbor answers on a new stream with a complete snapshot. + /// The prefix `node` asks its neighbors for announcements under. Under the + /// distance rule each neighbor answers on a new stream with a complete snapshot; + /// under the path rule the node edits what it asks each neighbour for. pub fn watch(&mut self, node: Node, prefix: &str) { - let prefix = p(prefix); - self.watch[node] = prefix.clone(); + self.watch_all(node, &[prefix]); + } + + /// `node`'s interest becomes `prefixes`, none inside another. The distance rule + /// takes one prefix. + pub fn watch_all(&mut self, node: Node, prefixes: &[&str]) { + let set: BTreeSet = prefixes.iter().map(|prefix| p(prefix)).collect(); + self.watch[node] = set.clone(); match &mut self.nodes[node] { - Relay::Distance(router) => router.watch(prefix), - Relay::Path(router) => router.watch(prefix), - }; - let peers: Vec = self.peers(node).collect(); - for peer in peers { - self.ask(node, peer); + Relay::Distance(router) => { + assert_eq!(set.len(), 1, "the distance rule watches one prefix"); + router.watch(set.into_iter().next().expect("one prefix")); + let peers: Vec = self.peers(node).collect(); + for peer in peers { + self.ask(node, peer); + } + } + Relay::Path(router) => router.watch(set).expect("no prefix inside another"), } self.flush(); } + /// The one prefix `node` watches. pub fn watching(&self, node: Node) -> &Path { + let mut prefixes = self.watch[node].iter(); + let prefix = prefixes.next().expect("a watched prefix"); + assert!(prefixes.next().is_none(), "node {node} watches several prefixes"); + prefix + } + + /// Every prefix `node` watches. + pub fn interest(&self, node: Node) -> &BTreeSet { &self.watch[node] } @@ -762,15 +769,7 @@ impl Sim { }, ) => router.batch(from, id, changes, floor, interest), (Relay::Distance(router), Msg::Request { id, seqno }) => router.request(from, id, seqno), - ( - Relay::Path(router), - Msg::Path { - id, - prefix, - reach, - interest, - }, - ) => router.route(from, id, prefix, reach, interest), + (Relay::Path(router), Msg::Path { id, prefix, reach }) => router.route(from, id, prefix, reach), ( Relay::Path(router), Msg::Claims { @@ -778,11 +777,13 @@ impl Sim { prefix, changes, floor, - interest, }, - ) => router.batch(from, id, prefix, changes, floor, interest), + ) => router.batch(from, id, prefix, changes, floor), (Relay::Path(router), Msg::Dead(dead)) => router.dead(from, dead, now), - (Relay::Path(router), Msg::Synced { interest }) => router.synced(from, interest), + // A router sends only edits it may: a refused one would close the session. + (Relay::Path(router), Msg::Ask(edit)) => router.edit(from, edit).expect("a valid edit"), + (Relay::Path(router), Msg::Edited) => router.edited(from), + (Relay::Path(router), Msg::Synced { prefix }) => router.synced(from, prefix), (_, msg) => panic!("node {to} cannot read {msg:?}"), } self.stats.delivered += 1; @@ -1006,16 +1007,7 @@ impl Sim { } self.stats.route_msgs += 1; self.stats.route_items += 1; - let interest = self.nodes[to].interest(); - ( - to, - Msg::Path { - id, - prefix, - reach, - interest, - }, - ) + (to, Msg::Path { id, prefix, reach }) } path::Out::Batch { to, @@ -1026,7 +1018,6 @@ impl Sim { } => { self.stats.claim_msgs += 1; self.stats.claim_items += changes.len(); - let interest = self.nodes[to].interest(); ( to, Msg::Claims { @@ -1034,7 +1025,6 @@ impl Sim { prefix, changes, floor, - interest, }, ) } @@ -1046,10 +1036,9 @@ impl Sim { }; (to, Msg::Dead(dead)) } - path::Out::Synced { to } => { - let interest = self.nodes[to].interest(); - (to, Msg::Synced { interest }) - } + path::Out::Ask { to, edit } => (to, Msg::Ask(edit)), + path::Out::Edited { to } => (to, Msg::Edited), + path::Out::Synced { to, prefix } => (to, Msg::Synced { prefix }), }; self.queues.entry((node, to)).or_default().push_back(msg); } diff --git a/rs/moq-net/src/route/tests/scope.rs b/rs/moq-net/src/route/tests/scope.rs index a1244e81352..b3b3f1b60f0 100644 --- a/rs/moq-net/src/route/tests/scope.rs +++ b/rs/moq-net/src/route/tests/scope.rs @@ -22,15 +22,16 @@ fn narrowing_keeps_the_newest_covering_withdrawal() { let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); for peer in [1, 2] { router.connect(peer, peer as Id, Link::default()).unwrap(); + router.edited(peer); } - router.route(1, 7, p(""), Some(offered(1, 0)), 0); - router.batch(1, 7, p(""), vec![(p("a/b/x"), Some(1))], Some(1), 0); - router.route(2, 7, p("a"), Some(offered(2, 0)), 0); - router.batch(2, 7, p("a"), Vec::new(), Some(3), 0); + router.route(1, 7, p(""), Some(offered(1, 0))); + router.batch(1, 7, p(""), vec![(p("a/b/x"), Some(1))], Some(1)); + router.route(2, 7, p("a"), Some(offered(2, 0))); + router.batch(2, 7, p("a"), Vec::new(), Some(3)); assert!(router.events().iter().any(|event| event.kind == Kind::End)); assert!(!router.shown().contains_key(&(7, p("a/b/x")))); - router.watch(p("a/b")); + router.watch(BTreeSet::from([p("a/b")])).unwrap(); assert!(!router.shown().contains_key(&(7, p("a/b/x")))); assert!(router.events().iter().all(|event| event.kind != Kind::Start)); } @@ -43,6 +44,7 @@ fn repeating_an_expired_dead_hop_does_not_restart_its_timer() { let hop = path::Hop { node: 4, session: 1 }; for peer in [1, 2] { router.connect(peer, peer as Id, Link::default()).unwrap(); + router.edited(peer); let path::Reach::Finite(mut offer) = offered(peer as Id, 0) else { unreachable!() }; @@ -56,7 +58,7 @@ fn repeating_an_expired_dead_hop_does_not_restart_its_timer() { }, ] .into(); - router.route(peer, 7, p(""), Some(path::Reach::Finite(offer)), 0); + router.route(peer, 7, p(""), Some(path::Reach::Finite(offer))); } let dead = path::Dead::Closed { hop, @@ -77,18 +79,19 @@ fn losing_a_wide_route_does_not_forget_a_kept_narrow_view() { let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); for peer in [1, 2] { router.connect(peer, peer as Id, Link::default()).unwrap(); + router.edited(peer); } - router.route(1, 7, p(""), Some(offered(1, 0)), 0); - router.batch(1, 7, p(""), vec![(p("a/x"), Some(1))], Some(1), 0); - router.route(2, 7, p("a"), Some(offered(2, 0)), 0); - router.batch(2, 7, p("a"), Vec::new(), Some(3), 0); + router.route(1, 7, p(""), Some(offered(1, 0))); + router.batch(1, 7, p(""), vec![(p("a/x"), Some(1))], Some(1)); + router.route(2, 7, p("a"), Some(offered(2, 0))); + router.batch(2, 7, p("a"), Vec::new(), Some(3)); let ended = router .events() .into_iter() .find(|event| event.kind == Kind::End) .unwrap(); - router.route(1, 7, p(""), None, 0); - router.batch(2, 7, p("a"), vec![(p("a/x"), Some(2))], Some(4), 0); + router.route(1, 7, p(""), None); + router.batch(2, 7, p("a"), vec![(p("a/x"), Some(2))], Some(4)); let started = router .events() .into_iter() @@ -101,6 +104,7 @@ fn losing_a_wide_route_does_not_forget_a_kept_narrow_view() { fn flapping_a_far_node_does_not_extend_older_dead_hops() { let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); router.connect(1, 1, Link::default()).unwrap(); + router.edited(1); let mut now = Instant::ZERO; let retained = (Doubt::DEFAULT.timer.as_nanos() / HOP.as_nanos()) as usize + 1; for session in 0..3 * retained { @@ -116,7 +120,7 @@ fn flapping_a_far_node_does_not_extend_older_dead_hops() { path::Hop { node: 1, session: 1 }, ] .into(); - router.route(1, 7, p(""), Some(path::Reach::Finite(offer)), 0); + router.route(1, 7, p(""), Some(path::Reach::Finite(offer))); router.dead( 1, path::Dead::Closed { @@ -144,6 +148,7 @@ fn a_dead_repath_withdraws_its_claims_before_ending_the_sent_route() { let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); for peer in [1, 2, 3] { router.connect(peer, peer as Id, Link::default()).unwrap(); + router.edited(peer); } let path::Reach::Finite(mut old) = offered(1, 0) else { unreachable!() @@ -156,10 +161,10 @@ fn a_dead_repath_withdraws_its_claims_before_ending_the_sent_route() { ] .into(); for prefix in ["", "a"] { - router.route(1, 7, p(prefix), Some(path::Reach::Finite(old.clone())), 0); - router.batch(1, 7, p(prefix), vec![(p("a/x"), Some(1))], Some(1), 0); + router.route(1, 7, p(prefix), Some(path::Reach::Finite(old.clone()))); + router.batch(1, 7, p(prefix), vec![(p("a/x"), Some(1))], Some(1)); } - router.peer_interest(3, Some(p("a"))).unwrap(); + router.edit(3, path::Edit { remove: Vec::new(), add: vec![p("a")] }).unwrap(); // A child still names this relay, so its newer empty view is not believed yet. let mut child = old; child.path = [ @@ -170,8 +175,8 @@ fn a_dead_repath_withdraws_its_claims_before_ending_the_sent_route() { path::Hop { node: 2, session: 1 }, ] .into(); - router.route(2, 7, p("a"), Some(path::Reach::Finite(child)), 0); - router.batch(2, 7, p("a"), Vec::new(), Some(3), 0); + router.route(2, 7, p("a"), Some(path::Reach::Finite(child))); + router.batch(2, 7, p("a"), Vec::new(), Some(3)); router.drain(); let path::Reach::Finite(offer) = offered(2, 1) else { unreachable!() @@ -284,12 +289,15 @@ fn drain_news_is_linear_in_neighbours() { let leaving: Id = 500; let idle = 2 * peers; router.connect(idle, 3000, Link::default()).unwrap(); + router.edited(idle); for i in 0..peers { let (up, down) = (i, peers + i); router.connect(up, 1000 + i as Id, Link::default()).unwrap(); + router.edited(up); router.connect(down, 2000 + i as Id, Link::default()).unwrap(); + router.edited(down); let prefix = p(&format!("live/{i}")); - router.peer_interest(down, Some(prefix.clone())).unwrap(); + router.edit(down, path::Edit { remove: Vec::new(), add: vec![prefix.clone()] }).unwrap(); let path = [ path::Hop { node: 7, session: 0 }, path::Hop { @@ -309,8 +317,8 @@ fn drain_news_is_linear_in_neighbours() { down_only: false, rtt: None, }; - router.route(up, 7, prefix.clone(), Some(path::Reach::Finite(offer)), 0); - router.batch(up, 7, prefix, vec![(p(&format!("live/{i}/x")), Some(1))], Some(1), 0); + router.route(up, 7, prefix.clone(), Some(path::Reach::Finite(offer))); + router.batch(up, 7, prefix, vec![(p(&format!("live/{i}/x")), Some(1))], Some(1)); } router.drain(); let before = router.work(); @@ -361,11 +369,12 @@ fn following_a_prefix_does_not_revisit_its_siblings() { let before = router.work(); let prefix = p(&format!("live/{scope}")); router.connect(scope, 1, Link::default()).unwrap(); - router.route(scope, 7, prefix.clone(), Some(offered(1, 0)), 0); + router.edited(scope); + router.route(scope, 7, prefix.clone(), Some(offered(1, 0))); let changes = (0..claims) .map(|claim| (p(&format!("live/{scope}/{claim}")), Some(1))) .collect(); - router.batch(scope, 7, prefix, changes, Some(1), 0); + router.batch(scope, 7, prefix, changes, Some(1)); let work = router.work() - before; let expected = *first.get_or_insert(work); assert_eq!( @@ -908,6 +917,7 @@ fn unrelated_dead_messages_do_not_allocate_or_poison_routes() { let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); for peer in [1, 2] { router.connect(peer, peer as Id, Link::default()).unwrap(); + router.edited(peer); } let now = Instant::ZERO; for node in 100..1100 { @@ -923,8 +933,8 @@ fn unrelated_dead_messages_do_not_allocate_or_poison_routes() { ); } assert_eq!((router.retained().dead, router.retained().doubt), (0, 0)); - router.route(1, 7, p(""), Some(offered(1, 0)), 0); - router.batch(1, 7, p(""), vec![(p("a"), Some(1))], Some(1), 0); + router.route(1, 7, p(""), Some(offered(1, 0))); + router.batch(1, 7, p(""), vec![(p("a"), Some(1))], Some(1)); let hop = path::Hop { node: 1, session: 1 }; // Another peer has no credit for this hop, even though this node heard it. router.dead( @@ -971,8 +981,9 @@ fn dead_replacement_paths_obey_the_same_limit_as_route_paths() { for length in [0, MAX_HOPS - 1, MAX_HOPS] { let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); router.connect(1, 1, Link::default()).unwrap(); - router.route(1, 7, p(""), Some(offered(1, 0)), 0); - router.batch(1, 7, p(""), vec![(p("a"), Some(1))], Some(1), 0); + router.edited(1); + router.route(1, 7, p(""), Some(offered(1, 0))); + router.batch(1, 7, p(""), vec![(p("a"), Some(1))], Some(1)); let offer = path::Offer { path: (0..length) .map(|i| path::Hop { @@ -1025,28 +1036,30 @@ fn an_upstream_stream_uses_a_narrower_exportable_source() { }, ) .unwrap(); + router.edited(peer); } - router.route(1, 7, p(""), Some(offered(1, 0)), 0); - router.batch(1, 7, p(""), vec![(p("a/x"), Some(1))], Some(1), 0); + router.route(1, 7, p(""), Some(offered(1, 0))); + router.batch(1, 7, p(""), vec![(p("a/x"), Some(1))], Some(1)); let path::Reach::Finite(mut narrow) = offered(2, 0) else { unreachable!() }; narrow.metric = 2; - router.route(2, 7, p("a"), Some(path::Reach::Finite(narrow)), 0); - router.peer_interest(3, Some(p("a"))).unwrap(); + router.route(2, 7, p("a"), Some(path::Reach::Finite(narrow))); + router.edit(3, path::Edit { remove: Vec::new(), add: vec![p("a")] }).unwrap(); assert!(router.drain().iter().any(|msg| matches!(msg, path::Out::Route { to: 3, prefix, reach: Some(path::Reach::Finite(offer)), .. } if *prefix == p("a") && !offer.down_only && offer.path[1].node == 2))); } #[test] fn replacing_a_childs_interest_keeps_the_hold_until_its_answer() { - for interest in [None, Some(p("a"))] { + for add in [Vec::new(), vec![p("a")]] { let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); for peer in [1, 2] { router.connect(peer, peer as Id, Link::default()).unwrap(); + router.edited(peer); } - router.route(1, 7, p(""), Some(offered(1, 0)), 0); - router.batch(1, 7, p(""), vec![(p("a/x"), Some(1))], Some(1), 0); - router.peer_interest(2, Some(p(""))).unwrap(); + router.route(1, 7, p(""), Some(offered(1, 0))); + router.batch(1, 7, p(""), vec![(p("a/x"), Some(1))], Some(1)); + router.edit(2, path::Edit { remove: Vec::new(), add: vec![p("")] }).unwrap(); let path::Reach::Finite(mut child) = offered(2, 0) else { unreachable!() }; @@ -1057,17 +1070,18 @@ fn replacing_a_childs_interest_keeps_the_hold_until_its_answer() { path::Hop { node: 2, session: 1 }, ] .into(); - router.route(2, 7, p(""), Some(path::Reach::Finite(child)), 0); + router.route(2, 7, p(""), Some(path::Reach::Finite(child))); router.disconnect([1], Instant::ZERO); assert!(!router.shown()[&(7, p("a/x"))].available); router.events(); - router.peer_interest(2, interest).unwrap(); + let remove = vec![p("")]; + router.edit(2, path::Edit { remove, add }).unwrap(); assert!( !router.shown()[&(7, p("a/x"))].available, "the child still owes an answer" ); assert!(router.events().is_empty()); - router.route(2, 7, p(""), Some(offered(2, 0)), 0); + router.route(2, 7, p(""), Some(offered(2, 0))); assert!(router.shown()[&(7, p("a/x"))].available); assert!(router.events().iter().all(|event| event.kind == Kind::Available)); } @@ -1090,16 +1104,17 @@ fn an_upstream_stream_keeps_a_hold_over_an_unexportable_live_source() { }, ) .unwrap(); + router.edited(peer); } - router.route(1, 7, p(live), Some(offered(1, 0)), 0); - router.batch(1, 7, p(live), vec![(p("a/x"), Some(1))], Some(1), 0); + router.route(1, 7, p(live), Some(offered(1, 0))); + router.batch(1, 7, p(live), vec![(p("a/x"), Some(1))], Some(1)); let path::Reach::Finite(mut offer) = offered(2, 0) else { unreachable!() }; offer.metric = 2; - router.route(2, 7, p(held), Some(path::Reach::Finite(offer)), 0); - router.batch(2, 7, p(held), vec![(p("a/x"), Some(1))], Some(1), 0); - router.peer_interest(3, Some(p("a"))).unwrap(); + router.route(2, 7, p(held), Some(path::Reach::Finite(offer))); + router.batch(2, 7, p(held), vec![(p("a/x"), Some(1))], Some(1)); + router.edit(3, path::Edit { remove: Vec::new(), add: vec![p("a")] }).unwrap(); let path::Reach::Finite(mut child) = offered(3, 0) else { unreachable!() }; @@ -1110,7 +1125,7 @@ fn an_upstream_stream_keeps_a_hold_over_an_unexportable_live_source() { path::Hop { node: 3, session: 1 }, ] .into(); - router.route(3, 7, p("a"), Some(path::Reach::Finite(child)), 0); + router.route(3, 7, p("a"), Some(path::Reach::Finite(child))); router.drain(); router.disconnect([2], Instant::ZERO); let messages = router.drain(); From e81ece0c78157e0ba5d1099a94ea75ce33005ab0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 03:41:35 -0800 Subject: [PATCH 03/11] wip(route): edit fixtures, sweeps over edited sets, carry own routes at the marker Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/path.rs | 68 ++++-- rs/moq-net/src/route/tests/handoff.rs | 60 +++-- rs/moq-net/src/route/tests/interest.rs | 298 +++++++++++++++++++++++++ rs/moq-net/src/route/tests/mod.rs | 1 + 4 files changed, 386 insertions(+), 41 deletions(-) create mode 100644 rs/moq-net/src/route/tests/interest.rs diff --git a/rs/moq-net/src/route/path.rs b/rs/moq-net/src/route/path.rs index 5caf0f15e1c..2b68a71e4f5 100644 --- a/rs/moq-net/src/route/path.rs +++ b/rs/moq-net/src/route/path.rs @@ -634,10 +634,18 @@ impl Carried for Sent { } } +/// What [`rekey`] leaves of a stream's routes. +struct Rekeyed { + routes: BTreeMap, + /// The keys it no longer carries a route under. + gone: Vec, + /// Each route that moved, with its offer, to another key: from, to. + moved: Vec<(RouteId, RouteId)>, +} + /// The routes one stream carries once an edit of its prefixes from `old` to `new` -/// takes effect at its marker, and the keys it no longer carries. Both sides apply -/// it alike to what was sent and what was heard, so they stay in step and nothing -/// both sets cover is sent again. +/// takes effect at its marker. Both sides apply it alike to what was sent and what +/// was heard, so they stay in step and nothing both sets cover is sent again. /// /// A route inside a prefix the new set covers stays. A route at a removed prefix /// that an added prefix widens moves to the wider prefix: the first by key keeps @@ -646,13 +654,10 @@ impl Carried for Sent { /// claim and the floors certified only the narrower prefix. A route at a removed /// prefix is carried, as presented there, to each added prefix inside it that no /// route keys, so its floor carries too. The rest go. -fn rekey>( - old: &BTreeSet, - new: &BTreeSet, - routes: BTreeMap, -) -> (BTreeMap, Vec) { +fn rekey>(old: &BTreeSet, new: &BTreeSet, routes: BTreeMap) -> Rekeyed { let mut kept: BTreeMap = BTreeMap::new(); let mut gone = Vec::new(); + let mut moved = Vec::new(); for (key, mut route) in routes.iter().map(|(key, route)| (key.clone(), route.clone())) { let (id, prefix) = &key; let Some(region) = ancestors(prefix).find(|above| old.contains(above)) else { @@ -676,6 +681,7 @@ fn rekey>( Some(first) => first.view().expect("a carried route").items.extend(items), None => { route.view().expect("a carried route").items = items; + moved.push((key.clone(), (*id, wider.clone()))); kept.insert((*id, wider), route); } } @@ -695,10 +701,15 @@ fn rekey>( } let mut copy = route.clone(); *copy.view().expect("a carried route") = view.present(inside); + moved.push(((*id, region.clone()), key.clone())); kept.insert(key, copy); } } - (kept, gone) + Rekeyed { + routes: kept, + gone, + moved, + } } /// The entries of `map` under identity `id`. @@ -848,16 +859,16 @@ impl Router { } self.unreflect(neighbor, key, &sent.src); } - let (rekeyed, gone) = rekey(&old, &new, tx.clone()); + let rekeyed = rekey(&old, &new, tx.clone()); // A child that learned a route this stream no longer carries under its key // may still owe an answer to the route it reflected. - for key in gone { + for key in rekeyed.gone { if self.child_pending(key.0, neighbor, &key.1) { let src = (key.0, tx[&key].src.clone()); self.waiting.entry(src).or_default().insert((neighbor, key.1)); } } - let tx = rekeyed; + let tx = rekeyed.routes; for (key, sent) in &tx { if let Some(offer) = sent.offer() { self.index_sent(neighbor, key, &offer.path); @@ -1250,16 +1261,29 @@ impl Router { for key in &keys { self.forget_heard(key, from); } - let (heard, _) = rekey(&old, &new, heard); - for (key, heard) in heard { + let rekeyed = rekey(&old, &new, heard); + for (key, heard) in rekeyed.routes { self.hear(key, from, heard); } + // A route this node used or held through `from` moves with the route it heard, + // so a hold or a doubt carries over rather than letting go of its claims. + for (was, now) in rekeyed.moved { + let Some(route) = self.routes.get(&was).filter(|route| route.next() == Some(from)) else { + continue; + }; + if !self.routes.contains_key(&now) { + let route = route.clone(); + self.scopes.entry(now.0).or_default().insert(now.1.clone()); + self.routes.insert(now, route); + } + } for (id, named) in filed { if of(&self.rx, id).any(|(_, routes)| routes.contains_key(&from)) { self.filed.insert((id, from), named); } } - self.update(ids, Vec::new(), None); + // A carried route may already have changed what the reader sees. + self.refresh(ids); } /// `from` sent the initial set of `prefix`, which this node asks it for. @@ -1946,13 +1970,13 @@ impl Router { Some((prefix.clone(), view.floors.get(prefix).copied())) }) .collect(); + // A view no route covers goes only once the narrower views a route still covers + // have learned what it knew. + let mut lost = Vec::new(); for prefix in keys { let key = (id, prefix.clone()); if !self.presence(id, prefix) { - if self.views.remove(&key).is_some() { - self.forget(id, prefix, &[]); - dirty.regions.insert(prefix.clone()); - } + lost.push(key); continue; } // Taken out while it is brought up to date, and always put back. @@ -2018,6 +2042,12 @@ impl Router { } self.views.insert(key, view); } + for key in lost { + if self.views.remove(&key).is_some() { + self.forget(id, &key.1, &[]); + dirty.regions.insert(key.1); + } + } // Which view the reader sees a claim at a view's own prefix from turns on floors. let floors: Vec<(Path, Option)> = keys .iter() diff --git a/rs/moq-net/src/route/tests/handoff.rs b/rs/moq-net/src/route/tests/handoff.rs index f75b5068a11..8c2b950e8f0 100644 --- a/rs/moq-net/src/route/tests/handoff.rs +++ b/rs/moq-net/src/route/tests/handoff.rs @@ -9,6 +9,7 @@ use std::collections::BTreeSet; +use super::path::meet; use super::*; both!( @@ -1150,10 +1151,22 @@ fn announcement_cost_follows_sessions_and_live_claims(rule: Rule) { /// grant under one subject, so both feed one qualified view. Under churn every /// view stays some revision of the origin's truth, the widening guard stays /// bounded, no claim ends while the origin still has it, and once churn stops -/// the client shows exactly the claims its interest and the scopes admit. +/// the client shows exactly the claims its interest and the scopes admit. Under +/// the path rule the client's interest is a set, edited in place. fn sweep_scoped_handoff(rule: Rule) { const CLAIMS: [&str; 5] = ["live/a/x", "live/a/y/", "live/b/z", "live/c", "live/"]; - const WATCHES: [&str; 5] = ["", "live/", "live/a/", "live/b/z", "live/c"]; + const WATCHES: [&[&str]; 5] = [&[""], &["live/"], &["live/a/"], &["live/b/z"], &["live/c"]]; + const SETS: [&[&str]; 8] = [ + &[""], + &["live/"], + &["live/a/"], + &["live/b/z"], + &["live/c"], + &["live/a/", "live/b/z"], + &["live/a/x", "live/a/y", "live/c"], + &["live/a/", "live/b/", "live/c"], + ]; + let watches: &[&[&str]] = if rule == Rule::Distance { &WATCHES } else { &SETS }; const SCOPES: [&str; 2] = ["live/a/", "live/b/"]; let (origin, r1, r2, client) = (0, 1, 2, 3); let mut rng = XorShift(0x5c0b_ed00); @@ -1185,7 +1198,7 @@ fn sweep_scoped_handoff(rule: Rule) { sim.unannounce(origin, claim); } } - 3 => sim.watch(client, WATCHES[rng.next(WATCHES.len() as u64) as usize]), + 3 => sim.watch_all(client, watches[rng.next(watches.len() as u64) as usize]), 4 => { let (a, b) = links[rng.next(links.len() as u64) as usize]; if sim.connected(a, b) { @@ -1227,21 +1240,13 @@ fn sweep_scoped_handoff(rule: Rule) { } assert!(sim.run()); sim.check().unwrap_or_else(|err| panic!("scenario {scenario}: {err}")); - let watch = sim.watching(client).clone(); + let watch = sim.interest(client).clone(); let rev = sim.revision(7); - // Each relay presents its scope to the client's interest. + // Each relay presents its scope to each prefix of the client's interest. let expected: BTreeSet = SCOPES .iter() .map(|scope| p(scope)) - .filter_map(|scope| { - if within(&scope, &watch) { - Some(scope) - } else if within(&watch, &scope) { - Some(watch.clone()) - } else { - None - } - }) + .flat_map(|scope| watch.iter().filter_map(move |watch| meet(&scope, watch))) .flat_map(|region| sim.presented(7, ®ion, rev).into_keys()) .collect(); let id = qualified("local", 7); @@ -1274,7 +1279,7 @@ const CLAIMS: [&str; 4] = ["live/", "live/a/", "live/a/x", "live/b"]; /// shows every live origin's current claims and holds nothing for a dead one. fn sweep_handoff(rule: Rule) { for scenario in 0..scenarios() { - let mut sim = churn(rule, scenario); + let mut sim = churn(rule, scenario, rule != Rule::Distance); let label = format!("scenario {scenario}"); assert_eq!(revivals(&sim.events), 0, "{label}"); assert_eq!(false_ends(&sim.events), 0, "{label}"); @@ -1321,7 +1326,7 @@ fn the_rules_agree_on_every_script() { for scenario in 0..scenarios() { let label = format!("scenario {scenario}"); let sims = [Rule::Distance, Rule::PATH].map(|rule| { - let mut sim = churn(rule, scenario); + let mut sim = churn(rule, scenario, false); assert_eq!(revivals(&sim.events), 0, "{label}: {rule:?}"); assert_eq!(false_ends(&sim.events), 0, "{label}: {rule:?}"); settled(&mut sim, &label); @@ -1335,8 +1340,17 @@ fn the_rules_agree_on_every_script() { /// Origins sit at `RELAYS` and `RELAYS + 1`, the client at `RELAYS + 2`. The script, /// every change made from outside, depends on `scenario` alone, so both rules run -/// the same one; which queued message arrives next is drawn apart from it. -fn churn(rule: Rule, scenario: usize) -> Sim { +/// the same one; which queued message arrives next is drawn apart from it. With +/// `sets`, which only the path rule takes, the client watches sets of prefixes. +fn churn(rule: Rule, scenario: usize, sets: bool) -> Sim { + const SETS: [&[&str]; 6] = [ + &[""], + &["live/a/"], + &["live/b"], + &["live/a/", "live/b"], + &["live/a/x", "live/b"], + &["live/a/x", "other"], + ]; let rng = &mut XorShift(0x4a4d_0ff5 ^ (scenario as u64 + 1).wrapping_mul(0x9e37_79b9_7f4a_7c15) | 1); let mut pick = XorShift(0x5eed_0003 + scenario as u64); let (first, client) = (RELAYS, RELAYS + 2); @@ -1393,9 +1407,8 @@ fn churn(rule: Rule, scenario: usize) -> Sim { sim.release(from, to); } } - 6 => { - sim.watch(client, ["", "live/a/", "live/b"][rng.next(3) as usize]); - } + 6 if sets => sim.watch_all(client, SETS[rng.next(SETS.len() as u64) as usize]), + 6 => sim.watch(client, ["", "live/a/", "live/b"][rng.next(3) as usize]), 7 if rng.chance(50) => { // Collection is allowed once a node holds no route to an identity. let node = rng.next(RELAYS as u64 + 3) as usize; @@ -1453,7 +1466,9 @@ fn settled(sim: &mut Sim, label: &str) { for node in 0..RELAYS + 3 { let mut expected = BTreeMap::new(); for (origin, &id) in (RELAYS..).zip(&live) { - let presented = sim.presented(id, sim.watching(node), sim.revision(id)); + let presented: BTreeMap = (sim.interest(node).iter()) + .flat_map(|watch| sim.presented(id, watch, sim.revision(id))) + .collect(); // A route reaches a node only while a claim meets its interest. assert_eq!( sim.routed(node, id), @@ -1810,3 +1825,4 @@ fn a_goaway_two_hops_from_a_ring_never_loops_a_request() { sim.join(leaving, hub, 1); goaway_around_a_ring(&mut sim, hub, leaving); } + diff --git a/rs/moq-net/src/route/tests/interest.rs b/rs/moq-net/src/route/tests/interest.rs new file mode 100644 index 00000000000..4476f214893 --- /dev/null +++ b/rs/moq-net/src/route/tests/interest.rs @@ -0,0 +1,298 @@ +//! An interest as a set of prefixes edited in place, under [`path::Router`]: what +//! an edit keeps, what it sends, and how it orders against everything else a +//! neighbour says on its one stream. + +use super::*; + +const ROOMS: [&str; 3] = ["live/a", "live/b", "live/c"]; + +/// The origin behind one relay, and a leaf client of the relay watching +/// `watch`, with `per` claims in each of `ROOMS`. +fn chain(watch: &[&str], per: usize) -> Sim { + let (origin, relay, client) = (0, 1, 2); + let mut sim = Sim::with(Rule::PATH, 3); + sim.leaf(origin); + sim.leaf(client); + sim.join(origin, relay, 1); + sim.join(relay, client, 1); + sim.watch_all(client, watch); + sim.originate(origin, 7, 0); + for room in ROOMS { + for i in 0..per { + sim.announce(origin, &format!("{room}/{i}"), 0, None); + } + } + assert!(sim.run()); + sim +} + +/// The claims `client` shows, available. +fn shown(sim: &Sim, client: Node) -> BTreeSet { + (sim.shown(client).iter()) + .filter(|(_, shown)| shown.available) + .map(|((_, claim), _)| claim.clone()) + .collect() +} + +fn rooms(rooms: &[&str], per: usize) -> BTreeSet { + (rooms.iter()) + .flat_map(|room| (0..per).map(move |i| p(&format!("{room}/{i}")))) + .collect() +} + +/// Swapping two prefixes for the one covering both, as the maintainer put it +/// ("fleet absorbing the previous two without retransmitting everything"), sends +/// only what the wider prefix newly covers, and the reader sees only that start. +#[test] +fn widening_swaps_prefixes_without_resending_what_both_cover() { + const PER: usize = 20; + let client = 2; + let mut sim = chain(&["live/a", "live/b"], PER); + assert_eq!(shown(&sim, client), rooms(&["live/a", "live/b"], PER)); + let (items, before) = (sim.stats.claim_items, sim.events.len()); + + sim.watch(client, "live"); + assert!(sim.run()); + assert_eq!(shown(&sim, client), rooms(&ROOMS, PER)); + assert_eq!(sim.stats.claim_items - items, PER, "only live/c is sent"); + let events = &sim.events[before..]; + assert!( + events.iter().all(|event| event.kind == Kind::Start && event.claim.starts_with("live/c/")), + "{events:?}" + ); + assert_eq!(sim.path(client).is_synced(1, &p("live")), Some(true)); + sim.check().unwrap(); +} + +/// Swapping one prefix for two inside it carries the wider floor to each, so a +/// narrowing that changes nothing it keeps sends no claim, and the reader only +/// stops watching what it dropped. +#[test] +fn narrowing_carries_the_wider_floor() { + const PER: usize = 20; + let client = 2; + let mut sim = chain(&["live"], PER); + let (items, before) = (sim.stats.claim_items, sim.events.len()); + + sim.watch_all(client, &["live/a", "live/b"]); + assert!(sim.run()); + assert_eq!(shown(&sim, client), rooms(&["live/a", "live/b"], PER)); + assert_eq!(sim.stats.claim_items, items, "the floor carries, so nothing is sent"); + let events = &sim.events[before..]; + assert!( + events.iter().all(|event| event.kind == Kind::Unwatched && event.claim.starts_with("live/c/")), + "{events:?}" + ); + // The carried floor still certifies a withdrawal. + sim.unannounce(0, "live/a/0"); + assert!(sim.run()); + assert_eq!(life(&sim, client, "live/a/0"), [Kind::Start, Kind::End]); + sim.check().unwrap(); +} + +/// What the relay sent before the marker answers the old set: a withdrawal in a +/// region the edit drops is never shown, one in a region it keeps ends the claim, +/// and the wider set then shows the origin's claims, with nothing revived. +#[test] +fn an_edit_crossing_claim_changes_applies_them_to_the_set_they_answered() { + let (origin, relay, client) = (0, 1, 2); + for widen in [false, true] { + let mut sim = chain(&["live/a", "live/c"], 2); + sim.hold(relay, client); + sim.unannounce(origin, "live/a/0"); + sim.unannounce(origin, "live/c/0"); + sim.announce(origin, "live/b/9", 0, None); + assert!(sim.run()); + match widen { + true => sim.watch(client, "live"), + false => sim.watch_all(client, &["live/a"]), + } + assert!(sim.run()); + assert!(sim.queued(relay, client) > 0, "the old set's answer waits"); + sim.release(relay, client); + assert!(sim.run()); + let expected = match widen { + true => ["live/a/1", "live/b/0", "live/b/1", "live/b/9", "live/c/1"].as_slice(), + false => ["live/a/1"].as_slice(), + }; + assert_eq!(shown(&sim, client), expected.iter().map(|claim| p(claim)).collect()); + assert_eq!(life(&sim, client, "live/a/0"), [Kind::Start, Kind::End]); + let dropped: &[Kind] = match widen { + true => &[Kind::Start, Kind::End], + false => &[Kind::Start, Kind::Unwatched], + }; + assert_eq!(life(&sim, client, "live/c/0"), dropped); + assert_eq!(revivals(&sim.events), 0); + assert_eq!(false_ends(&sim.events), 0); + sim.check().unwrap(); + } +} + +/// An edit crosses a session closing behind the relay: the relay's hold on its +/// route and the withdrawal naming the dead hop ride the same stream as the +/// marker, in either order. The client holds the claim it kept through the +/// marker exactly as it does with no edit, then reaches it on the detour, with +/// nothing ended or revived. +#[test] +fn an_edit_crossing_a_dead_hop_and_a_hold_settles() { + let (origin, r1, r2, client) = (0, 1, 2, 3); + let mut baseline = None; + for edit in [None, Some(false), Some(true)] { + let edit_first = edit == Some(true); + let mut sim = Sim::with(Rule::PATH, 4); + sim.leaf(origin); + sim.leaf(client); + sim.join(origin, r1, 1); + sim.join(origin, r2, 3); + sim.join(r1, client, 1); + sim.join(r2, client, 1); + sim.join(r1, r2, 1); + sim.watch_all(client, &["live/a"]); + sim.originate(origin, 7, 0); + sim.announce(origin, "live/a/x", 0, None); + sim.announce(origin, "live/b/y", 0, None); + assert!(sim.run()); + assert_eq!(sim.next_hop(client, 7), Some(r1)); + + sim.hold(r1, client); + sim.hold(r2, client); + if edit_first { + sim.watch(client, "live"); + sim.disconnect(origin, r1); + } else { + sim.disconnect(origin, r1); + if edit.is_some() { + sim.watch(client, "live"); + } + } + assert!(sim.run()); + sim.release(r1, client); + sim.release(r2, client); + assert!(sim.run()); + if edit.is_some() { + assert_eq!(shown(&sim, client), [p("live/a/x"), p("live/b/y")].into()); + } + assert_eq!(sim.reaches(client, 7, &p("live/a/x")), Some(origin)); + let life = life(&sim, client, "live/a/x"); + assert_eq!(&life, baseline.get_or_insert_with(|| life.clone()), "edit {edit:?}"); + assert!( + life.iter().all(|kind| !matches!(kind, Kind::End | Kind::Offline)), + "{life:?}" + ); + assert_eq!(false_ends(&sim.events), 0); + assert_eq!(revivals(&sim.events), 0); + assert_eq!(sim.pending(), 0); + sim.check().unwrap(); + } +} + +/// Each added prefix is SYNCED on its own: one waiting behind a held stream is +/// not, while the other neighbour's already is. +#[test] +fn each_added_prefix_is_synced_on_its_own() { + let (origin, r1, r2, client) = (0, 1, 2, 3); + let mut sim = Sim::with(Rule::PATH, 4); + sim.leaf(origin); + sim.leaf(client); + sim.join(origin, r1, 1); + sim.join(origin, r2, 1); + sim.join(r1, client, 1); + sim.join(r2, client, 1); + sim.watch_all(client, &["live/a"]); + assert!(sim.run()); + sim.hold(r2, client); + sim.watch_all(client, &["live/a", "live/b"]); + assert!(sim.run()); + let router = sim.path(client); + assert_eq!(router.is_synced(r1, &p("live/b")), Some(true)); + assert_eq!(router.is_synced(r2, &p("live/a")), Some(true)); + assert_eq!(router.is_synced(r2, &p("live/b")), None, "its edit waits behind the marker"); + assert_eq!(router.is_synced(r1, &p("live/c")), None); + sim.release(r2, client); + assert!(sim.run()); + assert_eq!(sim.path(client).is_synced(r2, &p("live/b")), Some(true)); +} + +/// An edit that names a prefix it does not hold, or leaves one inside another, is +/// refused whole, and so is an interest with one prefix inside another. +#[test] +fn edits_that_nest_or_name_no_live_prefix_are_refused() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.connect(1, 1, Link::default()).unwrap(); + let edit = |remove: &[&str], add: &[&str]| path::Edit { + remove: remove.iter().map(|prefix| p(prefix)).collect(), + add: add.iter().map(|prefix| p(prefix)).collect(), + }; + router.edit(1, edit(&[], &["live/a"])).unwrap(); + assert_eq!(router.edit(1, edit(&["live/b"], &[])), Err(Error::UnknownPrefix)); + assert_eq!(router.edit(1, edit(&[], &["live/a/x"])), Err(Error::Nested)); + assert_eq!(router.edit(1, edit(&[], &["live"])), Err(Error::Nested)); + assert_eq!(router.edit(1, edit(&[], &["live/b", "live/b/x"])), Err(Error::Nested)); + assert_eq!(router.edit(2, edit(&[], &["live"])), Err(Error::UnknownNeighbor)); + let markers = |router: &mut path::Router| { + (router.drain().iter()) + .filter(|out| matches!(out, path::Out::Edited { .. })) + .count() + }; + assert_eq!(markers(&mut router), 1, "a refused edit sends no marker"); + // A swap in one edit is no nesting. + router.edit(1, edit(&["live/a"], &["live"])).unwrap(); + assert_eq!(markers(&mut router), 1); + assert_eq!( + router.watch(BTreeSet::from([p("live"), p("live/a")])), + Err(Error::Nested) + ); +} + +/// A scope with several regions asks for each where it meets the interest, so +/// each is certified on its own. +#[test] +fn a_scope_with_several_regions_asks_for_each() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.watch(BTreeSet::from([p("live")])).unwrap(); + let link = Link { + scope: Some(Scope { + regions: vec![p("live/a"), p("live/b/x"), p("other"), p("")], + qualify: Arc::new(|id| id | QUALIFIED), + }), + ..Link::default() + }; + router.connect(1, 1, link).unwrap(); + let asked: Vec = (router.drain().into_iter()) + .filter_map(|out| match out { + path::Out::Ask { edit, .. } => Some(edit), + _ => None, + }) + .collect(); + // The root region meets the interest at `live`, which covers the narrower two. + assert_eq!( + asked, + [path::Edit { + remove: Vec::new(), + add: vec![p("live")] + }] + ); + let link = Link { + scope: Some(Scope { + regions: vec![p("live/a"), p("live/b/x"), p("other")], + qualify: Arc::new(|id| id | QUALIFIED), + }), + ..Link::default() + }; + router.connect(2, 2, link).unwrap(); + let asked: Vec = (router.drain().into_iter()) + .filter_map(|out| match out { + path::Out::Ask { edit, .. } => Some(edit), + _ => None, + }) + .collect(); + assert_eq!( + asked, + [path::Edit { + remove: Vec::new(), + add: vec![p("live/a"), p("live/b/x")] + }] + ); +} + + diff --git a/rs/moq-net/src/route/tests/mod.rs b/rs/moq-net/src/route/tests/mod.rs index c666741a0af..095dac875e0 100644 --- a/rs/moq-net/src/route/tests/mod.rs +++ b/rs/moq-net/src/route/tests/mod.rs @@ -12,6 +12,7 @@ mod cache; mod compare; mod handoff; +mod interest; mod paths; mod residual; mod routing; From 5bd80cc7f0a39fb131af29ca3f72b26cd3dd8028 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 05:33:28 -0800 Subject: [PATCH 04/11] wip(route): model interest edits, move the fuzz harness to edits, retire the quest Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-route/SKILL.md | 18 +- quest/m2/cluster-routes/README.md | 4 +- quest/m2/cluster-routes/interest-set.md | 126 ------------ quest/m2/cluster-routes/route-stream.md | 5 +- quest/m2/cluster-routes/switch.md | 14 +- rs/moq-net/src/fuzz.rs | 48 ++++- rs/moq-net/src/route/mod.rs | 8 +- rs/moq-net/src/route/path.rs | 28 ++- rs/moq-net/src/route/tests/handoff.rs | 1 - rs/moq-net/src/route/tests/interest.rs | 16 +- rs/moq-net/src/route/tests/scope.rs | 50 ++++- rs/moq-net/verify/route/Blind.cfg | 2 + rs/moq-net/verify/route/Dead.cfg | 2 + rs/moq-net/verify/route/Drain.cfg | 2 + rs/moq-net/verify/route/DrainRing.cfg | 2 + rs/moq-net/verify/route/Early.cfg | 2 + rs/moq-net/verify/route/Edit.cfg | 38 ++++ rs/moq-net/verify/route/HoldCycle.cfg | 2 + rs/moq-net/verify/route/Lag.cfg | 2 + rs/moq-net/verify/route/LagWide.cfg | 2 + rs/moq-net/verify/route/Loop.cfg | 2 + rs/moq-net/verify/route/Overlap.cfg | 2 + rs/moq-net/verify/route/Prefer.cfg | 2 + rs/moq-net/verify/route/Ring.cfg | 2 + rs/moq-net/verify/route/RouteHandoff.cfg | 2 + rs/moq-net/verify/route/RouteHandoff.tla | 233 ++++++++++++++++++----- rs/moq-net/verify/route/Twice.cfg | 2 + 27 files changed, 391 insertions(+), 226 deletions(-) delete mode 100644 quest/m2/cluster-routes/interest-set.md create mode 100644 rs/moq-net/verify/route/Edit.cfg diff --git a/.claude/skills/verify-route/SKILL.md b/.claude/skills/verify-route/SKILL.md index c26aa3e6551..c3045b08cc2 100644 --- a/.claude/skills/verify-route/SKILL.md +++ b/.claude/skills/verify-route/SKILL.md @@ -5,7 +5,7 @@ description: Model-check the lite-07 route rule (`rs/moq-net/verify/route/RouteH # Verify the route rule -`rs/moq-net/verify/route/RouteHandoff.tla` models one publisher's routes across a mesh, under the rule the routes epic's Plan item 27 picked: a route names every relay it crossed with the session it learned it on, a closed session is named in one DEAD message per neighbour, a relay with no route holds until those that learned it from the relay answer, and only along the path it was built on, so holds never close a cycle, and a closed session casts doubt on its far end until a timer fires. A relay ranks routes by drain, then the lowest link priority on the path, then cost; an offer carries its drain, so no relay ranks a path above what its neighbour did. A relay that hears GOAWAY sends a DRAIN naming the leaving node, the DEAD message's other entry, to each neighbour it sent a route through that node, and each relay doubts that node, so none takes a stale offer through it. A draining path ranks worst but stays usable. It follows `route::path::Router` in `rs/moq-net/src/route/path.rs` with one publisher and a route per prefix (`"root"`, and `"a"` inside it, which relays in `Scoped` ask for alone), and keeps dead hops until their timer fires. Every entry point is one atomic step ending in `Update`; messages share one FIFO per direction. The node timer is `Lift`; each dead hop also expires independently through `Expire`, so repeated failures of one node cannot retain all its old hops. Either may fire at any moment, or with `Patient` only once no relevant DEAD is in flight. Sessions still run Babel's distance in `route::Router`: only `route::tests::Sim` drives the path router, under `Rule::PATH` (timer-only doubt), until [the switch](/quest/m2/cluster-routes/switch.md) puts it on the wire. +`rs/moq-net/verify/route/RouteHandoff.tla` models one publisher's routes across a mesh, under the rule the routes epic's Plan item 27 picked: a route names every relay it crossed with the session it learned it on, a closed session is named in one DEAD message per neighbour, a relay with no route holds until those that learned it from the relay answer, and only along the path it was built on, so holds never close a cycle, and a closed session casts doubt on its far end until a timer fires. A relay ranks routes by drain, then the lowest link priority on the path, then cost; an offer carries its drain, so no relay ranks a path above what its neighbour did. A relay that hears GOAWAY sends a DRAIN naming the leaving node, the DEAD message's other entry, to each neighbour it sent a route through that node, and each relay doubts that node, so none takes a stale offer through it. A draining path ranks worst but stays usable. It follows `route::path::Router` in `rs/moq-net/src/route/path.rs` with one publisher and a route per prefix (`"root"`, and `"a"` inside it, which relays in `Scoped` start out watching alone), and keeps dead hops until their timer fires. A relay asks each neighbour for the set of prefixes it watches, none inside another; a relay in `Editors` may change that set up to `MaxEdits` times. Each change is an edit, sent as an ask on the FIFO and answered by a marker on the neighbour's FIFO, where both sides rekey the routes between them alike (`Rekey`): a key the new set covers stays, the route at a removed `"a"` moves to an added `"root"` without its floor, and the route at a removed `"root"` carries its floor, presented at `"a"`, to an added `"a"`. A heard route is believed for a prefix only while the relay asked for it at the last marker and in every edit still in flight. Every entry point is one atomic step ending in `Update`; messages share one FIFO per direction. The node timer is `Lift`; each dead hop also expires independently through `Expire`, so repeated failures of one node cannot retain all its old hops. Either may fire at any moment, or with `Patient` only once no relevant DEAD is in flight. Sessions still run Babel's distance in `route::Router`: only `route::tests::Sim` drives the path router, under `Rule::PATH` (timer-only doubt), until [the switch](/quest/m2/cluster-routes/switch.md) puts it on the wire. ## Rules @@ -18,12 +18,12 @@ Each rule is an INVARIANT; the harm says why it matters. | `NoRevival` | A reader never sees START for an item it saw END for, unless it forgot it since. | An ended broadcast comes back. | | `NoFalseEnd` | A reader sees END only once the publisher withdrew that item. | A live broadcast is reported as ended rather than offline. | | `NoRegression` | A reader never sees an UPDATE to an older item. | A stale announcement replaces a newer one. | -| `Stable` | A step that only changes a route's path or lifts a doubt changes a shown claim only to a newer floor, or lets go of one it held unavailable. | A detour flaps claims, as a seqno advance did under the distance. | +| `Stable` | A step that only changes a route's path, lifts a doubt, or applies an interest edit (its ask or its marker) changes a shown claim only to a newer floor, or lets go of one it held unavailable. | A detour flaps claims, as a seqno advance did under the distance, or an edit that swaps prefixes ends or re-sends a claim both sets cover. | | `SimplePaths` | Every route held or in flight names a loop-free path from the publisher to its sender. | A relay took back a route that crossed it. | | `LoopsAreStale` | A request that loops crosses a relay whose route names a closed session. | Routing loops with every relay up to date. | | `StaleIsChased` | Every stale route a relay may use has the news that ends it on its way: queued toward the relay, or toward the neighbour it came from, and so on back. | A stale route, and any loop through it, lasts for good. | | `NoStaleReturn` | No relay may use a route through a hop it once learned had died. | A dead path comes back after a relay forgot it. | -| `QuietIsSettled` | Once nothing is in flight and every timer fired: each relay shows exactly the claims a path of open sessions carries to it, available, and a request for each reaches the publisher; no relay holds, keeps a stale route, or remembers a dead hop. | A hold never ends, a claim is shown that no path carries, or a relay behind a scoped neighbour never reaches what a farther one offers. | +| `QuietIsSettled` | Once nothing is in flight and every timer fired: each relay shows exactly the claims a path of open sessions through relays watching them carries to it, available, and a request for each reaches the publisher; no relay holds, keeps a stale route, or remembers a dead hop; and both ends of each session agree on what it asks for. | A hold never ends, a claim is shown that no path carries, a relay behind a scoped neighbour never reaches what a farther one offers, or an edit never takes effect. | | `Budget` | From one quiet point to the next, the messages sent stay within 4 each way over each configured session per prefix and per claim, for each external change. | One change sets off announce spam, or routing never goes quiet. | | `RequestsLoopFree` | A request never loops. | A request cycles through relays instead of reaching the publisher. | @@ -97,18 +97,16 @@ Sync commit: `31b215b112fb649cc3f63d4de3c76b5e87769a06`, the base of the scoped | `rs/moq-net/src/route/path.rs` `Offer::rank`, `Router::goaway`, `Router::drains`, `Router::report_draining`, `Router::dead` (`Dead::Draining`) | `Rank`, `Below`, `PathPrio`, `CostAt`, `GoAway`, `MarkDrain`, `Credited`, `ReportDrain`, `Recv` | | `rs/moq-net/src/route/path.rs` `Router::pending`, `Router::hopes`, `Router::doubts_path` | `Pending`, `Hopes`, `Doubted` | | `rs/moq-net/src/route/path.rs` `Router::src` | `SrcKey` | -| `rs/moq-net/src/route/path.rs` `Router::want`, `Router::advertise`, `Router::report` | `Want`, `Advertise`, `Report` | +| `rs/moq-net/src/route/path.rs` `Router::want`, `Router::stream`, `Router::advertise`, `Router::report` | `Want`, `Want0`, `Streams`, `Advertise`, `Report` | | `rs/moq-net/src/route/path.rs` `Router::route`, `Router::batch`, `Router::dead`, `Router::fail`, `Router::disconnect`, `Router::connect`, `Router::change` | `Recv`, `Fail`, `Disconnect`, `Connect`, `ChangeClaim` | | `rs/moq-net/src/route/path.rs` `Router::collect_doubt`, `Router::timeout` | `Collect`, `Lift`, `Expire`: each dead hop's own timer retires it independently of later failures of the same node; reporting an already retained hop again does not restart its expiry. | -| `rs/moq-net/src/route/path.rs` `Router::know`, `Router::shown_now`, `Router::show` | `Know`, `ShownNow` | +| `rs/moq-net/src/route/path.rs` `Router::know`, `Router::believed`, `Router::watched`, `Router::shown_now`, `Router::show` | `Know`, `NewView`, `Believed`, `Watched`, `ShownNow` | +| `rs/moq-net/src/route/path.rs` `Router::watch`, `Router::ask`, `Router::asks`, `Router::edit`, `Router::edited`, `Session::asking`, `rekey` | `Watch`, `Asks`, `Answer`, `Marker`, `Asking`, `Rekey`, `Moved`: the model's relays watch with no scope regions, so their asks are what they watch. | | `rs/moq-net/src/route/reader.rs` `Reader::apply` | `Events` | | `rs/moq-net/src/route/tests/mod.rs` `Sim::deliver` | `Deliver` | ## Not covered -The router's current watch contract supports one interest prefix per neighbour. -Concurrent disjoint regions of one permit need the -[interest set](/quest/m2/cluster-routes/interest-set.md), including per-prefix -SYNCED; neither this model nor the current scoped sweep covers them. +SYNCED is left out: the router only records it per prefix (`Router::synced`, `Router::is_synced`), and nothing it decides reads it. So are scope regions: a relay asks for what it watches, and `interest.rs`'s `a_scope_with_several_regions_asks_for_each` and the sweeps cover a scope met with the interest. -One publisher, so routes for several publishers are left out, and so are the copies and copy rank of cache-aware routing, which `rs/moq-net/src/route/tests/paths.rs` prototypes; the DEAD message's per-publisher batching reduces to one route per prefix. Two prefixes, one nested in the other, and a scoped relay asks every neighbour for the narrow one: no wildcards, interest edits, stream generations or cancellation, grants, qualified keys, or hidden claims. Leaves out restarts, priorities other than 0 and 1, a priority that differs between a link's two ends, costs other than 1 and 2, RTT, upstream/down-only links, and the hop limit. Messages come only from honest routers: validation of an unrelated DEAD or its far node, and empty or overlong replacement paths, is outside the model's input domain. Rust fixtures cover those guards and exportable covering-source selection, including keeping a covering hold when every live source is unexportable. Interest-edit fixtures also cover a cancelled stream retaining its pending child replies; the model has no stream cancellation. A claims message carries the sender's whole view for its prefix, equal to applying its changes on a FIFO stream. A route is sent on every stream whether or not the publisher has a claim there. The publisher ignores what it hears. `Overlap.cfg` also simulates five relays with every session up. No exhaustive pass has more than four relays or two session changes; `Sim`'s sweeps on the path rule (`sweep_handoff`, `sweep_scoped_handoff`, `sweep_scoped_meshes_show_what_the_rule_says`, and `compare.rs`'s churn) cover larger meshes, grants, qualified keys, interest edits, restarts, and churn. +One publisher, so routes for several publishers are left out, and so are the copies and copy rank of cache-aware routing, which `rs/moq-net/src/route/tests/paths.rs` prototypes; the DEAD message's per-publisher batching reduces to one route per prefix. Two prefixes, one nested in the other, so an edit adds, removes, widens, or narrows one prefix at a time: no wildcards, swaps of several prefixes for one, stream cancellation, grants, qualified keys, or hidden claims. Leaves out restarts, priorities other than 0 and 1, a priority that differs between a link's two ends, costs other than 1 and 2, RTT, upstream/down-only links, and the hop limit. Messages come only from honest routers: validation of an unrelated DEAD or its far node, and empty or overlong replacement paths, is outside the model's input domain. Rust fixtures cover those guards and exportable covering-source selection, including keeping a covering hold when every live source is unexportable. Interest-edit fixtures also cover a cancelled stream retaining its pending child replies; the model has no stream cancellation. A claims message carries the sender's whole view for its prefix, equal to applying its changes on a FIFO stream. A route is sent on every stream whether or not the publisher has a claim there. The publisher ignores what it hears. `Overlap.cfg` also simulates five relays with every session up. No exhaustive pass has more than four relays or two session changes; `Sim`'s sweeps on the path rule (`sweep_handoff`, `sweep_scoped_handoff`, `sweep_scoped_meshes_show_what_the_rule_says`, and `compare.rs`'s churn) cover larger meshes, grants, qualified keys, interest edits, restarts, and churn. diff --git a/quest/m2/cluster-routes/README.md b/quest/m2/cluster-routes/README.md index 2e7e945cab0..2f2e9e31718 100644 --- a/quest/m2/cluster-routes/README.md +++ b/quest/m2/cluster-routes/README.md @@ -37,7 +37,7 @@ The items below were decided 2026-11-10 (maintainer, /quest-plan on lite-07 rout 19. ✅ Asked whether to keep them: **ROUTE_HOLD stays, and the newest floor wins per identity.** Rejected: dropping holds, where each stall takes every claim behind the node offline and re-sends them all; and claims that follow only the selected next hop, which re-pulls a node's whole claim table on each next-hop change and showed revivals and false ENDs in the sweeps below. 20. ✅ Asked when an origin advances its seqno unprompted: **only when a route link changes**, implemented as a bump when a route link closes. Rejected: only to answer a ROUTE_REQUEST; keeping the per-stream bump. 21. ✅ Asked how one side's routing messages travel so none overtakes another: **one unidirectional Announce Stream per side**, carrying its requests, its answers, and its echoes in order ([the route stream](/quest/m2/cluster-routes/route-stream.md)). It replaces "There MAY be multiple Announce Streams". Rejected: one bidirectional Announce Stream per direction with the echo moved to the answering half; one bidirectional stream per session. -22. ✅ **A side's interest is a set of prefixes edited in place.** The live set never nests, and one edit may swap prefixes for a wider or narrower one without re-sending what both cover ([the interest set](/quest/m2/cluster-routes/interest-set.md)). A token-scope change is such an edit, with one prefix per region of the scope. This supersedes items 16 and 17, and the handoff contract's "An interest change opens a new stream" below, once the route stream lands. +22. ✅ **A side's interest is a set of prefixes edited in place.** The live set never nests, and one edit may swap prefixes for a wider or narrower one without re-sending what both cover ([the interest set](https://github.com/moq-dev/moq/pull/8728)). A token-scope change is such an edit, with one prefix per region of the scope. This supersedes items 16 and 17, and the handoff contract's "An interest change opens a new stream" below, once the route stream lands. 23. ✅ Asked what a floor covers with several prefixes on one stream: **one prefix, named by its id.** This amends item 10: the floor is still a marker and never a region list. Rejected: a floor over the whole set. 24. ✅ Asked how a side learns a newly added prefix's initial set is complete: **SYNCED per prefix, naming its id**, plus one marker per edit where it takes effect for both sides. Rejected: one SYNCED per edit, counted in order. 25. ✅ Asked how a relay safely forgets a publisher that is gone, after the model showed a per-neighbour echo is not enough once the origin sits behind a relay: **each route names the nodes it crossed, and a node never uses a route that names itself** ([moq-dev/moq#8487](https://github.com/moq-dev/moq/pull/8487), which holds the counterexamples and the model results). Feasibility still selects routes; the list is one per origin route, not the per-claim Hop ID list item 6 removed, and the echo and the drain rule stay as they are. As built: Babel's feasibility condition is unchanged, and a node never selects or certifies through such a route. Where the condition refuses it, it still counts as infeasible, so a relay whose neighbours all route through it holds its claims while one of them finds another way. Under a qualified key it counts only while its hops extend those of the route the node last sent under that key: a copy that a scoped peer hands back under a key derived from the node's own route never moves, so nothing waits on it. The origin names itself too, so a rotated identity's route is refused where it began, and a node names itself by a Hop ID that a rotation leaves alone. The hop limit is the wire's 32 and no lower. A change of hops or RTT alone lets go of no region, unless it changes whether the node may use the route: by crossing it, or by reaching the hop limit. This supersedes the claim, in the handoff contract and in decision 6 below, that the echo alone makes forgetting safe. First decided as a relay echoing a ROUTE_END only once its own downstream had; the model ran that the same day and it fails when a session closes before answering. Rejected: that deferred echo; never dropping the distance; a timer; relays as the only origin nodes. @@ -57,7 +57,7 @@ The items below were decided 2026-11-10 (maintainer, /quest-plan on lite-07 rout Measured on 24 relays (the mesh sweep's sparse shape, 16 seeds), against today's router, with the doubt lasting 4 hops of delay: a detour settles in 12 hops of delay where the distance takes 15.7, and in 6.2 where it does not run through the far end of the closed link; the publisher's relay dying costs 124 route messages where the distance costs 276, with no loop and 6 claims sent again; and a link closing in a clique costs 44 messages with one publisher behind it and 198 with eight, where the distance costs 621 and 5,024. Accepted: a detour through the far end of a closed link waits the timer out; selected routes can loop for a moment when two unrelated things fail at once (32 of 120,000 churn steps), which the distance never does; and holds and doubt have passed sweeps. The model covers both, but its no-stale-route property holds only for one failure and a patient timer; `StaleIsChased` bounds the rest. The timer's length is the model's and the router's to settle, from the maintainer's 1 s. Rejected: Babel's distance; paths with nothing else, which hunt through stale paths (65,219 messages for one publisher leaving a clique of 12); no doubt, which loops when a relay with several neighbours dies; and a doubt lifted by an epoch the node floods, which resolves in a couple of hops but costs a message per session on each relay-to-relay close ("Epoches are a bit scary in terms of messages"). -28. ✅ Decided 2026-11-12 (maintainer, /quest-plan on the quests still written for Babel), asked how item 27's removal is split and ordered: **the router first with no wire change, then a switch of the sessions, and the switch lands before the route stream.** [The router](https://github.com/moq-dev/moq/pull/8622) runs the path rule with a route per scope, sans-IO, beside today's router. [The switch](/quest/m2/cluster-routes/switch.md) puts it on the lite-07-wip wire, the draft, the Rust sessions, and `js/net`, and deletes Babel's router. [The interest set](/quest/m2/cluster-routes/interest-set.md) then edits the path router only, and [the route stream](/quest/m2/cluster-routes/route-stream.md) follows the switch with no echo or route request to carry. The switch's recommended wire puts the answer that ends a hold on the publisher half, behind the routes its sender sent first, so the path rule needs no order between streams that today's sessions lack for one prefix. [Acceptance](/quest/m2/cluster-routes/acceptance.md) is of the switched sessions, and the inductive invariant stays in m1, rewritten for the path, without `LoopFree` as an invariant. Rejected: the router quest in m1; one quest for router, wire, draft, sessions, and `js/net`; the route stream first, which builds the echo and ROUTE_REQUEST on the new stream and then deletes them; one wire change for the path rule and the one stream together; moving the inductive invariant to m4; deleting it. Reversed 2026-11-13 (maintainer, /quest-complete): ✅ the inductive invariant is deleted, since Apalache cannot step even one transition of the route model ([moq-dev/moq#8564](https://github.com/moq-dev/moq/pull/8564)), no Rust TLA+ checker qualifies ([moq-dev/moq#8600](https://github.com/moq-dev/moq/pull/8600), [moq-dev/moq#8669](https://github.com/moq-dev/moq/pull/8669)), and no more JVM model checking is wanted here (rejected: keeping it paused). +28. ✅ Decided 2026-11-12 (maintainer, /quest-plan on the quests still written for Babel), asked how item 27's removal is split and ordered: **the router first with no wire change, then a switch of the sessions, and the switch lands before the route stream.** [The router](https://github.com/moq-dev/moq/pull/8622) runs the path rule with a route per scope, sans-IO, beside today's router. [The switch](/quest/m2/cluster-routes/switch.md) puts it on the lite-07-wip wire, the draft, the Rust sessions, and `js/net`, and deletes Babel's router. [The interest set](https://github.com/moq-dev/moq/pull/8728) then edits the path router only, and [the route stream](/quest/m2/cluster-routes/route-stream.md) follows the switch with no echo or route request to carry. The switch's recommended wire puts the answer that ends a hold on the publisher half, behind the routes its sender sent first, so the path rule needs no order between streams that today's sessions lack for one prefix. [Acceptance](/quest/m2/cluster-routes/acceptance.md) is of the switched sessions, and the inductive invariant stays in m1, rewritten for the path, without `LoopFree` as an invariant. Rejected: the router quest in m1; one quest for router, wire, draft, sessions, and `js/net`; the route stream first, which builds the echo and ROUTE_REQUEST on the new stream and then deletes them; one wire change for the path rule and the one stream together; moving the inductive invariant to m4; deleting it. Reversed 2026-11-13 (maintainer, /quest-complete): ✅ the inductive invariant is deleted, since Apalache cannot step even one transition of the route model ([moq-dev/moq#8564](https://github.com/moq-dev/moq/pull/8564)), no Rust TLA+ checker qualifies ([moq-dev/moq#8600](https://github.com/moq-dev/moq/pull/8600), [moq-dev/moq#8669](https://github.com/moq-dev/moq/pull/8669)), and no more JVM model checking is wanted here (rejected: keeping it paused). The route layer's core landed in [moq-dev/moq#7676](https://github.com/moq-dev/moq/pull/7676): a diff --git a/quest/m2/cluster-routes/interest-set.md b/quest/m2/cluster-routes/interest-set.md deleted file mode 100644 index 5ecc2b01bdd..00000000000 --- a/quest/m2/cluster-routes/interest-set.md +++ /dev/null @@ -1,126 +0,0 @@ -# [L] The route model and router take an interest as an edited set of prefixes - -## Goal - -The route model (`rs/moq-net/verify/route/RouteHandoff.tla`) and the router -(`rs/moq-net/src/route/`) treat a neighbour as one session, keyed once for -both directions, whose interest is a set of prefixes edited in place. TLC -checks every ordering of an edit against the routes, withdrawals, batches, -floors, and hold answers sharing one FIFO per direction, and the existing -rules hold: an edit -that swaps prefixes for a wider or narrower one keeps every claim both cover, -ends nothing, revives nothing, and sends nothing twice. Nothing on the wire -or in the sessions changes here; [the route stream](/quest/m2/cluster-routes/route-stream.md) -carries it. - -## Plan - -Decided 2026-11-10 (maintainer, /quest-plan on lite-07 routing). The model -goes first because the set's edit rule is the one new rule in this redesign, -and every earlier routing bug was an ordering nobody had enumerated. - -Decided 2026-11-12 (maintainer, /quest-plan on the quests still written for -Babel): ✅ this edits the path router, after -[the route per scope](https://github.com/moq-dev/moq/pull/8622) builds it, -and the model as it has a route per prefix -(`rs/moq-net/verify/route/RouteHandoff.tla`) -(rejected: landing first on Babel's router, which -[the switch](/quest/m2/cluster-routes/switch.md) deletes). The path -rule has no echo and no route request -([epic](/quest/m2/cluster-routes/README.md) Plan item 27), so an edit -is checked against a withdrawal that names a dead hop and against a hold and -its answer instead. - -Settled: - -1. ✅ **A stamp covers one prefix, named by its id** (rejected: a stamp - covering the whole set). A floor is `{route, prefix, revision}`, so each - prefix is certified exactly as one stream per prefix was, and the merge - rule is unchanged. A whole-set stamp needs the receiver to track which - prefixes each stamp covers anyway, since a stamp sent before a prefix was - added says nothing about it, and a sender that knows only part of the set - could certify nothing. This amends Plan item 10 of the - [epic](/quest/m2/cluster-routes/README.md): the floor stays a - marker, never a region list, and gains the prefix it names. -2. ✅ **SYNCED is per prefix, naming its id** (rejected: one SYNCED per edit, - counted in order, which makes a small prefix wait behind a large one). - Each edit also gets one marker in the answering side's stream, the point - where it takes effect for both sides. -3. ✅ **The live set never holds a prefix inside another, and one edit may - swap prefixes without re-sending what both cover.** In the maintainer's - words: "we might have fleet/r7 and fleet/r2, then expand it to fleet ... - somehow we should support fleet absorbing the previous two without - retransmitting everything". Rejected: nested prefixes that each list - independently, where a peer multiplies a relay's work by repeating a - prefix, bounded only by message size; and removing the old prefixes only - after the new one is SYNCED, which holds every claim twice during the - handover and is what Plan items 16 and 17 of the epic did across streams. - -The edit rule those three imply, to prove rather than assume: - -- An edit removes live prefixes by id and adds prefixes, each taking the next - id. An edit that would leave one live prefix inside another is refused. -- At the edit's marker, a claim of a removed prefix stays live, with its id - and attributes, when an added prefix covers it and does not hide it; it is - that prefix's claim from then on. Every other claim of a removed prefix is - dropped. A claim presented at a removed prefix itself may stand for a - broader claim (Plan item 11), which the receiver cannot tell apart, so it - is always dropped and sent again if the new set still presents it. -- The sender then sends what the added prefixes newly cover, one floor per - route for each added prefix, and SYNCED for it. -- Widening: a kept claim stays certified by its old prefix's last floor until - its route's first floor for the new prefix. Narrowing: the wider floor - carries to each added prefix inside it. Decide, and prove, what a floor - that certifies nothing does to the narrower floors it replaces; - recommended: it replaces them, so there is one floor per route per prefix. -- A route stays on the stream while some live prefix presents a claim of it - (Plan item 15, over the set). One whose last claim an edit drops ends with - ROUTE_END after the marker, with no floor first, since nothing was - withdrawn. - -Work: - -- Extend the current single-prefix-per-neighbour interface to all the disjoint - regions of one permit. `peer_interest` currently replaces its one answer and - SYNCED names only the neighbour; several calls under that key cannot serve - several regions. Update `watch`/`synced`'s contract and cover a permit with - `fleet/r2` and `fleet/r7` together, including an edit that absorbs both into - `fleet`. The router and switch preceding this quest claim only one region - per neighbour. -- Start from the model and the router with the path rule. Everything a side - says to a neighbour shares one FIFO per direction, the hold answer - included: with one stream per side nothing overtakes anything. -- The model has two prefixes and a scoped relay since - it has a route per scope. - Give it edits in flight: add, remove, widen, narrow, and an edit that - crosses a claim change, a route change, a ROUTE_END, a withdrawal that - names a dead hop, and a hold with its answer. Keep it - tractable by bounding edits as it bounds flaps. Where Settled item 1 says - the merge rule is unchanged, read it as that quest's rule: a floor names - its route's prefix, and the newest floor wins among one publisher's routes. -- In the router, `Peer::asked` and `Peer::watch` become sets with a floor and - a synced flag per prefix, `batch` takes the prefix its floor names, and a - neighbour is one key for both directions, as `Sim` already has it. The - test-only `watch` and interest generation go: an interest change is an - edit, and a batch in flight before the marker applies to the old set - instead of being discarded. -- Redo the handoff fixtures that an interest change touches - (`widening_interest_needs_floors_scoped_to_what_they_cover`, - `cancelling_interest_retires_its_region`, - `widening_hides_what_was_frozen_until_a_covering_floor_arrives`, - `a_snapshot_queued_before_widening_keeps_the_frozen_rest_hidden`, and the - two scoped-grant widening fixtures) and both sweeps over edited sets. -- Update the `verify-route` skill's coverage table, its "Not covered" list, - and its sync commit. - -The session layer keeps one prefix per neighbour, and `route::Node` keeps -registering each announce stream under its own key, until the route stream -lands. The router here only gains the ability to hold both directions under -one key, which `Sim` and the tests use, so this changes no behaviour on its -own. - -Public API: none (the router is crate-private). Wire: none. - -## Related - -- [lite-07 sessions run the path router](/quest/m2/cluster-routes/switch.md) - deletes Babel's router; whichever lands second adapts diff --git a/quest/m2/cluster-routes/route-stream.md b/quest/m2/cluster-routes/route-stream.md index 36ba89e3311..ee4c6d77e60 100644 --- a/quest/m2/cluster-routes/route-stream.md +++ b/quest/m2/cluster-routes/route-stream.md @@ -46,8 +46,8 @@ Settled: client must open even when it wants nothing). 2. ✅ **Its interest is a set of prefixes edited in place**, with a floor per prefix, SYNCED per prefix, and swaps that keep what both sets cover: - [the interest set](/quest/m2/cluster-routes/interest-set.md) - settles and proves those rules, and this puts them on the wire. + [the interest set](https://github.com/moq-dev/moq/pull/8728) + settled and proved those rules, and this puts them on the wire. 3. ✅ **A token-scope change is an edit.** The subscriber's set is its local interests met with its scope's regions, one prefix per region, so a multi-region scope is certifiable. Plan items 16 and 17 of the @@ -136,7 +136,6 @@ Public API: none expected. Wire: lite-07-wip only. ## Required -- [The route model and router take an interest as an edited set of prefixes](/quest/m2/cluster-routes/interest-set.md) - the edit rule proven in the model and the router that implements it - [lite-07 sessions run the path router](/quest/m2/cluster-routes/switch.md) - the messages this moves onto one stream, with no echo or route request among them ## Related diff --git a/quest/m2/cluster-routes/switch.md b/quest/m2/cluster-routes/switch.md index e71904da705..9ddb6d326c4 100644 --- a/quest/m2/cluster-routes/switch.md +++ b/quest/m2/cluster-routes/switch.md @@ -85,12 +85,14 @@ Work: `carries` the claim. Honor `path::Router::new`'s fresh-hop-per-process contract, including when `Config::hop` is configured: the old seqno cannot recover reused path-hop names. - The driver also implements `watch`/`synced`'s single-prefix scope contract, - completing cancelled streams locally when no replacement can answer. This - stage supports one scope region per neighbour. Multi-region permits need - [the interest set](/quest/m2/cluster-routes/interest-set.md), including its - per-prefix SYNCED; requesting each region under the same neighbour key would - replace the previous answer. Do not claim that coverage before it lands. + The driver also implements the path router's interest set + ([the interest set](https://github.com/moq-dev/moq/pull/8728)): `watch` takes a set of prefixes, the router asks + each neighbour for one prefix per scope region it meets (`Out::Ask`), and the + driver carries each edit, its marker (`Out::Edited`, `edited`), and SYNCED + per prefix (`Out::Synced`, `synced`) under one neighbour key, completing + cancelled streams locally when no replacement can answer. Until the route + stream lands, the session layer may still open one announce stream per + prefix. - Delete today's router with what only it used: the feasibility distance and its drain, seqno requests and their pending state, the echo, and their fixtures. A fixture that states a claim rule moves to the new router if diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index 4b8b66df06e..e0306c431bb 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -879,7 +879,16 @@ impl PathRouteNode { } for (peer, prefix) in interests.iter().enumerate() { node.0.connect(peer, peer as u64 + 2, Default::default()).unwrap(); - node.0.peer_interest(peer, Some(Path::new(prefix).unwrap())).unwrap(); + let add = vec![Path::new(prefix).unwrap().to_owned()]; + node.0 + .edit( + peer, + crate::route::path::Edit { + remove: Vec::new(), + add, + }, + ) + .unwrap(); } node.drain(); node @@ -919,6 +928,7 @@ impl PathRouteRelay { revision: 1, }; relay.router.connect(0, 1, Default::default()).unwrap(); + relay.router.edited(0); relay.router.route( 0, 1, @@ -931,14 +941,23 @@ impl PathRouteRelay { down_only: false, rtt: None, })), - 0, ); let snapshot = (0..claims) .map(|i| (Path::new(&format!("live/{i}")).unwrap(), Some(1))) .collect(); - relay.router.batch(0, 1, Path::empty(), snapshot, Some(1), 0); + relay.router.batch(0, 1, Path::empty(), snapshot, Some(1)); relay.router.connect(1, 3, Default::default()).unwrap(); - relay.router.peer_interest(1, Some(Path::empty())).unwrap(); + let add = vec![Path::empty()]; + relay + .router + .edit( + 1, + crate::route::path::Edit { + remove: Vec::new(), + add, + }, + ) + .unwrap(); relay.drain(); relay } @@ -949,7 +968,7 @@ impl PathRouteRelay { let changes = (0..claims) .map(|i| (Path::new(&format!("live/{i}")).unwrap(), Some(self.revision))) .collect(); - self.router.batch(0, 1, Path::empty(), changes, Some(self.revision), 0); + self.router.batch(0, 1, Path::empty(), changes, Some(self.revision)); self.drain() } @@ -994,6 +1013,7 @@ impl PathRouteFollower { use crate::route::path::{Hop, Offer, Reach}; let prefix = self.prefixes[k].clone(); self.router.connect(k, 1, crate::route::Link::default()).unwrap(); + self.router.edited(k); self.router.route( k, 1, @@ -1006,12 +1026,11 @@ impl PathRouteFollower { down_only: false, rtt: None, })), - 0, ); let changes = (0..self.claims) .map(|j| (Path::new(&format!("live/{k}/{j}")).unwrap(), Some(()))) .collect(); - self.router.batch(k, 1, prefix, changes, Some(1), 0); + self.router.batch(k, 1, prefix, changes, Some(1)); } /// Close every session in one call, including the resulting timer work. @@ -1068,6 +1087,7 @@ impl PathRouteDrain { let prefix = Path::new(&format!("live/{i}")).unwrap(); let via = 1_000 + i as u64; router.connect(i, via, crate::route::Link::default()).unwrap(); + router.edited(i); let path = [ Hop { node: 1, session: 0 }, Hop { @@ -1088,17 +1108,25 @@ impl PathRouteDrain { down_only: false, rtt: None, })), - 0, ); let claim = Path::new(&format!("live/{i}/x")).unwrap(); - router.batch(i, 1, prefix, vec![(claim, Some(()))], Some(1), 0); + router.batch(i, 1, prefix, vec![(claim, Some(()))], Some(1)); } for r in 0..recipients { let peer = sessions + r; router .connect(peer, 5_000 + r as u64, crate::route::Link::default()) .unwrap(); - router.peer_interest(peer, Some(Path::empty())).unwrap(); + let add = vec![Path::empty()]; + router + .edit( + peer, + crate::route::path::Edit { + remove: Vec::new(), + add, + }, + ) + .unwrap(); } let idle = sessions + recipients; router.connect(idle, 9_000, crate::route::Link::default()).unwrap(); diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index 106b414b4cb..dcffd5fd279 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -1947,8 +1947,12 @@ impl Router { regions.as_ref().is_some_and(|regions| !within_any(claim, regions)) || !ctx.own && views.get(&id).is_none_or(|view| view.as_of(claim).is_none()) }; - self.reader - .apply(id, |claim| within(claim, &self.watch), now, |claim| ctx.live && !uncertified(claim)); + self.reader.apply( + id, + |claim| within(claim, &self.watch), + now, + |claim| ctx.live && !uncertified(claim), + ); } /// How the reader now sees each of `claims` of `id`, in `ctx`. diff --git a/rs/moq-net/src/route/path.rs b/rs/moq-net/src/route/path.rs index 2b68a71e4f5..62db9402f1b 100644 --- a/rs/moq-net/src/route/path.rs +++ b/rs/moq-net/src/route/path.rs @@ -654,7 +654,11 @@ struct Rekeyed { /// claim and the floors certified only the narrower prefix. A route at a removed /// prefix is carried, as presented there, to each added prefix inside it that no /// route keys, so its floor carries too. The rest go. -fn rekey>(old: &BTreeSet, new: &BTreeSet, routes: BTreeMap) -> Rekeyed { +fn rekey>( + old: &BTreeSet, + new: &BTreeSet, + routes: BTreeMap, +) -> Rekeyed { let mut kept: BTreeMap = BTreeMap::new(); let mut gone = Vec::new(); let mut moved = Vec::new(); @@ -2015,8 +2019,9 @@ impl Router { let heard = &self.rx[&route][&peer].view; // A route with no floor yet may hold claims an edit carried to it, which the // view never followed. - let follows = delta - .filter(|delta| delta.route == route && delta.from == peer && delta.was.is_some() && delta.was == floor); + let follows = delta.filter(|delta| { + delta.route == route && delta.from == peer && delta.was.is_some() && delta.was == floor + }); let view = match follows { Some(delta) => { let mut view = base; @@ -2717,7 +2722,12 @@ impl Router { for prefix in self.stream(id, peer, prefixes, region) { let key = (id, prefix); if let Some(sent) = answer.tx.get(&key).filter(|sent| sent.offer().is_some()) { - sources.push((peer, key.clone(), sent.src.clone(), self.src(id, &key.1, peer, prefixes))); + sources.push(( + peer, + key.clone(), + sent.src.clone(), + self.src(id, &key.1, peer, prefixes), + )); } } } @@ -2915,9 +2925,15 @@ impl Router { "answering sessions" ); for (peer, answer) in &self.answering { - assert!(!answer.prefixes.is_empty() && !nested(&answer.prefixes), "{peer:?}'s prefixes"); + assert!( + !answer.prefixes.is_empty() && !nested(&answer.prefixes), + "{peer:?}'s prefixes" + ); for (_, prefix) in answer.tx.keys() { - assert!(within_any(prefix, &answer.prefixes), "sent {prefix:?} outside {peer:?}'s prefixes"); + assert!( + within_any(prefix, &answer.prefixes), + "sent {prefix:?} outside {peer:?}'s prefixes" + ); } } for (peer, session) in &self.sessions { diff --git a/rs/moq-net/src/route/tests/handoff.rs b/rs/moq-net/src/route/tests/handoff.rs index 8c2b950e8f0..7f15cac727e 100644 --- a/rs/moq-net/src/route/tests/handoff.rs +++ b/rs/moq-net/src/route/tests/handoff.rs @@ -1825,4 +1825,3 @@ fn a_goaway_two_hops_from_a_ring_never_loops_a_request() { sim.join(leaving, hub, 1); goaway_around_a_ring(&mut sim, hub, leaving); } - diff --git a/rs/moq-net/src/route/tests/interest.rs b/rs/moq-net/src/route/tests/interest.rs index 4476f214893..26324b01061 100644 --- a/rs/moq-net/src/route/tests/interest.rs +++ b/rs/moq-net/src/route/tests/interest.rs @@ -57,7 +57,9 @@ fn widening_swaps_prefixes_without_resending_what_both_cover() { assert_eq!(sim.stats.claim_items - items, PER, "only live/c is sent"); let events = &sim.events[before..]; assert!( - events.iter().all(|event| event.kind == Kind::Start && event.claim.starts_with("live/c/")), + events + .iter() + .all(|event| event.kind == Kind::Start && event.claim.starts_with("live/c/")), "{events:?}" ); assert_eq!(sim.path(client).is_synced(1, &p("live")), Some(true)); @@ -80,7 +82,9 @@ fn narrowing_carries_the_wider_floor() { assert_eq!(sim.stats.claim_items, items, "the floor carries, so nothing is sent"); let events = &sim.events[before..]; assert!( - events.iter().all(|event| event.kind == Kind::Unwatched && event.claim.starts_with("live/c/")), + events + .iter() + .all(|event| event.kind == Kind::Unwatched && event.claim.starts_with("live/c/")), "{events:?}" ); // The carried floor still certifies a withdrawal. @@ -206,7 +210,11 @@ fn each_added_prefix_is_synced_on_its_own() { let router = sim.path(client); assert_eq!(router.is_synced(r1, &p("live/b")), Some(true)); assert_eq!(router.is_synced(r2, &p("live/a")), Some(true)); - assert_eq!(router.is_synced(r2, &p("live/b")), None, "its edit waits behind the marker"); + assert_eq!( + router.is_synced(r2, &p("live/b")), + None, + "its edit waits behind the marker" + ); assert_eq!(router.is_synced(r1, &p("live/c")), None); sim.release(r2, client); assert!(sim.run()); @@ -294,5 +302,3 @@ fn a_scope_with_several_regions_asks_for_each() { }] ); } - - diff --git a/rs/moq-net/src/route/tests/scope.rs b/rs/moq-net/src/route/tests/scope.rs index b3b3f1b60f0..6795c124180 100644 --- a/rs/moq-net/src/route/tests/scope.rs +++ b/rs/moq-net/src/route/tests/scope.rs @@ -164,7 +164,15 @@ fn a_dead_repath_withdraws_its_claims_before_ending_the_sent_route() { router.route(1, 7, p(prefix), Some(path::Reach::Finite(old.clone()))); router.batch(1, 7, p(prefix), vec![(p("a/x"), Some(1))], Some(1)); } - router.edit(3, path::Edit { remove: Vec::new(), add: vec![p("a")] }).unwrap(); + router + .edit( + 3, + path::Edit { + remove: Vec::new(), + add: vec![p("a")], + }, + ) + .unwrap(); // A child still names this relay, so its newer empty view is not believed yet. let mut child = old; child.path = [ @@ -297,7 +305,15 @@ fn drain_news_is_linear_in_neighbours() { router.connect(down, 2000 + i as Id, Link::default()).unwrap(); router.edited(down); let prefix = p(&format!("live/{i}")); - router.edit(down, path::Edit { remove: Vec::new(), add: vec![prefix.clone()] }).unwrap(); + router + .edit( + down, + path::Edit { + remove: Vec::new(), + add: vec![prefix.clone()], + }, + ) + .unwrap(); let path = [ path::Hop { node: 7, session: 0 }, path::Hop { @@ -1045,7 +1061,15 @@ fn an_upstream_stream_uses_a_narrower_exportable_source() { }; narrow.metric = 2; router.route(2, 7, p("a"), Some(path::Reach::Finite(narrow))); - router.edit(3, path::Edit { remove: Vec::new(), add: vec![p("a")] }).unwrap(); + router + .edit( + 3, + path::Edit { + remove: Vec::new(), + add: vec![p("a")], + }, + ) + .unwrap(); assert!(router.drain().iter().any(|msg| matches!(msg, path::Out::Route { to: 3, prefix, reach: Some(path::Reach::Finite(offer)), .. } if *prefix == p("a") && !offer.down_only && offer.path[1].node == 2))); } @@ -1059,7 +1083,15 @@ fn replacing_a_childs_interest_keeps_the_hold_until_its_answer() { } router.route(1, 7, p(""), Some(offered(1, 0))); router.batch(1, 7, p(""), vec![(p("a/x"), Some(1))], Some(1)); - router.edit(2, path::Edit { remove: Vec::new(), add: vec![p("")] }).unwrap(); + router + .edit( + 2, + path::Edit { + remove: Vec::new(), + add: vec![p("")], + }, + ) + .unwrap(); let path::Reach::Finite(mut child) = offered(2, 0) else { unreachable!() }; @@ -1114,7 +1146,15 @@ fn an_upstream_stream_keeps_a_hold_over_an_unexportable_live_source() { offer.metric = 2; router.route(2, 7, p(held), Some(path::Reach::Finite(offer))); router.batch(2, 7, p(held), vec![(p("a/x"), Some(1))], Some(1)); - router.edit(3, path::Edit { remove: Vec::new(), add: vec![p("a")] }).unwrap(); + router + .edit( + 3, + path::Edit { + remove: Vec::new(), + add: vec![p("a")], + }, + ) + .unwrap(); let path::Reach::Finite(mut child) = offered(3, 0) else { unreachable!() }; diff --git a/rs/moq-net/verify/route/Blind.cfg b/rs/moq-net/verify/route/Blind.cfg index aa709100bdb..ef0555fa007 100644 --- a/rs/moq-net/verify/route/Blind.cfg +++ b/rs/moq-net/verify/route/Blind.cfg @@ -13,10 +13,12 @@ CONSTANTS Preferred = {} Leaving = {} Scoped = {1} + Editors = {} Claims = {"ax", "bx"} Narrow = {"ax"} MaxRev = 2 MaxFlaps = 1 + MaxEdits = 0 Patient = TRUE SPECIFICATION Spec diff --git a/rs/moq-net/verify/route/Dead.cfg b/rs/moq-net/verify/route/Dead.cfg index 1e1fc4126c8..3a4c0e1994e 100644 --- a/rs/moq-net/verify/route/Dead.cfg +++ b/rs/moq-net/verify/route/Dead.cfg @@ -14,10 +14,12 @@ CONSTANTS Preferred = {} Leaving = {} Scoped = {1} + Editors = {} Claims = {"ax", "bx"} Narrow = {"ax"} MaxRev = 3 MaxFlaps = 1 + MaxEdits = 0 Patient = TRUE SPECIFICATION Spec diff --git a/rs/moq-net/verify/route/Drain.cfg b/rs/moq-net/verify/route/Drain.cfg index f3da411c468..00a20754e60 100644 --- a/rs/moq-net/verify/route/Drain.cfg +++ b/rs/moq-net/verify/route/Drain.cfg @@ -13,10 +13,12 @@ CONSTANTS Preferred = {} Leaving = {0} Scoped = {} + Editors = {} Claims = {"x"} Narrow = {} MaxRev = 1 MaxFlaps = 1 + MaxEdits = 0 Patient = TRUE SPECIFICATION Spec diff --git a/rs/moq-net/verify/route/DrainRing.cfg b/rs/moq-net/verify/route/DrainRing.cfg index 2698259d236..3c1269917b4 100644 --- a/rs/moq-net/verify/route/DrainRing.cfg +++ b/rs/moq-net/verify/route/DrainRing.cfg @@ -16,10 +16,12 @@ CONSTANTS Preferred = {} Leaving = {0} Scoped = {} + Editors = {} Claims = {"x"} Narrow = {} MaxRev = 1 MaxFlaps = 0 + MaxEdits = 0 Patient = TRUE SPECIFICATION Spec diff --git a/rs/moq-net/verify/route/Early.cfg b/rs/moq-net/verify/route/Early.cfg index 96b9e2e89f4..2d6b4fac416 100644 --- a/rs/moq-net/verify/route/Early.cfg +++ b/rs/moq-net/verify/route/Early.cfg @@ -10,10 +10,12 @@ CONSTANTS Preferred = {} Leaving = {} Scoped = {} + Editors = {} Claims = {"x"} Narrow = {} MaxRev = 1 MaxFlaps = 1 + MaxEdits = 0 Patient = FALSE SPECIFICATION Spec diff --git a/rs/moq-net/verify/route/Edit.cfg b/rs/moq-net/verify/route/Edit.cfg new file mode 100644 index 00000000000..eb8fd9b5584 --- /dev/null +++ b/rs/moq-net/verify/route/Edit.cfg @@ -0,0 +1,38 @@ +\* Interest edits in flight: relay 3, watching "a", hears the publisher (0) +\* through relays 1 and 2, and changes what it watches twice: widening, +\* narrowing, dropping everything, or asking again. Each edit crosses a claim +\* change, a session closing and its DEAD, the holds and their answers, and +\* whatever else the FIFOs carry. +CONSTANTS + Nodes = {0, 1, 2, 3} + Origin = 0 + Links = {{0, 1}, {0, 2}, {1, 3}, {2, 3}} + Dear = {} + Ring = {} + Preferred = {} + Leaving = {} + Scoped = {3} + Editors = {3} + Claims = {"ax", "bx"} + Narrow = {"ax"} + MaxRev = 1 + MaxFlaps = 1 + MaxEdits = 2 + Patient = TRUE + +SPECIFICATION Spec + +INVARIANTS + Truthful + NoRevival + NoFalseEnd + NoRegression + Stable + Seen + SimplePaths + LoopsAreStale + StaleIsChased + RequestsLoopFree + NoStaleReturn + QuietIsSettled + Budget diff --git a/rs/moq-net/verify/route/HoldCycle.cfg b/rs/moq-net/verify/route/HoldCycle.cfg index 6e26a5171ae..1dc908067b1 100644 --- a/rs/moq-net/verify/route/HoldCycle.cfg +++ b/rs/moq-net/verify/route/HoldCycle.cfg @@ -15,10 +15,12 @@ CONSTANTS Preferred = {} Leaving = {} Scoped = {4, 5} + Editors = {} Claims = {"ax"} Narrow = {"ax"} MaxRev = 1 MaxFlaps = 1 + MaxEdits = 0 Patient = TRUE SPECIFICATION Spec diff --git a/rs/moq-net/verify/route/Lag.cfg b/rs/moq-net/verify/route/Lag.cfg index 1b13967a558..0537a6f0bc0 100644 --- a/rs/moq-net/verify/route/Lag.cfg +++ b/rs/moq-net/verify/route/Lag.cfg @@ -11,10 +11,12 @@ CONSTANTS Preferred = {} Leaving = {} Scoped = {} + Editors = {} Claims = {"ax"} Narrow = {"ax"} MaxRev = 2 MaxFlaps = 1 + MaxEdits = 0 Patient = TRUE SPECIFICATION Spec diff --git a/rs/moq-net/verify/route/LagWide.cfg b/rs/moq-net/verify/route/LagWide.cfg index 4ebc496667d..a14ae7cd4b2 100644 --- a/rs/moq-net/verify/route/LagWide.cfg +++ b/rs/moq-net/verify/route/LagWide.cfg @@ -9,10 +9,12 @@ CONSTANTS Preferred = {} Leaving = {} Scoped = {1} + Editors = {} Claims = {"ax"} Narrow = {"ax"} MaxRev = 2 MaxFlaps = 1 + MaxEdits = 0 Patient = TRUE SPECIFICATION Spec diff --git a/rs/moq-net/verify/route/Loop.cfg b/rs/moq-net/verify/route/Loop.cfg index 5dc279c44d1..c8977766626 100644 --- a/rs/moq-net/verify/route/Loop.cfg +++ b/rs/moq-net/verify/route/Loop.cfg @@ -11,10 +11,12 @@ CONSTANTS Preferred = {} Leaving = {} Scoped = {} + Editors = {} Claims = {"x"} Narrow = {} MaxRev = 0 MaxFlaps = 3 + MaxEdits = 0 Patient = TRUE SPECIFICATION Spec diff --git a/rs/moq-net/verify/route/Overlap.cfg b/rs/moq-net/verify/route/Overlap.cfg index aeb6101abc3..40c11a87b67 100644 --- a/rs/moq-net/verify/route/Overlap.cfg +++ b/rs/moq-net/verify/route/Overlap.cfg @@ -12,10 +12,12 @@ CONSTANTS Preferred = {} Leaving = {} Scoped = {2, 3, 4} + Editors = {} Claims = {"x"} Narrow = {"x"} MaxRev = 1 MaxFlaps = 0 + MaxEdits = 0 Patient = TRUE SPECIFICATION Spec diff --git a/rs/moq-net/verify/route/Prefer.cfg b/rs/moq-net/verify/route/Prefer.cfg index d2047f90906..f1de6970274 100644 --- a/rs/moq-net/verify/route/Prefer.cfg +++ b/rs/moq-net/verify/route/Prefer.cfg @@ -12,10 +12,12 @@ CONSTANTS Preferred = {{1, 2}} Leaving = {} Scoped = {} + Editors = {} Claims = {"x"} Narrow = {} MaxRev = 1 MaxFlaps = 1 + MaxEdits = 0 Patient = TRUE SPECIFICATION Spec diff --git a/rs/moq-net/verify/route/Ring.cfg b/rs/moq-net/verify/route/Ring.cfg index c8234dd7ae1..da2c86c5ec4 100644 --- a/rs/moq-net/verify/route/Ring.cfg +++ b/rs/moq-net/verify/route/Ring.cfg @@ -11,10 +11,12 @@ CONSTANTS Preferred = {} Leaving = {} Scoped = {} + Editors = {} Claims = {"x"} Narrow = {} MaxRev = 1 MaxFlaps = 1 + MaxEdits = 0 Patient = FALSE SPECIFICATION Spec diff --git a/rs/moq-net/verify/route/RouteHandoff.cfg b/rs/moq-net/verify/route/RouteHandoff.cfg index 0c5dcdd120d..562851e0ed9 100644 --- a/rs/moq-net/verify/route/RouteHandoff.cfg +++ b/rs/moq-net/verify/route/RouteHandoff.cfg @@ -11,10 +11,12 @@ CONSTANTS Preferred = {} Leaving = {} Scoped = {} + Editors = {} Claims = {"x"} Narrow = {} MaxRev = 1 MaxFlaps = 1 + MaxEdits = 0 Patient = TRUE SPECIFICATION Spec diff --git a/rs/moq-net/verify/route/RouteHandoff.tla b/rs/moq-net/verify/route/RouteHandoff.tla index e5f2ec9ae35..a5cb9c8c131 100644 --- a/rs/moq-net/verify/route/RouteHandoff.tla +++ b/rs/moq-net/verify/route/RouteHandoff.tla @@ -5,8 +5,14 @@ (* rs/moq-net/src/route/path.rs runs it, with a route per scope. *) (* *) (* A route is keyed by the prefix it covers: "root", or "a" inside it. A *) -(* relay in `Scoped` asks every neighbour for "a" alone, so a stream to it *) -(* carries one route, keyed "a". A route names every relay it crossed, *) +(* relay watches a set of prefixes, none inside another, and asks every *) +(* neighbour for that set: a stream answering {"a"} carries one route, *) +(* keyed "a". A relay in `Editors` may change what it watches. Each change *) +(* is an edit, sent on the FIFO as an ask; the neighbour answers it with a *) +(* marker on its own FIFO, and both sides rekey the routes between them at *) +(* that point alike (path.rs: `rekey`). A route heard is believed for a *) +(* prefix only while the node asked for it at the last marker and in every *) +(* edit still in flight. A route names every relay it crossed, *) (* each with the session that relay learned it on, and a relay uses the *) (* best live route whose path does not name it: one crossing no draining *) (* session, then the highest lowest link priority, then the cheapest. An *) @@ -39,11 +45,13 @@ CONSTANTS \* next costs 1 there and 2 at the next, overriding `Dear` Preferred, \* the links both ends give priority 1; the rest have 0 Leaving, \* nodes that may send GOAWAY, once on each session - Scoped, \* relays that ask every neighbour for prefix "a" alone + Scoped, \* relays that start out watching prefix "a" alone + Editors, \* relays that may change what they watch Claims, \* claim paths Narrow, \* the claims inside prefix "a" MaxRev, \* the publisher's claim changes MaxFlaps, \* session opens and closes + MaxEdits, \* changes of what a relay watches Patient \* TRUE: a doubt lifts only once no news about its node is in flight Relays == Nodes \ {Origin} @@ -71,10 +79,15 @@ KeySeq == <<"root", "a">> Covers(k, c) == k = "root" \/ c \in Narrow \* The key whose view a node shows `c` from: the narrowest that covers it. KeyOf(c) == IF c \in Narrow THEN "a" ELSE "root" -\* What `n` asks its neighbours for. -Ask(n) == IF n \in Scoped THEN "a" ELSE "root" -\* The keys a stream to `p` carries. -Streams(p) == IF p \in Scoped THEN {"a"} ELSE Keys + +\* The sets of prefixes a node may watch or ask for: none inside another. +Sets == {{}, {"a"}, {"root"}} +\* Whether key `k` lies within a prefix of the set `S`. +Within(k, S) == "root" \in S \/ k \in S +\* Whether the set `S` covers claim `c`. +CoversSet(S, c) == \E k \in S : Covers(k, c) +\* What node n watches at the start: the publisher watches nothing. +Watch0(n) == IF n = Origin THEN {} ELSE IF n \in Scoped THEN {"a"} ELSE {"root"} NoItems == [c \in Claims |-> None] Restrict(items, k) == [c \in Claims |-> IF Covers(k, c) THEN items[c] ELSE None] @@ -99,7 +112,7 @@ NotShown == [item |-> None, avail |-> FALSE] \* Messages: one record shape so TLC can compare them. Blank == [t |-> "", k |-> "root", path |-> <<>>, drain |-> FALSE, rev |-> None, items |-> NoItems, - hop |-> OHop, moved |-> {}, gone |-> FALSE, node |-> None] + hop |-> OHop, moved |-> {}, gone |-> FALSE, node |-> None, set |-> {}] OfferMsg(k, path, drain) == [Blank EXCEPT !.t = "offer", !.k = k, !.path = path, !.drain = drain] HoldMsg(k) == [Blank EXCEPT !.t = "hold", !.k = k] EndMsg(k) == [Blank EXCEPT !.t = "end", !.k = k] @@ -108,6 +121,10 @@ DeadMsg(hop, moved, gone) == [Blank EXCEPT !.t = "dead", !.hop = hop, !.moved = moved, !.gone = gone] \* The DEAD message's DRAIN entry: `node` sent GOAWAY. DrainMsg(node) == [Blank EXCEPT !.t = "drain", !.node = node] +\* An edit of what the sender asks for, naming the set it asks for once applied. +AskMsg(set) == [Blank EXCEPT !.t = "ask", !.set = set] +\* The marker answering the oldest edit the receiver asked for. +EditedMsg == [Blank EXCEPT !.t = "edited"] VARIABLES up, \* open sessions @@ -124,11 +141,12 @@ VARIABLES unstable, \* claim changes a path change or a lifted doubt alone made learnt, \* learnt[n]: every hop n was ever told is dead flaps, + edits, \* changes of what a relay watches burst, \* messages sent since the last quiet point churn \* external changes since the last quiet point vars == <> + unstable, learnt, flaps, edits, burst, churn>> Dirs == {<> \in Nodes \X Nodes : {m, n} \in Links} Peers(n, ups) == {m \in Nodes : {n, m} \in ups} @@ -171,8 +189,12 @@ Drains(s, path) == \E i \in DOMAIN path : path[i][1] \in s.draining Usable(n, s, m, k) == s.rx[m][k].on /\ s.rx[m][k].state = "live" /\ ~Names(s.rx[m][k].path, n) +\* The keys a stream answering p's asks carries (path.rs: `stream`): every key +\* within them. +Streams(s, p) == {k \in Keys : Within(k, s.ans[p])} + \* The key of this node's own route an offer of key k to p reflects. A stream -\* to a scoped relay carries one route, "a": the node's route for "a" where it +\* answering {"a"} carries one route, "a": the node's route for "a" where it \* has a live one not naming the receiver, else its root route, which covers "a". \* A narrower path cannot outrank a better covering path: that preference can \* make neighbours oscillate between scopes. Equal ranks prefer the narrow key. @@ -183,7 +205,7 @@ Cheapest(s, ks) == \/ R(k) = R(j) /\ (k = "a" \/ j = "root") SrcKey(s, p, k) == - IF p \notin Scoped THEN k + IF k # "a" \/ "a" \notin s.ans[p] THEN k ELSE LET live == {j \in Keys : Live(s.rt[j])} usable == {j \in live : ~Names(s.rt[j].path, p)} IN IF usable # {} THEN Cheapest(s, usable) @@ -256,17 +278,26 @@ Select(n, s, gens) == (* route covering that key gave it, kept while it holds a covering route. *) Presence(s, k) == s.rt[k].on \/ (k = "a" /\ s.rt["root"].on) +\* The node keeps a view for key k (path.rs: `watched`): one within what it watches. +Watched(s, k) == Within(k, s.watch) -\* A route this node believes: live, and it did not cross this node. -Believed(n, s, m, j) == - s.rx[m][j].on /\ s.rx[m][j].state = "live" /\ ~Names(s.rx[m][j].path, n) +\* This node asked m for key k at the last marker and in every edit still in +\* flight (path.rs: `Session::asking`). +Asking(s, m, k) == + /\ Within(k, s.asked[m]) + /\ \A i \in DOMAIN s.edits[m] : Within(k, s.edits[m][i]) + +\* A route this node believes for its claims within key k: live, it did not +\* cross this node, and no edit could drop k from what m answers. +Believed(n, s, m, j, k) == + s.rx[m][j].on /\ s.rx[m][j].state = "live" /\ ~Names(s.rx[m][j].path, n) /\ Asking(s, m, k) NewView(n, s, k) == - IF ~Presence(s, k) THEN NoView + IF ~Presence(s, k) \/ ~Watched(s, k) THEN NoView ELSE LET base == IF s.view[k].floor = Absent THEN EmptyView ELSE s.view[k] src == {x \in Nodes \X Keys : /\ x[2] = k \/ x[2] = "root" - /\ Believed(n, s, x[1], x[2]) + /\ Believed(n, s, x[1], x[2], k) /\ s.rx[x[1]][x[2]].rev > base.floor} best == CHOOSE x \in src : \A y \in src : s.rx[x[1]][x[2]].rev >= s.rx[y[1]][y[2]].rev @@ -292,8 +323,9 @@ ShownNow(s) == \* What this node wants a neighbour p to hold for key k: a path, a hold, an \* end, or to keep what it was sent while the next hop this node holds under -\* waits for those behind it. -Want(n, s, k, p) == +\* waits for those behind it. A path or a hold goes only where this node keeps +\* a view: one it does not watch ends (path.rs: `Want::Empty` with no covering view). +Want0(n, s, k, p) == LET src == SrcKey(s, p, k) r == s.rt[src] IN IF ~r.on THEN [kind |-> "end", path |-> <<>>, drain |-> FALSE, src |-> src] @@ -306,12 +338,18 @@ Want(n, s, k, p) == THEN [kind |-> "end", path |-> <<>>, drain |-> FALSE, src |-> src] ELSE [kind |-> "hold", path |-> <<>>, drain |-> FALSE, src |-> src] +Want(n, s, k, p) == + LET w == Want0(n, s, k, p) + IN IF w.kind \in {"path", "hold"} /\ s.view[k].floor = Absent + THEN [w EXCEPT !.kind = "end", !.path = <<>>, !.drain = FALSE] + ELSE w + \* `advertise` for one key on the stream to p. Advertise1(n, s, p, k) == LET t == s.tx[p][k] w == Want(n, s, k, p) v == IF s.view[k].floor = Absent THEN EmptyView ELSE s.view[k] - IN CASE k \notin Streams(p) \/ w.kind = "keep" -> [tx |-> t, out |-> <<>>] + IN CASE k \notin Streams(s, p) \/ w.kind = "keep" -> [tx |-> t, out |-> <<>>] [] w.kind = "hold" -> IF t.on /\ ~t.held THEN [tx |-> [t EXCEPT !.held = TRUE], out |-> <>] ELSE [tx |-> t, out |-> <<>>] @@ -409,19 +447,20 @@ DeadReport(hop) == [NoReport EXCEPT !.on = TRUE, !.hop = hop] DrainReport(x) == [NoReport EXCEPT !.node = x] \* The reader's events: changes to `ended` and the counters. A claim leaving -\* while the node still holds a covering route is an END, else it went offline. +\* while the node still holds a covering route and watches it is an END, else +\* it went offline or is no longer watched, and the reader forgets it. \* `quiet` marks a step that only changed a path or lifted a doubt: in one, a \* claim the reader sees changes only to what a newer floor says, or goes \* offline after it was held unavailable. Events(s0, s, old, now, ov, eb, quiet) == - LET present(c) == Presence(s, KeyOf(c)) + LET present(c) == Presence(s, KeyOf(c)) /\ Watched(s, KeyOf(c)) isEnd(c) == old[c].item # None /\ now[c].item = None /\ present(c) isStart(c) == old[c].item = None /\ now[c].item # None isUpdate(c) == old[c].item # None /\ now[c].item # None /\ old[c].item # now[c].item \* Forgetting a claim clears what the reader saw end. eb1 == [c \in Claims |-> IF present(c) THEN eb[c] ELSE None] newer(c) == s.view[KeyOf(c)].floor > s0.view[KeyOf(c)].floor - before(c) == Presence(s0, KeyOf(c)) + before(c) == Presence(s0, KeyOf(c)) /\ Watched(s0, KeyOf(c)) \* Letting go of a claim it held unavailable ends a hold, not a path. release(c) == ~old[c].avail /\ ~present(c) IN [ended |-> [c \in Claims |-> IF isEnd(c) THEN old[c].item ELSE eb1[c]], @@ -447,8 +486,9 @@ Update(n, s0, ov, ups, gens, eb, rep, quiet) == dead |-> a.s.dead] \* The publisher ignores every message it hears for its own route, so none is -\* queued toward it. Dropping them changes no behaviour; `Budget` still counts them. -Heeded(d, m) == d[2] # Origin +\* queued toward it, but answers what it is asked for. Dropping them changes no +\* behaviour; `Budget` still counts them. It watches nothing, so asks nothing. +Heeded(d, m) == d[2] # Origin \/ m.t = "ask" \* The sum of f(x) for each x in the finite set S. \* FoldSet runs natively through the Community Modules' Java override; an @@ -489,10 +529,15 @@ Internal(sent) == /\ UNCHANGED churn ----------------------------------------------------------------------------- -(* Initial state: every session open, the publisher's route sent and no *) -(* claims yet. *) - -InitNode == [ +(* Initial state: every session open with its asks answered, the *) +(* publisher's route sent and no claims yet. *) + +\* Every session starts with its asks answered. +InitNode(n) == [ + watch |-> Watch0(n), + asked |-> [m \in Nodes |-> IF {n, m} \in Links THEN Watch0(n) ELSE {}], + edits |-> [m \in Nodes |-> <<>>], + ans |-> [m \in Nodes |-> IF {n, m} \in Links THEN Watch0(m) ELSE {}], rx |-> [m \in Nodes |-> [k \in Keys |-> NoHeard]], rt |-> [k \in Keys |-> NoRoute], view |-> [k \in Keys |-> NoView], @@ -510,7 +555,7 @@ InitNode == [ OV0 == [rev |-> 0, items |-> NoItems] Gen0 == [l \in Links |-> 0] -Start == Update(Origin, [InitNode EXCEPT !.rt = OwnRoutes], OV0, Links, Gen0, +Start == Update(Origin, [InitNode(Origin) EXCEPT !.rt = OwnRoutes], OV0, Links, Gen0, [c \in Claims |-> None], NoReport, FALSE) Init == @@ -520,7 +565,7 @@ Init == /\ rev = 0 /\ own = NoItems /\ hist = <> - /\ st = [n \in Nodes |-> IF n = Origin THEN Start.s ELSE InitNode] + /\ st = [n \in Nodes |-> IF n = Origin THEN Start.s ELSE InitNode(n)] /\ ended = [n \in Nodes |-> [c \in Claims |-> None]] /\ revivals = 0 /\ falseEnds = 0 @@ -528,6 +573,7 @@ Init == /\ unstable = 0 /\ learnt = [n \in Nodes |-> {}] /\ flaps = 0 + /\ edits = 0 /\ burst = 0 /\ churn = 1 @@ -545,7 +591,7 @@ ChangeClaim(c, item) == /\ hist' = Append(hist, own1) /\ Commit(rs, q) /\ External(Sent(rs)) - /\ UNCHANGED <> + /\ UNCHANGED <> Announce(c) == rev < MaxRev /\ ChangeClaim(c, rev + 1) Unannounce(c) == rev < MaxRev /\ own[c] # None /\ ChangeClaim(c, None) @@ -587,7 +633,8 @@ Credited(s, from, x) == \E k \in Keys : s.rx[from][k].on /\ Names(s.rx[from][k]. Recv(n, s, from, msg) == LET k == msg.k h == s.rx[from][k] - IN CASE msg.t = "offer" -> + IN CASE msg.t = "offer" /\ ~Within(k, s.asked[from]) -> s + [] msg.t = "offer" -> [s EXCEPT !.rx[from][k] = [on |-> TRUE, path |-> msg.path, drain |-> msg.drain, state |-> IF DeadIn(msg.path, s.dead) THEN "dead" ELSE "live", @@ -600,6 +647,50 @@ Recv(n, s, from, msg) == [] msg.t = "drain" -> IF Credited(s, from, msg.node) THEN MarkDrain(n, s, msg.node) ELSE s [] OTHER -> Fail(n, s, msg.hop, from, msg.moved, msg.gone) +\* The routes of one stream once an edit of its prefixes from `O` to `N` takes +\* effect at its marker (path.rs: `rekey`), with `Off(r)` what a gone entry +\* becomes. Both sides apply it alike, to what was sent and what was heard. A +\* key within N stays, unless it sits at a removed prefix. The route at a +\* removed "a" moves to an added "root" with its offer and state, and without +\* its floor, which certified only "a". The route at a removed "root" is +\* carried, presented at "a", to an added "a" no route keys, so its floor +\* carries too. The rest go, with no ROUTE_END. +Rekey(O, N, R, Off(_)) == + [k \in Keys |-> + IF k = "root" + THEN IF "root" \in O /\ "root" \in N THEN R["root"] + ELSE IF "a" \in O /\ "root" \in N /\ R["a"].on THEN [R["a"] EXCEPT !.rev = None] + ELSE Off(R["root"]) + ELSE IF R["a"].on /\ (("root" \in O /\ N # {}) \/ ("a" \in O /\ "a" \in N)) THEN R["a"] + ELSE IF "root" \in O /\ "a" \in N /\ R["root"].on /\ ~R["a"].on + THEN [R["root"] EXCEPT !.items = Restrict(@, "a")] + ELSE Off(R["a"])] + +\* The keys `Rekey` moved a route between, as <>. +Moved(O, N, R) == + (IF "a" \in O /\ "root" \in N /\ R["a"].on THEN {<<"a", "root">>} ELSE {}) + \cup (IF "root" \in O /\ "a" \in N /\ R["root"].on /\ ~R["a"].on THEN {<<"root", "a">>} ELSE {}) + +\* `edit`: m now asks node n for `set`. It takes effect at once, behind the +\* marker n sends first. +Answer(s, m, set) == + [s EXCEPT !.ans[m] = set, + !.tx[m] = Rekey(s.ans[m], set, s.tx[m], LAMBDA r : NoSent(r.src))] + +\* `edited`: m's marker for node n's oldest edit in flight. The routes m sent +\* are rekeyed as m rekeyed them, and a route n used or held through m moves +\* with the route it heard, so a hold carries over rather than letting go. +Marker(s, m) == + LET old == s.asked[m] + new == Head(s.edits[m]) + mv == Moved(old, new, s.rx[m]) + carry(k) == {x \in mv : x[2] = k /\ s.rt[x[1]].on /\ s.rt[x[1]].next = m /\ ~s.rt[k].on} + IN [s EXCEPT !.asked[m] = new, + !.edits[m] = Tail(@), + !.rx[m] = Rekey(old, new, s.rx[m], LAMBDA r : NoHeard), + !.rt = [k \in Keys |-> IF carry(k) # {} THEN s.rt[(CHOOSE x \in carry(k) : TRUE)[1]] + ELSE s.rt[k]]] + Deliver(d) == LET m == d[1] n == d[2] @@ -608,14 +699,22 @@ Deliver(d) == rep == CASE msg.t = "dead" -> DeadReport(msg.hop) [] msg.t = "drain" /\ msg.node # n /\ Credited(st[n], m, msg.node) -> DrainReport(msg.node) [] OTHER -> NoReport - \* Only the route's path, or how it ranks, changed. + \* Only the route's path, or how it ranks, changed; or an edit took effect, + \* which changes no claim both sets cover. quiet == \/ msg.t = "offer" /\ st[n].rx[m][msg.k].on - \/ msg.t = "drain" - rs == [x \in {n} |-> Update(n, Recv(n, st[n], m, msg), OV, up, gen, ended[n], rep, quiet)] + \/ msg.t \in {"drain", "ask", "edited"} + s1 == CASE msg.t = "ask" -> Answer(st[n], m, msg.set) + [] msg.t = "edited" -> Marker(st[n], m) + [] OTHER -> Recv(n, st[n], m, msg) + u == Update(n, s1, OV, up, gen, ended[n], rep, quiet) + \* The marker goes before anything the edit sends. + rs == [x \in {n} |-> IF msg.t = "ask" + THEN [u EXCEPT !.out[m] = <> \o @] + ELSE u] IN /\ q[d] # <<>> /\ Commit(rs, q0) /\ Internal(Sent(rs)) - /\ UNCHANGED <> + /\ UNCHANGED <> \* The timer node n started on x fires: the doubt on x lifts, and the dead hops \* whose session ended at x may be forgotten. With `Patient`, only once no DEAD @@ -630,7 +729,7 @@ Lift(n, x) == ~(q[d][i].t = "dead" /\ q[d][i].hop[2] = x) /\ ~(q[d][i].t = "drain" /\ q[d][i].node = x) /\ Commit(rs, q) /\ Internal(Sent(rs)) - /\ UNCHANGED <> + /\ UNCHANGED <> \* A hop's own timer can fire before a later failure's timer on the same far \* node. A new DEAD cannot extend every old hop's retention indefinitely. @@ -641,7 +740,7 @@ Expire(n, hop) == /\ Patient => \A d \in Dirs : \A i \in DOMAIN q[d] : ~(q[d][i].t = "dead" /\ q[d][i].hop = hop) /\ Commit(rs, q) /\ Internal(Sent(rs)) - /\ UNCHANGED <> + /\ UNCHANGED <> ----------------------------------------------------------------------------- (* Sessions open, drain, and close. *) @@ -657,22 +756,33 @@ GoAway(n, m) == /\ m \notin st[n].drain /\ Commit(rs, q) /\ External(Sent(rs)) - /\ UNCHANGED <> + /\ UNCHANGED <> \* A closed session: everything learned or sent on it goes. Closed(s, y) == - [s EXCEPT !.rx[y] = [k \in Keys |-> NoHeard], + [s EXCEPT !.asked[y] = {}, + !.edits[y] = <<>>, + !.ans[y] = {}, + !.rx[y] = [k \in Keys |-> NoHeard], !.tx[y] = [k \in Keys |-> NoSent(k)], !.told[y] = {}, !.toldDrain[y] = {}, !.drain = @ \ {y}] +\* `ask`: the edit bringing what node n asks m for to what it watches, if any. +Asks(s, m) == + LET requested == IF s.edits[m] = <<>> THEN s.asked[m] ELSE s.edits[m][Len(s.edits[m])] + IN IF requested = s.watch THEN [s |-> s, out |-> <<>>] + ELSE [s |-> [s EXCEPT !.edits[m] = Append(@, s.watch)], out |-> <>] + Connect(l) == LET up1 == up \cup {l} gen1 == [gen EXCEPT ![l] = @ + 1] rs == [x \in l |-> LET y == CHOOSE y \in l : y # x - IN Update(x, Closed(st[x], y), OV, up1, gen1, ended[x], NoReport, FALSE)] + a == Asks(Closed(st[x], y), y) + u == Update(x, a.s, OV, up1, gen1, ended[x], NoReport, FALSE) + IN [u EXCEPT !.out[y] = a.out \o @]] IN /\ l \notin up /\ flaps < MaxFlaps /\ up' = up1 @@ -680,7 +790,7 @@ Connect(l) == /\ flaps' = flaps + 1 /\ Commit(rs, q) /\ External(Sent(rs)) - /\ UNCHANGED <> + /\ UNCHANGED <> \* `disconnect`: each end names the hop it learned routes on there. Disconnect(l) == @@ -700,7 +810,25 @@ Disconnect(l) == /\ flaps' = flaps + 1 /\ Commit(rs, q0) /\ External(Sent(rs)) - /\ UNCHANGED <> + /\ UNCHANGED <> + +\* `watch`: relay n changes what it watches to `w`, forgets what it knew outside +\* it, and edits what it asks each neighbour for. +Watch(n, w) == + LET s0 == [st[n] EXCEPT !.watch = w] + peers == Peers(n, up) + a == [p \in peers |-> Asks(s0, p)] + s1 == [s0 EXCEPT !.edits = [p \in Nodes |-> IF p \in peers THEN a[p].s.edits[p] ELSE @[p]]] + u == Update(n, s1, OV, up, gen, ended[n], NoReport, FALSE) + rs == [x \in {n} |-> [u EXCEPT !.out = [p \in Nodes |-> + IF p \in peers THEN a[p].out \o u.out[p] ELSE u.out[p]]]] + IN /\ n \in Editors + /\ edits < MaxEdits + /\ w # st[n].watch + /\ edits' = edits + 1 + /\ Commit(rs, q) + /\ External(Sent(rs)) + /\ UNCHANGED <> \* Nothing is in flight and no doubt is left: the run may stop here. Done == @@ -715,6 +843,7 @@ Next == \/ \E n \in Relays : \E hop \in st[n].expiry : Expire(n, hop) \/ \E l \in Links : Connect(l) \/ Disconnect(l) \/ \E n \in Relays, m \in Leaving : GoAway(n, m) + \/ \E n \in Relays, w \in Sets : Watch(n, w) \/ Done Spec == Init /\ [][Next]_vars @@ -746,16 +875,17 @@ NoRegression == regressions = 0 Stable == unstable = 0 \* Whether node n shows claim c (Seen): exactly while it holds a route that -\* covers c, as the newest floor a covering route gave it says. +\* covers c and watches c, as the newest floor a believed covering route gave +\* it says. Seen == \A n \in Relays, c \in Claims : LET v == st[n].view[KeyOf(c)] IN /\ st[n].shown[c].item # None <=> (Presence(st[n], KeyOf(c)) /\ v.floor >= 0 /\ v.items[c] # None) /\ st[n].shown[c].item # None => st[n].shown[c].item = v.items[c] \* No believed covering route has a newer floor than the view. - /\ Presence(st[n], KeyOf(c)) => + /\ Presence(st[n], KeyOf(c)) /\ Watched(st[n], KeyOf(c)) => \A m \in Nodes, j \in Keys : - (j = KeyOf(c) \/ j = "root") /\ Believed(n, st[n], m, j) => + (j = KeyOf(c) \/ j = "root") /\ Believed(n, st[n], m, j, KeyOf(c)) => st[n].rx[m][j].rev <= v.floor \* The loop-free paths from the publisher over the configured links, as the @@ -854,16 +984,16 @@ StaleIsChased == Usable(n, st[n], m, k) /\ Stale(st[n].rx[m][k].path) => Chased(n, m, k, Cardinality(Nodes)) \* The nodes a path over open sessions from the publisher reaches, through -\* nodes whose interest covers claim c. +\* nodes that watch claim c. RECURSIVE Grow(_, _, _) Grow(R, c, i) == IF i = 0 THEN R - ELSE Grow(R \cup {n \in Relays : Covers(Ask(n), c) /\ \E m \in R : {m, n} \in up}, c, i - 1) + ELSE Grow(R \cup {n \in Relays : CoversSet(st[n].watch, c) /\ \E m \in R : {m, n} \in up}, c, i - 1) Carried(c) == Grow({Origin}, c, Cardinality(Nodes)) \* Once nothing is in flight and no doubt is left: each relay shows exactly the \* publisher's claims that a path carries to it, available; a request for each -\* reaches the publisher; no relay holds a route or keeps a stale one; and -\* every dead hop is forgotten. +\* reaches the publisher; no relay holds a route or keeps a stale one; every +\* dead hop is forgotten; and each neighbour answers what the relay watches. Settled == /\ \A n \in Relays : /\ \A c \in Claims : @@ -873,6 +1003,9 @@ Settled == /\ \A m \in Nodes, k \in Keys : st[n].rx[m][k].on => st[n].rx[m][k].state = "live" /\ ~Stale(st[n].rx[m][k].path) /\ st[n].dead = {} + \* Every edit took effect, and both ends of a session agree on it. + /\ \A m \in Peers(n, up) : + st[n].asked[m] = st[n].watch /\ st[n].edits[m] = <<>> /\ st[m].ans[n] = st[n].watch /\ \A n \in Relays, c \in Claims : st[n].shown[c].avail => Request(n, c) = "origin" QuietIsSettled == Quiet /\ NoDoubt => Settled diff --git a/rs/moq-net/verify/route/Twice.cfg b/rs/moq-net/verify/route/Twice.cfg index 2b277a48b58..f4e92bbe337 100644 --- a/rs/moq-net/verify/route/Twice.cfg +++ b/rs/moq-net/verify/route/Twice.cfg @@ -12,10 +12,12 @@ CONSTANTS Preferred = {} Leaving = {} Scoped = {} + Editors = {} Claims = {"x"} Narrow = {} MaxRev = 1 MaxFlaps = 2 + MaxEdits = 0 Patient = TRUE SPECIFICATION Spec From a44ee9372d9b63b71c0c9d4cc3d0e29eba8d94f8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 07:08:12 -0800 Subject: [PATCH 05/11] fix(route): rekey keeps the route the sender would reflect where several meet TLC found a narrowing that kept a stale route the receiver could not use at the added prefix and dropped the covering one it used, taking its claim offline until the next offer. Ended sent routes no longer count as a route at the key either, which left the two sides out of step. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-route/SKILL.md | 2 +- rs/moq-net/src/route/path.rs | 101 ++++++++++++++++------- rs/moq-net/src/route/tests/interest.rs | 49 +++++++++++ rs/moq-net/verify/route/Edit.cfg | 2 + rs/moq-net/verify/route/EditSmall.cfg | 38 +++++++++ rs/moq-net/verify/route/RouteHandoff.tla | 86 +++++++++++-------- 6 files changed, 209 insertions(+), 69 deletions(-) create mode 100644 rs/moq-net/verify/route/EditSmall.cfg diff --git a/.claude/skills/verify-route/SKILL.md b/.claude/skills/verify-route/SKILL.md index c3045b08cc2..a67f139ce9b 100644 --- a/.claude/skills/verify-route/SKILL.md +++ b/.claude/skills/verify-route/SKILL.md @@ -18,7 +18,7 @@ Each rule is an INVARIANT; the harm says why it matters. | `NoRevival` | A reader never sees START for an item it saw END for, unless it forgot it since. | An ended broadcast comes back. | | `NoFalseEnd` | A reader sees END only once the publisher withdrew that item. | A live broadcast is reported as ended rather than offline. | | `NoRegression` | A reader never sees an UPDATE to an older item. | A stale announcement replaces a newer one. | -| `Stable` | A step that only changes a route's path, lifts a doubt, or applies an interest edit (its ask or its marker) changes a shown claim only to a newer floor, or lets go of one it held unavailable. | A detour flaps claims, as a seqno advance did under the distance, or an edit that swaps prefixes ends or re-sends a claim both sets cover. | +| `Stable` | A step that only changes a route's path, lifts a doubt, or applies an interest edit (its ask, or its marker for the claims both of the edit's sets cover) changes a shown claim only to a newer floor, or lets go of one it held unavailable. | A detour flaps claims, as a seqno advance did under the distance, or an edit that swaps prefixes ends or re-sends a claim both sets cover. | | `SimplePaths` | Every route held or in flight names a loop-free path from the publisher to its sender. | A relay took back a route that crossed it. | | `LoopsAreStale` | A request that loops crosses a relay whose route names a closed session. | Routing loops with every relay up to date. | | `StaleIsChased` | Every stale route a relay may use has the news that ends it on its way: queued toward the relay, or toward the neighbour it came from, and so on back. | A stale route, and any loop through it, lasts for good. | diff --git a/rs/moq-net/src/route/path.rs b/rs/moq-net/src/route/path.rs index 62db9402f1b..1704b5bb81c 100644 --- a/rs/moq-net/src/route/path.rs +++ b/rs/moq-net/src/route/path.rs @@ -611,17 +611,27 @@ fn nested(set: &BTreeSet) -> bool { /// What a stream carries for a route, as [`rekey`] moves it. trait Carried: Clone { + /// The offer it carries, unless it was ended. + fn offered(&self) -> Option<&Offer>; /// Its claims, unless it was ended. fn view(&mut self) -> Option<&mut View>; } impl Carried for Heard { + fn offered(&self) -> Option<&Offer> { + Some(&self.offer) + } + fn view(&mut self) -> Option<&mut View> { Some(&mut self.view) } } impl Carried for Sent { + fn offered(&self) -> Option<&Offer> { + self.offer() + } + fn view(&mut self) -> Option<&mut View> { match &mut self.state { SentState::Live(sent) | SentState::Held(sent) => { @@ -637,61 +647,67 @@ impl Carried for Sent { /// What [`rekey`] leaves of a stream's routes. struct Rekeyed { routes: BTreeMap, - /// The keys it no longer carries a route under. + /// The keys whose route it no longer carries there. gone: Vec, /// Each route that moved, with its offer, to another key: from, to. moved: Vec<(RouteId, RouteId)>, } +/// Whether the sender would reflect `a` rather than `b` to `receiver` under one +/// key, as [`Router::src`] picks: one not naming the receiver first, then the +/// better rank. Equals keep `b`. Both sides see the same offers, so both agree. +fn preferred(a: &Offer, b: &Offer, receiver: Id) -> bool { + (names(&a.path, receiver), a.rank()) < (names(&b.path, receiver), b.rank()) +} + /// The routes one stream carries once an edit of its prefixes from `old` to `new` /// takes effect at its marker. Both sides apply it alike to what was sent and what /// was heard, so they stay in step and nothing both sets cover is sent again. +/// `receiver` names the stream's receiving node. /// /// A route inside a prefix the new set covers stays. A route at a removed prefix -/// that an added prefix widens moves to the wider prefix: the first by key keeps -/// its offer and state, the rest give only their claims, and the claim presented -/// at the removed prefix goes with every floor, since it may stand for a broader -/// claim and the floors certified only the narrower prefix. A route at a removed -/// prefix is carried, as presented there, to each added prefix inside it that no -/// route keys, so its floor carries too. The rest go. +/// that an added prefix widens moves to the wider prefix, with the claim presented +/// at the removed prefix and every floor gone, since that claim may stand for a +/// broader one and the floors certified only the narrower prefix. A route at a +/// removed prefix is carried, as presented there, to each added prefix inside it, +/// so its floor carries too. The rest go. +/// +/// Where several routes reach one key, the key keeps the offer and state of the one +/// the sender would reflect there ([`preferred`]), the narrowest or first by key +/// among equals, and the claims of every route that widened into it. Keeping +/// another could leave the receiver no usable route until the sender's next offer +/// for the key, and its claims offline in between. fn rekey>( old: &BTreeSet, new: &BTreeSet, routes: BTreeMap, + receiver: Id, ) -> Rekeyed { let mut kept: BTreeMap = BTreeMap::new(); let mut gone = Vec::new(); - let mut moved = Vec::new(); - for (key, mut route) in routes.iter().map(|(key, route)| (key.clone(), route.clone())) { - let (id, prefix) = &key; + // The routes moving to each key, as from where and what they carry there. + let mut arriving: BTreeMap> = BTreeMap::new(); + for (key, route) in &routes { + let (id, prefix) = key; let Some(region) = ancestors(prefix).find(|above| old.contains(above)) else { - gone.push(key); + gone.push(key.clone()); continue; }; if within_any(prefix, new) && (new.contains(®ion) || *prefix != region) { - kept.insert(key, route); + kept.insert(key.clone(), route.clone()); continue; } + gone.push(key.clone()); + let mut route = route.clone(); if let Some(wider) = ancestors(®ion).find(|above| new.contains(above)) { - // At the removed prefix, an added one widens it. - gone.push(key.clone()); let Some(view) = route.view() else { continue; }; view.items.remove(prefix); view.floors.clear(); - let items = std::mem::take(&mut view.items); - match kept.get_mut(&(*id, wider.clone())) { - Some(first) => first.view().expect("a carried route").items.extend(items), - None => { - route.view().expect("a carried route").items = items; - moved.push((key.clone(), (*id, wider.clone()))); - kept.insert((*id, wider), route); - } - } + arriving.entry((*id, wider)).or_default().push((key.clone(), route)); continue; } - gone.push(key.clone()); if *prefix != region { continue; } @@ -699,15 +715,36 @@ fn rekey>( continue; }; for inside in within_set(new, ®ion) { - let key = (*id, inside.clone()); - if routes.contains_key(&key) { - continue; - } let mut copy = route.clone(); *copy.view().expect("a carried route") = view.present(inside); - moved.push(((*id, region.clone()), key.clone())); - kept.insert(key, copy); + arriving.entry((*id, inside.clone())).or_default().push((key.clone(), copy)); + } + } + let mut moved = Vec::new(); + for (key, mut routes) in arriving { + let offer = |route: &T| route.offered().expect("an offered route").clone(); + let mut best = 0; + for (i, (_, route)) in routes.iter().enumerate().skip(1) { + if preferred(&offer(route), &offer(&routes[best].1), receiver) { + best = i; + } + } + let (from, mut route) = routes.remove(best); + // The claims of the rest stay, though no floor certifies them. + for (_, mut other) in routes { + let items = std::mem::take(&mut other.view().expect("a carried route").items); + route.view().expect("a carried route").items.extend(items); + } + // A route the key already carries stays, unless the sender would reflect this one. + let staying = kept.get(&key).and_then(|staying| staying.offered().cloned()); + if staying.is_some_and(|staying| !preferred(&offer(&route), &staying, receiver)) { + continue; + } + if kept.contains_key(&key) { + gone.push(key.clone()); } + moved.push((from, key.clone())); + kept.insert(key, route); } Rekeyed { routes: kept, @@ -863,7 +900,7 @@ impl Router { } self.unreflect(neighbor, key, &sent.src); } - let rekeyed = rekey(&old, &new, tx.clone()); + let rekeyed = rekey(&old, &new, tx.clone(), self.sessions[&neighbor].hop); // A child that learned a route this stream no longer carries under its key // may still owe an answer to the route it reflected. for key in rekeyed.gone { @@ -1265,7 +1302,7 @@ impl Router { for key in &keys { self.forget_heard(key, from); } - let rekeyed = rekey(&old, &new, heard); + let rekeyed = rekey(&old, &new, heard, self.hop); for (key, heard) in rekeyed.routes { self.hear(key, from, heard); } diff --git a/rs/moq-net/src/route/tests/interest.rs b/rs/moq-net/src/route/tests/interest.rs index 26324b01061..d0abc03235a 100644 --- a/rs/moq-net/src/route/tests/interest.rs +++ b/rs/moq-net/src/route/tests/interest.rs @@ -190,6 +190,55 @@ fn an_edit_crossing_a_dead_hop_and_a_hold_settles() { } } +/// A narrowing whose stream carried a route at the added prefix that the receiver +/// cannot use, here one the relay learned through the receiver's own dead session, +/// keeps the covering route the receiver used there instead, as the sender would +/// reflect it, so the claim stays available across the marker. Found by TLC +/// (`EditSmall.cfg`). +#[test] +fn narrowing_keeps_the_route_the_sender_would_reflect() { + let (origin, r1, client) = (0, 1, 2); + let mut sim = Sim::with(Rule::PATH, 3); + sim.leaf(origin); + sim.join(origin, r1, 1); + sim.join(origin, client, 1); + sim.join(r1, client, 1); + sim.watch_all(client, &["live/a"]); + sim.originate(origin, 7, 0); + sim.announce(origin, "live/a/x", 0, None); + assert!(sim.run()); + assert_eq!(sim.next_hop(client, 7), Some(origin)); + + // The relay answers the widening first. The client takes its root route, and + // hears the one the relay learned through the client only once that session + // closed, so it lets go of its own route there. + sim.hold(origin, client); + sim.hold(r1, client); + sim.watch(client, "live"); + assert!(sim.run()); + while sim.path(client).heard_offer(7, &p("live"), r1).is_none() { + assert!(sim.deliver(r1, client)); + } + assert!(sim.path(client).heard_offer(7, &p("live/a"), r1).is_none()); + sim.watch_all(client, &["live/a"]); + assert!(sim.run()); + sim.disconnect(origin, client); + assert!(sim.run()); + assert_eq!(sim.next_hop(client, 7), Some(r1)); + assert!(sim.queued(r1, client) > 0, "the relay's marker waits"); + while sim.deliver(r1, client) { + assert!(sim.shows(client, 7, "live/a/x").is_some(), "{:?}", life(&sim, client, "live/a/x")); + } + sim.release(r1, client); + assert!(sim.run()); + let life = life(&sim, client, "live/a/x"); + assert!( + life.iter().all(|kind| !matches!(kind, Kind::End | Kind::Offline)), + "{life:?}" + ); + sim.check().unwrap(); +} + /// Each added prefix is SYNCED on its own: one waiting behind a held stream is /// not, while the other neighbour's already is. #[test] diff --git a/rs/moq-net/verify/route/Edit.cfg b/rs/moq-net/verify/route/Edit.cfg index eb8fd9b5584..bacaa72c8bb 100644 --- a/rs/moq-net/verify/route/Edit.cfg +++ b/rs/moq-net/verify/route/Edit.cfg @@ -1,3 +1,5 @@ +\* Too large to check every ordering, so TLC samples it at random. +\* Simulate: -simulate num=10000 -depth 120 \* Interest edits in flight: relay 3, watching "a", hears the publisher (0) \* through relays 1 and 2, and changes what it watches twice: widening, \* narrowing, dropping everything, or asking again. Each edit crosses a claim diff --git a/rs/moq-net/verify/route/EditSmall.cfg b/rs/moq-net/verify/route/EditSmall.cfg new file mode 100644 index 00000000000..d5d052fc591 --- /dev/null +++ b/rs/moq-net/verify/route/EditSmall.cfg @@ -0,0 +1,38 @@ +\* Every ordering of an interest edit: relay 2, watching "a", hears the +\* publisher (0) through relay 1 and over a dear session of its own, and +\* changes what it watches twice: widening, narrowing, dropping everything, or +\* asking again. Each edit crosses a claim change, the 0-1 session closing with +\* its DEAD and the holds it starts, or the 1-2 session closing. +CONSTANTS + Nodes = {0, 1, 2} + Origin = 0 + Links = {{0, 1}, {0, 2}, {1, 2}} + Dear = {{0, 2}} + Ring = {} + Preferred = {} + Leaving = {} + Scoped = {2} + Editors = {2} + Claims = {"ax", "bx"} + Narrow = {"ax"} + MaxRev = 1 + MaxFlaps = 1 + MaxEdits = 2 + Patient = TRUE + +SPECIFICATION Spec + +INVARIANTS + Truthful + NoRevival + NoFalseEnd + NoRegression + Stable + Seen + SimplePaths + LoopsAreStale + StaleIsChased + RequestsLoopFree + NoStaleReturn + QuietIsSettled + Budget diff --git a/rs/moq-net/verify/route/RouteHandoff.tla b/rs/moq-net/verify/route/RouteHandoff.tla index a5cb9c8c131..8bba4f34212 100644 --- a/rs/moq-net/verify/route/RouteHandoff.tla +++ b/rs/moq-net/verify/route/RouteHandoff.tla @@ -449,9 +449,10 @@ DrainReport(x) == [NoReport EXCEPT !.node = x] \* The reader's events: changes to `ended` and the counters. A claim leaving \* while the node still holds a covering route and watches it is an END, else \* it went offline or is no longer watched, and the reader forgets it. -\* `quiet` marks a step that only changed a path or lifted a doubt: in one, a -\* claim the reader sees changes only to what a newer floor says, or goes -\* offline after it was held unavailable. +\* `quiet` holds the claims a step leaves alone but for their paths: one that +\* only changed a path, lifted a doubt, or applied an edit to what both its +\* sets cover. A claim in it the reader sees changes only to what a newer floor +\* says, or goes offline after it was held unavailable. Events(s0, s, old, now, ov, eb, quiet) == LET present(c) == Presence(s, KeyOf(c)) /\ Watched(s, KeyOf(c)) isEnd(c) == old[c].item # None /\ now[c].item = None /\ present(c) @@ -467,8 +468,7 @@ Events(s0, s, old, now, ov, eb, quiet) == revivals |-> Cardinality({c \in Claims : isStart(c) /\ eb[c] = now[c].item}), falseEnds |-> Cardinality({c \in Claims : isEnd(c) /\ ov.items[c] = old[c].item}), regressions |-> Cardinality({c \in Claims : isUpdate(c) /\ now[c].item < old[c].item}), - unstable |-> IF ~quiet THEN 0 - ELSE Cardinality({c \in Claims : old[c].item # now[c].item /\ before(c) /\ ~newer(c) /\ ~release(c)})] + unstable |-> Cardinality({c \in quiet : old[c].item # now[c].item /\ before(c) /\ ~newer(c) /\ ~release(c)})] Update(n, s0, ov, ups, gens, eb, rep, quiet) == LET s1 == Select(n, s0, gens) @@ -556,7 +556,7 @@ InitNode(n) == [ OV0 == [rev |-> 0, items |-> NoItems] Gen0 == [l \in Links |-> 0] Start == Update(Origin, [InitNode(Origin) EXCEPT !.rt = OwnRoutes], OV0, Links, Gen0, - [c \in Claims |-> None], NoReport, FALSE) + [c \in Claims |-> None], NoReport, {}) Init == /\ up = Links @@ -585,7 +585,7 @@ OV == [rev |-> rev, items |-> own] ChangeClaim(c, item) == LET own1 == [own EXCEPT ![c] = item] ov1 == [rev |-> rev + 1, items |-> own1] - rs == [n \in {Origin} |-> Update(Origin, st[Origin], ov1, up, gen, ended[Origin], NoReport, FALSE)] + rs == [n \in {Origin} |-> Update(Origin, st[Origin], ov1, up, gen, ended[Origin], NoReport, {})] IN /\ rev' = rev + 1 /\ own' = own1 /\ hist' = Append(hist, own1) @@ -647,47 +647,58 @@ Recv(n, s, from, msg) == [] msg.t = "drain" -> IF Credited(s, from, msg.node) THEN MarkDrain(n, s, msg.node) ELSE s [] OTHER -> Fail(n, s, msg.hop, from, msg.moved, msg.gone) -\* The routes of one stream once an edit of its prefixes from `O` to `N` takes -\* effect at its marker (path.rs: `rekey`), with `Off(r)` what a gone entry -\* becomes. Both sides apply it alike, to what was sent and what was heard. A -\* key within N stays, unless it sits at a removed prefix. The route at a -\* removed "a" moves to an added "root" with its offer and state, and without -\* its floor, which certified only "a". The route at a removed "root" is -\* carried, presented at "a", to an added "a" no route keys, so its floor -\* carries too. The rest go, with no ROUTE_END. -Rekey(O, N, R, Off(_)) == +\* Whether the sender would reflect route `x` rather than `y` to node r under +\* one key, as `SrcKey` picks: one not naming r first, then the better rank. +\* Both sides see the same offers, so both agree. +Reflects(x, y, r) == + \/ ~Names(x.path, r) /\ Names(y.path, r) + \/ Names(x.path, r) = Names(y.path, r) /\ Below(Rank(x.path, x.drain), Rank(y.path, y.drain)) + +\* The routes of one stream to node r once an edit of its prefixes from `O` to +\* `N` takes effect at its marker (path.rs: `rekey`), with `Off(e)` what a gone +\* entry becomes. Both sides apply it alike, to what was sent and what was +\* heard. A key within N stays, unless it sits at a removed prefix. The route +\* at a removed "a" moves to an added "root" with its offer and state, and +\* without its floor, which certified only "a". The route at a removed "root" +\* is carried, presented at "a", to an added "a", so its floor carries too, +\* unless the route "a" already carries is the one the sender would reflect +\* there. The rest go, with no ROUTE_END. +Carries(O, N, R, r) == + /\ "root" \in O /\ "a" \in N /\ R["root"].on + /\ ~R["a"].on \/ Reflects(R["root"], R["a"], r) + +Rekey(O, N, R, Off(_), r) == [k \in Keys |-> IF k = "root" THEN IF "root" \in O /\ "root" \in N THEN R["root"] ELSE IF "a" \in O /\ "root" \in N /\ R["a"].on THEN [R["a"] EXCEPT !.rev = None] ELSE Off(R["root"]) - ELSE IF R["a"].on /\ (("root" \in O /\ N # {}) \/ ("a" \in O /\ "a" \in N)) THEN R["a"] - ELSE IF "root" \in O /\ "a" \in N /\ R["root"].on /\ ~R["a"].on - THEN [R["root"] EXCEPT !.items = Restrict(@, "a")] + ELSE IF Carries(O, N, R, r) THEN [R["root"] EXCEPT !.items = Restrict(@, "a")] + ELSE IF R["a"].on /\ (("root" \in O /\ N # {}) \/ ("a" \in O /\ "a" \in N)) THEN R["a"] ELSE Off(R["a"])] \* The keys `Rekey` moved a route between, as <>. -Moved(O, N, R) == +Moved(O, N, R, r) == (IF "a" \in O /\ "root" \in N /\ R["a"].on THEN {<<"a", "root">>} ELSE {}) - \cup (IF "root" \in O /\ "a" \in N /\ R["root"].on /\ ~R["a"].on THEN {<<"root", "a">>} ELSE {}) + \cup (IF Carries(O, N, R, r) THEN {<<"root", "a">>} ELSE {}) \* `edit`: m now asks node n for `set`. It takes effect at once, behind the \* marker n sends first. Answer(s, m, set) == [s EXCEPT !.ans[m] = set, - !.tx[m] = Rekey(s.ans[m], set, s.tx[m], LAMBDA r : NoSent(r.src))] + !.tx[m] = Rekey(s.ans[m], set, s.tx[m], LAMBDA e : NoSent(e.src), m)] \* `edited`: m's marker for node n's oldest edit in flight. The routes m sent \* are rekeyed as m rekeyed them, and a route n used or held through m moves \* with the route it heard, so a hold carries over rather than letting go. -Marker(s, m) == +Marker(n, s, m) == LET old == s.asked[m] new == Head(s.edits[m]) - mv == Moved(old, new, s.rx[m]) + mv == Moved(old, new, s.rx[m], n) carry(k) == {x \in mv : x[2] = k /\ s.rt[x[1]].on /\ s.rt[x[1]].next = m /\ ~s.rt[k].on} IN [s EXCEPT !.asked[m] = new, !.edits[m] = Tail(@), - !.rx[m] = Rekey(old, new, s.rx[m], LAMBDA r : NoHeard), + !.rx[m] = Rekey(old, new, s.rx[m], LAMBDA e : NoHeard, n), !.rt = [k \in Keys |-> IF carry(k) # {} THEN s.rt[(CHOOSE x \in carry(k) : TRUE)[1]] ELSE s.rt[k]]] @@ -700,11 +711,14 @@ Deliver(d) == [] msg.t = "drain" /\ msg.node # n /\ Credited(st[n], m, msg.node) -> DrainReport(msg.node) [] OTHER -> NoReport \* Only the route's path, or how it ranks, changed; or an edit took effect, - \* which changes no claim both sets cover. - quiet == \/ msg.t = "offer" /\ st[n].rx[m][msg.k].on - \/ msg.t \in {"drain", "ask", "edited"} + \* which changes no claim both its sets cover. + quiet == CASE \/ msg.t = "offer" /\ st[n].rx[m][msg.k].on + \/ msg.t \in {"drain", "ask"} -> Claims + [] msg.t = "edited" -> + {c \in Claims : CoversSet(st[n].asked[m], c) /\ CoversSet(Head(st[n].edits[m]), c)} + [] OTHER -> {} s1 == CASE msg.t = "ask" -> Answer(st[n], m, msg.set) - [] msg.t = "edited" -> Marker(st[n], m) + [] msg.t = "edited" -> Marker(n, st[n], m) [] OTHER -> Recv(n, st[n], m, msg) u == Update(n, s1, OV, up, gen, ended[n], rep, quiet) \* The marker goes before anything the edit sends. @@ -723,7 +737,7 @@ Deliver(d) == Lift(n, x) == LET rs == [y \in {n} |-> Update(n, [st[n] EXCEPT !.doubt = @ \ {x}, !.wait = @ \ {x}, !.expiry = {h \in @ : h[2] # x}], - OV, up, gen, ended[n], NoReport, TRUE)] + OV, up, gen, ended[n], NoReport, Claims)] IN /\ x \in st[n].wait /\ Patient => \A d \in Dirs : \A i \in DOMAIN q[d] : ~(q[d][i].t = "dead" /\ q[d][i].hop[2] = x) /\ ~(q[d][i].t = "drain" /\ q[d][i].node = x) @@ -735,7 +749,7 @@ Lift(n, x) == \* node. A new DEAD cannot extend every old hop's retention indefinitely. Expire(n, hop) == LET rs == [y \in {n} |-> Update(n, [st[n] EXCEPT !.expiry = @ \ {hop}], - OV, up, gen, ended[n], NoReport, TRUE)] + OV, up, gen, ended[n], NoReport, Claims)] IN /\ hop \in st[n].expiry /\ Patient => \A d \in Dirs : \A i \in DOMAIN q[d] : ~(q[d][i].t = "dead" /\ q[d][i].hop = hop) /\ Commit(rs, q) @@ -751,7 +765,7 @@ Expire(n, hop) == \* neighbour it sent a route through m. GoAway(n, m) == LET rs == [x \in {n} |-> Update(n, MarkDrain(n, [st[n] EXCEPT !.drain = @ \cup {m}], m), - OV, up, gen, ended[n], DrainReport(m), TRUE)] + OV, up, gen, ended[n], DrainReport(m), Claims)] IN /\ {n, m} \in up /\ m \notin st[n].drain /\ Commit(rs, q) @@ -781,7 +795,7 @@ Connect(l) == rs == [x \in l |-> LET y == CHOOSE y \in l : y # x a == Asks(Closed(st[x], y), y) - u == Update(x, a.s, OV, up1, gen1, ended[x], NoReport, FALSE) + u == Update(x, a.s, OV, up1, gen1, ended[x], NoReport, {}) IN [u EXCEPT !.out[y] = a.out \o @]] IN /\ l \notin up /\ flaps < MaxFlaps @@ -801,9 +815,9 @@ Disconnect(l) == hop == <> s == Closed(st[x], y) IN IF x = Origin - THEN Update(x, s, OV, up1, gen, ended[x], NoReport, FALSE) + THEN Update(x, s, OV, up1, gen, ended[x], NoReport, {}) ELSE Update(x, Fail(x, s, hop, None, {}, FALSE), OV, up1, gen, ended[x], - DeadReport(hop), FALSE)] + DeadReport(hop), {})] IN /\ l \in up /\ flaps < MaxFlaps /\ up' = up1 @@ -819,7 +833,7 @@ Watch(n, w) == peers == Peers(n, up) a == [p \in peers |-> Asks(s0, p)] s1 == [s0 EXCEPT !.edits = [p \in Nodes |-> IF p \in peers THEN a[p].s.edits[p] ELSE @[p]]] - u == Update(n, s1, OV, up, gen, ended[n], NoReport, FALSE) + u == Update(n, s1, OV, up, gen, ended[n], NoReport, {}) rs == [x \in {n} |-> [u EXCEPT !.out = [p \in Nodes |-> IF p \in peers THEN a[p].out \o u.out[p] ELSE u.out[p]]]] IN /\ n \in Editors From 4fbf8f1a925a7a67a82f6373a446081d73a01060 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 08:20:04 -0800 Subject: [PATCH 06/11] test(route): exhaustive narrow and widen configs, two edits simulated Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/path.rs | 5 ++- rs/moq-net/src/route/tests/interest.rs | 6 ++- rs/moq-net/verify/route/Edit.cfg | 23 ++++++----- rs/moq-net/verify/route/EditNarrow.cfg | 38 +++++++++++++++++++ .../route/{EditSmall.cfg => EditWiden.cfg} | 12 +++--- 5 files changed, 66 insertions(+), 18 deletions(-) create mode 100644 rs/moq-net/verify/route/EditNarrow.cfg rename rs/moq-net/verify/route/{EditSmall.cfg => EditWiden.cfg} (64%) diff --git a/rs/moq-net/src/route/path.rs b/rs/moq-net/src/route/path.rs index 1704b5bb81c..4cd61bec432 100644 --- a/rs/moq-net/src/route/path.rs +++ b/rs/moq-net/src/route/path.rs @@ -717,7 +717,10 @@ fn rekey>( for inside in within_set(new, ®ion) { let mut copy = route.clone(); *copy.view().expect("a carried route") = view.present(inside); - arriving.entry((*id, inside.clone())).or_default().push((key.clone(), copy)); + arriving + .entry((*id, inside.clone())) + .or_default() + .push((key.clone(), copy)); } } let mut moved = Vec::new(); diff --git a/rs/moq-net/src/route/tests/interest.rs b/rs/moq-net/src/route/tests/interest.rs index d0abc03235a..c9dafd01b05 100644 --- a/rs/moq-net/src/route/tests/interest.rs +++ b/rs/moq-net/src/route/tests/interest.rs @@ -227,7 +227,11 @@ fn narrowing_keeps_the_route_the_sender_would_reflect() { assert_eq!(sim.next_hop(client, 7), Some(r1)); assert!(sim.queued(r1, client) > 0, "the relay's marker waits"); while sim.deliver(r1, client) { - assert!(sim.shows(client, 7, "live/a/x").is_some(), "{:?}", life(&sim, client, "live/a/x")); + assert!( + sim.shows(client, 7, "live/a/x").is_some(), + "{:?}", + life(&sim, client, "live/a/x") + ); } sim.release(r1, client); assert!(sim.run()); diff --git a/rs/moq-net/verify/route/Edit.cfg b/rs/moq-net/verify/route/Edit.cfg index bacaa72c8bb..99385990b50 100644 --- a/rs/moq-net/verify/route/Edit.cfg +++ b/rs/moq-net/verify/route/Edit.cfg @@ -1,20 +1,23 @@ \* Too large to check every ordering, so TLC samples it at random. \* Simulate: -simulate num=10000 -depth 120 -\* Interest edits in flight: relay 3, watching "a", hears the publisher (0) -\* through relays 1 and 2, and changes what it watches twice: widening, -\* narrowing, dropping everything, or asking again. Each edit crosses a claim -\* change, a session closing and its DEAD, the holds and their answers, and -\* whatever else the FIFOs carry. +\* Two interest edits in flight: relay 2, watching "a", hears the publisher (0) +\* through relay 1 and over a dear session of its own, and changes what it +\* watches twice: widening then narrowing back, dropping everything then asking +\* again, and so on. Before the rekey kept the route the sender would reflect, +\* a widening then a narrowing crossing the 0-2 session closing kept a route +\* relay 1 learned through relay 2's dead session and took "ax" offline, +\* breaking Stable at depth 16 +\* (`interest::narrowing_keeps_the_route_the_sender_would_reflect`). CONSTANTS - Nodes = {0, 1, 2, 3} + Nodes = {0, 1, 2} Origin = 0 - Links = {{0, 1}, {0, 2}, {1, 3}, {2, 3}} - Dear = {} + Links = {{0, 1}, {0, 2}, {1, 2}} + Dear = {{0, 2}} Ring = {} Preferred = {} Leaving = {} - Scoped = {3} - Editors = {3} + Scoped = {2} + Editors = {2} Claims = {"ax", "bx"} Narrow = {"ax"} MaxRev = 1 diff --git a/rs/moq-net/verify/route/EditNarrow.cfg b/rs/moq-net/verify/route/EditNarrow.cfg new file mode 100644 index 00000000000..f604d0b0b2e --- /dev/null +++ b/rs/moq-net/verify/route/EditNarrow.cfg @@ -0,0 +1,38 @@ +\* Every ordering of one interest edit: relay 2, watching everything, hears +\* the publisher (0) through relay 1 and over a dear session of its own, and +\* narrows to "a" or drops everything. The edit crosses a claim change, the +\* 0-1 or 0-2 session closing with its DEAD and the holds it starts, or the +\* 1-2 session closing. +CONSTANTS + Nodes = {0, 1, 2} + Origin = 0 + Links = {{0, 1}, {0, 2}, {1, 2}} + Dear = {{0, 2}} + Ring = {} + Preferred = {} + Leaving = {} + Scoped = {} + Editors = {2} + Claims = {"ax", "bx"} + Narrow = {"ax"} + MaxRev = 1 + MaxFlaps = 1 + MaxEdits = 1 + Patient = TRUE + +SPECIFICATION Spec + +INVARIANTS + Truthful + NoRevival + NoFalseEnd + NoRegression + Stable + Seen + SimplePaths + LoopsAreStale + StaleIsChased + RequestsLoopFree + NoStaleReturn + QuietIsSettled + Budget diff --git a/rs/moq-net/verify/route/EditSmall.cfg b/rs/moq-net/verify/route/EditWiden.cfg similarity index 64% rename from rs/moq-net/verify/route/EditSmall.cfg rename to rs/moq-net/verify/route/EditWiden.cfg index d5d052fc591..99317eabd3d 100644 --- a/rs/moq-net/verify/route/EditSmall.cfg +++ b/rs/moq-net/verify/route/EditWiden.cfg @@ -1,8 +1,8 @@ -\* Every ordering of an interest edit: relay 2, watching "a", hears the -\* publisher (0) through relay 1 and over a dear session of its own, and -\* changes what it watches twice: widening, narrowing, dropping everything, or -\* asking again. Each edit crosses a claim change, the 0-1 session closing with -\* its DEAD and the holds it starts, or the 1-2 session closing. +\* Every ordering of one interest edit: relay 2, watching "a", hears the +\* publisher (0) through relay 1 and over a dear session of its own, and widens +\* to everything or drops everything. The edit crosses a claim change, the 0-1 +\* or 0-2 session closing with its DEAD and the holds it starts, or the 1-2 +\* session closing. CONSTANTS Nodes = {0, 1, 2} Origin = 0 @@ -17,7 +17,7 @@ CONSTANTS Narrow = {"ax"} MaxRev = 1 MaxFlaps = 1 - MaxEdits = 2 + MaxEdits = 1 Patient = TRUE SPECIFICATION Spec From 5ce32caf1266d856f4cd587b03877ed8b5b72f82 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 09:27:06 -0800 Subject: [PATCH 07/11] docs(verify-route): the edit configs, and the sync commit at the interest set Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-route/SKILL.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.claude/skills/verify-route/SKILL.md b/.claude/skills/verify-route/SKILL.md index a67f139ce9b..9f2f4a07281 100644 --- a/.claude/skills/verify-route/SKILL.md +++ b/.claude/skills/verify-route/SKILL.md @@ -79,6 +79,9 @@ change; faster checkers are separate work. | `Drain.cfg` | The publisher joined to two relays joined to each other; both hear its GOAWAY. Ranking only a relay's own drain exceeded `Budget`. | 1 claim change, 1 session change, GOAWAY on each session | 648,218 | two minutes | | `DrainRing.cfg` | The publisher joined to three relays in a ring that is cheap one way round (`Ring`); all hear its GOAWAY. Without the DRAIN news and its doubt, a request looped round the ring with no session closed, breaking `LoopsAreStale`. Simulated. | 1 claim change, GOAWAY on each session | 1,000 behaviours per worker, depth 150, seed 1 | about a minute | | `Prefer.cfg` | The same triangle, where both relays prefer the link between them. Ranking only a relay's own link priority exceeded `Budget`. | 1 claim change, 1 session change | 14,630 | seconds | +| `EditNarrow.cfg` | The publisher with sessions to relays 1 and 2, the one to relay 2 dear; relay 2 watches everything and narrows to `"a"` or drops it once. | 1 claim change, 1 session change, 1 edit | 768,483 | three minutes | +| `EditWiden.cfg` | The same, where relay 2 watches `"a"` and widens to everything or drops it once. | 1 claim change, 1 session change, 1 edit | 3,655,573 | a quarter of an hour | +| `Edit.cfg` | The same with two edits, such as widening then narrowing back, or dropping everything then asking again. Before the rekey kept the route the sender would reflect, a widening then a narrowing crossing the close of relay 2's own session kept a route relay 1 learned through it and took a claim offline, breaking `Stable`. Simulated; an exhaustive pass covered every behaviour to depth 18 (10.6M states) before it was stopped. | 1 claim change, 1 session change, 2 edits | 40,000 behaviours, 4.8M states | four minutes | | `HoldCycle.cfg` | The publisher on a hub with two open relays, and two relays asking for `"a"` alone, each joined to both open relays and nearer the other's choice. Before a hold followed only the path it was built on, the publisher's session closing left four holds waiting on each other for good, breaking `QuietIsSettled` within seconds. Simulated. | 1 claim change, 1 session change | 2,000 behaviours per worker, depth 200, seed 1 | a few minutes | Run the affected config last, with at most one TLC process at a time. First run what answers in seconds: the router's tests (`quest gate just rs test -p moq-net --lib -E 'test(/route::tests::(handoff|residual|scope|compare)/)'`, with `MOQ_ROUTE_SCENARIOS` raising the sweeps' count), then TLC's simulation mode with raised bounds in a copy of a config, such as `-simulate num=3000 -depth 150`. @@ -89,7 +92,7 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `31b215b112fb649cc3f63d4de3c76b5e87769a06`, the base of the scoped path router in [moq-dev/moq#8622](https://github.com/moq-dev/moq/pull/8622). Run `git log 31b215b11.. -- rs/moq-net/src/route/path.rs rs/moq-net/src/route/reader.rs` to see what changed since. Prefix and reader indexes make the router's updates incremental; test-only full recomputation checks their result against the complete view. The model recomputes every prefix atomically and leaves that optimization out. `Rank` orders routes as `Offer::rank` does: drain, carried with each offer or known from a DRAIN (`Drains`), then the lowest link priority on the path (`Preferred`), then cost; `GoAway` marks a neighbour's session draining. `SrcKey` mirrors the router's preference for the best-ranked covering live path that does not name its receiver, with the narrower scope breaking ties; `Select` plans live routes and their reflected sources before deciding holds; a held route keeps its last live path, so `SelectKey` holds under a next hop only on the path it was built on and `Want` ends a held stream whose source now has another path; `Advertise1` forwards a newer floor even when the claims did not change. The router keeps doubt until its one-second timer fires (`Lift::Timer`, `Doubt::DEFAULT`); the model's `Collect` leaves it for `Lift`. The comparison-only `Rule::EARLY` lifts doubt once no heard route names its node and is outside this timer-only model. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. +Sync commit: `4fbf8f1a925a7a67a82f6373a446081d73a01060`, the interest set in [moq-dev/moq#8728](https://github.com/moq-dev/moq/pull/8728). Run `git log 4fbf8f1a9.. -- rs/moq-net/src/route/path.rs rs/moq-net/src/route/reader.rs` to see what changed since. Prefix and reader indexes make the router's updates incremental; test-only full recomputation checks their result against the complete view. The model recomputes every prefix atomically and leaves that optimization out. `Rank` orders routes as `Offer::rank` does: drain, carried with each offer or known from a DRAIN (`Drains`), then the lowest link priority on the path (`Preferred`), then cost; `GoAway` marks a neighbour's session draining. `SrcKey` mirrors the router's preference for the best-ranked covering live path that does not name its receiver, with the narrower scope breaking ties; `Select` plans live routes and their reflected sources before deciding holds; a held route keeps its last live path, so `SelectKey` holds under a next hop only on the path it was built on and `Want` ends a held stream whose source now has another path; `Advertise1` forwards a newer floor even when the claims did not change. The router keeps doubt until its one-second timer fires (`Lift::Timer`, `Doubt::DEFAULT`); the model's `Collect` leaves it for `Lift`. The comparison-only `Rule::EARLY` lifts doubt once no heard route names its node and is outside this timer-only model. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. | Code | Model | | --- | --- | From 0b1e8da7372a3ae3e8fbcb9bd557e6c9b5ce48c5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 12:29:32 -0800 Subject: [PATCH 08/11] fix(route): rekey prefers a live route, and a kept route the DEAD implies held is held A first DEAD makes the neighbour hold every route it did not send this node, so the sender records that hold too; the model's new Mirrored rule checks that both sides agree, holds included, whenever nothing is in flight. The sync commit moves to the PR base, and the HoldCycle stale return that random seeds find on the base becomes a quest. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-route/SKILL.md | 3 +- .../cluster-routes/holdcycle-stale-return.md | 28 +++++++++++ rs/moq-net/src/route/path.rs | 47 +++++++++++++++---- rs/moq-net/verify/route/Blind.cfg | 1 + rs/moq-net/verify/route/Dead.cfg | 1 + rs/moq-net/verify/route/Drain.cfg | 1 + rs/moq-net/verify/route/DrainRing.cfg | 1 + rs/moq-net/verify/route/Early.cfg | 1 + rs/moq-net/verify/route/Edit.cfg | 1 + rs/moq-net/verify/route/EditNarrow.cfg | 1 + rs/moq-net/verify/route/EditWiden.cfg | 1 + rs/moq-net/verify/route/HoldCycle.cfg | 1 + rs/moq-net/verify/route/Lag.cfg | 1 + rs/moq-net/verify/route/LagWide.cfg | 1 + rs/moq-net/verify/route/Loop.cfg | 1 + rs/moq-net/verify/route/Overlap.cfg | 1 + rs/moq-net/verify/route/Prefer.cfg | 1 + rs/moq-net/verify/route/Ring.cfg | 1 + rs/moq-net/verify/route/RouteHandoff.cfg | 1 + rs/moq-net/verify/route/RouteHandoff.tla | 45 ++++++++++++------ rs/moq-net/verify/route/Twice.cfg | 1 + 21 files changed, 116 insertions(+), 24 deletions(-) create mode 100644 quest/m2/cluster-routes/holdcycle-stale-return.md diff --git a/.claude/skills/verify-route/SKILL.md b/.claude/skills/verify-route/SKILL.md index 9f2f4a07281..822c288dc58 100644 --- a/.claude/skills/verify-route/SKILL.md +++ b/.claude/skills/verify-route/SKILL.md @@ -14,6 +14,7 @@ Each rule is an INVARIANT; the harm says why it matters. | Rule | Holds when | Harm if broken | | --- | --- | --- | | `Truthful` | Every view a relay keeps, and every floor it was offered, is the publisher's claims at that revision within its prefix. | A viewer sees a set of broadcasts the publisher never had. | +| `Mirrored` | With nothing in flight from one relay to another, what the first sent is what the second heard: the same routes, on the same paths, held exactly where the second holds them. | An edit rekeys the two sides differently, and they disagree about a route for good. | | `Seen` | A relay shows a claim exactly while it holds a route covering it and the newest floor a believed covering route gave it has the claim, as that floor says. | A reader sees a claim no route vouches for, or an older one than it learned. | | `NoRevival` | A reader never sees START for an item it saw END for, unless it forgot it since. | An ended broadcast comes back. | | `NoFalseEnd` | A reader sees END only once the publisher withdrew that item. | A live broadcast is reported as ended rather than offline. | @@ -92,7 +93,7 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `4fbf8f1a925a7a67a82f6373a446081d73a01060`, the interest set in [moq-dev/moq#8728](https://github.com/moq-dev/moq/pull/8728). Run `git log 4fbf8f1a9.. -- rs/moq-net/src/route/path.rs rs/moq-net/src/route/reader.rs` to see what changed since. Prefix and reader indexes make the router's updates incremental; test-only full recomputation checks their result against the complete view. The model recomputes every prefix atomically and leaves that optimization out. `Rank` orders routes as `Offer::rank` does: drain, carried with each offer or known from a DRAIN (`Drains`), then the lowest link priority on the path (`Preferred`), then cost; `GoAway` marks a neighbour's session draining. `SrcKey` mirrors the router's preference for the best-ranked covering live path that does not name its receiver, with the narrower scope breaking ties; `Select` plans live routes and their reflected sources before deciding holds; a held route keeps its last live path, so `SelectKey` holds under a next hop only on the path it was built on and `Want` ends a held stream whose source now has another path; `Advertise1` forwards a newer floor even when the claims did not change. The router keeps doubt until its one-second timer fires (`Lift::Timer`, `Doubt::DEFAULT`); the model's `Collect` leaves it for `Lift`. The comparison-only `Rule::EARLY` lifts doubt once no heard route names its node and is outside this timer-only model. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. +Sync commit: `12537d1c2272c39db154fbd210bf60b8f438da4d`, the base of the interest set in [moq-dev/moq#8728](https://github.com/moq-dev/moq/pull/8728). Run `git log 12537d1c2.. -- rs/moq-net/src/route/path.rs rs/moq-net/src/route/reader.rs` to see what changed since. Prefix and reader indexes make the router's updates incremental; test-only full recomputation checks their result against the complete view. The model recomputes every prefix atomically and leaves that optimization out. `Rank` orders routes as `Offer::rank` does: drain, carried with each offer or known from a DRAIN (`Drains`), then the lowest link priority on the path (`Preferred`), then cost; `GoAway` marks a neighbour's session draining. `SrcKey` mirrors the router's preference for the best-ranked covering live path that does not name its receiver, with the narrower scope breaking ties; `Select` plans live routes and their reflected sources before deciding holds; a held route keeps its last live path, so `SelectKey` holds under a next hop only on the path it was built on and `Want` ends a held stream whose source now has another path; `Advertise1` forwards a newer floor even when the claims did not change. The router keeps doubt until its one-second timer fires (`Lift::Timer`, `Doubt::DEFAULT`); the model's `Collect` leaves it for `Lift`. The comparison-only `Rule::EARLY` lifts doubt once no heard route names its node and is outside this timer-only model. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. | Code | Model | | --- | --- | diff --git a/quest/m2/cluster-routes/holdcycle-stale-return.md b/quest/m2/cluster-routes/holdcycle-stale-return.md new file mode 100644 index 00000000000..5b2368b0a1a --- /dev/null +++ b/quest/m2/cluster-routes/holdcycle-stale-return.md @@ -0,0 +1,28 @@ +# [M] HoldCycle keeps a dead path from returning off its pinned seed + +## Goal + +`HoldCycle.cfg` holds `NoStaleReturn` under any simulation seed, as its +bounds promise: one session change and a patient timer, where the +[verify-route skill](/.claude/skills/verify-route/SKILL.md) says the rule holds. +Today Nightly simulates it only with `-seed 1`, which passes. Random seeds +break `NoStaleReturn` within a few hundred thousand states (a 35-step +behaviour). The base of +[the interest set](https://github.com/moq-dev/moq/pull/8728) breaks it the +same way, so no change to interest edits caused it. + +## Plan + +Recommended: + +1. Rerun `HoldCycle.cfg` with `-simulate num=3000 -depth 100` and no seed + until it fails, then read the behaviour as the skill's Reading a failure + says. Decide whether the model misdescribes the router or the router lets a + relay use a route through a hop it already learned died. +2. Replay it on `route::tests::Sim` under `Rule::PATH`. If the code follows + the path, fix the code and pin the replay as a fixture. If it does not, + fix the model. +3. Drop `-seed 1` from `HoldCycle.cfg`'s `Simulate:` line once random seeds + pass, so Nightly samples new orderings each night. Do not drop + `NoStaleReturn` from the config unless the behaviour shows a second session + change that makes this `Twice.cfg`'s accepted case. diff --git a/rs/moq-net/src/route/path.rs b/rs/moq-net/src/route/path.rs index 4cd61bec432..d0a6a0d9ca2 100644 --- a/rs/moq-net/src/route/path.rs +++ b/rs/moq-net/src/route/path.rs @@ -613,6 +613,8 @@ fn nested(set: &BTreeSet) -> bool { trait Carried: Clone { /// The offer it carries, unless it was ended. fn offered(&self) -> Option<&Offer>; + /// Whether it is held: both sides agree, as both change it on the same messages. + fn held(&self) -> bool; /// Its claims, unless it was ended. fn view(&mut self) -> Option<&mut View>; } @@ -622,6 +624,10 @@ impl Carried for Heard { Some(&self.offer) } + fn held(&self) -> bool { + self.state == State::Held + } + fn view(&mut self) -> Option<&mut View> { Some(&mut self.view) } @@ -632,6 +638,10 @@ impl Carried for Sent { self.offer() } + fn held(&self) -> bool { + Sent::held(self) + } + fn view(&mut self) -> Option<&mut View> { match &mut self.state { SentState::Live(sent) | SentState::Held(sent) => { @@ -654,10 +664,15 @@ struct Rekeyed { } /// Whether the sender would reflect `a` rather than `b` to `receiver` under one -/// key, as [`Router::src`] picks: one not naming the receiver first, then the -/// better rank. Equals keep `b`. Both sides see the same offers, so both agree. -fn preferred(a: &Offer, b: &Offer, receiver: Id) -> bool { - (names(&a.path, receiver), a.rank()) < (names(&b.path, receiver), b.rank()) +/// key, as [`Router::src`] picks: a live one first, then one not naming the +/// receiver, then the better rank. Equals keep `b`. Both sides see the same +/// offers and holds, so both agree. +fn preferred>(a: &T, b: &T, receiver: Id) -> bool { + let key = |route: &T| { + let offer = route.offered().expect("an offered route"); + (route.held(), names(&offer.path, receiver), offer.rank()) + }; + key(a) < key(b) } /// The routes one stream carries once an edit of its prefixes from `old` to `new` @@ -725,10 +740,9 @@ fn rekey>( } let mut moved = Vec::new(); for (key, mut routes) in arriving { - let offer = |route: &T| route.offered().expect("an offered route").clone(); let mut best = 0; for (i, (_, route)) in routes.iter().enumerate().skip(1) { - if preferred(&offer(route), &offer(&routes[best].1), receiver) { + if preferred(route, &routes[best].1, receiver) { best = i; } } @@ -739,8 +753,8 @@ fn rekey>( route.view().expect("a carried route").items.extend(items); } // A route the key already carries stays, unless the sender would reflect this one. - let staying = kept.get(&key).and_then(|staying| staying.offered().cloned()); - if staying.is_some_and(|staying| !preferred(&offer(&route), &staying, receiver)) { + let staying = kept.get(&key).filter(|staying| staying.offered().is_some()); + if staying.is_some_and(|staying| !preferred(&route, staying, receiver)) { continue; } if kept.contains_key(&key) { @@ -2309,7 +2323,22 @@ impl Router { } rest += 1; } - Want::Keep => rest += 1, + Want::Keep => { + // The neighbour takes a first DEAD to mean this node holds every route it + // did not learn from the neighbour, so what was sent says so too. + if first && !sent.held() && !child(&sent.offer().expect("sent").path, self.sessions[&peer].hop) + { + let sent = self + .answering + .get_mut(&peer) + .expect("answering") + .tx + .get_mut(key) + .expect("sent"); + sent.hold(); + } + rest += 1; + } end @ (Want::End | Want::Empty) => { let batch = (matches!(end, Want::Empty) && self.covering_view(key.0, &key.1).is_some()).then(|| { diff --git a/rs/moq-net/verify/route/Blind.cfg b/rs/moq-net/verify/route/Blind.cfg index ef0555fa007..cb12ae926fa 100644 --- a/rs/moq-net/verify/route/Blind.cfg +++ b/rs/moq-net/verify/route/Blind.cfg @@ -25,6 +25,7 @@ SPECIFICATION Spec INVARIANTS Truthful + Mirrored NoRevival NoFalseEnd NoRegression diff --git a/rs/moq-net/verify/route/Dead.cfg b/rs/moq-net/verify/route/Dead.cfg index 3a4c0e1994e..6cbc4b87fce 100644 --- a/rs/moq-net/verify/route/Dead.cfg +++ b/rs/moq-net/verify/route/Dead.cfg @@ -26,6 +26,7 @@ SPECIFICATION Spec INVARIANTS Truthful + Mirrored NoRevival NoFalseEnd NoRegression diff --git a/rs/moq-net/verify/route/Drain.cfg b/rs/moq-net/verify/route/Drain.cfg index 00a20754e60..8f74661dd0e 100644 --- a/rs/moq-net/verify/route/Drain.cfg +++ b/rs/moq-net/verify/route/Drain.cfg @@ -25,6 +25,7 @@ SPECIFICATION Spec INVARIANTS Truthful + Mirrored NoRevival NoFalseEnd NoRegression diff --git a/rs/moq-net/verify/route/DrainRing.cfg b/rs/moq-net/verify/route/DrainRing.cfg index 3c1269917b4..99f4beb7d95 100644 --- a/rs/moq-net/verify/route/DrainRing.cfg +++ b/rs/moq-net/verify/route/DrainRing.cfg @@ -28,6 +28,7 @@ SPECIFICATION Spec INVARIANTS Truthful + Mirrored NoRevival NoFalseEnd NoRegression diff --git a/rs/moq-net/verify/route/Early.cfg b/rs/moq-net/verify/route/Early.cfg index 2d6b4fac416..4defd365574 100644 --- a/rs/moq-net/verify/route/Early.cfg +++ b/rs/moq-net/verify/route/Early.cfg @@ -22,6 +22,7 @@ SPECIFICATION Spec INVARIANTS Truthful + Mirrored NoRevival NoFalseEnd NoRegression diff --git a/rs/moq-net/verify/route/Edit.cfg b/rs/moq-net/verify/route/Edit.cfg index 99385990b50..b0bcf3684ba 100644 --- a/rs/moq-net/verify/route/Edit.cfg +++ b/rs/moq-net/verify/route/Edit.cfg @@ -29,6 +29,7 @@ SPECIFICATION Spec INVARIANTS Truthful + Mirrored NoRevival NoFalseEnd NoRegression diff --git a/rs/moq-net/verify/route/EditNarrow.cfg b/rs/moq-net/verify/route/EditNarrow.cfg index f604d0b0b2e..a5af453f665 100644 --- a/rs/moq-net/verify/route/EditNarrow.cfg +++ b/rs/moq-net/verify/route/EditNarrow.cfg @@ -24,6 +24,7 @@ SPECIFICATION Spec INVARIANTS Truthful + Mirrored NoRevival NoFalseEnd NoRegression diff --git a/rs/moq-net/verify/route/EditWiden.cfg b/rs/moq-net/verify/route/EditWiden.cfg index 99317eabd3d..82366c4ec2b 100644 --- a/rs/moq-net/verify/route/EditWiden.cfg +++ b/rs/moq-net/verify/route/EditWiden.cfg @@ -24,6 +24,7 @@ SPECIFICATION Spec INVARIANTS Truthful + Mirrored NoRevival NoFalseEnd NoRegression diff --git a/rs/moq-net/verify/route/HoldCycle.cfg b/rs/moq-net/verify/route/HoldCycle.cfg index 1dc908067b1..b3af2339860 100644 --- a/rs/moq-net/verify/route/HoldCycle.cfg +++ b/rs/moq-net/verify/route/HoldCycle.cfg @@ -27,6 +27,7 @@ SPECIFICATION Spec INVARIANTS Truthful + Mirrored NoRevival NoFalseEnd NoRegression diff --git a/rs/moq-net/verify/route/Lag.cfg b/rs/moq-net/verify/route/Lag.cfg index 0537a6f0bc0..18279bc1f34 100644 --- a/rs/moq-net/verify/route/Lag.cfg +++ b/rs/moq-net/verify/route/Lag.cfg @@ -23,6 +23,7 @@ SPECIFICATION Spec INVARIANTS Truthful + Mirrored NoRevival NoFalseEnd NoRegression diff --git a/rs/moq-net/verify/route/LagWide.cfg b/rs/moq-net/verify/route/LagWide.cfg index a14ae7cd4b2..029c045f88c 100644 --- a/rs/moq-net/verify/route/LagWide.cfg +++ b/rs/moq-net/verify/route/LagWide.cfg @@ -21,6 +21,7 @@ SPECIFICATION Spec INVARIANTS Truthful + Mirrored NoRevival NoFalseEnd NoRegression diff --git a/rs/moq-net/verify/route/Loop.cfg b/rs/moq-net/verify/route/Loop.cfg index c8977766626..62123c8c897 100644 --- a/rs/moq-net/verify/route/Loop.cfg +++ b/rs/moq-net/verify/route/Loop.cfg @@ -23,6 +23,7 @@ SPECIFICATION Spec INVARIANTS Truthful + Mirrored NoRevival NoFalseEnd NoRegression diff --git a/rs/moq-net/verify/route/Overlap.cfg b/rs/moq-net/verify/route/Overlap.cfg index 40c11a87b67..fe11d1bf7ab 100644 --- a/rs/moq-net/verify/route/Overlap.cfg +++ b/rs/moq-net/verify/route/Overlap.cfg @@ -24,6 +24,7 @@ SPECIFICATION Spec INVARIANTS Truthful + Mirrored NoRevival NoFalseEnd NoRegression diff --git a/rs/moq-net/verify/route/Prefer.cfg b/rs/moq-net/verify/route/Prefer.cfg index f1de6970274..39091a62f1a 100644 --- a/rs/moq-net/verify/route/Prefer.cfg +++ b/rs/moq-net/verify/route/Prefer.cfg @@ -24,6 +24,7 @@ SPECIFICATION Spec INVARIANTS Truthful + Mirrored NoRevival NoFalseEnd NoRegression diff --git a/rs/moq-net/verify/route/Ring.cfg b/rs/moq-net/verify/route/Ring.cfg index da2c86c5ec4..4b1eedcf83e 100644 --- a/rs/moq-net/verify/route/Ring.cfg +++ b/rs/moq-net/verify/route/Ring.cfg @@ -23,6 +23,7 @@ SPECIFICATION Spec INVARIANTS Truthful + Mirrored NoRevival NoFalseEnd NoRegression diff --git a/rs/moq-net/verify/route/RouteHandoff.cfg b/rs/moq-net/verify/route/RouteHandoff.cfg index 562851e0ed9..8a53686af0a 100644 --- a/rs/moq-net/verify/route/RouteHandoff.cfg +++ b/rs/moq-net/verify/route/RouteHandoff.cfg @@ -23,6 +23,7 @@ SPECIFICATION Spec INVARIANTS Truthful + Mirrored NoRevival NoFalseEnd NoRegression diff --git a/rs/moq-net/verify/route/RouteHandoff.tla b/rs/moq-net/verify/route/RouteHandoff.tla index 8bba4f34212..dc787779b22 100644 --- a/rs/moq-net/verify/route/RouteHandoff.tla +++ b/rs/moq-net/verify/route/RouteHandoff.tla @@ -398,6 +398,9 @@ Report1(n, s, p, hop) == !.held = FALSE, !.src = w[k].src] [] w[k].kind = "hold" -> [t[k] EXCEPT !.held = TRUE] [] w[k].kind = "end" -> NoSent(t[k].src) + \* p takes a first DEAD to mean every route it did not + \* send this node is held, so what was sent says so too. + [] first /\ ~Child(t[k].path, p) -> [t[k] EXCEPT !.held = TRUE] [] OTHER -> t[k]] out == IF first \/ (gone /\ ends # {}) THEN <> @@ -648,11 +651,14 @@ Recv(n, s, from, msg) == [] OTHER -> Fail(n, s, msg.hop, from, msg.moved, msg.gone) \* Whether the sender would reflect route `x` rather than `y` to node r under -\* one key, as `SrcKey` picks: one not naming r first, then the better rank. -\* Both sides see the same offers, so both agree. -Reflects(x, y, r) == - \/ ~Names(x.path, r) /\ Names(y.path, r) - \/ Names(x.path, r) = Names(y.path, r) /\ Below(Rank(x.path, x.drain), Rank(y.path, y.drain)) +\* one key, as `SrcKey` picks: a live one first, then one not naming r, then +\* the better rank. `Held(e)` says whether an entry is held. Both sides see the +\* same offers and holds (`Mirrored`), so both agree. +Reflects(x, y, r, Held(_)) == + \/ ~Held(x) /\ Held(y) + \/ Held(x) = Held(y) /\ ~Names(x.path, r) /\ Names(y.path, r) + \/ Held(x) = Held(y) /\ Names(x.path, r) = Names(y.path, r) + /\ Below(Rank(x.path, x.drain), Rank(y.path, y.drain)) \* The routes of one stream to node r once an edit of its prefixes from `O` to \* `N` takes effect at its marker (path.rs: `rekey`), with `Off(e)` what a gone @@ -663,30 +669,30 @@ Reflects(x, y, r) == \* is carried, presented at "a", to an added "a", so its floor carries too, \* unless the route "a" already carries is the one the sender would reflect \* there. The rest go, with no ROUTE_END. -Carries(O, N, R, r) == +Carries(O, N, R, r, Held(_)) == /\ "root" \in O /\ "a" \in N /\ R["root"].on - /\ ~R["a"].on \/ Reflects(R["root"], R["a"], r) + /\ ~R["a"].on \/ Reflects(R["root"], R["a"], r, Held) -Rekey(O, N, R, Off(_), r) == +Rekey(O, N, R, Off(_), r, Held(_)) == [k \in Keys |-> IF k = "root" THEN IF "root" \in O /\ "root" \in N THEN R["root"] ELSE IF "a" \in O /\ "root" \in N /\ R["a"].on THEN [R["a"] EXCEPT !.rev = None] ELSE Off(R["root"]) - ELSE IF Carries(O, N, R, r) THEN [R["root"] EXCEPT !.items = Restrict(@, "a")] + ELSE IF Carries(O, N, R, r, Held) THEN [R["root"] EXCEPT !.items = Restrict(@, "a")] ELSE IF R["a"].on /\ (("root" \in O /\ N # {}) \/ ("a" \in O /\ "a" \in N)) THEN R["a"] ELSE Off(R["a"])] \* The keys `Rekey` moved a route between, as <>. -Moved(O, N, R, r) == +Moved(O, N, R, r, Held(_)) == (IF "a" \in O /\ "root" \in N /\ R["a"].on THEN {<<"a", "root">>} ELSE {}) - \cup (IF Carries(O, N, R, r) THEN {<<"root", "a">>} ELSE {}) + \cup (IF Carries(O, N, R, r, Held) THEN {<<"root", "a">>} ELSE {}) \* `edit`: m now asks node n for `set`. It takes effect at once, behind the \* marker n sends first. Answer(s, m, set) == [s EXCEPT !.ans[m] = set, - !.tx[m] = Rekey(s.ans[m], set, s.tx[m], LAMBDA e : NoSent(e.src), m)] + !.tx[m] = Rekey(s.ans[m], set, s.tx[m], LAMBDA e : NoSent(e.src), m, LAMBDA e : e.held)] \* `edited`: m's marker for node n's oldest edit in flight. The routes m sent \* are rekeyed as m rekeyed them, and a route n used or held through m moves @@ -694,11 +700,11 @@ Answer(s, m, set) == Marker(n, s, m) == LET old == s.asked[m] new == Head(s.edits[m]) - mv == Moved(old, new, s.rx[m], n) + mv == Moved(old, new, s.rx[m], n, LAMBDA e : e.state = "held") carry(k) == {x \in mv : x[2] = k /\ s.rt[x[1]].on /\ s.rt[x[1]].next = m /\ ~s.rt[k].on} IN [s EXCEPT !.asked[m] = new, !.edits[m] = Tail(@), - !.rx[m] = Rekey(old, new, s.rx[m], LAMBDA e : NoHeard, n), + !.rx[m] = Rekey(old, new, s.rx[m], LAMBDA e : NoHeard, n, LAMBDA e : e.state = "held"), !.rt = [k \in Keys |-> IF carry(k) # {} THEN s.rt[(CHOOSE x \in carry(k) : TRUE)[1]] ELSE s.rt[k]]] @@ -875,6 +881,17 @@ Truthful == st[n].rx[m][k].on /\ st[n].rx[m][k].rev >= 0 => st[n].rx[m][k].items = Restrict(hist[st[n].rx[m][k].rev + 1], k) +\* With nothing in flight from m to n, what m sent n is what n heard from m, +\* on the same path and held exactly when n holds it, so an edit rekeys both +\* sides alike. +Mirrored == + \A d \in Dirs : d[2] # Origin /\ {d[1], d[2]} \in up /\ q[d] = <<>> => + LET t == st[d[1]].tx[d[2]] + h == st[d[2]].rx[d[1]] + IN \A k \in Keys : + /\ t[k].on = h[k].on + /\ t[k].on => t[k].path = h[k].path /\ (t[k].held <=> h[k].state = "held") + \* A reader never sees a broadcast start again after it saw it end. NoRevival == revivals = 0 diff --git a/rs/moq-net/verify/route/Twice.cfg b/rs/moq-net/verify/route/Twice.cfg index f4e92bbe337..b1191c9713d 100644 --- a/rs/moq-net/verify/route/Twice.cfg +++ b/rs/moq-net/verify/route/Twice.cfg @@ -24,6 +24,7 @@ SPECIFICATION Spec INVARIANTS Truthful + Mirrored NoRevival NoFalseEnd NoRegression From 97864500c4ecffb28f99519392ce044347f2fcc7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 13:29:48 -0800 Subject: [PATCH 09/11] fix(route): the receiver infers a first DEAD's holds by path alone, as the sender does A node qualifies every key a scoped neighbour sends, so its qualified-key child test filed a route it reflected back as held, while the neighbour kept it live. Both sides now decide by the path, which the neighbour can see. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/path.rs | 27 +++++++++++++---- rs/moq-net/src/route/tests/interest.rs | 2 +- rs/moq-net/src/route/tests/scope.rs | 40 ++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 6 deletions(-) diff --git a/rs/moq-net/src/route/path.rs b/rs/moq-net/src/route/path.rs index d0a6a0d9ca2..efd8aaf781a 100644 --- a/rs/moq-net/src/route/path.rs +++ b/rs/moq-net/src/route/path.rs @@ -1447,6 +1447,20 @@ impl Router { (heard.state == State::Live).then_some(&heard.offer) } + /// Whether this node files `from`'s route for `id` under `prefix` as held. + #[cfg(test)] + pub fn heard_held(&self, id: Id, prefix: &Path, from: N) -> Option { + let heard = self.rx.get(&(id, prefix.clone()))?.get(&from)?; + Some(heard.state == State::Held) + } + + /// Whether what this node last sent `peer` for `id` under `prefix` holds it. + #[cfg(test)] + pub fn sent_held(&self, peer: N, id: Id, prefix: &Path) -> Option { + let sent = self.answering.get(&peer)?.tx.get(&(id, prefix.clone()))?; + sent.offer().map(|_| sent.held()) + } + /// Every route this node uses for `id`, by prefix. pub fn routes(&self, id: Id) -> impl Iterator)> { of(&self.routes, id).map(|((_, prefix), route)| (prefix, route)) @@ -1740,14 +1754,16 @@ impl Router { if gone { self.forget_heard(&key, peer); } else { - let is_child = self.is_child(key.0, peer, &self.rx[&key][&peer].offer.path); + let hop = self.hop; let heard = self .rx .get_mut(&key) .and_then(|routes| routes.get_mut(&peer)) .expect("heard"); // A neighbour that holds under this node answers once those behind it have. - if !is_child { + // The neighbour cannot tell whether this node qualified the key, so + // both sides decide by the path alone, as the sender's report does. + if !child(&heard.offer.path, hop) { heard.state = State::Held; } } @@ -1938,9 +1954,10 @@ impl Router { /// The prefix of this node's own route that what it offers a stream answering /// `prefixes` under `prefix` reflects: the route itself inside one of them, and at - /// one the best-ranked live route covering it that does not name the neighbour. Prefer the narrower scope among equal ranks. Without a usable - /// path, reflect a live or held route so a child can answer its loss. Preferring - /// a narrower route regardless of cost can make covering paths oscillate. + /// one the best-ranked live route covering it that does not name the neighbour. + /// Prefer the narrower scope among equal ranks. Without a usable path, reflect a + /// live or held route so a child can answer its loss. Preferring a narrower route + /// regardless of cost can make covering paths oscillate. fn src(&self, id: Id, prefix: &Path, peer: N, prefixes: &BTreeSet) -> Path { if !prefixes.contains(prefix) { return prefix.clone(); diff --git a/rs/moq-net/src/route/tests/interest.rs b/rs/moq-net/src/route/tests/interest.rs index c9dafd01b05..1191dd32468 100644 --- a/rs/moq-net/src/route/tests/interest.rs +++ b/rs/moq-net/src/route/tests/interest.rs @@ -194,7 +194,7 @@ fn an_edit_crossing_a_dead_hop_and_a_hold_settles() { /// cannot use, here one the relay learned through the receiver's own dead session, /// keeps the covering route the receiver used there instead, as the sender would /// reflect it, so the claim stays available across the marker. Found by TLC -/// (`EditSmall.cfg`). +/// (`Edit.cfg`). #[test] fn narrowing_keeps_the_route_the_sender_would_reflect() { let (origin, r1, client) = (0, 1, 2); diff --git a/rs/moq-net/src/route/tests/scope.rs b/rs/moq-net/src/route/tests/scope.rs index 6795c124180..218219a927c 100644 --- a/rs/moq-net/src/route/tests/scope.rs +++ b/rs/moq-net/src/route/tests/scope.rs @@ -1192,3 +1192,43 @@ fn an_upstream_stream_keeps_a_hold_over_an_unexportable_live_source() { ); } } + +/// A node qualifies every key a scoped neighbour sends, so a route it reflected +/// comes back under another key. While doubt keeps that neighbour off another +/// path, it holds the route under this node and keeps it live through a first +/// DEAD, so this node files it live too. +#[test] +fn a_qualified_reflection_stays_live_through_a_first_dead() { + let (origin, relay, scoped, other, spare) = (0, 1, 2, 3, 4); + let mut sim = Sim::with(Rule::PATH, 5); + sim.join(origin, relay, 1); + sim.link( + relay, + scoped, + SimLink::cost(1).grant(Grant { + subject: "ga", + scope: "a", + }), + SimLink::cost(1), + ); + sim.join(origin, other, 5); + sim.join(other, scoped, 5); + sim.join(origin, spare, 5); + sim.join(spare, relay, 5); + sim.originate(origin, 7, 0); + sim.announce(origin, "a/x", 0, None); + assert!(sim.run()); + let key = qualified("ga", 7); + let at = p("a"); + assert_eq!(sim.path(scoped).sent_held(relay, 7, &at), Some(false)); + sim.disconnect(origin, relay); + assert!(sim.flow()); + // The neighbour holds under this node, waiting out its doubt of another path. + assert_eq!(sim.path(scoped).sent_held(relay, 7, &at), Some(false), "{}", mesh(&sim)); + assert_eq!( + sim.path(relay).heard_held(key, &at, scoped), + Some(false), + "{}", + mesh(&sim) + ); +} From f2a95a16e458d03d4803076bc676939e6dbe9fbb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 22:00:39 -0800 Subject: [PATCH 10/11] quest: the m2 triage links the interest set's PR, since it deletes the quest Co-Authored-By: Claude Opus 5.5 --- quest/m2/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quest/m2/README.md b/quest/m2/README.md index a7615b8a708..5d28644fcf8 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -236,7 +236,7 @@ what would need a breaking release had it landed right after the cut: start, and lite-07 by start rewords the draft. Rejected for each: staying in m2. - ✅ Stayed in m2 though not breaking itself: - [the interest set](/quest/m2/cluster-routes/interest-set.md), because + [the interest set](https://github.com/moq-dev/moq/pull/8728), because [the route stream](/quest/m2/cluster-routes/route-stream.md) needs it. The route stream stays because lite-07 would otherwise freeze with one router key per announce stream, which the route model doesn't check, and From 0e156ac396d5acc343df6db656f24be605f76f75 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 22:27:38 -0800 Subject: [PATCH 11/11] fix(route): widening several prefixes carries the route in use from any of them Rekey records every route absorbed into a key, not only the one whose offer it keeps, and the receiver carries the route it was using through the sender from any of them. Before, a cheaper offer through a doubted node won the key, and with no route carried, the claims of the prefix in use went offline. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-route/SKILL.md | 2 +- rs/moq-net/src/route/path.rs | 26 ++++++++++---- rs/moq-net/src/route/tests/interest.rs | 49 ++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 8 deletions(-) diff --git a/.claude/skills/verify-route/SKILL.md b/.claude/skills/verify-route/SKILL.md index f2d9e9164b3..e1d957aeac7 100644 --- a/.claude/skills/verify-route/SKILL.md +++ b/.claude/skills/verify-route/SKILL.md @@ -101,7 +101,7 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `6f8b12729b2bf2ba75994373834cbfbe88ddfcf2`, the base of the interest set in [moq-dev/moq#8728](https://github.com/moq-dev/moq/pull/8728). Run `git log 6f8b12729.. -- rs/moq-net/src/route/path.rs rs/moq-net/src/route/reader.rs` to see what changed since. Prefix and reader indexes make the router's updates incremental; test-only full recomputation checks their result against the complete view. The model recomputes every prefix atomically and leaves that optimization out. `Rank` orders routes as `Offer::rank` does: drain, carried with each offer or known from a DRAIN (`Drains`), then the lowest link priority on the path (`Preferred`), then cost; `GoAway` marks a neighbour's session draining. `SrcKey` mirrors the router's preference for the best-ranked covering live path that does not name its receiver, with the narrower scope breaking ties; `Select` plans live routes and their reflected sources before deciding holds; a held route keeps its last live path, so `SelectKey` holds under a next hop only on the path it was built on and `Want` ends a held stream whose source now has another path; `Advertise1` forwards a newer floor even when the claims did not change. The router keeps doubt until its one-second timer fires (`Lift::Timer`, `Doubt::DEFAULT`); the model's `Collect` leaves it for `Lift`. The comparison-only `Rule::EARLY` lifts doubt once no heard route names its node and is outside this timer-only model. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. +Sync commit: `cc17f589737dbedf156b86a8ba327593f3905074`, the base of the interest set in [moq-dev/moq#8728](https://github.com/moq-dev/moq/pull/8728). Run `git log cc17f5897.. -- rs/moq-net/src/route/path.rs rs/moq-net/src/route/reader.rs` to see what changed since. Prefix and reader indexes make the router's updates incremental; test-only full recomputation checks their result against the complete view. The model recomputes every prefix atomically and leaves that optimization out. `Rank` orders routes as `Offer::rank` does: drain, carried with each offer or known from a DRAIN (`Drains`), then the lowest link priority on the path (`Preferred`), then cost; `GoAway` marks a neighbour's session draining. `SrcKey` mirrors the router's preference for the best-ranked covering live path that does not name its receiver, with the narrower scope breaking ties; `Select` plans live routes and their reflected sources before deciding holds; a held route keeps its last live path, so `SelectKey` holds under a next hop only on the path it was built on and `Want` ends a held stream whose source now has another path; `Advertise1` forwards a newer floor even when the claims did not change. The router keeps doubt until its one-second timer fires (`Lift::Timer`, `Doubt::DEFAULT`); the model's `Collect` leaves it for `Lift`. The comparison-only `Rule::EARLY` lifts doubt once no heard route names its node and is outside this timer-only model. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. | Code | Model | | --- | --- | diff --git a/rs/moq-net/src/route/path.rs b/rs/moq-net/src/route/path.rs index b0f55b4d318..3cf5da46656 100644 --- a/rs/moq-net/src/route/path.rs +++ b/rs/moq-net/src/route/path.rs @@ -663,7 +663,8 @@ struct Rekeyed { routes: BTreeMap, /// The keys whose route it no longer carries there. gone: Vec, - /// Each route that moved, with its offer, to another key: from, to. + /// Each route that moved to another key, as from, to: every route absorbed into + /// the key, the one whose offer it keeps first. moved: Vec<(RouteId, RouteId)>, } @@ -751,10 +752,12 @@ fn rekey>( } } let (from, mut route) = routes.remove(best); + let mut absorbed = Vec::new(); // The claims of the rest stay, though no floor certifies them. - for (_, mut other) in routes { + for (other_from, mut other) in routes { let items = std::mem::take(&mut other.view().expect("a carried route").items); route.view().expect("a carried route").items.extend(items); + absorbed.push(other_from); } // A route the key already carries stays, unless the sender would reflect this one. let staying = kept.get(&key).filter(|staying| staying.offered().is_some()); @@ -765,6 +768,7 @@ fn rekey>( gone.push(key.clone()); } moved.push((from, key.clone())); + moved.extend(absorbed.into_iter().map(|other| (other, key.clone()))); kept.insert(key, route); } Rekeyed { @@ -1327,17 +1331,25 @@ impl Router { self.hear(key, from, heard); } // A route this node used or held through `from` moves with the route it heard, - // so a hold or a doubt carries over rather than letting go of its claims. + // so a hold or a doubt carries over rather than letting go of its claims. Of + // several moving to one key, the one in use goes: only it may stay on a path + // through a doubted node. + let mut carried: BTreeMap> = BTreeMap::new(); for (was, now) in rekeyed.moved { let Some(route) = self.routes.get(&was).filter(|route| route.next() == Some(from)) else { continue; }; - if !self.routes.contains_key(&now) { - let route = route.clone(); - self.scopes.entry(now.0).or_default().insert(now.1.clone()); - self.routes.insert(now, route); + if self.routes.contains_key(&now) { + continue; + } + if carried.get(&now).is_none_or(|kept| !kept.live() && route.live()) { + carried.insert(now, route.clone()); } } + for (now, route) in carried { + self.scopes.entry(now.0).or_default().insert(now.1.clone()); + self.routes.insert(now, route); + } for (id, named) in filed { if of(&self.rx, id).any(|(_, routes)| routes.contains_key(&from)) { self.filed.insert((id, from), named); diff --git a/rs/moq-net/src/route/tests/interest.rs b/rs/moq-net/src/route/tests/interest.rs index 1191dd32468..5c8d384c53d 100644 --- a/rs/moq-net/src/route/tests/interest.rs +++ b/rs/moq-net/src/route/tests/interest.rs @@ -355,3 +355,52 @@ fn a_scope_with_several_regions_asks_for_each() { }] ); } + +/// Widening several prefixes into one keeps the route this node was using at any of +/// them, though the sender reflects a cheaper one from another prefix there: that +/// one crosses a node this node doubts, which only a route already in use may stay +/// on, so carrying no route would take the used prefix's claims offline. +#[test] +fn widening_several_prefixes_keeps_the_route_in_use() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for peer in [2, 3] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + router.edited(peer); + } + router.watch(BTreeSet::from([p("a"), p("b")])).unwrap(); + router.edited(2); + router.route(2, 7, p("b"), Some(path::Reach::Finite(along(&[(7, 0), (2, 1)], 5)))); + router.batch(2, 7, p("b"), vec![(p("b/x"), Some(1))], Some(1)); + router.disconnect([3], Instant::ZERO); + router.route( + 2, + 7, + p("a"), + Some(path::Reach::Finite(along(&[(7, 0), (3, 4), (2, 1)], 1))), + ); + router.batch(2, 7, p("a"), vec![(p("a/x"), Some(1))], Some(1)); + assert!(matches!( + router.route_for(7, &p("b")), + Some(path::Route::Live { next: 2, .. }) + )); + assert!(router.route_for(7, &p("a")).and_then(path::Route::offer).is_none()); + router.events(); + router.watch(BTreeSet::from([Path::empty()])).unwrap(); + router.edited(2); + assert!( + matches!( + router.route_for(7, &Path::empty()), + Some(path::Route::Live { next: 2, .. }) + ), + "{}", + router.dump() + ); + assert!(router.shown()[&(7, p("b/x"))].available); + let events = router.events(); + assert!( + events + .iter() + .all(|event| event.claim != p("b/x") || event.kind != Kind::Offline), + "{events:?}" + ); +}