From 2a81ae4eb11f72bde5919bfef68c3a5c0b75690e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 04:49:54 -0800 Subject: [PATCH 01/14] chore(quest): claim quest/m2/cluster-routes/acceptance.md Quest-Claim: 311e8f5937987b5b From c48e3b4a0c55748a1cdbd4a703ae0356398cd7de Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 05:57:52 -0800 Subject: [PATCH 02/14] test(net): tiered-cluster and LAN partition routes on lite-07 sessions Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/tests/integration/main.rs | 1 + rs/moq-net/tests/integration/route_cluster.rs | 708 ++++++++++++++++++ 2 files changed, 709 insertions(+) create mode 100644 rs/moq-net/tests/integration/route_cluster.rs diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index 6acf754bf36..be2c86c5e10 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -65,6 +65,7 @@ mod relay_upstream_close; mod replacement; mod request_caps; mod route_change; +mod route_cluster; mod route_health; mod routing_contracts; mod segmented_frame; diff --git a/rs/moq-net/tests/integration/route_cluster.rs b/rs/moq-net/tests/integration/route_cluster.rs new file mode 100644 index 00000000000..86344983971 --- /dev/null +++ b/rs/moq-net/tests/integration/route_cluster.rs @@ -0,0 +1,708 @@ +#![cfg(feature = "async")] + +//! Lite-07 routes across a tiered cluster, end to end on real sessions with time mocked. +//! +//! Two regions each run two cores and two edges. Every edge dials both cores of its +//! region and marks them `upstream`, the cores of a region peer with each other, and +//! each core peers with its counterpart in the other region over a priced backbone link. +//! Publishers push to an edge of one region, and readers sit on an edge of the other. + +use crate::route_change::{Delivery, produce_origin, read, timed}; +use crate::support; + +use std::time::Duration; + +use futures::{StreamExt, channel::mpsc}; +use moq_net::{Error, Instance, Timestamp, Version, origin, track}; +use support::Kind; +use support::harness::{MockConnectOptions, MockPair, connect_mock}; + +/// How long a relay holds a route no neighbour answers for. +const HOLD: Duration = Duration::from_secs(10); + +/// How long a closed session's far end is doubted. +const DOUBT: Duration = Duration::from_secs(1); + +/// Frames per group. +const FRAMES: u64 = 4; + +fn lite07() -> Version { + "moq-lite-07-wip".parse().unwrap() +} + +/// Let every route settle, doubts and holds included. +async fn quiesce() { + kio::clock::sleep(HOLD + DOUBT).await; +} + +/// Wait long enough for messages in flight to land, without reaching the doubt. +async fn land() { + kio::clock::sleep(Duration::from_millis(100)).await; +} + +/// Close a session at both ends, as a dropped connection does. +fn cut(pair: &MockPair) { + pair.server.abort(Error::Cancel); + pair.client.abort(Error::Cancel); +} + +/// `dialer` connects to `acceptor`, each publishing and subscribing everything, the way +/// relays link. The dialer marks the link `link` on its side, ranks it `preference`, and +/// prices it `cost`; the acceptor sees a peer. +async fn join( + dialer: &origin::Producer, + link: origin::Link, + acceptor: &origin::Producer, + cost: u64, + preference: i8, +) -> MockPair { + let dialer = dialer.clone().with_link(link); + let acceptor = acceptor.clone().with_link(origin::Link::Peer); + let mut options = MockConnectOptions::new(lite07()); + options.client_publish = Some(dialer.consume().with_hidden(true)); + options.client_subscribe = Some(dialer.with_preference(origin::Preference::new(preference))); + options.server_publish = Some(acceptor.consume().with_hidden(true)); + options.server_subscribe = Some(acceptor); + options.cost = Some(cost); + connect_mock(options).await +} + +/// A client `publisher` pushes everything it publishes to `relay`, over a link it prices +/// `cost`. +async fn push(publisher: &origin::Producer, relay: &origin::Producer, cost: u64) -> MockPair { + let mut options = MockConnectOptions::new(lite07()); + options.client_publish = Some(publisher.consume()); + options.server_subscribe = Some(relay.clone()); + options.cost = Some(cost); + connect_mock(options).await +} + +/// Relays and the sessions between them, each filed under both ends' hops. +#[derive(Default)] +struct Links(Vec<(u64, u64, MockPair)>); + +impl Links { + /// `dialer` connects to `acceptor`; see [`join`]. + async fn join( + &mut self, + dialer: &origin::Producer, + link: origin::Link, + acceptor: &origin::Producer, + cost: u64, + preference: i8, + ) { + let pair = join(dialer, link, acceptor, cost, preference).await; + self.0.push((dialer.hop().get(), acceptor.hop().get(), pair)); + } + + /// Close the session between `a` and `b`, and forget it. + fn cut(&mut self, a: &origin::Producer, b: &origin::Producer) { + let (a, b) = (a.hop().get(), b.hop().get()); + let between = |(x, y, _): &(u64, u64, MockPair)| (*x, *y) == (a, b) || (*x, *y) == (b, a); + let (_, _, pair) = self.0.iter().find(|link| between(link)).expect("no such link"); + cut(pair); + self.0.retain(|link| !between(link)); + } + + /// `node` dies: every session it holds closes. + fn kill(&mut self, node: &origin::Producer) { + let hop = node.hop().get(); + let touches = |(x, y, _): &(u64, u64, MockPair)| *x == hop || *y == hop; + self.0.iter().filter(|link| touches(link)).for_each(|(_, _, pair)| cut(pair)); + self.0.retain(|link| !touches(link)); + } +} + +/// One change a node showed at a path. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Seen { + kind: Kind, + available: bool, + cost: u64, + instance: Option, + at: kio::clock::Instant, +} + +/// Every change a node shows at one path, the current route included. +struct Watch { + rx: mpsc::UnboundedReceiver, +} + +impl Watch { + async fn new(node: &origin::Producer, path: &'static str) -> Self { + let mut routes = node.consume().routes(); + let (tx, rx) = mpsc::unbounded(); + kio::clock::sim::spawn(async move { + while let Some(event) = routes.next().await { + if event.announce.prefix.as_str() != path { + continue; + } + let kind = match event.kind { + origin::RouteKind::Start => Kind::Start, + origin::RouteKind::Update => Kind::Update, + origin::RouteKind::Restart => Kind::Restart, + origin::RouteKind::End => Kind::End, + origin::RouteKind::Offline => Kind::Offline, + }; + let route = &event.announce.route; + let live = matches!(kind, Kind::Start | Kind::Update | Kind::Restart); + let seen = Seen { + kind, + available: route.available(), + cost: route.cost().value(), + instance: live.then(|| route.instance().clone()), + at: kio::clock::now(), + }; + if tx.unbounded_send(seen).is_err() { + break; + } + } + }); + land().await; + Self { rx } + } + + /// What the node showed since the last call. + fn seen(&mut self) -> Vec { + std::iter::from_fn(|| self.rx.try_recv().ok()).collect() + } +} + +fn kinds(seen: &[Seen]) -> Vec { + seen.iter().map(|seen| seen.kind).collect() +} + +fn tagged(name: &str, group: u64, frame: u64) -> Vec { + format!("{name}:{group}.{frame}").into_bytes() +} + +/// The next frame the reader reports, failing on an error or a hang. +async fn next(rx: &mut mpsc::UnboundedReceiver) -> (u64, Vec) { + let (group, frame) = kio::clock::timeout(Duration::from_secs(10), rx.next()) + .await + .expect("reader hung") + .expect("reader ended"); + let (_, payload) = frame.unwrap_or_else(|err| panic!("group {group} failed: {err}")); + (group, payload) +} + +/// Read group `key` in full, as `name` wrote it. +async fn read_group(rx: &mut mpsc::UnboundedReceiver, name: &str, key: u64) { + for frame in 0..FRAMES { + assert_eq!(next(rx).await, (key, tagged(name, key, frame))); + } +} + +/// Subscribe `video` at `path` on `node`, reading every group in full. +async fn subscribe(node: &origin::Producer, path: &str) -> (moq_net::broadcast::Consumer, mpsc::UnboundedReceiver) { + let broadcast = (node.consume().follow(path, None).unwrap().available().await) + .unwrap() + .unwrap(); + let prefs = track::Subscription::default().with_delay(track::Delay::default().with_max(Duration::from_secs(60))); + let sub = broadcast.track("video").unwrap().subscribe(prefs).await.unwrap(); + (broadcast, read(sub)) +} + +/// One publisher of a broadcast, writing frames tagged with its name. +struct Publisher { + name: &'static str, + _origin: origin::Producer, + route: origin::Route, + track: track::Producer, + link: MockPair, +} + +impl Publisher { + /// Publish `path` on a fresh origin `hop`, under `instance` at epoch 0 when given, + /// pushed to `edge` over a link priced `cost`. + async fn new( + name: &'static str, + hop: u64, + path: &str, + instance: Option<&Instance>, + edge: &origin::Producer, + cost: u64, + ) -> Self { + let origin = produce_origin(hop); + let mut options = origin::RouteOptions::hidden(); + if let Some(instance) = instance { + options = options.with_instance(instance.clone(), 0); + } + let route = origin.route(path, options).unwrap(); + let track = route.broadcast().create_track("video", timed()).unwrap(); + route.announce().unwrap(); + let link = push(&origin, edge, cost).await; + Self { + name, + _origin: origin, + route, + track, + link, + } + } + + /// Write group `key` in full. + fn group(&self, key: u64) { + let mut group = self.track.create_group(moq_net::group::Info { start: key }).unwrap(); + for frame in 0..FRAMES { + let timestamp = Timestamp::from_micros(1_000_000 + key * 100_000 + frame * 1_000).unwrap(); + group.write_frame(timestamp, tagged(self.name, key, frame)).unwrap(); + } + group.close().unwrap(); + } +} + +/// The cost of a backbone link between regions. +const BACKBONE: u64 = 100; + +/// What an edge pays for its second core, so its first is preferred. +const SECOND_CORE: u64 = 5; + +/// What a route from a publisher pushing to edge `A[0]` costs at edge `B[0]`, through the +/// first core of each region: the push, the edge's first core, the backbone, and the far +/// edge's first core. +const VIA_FIRST_CORES: u64 = 1 + 1 + BACKBONE + 1; + +/// Two regions `A` (0) and `B` (1), each with cores `[0]` and `[1]` and edges `[0]` and `[1]`. +struct Regions { + cores: [[origin::Producer; 2]; 2], + edges: [[origin::Producer; 2]; 2], + links: Links, +} + +impl Regions { + /// Every edge dials both cores of its region, `[0]` at cost 1 and `[1]` at + /// [`SECOND_CORE`], marking them upstream. The cores of a region peer at cost 1, and + /// core `[i]` of `A` peers with core `[i]` of `B` at [`BACKBONE`]. + async fn new() -> Self { + let cores = [[10, 11], [20, 21]].map(|hops| hops.map(produce_origin)); + let edges = [[12, 13], [22, 23]].map(|hops| hops.map(produce_origin)); + let mut links = Links::default(); + for region in 0..2 { + for edge in &edges[region] { + links.join(edge, origin::Link::Upstream, &cores[region][0], 1, 0).await; + (links.join(edge, origin::Link::Upstream, &cores[region][1], SECOND_CORE, 0)).await; + } + (links.join(&cores[region][0], origin::Link::Peer, &cores[region][1], 1, 0)).await; + } + for (a, b) in cores[0].iter().zip(&cores[1]) { + links.join(a, origin::Link::Peer, b, BACKBONE, 0).await; + } + Self { cores, edges, links } + } + + /// Every relay, cores first. + fn relays(&self) -> impl Iterator { + self.cores.iter().chain(&self.edges).flatten() + } +} + +/// A publisher pushes `live` to an edge of `A`, and a reader on an edge of `B` reads it +/// through both regions' first cores. The publisher withdraws it: every relay, the far +/// edge included, sees it end once, with nothing shown before or after. +#[kio::test] +async fn a_far_edge_reads_a_claim_and_sees_its_withdrawal_once() { + let regions = Regions::new().await; + let publisher = Publisher::new("P", 1, "live", None, ®ions.edges[0][0], 1).await; + quiesce().await; + + let mut watched = Vec::new(); + for relay in regions.relays() { + watched.push(Watch::new(relay, "live").await); + } + let far = ®ions.edges[1][0]; + let (_broadcast, mut rx) = subscribe(far, "live").await; + publisher.group(0); + read_group(&mut rx, "P", 0).await; + + let at_far = watched[6].seen(); + assert!( + matches!(at_far.as_slice(), [Seen { kind: Kind::Start, available: true, cost: VIA_FIRST_CORES, .. }]), + "the far edge reads through the first cores: {at_far:?}" + ); + for watch in &mut watched { + watch.seen(); + } + + publisher.route.withdraw(); + quiesce().await; + for (relay, watch) in regions.relays().zip(&mut watched) { + let seen = watch.seen(); + assert_eq!(kinds(&seen), [Kind::End], "relay {}: {seen:?}", relay.hop()); + } +} + +/// A reader on an edge of `B` reads `live` through the first core of `A`, which then dies. +/// The second core of `A` reaches the publisher's edge on its own, so every relay moves +/// onto a path avoiding the dead core at once: the far edge sees the route updated in +/// place under the same instance, never restarted, ended, or offline, and its reader +/// reads every frame once. +#[kio::test] +async fn losing_a_core_keeps_the_far_edge_reading() { + let mut regions = Regions::new().await; + let publisher = Publisher::new("P", 1, "live", None, ®ions.edges[0][0], 1).await; + quiesce().await; + + let far = regions.edges[1][0].clone(); + let mut watch = Watch::new(&far, "live").await; + let (broadcast, mut rx) = subscribe(&far, "live").await; + publisher.group(0); + read_group(&mut rx, "P", 0).await; + let start = watch.seen(); + assert_eq!(start.len(), 1, "{start:?}"); + assert_eq!(start[0].cost, VIA_FIRST_CORES, "{start:?}"); + + let dying = regions.cores[0][0].clone(); + regions.links.kill(&dying); + land().await; + publisher.group(1); + read_group(&mut rx, "P", 1).await; + quiesce().await; + publisher.group(2); + read_group(&mut rx, "P", 2).await; + + let seen = watch.seen(); + assert!(!seen.is_empty(), "the far edge's route moved off the dead core"); + assert!( + seen.iter() + .all(|seen| seen.kind == Kind::Update && seen.instance == start[0].instance), + "updated in place under one instance: {seen:?}" + ); + assert!( + seen.last().unwrap().available && seen.last().unwrap().cost > VIA_FIRST_CORES, + "settled on a longer path: {seen:?}" + ); + land().await; + assert!(rx.try_recv().is_err(), "trailing delivery"); + drop(broadcast); +} + +/// Two publishers on the two edges of `A` serve `live` under one explicit instance at the +/// same epoch, the first cheaper. A reader on an edge of `B` reads the first's copy until +/// the first becomes unreachable, then reads on through the second's: the far edge keeps +/// one broadcast, sees the route updated in place rather than restarted, and its reader +/// takes the next group from the second with nothing lost or repeated. +#[kio::test] +async fn a_matching_epoch_takes_over_at_the_far_edge() { + let regions = Regions::new().await; + let instance = Instance::random(); + let first = Publisher::new("P1", 1, "live", Some(&instance), ®ions.edges[0][0], 1).await; + let second = Publisher::new("P2", 2, "live", Some(&instance), ®ions.edges[0][1], 10).await; + quiesce().await; + + let far = regions.edges[1][0].clone(); + let mut watch = Watch::new(&far, "live").await; + let (broadcast, mut rx) = subscribe(&far, "live").await; + first.group(0); + second.group(0); + read_group(&mut rx, "P1", 0).await; + let start = watch.seen(); + assert_eq!(kinds(&start), [Kind::Start], "{start:?}"); + + cut(&first.link); + quiesce().await; + second.group(1); + read_group(&mut rx, "P2", 1).await; + + let seen = watch.seen(); + assert!( + seen.iter() + .all(|seen| seen.kind == Kind::Update && seen.instance.as_ref() == Some(&instance)), + "one broadcast, updated in place: {seen:?}" + ); + assert!(seen.last().is_some_and(|seen| seen.available), "{seen:?}"); + let now = (far.consume().follow("live", None).unwrap().available().await) + .unwrap() + .unwrap(); + assert!(now.is_clone(&broadcast), "the far edge kept its broadcast"); + land().await; + assert!(rx.try_recv().is_err(), "trailing delivery"); + drop((first, second)); +} + +/// What the CDN's own link between its relays costs, well above the LAN's. +const CDN_INTERNAL: u64 = 10; + +/// A local cluster beside a CDN: LAN relays `lan[0]` and `lan[1]` peer at cost 1, and each +/// dials its own CDN relay, `cdn[0]` and `cdn[1]`, marking it upstream, while the CDN's +/// relays peer at [`CDN_INTERNAL`]. Without the down-only flag the CDN would reach itself +/// through the cheaper LAN. +struct Lan { + cdn: [origin::Producer; 2], + lan: [origin::Producer; 2], + links: Links, +} + +impl Lan { + async fn new() -> Self { + Self::with_uplinks(origin::Link::Upstream).await + } + + /// [`Self::new`], with each LAN relay marking its CDN link `uplink`. + async fn with_uplinks(uplink: origin::Link) -> Self { + let cdn = [30, 31].map(produce_origin); + let lan = [32, 33].map(produce_origin); + let mut links = Links::default(); + links.join(&cdn[0], origin::Link::Peer, &cdn[1], CDN_INTERNAL, 0).await; + links.join(&lan[0], origin::Link::Peer, &lan[1], 1, 0).await; + for (lan, cdn) in lan.iter().zip(&cdn) { + links.join(lan, uplink, cdn, 1, 0).await; + } + Self { cdn, lan, links } + } + + /// Every relay, the CDN's first. + fn relays(&self) -> impl Iterator { + self.cdn.iter().chain(&self.lan) + } +} + +/// A local publisher on `lan[1]` and a CDN publisher on `cdn[0]`. The LAN partitions, and +/// `lan[0]` reaches the local broadcast through the CDN under the same instance, then +/// moves back onto the LAN in place once it heals. Meanwhile `cdn[1]` reaches the CDN +/// publisher only over the CDN's own expensive link, never through the LAN. +/// +/// PINNED, contradicting the contract ("a node ... holds ... showing the announcements as +/// unavailable"; "the reader's announcement stays, unavailable"): at the cut `lan[0]`'s +/// router holds the route through the doubt on `lan[1]`, but its origin shows the +/// broadcast Offline, then Start through the CDN once the doubt and the origin's +/// two-second retraction quarantine pass. The held claim's only home was the closed LAN +/// session's stream, which `route::Node::close` drops, and nothing re-homes it. The +/// reader's track ends with the error either way, as a held route ends a forwarded +/// subscription, so a fresh one reads on through the CDN. +#[kio::test] +async fn a_partitioned_lan_reaches_itself_through_the_cdn_and_heals() { + let mut lan = Lan::new().await; + let local = Publisher::new("O", 2, "local", None, &lan.lan[1], 1).await; + let _cdn = Publisher::new("Q", 3, "cdn", None, &lan.cdn[0], 1).await; + quiesce().await; + + let mut at_cdn = Watch::new(&lan.cdn[1], "cdn").await; + let mut island = Watch::new(&lan.lan[0], "local").await; + let (_broadcast, mut rx) = subscribe(&lan.lan[0], "local").await; + local.group(0); + read_group(&mut rx, "O", 0).await; + let start = island.seen(); + assert!( + matches!(start.as_slice(), [Seen { kind: Kind::Start, cost: 2, .. }]), + "over the LAN: {start:?}" + ); + let instance = start[0].instance.clone(); + + let [lan0, lan1] = lan.lan.clone(); + let cut_at = kio::clock::now(); + lan.links.cut(&lan0, &lan1); + quiesce().await; + let partitioned = island.seen(); + // The local publisher's push, `lan[1]`'s uplink, the CDN's link, and `lan[0]`'s uplink. + let through_cdn = 1 + 1 + CDN_INTERNAL + 1; + let [offline, back] = partitioned.as_slice() else { + panic!("the island let go and came back: {partitioned:?}"); + }; + assert!( + offline.kind == Kind::Offline && offline.at == cut_at, + "PINNED: let go at the cut, not held: {partitioned:?}" + ); + assert!( + back.kind == Kind::Start + && back.available + && back.cost == through_cdn + && back.instance == instance + && back.at.saturating_duration_since(cut_at) >= DOUBT, + "through the CDN once the doubt lifts: {partitioned:?}" + ); + let ended = kio::clock::timeout(Duration::from_secs(1), rx.next()).await; + assert!( + matches!(ended, Ok(Some((_, Err(_))))), + "the held route ended the track: {ended:?}" + ); + // A fresh subscription joins at the latest group, the one already read. + let (_broadcast, mut rx) = subscribe(&lan.lan[0], "local").await; + read_group(&mut rx, "O", 0).await; + local.group(1); + read_group(&mut rx, "O", 1).await; + + lan.links.join(&lan0, origin::Link::Peer, &lan1, 1, 0).await; + quiesce().await; + let healed = island.seen(); + assert!( + matches!(healed.as_slice(), [Seen { kind: Kind::Update, available: true, cost: 2, .. }]), + "back on the LAN in place: {healed:?}" + ); + local.group(2); + read_group(&mut rx, "O", 2).await; + + let cdn = at_cdn.seen(); + assert!( + matches!(cdn.as_slice(), [Seen { kind: Kind::Start, cost, .. }] if *cost == 1 + CDN_INTERNAL), + "the CDN's own link, throughout: {cdn:?}" + ); +} + +/// The CDN's own link costs more than a detour through the LAN, but every route the LAN +/// learned from the CDN is down-only, so `cdn[1]` reaches the CDN publisher over that +/// link, and once it closes lets the broadcast go rather than reaching `cdn[0]` through +/// the LAN. The control leaves the LAN's uplinks unmarked: `cdn[1]` then takes the LAN +/// from the start and keeps it when the link closes. +#[kio::test] +async fn the_cdn_never_reaches_itself_through_the_lan() { + for marked in [true, false] { + let uplink = if marked { origin::Link::Upstream } else { origin::Link::Peer }; + let mut lan = Lan::with_uplinks(uplink).await; + let _cdn = Publisher::new("Q", 3, "cdn", None, &lan.cdn[0], 1).await; + quiesce().await; + + let mut watch = Watch::new(&lan.cdn[1], "cdn").await; + let start = watch.seen(); + // The publisher's push, then the CDN's own link, or the LAN's three links. + let cost = if marked { 1 + CDN_INTERNAL } else { 1 + 3 }; + assert!( + matches!(start.as_slice(), [Seen { kind: Kind::Start, cost: c, .. }] if *c == cost), + "marked {marked}: {start:?}" + ); + let [cdn0, cdn1] = lan.cdn.clone(); + lan.links.cut(&cdn0, &cdn1); + quiesce().await; + let seen = watch.seen(); + let expected: &[Kind] = if marked { &[Kind::Offline] } else { &[] }; + assert_eq!(kinds(&seen), expected, "marked {marked}: {seen:?}"); + } +} + +/// Every request a claim hears, by path. +fn heard(claim: origin::Claim) -> mpsc::UnboundedReceiver { + let (tx, rx) = mpsc::unbounded(); + kio::clock::sim::spawn(async move { + while let Some(request) = claim.requested().await { + if tx.unbounded_send(request.path().to_string()).is_err() { + return; + } + } + }); + rx +} + +/// One origin claims `live` from `lan[1]` and another `live/a` from `cdn[0]`. On every +/// relay a path under `live/a` resolves to the narrower claim and one under `live/b` to +/// the wider, and each request reaches the claim it resolved to. +#[kio::test] +async fn overlapping_wildcards_resolve_to_the_most_specific_claim_everywhere() { + let lan = Lan::new().await; + let wide = produce_origin(4); + let narrow = produce_origin(5); + let wide_claim = wide.claim("live", origin::RouteOptions::default()).unwrap(); + let narrow_claim = narrow.claim("live/a", origin::RouteOptions::default()).unwrap(); + let (wide_instance, narrow_instance) = (wide_claim.instance(), narrow_claim.instance()); + let mut wide_heard = heard(wide_claim); + let mut narrow_heard = heard(narrow_claim); + let _pushes = [push(&wide, &lan.lan[1], 1).await, push(&narrow, &lan.cdn[0], 1).await]; + quiesce().await; + + let mut subscribing = Vec::new(); + for relay in lan.relays() { + for (under, instance) in [("a", &narrow_instance), ("b", &wide_instance)] { + let path = format!("live/{under}/{}", relay.hop()); + let broadcast = relay.consume().request(path.as_str(), None).await.unwrap(); + assert_eq!(broadcast.info().instance.as_ref(), Some(instance), "{path}"); + // Reading a track sends the request on to the claim. + let track = broadcast.track("video").unwrap(); + subscribing.push(kio::clock::sim::spawn(async move { track.subscribe(None).await.map(drop) })); + } + } + land().await; + let mut wide_paths: Vec<_> = std::iter::from_fn(|| wide_heard.try_recv().ok()).collect(); + let mut narrow_paths: Vec<_> = std::iter::from_fn(|| narrow_heard.try_recv().ok()).collect(); + wide_paths.sort(); + narrow_paths.sort(); + let expected = |under: &str| { + let mut paths: Vec<_> = (lan.relays()).map(|relay| format!("live/{under}/{}", relay.hop())).collect(); + paths.sort(); + paths + }; + assert_eq!(narrow_paths, expected("a")); + assert_eq!(wide_paths, expected("b")); + drop(subscribing); +} + +/// The contract's accepted cost of one selection per publisher and prefix: a relay whose +/// preferred route is down-only exports nothing, so a neighbour cannot use an exportable +/// route that relay also holds. +/// +/// A local publisher on `L1`, whose uplink is `C1`. `L2` reaches it over the LAN at +/// `lan_cost` and through its uplink `C2`, which hears it from `C1`, and `L2` also dials +/// a far CDN relay `C3`, linked to `C2` at 50. At a LAN cost of 13 `L2` prefers the +/// down-only route through `C2` (13 against 14), and `C3` hauls the broadcast across the +/// CDN at 62 although `L2`'s LAN route would reach it at 15. At 11 `L2` prefers the LAN, +/// exports it, and `C3` takes it at 13. Pinned, not endorsed: the epic accepts it, and +/// ranking local peers above a backup uplink avoids it. +#[kio::test] +async fn a_preferred_down_only_route_hides_the_exportable_alternative() { + for (lan_cost, at_far) in [(13, 1 + 1 + CDN_INTERNAL + 50), (11, 1 + 11 + 1)] { + let [c1, c2, c3] = [30, 31, 34].map(produce_origin); + let [l1, l2] = [32, 33].map(produce_origin); + let mut links = Links::default(); + links.join(&c1, origin::Link::Peer, &c2, CDN_INTERNAL, 0).await; + links.join(&c2, origin::Link::Peer, &c3, 50, 0).await; + links.join(&l1, origin::Link::Upstream, &c1, 1, 0).await; + links.join(&l2, origin::Link::Upstream, &c2, 1, 0).await; + links.join(&l2, origin::Link::Upstream, &c3, 1, 0).await; + links.join(&l1, origin::Link::Peer, &l2, lan_cost, 0).await; + let _local = Publisher::new("O", 2, "local", None, &l1, 1).await; + quiesce().await; + + let mut watch = Watch::new(&c3, "local").await; + let seen = watch.seen(); + assert!( + matches!(seen.as_slice(), [Seen { kind: Kind::Start, cost, .. }] if *cost == at_far), + "LAN at {lan_cost}: {seen:?}" + ); + } +} + +/// Two local publishers serve `live`, each under its own minted instance: `O1` on `lan[1]` +/// and, priced to stand by, `O2` on `lan[0]`. `O1` becomes unreachable, and every relay +/// moves the path onto `O2`'s instance as a Restart, never as an Update and never through +/// an End. +#[kio::test] +async fn an_epochless_replacement_restarts_everywhere() { + let lan = Lan::new().await; + let first = Publisher::new("O1", 6, "live", None, &lan.lan[1], 1).await; + let second = Publisher::new("O2", 7, "live", None, &lan.lan[0], 10).await; + quiesce().await; + + let mut watched = Vec::new(); + for relay in lan.relays() { + let mut watch = Watch::new(relay, "live").await; + let start = watch.seen(); + assert!( + matches!(start.as_slice(), [Seen { kind: Kind::Start, .. }]), + "relay {}: {start:?}", + relay.hop() + ); + watched.push((watch, start[0].instance.clone())); + } + + cut(&first.link); + quiesce().await; + for (relay, (watch, before)) in lan.relays().zip(&mut watched) { + let seen = watch.seen(); + let hop = relay.hop(); + assert!( + seen.iter().all(|seen| seen.kind != Kind::End && seen.kind != Kind::Offline), + "relay {hop}: {seen:?}" + ); + let restart = seen.iter().position(|seen| seen.kind == Kind::Restart); + let Some(restart) = restart else { + panic!("relay {hop}: no restart: {seen:?}"); + }; + assert!( + seen[restart].available && seen[restart].instance != *before, + "relay {hop}: {seen:?}" + ); + assert!( + seen[restart..].iter().all(|later| later.instance == seen[restart].instance), + "relay {hop}: {seen:?}" + ); + } + drop(second); +} From aa88b01cafa79408b3df7f5b81168825f1e1a823 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 06:15:14 -0800 Subject: [PATCH 03/14] test(net): route layer acceptance fixtures for topology and prior art Real-session lite-07 fixtures, time mocked: a link flap costs route messages independent of the broadcasts behind it, a stale neighbour in a ring of three cannot revive an ended broadcast, a five-relay ring moves to its backup at once or after the doubt, two relays routing through each other let go on each other's DEAD, and the three prior-art fixtures (a relay noticed a keep-alive apart, a dispute wheel, a primary that returns). A scripted-stream test covers every message naming an unassigned or retired Route ID. Pinned against the contract: a relay whose own session closed with no route left shows its held broadcast offline locally, then starts it again after the retraction quarantine. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/subscriber.rs | 60 ++ rs/moq-net/tests/integration/main.rs | 1 + rs/moq-net/tests/integration/path_routes.rs | 42 +- .../tests/integration/route_acceptance.rs | 549 ++++++++++++++++++ 4 files changed, 634 insertions(+), 18 deletions(-) create mode 100644 rs/moq-net/tests/integration/route_acceptance.rs diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 913db2c7b00..847c786f73f 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -10709,6 +10709,66 @@ mod tests { } } + /// Every well-formed message naming a Route ID closes the session when the neighbour + /// never assigned it (5, past its two routes) or already retired it (0, after its + /// ROUTE_END), as the draft requires: a route's state is the neighbour's to keep in + /// step, so a stray ID means the two ends disagree on which routes live. + #[kio::test] + async fn an_unassigned_or_retired_route_id_closes_the_session() { + type Encode = Box)>; + let encode = |message: lite::AnnounceBroadcast| -> Encode { + Box::new(move |w| message.encode(w, Version::Lite07).unwrap()) + }; + let offer = crate::route::path::Offer { + path: [crate::route::path::Hop { node: 7, session: 0 }].into(), + draining: false, + priority: i8::MAX, + metric: 0, + down_only: false, + rtt: None, + }; + let naming = |id: u64| -> [(&str, Encode); 6] { + [ + ( + "ROUTE_UPDATE", + Box::new(raw(ROUTE_UPDATE, offer_body(&[id], 1, 7, 0, &[]))), + ), + ("ROUTE_HOLD", encode(lite::AnnounceBroadcast::RouteHold { id })), + ("ROUTE_END", encode(lite::AnnounceBroadcast::RouteEnd { id })), + ("ANNOUNCE_FLOOR", Box::new(floor(id, Some(1)))), + ("ANNOUNCE_START", Box::new(claim_on(id, "x"))), + ( + "DEAD", + encode(lite::AnnounceBroadcast::Dead(lite::Dead::Closed { + hop: crate::route::path::Hop { node: 7, session: 0 }, + far: 7, + gone: false, + more: false, + moved: vec![(id, Ok(offer.clone()))], + })), + ), + ] + }; + let end = encode(lite::AnnounceBroadcast::RouteEnd { id: 0 }); + for (id, before, what) in [(5, None, "unassigned"), (0, Some(&end), "retired")] { + for (message, naming) in naming(id) { + let mut script = two_routes(); + let messages: Vec<&dyn Fn(&mut crate::coding::Encoder<'_>)> = + before.into_iter().map(|end| &**end).chain([&*naming]).collect(); + script.extend(announce_script(Version::Lite07, &messages)); + let (mut prefix, _consumer) = root_prefix( + Version::Lite07, + crate::lite::test_transport::ScriptedSession::new(script), + ); + let err = kio::wait(|waiter| prefix.poll_run(waiter)).await.unwrap_err(); + assert!( + matches!(err, Error::ProtocolViolation), + "{message} of a {what} route: {err}" + ); + } + } + } + #[kio::test] async fn a_duplicate_synced_is_refused() { const VERSION: Version = Version::Lite07; diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index 6acf754bf36..2d07b02578d 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -64,6 +64,7 @@ mod rejoin; mod relay_upstream_close; mod replacement; mod request_caps; +mod route_acceptance; mod route_change; mod route_health; mod routing_contracts; diff --git a/rs/moq-net/tests/integration/path_routes.rs b/rs/moq-net/tests/integration/path_routes.rs index a5db1c75806..2eadc0ae948 100644 --- a/rs/moq-net/tests/integration/path_routes.rs +++ b/rs/moq-net/tests/integration/path_routes.rs @@ -20,44 +20,48 @@ use support::harness::{MockConnectOptions, MockPair, connect_mock, peer, peer_wi /// How long a relay holds a route no neighbour answers for, and lets an announce /// stream stall before closing its session. -const HOLD: Duration = Duration::from_secs(10); +pub(super) const HOLD: Duration = Duration::from_secs(10); /// How long a closed session's far end is doubted. -const DOUBT: Duration = Duration::from_secs(1); +pub(super) const DOUBT: Duration = Duration::from_secs(1); -fn lite07() -> Version { +/// The wip version the path rule runs on. +pub(super) fn lite07() -> Version { "moq-lite-07-wip".parse().unwrap() } /// One change a node showed at a path: how long after the cut, its kind, and whether a /// usable route reached the publisher. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct Seen { - at: Duration, - kind: support::Kind, - available: bool, +pub(super) struct Seen { + pub at: Duration, + pub kind: support::Kind, + pub available: bool, + /// The cost of the route shown, which tells its path apart. + pub cost: u64, } /// The changes a node shows at a path. -struct Watch { - rx: mpsc::UnboundedReceiver<(kio::clock::Instant, support::Kind, bool)>, +pub(super) struct Watch { + rx: mpsc::UnboundedReceiver<(kio::clock::Instant, support::Kind, bool, u64)>, } impl Watch { /// What the node showed since the last call, timed from `start`. - fn seen(&mut self, start: kio::clock::Instant) -> Vec { + pub(super) fn seen(&mut self, start: kio::clock::Instant) -> Vec { std::iter::from_fn(|| self.rx.try_recv().ok()) - .map(|(at, kind, available)| Seen { + .map(|(at, kind, available, cost)| Seen { at: at.saturating_duration_since(start), kind, available, + cost, }) .collect() } } /// Watch every change `node` shows at `path`, past what it shows now. -async fn record(node: &origin::Producer, path: &'static str) -> Watch { +pub(super) async fn record(node: &origin::Producer, path: &'static str) -> Watch { let mut routes = node.consume().routes(); let (tx, rx) = mpsc::unbounded(); kio::clock::sim::spawn(async move { @@ -72,7 +76,8 @@ async fn record(node: &origin::Producer, path: &'static str) -> Watch { origin::RouteKind::End => support::Kind::End, origin::RouteKind::Offline => support::Kind::Offline, }; - let seen = (kio::clock::now(), kind, event.announce.route.available()); + let route = &event.announce.route; + let seen = (kio::clock::now(), kind, route.available(), route.cost().value()); if tx.unbounded_send(seen).is_err() { break; } @@ -84,30 +89,31 @@ async fn record(node: &origin::Producer, path: &'static str) -> Watch { watch } -fn withdrawn(seen: &Seen) -> bool { +/// Whether the node let go of the path, ended or offline. +pub(super) fn withdrawn(seen: &Seen) -> bool { matches!(seen.kind, support::Kind::End | support::Kind::Offline) } /// Close a session at both ends, as a dropped connection does. -fn cut(pair: &MockPair) { +pub(super) fn cut(pair: &MockPair) { pair.server.abort(Error::Cancel); pair.client.abort(Error::Cancel); } /// Publish `path` on `node`. -fn publish(node: &origin::Producer, path: &str) -> origin::Route { +pub(super) fn publish(node: &origin::Producer, path: &str) -> origin::Route { let route = node.route(path, origin::RouteOptions::hidden()).unwrap(); route.announce().unwrap(); route } /// Wait long enough for every message in flight to land, without reaching the doubt. -async fn land() { +pub(super) async fn land() { kio::clock::sleep(Duration::from_millis(100)).await; } /// Let the network settle, doubts and holds included. -async fn quiesce() { +pub(super) async fn quiesce() { kio::clock::sleep(HOLD + DOUBT).await; } diff --git a/rs/moq-net/tests/integration/route_acceptance.rs b/rs/moq-net/tests/integration/route_acceptance.rs new file mode 100644 index 00000000000..b34921ddca4 --- /dev/null +++ b/rs/moq-net/tests/integration/route_acceptance.rs @@ -0,0 +1,549 @@ +#![cfg(feature = "async")] + +//! Acceptance of lite-07's path rule on real sessions with time mocked: the topology +//! fixtures of the routes epic's acceptance quest (`quest/m2/cluster-routes/acceptance.md`) +//! and the three from routing prior art (the epic's Plan item 30). +//! +//! The rule's own fixtures live in `path_routes.rs`, whose helpers these share. + +use crate::path_routes::{DOUBT, HOLD, Seen, cut, land, lite07, publish, quiesce, record, withdrawn}; +use crate::route_change::{link, produce_origin}; +use crate::support; + +use std::time::Duration; + +use futures::FutureExt; +use moq_net::{ + Version, origin, + stats::{Message, Registry, Tier, Totals}, +}; +use support::harness::{MockConnectOptions, MockPair, connect_mock, peer, peer_with_latency}; + +/// How a [`join`] is configured beyond [`peer`]'s defaults. +#[derive(Default)] +struct Join { + /// What the link costs both ways, when not the default of 1. + cost: Option, + /// How each side ranks the routes it receives over the link: the first node's, then + /// the second's. + preference: [i8; 2], + /// Where the second node's session counts its control messages. + stats: moq_net::stats::Session, +} + +/// [`peer`] `a` and `b`, configured by `join`. +async fn join(version: Version, a: &origin::Producer, b: &origin::Producer, join: Join) -> MockPair { + let [a, b] = [(a, join.preference[0]), (b, join.preference[1])].map(|(node, preference)| { + node.clone() + .with_link(origin::Link::Peer) + .with_preference(origin::Preference::new(preference)) + }); + let mut options = MockConnectOptions::new(version); + options.client_publish = Some(a.consume().with_hidden(true)); + options.client_subscribe = Some(a); + options.server_publish = Some(b.consume().with_hidden(true)); + options.server_subscribe = Some(b); + options.cost = join.cost; + options.server_stats = join.stats; + connect_mock(options).await +} + +/// Whether a session has closed, without waiting for it to. +fn closed(pair: &MockPair) -> bool { + pair.server.closed().now_or_never().is_some() || pair.client.closed().now_or_never().is_some() +} + +/// The announce stream messages counted under `tier`, both ways. +fn announces(registry: &Registry, tier: &Tier) -> u64 { + (registry.snapshot().control().into_iter()) + .filter(|(row, class, _)| row == tier && *class == Message::Announce) + .map( + |( + _, + _, + Totals { + tx_messages, + rx_messages, + .. + }, + )| tx_messages + rx_messages, + ) + .sum() +} + +/// What a link flap cost, in announce stream messages on the sessions beside the link. +#[derive(Debug, PartialEq, Eq)] +struct Flap { + /// While the link is down: on `B`-`C`, then on `C`-`D`. + down: [u64; 2], + /// Once it reopens: on `B`-`C`, then on `C`-`D`. + up: [u64; 2], +} + +/// `O` publishes `broadcasts` broadcasts and peers with `A`, which peers with `B` and `C`, +/// which peer with each other. `D` reads from `C`. The link from `A` to `C` closes and +/// opens again. Returns what that cost beside the link, after checking that `D` never saw +/// a broadcast end, restart, or start again. +async fn flap(broadcasts: usize) -> Flap { + let version = lite07(); + let [o, a, b, c, d] = [1, 2, 3, 4, 5].map(produce_origin); + let _live: Vec<_> = (0..broadcasts).map(|i| publish(&o, &format!("live/{i}"))).collect(); + let registry = Registry::new(Default::default()); + let tiers = ["b-c", "c-d"].map(|label| Tier::new(moq_net::Path::from_static(label))); + let counted = |tier: &Tier| { + registry + .tier(tier.clone()) + .session(moq_net::stats::SessionInfo::new(moq_net::Path::from_static(""))) + }; + let _o_a = peer(version, &o, &a).await; + let _a_b = peer(version, &a, &b).await; + let a_c = peer(version, &a, &c).await; + let _b_c = join( + version, + &b, + &c, + Join { + stats: counted(&tiers[0]), + ..Default::default() + }, + ) + .await; + let mut options = MockConnectOptions::new(version); + options.server_publish = Some(c.consume()); + options.client_subscribe = Some(d.clone()); + options.server_stats = counted(&tiers[1]); + let _c_d = connect_mock(options).await; + quiesce().await; + + let mut events = support::watch(d.consume().routes()); + land().await; + let shown = support::drain(&mut events); + assert_eq!(shown.len(), broadcasts, "D shows every broadcast"); + + let count = || tiers.each_ref().map(|tier| announces(®istry, tier)); + let before = count(); + cut(&a_c); + quiesce().await; + let after = count(); + let _a_c = peer(version, &a, &c).await; + quiesce().await; + let reopened = count(); + + let churn: Vec<_> = (support::drain(&mut events).into_iter()) + .filter(|(_, kinds)| kinds.iter().any(|kind| *kind != support::Kind::Update)) + .collect(); + assert!(churn.is_empty(), "D keeps every broadcast: {churn:?}"); + Flap { + down: [0, 1].map(|i| after[i] - before[i]), + up: [0, 1].map(|i| reopened[i] - after[i]), + } +} + +/// A link flap with many broadcasts behind one origin costs ROUTE changes per link, not +/// a message per broadcast. `C` loses its link to `A`, holds its route while it doubts `A`, +/// moves onto `B`'s, and moves back once the link reopens: on the sessions beside the link, +/// one broadcast and 64 cost the same messages, and `D` keeps every broadcast throughout. +/// The router's own count, which also bounds the reopened session's snapshot, is +/// `route::tests::handoff::a_link_flap_costs_routes_not_broadcasts`. +#[kio::test] +async fn a_link_flap_costs_routes_not_broadcasts() { + let one = flap(1).await; + let many = flap(64).await; + // Going down, each session beside the link carries two route messages (`C` holds, + // then moves onto `B`'s route); coming back, one (`C` moves back). + let routes = Flap { + down: [2, 2], + up: [1, 1], + }; + assert_eq!(one, routes, "one broadcast"); + assert_eq!(many, routes, "the cost does not follow the broadcasts"); +} + +/// `R1` publishes in a ring of three, where the link from `R2` to `R3` lags by [`STALE`]. +/// The broadcast ends while `R3`'s route and claim are still on their way to `R2`, and +/// `R2`'s to `R3`: each arrives after the withdrawal it predates, and neither relay shows +/// the broadcast again. A claim's view is versioned by its publisher's revision, so an +/// older one never overwrites a newer withdrawal, whichever neighbour sends it. +#[kio::test] +async fn a_stale_neighbour_cannot_revive_an_ended_broadcast() { + const STALE: Duration = Duration::from_secs(2); + let version = lite07(); + let [r1, r2, r3] = [1, 2, 3].map(produce_origin); + let _r1_r2 = peer(version, &r1, &r2).await; + let _r1_r3 = peer(version, &r1, &r3).await; + let _r2_r3 = peer_with_latency(version, &r2, &r3, STALE).await; + let mut watched = Vec::new(); + for relay in [&r2, &r3] { + watched.push(record(relay, "live").await); + } + + let start = kio::clock::now(); + let live = publish(&r1, "live"); + kio::clock::sleep(STALE / 2).await; + drop(live); + quiesce().await; + for (i, watch) in watched.iter_mut().enumerate() { + let seen = watch.seen(start); + let ended = seen.iter().position(withdrawn).expect("ended"); + assert_eq!(ended, seen.len() - 1, "R{}: nothing after the end: {seen:?}", i + 2); + assert!( + seen[ended].at < STALE, + "R{}: before the stale route lands: {seen:?}", + i + 2 + ); + } +} + +/// A ring of five relays, `R0` to `R4`, each link of cost 1, with `P` publishing behind +/// `R0`. Returns `P`'s route and the ring's sessions, `Ri` to `Ri+1` at index `i`. +async fn ring() -> ( + origin::Producer, + origin::Route, + Vec, + Vec, + MockPair, +) { + let version = lite07(); + let p = produce_origin(10); + let live = publish(&p, "live"); + let relays: Vec<_> = (1..=5).map(produce_origin).collect(); + let p_r0 = link(version, &p, &relays[0]).await; + let mut links = Vec::new(); + for i in 0..5 { + links.push(peer(version, &relays[i], &relays[(i + 1) % 5]).await); + } + quiesce().await; + (p, live, relays, links, p_r0) +} + +/// [`ring`]: the link from `R1` to `R2` closes. `R2` preferred the path through `R1`, of +/// cost 3, and its backup runs the other way round the ring, of cost 4, which avoids `R1`, +/// the closed link's far end: it moves there at once (within the origin's update hold, +/// well before a doubt could lift), the broadcast available throughout. Every other +/// session stays up. +#[kio::test] +async fn a_ring_moves_at_once_to_a_backup_avoiding_the_far_end() { + let (_p, _live, relays, links, _p_r0) = ring().await; + let mut watch = record(&relays[2], "live").await; + let start = kio::clock::now(); + cut(&links[1]); + kio::clock::sleep(DOUBT / 2).await; + let seen = watch.seen(start); + assert!(seen.iter().all(|seen| seen.available && !withdrawn(seen)), "{seen:?}"); + assert_eq!(seen.last().map(|seen| seen.cost), Some(4), "on the backup: {seen:?}"); + quiesce().await; + assert_eq!(watch.seen(start), [], "and stays there"); + for (i, pair) in links.iter().enumerate().filter(|(i, _)| *i != 1) { + assert!(!closed(pair), "R{i}-R{} stays up", (i + 1) % 5); + } +} + +/// [`ring`]: the link from `R0` to `R1` closes. Every backup runs the other way round the +/// ring, through `R0`, the closed link's far end, which `R1` cannot tell from a dead relay +/// whose news is still on its way, and which `R1`'s DEAD names to `R2`. `R2` holds the +/// broadcast unavailable, then moves onto its backup, of cost 4, once the doubt lifts, and +/// `R1` onto `R2`'s, of cost 5. Every session of the backup stays up. +/// +/// Pinned, against the contract ("while held, the reader sees them unavailable"): `R1`, +/// whose own session closed, tells `R2` it holds, but its own origin shows the broadcast +/// offline at once, since a closed stream takes what it served with it +/// (`route::node::State::close`). Its readers then see it start again, as a new +/// announcement rather than an update, only once the origin's retraction quarantine ends, +/// a second after the doubt lifted. +#[kio::test] +async fn a_ring_waits_out_the_doubt_for_a_backup_through_the_far_end() { + let (_p, _live, relays, links, _p_r0) = ring().await; + let mut far = record(&relays[1], "live").await; + let mut near = record(&relays[2], "live").await; + let start = kio::clock::now(); + cut(&links[0]); + quiesce().await; + + let seen = near.seen(start); + assert!(!seen.iter().any(withdrawn), "R2 holds, never withdraws: {seen:?}"); + let held = seen.iter().find(|seen| !seen.available).expect("R2 holds meanwhile"); + assert!(held.at < DOUBT, "{seen:?}"); + let moved = seen.last().expect("R2 moves"); + assert!(moved.available && moved.cost == 4, "R2 ends on its backup: {seen:?}"); + assert!(moved.at >= DOUBT, "not before the doubt lifts: {seen:?}"); + + let seen = far.seen(start); + let kinds: Vec<_> = seen.iter().map(|seen| (seen.kind, seen.available)).collect(); + assert_eq!( + kinds, + [(support::Kind::Offline, true), (support::Kind::Start, true)], + "pinned: R1 shows its held broadcast offline: {seen:?}" + ); + assert_eq!(seen[0].at, Duration::ZERO, "{seen:?}"); + assert!(seen[1].at >= 2 * DOUBT && seen[1].cost == 5, "{seen:?}"); + for (i, pair) in links.iter().enumerate().skip(1) { + assert!(!closed(pair), "R{i}-R{} stays up", (i + 1) % 5); + } +} + +/// `A` and `B` each reach `P` directly and peer with each other, over a link lagging by +/// [`LAG`]. `P` dies: each relay's only route left runs through the other, which lost `P` +/// too. Each holds for the other, which learned its route, and the other's DEAD answers +/// it: both let go one lag after the cut, long before a hold's deadline, neither waiting +/// on the other. +#[kio::test] +async fn two_relays_routing_through_each_other_both_let_go() { + const LAG: Duration = Duration::from_millis(200); + let version = lite07(); + let [p, a, b] = [1, 2, 3].map(produce_origin); + let _live = publish(&p, "live"); + let dying = [peer(version, &p, &a).await, peer(version, &p, &b).await]; + let _a_b = peer_with_latency(version, &a, &b, LAG).await; + quiesce().await; + + let mut watched = [record(&a, "live").await, record(&b, "live").await]; + let start = kio::clock::now(); + dying.iter().for_each(cut); + quiesce().await; + for (name, watch) in ["A", "B"].into_iter().zip(&mut watched) { + let seen = watch.seen(start); + assert!(!seen.iter().any(|seen| seen.available), "{name}: {seen:?}"); + assert_eq!( + seen.iter().filter(|seen| withdrawn(seen)).count(), + 1, + "{name}: {seen:?}" + ); + let ended = seen.last().unwrap(); + assert!(withdrawn(ended), "{name}: {seen:?}"); + assert!( + ended.at >= LAG && ended.at < 2 * LAG, + "{name}: let go at {:?}: {seen:?}", + ended.at + ); + } +} + +/// How far apart [`a_vanished_relay_is_noticed_a_keep_alive_apart`]'s neighbours notice. +const KEEP_ALIVE: Duration = Duration::from_secs(3); + +/// Prior art (the epic's Plan item 30): `X` is `P`'s only way out and neighbours `R1`, +/// `R2` and `R3`, which mesh with each other. `X` vanishes: it sends nothing more, and its +/// neighbours notice one at a time, `R1` at once, `R2` half a [`KEEP_ALIVE`] later, `R3` a +/// whole one later, as keep-alives spaced that far apart would. +/// +/// Measured, each move shown 300 ms later through the origin's update hold: no relay moves +/// onto a route through `X` before its own doubt lifts, but the one-second [`DOUBT`] is +/// shorter than the three-second spread, so the doubt lifts while a neighbour still +/// offers a route through the dead relay. `R1` holds from 0 s, moves onto such a route at +/// 1 s, holds again when `R2`'s DEAD names `X` at 1.5 s, and moves onto `R3`'s at 2.5 s, +/// as `R2` does. `R3` notices at 3 s, and all three go offline then, once each: the +/// broadcast is never withdrawn early, but readers on `R1` and `R2` see it turn available +/// twice and once while nothing can serve it, and a request made then follows a route into +/// the dead relay. Everyone lets go only when the last neighbour notices. +/// +/// So the doubt keeps a relay off a dead relay reported one neighbour at a time only when +/// it outlasts the spread of its neighbours' detection: the doubt must follow the +/// keep-alive (the idle timeout that detects a silent peer), or relay keep-alives must +/// shorten below the doubt. Which is the maintainer's to decide; this pins the hunt. +#[kio::test] +async fn a_vanished_relay_is_noticed_a_keep_alive_apart() { + let version = lite07(); + let [p, x, r1, r2, r3] = [1, 2, 3, 4, 5].map(produce_origin); + let _live = publish(&p, "live"); + let p_x = peer(version, &p, &x).await; + let mut noticed = Vec::new(); + for r in [&r1, &r2, &r3] { + noticed.push(peer(version, &x, r).await); + } + let _mesh = [ + peer(version, &r1, &r2).await, + peer(version, &r2, &r3).await, + peer(version, &r1, &r3).await, + ]; + quiesce().await; + + let mut watched = Vec::new(); + for r in [&r1, &r2, &r3] { + watched.push(record(r, "live").await); + } + let start = kio::clock::now(); + // `X` is the client of each of its sessions but `P`'s: nothing it writes leaves it. + p_x.server_transport.hold_writes(); + for pair in ¬iced { + pair.client_transport.hold_writes(); + } + for (i, pair) in noticed.iter().enumerate() { + kio::clock::sleep_until(start.saturating_add(KEEP_ALIVE * i as u32 / 2)).await; + cut(pair); + } + quiesce().await; + + let seen: Vec> = watched.iter_mut().map(|watch| watch.seen(start)).collect(); + for (i, seen) in seen.iter().enumerate() { + let notice = KEEP_ALIVE * i as u32 / 2; + assert!( + seen.iter().all(|seen| seen.at >= notice), + "R{}: nothing before it notices: {seen:?}", + i + 1 + ); + assert!( + !seen + .iter() + .any(|seen| seen.available && !withdrawn(seen) && seen.at < notice + DOUBT), + "R{}: no move through X before the doubt lifts: {seen:?}", + i + 1 + ); + assert_eq!( + seen.iter().filter(|seen| withdrawn(seen)).count(), + 1, + "R{}: {seen:?}", + i + 1 + ); + let ended = seen.last().unwrap(); + assert!( + withdrawn(ended) && ended.at == KEEP_ALIVE, + "R{}: let go when R3 notices: {seen:?}", + i + 1 + ); + } + // Pinned: each move back to available is onto a route through the dead `X`, in + // milliseconds after `X` vanished. + let moves = |seen: &[Seen]| -> Vec { + (seen.iter()) + .filter(|seen| seen.available && !withdrawn(seen)) + .map(|seen| seen.at.as_millis()) + .collect() + }; + assert_eq!(moves(&seen[0]), [1_300, 2_800], "R1: {:?}", seen[0]); + assert_eq!(moves(&seen[1]), [2_800], "R2: {:?}", seen[1]); + assert_eq!(moves(&seen[2]), [], "R3: {:?}", seen[2]); +} + +/// Prior art (the epic's Plan item 30, RFC 4264's dispute wheel): `R1`, `R2` and `R3` +/// each reach `P` directly and form a ring, and each demotes its own link to `P` and +/// promotes the ring link to the next relay, preferring a path through its neighbour to +/// its own. Under BGP's local preference each would take the next relay's direct route +/// and flip back as that relay does the same, forever. A lite-07 path ranks by the lowest +/// preference on it, so every path to `P` carries some relay's demoted link and ranks the +/// same: cost decides, and each settles direct, then stays. +#[kio::test] +async fn a_dispute_wheel_settles() { + let version = lite07(); + let p = produce_origin(10); + let _live = publish(&p, "live"); + let relays: Vec<_> = (1..=3).map(produce_origin).collect(); + let mut watched = Vec::new(); + for relay in &relays { + watched.push(record(relay, "live").await); + } + let start = kio::clock::now(); + let mut links = Vec::new(); + for (i, relay) in relays.iter().enumerate() { + let preference = [0, -1]; + links.push( + join( + version, + &p, + relay, + Join { + preference, + ..Default::default() + }, + ) + .await, + ); + let next = &relays[(i + 1) % 3]; + let preference = [0, 1]; + links.push( + join( + version, + next, + relay, + Join { + preference, + ..Default::default() + }, + ) + .await, + ); + } + quiesce().await; + for (i, watch) in watched.iter_mut().enumerate() { + let seen = watch.seen(start); + let last = seen.last().expect("reached"); + assert!(last.available && last.cost == 1, "R{} settles direct: {seen:?}", i + 1); + } + quiesce().await; + for (i, watch) in watched.iter_mut().enumerate() { + assert_eq!(watch.seen(start), [], "R{} stays", i + 1); + } +} + +/// Prior art (the epic's Plan item 30): `P` reaches `A` over a primary link of cost 1 and +/// `B` over a backup of cost 4, and `A`, `B` and `C` mesh. The primary fails and returns, +/// either within the doubt of `P` or after every relay moved onto the backup: every relay +/// ends back on the primary, at the cost it had before. +/// +/// `A`'s DEAD names `P`, the far end, so `B` and `C` doubt the publisher itself and hold +/// for [`DOUBT`], unavailable, though `B`'s own session to `P` is healthy: every backup runs +/// through `P`. Returning within the doubt, the primary is back before anyone moves. +/// +/// Pinned, against the contract (see +/// [`a_ring_waits_out_the_doubt_for_a_backup_through_the_far_end`]): `A`, left with no +/// route, shows the broadcast offline at once rather than held, and starts it again. +#[kio::test] +async fn a_primary_that_fails_and_returns_takes_every_relay_back() { + for back in [DOUBT / 2, HOLD + DOUBT] { + let version = lite07(); + let [p, a, b, c] = [1, 2, 3, 4].map(produce_origin); + let _live = publish(&p, "live"); + let primary = || Join { + cost: Some(1), + ..Default::default() + }; + let p_a = join(version, &p, &a, primary()).await; + let backup = Join { + cost: Some(4), + ..Default::default() + }; + let _p_b = join(version, &p, &b, backup).await; + let _mesh = [ + peer(version, &a, &b).await, + peer(version, &a, &c).await, + peer(version, &b, &c).await, + ]; + quiesce().await; + + let mut watched = Vec::new(); + for relay in [&a, &b, &c] { + watched.push(record(relay, "live").await); + } + let start = kio::clock::now(); + cut(&p_a); + kio::clock::sleep(back).await; + let _p_a = join(version, &p, &a, primary()).await; + quiesce().await; + let seen: Vec> = watched.iter_mut().map(|watch| watch.seen(start)).collect(); + for ((name, cost), seen) in [("A", 1), ("B", 2), ("C", 2)].into_iter().zip(&seen) { + let last = seen.last().expect("moved"); + assert!( + last.available && !withdrawn(last) && last.cost == cost, + "{name} after {back:?}: back on the primary: {seen:?}" + ); + } + for (name, seen) in ["B", "C"].into_iter().zip(&seen[1..]) { + assert!(!seen.iter().any(withdrawn), "{name} after {back:?}: {seen:?}"); + let held = seen.iter().position(|seen| !seen.available).expect("held"); + let usable = seen.iter().position(|seen| seen.available).expect("usable"); + assert!( + held < usable && seen[usable].at >= DOUBT, + "{name} after {back:?}: {seen:?}" + ); + // Within the doubt, the primary returns before anyone moves onto the backup. + let moved = seen.iter().any(|seen| seen.available && seen.cost != 2); + assert_eq!(moved, back > DOUBT, "{name} after {back:?}: {seen:?}"); + } + let kinds: Vec<_> = seen[0].iter().map(|seen| seen.kind).collect(); + assert_eq!( + kinds[..2], + [support::Kind::Offline, support::Kind::Start], + "pinned: A shows its held broadcast offline: {:?}", + seen[0] + ); + } +} From 28b0301b9a9639f06dd56342cad2ed2b668b1c39 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 06:19:52 -0800 Subject: [PATCH 04/14] test(net): failover acceptance measured on live readers, and a narrower route fails over to a wider one Adds rs/moq-net/tests/integration/route_failover.rs: next-hop moves under a live reader, a new next hop whose ROUTE carries none of the read claim, and moq-dev/moq#4644's live-mesh withdrawal and link-failure tests ported to lite-07 and the path rule. Fixes a reader at a relay losing its subscription when the session serving a claim's narrowest route closes while a wider route still covers it: the origin's reader followed only primary lanes, so it saw no route until the node promoted the wider one, and the claim went Offline, then Start two seconds later. A wider lane now serves, ranked after every primary one. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/origin.rs | 13 +- rs/moq-net/tests/integration/main.rs | 1 + rs/moq-net/tests/integration/mesh_withdraw.rs | 6 +- .../tests/integration/route_failover.rs | 661 ++++++++++++++++++ 4 files changed, 675 insertions(+), 6 deletions(-) create mode 100644 rs/moq-net/tests/integration/route_failover.rs diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index dd5c6ca9784..1aefd76e35e 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -548,7 +548,8 @@ impl RouteInfo { /// Which of the lite-07 route layer's routes to a publisher a route follows: one per /// prefix covering the claim. Only the narrowest live one is primary, which a reader -/// here follows; a request a peer forwards follows the one it names. +/// here follows, falling back on a wider one only while none is; a request a peer +/// forwards follows the one it names. #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub(crate) struct Lane { /// How many segments the route's prefix has, absolute. @@ -885,6 +886,10 @@ fn route_order(path: &Path, entry: &RouteEntry) -> RouteOrder { None => (entry.is_anonymous(), entry.hops.len()), }; ( + // The route layer's narrowest route ahead of a wider one covering the claim. The + // wider one serves only between a closing session taking the narrowest away and the + // node promoting the next, so a reader fails over then rather than seeing an end. + !entry.lane.primary, entry.draining || !entry.available, // The route layer's selected route ahead of its alternates. !entry.lane.selected, @@ -912,6 +917,7 @@ fn route_order(path: &Path, entry: &RouteEntry) -> RouteOrder { /// The key [`route_order`] ranks an entry by: lower wins. type RouteOrder = ( + bool, bool, bool, Reverse, @@ -1588,7 +1594,8 @@ struct Horizon { /// through that node's routes. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub(crate) enum NodeFilter { - /// Every route, of the route layer's only the primary ones (see [`Lane`]). + /// Every route. Of the route layer's, the primary ones rank first (see [`Lane`]), so + /// a wider covering route serves only while no narrower one does. #[default] Any, /// Only the routes this node serves itself. @@ -1605,7 +1612,7 @@ pub(crate) enum NodeFilter { impl NodeFilter { fn admits(self, node: Option, lane: &Lane) -> bool { match self { - Self::Any => node.is_none() || lane.primary, + Self::Any => true, Self::Own => node.is_none(), Self::Route { node: id, depth } => node == Some(id) && lane.depth == depth, } diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index 6acf754bf36..26bdbb913cc 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -65,6 +65,7 @@ mod relay_upstream_close; mod replacement; mod request_caps; mod route_change; +mod route_failover; mod route_health; mod routing_contracts; mod segmented_frame; diff --git a/rs/moq-net/tests/integration/mesh_withdraw.rs b/rs/moq-net/tests/integration/mesh_withdraw.rs index b1c3d775ecc..d0ed803addb 100644 --- a/rs/moq-net/tests/integration/mesh_withdraw.rs +++ b/rs/moq-net/tests/integration/mesh_withdraw.rs @@ -17,7 +17,7 @@ use moq_net::{Version, cache, origin}; use support::harness::{MockConnectOptions, MockPair, connect_mock, peer, peer_with_latency}; use support::{Kind, Rng, drain, watch}; -fn produce_origin(hop: u64) -> origin::Producer { +pub(super) fn produce_origin(hop: u64) -> origin::Producer { produce(hop, origin::Config::default().update_hold) } @@ -138,7 +138,7 @@ async fn partial_mesh_withdraw_then_republish() { /// relays in neighboring PoPs peer and most reach a publisher's relay through /// others. #[rustfmt::skip] -const PRODUCTION: &[(usize, usize)] = &[ +pub(super) const PRODUCTION: &[(usize, usize)] = &[ (0, 20), (0, 22), (0, 29), (1, 3), (1, 12), (1, 14), (1, 21), (2, 20), (2, 26), (2, 32), (3, 8), (3, 11), (3, 15), (3, 16), (3, 17), (3, 18), (3, 21), (3, 22), (3, 23), (3, 24), (3, 29), (3, 30), (3, 31), (4, 7), (4, 20), (4, 22), (4, 25), (4, 29), (5, 20), (5, 27), (5, 33), (6, 23), (6, 24), @@ -159,7 +159,7 @@ const PRODUCTION: &[(usize, usize)] = &[ /// One-way latency in ms for each [`PRODUCTION`] link, from the great-circle /// distance between its PoPs at 200 km/ms over a path 1.5 times as long, plus 1 ms. #[rustfmt::skip] -const PRODUCTION_LATENCY_MS: &[u64] = &[ +pub(super) const PRODUCTION_LATENCY_MS: &[u64] = &[ 10, 7, 7, 50, 30, 51, 50, 9, 13, 13, 5, 4, 6, 8, 3, 4, 1, 50, 6, 4, 50, 6, 4, 50, 63, 58, 50, 58, 16, 5, 5, 10, 10, 10, 10, 12, 1, 12, 6, 3, 5, 8, 8, 5, 4, 43, 5, 4, 43, 6, 11, 8, 49, 22, 23, 6, 8, 49, 22, 23, 3, 5, 4, 41, 103, 103, 60, 91, 91, 59, 64, 59, 64, 6, 9, 8, 3, 6, 6, 4, 48, 4, 3, diff --git a/rs/moq-net/tests/integration/route_failover.rs b/rs/moq-net/tests/integration/route_failover.rs new file mode 100644 index 00000000000..2dbf8707cb2 --- /dev/null +++ b/rs/moq-net/tests/integration/route_failover.rs @@ -0,0 +1,661 @@ +#![cfg(feature = "async")] + +//! Failover measured on a live broadcast while next hops change, on lite-07. +//! +//! A publisher writes a frame every [`STEP`] while relays lose and gain the sessions its +//! route crosses. Eventual route recovery alone does not show a move is seamless, so each +//! fixture asserts on the frames its readers get as well as on what their route streams +//! show, and reports what the moves cost on the wire: per-path retractions (End or +//! Offline), re-announcements (Start or Restart after the first), and the announce-stream +//! messages each session sent, which carry every ROUTE, DEAD, and claim batch. +//! +//! The mesh fixtures port moq-dev/moq#4644's live-graph withdrawal and failover tests, +//! which ran on lite-06 under a hold-down, to the path rule. + +use crate::mesh_withdraw::{PRODUCTION, PRODUCTION_LATENCY_MS}; +use crate::route_change::{payload, produce_origin, timed}; +use crate::support; + +use std::collections::BTreeMap; +use std::time::Duration; + +use futures::channel::mpsc; +use moq_net::{Error, Pattern, Patterns, Timestamp, Version, group, origin, stats, track}; +use support::Kind; +use support::harness::{MockConnectOptions, MockPair, connect_mock, peer_with_latency}; + +fn lite07() -> Version { + "moq-lite-07-wip".parse().unwrap() +} + +/// How often a publisher writes a frame. +const STEP: Duration = Duration::from_millis(20); + +/// How often a publisher writes a frame to the mesh's 33 readers: often enough that a +/// reader's gap shows, rarely enough that the mesh fixtures stay cheap. +const MESH_STEP: Duration = Duration::from_millis(100); + +/// Frames per group. +const FRAMES: u64 = 5; + +/// How long a closed session's far end is doubted. +const DOUBT: Duration = Duration::from_secs(1); + +/// How long a relay holds a route no neighbour answers for. +const HOLD: Duration = Duration::from_secs(10); + +/// One change a node showed at a path: how long after the start, its kind, and whether a +/// usable route reached the publisher. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Seen { + at: Duration, + kind: Kind, + available: bool, +} + +/// Every change a node shows at a path, from now on. +struct Watch { + rx: mpsc::UnboundedReceiver, +} + +impl Watch { + fn new(node: &origin::Producer, path: &str, start: kio::clock::Instant) -> Self { + let path = path.to_string(); + let mut routes = node.consume().routes(); + let (tx, rx) = mpsc::unbounded(); + kio::clock::sim::spawn(async move { + while let Some(event) = routes.next().await { + if event.announce.prefix.as_str() != path { + continue; + } + let kind = match event.kind { + origin::RouteKind::Start => Kind::Start, + origin::RouteKind::Update => Kind::Update, + origin::RouteKind::Restart => Kind::Restart, + origin::RouteKind::End => Kind::End, + origin::RouteKind::Offline => Kind::Offline, + }; + let seen = Seen { + at: kio::clock::now().saturating_duration_since(start), + kind, + available: event.announce.route.available(), + }; + if tx.unbounded_send(seen).is_err() { + break; + } + } + }); + Self { rx } + } + + /// What the node showed since the last call. + fn seen(&mut self) -> Vec { + std::iter::from_fn(|| self.rx.try_recv().ok()).collect() + } +} + +/// What a route stream showed of one path: retractions, re-announcements, updates, and the +/// longest stretch it went without a usable route. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct Churn { + retractions: usize, + reannouncements: usize, + updates: usize, + dark: Duration, +} + +impl Churn { + /// Tally `seen`, which follows a moment the path was shown available, until `end`. + fn of(seen: &[Seen], end: Duration) -> Self { + let mut churn = Self::default(); + let mut dark_since = None; + for seen in seen { + match seen.kind { + Kind::End | Kind::Offline => churn.retractions += 1, + Kind::Start | Kind::Restart => churn.reannouncements += 1, + Kind::Update => churn.updates += 1, + } + let live = seen.available && !matches!(seen.kind, Kind::End | Kind::Offline); + match (live, dark_since) { + (false, None) => dark_since = Some(seen.at), + (true, Some(since)) => { + churn.dark = churn.dark.max(seen.at - since); + dark_since = None; + } + _ => {} + } + } + if let Some(since) = dark_since { + churn.dark = churn.dark.max(end.saturating_sub(since)); + } + churn + } +} + +/// A broadcast's `video` track, written one frame every step, each payload naming the +/// frame's index. +struct Feed { + track: track::Producer, + /// How often it writes a frame. + step: Duration, + group: Option, + /// When each frame was written, by index, since the start. + written: Vec, + start: kio::clock::Instant, +} + +impl Feed { + fn new(track: track::Producer, start: kio::clock::Instant, step: Duration) -> Self { + Self { + track, + step, + group: None, + written: Vec::new(), + start, + } + } + + /// Write `count` frames, one every step. + async fn run(&mut self, count: u64) { + for _ in 0..count { + let index = self.written.len() as u64; + if index % FRAMES == 0 { + if let Some(group) = self.group.take() { + group.close().unwrap(); + } + self.group = Some(self.track.append_group(index / FRAMES).unwrap()); + } + let group = self.group.as_mut().unwrap(); + let timestamp = Timestamp::from_millis(index * self.step.as_millis() as u64).unwrap(); + group + .write_frame(timestamp, payload(index / FRAMES, index % FRAMES)) + .unwrap(); + self.written + .push(kio::clock::now().saturating_duration_since(self.start)); + kio::clock::sleep(self.step).await; + } + } +} + +/// A frame a reader got, by index, and when; or the error that ended its subscription. +type Got = moq_net::Result<(u64, Duration)>; + +/// Read every frame of `video` at `path` through `node`, following it from the start. +async fn reader(node: &origin::Producer, path: &str, start: kio::clock::Instant) -> mpsc::UnboundedReceiver { + let remote = node.consume().request(path, None).await.unwrap(); + let prefs = track::Subscription::default().with_delay(track::Delay::default().with_max(Duration::from_secs(60))); + let mut sub = remote.track("video").unwrap().subscribe(prefs).await.unwrap(); + let (tx, rx) = mpsc::unbounded(); + kio::clock::sim::spawn(async move { + let _keep = remote; + loop { + let mut group = match sub.recv_group().await { + Ok(Some(group)) => group, + Ok(None) => return, + Err(err) => { + let _ = tx.unbounded_send(Err(err)); + return; + } + }; + loop { + match group.read_frame().await { + Ok(Some(frame)) => { + let at = kio::clock::now().saturating_duration_since(start); + let text = String::from_utf8(frame.payload.to_vec()).unwrap(); + let (key, index) = text.split_once('.').unwrap(); + let index = key.parse::().unwrap() * FRAMES + index.parse::().unwrap(); + if tx.unbounded_send(Ok((index, at))).is_err() { + return; + } + } + Ok(None) => break, + Err(err) => { + let _ = tx.unbounded_send(Err(err)); + return; + } + } + } + } + }); + rx +} + +/// What a reader got of a feed: frames it missed, frames out of order or twice, the error +/// that ended it, and the longest any frame took to reach it. +#[derive(Debug, Default)] +struct Liveness { + lost: u64, + misordered: u64, + ended: Option, + worst: Duration, +} + +impl Liveness { + fn of(rx: &mut mpsc::UnboundedReceiver, feed: &Feed) -> Self { + let mut live = Self::default(); + let mut next = 0; + while let Ok(got) = rx.try_recv() { + match got { + Ok((index, at)) => { + match index.cmp(&next) { + std::cmp::Ordering::Less => live.misordered += 1, + std::cmp::Ordering::Equal => next += 1, + std::cmp::Ordering::Greater => { + live.lost += index - next; + next = index + 1; + } + } + live.worst = live.worst.max(at.saturating_sub(feed.written[index as usize])); + } + Err(err) => live.ended = Some(err), + } + } + live.lost += feed.written.len() as u64 - next; + live + } + + /// Every frame arrived, once and in order, and the subscription stands. + fn seamless(&self) -> bool { + self.lost == 0 && self.misordered == 0 && self.ended.is_none() + } +} + +/// Have `to` pull everything `from` publishes, over a link `to` prices at `cost`, counting +/// what `from` sends under `name`. +struct Links { + registry: stats::Registry, +} + +impl Links { + fn new() -> Self { + Self { + registry: stats::Registry::new(Default::default()), + } + } + + fn options(&self, name: &'static str, from: &origin::Producer, to: &origin::Producer) -> MockConnectOptions { + let mut options = MockConnectOptions::new(lite07()); + options.server_publish = Some(from.consume()); + options.client_subscribe = Some(to.clone()); + options.server_stats = self + .registry + .tier(stats::Tier::new(moq_net::Path::from_static(name))) + .session(stats::SessionInfo::new(moq_net::Path::from_static(""))); + options + } + + async fn link(&self, name: &'static str, from: &origin::Producer, to: &origin::Producer, cost: u64) -> MockPair { + let mut options = self.options(name, from, to); + options.cost = Some(cost); + connect_mock(options).await + } + + /// Announce-stream messages each link sent, by name. + fn sent(&self) -> BTreeMap { + let mut sent = BTreeMap::new(); + for (tier, class, totals) in self.registry.snapshot().control() { + if class == stats::Message::Announce { + *sent.entry(tier.as_str().to_string()).or_default() += totals.tx_messages; + } + } + sent + } +} + +/// What each link sent between `before` and `after`. +fn delta(before: &BTreeMap, after: &BTreeMap) -> BTreeMap { + (after.iter()) + .map(|(name, count)| (name.clone(), count - before.get(name).copied().unwrap_or(0))) + .filter(|(_, count)| *count > 0) + .collect() +} + +/// Close a session at both ends, as a dropped connection does. +fn cut(pair: &MockPair) { + pair.server.abort(Error::Cancel); + pair.client.abort(Error::Cancel); +} + +/// How long the moves in [`next_hop_moves_keep_a_live_reader`] take to land: past the +/// origin's update hold, and the new next hop's upstream latency both ways. +const MOVE: u64 = 75; + +/// `P` publishes `live` to `A` and `B`, and the relay `R` pulls both, preferring `A`. A +/// reader on `D`, one hop past `R`, follows `live` while `R`'s route to `P` moves four +/// times between next hops, each a move to a usable route with the same instance: +/// +/// 1. `C` joins, cheaper than `A`, though its own upstream to `P` lags 150 ms each way, +/// so its ROUTE reaches `R` well after `A`'s; +/// 2. `C`'s session to `R` closes, and `R` falls back to `A`, whose path avoids `C`; +/// 3. `A`'s session to `R` closes, and `R` falls back to `B`; +/// 4. `A` reconnects, and `R` moves back to it. +/// +/// Proves the reader gets every frame once, in order, with its subscription intact; that +/// `R` and `D` each show the moves as updates in place, never a retraction, a restart, or +/// an unavailable route; and that each move costs `D` one ROUTE message. +#[kio::test] +async fn next_hop_moves_keep_a_live_reader() { + let links = Links::new(); + let [p, a, b, c, r, d] = [1, 2, 3, 4, 5, 6].map(produce_origin); + let broadcast = p.route("live", origin::RouteOptions::hidden()).unwrap(); + let track = broadcast.broadcast().create_track("video", timed()).unwrap(); + broadcast.announce().unwrap(); + let _p_a = links.link("p-a", &p, &a, 1).await; + let _p_b = links.link("p-b", &p, &b, 1).await; + let a_r = links.link("a-r", &a, &r, 10).await; + let _b_r = links.link("b-r", &b, &r, 100).await; + let _r_d = links.link("r-d", &r, &d, 1).await; + kio::clock::sleep(Duration::from_secs(1)).await; + + let start = kio::clock::now(); + let mut at_r = Watch::new(&r, "live", start); + let mut at_d = Watch::new(&d, "live", start); + let mut feed = Feed::new(track, start, STEP); + let mut rx = reader(&d, "live", start).await; + feed.run(MOVE).await; + let before = links.sent(); + + let mut options = links.options("p-c", &p, &c); + options.latency = Duration::from_millis(150); + let _p_c = connect_mock(options).await; + let c_r = links.link("c-r", &c, &r, 1).await; + feed.run(MOVE).await; + cut(&c_r); + feed.run(MOVE).await; + cut(&a_r); + feed.run(MOVE).await; + let _a_r = links.link("a-r2", &a, &r, 10).await; + feed.run(MOVE).await; + kio::clock::sleep(Duration::from_secs(1)).await; + + let sent = delta(&before, &links.sent()); + let live = Liveness::of(&mut rx, &feed); + let end = kio::clock::now().saturating_duration_since(start); + let (seen_r, seen_d) = (at_r.seen(), at_d.seen()); + let (churn_r, churn_d) = (Churn::of(&seen_r[1..], end), Churn::of(&seen_d[1..], end)); + println!("next hop moves: reader {live:?}; R {churn_r:?}; D {churn_d:?}; sent {sent:?}"); + + assert!(live.seamless(), "{live:?}"); + for (node, seen) in [("R", &seen_r), ("D", &seen_d)] { + assert_eq!(seen[0].kind, Kind::Start, "{node}: {seen:?}"); + assert!( + seen[1..].iter().all(|seen| seen.kind == Kind::Update && seen.available), + "{node}: {seen:?}" + ); + assert_eq!(seen.len(), 5, "{node}: one update per move: {seen:?}"); + } + assert_eq!(sent.get("r-d"), Some(&4), "one ROUTE message per move: {sent:?}"); + drop(broadcast); +} + +/// `P` publishes `live` and `other`, and the relay `R` pulls both from `A`. `C` asks `P` +/// only for `other`, as a client does for what its readers watch, so its route to `P` +/// covers `other` alone. `C` then joins `R`, cheaper than `A`: its ROUTE to `P` reaches `R` +/// while its announce set carries no `live`. Readers on `R` and on `D`, one hop past it, +/// follow both broadcasts. Then `C`'s session to `R` closes. +/// +/// Proves the new next hop moves only what it announced: `other` onto `C`, while `live` +/// stays on `A` untouched. Then `other`, served by `C`'s narrower route, fails over at once +/// to `A`'s wider one, which covers it all along: an update in place, never a retraction, +/// and every reader keeps every frame. Before the fix, `R`'s readers followed only the +/// narrowest route, so the session closing left them none until the node promoted `A`'s: +/// `R` showed `other` Offline, then Start again two seconds later, and its reader there +/// lost the subscription. +#[kio::test] +async fn a_narrower_next_hop_fails_over_to_a_wider_route() { + const PATHS: [&str; 2] = ["live", "other"]; + let links = Links::new(); + let [p, a, c, r, d] = [1, 2, 3, 4, 5].map(produce_origin); + let start = kio::clock::now(); + let mut feeds = Vec::new(); + let mut broadcasts = Vec::new(); + for path in PATHS { + let broadcast = p.route(path, origin::RouteOptions::hidden()).unwrap(); + let track = broadcast.broadcast().create_track("video", timed()).unwrap(); + feeds.push(Feed::new(track, start, STEP)); + broadcast.announce().unwrap(); + broadcasts.push(broadcast); + } + let _p_a = links.link("p-a", &p, &a, 1).await; + let _a_r = links.link("a-r", &a, &r, 10).await; + let _r_d = links.link("r-d", &r, &d, 1).await; + let mut options = links.options("p-c", &p, &c); + options.client_discovers_all = false; + let _p_c = connect_mock(options).await; + let _c_reads_other = c + .consume() + .scope("other", &Patterns::from(Pattern::all())) + .unwrap() + .routes(); + kio::clock::sleep(Duration::from_secs(1)).await; + + let mut watched = Vec::new(); + let mut readers = Vec::new(); + for (index, path) in PATHS.into_iter().enumerate() { + for (name, node) in [("R", &r), ("D", &d)] { + watched.push((path, name, Watch::new(node, path, start))); + readers.push((index, name, reader(node, path, start).await)); + } + } + let run = async |feeds: &mut Vec, count| { + futures::future::join_all(feeds.iter_mut().map(|feed| feed.run(count))).await; + }; + run(&mut feeds, MOVE).await; + let before = links.sent(); + let c_r = links.link("c-r", &c, &r, 1).await; + run(&mut feeds, MOVE).await; + let joined = links.sent(); + cut(&c_r); + run(&mut feeds, MOVE).await; + kio::clock::sleep(Duration::from_secs(1)).await; + let end = kio::clock::now().saturating_duration_since(start); + + let lives: Vec<_> = (readers.iter_mut()) + .map(|(index, node, rx)| (PATHS[*index], *node, Liveness::of(rx, &feeds[*index]))) + .collect(); + let seens: Vec<_> = (watched.iter_mut()) + .map(|(path, node, watch)| (*path, *node, watch.seen())) + .collect(); + for ((path, node, live), (_, _, seen)) in lives.iter().zip(&seens) { + println!( + "narrower next hop, {path} at {node}: reader {live:?}; {:?}", + Churn::of(&seen[1..], end) + ); + } + let sent = (delta(&before, &joined), delta(&joined, &links.sent())); + println!("narrower next hop: sent joining {:?}, closing {:?}", sent.0, sent.1); + + for (path, node, live) in &lives { + assert!(live.seamless(), "{path} at {node}: {live:?}"); + } + for (path, node, seen) in &seens { + assert_eq!(seen[0].kind, Kind::Start, "{path} at {node}: {seen:?}"); + assert!( + seen[1..].iter().all(|seen| seen.kind == Kind::Update && seen.available), + "{path} at {node}: {seen:?}" + ); + let moves = match *path { + // `C` never announced it, so nothing moved. + "live" => 0, + // Onto `C`, then back onto `A`. + _ => 2, + }; + assert_eq!(seen.len(), 1 + moves, "{path} at {node}: {seen:?}"); + } + drop(broadcasts); +} + +/// The publishers the mesh fixtures try, as many as `partial_mesh_withdraw_retracts_once` +/// does: each costs a round of the whole mesh, about a second of debug-build CPU for a +/// withdrawal and four for a failure. +const PUBLISHERS: [usize; 5] = [5, 13, 22, 26, 32]; + +/// moq.pro's 34-relay mesh on lite-07 with no link latency: each relay in [`PUBLISHERS`] +/// publishes a broadcast, then withdraws it. +/// +/// Ports moq-dev/moq#4644's `live_withdraw_retracts_once`, which accepted updates before +/// the retraction while a relay stepped through paths it held. The path rule drops every +/// route through the withdrawing node at once, so nothing steps: proves every relay shows +/// the withdrawal as one retraction and nothing else. With the links' estimated latency, +/// this is `mesh_withdraw::partial_mesh_withdraw_retracts_once`. +#[kio::test] +async fn live_mesh_withdraw_retracts_once_without_latency() { + let nodes: Vec<_> = (1..=34).map(crate::mesh_withdraw::produce_origin).collect(); + let mut pairs = Vec::new(); + for &(a, b) in PRODUCTION { + pairs.push(peer_with_latency(lite07(), &nodes[a], &nodes[b], Duration::ZERO).await); + } + let mut watched: Vec<_> = nodes + .iter() + .map(|node| support::watch(node.consume().routes())) + .collect(); + + for publisher in PUBLISHERS { + let name = format!("room/{publisher}"); + let broadcast = nodes[publisher] + .route(name.clone(), origin::RouteOptions::hidden()) + .unwrap(); + broadcast.announce().unwrap(); + kio::clock::sim::idle().await; + for (relay, watched) in watched.iter_mut().enumerate() { + let updates = support::drain(watched); + assert_eq!( + updates.get(&name).map(Vec::as_slice), + Some(&[Kind::Start][..]), + "publisher {publisher}, relay {relay}" + ); + } + + drop(broadcast); + kio::clock::sim::idle().await; + let mut hunted = Vec::new(); + for (relay, watched) in watched.iter_mut().enumerate() { + let updates = support::drain(watched); + match updates.get(&name) { + Some(kinds) if kinds == &[Kind::End] => {} + kinds => hunted.push((relay, kinds.cloned())), + } + } + assert!(hunted.is_empty(), "publisher {publisher}: {hunted:?}"); + } +} + +/// moq.pro's 34-relay mesh on lite-07 at its estimated latencies, a reader on every relay. +/// For each publisher in [`PUBLISHERS`], every link of its relay closes but the slowest, +/// while the broadcast stays live, and then the links come back. +/// +/// Ports moq-dev/moq#4644's `live_link_failure_fails_over`, which bounded the longest any +/// relay went without a path by two hold-downs. Proves every relay serves the broadcast +/// again with at most one retraction and one re-announcement each (no hunting), and every +/// reader on a relay whose route stayed usable gets every frame. +/// +/// Pins behaviour the contract's continuity goal contradicts, pending a decision: every +/// route left crosses the publisher's relay, the far end of each closed link, which every +/// relay that heard a DEAD doubts for [`DOUBT`]. Each such relay holds the broadcast with +/// no usable route, its neighbours are in the same state and answer with nothing, so the +/// holds end and the relays let go before the doubt lifts: the broadcast goes Offline, +/// then starts again once the doubt lifts, after the origin's two-second quarantine +/// (moq-dev/moq#5375). The router alone does the same: Unavailable, Offline, Start. +#[kio::test] +async fn live_mesh_link_failure_keeps_presence() { + let version = lite07(); + let nodes: Vec<_> = (1..=34).map(crate::mesh_withdraw::produce_origin).collect(); + let mut links: Vec> = Vec::new(); + for (&(a, b), &ms) in PRODUCTION.iter().zip(PRODUCTION_LATENCY_MS) { + let latency = Duration::from_millis(ms); + links.push(Some(peer_with_latency(version, &nodes[a], &nodes[b], latency).await)); + } + + let mut worst = Duration::ZERO; + let (mut dark_relays, mut retracting, mut lost_readers) = (0, 0, 0); + for publisher in PUBLISHERS { + let name = format!("room/{publisher}"); + let start = kio::clock::now(); + let broadcast = nodes[publisher] + .route(name.clone(), origin::RouteOptions::hidden()) + .unwrap(); + let track = broadcast.broadcast().create_track("video", timed()).unwrap(); + let mut feed = Feed::new(track, start, MESH_STEP); + broadcast.announce().unwrap(); + kio::clock::sleep(Duration::from_secs(1)).await; + let mut watched: Vec<_> = nodes.iter().map(|node| Watch::new(node, &name, start)).collect(); + let mut readers = Vec::new(); + for (relay, node) in nodes.iter().enumerate() { + if relay != publisher { + readers.push((relay, reader(node, &name, start).await)); + } + } + feed.run(5).await; + + // Cut every link of the publisher's relay but its slowest. + let touching: Vec<_> = (0..PRODUCTION.len()) + .filter(|&index| PRODUCTION[index].0 == publisher || PRODUCTION[index].1 == publisher) + .collect(); + let kept = *touching + .iter() + .max_by_key(|&&index| PRODUCTION_LATENCY_MS[index]) + .unwrap(); + let cut_links: Vec<_> = touching.into_iter().filter(|&index| index != kept).collect(); + for &index in &cut_links { + cut(links[index].as_ref().unwrap()); + links[index] = None; + } + feed.run(((HOLD + DOUBT).as_millis() / MESH_STEP.as_millis()) as u64) + .await; + // The last frames cross the mesh. + kio::clock::sleep(Duration::from_secs(1)).await; + let end = kio::clock::now().saturating_duration_since(start); + + let mut dark = Vec::new(); + for (relay, watch) in watched.iter_mut().enumerate() { + let seen = watch.seen(); + assert!( + seen.first() + .is_some_and(|seen| seen.kind == Kind::Start && seen.available), + "publisher {publisher}, relay {relay}: {seen:?}" + ); + let churn = Churn::of(&seen[1..], end); + assert!( + churn.retractions <= 1 && churn.reannouncements <= churn.retractions, + "publisher {publisher}, relay {relay} hunted: {seen:?}" + ); + let last = seen.last().unwrap(); + assert!( + last.available && !matches!(last.kind, Kind::End | Kind::Offline), + "publisher {publisher}, relay {relay}: {seen:?}" + ); + worst = worst.max(churn.dark); + retracting += churn.retractions; + if seen[1..] + .iter() + .any(|seen| !seen.available || seen.kind == Kind::Offline) + { + dark.push(relay); + } + } + dark_relays += dark.len(); + for (relay, rx) in &mut readers { + let live = Liveness::of(rx, &feed); + if !live.seamless() { + lost_readers += 1; + assert!( + dark.contains(relay), + "publisher {publisher}: relay {relay} kept a usable route but its reader got {live:?}" + ); + } + } + + drop(broadcast); + for &index in &cut_links { + let (a, b) = PRODUCTION[index]; + let latency = Duration::from_millis(PRODUCTION_LATENCY_MS[index]); + links[index] = Some(peer_with_latency(version, &nodes[a], &nodes[b], latency).await); + } + kio::clock::sleep(HOLD + DOUBT).await; + } + println!( + "live mesh link failure over {} publishers: {dark_relays} relays went unavailable, {retracting} retracted and \ + announced again, {lost_readers} readers lost their subscription, longest without a path {worst:?}", + PUBLISHERS.len() + ); +} From 45e129e750095f8f91298d82e8db96c34bc74623 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 06:28:51 -0800 Subject: [PATCH 05/14] test(net): a claim covering a mount's target continues beneath the mount point The worker's answer arrives beneath the mount exactly as it does unmounted, on lite-06 and lite-07, reusing claim_continues' worker. Co-Authored-By: Claude Opus 5.5 --- .../tests/integration/claim_continues.rs | 18 ++--- rs/moq-net/tests/integration/mount.rs | 70 ++++++++++++++++--- .../tests/integration/route_failover.rs | 2 +- 3 files changed, 72 insertions(+), 18 deletions(-) diff --git a/rs/moq-net/tests/integration/claim_continues.rs b/rs/moq-net/tests/integration/claim_continues.rs index 80097ae9856..3dd88d12a5e 100644 --- a/rs/moq-net/tests/integration/claim_continues.rs +++ b/rs/moq-net/tests/integration/claim_continues.rs @@ -42,20 +42,20 @@ async fn link(version: Version, from: &origin::Producer, to: &origin::Producer, } /// Let every task settle. Time is paused, so this returns once the runtime is idle. -async fn settle() { +pub(super) async fn settle() { kio::clock::sleep(Duration::from_millis(500)).await; } /// A worker claiming `claim`, answering each request with an output whose `video` track /// carries the worker's name and how many outputs it made before. -struct Worker { - claim: Rc, +pub(super) struct Worker { + pub(super) claim: Rc, outputs: Rc>>, - origin: origin::Producer, + pub(super) origin: origin::Producer, } impl Worker { - fn new(hop: u64, claim: &str, name: &'static str) -> Self { + pub(super) fn new(hop: u64, claim: &str, name: &'static str) -> Self { let origin = produce_origin(hop); let claim = Rc::new(origin.claim(claim, origin::RouteOptions::default()).unwrap()); let outputs = Rc::new(RefCell::new(Vec::new())); @@ -82,7 +82,7 @@ impl Worker { } /// How many outputs the worker was asked for. - fn served(&self) -> usize { + pub(super) fn served(&self) -> usize { self.outputs.borrow().len() } @@ -94,7 +94,7 @@ impl Worker { } /// Every announce event already delivered, as `(kind, prefix, instance)`. -async fn drain(announced: &mut origin::Routes) -> Vec<(&'static str, String, Instance)> { +pub(super) async fn drain(announced: &mut origin::Routes) -> Vec<(&'static str, String, Instance)> { let mut events = Vec::new(); while let Ok(event) = kio::clock::timeout(Duration::from_secs(1), announced.next()).await { let (kind, announce) = match event.expect("announce ended") { @@ -125,7 +125,7 @@ async fn drain(announced: &mut origin::Routes) -> Vec<(&'static str, String, Ins } /// The events without their instances. -fn kinds(events: &[(&'static str, String, Instance)]) -> Vec<(&'static str, String)> { +pub(super) fn kinds(events: &[(&'static str, String, Instance)]) -> Vec<(&'static str, String)> { events.iter().map(|(kind, prefix, _)| (*kind, prefix.clone())).collect() } @@ -144,7 +144,7 @@ async fn first(origin: &origin::Producer, path: &str) -> Option { /// The first frame at `path` through `origin`. Unless the first subscription through the /// claim `continues`, it resets, and the next one reads. -async fn watch(continues: bool, origin: &origin::Producer, path: &str) -> String { +pub(super) async fn watch(continues: bool, origin: &origin::Producer, path: &str) -> String { if !continues { assert_eq!(first(origin, path).await, None, "the first subscription continued"); settle().await; diff --git a/rs/moq-net/tests/integration/mount.rs b/rs/moq-net/tests/integration/mount.rs index 59f59000080..c10df460ccc 100644 --- a/rs/moq-net/tests/integration/mount.rs +++ b/rs/moq-net/tests/integration/mount.rs @@ -176,12 +176,62 @@ async fn relayed(version: &str) { ); } +/// `W -> R -> V`: `W`'s worker claims `tenant-a`, which covers the mount's target, and +/// answers a request for `tenant-a/public/cam` with that exact path under the claim's +/// instance. `V` reads `R` through tenant B's mounted handle, or plain when `mounted` is +/// false, and asks for the camera by the name its handle gives it. +/// +/// Proves the answer arrives beneath the mount point exactly as it does unmounted, as +/// `claim_continues::chain` pins it: the claim starts at the mount point, the answer +/// starts beneath it under the lent instance, and the first subscription continues on +/// lite-07 while an older wire's resets once. +async fn claimed(version: &str, mounted: bool) { + use crate::claim_continues::{Worker, drain, kinds, settle, watch}; + + let version: Version = version.parse().unwrap(); + let worker = Worker::new(1, "tenant-a", "w"); + let relay = produce_origin(2); + let viewer = produce_origin(3); + let _upstream = link(version, worker.origin.consume(), &relay).await; + let (handle, claim, answer) = match mounted { + true => (tenant(&relay, "tenant-b"), "shared", "shared/cam"), + false => (relay.consume(), "tenant-a", "tenant-a/public/cam"), + }; + let _downstream = link(version, handle, &viewer).await; + settle().await; + + let mut routes = viewer.consume().routes(); + assert_eq!( + kinds(&drain(&mut routes).await), + [("start", claim.to_string())], + "{version}" + ); + + let continues = version.to_string() == "moq-lite-07-wip"; + assert_eq!(watch(continues, &viewer, answer).await, "w0", "{version}"); + assert_eq!(worker.served(), 1, "{version}"); + + let events = drain(&mut routes).await; + assert_eq!(kinds(&events), [("start", answer.to_string())], "{version}"); + if continues { + assert_eq!(events[0].2, worker.claim.instance(), "{version}"); + } + + // A late snapshot starts what stands. + let mut late = viewer.consume().routes(); + assert_eq!( + kinds(&drain(&mut late).await), + [("start", claim.to_string()), ("start", answer.to_string())], + "{version}: a late snapshot" + ); +} + macro_rules! mounted_tests { - ($($name:ident: $test:ident, $version:literal,)*) => { + ($($name:ident: $test:ident($($arg:expr),*),)*) => { $( #[kio::test] async fn $name() { - kio::clock::timeout(TEST_TIMEOUT, $test($version)) + kio::clock::timeout(TEST_TIMEOUT, $test($($arg),*)) .await .expect("timed out"); } @@ -190,10 +240,14 @@ macro_rules! mounted_tests { } mounted_tests! { - mounted_lite06: mounted, "moq-lite-06", - mounted_lite07: mounted, "moq-lite-07-wip", - rooted_inside_lite06: rooted_inside, "moq-lite-06", - rooted_inside_lite07: rooted_inside, "moq-lite-07-wip", - relayed_lite06: relayed, "moq-lite-06", - relayed_lite07: relayed, "moq-lite-07-wip", + mounted_lite06: mounted("moq-lite-06"), + mounted_lite07: mounted("moq-lite-07-wip"), + rooted_inside_lite06: rooted_inside("moq-lite-06"), + rooted_inside_lite07: rooted_inside("moq-lite-07-wip"), + relayed_lite06: relayed("moq-lite-06"), + relayed_lite07: relayed("moq-lite-07-wip"), + claimed_unmounted_lite06: claimed("moq-lite-06", false), + claimed_unmounted_lite07: claimed("moq-lite-07-wip", false), + claimed_beneath_the_mount_lite06: claimed("moq-lite-06", true), + claimed_beneath_the_mount_lite07: claimed("moq-lite-07-wip", true), } diff --git a/rs/moq-net/tests/integration/route_failover.rs b/rs/moq-net/tests/integration/route_failover.rs index 2dbf8707cb2..0b372b4bb6b 100644 --- a/rs/moq-net/tests/integration/route_failover.rs +++ b/rs/moq-net/tests/integration/route_failover.rs @@ -159,7 +159,7 @@ impl Feed { async fn run(&mut self, count: u64) { for _ in 0..count { let index = self.written.len() as u64; - if index % FRAMES == 0 { + if index.is_multiple_of(FRAMES) { if let Some(group) = self.group.take() { group.close().unwrap(); } From 889df725b86996bf0ab231b1fcf8541ef35f3c4e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 06:37:18 -0800 Subject: [PATCH 06/14] fix(net): a scope widened to the root keeps the routes it held When a lite-07 session's limit widened from one region to the root, the relay reopened its announce request above the region and the peer's claims moved to a route at another prefix, under a bare identity. The replaced request dropped what it served before the new one served it, so readers at the relay saw Offline then Start. The old request's served claims now stay in the origin until the replacement's first outputs are in. Adds the grant-union cluster test that found it and a focused auth test. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/subscriber.rs | 19 ++- rs/moq-net/src/lite/subscriber/routed.rs | 13 ++ rs/moq-net/tests/integration/auth.rs | 34 ++++ rs/moq-net/tests/integration/route_cluster.rs | 157 +++++++++++++++++- 4 files changed, 219 insertions(+), 4 deletions(-) diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 913db2c7b00..c6c47a7020f 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -2739,6 +2739,15 @@ impl AnnouncePrefix { } } + /// Let go of this request, all but the claims its route layer serves, which stay in + /// the origin until the caller drops them. + fn retire(mut self: Box) -> Option { + match &mut self.state { + PrefixState::Run(step) => step.run.routed.as_mut().map(routed::Routed::retire), + _ => None, + } + } + /// Fails the session on any error, [`Error::TooManyRequests`] included: a peer /// announcing more than the session allows loses the session. fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { @@ -2908,17 +2917,23 @@ impl AnnouncePrefix { match stream.reader.poll_decode_maybe::(waiter) { Poll::Ready(Ok(Some(announce))) => { if matches!(announce, lite::AnnounceBroadcast::Synced) { + // This request now carries what the one it replaces did. The old + // stream closes first, so the node hands this one its claims, but + // what it served leaves the origin only once this one serves it: + // a claim this request files under another route prefix or + // identity, such as a region widened to the root, would otherwise + // be let go in between. + let retired = self.replacing.take().and_then(|old| old.retire()); // What the initial set put in the node is served before the // snapshot lets readers see the origin synced. if let Some(routed) = &mut run.routed { routed.outputs(&self.subscriber); } + drop(retired); if std::mem::replace(&mut run.synced, true) { return Poll::Ready(Err(Error::ProtocolViolation)); } self.snapshot = None; - // This request now carries what the one it replaces did. - self.replacing = None; continue; } if run.initial > 0 { diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 01bdfa9cecf..ced5e8804f0 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -120,6 +120,11 @@ struct ServedClaim { /// A served claim's identity, its path, and the prefix of the route it is served along. type ServedKey = (route::Id, Path, Path); +/// What a replaced request's stream served, kept in the origin until dropped. +pub(super) struct Retired { + _served: HashMap, +} + /// The route layer one lite-07 announce stream feeds. pub(super) struct Routed { node: node::Node, @@ -677,6 +682,14 @@ impl Routed { Ok(()) } + /// Take what the stream serves, for the caller to let go of once the request replacing + /// this one serves it too. + pub(super) fn retire(&mut self) -> Retired { + let mut served = std::mem::take(&mut self.served); + served.extend(self.previous.drain()); + Retired { _served: served } + } + /// Move what the stream holds onto a budget of its own with the session's cap, while a /// request replacing it takes the same claims over on the session's. The session's /// count is then the replacement's alone, and a peer flooding this stream fails it diff --git a/rs/moq-net/tests/integration/auth.rs b/rs/moq-net/tests/integration/auth.rs index 811dc051ca2..d8e3a192c46 100644 --- a/rs/moq-net/tests/integration/auth.rs +++ b/rs/moq-net/tests/integration/auth.rs @@ -863,6 +863,40 @@ async fn a_scope_change_reopens_without_letting_go(version: &'static str) { .expect("timed out"); } +/// Widening the limit to the root reopens the relay's request there and files the peer's +/// claims under bare identities, along a route at another prefix. The new request takes +/// `a/x` over in place, with no let go between the two. +#[kio::test] +async fn lite_07_a_scope_widened_to_the_root_reopens_without_letting_go() { + within(async { + let publisher = produce_origin(2); + let relay = produce_origin(1); + let pair = connect(Options { + version: Some(LITE_07), + client_publish: Some(publisher.clone()), + server_subscribe: Some(relay.clone()), + ..Default::default() + }) + .await; + pair.server.auth().authorize(&grant(&["a"], &[])); + + let mut routes = relay.consume().routes(); + let a = publisher.route("a/x", moq_net::origin::RouteOptions::hidden()).unwrap(); + a.announce().unwrap(); + kio::clock::sleep(Duration::from_secs(1)).await; + assert_eq!(kinds(&mut routes, "a/x"), [origin::RouteKind::Start]); + + pair.server.auth().authorize(&grant(&[""], &[])); + kio::clock::sleep(Duration::from_secs(1)).await; + assert!( + (kinds(&mut routes, "a/x").iter()).all(|kind| *kind == origin::RouteKind::Update), + "the widening let go of a/x" + ); + }) + .await + .expect("timed out"); +} + /// A session holding all the announce slots it may has no room to hear its peer's claims /// twice, so the old request gives its slots to the new one during a scope change, /// rather than the handover running the peer past its cap. diff --git a/rs/moq-net/tests/integration/route_cluster.rs b/rs/moq-net/tests/integration/route_cluster.rs index 86344983971..8ab1afdb496 100644 --- a/rs/moq-net/tests/integration/route_cluster.rs +++ b/rs/moq-net/tests/integration/route_cluster.rs @@ -13,9 +13,9 @@ use crate::support; use std::time::Duration; use futures::{StreamExt, channel::mpsc}; -use moq_net::{Error, Instance, Timestamp, Version, origin, track}; +use moq_net::{Error, Instance, Pattern, Patterns, Timestamp, Version, auth, origin, track}; use support::Kind; -use support::harness::{MockConnectOptions, MockPair, connect_mock}; +use support::harness::{MockConnectOptions, MockPair, connect_mock, now, spawn}; /// How long a relay holds a route no neighbour answers for. const HOLD: Duration = Duration::from_secs(10); @@ -706,3 +706,156 @@ async fn an_epochless_replacement_restarts_everywhere() { } drop(second); } + +/// `dialer` dials `acceptor` as an uplink, the way [`join`] does, with the acceptor taking +/// the tokens the dialer presents through the returned requests. +async fn tokened_uplink(dialer: &origin::Producer, acceptor: &origin::Producer) -> (MockPair, auth::Requests) { + let version = lite07(); + let mut transport = moq_net::transport::memory::Config::default(); + transport.protocol = Some(version.alpn().to_string()); + let (client_transport, server_transport) = support::mock::create_mock_session_pair(transport); + let dialer = dialer.clone().with_link(origin::Link::Upstream); + let acceptor = acceptor.clone().with_link(origin::Link::Peer); + let client = moq_net::Client::new() + .with_versions(version.into()) + .with_publisher(dialer.consume().with_hidden(true)) + .with_subscriber(dialer.clone()); + let server = moq_net::Server::new() + .with_versions(version.into()) + .with_publisher(acceptor.consume().with_hidden(true)) + .with_subscriber(acceptor); + let client_fut = async { + let (session, driver) = (client.connect(now(), client_transport.clone()).await).expect("client handshake"); + spawn(driver); + session + }; + let server_fut = async { + let mut handshake = (server.accept_request(now(), server_transport.clone()).await).expect("server handshake"); + let requests = handshake.requests(); + let (session, driver) = handshake.ok().await.expect("server accept"); + spawn(driver); + (session, requests) + }; + let (client, (server, requests)) = futures::join!(client_fut, server_fut); + let pair = MockPair { + client, + server, + client_transport, + server_transport, + client_discovery: Some(dialer.consume().routes()), + }; + (pair, requests) +} + +/// A grant of the `publish` subtrees, or of everything for `**`, subscribing to +/// everything and lapsing at `expires`. +fn grant(publish: &[&str], expires: Option) -> auth::Grant { + let mut grant = auth::Grant::all(); + grant.publish = (publish.iter()) + .map(|prefix| match *prefix { + "**" => Pattern::all(), + prefix => Pattern::subtree(prefix).unwrap(), + }) + .collect::(); + grant.expires = expires; + grant +} + +/// Whether a route only moved in place: each change an Update that keeps it available. +/// Refiling it under another identity or route prefix is an Update, never a let go. +fn moved(seen: &[Seen]) -> bool { + seen.iter().all(|seen| seen.kind == Kind::Update && seen.available) +} + +/// A LAN relay `L` dials its uplink `C1`, which grants its connection credential publish +/// rights to `local` alone, and `C1` peers with `C2`. A local publisher on `L` serves +/// `local/x`, inside that scope, and `other/z`, outside it. `L` presents a second token +/// that `C1` grants publish rights to `**`, until the token expires and `C1` revokes it. +/// The union crosses the scoped boundary each way: `other/z` appears at both CDN relays +/// once the wide token lands and goes Offline once it expires, as the narrowed floor +/// cannot certify a withdrawal outside `local`. `local/x` only moves in place across both +/// changes, refiled from a qualified identity to a bare one and back with no End or +/// Offline, and still ends when its publisher withdraws it. Widening to the root once +/// let `local/x` go at `C1` while its request reopened above `local`. +#[kio::test] +async fn a_grant_union_crosses_the_scoped_boundary_each_way() { + let [c1, c2, l] = [30, 31, 32].map(produce_origin); + let mut links = Links::default(); + links.join(&c1, origin::Link::Peer, &c2, CDN_INTERNAL, 0).await; + let (uplink, mut requests) = tokened_uplink(&l, &c1).await; + // The connection credential's grant, set the way a relay sets it: moq-net widens this + // scope by every grant the acceptor issues, and floors certify only inside it. + uplink.server.auth().authorize(&grant(&["local"], None)); + let wide_expires = kio::clock::now().saturating_add(Duration::from_secs(30)); + kio::clock::sim::spawn(async move { + let mut held = Vec::new(); + while let Some(request) = requests.next().await { + match request.token().as_ref() { + b"wide" => { + let issued = request.accept(grant(&["**"], Some(wide_expires))); + // Revoking a token's grant once it expires is the acceptor's job. + kio::clock::sim::spawn(async move { + kio::clock::sleep_until(wide_expires).await; + issued.revoke(moq_net::SessionError::Expired, "expired"); + }); + } + _ => held.push(request.accept(grant(&["local"], None))), + } + } + }); + let publisher = produce_origin(2); + let inside = publisher.route("local/x", origin::RouteOptions::hidden()).unwrap(); + inside.announce().unwrap(); + let _push = push(&publisher, &l, 1).await; + quiesce().await; + + let mut watched = Vec::new(); + for relay in [&c1, &c2] { + watched.push((Watch::new(relay, "local/x").await, Watch::new(relay, "other/z").await)); + } + for (inside, outside) in &mut watched { + let (inside, outside) = (inside.seen(), outside.seen()); + assert!( + matches!(inside.as_slice(), [Seen { kind: Kind::Start, available: true, .. }]) && outside.is_empty(), + "scoped: inside {inside:?} outside {outside:?}" + ); + } + + let wide = uplink.client.auth().add("wide").unwrap(); + wide.answered().await.unwrap(); + // A dialer announcing outside its union closes its session, so `other/z` waits on the + // wide grant. + let outside = publisher.route("other/z", origin::RouteOptions::hidden()).unwrap(); + outside.announce().unwrap(); + kio::clock::sleep(Duration::from_secs(5)).await; + for (inside, outside) in &mut watched { + let (inside, outside) = (inside.seen(), outside.seen()); + assert!( + moved(&inside) && matches!(outside.as_slice(), [Seen { kind: Kind::Start, available: true, .. }]), + "widened: inside {inside:?} outside {outside:?}" + ); + } + + // The narrowed floor certifies nothing outside `local`, so `other/z` is let go + // (Offline), not ended: its publisher never withdrew it. + kio::clock::sleep_until(wide_expires).await; + quiesce().await; + for (inside, outside) in &mut watched { + let (inside, outside) = (inside.seen(), kinds(&outside.seen())); + assert!( + moved(&inside) && outside == [Kind::Offline], + "expired: inside {inside:?} outside {outside:?}" + ); + } + assert_eq!(uplink.client_transport.close_reason(), None, "the session outlived the token"); + + // Inside the scope the floor still certifies a withdrawal. + inside.withdraw(); + kio::clock::sleep(Duration::from_secs(1)).await; + for (inside, outside) in &mut watched { + let (inside, outside) = (kinds(&inside.seen()), outside.seen()); + assert!(inside == [Kind::End] && outside.is_empty(), "withdrawn: {inside:?}"); + } + drop((wide, uplink, inside, outside, links)); +} + From a06deba940f08b09c09293aa87d00140d015bdfc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 06:48:21 -0800 Subject: [PATCH 07/14] test(net): a client holds routes only to origins meeting its interest Session fixtures for the epic's Plan item 15 on lite-07: a client listing one prefix of a relay never hears an origin outside it, a request outside that interest still resolves through the relay, and an origin leaving the interest ends its route behind its floor, End where the relay certifies the request and Offline where its grant knows only part of it. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/tests/integration/route_cluster.rs | 254 +++++++++++++++++- 1 file changed, 241 insertions(+), 13 deletions(-) diff --git a/rs/moq-net/tests/integration/route_cluster.rs b/rs/moq-net/tests/integration/route_cluster.rs index 8ab1afdb496..07615d9b5fd 100644 --- a/rs/moq-net/tests/integration/route_cluster.rs +++ b/rs/moq-net/tests/integration/route_cluster.rs @@ -6,6 +6,10 @@ //! region and marks them `upstream`, the cores of a region peer with each other, and //! each core peers with its counterpart in the other region over a priced backbone link. //! Publishers push to an edge of one region, and readers sit on an edge of the other. +//! +//! A local mesh of two LAN relays, each with its own CDN uplink, covers partitions, +//! overlapping claims, restarts, and a grant union crossing a scoped boundary. Last, a +//! client listing one prefix of a relay holds routes only to the origins meeting it. use crate::route_change::{Delivery, produce_origin, read, timed}; use crate::support; @@ -13,6 +17,7 @@ use crate::support; use std::time::Duration; use futures::{StreamExt, channel::mpsc}; +use moq_net::stats::{Message, Registry, SessionInfo, Tier}; use moq_net::{Error, Instance, Pattern, Patterns, Timestamp, Version, auth, origin, track}; use support::Kind; use support::harness::{MockConnectOptions, MockPair, connect_mock, now, spawn}; @@ -108,7 +113,10 @@ impl Links { fn kill(&mut self, node: &origin::Producer) { let hop = node.hop().get(); let touches = |(x, y, _): &(u64, u64, MockPair)| *x == hop || *y == hop; - self.0.iter().filter(|link| touches(link)).for_each(|(_, _, pair)| cut(pair)); + self.0 + .iter() + .filter(|link| touches(link)) + .for_each(|(_, _, pair)| cut(pair)); self.0.retain(|link| !touches(link)); } } @@ -194,7 +202,10 @@ async fn read_group(rx: &mut mpsc::UnboundedReceiver, name: &str, key: } /// Subscribe `video` at `path` on `node`, reading every group in full. -async fn subscribe(node: &origin::Producer, path: &str) -> (moq_net::broadcast::Consumer, mpsc::UnboundedReceiver) { +async fn subscribe( + node: &origin::Producer, + path: &str, +) -> (moq_net::broadcast::Consumer, mpsc::UnboundedReceiver) { let broadcast = (node.consume().follow(path, None).unwrap().available().await) .unwrap() .unwrap(); @@ -317,7 +328,15 @@ async fn a_far_edge_reads_a_claim_and_sees_its_withdrawal_once() { let at_far = watched[6].seen(); assert!( - matches!(at_far.as_slice(), [Seen { kind: Kind::Start, available: true, cost: VIA_FIRST_CORES, .. }]), + matches!( + at_far.as_slice(), + [Seen { + kind: Kind::Start, + available: true, + cost: VIA_FIRST_CORES, + .. + }] + ), "the far edge reads through the first cores: {at_far:?}" ); for watch in &mut watched { @@ -484,7 +503,14 @@ async fn a_partitioned_lan_reaches_itself_through_the_cdn_and_heals() { read_group(&mut rx, "O", 0).await; let start = island.seen(); assert!( - matches!(start.as_slice(), [Seen { kind: Kind::Start, cost: 2, .. }]), + matches!( + start.as_slice(), + [Seen { + kind: Kind::Start, + cost: 2, + .. + }] + ), "over the LAN: {start:?}" ); let instance = start[0].instance.clone(); @@ -526,7 +552,15 @@ async fn a_partitioned_lan_reaches_itself_through_the_cdn_and_heals() { quiesce().await; let healed = island.seen(); assert!( - matches!(healed.as_slice(), [Seen { kind: Kind::Update, available: true, cost: 2, .. }]), + matches!( + healed.as_slice(), + [Seen { + kind: Kind::Update, + available: true, + cost: 2, + .. + }] + ), "back on the LAN in place: {healed:?}" ); local.group(2); @@ -547,7 +581,11 @@ async fn a_partitioned_lan_reaches_itself_through_the_cdn_and_heals() { #[kio::test] async fn the_cdn_never_reaches_itself_through_the_lan() { for marked in [true, false] { - let uplink = if marked { origin::Link::Upstream } else { origin::Link::Peer }; + let uplink = if marked { + origin::Link::Upstream + } else { + origin::Link::Peer + }; let mut lan = Lan::with_uplinks(uplink).await; let _cdn = Publisher::new("Q", 3, "cdn", None, &lan.cdn[0], 1).await; quiesce().await; @@ -606,7 +644,9 @@ async fn overlapping_wildcards_resolve_to_the_most_specific_claim_everywhere() { assert_eq!(broadcast.info().instance.as_ref(), Some(instance), "{path}"); // Reading a track sends the request on to the claim. let track = broadcast.track("video").unwrap(); - subscribing.push(kio::clock::sim::spawn(async move { track.subscribe(None).await.map(drop) })); + subscribing.push(kio::clock::sim::spawn( + async move { track.subscribe(None).await.map(drop) }, + )); } } land().await; @@ -615,7 +655,9 @@ async fn overlapping_wildcards_resolve_to_the_most_specific_claim_everywhere() { wide_paths.sort(); narrow_paths.sort(); let expected = |under: &str| { - let mut paths: Vec<_> = (lan.relays()).map(|relay| format!("live/{under}/{}", relay.hop())).collect(); + let mut paths: Vec<_> = (lan.relays()) + .map(|relay| format!("live/{under}/{}", relay.hop())) + .collect(); paths.sort(); paths }; @@ -688,7 +730,8 @@ async fn an_epochless_replacement_restarts_everywhere() { let seen = watch.seen(); let hop = relay.hop(); assert!( - seen.iter().all(|seen| seen.kind != Kind::End && seen.kind != Kind::Offline), + seen.iter() + .all(|seen| seen.kind != Kind::End && seen.kind != Kind::Offline), "relay {hop}: {seen:?}" ); let restart = seen.iter().position(|seen| seen.kind == Kind::Restart); @@ -700,7 +743,9 @@ async fn an_epochless_replacement_restarts_everywhere() { "relay {hop}: {seen:?}" ); assert!( - seen[restart..].iter().all(|later| later.instance == seen[restart].instance), + seen[restart..] + .iter() + .all(|later| later.instance == seen[restart].instance), "relay {hop}: {seen:?}" ); } @@ -816,7 +861,14 @@ async fn a_grant_union_crosses_the_scoped_boundary_each_way() { for (inside, outside) in &mut watched { let (inside, outside) = (inside.seen(), outside.seen()); assert!( - matches!(inside.as_slice(), [Seen { kind: Kind::Start, available: true, .. }]) && outside.is_empty(), + matches!( + inside.as_slice(), + [Seen { + kind: Kind::Start, + available: true, + .. + }] + ) && outside.is_empty(), "scoped: inside {inside:?} outside {outside:?}" ); } @@ -831,7 +883,15 @@ async fn a_grant_union_crosses_the_scoped_boundary_each_way() { for (inside, outside) in &mut watched { let (inside, outside) = (inside.seen(), outside.seen()); assert!( - moved(&inside) && matches!(outside.as_slice(), [Seen { kind: Kind::Start, available: true, .. }]), + moved(&inside) + && matches!( + outside.as_slice(), + [Seen { + kind: Kind::Start, + available: true, + .. + }] + ), "widened: inside {inside:?} outside {outside:?}" ); } @@ -847,7 +907,11 @@ async fn a_grant_union_crosses_the_scoped_boundary_each_way() { "expired: inside {inside:?} outside {outside:?}" ); } - assert_eq!(uplink.client_transport.close_reason(), None, "the session outlived the token"); + assert_eq!( + uplink.client_transport.close_reason(), + None, + "the session outlived the token" + ); // Inside the scope the floor still certifies a withdrawal. inside.withdraw(); @@ -859,3 +923,167 @@ async fn a_grant_union_crosses_the_scoped_boundary_each_way() { drop((wide, uplink, inside, outside, links)); } +/// A client of `relay` asking it only about what the client's readers watch, with the +/// relay's side of the session counted. +async fn reader(relay: &origin::Producer, client: &origin::Producer) -> (MockPair, Counted) { + let registry = Registry::new(Default::default()); + let tier = Tier::new(moq_net::Path::from_static("client")); + let mut options = MockConnectOptions::new(lite07()); + options.server_publish = Some(relay.consume()); + options.client_subscribe = Some(client.clone()); + options.client_discovers_all = false; + options.server_stats = (registry.tier(tier.clone())).session(SessionInfo::new(moq_net::Path::from_static(""))); + (connect_mock(options).await, Counted { registry, tier }) +} + +/// The control bytes a relay sent its client. +struct Counted { + registry: Registry, + tier: Tier, +} + +impl Counted { + /// The announce bytes sent so far: routes, claims, floors, and their ends. + fn announced(&self) -> u64 { + (self.registry.snapshot().control().iter()) + .filter(|(tier, class, _)| *tier == self.tier && *class == Message::Announce) + .map(|(_, _, bytes)| bytes.tx) + .sum() + } +} + +/// A reader listing `prefix` on `node`, which is all a client's discovery follows. +struct Listing { + routes: origin::Routes, +} + +impl Listing { + async fn new(node: &origin::Producer, prefix: &str) -> Self { + let routes = (node.consume().scope(prefix, &Patterns::from(Pattern::all()))) + .unwrap() + .routes(); + let listing = Self { routes }; + // The request, its answer, and the initial set. + kio::clock::sleep(DOUBT).await; + listing + } + + /// Each event since the last call, as the path relative to the prefix and its kind. + fn seen(&mut self) -> Vec<(String, Kind)> { + let mut seen = Vec::new(); + while let std::task::Poll::Ready(Some(event)) = self.routes.poll_next(&kio::Waiter::noop()) { + let kind = match event.kind { + origin::RouteKind::Start => Kind::Start, + origin::RouteKind::Update => Kind::Update, + origin::RouteKind::Restart => Kind::Restart, + origin::RouteKind::End => Kind::End, + origin::RouteKind::Offline => Kind::Offline, + }; + seen.push((event.announce.prefix.to_string(), kind)); + } + seen + } +} + +/// An origin `hop` publishing each of `paths` over its own session to `relay`. +struct Origin { + node: origin::Producer, + routes: Vec, + link: MockPair, +} + +impl Origin { + async fn new(hop: u64, paths: &[&str], relay: &origin::Producer) -> Self { + let node = produce_origin(hop); + let routes = (paths.iter()) + .map(|path| { + let route = node.route(*path, origin::RouteOptions::hidden()).unwrap(); + route.announce().unwrap(); + route + }) + .collect(); + let link = push(&node, relay, 1).await; + Self { node, routes, link } + } + + /// Announce `path` and withdraw it again, a few times over. + async fn churn(&self, path: &str) { + for _ in 0..3 { + let route = self.node.route(path, origin::RouteOptions::hidden()).unwrap(); + route.announce().unwrap(); + kio::clock::sleep(DOUBT).await; + route.withdraw(); + kio::clock::sleep(DOUBT).await; + } + } +} + +/// A relay hears `L`, claiming `live/a` and `vod/l`, and `V`, claiming only `vod/b`, each +/// over its own session. A client lists only `live`. It holds `L`'s route and never `V`'s: +/// `V` churning costs the client no byte. A request outside that interest still resolves +/// through the relay. A Rust client's request discovers its own path, so it names `V`'s +/// node; Node ID 0, the publisher selecting, comes only from other implementations and +/// is covered at the publisher (`an_answered_track_stream_holds_the_track_until_the_requester_closes`). +#[kio::test] +async fn a_client_watching_one_prefix_holds_only_the_origins_meeting_it() { + let relay = produce_origin(1); + let _l = Origin::new(2, &["live/a", "vod/l"], &relay).await; + let v = Origin::new(3, &["vod/b"], &relay).await; + let client = produce_origin(9); + let (_pair, counted) = reader(&relay, &client).await; + quiesce().await; + let mut live = Listing::new(&client, "live").await; + assert_eq!(live.seen(), [("a".to_string(), Kind::Start)]); + + let before = counted.announced(); + v.churn("vod/c").await; + assert_eq!( + counted.announced(), + before, + "an origin outside the interest reached the client" + ); + + let broadcast = kio::clock::timeout(HOLD, client.consume().request("vod/b", None)) + .await + .expect("answered") + .expect("resolved through the relay"); + assert_eq!(broadcast.info().instance, Some(v.routes[0].instance())); + assert!(live.seen().is_empty()); +} + +/// `L` claims `live/a` and `vod/l` at a relay whose client lists `live`, then withdraws +/// `live/a`, its last claim there. Its route to the client ends behind that batch, closed +/// by its floor: the client sees `live/a` End where the relay certifies the request, and +/// Offline where the relay's grant to `L` (`live/a` and `vod`) knows only part of it. +/// Either way `L`'s later churn under `vod` costs the client nothing, so the route is gone. +#[kio::test] +async fn an_origin_leaving_the_interest_ends_its_route_by_its_floor() { + for (scoped, gone) in [(false, Kind::End), (true, Kind::Offline)] { + let relay = produce_origin(1); + let l = Origin::new(2, &["live/a", "vod/l"], &relay).await; + if scoped { + let mut grant = auth::Grant::default(); + grant.publish = [Pattern::subtree("live/a").unwrap(), Pattern::subtree("vod").unwrap()] + .into_iter() + .collect(); + l.link.server.auth().authorize(&grant); + } + let client = produce_origin(9); + let (_pair, counted) = reader(&relay, &client).await; + quiesce().await; + let mut live = Listing::new(&client, "live").await; + assert_eq!(live.seen(), [("a".to_string(), Kind::Start)], "scoped {scoped}"); + + l.routes[0].withdraw(); + kio::clock::sleep(DOUBT).await; + assert_eq!(live.seen(), [("a".to_string(), gone)], "scoped {scoped}"); + + let before = counted.announced(); + l.churn("vod/m").await; + assert_eq!( + counted.announced(), + before, + "scoped {scoped}: the route outlived the interest" + ); + } +} From a4dc7d05eb7f1df4fc9e817cf7d5c75638346bb0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 06:51:14 -0800 Subject: [PATCH 08/14] test(route): operation counts for a route change along each fan-out axis Sweep origins, claims per origin (one included), neighbours, and the streams a relay answers, watched or missed, for a repath, a link RTT change, and a stream's close, and bound what same-clock churn leaves remembered. Residual tests pin the axes that still grow: unrelated streams, per-claim re-serves, and closed sessions kept until the doubt timer. A repath now moves a heard route's claims instead of copying them, and a closed hop's report visits only the neighbours sent a route through it. routes/path/repath and routes/path/relink bench the change. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/benches/routes.rs | 43 +++- rs/moq-net/src/fuzz.rs | 97 +++++++ rs/moq-net/src/route/node/tests.rs | 121 +++++++++ rs/moq-net/src/route/path.rs | 64 +++-- rs/moq-net/src/route/tests/cost.rs | 401 +++++++++++++++++++++++++++++ rs/moq-net/src/route/tests/mod.rs | 1 + 6 files changed, 691 insertions(+), 36 deletions(-) create mode 100644 rs/moq-net/src/route/tests/cost.rs diff --git a/rs/moq-net/benches/routes.rs b/rs/moq-net/benches/routes.rs index b3c35f9f339..84c4d3774ce 100644 --- a/rs/moq-net/benches/routes.rs +++ b/rs/moq-net/benches/routes.rs @@ -33,6 +33,13 @@ //! - path/drained: after the drain, an RTT change on a session with no routes. Swept //! over the same grid: the streams axis is expected to grow, since each stream's //! DRAIN notice is kept; the sessions axis is not. +//! - path/repath: a relay hears one publisher's route move onto another path, and +//! answers streams the route misses. Swept over the claims behind the route and those +//! streams: neither axis is expected to grow. +//! - path/relink: the RTT of the link every route arrives on changes. Swept over the +//! publishers behind the link and the streams their routes miss: the publishers axis +//! is expected to grow, since each route changes; the streams axis is not, though today +//! it does (`route::tests::cost::residual_a_link_update_visits_every_unrelated_stream`). //! //! The route layer is crate-private, so this reaches it through the `fuzz` module. //! @@ -57,6 +64,12 @@ const FOLLOWED: [(usize, usize); 4] = [(16, 1), (1_024, 1), (16, 16), (1_024, 16 /// sessions drain. const DRAINING: [(usize, usize); 4] = [(4, 4), (128, 4), (4, 128), (128, 128)]; +/// `(claims, streams)` shapes for one route's change, with streams it misses. +const REPATH: [(usize, usize); 4] = [(1, 10), (10_000, 10), (1, 1_000), (10_000, 1_000)]; + +/// `(publishers, streams)` shapes for a link's change, with streams its routes miss. +const RELINK: [(usize, usize); 4] = [(1, 10), (1_000, 10), (1, 1_000), (1_000, 1_000)]; + /// `(claims, lag)` shapes for a relay hearing a stream that lags another. const LAGGING: [(usize, usize); 4] = [(100, 1), (10_000, 1), (100, 100), (10_000, 10_000)]; @@ -283,6 +296,32 @@ fn bench_path_drained(c: &mut Criterion) { group.finish(); } +fn bench_path_repath(c: &mut Criterion) { + let mut group = c.benchmark_group("routes/path/repath"); + for (claims, streams) in REPATH { + let id = BenchmarkId::from_parameter(format!("{claims}c_{streams}s")); + let mut fixture = None; + group.bench_function(id, |b| { + let node = fixture.get_or_insert_with(|| fuzz::PathRouteChange::new(1, claims, streams)); + b.iter(|| node.repath()); + }); + } + group.finish(); +} + +fn bench_path_relink(c: &mut Criterion) { + let mut group = c.benchmark_group("routes/path/relink"); + for (publishers, streams) in RELINK { + let id = BenchmarkId::from_parameter(format!("{publishers}p_{streams}s")); + let mut fixture = None; + group.bench_function(id, |b| { + let node = fixture.get_or_insert_with(|| fuzz::PathRouteChange::new(publishers, 1, streams)); + b.iter(|| node.relink()); + }); + } + group.finish(); +} + criterion_group!( benches, bench_watched, @@ -299,6 +338,8 @@ criterion_group!( bench_path_teardown, bench_path_refollow, bench_path_drain, - bench_path_drained + bench_path_drained, + bench_path_repath, + bench_path_relink ); criterion_main!(benches); diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index b6a50cca9c9..afe1cd97462 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -1205,6 +1205,103 @@ impl PathRouteDrain { } } +/// A path router hearing `origins` publishers from one neighbour, each under a prefix of +/// its own with `claims` claims, and answering `streams` streams that each watch a +/// prefix no publisher claims under, for the routes bench. +pub struct PathRouteChange { + router: crate::route::path::Router, + far: u64, + rtt: u64, +} + +impl PathRouteChange { + const UP: u64 = 1_000; + + fn reach(id: u64, far: u64) -> crate::route::path::Reach { + use crate::route::path::{Hop, Offer, Reach}; + let path = [ + Hop { node: id, session: 0 }, + Hop { node: far, session: 1 }, + Hop { + node: Self::UP, + session: 1, + }, + ]; + Reach::Finite(Offer { + path: path.into(), + draining: false, + priority: i8::MAX, + metric: 1, + down_only: false, + rtt: Some(std::time::Duration::ZERO), + }) + } + + /// Hear `origins` publishers with `claims` claims each, and answer `streams` streams + /// the routes miss. + pub fn new(origins: usize, claims: usize, streams: usize) -> Self { + let mut router = crate::route::path::Router::new(2, 2, crate::route::path::Doubt::default()); + router.connect(0, Self::UP, crate::route::Link::default()).unwrap(); + router.edited(0); + for o in 0..origins { + let id = 1 + o as u64; + let prefix = Path::new(&format!("o{o}")).unwrap(); + router.route(0, id, prefix.clone(), Some(Self::reach(id, 500))); + let changes = (0..claims) + .map(|c| (Path::new(&format!("o{o}/{c}")).unwrap(), Some(()))) + .collect(); + router.batch(0, id, prefix, changes, Some(1)); + } + for s in 0..streams { + let peer = 1 + s; + router + .connect(peer, 5_000 + s as u64, crate::route::Link::default()) + .unwrap(); + let add = vec![Path::new(&format!("x{s}")).unwrap().to_owned()]; + router + .edit( + peer, + crate::route::path::Edit { + remove: Vec::new(), + add, + }, + ) + .unwrap(); + } + let mut node = Self { + router, + far: 500, + rtt: 0, + }; + node.flush(); + node + } + + /// The neighbour moves the first publisher's route onto another path, which no + /// stream watches. + pub fn repath(&mut self) { + self.far = if self.far == 500 { 501 } else { 500 }; + self.router + .route(0, 1, Path::new("o0").unwrap(), Some(Self::reach(1, self.far))); + assert_eq!(self.flush(), 0); + } + + /// The RTT of the link to the neighbour changes, and with it every route through it, + /// which no stream watches. + pub fn relink(&mut self) { + self.rtt = self.rtt % 8 + 1; + self.router.rtt(0, Some(std::time::Duration::from_millis(self.rtt))); + assert_eq!(self.flush(), 0); + } + + fn flush(&mut self) -> usize { + let sent = self.router.drain().len(); + self.router.events(); + self.router.changed(); + sent + } +} + /// What a client session fed into an origin hears it should ask its peer about, for /// the interests bench. pub struct Interests(crate::model::Interests); diff --git a/rs/moq-net/src/route/node/tests.rs b/rs/moq-net/src/route/node/tests.rs index ea98a73b29f..6a63857035a 100644 --- a/rs/moq-net/src/route/node/tests.rs +++ b/rs/moq-net/src/route/node/tests.rs @@ -806,3 +806,124 @@ fn a_release_closes_with_the_newest_floor_after_older_batches_are_read() { }); assert_eq!(last, Some((vec![p("live/y")], Some(6)))); } + +/// What one route update makes a relay do, behind an origin with `claims` claims beside +/// another origin with `others`: the serve commands its session applies to the origin, +/// for a repath and then for an RTT change, and the ROUTE and other messages a stream +/// watching the root is sent for both. +fn reserved(claims: usize, others: usize) -> (usize, usize, usize, usize) { + let instance = Instance::random(); + let relay = Node::new(2, 2, clock()); + let edge = relay.answer(peer(3, 3), Link::default(), Path::empty()).unwrap(); + let up = relay.receive(peer(1, 10), Link::default()).unwrap(); + let offer = |id: Id, far: Id| { + let path = [ + Hop { node: id, session: 0 }, + Hop { node: far, session: 1 }, + Hop { node: 10, session: 1 }, + ]; + let offer = Offer { + path: path.into(), + draining: false, + priority: i8::MAX, + metric: 1, + down_only: false, + rtt: Some(Duration::ZERO), + }; + Some(Reach::Finite(offer)) + }; + let live = claim(&instance, 0); + for (id, count) in [(5, claims), (6, others)] { + let prefix = Path::new(&format!("o{id}")).unwrap(); + up.route(id, prefix.clone(), offer(id, 500)); + let changes = (0..count) + .map(|i| (Path::new(&format!("o{id}/{i}")).unwrap(), Some(live.clone()))) + .collect(); + up.batch(id, prefix, changes, Some(1)); + } + serve_all(&up); + outputs(&edge); + // Read after each update, since a route's unread changes coalesce. + let mut sent = Vec::new(); + up.route(5, p("o5"), offer(5, 501)); + let repath = served(outputs(&up)).len(); + sent.extend(wire(outputs(&edge))); + up.rtt(Some(Duration::from_millis(40))); + let rtt = served(outputs(&up)).len(); + sent.extend(wire(outputs(&edge))); + let routes = (sent.iter()).filter(|out| matches!(out, Output::Route { .. })).count(); + (repath, rtt, routes, sent.len() - routes) +} + +/// A route update re-serves every claim behind it: the session rewrites the origin's +/// route table once per broadcast, each a `sync_route` that wakes the fronts and cursors +/// below it, though no claim changed. The wire stays flat: a stream watching the route +/// is sent one ROUTE per update and no claim, and a repath leaves other origins alone. +/// This should be one write per route, which its broadcasts refer to. +#[test] +fn residual_a_route_update_reserves_every_claim() { + for claims in [1, 8, 64] { + for others in [0, 64] { + let (repath, rtt, routes, other) = reserved(claims, others); + // The RTT change moves both origins' routes, which share the link. + assert_eq!( + (repath, rtt), + (claims, claims + others), + "{claims} claims, {others} beside" + ); + let changed = 2 + usize::from(others > 0); + assert_eq!((routes, other), (changed, 0), "a ROUTE per route changed, and no claim"); + } + } +} + +/// Each closed session is remembered until the doubt timer, even one that sent this node +/// no route, as a viewer's does: a dead hop for its session, and a doubt on its node when +/// it declared one. Viewer churn grows both by the sessions closed inside one timer. A +/// session no route was learned on has no hop any path can name, so it need not be +/// remembered. +#[test] +fn residual_each_closed_session_is_remembered_until_the_doubt_timer() { + const CLOSED: u64 = 64; + for declared in [false, true] { + let instance = Instance::random(); + let clock = clock(); + let relay = Node::new(2, 2, clock.clone()); + let up = relay.receive(peer(1, 10), Link::default()).unwrap(); + let offer = Offer { + path: [Hop { node: 5, session: 0 }, Hop { node: 10, session: 1 }].into(), + draining: false, + priority: i8::MAX, + metric: 1, + down_only: false, + rtt: None, + }; + up.route(5, Path::empty(), Some(Reach::Finite(offer))); + up.batch( + 5, + Path::empty(), + vec![(p("live/x"), Some(claim(&instance, 0)))], + Some(1), + ); + serve_all(&up); + for viewer in 0..CLOSED { + let node = if declared { 1000 + viewer } else { 0 }; + let stream = (relay.answer(peer(100 + viewer, node), Link::default(), Path::empty())).unwrap(); + outputs(&stream); + let closing = relay.closing(100 + viewer); + drop(stream); + drop(closing); + } + let kept = relay.lock().router.retained(); + let doubts = if declared { CLOSED as usize } else { 1 }; + assert_eq!( + (kept.dead, kept.doubt), + (CLOSED as usize, doubts), + "declared {declared}" + ); + clock.advance(Instant::ZERO.saturating_add(Doubt::DEFAULT.timer)); + relay.timeout(); + let kept = relay.lock().router.retained(); + assert_eq!((kept.dead, kept.doubt), (0, 0), "declared {declared}"); + } +} diff --git a/rs/moq-net/src/route/path.rs b/rs/moq-net/src/route/path.rs index 2707df16d36..cb91d5cfc64 100644 --- a/rs/moq-net/src/route/path.rs +++ b/rs/moq-net/src/route/path.rs @@ -1322,14 +1322,12 @@ impl Router { } } let key = (id, prefix); - let entry = self.rx.get(&key).and_then(|routes| routes.get(&from)); match reach { // A path a node could not extend is refused, as if ended. Some(Reach::Finite(offer)) if offer.extendable() => { - let (view, batched) = entry.map_or_else(Default::default, |entry| (entry.view.clone(), entry.batched)); - if entry.is_some() { - self.forget_heard(&key, from); - } + // A new path keeps the claims the route carries. + let (view, batched) = + (self.forget_heard(&key, from)).map_or_else(Default::default, |heard| (heard.view, heard.batched)); let state = match self.dead_in(&offer.path) { true => State::Dead, false => State::Live, @@ -1353,10 +1351,9 @@ impl Router { heard.state = State::Held; } Some(Reach::Finite(_)) | None => { - if entry.is_none() { + if self.forget_heard(&key, from).is_none() { return; } - self.forget_heard(&key, from); } } self.update_route(key); @@ -1951,13 +1948,11 @@ impl Router { self.rx.entry(key).or_default().insert(from, heard); } - fn forget_heard(&mut self, key: &RouteId, from: N) { - let Some(routes) = self.rx.get_mut(key) else { - return; - }; - let Some(heard) = routes.remove(&from) else { - return; - }; + /// Forget the route `from` sent under `key`, returning it, so a caller keeping its + /// claims moves them rather than copying every one. + fn forget_heard(&mut self, key: &RouteId, from: N) -> Option> { + let routes = self.rx.get_mut(key)?; + let heard = routes.remove(&from)?; if routes.is_empty() { self.rx.remove(key); } @@ -1985,13 +1980,13 @@ impl Router { self.filed.remove(&(key.0, from)); } } + Some(heard) } /// Re-path a heard route in place, keeping its claims. fn repath(&mut self, key: &RouteId, from: N, offer: Offer) { let named = self.name(from, key.0); - let mut heard = self.rx[key][&from].clone(); - self.forget_heard(key, from); + let mut heard = self.forget_heard(key, from).expect("heard"); heard.state = match self.dead_in(&offer.path) { true => State::Dead, false => State::Live, @@ -2093,7 +2088,7 @@ impl Router { if let Some(offer) = moved { match offer.extendable() { true => self.repath(key, peer, offer.clone()), - false => self.forget_heard(key, peer), + false => drop(self.forget_heard(key, peer)), } return; } @@ -2699,7 +2694,11 @@ impl Router { /// Tell each neighbour that was sent a route through `hop` what became of those /// routes, in one [`Dead`] where it is the first word on that hop. fn report(&mut self, hop: Hop) { - let peers: Vec = self.sessions.keys().copied().collect(); + // Only the neighbours sent a route through the hop are told, so only they are visited, + // not every session. + let peers: BTreeSet = (self.tx_hops.get(&hop).into_iter().flatten()) + .map(|(peer, _)| *peer) + .collect(); for peer in peers { let lo = (peer, (0, Path::empty())); let through: Vec = (self.tx_hops.get(&hop).into_iter()) @@ -2707,9 +2706,6 @@ impl Router { .take_while(|(to, _)| *to == peer) .map(|(_, key)| key.clone()) .collect(); - if through.is_empty() { - continue; - } let Some(answer) = self.answering.get(&peer) else { continue; }; @@ -2910,29 +2906,26 @@ impl Router { /// Bring what each watching neighbour was sent for `id` up to date, with the `dirty` /// claims changed since. fn advertise(&mut self, id: Id, dirty: &Dirty, region: Option<&Path>) { - let peers: Vec<(N, BTreeSet)> = self - .answering - .iter() - .map(|(peer, answer)| (*peer, answer.prefixes.clone())) - .collect(); // Plan every stream against the state as it stands, then send. let mut plans = Vec::new(); let mut planned: BTreeMap> = BTreeMap::new(); - for (peer, prefixes) in &peers { - for prefix in self.stream(id, *peer, prefixes, region) { - planned.entry(*peer).or_default().insert(prefix.clone()); + for (&peer, answer) in &self.answering { + // Every stream is visited, whether or not the change meets it. + self.step(); + let prefixes = &answer.prefixes; + for prefix in self.stream(id, peer, prefixes, region) { + planned.entry(peer).or_default().insert(prefix.clone()); self.step(); - let want = self.want(id, &prefix, *peer, prefixes); - let on = - (self.answering[peer].tx.get(&(id, prefix.clone()))).is_some_and(|sent| sent.offer().is_some()); + let want = self.want(id, &prefix, peer, prefixes); + let on = (answer.tx.get(&(id, prefix.clone()))).is_some_and(|sent| sent.offer().is_some()); let batch = match &want { - Want::Path(..) => Some(self.present(id, *peer, &prefix, dirty)), + Want::Path(..) => Some(self.present(id, peer, &prefix, dirty)), Want::Empty if on && self.covering_view(id, &prefix).is_some() => { - Some(self.present(id, *peer, &prefix, dirty)) + Some(self.present(id, peer, &prefix, dirty)) } _ => None, }; - plans.push((*peer, prefix, want, batch)); + plans.push((peer, prefix, want, batch)); } } for (peer, prefix, want, batch) in plans { @@ -3224,6 +3217,7 @@ impl Router { // Reflected sources must move before old holds ask whether a child answered. let mut sources = Vec::new(); for (&peer, answer) in self.answering.iter().filter(|_| route_changed) { + self.step(); let prefixes = &answer.prefixes; for &id in &ids { let region = scope.as_ref().filter(|(key, _)| *key == id).map(|(_, prefix)| prefix); diff --git a/rs/moq-net/src/route/tests/cost.rs b/rs/moq-net/src/route/tests/cost.rs new file mode 100644 index 00000000000..08314155681 --- /dev/null +++ b/rs/moq-net/src/route/tests/cost.rs @@ -0,0 +1,401 @@ +//! What one route change costs a relay, swept along each axis it could fan out over: +//! the origins it hears, the claims behind each origin (one included), the neighbours +//! offering the same route, and the streams it answers, each watching the change or a +//! prefix the change misses. A sweep holds the other axes fixed and counts +//! [`path::Router::work`], what the change sends, and what the reader sees, so a change +//! that expands into per-claim rewrites or visits streams it cannot reach fails here +//! rather than only timing slower. The `residual_` tests pin axes that still grow and +//! must not: a fix flips them. + +use super::*; + +/// This relay, in paths. +const RELAY: Id = 99; +/// The far node a heard route crosses before its sender, and the one a repath moves to. +const FAR: [Id; 2] = [500, 501]; +/// The first key of the streams this relay answers. +const DOWN: Node = 10_000; + +#[derive(Clone, Copy, Debug)] +struct Shape { + /// Origins heard, each under a prefix of its own. + origins: usize, + /// Claims behind each origin. + claims: usize, + /// Upstream neighbours, each offering every origin's route. + neighbours: usize, + /// Streams this relay answers. + streams: usize, + /// Whether each stream watches the root, covering every origin, or a prefix no + /// origin claims under. + watched: bool, +} + +const BASE: Shape = Shape { + origins: 4, + claims: 4, + neighbours: 2, + streams: 4, + watched: true, +}; + +/// What one change cost this relay. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct Cost { + /// Routes, prefixes, streams, and reader claims visited. + work: usize, + /// Identities recomputed. + touched: usize, + /// ROUTE messages sent. + routes: usize, + /// Claims sent in batches. + claims: usize, + /// Any other message sent. + other: usize, + /// Shown claims whose state changed. + shown: usize, +} + +struct Relay { + router: path::Router, + far: usize, + rtt: u64, + closed: usize, +} + +/// The route `neighbour` offers for origin `id` through `far`, carrying an RTT so a +/// change to the link's RTT changes the offer this relay sends on. +fn offer(id: Id, far: Id, neighbour: usize) -> path::Reach { + let mut offer = along( + &[(id, 0), (far, 1), (1000 + neighbour as Id, 1)], + 1 + neighbour as Metric, + ); + offer.rtt = Some(Duration::ZERO); + path::Reach::Finite(offer) +} + +fn origin(origin: usize) -> Path { + p(&format!("o{origin}")) +} + +impl Relay { + fn new(shape: Shape) -> Self { + let mut router = path::Router::new(RELAY, 1, Doubt::DEFAULT); + for up in 0..shape.neighbours { + router.connect(up, 1000 + up as Id, Link::default()).unwrap(); + router.edited(up); + } + for stream in 0..shape.streams { + let down = DOWN + stream; + router.connect(down, 2000 + stream as Id, Link::default()).unwrap(); + router.edited(down); + let prefix = match shape.watched { + true => String::new(), + false => format!("x{stream}"), + }; + reask(&mut router, down, &[&prefix]).unwrap(); + } + for o in 0..shape.origins { + let id = 1 + o as Id; + for up in 0..shape.neighbours { + router.route(up, id, origin(o), Some(offer(id, FAR[0], up))); + let changes = (0..shape.claims) + .map(|claim| (p(&format!("o{o}/b{claim}")), Some(1))) + .collect(); + router.batch(up, id, origin(o), changes, Some(1)); + } + } + let mut relay = Self { + router, + far: 0, + rtt: 0, + closed: 0, + }; + relay.measure(|_| {}); + relay + } + + /// The first neighbour moves the first origin's route onto a path through the other + /// far node: one route changes, and none of its claims. + fn repath(&mut self) -> Cost { + self.far = 1 - self.far; + let reach = offer(1, FAR[self.far], 0); + self.measure(|router| router.route(0, 1, origin(0), Some(reach))) + } + + /// The RTT of the link to the first neighbour changes: every route it offers does. + fn relink(&mut self) -> Cost { + self.rtt = self.rtt % 2 + 1; + let rtt = Duration::from_millis(5 * self.rtt); + self.measure(|router| router.rtt(0, Some(rtt))) + } + + /// The session of the next stream this relay answers closes. + fn close(&mut self) -> Cost { + let down = DOWN + self.closed; + self.closed += 1; + self.measure(|router| router.disconnect([down], Instant::ZERO)) + } + + fn measure(&mut self, change: impl FnOnce(&mut path::Router)) -> Cost { + let (work, touched) = (self.router.work(), self.router.touched()); + change(&mut self.router); + let mut cost = Cost { + work: self.router.work() - work, + touched: self.router.touched() - touched, + ..Cost::default() + }; + for out in self.router.drain() { + match out { + path::Out::Route { .. } => cost.routes += 1, + path::Out::Batch { changes, .. } if !changes.is_empty() => cost.claims += changes.len(), + _ => cost.other += 1, + } + } + self.router.events(); + cost.shown = self.router.changed().values().map(BTreeSet::len).sum(); + cost + } +} + +/// The cost of `change` on relays shaped as `BASE` with one axis set to each size, +/// measured after one warm-up change so first-time effects are left out. +fn sweep(axis: fn(&mut Shape, usize), sizes: [usize; 3], change: fn(&mut Relay) -> Cost) -> [Cost; 3] { + sizes.map(|size| { + let mut shape = BASE; + axis(&mut shape, size); + let mut relay = Relay::new(shape); + change(&mut relay); + change(&mut relay) + }) +} + +fn origins(shape: &mut Shape, n: usize) { + shape.origins = n; +} + +fn claims(shape: &mut Shape, n: usize) { + shape.claims = n; +} + +fn neighbours(shape: &mut Shape, n: usize) { + shape.neighbours = n; +} + +fn watched(shape: &mut Shape, n: usize) { + shape.streams = n; +} + +fn unwatched(shape: &mut Shape, n: usize) { + shape.streams = n; + shape.watched = false; +} + +/// One broadcast per origin, a few, and many. +const FLAT: [usize; 3] = [1, 8, 64]; +/// Doublings, for axes that may grow linearly. +const LINEAR: [usize; 3] = [8, 16, 32]; + +fn assert_flat(name: &str, costs: [Cost; 3]) { + assert!( + costs.iter().all(|cost| *cost == costs[0]), + "{name} must not grow: {costs:?}" + ); +} + +fn assert_linear(name: &str, costs: [Cost; 3]) { + for pair in costs.windows(2) { + // Quadratic work would quadruple. + assert!( + pair[1].work <= 5 * pair[0].work / 2, + "{name} grows faster than linear: {costs:?}" + ); + } +} + +/// Moving one route onto another path sends each stream watching it one ROUTE, and no +/// claim: the work is flat in the origins and the claims behind the route, and grows +/// linearly in the streams it reaches and the neighbours offering the same route, which +/// it ranks. +#[test] +fn a_repath_costs_the_streams_it_reaches_and_nothing_per_claim() { + assert_flat("origins", sweep(origins, FLAT, Relay::repath)); + assert_flat("claims", sweep(claims, FLAT, Relay::repath)); + let reached = sweep(watched, LINEAR, Relay::repath); + assert_linear("streams the change reaches", reached); + assert_eq!(reached.map(|cost| cost.routes), LINEAR, "a ROUTE per stream"); + assert_linear("neighbours", sweep(neighbours, LINEAR, Relay::repath)); + for cost in reached { + assert_eq!( + (cost.touched, cost.claims, cost.other, cost.shown), + (1, 0, 0, 0), + "{cost:?}" + ); + } +} + +/// An RTT change on one link re-offers each route through it to each stream watching +/// that route, and sends no claim: the work is flat in the claims behind each route, and +/// grows linearly in the routes through the link and the streams they reach. +#[test] +fn a_link_update_costs_the_routes_through_it_and_nothing_per_claim() { + assert_flat("claims", sweep(claims, FLAT, Relay::relink)); + let routes = sweep(origins, LINEAR, Relay::relink); + assert_linear("routes through the link", routes); + assert_eq!(routes.map(|cost| cost.touched), LINEAR); + assert_eq!(routes.map(|cost| cost.routes), LINEAR.map(|n| n * BASE.streams)); + let reached = sweep(watched, LINEAR, Relay::relink); + assert_linear("streams the change reaches", reached); + assert_eq!(reached.map(|cost| cost.routes), LINEAR.map(|n| n * BASE.origins)); + for cost in routes.into_iter().chain(reached) { + assert_eq!((cost.claims, cost.other, cost.shown), (0, 0, 0), "{cost:?}"); + } +} + +/// A route change visits every stream this relay answers for each identity it +/// recomputes, even a stream whose prefix no route meets: `advertise` walks every +/// answering stream, and so does `run` for the sources a selection change moves. An +/// update with no single route in scope (an RTT change, a DEAD, a drain, a closed +/// session) also plans every prefix of each, since `stream` returns all of them when no +/// region narrows it. Both should be flat in the streams the change misses, which needs +/// an index of the streams by the prefixes they ask for and the routes they were sent. +#[test] +fn residual_a_route_change_visits_every_unrelated_stream() { + for change in [Relay::repath as fn(&mut Relay) -> Cost, Relay::relink] { + let costs = sweep(unwatched, FLAT, change); + assert!(costs.iter().all(|cost| cost.routes == 0), "{costs:?}"); + assert!( + costs.windows(2).all(|pair| pair[1].work > pair[0].work), + "fixed, so flip this residual: {costs:?}" + ); + } +} + +/// Closing one stream's session, which sent this relay nothing, re-plans every route +/// it was sent for every other stream, which then sends nothing: viewer churn costs +/// each close the identities it watched times all the streams left. This should be +/// flat in the other streams. +#[test] +fn residual_closing_a_stream_revisits_every_other_stream() { + let costs = sweep(watched, LINEAR, Relay::close); + assert!( + costs.iter().all(|cost| cost.routes == 0 && cost.other == 0), + "{costs:?}" + ); + assert!( + costs.windows(2).all(|pair| pair[1].work > pair[0].work), + "fixed, so flip this residual: {costs:?}" + ); +} + +/// The routes `router` holds for `ids` with no live path. +fn held(router: &path::Router, ids: impl IntoIterator) -> usize { + (ids.into_iter()) + .map(|id| router.routes(id).filter(|(_, route)| route.offer().is_none()).count()) + .sum() +} + +/// Churn while no timer fires: a far session that flaps, and publishers that restart +/// under fresh identities, sent on to a stream that never answers. A doubt is kept per +/// far node however often its sessions flap, and a repeated DEAD adds nothing, but each +/// closed session's hop, each restarted identity's doubt, and each lost identity's hold +/// is kept until its own timer: what a node remembers grows with the churn inside one +/// timer, not with the live topology. Once the timers pass, only the live topology is +/// left. A peer can only make a node remember hops on paths it offered (`credits`). +#[test] +fn residual_same_clock_churn_is_remembered_until_its_timer() { + const FLAPS: u64 = 64; + let now = Instant::ZERO; + let mut router = path::Router::::new(RELAY, 1, Doubt::DEFAULT); + router.connect(0, 1000, Link::default()).unwrap(); + router.edited(0); + router.connect(DOWN, 2000, Link::default()).unwrap(); + router.edited(DOWN); + reask(&mut router, DOWN, &[""]).unwrap(); + router.route(0, 7, p(""), Some(path::Reach::Finite(along(&[(7, 0), (1000, 0)], 1)))); + router.batch(0, 7, p(""), vec![(p("a"), Some(1))], Some(1)); + // The neighbour's session to publisher 7 flaps, and its route moves to the next. + let flap = |router: &mut path::Router, session: u64| { + let moved = vec![path::Moved { + id: 7, + prefix: p(""), + offer: along(&[(7, 0), (1000, session + 1)], 1), + }]; + let hop = path::Hop { node: 1000, session }; + let dead = path::Dead::Closed { + hop, + far: 7, + moved, + gone: false, + }; + router.dead(0, dead, now); + router.timeout(now); + router.drain(); + }; + for session in 0..FLAPS { + flap(&mut router, session); + } + let kept = router.retained(); + assert_eq!((kept.doubt, kept.draining), (1, 0), "one doubt per far node"); + assert_eq!(kept.dead, FLAPS as usize, "a dead hop per closed session"); + flap(&mut router, FLAPS - 1); + assert_eq!(router.retained(), kept, "a repeated DEAD adds nothing"); + + // Publishers behind the neighbour restart under fresh identities, each losing its + // session to the neighbour. Each was sent on to a stream that never answers, so each + // is held until the hold timer. + let restarts = 3000..3000 + FLAPS; + for id in restarts.clone() { + let hop = path::Hop { + node: 1000, + session: id, + }; + router.route( + 0, + id, + p(""), + Some(path::Reach::Finite(along(&[(id, 0), (1000, id)], 1))), + ); + router.batch(0, id, p(""), vec![(p("a"), Some(1))], Some(1)); + router.drain(); + let dead = path::Dead::Closed { + hop, + far: id, + moved: Vec::new(), + gone: false, + }; + router.dead(0, dead, now); + router.timeout(now); + router.drain(); + } + let kept = router.retained(); + assert_eq!( + held(&router, restarts.clone()), + FLAPS as usize, + "a hold per lost identity" + ); + assert_eq!(kept.doubt, 1 + FLAPS as usize, "a doubt per restarted identity"); + assert_eq!(kept.dead, 2 * FLAPS as usize); + + // Once the doubt timer fires, only the hops a held route still names are left. + let later = now.saturating_add(Doubt::DEFAULT.timer); + router.timeout(later); + let kept = router.retained(); + assert_eq!((kept.dead, kept.doubt), (FLAPS as usize, 0), "{kept:?}"); + assert!(live(&router, 7), "the moved route is used once the doubt lifts"); + // Once the hold timer fires, this relay lets go; once the neighbour lets go too, only + // the live topology is left. + router.timeout(later.saturating_add(path::HOLD)); + assert_eq!(held(&router, restarts.clone()), 0); + for id in restarts { + router.route(0, id, p(""), None); + } + router.drain(); + let kept = router.retained(); + assert_eq!((kept.routes, kept.dead, kept.doubt), (1, 0, 0), "{kept:?}"); +} + +/// Whether `router` uses a live route for publisher `id`. +fn live(router: &path::Router, id: Id) -> bool { + router.routes(id).any(|(_, route)| route.offer().is_some()) +} diff --git a/rs/moq-net/src/route/tests/mod.rs b/rs/moq-net/src/route/tests/mod.rs index f52b824c8ea..0d0eda6c457 100644 --- a/rs/moq-net/src/route/tests/mod.rs +++ b/rs/moq-net/src/route/tests/mod.rs @@ -10,6 +10,7 @@ mod cache; mod compare; +mod cost; mod handoff; mod interest; mod residual; From 94496e3a899058d90b59d0134d908b2885649538 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 07:02:41 -0800 Subject: [PATCH 09/14] docs(bench): name the residual the repath and relink groups pin Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/benches/routes.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rs/moq-net/benches/routes.rs b/rs/moq-net/benches/routes.rs index 84c4d3774ce..97b0b344df5 100644 --- a/rs/moq-net/benches/routes.rs +++ b/rs/moq-net/benches/routes.rs @@ -35,11 +35,12 @@ //! DRAIN notice is kept; the sessions axis is not. //! - path/repath: a relay hears one publisher's route move onto another path, and //! answers streams the route misses. Swept over the claims behind the route and those -//! streams: neither axis is expected to grow. +//! streams: neither axis is expected to grow, though the streams axis does today +//! (`route::tests::cost::residual_a_route_change_visits_every_unrelated_stream`). //! - path/relink: the RTT of the link every route arrives on changes. Swept over the //! publishers behind the link and the streams their routes miss: the publishers axis -//! is expected to grow, since each route changes; the streams axis is not, though today -//! it does (`route::tests::cost::residual_a_link_update_visits_every_unrelated_stream`). +//! is expected to grow, since each route changes; the streams axis is not, though it +//! does today, as above. //! //! The route layer is crate-private, so this reaches it through the `fuzz` module. //! From 714e3334fb29cb63cc40679a408d10f978b9d3b8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 07:31:04 -0800 Subject: [PATCH 10/14] test(route): forwarding fixtures run on the path router, and the reference model goes The routing contract's forwarding, export, and qualified-identity fixtures move from the Babel-era reference model to route/tests/forward.rs on the path router, and pool ties to real sessions. The model and its Babel-only fixtures are deleted. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/tests/forward.rs | 509 ++++ rs/moq-net/src/route/tests/mod.rs | 1 + rs/moq-net/tests/integration/main.rs | 1 - .../tests/integration/route_acceptance.rs | 50 + .../integration/routing_contracts/handoff.rs | 1493 ----------- .../integration/routing_contracts/mod.rs | 1353 ---------- .../integration/routing_contracts/model.rs | 2275 ----------------- 7 files changed, 560 insertions(+), 5122 deletions(-) create mode 100644 rs/moq-net/src/route/tests/forward.rs delete mode 100644 rs/moq-net/tests/integration/routing_contracts/handoff.rs delete mode 100644 rs/moq-net/tests/integration/routing_contracts/mod.rs delete mode 100644 rs/moq-net/tests/integration/routing_contracts/model.rs diff --git a/rs/moq-net/src/route/tests/forward.rs b/rs/moq-net/src/route/tests/forward.rs new file mode 100644 index 00000000000..1d2423a913d --- /dev/null +++ b/rs/moq-net/src/route/tests/forward.rs @@ -0,0 +1,509 @@ +//! Where a request goes, on the path rule: the forwarding, export, and qualified +//! identity fixtures the reference model ran before the router existed. +//! +//! A lite-07 request names its publisher, and each relay follows the route it offered +//! the neighbour the request came from. No transit relay reselects, so the model's +//! counterexamples for path-only requests (local priorities circling, nested claims +//! looping where a relay has not heard the narrower ones, joins deadlocking, another +//! publisher spliced in) have nothing to run on here: each fixture checks that a named +//! request arrives instead. The model's Babel fixtures (feasibility, seqno requests, the +//! distance a node keeps) went with Babel. + +use super::*; + +const LOCAL: Grant = Grant { + subject: "local", + scope: "local/", +}; + +/// `node` publishes as `id`, claiming each of `claims`. +fn publish(sim: &mut Sim, node: Node, id: Id, claims: &[&str]) { + sim.originate(node, id); + for claim in claims { + sim.announce(node, claim, 0, None); + } +} + +/// Where a reader's request at `node` ends when every relay on the way serves it from +/// its own subscription to the named publisher: each hop follows its own reader's +/// route, as a request joining that subscription does. `None` where one stops short. +/// A cycle would leave every subscription in it waiting on the next, so it panics. +fn joined(sim: &Sim, node: Node, id: Id, claim: &Path) -> Option { + let (mut at, mut id) = (node, id); + let mut seen = BTreeSet::new(); + loop { + assert!( + seen.insert((at, id)), + "subscriptions to {id} {claim:?} join round a cycle from node {node}" + ); + match sim.path(at).forward(id, claim, None)? { + path::Forward::Here => return Some(at), + path::Forward::Next { next, named, .. } => (at, id) = (next, named), + } + } +} + +/// Three relays in a ring, each next to its own publisher of `live`, and each giving +/// its clockwise link `priority`. +fn priority_ring(priority: i8) -> Sim { + let mut sim = Sim::new(6); + for publisher in 3..6 { + sim.leaf(publisher); + } + for relay in 0..3 { + sim.link( + relay, + (relay + 1) % 3, + SimLink::cost(1).priority(priority), + SimLink::cost(1), + ); + sim.join(relay + 3, relay, 1); + } + for publisher in 3..6 { + publish(&mut sim, publisher, publisher as Id, &["live/"]); + } + assert!(sim.run()); + sim +} + +/// Local priorities that each favour the clockwise link. In the model a request that +/// named only its path picked another publisher at every relay and circled the ring, +/// and three relays subscribing at once joined each other's subscriptions and starved. +/// A request naming its publisher reaches it from every relay, alone or joined. +#[test] +fn local_priorities_never_turn_a_named_request() { + for priority in [0, 1] { + let sim = priority_ring(priority); + let claim = p("live/x"); + for relay in 0..3 { + for publisher in 3..6 { + let id = publisher as Id; + let label = format!("priority {priority}, relay {relay}, publisher {publisher}"); + assert_eq!(sim.reaches(relay, id, &claim), Some(publisher), "{label}"); + assert_eq!(joined(&sim, relay, id, &claim), Some(publisher), "{label}"); + } + } + } +} + +/// Two publishers of `live/x` behind one core, the second cheaper. A relay that picked +/// the first keeps it however the core would rank them: the core forwards the request +/// it was handed rather than splicing in the publisher it would pick itself. +#[test] +fn a_transit_relay_never_swaps_the_named_publisher() { + let (edge, core, first, second) = (0, 1, 2, 3); + let mut sim = Sim::new(4); + sim.leaf(first); + sim.leaf(second); + sim.join(edge, core, 1); + sim.join(first, core, 2); + sim.join(second, core, 1); + publish(&mut sim, first, 2, &["live/x"]); + publish(&mut sim, second, 3, &["live/x"]); + assert!(sim.run()); + let claim = p("live/x"); + assert_eq!(sim.reaches(edge, 2, &claim), Some(first)); + assert_eq!(sim.reaches(edge, 3, &claim), Some(second)); + assert_eq!(joined(&sim, edge, 2, &claim), Some(first)); +} + +/// Nested claims `live`, `live/a`, and `live/a/x` from three publishers on a ring with a +/// chord. In the model each relay took the longest claim it had heard, so relay 0, +/// missing the narrower claims, forwarded toward the wildcard while its neighbours +/// forwarded back through it toward a narrower one: a three-hop loop that requester +/// exclusion missed. Here relay 0 hears the narrower claims late, and a request for +/// any of the three publishers reaches it from every relay, before and after. +#[test] +fn nested_claims_never_loop_a_named_request() { + let mut sim = Sim::new(7); + for publisher in 4..7 { + sim.leaf(publisher); + } + sim.join(0, 1, 2); + sim.join(1, 2, 1); + sim.join(2, 3, 3); + sim.join(3, 0, 1); + sim.join(0, 2, 2); + sim.join(4, 2, 1); + sim.join(5, 1, 1); + sim.join(6, 3, 1); + publish(&mut sim, 4, 4, &["live/"]); + assert!(sim.run()); + // Relay 0 hears nothing more until the narrower claims are everywhere else. + for peer in [1, 2, 3] { + sim.hold(peer, 0); + } + publish(&mut sim, 5, 5, &["live/a/"]); + publish(&mut sim, 6, 6, &["live/a/x"]); + assert!(sim.run()); + let claim = p("live/a/x"); + let check = |sim: &Sim, label: &str| { + for relay in 0..4 { + for publisher in 4..7 { + let id = publisher as Id; + if relay == 0 && sim.queued(1, 0) + sim.queued(2, 0) + sim.queued(3, 0) > 0 && publisher > 4 { + // Relay 0 has no route to the narrower publishers yet. + continue; + } + assert_eq!( + sim.reaches(relay, id, &claim), + Some(publisher), + "{label}: relay {relay}" + ); + assert_eq!( + joined(sim, relay, id, &claim), + Some(publisher), + "{label}: relay {relay}" + ); + } + } + }; + check(&sim, "relay 0 behind"); + for peer in [1, 2, 3] { + sim.release(peer, 0); + } + assert!(sim.run()); + check(&sim, "settled"); +} + +/// Five relays on a ring with chords, random costs and priorities, and three +/// publishers each claiming `live`, `live/a`, or `live/b`. Once settled, every relay's +/// request for every publisher reaches it, alone or joined. Then a ring link closes, and at +/// every round until it settles again no request loops except through a stale route, +/// and once settled every relay reaches every publisher again. +#[test] +fn sweep_named_requests_arrive() { + const RELAYS: usize = 5; + let mut rng = XorShift(0x5eed_c0de); + let prefixes = ["live/", "live/a/", "live/b"]; + for scenario in 0..scenarios() { + let mut sim = Sim::new(RELAYS + 3); + for publisher in RELAYS..RELAYS + 3 { + sim.leaf(publisher); + } + for relay in 0..RELAYS { + let side = |rng: &mut XorShift| SimLink::cost(1 + rng.next(3)).priority(if rng.chance(25) { 1 } else { 0 }); + let (ab, ba) = (side(&mut rng), side(&mut rng)); + sim.link(relay, (relay + 1) % RELAYS, ab, ba); + } + for _ in 0..2 { + let (a, b) = (rng.next(RELAYS as u64) as usize, rng.next(RELAYS as u64) as usize); + if a.abs_diff(b) > 1 && a.abs_diff(b) != RELAYS - 1 && !sim.connected(a, b) { + sim.join(a, b, 1 + rng.next(3)); + } + } + for publisher in RELAYS..RELAYS + 3 { + sim.join(publisher, rng.next(RELAYS as u64) as usize, 1); + let prefix = prefixes[rng.next(3) as usize]; + publish(&mut sim, publisher, publisher as Id, &[prefix]); + } + assert!(sim.run(), "scenario {scenario} settles"); + + // Every relay watches the root, so its route covers any path: the request reaches + // the publisher, which answers for what it does not claim. + let arrive = |sim: &Sim, label: &str| { + for at in 0..RELAYS { + for publisher in RELAYS..RELAYS + 3 { + let id = publisher as Id; + for path in ["live/a/x", "live/b"] { + let claim = p(path); + let label = format!("scenario {scenario} {label}, relay {at}, publisher {publisher}, {path}"); + assert_eq!(sim.reaches(at, id, &claim), Some(publisher), "{label}"); + assert_eq!(joined(sim, at, id, &claim), Some(publisher), "{label}"); + } + } + } + }; + arrive(&sim, "settled"); + + let cut = rng.next(RELAYS as u64) as usize; + sim.disconnect(cut, (cut + 1) % RELAYS); + let mut quiet = false; + for _ in 0..1_000 { + for publisher in RELAYS..RELAYS + 3 { + assert_loop_free(&sim, RELAYS + 3, publisher as Id); + } + if sim.busy().is_empty() && sim.deadline().is_none() { + quiet = true; + break; + } + sim.settle(1); + } + assert!(quiet, "scenario {scenario} settles after the cut"); + arrive(&sim, "after the cut"); + } +} + +/// A scoped peer declares a victim's identity as its own and claims what the victim +/// claims, more cheaply. Admitted unscoped, its routes merge with the victim's and +/// requests for the victim's claims outside the forger's namespace go to the forger. +/// Admitted under a scoped grant, its identity is filed under a key derived from the +/// grant's subject, which carries only claims inside its scope, so the victim keeps +/// its own claims. +#[test] +fn a_forged_identity_is_contained_by_a_scoped_admission() { + const FORGER: Grant = Grant { + subject: "forger", + scope: "forger/", + }; + for scoped in [false, true] { + let (victim, core, edge, forger) = (0, 1, 2, 3); + let mut sim = Sim::new(4); + sim.leaf(victim); + sim.leaf(forger); + sim.join(victim, core, 5); + sim.join(core, edge, 1); + let admission = match scoped { + true => SimLink::cost(1).grant(FORGER), + false => SimLink::cost(1), + }; + sim.link(edge, forger, admission, SimLink::cost(1)); + publish(&mut sim, victim, 7, &["other/", "forger/"]); + publish(&mut sim, forger, 7, &["other/", "forger/"]); + assert!(sim.run()); + + let other = p("other/x"); + match scoped { + false => assert_eq!( + sim.reaches(core, 7, &other), + Some(forger), + "the forger serves another namespace" + ), + true => { + assert_eq!(sim.reaches(core, 7, &other), Some(victim)); + let key = qualified(FORGER.subject, 7); + assert_eq!(sim.shows(core, key, "other"), None, "outside the forger's scope"); + assert_eq!( + sim.reaches(core, key, &p("forger/x")), + Some(forger), + "the forgery stays inside its own scope" + ); + } + } + } +} + +/// A local mesh publishing through two uplinks, each admitting its local relay under +/// the same subject, so both CDN relays file the local publisher under one key. It +/// survives losing either uplink: the CDN keeps the key and a request follows the +/// other uplink. The local relay's own identity is filed the same way and fails over +/// alike. Claims outside the scope are refused under that key. +#[test] +fn scoped_delivery_through_two_uplinks_reaches_one_key() { + let (c1, c2, l1, l2, origin) = (0, 1, 2, 3, 4); + let mut sim = Sim::new(5); + sim.leaf(origin); + sim.join(c1, c2, 10); + sim.join(l1, l2, 1); + let admit = SimLink::cost(1).grant(LOCAL); + sim.link(l1, c1, SimLink::cost(1).upstream(), admit); + sim.link(l2, c2, SimLink::cost(1).upstream(), admit); + sim.join(origin, l1, 1); + publish(&mut sim, origin, 9, &["local/", "cdn/"]); + publish(&mut sim, l1, 20, &["local/l1/"]); + assert!(sim.run()); + + let key = qualified(LOCAL.subject, 9); + let own = qualified(LOCAL.subject, 20); + assert!(!sim.routed(c1, 9), "a scoped peer's identities are qualified"); + assert!(!sim.routed(c1, 20), "its own included"); + assert_eq!(sim.next_hop(c1, key), Some(l1)); + assert_eq!(sim.next_hop(c2, key), Some(l2)); + assert_eq!(sim.next_hop(c1, own), Some(l1)); + assert_eq!(sim.shows(c2, key, "cdn"), None, "outside the scope"); + assert_eq!(sim.reaches(l1, 9, &p("cdn/x")), Some(origin)); + + sim.disconnect(l1, c1); + assert!(sim.run()); + assert_eq!(sim.next_hop(c1, own), Some(c2)); + assert_eq!(sim.reaches(c1, own, &p("local/l1/x")), Some(l1)); + assert_eq!(sim.reaches(c1, key, &p("local/x")), Some(origin)); + + sim.connect(l1, c1); + sim.disconnect(l2, c2); + assert!(sim.run()); + assert_eq!(sim.next_hop(c2, key), Some(c1)); + assert_eq!(sim.reaches(c2, key, &p("local/x")), Some(origin)); +} + +/// Two CDN relays, each the uplink of one relay in a local mesh, and a CDN publisher +/// on the first. The CDN admits each local relay under `LOCAL`, and the second local +/// relay forgets to mark its uplink upstream. +fn leaky_uplinks() -> Sim { + let (c1, c2, l1, l2, origin) = (0, 1, 2, 3, 4); + let mut sim = Sim::new(5); + sim.leaf(origin); + sim.join(c1, c2, 10); + sim.join(l1, l2, 1); + let admit = SimLink::cost(1).grant(LOCAL); + sim.link(l1, c1, SimLink::cost(1).upstream(), admit); + sim.link(l2, c2, SimLink::cost(1), admit); + sim.join(origin, c1, 1); + publish(&mut sim, origin, 7, &[""]); + sim +} + +/// The leaked CDN route comes back to the CDN filed under the local subject's key, and +/// that key's route then leaks down and up again. A scoped connection forwarding an +/// already qualified key has that route refused, so the lap ends after one filing +/// instead of minting a new identity on every lap. +#[test] +fn a_qualified_identity_is_never_qualified_again() { + let (c1, c2, l2) = (0, 1, 3); + let mut sim = leaky_uplinks(); + assert!(sim.run(), "the leak settles"); + let key = qualified(LOCAL.subject, 7); + assert_eq!(sim.next_hop(c2, 7), Some(c1)); + assert_eq!(sim.next_hop(c2, key), Some(l2), "the leak is filed apart once"); + for node in [c1, c2] { + assert!( + !sim.routed(node, qualified(LOCAL.subject, key)), + "node {node} never files the key again" + ); + } +} + +/// Redundant cores: an edge losing the core it routes through moves to the other at +/// once, since the other core's route avoids the closed session's far end. +#[test] +fn redundant_cores_fail_over_at_once() { + let (ingest, core_a, core_b, edge) = (0, 1, 2, 3); + let mut sim = Sim::new(4); + sim.leaf(ingest); + sim.join(ingest, core_a, 1); + sim.join(ingest, core_b, 1); + sim.join(core_a, core_b, 1); + sim.link(edge, core_a, SimLink::cost(1).upstream(), SimLink::cost(1)); + sim.link(edge, core_b, SimLink::cost(1).upstream(), SimLink::cost(1)); + publish(&mut sim, ingest, 7, &[""]); + assert!(sim.run()); + let primary = sim.next_hop(edge, 7).expect("a route"); + let other = if primary == core_a { core_b } else { core_a }; + + sim.disconnect(edge, primary); + assert_eq!(sim.next_hop(edge, 7), Some(other)); + assert_eq!(sim.shows(edge, 7, ""), Some(true), "never unavailable"); + assert_eq!(sim.reaches(edge, 7, &p("x")), Some(ingest)); +} + +/// A publisher reached directly over a preferred peer-to-peer session and through the +/// CDN, by a subscriber that is a browser or a routing node. Losing the direct session +/// keeps the claim shown, and the subscriber ends up on the CDN route either way. +#[test] +fn a_lost_p2p_session_falls_back_to_the_cdn() { + for transit in [false, true] { + let (publisher, edge, subscriber) = (0, 1, 2); + let mut sim = Sim::new(3); + sim.leaf(publisher); + if !transit { + sim.leaf(subscriber); + } + sim.join(publisher, edge, 1); + sim.link(subscriber, edge, SimLink::cost(1).upstream(), SimLink::cost(1)); + sim.link(subscriber, publisher, SimLink::cost(1).priority(1), SimLink::cost(1)); + publish(&mut sim, publisher, 9, &[""]); + assert!(sim.run()); + assert_eq!(sim.next_hop(subscriber, 9), Some(publisher), "transit {transit}"); + + sim.disconnect(subscriber, publisher); + assert!(sim.shows(subscriber, 9, "").is_some(), "transit {transit}: still shown"); + assert!(sim.run()); + assert_eq!(sim.next_hop(subscriber, 9), Some(edge), "transit {transit}"); + assert_eq!(sim.shows(subscriber, 9, ""), Some(true), "transit {transit}"); + } +} + +/// A local relay whose own uplink is down still consumes CDN content and publishes +/// local content through its LAN peer: the down-only route flows down the LAN, and +/// the exportable one up the other uplink. +#[test] +fn a_dead_uplink_consumes_and_publishes_through_a_lan_peer() { + let (c1, c2, l1, l2, cdn, local) = (0, 1, 2, 3, 4, 5); + let mut sim = Sim::new(6); + sim.leaf(cdn); + sim.leaf(local); + sim.join(c1, c2, 10); + sim.join(l1, l2, 1); + sim.link(l1, c1, SimLink::cost(1).upstream(), SimLink::cost(1)); + sim.link(l2, c2, SimLink::cost(1).upstream(), SimLink::cost(1)); + sim.disconnect(l2, c2); + sim.join(cdn, c1, 1); + sim.join(local, l2, 1); + publish(&mut sim, cdn, 7, &["cdn/"]); + publish(&mut sim, local, 9, &["local/"]); + assert!(sim.run()); + + assert_eq!(sim.next_hop(l2, 7), Some(l1)); + assert_eq!(sim.reaches(l2, 7, &p("cdn/x")), Some(cdn)); + assert_eq!(sim.next_hop(c2, 9), Some(c1)); + assert_eq!(sim.reaches(c2, 9, &p("local/x")), Some(local)); +} + +/// `L2` reaches a local publisher over the LAN through `L1`, and `C2`, which `L2` marks +/// upstream, routes through `L2` until the CDN hears the publisher itself. Then `L2`'s +/// cheapest route runs down from `C2`, so it is down-only and never offered back up: +/// `C2` moves onto the CDN, and a request `C2` forwarded to `L2` before stops at `L2` +/// rather than following a route back up through `C2`. +#[test] +fn an_upstream_request_stops_once_the_route_turns_down_only() { + let (c1, c2, l1, l2, origin) = (0, 1, 2, 3, 4); + let mut sim = Sim::new(5); + sim.leaf(origin); + sim.join(c1, c2, 1); + sim.join(origin, l1, 10); + sim.join(l2, l1, 1); + sim.link(l2, c2, SimLink::cost(1).upstream(), SimLink::cost(1)); + publish(&mut sim, origin, 7, &["local/"]); + assert!(sim.run()); + assert_eq!(sim.next_hop(c2, 7), Some(l2)); + let claim = p("local/x"); + assert_eq!(sim.reaches(c2, 7, &claim), Some(origin)); + let Some(path::Forward::Next { prefix, .. }) = sim.path(c2).forward(7, &claim, None) else { + panic!("C2 forwards to L2"); + }; + + sim.join(origin, c1, 1); + assert!(sim.run()); + assert_eq!(sim.next_hop(l2, 7), Some(c2)); + let route = sim.path(l2).route_for(7, &Path::empty()).and_then(path::Route::offer); + assert!(route.expect("a route").down_only); + assert_eq!(sim.next_hop(c2, 7), Some(c1), "C2 re-resolves off the LAN"); + assert_eq!(sim.reaches(c2, 7, &claim), Some(origin)); + assert!( + sim.path(l2).forward(7, &claim, Some((c2, prefix))).is_none(), + "L2 stops the request it accepted from upstream" + ); +} + +/// A local relay `L2` reaching a local publisher over the LAN and through its uplink, +/// with the LAN or the uplink coming up first. The path rule ranks each route by the +/// lowest link priority on it and then by cost, with no feasibility condition, so the +/// same topology settles on the same route whichever session came up first. Under +/// Babel a transit relay kept a longer route it heard first. +#[test] +fn transit_selection_does_not_depend_on_arrival_order() { + let (c1, c2, l1, l2, origin) = (0, 1, 2, 3, 4); + for prefer_uplink in [false, true] { + let settled = |lan_first: bool| { + let mut sim = Sim::new(5); + sim.leaf(origin); + sim.join(c1, c2, 10); + sim.join(origin, l1, 1); + publish(&mut sim, origin, 7, &["local/"]); + let (uplink, lan) = if prefer_uplink { (1, 0) } else { (0, 1) }; + let lan = |sim: &mut Sim| sim.link(l2, l1, SimLink::cost(1).priority(lan), SimLink::cost(1)); + if lan_first { + lan(&mut sim); + } + sim.link(l1, c1, SimLink::cost(1).upstream(), SimLink::cost(1)); + sim.link(l2, c2, SimLink::cost(1).upstream().priority(uplink), SimLink::cost(1)); + assert!(sim.run()); + if !lan_first { + lan(&mut sim); + } + assert!(sim.run()); + sim.next_hop(l2, 7) + }; + assert_eq!(settled(false), settled(true), "prefer uplink {prefer_uplink}"); + } +} diff --git a/rs/moq-net/src/route/tests/mod.rs b/rs/moq-net/src/route/tests/mod.rs index 0d0eda6c457..5a89775b228 100644 --- a/rs/moq-net/src/route/tests/mod.rs +++ b/rs/moq-net/src/route/tests/mod.rs @@ -11,6 +11,7 @@ mod cache; mod compare; mod cost; +mod forward; mod handoff; mod interest; mod residual; diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index c2e26d98c21..0d2bb9be13e 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -69,7 +69,6 @@ mod route_change; mod route_cluster; mod route_failover; mod route_health; -mod routing_contracts; mod segmented_frame; mod session_close; mod session_panic; diff --git a/rs/moq-net/tests/integration/route_acceptance.rs b/rs/moq-net/tests/integration/route_acceptance.rs index b34921ddca4..1a6f065e6bc 100644 --- a/rs/moq-net/tests/integration/route_acceptance.rs +++ b/rs/moq-net/tests/integration/route_acceptance.rs @@ -547,3 +547,53 @@ async fn a_primary_that_fails_and_returns_takes_every_relay_back() { ); } } + +/// First-hop selection across a pool: three members each claim `transcode` through two +/// hubs, and two edges hang off one hub each, so every edge reaches every member at the +/// same cost and the tie decides. The tie hashes the path and the member's node rather +/// than anything local to the edge, so both edges pick the same member for every path, +/// and the picks spread over the pool. +#[kio::test] +async fn pool_ties_agree_on_every_edge() { + let version = lite07(); + let [edge_a, edge_b, hub_a, hub_b] = [1, 2, 3, 4].map(produce_origin); + let members = [5, 6, 7].map(produce_origin); + let mut pairs = vec![ + peer(version, &edge_a, &hub_a).await, + peer(version, &edge_b, &hub_b).await, + peer(version, &hub_a, &hub_b).await, + ]; + let mut claims = Vec::new(); + for member in &members { + claims.push(member.claim("transcode", origin::RouteOptions::default()).unwrap()); + pairs.push(peer(version, member, &hub_a).await); + pairs.push(peer(version, member, &hub_b).await); + } + quiesce().await; + + let instances: Vec<_> = claims.iter().map(|claim| Some(claim.instance())).collect(); + let mut picked = std::collections::BTreeSet::new(); + for i in 0..32 { + let path = format!("transcode/{i}"); + let a = edge_a + .consume() + .request(path.as_str(), None) + .await + .unwrap() + .info() + .instance + .clone(); + let b = edge_b + .consume() + .request(path.as_str(), None) + .await + .unwrap() + .info() + .instance + .clone(); + assert_eq!(a, b, "{path}"); + picked.insert(instances.iter().position(|instance| *instance == a).expect("a member")); + } + assert!(picked.len() > 1, "every path picked one member: {picked:?}"); + drop(pairs); +} diff --git a/rs/moq-net/tests/integration/routing_contracts/handoff.rs b/rs/moq-net/tests/integration/routing_contracts/handoff.rs deleted file mode 100644 index 37dd0597cd1..00000000000 --- a/rs/moq-net/tests/integration/routing_contracts/handoff.rs +++ /dev/null @@ -1,1493 +0,0 @@ -//! Next-hop handoff: complete and fresh announcements across route changes. -//! -//! Each origin versions its own claims with a revision that starts at zero for -//! each routing identity. A node sends each neighbor its view of an origin's -//! claims within that neighbor's interest, closed by floors that certify the -//! view complete as of a revision for the region they cover. Receivers merge -//! every neighbor's view claim by claim, keeping whichever knowledge is newer, -//! and keep showing an origin's claims while any route to it is usable or -//! recovering. See `quest/m2/cluster-routes/README.md`. - -use std::collections::BTreeSet; - -use super::XorShift; -use super::model::{ - Config, Event, Freshness, Grant, Id, Kind, Link, Net, Requests, VARINT_MAX, overlaps, qualified, within, -}; - -const NEXT_HOP: Freshness = Freshness::NextHop; -const BARRIER: Freshness = Freshness::Barrier; -const FLOORS: Freshness = Freshness::Floors { scoped: true }; - -fn config(freshness: Freshness, requests: Requests) -> Config { - Config { - freshness, - requests, - ..Config::default() - } -} - -fn at(net: &Net, node: usize) -> Vec { - net.events.iter().filter(|event| event.node == node).copied().collect() -} - -fn kinds(events: &[Event]) -> Vec { - events.iter().map(|event| event.kind).collect() -} - -/// Starts of a claim's life that the same node had already shown ending. -fn revivals(events: &[Event]) -> usize { - let mut ended = BTreeSet::new(); - let mut count = 0; - for event in events { - let life = (event.node, event.forgot, event.id, event.claim, event.since); - match event.kind { - Kind::End => { - ended.insert(life); - } - Kind::Start if ended.contains(&life) => count += 1, - _ => {} - } - } - count -} - -/// Ends shown while the origin still had that life of the claim. -fn false_ends(events: &[Event]) -> usize { - events - .iter() - .filter(|event| event.kind == Kind::End && !event.withdrawn) - .count() -} - -/// Starts of a claim's life that the same node had already shown going away. -fn reannouncements(events: &[Event]) -> usize { - let mut gone = BTreeSet::new(); - let mut count = 0; - for event in events { - let life = (event.node, event.forgot, event.id, event.claim, event.since); - match event.kind { - Kind::End | Kind::Offline => { - gone.insert(life); - } - Kind::Start if gone.contains(&life) => count += 1, - _ => {} - } - } - count -} - -/// An origin reaching a node through two relays, the first preferred and the -/// second a feasible successor. -fn two_relays(freshness: Freshness) -> Net { - let (origin, a, b, node) = (0, 1, 2, 3); - let mut net = Net::new(config(freshness, Requests::Off), 4); - net.leaf(origin); - net.join(origin, a, 1); - net.join(origin, b, 1); - net.join(node, a, 1); - net.join(node, b, 2); - net.originate(origin, 7, 0); - assert!(net.run()); - net -} - -/// A withdrawal reaches the node through its next hop while the alternative's -/// copy is still in flight. When the next hop goes, the alternative's complete -/// snapshot is older than what the node already knows: adopting it, by next-hop -/// authority or behind a snapshot barrier, revives the claim until the -/// withdrawal catches up. Floors refuse the older snapshot. -#[test] -fn an_older_complete_snapshot_revives_a_known_withdrawal() { - let (origin, a, b, node) = (0, 1, 2, 3); - for freshness in [NEXT_HOP, BARRIER, FLOORS] { - let mut net = two_relays(freshness); - net.announce(origin, "live/x", 0, None); - assert!(net.run()); - assert_eq!(net.shows(node, 7, "live/x"), Some(true)); - - net.hold(origin, b); - net.unannounce(origin, "live/x"); - assert!(net.run()); - assert_eq!(net.shows(node, 7, "live/x"), None, "{freshness:?}"); - // Forgetting another origin excuses no replay of this one. - net.collect(node, 99); - - net.disconnect(node, a); - assert!(net.run()); - assert_eq!(net.route(node, 7).and_then(|route| route.next), Some(b)); - let revived = net.shows(node, 7, "live/x").is_some(); - assert_eq!(revived, freshness != FLOORS, "{freshness:?}"); - - net.release(origin, b); - assert!(net.run()); - assert_eq!(net.shows(node, 7, "live/x"), None); - let events = at(&net, node); - assert_eq!(revivals(&events), usize::from(freshness != FLOORS), "{freshness:?}"); - if freshness == FLOORS { - assert_eq!(kinds(&events), [Kind::Start, Kind::End]); - net.check().unwrap(); - } - } -} - -/// A new next hop's ROUTE arrives before its announcements. Next-hop authority -/// shows the new next hop's empty set, a false END followed by a -/// re-announcement; merged views keep what the node knows and the reader sees -/// nothing change. -#[test] -fn next_hop_authority_reports_false_absence_while_announcements_trail() { - let (origin, a, b, node) = (0, 1, 2, 3); - for freshness in [NEXT_HOP, FLOORS] { - let mut net = two_relays(freshness); - net.disconnect(node, b); - net.announce(origin, "live/x", 0, None); - assert!(net.run()); - let before = net.events.len(); - - net.hold(b, node); - net.connect(node, b); - assert!(net.run()); - assert!(net.queued(b, node) >= 2, "a ROUTE then its announcements"); - assert!(net.deliver(b, node), "only the ROUTE"); - net.disconnect(node, a); - assert!(net.run()); - assert_eq!(net.route(node, 7).and_then(|route| route.next), Some(b)); - let trailing = net.shows(node, 7, "live/x"); - - net.release(b, node); - assert!(net.run()); - assert_eq!(net.shows(node, 7, "live/x"), Some(true)); - let events: Vec = net.events[before..] - .iter() - .filter(|e| e.node == node) - .copied() - .collect(); - if freshness == NEXT_HOP { - assert_eq!(trailing, None); - assert_eq!(false_ends(&events), 1); - assert_eq!(reannouncements(&events), 1); - } else { - assert_eq!(trailing, Some(true)); - assert!(events.is_empty(), "{events:?}"); - } - } -} - -/// The partitioned local mesh from the routing contract, with a client behind -/// the starved relay. The CDN detour is infeasible until the origin advances -/// its seqno, and through that window the relay and its client keep the claim -/// as unavailable rather than ending it. Next-hop authority ends it. With seqno -/// requests the relay recovers through the CDN by itself. -#[test] -fn a_starved_relay_keeps_presence_unavailable_until_its_request_recovers_it() { - let (c1, c2, l1, l2, origin, client) = (0, 1, 2, 3, 4, 5); - let build = |freshness, requests| { - let mut net = Net::new(config(freshness, requests), 6); - net.join(c1, c2, 10); - net.join(l1, l2, 1); - net.link(l1, c1, Link::cost(1).upstream(), Link::cost(1)); - net.link(l2, c2, Link::cost(1).upstream(), Link::cost(1)); - net.leaf(origin); - net.leaf(client); - net.join(origin, l2, 1); - net.join(client, l1, 1); - net.originate(origin, 7, 0); - net.announce(origin, "local/x", 0, None); - assert!(net.run()); - assert_eq!(net.shows(client, 7, "local/x"), Some(true)); - net - }; - - let mut net = build(NEXT_HOP, Requests::Off); - net.disconnect(l1, l2); - assert!(net.run()); - assert!(net.starved(l1, 7)); - assert_eq!(false_ends(&net.events), 2, "the relay and its client"); - - let mut net = build(FLOORS, Requests::Off); - net.disconnect(l1, l2); - assert!(net.run()); - assert!(net.starved(l1, 7)); - assert!(net.grounded(client, 7)); - for node in [l1, client] { - assert_eq!(net.shows(node, 7, "local/x"), Some(false), "node {node}"); - } - assert_eq!(false_ends(&net.events), 0); - - let mut net = build(FLOORS, Requests::HopByHop); - let before = net.events.len(); - net.disconnect(l1, l2); - assert!(net.run()); - assert_eq!(net.seqno(origin), 1, "the request reached the origin"); - assert_eq!(net.route(l1, 7).and_then(|route| route.next), Some(c1)); - let events: Vec = net.events[before..] - .iter() - .filter(|e| e.node == client) - .copied() - .collect(); - assert_eq!(kinds(&events), [Kind::Unavailable, Kind::Available]); -} - -/// A client of two relays, one lagging behind its origin. -fn lagging(freshness: Freshness, watch: &'static str) -> Net { - let (origin, r1, r2, client) = (0, 1, 2, 3); - let mut net = Net::new(config(freshness, Requests::Off), 4); - net.leaf(origin); - net.leaf(client); - net.join(origin, r1, 1); - net.join(origin, r2, 1); - net.join(client, r1, 1); - net.join(client, r2, 1); - net.watch(client, watch); - net.originate(origin, 7, 0); - net -} - -/// A client watching `live/a/` knows `live/a/x` ended at revision 3, then widens -/// to everything. The lagging relay's wider snapshot, at revision 2, must not -/// revive `live/a/x`, yet is the first to report `live/b/y`. Floors scoped to -/// the region they cover do both. One floor per origin either refuses the wide -/// snapshot, leaving `live/b/y` missing until the origin changes something, or -/// would accept it and revive `live/a/x`. -#[test] -fn widening_interest_needs_floors_scoped_to_what_they_cover() { - let (origin, r1, r2, client) = (0, 1, 2, 3); - for scoped in [false, true] { - let mut net = lagging(Freshness::Floors { scoped }, "live/a/"); - net.announce(origin, "live/a/x", 0, None); - net.announce(origin, "live/b/y", 0, None); - assert!(net.run()); - net.hold(origin, r2); - net.unannounce(origin, "live/a/x"); - assert!(net.run()); - assert!(net.shown(client).is_empty()); - - net.hold(r1, client); - net.watch(client, ""); - assert!(net.run()); - net.release(r1, client); - assert!(net.run()); - assert_eq!(net.shows(client, 7, "live/a/x"), None); - assert_eq!(net.shows(client, 7, "live/b/y").is_some(), scoped, "scoped {scoped}"); - assert_eq!(revivals(&net.events), 0); - if scoped { - net.check().unwrap(); - } - } -} - -/// A client narrows its interest and later widens it again. Narrowing retires -/// what it knew outside the new interest, so its state follows the current -/// interest rather than every prefix it ever watched. On rejoin a lagging -/// relay's snapshot replays a withdrawal the client saw before it narrowed, -/// as after a collection, and only until the newer snapshot behind it lands. -#[test] -fn cancelling_interest_retires_its_region() { - let (origin, r1, r2, client) = (0, 1, 2, 3); - let mut net = lagging(FLOORS, ""); - net.announce(origin, "live/a/x", 0, None); - assert!(net.run()); - net.hold(origin, r2); - net.unannounce(origin, "live/a/x"); - assert!(net.run()); - assert_eq!(net.shows(client, 7, "live/a/x"), None); - - net.watch(client, "live/b/"); - assert!(net.run()); - assert_eq!(net.floors(client, 7), 1, "only the watched region"); - net.hold(r1, client); - net.watch(client, ""); - assert!(net.run()); - assert_eq!( - net.shows(client, 7, "live/a/x"), - Some(true), - "the lagging relay replays" - ); - net.release(r1, client); - assert!(net.run()); - assert_eq!(net.shows(client, 7, "live/a/x"), None); - net.release(origin, r2); - assert!(net.run()); - let life: Vec = at(&net, client) - .iter() - .filter(|event| event.claim == "live/a/x") - .map(|event| event.kind) - .collect(); - assert_eq!(life, [Kind::Start, Kind::End, Kind::Start, Kind::End]); - assert_eq!(revivals(&net.events), 0, "a retired region, like a collection"); - assert_eq!(net.floors(client, 7), 1); - net.check().unwrap(); -} - -/// A client narrows its interest, and a claim outside it ends unseen. On -/// widening, what it kept there is frozen at a floor it no longer refreshed, -/// so it shows nothing in the widened region until a floor covering it -/// arrives, rather than a stale Start the next snapshot ends. -#[test] -fn widening_hides_what_was_frozen_until_a_covering_floor_arrives() { - let (origin, r1, r2, client) = (0, 1, 2, 3); - let mut net = lagging(FLOORS, ""); - net.announce(origin, "live/a/x", 0, None); - net.announce(origin, "live/b/y", 0, None); - assert!(net.run()); - net.watch(client, "live/b/"); - assert!(net.run()); - net.unannounce(origin, "live/a/x"); - assert!(net.run()); - - net.hold(r1, client); - net.hold(r2, client); - net.watch(client, ""); - assert_eq!(net.shows(client, 7, "live/a/x"), None, "frozen while unwatched"); - assert_eq!(net.shows(client, 7, "live/b/y"), Some(true), "still current"); - - net.release(r1, client); - net.release(r2, client); - assert!(net.run()); - let life: Vec = at(&net, client) - .iter() - .filter(|event| event.claim == "live/a/x") - .map(|event| event.kind) - .collect(); - assert_eq!(life, [Kind::Start, Kind::Unwatched]); - net.check().unwrap(); -} - -/// A snapshot sent before the client narrowed still waits on one session when -/// it widens again. Its floor covers the widened region but answers the old -/// interest, so it must not lift the guard on what stayed frozen. -#[test] -fn a_snapshot_queued_before_widening_keeps_the_frozen_rest_hidden() { - let (origin, r1, r2, client) = (0, 1, 2, 3); - let mut net = lagging(FLOORS, ""); - net.announce(origin, "live/a/x", 0, None); - net.announce(origin, "live/b/y", 0, None); - assert!(net.run()); - net.hold(r2, client); - net.announce(origin, "live/b/z", 0, None); - assert!(net.run()); - assert!(net.queued(r2, client) > 0, "a root snapshot waits"); - net.watch(client, "live/b/"); - assert!(net.run()); - net.unannounce(origin, "live/a/x"); - assert!(net.run()); - - net.hold(r1, client); - net.watch(client, ""); - assert!(net.deliver(r2, client), "the root snapshot from before narrowing"); - assert_eq!(net.shows(client, 7, "live/a/x"), None, "it answers the old interest"); - - net.release(r1, client); - net.release(r2, client); - assert!(net.run()); - let life: Vec = at(&net, client) - .iter() - .filter(|event| event.claim == "live/a/x") - .map(|event| event.kind) - .collect(); - assert_eq!(life, [Kind::Start, Kind::Unwatched]); - net.check().unwrap(); -} - -/// A client behind a scoped grant widens its interest. Every batch it gets is -/// cut to the grant's scope, so no floor ever covers the widened interest; -/// the batch answering it is still what shows the newly covered claims. The -/// certified region is kept once however many batches repeat it, and the -/// freshness oracle checks the qualified view against the origin's history. -#[test] -fn widening_behind_a_scoped_grant_shows_the_rest_of_the_scope() { - let (origin, relay, client) = (0, 1, 2); - let mut net = Net::new(config(FLOORS, Requests::Off), 3); - net.leaf(origin); - net.leaf(client); - net.join(origin, relay, 1); - let grant = Grant { - subject: "local", - scope: "live/a/", - }; - net.link(client, relay, Link::cost(1).grant(grant), Link::cost(1)); - net.watch(client, "live/a/x/"); - net.originate(origin, 7, 0); - net.announce(origin, "live/a/x/1", 0, None); - net.announce(origin, "live/a/y", 0, None); - assert!(net.run()); - let id = qualified("local", 7); - assert_eq!(net.shows(client, id, "live/a/x/1"), Some(true)); - assert_eq!(net.shows(client, id, "live/a/y"), None); - - net.watch(client, ""); - assert!(net.run()); - assert_eq!(net.shows(client, id, "live/a/y"), Some(true)); - let certified = net.retained().certified; - for _ in 0..8 { - net.announce(origin, "live/a/z", 0, None); - net.unannounce(origin, "live/a/z"); - assert!(net.run()); - } - assert_eq!(net.retained().certified, certified, "claim churn adds no state"); - assert!( - net.check().unwrap().contains(&(client, id)), - "the qualified view is checked" - ); -} - -/// Two scoped grants qualify one origin under the same subject, so their -/// batches merge into one view. After widening, a batch through the first -/// grant certifies only its own scope: a claim in the second scope that ended -/// while unwatched stays hidden until the second grant answers too. -#[test] -fn widening_behind_two_grants_waits_for_each_scope() { - let (origin, r1, r2, client) = (0, 1, 2, 3); - let mut net = Net::new(config(FLOORS, Requests::Off), 4); - net.leaf(origin); - net.leaf(client); - net.join(origin, r1, 1); - net.join(origin, r2, 1); - for (relay, scope) in [(r1, "live/a/"), (r2, "live/b/")] { - let grant = Grant { - subject: "local", - scope, - }; - net.link(client, relay, Link::cost(1).grant(grant), Link::cost(1)); - } - net.originate(origin, 7, 0); - net.announce(origin, "live/a/x/1", 0, None); - net.announce(origin, "live/b/z", 0, None); - assert!(net.run()); - let id = qualified("local", 7); - assert_eq!(net.shows(client, id, "live/b/z"), Some(true)); - - net.hold(r1, client); - net.hold(r2, client); - net.watch(client, "live/a/x/"); - net.unannounce(origin, "live/b/z"); - assert!(net.run()); - net.watch(client, ""); - while net.deliver(r1, client) {} - assert_eq!(net.shows(client, id, "live/a/x/1"), Some(true)); - assert_eq!(net.shows(client, id, "live/b/z"), None, "only the first scope answered"); - - net.release(r1, client); - net.release(r2, client); - assert!(net.run()); - let life: Vec = at(&net, client) - .iter() - .filter(|event| event.claim == "live/b/z") - .map(|event| event.kind) - .collect(); - assert_eq!(life, [Kind::Start, Kind::Unwatched]); - net.check().unwrap(); -} - -/// One key arrives through two grants under one subject. When the session -/// for one scope closes, nothing certifies that scope any more, so its claims -/// go offline rather than staying shown through the other grant, which could -/// never deliver their withdrawal. The other scope stays available. -#[test] -fn a_closed_grant_lets_its_scope_go() { - let (origin, r1, r2, client) = (0, 1, 2, 3); - let mut net = Net::new(config(FLOORS, Requests::Off), 4); - net.leaf(origin); - net.leaf(client); - net.join(origin, r1, 1); - net.join(origin, r2, 1); - for (relay, scope) in [(r1, "live/a/"), (r2, "live/b/")] { - let grant = Grant { - subject: "local", - scope, - }; - net.link(client, relay, Link::cost(1).grant(grant), Link::cost(1)); - } - net.originate(origin, 7, 0); - net.announce(origin, "live/a/x", 0, None); - net.announce(origin, "live/b/z", 0, None); - assert!(net.run()); - let id = qualified("local", 7); - assert_eq!(net.shows(client, id, "live/a/x"), Some(true)); - - net.disconnect(client, r1); - assert!(net.run()); - assert_eq!(net.shows(client, id, "live/a/x"), None); - assert_eq!(net.shows(client, id, "live/b/z"), Some(true)); - net.unannounce(origin, "live/a/x"); - net.announce(origin, "live/a/y", 0, None); - assert!(net.run()); - net.connect(client, r1); - assert!(net.run()); - assert_eq!(net.shows(client, id, "live/a/x"), None); - assert_eq!(net.shows(client, id, "live/a/y"), Some(true)); - let life: Vec = at(&net, client) - .iter() - .filter(|event| event.claim == "live/a/x") - .map(|event| event.kind) - .collect(); - assert_eq!(life, [Kind::Start, Kind::Offline]); - assert_eq!(false_ends(&net.events), 0); - net.check().unwrap(); -} - -/// A relay forwards a key whose floors are all grant scopes, and its peer -/// moves to an interest no scope meets. The relay still answers the new -/// stream, certifying nothing, so the peer lets go of a wildcard it kept from -/// the scoped view instead of waiting on an answer that never comes. -#[test] -fn a_disjoint_interest_is_answered_with_nothing() { - let (origin, local, x, y, z) = (0, 1, 2, 3, 4); - let mut net = Net::new(config(FLOORS, Requests::Off), 5); - net.leaf(origin); - net.join(origin, local, 1); - for (relay, scope) in [(x, "live/a/"), (y, "live/b/")] { - let grant = Grant { - subject: "local", - scope, - }; - net.link(relay, local, Link::cost(1).grant(grant), Link::cost(1)); - net.join(relay, z, 1); - } - net.originate(origin, 7, 0); - net.announce(origin, "live/", 0, None); - assert!(net.run()); - let id = qualified("local", 7); - assert_eq!(net.shows(z, id, "live/"), Some(true)); - - net.watch(z, "live/c"); - assert!(net.run()); - assert_eq!(net.shows(z, id, "live/"), None); - let life: Vec = at(&net, z) - .iter() - .filter(|event| event.id == id && event.claim == "live/") - .map(|event| event.kind) - .collect(); - assert_eq!(life, [Kind::Start, Kind::Offline]); - net.check().unwrap(); -} - -/// An interest matches whole path segments: watching `live/a` sees what lies -/// below it, never a sibling `live/ab` that merely shares its bytes. -#[test] -fn an_interest_matches_whole_segments() { - let (origin, client) = (0, 1); - let mut net = Net::new(config(FLOORS, Requests::Off), 2); - net.leaf(origin); - net.join(origin, client, 1); - net.originate(origin, 7, 0); - net.announce(origin, "live/a/x", 0, None); - net.announce(origin, "live/ab", 0, None); - net.watch(client, "live/a"); - assert!(net.run()); - assert_eq!(net.shows(client, 7, "live/a/x"), Some(true)); - assert_eq!(net.shows(client, 7, "live/ab"), None); - net.check().unwrap(); -} - -/// One key arrives through a wide grant and a narrow one under one subject. -/// The narrow session lags behind a withdrawal the wide one delivered; when -/// the wide session closes, only what lies outside the narrow scope goes -/// offline, and the newer knowledge inside it stays rather than reviving the -/// withdrawn claim from the lagging session. -#[test] -fn a_lost_scope_keeps_newer_knowledge_where_still_certified() { - let (origin, r1, r2, client) = (0, 1, 2, 3); - let mut net = Net::new(config(FLOORS, Requests::Off), 4); - net.leaf(origin); - net.leaf(client); - net.join(origin, r1, 1); - net.join(origin, r2, 1); - for (relay, scope) in [(r1, "live/"), (r2, "live/b/")] { - let grant = Grant { - subject: "local", - scope, - }; - net.link(client, relay, Link::cost(1).grant(grant), Link::cost(1)); - } - net.originate(origin, 7, 0); - net.announce(origin, "live/a/x", 0, None); - net.announce(origin, "live/b/z", 0, None); - assert!(net.run()); - let id = qualified("local", 7); - assert_eq!(net.shows(client, id, "live/b/z"), Some(true)); - - net.hold(r2, client); - net.unannounce(origin, "live/b/z"); - assert!(net.run()); - assert_eq!(net.shows(client, id, "live/b/z"), None); - net.disconnect(client, r1); - assert!(net.run()); - assert_eq!(net.shows(client, id, "live/a/x"), None, "outside the scope left"); - assert_eq!( - net.shows(client, id, "live/b/z"), - None, - "the lagging scope revives nothing" - ); - net.check().unwrap(); - - net.release(r2, client); - assert!(net.run()); - assert_eq!(net.shows(client, id, "live/b/z"), None); - assert_eq!(revivals(&at(&net, client)), 0); - net.check().unwrap(); -} - -/// Two relays each admit an uplink scoped to a different prefix under one -/// subject and forward the key to a third, whose view then carries one floor -/// per scope. When one uplink closes, that relay is starved and holds, so the -/// third shows that scope unavailable while the other stays available; once -/// a seqno request recovers a route, nothing certifies the lost scope and the -/// third lets it go. Certifiers are feasible neighbors only, so the starved -/// relay and the third never keep the scope alive through each other. -#[test] -fn a_relay_lets_go_of_a_scope_its_uplinks_lost() { - for (requests, lost) in [(Requests::Off, Some(false)), (Requests::HopByHop, None)] { - let (origin, local, x, y, z) = (0, 1, 2, 3, 4); - let mut net = Net::new(config(FLOORS, requests), 5); - net.leaf(origin); - net.join(origin, local, 1); - for (relay, scope) in [(x, "live/a/"), (y, "live/b/")] { - let grant = Grant { - subject: "local", - scope, - }; - net.link(relay, local, Link::cost(1).grant(grant), Link::cost(1)); - net.join(relay, z, 1); - } - net.originate(origin, 7, 0); - net.announce(origin, "live/a/x", 0, None); - net.announce(origin, "live/b/z", 0, None); - assert!(net.run()); - let id = qualified("local", 7); - assert_eq!(net.shows(z, id, "live/a/x"), Some(true)); - assert_eq!(net.shows(z, id, "live/b/z"), Some(true)); - - net.disconnect(x, local); - net.unannounce(origin, "live/a/x"); - assert!(net.run()); - assert_eq!(net.shows(z, id, "live/a/x"), lost, "{requests:?}"); - assert_eq!(net.shows(z, id, "live/b/z"), Some(true), "{requests:?}"); - assert_eq!(false_ends(&net.events), 0, "{requests:?}"); - net.check().unwrap(); - } -} - -/// An exact claim covers only its own path, so it is outside an interest -/// below it, and only a wildcard ancestor is inside. -#[test] -fn only_a_wildcard_claim_overlaps_an_interest_below_it() { - assert!(!overlaps("live/a", "live/a/x")); - assert!(overlaps("live/a/", "live/a/x")); - assert!(overlaps("live/a/x", "live/a/")); - assert!(overlaps("live/a", "live/a")); - - let (origin, client) = (0, 3); - let mut net = lagging(FLOORS, "live/a/x"); - net.announce(origin, "live/a", 0, None); - net.announce(origin, "live/a/", 0, None); - assert!(net.run()); - assert_eq!(net.shows(client, 7, "live/a"), None); - assert_eq!(net.shows(client, 7, "live/a/"), Some(true)); - net.check().unwrap(); -} - -/// Two origins hold the same wildcard capability for one instance, and a client -/// watches a region below it. Withdrawing one origin's wildcard ends only that -/// origin's claim. The client's floor for `live/a/` certifies the wildcard above -/// it, so a lagging relay's older copy of the withdrawn wildcard is refused -/// after the client's next hop moves there. -#[test] -fn wildcard_claims_end_per_origin_and_are_certified_below() { - let (o1, o2, r1, r2, client) = (0, 1, 2, 3, 4); - for freshness in [BARRIER, FLOORS] { - let mut net = Net::new(config(freshness, Requests::Off), 5); - for origin in [o1, o2] { - net.leaf(origin); - net.join(origin, r1, 1); - net.join(origin, r2, 1); - } - net.leaf(client); - net.join(client, r1, 1); - net.join(client, r2, 1); - net.watch(client, "live/a/"); - net.originate(o1, 10, 0); - net.originate(o2, 11, 0); - net.announce(o1, "live/", 0, Some(1)); - net.announce(o2, "live/", 0, Some(1)); - assert!(net.run()); - assert_eq!(net.shown(client).len(), 2); - - net.hold(o1, r2); - net.unannounce(o1, "live/"); - assert!(net.run()); - assert_eq!(net.shows(client, 10, "live/"), None); - assert_eq!( - net.shows(client, 11, "live/"), - Some(true), - "the other origin's claim stays" - ); - - net.disconnect(client, r1); - assert!(net.run()); - assert_eq!(net.shows(client, 10, "live/").is_some(), freshness == BARRIER); - assert_eq!(net.shows(client, 11, "live/"), Some(true)); - } -} - -/// An origin process restarts. With a fresh identity its claims start a new -/// revision history and the old identity's go offline once nothing can recover -/// them. A restart that kept its identity, even with a persisted seqno, would -/// restart its revision below the floors other nodes hold: they keep showing the -/// dead process's claims and refuse the new ones. -#[test] -fn origin_restart_takes_a_fresh_identity_for_its_claims() { - let (origin, a, b) = (0, 1, 2); - for fresh in [false, true] { - let mut net = Net::new(Config::default(), 3); - net.leaf(origin); - net.join(origin, a, 1); - net.join(a, b, 1); - net.originate(origin, 7, 0); - net.announce(origin, "live/x", 0, None); - net.unannounce(origin, "live/x"); - net.announce(origin, "live/y", 0, None); - assert!(net.run()); - - net.kill(origin); - assert!(net.run()); - for node in [a, b] { - assert!(net.shown(node).is_empty()); - assert!(!net.holds(node, 7), "nothing can recover the dead identity"); - } - let offline = net.events.iter().filter(|e| e.kind == Kind::Offline).count(); - assert_eq!(offline, 2, "offline, not ended by the origin"); - assert_eq!(false_ends(&net.events), 0); - - let id = if fresh { 8 } else { 7 }; - net.originate(origin, id, 10); - net.reconnect(origin); - net.announce(origin, "live/x", 0, None); - assert!(net.run()); - for node in [a, b] { - if fresh { - assert_eq!(net.shows(node, 8, "live/x"), Some(true)); - assert_eq!(net.shown(node).len(), 1); - } else { - assert_eq!(net.shows(node, 7, "live/y"), Some(true), "the dead process's claim"); - assert_eq!(net.shows(node, 7, "live/x"), None); - } - } - } -} - -/// The route contract's collected incarnation, with announcements. Node b -/// learned that `live/x` ended, then lost every route and collected the origin. -/// A session still delivering older messages brings the claim back, but only -/// until the withdrawal queued behind it on that same session lands. Without -/// collection, b's retained floor refuses the replay outright. -#[test] -fn stale_replay_after_collection_lasts_only_until_the_withdrawal_behind_it() { - let (origin, a, b, c) = (0, 1, 2, 3); - for collect in [false, true] { - let mut net = Net::new(Config::default(), 4); - net.leaf(origin); - net.join(origin, a, 1); - net.join(a, b, 1); - net.join(a, c, 1); - net.join(c, b, 1); - net.hold(a, b); - net.originate(origin, 7, 0); - net.announce(origin, "live/x", 0, None); - assert!(net.run()); - net.unannounce(origin, "live/x"); - assert!(net.run()); - assert_eq!(kinds(&at(&net, b)), [Kind::Start, Kind::End]); - - net.kill(origin); - assert!(net.run()); - assert!(!net.holds(b, 7)); - if collect { - net.collect(b, 7); - } - // The held ROUTE, an empty snapshot, then the one carrying `live/x`. - for _ in 0..3 { - assert!(net.deliver(a, b)); - } - assert_eq!(net.shows(b, 7, "live/x").is_some(), collect); - assert!(net.deliver(a, b), "the withdrawal"); - assert_eq!(net.shows(b, 7, "live/x"), None); - net.release(a, b); - assert!(net.run()); - let replayed: &[Kind] = if collect { - &[Kind::Start, Kind::End, Kind::Start, Kind::End] - } else { - &[Kind::Start, Kind::End] - }; - assert_eq!(kinds(&at(&net, b)), replayed); - assert!(!net.holds(b, 7)); - } -} - -/// A relay whose own route went away holds its routes, and its downstream -/// nodes hold through it, only along feasible advertisements, so holds cannot -/// prop each other up. When the origin dies, every node in a ring goes -/// offline and keeps no route state, with no timer, and none of them reports -/// an END the origin did not send. -#[test] -fn holds_retire_once_nothing_can_recover() { - let origin = 4; - for requests in [Requests::Off, Requests::HopByHop] { - let mut net = Net::new(config(FLOORS, requests), 5); - for relay in 0..4 { - net.join(relay, (relay + 1) % 4, 1); - } - net.leaf(origin); - net.join(origin, 0, 1); - net.originate(origin, 7, 0); - net.announce(origin, "live/x", 0, None); - assert!(net.run()); - - net.kill(origin); - assert!(net.run()); - for relay in 0..4 { - assert!(net.shown(relay).is_empty()); - assert!(!net.holds(relay, 7), "relay {relay} under {requests:?}"); - } - assert_eq!(false_ends(&net.events), 0); - assert_eq!(net.retained().pending, 0); - } -} - -/// Seqnos and revisions are varints. An origin never advertises the last -/// value: a bump, a claim change, or a request that would reach it rotates the -/// identity instead, so every node lets the old one go and sees the claims -/// again under the new one. Nothing wraps. A session proves one key once, so -/// rotating closes and reopens every session the origin had up; proving a -/// second key on a live session fails. -#[test] -fn exhausted_counters_rotate_the_identity() { - let (origin, a, b, c, client) = (0, 1, 2, 3, 4); - let rotated = |net: &Net| { - let fresh = net.proven(origin); - assert_ne!(fresh, 7, "a fresh identity"); - assert_eq!(net.shows(client, 7, "live/x"), None); - assert_eq!(net.shows(client, fresh, "live/x"), Some(true)); - assert!(net.route(a, 7).is_none()); - net.check().unwrap(); - }; - let build = |seqno| { - let mut net = Net::new(config(FLOORS, Requests::HopByHop), 5); - net.leaf(origin); - net.leaf(client); - net.join(origin, a, 1); - net.join(a, b, 1); - net.join(b, c, 1); - net.join(origin, c, 1); - net.join(client, a, 1); - net.disconnect(origin, c); - net.originate(origin, 7, seqno); - net.announce(origin, "live/x", 0, None); - assert!(net.run()); - net - }; - - let mut net = build(VARINT_MAX - 2); - net.bump(origin); - assert!(net.run()); - assert_eq!(net.proven(origin), 7); - net.bump(origin); - assert!(net.run()); - rotated(&net); - - let mut net = build(0); - net.age(origin, VARINT_MAX - 2); - net.announce(origin, "live/y", 0, None); - assert!(net.run()); - assert_eq!(net.proven(origin), 7); - net.unannounce(origin, "live/y"); - assert!(net.run()); - rotated(&net); - - // The origin moves to a farther attachment, and the starved node asks for - // the last seqno. - let mut net = build(VARINT_MAX - 1); - net.disconnect(origin, a); - net.connect(origin, c); - assert!(net.run()); - rotated(&net); - assert!(net.route(a, net.proven(origin)).is_some()); -} - -/// The seqno request sketch: a starved node resends its request a few times -/// and then gives up until an infeasible update arrives. Node b forwards along -/// its route through x, whose own route just failed but whose retraction to b -/// is still in flight, so x drops every attempt. When the retraction lands, b -/// moves to an equal-metric route through y and advertises nothing new, so n -/// stays starved behind a quiet origin. Keeping the request at every hop that -/// forwarded it, and resending it when that hop's next hop changes, reaches the -/// origin with no timer. -#[test] -fn a_quiet_origin_stays_starved_after_bounded_retries() { - let (origin, w, x, y2, y, b, n, z, client) = (0, 1, 2, 3, 4, 5, 6, 7, 8); - for requests in [Requests::Retries(2), Requests::HopByHop] { - let mut net = Net::new(config(FLOORS, requests), 9); - net.leaf(origin); - net.leaf(client); - net.join(origin, w, 1); - net.join(w, x, 1); - net.join(origin, y2, 1); - net.join(y2, y, 1); - net.link(b, x, Link::cost(1).upstream(), Link::cost(1)); - net.link(b, y, Link::cost(1).upstream(), Link::cost(1)); - net.join(b, n, 1); - net.join(n, z, 1); - net.join(z, origin, 1); - net.join(n, client, 1); - net.originate(origin, 7, 0); - net.announce(origin, "live/x", 0, None); - assert!(net.run()); - assert_eq!(net.route(n, 7).and_then(|route| route.next), Some(z)); - assert_eq!(net.route(b, 7).and_then(|route| route.next), Some(x)); - - net.disconnect(n, z); - net.hold(x, b); - net.disconnect(w, x); - assert!(net.run()); - assert!(net.starved(n, 7)); - for _ in 0..3 { - net.tick(); - assert!(net.run()); - } - net.release(x, b); - assert!(net.run()); - assert_eq!(net.route(b, 7).and_then(|route| route.next), Some(y)); - assert_eq!(net.shows(client, 7, "live/x"), Some(requests == Requests::HopByHop)); - if requests == Requests::HopByHop { - assert_eq!(net.seqno(origin), 1); - assert!(net.route(n, 7).is_some()); - } else { - assert_eq!(net.seqno(origin), 0); - assert!(net.starved(n, 7), "nothing will ever ask the origin again"); - } - } -} - -/// Random meshes churn sessions and hold and deliver messages out of step, then -/// release every hold without reconnecting anything. Every relay that can -/// still reach a quiet origin must end with a route to it, which only a seqno -/// request can bring. A forwarder that skipped its requesters stranded some: -/// its only next hop was a requester that had long stopped waiting, and no -/// message ever cancels a request. -#[test] -fn sweep_requests_reach_a_quiet_origin() { - const RELAYS: usize = 5; - let mut stranded = 0; - for seed in 0..1000u64 { - let mut rng = XorShift(0x9e37_79b9 ^ (seed * 0x1234_5678_9abc)); - let nodes = RELAYS + 2; - let mut net = Net::new(config(FLOORS, Requests::HopByHop), nodes); - let mut links = Vec::new(); - for a in 0..RELAYS { - for b in a + 1..RELAYS { - if b == a + 1 || rng.chance(40) { - net.join(a, b, 1 + rng.next(3)); - links.push((a, b)); - } - } - } - for origin in RELAYS..nodes { - net.leaf(origin); - for _ in 0..2 { - let relay = rng.next(RELAYS as u64) as usize; - if !links.contains(&(relay, origin)) { - net.join(origin, relay, 1 + rng.next(3)); - links.push((relay, origin)); - } - } - net.originate(origin, origin as u64, 0); - } - assert!(net.run()); - for _ in 0..30 { - match rng.next(6) { - 0 | 1 => { - let (a, b) = links[rng.next(links.len() as u64) as usize]; - if net.connected(a, b) { - net.disconnect(a, b); - } else { - net.connect(a, b); - } - } - 2 | 3 => { - let (a, b) = links[rng.next(links.len() as u64) as usize]; - let (from, to) = if rng.chance(50) { (a, b) } else { (b, a) }; - if rng.chance(60) { - net.hold(from, to); - } else { - net.release(from, to); - } - } - _ => { - let busy = net.busy(); - if !busy.is_empty() { - let (from, to) = busy[rng.next(busy.len() as u64) as usize]; - net.deliver(from, to); - } - } - } - net.settle(1); - } - for &(a, b) in &links { - net.release(a, b); - net.release(b, a); - } - assert!(net.run()); - for origin in RELAYS..nodes { - // The relays an up session path still joins to the origin. - let mut reach = BTreeSet::from([origin]); - let mut frontier = vec![origin]; - while let Some(at) = frontier.pop() { - for &(a, b) in &links { - let next = match at { - _ if at == a => b, - _ if at == b => a, - _ => continue, - }; - if next < RELAYS && net.connected(a, b) && reach.insert(next) { - frontier.push(next); - } - } - } - stranded += reach - .iter() - .filter(|&&node| node < RELAYS && net.route(node, origin as Id).is_none()) - .count(); - } - } - assert_eq!(stranded, 0, "relays left without a route to a reachable origin"); -} - -/// One change to a claim costs one batch per session carrying its origin's -/// route and one event per node watching it; retained state follows sessions, -/// live claims, and current interest, and neither withdrawal history nor -/// interest history leaves anything behind. -#[test] -fn announcement_cost_follows_sessions_and_live_claims() { - const PREFIXES: [&str; 4] = ["live/a", "live/b", "live/c", "live/d"]; - let build = |relays: usize, claims: usize| { - let mut net = Net::new(Config::default(), relays + 2); - for relay in 0..relays { - net.join(relay, (relay + 1) % relays, 1); - } - let origin = relays; - net.leaf(origin); - net.join(origin, 0, 1); - // A client watching a region with no claims yet. - let client = relays + 1; - net.leaf(client); - net.join(client, relays / 2, 1); - net.watch(client, "none/"); - net.originate(origin, 7, 0); - for prefix in &PREFIXES[..claims] { - net.announce(origin, prefix, 0, None); - } - assert!(net.run()); - net - }; - for (relays, claims) in [(4, 2), (8, 2), (4, 4)] { - let mut net = build(relays, claims); - let origin = relays; - // Each relay sends to both ring neighbors, and the origin and its relay to - // each other: without split horizon the relay echoes the origin's own claims. - let sessions = 2 * relays + 2; - let retained = net.retained(); - assert_eq!(retained.items, relays * claims); - assert_eq!(retained.floors, relays + 1); - assert_eq!(retained.heard, sessions * claims); - // Both ends of every session cache the same view. - assert_eq!( - (retained.sent, retained.sent_floors), - (retained.heard, retained.heard_floors) - ); - - let (stats, events) = (net.stats, net.events.len()); - net.unannounce(origin, PREFIXES[0]); - assert!(net.run()); - // The client's session carries only the advanced floor. - assert_eq!(net.stats.claim_msgs - stats.claim_msgs, sessions + 1); - assert_eq!(net.stats.claim_items - stats.claim_items, sessions); - assert_eq!(net.events.len() - events, relays + 1, "once per node"); - - for _ in 0..50 { - net.announce(origin, PREFIXES[0], 0, None); - net.unannounce(origin, PREFIXES[0]); - } - assert!(net.run()); - let after = net.retained(); - assert_eq!(after.items, relays * (claims - 1)); - assert_eq!(after.floors, retained.floors, "no tombstones"); - assert_eq!(after.echoes, 0, "every ROUTE_END echo drained"); - net.check().unwrap(); - - // The client cycles through disjoint regions, one with a live claim. - let client = relays + 1; - for cycles in [1, 16] { - for room in 0..cycles { - let region: &'static str = Box::leak(format!("room/{room}/").into_boxed_str()); - net.watch(client, region); - assert!(net.run()); - net.watch(client, PREFIXES[1]); - assert!(net.run()); - assert_eq!(net.shows(client, 7, PREFIXES[1]), Some(true)); - } - net.watch(client, "none/"); - assert!(net.run()); - assert_eq!(net.retained(), after, "interest history after {cycles} cycles"); - } - net.check().unwrap(); - } -} - -/// A client reaches one origin through two relays, each session scoped by a -/// grant under the same subject, so both feed one qualified view. Claims -/// change in and around both scopes, the client's interest moves, sessions -/// drop and return, and messages are held and delivered out of step. Every -/// view stays some revision of the origin's truth after every step, the -/// widening guard stays bounded, no claim ends while the origin still has it, -/// and once churn stops the client shows exactly the claims its interest and -/// the scopes admit. -#[test] -fn sweep_scoped_handoff() { - const CLAIMS: [&str; 5] = ["live/a/x", "live/a/y/", "live/b/z", "live/c", "live/"]; - const WATCHES: [&str; 5] = ["", "live/", "live/a/", "live/b/z", "live/c"]; - const SCOPES: [&str; 2] = ["live/a/", "live/b/"]; - let (origin, r1, r2, client) = (0, 1, 2, 3); - let mut rng = XorShift(0x5c0b_ed00); - let mut live_starts = 0; - for scenario in 0..150 { - let mut net = Net::new(config(FLOORS, Requests::HopByHop), 4); - net.leaf(origin); - net.leaf(client); - net.join(origin, r1, 1); - net.join(origin, r2, 1); - net.join(r1, r2, 1); - for (relay, scope) in [(r1, SCOPES[0]), (r2, SCOPES[1])] { - let grant = Grant { - subject: "local", - scope, - }; - net.link(client, relay, Link::cost(1).grant(grant), Link::cost(1)); - } - let links = [(origin, r1), (origin, r2), (r1, r2), (client, r1), (client, r2)]; - net.originate(origin, 7, 0); - assert!(net.run()); - for _ in 0..40 { - match rng.next(8) { - 0..=2 => { - let claim = CLAIMS[rng.next(CLAIMS.len() as u64) as usize]; - if rng.chance(60) { - net.announce(origin, claim, rng.next(3), None); - } else { - net.unannounce(origin, claim); - } - } - 3 => net.watch(client, WATCHES[rng.next(WATCHES.len() as u64) as usize]), - 4 => { - let (a, b) = links[rng.next(links.len() as u64) as usize]; - if net.connected(a, b) { - net.disconnect(a, b); - } else { - net.connect(a, b); - } - } - 5 => { - let (a, b) = links[rng.next(links.len() as u64) as usize]; - let (from, to) = if rng.chance(50) { (a, b) } else { (b, a) }; - if rng.chance(60) { - net.hold(from, to); - } else { - net.release(from, to); - } - } - _ => { - let busy = net.busy(); - if !busy.is_empty() { - let (from, to) = busy[rng.next(busy.len() as u64) as usize]; - net.deliver(from, to); - } - } - } - net.settle(1); - net.check().unwrap_or_else(|err| panic!("scenario {scenario}: {err}")); - assert!( - net.retained().certified <= SCOPES.len() + 1, - "scenario {scenario}: the widening guard grows" - ); - } - for (a, b) in links { - net.release(a, b); - net.release(b, a); - if !net.connected(a, b) { - net.connect(a, b); - } - } - assert!(net.run()); - net.check().unwrap_or_else(|err| panic!("scenario {scenario}: {err}")); - let watch = net.watching(client); - let rev = net.revision(7); - let expected: BTreeSet<&str> = CLAIMS - .into_iter() - .filter(|claim| net.truth(7, claim, rev).is_some()) - .filter(|claim| { - SCOPES.iter().any(|scope| { - let region = match () { - _ if within(scope, watch) => *scope, - _ if within(watch, scope) => watch, - _ => return false, - }; - overlaps(claim, region) - }) - }) - .collect(); - let id = qualified("local", 7); - let shown: BTreeSet<&str> = net - .shown(client) - .iter() - .filter(|((shown, _), view)| *shown == id && view.available) - .map(|((_, claim), _)| *claim) - .collect(); - assert_eq!(shown, expected, "scenario {scenario} watching {watch:?}"); - // A scope its last session left, or an interest no scope meets, goes - // offline rather than ending. - assert_eq!(false_ends(&net.events), 0, "scenario {scenario}"); - live_starts += net - .events - .iter() - .filter(|event| event.id == id && event.kind == Kind::Start && !event.withdrawn) - .count(); - } - // Qualified events are judged against the origin's history, so a live - // claim's Start is not mistaken for a withdrawn one. - assert!(live_starts > 0); -} - -/// Random meshes with two origins and a client under random churn: claims -/// change, sessions drop and return, messages are held and replayed out of -/// step, the client changes interest, and an origin restarts with a fresh -/// identity. Under every rule, selected routes never loop, even right after a -/// drained feasibility distance is dropped. Under merged floors, every view is always some revision of the -/// origin's truth, nothing revives and nothing ends falsely, and once churn -/// stops every node shows every live origin's current claims and holds -/// nothing for a dead one. A node that collected an origin with no route left, -/// or restarted, may see a stale view return until the withdrawal behind it -/// lands; nothing else revives. Next-hop authority and a snapshot barrier both -/// revive withdrawals and report false ENDs on the same scenarios. -#[test] -fn sweep_handoff() { - let mut totals = Vec::new(); - for freshness in [NEXT_HOP, BARRIER, FLOORS] { - let mut rng = XorShift(0x4a4d_0ff5); - let mut total = Totals::default(); - for scenario in 0..150 { - let mut net = churn(&mut rng, freshness); - let label = format!("scenario {scenario} under {freshness:?}"); - total.events += net.events.len(); - total.revivals += revivals(&net.events); - total.false_ends += false_ends(&net.events); - total.reannouncements += reannouncements(&net.events); - total.claim_bytes += net.stats.claim_bytes; - total.route_msgs += net.stats.route_msgs; - total.requests += net.stats.requests; - if freshness != FLOORS { - continue; - } - assert_eq!(revivals(&net.events), 0, "{label}"); - assert_eq!(false_ends(&net.events), 0, "{label}"); - settled(&mut net, &label); - } - totals.push((freshness, total)); - } - for (freshness, total) in &totals { - eprintln!("{freshness:?}: {total:?}"); - } - let [(_, next_hop), (_, barrier), _] = &totals[..] else { - unreachable!() - }; - assert!(next_hop.revivals > 0 && next_hop.false_ends > 0); - assert!(barrier.revivals > 0); -} - -#[derive(Debug, Default)] -struct Totals { - events: usize, - revivals: usize, - false_ends: usize, - reannouncements: usize, - claim_bytes: usize, - route_msgs: usize, - requests: usize, -} - -const RELAYS: usize = 5; -const CLAIMS: [&str; 4] = ["live/", "live/a/", "live/a/x", "live/b"]; - -/// Origins sit at `RELAYS` and `RELAYS + 1`, the client at `RELAYS + 2`. -fn churn(rng: &mut XorShift, freshness: Freshness) -> Net { - let (first, client) = (RELAYS, RELAYS + 2); - let mut net = Net::new(config(freshness, Requests::HopByHop), RELAYS + 3); - let mut links = Vec::new(); - for relay in 0..RELAYS { - links.push((relay, (relay + 1) % RELAYS)); - } - let (a, b) = (rng.next(RELAYS as u64) as usize, rng.next(RELAYS as u64) as usize); - if a.abs_diff(b) > 1 && a.abs_diff(b) != RELAYS - 1 { - links.push((a, b)); - } - for leaf in [first, first + 1, client] { - net.leaf(leaf); - let a = rng.next(RELAYS as u64) as usize; - links.push((leaf, a)); - let b = (a + 1 + rng.next(RELAYS as u64 - 1) as usize) % RELAYS; - links.push((leaf, b)); - } - for &(a, b) in &links { - net.join(a, b, 1 + rng.next(3)); - } - let mut ids = [100, 200]; - net.originate(first, ids[0], 0); - net.originate(first + 1, ids[1], 0); - assert!(net.run()); - - for _ in 0..40 { - match rng.next(10) { - 0..=3 => { - let origin = rng.next(2) as usize; - let claim = CLAIMS[rng.next(CLAIMS.len() as u64) as usize]; - if rng.chance(50) { - net.announce(first + origin, claim, rng.next(3), None); - } else { - net.unannounce(first + origin, claim); - } - } - 4 => { - let (a, b) = links[rng.next(links.len() as u64) as usize]; - if net.connected(a, b) { - net.disconnect(a, b); - } else { - net.connect(a, b); - } - } - 5 => { - let (a, b) = links[rng.next(links.len() as u64) as usize]; - let (from, to) = if rng.chance(50) { (a, b) } else { (b, a) }; - if rng.chance(70) { - net.hold(from, to); - } else { - net.release(from, to); - } - } - 6 => { - net.watch(client, ["", "live/a/", "live/b"][rng.next(3) as usize]); - } - 7 if rng.chance(50) => { - // Collection of routes and announcements is allowed once a node holds no route to an identity. - let node = rng.next(RELAYS as u64 + 3) as usize; - let id = ids[rng.next(2) as usize]; - if !net.holds(node, id) && (RELAYS..RELAYS + 2).all(|origin| node != origin) { - net.collect(node, id); - } - } - 7 if rng.chance(40) => { - let origin = rng.next(2) as usize; - net.kill(first + origin); - ids[origin] += 1; - net.originate(first + origin, ids[origin], 0); - net.reconnect(first + origin); - } - _ => { - // Echoed ROUTE_ENDs ride the route's own stream, so they interleave - // freely with the echoing node's routes. - for _ in 0..rng.next(6) { - let busy: Vec<(bool, usize, usize)> = (net.busy().into_iter().map(|(from, to)| (false, from, to))) - .chain(net.busy_acks().into_iter().map(|(from, to)| (true, from, to))) - .collect(); - if busy.is_empty() { - break; - } - match busy[rng.next(busy.len() as u64) as usize] { - (false, from, to) => net.deliver(from, to), - (true, from, to) => net.deliver_ack(from, to), - }; - } - } - } - if freshness == FLOORS { - net.check().unwrap(); - } - loop_free(&net, ids); - } - - for &(a, b) in &links { - net.release(a, b); - net.release(b, a); - if !net.connected(a, b) { - net.connect(a, b); - } - } - assert!(net.run()); - net -} - -/// A node drops a feasibility distance only once every neighbor echoed its -/// ROUTE_END, so an older advertisement of its own never returns as a loop. -/// `ids` are the origins' current identities; none past them exists yet. -fn loop_free(net: &Net, ids: [Id; 2]) { - for id in (100..110.min(ids[0] + 1)).chain(200..210.min(ids[1] + 1)) { - for node in 0..RELAYS + 3 { - let mut at = node; - for hop in 0.. { - let Some(next) = net.route(at, id).and_then(|selected| selected.next) else { - break; - }; - assert!(hop <= RELAYS + 3, "{id} loops from node {node}"); - at = next; - } - } - } -} - -/// After churn stops with every session up, each node shows exactly the live -/// origins' current claims within its interest, available, and keeps nothing -/// for an identity whose process is gone. -fn settled(net: &mut Net, label: &str) { - net.check().unwrap(); - let live: Vec = (RELAYS..RELAYS + 2).map(|origin| net.proven(origin)).collect(); - for node in 0..RELAYS + 3 { - let mut expected = BTreeSet::new(); - for &id in &live { - assert!( - net.route(node, id).is_some(), - "{label}: node {node} has no route to {id}" - ); - for claim in CLAIMS { - if overlaps(claim, net.watching(node)) && net.truth(id, claim, net.revision(id)).is_some() { - expected.insert((id, claim)); - } - } - } - let shown: BTreeSet<(Id, &str)> = net.shown(node).keys().copied().collect(); - assert_eq!(shown, expected, "{label}: node {node}"); - for (&(id, claim), shown) in net.shown(node) { - assert!(shown.available, "{label}: node {node} {id} {claim}"); - assert_eq!( - net.truth(id, claim, net.revision(id)), - Some(shown.item), - "{label}: node {node} {id} {claim}" - ); - } - for id in (100..110).chain(200..210).filter(|id| !live.contains(id)) { - assert!(!net.holds(node, id), "{label}: node {node} holds dead {id}"); - assert!( - !net.remembers(node, id), - "{label}: node {node} keeps dead {id}'s distance" - ); - } - } - assert_eq!(net.retained().pending, 0, "{label}"); -} diff --git a/rs/moq-net/tests/integration/routing_contracts/mod.rs b/rs/moq-net/tests/integration/routing_contracts/mod.rs deleted file mode 100644 index 76dc6a3283d..00000000000 --- a/rs/moq-net/tests/integration/routing_contracts/mod.rs +++ /dev/null @@ -1,1353 +0,0 @@ -//! Deterministic counterexamples for the per-origin routing contract. -//! -//! Each fixture runs one scenario from `quest/m2/cluster-routes/README.md` -//! against [`model`], under the rule that fails and the rule that holds. The -//! model is a reference for the contract, not the relay: when lite-07's route -//! layer lands, these topologies become its acceptance tests. Failover fixtures -//! here advance the origin's seqno by hand where the seqno request would; -//! [`handoff`] runs the request exchange and the announcement layer. It sits in -//! `tests/` rather than inline because it tests no moq-net module. - -mod handoff; -mod model; - -use model::{Aggregate, Config, Export, Forward, Grant, Link, Metric, Net, Outcome, Rule, TieBreak, qualified}; - -const PATH_ONLY: Forward = Forward::PathOnly { - exclude_requester: true, -}; - -/// The CDN's grant to a local relay: its credential subject, publishing under `local/`. -const LOCAL: Grant = Grant { - subject: "local", - scope: "local/", -}; - -fn served_by(outcome: &Outcome) -> Option<(u64, usize)> { - match outcome { - Outcome::Served { by, path } => Some((*by, *path.last().unwrap())), - _ => None, - } -} - -fn loops(outcomes: &[Outcome]) -> bool { - outcomes.iter().any(|outcome| matches!(outcome, Outcome::Loop { .. })) -} - -// Identity: a fresh routing identity per process start, seqno per identity. - -/// A restart that reuses its routing identity restarts its seqno below the -/// feasibility distance every other node still holds, so its routes are refused -/// until the new process counts past the old one. This is why a pinned -/// `cluster.id` cannot stay the routing identity. -#[test] -fn restart_reusing_identity_is_refused() { - let (origin, a, b) = (0, 1, 2); - let mut net = Net::new(Config::default(), 3); - net.join(origin, a, 1); - net.join(a, b, 1); - net.originate(origin, 7, 0); - for _ in 0..3 { - net.bump(origin); - } - assert!(net.run()); - assert!(net.route(b, 7).is_some()); - - net.kill(origin); - net.originate(origin, 7, 0); - net.reconnect(origin); - assert!(net.run()); - assert!(net.starved(a, 7), "seqno 0 is older than the 3 a remembers"); - assert!(net.route(b, 7).is_none()); - - net.kill(origin); - net.originate(origin, 8, 0); - net.reconnect(origin); - assert!(net.run()); - assert!(net.route(b, 8).is_some(), "a fresh identity has no history to lose to"); -} - -/// The same process moving to a farther attachment keeps its identity and -/// seqno, so the node next to its old attachment holds a feasibility distance -/// shorter than any route it now has and starves. The origin must advance its -/// seqno whenever its own sessions change. -#[test] -fn reconnect_without_restart_needs_a_seqno_advance() { - let (origin, a, b, c) = (0, 1, 2, 3); - let mut net = Net::new(Config::default(), 4); - net.join(a, b, 1); - net.join(b, c, 1); - net.join(origin, a, 1); - net.join(origin, c, 1); - net.disconnect(origin, c); - net.originate(origin, 7, 0); - assert!(net.run()); - - net.disconnect(origin, a); - net.connect(origin, c); - assert!(net.run()); - assert!(net.starved(a, 7), "a advertised one hop; every new route is longer"); - - net.bump(origin); - assert!(net.run()); - assert_eq!(net.route(a, 7).and_then(|route| route.next), Some(b)); -} - -/// A dead incarnation's route can return after a node collects it, but only -/// from a message already in flight on a live session, and that session then -/// delivers the retraction behind it. Ordered sessions, session-scoped state, -/// and never advertising an unreachable route are what bound the revival, not -/// the random identity. -#[test] -fn collected_incarnation_returns_only_while_its_retraction_is_in_flight() { - let (origin, a, b, c) = (0, 1, 2, 3); - let mut net = Net::new(Config::default(), 4); - net.join(origin, a, 1); - net.join(a, b, 1); - net.join(a, c, 1); - net.join(c, b, 1); - // Everything a sends b waits: first the route, then its retraction. - net.hold(a, b); - net.originate(origin, 7, 0); - assert!(net.run()); - assert_eq!(net.route(b, 7).and_then(|route| route.next), Some(c)); - - net.kill(origin); - assert!(net.run()); - assert!(net.route(b, 7).is_none()); - - net.collect(b, 7); - assert!(net.deliver(a, b), "the held update"); - assert!(net.route(b, 7).is_some(), "the dead incarnation is back at b"); - assert!(net.run()); - // c drained its distance, so b's copy spreads to it, but the path ends at a, - // which let go: a stale dead end until the retraction lands, never a loop. - assert_eq!(net.route(c, 7).and_then(|route| route.next), Some(b)); - assert!(net.route(a, 7).is_none()); - - net.release(a, b); - assert!(net.run()); - for node in 0..4 { - assert!( - net.route(node, 7).is_none(), - "node {node} still routes to a dead incarnation" - ); - } -} - -/// No route entry is not proof that no advertisement derived from this node's -/// own is still in flight. b's echo from c waits on c's session while b loses -/// its only entry. b keeps its feasibility distance until every neighbor it -/// advertised to echoes the ROUTE_END back, so the queued echo is refused -/// rather than letting b route via c while c still routes via b. A leaf never -/// advertises the identity back, so only an explicit echo of the ROUTE_END, -/// which it sends too, ends the drain. Once the echoes land nothing is left. -#[test] -fn collection_keeps_the_feasibility_distance_against_a_queued_echo() { - let (origin, b, c, leaf) = (0, 1, 2, 3); - let mut net = Net::new(Config::default(), 4); - net.leaf(leaf); - net.join(origin, b, 1); - net.join(b, c, 1); - net.join(b, leaf, 1); - net.hold(c, b); - net.originate(origin, 7, 0); - assert!(net.run()); - assert_eq!(net.route(c, 7).and_then(|route| route.next), Some(b)); - assert!(net.queued(c, b) > 0, "c's echo waits"); - - net.hold(b, c); - net.hold(leaf, b); - net.disconnect(origin, b); - assert!(!net.holds(b, 7)); - net.collect(b, 7); - assert!(net.remembers(b, 7), "draining: the ROUTE_ENDs are not echoed yet"); - assert!(net.deliver(c, b), "the echo of b's own advertisement"); - assert!(net.route(b, 7).is_none(), "b refuses its own echo"); - assert!(net.starved(b, 7)); - - net.release(b, c); - net.release(c, b); - assert!(net.run()); - assert!(net.queued_acks(leaf, b) > 0, "the leaf's echo waits"); - assert!(net.remembers(b, 7), "the leaf may still hold b's advertisement"); - net.release(leaf, b); - assert!(net.run()); - for node in [b, c, leaf] { - assert!( - net.route(node, 7).is_none(), - "node {node} routes to an unreachable origin" - ); - assert!(!net.holds(node, 7), "node {node}"); - assert!(!net.remembers(node, 7), "node {node} keeps a drained distance"); - } -} - -/// Every node's next-hop chain toward `id` ends without revisiting a node. -fn assert_loop_free(net: &Net, nodes: usize, id: u64) { - for node in 0..nodes { - let mut at = node; - for hop in 0.. { - let Some(next) = net.route(at, id).and_then(|route| route.next) else { - break; - }; - assert!(hop < nodes, "{id} loops from node {node}"); - at = next; - } - } -} - -/// A restarted relay loses every feasibility distance, yet stays loop-free: -/// its sessions close with the process, and each neighbor drops what it -/// learned on them, so no neighbor still holds an advertisement the new -/// process never made. A stale echo still reaches it through a third node's -/// session that outlived it, and is safe, because every node on that echo's -/// path kept its own distance. -#[test] -fn restart_drops_the_feasibility_distance_with_every_session() { - let (origin, b, c, d) = (0, 1, 2, 3); - let mut net = Net::new(Config::default(), 4); - net.join(origin, b, 1); - net.join(b, c, 1); - net.join(c, d, 1); - net.join(d, b, 10); - net.originate(origin, 7, 0); - assert!(net.run()); - assert_eq!(net.route(d, 7).and_then(|route| route.next), Some(c)); - - // The update c sends d after losing b waits on a session that outlives b. - net.hold(c, d); - net.kill(b); - assert!(net.run()); - assert_loop_free(&net, 4, 7); - assert!(net.route(c, 7).is_none()); - assert_eq!(net.route(d, 7).and_then(|route| route.next), Some(c), "stale"); - - net.connect(b, d); - assert!(net.run()); - assert_eq!( - net.route(b, 7).and_then(|route| route.next), - Some(d), - "the echo of the old advertisement reaches the new process" - ); - assert_loop_free(&net, 4, 7); - net.connect(b, c); - assert!(net.run()); - assert!(net.route(c, 7).is_none(), "the distance at c refuses the longer copy"); - assert_loop_free(&net, 4, 7); - - net.release(c, d); - assert!(net.run()); - assert_loop_free(&net, 4, 7); - net.connect(origin, b); - assert!(net.run()); - assert_loop_free(&net, 4, 7); - for node in [b, c, d] { - assert!(net.route(node, 7).is_some(), "node {node} recovers"); - } -} - -/// Random meshes churn links, hold and deliver messages out of step, and -/// collect route-less identities, then drop the feasibility distance. Dropped -/// once no route entry is left, it loops; dropped once every neighbor has -/// processed the retraction, as its echoed ROUTE_ENDs or a restart -/// prove, it never does. Only the edge to each neighbor matters, since every -/// node farther away keeps its own distance. -#[test] -fn sweep_forgetting_distances() { - let mut loops = [0; 2]; - let mut forgotten = [0; 2]; - for (acked, (loops, forgotten)) in [false, true].into_iter().zip(loops.iter_mut().zip(&mut forgotten)) { - let mut rng = XorShift(0x5eed_f0d5); - for _ in 0..300 { - let nodes = RELAYS + 2; - let mut net = Net::new(Config::default(), nodes); - let mut links = Vec::new(); - for a in 0..RELAYS { - for b in a + 1..RELAYS { - if b == a + 1 || rng.chance(40) { - net.join(a, b, 1 + rng.next(3)); - links.push((a, b)); - } - } - } - let ids = [RELAYS as u64, RELAYS as u64 + 1]; - for origin in RELAYS..nodes { - let relay = rng.next(RELAYS as u64) as usize; - net.join(origin, relay, 1); - links.push((relay, origin)); - net.originate(origin, origin as u64, 0); - } - assert!(net.run()); - - for _ in 0..60 { - match rng.next(8) { - 0 => { - let (a, b) = links[rng.next(links.len() as u64) as usize]; - if net.connected(a, b) { - net.disconnect(a, b); - } else { - net.connect(a, b); - } - } - 1 | 2 => { - let (a, b) = links[rng.next(links.len() as u64) as usize]; - let (from, to) = if rng.chance(50) { (a, b) } else { (b, a) }; - if rng.chance(60) { - net.hold(from, to); - } else { - net.release(from, to); - } - } - 3 => { - let busy = net.busy(); - if !busy.is_empty() { - let (from, to) = busy[rng.next(busy.len() as u64) as usize]; - net.deliver(from, to); - } - } - 4 => net.bump(RELAYS + rng.next(2) as usize), - _ => { - let node = rng.next(RELAYS as u64) as usize; - let id = ids[rng.next(2) as usize]; - if !net.holds(node, id) { - net.collect(node, id); - if !acked || net.retracted(node, id) { - net.forget(node, id); - *forgotten += 1; - } - } - } - } - net.settle(1); - for &id in &ids { - for node in 0..nodes { - let mut at = node; - let mut hops = 0; - while let Some(next) = net.route(at, id).and_then(|route| route.next) { - hops += 1; - if hops > nodes { - *loops += 1; - break; - } - at = next; - } - } - } - } - } - } - println!("distances dropped {forgotten:?}, loops {loops:?}: at no entry, then once retracted"); - assert!(loops[0] > 0, "the sweep must reach the unsafe drop"); - assert!(forgotten[1] > 0); - assert_eq!(loops[1], 0); -} - -// Metric: strictly positive link costs, checked, with no hop count. - -/// An origin two links from node 2, then an unrelated session at node 2 that -/// makes it recompute its selection. -fn chain(rule: Rule, origin_cost: Metric, sibling_cost: Metric) -> Net { - let (origin, a, b, late) = (0, 1, 2, 3); - let mut net = Net::new( - Config { - rule, - ..Config::default() - }, - 4, - ); - net.join(origin, a, origin_cost); - net.join(a, b, sibling_cost); - net.originate(origin, 7, 0); - assert!(net.run()); - net.join(b, late, 1); - assert!(net.run()); - net -} - -/// Babel's feasibility needs every link to increase the metric (RFC 8966 -/// s3.5.2). A zero-cost link, which `?cost=0` allows today, leaves a route -/// equal to its own feasibility distance, so the next recomputation refuses -/// it. Strictly positive costs increase on every link without a hop count, so -/// a zero-cost link is refused when it is configured. -#[test] -#[should_panic(expected = "zero-cost link 1-2 refused")] -fn zero_cost_links_are_refused() { - assert!(chain(Rule::Saturating { ceiling: Metric::MAX }, 10, 0).starved(2, 7)); - assert_eq!( - chain(Rule::CHECKED, 10, 1).route(2, 7).map(|route| route.metric), - Some(11) - ); - chain(Rule::CHECKED, 10, 0); -} - -/// A total beyond the ceiling is unreachable under checked addition. A -/// saturating cost instead caps it at the ceiling, equal to the next hop's -/// metric, so the route is accepted and then starves on its next recomputation. -#[test] -fn overflow_is_unreachable() { - let ceiling = 100; - assert!(chain(Rule::Saturating { ceiling }, 100, 5).starved(2, 7)); - - let net = chain(Rule::Checked { ceiling }, 100, 5); - assert_eq!(net.route(1, 7).map(|route| route.metric), Some(100)); - assert!(net.route(2, 7).is_none()); - assert!(!net.starved(2, 7), "unreachable, not waiting on a seqno"); - assert!(net.route(3, 7).is_none()); -} - -// Forwarding progress: path-only requests reselect the origin at every hop. - -/// Three relays in a ring, each next to its own importer of one instance, and each -/// giving its clockwise link `priority`. -fn priority_ring(priority: i8) -> Net { - let mut net = Net::new(Config::default(), 6); - for relay in 0..3 { - let next = (relay + 1) % 3; - net.link(relay, next, Link::cost(1).priority(priority), Link::cost(1)); - let importer = relay + 3; - net.leaf(importer); - net.join(importer, relay, 1); - net.originate(importer, importer as u64, 0); - net.claim(importer as u64, "live/", 0, Some(1)); - } - assert!(net.run()); - net -} - -/// Per-origin routes are loop-free, yet a path-only request picks a different -/// origin at every hop and circles, past requester exclusion. Naming the -/// identity or bounding the total delivers. -#[test] -fn local_priorities_loop_path_only_requests() { - let net = priority_ring(1); - assert!(matches!(net.request(0, "live/x", PATH_ONLY), Outcome::Loop { .. })); - assert_eq!(served_by(&net.request(0, "live/x", Forward::Pinned)), Some((4, 4))); - assert_eq!(served_by(&net.request(0, "live/x", Forward::Bounded)), Some((4, 4))); - // The downstream condition needs no wire field but overrides the priority. - assert!(served_by(&net.request(0, "live/x", Forward::Downhill)).is_some()); - - // Without priorities the total is a potential that strictly decreases. - let net = priority_ring(0); - assert_eq!(served_by(&net.request(0, "live/x", PATH_ONLY)), Some((3, 3))); -} - -/// The same ring with every relay subscribing at once. Each forwarded request -/// joins the subscription its neighbor already opened, and no source feeds the -/// cycle. Joining by path or by instance deadlocks; joining only a subscription to -/// the same identity, or one within the request's bound, does not. -#[test] -fn aggregation_cycles_starve_without_a_forwarding_constraint() { - let net = priority_ring(1); - let all = [0, 1, 2]; - assert!(loops(&net.subscribe_all("live/x", &all, PATH_ONLY, Aggregate::Path))); - assert!(loops(&net.subscribe_all( - "live/x", - &all, - Forward::Pinned, - Aggregate::Instance - ))); - for (mode, aggregate) in [ - (Forward::Pinned, Aggregate::Identity), - (Forward::Bounded, Aggregate::Bound), - ] { - let outcomes = net.subscribe_all("live/x", &all, mode, aggregate); - assert!( - outcomes.iter().all(|outcome| served_by(outcome).is_some()), - "{outcomes:?}" - ); - } -} - -/// Under another instance, the serving identity is the instance. A relay that picks -/// another unnamed origin within the requester's bound splices a different -/// publisher under the requester's announced one, with no Restart. Only a -/// request naming the identity keeps the instance the requester announced. -#[test] -fn reselection_without_an_instance_splices_another_publisher() { - let (edge, core, first, second) = (0, 1, 2, 3); - let mut net = Net::new(Config::default(), 4); - net.join(edge, core, 1); - net.leaf(first); - net.leaf(second); - net.join(first, core, 2); - net.join(second, core, 1); - net.originate(first, 2, 0); - net.originate(second, 3, 0); - net.claim(2, "live/x", 0, None); - let late = net.claim(3, "live/x", 0, None); - // The edge has not heard the second claim yet. - net.hide(edge, late); - assert!(net.run()); - - assert_eq!(net.resolve(edge, "live/x"), Some(2)); - for mode in [PATH_ONLY, Forward::Downhill, Forward::Bounded] { - assert_eq!(served_by(&net.request(edge, "live/x", mode)), Some((3, 3)), "{mode:?}"); - } - assert_eq!(served_by(&net.request(edge, "live/x", Forward::Pinned)), Some((2, 2))); -} - -/// Nested claims `live/`, `live/a/`, and `live/a/x` in a ring with a chord, -/// with no priorities and symmetric costs. Each relay takes the longest claim -/// it has heard, so a relay missing the specific claims forwards toward the -/// wildcard while its neighbor forwards toward a more specific origin back -/// through it (RFC 8966 s2.8). The cycle has three hops, so requester -/// exclusion misses it. A request naming the identity follows one origin's -/// loop-free routes. -#[test] -fn overlapping_wildcards_loop_where_specific_claims_are_unheard() { - let mut net = Net::new(Config::default(), 7); - net.join(0, 1, 2); - net.join(1, 2, 1); - net.join(2, 3, 3); - net.join(3, 0, 1); - net.join(0, 2, 2); - let mut claims = Vec::new(); - for (origin, relay, prefix, seed) in [(4, 2, "live/", 1), (5, 1, "live/a/", 1), (6, 3, "live/a/x", 0)] { - net.leaf(origin); - net.join(origin, relay, 1); - net.originate(origin, origin as u64, 0); - claims.push(net.claim(origin as u64, prefix, seed, None)); - } - // Relay 0 has heard only the wildcard, relay 2 not the exact path. - net.hide(0, claims[1]); - net.hide(0, claims[2]); - net.hide(2, claims[2]); - assert!(net.run()); - - assert_eq!( - net.request(0, "live/a/x", PATH_ONLY), - Outcome::Loop { - path: vec![0, 2, 1, 0, 2] - } - ); - assert_eq!(served_by(&net.request(0, "live/a/x", Forward::Pinned)), Some((4, 4))); - assert_eq!(served_by(&net.request(1, "live/a/x", Forward::Pinned)), Some((6, 6))); -} - -/// A deterministic sweep over small random meshes: priorities, overlapping -/// wildcards, instances, and claims some nodes have not heard. Pinned and bounded -/// requests never loop or deadlock; path-only requests and the wire-free -/// downstream condition do. A pinned request always arrives, while a bounded -/// one stalls where a transit hop has not heard the claim. -#[test] -fn sweep_forwarding_progress() { - let mut rng = XorShift(0x5eed_c0de); - let (mut path_only_loops, mut downhill_loops, mut path_deadlocks, mut bounded_stalls) = (0, 0, 0, 0); - for scenario in 0..400 { - let net = random_mesh(&mut rng); - for at in 0..RELAYS { - for path in ["live/a/x", "live/b"] { - let label = format!("scenario {scenario} at {at} for {path}"); - for mode in [Forward::Pinned, Forward::Bounded] { - assert!( - !matches!(net.request(at, path, mode), Outcome::Loop { .. }), - "{mode:?} {label}" - ); - } - if net.resolve(at, path).is_some() { - // A pinned hop needs only a route; a bounded one needs the claim too. - assert!(served_by(&net.request(at, path, Forward::Pinned)).is_some(), "{label}"); - bounded_stalls += - matches!(net.request(at, path, Forward::Bounded), Outcome::Stalled { .. }) as usize; - } - path_only_loops += matches!(net.request(at, path, PATH_ONLY), Outcome::Loop { .. }) as usize; - downhill_loops += matches!(net.request(at, path, Forward::Downhill), Outcome::Loop { .. }) as usize; - } - } - let all: Vec = (0..RELAYS).collect(); - for path in ["live/a/x", "live/b"] { - // Joining by identity delivers exactly what each request would alone. - let alone: Vec = all.iter().map(|&at| net.request(at, path, Forward::Pinned)).collect(); - assert_eq!( - net.subscribe_all(path, &all, Forward::Pinned, Aggregate::Identity), - alone, - "scenario {scenario} for {path}" - ); - assert!(!loops(&net.subscribe_all( - path, - &all, - Forward::Bounded, - Aggregate::Bound - ))); - path_deadlocks += loops(&net.subscribe_all(path, &all, PATH_ONLY, Aggregate::Path)) as usize; - } - } - assert!(path_only_loops > 0, "the sweep must reach a path-only loop"); - assert!(downhill_loops > 0, "the sweep must reach a downstream-condition loop"); - assert!(path_deadlocks > 0, "the sweep must reach a path aggregation deadlock"); - assert!( - bounded_stalls > 0, - "a bounded hop that has not heard the claim must stall" - ); -} - -const RELAYS: usize = 5; - -struct XorShift(u64); - -impl XorShift { - fn next(&mut self, below: u64) -> u64 { - self.0 ^= self.0 << 13; - self.0 ^= self.0 >> 7; - self.0 ^= self.0 << 17; - self.0 % below - } - - fn chance(&mut self, percent: u64) -> bool { - self.next(100) < percent - } -} - -fn random_mesh(rng: &mut XorShift) -> Net { - let origins = 3; - let mut net = Net::new(Config::default(), RELAYS + origins); - // A ring keeps it connected; chords add alternatives. - for relay in 0..RELAYS { - let next = (relay + 1) % RELAYS; - let link = |rng: &mut XorShift| Link::cost(1 + rng.next(3)).priority(if rng.chance(25) { 1 } else { 0 }); - let (ab, ba) = (link(rng), link(rng)); - net.link(relay, next, ab, ba); - } - for _ in 0..2 { - let (a, b) = (rng.next(RELAYS as u64) as usize, rng.next(RELAYS as u64) as usize); - if a.abs_diff(b) > 1 && a.abs_diff(b) != RELAYS - 1 { - net.join(a, b, 1 + rng.next(3)); - } - } - let prefixes = ["live/", "live/a/", "live/b"]; - for origin in RELAYS..RELAYS + origins { - net.leaf(origin); - net.join(origin, rng.next(RELAYS as u64) as usize, 1); - net.originate(origin, origin as u64, 0); - let prefix = prefixes[rng.next(3) as usize]; - let instance = if rng.chance(50) { Some(1) } else { None }; - let claim = net.claim(origin as u64, prefix, rng.next(3), instance); - for relay in 0..RELAYS { - if rng.chance(20) { - net.hide(relay, claim); - } - } - } - assert!(net.run()); - net -} - -// Failover: Babel feasibility against longer detours. - -/// Redundant cores are feasible successors: losing one fails over at once. -#[test] -fn redundant_cores_fail_over_without_a_seqno_request() { - let (ingest, core_a, core_b, edge) = (0, 1, 2, 3); - let mut net = Net::new(Config::default(), 4); - net.join(ingest, core_a, 1); - net.join(ingest, core_b, 1); - net.join(core_a, core_b, 1); - net.link(edge, core_a, Link::cost(1).upstream(), Link::cost(1)); - net.link(edge, core_b, Link::cost(1).upstream(), Link::cost(1)); - net.originate(ingest, 7, 0); - assert!(net.run()); - let primary = net.route(edge, 7).and_then(|route| route.next).unwrap(); - - net.disconnect(edge, primary); - assert!(net.run()); - assert!( - net.route(edge, 7).is_some(), - "the other core advertises no more than the lost one" - ); -} - -/// A publisher reached directly over P2P and through the CDN. A browser selects -/// freely because nothing routes through it, so losing the P2P session falls -/// back at once. An opted-in native routing node advertised the one-hop route, -/// so the CDN detour is infeasible until the publisher advances its seqno, which -/// it can do because it saw its own P2P session close. -#[test] -fn p2p_fallback_is_immediate_for_leaves_and_needs_a_seqno_for_transit() { - for transit in [false, true] { - let (publisher, edge, subscriber) = (0, 1, 2); - let mut net = Net::new(Config::default(), 3); - net.leaf(publisher); - if !transit { - net.leaf(subscriber); - } - net.join(publisher, edge, 1); - net.link(subscriber, edge, Link::cost(1).upstream(), Link::cost(1)); - net.link(subscriber, publisher, Link::cost(1).priority(1), Link::cost(1)); - net.originate(publisher, 9, 0); - assert!(net.run()); - assert_eq!(net.route(subscriber, 9).and_then(|route| route.next), Some(publisher)); - - net.disconnect(subscriber, publisher); - assert!(net.run()); - assert_eq!(net.starved(subscriber, 9), transit); - - net.bump(publisher); - assert!(net.run()); - assert_eq!(net.route(subscriber, 9).and_then(|route| route.next), Some(edge)); - } -} - -/// A relay whose only peers are upstream sends no update for its down-only -/// route, so it holds no feasibility distance (RFC 8966 s3.7.3) and takes a -/// longer route at the same seqno once its best uplink goes, with no seqno -/// request. -#[test] -fn feasibility_tracks_only_updates_actually_sent() { - let (relay, c1, c2, origin) = (0, 1, 2, 3); - let mut net = Net::new(Config::default(), 4); - net.link(relay, c1, Link::cost(1).upstream(), Link::cost(1)); - net.link(relay, c2, Link::cost(1).upstream(), Link::cost(1)); - net.leaf(origin); - net.join(origin, c1, 1); - net.join(origin, c2, 5); - net.originate(origin, 7, 0); - assert!(net.run()); - assert_eq!(net.route(relay, 7).and_then(|route| route.next), Some(c1)); - - net.disconnect(relay, c1); - assert!(net.run()); - assert_eq!(net.route(relay, 7).and_then(|route| route.next), Some(c2)); -} - -// GOAWAY: the lowest preference, never a saturated metric. - -/// An edge with two upstream cores, preferring the first. A GOAWAY from that -/// core ranks it below every eligible peer, priority included, and the edge -/// fails over at once to the other core, which is feasible. The draining core -/// stays a last resort while its session is up. -#[test] -fn goaway_fails_over_and_keeps_the_draining_peer_last() { - let (ingest, core_a, core_b, edge) = (0, 1, 2, 3); - let mut net = Net::new(Config::default(), 4); - net.join(ingest, core_a, 1); - net.join(ingest, core_b, 1); - net.link(edge, core_a, Link::cost(1).upstream().priority(1), Link::cost(1)); - net.link(edge, core_b, Link::cost(1).upstream(), Link::cost(1)); - net.originate(ingest, 7, 0); - assert!(net.run()); - let next = |net: &Net| net.route(edge, 7).and_then(|route| route.next); - assert_eq!(next(&net), Some(core_a)); - - net.goaway(core_a); - assert!(net.run()); - assert_eq!(next(&net), Some(core_b)); - - net.disconnect(edge, core_b); - assert!(net.run()); - assert_eq!(next(&net), Some(core_a), "the draining core is the last resort"); -} - -/// An edge serving a client through two uplinks, where the second uplink's -/// route is longer than the one the edge advertised. Retracting the first -/// route, as a saturated drain metric would, starves the edge and the client -/// until the origin refreshes its seqno. A GOAWAY instead keeps the draining -/// uplink serving while the alternative is infeasible. -/// -/// The refresh reaches the edge through both uplinks. If the alternative's -/// arrives first, the edge moves. If the draining uplink's arrives first, the -/// edge selects it at the new seqno and lowers its feasibility distance, so the -/// alternative stays infeasible and the edge keeps the draining uplink until its -/// session closes. -#[test] -fn goaway_keeps_serving_until_a_refresh_makes_the_alternative_feasible() { - let (origin, a, b, edge, client) = (0, 1, 2, 3, 4); - let build = || { - let mut net = Net::new(Config::default(), 5); - net.leaf(origin); - net.leaf(client); - net.join(origin, a, 1); - net.join(origin, b, 5); - net.link(edge, a, Link::cost(1).upstream(), Link::cost(1)); - net.link(edge, b, Link::cost(1).upstream(), Link::cost(1)); - net.join(edge, client, 1); - net.originate(origin, 7, 0); - assert!(net.run()); - net - }; - let next = |net: &Net| net.route(edge, 7).and_then(|route| route.next); - - let mut net = build(); - net.disconnect(edge, a); - assert!(net.run()); - assert!(net.starved(edge, 7)); - assert!(net.route(client, 7).is_none()); - - for draining_first in [false, true] { - let mut net = build(); - net.goaway(a); - assert!(net.run()); - assert_eq!(next(&net), Some(a), "b advertised 5, above the edge's distance of 2"); - assert!(net.route(client, 7).is_some()); - - let late = if draining_first { b } else { a }; - net.hold(late, edge); - net.bump(origin); - assert!(net.run()); - net.release(late, edge); - assert!(net.run()); - assert!(net.route(client, 7).is_some()); - if draining_first { - assert_eq!(next(&net), Some(a), "the refresh through a lowered the distance first"); - net.disconnect(edge, a); - assert!(net.run()); - assert!(net.starved(edge, 7), "only another refresh moves the edge to b"); - } else { - assert_eq!(next(&net), Some(b)); - } - } -} - -// Export: one selection per identity, with a down-only bit. - -/// Two CDN relays, each the uplink of one relay in a local mesh, reaching a CDN -/// origin claiming `cdn/`. The CDN admits each local relay under `grant`. The -/// caller runs it. -fn two_uplinks(cfg: Config, l2_marks_upstream: bool, grant: Option) -> Net { - let (c1, c2, l1, l2, origin) = (0, 1, 2, 3, 4); - let mut net = Net::new(cfg, 5); - net.join(c1, c2, 10); - net.join(l1, l2, 1); - let admit = Link { grant, ..Link::cost(1) }; - let up = if l2_marks_upstream { - Link::cost(1).upstream() - } else { - Link::cost(1) - }; - net.link(l1, c1, Link::cost(1).upstream(), admit); - net.link(l2, c2, up, admit); - net.leaf(origin); - net.join(origin, c1, 1); - net.originate(origin, 7, 0); - net.claim(7, "cdn/", 0, None); - net -} - -/// Without export policy the CDN routes between its own relays through the -/// LAN; the down-only bit keeps it on the CDN link while the local relays still -/// reach the CDN origin. -#[test] -fn two_uplinks_never_carry_cdn_transit() { - let (c1, c2, l2) = (0, 1, 3); - let next = |net: &Net| net.route(c2, 7).and_then(|route| route.next); - let mut net = two_uplinks( - Config { - export: Export::Anything, - ..Config::default() - }, - true, - None, - ); - assert!(net.run()); - assert_eq!(next(&net), Some(l2)); - let mut net = two_uplinks(Config::default(), true, None); - assert!(net.run()); - assert_eq!(next(&net), Some(c1)); - assert!(net.route(l2, 7).is_some()); -} - -/// The topology that broke the two-view export model: C1-C2 at 100, a publisher -/// linked to C1 at 1 and to L1 at 10, the LAN L1-L2 at 1, and each local relay -/// marking its CDN link upstream. `uplink` decides whether L1's is up yet. -fn lan_beside_the_cdn(cfg: Config, uplink: bool) -> Net { - let (c1, c2, l1, l2, origin) = (0, 1, 2, 3, 4); - let mut net = Net::new(cfg, 5); - net.join(c1, c2, 100); - net.leaf(origin); - net.join(origin, c1, 1); - net.join(origin, l1, 10); - net.join(l1, l2, 1); - net.link(l2, c2, Link::cost(1).upstream(), Link::cost(1)); - net.originate(origin, 7, 0); - net.claim(7, "live/", 0, None); - assert!(net.run()); - if uplink { - net.link(l1, c1, Link::cost(1).upstream(), Link::cost(1)); - assert!(net.run()); - } - net -} - -/// L1's best route runs through C1, so it is down-only and L2 exports nothing -/// to C2, which pays its internal link. With no down-only bit, as with the -/// export view that lost its restriction one LAN hop after the upstream -/// ingress, C2 reaches C1 through the LAN. -#[test] -fn cdn_never_reaches_its_own_relay_through_the_lan() { - let (c1, c2, l1, l2, origin) = (0, 1, 2, 3, 4); - let served = |export| { - let cfg = Config { - export, - ..Config::default() - }; - lan_beside_the_cdn(cfg, true).request(c2, "live/x", Forward::Pinned) - }; - assert_eq!( - served(Export::Anything), - Outcome::Served { - by: 7, - path: vec![c2, l2, l1, c1, origin] - } - ); - assert_eq!( - served(Export::DownOnly), - Outcome::Served { - by: 7, - path: vec![c2, c1, origin] - } - ); -} - -/// Before L1's uplink comes up, L1 reaches the publisher directly and C2 -/// subscribes through the LAN. The uplink then turns L1's route down-only -/// through C1. While L2's retraction is still in flight to C2, L2, which -/// accepted C2's request from an upstream link, must end it. L1 cannot tell the -/// request came from upstream, since no request bit goes on the wire, and -/// would forward it to C1. Once the retraction lands, C2 re-resolves. -#[test] -fn upstream_ingress_ends_a_request_whose_route_turns_down_only() { - let (c1, c2, l1, l2, origin) = (0, 1, 2, 3, 4); - for recheck in [false, true] { - let cfg = Config { - recheck, - ..Config::default() - }; - let mut net = lan_beside_the_cdn(cfg, false); - assert_eq!( - net.request(c2, "live/x", Forward::Pinned), - Outcome::Served { - by: 7, - path: vec![c2, l2, l1, origin] - } - ); - - net.hold(l2, c2); - net.link(l1, c1, Link::cost(1).upstream(), Link::cost(1)); - assert!(net.run()); - assert_eq!(net.route(c2, 7).and_then(|route| route.next), Some(l2)); - let outcome = net.request(c2, "live/x", Forward::Pinned); - if recheck { - assert_eq!(outcome, Outcome::Stalled { path: vec![c2, l2] }); - } else { - assert_eq!( - outcome, - Outcome::Served { - by: 7, - path: vec![c2, l2, l1, c1, origin] - } - ); - } - // L2's own reader follows the down-only route, and C2's request must not - // join that subscription either. - assert_eq!( - net.subscribe_all("live/x", &[c2, l2], Forward::Pinned, Aggregate::Identity), - vec![ - outcome, - Outcome::Served { - by: 7, - path: vec![l2, l1, c1, origin] - } - ] - ); - - net.release(l2, c2); - assert!(net.run()); - assert_eq!( - net.request(c2, "live/x", Forward::Pinned), - Outcome::Served { - by: 7, - path: vec![c2, c1, origin] - } - ); - } -} - -/// A local relay whose own uplink is down still consumes CDN content and -/// publishes local content through its LAN peer: down-only routes flow down -/// the LAN, and exportable ones up the other uplink. -#[test] -fn dead_uplink_consumes_and_publishes_through_a_lan_peer() { - let (c1, c2, l1, l2, cdn, local) = (0, 1, 2, 3, 4, 5); - let mut net = Net::new(Config::default(), 6); - net.join(c1, c2, 10); - net.join(l1, l2, 1); - net.link(l1, c1, Link::cost(1).upstream(), Link::cost(1)); - net.link(l2, c2, Link::cost(1).upstream(), Link::cost(1)); - net.disconnect(l2, c2); - for (origin, relay, id, prefix) in [(cdn, c1, 7, "cdn/"), (local, l2, 9, "local/")] { - net.leaf(origin); - net.join(origin, relay, 1); - net.originate(origin, id, 0); - net.claim(id, prefix, 0, None); - } - assert!(net.run()); - - assert_eq!( - net.request(l2, "cdn/x", Forward::Pinned), - Outcome::Served { - by: 7, - path: vec![l2, l1, c1, cdn] - } - ); - assert_eq!( - net.request(c2, "local/x", Forward::Pinned), - Outcome::Served { - by: 9, - path: vec![c2, c1, l1, l2, local] - } - ); -} - -/// A partitioned local mesh reaches its other half through the CDN, and heals -/// back onto the LAN. The detour is longer than the route each island -/// advertised, so it waits on a seqno request answered by the origin. -#[test] -fn partitioned_local_mesh_reaches_itself_through_the_cdn() { - let (c1, c2, l1, l2, origin) = (0, 1, 2, 3, 4); - let mut net = Net::new(Config::default(), 5); - net.join(c1, c2, 10); - net.join(l1, l2, 1); - net.link(l1, c1, Link::cost(1).upstream(), Link::cost(1)); - net.link(l2, c2, Link::cost(1).upstream(), Link::cost(1)); - net.leaf(origin); - net.join(origin, l2, 1); - net.originate(origin, 7, 0); - net.claim(7, "local/", 0, None); - assert!(net.run()); - assert_eq!(net.route(l1, 7).and_then(|route| route.next), Some(l2)); - - net.disconnect(l1, l2); - assert!(net.run()); - assert!(net.starved(l1, 7), "the CDN detour is longer than the LAN route"); - - net.bump(origin); - assert!(net.run()); - assert_eq!(net.route(l1, 7).and_then(|route| route.next), Some(c1)); - assert_eq!( - served_by(&net.request(l1, "local/x", Forward::Pinned)), - Some((7, origin)) - ); - - net.connect(l1, l2); - assert!(net.run()); - assert_eq!(net.route(l1, 7).and_then(|route| route.next), Some(l2), "heals"); -} - -/// A local relay L2 whose LAN path to a local origin is cheap, behind a CDN -/// whose internal link is expensive. `lan_first` decides which session comes -/// up first, and `prefer_uplink` whether L2's application ranks its uplink -/// above the LAN or the LAN above it. -fn preferred_uplink(lan_first: bool, prefer_uplink: bool) -> Net { - let (c1, c2, l1, l2, origin) = (0, 1, 2, 3, 4); - let mut net = Net::new(Config::default(), 5); - net.join(c1, c2, 10); - net.leaf(origin); - net.join(origin, l1, 1); - net.originate(origin, 7, 0); - net.claim(7, "local/", 0, None); - let (uplink, lan) = if prefer_uplink { (1, 0) } else { (0, 1) }; - let lan = |net: &mut Net| net.link(l2, l1, Link::cost(1).priority(lan), Link::cost(1)); - if lan_first { - lan(&mut net); - } - net.link(l1, c1, Link::cost(1).upstream(), Link::cost(1)); - net.link(l2, c2, Link::cost(1).upstream().priority(uplink), Link::cost(1)); - assert!(net.run()); - if !lan_first { - lan(&mut net); - } - assert!(net.run()); - net -} - -/// With one selection per identity, a relay whose preferred route is down-only -/// exports nothing, so the CDN hauls the origin across its expensive internal -/// link while a LAN path costing 3 sits unadvertised. This is accepted; ranking -/// local peers above a backup uplink avoids it. -#[test] -fn preferred_down_only_route_hides_the_exportable_alternative() { - let (c1, c2, l1, l2, origin) = (0, 1, 2, 3, 4); - let next = |net: &Net, node| net.route(node, 7).and_then(|route| route.next); - - let net = preferred_uplink(false, true); - assert_eq!(next(&net, l2), Some(c2)); - assert!(net.export_route(l2, 7).is_none()); - assert_eq!(next(&net, c2), Some(c1)); - assert_eq!(net.route(c2, 7).map(|route| route.metric), Some(12)); - - let net = preferred_uplink(false, false); - assert_eq!(next(&net, l2), Some(l1)); - assert_eq!( - net.request(c2, "local/x", Forward::Pinned), - Outcome::Served { - by: 7, - path: vec![c2, l2, l1, origin] - } - ); -} - -/// Babel selects only feasible routes, so a transit relay's priority for a -/// longer route holds only if that route arrived before the shorter one was -/// advertised. The same topology settles on different routes depending on -/// which session came up first. -#[test] -fn transit_priority_depends_on_arrival_order() { - let (c2, l1, l2) = (1, 2, 3); - let next = |lan_first| { - preferred_uplink(lan_first, true) - .route(l2, 7) - .and_then(|route| route.next) - }; - assert_eq!(next(false), Some(c2)); - assert_eq!(next(true), Some(l1)); -} - -// Subject-qualified identities: a forwarded identity cannot reach beyond its admission. - -/// A scoped peer forwards a victim's identity with a newer seqno. Routes are -/// shared by every claim of an identity, so unscoped, the forgery captures -/// claims outside the forger's namespace, and its seqno outlives it: after it -/// leaves, the victim's real routes are refused until the victim counts past -/// the forged seqno. Filed under `hash(subject, id)`, the forgery serves only -/// claims inside the forger's scope. The forger's own identity is filed the -/// same way, so declaring the victim's identity as its own gains it nothing more. -#[test] -fn forged_identity_hijacks_claims_outside_the_forgers_scope() { - for scoped_admission in [false, true] { - let (victim, core, edge, forger) = (0, 1, 2, 3); - let mut net = Net::new(Config::default(), 4); - net.leaf(victim); - net.leaf(forger); - net.join(victim, core, 5); - net.join(core, edge, 1); - let admission = if scoped_admission { - Link::cost(1).grant(Grant { - subject: "forger", - scope: "forger/", - }) - } else { - Link::cost(1) - }; - net.link(edge, forger, admission, Link::cost(1)); - net.originate(victim, 7, 0); - net.originate(forger, 3, 0); - net.claim(7, "other/", 0, None); - net.claim(7, "forger/", 0, None); - assert!(net.run()); - - net.forge(forger, 7, 1000); - assert!(net.run()); - let own = if scoped_admission { qualified("forger", 3) } else { 3 }; - assert!(net.route(core, own).is_some()); - assert_eq!( - net.route(core, 3).is_some(), - !scoped_admission, - "a scoped connection's own identity is qualified" - ); - let served = served_by(&net.request(core, "other/x", Forward::Pinned)); - if scoped_admission { - assert_eq!(served, Some((7, victim))); - assert_eq!(net.resolve(core, "forger/x"), Some(qualified("forger", 7))); - assert_eq!( - served_by(&net.request(core, "forger/x", Forward::Pinned)), - Some((7, forger)), - "the forgery is contained in its own scope" - ); - } else { - assert_eq!(served, Some((7, forger)), "the forger serves another namespace"); - } - - net.kill(forger); - assert!(net.run()); - assert_eq!( - net.starved(core, 7), - !scoped_admission, - "the forged seqno outlives the forger" - ); - } -} - -/// A local mesh publishing through two uplinks, each admitted under the same -/// subject. Both CDN relays file the local origin under one key, so it survives -/// losing an uplink: the first hop keeps its identity and the request follows -/// the other uplink, named as the local relay knows it. Claims outside the -/// scope are refused under that key. The local relay's own identity is filed -/// under the same key, so its own broadcast fails over the same way. -#[test] -fn scoped_delivery_through_two_uplinks_reaches_one_key() { - let (c1, c2, l1, l2, origin) = (0, 1, 2, 3, 4); - let mut net = Net::new(Config::default(), 5); - net.join(c1, c2, 10); - net.join(l1, l2, 1); - let admit = Link::cost(1).grant(LOCAL); - net.link(l1, c1, Link::cost(1).upstream(), admit); - net.link(l2, c2, Link::cost(1).upstream(), admit); - net.leaf(origin); - net.join(origin, l1, 1); - net.originate(origin, 9, 0); - net.originate(l1, 20, 0); - net.claim(9, "local/", 0, None); - net.claim(9, "cdn/", 0, None); - net.claim(20, "local/l1/", 0, None); - assert!(net.run()); - - let key = qualified(LOCAL.subject, 9); - assert!(net.route(c1, 9).is_none()); - assert_eq!(net.route(c1, key).and_then(|route| route.next), Some(l1)); - assert_eq!(net.route(c2, key).and_then(|route| route.next), Some(l2)); - let own = qualified(LOCAL.subject, 20); - assert!( - net.route(c1, 20).is_none(), - "the local relay's own identity is qualified" - ); - assert_eq!(net.route(c1, own).and_then(|route| route.next), Some(l1)); - assert_eq!(net.resolve(c2, "local/l1/x"), Some(own)); - - // The local relay loses its direct uplink, and its own broadcast fails over through its sibling. - net.disconnect(l1, c1); - assert!(net.run()); - assert!(net.starved(c1, own), "the detour is longer than the lost uplink"); - // Its own set of sessions changed, so the origin advances its seqno. - net.bump(l1); - assert!(net.run()); - assert_eq!(net.resolve(c1, "local/l1/x"), Some(own)); - assert_eq!( - net.request(c1, "local/l1/x", Forward::Pinned), - Outcome::Served { - by: 20, - path: vec![c1, c2, l2, l1] - } - ); - net.connect(l1, c1); - assert!(net.run()); - assert_eq!(net.resolve(c2, "local/x"), Some(key)); - assert_eq!( - net.request(c2, "local/x", Forward::Pinned), - Outcome::Served { - by: 9, - path: vec![c2, l2, l1, origin] - } - ); - assert_eq!(net.resolve(c2, "cdn/x"), None, "outside the scope"); - assert_eq!(net.resolve(l1, "cdn/x"), Some(9)); - - net.disconnect(l2, c2); - assert!(net.run()); - assert_eq!(net.route(c2, key).and_then(|route| route.next), Some(c1)); - assert_eq!( - net.request(c2, "local/x", Forward::Pinned), - Outcome::Served { - by: 9, - path: vec![c2, c1, l1, origin] - } - ); - - // A retracted identity leaves no alias behind. - assert_eq!(net.aliases(c1), 2, "the origin's and the local relay's own"); - net.kill(origin); - assert!(net.run()); - assert_eq!(net.aliases(c1), 1); -} - -/// A local relay missing its `upstream` mark leaks CDN routes back to the CDN, -/// which then reaches its own origin through the LAN. Admitting the local relay -/// under a scoped grant files the leaked identity apart by its subject, where the CDN origin's claims -/// do not count, so the CDN keeps its own route. -#[test] -fn misconfigured_uplink_is_contained_by_qualified_identities() { - let (c1, c2, l1, l2, origin) = (0, 1, 2, 3, 4); - let mut net = two_uplinks(Config::default(), false, None); - assert!(net.run()); - assert_eq!(net.route(c2, 7).and_then(|route| route.next), Some(l2)); - assert_eq!( - net.request(c2, "cdn/x", Forward::Pinned), - Outcome::Served { - by: 7, - path: vec![c2, l2, l1, c1, origin] - } - ); - - let mut net = two_uplinks(Config::default(), false, Some(LOCAL)); - assert!(net.run()); - assert_eq!(net.route(c2, 7).and_then(|route| route.next), Some(c1)); - assert_eq!( - net.route(c2, qualified(LOCAL.subject, 7)).and_then(|route| route.next), - Some(l2), - "the leak is filed apart" - ); - assert_eq!( - net.request(c2, "cdn/x", Forward::Pinned), - Outcome::Served { - by: 7, - path: vec![c2, c1, origin] - } - ); -} - -/// A qualified key the CDN sent down comes back up through the misconfigured -/// uplink. Filed again as `hash(subject, hash(subject, id))`, it is a new -/// identity on every lap, so the CDN never settles. Qualified keys are -/// recognizable, and a scoped connection forwarding one has that route refused, so the lap ends and -/// the CDN holds what correctly marked uplinks give it. Nested scoped -/// federation is therefore out: the inner level must be unscoped. -#[test] -fn scoped_connections_cannot_requalify_a_qualified_identity() { - let c2 = 1; - let mut net = two_uplinks(Config::default(), true, Some(LOCAL)); - assert!(net.run()); - let marked = net.identities(c2); - - let mut net = two_uplinks(Config::default(), false, Some(LOCAL)); - assert!(net.run()); - assert_eq!(net.identities(c2), marked + 1, "the leak is filed once, never again"); - - let cfg = Config { - requalify: true, - ..Config::default() - }; - let mut net = two_uplinks(cfg, false, Some(LOCAL)); - let mut known = Vec::new(); - for _ in 0..4 { - assert!(!net.settle(10)); - known.push(net.identities(c2)); - } - assert!(known.windows(2).all(|pair| pair[0] < pair[1]), "{known:?}"); - assert!(known[0] > marked, "{known:?} against {marked}"); -} - -// Selection: every node agrees on a pool member. - -/// Cross-origin ties hash the path and identity, so every node agrees on a -/// pool member; hashing a node-local view does not. -#[test] -fn pool_ties_agree_only_on_a_path_and_identity_hash() { - let disagreements = |tie| { - let (edge_a, edge_b, hub_a, hub_b) = (0, 1, 2, 3); - let mut net = Net::new( - Config { - tie, - ..Config::default() - }, - 7, - ); - net.join(edge_a, hub_a, 1); - net.join(edge_b, hub_b, 1); - net.join(hub_a, hub_b, 1); - for member in 4..7 { - net.leaf(member); - net.join(member, hub_a, 1); - net.join(member, hub_b, 1); - net.originate(member, member as u64, 0); - net.claim(member as u64, "transcode/", 0, None); - } - assert!(net.run()); - (0..64) - .filter(|i| { - let path = format!("transcode/{i}"); - net.resolve(edge_a, &path) != net.resolve(edge_b, &path) - }) - .count() - }; - assert_eq!(disagreements(TieBreak::PathIdentity), 0); - assert!(disagreements(TieBreak::PathHere) > 0); -} diff --git a/rs/moq-net/tests/integration/routing_contracts/model.rs b/rs/moq-net/tests/integration/routing_contracts/model.rs deleted file mode 100644 index 04e47064a1d..00000000000 --- a/rs/moq-net/tests/integration/routing_contracts/model.rs +++ /dev/null @@ -1,2275 +0,0 @@ -//! A reference model of the proposed per-origin route layer, small enough to read. -//! -//! It is not the implementation. It exists so the routing contract in -//! `quest/m2/cluster-routes/README.md` has deterministic counterexamples before -//! the lite-07 wire is fixed: every rule the contract depends on is a switch -//! here, and each fixture runs a scenario under the rule that fails and the rule -//! that holds. -//! -//! Routing is Babel (RFC 8966) keyed by routing identity: each node selects one -//! feasible route per identity, advertises it to every neighbor whose session is -//! up, and only the identity's own node advances its seqno. Messages travel in -//! order on each directed session and can be held to replay them late. -//! -//! Claims come in two forms. The forwarding fixtures use global claims (an -//! identity serving a prefix, at a seed cost and optional instance) known -//! everywhere unless a fixture hides one from a node, which stands in for -//! scoping, interest, or a claim that has not arrived yet. The handoff fixtures -//! use announcements instead: an origin versions its own claims, and they travel -//! with its route as certified views that each node merges and shows. - -use std::{ - cmp::Reverse, - collections::{BTreeMap, BTreeSet, VecDeque}, - iter, -}; - -pub type Node = usize; - -/// An opaque routing identity: the hash of a per-process key, or [`qualified`]. -pub type Id = u64; - -/// Set on every [`qualified`] identity and never on a key-derived one, so any -/// relay recognizes a qualified key. -const QUALIFIED: Id = 1 << 63; - -/// A route metric: the sum of the configured link costs. -pub type Metric = u64; - -/// An origin's claim revision: it advances on every change to the origin's own -/// claims, starting from zero with each routing identity. -pub type Rev = u64; - -/// An origin's route seqno. -pub type Seqno = u64; - -/// The largest lite-07 varint, which carries both a seqno and a revision. An -/// origin never advertises it: reaching it rotates the identity instead, so -/// every advertised seqno leaves room for the successor a request asks for. -pub const VARINT_MAX: u64 = u64::MAX; - -/// How link costs accumulate. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Rule { - /// Today's single cost: any link cost, saturating at the ceiling, so a - /// zero-cost link or a saturated total does not increase the metric. - Saturating { ceiling: Metric }, - /// Strictly positive link costs with checked addition: a total beyond the - /// ceiling is unreachable rather than capped. - Checked { ceiling: Metric }, -} - -impl Rule { - /// The checked rule up to the largest QUIC varint. - pub const CHECKED: Self = Self::Checked { ceiling: (1 << 62) - 1 }; - - fn admits(self, cost: Metric) -> bool { - match self { - Self::Saturating { .. } => true, - Self::Checked { .. } => cost > 0, - } - } - - /// The metric after adding `cost`, or `None` when unreachable. - fn add(self, metric: Metric, cost: Metric) -> Option { - match self { - Self::Saturating { ceiling } => Some(metric.saturating_add(cost).min(ceiling)), - Self::Checked { ceiling } => metric.checked_add(cost).filter(|metric| *metric <= ceiling), - } - } -} - -/// What a node may advertise to an upstream link. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Export { - /// No restriction: every link transits. - Anything, - /// One selected route per identity, carrying a down-only bit set when it was - /// learned on an upstream link or from a down-only advertisement, and never - /// sent to an upstream link. - DownOnly, -} - -/// The cross-origin tie-break after cost. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum TieBreak { - /// Hash the path and the serving identity: every node agrees. - PathIdentity, - /// Hash the path, identity, next hop, and this node, standing in for - /// today's hash over a hop chain that differs per node: nodes may disagree. - PathHere, -} - -/// How a node adopts the announcements its neighbors send. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Freshness { - /// Next-hop authority: show what the selected next hop has sent so far, and - /// nothing while there is no selected route. - NextHop, - /// A snapshot barrier alone: keep the last adopted view until the new next - /// hop's complete view arrives, then adopt it whole. - Barrier, - /// Merge every neighbor's certified view, claim by claim, keeping whichever - /// knowledge is at the newer origin revision. A view's floors certify only the - /// claims overlapping their region; `scoped: false` lets any floor of the - /// origin vouch for any claim. - Floors { scoped: bool }, -} - -/// How a starved node gets its origin to advance the seqno. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Requests { - /// No seqno requests: fixtures advance the origin by hand. - Off, - /// The earlier sketch: a starved node sends a request when it starves and on - /// each [`Net::tick`] up to this many times, then gives up until an infeasible - /// update arrives. Forwarders keep no state. - Retries(u32), - /// Every node holding a request, its own or forwarded, keeps it until a route - /// at least that new is selected or no route is left, sends it to its next - /// hop even when that is a requester, and sends it again whenever that - /// neighbor changes. No timers. - HopByHop, -} - -#[derive(Clone, Copy, Debug)] -pub struct Config { - pub rule: Rule, - pub export: Export, - pub tie: TieBreak, - /// A relay that accepted a request from an upstream link ends it once its - /// route turns down-only. Without it the relay keeps following that route. - pub recheck: bool, - /// A scoped connection may forward an already qualified identity, which - /// the receiver qualifies again by its own subject. Otherwise that route is - /// refused. - pub requalify: bool, - pub freshness: Freshness, - pub requests: Requests, -} - -impl Default for Config { - fn default() -> Self { - Self { - rule: Rule::CHECKED, - export: Export::DownOnly, - tie: TieBreak::PathIdentity, - recheck: true, - requalify: false, - freshness: Freshness::Floors { scoped: true }, - requests: Requests::Off, - } - } -} - -/// One side's configuration of a link to a neighbor. -#[derive(Clone, Copy, Debug)] -pub struct Link { - pub cost: Metric, - /// The application's local priority for this connection; higher wins. - pub priority: i8, - /// This side marks the neighbor upstream. - pub upstream: bool, - /// This side admitted the neighbor under a scoped publish grant: an - /// identity the neighbor forwards, other than its own, is filed under - /// [`qualified`] by the grant's subject, and claims count under that key - /// only inside the grant's scope. - pub grant: Option, -} - -impl Default for Link { - fn default() -> Self { - Self { - cost: 1, - priority: 0, - upstream: false, - grant: None, - } - } -} - -impl Link { - pub fn cost(cost: Metric) -> Self { - Self { - cost, - ..Self::default() - } - } - - pub fn priority(mut self, priority: i8) -> Self { - self.priority = priority; - self - } - - pub fn upstream(mut self) -> Self { - self.upstream = true; - self - } - - pub fn grant(mut self, grant: Grant) -> Self { - self.grant = Some(grant); - self - } -} - -/// A scoped publish grant: the credential subject that qualifies forwarded -/// identities, and the prefix standing in for the grant's pattern union. -#[derive(Clone, Copy, Debug)] -pub struct Grant { - pub subject: &'static str, - pub scope: &'static str, -} - -/// What a node advertises for one identity. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct Advert { - seqno: Seqno, - metric: Metric, - down_only: bool, -} - -/// A ROUTE's state on a session. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Reach { - /// A usable route. - Finite(Advert), - /// An infinite metric: unusable, but the sender still holds the identity's - /// announcements while it waits for recovery. Only follows a finite route. - Held, -} - -/// One message on a directed session. -#[derive(Clone, Debug)] -enum Msg { - /// ROUTE_START or ROUTE_UPDATE, or `None` for ROUTE_END. - Route { id: Id, reach: Option }, - /// A batch of ANNOUNCEs closed by its floor: the sender's complete view of - /// this identity's claims within the receiver's interest. `interest` names - /// the interest it answers, as the request a stream belongs to does. - Claims { id: Id, view: View, interest: u32 }, - /// ROUTE_REQUEST, toward the neighbor whose route the sender would use. - Request { id: Id, seqno: Seqno }, -} - -/// A route a node holds from one neighbor. -#[derive(Clone, Copy, Debug)] -struct Entry { - /// The last finite advertisement. - advert: Advert, - held: bool, -} - -/// The route a node selected for an identity. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Selected { - /// The neighbor it goes through, or `None` when this node is the identity. - pub next: Option, - pub seqno: Seqno, - /// This node's metric. - pub metric: Metric, - /// The metric the next hop advertised. - pub advertised: Metric, - /// Whether an upstream link is on the path (only under [`Export::DownOnly`]). - pub down_only: bool, -} - -/// A seqno request a node keeps. -#[derive(Clone, Debug)] -struct Pending { - seqno: Seqno, - /// Neighbors it forwards the request for; empty when only its own. - from: BTreeSet, - /// The neighbor it last sent the request to. - asked: Option, - /// Resends left under [`Requests::Retries`]. - retries: u32, -} - -/// One announced claim: the attributes the wire carries, plus the revision its -/// life began at so fixtures can tell a revival from a fresh announcement. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Item { - pub seed: Metric, - pub instance: Option, - pub since: Rev, -} - -/// What a node knows of one origin's claims: each region's floor certifies -/// that the items overlapping it are exactly the origin's claims there as of -/// that revision. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct View { - floors: BTreeMap<&'static str, Rev>, - items: BTreeMap<&'static str, Item>, -} - -/// Whether a claim on `prefix` covers `path`: itself, or below it for a wildcard. -fn covers(prefix: &str, path: &str) -> bool { - path == prefix || (prefix.ends_with('/') && path.starts_with(prefix)) -} - -/// Whether `path` is `prefix` or lies below it, matching whole path segments. -pub fn within(path: &str, prefix: &str) -> bool { - path.strip_prefix(prefix) - .is_some_and(|rest| prefix.is_empty() || prefix.ends_with('/') || rest.is_empty() || rest.starts_with('/')) -} - -/// Whether `claim` can concern a path under `interest`: anything inside it, or -/// a wildcard above it. -pub fn overlaps(claim: &str, interest: &str) -> bool { - within(claim, interest) || covers(claim, interest) -} - -impl View { - /// What a node keeps of its view once its interest is `watch`. A floor - /// outside the interest may certify a wildcard above both at a newer - /// revision than any floor inside, so it stays, with what it certifies, - /// until a floor inside catches up; dropping it would leave that wildcard - /// known only at an older revision than the item it holds. Once no - /// neighbor certifies it, [`Net::prune`] lets it go. - fn retire(&self, watch: &'static str) -> View { - let mut kept = self.restrict(watch); - let newest = kept.floors.values().max().copied(); - for (®ion, &floor) in &self.floors { - if Some(floor) > newest && !kept.floors.contains_key(region) { - kept.floors.insert(region, floor); - for (&claim, &item) in &self.items { - if overlaps(claim, region) { - kept.items.insert(claim, item); - } - } - } - } - kept - } - - /// The view as a session to a peer interested in `prefix` carries it. - fn restrict(&self, prefix: &'static str) -> View { - let mut floors = BTreeMap::new(); - for (®ion, &floor) in &self.floors { - // Two interest regions meet when either contains the other. - if within(region, prefix) || within(prefix, region) { - let region = if region.len() > prefix.len() { region } else { prefix }; - let entry = floors.entry(region).or_insert(floor); - *entry = (*entry).max(floor); - } - } - // An item stays only while a kept floor still certifies it. - let items = self - .items - .iter() - .filter(|(claim, _)| floors.keys().any(|region| overlaps(claim, region))) - .map(|(claim, item)| (*claim, *item)) - .collect(); - View { floors, items } - } - - /// The newest revision this view knows `claim` at, if any region covers it. - fn as_of(&self, claim: &str, scoped: bool) -> Option { - self.floors - .iter() - .filter(|(region, _)| !scoped || overlaps(claim, region)) - .map(|(_, floor)| *floor) - .max() - } - - /// Take `other`'s state for every claim it knows at a newer revision, then - /// raise the floors and drop any region a wider one at least as new covers. - fn merge(&mut self, other: &View, scoped: bool) { - let claims: BTreeSet<&'static str> = self.items.keys().chain(other.items.keys()).copied().collect(); - for claim in claims { - if other.as_of(claim, scoped) > self.as_of(claim, scoped) { - match other.items.get(claim) { - Some(item) => self.items.insert(claim, *item), - None => self.items.remove(claim), - }; - } - } - for (®ion, &floor) in &other.floors { - let entry = self.floors.entry(region).or_insert(floor); - *entry = (*entry).max(floor); - } - let floors = self.floors.clone(); - self.floors.retain(|region, floor| { - !floors - .iter() - .any(|(wider, newer)| wider != region && within(region, wider) && newer >= floor) - }); - } - - /// Estimated bytes to move a receiver watching `interest` from `old` to - /// `self`: every floor and the changed or removed items, each with its - /// path as a suffix of the interest and a varint or two. - fn diff_bytes(&self, old: Option<&View>, interest: &str) -> (usize, usize) { - let empty = View::default(); - let old = old.unwrap_or(&empty); - // Every path overlaps the interest, so one not under it covers it and - // goes out as an empty suffix. The separator after the interest is - // model notation, not an encoded byte. - let suffix = |path: &str| { - if within(path, interest) { - path[interest.len()..].trim_start_matches('/').len() - } else { - 0 - } - }; - // One ANNOUNCE_FLOOR lists every region, changed or not: a count, then a - // suffix and a revision per region. - let floors = 1 + self.floors.keys().map(|region| suffix(region) + 4).sum::(); - let changed: Vec = self - .items - .iter() - .filter(|(claim, item)| old.items.get(*claim) != Some(item)) - .map(|(claim, _)| suffix(claim) + 8) - .chain( - old.items - .keys() - .filter(|claim| !self.items.contains_key(*claim)) - .map(|claim| suffix(claim)), - ) - .collect(); - (changed.len(), 9 + floors + changed.iter().sum::()) - } -} - -/// A path costs its suffix bytes however the interest above it is spelled. -#[test] -fn a_suffix_costs_the_same_however_its_interest_is_spelled() { - let item = Item { - seed: 0, - instance: None, - since: 1, - }; - let view = View { - floors: BTreeMap::from([("live/a/x", 1)]), - items: BTreeMap::from([("live/a/x", item)]), - }; - // A batch header, then a region count, `x` and its revision, then `x` and its attributes. - assert_eq!(view.diff_bytes(None, "live/a/"), (1, 9 + 1 + 5 + 9)); - assert_eq!(view.diff_bytes(None, "live/a"), (1, 9 + 1 + 5 + 9)); -} - -/// One change an origin made to its own claims. -struct Change { - rev: Rev, - claim: &'static str, - item: Option, -} - -/// How a node's reader sees one announced claim. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Shown { - pub item: Item, - /// Whether the node has a usable route to the claim's origin. - pub available: bool, -} - -/// A change a node's announcement reader sees. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Kind { - Start, - /// The claim's attributes changed. - Update, - /// The node, still holding the origin, no longer shows the claim. - End, - /// The node let go of the origin: it has no usable or recovering route left. - Offline, - /// The node stopped watching the claim. - Unwatched, - Available, - Unavailable, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Event { - pub node: Node, - pub id: Id, - pub claim: &'static str, - pub kind: Kind, - pub since: Rev, - /// How many times the node had restarted, and then forgotten this claim of - /// this origin by collecting or retiring it, before this event. - pub forgot: (u32, u32), - /// Whether the origin had withdrawn or replaced this life of the claim by - /// then. An `End` without it is a false END. - pub withdrawn: bool, -} - -/// Control traffic and work, as estimates: these are protocol counts in -/// virtual time, not runtime benchmarks. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub struct Stats { - pub route_msgs: usize, - pub claim_msgs: usize, - /// Changed or removed items carried by claim batches: the receivers' replay work. - pub claim_items: usize, - pub claim_bytes: usize, - pub requests: usize, -} - -/// State kept for announcements, summed over every node. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub struct Retained { - /// Items in each node's own merged or adopted views. - pub items: usize, - /// Region floors in those views. - pub floors: usize, - /// Items in the per-session views a receiver keeps to apply the next batch. - pub heard: usize, - /// Region floors in those per-session views. - pub heard_floors: usize, - /// Items in the per-session views a sender keeps to diff the next batch. - pub sent: usize, - /// Region floors in those per-session views. - pub sent_floors: usize, - /// ROUTE_ENDs sent and not yet echoed back. - pub echoes: usize, - pub pending: usize, - /// Feasibility distances for identities a node does not serve. - pub distances: usize, - /// Regions certified since an interest widened, still awaiting the rest. - pub certified: usize, -} - -#[derive(Default)] -struct State { - /// This node's own identity, if it originates one. - proven: Option, - /// Identities this node serves itself, with their seqnos: its own, plus any - /// it forges. - origins: BTreeMap, - /// Whether this node advertises identities it does not serve. - transit: bool, - rx: BTreeMap>, - /// For a key this node qualified, the identity the neighbor named. - filed: BTreeMap<(Node, Id), Id>, - fd: BTreeMap, - sent: BTreeMap<(Node, Id), Reach>, - /// ROUTE_ENDs sent to each neighbor that it has not echoed back yet. Until - /// they all return, a neighbor may still hold this node's advertisement, - /// so the feasibility distance stays. - unacked: BTreeMap<(Node, Id), u32>, - pending: BTreeMap, - /// Identities whose own request [`Requests::Retries`] gave up on. - gave_up: BTreeSet, - /// The prefix this node asks its neighbors for announcements under. - watch: &'static str, - /// Counts this node's interest changes. - interest: u32, - /// Each origin's claims as this node knows them; authoritative for its own. - views: BTreeMap, - /// After the interest widened, the regions each view is known current in: - /// the old interest, plus what each batch answering the new one certified. - /// The rest is hidden until certified. - current: BTreeMap>, - /// The last view each neighbor sent, per identity. - heard: BTreeMap<(Node, Id), View>, - /// The interest stream each heard view arrived on. - heard_on: BTreeMap<(Node, Id), u32>, - sent_views: BTreeMap<(Node, Id), View>, - shown: BTreeMap<(Id, &'static str), Shown>, - /// How many times this node restarted. - restarts: u32, - /// Per claim this node has shown, how many times it forgot that claim, so a - /// stale replay is excused only for a claim it actually forgot. Fixture - /// bookkeeping, not relay state: it labels reader events. - forgot: BTreeMap<(Id, &'static str), u32>, -} - -/// Counts a forget against each shown claim of `id` overlapping none of `kept`. -fn forget(forgot: &mut BTreeMap<(Id, &'static str), u32>, id: Id, kept: &[&str]) { - for ((key, claim), count) in forgot.iter_mut() { - if *key == id && !kept.iter().any(|region| overlaps(claim, region)) { - *count += 1; - } - } -} - -/// An identity serving everything under a prefix. -#[derive(Clone, Copy, Debug)] -pub struct Claim { - pub id: Id, - /// A trailing `/` claims the subtree; otherwise only the exact path. - pub prefix: &'static str, - pub seed: Metric, - pub instance: Option, -} - -impl Claim { - fn covers(&self, path: &str) -> bool { - covers(self.prefix, path) - } -} - -/// How a transit hop treats a request it forwards. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Forward { - /// Every hop reselects among all origins; optionally never back to the requester. - PathOnly { exclude_requester: bool }, - /// No wire change: a transit hop forwards only to a candidate whose - /// advertised total is strictly below its own best total (RFC 5286's - /// downstream condition). - Downhill, - /// The first hop names the identity; later hops follow their route to it. - Pinned, - /// The first hop names a bound; later hops reselect within it. - Bounded, -} - -/// What a forwarded request carries. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] -struct Carried { - pinned: Option, - bound: Option, - instance: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum Outcome { - Served { by: Id, path: Vec }, - Loop { path: Vec }, - Stalled { path: Vec }, -} - -/// Which existing upstream subscription a forwarded request may join. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Aggregate { - /// Any subscription for the path, as today. - Path, - /// One serving the same explicit instance. - Instance, - /// One following a route to the same identity. - Identity, - /// One whose total is within the request's bound. - Bound, -} - -#[derive(Clone, Copy, Debug)] -struct Candidate { - id: Id, - next: Option, - seed: Metric, - total: Metric, - advertised: Metric, - draining: bool, - priority: i8, - prefix: usize, - instance: Option, - down_only: bool, -} - -#[derive(Clone, Copy, Debug)] -enum Up { - Serve, - Stalled, - Pending(Node, Node, Carried), - Joined(usize), -} - -#[derive(Clone, Copy, Debug)] -struct Front { - node: Node, - id: Id, - instance: Option, - total: Metric, - /// Whether the route it follows is down-only. - down_only: bool, - up: Up, -} - -pub struct Net { - cfg: Config, - nodes: Vec, - /// `links[a][b]`: `a`'s side of its link to `b`. Dense rather than a map, - /// since selection looks a side up per route and the sweeps select constantly. - links: Vec>>, - /// Every `(subject, scope)` a configured link grants. - grants: BTreeSet<(&'static str, &'static str)>, - up: BTreeSet<(Node, Node)>, - /// `(at, peer)`: `peer` sent `at` a GOAWAY on their current session. - draining: BTreeSet<(Node, Node)>, - queues: BTreeMap<(Node, Node), VecDeque>, - /// ROUTE_ENDs echoed back, on the stream of the route they end: unordered - /// against the echoing node's own routes in `queues`. - acks: BTreeMap<(Node, Node), VecDeque>, - held: BTreeSet<(Node, Node)>, - claims: Vec, - hidden: BTreeSet<(Node, usize)>, - /// Every change each origin made to its own claims, in revision order. - history: BTreeMap>, - pub stats: Stats, - pub events: Vec, -} - -impl Net { - pub fn new(cfg: Config, nodes: usize) -> Self { - Self { - cfg, - nodes: (0..nodes) - .map(|_| State { - transit: true, - ..State::default() - }) - .collect(), - links: vec![vec![None; nodes]; nodes], - grants: BTreeSet::new(), - up: BTreeSet::new(), - draining: BTreeSet::new(), - queues: BTreeMap::new(), - acks: BTreeMap::new(), - held: BTreeSet::new(), - claims: Vec::new(), - hidden: BTreeSet::new(), - history: BTreeMap::new(), - stats: Stats::default(), - events: Vec::new(), - } - } - - /// A node that advertises only identities it serves, like a browser. - pub fn leaf(&mut self, node: Node) { - self.nodes[node].transit = false; - } - - /// Configure both sides of a link and open its session. - pub fn link(&mut self, a: Node, b: Node, ab: Link, ba: Link) { - let rule = self.cfg.rule; - assert!( - rule.admits(ab.cost) && rule.admits(ba.cost), - "zero-cost link {a}-{b} refused" - ); - self.links[a][b] = Some(ab); - self.links[b][a] = Some(ba); - self.grants = self - .links - .iter() - .flatten() - .flatten() - .filter_map(|link| link.grant) - .map(|grant| (grant.subject, grant.scope)) - .collect(); - self.connect(a, b); - } - - /// A symmetric link of `cost`. - pub fn join(&mut self, a: Node, b: Node, cost: Metric) { - self.link(a, b, Link::cost(cost), Link::cost(cost)); - } - - /// `node`'s side of its configured link to `peer`. - fn side(&self, node: Node, peer: Node) -> &Link { - self.links[node][peer].as_ref().expect("a configured link") - } - - fn pair(a: Node, b: Node) -> (Node, Node) { - (a.min(b), a.max(b)) - } - - /// Open a session; each side replays what it currently advertises. - pub fn connect(&mut self, a: Node, b: Node) { - assert!(self.links[a][b].is_some(), "unknown link {a}-{b}"); - self.up.insert(Self::pair(a, b)); - self.advertise(a); - self.advertise(b); - } - - pub fn connected(&self, a: Node, b: Node) -> bool { - self.up.contains(&Self::pair(a, b)) - } - - /// Close a session: in-flight messages and everything learned on it go. - pub fn disconnect(&mut self, a: Node, b: Node) { - self.up.remove(&Self::pair(a, b)); - for (x, y) in [(a, b), (b, a)] { - self.queues.remove(&(x, y)); - self.acks.remove(&(x, y)); - self.held.remove(&(x, y)); - self.draining.remove(&(y, x)); - let state = &mut self.nodes[y]; - for routes in state.rx.values_mut() { - routes.remove(&x); - } - state.rx.retain(|_, routes| !routes.is_empty()); - state.filed.retain(|(from, _), _| *from != x); - state.heard.retain(|(from, _), _| *from != x); - state.heard_on.retain(|(from, _), _| *from != x); - for pending in state.pending.values_mut() { - pending.from.remove(&x); - } - let state = &mut self.nodes[x]; - // A closed session is an implicit echo: it held nothing past the close. - state.unacked.retain(|(n, _), _| *n != y); - state.sent.retain(|(n, _), _| *n != y); - state.sent_views.retain(|(n, _), _| *n != y); - } - self.advertise(a); - self.advertise(b); - } - - /// The process at `node` exits: its sessions close and its state is lost. - pub fn kill(&mut self, node: Node) { - let peers: Vec = self.peers(node).collect(); - for peer in peers { - self.disconnect(node, peer); - } - let old = std::mem::take(&mut self.nodes[node]); - self.nodes[node] = State { - transit: old.transit, - watch: old.watch, - interest: old.interest, - restarts: old.restarts + 1, - ..State::default() - }; - self.show(node); - } - - /// Re-open every configured link of `node`. - pub fn reconnect(&mut self, node: Node) { - let peers: Vec = (0..self.nodes.len()) - .filter(|&b| self.links[node][b].is_some()) - .collect(); - for peer in peers { - self.connect(node, peer); - } - } - - /// Every neighbor `node` has a session up with, in ascending order. - fn peers(&self, node: Node) -> impl Iterator + '_ { - (0..self.nodes.len()).filter(move |&peer| self.links[node][peer].is_some() && self.connected(node, peer)) - } - - /// `node` starts originating `id` at seqno `seqno`. - pub fn originate(&mut self, node: Node, id: Id, seqno: Seqno) { - assert_eq!(id & QUALIFIED, 0, "a key-derived identity is never qualified"); - let state = &mut self.nodes[node]; - state.proven = Some(id); - state.origins.insert(id, seqno); - state.views.insert( - id, - View { - floors: BTreeMap::from([("", 0)]), - items: BTreeMap::new(), - }, - ); - self.advertise(node); - } - - /// `node` advertises `id` as if forwarding it. - pub fn forge(&mut self, node: Node, id: Id, seqno: Seqno) { - self.nodes[node].origins.insert(id, seqno); - self.advertise(node); - } - - /// The origin advances its seqno. - pub fn bump(&mut self, node: Node) { - let state = &mut self.nodes[node]; - let id = state.proven.expect("not an origin"); - let seqno = state.origins.get_mut(&id).unwrap(); - if *seqno + 1 == VARINT_MAX { - return self.rotate(node); - } - *seqno += 1; - self.advertise(node); - } - - /// The origin at `node` retires its identity for a fresh one and announces - /// its claims again there: what it does instead of advancing a seqno or a - /// revision to [`VARINT_MAX`]. Every session closes and reopens, so every - /// node lets the old identity go. - fn rotate(&mut self, node: Node) { - let peers: Vec = self.peers(node).collect(); - for &peer in &peers { - self.disconnect(node, peer); - } - let state = &mut self.nodes[node]; - let old = state.proven.expect("not an origin"); - state.origins.remove(&old); - let claims = state.views.remove(&old).map(|view| view.items).unwrap_or_default(); - self.originate(node, fnv("rotate", &[old]) & !QUALIFIED, 0); - for (claim, item) in claims { - self.change(node, claim, Some((item.seed, item.instance))); - } - for peer in peers { - self.connect(node, peer); - } - } - - /// Jump the origin's revision to `rev`, as a long-lived origin's claim - /// changes reach it. - pub fn age(&mut self, node: Node, rev: Rev) { - let id = self.proven(node); - self.nodes[node].views.get_mut(&id).unwrap().floors.insert("", rev); - } - - /// The identity `node` originates. - pub fn proven(&self, node: Node) -> Id { - self.nodes[node].proven.expect("not an origin") - } - - /// The prefix `node` watches. - pub fn watching(&self, node: Node) -> &'static str { - self.nodes[node].watch - } - - /// The origin's current seqno. - pub fn seqno(&self, node: Node) -> Seqno { - let state = &self.nodes[node]; - state.origins[&state.proven.expect("not an origin")] - } - - /// The origin at `node` starts or updates a claim, advancing its revision. - pub fn announce(&mut self, node: Node, claim: &'static str, seed: Metric, instance: Option) { - self.change(node, claim, Some((seed, instance))); - } - - /// The origin at `node` withdraws a claim, advancing its revision. - pub fn unannounce(&mut self, node: Node, claim: &'static str) { - self.change(node, claim, None); - } - - fn change(&mut self, node: Node, claim: &'static str, attrs: Option<(Metric, Option)>) { - let id = self.nodes[node].proven.expect("not an origin"); - let view = self.nodes[node].views.get_mut(&id).unwrap(); - let rev = view.floors[""] + 1; - if rev == VARINT_MAX { - self.rotate(node); - return self.change(node, claim, attrs); - } - view.floors.insert("", rev); - let item = attrs.map(|(seed, instance)| Item { - seed, - instance, - since: view.items.get(claim).map_or(rev, |item| item.since), - }); - match item { - Some(item) => view.items.insert(claim, item), - None => view.items.remove(claim), - }; - self.history.entry(id).or_default().push(Change { rev, claim, item }); - self.advertise(node); - } - - /// The prefix `node` asks its neighbors for announcements under. - /// - /// A new interest is a new stream: the old stream's batches still in flight - /// are discarded, and each neighbor starts the new stream with a complete - /// snapshot, which retires what the node knew outside the new interest. - pub fn watch(&mut self, node: Node, prefix: &'static str) { - let state = &mut self.nodes[node]; - if !within(prefix, state.watch) { - let old = state.watch; - for id in state.views.keys().filter(|id| !state.origins.contains_key(id)) { - state.current.entry(*id).or_insert_with(|| vec![old]); - } - } - state - .current - .retain(|_, regions| !regions.iter().any(|region| within(prefix, region))); - state.watch = prefix; - state.interest += 1; - let peers: Vec = self.peers(node).collect(); - for peer in peers { - self.nodes[peer].sent_views.retain(|(to, _), _| *to != node); - self.advertise(peer); - } - self.advertise(node); - } - - /// `node` sends GOAWAY on every session: each peer ranks it below every - /// other eligible route while the session stays up. - pub fn goaway(&mut self, node: Node) { - let peers: Vec = self.peers(node).collect(); - for peer in peers { - self.draining.insert((peer, node)); - self.advertise(peer); - } - } - - /// Forget `id`'s routes and announcements at `node`. The feasibility - /// distance goes on its own once drained (see [`Net::advertise`]). - pub fn collect(&mut self, node: Node, id: Id) { - let state = &mut self.nodes[node]; - state.rx.remove(&id); - state.filed.retain(|(_, key), _| *key != id); - state.views.remove(&id); - state.current.remove(&id); - state.heard.retain(|(_, key), _| *key != id); - state.heard_on.retain(|(_, key), _| *key != id); - state.pending.remove(&id); - state.gave_up.remove(&id); - forget(&mut state.forgot, id, &[]); - self.advertise(node); - } - - /// Whether every neighbor has processed `node`'s retraction of `id`: none - /// holds an entry from it and no route message from it is in flight. The - /// echoed ROUTE_ENDs tell `node` this; a restart gets it by - /// closing every session. - pub fn retracted(&self, node: Node, id: Id) -> bool { - self.peers(node).all(|peer| { - let queued = self.queues.get(&(node, peer)).is_some_and(|queue| { - queue - .iter() - .any(|msg| matches!(msg, Msg::Route { id: sent, .. } if *sent == id)) - }); - let held = self - .key(peer, node, id) - .and_then(|key| self.nodes[peer].rx.get(&key)) - .is_some_and(|routes| routes.contains_key(&node)); - !queued && !held - }) - } - - /// Drop `node`'s feasibility distance for `id`. - pub fn forget(&mut self, node: Node, id: Id) { - self.nodes[node].fd.remove(&id); - } - - pub fn claim(&mut self, id: Id, prefix: &'static str, seed: Metric, instance: Option) -> usize { - self.claims.push(Claim { - id, - prefix, - seed, - instance, - }); - self.claims.len() - 1 - } - - /// `node` does not know claim `index`. - pub fn hide(&mut self, node: Node, index: usize) { - self.hidden.insert((node, index)); - } - - pub fn hold(&mut self, from: Node, to: Node) { - self.held.insert((from, to)); - } - - pub fn release(&mut self, from: Node, to: Node) { - self.held.remove(&(from, to)); - } - - /// Deliver one in-order message from `from` to `to`, even if held. - pub fn deliver(&mut self, from: Node, to: Node) -> bool { - let Some(msg) = self.queues.get_mut(&(from, to)).and_then(VecDeque::pop_front) else { - return false; - }; - self.receive(to, from, msg); - true - } - - /// Deliver one ROUTE_END echoed back from `from` to `to`, even if held. - pub fn deliver_ack(&mut self, from: Node, to: Node) -> bool { - let Some(id) = self.acks.get_mut(&(from, to)).and_then(VecDeque::pop_front) else { - return false; - }; - let state = &mut self.nodes[to]; - let left = state.unacked.get_mut(&(from, id)).expect("an echo answers a ROUTE_END"); - *left -= 1; - if *left == 0 { - state.unacked.remove(&(from, id)); - } - self.advertise(to); - true - } - - /// The ROUTE_END echoes waiting from `from` to `to`. - pub fn queued_acks(&self, from: Node, to: Node) -> usize { - self.acks.get(&(from, to)).map_or(0, VecDeque::len) - } - - /// Every directed session with an echo waiting, held or not. - pub fn busy_acks(&self) -> Vec<(Node, Node)> { - self.acks - .iter() - .filter(|(_, queue)| !queue.is_empty()) - .map(|(key, _)| *key) - .collect() - } - - /// Whether `node` keeps a feasibility distance for `id`. - pub fn remembers(&self, node: Node, id: Id) -> bool { - self.nodes[node].fd.contains_key(&id) - } - - /// The messages waiting from `from` to `to`. - pub fn queued(&self, from: Node, to: Node) -> usize { - self.queues.get(&(from, to)).map_or(0, VecDeque::len) - } - - /// Every directed session with a message waiting, held or not. - pub fn busy(&self) -> Vec<(Node, Node)> { - self.queues - .iter() - .filter(|(_, queue)| !queue.is_empty()) - .map(|(key, _)| *key) - .collect() - } - - /// Deliver round-robin until every unheld queue is empty. False if it never - /// settles within a generous budget, which is how oscillation shows up. - pub fn run(&mut self) -> bool { - self.settle(10_000) - } - - /// [`Net::run`] for at most `rounds` rounds. - pub fn settle(&mut self, rounds: usize) -> bool { - for _ in 0..rounds { - let ready: Vec<(Node, Node)> = self - .queues - .iter() - .filter(|(key, queue)| !queue.is_empty() && !self.held.contains(key)) - .map(|(key, _)| *key) - .collect(); - let acks: Vec<(Node, Node)> = self - .acks - .iter() - .filter(|(key, queue)| !queue.is_empty() && !self.held.contains(key)) - .map(|(key, _)| *key) - .collect(); - if ready.is_empty() && acks.is_empty() { - return true; - } - for (from, to) in ready { - self.deliver(from, to); - } - for (from, to) in acks { - self.deliver_ack(from, to); - } - } - false - } - - /// A resend timer fires everywhere: under [`Requests::Retries`], each node - /// resends its own pending request, or gives up once its retries are spent. - pub fn tick(&mut self) { - if !matches!(self.cfg.requests, Requests::Retries(_)) { - return; - } - for node in 0..self.nodes.len() { - let own: Vec = self.nodes[node] - .pending - .iter() - .filter(|(_, pending)| pending.from.is_empty()) - .map(|(id, _)| *id) - .collect(); - for id in own { - let pending = self.nodes[node].pending.get_mut(&id).unwrap(); - if pending.retries == 0 { - self.nodes[node].pending.remove(&id); - self.nodes[node].gave_up.insert(id); - continue; - } - pending.retries -= 1; - let seqno = pending.seqno; - if let Some(target) = self.target(node, id, &BTreeSet::new()) { - self.send_request(node, target, id, seqno); - } - } - } - } - - /// The key `at` files `id` under when `from` names it, or `None` when refused. - /// - /// Under a scoped grant every identity is qualified, the sender's own included, by - /// the grant's subject, which stands in for the route key the sender proved. - fn key(&self, at: Node, from: Node, id: Id) -> Option { - match self.side(at, from).grant { - Some(grant) => { - if id & QUALIFIED != 0 && !self.cfg.requalify { - return None; - } - Some(qualified(grant.subject, id)) - } - _ => Some(id), - } - } - - fn receive(&mut self, at: Node, from: Node, msg: Msg) { - match msg { - Msg::Route { id: named, reach } => { - // Echo every ROUTE_END once it is applied, filed or refused. - if reach.is_none() { - self.stats.route_msgs += 1; - self.acks.entry((at, from)).or_default().push_back(named); - } - let Some(id) = self.key(at, from, named) else { - return; - }; - if id != named { - match reach { - Some(_) => self.nodes[at].filed.insert((from, id), named), - None => self.nodes[at].filed.remove(&(from, id)), - }; - } - if let (Requests::Retries(_), Some(Reach::Finite(advert))) = (self.cfg.requests, reach) - && !self.feasible(at, id, advert) - { - self.nodes[at].gave_up.remove(&id); - } - let state = &mut self.nodes[at]; - let routes = state.rx.entry(id).or_default(); - match reach { - Some(Reach::Finite(advert)) => { - routes.insert(from, Entry { advert, held: false }); - } - Some(Reach::Held) => { - if let Some(entry) = routes.get_mut(&from) { - entry.held = true; - } - } - None => { - routes.remove(&from); - } - } - if routes.is_empty() { - state.rx.remove(&id); - } - } - Msg::Claims { id, view, interest } => { - let Some(id) = self.key(at, from, id) else { - return; - }; - let scope = match self.side(at, from).grant { - Some(grant) if id & QUALIFIED != 0 => Some(grant.scope), - _ => None, - }; - let view = match scope { - Some(scope) => view.restrict(scope), - None => view, - }; - let state = &mut self.nodes[at]; - if interest != state.interest { - // A batch from a stream the node has since replaced. - return; - } - // The batch certifies the regions its floors name. - let watch = state.watch; - for ®ion in view.floors.keys() { - let Some(regions) = state.current.get_mut(&id) else { - break; - }; - if region == watch { - state.current.remove(&id); - } else if !regions.iter().any(|known| within(region, known)) { - regions.retain(|known| !within(known, region)); - regions.push(region); - } - } - if let Freshness::Floors { scoped } = self.cfg.freshness - && !state.origins.contains_key(&id) - { - let merged = state.views.entry(id).or_default(); - merged.merge(&view, scoped); - // The batch certifies the interest, so what lies outside can go. - let kept = merged.retire(state.watch); - if kept != *merged { - *merged = kept; - forget(&mut state.forgot, id, &[state.watch]); - } - } - // The batch's floor list replaces the session's regions: one left - // out is no longer certified, and an empty list certifies nothing. - state.heard.insert((from, id), view); - state.heard_on.insert((from, id), interest); - } - Msg::Request { id, seqno } => self.request_received(at, from, id, seqno), - } - self.advertise(at); - } - - fn request_received(&mut self, at: Node, from: Node, id: Id, seqno: Seqno) { - let state = &mut self.nodes[at]; - if let Some(own) = state.origins.get_mut(&id) { - // Only the origin advances its seqno to answer. - if seqno == VARINT_MAX { - assert_eq!(state.proven, Some(id), "a forged identity ran out of seqnos"); - self.rotate(at); - } else { - *own = (*own).max(seqno); - } - return; - } - if self.route(at, id).is_some_and(|route| route.seqno >= seqno) { - // Already advertised to the requester. - return; - } - match self.cfg.requests { - Requests::Off => {} - Requests::Retries(_) => { - if let Some(target) = self.target(at, id, &BTreeSet::from([from])) { - self.send_request(at, target, id, seqno); - } - } - Requests::HopByHop => { - let pending = self.nodes[at].pending.entry(id).or_insert(Pending { - seqno, - from: BTreeSet::new(), - asked: None, - retries: 0, - }); - if seqno > pending.seqno { - pending.seqno = seqno; - pending.asked = None; - } - pending.from.insert(from); - } - } - } - - fn send_request(&mut self, node: Node, to: Node, id: Id, seqno: Seqno) { - self.stats.requests += 1; - let id = self.name(node, to, id); - self.queues - .entry((node, to)) - .or_default() - .push_back(Msg::Request { id, seqno }); - } - - /// The identity `node` names for `id` in a request it sends `next`: the one - /// `next` advertised, if `node` qualified it by `next`'s subject. - fn name(&self, node: Node, next: Node, id: Id) -> Id { - self.nodes[node].filed.get(&(next, id)).copied().unwrap_or(id) - } - - /// How many qualified keys `node` remembers the forwarded name of. - pub fn aliases(&self, node: Node) -> usize { - self.nodes[node].filed.len() - } - - /// How many identities `node` holds routes to. - pub fn identities(&self, node: Node) -> usize { - self.nodes[node].rx.len() - } - - /// How many region floors `node` keeps for `id`. - pub fn floors(&self, node: Node, id: Id) -> usize { - self.nodes[node].views.get(&id).map_or(0, |view| view.floors.len()) - } - - /// Whether `node` holds any route state for `id`. - pub fn holds(&self, node: Node, id: Id) -> bool { - self.nodes[node].rx.contains_key(&id) - } - - /// Babel's feasibility condition against `node`'s feasibility distance. - fn feasible(&self, node: Node, id: Id, advert: Advert) -> bool { - match self.nodes[node].fd.get(&id) { - None => true, - Some(&(seqno, metric)) => advert.seqno > seqno || (advert.seqno == seqno && advert.metric < metric), - } - } - - /// The route `node` selected for `id`. - pub fn route(&self, node: Node, id: Id) -> Option { - self.route_except(node, id, &BTreeSet::new()) - } - - /// The best feasible usable route that avoids every neighbor in `except`. - fn route_except(&self, node: Node, id: Id, except: &BTreeSet) -> Option { - let state = &self.nodes[node]; - if let Some(&seqno) = state.origins.get(&id) { - return Some(Selected { - next: None, - seqno, - metric: 0, - advertised: 0, - down_only: false, - }); - } - // A plain loop rather than an iterator chain: selection dominates the - // sweeps, and unoptimized test builds pay for every adapter. - type Rank = (bool, Reverse, Metric, Node); - let mut best: Option<(Rank, Selected)> = None; - for (&next, entry) in state.rx.get(&id)? { - if entry.held || except.contains(&next) || !self.feasible(node, id, entry.advert) { - continue; - } - let advert = entry.advert; - let link = self.side(node, next); - let Some(metric) = self.cfg.rule.add(advert.metric, link.cost) else { - continue; - }; - let rank = ( - self.draining.contains(&(node, next)), - Reverse(link.priority), - metric, - next, - ); - if best.as_ref().is_some_and(|(best, _)| *best <= rank) { - continue; - } - let selected = Selected { - next: Some(next), - seqno: advert.seqno, - metric, - advertised: advert.metric, - down_only: self.cfg.export == Export::DownOnly && (link.upstream || advert.down_only), - }; - best = Some((rank, selected)); - } - best.map(|(_, selected)| selected) - } - - /// The usable but infeasible routes `node` holds for `id`, best first. - fn infeasible(&self, node: Node, id: Id) -> Vec { - let mut out: Vec<(Metric, Node)> = self.infeasible_unordered(node, id).collect(); - out.sort(); - out.into_iter().map(|(_, next)| next).collect() - } - - /// [`Net::infeasible`] with each route's metric, in neighbor order. - fn infeasible_unordered(&self, node: Node, id: Id) -> impl Iterator + '_ { - self.nodes[node] - .rx - .get(&id) - .into_iter() - .flatten() - .filter(move |(_, entry)| !entry.held && !self.feasible(node, id, entry.advert)) - .filter_map(move |(&next, entry)| { - let metric = self.cfg.rule.add(entry.advert.metric, self.side(node, next).cost)?; - Some((metric, next)) - }) - } - - /// Where `node` sends a request for `id` on behalf of `from`: a feasible - /// route's next hop if one avoids them, else an infeasible route's (RFC 8966 - /// s3.8.1.2). - fn target(&self, node: Node, id: Id, from: &BTreeSet) -> Option { - self.route_except(node, id, from) - .and_then(|route| route.next) - .or_else(|| self.infeasible(node, id).into_iter().find(|next| !from.contains(next))) - } - - /// The route `node` would offer an upstream link for `id`. - pub fn export_route(&self, node: Node, id: Id) -> Option { - self.route(node, id).filter(|route| !route.down_only) - } - - /// The route `node` follows for `id` on behalf of a request from `from`. - fn route_from(&self, node: Node, id: Id, from: Option) -> Option { - if self.exports_only(node, from) { - self.export_route(node, id) - } else { - self.route(node, id) - } - } - - /// Whether `node` may follow only exportable routes for a request from `from`. - fn exports_only(&self, node: Node, from: Option) -> bool { - self.cfg.recheck && from.is_some_and(|from| self.side(node, from).upstream) - } - - /// Whether `node` holds a reachable route to `id` but none is feasible: it - /// needs the origin to advance its seqno. - pub fn starved(&self, node: Node, id: Id) -> bool { - self.route(node, id).is_none() && self.infeasible_unordered(node, id).next().is_some() - } - - /// Whether `node`, with no usable route to `id`, holds a held route it could - /// recover through. - pub fn grounded(&self, node: Node, id: Id) -> bool { - self.route(node, id).is_none() && self.recoverable(node, id) - } - - /// Whether `node` holds a held route to `id`. One counts only if its last - /// finite advertisement is feasible here, so holds follow a strictly - /// decreasing metric toward a starved node and cannot hold each other up. - fn recoverable(&self, node: Node, id: Id) -> bool { - self.nodes[node].rx.get(&id).is_some_and(|routes| { - routes - .values() - .any(|entry| entry.held && self.feasible(node, id, entry.advert)) - }) - } - - /// Whether `node`, having selected `route` for `id`, keeps showing `id`'s - /// announcements: it is routed, [starved](Self::starved), or - /// [grounded](Self::grounded). The caller passes the route it already - /// selected, since selecting dominates the sweeps. - fn live(&self, node: Node, id: Id, route: Option) -> bool { - route.is_some() || self.infeasible_unordered(node, id).next().is_some() || self.recoverable(node, id) - } - - /// The neighbors whose views can certify `id` at `node`: those whose last - /// finite advertisement is feasible here, held or not. Feasible edges - /// strictly decrease the metric toward the origin, so certification never - /// circulates: two nodes cannot keep a region alive through each other. - fn certifiers(&self, node: Node, id: Id) -> Vec { - self.nodes[node].rx.get(&id).map_or_else(Vec::new, |routes| { - routes - .iter() - .filter(|(_, entry)| self.feasible(node, id, entry.advert)) - .map(|(&next, _)| next) - .collect() - }) - } - - /// The regions of `id` that `node`'s certifiers certify, those whose route - /// is held only when `held`, or `None` while it holds no route or a - /// certifier has not yet answered the current interest: its announcements - /// may still trail its route. - fn certified(&self, node: Node, id: Id, held: bool) -> Option> { - let state = &self.nodes[node]; - let routes = state.rx.get(&id)?; - let mut regions = Vec::new(); - for next in self.certifiers(node, id) { - if state.heard_on.get(&(next, id)) != Some(&state.interest) { - return None; - } - if held || !routes[&next].held { - regions.extend(state.heard[&(next, id)].floors.keys()); - } - } - Some(regions) - } - - /// Let go of every region of `id`'s merged view that no routed neighbor - /// certifies any more, once each has answered the current interest: its - /// last certifier closed, ended its route, or no longer meets the interest. - /// The reader sees that region go offline. - fn prune(&mut self, node: Node, id: Id) { - let Freshness::Floors { scoped } = self.cfg.freshness else { - return; - }; - // A starved or held node keeps presence until it recovers or lets go. - if self.nodes[node].origins.contains_key(&id) || self.route(node, id).is_none() { - return; - } - let Some(regions) = self.certified(node, id, true) else { - return; - }; - let state = &self.nodes[node]; - let Some(merged) = state.views.get(&id) else { - return; - }; - let lost = merged - .floors - .iter() - .filter(|(floor, _)| !regions.iter().any(|region| within(floor, region))) - .map(|(_, &rev)| rev) - .max(); - let Some(lost) = lost else { - return; - }; - // Keep what the view knows within the regions still certified. That is - // sound only if no lost floor is newer than every kept one: otherwise a - // wildcard above both was last learned through the lost floor, and is - // unknown at the revision the kept floors certify. - let mut kept = View::default(); - for ®ion in ®ions { - kept.merge(&merged.restrict(region), scoped); - } - let (rebuilt, remembered) = if kept.floors.values().all(|&floor| floor >= lost) { - (kept, regions) - } else { - let mut rebuilt = View::default(); - for next in self.certifiers(node, id) { - rebuilt.merge(&self.nodes[node].heard[&(next, id)], scoped); - } - (rebuilt, Vec::new()) - }; - let state = &mut self.nodes[node]; - state.views.insert(id, rebuilt); - forget(&mut state.forgot, id, &remembered); - } - - /// The view `node` shows and forwards for `id`, having selected `route` for it. - fn adopted(&self, node: Node, id: Id, route: Option) -> Option<&View> { - let state = &self.nodes[node]; - if state.origins.contains_key(&id) { - return state.views.get(&id); - } - match self.cfg.freshness { - Freshness::NextHop => route - .and_then(|route| route.next) - .and_then(|next| state.heard.get(&(next, id))), - Freshness::Barrier | Freshness::Floors { .. } => { - self.live(node, id, route).then(|| state.views.get(&id)).flatten() - } - } - } - - /// What `node`'s announcement reader sees. - pub fn shown(&self, node: Node) -> &BTreeMap<(Id, &'static str), Shown> { - &self.nodes[node].shown - } - - /// Whether `node` shows `claim` from `id`, and whether it is available. - pub fn shows(&self, node: Node, id: Id, claim: &str) -> Option { - self.nodes[node] - .shown - .iter() - .find(|((shown, prefix), _)| *shown == id && *prefix == claim) - .map(|(_, shown)| shown.available) - } - - /// Recompute every selection at `node`, record feasibility distances for what - /// it advertises, and queue any change for each neighbor. - fn advertise(&mut self, node: Node) { - let ids: Vec = self.nodes[node].views.keys().copied().collect(); - for id in ids { - self.prune(node, id); - } - let state = &self.nodes[node]; - let mut ids: BTreeSet = state.rx.keys().copied().collect(); - ids.extend(state.origins.keys()); - ids.extend(state.sent.keys().map(|(_, id)| *id)); - let peers: Vec = self.peers(node).collect(); - - for id in ids { - let selected = self.route(node, id); - if self.cfg.freshness == Freshness::Barrier - && let Some(next) = selected.and_then(|route| route.next) - && let Some(view) = self.nodes[node].heard.get(&(next, id)) - { - let view = view.clone(); - self.nodes[node].views.insert(id, view); - } - // A leaf advertises only what it serves, and ends what it stopped serving. - let serves = self.nodes[node].transit || self.nodes[node].origins.contains_key(&id); - if !serves && !self.nodes[node].sent.keys().any(|(_, sent)| *sent == id) { - continue; - } - let selected = selected.filter(|_| serves); - // The feasibility distance tracks finite updates actually sent (RFC 8966 s3.7.3). - if let Some(selected) = selected - && peers - .iter() - .any(|&peer| !(self.side(node, peer).upstream && selected.down_only)) - { - self.record(node, id, selected); - } - // Recording may have tightened the feasibility distance: select again. - let route = self.route(node, id); - let live = serves && self.live(node, id, route); - let adopted = self.adopted(node, id, route); - // A sender that certifies anything answers every stream, even one - // whose interest none of its regions meets, so the receiver knows it - // certifies nothing there. - let answers = adopted.is_some_and(|view| !view.floors.is_empty()); - // Each floor names its region: the stream's interest, or a scope - // below it. Peers mostly share an interest, so restrict once per interest. - let mut restricted: Vec<(&'static str, View)> = Vec::new(); - for &peer in &peers { - let watch = self.nodes[peer].watch; - if !restricted.iter().any(|(interest, _)| *interest == watch) { - restricted.push((watch, adopted.map_or_else(View::default, |view| view.restrict(watch)))); - } - } - for &peer in &peers { - let upstream = self.side(node, peer).upstream; - let state = &self.nodes[node]; - let reach = match selected { - Some(route) if !(upstream && route.down_only) => Some(Reach::Finite(Advert { - seqno: route.seqno, - metric: route.metric, - down_only: route.down_only, - })), - None if live && state.sent.contains_key(&(peer, id)) => Some(Reach::Held), - _ => None, - }; - if state.sent.get(&(peer, id)).copied() != reach { - let state = &mut self.nodes[node]; - match reach { - Some(reach) => state.sent.insert((peer, id), reach), - None => { - *state.unacked.entry((peer, id)).or_default() += 1; - state.sent.remove(&(peer, id)) - } - }; - self.stats.route_msgs += 1; - self.queues - .entry((node, peer)) - .or_default() - .push_back(Msg::Route { id, reach }); - } - // Announcements ride on the route and end with it. - let state = &mut self.nodes[node]; - if reach.is_none() { - state.sent_views.remove(&(peer, id)); - continue; - } - let watch = self.nodes[peer].watch; - let (_, view) = restricted.iter().find(|(interest, _)| *interest == watch).unwrap(); - let state = &mut self.nodes[node]; - let unchanged = match state.sent_views.get(&(peer, id)) { - Some(sent) => sent == view, - None => !answers, - }; - if unchanged { - continue; - } - let (items, bytes) = view.diff_bytes(state.sent_views.get(&(peer, id)), watch); - state.sent_views.insert((peer, id), view.clone()); - self.stats.claim_msgs += 1; - self.stats.claim_items += items; - self.stats.claim_bytes += bytes; - self.queues.entry((node, peer)).or_default().push_back(Msg::Claims { - id, - view: view.clone(), - interest: self.nodes[peer].interest, - }); - } - } - // Drop a feasibility distance once nothing here routes or advertises the - // identity and every neighbor echoed its ROUTE_END: no node then holds an - // advertisement from this one, so no older one can return as a loop. - let state = &mut self.nodes[node]; - state.fd.retain(|id, _| { - state.rx.contains_key(id) - || state.origins.contains_key(id) - || state.sent.keys().any(|(_, sent)| sent == id) - || state.unacked.keys().any(|(_, unacked)| unacked == id) - }); - self.requests(node); - self.show(node); - } - - /// Start, resend, or clear `node`'s seqno requests. - fn requests(&mut self, node: Node) { - if self.cfg.requests == Requests::Off { - return; - } - // Every pending or starved identity; one neither pending nor starved - // falls through untouched below. - let state = &self.nodes[node]; - let mut ids: BTreeSet = state.pending.keys().copied().collect(); - ids.extend(state.rx.keys()); - for id in ids { - let state = &self.nodes[node]; - if state.origins.contains_key(&id) { - self.nodes[node].pending.remove(&id); - continue; - } - // Nothing below changes a selection, so one serves the whole pass. - let route = self.route(node, id); - let starved = route.is_none() && self.infeasible_unordered(node, id).next().is_some(); - if starved { - let want = state.fd[&id].0 + 1; - match self.cfg.requests { - Requests::HopByHop => { - let pending = self.nodes[node].pending.entry(id).or_insert(Pending { - seqno: want, - from: BTreeSet::new(), - asked: None, - retries: 0, - }); - if pending.seqno < want { - pending.seqno = want; - pending.asked = None; - } - } - Requests::Retries(retries) => { - if !state.pending.contains_key(&id) && !state.gave_up.contains(&id) { - self.nodes[node].pending.insert( - id, - Pending { - seqno: want, - from: BTreeSet::new(), - asked: None, - retries, - }, - ); - } - } - Requests::Off => unreachable!(), - } - } - let Some(pending) = self.nodes[node].pending.get(&id).cloned() else { - continue; - }; - let answered = route.is_some_and(|route| route.seqno >= pending.seqno); - if answered || (pending.from.is_empty() && !starved) { - self.nodes[node].pending.remove(&id); - continue; - } - // Hop by hop, a request follows the next hop even back through a - // requester: next hops form no loop, one pending request per hop stops - // a bounce, and no requester ever cancels, so excluding one can strand - // a request while every route still runs through it. - let except = match self.cfg.requests { - Requests::HopByHop => BTreeSet::new(), - _ => pending.from.clone(), - }; - let Some(target) = self.target(node, id, &except) else { - self.nodes[node].pending.remove(&id); - continue; - }; - let resend = match self.cfg.requests { - Requests::HopByHop => Some(target) != pending.asked, - _ => pending.asked.is_none(), - }; - if resend { - self.nodes[node].pending.get_mut(&id).unwrap().asked = Some(target); - self.send_request(node, target, id, pending.seqno); - } - } - } - - /// Recompute what `node`'s reader sees and log each change. - fn show(&mut self, node: Node) { - let state = &self.nodes[node]; - let mut ids: BTreeSet = state.views.keys().copied().collect(); - ids.extend(state.heard.keys().map(|(_, id)| *id)); - let mut shown = BTreeMap::new(); - for id in ids { - let route = self.route(node, id); - let Some(view) = self.adopted(node, id, route) else { - continue; - }; - let usable = route.is_some(); - // A claim is available only through a certifier that is not held: - // another scope's usable route cannot serve it. Only claims behind a - // usable route need the certifiers. - let serving: Option> = match self.cfg.freshness { - Freshness::Floors { .. } if usable && !view.items.is_empty() && !state.origins.contains_key(&id) => { - self.certified(node, id, false) - } - _ => None, - }; - let current = state.current.get(&id); - for (&claim, &item) in &view.items { - let certified = current.is_none_or(|regions| regions.iter().any(|region| overlaps(claim, region))); - let available = usable - && serving - .as_ref() - .is_none_or(|regions| regions.iter().any(|region| overlaps(claim, region))); - if overlaps(claim, state.watch) && certified { - shown.insert((id, claim), Shown { item, available }); - } - } - } - let old = std::mem::take(&mut self.nodes[node].shown); - let mut log = Vec::new(); - for (&(id, claim), now) in &shown { - match old.get(&(id, claim)) { - None => log.push((id, claim, Kind::Start, now.item.since)), - Some(was) => { - if was.item != now.item { - log.push((id, claim, Kind::Update, now.item.since)); - } - if was.available != now.available { - let kind = if now.available { - Kind::Available - } else { - Kind::Unavailable - }; - log.push((id, claim, kind, now.item.since)); - } - } - } - } - for (&(id, claim), was) in &old { - if shown.contains_key(&(id, claim)) { - continue; - } - // A claim no routed neighbor's region covers was let go, not ended. - let uncertified = matches!(self.cfg.freshness, Freshness::Floors { .. }) - && self - .certified(node, id, true) - .is_some_and(|regions| !regions.iter().any(|region| overlaps(claim, region))); - let kind = if !overlaps(claim, self.nodes[node].watch) { - Kind::Unwatched - } else if self.live(node, id, self.route(node, id)) && !uncertified { - Kind::End - } else { - Kind::Offline - }; - log.push((id, claim, kind, was.item.since)); - } - let events: Vec = log - .into_iter() - .map(|(id, claim, kind, since)| { - // A qualified identity answers to the history of the origin it names. - let origin = self.source(id).unwrap_or(id); - Event { - node, - id, - claim, - kind, - since, - forgot: ( - self.nodes[node].restarts, - self.nodes[node].forgot.get(&(id, claim)).copied().unwrap_or(0), - ), - withdrawn: self - .truth(origin, claim, self.revision(origin)) - .is_none_or(|item| item.since != since), - } - }) - .collect(); - for event in &events { - self.nodes[node].forgot.entry((event.id, event.claim)).or_insert(0); - } - self.events.extend(events); - self.nodes[node].shown = shown; - } - - /// Babel's feasibility distance: reset by a newer seqno, lowered by a better metric. - fn record(&mut self, node: Node, id: Id, selected: Selected) { - let fd = self.nodes[node] - .fd - .entry(id) - .or_insert((selected.seqno, selected.metric)); - if selected.seqno > fd.0 || (selected.seqno == fd.0 && selected.metric < fd.1) { - *fd = (selected.seqno, selected.metric); - } - } - - /// The origin's state of `claim` as of revision `rev`. - pub fn truth(&self, id: Id, claim: &str, rev: Rev) -> Option { - self.history - .get(&id)? - .iter() - .rev() - .find(|change| change.rev <= rev && change.claim == claim) - .and_then(|change| change.item) - } - - /// The origin's latest revision for `id`. - pub fn revision(&self, id: Id) -> Rev { - self.history - .get(&id) - .and_then(|history| history.last()) - .map_or(0, |change| change.rev) - } - - /// Every claim each node knows must be the origin's state at the revision it - /// knows that claim at: no view holds a mix of stale and fresh knowledge. - /// Qualified views are checked against their origin. Returns the views checked. - pub fn check(&self) -> Result, String> { - let mut checked = BTreeSet::new(); - for (node, state) in self.nodes.iter().enumerate() { - for (&key, view) in &state.views { - if state.origins.contains_key(&key) { - continue; - } - let Some(id) = self.source(key) else { - continue; - }; - let history = &self.history[&id]; - checked.insert((node, key)); - let claims: BTreeSet<&str> = history.iter().map(|change| change.claim).collect(); - for claim in claims { - let Some(rev) = view.as_of(claim, true) else { - if view.items.contains_key(claim) { - return Err(format!("node {node} holds {id}'s {claim} with no floor")); - } - continue; - }; - let known = view.items.get(claim).copied(); - if known != self.truth(id, claim, rev) { - return Err(format!( - "node {node} holds {key}'s {claim} as {known:?} at revision {rev}, where the origin had {:?}", - self.truth(id, claim, rev) - )); - } - } - } - } - Ok(checked) - } - - /// The origin identity whose claims `key` carries: itself, or what a grant's - /// subject qualified, unwound through any requalification. - fn source(&self, key: Id) -> Option { - if self.history.contains_key(&key) { - return Some(key); - } - if key & QUALIFIED == 0 { - return None; - } - let subjects: BTreeSet<&str> = self.grants.iter().map(|(subject, _)| *subject).collect(); - let known: BTreeSet = self - .nodes - .iter() - .flat_map(|state| state.views.keys().chain(state.rx.keys())) - .copied() - .chain(self.history.keys().copied()) - .collect(); - known - .into_iter() - .filter(|&inner| inner != key) - .find(|&inner| subjects.iter().any(|subject| qualified(subject, inner) == key)) - .and_then(|inner| self.source(inner)) - } - - /// Announcement and route state kept across every node. - pub fn retained(&self) -> Retained { - let mut out = Retained::default(); - for state in &self.nodes { - for (id, view) in &state.views { - if !state.origins.contains_key(id) { - out.items += view.items.len(); - out.floors += view.floors.len(); - } - } - out.heard += state.heard.values().map(|view| view.items.len()).sum::(); - out.heard_floors += state.heard.values().map(|view| view.floors.len()).sum::(); - out.sent += state.sent_views.values().map(|view| view.items.len()).sum::(); - out.sent_floors += state.sent_views.values().map(|view| view.floors.len()).sum::(); - out.echoes += state.unacked.values().map(|&count| count as usize).sum::(); - out.pending += state.pending.len(); - out.distances += state.fd.keys().filter(|id| !state.origins.contains_key(id)).count(); - out.certified += state.current.values().map(Vec::len).sum::(); - } - out - } - - fn candidates(&self, node: Node, path: &str, from: Option) -> Vec { - let mut out = Vec::new(); - for (index, claim) in self.claims.iter().enumerate() { - if !claim.covers(path) || self.hidden.contains(&(node, index)) { - continue; - } - // A claim counts under its bare identity, and under a qualified key only inside that grant's scope. - let inside = self.grants.iter().filter(|(_, scope)| within(claim.prefix, scope)); - for id in iter::once(claim.id).chain(inside.map(|(subject, _)| qualified(subject, claim.id))) { - let Some(route) = self.route_from(node, id, from) else { - continue; - }; - let Some(total) = self.cfg.rule.add(route.metric, claim.seed) else { - continue; - }; - out.push(Candidate { - id, - next: route.next, - seed: claim.seed, - total, - advertised: route.advertised, - draining: route.next.is_some_and(|next| self.draining.contains(&(node, next))), - priority: route.next.map_or(0, |next| self.side(node, next).priority), - prefix: claim.prefix.len(), - instance: claim.instance, - down_only: route.down_only, - }); - } - } - out - } - - #[expect( - clippy::type_complexity, - reason = "a test model's sort key; an alias would only move the type" - )] - fn rank( - &self, - node: Node, - path: &str, - c: &Candidate, - ) -> (Reverse, Reverse>, bool, Reverse, Metric, u64) { - let tie = match self.cfg.tie { - TieBreak::PathIdentity => fnv(path, &[c.id]), - TieBreak::PathHere => fnv(path, &[c.id, c.next.unwrap_or(node) as u64, node as u64]), - }; - ( - Reverse(c.prefix), - Reverse(c.instance), - c.draining, - Reverse(c.priority), - c.total, - tie, - ) - } - - /// The candidate `node` uses for a request arriving from `from` carrying `carried`. - fn choose(&self, node: Node, path: &str, from: Option, mode: Forward, carried: Carried) -> Option { - if let Some(pinned) = carried.pinned { - // A pinned request needs a route to the identity, not knowledge of its claim. - let route = self.route_from(node, pinned, from)?; - return Some(Candidate { - id: pinned, - next: route.next, - seed: 0, - total: route.metric, - advertised: route.advertised, - draining: false, - priority: 0, - prefix: 0, - instance: carried.instance, - down_only: route.down_only, - }); - } - let mut candidates = self.candidates(node, path, from); - match mode { - Forward::PathOnly { - exclude_requester: true, - } => candidates.retain(|c| from.is_none() || c.next != from), - Forward::Downhill if from.is_some() => { - let best = candidates.iter().map(|c| c.total).min(); - candidates.retain(|c| c.next.is_none() || best.is_some_and(|best| c.advertised + c.seed < best)); - } - _ => {} - } - if let Some(bound) = carried.bound { - candidates.retain(|c| c.total <= bound); - } - candidates.into_iter().min_by_key(|c| self.rank(node, path, c)) - } - - /// What `node` sends `next` for `chosen`: a pin or bound is set by the first - /// hop and kept or tightened after, and a pin is named as `next` knows it. - fn carry( - &self, - node: Node, - next: Node, - mode: Forward, - first: bool, - chosen: &Candidate, - carried: Carried, - ) -> Carried { - let carried = if first || mode == Forward::Bounded { - Carried { - pinned: (mode == Forward::Pinned).then_some(chosen.id), - bound: (mode == Forward::Bounded).then_some(chosen.advertised + chosen.seed), - instance: chosen.instance, - } - } else { - carried - }; - Carried { - pinned: carried.pinned.map(|id| self.name(node, next, id)), - ..carried - } - } - - /// The identity a reader at `at` would pick for `path`. - pub fn resolve(&self, at: Node, path: &str) -> Option { - self.choose(at, path, None, Forward::Pinned, Carried::default()) - .map(|chosen| chosen.id) - } - - /// Follow a request for `path` that a reader at `at` makes. - pub fn request(&self, at: Node, path: &str, mode: Forward) -> Outcome { - let mut path_nodes = vec![at]; - let mut node = at; - let mut from = None; - let mut carried = Carried::default(); - let mut seen = BTreeSet::new(); - loop { - if !seen.insert((node, from, carried)) { - return Outcome::Loop { path: path_nodes }; - } - let Some(chosen) = self.choose(node, path, from, mode, carried) else { - return Outcome::Stalled { path: path_nodes }; - }; - let Some(next) = chosen.next else { - return Outcome::Served { - by: chosen.id, - path: path_nodes, - }; - }; - carried = self.carry(node, next, mode, from.is_none(), &chosen, carried); - from = Some(node); - node = next; - path_nodes.push(node); - } - } - - /// Readers at every node in `openers` subscribe to `path` at once; each - /// relay's forwarded request may join a subscription the relay already has, - /// if the request may follow that subscription's route. Returns each - /// opener's outcome; a subscription cycle that no source feeds is a loop. - pub fn subscribe_all(&self, path: &str, openers: &[Node], mode: Forward, aggregate: Aggregate) -> Vec { - let mut fronts: Vec = Vec::new(); - let open = |net: &Self, node: Node, from: Option, carried: Carried| -> Front { - match net.choose(node, path, from, mode, carried) { - None => Front { - node, - id: 0, - instance: None, - total: 0, - down_only: false, - up: Up::Stalled, - }, - Some(chosen) => Front { - node, - id: chosen.id, - instance: chosen.instance, - total: chosen.total, - down_only: chosen.down_only, - up: match chosen.next { - None => Up::Serve, - Some(next) => Up::Pending( - node, - next, - net.carry(node, next, mode, from.is_none(), &chosen, carried), - ), - }, - }, - } - }; - for &node in openers { - fronts.push(open(self, node, None, Carried::default())); - } - - let mut index = 0; - while index < fronts.len() { - if let Up::Pending(from, to, carried) = fronts[index].up { - let exports_only = self.exports_only(to, Some(from)); - let joins = fronts.iter().position(|front| { - front.node == to - && !matches!(front.up, Up::Stalled) - && !(exports_only && front.down_only) - && match aggregate { - Aggregate::Path => true, - Aggregate::Instance => carried.instance.is_some() && front.instance == carried.instance, - Aggregate::Identity => carried.pinned == Some(front.id), - Aggregate::Bound => carried.bound.is_some_and(|bound| front.total <= bound), - } - }); - let target = joins.unwrap_or_else(|| { - fronts.push(open(self, to, Some(from), carried)); - fronts.len() - 1 - }); - fronts[index].up = Up::Joined(target); - } - index += 1; - } - - // Every front hangs off an opener's chain, so any cycle shows up as a loop here. - (0..openers.len()) - .map(|mut at| { - let mut seen = Vec::new(); - loop { - let again = seen.contains(&at); - seen.push(at); - let path = seen.iter().map(|&i| fronts[i].node).collect(); - if again { - return Outcome::Loop { path }; - } - match fronts[at].up { - Up::Joined(next) => at = next, - Up::Serve => { - return Outcome::Served { - by: fronts[at].id, - path, - }; - } - Up::Stalled => return Outcome::Stalled { path }, - Up::Pending(..) => unreachable!("every pending front is resolved"), - } - } - }) - .collect() - } -} - -/// The key a relay files `id` under when a connection admitted under a -/// scoped grant for `subject` forwards it. It depends on the subject, not the -/// connection or the grant's patterns, so every uplink of one subject reaches -/// the same key across token refresh. FNV keeps the model readable; the -/// contract needs a collision-resistant hash, since the forwarder picks `id`. -pub fn qualified(subject: &str, id: Id) -> Id { - fnv(subject, &[id]) | QUALIFIED -} - -/// FNV-1a, as `origin::fnv_key` uses, over a path and some ids. -pub fn fnv(path: &str, ids: &[u64]) -> u64 { - const PRIME: u64 = 0x0000_0100_0000_01b3; - let mut hash: u64 = 0x420C0DECB00B; - for byte in path.bytes().chain(ids.iter().flat_map(|id| id.to_le_bytes())) { - hash = (hash ^ u64::from(byte)).wrapping_mul(PRIME); - } - hash -} From eec5830e61f86c66eb711bd85b4b204c858a18b6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 07:32:44 -0800 Subject: [PATCH 11/14] quest: acceptance keeps the mitigations' retirement, and its findings become quests Co-Authored-By: Claude Opus 5.5 --- quest/m2/cluster-routes/README.md | 38 +++--- quest/m2/cluster-routes/acceptance.md | 142 ++++++-------------- quest/m2/cluster-routes/doubt-holds.md | 34 +++++ quest/m2/cluster-routes/doubt-keepalive.md | 31 +++++ quest/m2/cluster-routes/held-after-close.md | 33 +++++ quest/m2/cluster-routes/route-fanout.md | 44 ++++++ quest/m3/cluster-routing/federation.md | 2 +- 7 files changed, 202 insertions(+), 122 deletions(-) create mode 100644 quest/m2/cluster-routes/doubt-holds.md create mode 100644 quest/m2/cluster-routes/doubt-keepalive.md create mode 100644 quest/m2/cluster-routes/held-after-close.md create mode 100644 quest/m2/cluster-routes/route-fanout.md diff --git a/quest/m2/cluster-routes/README.md b/quest/m2/cluster-routes/README.md index 01ff7e29b8e..8586ba1e5dd 100644 --- a/quest/m2/cluster-routes/README.md +++ b/quest/m2/cluster-routes/README.md @@ -91,10 +91,11 @@ lite-07 session in Rust and `js/net` drives the path router deleted with its distance, seqno requests, and echo. The handoff fixtures, both sweeps, and the route-layer fixtures of the contract run against it in `route/tests/`, and real sessions with mocked time in -`tests/integration/path_routes.rs`; the forwarding and first-hop selection -fixtures still run only on the Babel-era reference model in -`rs/moq-net/tests/integration/routing_contracts/`, which -[acceptance](/quest/m2/cluster-routes/acceptance.md) deletes. The children below carry the rest, in order, and land in +`tests/integration/path_routes.rs`. The forwarding fixtures run on the router +in `route/tests/forward.rs`, and the topology, failover, and cluster +acceptance fixtures on real sessions in `tests/integration/route_acceptance.rs`, +`route_failover.rs`, and `route_cluster.rs`; the Babel-era reference model is +deleted. The children below carry the rest, in order, and land in one release: lite-07 carries the route layer as a whole ([lite-07 finalize](/quest/m2/lite07-finalize.md) waits on this epic). This README holds the routing contract every child implements. @@ -121,10 +122,7 @@ Settled by the core, for review: Keep the 2026-10-01 two-layer direction, reconciled with the maintainer's 2026-10-13 [decisions](/quest/m3/cluster-routing/README.md) and the routing -contract below. The routing contract's remaining model fixtures live in -`rs/moq-net/tests/integration/routing_contracts/` and run against a small reference model; -port each topology to the implementation as acceptance, then delete the model. -The handoff contract below, decided 2026-10-16, settles claim freshness: +contract below. The handoff contract below, decided 2026-10-16, settles claim freshness: The lite-07 wire is specified in `drafts/draft-lcurley-moq-lite.md` (Announce, Floors, and Routing), and `rs/moq-net/src/lite/announce.rs` encodes and decodes @@ -194,9 +192,8 @@ the bullets below are the contract it carries. id, when the connection's publish grant is scoped, and bare when it is unscoped (publish `**` at the root). Its own identity is no exception (2026-10-26): the bare exemption relied on the dropped proof, so an - unproven own identity could otherwise claim a victim's. Extend the fixture - below to a forger declaring the victim's id as its own, and drop the - reference model's `proved` exemption. The key has its own hash domain, so + unproven own identity could otherwise claim a victim's, as the fixture + below's forger does. The key has its own hash domain, so it cannot collide with a key-derived identity, and is collision-resistant at the identity width, since the forwarder picks `id` and could otherwise grind another peer's key. The relay that files it translates it back when @@ -204,7 +201,7 @@ the bullets below are the contract it carries. outside the grant's scope. Routes are shared by all of an identity's claims, so otherwise a scoped peer forging an identity, or declaring a victim's id as its own, captures claims outside its namespace - (`forged_identity_hijacks_claims_outside_the_forgers_scope`); with it, a + (`a_forged_identity_is_contained_by_a_scoped_admission`); with it, a forger or a misconfigured uplink only affects its own scope. The `peer` claim no longer decides transit trust. Legitimate scoped delivery survives losing one of two uplinks under one key @@ -219,7 +216,7 @@ the bullets below are the contract it carries. and a scoped connection forwarding an already qualified identity has that route refused, with no wire change. Otherwise a key sent down through a misconfigured uplink comes back and is filed again as a new identity on - every lap (`scoped_connections_cannot_requalify_a_qualified_identity`). + every lap (`a_qualified_identity_is_never_qualified_again`). Nested scoped federation is therefore out: the inner level must be unscoped. Revisit with a wire change if a use case appears. Rejected: keying a forwarded identity by its bare id and the narrowest scope it @@ -280,13 +277,14 @@ the bullets below are the contract it carries. when its selected route turns down-only, and the first hop re-resolves; no request bit goes on the wire. One selection hides a cheaper exportable route whenever the preferred route is down-only - (`preferred_down_only_route_hides_the_exportable_alternative`); this is - accepted, and ranking local peers above a backup uplink avoids it. The - two-view model it replaces lost its export restriction one LAN hop after an - upstream ingress, and identity-only aggregation deadlocked across views - (`cdn_never_reaches_its_own_relay_through_the_lan`, - `upstream_ingress_ends_a_request_whose_route_turns_down_only`, - `dead_uplink_consumes_and_publishes_through_a_lan_peer`). A + (`a_preferred_down_only_route_hides_the_exportable_alternative`); this is + accepted, and ranking local peers above a backup uplink avoids it, though a + route's priority is the lowest on its path, so it must outrank the whole + path. The two-view model it replaces lost its export restriction one LAN hop + after an upstream ingress, and identity-only aggregation deadlocked across + views (`the_cdn_never_reaches_itself_through_the_lan`, + `an_upstream_request_stops_once_the_route_turns_down_only`, + `a_dead_uplink_consumes_and_publishes_through_a_lan_peer`). A misconfigured uplink that leaks CDN routes back is contained by qualified identities rather than a downstream mark (`misconfigured_uplink_is_contained_by_qualified_identities`). diff --git a/quest/m2/cluster-routes/acceptance.md b/quest/m2/cluster-routes/acceptance.md index 5923f82007d..dbaf6f6cae1 100644 --- a/quest/m2/cluster-routes/acceptance.md +++ b/quest/m2/cluster-routes/acceptance.md @@ -1,14 +1,11 @@ -# [L] Route layer acceptance +# [M] Route layer acceptance ## Goal -The route layer is proven on the implementation, not the model: the -forwarding and first-hop selection fixtures run against the router and the -session, and the reference model in -`rs/moq-net/tests/integration/routing_contracts/` is deleted. The incident -fixtures, tiered-cluster and partition end-to-end checks, failover -measurements, and benchmarks below pass in CI, and the mitigations they -replace are retired. +The route layer is proven on the implementation, and the mitigations its +fixtures replace are retired: [moq-dev/moq#5375](https://github.com/moq-dev/moq/pull/5375)'s +retraction quarantine, [moq-dev/moq#5391](https://github.com/moq-dev/moq/pull/5391)'s +same-clock queue dedup, and the 300 ms update hold (`origin::Config::update_hold`). ## Plan @@ -19,100 +16,43 @@ tier and partition tests run before the cut (rejected: moving it to m3 and fixing late routing flaws in lite-08). A flaw found here is fixed on lite-07-wip before it freezes. -Decided 2026-11-12 (maintainer, /quest-plan on the quests still written for -Babel): acceptance is of the path rule -([epic](/quest/m2/cluster-routes/README.md) Plan item 27), on the -sessions [the switch](https://github.com/moq-dev/moq/pull/8731) left. -The seqno request fixtures are replaced below, since the rule has no seqno -and no request. - -- Port the forwarding fixtures (`Forward`, `Aggregate`, pool ties, and the - sweeps over them) to the implementation, then delete the model. -- The earlier simulator persisted seqnos across restart, and the path rule - has none. Re-run with a fresh routing identity per restart and the - stale-state rules; do not carry the old assumption into acceptance. -- Retire [moq-dev/moq#5375](https://github.com/moq-dev/moq/pull/5375)'s - quarantine, [moq-dev/moq#5391](https://github.com/moq-dev/moq/pull/5391)'s - dedup, and the 300 ms hold only once the fixtures below demonstrate their - replacement (see the epic's hold-down note). -- Add a session-level fixture for routes scoped to interested origins - (Plan item 15 of the epic, implemented on the router): a client watching - one prefix holds routes only to the origins whose claims meet it, a - request for a path outside its interest still resolves through the relay - (Node ID 0), and an origin leaving the interest ends its route after the - batch withdrawing its last claim there, closed by its floor: one that - certifies the withdrawal where the relay certifies the request, and one - certifying nothing where it knows only part of it. - -The counterexamples from #4349 (closed; this epic still owns the fix) are the `residual_*` tests in -`rs/moq-net/tests/integration/mesh_withdraw.rs`. They pin today's hunting on lite-06: -a stale path outlasting the 300 ms hold, a revival the quarantine misses, -and 179 hunting relays (3 reviving) over 200 seeded meshes. Turn each into a -single retraction everywhere, and keep `cross_node_close_retracts_once` -passing. On `main` an epochless switch to a stale path is a Restart, which a -lite-06 client sees as a retraction and an announcement; the release sent an -in-place update. - -Publishing toggles too, decided 2026-10-13: a longer path arrives first and -the better route then replaces it. `late_better_path_updates_in_place` pins -the lite-07 mesh, where the move keeps the instance and a lite-06 or lite-07 -client sees an update. `residual_late_better_path_restarts_on_lite06` pins the -lite-06 mesh, where each relay mints its own instance and the client sees -`Start, End, Start` while the broadcast stays live; turn it into an update. - -Tests, time mocked: the handoff fixtures and sweeps, already on the router; -[moq-dev/moq#4644](https://github.com/moq-dev/moq/pull/4644)'s live-graph withdrawal and failover tests (the PR closed and -its branch is gone; fetch `refs/pull/4644/head`) as acceptance, run on the wip version; a -link flap with many broadcasts behind one origin costs one ROUTE change per -link; a ring of three where a stale neighbour cannot revive an ended -broadcast; a two-uplink mesh that never carries CDN routes back to the CDN; -an island reached through the CDN after a partition; an unknown Route ID -refused; a five-node ring whose preferred origin link fails moves to its -longer backup, at once where the backup's path avoids the far end of the -closed link and once the doubt lifts where it runs through it, while that -backup never loses its session; a relay with several neighbours that dies -is named in one withdrawal per session, and no relay moves onto a route -through it before the doubt lifts; two relays whose only routes run through -each other both let go, neither holding for the other; and a relay that -holds lets go once the neighbours behind it have answered, including one -whose session closes without an answer. Failover is measured -on broadcast liveness while the next hop changes, including a new next hop -whose ROUTE arrives before its ANNOUNCE set, and reports per-path -retractions and re-announcements beside the ROUTE budget; eventual route -recovery alone does not show it is seamless. - -Tiered-cluster and partition/healing end-to-end checks belong here, -not after lite-07 can be finalized: two regions with redundant cores, publish, -claim withdrawal, core loss, and matching-epoch failover reaching a far edge; -a two-uplink mesh reached through CDN during partition. Include overlapping -wildcards, a hidden exportable alternative under one selection, epochless Restart, and a -grant union crossing the scoped boundary each way (a `**` token added, then expiring). Fixtures are -public and time-mocked, wired into CI alongside Rust/JS interop. - -Mounts get the same continuity checks (`rs/moq-net/tests/integration/mount.rs`): -a claim whose predecessor covers a mount's target arrives beneath the mount -point exactly as it does unmounted, on lite-06 and lite-07. This replaces -the deleted mount-takeover quest, whose `Takeover` event -[moq-dev/moq#8063](https://github.com/moq-dev/moq/pull/8063) removes in -favor of the claim's instance; if #8063 closes unmerged, restore that quest. - -Benchmark origins, broadcasts per origin, neighbors, active tracks/fronts, -and announce cursors independently. A route update must not expand back into -per-broadcast table rewrites or poll unrelated fronts. Measure messages, bytes, -wakes/select calls, CPU, replay work, and retained state. One broadcast per -origin is an explicit case. Coordinate with Route wakes and Announce replay; -benchmark smoke alone does not enforce complexity, so add operation-count -regressions. Bound held routes, doubts, and remembered dead hops under -same-clock churn. - -Decided 2026-11-12 (maintainer, /quest-audit; epic Plan item 30): three -fixtures from routing prior art, as tests before any design change. A -relay vanishes and its neighbours notice up to the 3 s keep-alive apart, -which shows whether the doubt timer must follow the keep-alive or relay -keep-alives must shorten. Three relays whose local priorities form a dispute -wheel settle, or the test pins how they fail (RFC 4264). A primary route -fails and returns, and every relay ends back on it. +The fixtures landed in [moq-dev/moq#8832](https://github.com/moq-dev/moq/pull/8832), +and the reference model is deleted: forwarding on the router +(`rs/moq-net/src/route/tests/forward.rs`), and on real sessions with time mocked +the topology and prior-art fixtures (`tests/integration/route_acceptance.rs`), +failover liveness and the live-graph tests of +[moq-dev/moq#4644](https://github.com/moq-dev/moq/pull/4644) (`route_failover.rs`), +the tiered cluster, partition, and scoped-interest checks (`route_cluster.rs`), +mount continuity (`mount.rs`), and operation-count regressions +(`route/tests/cost.rs`, `route/node/tests.rs`). They found the flaws the +quests under Required fix; the tests that pin each flaw say so in their docs. + +What remains: + +- Retire the quarantine, the dedup, and the hold once the Required quests + turn their pinned fixtures into the contract's behaviour. The quarantine + is what turns the flaws into a 2 s gap: a relay that shows a held claim + offline announces it again only after `RETRACT_QUARANTINE`. The hold and + the quarantine sit on the announce cursor, which every version shares, so + check what a lite-06 mesh loses without them + (`mesh_withdraw::cross_node_close_without_hold_reannounces` is its + positive control) before deleting them; if a lite-06 mesh still needs + them, retire them for lite-07 routes only. +- `mesh_withdraw::residual_late_better_path_restarts_on_lite06` pins a + lite-06 mesh where each relay mints its own instance, so a client sees + `Start, End, Start` while the broadcast stays live. Turn it into an update, + or record why a lite-06 mesh keeps it. +- A request naming Node ID 0 on a lite-07 session: Rust clients always name + a node, so only publisher unit tests cover a relay answering 0. Cover it + where `js/net` sends it, in `just test interop`. + +## Required + +- [Held after close](/quest/m2/cluster-routes/held-after-close.md) - a relay whose own session closed shows its held claims unavailable, not offline +- [Doubt holds](/quest/m2/cluster-routes/doubt-holds.md) - a relay keeps holding while doubt alone keeps it off its routes +- [Route fan-out](/quest/m2/cluster-routes/route-fanout.md) - a route change costs only the streams and broadcasts it touches ## Related - [Routes and announces](/quest/m2/cluster-routes/README.md) - the contract and its fixtures +- [Doubt and keep-alive](/quest/m2/cluster-routes/doubt-keepalive.md) - whether the doubt timer follows the keep-alive diff --git a/quest/m2/cluster-routes/doubt-holds.md b/quest/m2/cluster-routes/doubt-holds.md new file mode 100644 index 00000000000..c46a5eb4875 --- /dev/null +++ b/quest/m2/cluster-routes/doubt-holds.md @@ -0,0 +1,34 @@ +# [M] Doubt holds + +## Goal + +A relay whose only routes left are kept off by doubt holds its claims, +unavailable, until the doubt lifts and it moves onto one, instead of letting +go while its neighbours answer that they have nothing. + +## Plan + +Found by [acceptance](/quest/m2/cluster-routes/acceptance.md): +`route_failover::live_mesh_link_failure_keeps_presence` pins it on a +34-relay mesh, and the router alone does the same. Cutting every link of the +publisher's relay but its slowest leaves only routes through that relay, +which every relay that heard a link die doubts for the timer. A relay with +no usable route holds, its neighbours are in the same state and answer that +they have nothing, and it lets go before the doubt lifts: the broadcast goes +Unavailable, Offline, then Start. Over 5 publishers, 132 relays went +unavailable, 52 retracted and announced again, 98 readers lost their +subscription, and the longest gap was 2.0 s (the doubt plus the retraction +quarantine). + +The contract already says a node holds "while a neighbour offers a route that +only the doubt keeps it off". Find why the answers end the hold first: a +neighbour that holds only because of doubt should answer with ROUTE_HOLD, not +nothing. Check the rule in `RouteHandoff.tla` with the `verify-route` skill +before changing `rs/moq-net/src/route/path.rs`, and build on +[moq-dev/moq#8734](https://github.com/moq-dev/moq/pull/8734), which changes +which routes doubt blocks. + +## Related + +- [Routes and announces](/quest/m2/cluster-routes/README.md) - Plan item 27 and the hold rule +- [Held after close](/quest/m2/cluster-routes/held-after-close.md) - the same symptom at the relay whose own session closed diff --git a/quest/m2/cluster-routes/doubt-keepalive.md b/quest/m2/cluster-routes/doubt-keepalive.md new file mode 100644 index 00000000000..d0595c25f5f --- /dev/null +++ b/quest/m2/cluster-routes/doubt-keepalive.md @@ -0,0 +1,31 @@ +# [S] Doubt and keep-alive + +## Goal + +No relay moves onto a route through a relay that died silently, however far +apart its neighbours notice the death within the keep-alive. + +## Plan + +Found by [acceptance](/quest/m2/cluster-routes/acceptance.md) (epic Plan +item 30's first prior-art fixture): +`route_acceptance::a_vanished_relay_is_noticed_a_keep_alive_apart` kills +`X` silently, and its neighbours `R1`, `R2`, `R3` notice at 0 s, 1.5 s, and +3 s. With the 1 s doubt, `R1` moves onto a neighbour's route through the +dead `X` at 1 s, holds again at 1.5 s, and moves onto `R3`'s route through +`X` at 2.5 s; `R2` does the same at 2.5 s. All three go offline exactly once, +at 3 s. Readers see each move as available for a moment, and a request in +that window goes to a dead relay. A 1 s doubt does not cover a 3 s spread. + +## Waiting + +- Maintainer: how the doubt covers a silent death. Options: (a) the doubt + lasts at least the configured idle timeout, so it covers the spread (the + recommendation: no wire change, and a relay already knows its idle + timeout); (b) relay keep-alives shorten below the doubt, which costs + traffic on every relay link; (c) accept the window and keep the test + pinning it. Check with `/quest-grill` or the PR that picks it up. + +## Related + +- [Routes and announces](/quest/m2/cluster-routes/README.md) - Plan item 27's doubt and item 30 diff --git a/quest/m2/cluster-routes/held-after-close.md b/quest/m2/cluster-routes/held-after-close.md new file mode 100644 index 00000000000..cc5121548bd --- /dev/null +++ b/quest/m2/cluster-routes/held-after-close.md @@ -0,0 +1,33 @@ +# [M] Held after close + +## Goal + +A relay whose own session to its next hop closes, leaving it no usable +route, shows its claims held and unavailable to its own readers, as its +neighbours already do, instead of offline followed by a fresh Start once the +route recovers. + +## Plan + +Found by [acceptance](/quest/m2/cluster-routes/acceptance.md) on real +lite-07 sessions, three times over: +`route_acceptance::a_ring_waits_out_the_doubt_for_a_backup_through_the_far_end`, +`route_acceptance::a_primary_that_fails_and_returns_takes_every_relay_back`, +and `route_cluster::a_partitioned_lan_reaches_itself_through_the_cdn_and_heals` +pin it. The router holds the route correctly; the relay's readers see +Offline at the cut, then Start about 2 s later, after the origin's +retraction quarantine. Subscriptions end and a fresh one reads on. + +The cause is in `rs/moq-net/src/route/node.rs` (`Node::close` and +`State::close`): a held claim's only slot in the origin was on the closed +session's stream, and closing the stream withdraws everything it served, so +nothing is left to show the claim unavailable. + +Recommended: when a held claim loses its last slot to a closing stream, +move it, unavailable, onto a stream that still hears the held route, or show +it through the node itself, before the closing stream withdraws it. Turn the +three pinned fixtures into the contract's held-then-available sequence. + +## Related + +- [Routes and announces](/quest/m2/cluster-routes/README.md) - the hold rule and "Presence and availability are separate" diff --git a/quest/m2/cluster-routes/route-fanout.md b/quest/m2/cluster-routes/route-fanout.md new file mode 100644 index 00000000000..62226d4435d --- /dev/null +++ b/quest/m2/cluster-routes/route-fanout.md @@ -0,0 +1,44 @@ +# [L] Route fan-out + +## Goal + +A route change costs a relay work in the streams and broadcasts it touches: +not every answering stream, not every claim behind the route, and not state +remembered for sessions nothing was learned on. + +## Plan + +Found by [acceptance](/quest/m2/cluster-routes/acceptance.md)'s +operation-count regressions, each pinned by a `residual_*` test that fails +once fixed: + +- Unrelated streams (`route::tests::cost::residual_a_route_change_visits_every_unrelated_stream`, + `residual_closing_a_stream_revisits_every_other_stream`): every route change + visits every answering stream for each identity it recomputes, and changes + with no single route in scope (RTT, DEAD, drain, close) plan every prefix + of each stream. Closing one viewer costs the origins it watched times all + other streams, so viewer churn is quadratic. Measured: an RTT change with + 1,000 origins and 1,000 unrelated streams takes 847 ms. Recommended: index + the answering streams in `rs/moq-net/src/route/path.rs` by the prefixes + they ask for and by the routes sent to them. +- Per-broadcast rewrites (`route::node::tests::residual_a_route_update_reserves_every_claim`): + in `rs/moq-net/src/route/node.rs`, any change to a route, its RTT included, + re-serves every claim behind it into the origin (`Node::home`), a + route-table write per broadcast that wakes the fronts below it. The wire + stays flat. Recommended: one shared record per route in the origin that its + broadcasts refer to, coordinated with [route wakes](/quest/m4/route-wakes.md). +- History-bound state (`residual_same_clock_churn_is_remembered_until_its_timer`, + `route::node::tests::residual_each_closed_session_is_remembered_until_the_doubt_timer`): + dead hops, doubts, and held routes grow with the churn inside one timer, + and a closed viewer session that sent no route still leaves a dead hop (and + a doubt if its client declared a node). Skipping the dead hop for a session + nothing was learned on looks safe, since no path can name it; check it with + the `verify-route` skill first. + +Build on [moq-dev/moq#8734](https://github.com/moq-dev/moq/pull/8734), +which also edits `path.rs`. + +## Related + +- [Route wakes](/quest/m4/route-wakes.md) - origin-side wakes per route change +- [Routes and announces](/quest/m2/cluster-routes/README.md) - the contract's cost bullets diff --git a/quest/m3/cluster-routing/federation.md b/quest/m3/cluster-routing/federation.md index 97be5261ea1..863f28599a2 100644 --- a/quest/m3/cluster-routing/federation.md +++ b/quest/m3/cluster-routing/federation.md @@ -23,7 +23,7 @@ announces](/quest/m2/cluster-routes/README.md)). Decided 2026-11-04 read from a key file the local relay's uplinks share ([moq-dev/moq#8023](https://github.com/moq-dev/moq/pull/8023)). Without a key, a scoped forger declaring a victim's id captures claims outside its scope -(`forged_identity_hijacks_claims_outside_the_forgers_scope`). The scope itself +(`a_forged_identity_is_contained_by_a_scoped_admission`). The scope itself (`hash(scope, id)`) was rejected 2026-10-15, and the grant's issuer-qualified subject (`hash(subject, id)`) was dropped 2026-11-02 (maintainer). The key must survive token refresh and be shared by the local relay's two uplinks. A From 82847667fd3ed09235570ee8617e9c41e0c891a0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 12:46:35 -0800 Subject: [PATCH 12/14] test(route): acceptance follows main's doubt rule, and answers its review Merging main brought #8734's doubt rule, under which a route that begins at the doubted node is spared. Three acceptance fixtures pinned the old rule: - a_primary_that_fails_and_returns: B and C now move onto the backup at once and return only once the doubt lifts. - two_relays_routing_through_each_other: each moves onto the other's stale direct route first, so both let go two lags after the cut, not one. - live_mesh_link_failure_keeps_presence: 6 readers lost instead of 98, but 3 on relays that stayed available end Unroutable; pinned for request-loop. The mesh fixture no longer reproduces doubt-holds, so a router test pins it with the publisher behind a transit relay. The four follow-ups record the maintainer's keep, and doubt-keepalive its deferred decision. Review: route_order's doc names the primary lane and selection keys, and route_failover and route_cluster share path_routes' timers and helpers. Co-Authored-By: Claude Opus 5.5 --- quest/m2/cluster-routes/doubt-holds.md | 32 ++++++++------ quest/m2/cluster-routes/doubt-keepalive.md | 8 +++- quest/m2/cluster-routes/held-after-close.md | 4 ++ quest/m2/cluster-routes/route-fanout.md | 4 ++ quest/m3/cluster-routing/request-loop.md | 10 +++++ rs/moq-net/src/model/origin.rs | 12 ++++-- rs/moq-net/src/route/tests/flap.rs | 36 ++++++++++++++++ .../tests/integration/route_acceptance.rs | 42 +++++++++++-------- rs/moq-net/tests/integration/route_cluster.rs | 23 +--------- .../tests/integration/route_failover.rs | 36 +++++++--------- 10 files changed, 130 insertions(+), 77 deletions(-) diff --git a/quest/m2/cluster-routes/doubt-holds.md b/quest/m2/cluster-routes/doubt-holds.md index c46a5eb4875..2898d468152 100644 --- a/quest/m2/cluster-routes/doubt-holds.md +++ b/quest/m2/cluster-routes/doubt-holds.md @@ -8,25 +8,31 @@ go while its neighbours answer that they have nothing. ## Plan +Kept 2026-11-15 (maintainer, /quest-complete on +[moq-dev/moq#8832](https://github.com/moq-dev/moq/pull/8832)): ✅ keep it in +m2 (rejected: drop; moving it to m3). + Found by [acceptance](/quest/m2/cluster-routes/acceptance.md): -`route_failover::live_mesh_link_failure_keeps_presence` pins it on a -34-relay mesh, and the router alone does the same. Cutting every link of the -publisher's relay but its slowest leaves only routes through that relay, -which every relay that heard a link die doubts for the timer. A relay with -no usable route holds, its neighbours are in the same state and answer that -they have nothing, and it lets go before the doubt lifts: the broadcast goes -Unavailable, Offline, then Start. Over 5 publishers, 132 relays went -unavailable, 52 retracted and announced again, 98 readers lost their -subscription, and the longest gap was 2.0 s (the doubt plus the retraction -quarantine). +`route::tests::flap::residual_a_relay_kept_off_by_doubt_lets_go_before_it_lifts` +pins it on the router. The publisher sits behind a transit relay that loses +most of its links, leaving only routes through it, which every relay that +heard a link die doubts for the timer. A relay with no usable route holds, +its neighbours are in the same state and answer that they have nothing, and +it lets go a hop after the cut: the claim goes Unavailable, then gone, then +back once the doubt lifts. + +`route_failover::live_mesh_link_failure_keeps_presence` measured it on a +34-relay mesh before [moq-dev/moq#8734](https://github.com/moq-dev/moq/pull/8734): +over 5 publishers, 98 readers lost their subscription. That fixture publishes +on the relay itself, so since #8734 doubt no longer blocks its routes, and it +loses 6. A real-session fixture for this quest needs the publisher behind the +relay whose links close. The contract already says a node holds "while a neighbour offers a route that only the doubt keeps it off". Find why the answers end the hold first: a neighbour that holds only because of doubt should answer with ROUTE_HOLD, not nothing. Check the rule in `RouteHandoff.tla` with the `verify-route` skill -before changing `rs/moq-net/src/route/path.rs`, and build on -[moq-dev/moq#8734](https://github.com/moq-dev/moq/pull/8734), which changes -which routes doubt blocks. +before changing `rs/moq-net/src/route/path.rs`. ## Related diff --git a/quest/m2/cluster-routes/doubt-keepalive.md b/quest/m2/cluster-routes/doubt-keepalive.md index d0595c25f5f..859b9212d17 100644 --- a/quest/m2/cluster-routes/doubt-keepalive.md +++ b/quest/m2/cluster-routes/doubt-keepalive.md @@ -7,6 +7,10 @@ apart its neighbours notice the death within the keep-alive. ## Plan +Kept 2026-11-15 (maintainer, /quest-complete on +[moq-dev/moq#8832](https://github.com/moq-dev/moq/pull/8832)): ✅ keep it in +m2 (rejected: drop; moving it to m3). + Found by [acceptance](/quest/m2/cluster-routes/acceptance.md) (epic Plan item 30's first prior-art fixture): `route_acceptance::a_vanished_relay_is_noticed_a_keep_alive_apart` kills @@ -24,7 +28,9 @@ that window goes to a dead relay. A 1 s doubt does not cover a 3 s spread. recommendation: no wire change, and a relay already knows its idle timeout); (b) relay keep-alives shorten below the doubt, which costs traffic on every relay link; (c) accept the window and keep the test - pinning it. Check with `/quest-grill` or the PR that picks it up. + pinning it. Check with `/quest-grill` or the PR that picks it up. The + maintainer deferred this decision on 2026-11-15, not yet sure about the + doubt design. ## Related diff --git a/quest/m2/cluster-routes/held-after-close.md b/quest/m2/cluster-routes/held-after-close.md index cc5121548bd..829a4336926 100644 --- a/quest/m2/cluster-routes/held-after-close.md +++ b/quest/m2/cluster-routes/held-after-close.md @@ -9,6 +9,10 @@ route recovers. ## Plan +Kept 2026-11-15 (maintainer, /quest-complete on +[moq-dev/moq#8832](https://github.com/moq-dev/moq/pull/8832)): ✅ keep it in +m2 (rejected: drop; moving it to m3). + Found by [acceptance](/quest/m2/cluster-routes/acceptance.md) on real lite-07 sessions, three times over: `route_acceptance::a_ring_waits_out_the_doubt_for_a_backup_through_the_far_end`, diff --git a/quest/m2/cluster-routes/route-fanout.md b/quest/m2/cluster-routes/route-fanout.md index 62226d4435d..365638b99af 100644 --- a/quest/m2/cluster-routes/route-fanout.md +++ b/quest/m2/cluster-routes/route-fanout.md @@ -8,6 +8,10 @@ remembered for sessions nothing was learned on. ## Plan +Kept 2026-11-15 (maintainer, /quest-complete on +[moq-dev/moq#8832](https://github.com/moq-dev/moq/pull/8832)): ✅ keep it in +m2 (rejected: drop; moving it to m3). + Found by [acceptance](/quest/m2/cluster-routes/acceptance.md)'s operation-count regressions, each pinned by a `residual_*` test that fails once fixed: diff --git a/quest/m3/cluster-routing/request-loop.md b/quest/m3/cluster-routing/request-loop.md index 34be16918a9..b704f4ce35b 100644 --- a/quest/m3/cluster-routing/request-loop.md +++ b/quest/m3/cluster-routing/request-loop.md @@ -33,6 +33,16 @@ Decided 2026-11-13 (maintainer, /quest-plan on the BGP failure modes): error the subscriber sees). Epic Plan item 27 already rejected keeping a request from going back through the relay that sent it. +Found again by [acceptance](/quest/m2/cluster-routes/acceptance.md) once +[moq-dev/moq#8734](https://github.com/moq-dev/moq/pull/8734) merged: +`route_failover::live_mesh_link_failure_keeps_presence` cuts all but one of a +publishing relay's links on a 34-relay mesh, and 3 readers on relays that +stayed available end with `Unroutable` instead of hanging, all for one +publisher. Each such relay moved onto a neighbour's direct route that the +neighbour's own close made stale; the replaced doubt rule held them and +ended none. The test pins those readers: flip it once a pending request +follows its route and they read on. + The fixture runs real lite-07 relays with mocked time. Form the loop the churn sweeps found (two unrelated failures at once, so two relays briefly route a publisher through each other), subscribe at both while it lasts, diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 1aefd76e35e..ce969ee8a3d 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -858,10 +858,14 @@ impl<'a> Epochs<'a> { } /// Ordering key for an eligible route entry resolving `path` (see [`Epochs`]). -/// Lower wins: first a route that is neither draining (see [`RouteInfo::draining`]) nor -/// unavailable (see [`RouteInfo::available`]), so a peer that sent a GOAWAY or a held -/// route ranks below every other eligible route and serves only as a last resort. Then the higher link [`Preference`], so metrics from different operators -/// are never compared. Then an identified chain (no 0) outranks an anonymous one regardless +/// Lower wins: first the route layer's narrowest route covering the claim, so a wider +/// lite-07 route ranks below every narrowest one, even a draining or unavailable one, and +/// serves only while a closing session leaves no narrowest route. Then a route that is +/// neither draining (see [`RouteInfo::draining`]) nor unavailable (see +/// [`RouteInfo::available`]), so a peer that sent a GOAWAY or a held route ranks below +/// every other eligible route and serves only as a last resort. Then the route layer's +/// selected route ahead of its alternates. Then the higher link [`Preference`], so +/// metrics from different operators are never compared. Then an identified chain (no 0) outranks an anonymous one regardless /// of cost, then the cheapest cost, then a broadcast published on this origin (it serves /// what is here, not a claim that has to ask), then the shortest hop chain, then a hash /// of the instance and `path`, so every node converges on the same winner, then, among diff --git a/rs/moq-net/src/route/tests/flap.rs b/rs/moq-net/src/route/tests/flap.rs index c123bd5231a..e13c2ba97f1 100644 --- a/rs/moq-net/src/route/tests/flap.rs +++ b/rs/moq-net/src/route/tests/flap.rs @@ -191,3 +191,39 @@ fn a_link_opened_before_the_doubt_is_taken_at_once() { ); } } + +/// The publisher behind relay 1, which reaches relays 2 and 3 cheaply and relay 4 +/// dearly, and 2, 3 and 4 mesh. Relay 1 loses 2 and 3: every route left runs through +/// 1, which both doubt, while 4 still reaches it. Relay 3 should hold the claim, +/// unavailable, until the doubt lifts and it moves onto 4's route, but 2 answers that +/// it has nothing, so 3 lets go a hop after the cut and shows the claim afresh once the +/// doubt lifts. Either rule does this, since 1 is not the publisher. Pins the +/// doubt-holds quest's flaw: flip it to a hold once fixed. +#[test] +fn residual_a_relay_kept_off_by_doubt_lets_go_before_it_lifts() { + let (origin, x, r2, r3, r4) = (0, 1, 2, 3, 4); + for rule in [Rule::ORIGIN, Rule::PATH] { + let mut sim = Sim::with(rule, 5); + sim.leaf(origin); + sim.join(origin, x, 1); + sim.join(x, r2, 1); + sim.join(x, r3, 1); + sim.join(x, r4, 5); + sim.join(r2, r3, 1); + sim.join(r3, r4, 1); + sim.join(r2, r4, 1); + sim.originate(origin, 7); + sim.announce(origin, "live/x", 0, None); + assert!(sim.run()); + let cut = sim.now(); + sim.disconnect(x, r2); + sim.disconnect(x, r3); + assert_eq!(sim.shows(r3, 7, "live/x"), Some(false), "{}: held at the cut", rule.as_str()); + assert!(sim.flow()); + assert!(sim.now() < cut.saturating_add(Doubt::DEFAULT.timer), "{}", rule.as_str()); + assert_eq!(sim.shows(r3, 7, "live/x"), None, "{}: let go in doubt", rule.as_str()); + assert!(sim.run()); + assert_eq!(sim.toward(r3, 7), Some(Some(r4)), "{}: after the doubt", rule.as_str()); + assert_eq!(sim.shows(r3, 7, "live/x"), Some(true), "{}", rule.as_str()); + } +} diff --git a/rs/moq-net/tests/integration/route_acceptance.rs b/rs/moq-net/tests/integration/route_acceptance.rs index 1a6f065e6bc..14d22f1ffc5 100644 --- a/rs/moq-net/tests/integration/route_acceptance.rs +++ b/rs/moq-net/tests/integration/route_acceptance.rs @@ -282,10 +282,11 @@ async fn a_ring_waits_out_the_doubt_for_a_backup_through_the_far_end() { } /// `A` and `B` each reach `P` directly and peer with each other, over a link lagging by -/// [`LAG`]. `P` dies: each relay's only route left runs through the other, which lost `P` -/// too. Each holds for the other, which learned its route, and the other's DEAD answers -/// it: both let go one lag after the cut, long before a hold's deadline, neither waiting -/// on the other. +/// [`LAG`]. `P` dies: each relay's only route left is the other's direct route, which +/// begins at `P` and so is not doubted, and which the other's own close made stale. Each +/// moves onto it until the other's DEAD retracts it one lag after the cut, then holds for +/// the other's answer, a lag later: both let go two lags after the cut, long before a +/// hold's deadline. The router alone does the same. #[kio::test] async fn two_relays_routing_through_each_other_both_let_go() { const LAG: Duration = Duration::from_millis(200); @@ -311,7 +312,7 @@ async fn two_relays_routing_through_each_other_both_let_go() { let ended = seen.last().unwrap(); assert!(withdrawn(ended), "{name}: {seen:?}"); assert!( - ended.at >= LAG && ended.at < 2 * LAG, + ended.at >= 2 * LAG && ended.at < 3 * LAG, "{name}: let go at {:?}: {seen:?}", ended.at ); @@ -476,12 +477,14 @@ async fn a_dispute_wheel_settles() { /// Prior art (the epic's Plan item 30): `P` reaches `A` over a primary link of cost 1 and /// `B` over a backup of cost 4, and `A`, `B` and `C` mesh. The primary fails and returns, -/// either within the doubt of `P` or after every relay moved onto the backup: every relay -/// ends back on the primary, at the cost it had before. +/// either within the doubt of `P` or after it lifted: every relay ends back on the +/// primary, at the cost it had before. /// -/// `A`'s DEAD names `P`, the far end, so `B` and `C` doubt the publisher itself and hold -/// for [`DOUBT`], unavailable, though `B`'s own session to `P` is healthy: every backup runs -/// through `P`. Returning within the doubt, the primary is back before anyone moves. +/// `A`'s DEAD names `P`, the far end, so `B` and `C` doubt the publisher. The backup begins +/// at `P`, so the doubt does not keep them off it: both move onto it at once, staying +/// available, as soon as the cut's news arrives. The returning primary is a session to `P` +/// opened since the doubt began, so it waits out [`DOUBT`], which damps a flapping link: +/// no relay moves back before the doubt lifts, however soon the primary returns. /// /// Pinned, against the contract (see /// [`a_ring_waits_out_the_doubt_for_a_backup_through_the_far_end`]): `A`, left with no @@ -526,17 +529,20 @@ async fn a_primary_that_fails_and_returns_takes_every_relay_back() { "{name} after {back:?}: back on the primary: {seen:?}" ); } - for (name, seen) in ["B", "C"].into_iter().zip(&seen[1..]) { - assert!(!seen.iter().any(withdrawn), "{name} after {back:?}: {seen:?}"); - let held = seen.iter().position(|seen| !seen.available).expect("held"); - let usable = seen.iter().position(|seen| seen.available).expect("usable"); + for ((name, backup), seen) in [("B", 4), ("C", 5)].into_iter().zip(&seen[1..]) { assert!( - held < usable && seen[usable].at >= DOUBT, + seen.iter().all(|seen| seen.available && !withdrawn(seen)), "{name} after {back:?}: {seen:?}" ); - // Within the doubt, the primary returns before anyone moves onto the backup. - let moved = seen.iter().any(|seen| seen.available && seen.cost != 2); - assert_eq!(moved, back > DOUBT, "{name} after {back:?}: {seen:?}"); + assert!( + seen[0].cost == backup && seen[0].at < DOUBT, + "{name} after {back:?}: onto the backup before the doubt lifts: {seen:?}" + ); + let returned = seen.iter().position(|seen| seen.cost == 2).expect("returned"); + assert!( + seen[returned].at >= back.max(DOUBT), + "{name} after {back:?}: not back before the doubt lifts: {seen:?}" + ); } let kinds: Vec<_> = seen[0].iter().map(|seen| seen.kind).collect(); assert_eq!( diff --git a/rs/moq-net/tests/integration/route_cluster.rs b/rs/moq-net/tests/integration/route_cluster.rs index 07615d9b5fd..e0147ba7721 100644 --- a/rs/moq-net/tests/integration/route_cluster.rs +++ b/rs/moq-net/tests/integration/route_cluster.rs @@ -11,6 +11,7 @@ //! overlapping claims, restarts, and a grant union crossing a scoped boundary. Last, a //! client listing one prefix of a relay holds routes only to the origins meeting it. +use crate::path_routes::{DOUBT, HOLD, land, lite07, quiesce}; use crate::route_change::{Delivery, produce_origin, read, timed}; use crate::support; @@ -18,33 +19,13 @@ use std::time::Duration; use futures::{StreamExt, channel::mpsc}; use moq_net::stats::{Message, Registry, SessionInfo, Tier}; -use moq_net::{Error, Instance, Pattern, Patterns, Timestamp, Version, auth, origin, track}; +use moq_net::{Error, Instance, Pattern, Patterns, Timestamp, auth, origin, track}; use support::Kind; use support::harness::{MockConnectOptions, MockPair, connect_mock, now, spawn}; -/// How long a relay holds a route no neighbour answers for. -const HOLD: Duration = Duration::from_secs(10); - -/// How long a closed session's far end is doubted. -const DOUBT: Duration = Duration::from_secs(1); - /// Frames per group. const FRAMES: u64 = 4; -fn lite07() -> Version { - "moq-lite-07-wip".parse().unwrap() -} - -/// Let every route settle, doubts and holds included. -async fn quiesce() { - kio::clock::sleep(HOLD + DOUBT).await; -} - -/// Wait long enough for messages in flight to land, without reaching the doubt. -async fn land() { - kio::clock::sleep(Duration::from_millis(100)).await; -} - /// Close a session at both ends, as a dropped connection does. fn cut(pair: &MockPair) { pair.server.abort(Error::Cancel); diff --git a/rs/moq-net/tests/integration/route_failover.rs b/rs/moq-net/tests/integration/route_failover.rs index 0b372b4bb6b..a8b68fe2048 100644 --- a/rs/moq-net/tests/integration/route_failover.rs +++ b/rs/moq-net/tests/integration/route_failover.rs @@ -13,6 +13,7 @@ //! which ran on lite-06 under a hold-down, to the path rule. use crate::mesh_withdraw::{PRODUCTION, PRODUCTION_LATENCY_MS}; +use crate::path_routes::{DOUBT, HOLD, lite07}; use crate::route_change::{payload, produce_origin, timed}; use crate::support; @@ -20,14 +21,10 @@ use std::collections::BTreeMap; use std::time::Duration; use futures::channel::mpsc; -use moq_net::{Error, Pattern, Patterns, Timestamp, Version, group, origin, stats, track}; +use moq_net::{Error, Pattern, Patterns, StreamError, Timestamp, group, origin, stats, track}; use support::Kind; use support::harness::{MockConnectOptions, MockPair, connect_mock, peer_with_latency}; -fn lite07() -> Version { - "moq-lite-07-wip".parse().unwrap() -} - /// How often a publisher writes a frame. const STEP: Duration = Duration::from_millis(20); @@ -38,12 +35,6 @@ const MESH_STEP: Duration = Duration::from_millis(100); /// Frames per group. const FRAMES: u64 = 5; -/// How long a closed session's far end is doubted. -const DOUBT: Duration = Duration::from_secs(1); - -/// How long a relay holds a route no neighbour answers for. -const HOLD: Duration = Duration::from_secs(10); - /// One change a node showed at a path: how long after the start, its kind, and whether a /// usable route reached the publisher. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -547,15 +538,18 @@ async fn live_mesh_withdraw_retracts_once_without_latency() { /// Ports moq-dev/moq#4644's `live_link_failure_fails_over`, which bounded the longest any /// relay went without a path by two hold-downs. Proves every relay serves the broadcast /// again with at most one retraction and one re-announcement each (no hunting), and every -/// reader on a relay whose route stayed usable gets every frame. +/// reader on a relay whose route stayed usable gets every frame but the pinned ones below. +/// +/// Every route left begins at the publisher's relay, the far end of each closed link, so +/// doubt keeps no relay off one. A relay whose own link closed moves onto a neighbour's +/// direct route, which may be stale too, until the news arrives. Measured over the five +/// publishers: 18 relays retracted and announced again, most within milliseconds, and 6 +/// readers lost their subscription. /// -/// Pins behaviour the contract's continuity goal contradicts, pending a decision: every -/// route left crosses the publisher's relay, the far end of each closed link, which every -/// relay that heard a DEAD doubts for [`DOUBT`]. Each such relay holds the broadcast with -/// no usable route, its neighbours are in the same state and answer with nothing, so the -/// holds end and the relays let go before the doubt lifts: the broadcast goes Offline, -/// then starts again once the doubt lifts, after the origin's two-second quarantine -/// (moq-dev/moq#5375). The router alone does the same: Unavailable, Offline, Start. +/// Pins behaviour the contract's continuity goal contradicts, owned by the request-loop +/// quest: a reader on a relay that stayed available may end with `Unroutable` (3 readers, +/// all for publisher 22). The replaced doubt rule, which held those relays instead, ended +/// none of them. #[kio::test] async fn live_mesh_link_failure_keeps_presence() { let version = lite07(); @@ -638,8 +632,10 @@ async fn live_mesh_link_failure_keeps_presence() { let live = Liveness::of(rx, &feed); if !live.seamless() { lost_readers += 1; + // Pinned: the request met a stale route. + let unroutable = matches!(live.ended, Some(Error::Stream(StreamError::Unroutable))); assert!( - dark.contains(relay), + dark.contains(relay) || unroutable, "publisher {publisher}: relay {relay} kept a usable route but its reader got {live:?}" ); } From 2b61b2d0ecc690f98c2386726ef185fb8e7774b5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 12:50:23 -0800 Subject: [PATCH 13/14] test(route): pin only the three known Unroutable readers in the mesh fixture Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/tests/flap.rs | 13 +++++++++++-- rs/moq-net/tests/integration/route_failover.rs | 15 ++++++++++----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/rs/moq-net/src/route/tests/flap.rs b/rs/moq-net/src/route/tests/flap.rs index e13c2ba97f1..a01adf7f769 100644 --- a/rs/moq-net/src/route/tests/flap.rs +++ b/rs/moq-net/src/route/tests/flap.rs @@ -218,9 +218,18 @@ fn residual_a_relay_kept_off_by_doubt_lets_go_before_it_lifts() { let cut = sim.now(); sim.disconnect(x, r2); sim.disconnect(x, r3); - assert_eq!(sim.shows(r3, 7, "live/x"), Some(false), "{}: held at the cut", rule.as_str()); + assert_eq!( + sim.shows(r3, 7, "live/x"), + Some(false), + "{}: held at the cut", + rule.as_str() + ); assert!(sim.flow()); - assert!(sim.now() < cut.saturating_add(Doubt::DEFAULT.timer), "{}", rule.as_str()); + assert!( + sim.now() < cut.saturating_add(Doubt::DEFAULT.timer), + "{}", + rule.as_str() + ); assert_eq!(sim.shows(r3, 7, "live/x"), None, "{}: let go in doubt", rule.as_str()); assert!(sim.run()); assert_eq!(sim.toward(r3, 7), Some(Some(r4)), "{}: after the doubt", rule.as_str()); diff --git a/rs/moq-net/tests/integration/route_failover.rs b/rs/moq-net/tests/integration/route_failover.rs index a8b68fe2048..09807194186 100644 --- a/rs/moq-net/tests/integration/route_failover.rs +++ b/rs/moq-net/tests/integration/route_failover.rs @@ -548,8 +548,12 @@ async fn live_mesh_withdraw_retracts_once_without_latency() { /// /// Pins behaviour the contract's continuity goal contradicts, owned by the request-loop /// quest: a reader on a relay that stayed available may end with `Unroutable` (3 readers, -/// all for publisher 22). The replaced doubt rule, which held those relays instead, ended -/// none of them. +/// all for publisher 22, listed in [`UNROUTABLE`]). The replaced doubt rule, which held +/// those relays instead, ended none of them. +/// The (publisher, relay) readers [`live_mesh_link_failure_keeps_presence`] pins as ending +/// with `Unroutable` though their relay stayed available. +const UNROUTABLE: [(usize, usize); 3] = [(22, 17), (22, 24), (22, 31)]; + #[kio::test] async fn live_mesh_link_failure_keeps_presence() { let version = lite07(); @@ -632,10 +636,11 @@ async fn live_mesh_link_failure_keeps_presence() { let live = Liveness::of(rx, &feed); if !live.seamless() { lost_readers += 1; - // Pinned: the request met a stale route. - let unroutable = matches!(live.ended, Some(Error::Stream(StreamError::Unroutable))); + // Pinned: only these readers end with `Unroutable` on a relay that stayed available. + let pinned = UNROUTABLE.contains(&(publisher, *relay)) + && matches!(live.ended, Some(Error::Stream(StreamError::Unroutable))); assert!( - dark.contains(relay) || unroutable, + dark.contains(relay) || pinned, "publisher {publisher}: relay {relay} kept a usable route but its reader got {live:?}" ); } From c7e9d8d8c5312833074c40f7342fd64629d15bc5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 13:02:47 -0800 Subject: [PATCH 14/14] quest: the request loop returns to m2's cluster-routes epic The maintainer moved it back on 2026-11-15: relays that show a broadcast available while ending readers' subscriptions should block the milestone. Co-Authored-By: Claude Opus 5.5 --- quest/m2/README.md | 6 ++++-- .../cluster-routes}/request-loop.md | 9 ++++++++- quest/m2/lite07-finalize.md | 3 ++- 3 files changed, 14 insertions(+), 4 deletions(-) rename quest/{m3/cluster-routing => m2/cluster-routes}/request-loop.md (89%) diff --git a/quest/m2/README.md b/quest/m2/README.md index 200f944be7d..7fb904262da 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -222,8 +222,10 @@ maintainer asked that what remains be unblocked or punted, and picked these on [moq-dev/moq#8768](https://github.com/moq-dev/moq/pull/8768)). m2 keeps only what would need a breaking release had it landed right after the cut: -- ✅ Left m2: [the request loop](/quest/m3/cluster-routing/request-loop.md) - to m3's cluster-routing epic (a fixture and a fix, no wire); +- ✅ Left m2: [the request loop](/quest/m2/cluster-routes/request-loop.md) + to m3's cluster-routing epic (a fixture and a fix, no wire), until it + returned to m2's cluster-routes epic on 2026-11-15 (maintainer, on + [moq-dev/moq#8832](https://github.com/moq-dev/moq/pull/8832)); [media consumers](/quest/m3/time-addressing-consumers.md) to m3 (its API breaks ride the model; dropping hang's markers is tolerated by readers); [moq-transport time addressing](/quest/m4/ietf-time-addressing.md) to m4 diff --git a/quest/m3/cluster-routing/request-loop.md b/quest/m2/cluster-routes/request-loop.md similarity index 89% rename from quest/m3/cluster-routing/request-loop.md rename to quest/m2/cluster-routes/request-loop.md index b704f4ce35b..abf8f591f7d 100644 --- a/quest/m3/cluster-routing/request-loop.md +++ b/quest/m2/cluster-routes/request-loop.md @@ -3,7 +3,8 @@ ## Goal A subscription that loops between relays while routes settle reaches the -publisher once the next hops stop looping, instead of hanging silently. +publisher once the next hops stop looping, instead of hanging silently or +ending with `Unroutable` while its relay shows the broadcast available. Under the path rule, selected next hops can loop for a moment ([epic](/quest/m2/cluster-routes/README.md) Plan item 27). A request forwarded around that loop meets a relay already requesting the same track @@ -13,6 +14,12 @@ no data and no failure. ## Plan +Decided 2026-11-15 (maintainer, on +[moq-dev/moq#8832](https://github.com/moq-dev/moq/pull/8832)): ✅ move it +back to m2, under the cluster-routes epic (rejected: keep it in m3). Relays +that show a broadcast available while ending readers' subscriptions should +block the milestone. + Moved from m2 to m3 2026-11-14 (the m2 triage before the cut, under /quest-afk, for the maintainer's review): ✅ m3, under the cluster-routing epic (rejected: m2). A fixture and, if it hangs, a fix to where a relay's diff --git a/quest/m2/lite07-finalize.md b/quest/m2/lite07-finalize.md index 6deca5174ff..63d6fc6479b 100644 --- a/quest/m2/lite07-finalize.md +++ b/quest/m2/lite07-finalize.md @@ -129,8 +129,9 @@ maintainer picked these on [moq-dev/moq#8768](https://github.com/moq-dev/moq/pull/8768)), applying the maintainer's rule that m2 keeps only what would need a breaking release had it landed right after the cut: -- ✅ [The request loop](/quest/m3/cluster-routing/request-loop.md) moved +- ✅ [The request loop](/quest/m2/cluster-routes/request-loop.md) moved to m3 and no longer gates this: a fixture and a fix with no wire change. + It returned to m2 on 2026-11-15 and still does not gate this. - ✅ [Route layer acceptance](/quest/m2/cluster-routes/acceptance.md) stays in [Routes and announces](/quest/m2/cluster-routes/README.md), so this still waits on it (maintainer, /quest-complete on