From d6dbb1410401ae7f06e9ce3c1ea61c3af0b37e77 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 02:48:32 -0800 Subject: [PATCH 01/19] chore(quest): claim quest/m2/multicast/warmth.md Quest-Claim: cdde53f359478188 From 780e9ea27940c0442d60896c43b436f1a148c061 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 04:00:50 -0800 Subject: [PATCH 02/19] test(moq-net): measure three ways to find a warm copy over the multicast prototype (wip) Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/tests/mod.rs | 2 + rs/moq-net/src/route/tests/multi.rs | 90 +- rs/moq-net/src/route/tests/warm.rs | 1222 ++++++++++++++++++++++++++ rs/moq-net/src/route/tests/warmth.rs | 765 ++++++++++++++++ 4 files changed, 2046 insertions(+), 33 deletions(-) create mode 100644 rs/moq-net/src/route/tests/warm.rs create mode 100644 rs/moq-net/src/route/tests/warmth.rs diff --git a/rs/moq-net/src/route/tests/mod.rs b/rs/moq-net/src/route/tests/mod.rs index 74a6b2326b3..bbc4cc97955 100644 --- a/rs/moq-net/src/route/tests/mod.rs +++ b/rs/moq-net/src/route/tests/mod.rs @@ -21,6 +21,8 @@ mod residual; mod routing; mod scope; mod search; +mod warm; +mod warmth; use std::{ collections::{BTreeMap, BTreeSet, VecDeque}, diff --git a/rs/moq-net/src/route/tests/multi.rs b/rs/moq-net/src/route/tests/multi.rs index be57a98471b..fac6284b56e 100644 --- a/rs/moq-net/src/route/tests/multi.rs +++ b/rs/moq-net/src/route/tests/multi.rs @@ -888,46 +888,48 @@ impl Router { } } + /// The route to `origin` through `via` as it stands here, unless it is unusable. + fn candidate(&self, origin: Id, via: N, heard: &Heard) -> Option> { + let peer = self.peers.get(&via)?; + let offer = &heard.offer; + let floored = offer.path.windows(2).any(|pair| { + let (key, at) = self.floor_of(pair[1], pair[0].node); + self.closed.get(&key).is_some_and(|highest| at <= *highest) + }); + if heard.ended + || floored + || offer.names(self.me) + || offer.path.len() >= MAX_HOPS + || offer.path.iter().any(|hop| self.dead.contains_key(hop)) + || (self.tune.agree && !self.agrees(origin, via, offer)) + { + return None; + } + let named = |nodes: &dyn Fn(&Id) -> bool| offer.path.iter().any(|hop| nodes(&hop.node)); + let suspect = named(&|node| self.suspects.contains_key(node)); + let flagged = suspect && self.tune.suspect == Suspect::Flagged; + Some(Best { + via, + offer: Offer { + path: offer.path.clone(), + draining: offer.draining || peer.draining || flagged || named(&|node| self.draining.contains(node)), + priority: offer.priority.min(peer.port.priority), + cost: offer.cost + peer.port.cost, + marked: offer.marked || peer.port.upstream, + }, + suspect, + }) + } + /// Pick the route to `origin` and tell every neighbour what changed. fn reselect(&mut self, origin: Id) { let current = self.best.get(&origin).map(|best| best.via); let mut pick: Option<(Best, Order)> = None; + self.work += self.heard.get(&origin).map_or(0, BTreeMap::len); for (&via, heard) in self.heard.get(&origin).into_iter().flatten() { - self.work += 1; - let Some(peer) = self.peers.get(&via) else { + let Some(mine) = self.candidate(origin, via, heard) else { continue; }; - let offer = &heard.offer; - let floored = offer.path.windows(2).any(|pair| { - let (key, at) = self.floor_of(pair[1], pair[0].node); - self.closed.get(&key).is_some_and(|highest| at <= *highest) - }); - if heard.ended || floored { - continue; - } - if offer.names(self.me) - || offer.path.len() >= MAX_HOPS - || offer.path.iter().any(|hop| self.dead.contains_key(hop)) - { - continue; - } - if self.tune.agree && !self.agrees(origin, via, offer) { - continue; - } - let named = |nodes: &dyn Fn(&Id) -> bool| offer.path.iter().any(|hop| nodes(&hop.node)); - let suspect = named(&|node| self.suspects.contains_key(node)); - let flagged = suspect && self.tune.suspect == Suspect::Flagged; - let mine = Best { - via, - offer: Offer { - path: offer.path.clone(), - draining: offer.draining || peer.draining || flagged || named(&|node| self.draining.contains(node)), - priority: offer.priority.min(peer.port.priority), - cost: offer.cost + peer.port.cost, - marked: offer.marked || peer.port.upstream, - }, - suspect, - }; let (draining, priority, cost) = mine.rank(); // Among equal ranks a node stays on its next hop. let key = ( @@ -1076,6 +1078,28 @@ impl Router { .map(|((_, _, origin), claim)| (origin, claim)) } + /// The summed cost from here to the claim [`Self::pick`] names, as it ranks it. + pub fn cost(&self, broadcast: &Broadcast) -> Option { + let (origin, claim) = self.closest(broadcast)?; + match origin == self.me { + true => Some(claim.cost), + false => Some(self.best.get(&origin)?.offer.cost + claim.cost), + } + } + + /// Every neighbour whose route to `origin` ranks with the one in use: the equal + /// next hops a request may take. + pub fn ties(&self, origin: Id) -> Vec { + let Some(best) = self.best.get(&origin) else { + return Vec::new(); + }; + (self.heard.get(&origin).into_iter().flatten()) + .filter_map(|(&via, heard)| self.candidate(origin, via, heard)) + .filter(|candidate| candidate.rank() == best.rank()) + .map(|candidate| candidate.via) + .collect() + } + /// Where a request for `broadcast` naming `origin` goes from here. `None` fails it. pub fn step(&self, origin: Id, broadcast: &Broadcast) -> Option> { match origin == self.me { diff --git a/rs/moq-net/src/route/tests/warm.rs b/rs/moq-net/src/route/tests/warm.rs new file mode 100644 index 00000000000..64257c9dc14 --- /dev/null +++ b/rs/moq-net/src/route/tests/warm.rs @@ -0,0 +1,1222 @@ +//! A data plane over the multicast prototype ([`MultiSim`]), to measure how a relay +//! learns that a neighbour already has demand for a broadcast +//! (`quest/m2/multicast/warmth.md`). Nothing here ships; `warmth.rs` holds the +//! shapes, the pins, and the tables. +//! +//! A relay has demand for a broadcast while it has a reader or a neighbour's payload +//! subscription, and pulls it through one parent. It is warm while it has demand and +//! its parent's answer confirms the pull. Every link has a cold price, what a copy +//! across it costs and what the route layer ranks by, and a warm price no higher and at +//! least 1. Costs are summed from the publisher along the path the data takes: +//! +//! - A relay's own cost is its parent's offer plus the warm price of the link between. +//! - What it offers a neighbour is that cost while it has demand besides that +//! neighbour, and otherwise its parent's offer plus the link's cold price, since the +//! copy exists only for the asker. An offer is never below its sender's own cost. +//! - With no demand a relay prices its route at the route layer's summed cold cost, +//! and a warm neighbour at its offer plus the cold price of the link to it. +//! - A pulling relay moves only to a source strictly cheaper than its own cost. Two +//! relays then never each prefer the other: with `o(x)` an offer and `d(x)` an own +//! cost, that needs `o(b) + link < d(a) <= o(a)` and `o(a) + link < d(b) <= o(b)`. +//! +//! What a relay hears of its neighbours is the [`Mode`]. Every move is make before +//! break: the old parent feeds until the new one's answer confirms. A relay never +//! pulls through a neighbour that pulls through it. +//! +//! The route layer runs underneath, unchanged, under the bake-off's pick +//! ([`super::compare::Arm::KEPT`]), its messages interleaved with these at random. + +use std::{ + cmp::Reverse, + collections::{BTreeMap, BTreeSet, VecDeque}, +}; + +use super::cache::rendezvous; +use super::compare::Arm as Rule; +use super::multi::{Broadcast, MultiSim, Next, Tie, Tune}; +use super::*; + +/// How a relay hears that a neighbour has demand. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Mode { + /// Not at all: a join follows the route and stops at the first relay with demand, + /// equal next hops picked by the rendezvous hash. + Path, + /// A relay with demand holds a metadata-only subscription to this many neighbours, + /// the cheapest links first. Each answers, and updates, whether it is warm and at + /// what cost. A neighbour with no demand forwards the subscription along its route, + /// as a metadata-only subscription does. + Ask(usize), + /// A warm relay tells every neighbour, and each change, on its announce stream. + Advertise, +} + +/// One way to find a warm copy, as measured. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Arm { + pub mode: Mode, + /// An offer carries the relays its data crossed, and a relay never pulls from one + /// that names it. Without it a stale cost is all a cycle has to count up through. + pub hops: bool, + /// Relays holding equal-cost copies merge: the lower rendezvous hash joins the + /// higher, when nothing else is cheaper once it has. + pub merge: bool, +} + +impl Arm { + pub fn name(self) -> String { + let mode = match self.mode { + Mode::Path => "path".to_string(), + Mode::Ask(usize::MAX) => "ask_all".into(), + Mode::Ask(count) => format!("ask_{count}"), + Mode::Advertise => "advertise".into(), + }; + let hops = if self.hops { "+hops" } else { "" }; + let merge = if self.merge { "+merge" } else { "" }; + format!("{mode}{hops}{merge}") + } +} + +/// A link's two prices: a copy starting across it, and one that already crosses it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Price { + pub cold: Metric, + pub warm: Metric, +} + +impl Price { + pub fn new(cold: Metric, warm: Metric) -> Self { + assert!(1 <= warm && warm <= cold, "a warm price of {warm} against {cold}"); + Self { cold, warm } + } + + pub fn flat(cost: Metric) -> Self { + Self::new(cost, cost) + } +} + +/// What a relay offers a neighbour: its summed cost and, with [`Arm::hops`], the relays +/// the data crossed, the publisher's own relay first. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct Signal { + pub cost: Metric, + pub path: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum State { + /// Sent, not yet answered. + Pending, + Live(Signal), + /// The parent lost its own upstream. + Lost, +} + +/// A payload subscription this relay holds. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Pull { + via: Node, + stream: u64, + state: State, +} + +#[derive(Clone, Debug)] +enum Msg { + Subscribe { + stream: u64, + open: bool, + }, + /// A parent's answer and each change to it: its offer, or that it lost its upstream. + Answer { + stream: u64, + offer: Option, + }, + /// The metadata-only subscription that asks, opening or closing. + Ask(bool), + /// The asked neighbour's word and each change to it. + Told(Option), + Advert(Option), +} + +/// The stream a message travels on: each subscription and query is a stream of its +/// own, unordered against the rest, and adverts share a session's announce stream. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +enum Lane { + Pull, + Ask, + Announce, +} + +/// A queue: sender, receiver, lane, and broadcast. +type Key = (Node, Node, Lane, usize); + +/// One relay's state for one broadcast. +#[derive(Clone, Debug, Default)] +struct Carry { + reader: bool, + /// The round its reader started, until its first frame. + since: Option, + /// Neighbours pulling through this relay, by stream. + down: BTreeMap, + up: Option, + /// A move under way: it replaces `up` once answered. + next: Option, + /// What each neighbour pulling through it was last answered. + answered: BTreeMap>, + /// Neighbours it asks, with each one's latest word. + asks: BTreeMap>, + /// Those yet to answer: a relay with no pull waits for them before it starts one. + unanswered: BTreeSet, + asked: BTreeSet, + told: BTreeMap>, + heard: BTreeMap, + advertised: BTreeMap, +} + +impl Carry { + fn demand(&self) -> bool { + self.reader || !self.down.is_empty() + } + + /// Demand besides `peer`'s. + fn others(&self, peer: Node) -> bool { + self.reader || self.down.keys().any(|&child| child != peer) + } +} + +/// Messages this layer sent, by kind. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) struct Sent { + /// Payload subscriptions opened and closed. + pub subscribes: usize, + /// Answers to them and their updates. + pub answers: usize, + /// Metadata-only subscriptions opened and closed. + pub asks: usize, + /// Their answers and updates. + pub told: usize, + pub adverts: usize, +} + +impl Sent { + pub fn total(self) -> usize { + self.subscribes + self.answers + self.asks + self.told + self.adverts + } + + /// What only a warm signal costs: queries, their answers, and adverts. + pub fn warm(self) -> usize { + self.asks + self.told + self.adverts + } +} + +impl std::ops::Add for Sent { + type Output = Self; + + fn add(self, more: Self) -> Self { + Self { + subscribes: self.subscribes + more.subscribes, + answers: self.answers + more.answers, + asks: self.asks + more.asks, + told: self.told + more.told, + adverts: self.adverts + more.adverts, + } + } +} + +impl std::ops::Sub for Sent { + type Output = Self; + + fn sub(self, was: Self) -> Self { + Self { + subscribes: self.subscribes - was.subscribes, + answers: self.answers - was.answers, + asks: self.asks - was.asks, + told: self.told - was.told, + adverts: self.adverts - was.adverts, + } + } +} + +/// What went wrong while messages moved. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) struct Watch { + /// Deliveries observed. + pub steps: usize, + /// Deliveries after which some broadcast's pulls formed a cycle. + pub cycles: usize, + /// Of those, deliveries after which a reader's pulls led into it. + pub starved: usize, + /// The most deliveries in a row a cycle lasted. + pub longest: usize, + run: usize, + /// Readers, summed over rounds, unfed while a path to the publisher existed. + pub unfed: usize, + /// Copies past the first leaving the publisher's relay, summed over rounds: the + /// cost of a race left standing. + pub extra: usize, +} + +impl Watch { + /// What changed since `was`, with the longest cycle so far. + pub fn since(self, was: Self) -> Self { + Self { + steps: self.steps - was.steps, + cycles: self.cycles - was.cycles, + starved: self.starved - was.starved, + longest: self.longest, + run: 0, + unfed: self.unfed - was.unfed, + extra: self.extra - was.extra, + } + } +} + +/// A broadcast's tree once it settled. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct Tree { + /// Links carrying it, parent first. + pub edges: Vec<(Node, Node)>, + /// Its links' cold prices summed: what the copies cost. + pub cost: Metric, + /// The most copies of it one link carries, both ways counted. + pub copies: usize, + /// Copies leaving the publisher's relay. + pub uplinks: usize, +} + +/// The tune of the route rule the bake-off picked. +fn kept() -> Tune { + match Rule::KEPT { + Rule::Multi(tune) => tune, + Rule::Path(_) => unreachable!(), + } +} + +fn pair(a: Node, b: Node) -> (Node, Node) { + (a.min(b), a.max(b)) +} + +/// A source a relay can pull from. +#[derive(Clone, Debug)] +struct Choice { + via: Node, + cost: Metric, + /// The neighbour's offer, when it is a warm copy rather than the route. + offered: Option, +} + +pub(super) struct WarmSim { + pub arm: Arm, + pub net: MultiSim, + relays: usize, + nodes: usize, + /// Each publisher's endpoint node. + endpoints: BTreeMap, + prices: BTreeMap<(Node, Node), Price>, + open: BTreeSet<(Node, Node)>, + casts: Vec, + carry: Vec>, + queues: BTreeMap>, + streams: u64, + pub sent: Sent, + pub watch: Watch, + pub round: usize, + order: XorShift, + /// Rounds each reader waited for its first frame. + pub waits: Vec, +} + +impl WarmSim { + /// `relays` relays, numbered first, and `publishers` endpoints after them. + pub fn new(arm: Arm, relays: usize, publishers: usize, seed: u64) -> Self { + let nodes = relays + publishers; + Self { + arm, + net: MultiSim::new(kept(), nodes), + relays, + nodes, + endpoints: BTreeMap::new(), + prices: BTreeMap::new(), + open: BTreeSet::new(), + casts: Vec::new(), + carry: vec![BTreeMap::new(); nodes], + queues: BTreeMap::new(), + streams: 0, + sent: Sent::default(), + watch: Watch::default(), + round: 0, + order: XorShift(seed | 1), + waits: Vec::new(), + } + } + + fn relay(&self, node: Node) -> bool { + node < self.relays + } + + pub fn link(&mut self, a: Node, b: Node, price: Price, tie: Tie) { + self.prices.insert(pair(a, b), price); + self.open.insert(pair(a, b)); + self.net.link(a, b, price.cold, tie); + } + + /// Publisher `index` joins relay `home` and announces `claims` broadcasts. Returns + /// each broadcast's index. + pub fn publish(&mut self, index: usize, home: Node, claims: usize) -> Vec { + let node = self.relays + index; + let id = 100 + index as Id; + self.net.client(node); + self.endpoints.insert(id, node); + self.prices.insert(pair(node, home), Price::flat(1)); + self.open.insert(pair(node, home)); + self.net.link(node, home, 1, Tie::Peer); + self.net.publish(node, id); + (0..claims) + .map(|claim| { + let path = format!("live/{claim}"); + self.net.claim(node, &path, true); + self.casts.push((id, p(&path))); + self.casts.len() - 1 + }) + .collect() + } + + pub fn casts(&self) -> usize { + self.casts.len() + } + + fn price(&self, a: Node, b: Node) -> Price { + self.prices[&pair(a, b)] + } + + pub fn is_open(&self, a: Node, b: Node) -> bool { + self.open.contains(&pair(a, b)) + } + + fn peers(&self, node: Node) -> Vec { + (self.open.iter()) + .filter_map(|&(a, b)| match node { + _ if a == node => Some(b), + _ if b == node => Some(a), + _ => None, + }) + .collect() + } + + fn hash(&self, cast: usize, node: Node) -> u64 { + rendezvous(&self.casts[cast].1, node as Id) + } + + fn carry(&self, node: Node, cast: usize) -> Option<&Carry> { + self.carry[node].get(&cast) + } + + fn entry(&mut self, node: Node, cast: usize) -> &mut Carry { + self.carry[node].entry(cast).or_default() + } + + fn send(&mut self, from: Node, to: Node, lane: Lane, cast: usize, msg: Msg) { + if !self.is_open(from, to) { + return; + } + match &msg { + Msg::Subscribe { .. } => self.sent.subscribes += 1, + Msg::Answer { .. } => self.sent.answers += 1, + Msg::Ask(_) => self.sent.asks += 1, + Msg::Told(_) => self.sent.told += 1, + Msg::Advert(_) => self.sent.adverts += 1, + } + // The announce stream is one per session, whatever the broadcast. + let stream = match lane { + Lane::Announce => usize::MAX, + Lane::Pull | Lane::Ask => cast, + }; + self.queues + .entry((from, to, lane, stream)) + .or_default() + .push_back((cast, msg)); + } + + // --- What a relay knows --- + + /// The route layer's next hop for `cast` at `node`, the equal ones split by the + /// rendezvous hash, and its summed cold cost. + fn route(&self, node: Node, cast: usize) -> Option<(Node, Metric)> { + let router = self.net.router(node)?; + let broadcast = &self.casts[cast]; + let origin = router.pick(broadcast)?; + let cost = router.cost(broadcast)?; + let next = match router.step(origin, broadcast)? { + Next::Local(endpoint) => endpoint, + Next::Via(next) => (router.ties(origin).into_iter()) + .max_by_key(|&tie| (self.hash(cast, tie), tie)) + .unwrap_or(next), + }; + Some((next, cost)) + } + + /// The confirmed offer `node` pulls on, unless it names `node`. + fn live(&self, node: Node, cast: usize) -> Option<&Signal> { + match &self.carry(node, cast)?.up.as_ref()?.state { + State::Live(signal) if !(self.arm.hops && signal.path.contains(&node)) => Some(signal), + _ => None, + } + } + + /// What `node` offers `peer` for `cast`, if it is fed. + fn offer(&self, node: Node, cast: usize, peer: Node) -> Option { + if !self.relay(node) { + return Some(Signal { + cost: 0, + path: Vec::new(), + }); + } + let signal = self.live(node, cast)?; + let carry = self.carry(node, cast)?; + let price = self.price(carry.up.as_ref()?.via, node); + let cost = match (self.arm.mode, carry.others(peer)) { + // A join along the route needs no cost, only the confirmation. + (Mode::Path, _) => 0, + (_, true) => signal.cost + price.warm, + (_, false) => signal.cost + price.cold, + }; + let path = match self.arm.hops { + true => signal.path.iter().copied().chain([node]).collect(), + false => Vec::new(), + }; + Some(Signal { cost, path }) + } + + /// The warm signal `node` gives `peer`: its offer, while it has demand besides `peer`. + fn warm(&self, node: Node, cast: usize, peer: Node) -> Option { + self.carry(node, cast).filter(|carry| carry.others(peer))?; + self.offer(node, cast, peer) + } + + /// Every source `node` could pull `cast` from: its route, and each warm neighbour it + /// heard of. Never a neighbour that pulls through it. + fn options(&self, node: Node, cast: usize) -> Vec { + let Some(carry) = self.carry(node, cast) else { + return Vec::new(); + }; + let mut out = Vec::new(); + if let Some((via, cost)) = self.route(node, cast) + && !carry.down.contains_key(&via) + { + out.push(Choice { + via, + cost, + offered: None, + }); + } + let heard: Vec<(Node, &Signal)> = match self.arm.mode { + Mode::Path => Vec::new(), + Mode::Ask(_) => (carry.asks.iter()) + .filter_map(|(&peer, signal)| Some((peer, signal.as_ref()?))) + .collect(), + Mode::Advertise => carry.heard.iter().map(|(&peer, signal)| (peer, signal)).collect(), + }; + for (via, signal) in heard { + if !self.is_open(node, via) + || carry.down.contains_key(&via) + || (self.arm.hops && signal.path.contains(&node)) + { + continue; + } + out.push(Choice { + via, + cost: signal.cost + self.price(via, node).cold, + offered: Some(signal.cost), + }); + } + out + } + + /// The cheapest of `options`, a warm copy before the route, then the higher hash. + fn best<'a>(&self, cast: usize, options: &'a [Choice]) -> Option<&'a Choice> { + (options.iter()).min_by_key(|choice| { + ( + choice.cost, + choice.offered.is_none(), + Reverse(self.hash(cast, choice.via)), + choice.via, + ) + }) + } + + // --- What a relay does --- + + /// Bring `node`'s state for `cast` up to date with what it knows. + fn react(&mut self, node: Node, cast: usize) { + if !self.relay(node) { + return; + } + self.queries(node, cast); + self.pull(node, cast); + self.answer(node, cast); + self.tell(node, cast); + self.advertise(node, cast); + } + + /// Open and close the metadata-only subscriptions that ask. + fn queries(&mut self, node: Node, cast: usize) { + let Mode::Ask(count) = self.arm.mode else { + return; + }; + let carry = self.entry(node, cast).clone(); + let want: BTreeSet = match () { + _ if carry.demand() => { + let mut peers: Vec = self.peers(node).into_iter().filter(|&peer| self.relay(peer)).collect(); + peers.sort_by_key(|&peer| (self.price(node, peer).cold, Reverse(self.hash(cast, peer)), peer)); + peers.into_iter().take(count).collect() + } + // Asked with no demand of its own, it forwards the subscription along its route. + _ if !carry.asked.is_empty() => (self.route(node, cast).into_iter()) + .map(|(next, _)| next) + .filter(|&next| self.relay(next) && !carry.asked.contains(&next)) + .collect(), + _ => BTreeSet::new(), + }; + for &gone in carry.asks.keys().filter(|peer| !want.contains(peer)) { + let carry = self.entry(node, cast); + carry.asks.remove(&gone); + carry.unanswered.remove(&gone); + self.send(node, gone, Lane::Ask, cast, Msg::Ask(false)); + } + for &new in want.iter().filter(|peer| !carry.asks.contains_key(peer)) { + let carry = self.entry(node, cast); + carry.asks.insert(new, None); + carry.unanswered.insert(new); + self.send(node, new, Lane::Ask, cast, Msg::Ask(true)); + } + } + + fn subscribe(&mut self, node: Node, cast: usize, via: Node) -> Pull { + self.streams += 1; + let stream = self.streams; + self.send(node, via, Lane::Pull, cast, Msg::Subscribe { stream, open: true }); + Pull { + via, + stream, + state: State::Pending, + } + } + + fn unsubscribe(&mut self, node: Node, cast: usize, pull: Option) { + if let Some(pull) = pull { + let stream = pull.stream; + self.send(node, pull.via, Lane::Pull, cast, Msg::Subscribe { stream, open: false }); + } + } + + /// Pull from the source the rule picks, moving make before break. + fn pull(&mut self, node: Node, cast: usize) { + let hops = self.arm.hops; + let carry = self.entry(node, cast); + if !carry.demand() { + let (up, next) = (carry.up.take(), carry.next.take()); + self.unsubscribe(node, cast, up); + self.unsubscribe(node, cast, next); + return; + } + // A move completes once answered, and is dropped if refused. + if let Some(next) = &carry.next { + match &next.state { + State::Live(signal) if !(hops && signal.path.contains(&node)) => { + let old = std::mem::replace(&mut carry.up, carry.next.take()); + self.unsubscribe(node, cast, old); + } + State::Pending => {} + _ => { + let next = carry.next.take(); + self.unsubscribe(node, cast, next); + } + } + } + let options = self.options(node, cast); + let best = self.best(cast, &options).cloned(); + let carry = self.entry(node, cast); + let Some(up) = carry.up.clone() else { + // It asks before it pulls, so the first answers can steer the first pull. + if let Some(best) = best.filter(|_| carry.unanswered.is_empty()) { + let pull = self.subscribe(node, cast, best.via); + self.entry(node, cast).up = Some(pull); + } + return; + }; + let target = match &up.state { + // Nothing flows yet, so a better source replaces it outright. + State::Pending => { + let current = (options.iter()) + .find(|choice| choice.via == up.via) + .map(|choice| choice.cost); + match best { + Some(best) if best.via != up.via && current.is_none_or(|cost| best.cost < cost) => { + let pull = self.subscribe(node, cast, best.via); + let old = self.entry(node, cast).up.replace(pull); + self.unsubscribe(node, cast, old); + } + // A request whose source is gone fails closed: two relays that asked + // each other at once would otherwise wait on each other for good. + None => { + let old = self.entry(node, cast).up.take(); + self.unsubscribe(node, cast, old); + } + Some(_) => {} + } + return; + } + State::Live(signal) if !(hops && signal.path.contains(&node)) => match self.arm.mode { + // A join follows its route wherever it goes. + Mode::Path => best.map(|best| best.via), + _ => { + let own = signal.cost + self.price(up.via, node).warm; + let back = signal.cost + self.price(up.via, node).cold; + let merge = self.merge(node, cast, &options, own, back); + match best { + Some(best) if best.cost < own => Some(best.via), + _ => merge.or(Some(up.via)), + } + } + }, + // Lost, or fed through itself: anything usable beats it. + _ => best.map(|best| best.via), + }; + self.toward(node, cast, target.unwrap_or(up.via)); + } + + /// The relay `node` merges onto: a warm neighbour whose copy costs what its own does, + /// with a higher hash, once nothing else, its old parent included, is cheaper. + fn merge(&self, node: Node, cast: usize, options: &[Choice], own: Metric, back: Metric) -> Option { + if !self.arm.merge { + return None; + } + let mine = self.hash(cast, node); + let up = self.carry(node, cast)?.up.as_ref()?.via; + let target = (options.iter()) + .filter(|choice| choice.via != up && choice.offered == Some(own) && self.hash(cast, choice.via) > mine) + .max_by_key(|choice| self.hash(cast, choice.via))?; + let after = own + self.price(target.via, node).warm; + let stays = after < back && (options.iter()).all(|choice| choice.via == target.via || after < choice.cost); + stays.then_some(target.via) + } + + /// Move toward `target` unless already there or on the way. + fn toward(&mut self, node: Node, cast: usize, target: Node) { + let carry = self.entry(node, cast); + let up = carry.up.as_ref().map(|pull| pull.via); + let next = carry.next.as_ref().map(|pull| pull.via); + if up == Some(target) { + let next = carry.next.take(); + self.unsubscribe(node, cast, next); + return; + } + if next == Some(target) { + return; + } + let old = self.entry(node, cast).next.take(); + self.unsubscribe(node, cast, old); + let pull = self.subscribe(node, cast, target); + self.entry(node, cast).next = Some(pull); + } + + /// Tell each neighbour pulling through `node` what it offers now. + fn answer(&mut self, node: Node, cast: usize) { + let carry = self.entry(node, cast).clone(); + for (&child, &stream) in &carry.down { + let offer = self.offer(node, cast, child); + let was = carry.answered.get(&child); + // Unanswered until confirmed, then every change. + if (was.is_none() && offer.is_none()) || was == Some(&offer) { + continue; + } + self.entry(node, cast).answered.insert(child, offer.clone()); + self.send(node, child, Lane::Pull, cast, Msg::Answer { stream, offer }); + } + } + + /// Answer each neighbour asking `node`, and each change. + fn tell(&mut self, node: Node, cast: usize) { + let carry = self.entry(node, cast).clone(); + for &asker in &carry.asked { + let word = self.warm(node, cast, asker); + if carry.told.get(&asker) == Some(&word) { + continue; + } + self.entry(node, cast).told.insert(asker, word.clone()); + self.send(node, asker, Lane::Ask, cast, Msg::Told(word)); + } + } + + /// Tell every neighbour whether `node` is warm, on a change. + fn advertise(&mut self, node: Node, cast: usize) { + if self.arm.mode != Mode::Advertise { + return; + } + let peers: Vec = self.peers(node).into_iter().filter(|&peer| self.relay(peer)).collect(); + for peer in peers { + let want = self.warm(node, cast, peer); + let was = self.entry(node, cast).advertised.get(&peer).cloned(); + if was == want { + continue; + } + match &want { + Some(signal) => self.entry(node, cast).advertised.insert(peer, signal.clone()), + None => self.entry(node, cast).advertised.remove(&peer), + }; + self.send(node, peer, Lane::Announce, cast, Msg::Advert(want)); + } + } + + /// Every relay reacts to what changed. + fn step(&mut self) { + for node in 0..self.relays { + for cast in 0..self.casts.len() { + self.react(node, cast); + } + } + } + + fn recv(&mut self, from: Node, to: Node, cast: usize, msg: Msg) { + // A publisher answers at once. + if let Msg::Subscribe { stream, open: true } = msg + && !self.relay(to) + { + let offer = self.offer(to, cast, from); + self.send(to, from, Lane::Pull, cast, Msg::Answer { stream, offer }); + } + let carry = self.entry(to, cast); + match msg { + Msg::Subscribe { stream, open: true } => { + carry.down.insert(from, stream); + carry.answered.remove(&from); + } + Msg::Subscribe { stream, open: false } => { + if carry.down.get(&from) == Some(&stream) { + carry.down.remove(&from); + carry.answered.remove(&from); + } + } + Msg::Answer { stream, offer } => { + for pull in [&mut carry.up, &mut carry.next].into_iter().flatten() { + if (pull.via, pull.stream) == (from, stream) { + pull.state = match &offer { + Some(signal) => State::Live(signal.clone()), + None => State::Lost, + }; + } + } + } + Msg::Ask(true) => { + carry.asked.insert(from); + } + Msg::Ask(false) => { + carry.asked.remove(&from); + carry.told.remove(&from); + } + Msg::Told(word) => { + if let Some(heard) = carry.asks.get_mut(&from) { + *heard = word; + carry.unanswered.remove(&from); + } + } + Msg::Advert(Some(signal)) => { + carry.heard.insert(from, signal); + } + Msg::Advert(None) => { + carry.heard.remove(&from); + } + } + } + + // --- What the network does --- + + /// A reader at `node` starts or stops reading broadcast `cast`. + pub fn read(&mut self, node: Node, cast: usize, reading: bool) { + let round = self.round; + let carry = self.entry(node, cast); + carry.reader = reading; + carry.since = reading.then_some(round); + self.step(); + } + + pub fn reading(&self, node: Node, cast: usize) -> bool { + self.carry(node, cast).is_some_and(|carry| carry.reader) + } + + /// The session between `a` and `b` closes, and both notice. + pub fn close(&mut self, a: Node, b: Node) { + self.open.remove(&pair(a, b)); + self.net.close(a, b); + self.queues.retain(|&(from, to, ..), _| pair(from, to) != pair(a, b)); + for (x, y) in [(a, b), (b, a)] { + for carry in self.carry[x].values_mut() { + if carry.up.as_ref().is_some_and(|pull| pull.via == y) { + // A move under way becomes the pull; otherwise it went with the session. + carry.up = carry.next.take(); + } + if carry.next.as_ref().is_some_and(|pull| pull.via == y) { + carry.next = None; + } + carry.down.remove(&y); + carry.answered.remove(&y); + carry.asks.remove(&y); + carry.unanswered.remove(&y); + carry.asked.remove(&y); + carry.told.remove(&y); + carry.heard.remove(&y); + carry.advertised.remove(&y); + } + } + self.step(); + } + + pub fn reopen(&mut self, a: Node, b: Node) { + self.open.insert(pair(a, b)); + self.net.open(a, b); + self.step(); + } + + /// `node` restarts: every session of its closes, it forgets everything but who reads + /// there, and its sessions reopen. + pub fn restart(&mut self, node: Node) { + let peers = self.peers(node); + for &peer in &peers { + self.close(node, peer); + } + self.net.reset(node); + for carry in self.carry[node].values_mut() { + *carry = Carry { + reader: carry.reader, + since: carry.since, + ..Carry::default() + }; + } + for peer in peers { + self.reopen(node, peer); + } + } + + /// Deliver everything queued now, in a seeded random order: a hop of delay. False + /// if nothing was. + pub fn round(&mut self) -> bool { + enum Queue { + Route(Node, Node), + Data(Key), + } + let mut ready: Vec<(Queue, usize)> = (self.net.ready().into_iter()) + .map(|((from, to), count)| (Queue::Route(from, to), count)) + .chain( + (self.queues.iter()) + .filter(|(_, queue)| !queue.is_empty()) + .map(|(&key, queue)| (Queue::Data(key), queue.len())), + ) + .collect(); + if ready.is_empty() { + return false; + } + for i in (1..ready.len()).rev() { + ready.swap(i, self.order.next(i as u64 + 1) as usize); + } + for (queue, count) in ready { + for _ in 0..count { + match queue { + Queue::Route(from, to) => { + self.net.deliver(from, to); + } + Queue::Data(key) => { + let Some((cast, msg)) = self.queues.get_mut(&key).and_then(VecDeque::pop_front) else { + break; + }; + self.recv(key.0, key.1, cast, msg); + } + } + self.step(); + self.observe(); + } + } + self.net.flush(); + self.step(); + self.round += 1; + self.tally(); + true + } + + /// Deliver in rounds until nothing is left. False if that never happens. + pub fn settle(&mut self) -> bool { + for _ in 0..2_000 { + if !self.round() { + return true; + } + } + false + } + + /// Deliver one message from a queue `pick` chooses, as churn does. False if none waits. + pub fn deliver(&mut self, pick: &mut XorShift) -> bool { + let routes = self.net.ready(); + let data: Vec = (self.queues.iter()) + .filter(|(_, queue)| !queue.is_empty()) + .map(|(&key, _)| key) + .collect(); + let total = routes.len() + data.len(); + if total == 0 { + return false; + } + let at = pick.next(total as u64) as usize; + match at < routes.len() { + true => { + let ((from, to), _) = routes[at]; + self.net.deliver(from, to); + self.net.flush(); + } + false => { + let key = data[at - routes.len()]; + let (cast, msg) = self + .queues + .get_mut(&key) + .and_then(VecDeque::pop_front) + .expect("a message"); + self.recv(key.0, key.1, cast, msg); + } + } + self.step(); + self.observe(); + true + } + + /// A hop of delay passes for churn, which delivers one message at a time: count + /// unfed readers and first frames. + pub fn tick(&mut self) { + self.round += 1; + self.tally(); + } + + /// Each relay's state for every broadcast, for a failure report. + pub fn dump(&self) -> String { + let mut out = String::new(); + for node in 0..self.relays { + for (cast, carry) in &self.carry[node] { + let route = self.route(node, *cast); + out += &format!(" {node} #{cast} route {route:?}: {carry:?}\n"); + } + } + out += &format!(" open {:?}\n", self.open); + for node in 0..self.relays { + let router = self.net.router(node).expect("a relay"); + let picks: Vec<_> = self.casts.iter().map(|cast| router.pick(cast)).collect(); + out += &format!(" {node} picks {picks:?}\n"); + } + out + } + + // --- What is true --- + + /// Whether `node` is fed `cast`: its pulls lead to the publisher, each received. + pub fn fed(&self, node: Node, cast: usize) -> bool { + let mut at = node; + for _ in 0..=self.nodes { + if !self.relay(at) { + return true; + } + let Some(pull) = self.carry(at, cast).and_then(|carry| carry.up.as_ref()) else { + return false; + }; + let held = self + .carry(pull.via, cast) + .is_some_and(|carry| carry.down.get(&at) == Some(&pull.stream)); + if !held || !self.is_open(at, pull.via) { + return false; + } + at = pull.via; + } + false + } + + fn parent(&self, node: Node, cast: usize) -> Option { + Some(self.carry(node, cast)?.up.as_ref()?.via) + } + + /// Whether following `node`'s pulls for `cast` goes round a cycle. + fn circles(&self, node: Node, cast: usize) -> bool { + let mut at = node; + for _ in 0..=self.nodes { + match self.parent(at, cast) { + Some(next) => at = next, + None => return false, + } + } + true + } + + fn readers(&self) -> Vec<(Node, usize)> { + (0..self.relays) + .flat_map(|node| { + self.carry[node] + .iter() + .filter(|(_, carry)| carry.reader) + .map(move |(&cast, _)| (node, cast)) + }) + .collect() + } + + /// Count a cycle after a delivery. + fn observe(&mut self) { + self.watch.steps += 1; + let cycle = (0..self.casts.len()).any(|cast| (0..self.relays).any(|node| self.circles(node, cast))); + let starved = cycle && self.readers().into_iter().any(|(node, cast)| self.circles(node, cast)); + self.watch.cycles += usize::from(cycle); + self.watch.starved += usize::from(starved); + self.watch.run = if cycle { self.watch.run + 1 } else { 0 }; + self.watch.longest = self.watch.longest.max(self.watch.run); + } + + /// The relays a path over open links joins to `cast`'s publisher. + fn reach(&self, cast: usize) -> BTreeSet { + let start = self.endpoints[&self.casts[cast].0]; + let mut seen = BTreeSet::from([start]); + let mut queue = vec![start]; + while let Some(at) = queue.pop() { + for peer in self.peers(at) { + if self.relay(peer) && seen.insert(peer) { + queue.push(peer); + } + } + } + seen + } + + /// Count unfed readers as a round ends, and the first frames that arrived. + fn tally(&mut self) { + let round = self.round; + let mut reach: BTreeMap> = BTreeMap::new(); + for (node, cast) in self.readers() { + let fed = self.fed(node, cast) && self.live(node, cast).is_some(); + let path = reach.entry(cast).or_insert_with(|| self.reach(cast)).contains(&node); + self.watch.unfed += usize::from(path && !fed); + let carry = self.entry(node, cast); + if fed && let Some(since) = carry.since.take() { + self.waits.push(round - since); + } + } + for cast in 0..self.casts.len() { + let home = self.home(cast); + let copies = self.carry(home, cast).map_or(0, |carry| carry.down.len()); + self.watch.extra += copies.saturating_sub(1); + } + } + + /// Every reader with a path to its publisher is fed, no pulls circle, and nothing is + /// left to deliver. + pub fn check(&self) -> Result<(), String> { + if self.queues.values().any(|queue| !queue.is_empty()) || !self.net.ready().is_empty() { + return Err("still busy".into()); + } + for cast in 0..self.casts.len() { + let reach = self.reach(cast); + for node in 0..self.relays { + if self.circles(node, cast) { + return Err(format!("{}: broadcast {cast} circles at relay {node}", self.arm.name())); + } + if self.reading(node, cast) + && reach.contains(&node) + && !(self.fed(node, cast) && self.live(node, cast).is_some()) + { + return Err(format!("{}: relay {node} unfed for broadcast {cast}", self.arm.name())); + } + let carry = self.carry(node, cast); + if carry.is_some_and(|carry| !carry.demand() && carry.up.is_some()) { + return Err(format!("{}: relay {node} pulls {cast} for nobody", self.arm.name())); + } + } + } + Ok(()) + } + + /// Relays besides the publisher's pulling `cast` with no reader of their own: what + /// other relays' joins cost them. + pub fn carriers(&self, cast: usize) -> Vec { + let home = self.home(cast); + (0..self.relays) + .filter(|&node| node != home) + .filter(|&node| { + self.carry(node, cast) + .is_some_and(|carry| !carry.reader && carry.up.is_some()) + }) + .collect() + } + + /// The relay publishing `cast` is joined to. + pub fn home(&self, cast: usize) -> Node { + let endpoint = self.endpoints[&self.casts[cast].0]; + self.peers(endpoint)[0] + } + + pub fn tree(&self, cast: usize) -> Tree { + let home = self.home(cast); + let edges: Vec<(Node, Node)> = (0..self.relays) + .filter(|&node| self.fed(node, cast)) + .filter_map(|node| Some((self.parent(node, cast)?, node))) + .collect(); + let mut links: BTreeMap<(Node, Node), usize> = BTreeMap::new(); + for &(a, b) in &edges { + *links.entry(pair(a, b)).or_default() += 1; + } + Tree { + cost: edges.iter().map(|&(a, b)| self.price(a, b).cold).sum(), + copies: links.values().copied().max().unwrap_or(0), + uplinks: edges.iter().filter(|&&(parent, _)| parent == home).count(), + edges, + } + } + + /// The cheapest tree joining `cast`'s publisher to every relay reading it, over open + /// links at their cold prices: Dreyfus and Wagner's exact dynamic program. + pub fn cheapest(&self, cast: usize) -> Metric { + let endpoint = self.endpoints[&self.casts[cast].0]; + let mut terminals = vec![endpoint]; + terminals.extend((0..self.relays).filter(|&node| self.reading(node, cast))); + let n = self.nodes; + let mut dist = vec![vec![Metric::MAX / 4; n]; n]; + for (node, row) in dist.iter_mut().enumerate() { + row[node] = 0; + } + for &(a, b) in &self.open { + let cost = self.price(a, b).cold; + // Only relays forward. + dist[a][b] = dist[a][b].min(cost); + dist[b][a] = dist[b][a].min(cost); + } + for via in (0..n).filter(|&node| self.relay(node)) { + for a in 0..n { + for b in 0..n { + dist[a][b] = dist[a][b].min(dist[a][via] + dist[via][b]); + } + } + } + let count = terminals.len(); + let full = (1usize << count) - 1; + let mut tree = vec![vec![Metric::MAX / 4; n]; full + 1]; + for (index, &terminal) in terminals.iter().enumerate() { + tree[1 << index][..n].copy_from_slice(&dist[terminal]); + } + for mask in 1..=full { + if mask.count_ones() < 2 { + continue; + } + for at in 0..n { + let mut sub = (mask - 1) & mask; + while sub > 0 { + let joined = tree[sub][at] + tree[mask ^ sub][at]; + tree[mask][at] = tree[mask][at].min(joined); + sub = (sub - 1) & mask; + } + } + let row = tree[mask].clone(); + for at in 0..n { + tree[mask][at] = (0..n).map(|from| row[from] + dist[from][at]).min().unwrap_or(row[at]); + } + } + tree[full][endpoint] + } +} diff --git a/rs/moq-net/src/route/tests/warmth.rs b/rs/moq-net/src/route/tests/warmth.rs new file mode 100644 index 00000000000..1d81415d833 --- /dev/null +++ b/rs/moq-net/src/route/tests/warmth.rs @@ -0,0 +1,765 @@ +//! Three ways to find a warm copy, measured on [`WarmSim`] over the multicast +//! prototype's route rule (`quest/m2/multicast/warmth.md`). +//! +//! Each shape is built under every [`Arm`], then a script of reader changes and link +//! closes runs in phases, each settled before the next: what each broadcast's tree +//! costs against the cheapest, how many copies leave the publisher's relay, the +//! messages and rounds it took, and whether pulls ever formed a cycle or left a reader +//! unfed. The ignored benchmarks print the tables; the tests pin what they show. + +use super::multi::Tie; +use super::warm::*; +use super::*; + +/// Every arm measured, in the report's order. +fn arms() -> Vec { + let mut out = Vec::new(); + for hops in [true, false] { + out.push(Arm { + mode: Mode::Path, + hops, + merge: false, + }); + } + for mode in [Mode::Ask(1), Mode::Ask(2), Mode::Ask(usize::MAX), Mode::Advertise] { + for hops in [true, false] { + for merge in [false, true] { + out.push(Arm { mode, hops, merge }); + } + } + } + out +} + +const fn arm(mode: Mode, merge: bool) -> Arm { + Arm { + mode, + hops: true, + merge, + } +} + +const PATH: Arm = arm(Mode::Path, false); +const ASK: Arm = arm(Mode::Ask(usize::MAX), false); +const ASK_MERGE: Arm = arm(Mode::Ask(usize::MAX), true); +const ADVERTISE: Arm = arm(Mode::Advertise, false); +const ADVERTISE_MERGE: Arm = arm(Mode::Advertise, true); + +/// A mesh of relays with one publisher. +#[derive(Clone, Debug)] +struct Shape { + name: &'static str, + relays: usize, + links: Vec<(Node, Node, Price, Tie)>, + /// The relay the publisher joins. + home: Node, + claims: usize, +} + +impl Shape { + fn build(&self, arm: Arm, seed: u64) -> WarmSim { + let mut sim = WarmSim::new(arm, self.relays, 1, seed); + for &(a, b, price, tie) in &self.links { + sim.link(a, b, price, tie); + } + assert!(sim.settle(), "{} never builds", self.name); + sim.publish(0, self.home, self.claims); + assert!(sim.settle(), "{} never announces", self.name); + sim + } +} + +/// One change a script makes. +#[derive(Clone, Copy, Debug)] +enum Step { + /// A reader at a relay starts or stops reading a broadcast. + Read(Node, usize, bool), + Close(Node, Node), + /// Hops of delay pass, delivering what is queued. + Pass(usize), +} + +/// Steps applied together, then settled. +#[derive(Clone, Debug)] +struct Phase { + name: &'static str, + steps: Vec, +} + +fn phase(name: &'static str, steps: Vec) -> Phase { + Phase { name, steps } +} + +/// What a phase cost and left. +#[derive(Clone, Debug)] +struct Row { + phase: &'static str, + settled: bool, + right: Result<(), String>, + trees: Vec, + /// The cheapest tree per broadcast. + cheapest: Vec, + sent: Sent, + /// Reader starts and stops in the phase. + changes: usize, + rounds: usize, + /// Rounds each new reader waited for its first frame. + waits: Vec, + watch: Watch, + /// Relays with demand and no reader, per broadcast: what a join cost other relays. + carriers: Vec>, +} + +impl Row { + fn uplinks(&self) -> usize { + self.trees.iter().map(|tree| tree.uplinks).sum() + } + + fn cost(&self) -> Metric { + self.trees.iter().map(|tree| tree.cost).sum() + } + + fn copies(&self) -> usize { + self.trees.iter().map(|tree| tree.copies).max().unwrap_or(0) + } +} + +fn run(shape: &Shape, arm: Arm, phases: &[Phase], seed: u64) -> Vec { + let mut sim = shape.build(arm, seed); + let mut rows = Vec::new(); + for phase in phases { + let (sent, watch, round, waits) = (sim.sent, sim.watch, sim.round, sim.waits.len()); + let mut changes = 0; + for &step in &phase.steps { + match step { + Step::Read(node, cast, reading) => { + changes += 1; + sim.read(node, cast, reading); + } + Step::Close(a, b) => sim.close(a, b), + Step::Pass(rounds) => { + for _ in 0..rounds { + sim.round(); + } + } + } + } + let settled = sim.settle(); + let casts = 0..sim.casts(); + rows.push(Row { + phase: phase.name, + settled, + right: if settled { + sim.check() + } else { + Err("never settled".into()) + }, + trees: casts.clone().map(|cast| sim.tree(cast)).collect(), + cheapest: casts.clone().map(|cast| sim.cheapest(cast)).collect(), + sent: sim.sent - sent, + changes, + rounds: sim.round - round, + waits: sim.waits[waits..].to_vec(), + watch: sim.watch.since(watch), + carriers: casts.map(|cast| sim.carriers(cast)).collect(), + }); + } + rows +} + +// --- The shapes --- + +/// Pops on a line from the publisher's, `0`, with long links from `0` that pass over +/// the pops between. A long link is cheaper cold than the pops it passes over, and +/// dearer warm than joining a warm pop beside its far end. +fn pops() -> Shape { + let links = vec![ + (0, 1, Price::new(6, 1), Tie::Peer), + (1, 2, Price::new(4, 1), Tie::Peer), + (2, 3, Price::new(4, 1), Tie::Peer), + (0, 2, Price::flat(6), Tie::Peer), + (0, 3, Price::flat(8), Tie::Peer), + ]; + Shape { + name: "pops", + relays: 4, + links, + home: 0, + claims: 1, + } +} + +fn pop_phases() -> Vec { + use Step::Read; + vec![ + phase("far_pop_reads", vec![Read(3, 0, true)]), + phase("near_pop_reads", vec![Read(1, 0, true)]), + phase("middle_pop_reads", vec![Read(2, 0, true)]), + phase("near_pop_leaves", vec![Read(1, 0, false)]), + phase("middle_pop_leaves", vec![Read(2, 0, false)]), + ] +} + +/// The relay `0` joins the CDN, where the publisher is; `drones` more are meshed +/// with each other over cheap links, each with a costly uplink to it. +fn drones(drones: usize) -> Shape { + let mut links = Vec::new(); + for a in 1..=drones { + links.push((0, a, Price::new(10, 4), Tie::Peer)); + for b in a + 1..=drones { + links.push((a, b, Price::new(2, 1), Tie::Peer)); + } + } + Shape { + name: "drones", + relays: drones + 1, + links, + home: 0, + claims: 1, + } +} + +/// The drone carrying the broadcast's only uplink, once every drone reads it. +fn uplinked(rows: &[Row]) -> Option { + let tree = rows.last()?.trees.first()?; + let mut uplinks = tree.edges.iter().filter(|&&(parent, _)| parent == 0); + let (_, drone) = uplinks.next()?; + uplinks.next().is_none().then_some(*drone) +} + +fn drone_phases(drones: usize, together: bool) -> Vec { + let reads: Vec = (1..=drones).map(|drone| Step::Read(drone, 0, true)).collect(); + match together { + true => vec![phase("every_drone_reads_at_once", reads)], + false => reads + .into_iter() + .map(|step| phase("one_more_drone_reads", vec![step])) + .collect(), + } +} + +/// A publisher on `0`, two cores `1` and `2` linked to it and each other, and two +/// edges, `3` nearer `1` and `4` nearer `2`. +fn edges() -> Shape { + let links = vec![ + (0, 1, Price::new(3, 1), Tie::Peer), + (0, 2, Price::new(3, 1), Tie::Peer), + (1, 2, Price::new(2, 1), Tie::Peer), + (3, 1, Price::new(1, 1), Tie::Up), + (3, 2, Price::new(2, 1), Tie::Up), + (4, 2, Price::new(1, 1), Tie::Up), + (4, 1, Price::new(2, 1), Tie::Up), + ]; + Shape { + name: "edges", + relays: 5, + links, + home: 0, + claims: 1, + } +} + +/// A ring of `size` relays, each with its own uplink to the publisher's relay `0`, +/// carrying `claims` broadcasts. +fn ring(size: usize, claims: usize) -> Shape { + let mut links = Vec::new(); + for at in 1..=size { + links.push((0, at, Price::new(8, 2), Tie::Peer)); + links.push((at, at % size + 1, Price::new(1, 1), Tie::Peer)); + } + Shape { + name: "ring", + relays: size + 1, + links, + home: 0, + claims, + } +} + +fn ring_phases(size: usize, claims: usize, together: bool) -> Vec { + let all: Vec = (0..claims) + .flat_map(|cast| (1..=size).map(move |at| Step::Read(at, cast, true))) + .collect(); + match together { + true => vec![phase("every_relay_reads_at_once", all)], + false => all + .into_iter() + .map(|step| phase("one_more_reads", vec![step])) + .collect(), + } +} + +/// CDN tiers: `regions` regions of two cores each, linked within a region and to +/// the next region's cores, and `edges` edges per region on both of its cores, +/// peered with each other. The publisher is on a core of the first region. +fn tiers(regions: usize, edges: usize, claims: usize) -> Shape { + let core = |region: usize, side: usize| 2 * region + side; + let edge = |region: usize, index: usize| 2 * regions + region * edges + index; + let mut links = Vec::new(); + for region in 0..regions { + links.push((core(region, 0), core(region, 1), Price::new(2, 1), Tie::Peer)); + if region + 1 < regions { + for (a, b) in [(0, 0), (0, 1), (1, 0), (1, 1)] { + links.push((core(region, a), core(region + 1, b), Price::new(6, 2), Tie::Peer)); + } + } + for index in 0..edges { + for side in 0..2 { + links.push((edge(region, index), core(region, side), Price::new(3, 1), Tie::Up)); + } + for other in index + 1..edges { + links.push((edge(region, index), edge(region, other), Price::new(1, 1), Tie::Peer)); + } + } + } + Shape { + name: "tiers", + relays: regions * (2 + edges), + links, + home: 0, + claims, + } +} + +/// The first edge relay of a [`tiers`] shape. +fn first_edge(regions: usize) -> Node { + 2 * regions +} + +/// Readers start on a seeded random half of `relays` for each broadcast, one at a +/// time, then leave in another order. +fn crowd(relays: std::ops::Range, claims: usize, seed: u64) -> Vec { + let mut rng = XorShift(seed); + let mut joins = Vec::new(); + for cast in 0..claims { + for relay in relays.clone() { + if rng.chance(50) { + joins.push(Step::Read(relay, cast, true)); + } + } + } + for i in (1..joins.len()).rev() { + joins.swap(i, rng.next(i as u64 + 1) as usize); + } + let mut leaves: Vec = (joins.iter()) + .map(|&step| match step { + Step::Read(relay, cast, _) => Step::Read(relay, cast, false), + step => step, + }) + .collect(); + for i in (1..leaves.len()).rev() { + leaves.swap(i, rng.next(i as u64 + 1) as usize); + } + let half = leaves.len() / 2; + let mut phases: Vec = joins + .into_iter() + .map(|step| phase("reader_joins", vec![step])) + .collect(); + phases.extend( + leaves + .into_iter() + .take(half) + .map(|step| phase("reader_leaves", vec![step])), + ); + phases +} + +// --- The pins --- + +fn last(rows: &[Row], name: &str) -> Row { + rows.iter() + .rev() + .find(|row| row.phase == name) + .expect("a phase") + .clone() +} + +fn settles(rows: &[Row], case: &str) { + for row in rows { + assert_eq!(row.right, Ok(()), "{case}: {}", row.phase); + } +} + +/// A far pop's join skips every pop without demand, whatever the arm: the long link +/// is the route. Path joins never learn of the near pops' demand later; asking and +/// advertising move the far pops through them and back once it ends. +#[test] +fn a_join_skips_pops_without_demand_and_warm_ones_draw_it() { + for arm in arms() { + let rows = run(&pops(), arm, &pop_phases(), 1); + let case = arm.name(); + settles(&rows, &case); + let far = last(&rows, "far_pop_reads"); + assert_eq!(far.carriers[0], Vec::::new(), "{case}"); + assert_eq!(far.trees[0].edges, vec![(4, 0), (0, 3)], "{case}"); + // The middle pop's reader lands on the near pop's copy, and the far pop follows. + let middle = last(&rows, "middle_pop_reads"); + let edges: BTreeSet<(Node, Node)> = middle.trees[0].edges.iter().copied().collect(); + let joined = BTreeSet::from([(4, 0), (0, 1), (1, 2), (2, 3)]); + let skip = BTreeSet::from([(4, 0), (0, 1), (0, 2), (0, 3)]); + match arm.mode { + Mode::Path => assert_eq!(edges, skip, "{case}"), + // One query goes to the cheapest link, which for the middle pop is the near + // pop's and for the far pop the middle's. + _ => assert_eq!(edges, joined, "{case}"), + } + // The near pop's reader leaves and the middle pop returns to its long link. + let near = last(&rows, "near_pop_leaves"); + assert!( + near.trees[0].edges.contains(&(0, 2)), + "{case}: {:?}", + near.trees[0].edges + ); + assert!(near.carriers[0].is_empty(), "{case}: {:?}", near.carriers[0]); + } +} + +/// Drones that read one after another share one uplink when they can hear of each +/// other, and path joins take one each. +#[test] +fn drones_reading_in_turn_share_one_uplink() { + for arm in arms() { + let rows = run(&drones(4), arm, &drone_phases(4, false), 2); + let case = arm.name(); + settles(&rows, &case); + let uplinks = rows.last().expect("a phase").uplinks(); + match arm.mode { + Mode::Path => assert_eq!(uplinks, 4, "{case}"), + _ => assert_eq!(uplinks, 1, "{case}"), + } + } +} + +/// Drones that read at once each reach the CDN before any is warm. Summed costs +/// alone keep every uplink: no drone's copy plus the mesh beats its own. Merging on +/// the hash leaves one. +#[test] +fn drones_reading_at_once_merge_only_on_the_hash() { + for arm in arms() { + let rows = run(&drones(4), arm, &drone_phases(4, true), 3); + let case = arm.name(); + settles(&rows, &case); + let uplinks = rows.last().expect("a phase").uplinks(); + match arm.merge { + true => assert_eq!(uplinks, 1, "{case}"), + false => assert_eq!(uplinks, 4, "{case}"), + } + } +} + +/// The drone holding the one uplink loses it. It can recover only through a drone +/// it feeds, so every one of them takes its own uplink at once: the race again, +/// which only the merge undoes, and only for a relay that hears of more than one +/// neighbour. +#[test] +fn drones_take_over_a_lost_uplink() { + for arm in arms().into_iter().filter(|arm| arm.mode != Mode::Path) { + let mut phases = drone_phases(4, false); + let built = run(&drones(4), arm, &phases, 4); + let holder = uplinked(&built).expect("one uplink"); + phases.push(phase("its_uplink_closes", vec![Step::Close(0, holder)])); + let rows = run(&drones(4), arm, &phases, 4); + let case = arm.name(); + settles(&rows, &case); + let uplinks = rows.last().expect("a phase").uplinks(); + let merged = arm.merge && arm.mode != Mode::Ask(1); + assert_eq!(uplinks, if merged { 1 } else { 3 }, "{case}"); + } +} + +// --- Churn --- + +/// What churn showed for one arm, summed over its cases. +#[derive(Clone, Debug, Default)] +struct Churned { + cases: usize, + watch: Watch, + sent: Sent, + changes: usize, + /// Cases that never settled or settled wrong once churn stopped, and the first. + failed: usize, + first: Option, + /// Rounds each case took to settle once churn stopped, summed. + settle: usize, +} + +/// A random mesh of four to seven relays, a chain and about two in five of the +/// other links, each with a random cold price up to 6 and warm price up to that. +fn random(rng: &mut XorShift) -> Shape { + let relays = 4 + rng.next(4) as usize; + let mut links = Vec::new(); + for a in 0..relays { + for b in a + 1..relays { + if b == a + 1 || rng.chance(40) { + let cold = 1 + rng.next(6); + links.push((a, b, Price::new(cold, 1 + rng.next(cold)), Tie::Peer)); + } + } + } + Shape { + name: "random", + relays, + home: rng.next(relays as u64) as usize, + links, + claims: 2, + } +} + +/// Readers come and go, links flap, relays restart, and every message is delivered +/// in a random order: 300 steps a case, the same for every arm. A step is one +/// delivery, so a hop of delay is eight of them. Once churn stops the mesh must +/// settle with every reader fed and no pulls circling. +fn churn(arm: Arm, cases: std::ops::Range, shape: fn(&mut XorShift) -> Shape) -> Churned { + let mut out = Churned::default(); + for case in cases { + let mut script = XorShift(0x0bad_c0de + case as u64 * 0x9e37_79b9); + let mut pick = XorShift(0x5eed_0002 + case as u64); + let shape = shape(&mut script); + let mut sim = shape.build(arm, 0x5eed_0003 + case as u64); + let (sent, watch) = (sim.sent, sim.watch); + for _ in 0..2 { + let relay = script.next(shape.relays as u64) as usize; + sim.read(relay, 0, true); + } + for step in 0..300 { + match script.next(20) { + 0..=2 => { + let relay = script.next(shape.relays as u64) as usize; + let cast = script.next(shape.claims as u64) as usize; + let reading = !sim.reading(relay, cast); + out.changes += 1; + sim.read(relay, cast, reading); + } + 3 => { + let (a, b, ..) = shape.links[script.next(shape.links.len() as u64) as usize]; + match sim.is_open(a, b) { + true => sim.close(a, b), + false => sim.reopen(a, b), + } + } + 4 if case % 4 == 0 && script.chance(30) => sim.restart(script.next(shape.relays as u64) as usize), + _ => { + sim.deliver(&mut pick); + } + } + if step % 8 == 7 { + sim.tick(); + } + } + let seen = sim.watch.since(watch); + let round = sim.round; + let settled = sim.settle(); + out.settle += sim.round - round; + let right = match settled { + true => sim.check(), + false => Err("never settled".into()), + }; + if right.is_err() && std::env::var("WARM_DUMP").is_ok() { + println!("{} case {case} on {shape:?}\n{}", arm.name(), sim.dump()); + } + if let Err(err) = right { + out.failed += 1; + out.first.get_or_insert(format!("case {case}: {err}")); + } + out.cases += 1; + out.watch.steps += seen.steps; + out.watch.cycles += seen.cycles; + out.watch.starved += seen.starved; + out.watch.longest = out.watch.longest.max(seen.longest); + out.watch.unfed += seen.unfed; + out.watch.extra += seen.extra; + out.sent = out.sent + (sim.sent - sent); + } + out +} + +/// Drones with readers coming and going and uplinks flapping. +fn flying(rng: &mut XorShift) -> Shape { + drones(3 + rng.next(3) as usize) +} + +/// Under churn every arm settles once it stops, with every reader fed and no pulls +/// circling: the cycles it formed lasted only while messages were in flight. +#[test] +fn every_arm_settles_after_churn() { + for arm in [PATH, ASK, ASK_MERGE, ADVERTISE, ADVERTISE_MERGE] { + for shape in [random as fn(&mut XorShift) -> Shape, flying] { + let out = churn(arm, 0..12, shape); + assert_eq!(out.failed, 0, "{}: {:?}", arm.name(), out.first); + } + } +} + +// --- The tables --- + +/// One measured script on one shape. +#[derive(Clone)] +struct Case { + name: String, + shape: Shape, + phases: Vec, +} + +fn case(name: &str, shape: Shape, phases: Vec) -> Case { + Case { + name: name.into(), + shape, + phases, + } +} + +/// Every case the quest names, sized as it does. +fn cases() -> Vec { + let regions = 3; + let per = 3; + let first = first_edge(regions); + let last = first + regions * per; + let mut rng = XorShift(0x7761_726d); + let single: Vec = (0..16) + .map(|cast| { + let relay = first + rng.next((last - first) as u64) as usize; + phase("one_viewer", vec![Step::Read(relay, cast, true)]) + }) + .collect(); + let at_once = |gap: usize| -> Vec { + let mut steps = Vec::new(); + for drone in 1..=4 { + steps.push(Step::Read(drone, 0, true)); + steps.push(Step::Pass(gap)); + } + vec![phase("drones_read_a_gap_apart", steps)] + }; + vec![ + case("pops", pops(), pop_phases()), + case("drones_in_turn", drones(5), drone_phases(5, false)), + case("drones_at_once", drones(5), drone_phases(5, true)), + case("drones_one_hop_apart", drones(4), at_once(1)), + case("drones_two_hops_apart", drones(4), at_once(2)), + case( + "edges_in_turn", + edges(), + vec![ + phase("near_core_edge", vec![Step::Read(3, 0, true)]), + phase("other_edge", vec![Step::Read(4, 0, true)]), + ], + ), + case( + "edges_at_once", + edges(), + vec![phase( + "both_edges", + vec![Step::Read(3, 0, true), Step::Read(4, 0, true)], + )], + ), + case("ring_in_turn", ring(6, 8), ring_phases(6, 8, false)), + case("ring_at_once", ring(6, 8), ring_phases(6, 8, true)), + case( + "tiers_crowd", + tiers(regions, per, 4), + crowd(first..last, 4, 0x6372_6f77), + ), + case("tiers_one_viewer", tiers(regions, per, 16), single), + ] +} + +/// The drone shape's last phase closes the uplink the arm settled on. +fn takeover(arm: Arm) -> Case { + let mut phases = drone_phases(5, false); + let holder = uplinked(&run(&drones(5), arm, &phases, 4)); + if let Some(holder) = holder { + phases.push(phase("its_uplink_closes", vec![Step::Close(0, holder)])); + } + case("drones_lose_the_uplink", drones(5), phases) +} + +fn mean(values: &[usize]) -> f64 { + match values.len() { + 0 => 0.0, + count => values.iter().sum::() as f64 / count as f64, + } +} + +/// Per case and arm: what the trees cost against the path joins and the cheapest, +/// summed over every phase; the copies left at the end; messages; the wait for a +/// first frame; and cycles and unfed readers while it moved. +#[test] +#[ignore = "a benchmark; Nightly runs it from test/route-sweep.sh"] +fn benchmark_warmth_cases() { + println!( + "case,arm,phases,tree_cost,path_cost,cheapest_cost,uplinks,max_copies,carriers,messages,warm_messages,messages_per_reader_change,warm_per_reader_change,first_frame_ms,cycle_steps,longest_cycle,unfed_reader_rounds,right" + ); + let ms = |rounds: f64| rounds * HOP.as_secs_f64() * 1000.0; + for base in cases().into_iter().map(Some).chain([None]) { + let mut path_cost = None; + for arm in arms() { + let case = base.as_ref().map_or_else(|| takeover(arm), |case| case.clone()); + let rows = run(&case.shape, arm, &case.phases, 7); + let sum = |field: fn(&Row) -> usize| rows.iter().map(field).sum::(); + let cost: Metric = rows.iter().map(Row::cost).sum(); + let cheapest: Metric = rows.iter().flat_map(|row| row.cheapest.iter()).sum(); + if arm == PATH { + path_cost = Some(cost); + } + let sent = rows.iter().fold(Sent::default(), |all, row| all + row.sent); + let changes = sum(|row| row.changes).max(1) as f64; + let waits: Vec = rows.iter().flat_map(|row| row.waits.iter().copied()).collect(); + let end = rows.last().expect("a phase"); + let right = rows.iter().find_map(|row| row.right.clone().err()); + println!( + "{},{},{},{cost},{},{cheapest},{},{},{},{},{},{:.2},{:.2},{:.0},{},{},{},{}", + case.name, + arm.name(), + rows.len(), + path_cost.map_or("-".into(), |cost| cost.to_string()), + end.uplinks(), + end.copies(), + end.carriers.iter().map(Vec::len).sum::(), + sent.total(), + sent.warm(), + sent.total() as f64 / changes, + sent.warm() as f64 / changes, + ms(mean(&waits)), + sum(|row| row.watch.cycles), + rows.iter().map(|row| row.watch.longest).max().unwrap_or(0), + sum(|row| row.watch.unfed), + right.map_or("yes".into(), |err| err.replace(',', ";")), + ); + } + } +} + +/// Per arm, random meshes and drone meshes under churn: cycles, unfed readers, and +/// standing races while it churned, and whether each settled right once it stopped. +#[test] +#[ignore = "a benchmark; Nightly runs it from test/route-sweep.sh"] +fn benchmark_warmth_churn() { + println!( + "mesh,arm,cases,deliveries,cycle_deliveries,starved_deliveries,longest_cycle,unfed_reader_hops,extra_uplink_hops,reader_changes,messages,warm_messages,settle_rounds,failed" + ); + let cases: usize = std::env::var("MOQ_WARMTH_CASES").map_or(400, |cases| cases.parse().expect("MOQ_WARMTH_CASES")); + for (name, shape) in [("random", random as fn(&mut XorShift) -> Shape), ("drones", flying)] { + for arm in arms() { + let out = churn(arm, 0..cases, shape); + println!( + "{name},{},{},{},{},{},{},{},{},{},{},{},{},{}", + arm.name(), + out.cases, + out.watch.steps, + out.watch.cycles, + out.watch.starved, + out.watch.longest, + out.watch.unfed, + out.watch.extra, + out.changes, + out.sent.total(), + out.sent.warm(), + out.settle, + out.first.map_or("0".into(), |first| format!( + "{}: {}", + out.failed, + first.replace(',', ";") + )), + ); + } + } +} From 0400b9443dd46163efbf2e404819d133c9c80928 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 05:24:56 -0800 Subject: [PATCH 03/19] feat(moq-net): measure three ways to find a warm copy, and recommend advertising The warmth simulator runs path joins, asking on subscribe, and advertising over the multicast prototype, with offers that may name their data path and an optional rendezvous merge for simultaneous starts. It counts tree cost against the cheapest tree, uplinks, messages, first frames, and cycles and unfed readers under churn, including cycles that run through a pull the merge made. The recommendation, awaiting the maintainer: advertise with the path in each offer and leave the race standing, since the merge adds transient cycles under adverts. quest/m3/cache-routing/select.md is rewritten for it, and the nightly route sweep runs the warmth benchmarks. Co-Authored-By: Claude Opus 5.5 --- quest/m2/multicast/README.md | 4 +- quest/m2/multicast/proof.md | 3 +- quest/m2/multicast/warmth.md | 110 --------------------- quest/m3/cache-routing/README.md | 9 +- quest/m3/cache-routing/select.md | 143 ++++++++++++++++++++++----- quest/m3/route-health/README.md | 6 +- rs/moq-net/src/route/tests/warm.rs | 102 +++++++++++++++---- rs/moq-net/src/route/tests/warmth.rs | 71 +++++++++---- test/route-sweep.sh | 2 + 9 files changed, 270 insertions(+), 180 deletions(-) delete mode 100644 quest/m2/multicast/warmth.md diff --git a/quest/m2/multicast/README.md b/quest/m2/multicast/README.md index 0de6d4eb75f..46b16c7cbb2 100644 --- a/quest/m2/multicast/README.md +++ b/quest/m2/multicast/README.md @@ -106,7 +106,7 @@ Settled: 27's doubt (rejected: keeping holds and doubt with the doubt tied to the idle timeout; a doubt lifted by an epoch the surviving node floods). 5. ✅ **Three ways to find a warm copy are measured before one is picked** - ([warmth](/quest/m2/multicast/warmth.md)): joins that stop at the first + ([warmth](https://github.com/moq-dev/moq/pull/8903)): joins that stop at the first node carrying the Instance and nothing more; that plus a neighbour saying on each SUBSCRIBE answer whether it carries the Instance and at what cost; and carrying nodes advertising it. One constraint binds all three: a warm @@ -146,7 +146,7 @@ flagged as costly, on uplink each, "the warm cost to a drone beats the cold cost to the CDN" and one uplink carries the broadcast (rejected: narrowing 8 to joins that stop at a carrier already on the path, which delivers neither case). - [Warmth](/quest/m2/multicast/warmth.md) measures both. + [Warmth](https://github.com/moq-dev/moq/pull/8903) measured both. 10. ✅ **A moq-transport session is an endpoint, never transit.** Its wire has no path, chain, or Instance, so a moq-transport relay announcing back a name it learned from us cannot be told from a nearer publisher. diff --git a/quest/m2/multicast/proof.md b/quest/m2/multicast/proof.md index 3ef8108dfb5..6a28560edcf 100644 --- a/quest/m2/multicast/proof.md +++ b/quest/m2/multicast/proof.md @@ -66,7 +66,8 @@ State plainly what the theorem does not give, and where each is covered: - No loop at every instant: [requests](https://github.com/moq-dev/moq/pull/8881) fail closed instead. - A cost that depends on who subscribed: - [warmth](/quest/m2/multicast/warmth.md) argues it separately. + [cache-aware routing](/quest/m3/cache-routing/select.md) argues it + separately, and [warmth](https://github.com/moq-dev/moq/pull/8903) swept it. - Liveness under churn that never stops. - That the tree is loop-free once routes settle: one paragraph, since next hops toward one origin node at a fixed point cannot cycle and a request diff --git a/quest/m2/multicast/warmth.md b/quest/m2/multicast/warmth.md deleted file mode 100644 index 085c6ca7447..00000000000 --- a/quest/m2/multicast/warmth.md +++ /dev/null @@ -1,110 +0,0 @@ -# [L] Three ways to find a warm copy are measured - -## Goal - -The [bake-off](https://github.com/moq-dev/moq/pull/8880)'s harness runs three ways for -a relay to learn that a neighbour already has demand for a broadcast, over -the prototype route rule, and the PR reports for each: whether a CDN join -skips every pop without demand and whether drones with an uplink each settle -on one; the copies each link carries and the total link cost of each -broadcast's tree, against plain shortest-path joins and against the cheapest -tree; control messages per reader change; the time to a first frame; and -unfed readers and pull cycles under churn. The maintainer picks one, and the -PR rewrites [the implementation quest](/quest/m3/cache-routing/select.md) -for it. - -## Plan - -Decided 2026-11-15 (maintainer, /quest-plan): ✅ measure all three, then -pick (rejected: picking one unmeasured). The warm wire was stripped from -lite-07 on 2026-11-14 because its benefit over plain path routing was never -measured. - -Decided 2026-11-15 (maintainer, on -[moq-dev/moq#8873](https://github.com/moq-dev/moq/pull/8873); -[epic](/quest/m2/multicast/README.md) item 9): - -- ✅ A node is warm for an Instance while it has demand for it: something - downstream asked and its own upstream subscription is confirmed. Groups - left in a cache do not make it warm (rejected: a yes/no cache state). - Whether the signal is that fact alone or says how much demand stands - behind it is for the measurement. -- ✅ Two cases must work, so the pick is the cheapest mechanism that - delivers both: - - **Pop skipping.** Pops joined by links that pass over other pops. A join - for a broadcast no pop in between has demand for takes the long link and - costs those pops nothing. Once one of them has demand, a farther pop - joins through it when that is cheaper than the long link, and goes back - when the demand ends. - - **One uplink per group of drones.** Drones meshed with each other, each - with a costly uplink to the CDN. The first with demand reaches the CDN, - and the rest join it over the mesh, because a warm drone costs less than - a cold uplink. Two that reach out at once end on one uplink if the - tie-break below holds up, and when the drone holding the uplink loses - it, another takes over. - -The three: - -1. **On the path only.** A join stops at the first relay with demand for the - Instance, and equal-cost next hops are chosen by the rendezvous hash, so - siblings facing the same choice converge on one. No warm signal. A join - never learns of demand off its own path, so this delivers neither case: - it is the baseline for cost, not a candidate. -2. **Ask on subscribe.** Each SUBSCRIBE answer, payload or metadata-only, - says whether the neighbour has demand for the Instance and its summed - cost from the source, and the relay takes the cheapest. A - [metadata-only subscription](/quest/m3/route-health/meta.md) doubles as - the query, so nothing is sent for a broadcast nobody asks about. Sweep how - many neighbours a relay asks. -3. **Advertise.** A relay with demand announces it on the announce stream. - -The constraint the maintainer set for all three: a warm link costs its warm -price and a cold link its cold price, both at least 1, summed from the source -along the path the data takes, so a route never ranks better for being -extended. The stripped rule broke it: a carrier offered its copy at no cost -and a hash ranked carriers, and pulls formed a cycle in 3.3% of churn steps, -most leaving a reader unfed. - -What that constraint gives and what it leaves: with each relay's cost summed -along the path its data takes, two relays can never each prefer the other, -since one would need `b + link < a` and the other `a + link < b`. For the -same reason two drones that reached out at once, each at the cost of its own -warm uplink, never merge. - -Decided 2026-11-15 (maintainer, after -[moq-dev/moq#8873](https://github.com/moq-dev/moq/pull/8873)): ✅ merge them -with the tie-break routing already has, "the same hash based tiebreaker if -possible and one drone renegs". Among relays that hold equal-cost copies of -one Instance and can reach each other, the rendezvous hash that picks -between equal next hops names the one that keeps its uplink. The others -join it and drop their own uplink once the new copy delivers (rejected: a -second ordering only for this case). If the sweep shows that rule adding a -cycle, an unfed reader, or more messages than the second uplink costs, leave -the race: "I'm also fine leaving it as an acceptable race". Simultaneous -starts then stay on two uplinks until one drone's demand ends, and the PR -reports how often and for how long. - -Joining the cheapest point of the tree that exists is the greedy online -Steiner tree, which stays within a logarithmic factor of the cheapest tree -when readers only join (Imase and Waxman, 1991). Leaves and moves are what -the sweep adds to that. - -A cost that depends on who subscribed is outside the -[proof](/quest/m2/multicast/proof.md)'s theorem. Argue it separately and -sweep it: taking a route only makes it cheaper for others, and leaving one -only worsens an offer nobody uses. A cycle that outlasts the messages in -flight fails the mechanism; count the transient ones with the readers they -leave unfed. - -Shapes where the three differ: the two cases above; two edges that each -prefer a different core; a ring with a far source; and one viewer per -broadcast, where the first must cost no message at all. Earlier sweeps to -compare with: a LAN mesh of 6 pulled one uplink copy per broadcast, a ring -of 6 kept 12 copies for 8 broadcasts, and warm routes cost 3.1 to 4.5 -messages per reader change. - -The cut waits on the pick, not on the implementation. An additive field -lands in whichever wip is current with its implementation, as decided -2026-11-14; the PR says whether the pick changes bytes lite-07 would freeze. - -Public API: none. Wire: none. diff --git a/quest/m3/cache-routing/README.md b/quest/m3/cache-routing/README.md index de9760d3e6a..e4981209334 100644 --- a/quest/m3/cache-routing/README.md +++ b/quest/m3/cache-routing/README.md @@ -25,7 +25,7 @@ a node that says it carries a broadcast). What stands and what goes: layer gives with no message ([requests](https://github.com/moq-dev/moq/pull/8881)). - ✅ How a relay learns of a warm neighbour off its path is picked from - measurements ([warmth](/quest/m2/multicast/warmth.md)), which + measurements ([warmth](https://github.com/moq-dev/moq/pull/8903)), which [the implementation](/quest/m3/cache-routing/select.md) then builds. - ✅ A warm cost is strictly positive and summed from the source along the path the data takes. This replaces item 14's rule below, under which a @@ -57,9 +57,10 @@ This README's own work, once its child lands: plus a skip link: a cold broadcast takes the skip, a reader on the skipped relay makes the far relay join it, and that reader leaving sends it back. Two LAN relays with their own uplinks and readers on both pull one copy - over one uplink when the readers arrive in turn, and when they arrive at - once if [warmth](/quest/m2/multicast/warmth.md) kept its tie-break. The - other uplink takes over when that one is lost. + over one uplink when the readers arrive in turn. When they arrive at once, + both uplinks stay until one relay's readers leave, unless the maintainer + keeps the tie-break [warmth](https://github.com/moq-dev/moq/pull/8903) measured. The other uplink takes over + when that one is lost. - Docs: warm and cold costs in `doc/bin/relay/cluster.md`, and a section on the routing concept page once the [routing guide](/quest/m3/routing-guide.md) has added it. diff --git a/quest/m3/cache-routing/select.md b/quest/m3/cache-routing/select.md index 05685a6cf14..d5d6762dc01 100644 --- a/quest/m3/cache-routing/select.md +++ b/quest/m3/cache-routing/select.md @@ -2,38 +2,133 @@ ## Goal -Relays learn of warm copies the way -[warmth](/quest/m2/multicast/warmth.md) picked, and each pulls every demanded -broadcast from the cheapest copy of its Instance, moving an existing pull -only when the route it uses gets worse or ends, or another is strictly -cheaper in summed cost. A peer that sends no warm signal is treated as cold -and behaves as it does today. Relays export how many upstream copies they -hold and how many moves a warm copy caused. +A relay with demand for an Instance advertises it to its neighbours on the +announce stream, and each relay pulls every demanded broadcast from the +cheapest copy of its Instance, moving an existing pull only when the route it +uses gets worse or ends, or another is strictly cheaper in summed cost. A peer +that sends no warm signal is treated as cold and behaves as it does today. +Relays export how many upstream copies they hold and how many moves a warm +copy caused. ## Plan Rewritten 2026-11-15 (maintainer, /quest-plan on [cache-aware multicast](/quest/m2/multicast/README.md)): the rule this was to build, item 14 of the [epic](/quest/m3/cache-routing/README.md)'s earlier -Plan, is replaced. The warmth quest's PR fills in the mechanism here once -the maintainer has picked it. +Plan, is replaced. -What holds whichever is picked: +Recommended 2026-11-16 on +[moq-dev/moq#8903](https://github.com/moq-dev/moq/pull/8903), awaiting the +maintainer's confirmation at merge: **advertise, with the data path in every +offer, and the race left standing** (rejected: asking the cheapest +neighbours on each SUBSCRIBE, which doubled the time to a first frame, sent +a quarter to a third more messages, and left nearly twice the readers unfed +under churn; joins along the path only, the cost baseline, which delivers neither +case). That PR's simulator, `rs/moq-net/src/route/tests/warm.rs`, runs every +rule below, and its `warmth.rs` holds the shapes and the churn sweep this +ports onto production code. + +### What a relay sends + +- **The offer.** What a relay R with a live upstream through parent P offers + neighbour N: `O(R→N) = O(P→R) + warm(P, R)` while R has demand besides N, + otherwise `O(P→R) + cold(P, R)`, the chain priced unshared. The publisher's + relay offers 0. An offer is never below its sender's own cost. +- **The path.** Every offer names the relays the data passes from the + publisher's relay to the sender. It cut transient cycles under churn by a + quarter on random meshes and an eighth on drone meshes, and the merge below + cannot work without it: it never settled on tiered meshes and failed 7 to + 16 of 400 churned drone meshes. +- **Adverts.** While R has demand for an Instance besides N, it tells N its + offer and path on the announce stream, again on each change, and withdraws + it when the demand or its upstream ends. Every relay neighbour gets one, + parent and children included: dropping those two left 8 of 400 churned + drone meshes unsettled and nearly doubled the ring's uplinks. +- **Answers.** The subscribe answer carries the parent's offer and path to the + subscriber, and an update on each change. A subscriber waits for that answer + before it counts the pull as live. + +### What a relay chooses + +- Its own cost is `D(R) = O(P→R) + warm(P, R)`. A candidate costs its offer + plus the link's cold price, and the route costs the route layer's summed + cold cost, so a route never ranks better for being extended. +- It pulls from a neighbour only when that is strictly cheaper than `D(R)`. + It never pulls from a neighbour that pulls through it, nor from one whose + path names it. +- A move is make before break: the old pull stays until the new one is + answered live, and a move under way stays while it is no dearer than + staying. Its own subscription shares the old parent, which lowers what + staying costs, and dropping the move for that raises it again: a random + mesh oscillated for good until this rule. +- A pending pull whose source left its options fails closed. Two relays that + asked each other at once on stale adverts otherwise wait on each other for + good. +- Equal candidates fall to a warm copy before the route, then the rendezvous + hash, as `route_order` in `rs/moq-net/src/model/origin.rs` already ranks + equal routes. + +Why no cycle outlasts the messages in flight: a relay's cost is its parent's +offer plus a warm price of at least 1, and no offer is below its sender's +cost, so cost rises strictly along every pull chain. Two relays can never +each prefer the other on current offers, and a cycle needs one stale offer. +Gaining demand only lowers offers to others; losing it raises only the offer +to the neighbour that still pulls, which then moves only to something +strictly cheaper. The sweep agrees: every churned mesh settled, and cycles +lasted at most 42 deliveries, each while some answer was in flight. + +### Simultaneous starts + +Two drones reaching the CDN at once each pay their own warm uplink, so +neither offers the other anything cheaper, and both uplinks stay until one +drone's demand ends. The maintainer's rule, decided 2026-11-15: merge them +on the rendezvous hash unless that adds a cycle, an unfed reader, or more +messages than the second uplink costs. Under adverts it does add cycles: +in 400 churned drone meshes, 53 deliveries had a pull the merge made inside +a cycle. Each was a pending cycle. A merged drone's new parent lost its +uplink and rejoined through a drone that had trusted the merged drone's +advert, before the lost parent's answer arrived. The merge also added about +2% unfed reader hops. Under asking on subscribe, no merged pull was ever in a +cycle. + +So the race stands. A drone that loses the one uplink leaves every drone +it fed taking its own uplink at once: 4 uplinks on the four-drone shape, +1 with the merge. If the maintainer takes the merge instead, it is: among +equal-cost copies a drone can reach, it joins the neighbour with the higher +rendezvous hash whose offer equals its own cost, once that leaves it no +dearer than its unshared cost and every other option. It never joins a +sibling under a parent other than the publisher's relay: leaving that parent +raises the parent's offer to the sibling the merge relied on. It needs the +path in each offer. + +### What holds - A warm link costs its warm price and a cold one its cold price, both at least 1, summed from the source along the path the data takes. -- Extend today's selection rather than adding a second selector. - `route_order` in `rs/moq-net/src/model/origin.rs` already hashes equal - routes per path, and a front already moves to a better route of its own - Instance and resplices (`rs/moq-net/src/model/front.rs`). A front moves - only to a copy of its own Instance. +- Extend today's selection rather than adding a second selector. A front + already moves to a better route of its own Instance and resplices + (`rs/moq-net/src/model/front.rs`). A front moves only to a copy of its own + Instance. - The rule runs in production code that the fixtures and sweeps drive. The stripped rule lived only in a test simulator that checked it against - itself, so a rule that never offered a copy passed. -- Check the Goal's efficiency: copies held against plain path routing. -- Bound the warm state a peer can make a relay hold by design. -- Export upstream copies held and moves by cause through the relay's - existing metrics. + itself, so a rule that never offered a copy passed. Port the warmth shapes + and the churn sweep onto production relays: pops, drones in turn and at + once, the lost uplink, the ring, and the tiers. +- Bound the warm state a peer can make a relay hold: one advert per + neighbour per Instance the relay has a route for, a path no longer than the + route layer's hop limit, and adverts for anything else dropped. +- Check the Goal's efficiency against plain path routing on the same shapes. + On the simulator: pops 61 against 69 (cheapest tree 59), drones in turn one + uplink against five, a ring of 48 relays 8 uplinks against 48, tiers 1115 + against 1191 (cheapest 1095), at 7 to 13 messages per reader change against + 2 to 3. +- A far pop keeps pulling through a nearer pop after that pop's reader + leaves, while the nearer pop's unshared offer plus the warm link stays + below the far pop's own long link. That is the cheaper tree, and the + hysteresis that keeps it from flapping. +- Export upstream copies held, moves by cause (strictly cheaper, route lost + or worse, parent lost its upstream), and adverts sent and held, through the + relay's existing metrics. - `rs/moq-net/AGENTS.md` says a relay behaves the same warm or cold. That covers answers to requests; this changes which neighbour a request goes to. Bring a rewording to the maintainer if the two contradict. @@ -44,13 +139,15 @@ freezes without a warm wire, so whatever wire the pick needs lands in the then-current wip version with this. Public API: report any relay configuration added, such as a warm cost per -link. Wire: whatever the pick needs, in the then-current wip version. +link. Wire, in the then-current wip version and all additive: an advert on +the announce stream carrying an Instance's offer and path, and the same +offer and path on the subscribe answer and its updates. Nothing lite-07 +freezes changes. ## Required -- [Warmth](/quest/m2/multicast/warmth.md) - picks the mechanism this builds - [The switch](/quest/m2/multicast/switch.md) - the route layer it ranks on ## Related -- [Route health](/quest/m3/route-health/README.md) - its metadata-only subscription is the query if the pick is to ask on subscribe +- [Route health](/quest/m3/route-health/README.md) - its metadata-only subscription answers the same way diff --git a/quest/m3/route-health/README.md b/quest/m3/route-health/README.md index 0a95e792f0b..a6c6abe1d8b 100644 --- a/quest/m3/route-health/README.md +++ b/quest/m3/route-health/README.md @@ -65,9 +65,9 @@ New on 2026-11-15, from the prior art this is closest to: - A relay stops asking its standby when the standby is the only thing it serves that track to, so two relays never keep each other alive (the same RFC's rule for its secondary). -- The answer to a metadata-only subscription is also where a neighbour says - whether it carries the Instance, if - [warmth](/quest/m2/multicast/warmth.md) picks that. +- The answer to a metadata-only subscription carries the neighbour's offer + and data path for the Instance, as every subscribe answer does under + [cache-aware routing](/quest/m3/cache-routing/select.md). This README's own work, once its children land: end-to-end validation across a multi-hop relay network. Stalled video moves while healthy audio stays, a diff --git a/rs/moq-net/src/route/tests/warm.rs b/rs/moq-net/src/route/tests/warm.rs index 64257c9dc14..283a24c70aa 100644 --- a/rs/moq-net/src/route/tests/warm.rs +++ b/rs/moq-net/src/route/tests/warm.rs @@ -1,6 +1,6 @@ //! A data plane over the multicast prototype ([`MultiSim`]), to measure how a relay //! learns that a neighbour already has demand for a broadcast -//! (`quest/m2/multicast/warmth.md`). Nothing here ships; `warmth.rs` holds the +//! (`quest/m3/cache-routing/select.md`). Nothing here ships; `warmth.rs` holds the //! shapes, the pins, and the tables. //! //! A relay has demand for a broadcast while it has a reader or a neighbour's payload @@ -118,6 +118,8 @@ struct Pull { via: Node, stream: u64, state: State, + /// The merge made it, so a cycle through it is one the merge added. + merged: bool, } #[derive(Clone, Debug)] @@ -246,6 +248,8 @@ pub(super) struct Watch { pub cycles: usize, /// Of those, deliveries after which a reader's pulls led into it. pub starved: usize, + /// Of those, deliveries after which a cycle ran through a pull the merge made. + pub merged: usize, /// The most deliveries in a row a cycle lasted. pub longest: usize, run: usize, @@ -263,6 +267,7 @@ impl Watch { steps: self.steps - was.steps, cycles: self.cycles - was.cycles, starved: self.starved - was.starved, + merged: self.merged - was.merged, longest: self.longest, run: 0, unfed: self.unfed - was.unfed, @@ -310,8 +315,9 @@ pub(super) struct WarmSim { pub net: MultiSim, relays: usize, nodes: usize, - /// Each publisher's endpoint node. + /// Each publisher's endpoint node, and the relay it joined. endpoints: BTreeMap, + homes: BTreeMap, prices: BTreeMap<(Node, Node), Price>, open: BTreeSet<(Node, Node)>, casts: Vec, @@ -336,6 +342,7 @@ impl WarmSim { relays, nodes, endpoints: BTreeMap::new(), + homes: BTreeMap::new(), prices: BTreeMap::new(), open: BTreeSet::new(), casts: Vec::new(), @@ -367,6 +374,7 @@ impl WarmSim { let id = 100 + index as Id; self.net.client(node); self.endpoints.insert(id, node); + self.homes.insert(id, home); self.prices.insert(pair(node, home), Price::flat(1)); self.open.insert(pair(node, home)); self.net.link(node, home, 1, Tie::Peer); @@ -599,6 +607,7 @@ impl WarmSim { via, stream, state: State::Pending, + merged: false, } } @@ -644,6 +653,7 @@ impl WarmSim { } return; }; + let moving = carry.next.as_ref().map(|next| next.via); let target = match &up.state { // Nothing flows yet, so a better source replaces it outright. State::Pending => { @@ -673,36 +683,60 @@ impl WarmSim { let own = signal.cost + self.price(up.via, node).warm; let back = signal.cost + self.price(up.via, node).cold; let merge = self.merge(node, cast, &options, own, back); - match best { - Some(best) if best.cost < own => Some(best.via), - _ => merge.or(Some(up.via)), + // A move under way stays while it is no dearer: its own subscription + // shares the old parent, which lowers what staying costs, and dropping + // it for that would raise it again. + let moving = (moving) + .and_then(|via| options.iter().find(|choice| choice.via == via)) + .filter(|choice| choice.cost <= own) + .map(|choice| choice.via); + match (best, merge) { + (Some(best), _) if best.cost < own => Some(best.via), + (_, Some(merge)) => { + self.toward(node, cast, merge, true); + return; + } + _ => moving.or(Some(up.via)), } } }, // Lost, or fed through itself: anything usable beats it. _ => best.map(|best| best.via), }; - self.toward(node, cast, target.unwrap_or(up.via)); + self.toward(node, cast, target.unwrap_or(up.via), false); } /// The relay `node` merges onto: a warm neighbour whose copy costs what its own does, - /// with a higher hash, once nothing else, its old parent included, is cheaper. + /// with a higher hash, once nothing else, its old parent included, is cheaper. With + /// [`Arm::hops`], never a sibling under a parent other than the publisher's relay: + /// leaving that parent raises its offer to the sibling, which the merge relied on. fn merge(&self, node: Node, cast: usize, options: &[Choice], own: Metric, back: Metric) -> Option { if !self.arm.merge { return None; } let mine = self.hash(cast, node); let up = self.carry(node, cast)?.up.as_ref()?.via; + let home = self.home(cast); + let carry = self.carry(node, cast)?; + let sibling = |via: Node| { + let signal = match self.arm.mode { + Mode::Ask(_) => carry.asks.get(&via).cloned().flatten(), + _ => carry.heard.get(&via).cloned(), + }; + up != home && signal.is_some_and(|signal| signal.path.contains(&up)) + }; let target = (options.iter()) .filter(|choice| choice.via != up && choice.offered == Some(own) && self.hash(cast, choice.via) > mine) + .filter(|choice| !sibling(choice.via)) .max_by_key(|choice| self.hash(cast, choice.via))?; let after = own + self.price(target.via, node).warm; - let stays = after < back && (options.iter()).all(|choice| choice.via == target.via || after < choice.cost); + let stays = after <= back && (options.iter()).all(|choice| choice.via == target.via || after <= choice.cost); stays.then_some(target.via) } - /// Move toward `target` unless already there or on the way. - fn toward(&mut self, node: Node, cast: usize, target: Node) { + /// Move toward `target` unless already there or on the way, marking a new pull the + /// merge asked for. + fn toward(&mut self, node: Node, cast: usize, target: Node, merged: bool) { let carry = self.entry(node, cast); let up = carry.up.as_ref().map(|pull| pull.via); let next = carry.next.as_ref().map(|pull| pull.via); @@ -716,7 +750,10 @@ impl WarmSim { } let old = self.entry(node, cast).next.take(); self.unsubscribe(node, cast, old); - let pull = self.subscribe(node, cast, target); + let pull = Pull { + merged, + ..self.subscribe(node, cast, target) + }; self.entry(node, cast).next = Some(pull); } @@ -1049,6 +1086,30 @@ impl WarmSim { true } + /// Whether the cycle `node`'s pulls for `cast` run into holds a pull the merge made. + fn merged(&self, node: Node, cast: usize) -> bool { + let mut at = node; + for _ in 0..=self.nodes { + match self.parent(at, cast) { + Some(next) => at = next, + None => return false, + } + } + let start = at; + loop { + let Some(pull) = self.carry(at, cast).and_then(|carry| carry.up.as_ref()) else { + return false; + }; + if pull.merged { + return true; + } + at = pull.via; + if at == start { + return false; + } + } + } + fn readers(&self) -> Vec<(Node, usize)> { (0..self.relays) .flat_map(|node| { @@ -1065,7 +1126,11 @@ impl WarmSim { self.watch.steps += 1; let cycle = (0..self.casts.len()).any(|cast| (0..self.relays).any(|node| self.circles(node, cast))); let starved = cycle && self.readers().into_iter().any(|(node, cast)| self.circles(node, cast)); + let merged = cycle + && (0..self.casts.len()) + .any(|cast| (0..self.relays).any(|node| self.circles(node, cast) && self.merged(node, cast))); self.watch.cycles += usize::from(cycle); + self.watch.merged += usize::from(merged); self.watch.starved += usize::from(starved); self.watch.run = if cycle { self.watch.run + 1 } else { 0 }; self.watch.longest = self.watch.longest.max(self.watch.run); @@ -1148,8 +1213,7 @@ impl WarmSim { /// The relay publishing `cast` is joined to. pub fn home(&self, cast: usize) -> Node { - let endpoint = self.endpoints[&self.casts[cast].0]; - self.peers(endpoint)[0] + self.homes[&self.casts[cast].0] } pub fn tree(&self, cast: usize) -> Tree { @@ -1204,13 +1268,13 @@ impl WarmSim { if mask.count_ones() < 2 { continue; } - for at in 0..n { - let mut sub = (mask - 1) & mask; - while sub > 0 { - let joined = tree[sub][at] + tree[mask ^ sub][at]; - tree[mask][at] = tree[mask][at].min(joined); - sub = (sub - 1) & mask; + let mut sub = (mask - 1) & mask; + while sub > 0 { + let joined: Vec = tree[sub].iter().zip(&tree[mask ^ sub]).map(|(a, b)| a + b).collect(); + for (slot, joined) in tree[mask].iter_mut().zip(joined) { + *slot = (*slot).min(joined); } + sub = (sub - 1) & mask; } let row = tree[mask].clone(); for at in 0..n { diff --git a/rs/moq-net/src/route/tests/warmth.rs b/rs/moq-net/src/route/tests/warmth.rs index 1d81415d833..43a0096cd03 100644 --- a/rs/moq-net/src/route/tests/warmth.rs +++ b/rs/moq-net/src/route/tests/warmth.rs @@ -1,5 +1,5 @@ //! Three ways to find a warm copy, measured on [`WarmSim`] over the multicast -//! prototype's route rule (`quest/m2/multicast/warmth.md`). +//! prototype's route rule, for the pick `quest/m3/cache-routing/select.md` builds. //! //! Each shape is built under every [`Arm`], then a script of reader changes and link //! closes runs in phases, each settled before the next: what each broadcast's tree @@ -94,7 +94,6 @@ fn phase(name: &'static str, steps: Vec) -> Phase { #[derive(Clone, Debug)] struct Row { phase: &'static str, - settled: bool, right: Result<(), String>, trees: Vec, /// The cheapest tree per broadcast. @@ -102,7 +101,6 @@ struct Row { sent: Sent, /// Reader starts and stops in the phase. changes: usize, - rounds: usize, /// Rounds each new reader waited for its first frame. waits: Vec, watch: Watch, @@ -128,7 +126,7 @@ fn run(shape: &Shape, arm: Arm, phases: &[Phase], seed: u64) -> Vec { let mut sim = shape.build(arm, seed); let mut rows = Vec::new(); for phase in phases { - let (sent, watch, round, waits) = (sim.sent, sim.watch, sim.round, sim.waits.len()); + let (sent, watch, waits) = (sim.sent, sim.watch, sim.waits.len()); let mut changes = 0; for &step in &phase.steps { match step { @@ -144,21 +142,18 @@ fn run(shape: &Shape, arm: Arm, phases: &[Phase], seed: u64) -> Vec { } } } - let settled = sim.settle(); + let right = match sim.settle() { + true => sim.check(), + false => Err("never settled".into()), + }; let casts = 0..sim.casts(); rows.push(Row { phase: phase.name, - settled, - right: if settled { - sim.check() - } else { - Err("never settled".into()) - }, + right, trees: casts.clone().map(|cast| sim.tree(cast)).collect(), cheapest: casts.clone().map(|cast| sim.cheapest(cast)).collect(), sent: sim.sent - sent, changes, - rounds: sim.round - round, waits: sim.waits[waits..].to_vec(), watch: sim.watch.since(watch), carriers: casts.map(|cast| sim.carriers(cast)).collect(), @@ -554,17 +549,16 @@ fn churn(arm: Arm, cases: std::ops::Range, shape: fn(&mut XorShift) -> Sh true => sim.check(), false => Err("never settled".into()), }; - if right.is_err() && std::env::var("WARM_DUMP").is_ok() { - println!("{} case {case} on {shape:?}\n{}", arm.name(), sim.dump()); - } if let Err(err) = right { out.failed += 1; - out.first.get_or_insert(format!("case {case}: {err}")); + out.first + .get_or_insert_with(|| format!("case {case}: {err} on {shape:?}\n{}", sim.dump())); } out.cases += 1; out.watch.steps += seen.steps; out.watch.cycles += seen.cycles; out.watch.starved += seen.starved; + out.watch.merged += seen.merged; out.watch.longest = out.watch.longest.max(seen.longest); out.watch.unfed += seen.unfed; out.watch.extra += seen.extra; @@ -590,6 +584,46 @@ fn every_arm_settles_after_churn() { } } +/// One viewer per broadcast on CDN tiers: a join that follows its route costs no +/// message beyond its subscriptions, and both ways to hear of warmth spend some on +/// every first viewer. +#[test] +fn a_first_viewer_costs_warm_messages_unless_joins_follow_the_route() { + let case = cases() + .into_iter() + .find(|case| case.name == "tiers_one_viewer") + .expect("the case"); + for arm in [PATH, ASK, ADVERTISE] { + let rows = run(&case.shape, arm, &case.phases, 7); + settles(&rows, &arm.name()); + for row in &rows { + match arm.mode { + Mode::Path => assert_eq!(row.sent.warm(), 0, "{}", arm.name()), + _ => assert!(row.sent.warm() > 0, "{}", arm.name()), + } + } + } +} + +/// On CDN tiers two edges under one core cost the same, and a merge between them +/// leaves the core with one child, whose offer then rises: without hops to tell a +/// sibling the merge undoes itself for good. With them it settles on a cheaper tree. +#[test] +fn a_merge_settles_on_tiers_once_offers_name_their_hops() { + let case = cases() + .into_iter() + .find(|case| case.name == "tiers_crowd") + .expect("the case"); + for (alone, merged) in [(ASK, ASK_MERGE), (ADVERTISE, ADVERTISE_MERGE)] { + let cost = |arm: Arm| { + let rows = run(&case.shape, arm, &case.phases, 7); + settles(&rows, &arm.name()); + rows.iter().map(Row::cost).sum::() + }; + assert!(cost(merged) <= cost(alone), "{}", merged.name()); + } +} + // --- The tables --- /// One measured script on one shape. @@ -734,19 +768,20 @@ fn benchmark_warmth_cases() { #[ignore = "a benchmark; Nightly runs it from test/route-sweep.sh"] fn benchmark_warmth_churn() { println!( - "mesh,arm,cases,deliveries,cycle_deliveries,starved_deliveries,longest_cycle,unfed_reader_hops,extra_uplink_hops,reader_changes,messages,warm_messages,settle_rounds,failed" + "mesh,arm,cases,deliveries,cycle_deliveries,starved_deliveries,merge_cycle_deliveries,longest_cycle,unfed_reader_hops,extra_uplink_hops,reader_changes,messages,warm_messages,settle_rounds,failed" ); let cases: usize = std::env::var("MOQ_WARMTH_CASES").map_or(400, |cases| cases.parse().expect("MOQ_WARMTH_CASES")); for (name, shape) in [("random", random as fn(&mut XorShift) -> Shape), ("drones", flying)] { for arm in arms() { let out = churn(arm, 0..cases, shape); println!( - "{name},{},{},{},{},{},{},{},{},{},{},{},{},{}", + "{name},{},{},{},{},{},{},{},{},{},{},{},{},{},{}", arm.name(), out.cases, out.watch.steps, out.watch.cycles, out.watch.starved, + out.watch.merged, out.watch.longest, out.watch.unfed, out.watch.extra, diff --git a/test/route-sweep.sh b/test/route-sweep.sh index 3d82af799b4..c3f9c13702a 100755 --- a/test/route-sweep.sh +++ b/test/route-sweep.sh @@ -18,3 +18,5 @@ just rs test --profile=sweep -p moq-net --release --run-ignored only -E 'test(/^ MOQ_ROUTE_CASES=3000 just rs test -P sweep -p moq-net --release --run-ignored only -E 'test(=route::tests::cache::sweep_cache_routing_long)' # The multicast bake-off: its prototype carries every failure on every shape at 32 relays, and its tables against the path rule at 12. MOQ_BAKEOFF_RELAYS=12 just rs test --profile=sweep -p moq-net --release --run-ignored only -E 'test(/^route::tests::bakeoff::(sweep|benchmark)_/)' +# Warmth: path joins, asking, and advertising, each over fixed shapes and 400 churned meshes per arm, about 30 seconds. +just rs test --profile=sweep -p moq-net --release --run-ignored only -E 'test(/^route::tests::warmth::benchmark_/)' From c7704dea4759a7dde2aca7ce96e1124b5f270787 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 06:20:18 -0800 Subject: [PATCH 04/19] feat(moq-net): sweep warmth growth over relays and broadcasts, and advertise only where useful The warmth simulator reacts only the relay and broadcast a delivery touched and looks for cycles only in broadcasts whose pulls changed, which leaves every existing table unchanged and lets it carry 128 relays and 4096 broadcasts. benchmark_warmth_growth sweeps CDN tiers with a full core mesh (12 to 128 relays) and drone meshes (5 to 64 drones) against demanded broadcasts, for one viewer per broadcast and a crowd arriving in turn. Advertising to every neighbour grows with the neighbours of each warm relay. Advertising only where offer plus link beats the neighbour's route stays flat in cores on the CDN, keeps both required cases, and settles every churned mesh. Asking stays bounded only because it stops finding the warm drone on larger drone meshes. Two other cheap variants are measured and lose: advertising only once two others want the copy, and never over links marked upstream. The recommendation moves to the useful-only adverts, and quest/m3/cache-routing/select.md says so with the growth laws. Co-Authored-By: Claude Opus 5.5 --- quest/m3/cache-routing/select.md | 85 +++++++--- rs/moq-net/src/route/tests/warm.rs | 222 +++++++++++++++++++----- rs/moq-net/src/route/tests/warmth.rs | 241 ++++++++++++++++++++++++++- test/route-sweep.sh | 2 +- 4 files changed, 476 insertions(+), 74 deletions(-) diff --git a/quest/m3/cache-routing/select.md b/quest/m3/cache-routing/select.md index d5d6762dc01..f7b574dca50 100644 --- a/quest/m3/cache-routing/select.md +++ b/quest/m3/cache-routing/select.md @@ -2,8 +2,8 @@ ## Goal -A relay with demand for an Instance advertises it to its neighbours on the -announce stream, and each relay pulls every demanded broadcast from the +A relay with demand for an Instance advertises it on the announce stream to +each neighbour that could take it, and each relay pulls every demanded broadcast from the cheapest copy of its Instance, moving an existing pull only when the route it uses gets worse or ends, or another is strictly cheaper in summed cost. A peer that sends no warm signal is treated as cold and behaves as it does today. @@ -19,14 +19,28 @@ Plan, is replaced. Recommended 2026-11-16 on [moq-dev/moq#8903](https://github.com/moq-dev/moq/pull/8903), awaiting the -maintainer's confirmation at merge: **advertise, with the data path in every -offer, and the race left standing** (rejected: asking the cheapest -neighbours on each SUBSCRIBE, which doubled the time to a first frame, sent -a quarter to a third more messages, and left nearly twice the readers unfed -under churn; joins along the path only, the cost baseline, which delivers neither -case). That PR's simulator, `rs/moq-net/src/route/tests/warm.rs`, runs every -rule below, and its `warmth.rs` holds the shapes and the churn sweep this -ports onto production code. +maintainer's confirmation at merge: **advertise only to a neighbour that +could take the copy, with the data path in every offer, and the race left +standing**. Rejected: + +- Advertising to every neighbour, which grows with the neighbours of every + relay on the warm chain: 11 to 76 warm messages per one-viewer reader + change as a full core mesh grows from 3 to 32 cores. +- Asking the k cheapest neighbours on each SUBSCRIBE. It stays bounded per + subscribe only because it stops finding the copy: on a full drone mesh + past 5 drones, `ask_1` shared no uplink at 64 drones and `ask_2` few. It + doubled the time to a first frame and left about twice the readers unfed + under churn. On CDN tiers `ask_1` sent 6.5 to 11 warm messages per + one-viewer reader change against the pick's 3 to 7, about as many for a + crowd (3.9 to 8.1 against 5.4 to 7.9), and built dearer crowd trees. +- Advertising only once two others want the copy, which lost both cases; + never advertising over a link marked upstream, which saves nothing in a + core mesh. +- Joins along the path only, the cost baseline, which delivers neither case. + +That PR's simulator, `rs/moq-net/src/route/tests/warm.rs`, runs every rule +below, and its `warmth.rs` holds the shapes, the churn sweep, and the growth +sweep this ports onto production code. ### What a relay sends @@ -35,15 +49,19 @@ ports onto production code. otherwise `O(P→R) + cold(P, R)`, the chain priced unshared. The publisher's relay offers 0. An offer is never below its sender's own cost. - **The path.** Every offer names the relays the data passes from the - publisher's relay to the sender. It cut transient cycles under churn by a - quarter on random meshes and an eighth on drone meshes, and the merge below - cannot work without it: it never settled on tiered meshes and failed 7 to - 16 of 400 churned drone meshes. + publisher's relay to the sender. Without it, 1 of 400 churned random meshes + never settled, and transient cycles rose by 74% on random meshes and 25% on + drone meshes. The merge below cannot work without it either. - **Adverts.** While R has demand for an Instance besides N, it tells N its - offer and path on the announce stream, again on each change, and withdraws - it when the demand or its upstream ends. Every relay neighbour gets one, - parent and children included: dropping those two left 8 of 400 churned - drone meshes unsettled and nearly doubled the ring's uplinks. + offer and path on the announce stream when `O(R→N) + cold(R, N)` is below + N's route cost for the Instance, or equals it through a next hop other than + R, since a tie prefers a warm copy. It tells again on each change, and + withdraws when the demand, its upstream, or that test ends. Nothing it + skips could ever be taken: a relay at rest costs no more than its own + route, and a tie through R is the path join already. R knows N's route cost + from what N announces it, and computes it when N's route runs through R. + Skipping the parent and children by rule instead left 8 of 400 churned + drone meshes unsettled. - **Answers.** The subscribe answer carries the parent's offer and path to the subscriber, and an update on each change. A subscriber waits for that answer before it counts the pull as live. @@ -84,12 +102,12 @@ neither offers the other anything cheaper, and both uplinks stay until one drone's demand ends. The maintainer's rule, decided 2026-11-15: merge them on the rendezvous hash unless that adds a cycle, an unfed reader, or more messages than the second uplink costs. Under adverts it does add cycles: -in 400 churned drone meshes, 53 deliveries had a pull the merge made inside -a cycle. Each was a pending cycle. A merged drone's new parent lost its -uplink and rejoined through a drone that had trusted the merged drone's -advert, before the lost parent's answer arrived. The merge also added about -2% unfed reader hops. Under asking on subscribe, no merged pull was ever in a -cycle. +in 400 churned drone meshes, 41 deliveries had a pull the merge made inside +a cycle (53 when advertising to every neighbour). Each was a pending cycle. +A merged drone's new parent lost its uplink and rejoined through a drone +that had trusted the merged drone's advert, before the lost parent's answer +arrived. The merge also added about 2% unfed reader hops. Under asking on +subscribe, no merged pull was ever in a cycle. So the race stands. A drone that loses the one uplink leaves every drone it fed taking its own uplink at once: 4 uplinks on the four-drone shape, @@ -116,12 +134,27 @@ path in each offer. once, the lost uplink, the ring, and the tiers. - Bound the warm state a peer can make a relay hold: one advert per neighbour per Instance the relay has a route for, a path no longer than the - route layer's hop limit, and adverts for anything else dropped. + route layer's hop limit, and adverts for anything else dropped. Adverts are + per broadcast like the claims the announce stream already carries over + every link, so a relay hears at most one per neighbour per demanded + broadcast. - Check the Goal's efficiency against plain path routing on the same shapes. On the simulator: pops 61 against 69 (cheapest tree 59), drones in turn one uplink against five, a ring of 48 relays 8 uplinks against 48, tiers 1115 - against 1191 (cheapest 1095), at 7 to 13 messages per reader change against + against 1191 (cheapest 1095), at 5 to 9 messages per reader change against 2 to 3. +- Port the growth sweep, and hold its laws. Per reader change, warm messages + are flat in the number of demanded broadcasts for every arm. On CDN tiers + with a full core mesh they are also flat in cores: 3 to 7.9 from 12 to 128 + relays and 1 to 4,096 broadcasts. A core with demand is never cheaper + for another core than that core's own route, so only the edges beside it + hear. On a full drone mesh they grow with the drones, n − 1 per first + viewer (63 at 64 drones), since every drone could take the copy; that is + what buys the cheapest tree there (432 against 1,328 to 1,456 for asking + and path joins, a crowd of 9 on 16 broadcasts). At rest the busiest relay + of 128 holds 755 adverts for 4,096 one-viewer broadcasts, against 5,405 + metadata-only subscriptions for `ask_1` and 151,552 adverts sent to every + neighbour. - A far pop keeps pulling through a nearer pop after that pop's reader leaves, while the nearer pop's unshared offer plus the warm link stays below the far pop's own long link. That is the cheaper tree, and the diff --git a/rs/moq-net/src/route/tests/warm.rs b/rs/moq-net/src/route/tests/warm.rs index 283a24c70aa..013efee32e2 100644 --- a/rs/moq-net/src/route/tests/warm.rs +++ b/rs/moq-net/src/route/tests/warm.rs @@ -27,6 +27,7 @@ //! ([`super::compare::Arm::KEPT`]), its messages interleaved with these at random. use std::{ + cell::RefCell, cmp::Reverse, collections::{BTreeMap, BTreeSet, VecDeque}, }; @@ -47,8 +48,25 @@ pub(super) enum Mode { /// what cost. A neighbour with no demand forwards the subscription along its route, /// as a metadata-only subscription does. Ask(usize), - /// A warm relay tells every neighbour, and each change, on its announce stream. - Advertise, + /// A warm relay tells neighbours, and each change, on its announce stream: which + /// ones is the [`Audience`]. + Advertise(Audience), +} + +/// Which neighbours a warm relay advertises to. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Audience { + /// Every relay neighbour. + All, + /// Only a neighbour for whom its offer plus the link beats that neighbour's route, + /// or ties it through another next hop, since a tie prefers a warm copy. Nothing + /// else could ever take it: a relay at rest costs no more than its own route, and + /// a tie through this relay is the path join already. + Useful, + /// Only once it has demand from two sources besides the neighbour. + Shared, + /// Never over a link marked upstream. + Down, } /// One way to find a warm copy, as measured. @@ -69,7 +87,10 @@ impl Arm { Mode::Path => "path".to_string(), Mode::Ask(usize::MAX) => "ask_all".into(), Mode::Ask(count) => format!("ask_{count}"), - Mode::Advertise => "advertise".into(), + Mode::Advertise(Audience::All) => "advertise".into(), + Mode::Advertise(Audience::Useful) => "advertise_useful".into(), + Mode::Advertise(Audience::Shared) => "advertise_shared".into(), + Mode::Advertise(Audience::Down) => "advertise_down".into(), }; let hops = if self.hops { "+hops" } else { "" }; let merge = if self.merge { "+merge" } else { "" }; @@ -153,7 +174,7 @@ enum Lane { type Key = (Node, Node, Lane, usize); /// One relay's state for one broadcast. -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, PartialEq, Eq)] struct Carry { reader: bool, /// The round its reader started, until its first frame. @@ -319,11 +340,25 @@ pub(super) struct WarmSim { endpoints: BTreeMap, homes: BTreeMap, prices: BTreeMap<(Node, Node), Price>, + /// Links marked upstream. + marked: BTreeSet<(Node, Node)>, open: BTreeSet<(Node, Node)>, + /// Each node's open links. + adjacent: Vec>, + /// Cheapest cold costs between nodes over open links, until one opens or closes. + distances: RefCell>>>, casts: Vec, carry: Vec>, queues: BTreeMap>, streams: u64, + /// What must react before the next delivery: a relay and broadcast whose inputs + /// changed. Nothing else can act, so nothing else is asked to. + dirty: BTreeSet<(Node, usize)>, + /// Broadcasts whose pulls changed since cycles were last looked for. + changed: BTreeSet, + /// Broadcasts whose pulls circle: whether a reader's lead into it, and whether it + /// holds a pull the merge made. + circling: BTreeMap, pub sent: Sent, pub watch: Watch, pub round: usize, @@ -344,11 +379,17 @@ impl WarmSim { endpoints: BTreeMap::new(), homes: BTreeMap::new(), prices: BTreeMap::new(), + marked: BTreeSet::new(), open: BTreeSet::new(), + adjacent: vec![BTreeSet::new(); nodes], + distances: RefCell::new(None), casts: Vec::new(), carry: vec![BTreeMap::new(); nodes], queues: BTreeMap::new(), streams: 0, + dirty: BTreeSet::new(), + changed: BTreeSet::new(), + circling: BTreeMap::new(), sent: Sent::default(), watch: Watch::default(), round: 0, @@ -363,10 +404,33 @@ impl WarmSim { pub fn link(&mut self, a: Node, b: Node, price: Price, tie: Tie) { self.prices.insert(pair(a, b), price); - self.open.insert(pair(a, b)); + if tie == Tie::Up { + self.marked.insert(pair(a, b)); + } + self.join(a, b); self.net.link(a, b, price.cold, tie); } + fn join(&mut self, a: Node, b: Node) { + self.open.insert(pair(a, b)); + self.adjacent[a].insert(b); + self.adjacent[b].insert(a); + self.distances.replace(None); + } + + fn part(&mut self, a: Node, b: Node) { + self.open.remove(&pair(a, b)); + self.adjacent[a].remove(&b); + self.adjacent[b].remove(&a); + self.distances.replace(None); + } + + /// Every broadcast `node` holds state for reacts next, as when its routes change. + fn stir(&mut self, node: Node) { + let casts: Vec = self.carry[node].keys().copied().collect(); + self.dirty.extend(casts.into_iter().map(|cast| (node, cast))); + } + /// Publisher `index` joins relay `home` and announces `claims` broadcasts. Returns /// each broadcast's index. pub fn publish(&mut self, index: usize, home: Node, claims: usize) -> Vec { @@ -376,7 +440,7 @@ impl WarmSim { self.endpoints.insert(id, node); self.homes.insert(id, home); self.prices.insert(pair(node, home), Price::flat(1)); - self.open.insert(pair(node, home)); + self.join(node, home); self.net.link(node, home, 1, Tie::Peer); self.net.publish(node, id); (0..claims) @@ -402,13 +466,7 @@ impl WarmSim { } fn peers(&self, node: Node) -> Vec { - (self.open.iter()) - .filter_map(|&(a, b)| match node { - _ if a == node => Some(b), - _ if b == node => Some(a), - _ => None, - }) - .collect() + self.adjacent[node].iter().copied().collect() } fn hash(&self, cast: usize, node: Node) -> u64 { @@ -522,7 +580,7 @@ impl WarmSim { Mode::Ask(_) => (carry.asks.iter()) .filter_map(|(&peer, signal)| Some((peer, signal.as_ref()?))) .collect(), - Mode::Advertise => carry.heard.iter().map(|(&peer, signal)| (peer, signal)).collect(), + Mode::Advertise(_) => carry.heard.iter().map(|(&peer, signal)| (peer, signal)).collect(), }; for (via, signal) in heard { if !self.is_open(node, via) @@ -787,12 +845,14 @@ impl WarmSim { /// Tell every neighbour whether `node` is warm, on a change. fn advertise(&mut self, node: Node, cast: usize) { - if self.arm.mode != Mode::Advertise { + let Mode::Advertise(audience) = self.arm.mode else { return; - } + }; let peers: Vec = self.peers(node).into_iter().filter(|&peer| self.relay(peer)).collect(); for peer in peers { - let want = self.warm(node, cast, peer); + let want = self + .warm(node, cast, peer) + .filter(|signal| self.scoped(audience, node, cast, peer, signal)); let was = self.entry(node, cast).advertised.get(&peer).cloned(); if was == want { continue; @@ -805,16 +865,45 @@ impl WarmSim { } } - /// Every relay reacts to what changed. + /// Whether `audience` lets `node` advertise `signal` to `peer`. + fn scoped(&self, audience: Audience, node: Node, cast: usize, peer: Node, signal: &Signal) -> bool { + match audience { + Audience::All => true, + Audience::Useful => match self.route(peer, cast) { + Some((via, cost)) => { + let price = signal.cost + self.price(node, peer).cold; + price < cost || (price == cost && via != node) + } + None => true, + }, + Audience::Shared => self.carry(node, cast).is_some_and(|carry| { + let children = carry.down.keys().filter(|&&child| child != peer).count(); + usize::from(carry.reader) + children >= 2 + }), + Audience::Down => !self.marked.contains(&pair(node, peer)), + } + } + + /// Each relay whose inputs changed reacts, in relay order, until it settles; a + /// relay with no state for a broadcast has nothing to do for it. fn step(&mut self) { - for node in 0..self.relays { - for cast in 0..self.casts.len() { + while let Some((node, cast)) = self.dirty.pop_first() { + self.changed.insert(cast); + for _ in 0..8 { + let was = self.carry(node, cast).cloned(); self.react(node, cast); + if self.carry(node, cast) == was.as_ref() { + break; + } + } + if self.carry(node, cast).is_some_and(|carry| *carry == Carry::default()) { + self.carry[node].remove(&cast); } } } fn recv(&mut self, from: Node, to: Node, cast: usize, msg: Msg) { + self.dirty.insert((to, cast)); // A publisher answers at once. if let Msg::Subscribe { stream, open: true } = msg && !self.relay(to) @@ -874,6 +963,7 @@ impl WarmSim { let carry = self.entry(node, cast); carry.reader = reading; carry.since = reading.then_some(round); + self.dirty.insert((node, cast)); self.step(); } @@ -883,7 +973,7 @@ impl WarmSim { /// The session between `a` and `b` closes, and both notice. pub fn close(&mut self, a: Node, b: Node) { - self.open.remove(&pair(a, b)); + self.part(a, b); self.net.close(a, b); self.queues.retain(|&(from, to, ..), _| pair(from, to) != pair(a, b)); for (x, y) in [(a, b), (b, a)] { @@ -904,13 +994,16 @@ impl WarmSim { carry.heard.remove(&y); carry.advertised.remove(&y); } + self.stir(x); } self.step(); } pub fn reopen(&mut self, a: Node, b: Node) { - self.open.insert(pair(a, b)); + self.join(a, b); self.net.open(a, b); + self.stir(a); + self.stir(b); self.step(); } @@ -929,6 +1022,7 @@ impl WarmSim { ..Carry::default() }; } + self.stir(node); for peer in peers { self.reopen(node, peer); } @@ -960,9 +1054,10 @@ impl WarmSim { match queue { Queue::Route(from, to) => { self.net.deliver(from, to); + self.stir(to); } Queue::Data(key) => { - let Some((cast, msg)) = self.queues.get_mut(&key).and_then(VecDeque::pop_front) else { + let Some((cast, msg)) = self.pop(key) else { break; }; self.recv(key.0, key.1, cast, msg); @@ -979,6 +1074,16 @@ impl WarmSim { true } + /// The next message on a queue, dropping the queue once empty. + fn pop(&mut self, key: Key) -> Option<(usize, Msg)> { + let queue = self.queues.get_mut(&key)?; + let next = queue.pop_front(); + if queue.is_empty() { + self.queues.remove(&key); + } + next + } + /// Deliver in rounds until nothing is left. False if that never happens. pub fn settle(&mut self) -> bool { for _ in 0..2_000 { @@ -1006,14 +1111,11 @@ impl WarmSim { let ((from, to), _) = routes[at]; self.net.deliver(from, to); self.net.flush(); + self.stir(to); } false => { let key = data[at - routes.len()]; - let (cast, msg) = self - .queues - .get_mut(&key) - .and_then(VecDeque::pop_front) - .expect("a message"); + let (cast, msg) = self.pop(key).expect("a message"); self.recv(key.0, key.1, cast, msg); } } @@ -1121,14 +1223,24 @@ impl WarmSim { .collect() } - /// Count a cycle after a delivery. + /// Count a cycle after a delivery, looking again only at broadcasts whose pulls + /// changed. fn observe(&mut self) { self.watch.steps += 1; - let cycle = (0..self.casts.len()).any(|cast| (0..self.relays).any(|node| self.circles(node, cast))); - let starved = cycle && self.readers().into_iter().any(|(node, cast)| self.circles(node, cast)); - let merged = cycle - && (0..self.casts.len()) - .any(|cast| (0..self.relays).any(|node| self.circles(node, cast) && self.merged(node, cast))); + for cast in std::mem::take(&mut self.changed) { + let circling: Vec = (0..self.relays).filter(|&node| self.circles(node, cast)).collect(); + match circling.is_empty() { + true => self.circling.remove(&cast), + false => { + let starved = circling.iter().any(|&node| self.reading(node, cast)); + let merged = circling.iter().any(|&node| self.merged(node, cast)); + self.circling.insert(cast, (starved, merged)) + } + }; + } + let cycle = !self.circling.is_empty(); + let starved = self.circling.values().any(|&(starved, _)| starved); + let merged = self.circling.values().any(|&(_, merged)| merged); self.watch.cycles += usize::from(cycle); self.watch.merged += usize::from(merged); self.watch.starved += usize::from(starved); @@ -1154,10 +1266,11 @@ impl WarmSim { /// Count unfed readers as a round ends, and the first frames that arrived. fn tally(&mut self) { let round = self.round; - let mut reach: BTreeMap> = BTreeMap::new(); + let mut reach: BTreeMap> = BTreeMap::new(); for (node, cast) in self.readers() { let fed = self.fed(node, cast) && self.live(node, cast).is_some(); - let path = reach.entry(cast).or_insert_with(|| self.reach(cast)).contains(&node); + let start = self.endpoints[&self.casts[cast].0]; + let path = reach.entry(start).or_insert_with(|| self.reach(cast)).contains(&node); self.watch.unfed += usize::from(path && !fed); let carry = self.entry(node, cast); if fed && let Some(since) = carry.since.take() { @@ -1198,6 +1311,22 @@ impl WarmSim { Ok(()) } + /// The warm state `node` keeps about its neighbours across every broadcast: adverts + /// it heard and sent, or metadata-only subscriptions it holds and serves. + pub fn held(&self, node: Node) -> usize { + (self.carry[node].values()) + .map(|carry| carry.heard.len() + carry.advertised.len() + carry.asks.len() + carry.asked.len()) + .sum() + } + + /// The most relay neighbours any relay has. + pub fn degree(&self) -> usize { + (0..self.relays) + .map(|node| self.adjacent[node].iter().filter(|&&peer| self.relay(peer)).count()) + .max() + .unwrap_or(0) + } + /// Relays besides the publisher's pulling `cast` with no reader of their own: what /// other relays' joins cost them. pub fn carriers(&self, cast: usize) -> Vec { @@ -1236,10 +1365,11 @@ impl WarmSim { /// The cheapest tree joining `cast`'s publisher to every relay reading it, over open /// links at their cold prices: Dreyfus and Wagner's exact dynamic program. - pub fn cheapest(&self, cast: usize) -> Metric { - let endpoint = self.endpoints[&self.casts[cast].0]; - let mut terminals = vec![endpoint]; - terminals.extend((0..self.relays).filter(|&node| self.reading(node, cast))); + /// The cheapest cold cost between every two nodes over open links, through relays. + fn distances(&self) -> Vec> { + if let Some(dist) = self.distances.borrow().as_ref() { + return dist.clone(); + } let n = self.nodes; let mut dist = vec![vec![Metric::MAX / 4; n]; n]; for (node, row) in dist.iter_mut().enumerate() { @@ -1247,10 +1377,10 @@ impl WarmSim { } for &(a, b) in &self.open { let cost = self.price(a, b).cold; - // Only relays forward. dist[a][b] = dist[a][b].min(cost); dist[b][a] = dist[b][a].min(cost); } + // Only relays forward. for via in (0..n).filter(|&node| self.relay(node)) { for a in 0..n { for b in 0..n { @@ -1258,6 +1388,16 @@ impl WarmSim { } } } + self.distances.replace(Some(dist.clone())); + dist + } + + pub fn cheapest(&self, cast: usize) -> Metric { + let endpoint = self.endpoints[&self.casts[cast].0]; + let mut terminals = vec![endpoint]; + terminals.extend((0..self.relays).filter(|&node| self.reading(node, cast))); + let n = self.nodes; + let dist = self.distances(); let count = terminals.len(); let full = (1usize << count) - 1; let mut tree = vec![vec![Metric::MAX / 4; n]; full + 1]; diff --git a/rs/moq-net/src/route/tests/warmth.rs b/rs/moq-net/src/route/tests/warmth.rs index 43a0096cd03..5fe1fb6b067 100644 --- a/rs/moq-net/src/route/tests/warmth.rs +++ b/rs/moq-net/src/route/tests/warmth.rs @@ -21,7 +21,13 @@ fn arms() -> Vec { merge: false, }); } - for mode in [Mode::Ask(1), Mode::Ask(2), Mode::Ask(usize::MAX), Mode::Advertise] { + for mode in [ + Mode::Ask(1), + Mode::Ask(2), + Mode::Ask(usize::MAX), + Mode::Advertise(Audience::All), + Mode::Advertise(Audience::Useful), + ] { for hops in [true, false] { for merge in [false, true] { out.push(Arm { mode, hops, merge }); @@ -42,8 +48,9 @@ const fn arm(mode: Mode, merge: bool) -> Arm { const PATH: Arm = arm(Mode::Path, false); const ASK: Arm = arm(Mode::Ask(usize::MAX), false); const ASK_MERGE: Arm = arm(Mode::Ask(usize::MAX), true); -const ADVERTISE: Arm = arm(Mode::Advertise, false); -const ADVERTISE_MERGE: Arm = arm(Mode::Advertise, true); +const ADVERTISE: Arm = arm(Mode::Advertise(Audience::All), false); +const ADVERTISE_MERGE: Arm = arm(Mode::Advertise(Audience::All), true); +const USEFUL: Arm = arm(Mode::Advertise(Audience::Useful), false); /// A mesh of relays with one publisher. #[derive(Clone, Debug)] @@ -462,6 +469,228 @@ fn drones_take_over_a_lost_uplink() { } } +// --- Growth --- + +/// The cheaper ways to advertise that lost, measured beside the rest: telling only +/// once two others want it, and never over a link marked upstream. +fn audiences() -> Vec { + [Audience::Shared, Audience::Down] + .into_iter() + .map(|audience| arm(Mode::Advertise(audience), false)) + .collect() +} + +/// The arms swept for growth, every one with the path in its offers. +fn growing() -> Vec { + let mut arms = vec![ + PATH, + arm(Mode::Ask(1), false), + arm(Mode::Ask(2), false), + ASK, + ADVERTISE, + USEFUL, + ]; + arms.extend(audiences()); + arms +} + +/// A CDN tier: `cores` fully meshed, and `edges` edges per core, each joined to its +/// core and the next one over links marked upstream and meshed with the edges +/// beside it. The publisher joins core 0. +fn cdn(cores: usize, edges: usize, claims: usize) -> Shape { + let edge = |core: usize, index: usize| cores + core * edges + index; + let mut links = Vec::new(); + for a in 0..cores { + for b in a + 1..cores { + links.push((a, b, Price::new(6, 2), Tie::Peer)); + } + for index in 0..edges { + links.push((edge(a, index), a, Price::new(3, 1), Tie::Up)); + if cores > 1 { + links.push((edge(a, index), (a + 1) % cores, Price::new(3, 1), Tie::Up)); + } + for other in index + 1..edges { + links.push((edge(a, index), edge(a, other), Price::new(1, 1), Tie::Peer)); + } + } + } + Shape { + name: "cdn", + relays: cores * (1 + edges), + links, + home: 0, + claims, + } +} + +/// Who reads: one viewer per broadcast at a random reader relay, every one at once, +/// or a crowd of up to nine on each arriving and leaving in turn, as a crowd does. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Load { + OneViewer, + Crowd, +} + +impl Load { + fn name(self) -> &'static str { + match self { + Self::OneViewer => "one_viewer", + Self::Crowd => "crowd", + } + } + + /// Readers per broadcast, drawn from `relays`. + fn readers(self, relays: std::ops::Range, claims: usize, seed: u64) -> Vec<(Node, usize)> { + let mut rng = XorShift(seed); + let mut out = Vec::new(); + for cast in 0..claims { + let mut pool: Vec = relays.clone().collect(); + let count = match self { + Self::OneViewer => 1, + Self::Crowd => pool.len().min(9), + }; + for _ in 0..count { + let at = rng.next(pool.len() as u64) as usize; + out.push((pool.swap_remove(at), cast)); + } + } + out + } +} + +/// What one arm cost on one shape and load: every reader joins, then every one +/// leaves, each settled before the next when `in_turn`. +struct Grown { + changes: usize, + sent: Sent, + /// Warm state per relay with every reader in place: the mean and the most. + held: f64, + most: usize, + cost: Metric, + cheapest: Metric, + first: f64, + right: Result<(), String>, +} + +fn grow(shape: &Shape, readers: &[(Node, usize)], in_turn: bool, arm: Arm, cheapest: bool) -> Grown { + let mut sim = shape.build(arm, 11); + let (sent, waits) = (sim.sent, sim.waits.len()); + let mut settled = true; + for &(node, cast) in readers { + sim.read(node, cast, true); + if in_turn { + settled &= sim.settle(); + } + } + let mut right = match settled && sim.settle() { + true => sim.check(), + false => Err("never settled".into()), + }; + let held: Vec = (0..shape.relays).map(|node| sim.held(node)).collect(); + let casts = 0..sim.casts(); + let cost = casts.clone().map(|cast| sim.tree(cast).cost).sum(); + let cheapest = match cheapest { + true => casts.map(|cast| sim.cheapest(cast)).sum(), + false => 0, + }; + let first = mean(&sim.waits[waits..]); + for &(node, cast) in readers { + sim.read(node, cast, false); + if in_turn { + settled &= sim.settle(); + } + } + if right.is_ok() && !(settled && sim.settle()) { + right = Err("never settled after leaving".into()); + } + Grown { + changes: 2 * readers.len(), + sent: sim.sent - sent, + held: held.iter().sum::() as f64 / held.len() as f64, + most: held.into_iter().max().unwrap_or(0), + cost, + cheapest, + first, + right, + } +} + +/// A first viewer's adverts on CDN tiers: told to every neighbour they grow with +/// the core mesh, and told only where they could be taken they do not, since a core +/// with demand is never cheaper for another core than its own route. +#[test] +fn useful_adverts_do_not_grow_with_the_core_mesh() { + let warm = |arm: Arm, cores: usize| { + let shape = cdn(cores, 3, 1); + // The first edge under core 1, which the publisher's core is not. + let grown = grow(&shape, &[(cores + 3, 0)], false, arm, false); + assert_eq!(grown.right, Ok(()), "{}", arm.name()); + grown.sent.warm() + }; + assert_eq!(warm(USEFUL, 8), warm(USEFUL, 16)); + assert!(warm(ADVERTISE, 8) < warm(ADVERTISE, 16)); +} + +/// Warm messages and warm state as relays and demanded broadcasts grow, on CDN tiers +/// with their cores fully meshed and on drone meshes, for one viewer per broadcast +/// and for a crowd. `MOQ_WARMTH_GROWTH` caps the broadcasts. +#[test] +#[ignore = "a benchmark; Nightly runs it from test/route-sweep.sh"] +fn benchmark_warmth_growth() { + let most: usize = std::env::var("MOQ_WARMTH_GROWTH").map_or(4096, |most| most.parse().expect("MOQ_WARMTH_GROWTH")); + println!( + "shape,relays,neighbours,broadcasts,load,arm,reader_changes,messages,warm_messages,per_change_warm_messages,held_per_relay,held_at_most,tree_cost,cheapest_cost,first_frame_ms,right" + ); + let mut sizes: Vec<(Shape, std::ops::Range)> = Vec::new(); + for cores in [3, 8, 16, 32] { + let shape = cdn(cores, 3, 1); + sizes.push((shape, cores..cores * 4)); + } + for count in [5, 16, 64] { + sizes.push((drones(count), 1..count + 1)); + } + for (shape, readers) in sizes { + for load in [Load::OneViewer, Load::Crowd] { + let counts: &[usize] = match load { + Load::OneViewer => &[1, 16, 256, 4096], + Load::Crowd => &[1, 4, 16], + }; + for &claims in counts.iter().filter(|&&claims| claims <= most) { + let shape = Shape { + claims, + ..shape.clone() + }; + let reads = load.readers(readers.clone(), claims, 0x6772_6f77 + claims as u64); + let neighbours = shape.build(PATH, 11).degree(); + let mut cheapest = None; + for arm in growing() { + let grown = grow(&shape, &reads, load == Load::Crowd, arm, cheapest.is_none()); + let cheapest = *cheapest.get_or_insert(grown.cheapest); + println!( + "{},{},{neighbours},{claims},{},{},{},{},{},{:.2},{:.2},{},{},{cheapest},{:.0},{}", + shape.name, + shape.relays, + load.name(), + arm.name(), + grown.changes, + grown.sent.total(), + grown.sent.warm(), + grown.sent.warm() as f64 / grown.changes as f64, + grown.held, + grown.most, + grown.cost, + grown.first * HOP.as_secs_f64() * 1000.0, + match grown.right { + Ok(()) => "yes".into(), + Err(err) => err.replace(',', ";"), + }, + ); + } + } + } + } +} + // --- Churn --- /// What churn showed for one arm, summed over its cases. @@ -576,7 +805,7 @@ fn flying(rng: &mut XorShift) -> Shape { /// circling: the cycles it formed lasted only while messages were in flight. #[test] fn every_arm_settles_after_churn() { - for arm in [PATH, ASK, ASK_MERGE, ADVERTISE, ADVERTISE_MERGE] { + for arm in [PATH, ASK, ASK_MERGE, ADVERTISE, ADVERTISE_MERGE, USEFUL] { for shape in [random as fn(&mut XorShift) -> Shape, flying] { let out = churn(arm, 0..12, shape); assert_eq!(out.failed, 0, "{}: {:?}", arm.name(), out.first); @@ -725,7 +954,7 @@ fn benchmark_warmth_cases() { let ms = |rounds: f64| rounds * HOP.as_secs_f64() * 1000.0; for base in cases().into_iter().map(Some).chain([None]) { let mut path_cost = None; - for arm in arms() { + for arm in arms().into_iter().chain(audiences()) { let case = base.as_ref().map_or_else(|| takeover(arm), |case| case.clone()); let rows = run(&case.shape, arm, &case.phases, 7); let sum = |field: fn(&Row) -> usize| rows.iter().map(field).sum::(); @@ -772,7 +1001,7 @@ fn benchmark_warmth_churn() { ); let cases: usize = std::env::var("MOQ_WARMTH_CASES").map_or(400, |cases| cases.parse().expect("MOQ_WARMTH_CASES")); for (name, shape) in [("random", random as fn(&mut XorShift) -> Shape), ("drones", flying)] { - for arm in arms() { + for arm in arms().into_iter().chain(audiences()) { let out = churn(arm, 0..cases, shape); println!( "{name},{},{},{},{},{},{},{},{},{},{},{},{},{},{}", diff --git a/test/route-sweep.sh b/test/route-sweep.sh index c3f9c13702a..8c241d131b6 100755 --- a/test/route-sweep.sh +++ b/test/route-sweep.sh @@ -18,5 +18,5 @@ just rs test --profile=sweep -p moq-net --release --run-ignored only -E 'test(/^ MOQ_ROUTE_CASES=3000 just rs test -P sweep -p moq-net --release --run-ignored only -E 'test(=route::tests::cache::sweep_cache_routing_long)' # The multicast bake-off: its prototype carries every failure on every shape at 32 relays, and its tables against the path rule at 12. MOQ_BAKEOFF_RELAYS=12 just rs test --profile=sweep -p moq-net --release --run-ignored only -E 'test(/^route::tests::bakeoff::(sweep|benchmark)_/)' -# Warmth: path joins, asking, and advertising, each over fixed shapes and 400 churned meshes per arm, about 30 seconds. +# Warmth: path joins, asking, and advertising over fixed shapes, 400 churned meshes per arm, and growth to 128 relays and 4096 broadcasts, about 6 minutes. just rs test --profile=sweep -p moq-net --release --run-ignored only -E 'test(/^route::tests::warmth::benchmark_/)' From df19e53843f86329fd824f1ec2647248a2993970 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 07:07:45 -0800 Subject: [PATCH 05/19] fix(moq-net): warmth sim prices useful adverts from heard routes and keeps each subscription its own stream The useful audience now reads only the route a neighbour last offered the relay: a neighbour offers nothing back over a route through the relay, so an unheard one is priced at the relay's own route and the link. Each payload subscription and metadata-only query is a stream of its own, so a reopened one can overtake the old one's close, and receivers track them by identity. The tables are regenerated; the recommendation and the race verdict stand. Co-Authored-By: Claude Opus 5.5 --- quest/m3/cache-routing/select.md | 22 ++-- rs/moq-net/src/route/tests/multi.rs | 9 ++ rs/moq-net/src/route/tests/warm.rs | 151 ++++++++++++++++++---------- 3 files changed, 117 insertions(+), 65 deletions(-) diff --git a/quest/m3/cache-routing/select.md b/quest/m3/cache-routing/select.md index f7b574dca50..6c7b91c8332 100644 --- a/quest/m3/cache-routing/select.md +++ b/quest/m3/cache-routing/select.md @@ -32,7 +32,7 @@ standing**. Rejected: doubled the time to a first frame and left about twice the readers unfed under churn. On CDN tiers `ask_1` sent 6.5 to 11 warm messages per one-viewer reader change against the pick's 3 to 7, about as many for a - crowd (3.9 to 8.1 against 5.4 to 7.9), and built dearer crowd trees. + crowd (3.9 to 8.1 against 5.8 to 8), and built dearer crowd trees. - Advertising only once two others want the copy, which lost both cases; never advertising over a link marked upstream, which saves nothing in a core mesh. @@ -49,9 +49,9 @@ sweep this ports onto production code. otherwise `O(P→R) + cold(P, R)`, the chain priced unshared. The publisher's relay offers 0. An offer is never below its sender's own cost. - **The path.** Every offer names the relays the data passes from the - publisher's relay to the sender. Without it, 1 of 400 churned random meshes - never settled, and transient cycles rose by 74% on random meshes and 25% on - drone meshes. The merge below cannot work without it either. + publisher's relay to the sender. Without it, transient cycles under churn + rose by 73% on random meshes and 30% on drone meshes, and the merge below + cannot work. - **Adverts.** While R has demand for an Instance besides N, it tells N its offer and path on the announce stream when `O(R→N) + cold(R, N)` is below N's route cost for the Instance, or equals it through a next hop other than @@ -93,7 +93,7 @@ each prefer the other on current offers, and a cycle needs one stale offer. Gaining demand only lowers offers to others; losing it raises only the offer to the neighbour that still pulls, which then moves only to something strictly cheaper. The sweep agrees: every churned mesh settled, and cycles -lasted at most 42 deliveries, each while some answer was in flight. +lasted at most 57 deliveries, each while some answer was in flight. ### Simultaneous starts @@ -102,11 +102,11 @@ neither offers the other anything cheaper, and both uplinks stay until one drone's demand ends. The maintainer's rule, decided 2026-11-15: merge them on the rendezvous hash unless that adds a cycle, an unfed reader, or more messages than the second uplink costs. Under adverts it does add cycles: -in 400 churned drone meshes, 41 deliveries had a pull the merge made inside -a cycle (53 when advertising to every neighbour). Each was a pending cycle. -A merged drone's new parent lost its uplink and rejoined through a drone -that had trusted the merged drone's advert, before the lost parent's answer -arrived. The merge also added about 2% unfed reader hops. Under asking on +in 400 churned drone meshes, 145 deliveries had a pull the merge made inside +a cycle (8 when advertising to every neighbour), all transient. The one +traced was a pending cycle: a merged drone's new parent lost its uplink and +rejoined through a drone that had trusted the merged drone's advert, before +the lost parent's answer arrived. The merge also added about 2% unfed reader hops. Under asking on subscribe, no merged pull was ever in a cycle. So the race stands. A drone that loses the one uplink leaves every drone @@ -145,7 +145,7 @@ path in each offer. 2 to 3. - Port the growth sweep, and hold its laws. Per reader change, warm messages are flat in the number of demanded broadcasts for every arm. On CDN tiers - with a full core mesh they are also flat in cores: 3 to 7.9 from 12 to 128 + with a full core mesh they are also flat in cores: 3 to 8 from 12 to 128 relays and 1 to 4,096 broadcasts. A core with demand is never cheaper for another core than that core's own route, so only the edges beside it hear. On a full drone mesh they grow with the drones, n − 1 per first diff --git a/rs/moq-net/src/route/tests/multi.rs b/rs/moq-net/src/route/tests/multi.rs index fac6284b56e..33ae83c94f5 100644 --- a/rs/moq-net/src/route/tests/multi.rs +++ b/rs/moq-net/src/route/tests/multi.rs @@ -1087,6 +1087,15 @@ impl Router { } } + /// What `via` last offered this node toward `broadcast`'s closest claim: its own + /// summed cost to the broadcast, and whether its route runs through this node. + /// `None` when it offered nothing. + pub fn heard(&self, broadcast: &Broadcast, via: N) -> Option<(Metric, bool)> { + let (origin, claim) = self.closest(broadcast)?; + let heard = self.heard.get(&origin)?.get(&via)?; + Some((heard.offer.cost + claim.cost, heard.offer.names(self.me))) + } + /// Every neighbour whose route to `origin` ranks with the one in use: the equal /// next hops a request may take. pub fn ties(&self, origin: Id) -> Vec { diff --git a/rs/moq-net/src/route/tests/warm.rs b/rs/moq-net/src/route/tests/warm.rs index 013efee32e2..63cea17c8f4 100644 --- a/rs/moq-net/src/route/tests/warm.rs +++ b/rs/moq-net/src/route/tests/warm.rs @@ -61,7 +61,8 @@ pub(super) enum Audience { /// Only a neighbour for whom its offer plus the link beats that neighbour's route, /// or ties it through another next hop, since a tie prefers a warm copy. Nothing /// else could ever take it: a relay at rest costs no more than its own route, and - /// a tie through this relay is the path join already. + /// a tie through this relay is the path join already. The neighbour's route is + /// the one it last offered this relay, never its current one. Useful, /// Only once it has demand from two sources besides the neighbour. Shared, @@ -155,9 +156,15 @@ enum Msg { offer: Option, }, /// The metadata-only subscription that asks, opening or closing. - Ask(bool), + Ask { + stream: u64, + open: bool, + }, /// The asked neighbour's word and each change to it. - Told(Option), + Told { + stream: u64, + word: Option, + }, Advert(Option), } @@ -170,7 +177,8 @@ enum Lane { Announce, } -/// A queue: sender, receiver, lane, and broadcast. +/// A queue: sender, receiver, lane, and the stream: a subscription's or query's own, +/// or `usize::MAX` for the session's announce stream. type Key = (Node, Node, Lane, usize); /// One relay's state for one broadcast. @@ -179,19 +187,23 @@ struct Carry { reader: bool, /// The round its reader started, until its first frame. since: Option, - /// Neighbours pulling through this relay, by stream. - down: BTreeMap, + /// Neighbours pulling through this relay, and their subscriptions: a neighbour's + /// new one may arrive before its old one closes. + down: BTreeMap>, up: Option, /// A move under way: it replaces `up` once answered. next: Option, - /// What each neighbour pulling through it was last answered. - answered: BTreeMap>, + /// What each subscription through it was last answered. + answered: BTreeMap<(Node, u64), Option>, /// Neighbours it asks, with each one's latest word. asks: BTreeMap>, + /// The query it holds open to each. + asking: BTreeMap, /// Those yet to answer: a relay with no pull waits for them before it starts one. unanswered: BTreeSet, - asked: BTreeSet, - told: BTreeMap>, + /// Neighbours asking it, and their queries. + asked: BTreeMap>, + told: BTreeMap<(Node, u64), Option>, heard: BTreeMap, advertised: BTreeMap, } @@ -318,6 +330,16 @@ fn kept() -> Tune { } } +/// Close `stream` from `peer`, forgetting `peer` once none is left. +fn close(streams: &mut BTreeMap>, peer: Node, stream: u64) { + if let Some(open) = streams.get_mut(&peer) { + open.remove(&stream); + if open.is_empty() { + streams.remove(&peer); + } + } +} + fn pair(a: Node, b: Node) -> (Node, Node) { (a.min(b), a.max(b)) } @@ -488,14 +510,17 @@ impl WarmSim { match &msg { Msg::Subscribe { .. } => self.sent.subscribes += 1, Msg::Answer { .. } => self.sent.answers += 1, - Msg::Ask(_) => self.sent.asks += 1, - Msg::Told(_) => self.sent.told += 1, + Msg::Ask { .. } => self.sent.asks += 1, + Msg::Told { .. } => self.sent.told += 1, Msg::Advert(_) => self.sent.adverts += 1, } // The announce stream is one per session, whatever the broadcast. - let stream = match lane { - Lane::Announce => usize::MAX, - Lane::Pull | Lane::Ask => cast, + let stream = match &msg { + Msg::Subscribe { stream, .. } + | Msg::Answer { stream, .. } + | Msg::Ask { stream, .. } + | Msg::Told { stream, .. } => *stream as usize, + Msg::Advert(_) => usize::MAX, }; self.queues .entry((from, to, lane, stream)) @@ -639,7 +664,7 @@ impl WarmSim { // Asked with no demand of its own, it forwards the subscription along its route. _ if !carry.asked.is_empty() => (self.route(node, cast).into_iter()) .map(|(next, _)| next) - .filter(|&next| self.relay(next) && !carry.asked.contains(&next)) + .filter(|&next| self.relay(next) && !carry.asked.contains_key(&next)) .collect(), _ => BTreeSet::new(), }; @@ -647,13 +672,18 @@ impl WarmSim { let carry = self.entry(node, cast); carry.asks.remove(&gone); carry.unanswered.remove(&gone); - self.send(node, gone, Lane::Ask, cast, Msg::Ask(false)); + if let Some(stream) = carry.asking.remove(&gone) { + self.send(node, gone, Lane::Ask, cast, Msg::Ask { stream, open: false }); + } } for &new in want.iter().filter(|peer| !carry.asks.contains_key(peer)) { + self.streams += 1; + let stream = self.streams; let carry = self.entry(node, cast); carry.asks.insert(new, None); + carry.asking.insert(new, stream); carry.unanswered.insert(new); - self.send(node, new, Lane::Ask, cast, Msg::Ask(true)); + self.send(node, new, Lane::Ask, cast, Msg::Ask { stream, open: true }); } } @@ -818,28 +848,34 @@ impl WarmSim { /// Tell each neighbour pulling through `node` what it offers now. fn answer(&mut self, node: Node, cast: usize) { let carry = self.entry(node, cast).clone(); - for (&child, &stream) in &carry.down { + for (&child, streams) in &carry.down { let offer = self.offer(node, cast, child); - let was = carry.answered.get(&child); - // Unanswered until confirmed, then every change. - if (was.is_none() && offer.is_none()) || was == Some(&offer) { - continue; + for &stream in streams { + let was = carry.answered.get(&(child, stream)); + // Unanswered until confirmed, then every change. + if (was.is_none() && offer.is_none()) || was == Some(&offer) { + continue; + } + self.entry(node, cast).answered.insert((child, stream), offer.clone()); + let offer = offer.clone(); + self.send(node, child, Lane::Pull, cast, Msg::Answer { stream, offer }); } - self.entry(node, cast).answered.insert(child, offer.clone()); - self.send(node, child, Lane::Pull, cast, Msg::Answer { stream, offer }); } } /// Answer each neighbour asking `node`, and each change. fn tell(&mut self, node: Node, cast: usize) { let carry = self.entry(node, cast).clone(); - for &asker in &carry.asked { + for (&asker, streams) in &carry.asked { let word = self.warm(node, cast, asker); - if carry.told.get(&asker) == Some(&word) { - continue; + for &stream in streams { + if carry.told.get(&(asker, stream)) == Some(&word) { + continue; + } + self.entry(node, cast).told.insert((asker, stream), word.clone()); + let word = word.clone(); + self.send(node, asker, Lane::Ask, cast, Msg::Told { stream, word }); } - self.entry(node, cast).told.insert(asker, word.clone()); - self.send(node, asker, Lane::Ask, cast, Msg::Told(word)); } } @@ -869,13 +905,17 @@ impl WarmSim { fn scoped(&self, audience: Audience, node: Node, cast: usize, peer: Node, signal: &Signal) -> bool { match audience { Audience::All => true, - Audience::Useful => match self.route(peer, cast) { - Some((via, cost)) => { - let price = signal.cost + self.price(node, peer).cold; - price < cost || (price == cost && via != node) + Audience::Useful => { + let price = signal.cost + self.price(node, peer).cold; + let router = self.net.router(node); + match router.and_then(|router| router.heard(&self.casts[cast], peer)) { + Some((cost, false)) => price <= cost, + // Its route runs through this relay, which a neighbour offers nobody + // back: it costs this relay's route and the link, so only an offer + // below that route can beat it. + _ => self.route(node, cast).is_none_or(|(_, cost)| signal.cost < cost), } - None => true, - }, + } Audience::Shared => self.carry(node, cast).is_some_and(|carry| { let children = carry.down.keys().filter(|&&child| child != peer).count(); usize::from(carry.reader) + children >= 2 @@ -914,14 +954,11 @@ impl WarmSim { let carry = self.entry(to, cast); match msg { Msg::Subscribe { stream, open: true } => { - carry.down.insert(from, stream); - carry.answered.remove(&from); + carry.down.entry(from).or_default().insert(stream); } Msg::Subscribe { stream, open: false } => { - if carry.down.get(&from) == Some(&stream) { - carry.down.remove(&from); - carry.answered.remove(&from); - } + close(&mut carry.down, from, stream); + carry.answered.remove(&(from, stream)); } Msg::Answer { stream, offer } => { for pull in [&mut carry.up, &mut carry.next].into_iter().flatten() { @@ -933,15 +970,17 @@ impl WarmSim { } } } - Msg::Ask(true) => { - carry.asked.insert(from); + Msg::Ask { stream, open: true } => { + carry.asked.entry(from).or_default().insert(stream); } - Msg::Ask(false) => { - carry.asked.remove(&from); - carry.told.remove(&from); + Msg::Ask { stream, open: false } => { + close(&mut carry.asked, from, stream); + carry.told.remove(&(from, stream)); } - Msg::Told(word) => { - if let Some(heard) = carry.asks.get_mut(&from) { + Msg::Told { stream, word } => { + if carry.asking.get(&from) == Some(&stream) + && let Some(heard) = carry.asks.get_mut(&from) + { *heard = word; carry.unanswered.remove(&from); } @@ -986,11 +1025,12 @@ impl WarmSim { carry.next = None; } carry.down.remove(&y); - carry.answered.remove(&y); + carry.answered.retain(|&(child, _), _| child != y); carry.asks.remove(&y); carry.unanswered.remove(&y); + carry.asking.remove(&y); carry.asked.remove(&y); - carry.told.remove(&y); + carry.told.retain(|&(asker, _), _| asker != y); carry.heard.remove(&y); carry.advertised.remove(&y); } @@ -1161,9 +1201,12 @@ impl WarmSim { let Some(pull) = self.carry(at, cast).and_then(|carry| carry.up.as_ref()) else { return false; }; - let held = self - .carry(pull.via, cast) - .is_some_and(|carry| carry.down.get(&at) == Some(&pull.stream)); + let held = self.carry(pull.via, cast).is_some_and(|carry| { + carry + .down + .get(&at) + .is_some_and(|streams| streams.contains(&pull.stream)) + }); if !held || !self.is_open(at, pull.via) { return false; } From b7e6742d7ac043e92a1b4d53ad22fe5340174b66 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 09:25:21 -0800 Subject: [PATCH 06/19] feat(moq-net): sweep warmth over edges per core, keep adverts off the edges, and name the relays a pull feeds The growth sweep gains the edges-per-core axis (3 to 128 edges per core at 8 and 32 cores, edges peered or not), and useful adverts grow with it: a warm core tells every edge it could feed. Of the variants that keep adverts off marked links, only the one where an edge asks its other upstreams on subscribe keeps the unequal region on the cheapest tree, so it is the pick: flat in edges and cores, path joins' first frame. A new churned mesh, dual-homed regions on marked links, found a pull that never settles under every arm that offers a cost: a relay's own pull through another neighbour made the parent's offer to it warm. Each pull now names the relays whose demand alone it carries, and an offer to N leaves those out. The checker's reachability now follows the route layer's valley-free rule. Co-Authored-By: Claude Opus 5.5 --- quest/m3/cache-routing/select.md | 150 +++++++----- rs/moq-net/src/route/tests/warm.rs | 330 ++++++++++++++++++++------- rs/moq-net/src/route/tests/warmth.rs | 316 ++++++++++++++++++++----- test/route-sweep.sh | 2 +- 4 files changed, 607 insertions(+), 191 deletions(-) diff --git a/quest/m3/cache-routing/select.md b/quest/m3/cache-routing/select.md index 6c7b91c8332..d4255e597e0 100644 --- a/quest/m3/cache-routing/select.md +++ b/quest/m3/cache-routing/select.md @@ -3,7 +3,8 @@ ## Goal A relay with demand for an Instance advertises it on the announce stream to -each neighbour that could take it, and each relay pulls every demanded broadcast from the +each peer or upstream that could take it, an edge asks its other upstreams +when it gains demand, and each relay pulls every demanded broadcast from the cheapest copy of its Instance, moving an existing pull only when the route it uses gets worse or ends, or another is strictly cheaper in summed cost. A peer that sends no warm signal is treated as cold and behaves as it does today. @@ -19,23 +20,33 @@ Plan, is replaced. Recommended 2026-11-16 on [moq-dev/moq#8903](https://github.com/moq-dev/moq/pull/8903), awaiting the -maintainer's confirmation at merge: **advertise only to a neighbour that -could take the copy, with the data path in every offer, and the race left -standing**. Rejected: - -- Advertising to every neighbour, which grows with the neighbours of every - relay on the warm chain: 11 to 76 warm messages per one-viewer reader - change as a full core mesh grows from 3 to 32 cores. +maintainer's confirmation at merge: **adverts between peers and toward +upstreams, only where the copy could be taken; never down a link the +neighbour marked upstream, where the edge instead asks its other upstreams on +subscribe; the data path in every offer, the relays a pull feeds in every +subscription, and the race left standing**. Rejected: + +- Advertising to every neighbour that could take the copy, down marked links + too: it grows with a region's edges, since a warm core tells every edge it + could feed. 1.7 to 118 warm messages per one-viewer reader change as edges + per core grow from 3 to 128, against the pick's flat 1.6 to 1.8. +- Advertising to every neighbour, which also grows with the neighbours of + every relay on the warm chain: 9 to 66 per one-viewer reader change as a + full core mesh grows from 4 to 32 cores. +- Keeping adverts off the edges without the ask (never to a neighbour whose + route runs through the relay, never down a marked link, or asking only + between upstreams that tie): flat, but an edge whose cheaper uplink is to + the cold core never hears of the warm one, 29 against 24 on the unequal + region. - Asking the k cheapest neighbours on each SUBSCRIBE. It stays bounded per subscribe only because it stops finding the copy: on a full drone mesh past 5 drones, `ask_1` shared no uplink at 64 drones and `ask_2` few. It doubled the time to a first frame and left about twice the readers unfed - under churn. On CDN tiers `ask_1` sent 6.5 to 11 warm messages per - one-viewer reader change against the pick's 3 to 7, about as many for a - crowd (3.9 to 8.1 against 5.8 to 8), and built dearer crowd trees. + under churn. On CDN tiers `ask_1` sent 5.7 to 7 warm messages per + one-viewer reader change against the pick's 1.6 to 2.1. - Advertising only once two others want the copy, which lost both cases; - never advertising over a link marked upstream, which saves nothing in a - core mesh. + never advertising over a link the sender marked upstream, which saves + nothing in a core mesh. - Joins along the path only, the cost baseline, which delivers neither case. That PR's simulator, `rs/moq-net/src/route/tests/warm.rs`, runs every rule @@ -52,16 +63,32 @@ sweep this ports onto production code. publisher's relay to the sender. Without it, transient cycles under churn rose by 73% on random meshes and 30% on drone meshes, and the merge below cannot work. -- **Adverts.** While R has demand for an Instance besides N, it tells N its - offer and path on the announce stream when `O(R→N) + cold(R, N)` is below - N's route cost for the Instance, or equals it through a next hop other than - R, since a tie prefers a warm copy. It tells again on each change, and - withdraws when the demand, its upstream, or that test ends. Nothing it - skips could ever be taken: a relay at rest costs no more than its own - route, and a tie through R is the path join already. R knows N's route cost - from what N announces it, and computes it when N's route runs through R. - Skipping the parent and children by rule instead left 8 of 400 churned - drone meshes unsettled. +- **Adverts.** While R has demand for an Instance besides N, and N has not + marked R upstream, it tells N its offer and path on the announce stream + when `O(R→N) + cold(R, N)` is below N's route cost for the Instance, or + equals it through a next hop other than R, since a tie prefers a warm copy. + It tells again on each change, and withdraws when the demand, its upstream, + or that test ends. Nothing it skips between peers could ever be taken: a + relay at rest costs no more than its own route, and a tie through R is the + path join already. R knows N's route cost from what N announces it, and + computes it when N's route runs through R. Skipping the parent and children + by rule instead left 8 of 400 churned drone meshes unsettled. +- **Asks.** A relay with more than one upstream asks each upstream besides + the one its path join takes, on a metadata-only subscription opened when + it gains demand and closed when the demand ends. It does not wait for the + answers: it joins along its route at once and moves if an answer is + strictly cheaper. The asked upstream answers its offer and path, and each + change. An upstream cannot tell which of its upstreams a dual-homed edge's + route uses, since the edge announces it nothing, so only the edge can find + a warm core beside its route. +- **The pull's chain.** Every subscription names the relays whose demand + alone it carries: the subscriber, then, while the subscriber has no reader + and one neighbour pulling through it, that neighbour's chain. It updates + the chain on each change. A relay's demand besides N leaves out every pull + whose chain names N. Without it N's own pull, through another neighbour, + makes R's offer to N warm; N moves to R, the other pull ends, R's offer + turns cold, and N moves back for good. One of 400 churned regions never + settled for that reason, under the pick and most other warm arms. - **Answers.** The subscribe answer carries the parent's offer and path to the subscriber, and an update on each change. A subscriber waits for that answer before it counts the pull as live. @@ -90,10 +117,11 @@ Why no cycle outlasts the messages in flight: a relay's cost is its parent's offer plus a warm price of at least 1, and no offer is below its sender's cost, so cost rises strictly along every pull chain. Two relays can never each prefer the other on current offers, and a cycle needs one stale offer. -Gaining demand only lowers offers to others; losing it raises only the offer -to the neighbour that still pulls, which then moves only to something -strictly cheaper. The sweep agrees: every churned mesh settled, and cycles -lasted at most 57 deliveries, each while some answer was in flight. +Gaining demand lowers offers only to neighbours outside the chain that +brought it; losing it raises only the offer to the neighbour that still +pulls, which then moves only to something strictly cheaper. The sweep agrees: +every churned random mesh, drone mesh, and region settled, and cycles lasted +at most 38 deliveries under the pick, each while some answer was in flight. ### Simultaneous starts @@ -102,15 +130,15 @@ neither offers the other anything cheaper, and both uplinks stay until one drone's demand ends. The maintainer's rule, decided 2026-11-15: merge them on the rendezvous hash unless that adds a cycle, an unfed reader, or more messages than the second uplink costs. Under adverts it does add cycles: -in 400 churned drone meshes, 145 deliveries had a pull the merge made inside -a cycle (8 when advertising to every neighbour), all transient. The one -traced was a pending cycle: a merged drone's new parent lost its uplink and -rejoined through a drone that had trusted the merged drone's advert, before -the lost parent's answer arrived. The merge also added about 2% unfed reader hops. Under asking on -subscribe, no merged pull was ever in a cycle. +in 400 churned drone meshes, 20 deliveries had a pull the merge made inside +a cycle, all transient. The one traced was a pending cycle: a merged +drone's new parent lost its uplink and rejoined through a drone that had +trusted the merged drone's advert, before the lost parent's answer arrived. +The merge also added about 1% unfed reader hops. Under asking on subscribe, +no merged pull was ever in a cycle. So the race stands. A drone that loses the one uplink leaves every drone -it fed taking its own uplink at once: 4 uplinks on the four-drone shape, +it fed taking its own uplink at once: 4 uplinks on the five-drone shape, 1 with the merge. If the maintainer takes the merge instead, it is: among equal-cost copies a drone can reach, it joins the neighbour with the higher rendezvous hash whose offer equals its own cost, once that leaves it no @@ -137,24 +165,39 @@ path in each offer. route layer's hop limit, and adverts for anything else dropped. Adverts are per broadcast like the claims the announce stream already carries over every link, so a relay hears at most one per neighbour per demanded - broadcast. + broadcast. An upstream holds at most one ask per downstream per demanded + broadcast, and a pull's chain is no longer than the hop limit. - Check the Goal's efficiency against plain path routing on the same shapes. On the simulator: pops 61 against 69 (cheapest tree 59), drones in turn one - uplink against five, a ring of 48 relays 8 uplinks against 48, tiers 1115 - against 1191 (cheapest 1095), at 5 to 9 messages per reader change against - 2 to 3. -- Port the growth sweep, and hold its laws. Per reader change, warm messages - are flat in the number of demanded broadcasts for every arm. On CDN tiers - with a full core mesh they are also flat in cores: 3 to 8 from 12 to 128 - relays and 1 to 4,096 broadcasts. A core with demand is never cheaper - for another core than that core's own route, so only the edges beside it - hear. On a full drone mesh they grow with the drones, n − 1 per first - viewer (63 at 64 drones), since every drone could take the copy; that is - what buys the cheapest tree there (432 against 1,328 to 1,456 for asking - and path joins, a crowd of 9 on 16 broadcasts). At rest the busiest relay - of 128 holds 755 adverts for 4,096 one-viewer broadcasts, against 5,405 - metadata-only subscriptions for `ask_1` and 151,552 adverts sent to every - neighbour. + uplink against five, a ring of 48 relays 8 uplinks against 48, tiers 1095 + against 1191 (the cheapest), and an edge beside the warm core 24 against + 29, at 5 to 16.5 messages per reader change against 2 to 5.4. +- Port the growth sweep, and hold its laws, per link kind. Per reader change, + warm messages are flat in the number of demanded broadcasts for every arm. + - Between cores, a full peer mesh: flat in cores. A core with demand is + never cheaper for another core than that core's own route. + - Down a marked link: none. Up it: one ask per upstream beyond the first, + with its answer, flat in edges per core. Together 1.6 to 2.1 per + one-viewer reader change and 3.7 to 4.2 for a crowd, from 4 to 32 cores + and 3 to 128 edges per core, up to 2,080 relays. + - Between edges or drones that peer: n − 1 per first viewer (63 at 64 + drones), since every one could take the copy; that is what buys the + cheapest tree there (432 against 1,328 to 1,456 for asking and path + joins, a crowd of 9 on 16 broadcasts). + At rest the busiest relay of 80 holds 518 adverts and asks for 4,096 + one-viewer broadcasts, against 4,355 metadata-only subscriptions for + `ask_1` and 139,264 adverts sent to every neighbour. +- A region's edges do not peer every pair as it grows: they reach each other + through the region's cores. Peered, the region is a drone mesh and pays its + n − 1 per first viewer (65 at 64 edges per core); the tree it buys there, + 288 against 544 unpeered for a crowd, comes from pricing an edge-to-edge + link at a third of an uplink. +- Port the cases as acceptance tests, two of them new: drones starting at + once, whose uplinks the race keeps, and the same sweeps with each link + delivering in order at its own pace, rather than one control message per + delivery step across the mesh, which is what the simulator's unfed counts + and first frames assume. A first frame matches the path join's when readers + arrive in turn; when two edges start at once it took 262 ms against 150. - A far pop keeps pulling through a nearer pop after that pop's reader leaves, while the nearer pop's unshared offer plus the warm link stays below the far pop's own long link. That is the cheaper tree, and the @@ -173,9 +216,10 @@ then-current wip version with this. Public API: report any relay configuration added, such as a warm cost per link. Wire, in the then-current wip version and all additive: an advert on -the announce stream carrying an Instance's offer and path, and the same -offer and path on the subscribe answer and its updates. Nothing lite-07 -freezes changes. +the announce stream carrying an Instance's offer and path; the same offer and +path on the subscribe answer and its updates, and on the answer to a +metadata-only subscription; and the chain of relays a subscription feeds, on +the subscribe and an update. Nothing lite-07 freezes changes. ## Required diff --git a/rs/moq-net/src/route/tests/warm.rs b/rs/moq-net/src/route/tests/warm.rs index 63cea17c8f4..e7733b18766 100644 --- a/rs/moq-net/src/route/tests/warm.rs +++ b/rs/moq-net/src/route/tests/warm.rs @@ -27,9 +27,8 @@ //! ([`super::compare::Arm::KEPT`]), its messages interleaved with these at random. use std::{ - cell::RefCell, cmp::Reverse, - collections::{BTreeMap, BTreeSet, VecDeque}, + collections::{BTreeMap, BTreeSet, BinaryHeap, VecDeque}, }; use super::cache::rendezvous; @@ -68,6 +67,19 @@ pub(super) enum Audience { Shared, /// Never over a link marked upstream. Down, + /// As [`Audience::Useful`], but never to a neighbour whose route runs through this + /// relay: its path join lands here anyway, though it then prices this relay cold + /// against another warm neighbour. + NotNext, + /// As [`Audience::Useful`], but never over a link the neighbour marked upstream: an + /// edge relies on its path join and the rendezvous hash among equal upstreams. + NotDown, + /// As [`Audience::NotDown`], and a relay with demand whose route ties between + /// upstreams asks them on subscribe, without waiting for their answers. + NotDownAsk, + /// As [`Audience::NotDown`], and a relay with demand asks every upstream besides the + /// one its path join takes, on subscribe, without waiting for their answers. + NotDownAskUp, } /// One way to find a warm copy, as measured. @@ -92,6 +104,10 @@ impl Arm { Mode::Advertise(Audience::Useful) => "advertise_useful".into(), Mode::Advertise(Audience::Shared) => "advertise_shared".into(), Mode::Advertise(Audience::Down) => "advertise_down".into(), + Mode::Advertise(Audience::NotNext) => "advertise_not_next".into(), + Mode::Advertise(Audience::NotDown) => "advertise_not_down".into(), + Mode::Advertise(Audience::NotDownAsk) => "advertise_not_down_ask".into(), + Mode::Advertise(Audience::NotDownAskUp) => "advertise_not_down_ask_up".into(), }; let hops = if self.hops { "+hops" } else { "" }; let merge = if self.merge { "+merge" } else { "" }; @@ -142,13 +158,23 @@ struct Pull { state: State, /// The merge made it, so a cycle through it is one the merge added. merged: bool, + /// The relays whose demand alone it carries, as last told the parent. + chain: Vec, } #[derive(Clone, Debug)] enum Msg { + /// A payload subscription opening, with the relays whose demand alone it carries, + /// or closing. Subscribe { stream: u64, open: bool, + chain: Vec, + }, + /// A change to the relays whose demand alone a payload subscription carries. + Chain { + stream: u64, + chain: Vec, }, /// A parent's answer and each change to it: its offer, or that it lost its upstream. Answer { @@ -195,6 +221,8 @@ struct Carry { next: Option, /// What each subscription through it was last answered. answered: BTreeMap<(Node, u64), Option>, + /// The relays whose demand alone each subscription through it carries. + chains: BTreeMap<(Node, u64), Vec>, /// Neighbours it asks, with each one's latest word. asks: BTreeMap>, /// The query it holds open to each. @@ -213,9 +241,36 @@ impl Carry { self.reader || !self.down.is_empty() } - /// Demand besides `peer`'s. + /// Demand besides `peer`'s, counting a pull that carries only demand passing + /// through `peer` as `peer`'s: otherwise `peer`'s own pull through another neighbour + /// would make this relay's offer to `peer` warm, and `peer` would move here and back + /// for good. fn others(&self, peer: Node) -> bool { - self.reader || self.down.keys().any(|&child| child != peer) + self.reader + || (self.down.iter()).any(|(&child, streams)| { + child != peer + && (streams.iter()).any(|&stream| { + !self + .chains + .get(&(child, stream)) + .is_some_and(|chain| chain.contains(&peer)) + }) + }) + } + + /// The relays whose demand alone `node`'s pull carries: each below it with one + /// neighbour pulling and no reader, down to the first with a reader or more than one, + /// then `node`. + fn chain(&self, node: Node) -> Vec { + let mut children = self.down.iter(); + let mut chain = match (self.reader, children.next(), children.next()) { + (false, Some((&child, streams)), None) => (streams.last()) + .and_then(|&stream| self.chains.get(&(child, stream)).cloned()) + .unwrap_or_else(|| vec![child]), + _ => Vec::new(), + }; + chain.push(node); + chain } } @@ -231,16 +286,18 @@ pub(super) struct Sent { /// Their answers and updates. pub told: usize, pub adverts: usize, + /// Updates to the relays whose demand alone a payload subscription carries. + pub chains: usize, } impl Sent { pub fn total(self) -> usize { - self.subscribes + self.answers + self.asks + self.told + self.adverts + self.subscribes + self.answers + self.warm() } - /// What only a warm signal costs: queries, their answers, and adverts. + /// What only a warm signal costs: queries, their answers, adverts, and chain updates. pub fn warm(self) -> usize { - self.asks + self.told + self.adverts + self.asks + self.told + self.adverts + self.chains } } @@ -254,6 +311,7 @@ impl std::ops::Add for Sent { asks: self.asks + more.asks, told: self.told + more.told, adverts: self.adverts + more.adverts, + chains: self.chains + more.chains, } } } @@ -268,6 +326,7 @@ impl std::ops::Sub for Sent { asks: self.asks - was.asks, told: self.told - was.told, adverts: self.adverts - was.adverts, + chains: self.chains - was.chains, } } } @@ -364,11 +423,11 @@ pub(super) struct WarmSim { prices: BTreeMap<(Node, Node), Price>, /// Links marked upstream. marked: BTreeSet<(Node, Node)>, + /// The same, as the relay that marked it and its upstream. + upstream: BTreeSet<(Node, Node)>, open: BTreeSet<(Node, Node)>, /// Each node's open links. adjacent: Vec>, - /// Cheapest cold costs between nodes over open links, until one opens or closes. - distances: RefCell>>>, casts: Vec, carry: Vec>, queues: BTreeMap>, @@ -378,6 +437,9 @@ pub(super) struct WarmSim { dirty: BTreeSet<(Node, usize)>, /// Broadcasts whose pulls changed since cycles were last looked for. changed: BTreeSet, + /// The relays holding state for each broadcast, as of the last step: only they + /// can pull, so only they can circle. + members: BTreeMap>, /// Broadcasts whose pulls circle: whether a reader's lead into it, and whether it /// holds a pull the merge made. circling: BTreeMap, @@ -402,15 +464,16 @@ impl WarmSim { homes: BTreeMap::new(), prices: BTreeMap::new(), marked: BTreeSet::new(), + upstream: BTreeSet::new(), open: BTreeSet::new(), adjacent: vec![BTreeSet::new(); nodes], - distances: RefCell::new(None), casts: Vec::new(), carry: vec![BTreeMap::new(); nodes], queues: BTreeMap::new(), streams: 0, dirty: BTreeSet::new(), changed: BTreeSet::new(), + members: BTreeMap::new(), circling: BTreeMap::new(), sent: Sent::default(), watch: Watch::default(), @@ -428,6 +491,7 @@ impl WarmSim { self.prices.insert(pair(a, b), price); if tie == Tie::Up { self.marked.insert(pair(a, b)); + self.upstream.insert((a, b)); } self.join(a, b); self.net.link(a, b, price.cold, tie); @@ -437,14 +501,12 @@ impl WarmSim { self.open.insert(pair(a, b)); self.adjacent[a].insert(b); self.adjacent[b].insert(a); - self.distances.replace(None); } fn part(&mut self, a: Node, b: Node) { self.open.remove(&pair(a, b)); self.adjacent[a].remove(&b); self.adjacent[b].remove(&a); - self.distances.replace(None); } /// Every broadcast `node` holds state for reacts next, as when its routes change. @@ -509,6 +571,7 @@ impl WarmSim { } match &msg { Msg::Subscribe { .. } => self.sent.subscribes += 1, + Msg::Chain { .. } => self.sent.chains += 1, Msg::Answer { .. } => self.sent.answers += 1, Msg::Ask { .. } => self.sent.asks += 1, Msg::Told { .. } => self.sent.told += 1, @@ -517,6 +580,7 @@ impl WarmSim { // The announce stream is one per session, whatever the broadcast. let stream = match &msg { Msg::Subscribe { stream, .. } + | Msg::Chain { stream, .. } | Msg::Answer { stream, .. } | Msg::Ask { stream, .. } | Msg::Told { stream, .. } => *stream as usize, @@ -605,7 +669,10 @@ impl WarmSim { Mode::Ask(_) => (carry.asks.iter()) .filter_map(|(&peer, signal)| Some((peer, signal.as_ref()?))) .collect(), - Mode::Advertise(_) => carry.heard.iter().map(|(&peer, signal)| (peer, signal)).collect(), + // Adverts, and what any upstream it asked said. + Mode::Advertise(_) => (carry.heard.iter().map(|(&peer, signal)| (peer, signal))) + .chain((carry.asks.iter()).filter_map(|(&peer, signal)| Some((peer, signal.as_ref()?)))) + .collect(), }; for (via, signal) in heard { if !self.is_open(node, via) @@ -644,6 +711,7 @@ impl WarmSim { } self.queries(node, cast); self.pull(node, cast); + self.own(node, cast); self.answer(node, cast); self.tell(node, cast); self.advertise(node, cast); @@ -651,11 +719,20 @@ impl WarmSim { /// Open and close the metadata-only subscriptions that ask. fn queries(&mut self, node: Node, cast: usize) { - let Mode::Ask(count) = self.arm.mode else { - return; + let (count, patient) = match self.arm.mode { + Mode::Ask(count) => (count, true), + Mode::Advertise(Audience::NotDownAsk) => (0, false), + Mode::Advertise(Audience::NotDownAskUp) => (usize::MAX, false), + _ => return, }; let carry = self.entry(node, cast).clone(); let want: BTreeSet = match () { + // Its upstreams, when its route ties between more than one. + _ if !patient => match (carry.demand(), count) { + (false, _) => BTreeSet::new(), + (true, 0) => self.tied(node, cast), + (true, _) => self.upstreams(node, cast), + }, _ if carry.demand() => { let mut peers: Vec = self.peers(node).into_iter().filter(|&peer| self.relay(peer)).collect(); peers.sort_by_key(|&peer| (self.price(node, peer).cold, Reverse(self.hash(cast, peer)), peer)); @@ -682,27 +759,89 @@ impl WarmSim { let carry = self.entry(node, cast); carry.asks.insert(new, None); carry.asking.insert(new, stream); - carry.unanswered.insert(new); + if patient { + carry.unanswered.insert(new); + } self.send(node, new, Lane::Ask, cast, Msg::Ask { stream, open: true }); } } + /// The upstreams `node`'s route to `cast` ties between, if more than one. + fn tied(&self, node: Node, cast: usize) -> BTreeSet { + let Some(router) = self.net.router(node) else { + return BTreeSet::new(); + }; + let Some(origin) = router.pick(&self.casts[cast]) else { + return BTreeSet::new(); + }; + let tied: BTreeSet = (router.ties(origin).into_iter()) + .filter(|&peer| self.upstream.contains(&(node, peer))) + .collect(); + match tied.len() > 1 { + true => tied, + false => BTreeSet::new(), + } + } + + /// The upstreams of `node` besides the one its path join to `cast` takes. + fn upstreams(&self, node: Node, cast: usize) -> BTreeSet { + let next = self.route(node, cast).map(|(next, _)| next); + (self.peers(node).into_iter()) + .filter(|&peer| self.upstream.contains(&(node, peer)) && Some(peer) != next) + .collect() + } + fn subscribe(&mut self, node: Node, cast: usize, via: Node) -> Pull { self.streams += 1; let stream = self.streams; - self.send(node, via, Lane::Pull, cast, Msg::Subscribe { stream, open: true }); + let chain = self + .carry(node, cast) + .map_or_else(|| vec![node], |carry| carry.chain(node)); + let open = Msg::Subscribe { + stream, + open: true, + chain: chain.clone(), + }; + self.send(node, via, Lane::Pull, cast, open); Pull { via, stream, state: State::Pending, merged: false, + chain, + } + } + + /// Tell each parent when the relays whose demand alone its pull carries change. + fn own(&mut self, node: Node, cast: usize) { + // A join along the route prices nothing, so it needs no chain. + if self.arm.mode == Mode::Path { + return; + } + let carry = self.entry(node, cast); + let chain = carry.chain(node); + let mut told = Vec::new(); + for pull in [&mut carry.up, &mut carry.next].into_iter().flatten() { + if pull.chain != chain { + pull.chain = chain.clone(); + told.push((pull.via, pull.stream)); + } + } + for (via, stream) in told { + let chain = chain.clone(); + self.send(node, via, Lane::Pull, cast, Msg::Chain { stream, chain }); } } fn unsubscribe(&mut self, node: Node, cast: usize, pull: Option) { if let Some(pull) = pull { let stream = pull.stream; - self.send(node, pull.via, Lane::Pull, cast, Msg::Subscribe { stream, open: false }); + let close = Msg::Subscribe { + stream, + open: false, + chain: Vec::new(), + }; + self.send(node, pull.via, Lane::Pull, cast, close); } } @@ -903,18 +1042,24 @@ impl WarmSim { /// Whether `audience` lets `node` advertise `signal` to `peer`. fn scoped(&self, audience: Audience, node: Node, cast: usize, peer: Node, signal: &Signal) -> bool { + let price = signal.cost + self.price(node, peer).cold; + let heard = self + .net + .router(node) + .and_then(|router| router.heard(&self.casts[cast], peer)); + let useful = || match heard { + Some((cost, false)) => price <= cost, + // Its route runs through this relay, which a neighbour offers nobody back: it + // costs this relay's route and the link, so only an offer below that route + // can beat it. + _ => self.route(node, cast).is_none_or(|(_, cost)| signal.cost < cost), + }; match audience { Audience::All => true, - Audience::Useful => { - let price = signal.cost + self.price(node, peer).cold; - let router = self.net.router(node); - match router.and_then(|router| router.heard(&self.casts[cast], peer)) { - Some((cost, false)) => price <= cost, - // Its route runs through this relay, which a neighbour offers nobody - // back: it costs this relay's route and the link, so only an offer - // below that route can beat it. - _ => self.route(node, cast).is_none_or(|(_, cost)| signal.cost < cost), - } + Audience::Useful => useful(), + Audience::NotNext => matches!(heard, Some((cost, false)) if price <= cost), + Audience::NotDown | Audience::NotDownAsk | Audience::NotDownAskUp => { + !self.upstream.contains(&(peer, node)) && useful() } Audience::Shared => self.carry(node, cast).is_some_and(|carry| { let children = carry.down.keys().filter(|&&child| child != peer).count(); @@ -939,13 +1084,18 @@ impl WarmSim { if self.carry(node, cast).is_some_and(|carry| *carry == Carry::default()) { self.carry[node].remove(&cast); } + let members = self.members.entry(cast).or_default(); + match self.carry[node].contains_key(&cast) { + true => members.insert(node), + false => members.remove(&node), + }; } } fn recv(&mut self, from: Node, to: Node, cast: usize, msg: Msg) { self.dirty.insert((to, cast)); // A publisher answers at once. - if let Msg::Subscribe { stream, open: true } = msg + if let Msg::Subscribe { stream, open: true, .. } = msg && !self.relay(to) { let offer = self.offer(to, cast, from); @@ -953,12 +1103,26 @@ impl WarmSim { } let carry = self.entry(to, cast); match msg { - Msg::Subscribe { stream, open: true } => { + Msg::Subscribe { + stream, + open: true, + chain, + } => { carry.down.entry(from).or_default().insert(stream); + carry.chains.insert((from, stream), chain); + } + // Only while the subscription is open: it may have closed since. + Msg::Chain { stream, chain } => { + if carry.down.get(&from).is_some_and(|streams| streams.contains(&stream)) { + carry.chains.insert((from, stream), chain); + } } - Msg::Subscribe { stream, open: false } => { + Msg::Subscribe { + stream, open: false, .. + } => { close(&mut carry.down, from, stream); carry.answered.remove(&(from, stream)); + carry.chains.remove(&(from, stream)); } Msg::Answer { stream, offer } => { for pull in [&mut carry.up, &mut carry.next].into_iter().flatten() { @@ -1026,6 +1190,7 @@ impl WarmSim { } carry.down.remove(&y); carry.answered.retain(|&(child, _), _| child != y); + carry.chains.retain(|&(child, _), _| child != y); carry.asks.remove(&y); carry.unanswered.remove(&y); carry.asking.remove(&y); @@ -1221,26 +1386,25 @@ impl WarmSim { /// Whether following `node`'s pulls for `cast` goes round a cycle. fn circles(&self, node: Node, cast: usize) -> bool { + self.cycle(node, cast).is_some() + } + + /// A relay on the cycle `node`'s pulls for `cast` run into, if they do. + fn cycle(&self, node: Node, cast: usize) -> Option { + let mut seen = BTreeSet::new(); let mut at = node; - for _ in 0..=self.nodes { - match self.parent(at, cast) { - Some(next) => at = next, - None => return false, - } + while seen.insert(at) { + at = self.parent(at, cast)?; } - true + Some(at) } /// Whether the cycle `node`'s pulls for `cast` run into holds a pull the merge made. fn merged(&self, node: Node, cast: usize) -> bool { - let mut at = node; - for _ in 0..=self.nodes { - match self.parent(at, cast) { - Some(next) => at = next, - None => return false, - } - } - let start = at; + let Some(start) = self.cycle(node, cast) else { + return false; + }; + let mut at = start; loop { let Some(pull) = self.carry(at, cast).and_then(|carry| carry.up.as_ref()) else { return false; @@ -1271,7 +1435,8 @@ impl WarmSim { fn observe(&mut self) { self.watch.steps += 1; for cast in std::mem::take(&mut self.changed) { - let circling: Vec = (0..self.relays).filter(|&node| self.circles(node, cast)).collect(); + let members = self.members.get(&cast).into_iter().flatten(); + let circling: Vec = members.copied().filter(|&node| self.circles(node, cast)).collect(); match circling.is_empty() { true => self.circling.remove(&cast), false => { @@ -1294,16 +1459,21 @@ impl WarmSim { /// The relays a path over open links joins to `cast`'s publisher. fn reach(&self, cast: usize) -> BTreeSet { let start = self.endpoints[&self.casts[cast].0]; - let mut seen = BTreeSet::from([start]); - let mut queue = vec![start]; - while let Some(at) = queue.pop() { + // A route that came down a link marked upstream never goes back up one. + let mut seen = BTreeSet::from([(start, false)]); + let mut queue = vec![(start, false)]; + while let Some((at, marked)) = queue.pop() { for peer in self.peers(at) { - if self.relay(peer) && seen.insert(peer) { - queue.push(peer); + if !self.relay(peer) || (marked && self.upstream.contains(&(at, peer))) { + continue; + } + let next = (peer, marked || self.upstream.contains(&(peer, at))); + if seen.insert(next) { + queue.push(next); } } } - seen + seen.into_iter().map(|(node, _)| node).collect() } /// Count unfed readers as a round ends, and the first frames that arrived. @@ -1406,46 +1576,43 @@ impl WarmSim { } } - /// The cheapest tree joining `cast`'s publisher to every relay reading it, over open - /// links at their cold prices: Dreyfus and Wagner's exact dynamic program. - /// The cheapest cold cost between every two nodes over open links, through relays. - fn distances(&self) -> Vec> { - if let Some(dist) = self.distances.borrow().as_ref() { - return dist.clone(); - } - let n = self.nodes; - let mut dist = vec![vec![Metric::MAX / 4; n]; n]; - for (node, row) in dist.iter_mut().enumerate() { - row[node] = 0; - } - for &(a, b) in &self.open { - let cost = self.price(a, b).cold; - dist[a][b] = dist[a][b].min(cost); - dist[b][a] = dist[b][a].min(cost); - } - // Only relays forward. - for via in (0..n).filter(|&node| self.relay(node)) { - for a in 0..n { - for b in 0..n { - dist[a][b] = dist[a][b].min(dist[a][via] + dist[via][b]); + /// Lower each node's cost in `costs` to the cheapest reached from any node over open + /// links at their cold prices: Dijkstra from every node at once. + fn relax(&self, costs: &mut [Metric]) { + let mut heap: BinaryHeap> = (costs.iter().enumerate()) + .filter(|&(_, &cost)| cost < Metric::MAX / 4) + .map(|(node, &cost)| Reverse((cost, node))) + .collect(); + let start = costs.to_vec(); + while let Some(Reverse((cost, at))) = heap.pop() { + // Only relays carry a copy on; an endpoint only starts or ends one. + if cost > costs[at] || (!self.relay(at) && cost != start[at]) { + continue; + } + for &peer in &self.adjacent[at] { + let next = cost + self.price(at, peer).cold; + if next < costs[peer] { + costs[peer] = next; + heap.push(Reverse((next, peer))); } } } - self.distances.replace(Some(dist.clone())); - dist } + /// The cheapest tree joining `cast`'s publisher to every relay reading it, over open + /// links at their cold prices: Dreyfus and Wagner's exact dynamic program. pub fn cheapest(&self, cast: usize) -> Metric { let endpoint = self.endpoints[&self.casts[cast].0]; let mut terminals = vec![endpoint]; terminals.extend((0..self.relays).filter(|&node| self.reading(node, cast))); let n = self.nodes; - let dist = self.distances(); let count = terminals.len(); let full = (1usize << count) - 1; let mut tree = vec![vec![Metric::MAX / 4; n]; full + 1]; for (index, &terminal) in terminals.iter().enumerate() { - tree[1 << index][..n].copy_from_slice(&dist[terminal]); + let row = &mut tree[1 << index]; + row[terminal] = 0; + self.relax(row); } for mask in 1..=full { if mask.count_ones() < 2 { @@ -1459,10 +1626,9 @@ impl WarmSim { } sub = (sub - 1) & mask; } - let row = tree[mask].clone(); - for at in 0..n { - tree[mask][at] = (0..n).map(|from| row[from] + dist[from][at]).min().unwrap_or(row[at]); - } + let mut row = std::mem::take(&mut tree[mask]); + self.relax(&mut row); + tree[mask] = row; } tree[full][endpoint] } diff --git a/rs/moq-net/src/route/tests/warmth.rs b/rs/moq-net/src/route/tests/warmth.rs index 5fe1fb6b067..08999381491 100644 --- a/rs/moq-net/src/route/tests/warmth.rs +++ b/rs/moq-net/src/route/tests/warmth.rs @@ -387,7 +387,7 @@ fn settles(rows: &[Row], case: &str) { /// advertising move the far pops through them and back once it ends. #[test] fn a_join_skips_pops_without_demand_and_warm_ones_draw_it() { - for arm in arms() { + for arm in arms().into_iter().chain(narrower()) { let rows = run(&pops(), arm, &pop_phases(), 1); let case = arm.name(); settles(&rows, &case); @@ -420,7 +420,7 @@ fn a_join_skips_pops_without_demand_and_warm_ones_draw_it() { /// other, and path joins take one each. #[test] fn drones_reading_in_turn_share_one_uplink() { - for arm in arms() { + for arm in arms().into_iter().chain(narrower()) { let rows = run(&drones(4), arm, &drone_phases(4, false), 2); let case = arm.name(); settles(&rows, &case); @@ -471,16 +471,31 @@ fn drones_take_over_a_lost_uplink() { // --- Growth --- -/// The cheaper ways to advertise that lost, measured beside the rest: telling only -/// once two others want it, and never over a link marked upstream. +const NOT_NEXT: Arm = arm(Mode::Advertise(Audience::NotNext), false); +const NOT_DOWN: Arm = arm(Mode::Advertise(Audience::NotDown), false); +const NOT_DOWN_ASK: Arm = arm(Mode::Advertise(Audience::NotDownAsk), false); +const NOT_DOWN_ASK_UP: Arm = arm(Mode::Advertise(Audience::NotDownAskUp), false); + +/// The narrower audiences for useful adverts, which keep them off a CDN's edges. The +/// last, [`NOT_DOWN_ASK_UP`], is the pick. +fn narrower() -> Vec { + vec![NOT_NEXT, NOT_DOWN, NOT_DOWN_ASK, NOT_DOWN_ASK_UP] +} + +/// The cheaper ways to advertise measured beside the rest: the narrower audiences, +/// telling only once two others want it, and never over a link marked upstream. fn audiences() -> Vec { - [Audience::Shared, Audience::Down] - .into_iter() - .map(|audience| arm(Mode::Advertise(audience), false)) - .collect() + let mut arms = narrower(); + arms.extend( + [Audience::Shared, Audience::Down] + .into_iter() + .map(|audience| arm(Mode::Advertise(audience), false)), + ); + arms } -/// The arms swept for growth, every one with the path in its offers. +/// The arms swept as the core mesh and the drone mesh grow, every one with the path +/// in its offers. fn growing() -> Vec { let mut arms = vec![ PATH, @@ -494,29 +509,40 @@ fn growing() -> Vec { arms } -/// A CDN tier: `cores` fully meshed, and `edges` edges per core, each joined to its -/// core and the next one over links marked upstream and meshed with the edges -/// beside it. The publisher joins core 0. -fn cdn(cores: usize, edges: usize, claims: usize) -> Shape { - let edge = |core: usize, index: usize| cores + core * edges + index; +/// The arms swept as a region's edges grow. +fn widening() -> Vec { + let mut arms = vec![PATH, arm(Mode::Ask(1), false), arm(Mode::Ask(2), false), USEFUL]; + arms.extend(narrower()); + arms +} + +/// A CDN: `regions` pairs of cores, every core meshed with every other, and `edges` +/// edges per region, each homed on both of its region's cores over links it marks +/// upstream, so a region's edge has `edges` siblings under each core. With +/// `peered` a region's edges are meshed with each other too. The publisher joins +/// core 0. +fn cdn(regions: usize, edges: usize, peered: bool, claims: usize) -> Shape { + let cores = 2 * regions; + let edge = |region: usize, index: usize| cores + region * edges + index; let mut links = Vec::new(); for a in 0..cores { for b in a + 1..cores { links.push((a, b, Price::new(6, 2), Tie::Peer)); } + } + for region in 0..regions { for index in 0..edges { - links.push((edge(a, index), a, Price::new(3, 1), Tie::Up)); - if cores > 1 { - links.push((edge(a, index), (a + 1) % cores, Price::new(3, 1), Tie::Up)); + for core in [2 * region, 2 * region + 1] { + links.push((edge(region, index), core, Price::new(3, 1), Tie::Up)); } - for other in index + 1..edges { - links.push((edge(a, index), edge(a, other), Price::new(1, 1), Tie::Peer)); + for other in (index + 1..edges).filter(|_| peered) { + links.push((edge(region, index), edge(region, other), Price::new(1, 1), Tie::Peer)); } } } Shape { name: "cdn", - relays: cores * (1 + edges), + relays: cores + regions * edges, links, home: 0, claims, @@ -563,15 +589,22 @@ impl Load { struct Grown { changes: usize, sent: Sent, - /// Warm state per relay with every reader in place: the mean and the most. - held: f64, - most: usize, + /// Warm state each relay held with every reader in place. + held: Vec, cost: Metric, cheapest: Metric, first: f64, right: Result<(), String>, } +impl Grown { + /// The mean warm state held across `relays`. + fn held(&self, relays: std::ops::Range) -> f64 { + let count = relays.len().max(1); + self.held[relays].iter().sum::() as f64 / count as f64 + } +} + fn grow(shape: &Shape, readers: &[(Node, usize)], in_turn: bool, arm: Arm, cheapest: bool) -> Grown { let mut sim = shape.build(arm, 11); let (sent, waits) = (sim.sent, sim.waits.len()); @@ -606,8 +639,7 @@ fn grow(shape: &Shape, readers: &[(Node, usize)], in_turn: bool, arm: Arm, cheap Grown { changes: 2 * readers.len(), sent: sim.sent - sent, - held: held.iter().sum::() as f64 / held.len() as f64, - most: held.into_iter().max().unwrap_or(0), + held, cost, cheapest, first, @@ -615,60 +647,156 @@ fn grow(shape: &Shape, readers: &[(Node, usize)], in_turn: bool, arm: Arm, cheap } } -/// A first viewer's adverts on CDN tiers: told to every neighbour they grow with -/// the core mesh, and told only where they could be taken they do not, since a core +/// A first viewer's adverts on a CDN: told to every neighbour they grow with the +/// core mesh, and told only where they could be taken they do not, since a core /// with demand is never cheaper for another core than its own route. #[test] fn useful_adverts_do_not_grow_with_the_core_mesh() { - let warm = |arm: Arm, cores: usize| { - let shape = cdn(cores, 3, 1); - // The first edge under core 1, which the publisher's core is not. - let grown = grow(&shape, &[(cores + 3, 0)], false, arm, false); + let warm = |arm: Arm, regions: usize| { + let shape = cdn(regions, 3, false, 1); + // The first edge of region 1, which the publisher's core is not in. + let grown = grow(&shape, &[(2 * regions + 3, 0)], false, arm, false); assert_eq!(grown.right, Ok(()), "{}", arm.name()); grown.sent.warm() }; - assert_eq!(warm(USEFUL, 8), warm(USEFUL, 16)); - assert!(warm(ADVERTISE, 8) < warm(ADVERTISE, 16)); + assert_eq!(warm(USEFUL, 4), warm(USEFUL, 8)); + assert!(warm(ADVERTISE, 4) < warm(ADVERTISE, 8)); } -/// Warm messages and warm state as relays and demanded broadcasts grow, on CDN tiers -/// with their cores fully meshed and on drone meshes, for one viewer per broadcast -/// and for a crowd. `MOQ_WARMTH_GROWTH` caps the broadcasts. +/// Under the pick a warm core tells none of its region's edges, and each edge asks +/// the upstreams beside its path join on subscribe, so a first viewer's warm messages +/// stay flat as a region grows its edges and as the core mesh grows. Useful adverts +/// grow with the edges, since a warm core tells every edge it could feed. +#[test] +fn asks_toward_upstreams_do_not_grow_with_the_edges() { + let warm = |arm: Arm, regions: usize, edges: usize| { + let shape = cdn(regions, edges, false, 1); + // The first edge of region 1, which the publisher's core is not in. + let grown = grow(&shape, &[(2 * regions + edges, 0)], false, arm, false); + assert_eq!(grown.right, Ok(()), "{}", arm.name()); + grown.sent.warm() + }; + assert_eq!(warm(NOT_DOWN_ASK_UP, 4, 3), warm(NOT_DOWN_ASK_UP, 4, 16)); + assert_eq!(warm(NOT_DOWN_ASK_UP, 4, 3), warm(NOT_DOWN_ASK_UP, 8, 3)); + assert!(warm(USEFUL, 4, 3) < warm(USEFUL, 4, 16)); +} + +/// An edge whose cheaper uplink is to the cold core reaches the warm core only on a +/// warm signal over an upstream link: an advert down it, or the edge's own ask. +/// Skipping both leaves it on its path join. +#[test] +fn an_edge_finds_the_warm_core_beside_its_route() { + let cost = |arm: Arm| { + let rows = run(&unequal(), arm, &unequal_phases(), 7); + settles(&rows, &arm.name()); + let cost: Metric = rows.iter().map(Row::cost).sum(); + let cheapest: Metric = rows.iter().flat_map(|row| row.cheapest.iter()).sum(); + (cost, cheapest) + }; + for arm in [USEFUL, NOT_DOWN_ASK_UP] { + let (cost, cheapest) = cost(arm); + assert_eq!(cost, cheapest, "{}", arm.name()); + } + for arm in [NOT_NEXT, NOT_DOWN, NOT_DOWN_ASK] { + let (cost, cheapest) = cost(arm); + assert!(cost > cheapest, "{}", arm.name()); + } +} + +/// One CSV row of the growth sweeps. +struct Sized { + shape: Shape, + /// Cores and edges per core, or the CDN relay and its drones. + cores: usize, + edges: usize, + peered: bool, + /// Where one viewer per broadcast reads, and where a crowd does. + viewers: std::ops::Range, + crowd: std::ops::Range, + arms: Vec, + one_viewer: &'static [usize], + crowds: &'static [usize], +} + +/// Warm messages and warm state as relays and demanded broadcasts grow, for one +/// viewer per broadcast and for a crowd: CDN cores fully meshed with 3 edges per +/// core, then edges per core growing at 8 and 32 cores, with and without the edges +/// of a region peered, then full drone meshes. `MOQ_WARMTH_GROWTH` caps the +/// broadcasts. #[test] #[ignore = "a benchmark; Nightly runs it from test/route-sweep.sh"] fn benchmark_warmth_growth() { let most: usize = std::env::var("MOQ_WARMTH_GROWTH").map_or(4096, |most| most.parse().expect("MOQ_WARMTH_GROWTH")); println!( - "shape,relays,neighbours,broadcasts,load,arm,reader_changes,messages,warm_messages,per_change_warm_messages,held_per_relay,held_at_most,tree_cost,cheapest_cost,first_frame_ms,right" + "shape,cores,edges_per_core,peered,relays,neighbours,broadcasts,load,arm,reader_changes,messages,warm_messages,per_change_warm_messages,held_per_core,held_per_edge,held_at_most,tree_cost,cheapest_cost,first_frame_ms,right" ); - let mut sizes: Vec<(Shape, std::ops::Range)> = Vec::new(); - for cores in [3, 8, 16, 32] { - let shape = cdn(cores, 3, 1); - sizes.push((shape, cores..cores * 4)); + let mut sizes = Vec::new(); + let region = |regions: usize, edges: usize, peered: bool, arms: Vec, wide: bool| { + let cores = 2 * regions; + Sized { + shape: cdn(regions, edges, peered, 1), + cores, + edges, + peered, + viewers: cores..cores + regions * edges, + // A crowd in region 1, which the publisher's core is not in. + crowd: cores + edges..cores + 2 * edges, + arms, + one_viewer: if wide { &[256] } else { &[1, 16, 256, 4096] }, + crowds: if wide { &[16] } else { &[1, 4, 16] }, + } + }; + for regions in [2, 4, 8, 16] { + sizes.push(region(regions, 3, false, growing(), false)); + } + for regions in [4, 16] { + for edges in [3, 16, 64, 128] { + for peered in [false, true] { + // A region of a hundred edges does not peer every pair, so the full mesh + // is measured only while it stays small. + if peered && edges * regions > 256 { + continue; + } + sizes.push(region(regions, edges, peered, widening(), true)); + } + } } for count in [5, 16, 64] { - sizes.push((drones(count), 1..count + 1)); + sizes.push(Sized { + shape: drones(count), + cores: 1, + edges: count, + peered: true, + viewers: 1..count + 1, + crowd: 1..count + 1, + arms: growing(), + one_viewer: &[1, 16, 256, 4096], + crowds: &[1, 4, 16], + }); } - for (shape, readers) in sizes { + for size in sizes { for load in [Load::OneViewer, Load::Crowd] { - let counts: &[usize] = match load { - Load::OneViewer => &[1, 16, 256, 4096], - Load::Crowd => &[1, 4, 16], + let (counts, relays) = match load { + Load::OneViewer => (size.one_viewer, size.viewers.clone()), + Load::Crowd => (size.crowds, size.crowd.clone()), }; for &claims in counts.iter().filter(|&&claims| claims <= most) { let shape = Shape { claims, - ..shape.clone() + ..size.shape.clone() }; - let reads = load.readers(readers.clone(), claims, 0x6772_6f77 + claims as u64); + let reads = load.readers(relays.clone(), claims, 0x6772_6f77 + claims as u64); let neighbours = shape.build(PATH, 11).degree(); let mut cheapest = None; - for arm in growing() { + for &arm in &size.arms { let grown = grow(&shape, &reads, load == Load::Crowd, arm, cheapest.is_none()); let cheapest = *cheapest.get_or_insert(grown.cheapest); println!( - "{},{},{neighbours},{claims},{},{},{},{},{},{:.2},{:.2},{},{},{cheapest},{:.0},{}", + "{},{},{},{},{},{neighbours},{claims},{},{},{},{},{},{:.2},{:.2},{:.2},{},{},{cheapest},{:.0},{}", shape.name, + size.cores, + size.edges, + size.peered, shape.relays, load.name(), arm.name(), @@ -676,8 +804,9 @@ fn benchmark_warmth_growth() { grown.sent.total(), grown.sent.warm(), grown.sent.warm() as f64 / grown.changes as f64, - grown.held, - grown.most, + grown.held(0..size.cores), + grown.held(size.cores..shape.relays), + grown.held.iter().max().unwrap_or(&0), grown.cost, grown.first * HOP.as_secs_f64() * 1000.0, match grown.right { @@ -797,6 +926,41 @@ fn churn(arm: Arm, cases: std::ops::Range, shape: fn(&mut XorShift) -> Sh } /// Drones with readers coming and going and uplinks flapping. +/// A random CDN region: two or three cores fully meshed, and two to four edges, each +/// marking two cores upstream at random prices, some peering each other. +fn regional(rng: &mut XorShift) -> Shape { + let cores = 2 + rng.next(2) as usize; + let edges = 2 + rng.next(3) as usize; + let mut links = Vec::new(); + let price = |rng: &mut XorShift, low: u64, span: u64| { + let cold = low + rng.next(span); + Price::new(cold, 1 + rng.next(cold)) + }; + for a in 0..cores { + for b in a + 1..cores { + links.push((a, b, price(rng, 4, 4), Tie::Peer)); + } + } + for edge in cores..cores + edges { + let first = rng.next(cores as u64) as usize; + for core in [first, (first + 1) % cores] { + links.push((edge, core, price(rng, 2, 4), Tie::Up)); + } + for other in edge + 1..cores + edges { + if rng.chance(30) { + links.push((edge, other, price(rng, 1, 3), Tie::Peer)); + } + } + } + Shape { + name: "regional", + relays: cores + edges, + home: rng.next((cores + edges) as u64) as usize, + links, + claims: 2, + } +} + fn flying(rng: &mut XorShift) -> Shape { drones(3 + rng.next(3) as usize) } @@ -805,8 +969,16 @@ fn flying(rng: &mut XorShift) -> Shape { /// circling: the cycles it formed lasted only while messages were in flight. #[test] fn every_arm_settles_after_churn() { - for arm in [PATH, ASK, ASK_MERGE, ADVERTISE, ADVERTISE_MERGE, USEFUL] { - for shape in [random as fn(&mut XorShift) -> Shape, flying] { + for arm in [ + PATH, + ASK, + ASK_MERGE, + ADVERTISE, + ADVERTISE_MERGE, + USEFUL, + NOT_DOWN_ASK_UP, + ] { + for shape in [random as fn(&mut XorShift) -> Shape, flying, regional] { let out = churn(arm, 0..12, shape); assert_eq!(out.failed, 0, "{}: {:?}", arm.name(), out.first); } @@ -922,9 +1094,39 @@ fn cases() -> Vec { crowd(first..last, 4, 0x6372_6f77), ), case("tiers_one_viewer", tiers(regions, per, 16), single), + case("region_unequal", unequal(), unequal_phases()), + ] +} + +/// Each edge of [`unequal`] reads in turn. +fn unequal_phases() -> Vec { + vec![ + phase("first_edge", vec![Step::Read(3, 0, true)]), + phase("second_edge", vec![Step::Read(4, 0, true)]), ] } +/// One region of a CDN: the publisher's core 0, the region's cores 1 and 2, and its +/// edges 3 and 4, each with its cheaper uplink to a different core. No hash sends +/// both to one core; only a warm signal over an upstream link can. +fn unequal() -> Shape { + Shape { + name: "unequal", + relays: 5, + links: vec![ + (0, 1, Price::new(6, 2), Tie::Peer), + (0, 2, Price::new(6, 2), Tie::Peer), + (1, 2, Price::new(6, 2), Tie::Peer), + (3, 1, Price::new(3, 1), Tie::Up), + (3, 2, Price::new(4, 1), Tie::Up), + (4, 1, Price::new(4, 1), Tie::Up), + (4, 2, Price::new(3, 1), Tie::Up), + ], + home: 0, + claims: 1, + } +} + /// The drone shape's last phase closes the uplink the arm settled on. fn takeover(arm: Arm) -> Case { let mut phases = drone_phases(5, false); @@ -1000,7 +1202,11 @@ fn benchmark_warmth_churn() { "mesh,arm,cases,deliveries,cycle_deliveries,starved_deliveries,merge_cycle_deliveries,longest_cycle,unfed_reader_hops,extra_uplink_hops,reader_changes,messages,warm_messages,settle_rounds,failed" ); let cases: usize = std::env::var("MOQ_WARMTH_CASES").map_or(400, |cases| cases.parse().expect("MOQ_WARMTH_CASES")); - for (name, shape) in [("random", random as fn(&mut XorShift) -> Shape), ("drones", flying)] { + for (name, shape) in [ + ("random", random as fn(&mut XorShift) -> Shape), + ("drones", flying), + ("regional", regional), + ] { for arm in arms().into_iter().chain(audiences()) { let out = churn(arm, 0..cases, shape); println!( diff --git a/test/route-sweep.sh b/test/route-sweep.sh index 8c241d131b6..92a17c11434 100755 --- a/test/route-sweep.sh +++ b/test/route-sweep.sh @@ -18,5 +18,5 @@ just rs test --profile=sweep -p moq-net --release --run-ignored only -E 'test(/^ MOQ_ROUTE_CASES=3000 just rs test -P sweep -p moq-net --release --run-ignored only -E 'test(=route::tests::cache::sweep_cache_routing_long)' # The multicast bake-off: its prototype carries every failure on every shape at 32 relays, and its tables against the path rule at 12. MOQ_BAKEOFF_RELAYS=12 just rs test --profile=sweep -p moq-net --release --run-ignored only -E 'test(/^route::tests::bakeoff::(sweep|benchmark)_/)' -# Warmth: path joins, asking, and advertising over fixed shapes, 400 churned meshes per arm, and growth to 128 relays and 4096 broadcasts, about 6 minutes. +# Warmth: path joins, asking, and advertising over fixed shapes, 400 churned meshes per arm on three kinds of mesh, and growth over cores, edges per core, drones, and up to 4096 broadcasts, about 10 minutes. just rs test --profile=sweep -p moq-net --release --run-ignored only -E 'test(/^route::tests::warmth::benchmark_/)' From f4e9303e1a497703f93acbf5578890828f31c4dc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 11:17:03 -0800 Subject: [PATCH 07/19] feat(moq-net): make the warmth pick's ask a one-shot query and stop the reflection loop with a rule instead of a chain The edge's ask is now a query: one request, one answer of the upstream's warm offer or none, nothing forwarded or held upstream, and the answer dropped once a pull from that upstream is answered. It keeps the cheapest tree on the unequal region at one request and answer per extra upstream per first viewer, flat in edges and cores. A relay takes no warm signal from a relay on its own pull's path, which stops the loop where its own demand made a parent warm for it. That replaces the chain in every subscription: no wire field, fewer messages, and no churned mesh left unsettled in 2,000 each of random meshes, drone meshes, and regions, where the chain left 2 random meshes. The chain stays as a measured arm. A relay that has lost its upstream, or has no pull and no route, now tells a new subscriber so at once, and the checker counts a reader as reachable when the route layer gives its relay a route. `MOQ_WARMTH_ARMS` narrows the churn benchmark to named arms for longer sweeps. Co-Authored-By: Claude Opus 5.5 --- quest/m3/cache-routing/select.md | 178 ++++++++++++++++++--------- rs/moq-net/src/route/tests/warm.rs | 90 ++++++++++---- rs/moq-net/src/route/tests/warmth.rs | 70 ++++++++--- 3 files changed, 236 insertions(+), 102 deletions(-) diff --git a/quest/m3/cache-routing/select.md b/quest/m3/cache-routing/select.md index d4255e597e0..d8c58f0e219 100644 --- a/quest/m3/cache-routing/select.md +++ b/quest/m3/cache-routing/select.md @@ -3,7 +3,7 @@ ## Goal A relay with demand for an Instance advertises it on the announce stream to -each peer or upstream that could take it, an edge asks its other upstreams +each peer or upstream that could take it, an edge queries its other upstreams when it gains demand, and each relay pulls every demanded broadcast from the cheapest copy of its Instance, moving an existing pull only when the route it uses gets worse or ends, or another is strictly cheaper in summed cost. A peer @@ -21,29 +21,40 @@ Plan, is replaced. Recommended 2026-11-16 on [moq-dev/moq#8903](https://github.com/moq-dev/moq/pull/8903), awaiting the maintainer's confirmation at merge: **adverts between peers and toward -upstreams, only where the copy could be taken; never down a link the -neighbour marked upstream, where the edge instead asks its other upstreams on -subscribe; the data path in every offer, the relays a pull feeds in every -subscription, and the race left standing**. Rejected: +upstreams, only where the copy could be taken; none down a link the neighbour +marked upstream, where an edge instead queries its other upstreams once when +it gains demand; the data path in every offer; no warm signal taken from a +relay on the taker's own path; and the race left standing**. Four parts: +adverts, the path, the query, and one rule. + +The maintainer's decision is whether to keep the query, below. Rejected: - Advertising to every neighbour that could take the copy, down marked links too: it grows with a region's edges, since a warm core tells every edge it could feed. 1.7 to 118 warm messages per one-viewer reader change as edges - per core grow from 3 to 128, against the pick's flat 1.6 to 1.8. + per core grow from 3 to 128, against the pick's flat 1.05 to 1.3. - Advertising to every neighbour, which also grows with the neighbours of every relay on the warm chain: 9 to 66 per one-viewer reader change as a full core mesh grows from 4 to 32 cores. -- Keeping adverts off the edges without the ask (never to a neighbour whose - route runs through the relay, never down a marked link, or asking only - between upstreams that tie): flat, but an edge whose cheaper uplink is to - the cold core never hears of the warm one, 29 against 24 on the unequal - region. +- Asking only between upstreams that tie, or never advertising to a neighbour + whose route runs through the relay: as the pick without the query, at more + messages or none fewer. +- A standing query, held open for updates while the demand lasts: the same + trees as the one-shot query, but the upstream holds an entry per edge per + demanded broadcast (41 against 9 per core) and it left 1 of 400 churned + regions unsettled. +- Naming in every subscription the relays whose demand alone it carries, so + an offer leaves them out: a second fix for the loop the rule stops. It adds + a list and an update message on each change (6,913 to 8,469 per 400 + churned meshes), loses a shared drone uplink (41 against 33 with two drones + a hop apart), and left 2 of 2,000 churned random meshes unsettled where the + rule left none. - Asking the k cheapest neighbours on each SUBSCRIBE. It stays bounded per subscribe only because it stops finding the copy: on a full drone mesh past 5 drones, `ask_1` shared no uplink at 64 drones and `ask_2` few. It doubled the time to a first frame and left about twice the readers unfed under churn. On CDN tiers `ask_1` sent 5.7 to 7 warm messages per - one-viewer reader change against the pick's 1.6 to 2.1. + one-viewer reader change against the pick's 1.05 to 1.55. - Advertising only once two others want the copy, which lost both cases; never advertising over a link the sender marked upstream, which saves nothing in a core mesh. @@ -53,6 +64,50 @@ That PR's simulator, `rs/moq-net/src/route/tests/warm.rs`, runs every rule below, and its `warmth.rs` holds the shapes, the churn sweep, and the growth sweep this ports onto production code. +### Decision: the query + +The query is the one part that only an edge with more than one upstream +uses. Without it the pick is adverts between peers and toward upstreams, the +path, and the rule. With equal uplinks the query buys nothing: every edge of +a region ranks the two cores by the same rendezvous hash per broadcast, so +every edge's path join lands on the same core, always; path joins already +built the cheapest tree on every unpeered CDN shape swept. It pays only where an +edge's cheaper uplink is to the core that is cold. + +| shape | tree, with / without the query (cheapest) | warm messages per reader change, with / without | +|---|---|---| +| CDN, 8 cores, 16 edges per core, one viewer | 2236 / 2236 (2236) | 1.21 / 0.21 | +| CDN, 8 cores, 16 edges per core, crowd | 544 / 544 (544) | 3.00 / 2.00 | +| CDN, 32 cores, 3 edges per core, crowd | 256 / 256 (256) | 3.00 / 2.00 | +| tiers, crowd (edges peered) | 1095 / 1095 (1095) | 5.29 / 3.95 | +| tiers, one viewer | 1096 / 1096 (1096) | 5.50 / 3.50 | +| two edges, unequal uplinks, in turn | 12 / 13 (12) | 3.50 / 1.00 | +| two edges, unequal uplinks, at once | 9 / 9 (7) | 3.00 / 1.00 | +| one region, unequal uplinks | 24 / 29 (24) | 3.00 / 0.00 | +| 400 churned regions: unfed reader-hops, warm messages | 17,181 / 12,586 | 24,446 / 8,602 in all | + +Recommended: keep the query. Where uplinks are unequal, it saves the second +core's copy over the core mesh, the dearest link a region has. It costs one +request and one answer per extra upstream per first viewer at an edge, holds +nothing on the core, and carries no traffic. Without it an operator gets the +same tree by pricing each edge's uplinks equally. + +### What each part costs + +| part | on the wire | relay state | bounded by | without it | +|---|---|---|---|---| +| advert | offer and path on the announce stream, and its withdrawal | one heard per neighbour per demanded Instance | neighbours; the path by the hop limit | path joins: pops 69 against 61, drones in turn 5 uplinks against 1, a ring 48 against 8 | +| offer on the subscribe answer | a cost on the answer and its updates | one per pull | one per subscription | nothing can price a move | +| path in the offer | a list on the advert, answer, and query answer | as the offer | the hop limit | cycle deliveries up 77%, 26%, and 110% on random meshes, drones, and regions; 3 of 1,200 churned meshes unsettled | +| query | a request naming an Instance; one answer, offer and path or none | the edge keeps one answer per extra upstream per demanded Instance; the upstream none | upstreams beyond the first | an edge whose cheaper uplink is cold: 29 against 24 | +| no warm signal from a relay on the taker's path | none | none, the path of its own pull | none | 1 of 400 churned regions never settles | + +The rule replaces a chain in every subscription naming the relays whose +demand alone it carries. The offer's path cannot stand in for the chain on +the sender's side: the demand that comes back is downstream of the sender, +and its path names only relays upstream of it. On the taker's side the path +of its own pull is enough. + ### What a relay sends - **The offer.** What a relay R with a live upstream through parent P offers @@ -60,9 +115,10 @@ sweep this ports onto production code. otherwise `O(P→R) + cold(P, R)`, the chain priced unshared. The publisher's relay offers 0. An offer is never below its sender's own cost. - **The path.** Every offer names the relays the data passes from the - publisher's relay to the sender. Without it, transient cycles under churn - rose by 73% on random meshes and 30% on drone meshes, and the merge below - cannot work. + publisher's relay to the sender. Without it, cycle deliveries under churn + rose by 77% on random meshes, 26% on drone meshes, and 110% on regions, 1 + random mesh and 2 regions of 400 never settled, and the merge below cannot + work. - **Adverts.** While R has demand for an Instance besides N, and N has not marked R upstream, it tells N its offer and path on the announce stream when `O(R→N) + cold(R, N)` is below N's route cost for the Instance, or @@ -73,22 +129,21 @@ sweep this ports onto production code. path join already. R knows N's route cost from what N announces it, and computes it when N's route runs through R. Skipping the parent and children by rule instead left 8 of 400 churned drone meshes unsettled. -- **Asks.** A relay with more than one upstream asks each upstream besides - the one its path join takes, on a metadata-only subscription opened when - it gains demand and closed when the demand ends. It does not wait for the - answers: it joins along its route at once and moves if an answer is - strictly cheaper. The asked upstream answers its offer and path, and each - change. An upstream cannot tell which of its upstreams a dual-homed edge's - route uses, since the edge announces it nothing, so only the edge can find - a warm core beside its route. -- **The pull's chain.** Every subscription names the relays whose demand - alone it carries: the subscriber, then, while the subscriber has no reader - and one neighbour pulling through it, that neighbour's chain. It updates - the chain on each change. A relay's demand besides N leaves out every pull - whose chain names N. Without it N's own pull, through another neighbour, - makes R's offer to N warm; N moves to R, the other pull ends, R's offer - turns cold, and N moves back for good. One of 400 churned regions never - settled for that reason, under the pick and most other warm arms. +- **The query.** A relay with more than one upstream sends each upstream + besides the one its path join takes one query when it gains demand: one + request, one answer. The upstream answers its warm offer and path, or that + it has none, from what it holds, and keeps nothing: it forwards nothing, + holds no demand for it, and sends no update. The relay does not wait: it + joins along its route at once, moves if an answer is strictly cheaper, and + drops an answer once a pull from that upstream is answered, live or lost: + an answer kept longer went stale and held a move to a core that had lost + its upstream. It is not a + metadata-only subscription: one forwarded by a cold core would hold demand + of a kind across the core mesh for as long as the viewer stays, the + standby the epic keeps opt in and off inside the CDN. An upstream cannot + tell which of its upstreams a dual-homed edge's route uses, since the edge + announces it nothing, so only the edge can find a warm core beside its + route. - **Answers.** The subscribe answer carries the parent's offer and path to the subscriber, and an update on each change. A subscriber waits for that answer before it counts the pull as live. @@ -101,6 +156,15 @@ sweep this ports onto production code. - It pulls from a neighbour only when that is strictly cheaper than `D(R)`. It never pulls from a neighbour that pulls through it, nor from one whose path names it. +- It takes no advert or answer from a relay named on its own live pull's + path; its route still counts. That relay's warmth may be this relay's own + demand coming back: N pulls through X from R, so R has demand besides N + and offers N its warm cost; N moves to R, X's pull ends, R's offer turns + cold, N goes back through X, and so on for good. Without the rule, 1 of + 400 churned regions never settled that way under most warm arms; with it, none of 2,000 each of random meshes, drone meshes, + and regions under the pick. A relay on N's path is upstream of N's parent, + so N gives up only a move up its own branch, which its route still offers + when that is the cheaper way. - A move is make before break: the old pull stays until the new one is answered live, and a move under way stays while it is no dearer than staying. Its own subscription shares the old parent, which lowers what @@ -117,11 +181,12 @@ Why no cycle outlasts the messages in flight: a relay's cost is its parent's offer plus a warm price of at least 1, and no offer is below its sender's cost, so cost rises strictly along every pull chain. Two relays can never each prefer the other on current offers, and a cycle needs one stale offer. -Gaining demand lowers offers only to neighbours outside the chain that -brought it; losing it raises only the offer to the neighbour that still -pulls, which then moves only to something strictly cheaper. The sweep agrees: -every churned random mesh, drone mesh, and region settled, and cycles lasted -at most 38 deliveries under the pick, each while some answer was in flight. +Gaining demand lowers offers to others, and the one it lowers to the relay +whose own demand it is, that relay ignores; losing it raises only the offer to +the neighbour that still pulls, which then moves only to something strictly +cheaper. The sweep agrees, though it is no proof: every churned random mesh, +drone mesh, and region settled under the pick, and cycles lasted at most 61 +deliveries, each while some answer was in flight. ### Simultaneous starts @@ -130,11 +195,11 @@ neither offers the other anything cheaper, and both uplinks stay until one drone's demand ends. The maintainer's rule, decided 2026-11-15: merge them on the rendezvous hash unless that adds a cycle, an unfed reader, or more messages than the second uplink costs. Under adverts it does add cycles: -in 400 churned drone meshes, 20 deliveries had a pull the merge made inside +in 400 churned drone meshes, 95 deliveries had a pull the merge made inside a cycle, all transient. The one traced was a pending cycle: a merged drone's new parent lost its uplink and rejoined through a drone that had trusted the merged drone's advert, before the lost parent's answer arrived. -The merge also added about 1% unfed reader hops. Under asking on subscribe, +The merge also added about 2% unfed reader hops. Under asking on subscribe, no merged pull was ever in a cycle. So the race stands. A drone that loses the one uplink leaves every drone @@ -165,39 +230,42 @@ path in each offer. route layer's hop limit, and adverts for anything else dropped. Adverts are per broadcast like the claims the announce stream already carries over every link, so a relay hears at most one per neighbour per demanded - broadcast. An upstream holds at most one ask per downstream per demanded - broadcast, and a pull's chain is no longer than the hop limit. + broadcast. A query leaves nothing on the upstream; the edge keeps at most + one answer per upstream beyond the first per demanded broadcast. - Check the Goal's efficiency against plain path routing on the same shapes. On the simulator: pops 61 against 69 (cheapest tree 59), drones in turn one uplink against five, a ring of 48 relays 8 uplinks against 48, tiers 1095 against 1191 (the cheapest), and an edge beside the warm core 24 against - 29, at 5 to 16.5 messages per reader change against 2 to 5.4. + 29, at 4.8 to 11 messages per reader change against 2 to 5.4. - Port the growth sweep, and hold its laws, per link kind. Per reader change, warm messages are flat in the number of demanded broadcasts for every arm. - Between cores, a full peer mesh: flat in cores. A core with demand is never cheaper for another core than that core's own route. - - Down a marked link: none. Up it: one ask per upstream beyond the first, - with its answer, flat in edges per core. Together 1.6 to 2.1 per - one-viewer reader change and 3.7 to 4.2 for a crowd, from 4 to 32 cores - and 3 to 128 edges per core, up to 2,080 relays. + - Down a marked link: none. Up it: one query per upstream beyond the + first, with its answer, when an edge gains demand, flat in edges per + core. Together 1.05 to 1.55 per one-viewer reader change and 3 for a + crowd, from 4 to 32 cores and 3 to 128 edges per core, up to 2,080 + relays; the query is exactly 1 of it. - Between edges or drones that peer: n − 1 per first viewer (63 at 64 drones), since every one could take the copy; that is what buys the cheapest tree there (432 against 1,328 to 1,456 for asking and path joins, a crowd of 9 on 16 broadcasts). - At rest the busiest relay of 80 holds 518 adverts and asks for 4,096 - one-viewer broadcasts, against 4,355 metadata-only subscriptions for - `ask_1` and 139,264 adverts sent to every neighbour. + At rest the busiest relay of 80 holds 259 adverts for 4,096 one-viewer + broadcasts, against 4,355 metadata-only subscriptions for `ask_1` and + 139,264 adverts sent to every neighbour. - A region's edges do not peer every pair as it grows: they reach each other through the region's cores. Peered, the region is a drone mesh and pays its - n − 1 per first viewer (65 at 64 edges per core); the tree it buys there, + n − 1 per first viewer (64 at 64 edges per core); the tree it buys there, 288 against 544 unpeered for a crowd, comes from pricing an edge-to-edge link at a third of an uplink. - Port the cases as acceptance tests, two of them new: drones starting at once, whose uplinks the race keeps, and the same sweeps with each link delivering in order at its own pace, rather than one control message per delivery step across the mesh, which is what the simulator's unfed counts - and first frames assume. A first frame matches the path join's when readers - arrive in turn; when two edges start at once it took 262 ms against 150. + and first frames assume. A first frame matches the path join's everywhere + the sweeps measured, but two drones a hop apart (81 ms against 69). Two + edges starting at once each keep their path join (9 against 7), the race + again. - A far pop keeps pulling through a nearer pop after that pop's reader leaves, while the nearer pop's unshared offer plus the warm link stays below the far pop's own long link. That is the cheaper tree, and the @@ -216,10 +284,10 @@ then-current wip version with this. Public API: report any relay configuration added, such as a warm cost per link. Wire, in the then-current wip version and all additive: an advert on -the announce stream carrying an Instance's offer and path; the same offer and -path on the subscribe answer and its updates, and on the answer to a -metadata-only subscription; and the chain of relays a subscription feeds, on -the subscribe and an update. Nothing lite-07 freezes changes. +the announce stream carrying an Instance's offer and path, and its +withdrawal; the same offer and path on the subscribe answer and its updates; +and a query naming an Instance, answered once with an offer and path or +none. Nothing lite-07 freezes changes. ## Required diff --git a/rs/moq-net/src/route/tests/warm.rs b/rs/moq-net/src/route/tests/warm.rs index e7733b18766..6afb42c3c93 100644 --- a/rs/moq-net/src/route/tests/warm.rs +++ b/rs/moq-net/src/route/tests/warm.rs @@ -80,6 +80,10 @@ pub(super) enum Audience { /// As [`Audience::NotDown`], and a relay with demand asks every upstream besides the /// one its path join takes, on subscribe, without waiting for their answers. NotDownAskUp, + /// As [`Audience::NotDownAskUp`], but each ask is one request and one answer: the + /// upstream answers its warm offer once and keeps nothing, and the relay drops the + /// answer once a pull from that upstream is answered, live or lost. + NotDownQuery, } /// One way to find a warm copy, as measured. @@ -92,6 +96,11 @@ pub(super) struct Arm { /// Relays holding equal-cost copies merge: the lower rendezvous hash joins the /// higher, when nothing else is cheaper once it has. pub merge: bool, + /// A subscription names the relays whose demand alone it carries, and an offer + /// leaves out the demand that passes through its receiver. Without it, the rule + /// that stops the same loop: a relay never takes a warm signal from a relay named + /// on its own pull's path, whose warmth may be this relay's own demand. + pub chain: bool, } impl Arm { @@ -108,10 +117,12 @@ impl Arm { Mode::Advertise(Audience::NotDown) => "advertise_not_down".into(), Mode::Advertise(Audience::NotDownAsk) => "advertise_not_down_ask".into(), Mode::Advertise(Audience::NotDownAskUp) => "advertise_not_down_ask_up".into(), + Mode::Advertise(Audience::NotDownQuery) => "advertise_not_down_query".into(), }; let hops = if self.hops { "+hops" } else { "" }; let merge = if self.merge { "+merge" } else { "" }; - format!("{mode}{hops}{merge}") + let chain = if self.chain { "+chain" } else { "" }; + format!("{mode}{hops}{merge}{chain}") } } @@ -674,9 +685,17 @@ impl WarmSim { .chain((carry.asks.iter()).filter_map(|(&peer, signal)| Some((peer, signal.as_ref()?)))) .collect(), }; + // Without the chain, an ancestor's warmth may be this relay's own demand. + let ancestors: Vec = match self.arm.chain { + true => Vec::new(), + false => self + .live(node, cast) + .map_or_else(Vec::new, |signal| signal.path.clone()), + }; for (via, signal) in heard { if !self.is_open(node, via) || carry.down.contains_key(&via) + || ancestors.contains(&via) || (self.arm.hops && signal.path.contains(&node)) { continue; @@ -722,9 +741,24 @@ impl WarmSim { let (count, patient) = match self.arm.mode { Mode::Ask(count) => (count, true), Mode::Advertise(Audience::NotDownAsk) => (0, false), - Mode::Advertise(Audience::NotDownAskUp) => (usize::MAX, false), + Mode::Advertise(Audience::NotDownAskUp | Audience::NotDownQuery) => (usize::MAX, false), _ => return, }; + let once = self.arm.mode == Mode::Advertise(Audience::NotDownQuery); + // A one-shot answer is spent once a pull from that upstream is answered, live or + // lost: its subscribe answer says the rest. + if once { + let carry = self.entry(node, cast); + let answered: Vec = ([&carry.up, &carry.next].into_iter().flatten()) + .filter(|pull| pull.state != State::Pending) + .map(|pull| pull.via) + .collect(); + for via in answered { + if let Some(word) = carry.asks.get_mut(&via) { + *word = None; + } + } + } let carry = self.entry(node, cast).clone(); let want: BTreeSet = match () { // Its upstreams, when its route ties between more than one. @@ -749,7 +783,10 @@ impl WarmSim { let carry = self.entry(node, cast); carry.asks.remove(&gone); carry.unanswered.remove(&gone); - if let Some(stream) = carry.asking.remove(&gone) { + // The upstream keeps nothing for a one-shot query, so there is nothing to close. + if let Some(stream) = carry.asking.remove(&gone) + && !once + { self.send(node, gone, Lane::Ask, cast, Msg::Ask { stream, open: false }); } } @@ -794,9 +831,10 @@ impl WarmSim { fn subscribe(&mut self, node: Node, cast: usize, via: Node) -> Pull { self.streams += 1; let stream = self.streams; - let chain = self - .carry(node, cast) - .map_or_else(|| vec![node], |carry| carry.chain(node)); + let chain = match self.arm.chain { + true => (self.carry(node, cast)).map_or_else(|| vec![node], |carry| carry.chain(node)), + false => Vec::new(), + }; let open = Msg::Subscribe { stream, open: true, @@ -815,7 +853,7 @@ impl WarmSim { /// Tell each parent when the relays whose demand alone its pull carries change. fn own(&mut self, node: Node, cast: usize) { // A join along the route prices nothing, so it needs no chain. - if self.arm.mode == Mode::Path { + if self.arm.mode == Mode::Path || !self.arm.chain { return; } let carry = self.entry(node, cast); @@ -987,12 +1025,14 @@ impl WarmSim { /// Tell each neighbour pulling through `node` what it offers now. fn answer(&mut self, node: Node, cast: usize) { let carry = self.entry(node, cast).clone(); + // Neither pulling nor trying to: a new subscriber hears so at once. + let stuck = carry.next.is_none() && carry.up.as_ref().is_none_or(|pull| pull.state == State::Lost); for (&child, streams) in &carry.down { let offer = self.offer(node, cast, child); for &stream in streams { let was = carry.answered.get(&(child, stream)); - // Unanswered until confirmed, then every change. - if (was.is_none() && offer.is_none()) || was == Some(&offer) { + // Unanswered until confirmed or lost, then every change. + if (was.is_none() && offer.is_none() && !stuck) || was == Some(&offer) { continue; } self.entry(node, cast).answered.insert((child, stream), offer.clone()); @@ -1014,6 +1054,12 @@ impl WarmSim { self.entry(node, cast).told.insert((asker, stream), word.clone()); let word = word.clone(); self.send(node, asker, Lane::Ask, cast, Msg::Told { stream, word }); + // A one-shot query is answered once and forgotten. + if self.arm.mode == Mode::Advertise(Audience::NotDownQuery) { + let carry = self.entry(node, cast); + close(&mut carry.asked, asker, stream); + carry.told.remove(&(asker, stream)); + } } } } @@ -1058,7 +1104,7 @@ impl WarmSim { Audience::All => true, Audience::Useful => useful(), Audience::NotNext => matches!(heard, Some((cost, false)) if price <= cost), - Audience::NotDown | Audience::NotDownAsk | Audience::NotDownAskUp => { + Audience::NotDown | Audience::NotDownAsk | Audience::NotDownAskUp | Audience::NotDownQuery => { !self.upstream.contains(&(peer, node)) && useful() } Audience::Shared => self.carry(node, cast).is_some_and(|carry| { @@ -1456,24 +1502,14 @@ impl WarmSim { self.watch.longest = self.watch.longest.max(self.watch.run); } - /// The relays a path over open links joins to `cast`'s publisher. + /// The relays the route layer gives a route to `cast`'s publisher, which a pull + /// can follow. A path over open links is not enough: a relay announces only its + /// best route, and never one that came down a marked link back up one. fn reach(&self, cast: usize) -> BTreeSet { let start = self.endpoints[&self.casts[cast].0]; - // A route that came down a link marked upstream never goes back up one. - let mut seen = BTreeSet::from([(start, false)]); - let mut queue = vec![(start, false)]; - while let Some((at, marked)) = queue.pop() { - for peer in self.peers(at) { - if !self.relay(peer) || (marked && self.upstream.contains(&(at, peer))) { - continue; - } - let next = (peer, marked || self.upstream.contains(&(peer, at))); - if seen.insert(next) { - queue.push(next); - } - } - } - seen.into_iter().map(|(node, _)| node).collect() + (0..self.relays) + .filter(|&node| node == start || self.route(node, cast).is_some()) + .collect() } /// Count unfed readers as a round ends, and the first frames that arrived. @@ -1497,7 +1533,7 @@ impl WarmSim { } } - /// Every reader with a path to its publisher is fed, no pulls circle, and nothing is + /// Every reader with a route to its publisher is fed, no pulls circle, and nothing is /// left to deliver. pub fn check(&self) -> Result<(), String> { if self.queues.values().any(|queue| !queue.is_empty()) || !self.net.ready().is_empty() { diff --git a/rs/moq-net/src/route/tests/warmth.rs b/rs/moq-net/src/route/tests/warmth.rs index 08999381491..83e172296a8 100644 --- a/rs/moq-net/src/route/tests/warmth.rs +++ b/rs/moq-net/src/route/tests/warmth.rs @@ -19,6 +19,7 @@ fn arms() -> Vec { mode: Mode::Path, hops, merge: false, + chain: false, }); } for mode in [ @@ -30,7 +31,12 @@ fn arms() -> Vec { ] { for hops in [true, false] { for merge in [false, true] { - out.push(Arm { mode, hops, merge }); + out.push(Arm { + mode, + hops, + merge, + chain: false, + }); } } } @@ -42,6 +48,7 @@ const fn arm(mode: Mode, merge: bool) -> Arm { mode, hops: true, merge, + chain: false, } } @@ -475,11 +482,18 @@ const NOT_NEXT: Arm = arm(Mode::Advertise(Audience::NotNext), false); const NOT_DOWN: Arm = arm(Mode::Advertise(Audience::NotDown), false); const NOT_DOWN_ASK: Arm = arm(Mode::Advertise(Audience::NotDownAsk), false); const NOT_DOWN_ASK_UP: Arm = arm(Mode::Advertise(Audience::NotDownAskUp), false); - -/// The narrower audiences for useful adverts, which keep them off a CDN's edges. The -/// last, [`NOT_DOWN_ASK_UP`], is the pick. +const NOT_DOWN_QUERY: Arm = arm(Mode::Advertise(Audience::NotDownQuery), false); +/// The pick with the chain in place of the rule that a relay never takes a warm +/// signal from one named on its own pull's path. +const CHAIN: Arm = Arm { + chain: true, + ..NOT_DOWN_QUERY +}; + +/// The narrower audiences for useful adverts, which keep them off a CDN's edges. +/// [`NOT_DOWN_QUERY`] is the pick, and [`CHAIN`] the pick with the chain. fn narrower() -> Vec { - vec![NOT_NEXT, NOT_DOWN, NOT_DOWN_ASK, NOT_DOWN_ASK_UP] + vec![NOT_NEXT, NOT_DOWN, NOT_DOWN_ASK, NOT_DOWN_ASK_UP, NOT_DOWN_QUERY, CHAIN] } /// The cheaper ways to advertise measured beside the rest: the narrower audiences, @@ -663,12 +677,12 @@ fn useful_adverts_do_not_grow_with_the_core_mesh() { assert!(warm(ADVERTISE, 4) < warm(ADVERTISE, 8)); } -/// Under the pick a warm core tells none of its region's edges, and each edge asks -/// the upstreams beside its path join on subscribe, so a first viewer's warm messages -/// stay flat as a region grows its edges and as the core mesh grows. Useful adverts -/// grow with the edges, since a warm core tells every edge it could feed. +/// Under the pick a warm core tells none of its region's edges, and each edge queries +/// the upstreams beside its path join once on subscribe, so a first viewer's warm +/// messages stay flat as a region grows its edges and as the core mesh grows. Useful +/// adverts grow with the edges, since a warm core tells every edge it could feed. #[test] -fn asks_toward_upstreams_do_not_grow_with_the_edges() { +fn queries_toward_upstreams_do_not_grow_with_the_edges() { let warm = |arm: Arm, regions: usize, edges: usize| { let shape = cdn(regions, edges, false, 1); // The first edge of region 1, which the publisher's core is not in. @@ -676,13 +690,15 @@ fn asks_toward_upstreams_do_not_grow_with_the_edges() { assert_eq!(grown.right, Ok(()), "{}", arm.name()); grown.sent.warm() }; - assert_eq!(warm(NOT_DOWN_ASK_UP, 4, 3), warm(NOT_DOWN_ASK_UP, 4, 16)); - assert_eq!(warm(NOT_DOWN_ASK_UP, 4, 3), warm(NOT_DOWN_ASK_UP, 8, 3)); + for arm in [NOT_DOWN_QUERY, NOT_DOWN_ASK_UP] { + assert_eq!(warm(arm, 4, 3), warm(arm, 4, 16), "{}", arm.name()); + assert_eq!(warm(arm, 4, 3), warm(arm, 8, 3), "{}", arm.name()); + } assert!(warm(USEFUL, 4, 3) < warm(USEFUL, 4, 16)); } /// An edge whose cheaper uplink is to the cold core reaches the warm core only on a -/// warm signal over an upstream link: an advert down it, or the edge's own ask. +/// warm signal over an upstream link: an advert down it, or the edge's own query. /// Skipping both leaves it on its path join. #[test] fn an_edge_finds_the_warm_core_beside_its_route() { @@ -693,7 +709,7 @@ fn an_edge_finds_the_warm_core_beside_its_route() { let cheapest: Metric = rows.iter().flat_map(|row| row.cheapest.iter()).sum(); (cost, cheapest) }; - for arm in [USEFUL, NOT_DOWN_ASK_UP] { + for arm in [USEFUL, NOT_DOWN_ASK_UP, NOT_DOWN_QUERY] { let (cost, cheapest) = cost(arm); assert_eq!(cost, cheapest, "{}", arm.name()); } @@ -977,6 +993,8 @@ fn every_arm_settles_after_churn() { ADVERTISE_MERGE, USEFUL, NOT_DOWN_ASK_UP, + NOT_DOWN_QUERY, + CHAIN, ] { for shape in [random as fn(&mut XorShift) -> Shape, flying, regional] { let out = churn(arm, 0..12, shape); @@ -1151,7 +1169,7 @@ fn mean(values: &[usize]) -> f64 { #[ignore = "a benchmark; Nightly runs it from test/route-sweep.sh"] fn benchmark_warmth_cases() { println!( - "case,arm,phases,tree_cost,path_cost,cheapest_cost,uplinks,max_copies,carriers,messages,warm_messages,messages_per_reader_change,warm_per_reader_change,first_frame_ms,cycle_steps,longest_cycle,unfed_reader_rounds,right" + "case,arm,phases,tree_cost,path_cost,cheapest_cost,uplinks,max_copies,carriers,messages,warm_messages,messages_per_reader_change,warm_per_reader_change,first_frame_ms,cycle_steps,longest_cycle,unfed_reader_rounds,advert_messages,query_messages,chain_messages,right" ); let ms = |rounds: f64| rounds * HOP.as_secs_f64() * 1000.0; for base in cases().into_iter().map(Some).chain([None]) { @@ -1171,7 +1189,7 @@ fn benchmark_warmth_cases() { let end = rows.last().expect("a phase"); let right = rows.iter().find_map(|row| row.right.clone().err()); println!( - "{},{},{},{cost},{},{cheapest},{},{},{},{},{},{:.2},{:.2},{:.0},{},{},{},{}", + "{},{},{},{cost},{},{cheapest},{},{},{},{},{},{:.2},{:.2},{:.0},{},{},{},{},{},{},{}", case.name, arm.name(), rows.len(), @@ -1187,30 +1205,39 @@ fn benchmark_warmth_cases() { sum(|row| row.watch.cycles), rows.iter().map(|row| row.watch.longest).max().unwrap_or(0), sum(|row| row.watch.unfed), + sent.adverts, + sent.asks + sent.told, + sent.chains, right.map_or("yes".into(), |err| err.replace(',', ";")), ); } } } -/// Per arm, random meshes and drone meshes under churn: cycles, unfed readers, and +/// Per arm, random meshes, drone meshes, and CDN regions under churn: cycles, unfed readers, and /// standing races while it churned, and whether each settled right once it stopped. #[test] #[ignore = "a benchmark; Nightly runs it from test/route-sweep.sh"] fn benchmark_warmth_churn() { println!( - "mesh,arm,cases,deliveries,cycle_deliveries,starved_deliveries,merge_cycle_deliveries,longest_cycle,unfed_reader_hops,extra_uplink_hops,reader_changes,messages,warm_messages,settle_rounds,failed" + "mesh,arm,cases,deliveries,cycle_deliveries,starved_deliveries,merge_cycle_deliveries,longest_cycle,unfed_reader_hops,extra_uplink_hops,reader_changes,messages,warm_messages,advert_messages,query_messages,chain_messages,settle_rounds,failed" ); let cases: usize = std::env::var("MOQ_WARMTH_CASES").map_or(400, |cases| cases.parse().expect("MOQ_WARMTH_CASES")); + // A comma-separated list of arm names to run, for a longer sweep of a few. + let only = std::env::var("MOQ_WARMTH_ARMS").ok(); + let picked = |arm: &Arm| { + only.as_ref() + .is_none_or(|only| only.split(',').any(|name| name == arm.name())) + }; for (name, shape) in [ ("random", random as fn(&mut XorShift) -> Shape), ("drones", flying), ("regional", regional), ] { - for arm in arms().into_iter().chain(audiences()) { + for arm in arms().into_iter().chain(audiences()).filter(picked) { let out = churn(arm, 0..cases, shape); println!( - "{name},{},{},{},{},{},{},{},{},{},{},{},{},{},{}", + "{name},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}", arm.name(), out.cases, out.watch.steps, @@ -1223,6 +1250,9 @@ fn benchmark_warmth_churn() { out.changes, out.sent.total(), out.sent.warm(), + out.sent.adverts, + out.sent.asks + out.sent.told, + out.sent.chains, out.settle, out.first.map_or("0".into(), |first| format!( "{}: {}", From 1a6d59f52bd4652497160159fb66d2ed80f35a38 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 11:25:23 -0800 Subject: [PATCH 08/19] test(moq-net): rename the warmth growth sweep's Sized to Size so it no longer shadows the prelude trait Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/tests/warmth.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rs/moq-net/src/route/tests/warmth.rs b/rs/moq-net/src/route/tests/warmth.rs index 83e172296a8..03ca65e2573 100644 --- a/rs/moq-net/src/route/tests/warmth.rs +++ b/rs/moq-net/src/route/tests/warmth.rs @@ -720,7 +720,7 @@ fn an_edge_finds_the_warm_core_beside_its_route() { } /// One CSV row of the growth sweeps. -struct Sized { +struct Size { shape: Shape, /// Cores and edges per core, or the CDN relay and its drones. cores: usize, @@ -749,7 +749,7 @@ fn benchmark_warmth_growth() { let mut sizes = Vec::new(); let region = |regions: usize, edges: usize, peered: bool, arms: Vec, wide: bool| { let cores = 2 * regions; - Sized { + Size { shape: cdn(regions, edges, peered, 1), cores, edges, @@ -778,7 +778,7 @@ fn benchmark_warmth_growth() { } } for count in [5, 16, 64] { - sizes.push(Sized { + sizes.push(Size { shape: drones(count), cores: 1, edges: count, From 7ad6c0f1ba07be67b30ec9120e6f613a4ac10942 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 12:16:19 -0800 Subject: [PATCH 09/19] feat(moq-net): keep the warmth pick's adverts to the route layer's mark, count late routes as unfed, and price the ancestor rule The pick advertised one core's copy from an edge up to its other core, so the core pulled through the edge, a path the route layer never offers. An advert now obeys the route layer's mark: none up a marked link of a copy that came over one. On an unpeered CDN the pick sends exactly the query, 1.0 warm message per reader change, and a core holds nothing at rest. The churn tally counts a reader joined over open links as unfed while its route is still on the way, and churn reports the settled trees against the cheapest for the same readers. A fixture records what the ancestor rule gives up: a move up the taker's branch to a relay warm on its own account. Co-Authored-By: Claude Opus 5.5 --- quest/m3/cache-routing/select.md | 87 +++++++++++++++++----------- rs/moq-net/src/route/tests/warm.rs | 59 ++++++++++++++++--- rs/moq-net/src/route/tests/warmth.rs | 49 +++++++++++++++- 3 files changed, 149 insertions(+), 46 deletions(-) diff --git a/quest/m3/cache-routing/select.md b/quest/m3/cache-routing/select.md index d8c58f0e219..44bde028bf0 100644 --- a/quest/m3/cache-routing/select.md +++ b/quest/m3/cache-routing/select.md @@ -21,18 +21,20 @@ Plan, is replaced. Recommended 2026-11-16 on [moq-dev/moq#8903](https://github.com/moq-dev/moq/pull/8903), awaiting the maintainer's confirmation at merge: **adverts between peers and toward -upstreams, only where the copy could be taken; none down a link the neighbour -marked upstream, where an edge instead queries its other upstreams once when -it gains demand; the data path in every offer; no warm signal taken from a -relay on the taker's own path; and the race left standing**. Four parts: -adverts, the path, the query, and one rule. +upstreams, only where the copy could be taken, never down a link the neighbour +marked upstream, and never up one with a copy that came over one; an edge +instead queries its other upstreams once when it gains demand; the data path +in every offer; no warm signal taken from a relay on the taker's own path; and +the race left standing**. Four parts: adverts, the path, the query, and one +rule. Adverts reuse the route layer's mark: it passes no route from one marked +link to another, so an edge carries nothing between its cores. The maintainer's decision is whether to keep the query, below. Rejected: - Advertising to every neighbour that could take the copy, down marked links too: it grows with a region's edges, since a warm core tells every edge it could feed. 1.7 to 118 warm messages per one-viewer reader change as edges - per core grow from 3 to 128, against the pick's flat 1.05 to 1.3. + per core grow from 3 to 128, against the pick's flat 1.0. - Advertising to every neighbour, which also grows with the neighbours of every relay on the warm chain: 9 to 66 per one-viewer reader change as a full core mesh grows from 4 to 32 cores. @@ -41,8 +43,8 @@ The maintainer's decision is whether to keep the query, below. Rejected: messages or none fewer. - A standing query, held open for updates while the demand lasts: the same trees as the one-shot query, but the upstream holds an entry per edge per - demanded broadcast (41 against 9 per core) and it left 1 of 400 churned - regions unsettled. + demanded broadcast (32 per core, against none) and it left 1 of 400 + churned regions unsettled. - Naming in every subscription the relays whose demand alone it carries, so an offer leaves them out: a second fix for the loop the rule stops. It adds a list and an update message on each change (6,913 to 8,469 per 400 @@ -54,7 +56,7 @@ The maintainer's decision is whether to keep the query, below. Rejected: past 5 drones, `ask_1` shared no uplink at 64 drones and `ask_2` few. It doubled the time to a first frame and left about twice the readers unfed under churn. On CDN tiers `ask_1` sent 5.7 to 7 warm messages per - one-viewer reader change against the pick's 1.05 to 1.55. + one-viewer reader change against the pick's 1.0. - Advertising only once two others want the copy, which lost both cases; never advertising over a link the sender marked upstream, which saves nothing in a core mesh. @@ -67,8 +69,8 @@ sweep this ports onto production code. ### Decision: the query The query is the one part that only an edge with more than one upstream -uses. Without it the pick is adverts between peers and toward upstreams, the -path, and the rule. With equal uplinks the query buys nothing: every edge of +uses. Without it the pick is adverts, the path, and the rule, and an unpeered +CDN sends no warm message at all. With equal uplinks the query buys nothing: every edge of a region ranks the two cores by the same rendezvous hash per broadcast, so every edge's path join lands on the same core, always; path joins already built the cheapest tree on every unpeered CDN shape swept. It pays only where an @@ -76,31 +78,35 @@ edge's cheaper uplink is to the core that is cold. | shape | tree, with / without the query (cheapest) | warm messages per reader change, with / without | |---|---|---| -| CDN, 8 cores, 16 edges per core, one viewer | 2236 / 2236 (2236) | 1.21 / 0.21 | -| CDN, 8 cores, 16 edges per core, crowd | 544 / 544 (544) | 3.00 / 2.00 | -| CDN, 32 cores, 3 edges per core, crowd | 256 / 256 (256) | 3.00 / 2.00 | -| tiers, crowd (edges peered) | 1095 / 1095 (1095) | 5.29 / 3.95 | +| CDN, 8 cores, 16 edges per core, one viewer | 2236 / 2236 (2236) | 1.00 / 0.00 | +| CDN, 8 cores, 16 edges per core, crowd | 544 / 544 (544) | 1.00 / 0.00 | +| CDN, 32 cores, 3 edges per core, crowd | 256 / 256 (256) | 1.00 / 0.00 | +| tiers, crowd (edges peered) | 1095 / 1095 (1095) | 4.71 / 3.38 | | tiers, one viewer | 1096 / 1096 (1096) | 5.50 / 3.50 | -| two edges, unequal uplinks, in turn | 12 / 13 (12) | 3.50 / 1.00 | +| two edges, unequal uplinks, in turn | 12 / 13 (12) | 2.50 / 0.50 | | two edges, unequal uplinks, at once | 9 / 9 (7) | 3.00 / 1.00 | -| one region, unequal uplinks | 24 / 29 (24) | 3.00 / 0.00 | -| 400 churned regions: unfed reader-hops, warm messages | 17,181 / 12,586 | 24,446 / 8,602 in all | +| one region, unequal uplinks | 24 / 29 (24) | 2.00 / 0.00 | +| 400 churned regions: settled trees over cheapest; unfed reader-hops | +4.9% / +5.3%; 22,576 / 16,805 | 21,217 / 4,858 in all | Recommended: keep the query. Where uplinks are unequal, it saves the second core's copy over the core mesh, the dearest link a region has. It costs one request and one answer per extra upstream per first viewer at an edge, holds -nothing on the core, and carries no traffic. Without it an operator gets the -same tree by pricing each edge's uplinks equally. +nothing on the core, and carries no traffic. Under churn it is dearer: 4.4 +times the warm messages and 34% more unfed reader-hops, much of which is the +simulator delivering one control message per step across the whole mesh, so +every query delays a route or a subscribe somewhere else. Without it an +operator gets the same tree by pricing each edge's uplinks equally. ### What each part costs | part | on the wire | relay state | bounded by | without it | |---|---|---|---|---| | advert | offer and path on the announce stream, and its withdrawal | one heard per neighbour per demanded Instance | neighbours; the path by the hop limit | path joins: pops 69 against 61, drones in turn 5 uplinks against 1, a ring 48 against 8 | +| the route layer's mark on adverts | none: each link's mark is configured already | none | none | an edge carries a copy between its cores, which no route would: 51 readers in 400 churned regions fed that way, cycle deliveries 213 against 90, and 1.05 to 1.55 warm messages per one-viewer reader change on an unpeered CDN against 1.0 | | offer on the subscribe answer | a cost on the answer and its updates | one per pull | one per subscription | nothing can price a move | | path in the offer | a list on the advert, answer, and query answer | as the offer | the hop limit | cycle deliveries up 77%, 26%, and 110% on random meshes, drones, and regions; 3 of 1,200 churned meshes unsettled | | query | a request naming an Instance; one answer, offer and path or none | the edge keeps one answer per extra upstream per demanded Instance; the upstream none | upstreams beyond the first | an edge whose cheaper uplink is cold: 29 against 24 | -| no warm signal from a relay on the taker's path | none | none, the path of its own pull | none | 1 of 400 churned regions never settles | +| no warm signal from a relay on the taker's path | none | none, the path of its own pull | none | 1 of 400 churned regions never settles. Its price: a move up the taker's own branch to a relay warm on its own account, 14 against 13 in `an_ancestor_warm_on_its_own_is_passed_over`; under churn the chain, which keeps that move, settled on trees no cheaper (+2.9%, +8.7%, +5.2% over cheapest against the rule's +2.8%, +8.8%, +4.9%) | The rule replaces a chain in every subscription naming the relays whose demand alone it carries. The offer's path cannot stand in for the chain on @@ -119,8 +125,9 @@ of its own pull is enough. rose by 77% on random meshes, 26% on drone meshes, and 110% on regions, 1 random mesh and 2 regions of 400 never settled, and the merge below cannot work. -- **Adverts.** While R has demand for an Instance besides N, and N has not - marked R upstream, it tells N its offer and path on the announce stream +- **Adverts.** While R has demand for an Instance besides N, N has not + marked R upstream, and R has not both marked N upstream and taken its copy + over a marked link, it tells N its offer and path on the announce stream when `O(R→N) + cold(R, N)` is below N's route cost for the Instance, or equals it through a next hop other than R, since a tie prefers a warm copy. It tells again on each change, and withdraws when the demand, its upstream, @@ -128,7 +135,10 @@ of its own pull is enough. relay at rest costs no more than its own route, and a tie through R is the path join already. R knows N's route cost from what N announces it, and computes it when N's route runs through R. Skipping the parent and children - by rule instead left 8 of 400 churned drone meshes unsettled. + by rule instead left 8 of 400 churned drone meshes unsettled. The mark is + the route layer's own rule, a route from one marked link never passed to + another: without it an edge advertised one core's copy to its other core, + which then pulled through the edge, a path no route offers. - **The query.** A relay with more than one upstream sends each upstream besides the one its path join takes one query when it gains demand: one request, one answer. The upstream answers its warm offer and path, or that @@ -164,7 +174,11 @@ of its own pull is enough. 400 churned regions never settled that way under most warm arms; with it, none of 2,000 each of random meshes, drone meshes, and regions under the pick. A relay on N's path is upstream of N's parent, so N gives up only a move up its own branch, which its route still offers - when that is the cheaper way. + when that is the cheaper way. It costs a move to a relay on N's path that + is warm on its own account, which N cannot tell apart from its own demand: + in `an_ancestor_warm_on_its_own_is_passed_over` N keeps a tree of 14 where + 13 is cheapest, but under churn the trees it settled on cost no more than + the chain's, which keeps such moves. - A move is make before break: the old pull stays until the new one is answered live, and a move under way stays while it is no dearer than staying. Its own subscription shares the old parent, which lowers what @@ -185,7 +199,7 @@ Gaining demand lowers offers to others, and the one it lowers to the relay whose own demand it is, that relay ignores; losing it raises only the offer to the neighbour that still pulls, which then moves only to something strictly cheaper. The sweep agrees, though it is no proof: every churned random mesh, -drone mesh, and region settled under the pick, and cycles lasted at most 61 +drone mesh, and region settled under the pick, and cycles lasted at most 57 deliveries, each while some answer was in flight. ### Simultaneous starts @@ -199,7 +213,7 @@ in 400 churned drone meshes, 95 deliveries had a pull the merge made inside a cycle, all transient. The one traced was a pending cycle: a merged drone's new parent lost its uplink and rejoined through a drone that had trusted the merged drone's advert, before the lost parent's answer arrived. -The merge also added about 2% unfed reader hops. Under asking on subscribe, +The merge also added about 1.5% unfed reader hops. Under asking on subscribe, no merged pull was ever in a cycle. So the race stands. A drone that loses the one uplink leaves every drone @@ -236,23 +250,26 @@ path in each offer. On the simulator: pops 61 against 69 (cheapest tree 59), drones in turn one uplink against five, a ring of 48 relays 8 uplinks against 48, tiers 1095 against 1191 (the cheapest), and an edge beside the warm core 24 against - 29, at 4.8 to 11 messages per reader change against 2 to 5.4. + 29, at 4.3 to 11 messages per reader change against 2 to 5.4. Under churn, + the trees each case settled on cost 2.8%, 8.8%, and 4.9% over the cheapest + for the same readers on random meshes, drones, and regions, against path + joins' 2.7%, 28.9%, and 5.0%. - Port the growth sweep, and hold its laws, per link kind. Per reader change, warm messages are flat in the number of demanded broadcasts for every arm. - Between cores, a full peer mesh: flat in cores. A core with demand is never cheaper for another core than that core's own route. - Down a marked link: none. Up it: one query per upstream beyond the - first, with its answer, when an edge gains demand, flat in edges per - core. Together 1.05 to 1.55 per one-viewer reader change and 3 for a - crowd, from 4 to 32 cores and 3 to 128 edges per core, up to 2,080 - relays; the query is exactly 1 of it. + first, with its answer, when an edge gains demand, and no advert of a + copy that came down another. Exactly 1.0 per reader change, one viewer or + a crowd, flat from 4 to 32 cores and 3 to 128 edges per core, up to + 2,080 relays; a core holds nothing at rest. - Between edges or drones that peer: n − 1 per first viewer (63 at 64 drones), since every one could take the copy; that is what buys the cheapest tree there (432 against 1,328 to 1,456 for asking and path joins, a crowd of 9 on 16 broadcasts). - At rest the busiest relay of 80 holds 259 adverts for 4,096 one-viewer - broadcasts, against 4,355 metadata-only subscriptions for `ask_1` and - 139,264 adverts sent to every neighbour. + At rest the busiest relay of 80 holds 99 query answers for 4,096 + one-viewer broadcasts, against 4,355 metadata-only subscriptions for + `ask_1` and 139,264 adverts sent to every neighbour. - A region's edges do not peer every pair as it grows: they reach each other through the region's cores. Peered, the region is a drone mesh and pays its n − 1 per first viewer (64 at 64 edges per core); the tree it buys there, diff --git a/rs/moq-net/src/route/tests/warm.rs b/rs/moq-net/src/route/tests/warm.rs index 6afb42c3c93..187ead03dfc 100644 --- a/rs/moq-net/src/route/tests/warm.rs +++ b/rs/moq-net/src/route/tests/warm.rs @@ -72,7 +72,9 @@ pub(super) enum Audience { /// against another warm neighbour. NotNext, /// As [`Audience::Useful`], but never over a link the neighbour marked upstream: an - /// edge relies on its path join and the rendezvous hash among equal upstreams. + /// edge relies on its path join and the rendezvous hash among equal upstreams. Nor + /// up a marked link with a copy that came over one, as the route layer never passes + /// a route from one marked link to another: an edge carries nothing between cores. NotDown, /// As [`Audience::NotDown`], and a relay with demand whose route ties between /// upstreams asks them on subscribe, without waiting for their answers. @@ -1086,6 +1088,17 @@ impl WarmSim { } } + /// Whether `node`'s copy, which `signal` offers, came over a link its receiver marked + /// upstream: the route layer passes no such route on to another marked neighbour. + fn crossed(&self, node: Node, cast: usize, signal: &Signal) -> bool { + let up = self.carry(node, cast).and_then(|carry| carry.up.as_ref()); + up.is_some_and(|pull| self.upstream.contains(&(node, pull.via))) + || signal + .path + .windows(2) + .any(|hop| self.upstream.contains(&(hop[1], hop[0]))) + } + /// Whether `audience` lets `node` advertise `signal` to `peer`. fn scoped(&self, audience: Audience, node: Node, cast: usize, peer: Node, signal: &Signal) -> bool { let price = signal.cost + self.price(node, peer).cold; @@ -1105,7 +1118,9 @@ impl WarmSim { Audience::Useful => useful(), Audience::NotNext => matches!(heard, Some((cost, false)) if price <= cost), Audience::NotDown | Audience::NotDownAsk | Audience::NotDownAskUp | Audience::NotDownQuery => { - !self.upstream.contains(&(peer, node)) && useful() + !self.upstream.contains(&(peer, node)) + && !(self.upstream.contains(&(node, peer)) && self.crossed(node, cast, signal)) + && useful() } Audience::Shared => self.carry(node, cast).is_some_and(|carry| { let children = carry.down.keys().filter(|&&child| child != peer).count(); @@ -1502,9 +1517,25 @@ impl WarmSim { self.watch.longest = self.watch.longest.max(self.watch.run); } + /// The relays joined to `cast`'s publisher over open links through relays, whether + /// or not their route has arrived yet. + fn connected(&self, cast: usize) -> BTreeSet { + let start = self.endpoints[&self.casts[cast].0]; + let mut seen = BTreeSet::from([start]); + let mut queue = vec![start]; + while let Some(at) = queue.pop() { + for peer in self.peers(at) { + if self.relay(peer) && seen.insert(peer) { + queue.push(peer); + } + } + } + seen + } + /// The relays the route layer gives a route to `cast`'s publisher, which a pull - /// can follow. A path over open links is not enough: a relay announces only its - /// best route, and never one that came down a marked link back up one. + /// can follow once settled. A path over open links is not enough: a relay announces + /// only its best route, and never one that came down a marked link back up one. fn reach(&self, cast: usize) -> BTreeSet { let start = self.endpoints[&self.casts[cast].0]; (0..self.relays) @@ -1512,14 +1543,19 @@ impl WarmSim { .collect() } - /// Count unfed readers as a round ends, and the first frames that arrived. + /// Count unfed readers as a round ends, and the first frames that arrived. A reader + /// counts while joined over open links, so a late route is an outage as much as a + /// late copy; one the route layer never routes counts alike for every arm. fn tally(&mut self) { let round = self.round; let mut reach: BTreeMap> = BTreeMap::new(); for (node, cast) in self.readers() { let fed = self.fed(node, cast) && self.live(node, cast).is_some(); let start = self.endpoints[&self.casts[cast].0]; - let path = reach.entry(start).or_insert_with(|| self.reach(cast)).contains(&node); + let path = reach + .entry(start) + .or_insert_with(|| self.connected(cast)) + .contains(&node); self.watch.unfed += usize::from(path && !fed); let carry = self.entry(node, cast); if fed && let Some(since) = carry.since.take() { @@ -1635,12 +1671,17 @@ impl WarmSim { } } - /// The cheapest tree joining `cast`'s publisher to every relay reading it, over open - /// links at their cold prices: Dreyfus and Wagner's exact dynamic program. + /// The cheapest tree joining `cast`'s publisher to every relay reading it that has a + /// route or a copy, over open links at their cold prices: Dreyfus and Wagner's exact + /// dynamic program. pub fn cheapest(&self, cast: usize) -> Metric { let endpoint = self.endpoints[&self.casts[cast].0]; + let reach = self.reach(cast); let mut terminals = vec![endpoint]; - terminals.extend((0..self.relays).filter(|&node| self.reading(node, cast))); + terminals.extend( + (0..self.relays) + .filter(|&node| self.reading(node, cast) && (reach.contains(&node) || self.fed(node, cast))), + ); let n = self.nodes; let count = terminals.len(); let full = (1usize << count) - 1; diff --git a/rs/moq-net/src/route/tests/warmth.rs b/rs/moq-net/src/route/tests/warmth.rs index 03ca65e2573..adf0ef7aa3d 100644 --- a/rs/moq-net/src/route/tests/warmth.rs +++ b/rs/moq-net/src/route/tests/warmth.rs @@ -719,6 +719,41 @@ fn an_edge_finds_the_warm_core_beside_its_route() { } } +/// The price of taking no warm signal from a relay on the taker's own path. Relay 1 +/// is warm for a reader of its own, and 3 pulls through 1 and 2 at 4. Once the +/// direct link 1 to 3 opens, 1's warm copy is 3 away, but 1 is 3's ancestor, so the +/// pick stays: its tree costs 14 where 13 is cheapest, the publisher's link included. The chain, which tells 1 that +/// 2's pull is 3's demand, takes it. +#[test] +fn an_ancestor_warm_on_its_own_is_passed_over() { + let shape = Shape { + name: "ancestor", + relays: 4, + links: vec![ + (0, 1, Price::new(11, 2), Tie::Peer), + (1, 2, Price::new(1, 1), Tie::Peer), + (2, 3, Price::new(1, 1), Tie::Peer), + (1, 3, Price::new(1, 1), Tie::Peer), + ], + home: 0, + claims: 1, + }; + let tree = |arm: Arm| { + let mut sim = shape.build(arm, 7); + sim.close(1, 3); + sim.read(1, 0, true); + sim.read(3, 0, true); + assert!(sim.settle(), "{}", arm.name()); + assert_eq!(sim.tree(0).cost, 14, "{}", arm.name()); + sim.reopen(1, 3); + assert!(sim.settle(), "{}", arm.name()); + assert_eq!(sim.check(), Ok(()), "{}", arm.name()); + (sim.tree(0).cost, sim.cheapest(0)) + }; + assert_eq!(tree(NOT_DOWN_QUERY), (14, 13)); + assert_eq!(tree(CHAIN), (13, 13)); +} + /// One CSV row of the growth sweeps. struct Size { shape: Shape, @@ -850,6 +885,10 @@ struct Churned { first: Option, /// Rounds each case took to settle once churn stopped, summed. settle: usize, + /// The cost of the trees each case settled on, and of the cheapest trees for the + /// same readers, summed over the cases that settled right. + tree: Metric, + cheapest: Metric, } /// A random mesh of four to seven relays, a chain and about two in five of the @@ -923,6 +962,10 @@ fn churn(arm: Arm, cases: std::ops::Range, shape: fn(&mut XorShift) -> Sh true => sim.check(), false => Err("never settled".into()), }; + if right.is_ok() { + out.tree += (0..shape.claims).map(|cast| sim.tree(cast).cost).sum::(); + out.cheapest += (0..shape.claims).map(|cast| sim.cheapest(cast)).sum::(); + } if let Err(err) = right { out.failed += 1; out.first @@ -1220,7 +1263,7 @@ fn benchmark_warmth_cases() { #[ignore = "a benchmark; Nightly runs it from test/route-sweep.sh"] fn benchmark_warmth_churn() { println!( - "mesh,arm,cases,deliveries,cycle_deliveries,starved_deliveries,merge_cycle_deliveries,longest_cycle,unfed_reader_hops,extra_uplink_hops,reader_changes,messages,warm_messages,advert_messages,query_messages,chain_messages,settle_rounds,failed" + "mesh,arm,cases,deliveries,cycle_deliveries,starved_deliveries,merge_cycle_deliveries,longest_cycle,unfed_reader_hops,extra_uplink_hops,reader_changes,messages,warm_messages,advert_messages,query_messages,chain_messages,settle_rounds,settled_tree_cost,settled_cheapest_cost,failed" ); let cases: usize = std::env::var("MOQ_WARMTH_CASES").map_or(400, |cases| cases.parse().expect("MOQ_WARMTH_CASES")); // A comma-separated list of arm names to run, for a longer sweep of a few. @@ -1237,7 +1280,7 @@ fn benchmark_warmth_churn() { for arm in arms().into_iter().chain(audiences()).filter(picked) { let out = churn(arm, 0..cases, shape); println!( - "{name},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}", + "{name},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}", arm.name(), out.cases, out.watch.steps, @@ -1254,6 +1297,8 @@ fn benchmark_warmth_churn() { out.sent.asks + out.sent.told, out.sent.chains, out.settle, + out.tree, + out.cheapest, out.first.map_or("0".into(), |first| format!( "{}: {}", out.failed, From 00d0774c35e0e708b365572ac3c3d77c01d2d002 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 14:28:45 -0800 Subject: [PATCH 10/19] fix(moq-net): order the warmth sim's adverts with route messages, carry the mark on offers, and close an uplink for every arm Adverts and route messages share a session's announce stream, so the simulator stamps both from one clock and delivers a session's next advert or route message only after any older one. The mark an advert obeys now travels on the offer as a bit, as a route's offer carries it, instead of being read from the simulator's link configuration. drones_lose_the_uplink closes the first reader's uplink for arms that keep more than one, so path joins run the same six phases. Churn bounds each settle at a million deliveries, which a growing advert storm in a rejected arm otherwise spent without end. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/tests/multi.rs | 22 ++++- rs/moq-net/src/route/tests/warm.rs | 117 +++++++++++++++++++-------- rs/moq-net/src/route/tests/warmth.rs | 11 +-- 3 files changed, 107 insertions(+), 43 deletions(-) diff --git a/rs/moq-net/src/route/tests/multi.rs b/rs/moq-net/src/route/tests/multi.rs index 33ae83c94f5..031abfd178e 100644 --- a/rs/moq-net/src/route/tests/multi.rs +++ b/rs/moq-net/src/route/tests/multi.rs @@ -1267,7 +1267,10 @@ pub(super) struct MultiSim { restarts: Vec, ports: BTreeMap<(Node, Node), Port>, up: BTreeSet<(Node, Node)>, - queues: BTreeMap<(Node, Node), VecDeque>, + /// Each message with the clock when it was queued, so another queue on the same + /// session can keep their order. + queues: BTreeMap<(Node, Node), VecDeque<(u64, Msg)>>, + clock: u64, held: BTreeSet<(Node, Node)>, lives: BTreeMap, pub stats: Stats, @@ -1286,6 +1289,7 @@ impl MultiSim { ports: BTreeMap::new(), up: BTreeSet::new(), queues: BTreeMap::new(), + clock: 0, held: BTreeSet::new(), lives: BTreeMap::new(), stats: Stats::default(), @@ -1470,7 +1474,19 @@ impl MultiSim { self.stats.claim_items += changes.len(); } } - self.queues.entry((from, to)).or_default().push_back(msg); + let at = self.stamp(); + self.queues.entry((from, to)).or_default().push_back((at, msg)); + } + + /// The next tick of the clock queued messages carry. + pub fn stamp(&mut self) -> u64 { + self.clock += 1; + self.clock + } + + /// When the next message from `from` to `to` was queued, if one waits. + pub fn head(&self, from: Node, to: Node) -> Option { + self.queues.get(&(from, to))?.front().map(|&(at, _)| at) } pub fn hold(&mut self, from: Node, to: Node) { @@ -1485,7 +1501,7 @@ impl MultiSim { } pub fn deliver(&mut self, from: Node, to: Node) -> bool { - let Some(msg) = self.queues.get_mut(&(from, to)).and_then(VecDeque::pop_front) else { + let Some((_, msg)) = self.queues.get_mut(&(from, to)).and_then(VecDeque::pop_front) else { return false; }; if let Some(router) = self.relay(to) { diff --git a/rs/moq-net/src/route/tests/warm.rs b/rs/moq-net/src/route/tests/warm.rs index 187ead03dfc..761d8bc6a41 100644 --- a/rs/moq-net/src/route/tests/warm.rs +++ b/rs/moq-net/src/route/tests/warm.rs @@ -146,12 +146,14 @@ impl Price { } } -/// What a relay offers a neighbour: its summed cost and, with [`Arm::hops`], the relays -/// the data crossed, the publisher's own relay first. +/// What a relay offers a neighbour: its summed cost, with [`Arm::hops`] the relays the +/// data crossed, the publisher's own relay first, and whether the data came over a link +/// its receiver marked upstream, the bit a route's offer carries. #[derive(Clone, Debug, PartialEq, Eq)] pub(super) struct Signal { pub cost: Metric, pub path: Vec, + pub marked: bool, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -208,7 +210,8 @@ enum Msg { } /// The stream a message travels on: each subscription and query is a stream of its -/// own, unordered against the rest, and adverts share a session's announce stream. +/// own, unordered against the rest, and adverts share a session's announce stream, +/// in order with the route layer's messages. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] enum Lane { Pull, @@ -443,7 +446,9 @@ pub(super) struct WarmSim { adjacent: Vec>, casts: Vec, carry: Vec>, - queues: BTreeMap>, + /// Each message with the route layer's clock when it was queued: an advert keeps its + /// place among the route messages on the same session's announce stream. + queues: BTreeMap>, streams: u64, /// What must react before the next delivery: a relay and broadcast whose inputs /// changed. Nothing else can act, so nothing else is asked to. @@ -458,6 +463,9 @@ pub(super) struct WarmSim { circling: BTreeMap, pub sent: Sent, pub watch: Watch, + /// The deliveries [`WarmSim::settle`] may spend before it gives up, if bounded: a + /// storm that only grows would cost unbounded work. + pub budget: Option, pub round: usize, order: XorShift, /// Rounds each reader waited for its first frame. @@ -490,6 +498,7 @@ impl WarmSim { circling: BTreeMap::new(), sent: Sent::default(), watch: Watch::default(), + budget: None, round: 0, order: XorShift(seed | 1), waits: Vec::new(), @@ -599,10 +608,31 @@ impl WarmSim { | Msg::Told { stream, .. } => *stream as usize, Msg::Advert(_) => usize::MAX, }; + let at = self.net.stamp(); self.queues .entry((from, to, lane, stream)) .or_default() - .push_back((cast, msg)); + .push_back((at, cast, msg)); + } + + /// When the next advert from `from` to `to` was queued, if one waits. + fn advert(&self, from: Node, to: Node) -> Option { + let queue = self.queues.get(&(from, to, Lane::Announce, usize::MAX))?; + queue.front().map(|&(at, ..)| at) + } + + /// Whether `key`'s next message waits behind an older route message: adverts and + /// routes share the session's one announce stream. + fn blocked(&self, key: Key) -> bool { + let (from, to, lane, _) = key; + let route = self.net.head(from, to); + lane == Lane::Announce && route.is_some_and(|route| self.advert(from, to).is_some_and(|at| route < at)) + } + + /// Whether the next route message from `from` to `to` waits behind an older advert. + fn behind(&self, from: Node, to: Node) -> bool { + let route = self.net.head(from, to); + route.is_some_and(|route| self.advert(from, to).is_some_and(|at| at < route)) } // --- What a relay knows --- @@ -637,6 +667,7 @@ impl WarmSim { return Some(Signal { cost: 0, path: Vec::new(), + marked: false, }); } let signal = self.live(node, cast)?; @@ -652,7 +683,8 @@ impl WarmSim { true => signal.path.iter().copied().chain([node]).collect(), false => Vec::new(), }; - Some(Signal { cost, path }) + let marked = signal.marked || self.upstream.contains(&(node, carry.up.as_ref()?.via)); + Some(Signal { cost, path, marked }) } /// The warm signal `node` gives `peer`: its offer, while it has demand besides `peer`. @@ -1088,17 +1120,6 @@ impl WarmSim { } } - /// Whether `node`'s copy, which `signal` offers, came over a link its receiver marked - /// upstream: the route layer passes no such route on to another marked neighbour. - fn crossed(&self, node: Node, cast: usize, signal: &Signal) -> bool { - let up = self.carry(node, cast).and_then(|carry| carry.up.as_ref()); - up.is_some_and(|pull| self.upstream.contains(&(node, pull.via))) - || signal - .path - .windows(2) - .any(|hop| self.upstream.contains(&(hop[1], hop[0]))) - } - /// Whether `audience` lets `node` advertise `signal` to `peer`. fn scoped(&self, audience: Audience, node: Node, cast: usize, peer: Node, signal: &Signal) -> bool { let price = signal.cost + self.price(node, peer).cold; @@ -1119,7 +1140,7 @@ impl WarmSim { Audience::NotNext => matches!(heard, Some((cost, false)) if price <= cost), Audience::NotDown | Audience::NotDownAsk | Audience::NotDownAskUp | Audience::NotDownQuery => { !self.upstream.contains(&(peer, node)) - && !(self.upstream.contains(&(node, peer)) && self.crossed(node, cast, signal)) + && !(self.upstream.contains(&(node, peer)) && signal.marked) && useful() } Audience::Shared => self.carry(node, cast).is_some_and(|carry| { @@ -1297,6 +1318,7 @@ impl WarmSim { /// Deliver everything queued now, in a seeded random order: a hop of delay. False /// if nothing was. pub fn round(&mut self) -> bool { + #[derive(Clone, Copy)] enum Queue { Route(Node, Node), Data(Key), @@ -1315,22 +1337,40 @@ impl WarmSim { for i in (1..ready.len()).rev() { ready.swap(i, self.order.next(i as u64 + 1) as usize); } - for (queue, count) in ready { - for _ in 0..count { - match queue { - Queue::Route(from, to) => { - self.net.deliver(from, to); - self.stir(to); + // A queue whose next message waits behind an older one on the same announce + // stream takes its turn on a later pass, once that one is delivered. + loop { + let mut moved = false; + for (queue, count) in &mut ready { + while *count > 0 { + let blocked = match *queue { + Queue::Route(from, to) => self.behind(from, to), + Queue::Data(key) => self.blocked(key), + }; + if blocked { + break; } - Queue::Data(key) => { - let Some((cast, msg)) = self.pop(key) else { - break; - }; - self.recv(key.0, key.1, cast, msg); + *count -= 1; + moved = true; + match *queue { + Queue::Route(from, to) => { + self.net.deliver(from, to); + self.stir(to); + } + Queue::Data(key) => { + let Some((cast, msg)) = self.pop(key) else { + *count = 0; + break; + }; + self.recv(key.0, key.1, cast, msg); + } } + self.step(); + self.observe(); } - self.step(); - self.observe(); + } + if !moved { + break; } } self.net.flush(); @@ -1343,16 +1383,21 @@ impl WarmSim { /// The next message on a queue, dropping the queue once empty. fn pop(&mut self, key: Key) -> Option<(usize, Msg)> { let queue = self.queues.get_mut(&key)?; - let next = queue.pop_front(); + let next = queue.pop_front().map(|(_, cast, msg)| (cast, msg)); if queue.is_empty() { self.queues.remove(&key); } next } - /// Deliver in rounds until nothing is left. False if that never happens. + /// Deliver in rounds until nothing is left. False if that never happens, or once + /// it spends [`WarmSim::budget`]. pub fn settle(&mut self) -> bool { + let start = self.watch.steps; for _ in 0..2_000 { + if self.budget.is_some_and(|budget| self.watch.steps - start > budget) { + return false; + } if !self.round() { return true; } @@ -1362,9 +1407,11 @@ impl WarmSim { /// Deliver one message from a queue `pick` chooses, as churn does. False if none waits. pub fn deliver(&mut self, pick: &mut XorShift) -> bool { - let routes = self.net.ready(); + let routes: Vec<_> = (self.net.ready().into_iter()) + .filter(|&((from, to), _)| !self.behind(from, to)) + .collect(); let data: Vec = (self.queues.iter()) - .filter(|(_, queue)| !queue.is_empty()) + .filter(|&(&key, queue)| !queue.is_empty() && !self.blocked(key)) .map(|(&key, _)| key) .collect(); let total = routes.len() + data.len(); diff --git a/rs/moq-net/src/route/tests/warmth.rs b/rs/moq-net/src/route/tests/warmth.rs index adf0ef7aa3d..00b0d9c7d0d 100644 --- a/rs/moq-net/src/route/tests/warmth.rs +++ b/rs/moq-net/src/route/tests/warmth.rs @@ -924,6 +924,8 @@ fn churn(arm: Arm, cases: std::ops::Range, shape: fn(&mut XorShift) -> Sh let mut pick = XorShift(0x5eed_0002 + case as u64); let shape = shape(&mut script); let mut sim = shape.build(arm, 0x5eed_0003 + case as u64); + // Far beyond what any mesh here settles in, so only a storm that keeps growing. + sim.budget = Some(1_000_000); let (sent, watch) = (sim.sent, sim.watch); for _ in 0..2 { let relay = script.next(shape.relays as u64) as usize; @@ -1188,13 +1190,12 @@ fn unequal() -> Shape { } } -/// The drone shape's last phase closes the uplink the arm settled on. +/// The drone shape's last phase closes the uplink the arm settled on, or the first +/// reader's where the arm keeps more than one, as path joins keep one per drone. fn takeover(arm: Arm) -> Case { let mut phases = drone_phases(5, false); - let holder = uplinked(&run(&drones(5), arm, &phases, 4)); - if let Some(holder) = holder { - phases.push(phase("its_uplink_closes", vec![Step::Close(0, holder)])); - } + let holder = uplinked(&run(&drones(5), arm, &phases, 4)).unwrap_or(1); + phases.push(phase("its_uplink_closes", vec![Step::Close(0, holder)])); case("drones_lose_the_uplink", drones(5), phases) } From 89ec632bba76bab880e1f0b0c9a5a11bed1b2dbc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 15:00:23 -0800 Subject: [PATCH 11/19] docs(quest): update the cache-routing select quest for the ordered announce stream and the open loop in churned mesh 503 Co-Authored-By: Claude Opus 5.5 --- quest/m3/cache-routing/select.md | 56 +++++++++++++++++++------------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/quest/m3/cache-routing/select.md b/quest/m3/cache-routing/select.md index 44bde028bf0..91006b4c880 100644 --- a/quest/m3/cache-routing/select.md +++ b/quest/m3/cache-routing/select.md @@ -47,10 +47,10 @@ The maintainer's decision is whether to keep the query, below. Rejected: churned regions unsettled. - Naming in every subscription the relays whose demand alone it carries, so an offer leaves them out: a second fix for the loop the rule stops. It adds - a list and an update message on each change (6,913 to 8,469 per 400 + a list and an update message on each change (6,904 to 8,485 per 400 churned meshes), loses a shared drone uplink (41 against 33 with two drones - a hop apart), and left 2 of 2,000 churned random meshes unsettled where the - rule left none. + a hop apart), and left 1 of 2,000 churned random meshes unsettled, as the + rule does, though a different one. - Asking the k cheapest neighbours on each SUBSCRIBE. It stays bounded per subscribe only because it stops finding the copy: on a full drone mesh past 5 drones, `ask_1` shared no uplink at 64 drones and `ask_2` few. It @@ -86,12 +86,12 @@ edge's cheaper uplink is to the core that is cold. | two edges, unequal uplinks, in turn | 12 / 13 (12) | 2.50 / 0.50 | | two edges, unequal uplinks, at once | 9 / 9 (7) | 3.00 / 1.00 | | one region, unequal uplinks | 24 / 29 (24) | 2.00 / 0.00 | -| 400 churned regions: settled trees over cheapest; unfed reader-hops | +4.9% / +5.3%; 22,576 / 16,805 | 21,217 / 4,858 in all | +| 400 churned regions: settled trees over cheapest; unfed reader-hops | +4.9% / +5.3%; 22,477 / 16,810 | 21,182 / 4,879 in all | Recommended: keep the query. Where uplinks are unequal, it saves the second core's copy over the core mesh, the dearest link a region has. It costs one request and one answer per extra upstream per first viewer at an edge, holds -nothing on the core, and carries no traffic. Under churn it is dearer: 4.4 +nothing on the core, and carries no traffic. Under churn it is dearer: 4.3 times the warm messages and 34% more unfed reader-hops, much of which is the simulator delivering one control message per step across the whole mesh, so every query delays a route or a subscribe somewhere else. Without it an @@ -102,11 +102,11 @@ operator gets the same tree by pricing each edge's uplinks equally. | part | on the wire | relay state | bounded by | without it | |---|---|---|---|---| | advert | offer and path on the announce stream, and its withdrawal | one heard per neighbour per demanded Instance | neighbours; the path by the hop limit | path joins: pops 69 against 61, drones in turn 5 uplinks against 1, a ring 48 against 8 | -| the route layer's mark on adverts | none: each link's mark is configured already | none | none | an edge carries a copy between its cores, which no route would: 51 readers in 400 churned regions fed that way, cycle deliveries 213 against 90, and 1.05 to 1.55 warm messages per one-viewer reader change on an unpeered CDN against 1.0 | +| the route layer's mark on adverts | none: each link's mark is configured already | none | none | an edge carries a copy between its cores, which no route would: 51 readers in 400 churned regions fed that way, and 1.05 to 1.55 warm messages per one-viewer reader change on an unpeered CDN against 1.0 | | offer on the subscribe answer | a cost on the answer and its updates | one per pull | one per subscription | nothing can price a move | -| path in the offer | a list on the advert, answer, and query answer | as the offer | the hop limit | cycle deliveries up 77%, 26%, and 110% on random meshes, drones, and regions; 3 of 1,200 churned meshes unsettled | +| path in the offer | a list on the advert, answer, and query answer | as the offer | the hop limit | cycle deliveries up 72%, 7%, and 156% on random meshes, drones, and regions; 3 of 1,200 churned meshes unsettled | | query | a request naming an Instance; one answer, offer and path or none | the edge keeps one answer per extra upstream per demanded Instance; the upstream none | upstreams beyond the first | an edge whose cheaper uplink is cold: 29 against 24 | -| no warm signal from a relay on the taker's path | none | none, the path of its own pull | none | 1 of 400 churned regions never settles. Its price: a move up the taker's own branch to a relay warm on its own account, 14 against 13 in `an_ancestor_warm_on_its_own_is_passed_over`; under churn the chain, which keeps that move, settled on trees no cheaper (+2.9%, +8.7%, +5.2% over cheapest against the rule's +2.8%, +8.8%, +4.9%) | +| no warm signal from a relay on the taker's path | none | none, the path of its own pull | none | 1 of 400 churned regions never settles. Its price: a move up the taker's own branch to a relay warm on its own account, 14 against 13 in `an_ancestor_warm_on_its_own_is_passed_over`; under churn the chain, which keeps that move, settled on trees about as dear (+2.6%, +9.6%, +5.2% over cheapest against the rule's +2.7%, +9.4%, +4.9%) | The rule replaces a chain in every subscription naming the relays whose demand alone it carries. The offer's path cannot stand in for the chain on @@ -122,8 +122,8 @@ of its own pull is enough. relay offers 0. An offer is never below its sender's own cost. - **The path.** Every offer names the relays the data passes from the publisher's relay to the sender. Without it, cycle deliveries under churn - rose by 77% on random meshes, 26% on drone meshes, and 110% on regions, 1 - random mesh and 2 regions of 400 never settled, and the merge below cannot + rose by 72% on random meshes, 7% on drone meshes, and 156% on regions, 1 + drone mesh and 2 regions of 400 never settled, and the merge below cannot work. - **Adverts.** While R has demand for an Instance besides N, N has not marked R upstream, and R has not both marked N upstream and taken its copy @@ -131,7 +131,9 @@ of its own pull is enough. when `O(R→N) + cold(R, N)` is below N's route cost for the Instance, or equals it through a next hop other than R, since a tie prefers a warm copy. It tells again on each change, and withdraws when the demand, its upstream, - or that test ends. Nothing it skips between peers could ever be taken: a + or that test ends. Adverts keep their order with the route layer's + messages on the same announce stream: a withdrawn route never trails an + advert sent after it. Nothing it skips between peers could ever be taken: a relay at rest costs no more than its own route, and a tie through R is the path join already. R knows N's route cost from what N announces it, and computes it when N's route runs through R. Skipping the parent and children @@ -171,14 +173,21 @@ of its own pull is enough. demand coming back: N pulls through X from R, so R has demand besides N and offers N its warm cost; N moves to R, X's pull ends, R's offer turns cold, N goes back through X, and so on for good. Without the rule, 1 of - 400 churned regions never settled that way under most warm arms; with it, none of 2,000 each of random meshes, drone meshes, - and regions under the pick. A relay on N's path is upstream of N's parent, + 400 churned regions never settled that way under most warm arms; with it, + none of 2,000 drone meshes or regions and 1 of 2,000 random meshes under + the pick. That one, case 503, loops through a sibling: 6 pulls from 4, + which makes 4 warm for 3, so 3 offers 6 less than 4 does; 6 moves to 3, 4 + turns cold for 3, and 6 moves back. 3's offer crosses 6's path at 4, below + the publisher's relay, but refusing every offer that crosses it left a + region unsettled under five other arms and did not settle more under the + pick, so the rule stays as it is and the case is an acceptance test. + A relay on N's path is upstream of N's parent, so N gives up only a move up its own branch, which its route still offers when that is the cheaper way. It costs a move to a relay on N's path that is warm on its own account, which N cannot tell apart from its own demand: in `an_ancestor_warm_on_its_own_is_passed_over` N keeps a tree of 14 where - 13 is cheapest, but under churn the trees it settled on cost no more than - the chain's, which keeps such moves. + 13 is cheapest, but under churn the trees it settled on cost about what + the chain's did, which keeps such moves. - A move is make before break: the old pull stays until the new one is answered live, and a move under way stays while it is no dearer than staying. Its own subscription shares the old parent, which lowers what @@ -198,9 +207,11 @@ each prefer the other on current offers, and a cycle needs one stale offer. Gaining demand lowers offers to others, and the one it lowers to the relay whose own demand it is, that relay ignores; losing it raises only the offer to the neighbour that still pulls, which then moves only to something strictly -cheaper. The sweep agrees, though it is no proof: every churned random mesh, -drone mesh, and region settled under the pick, and cycles lasted at most 57 -deliveries, each while some answer was in flight. +cheaper. The argument misses what one relay's demand does to a price two +paths share, as case 503 above shows, and the sweep is no proof: all 400 +churned meshes of each kind settled under the pick, cycles lasted at most 39 +deliveries, each while some answer was in flight, and at 2,000 per kind one +random mesh never settled. ### Simultaneous starts @@ -209,11 +220,11 @@ neither offers the other anything cheaper, and both uplinks stay until one drone's demand ends. The maintainer's rule, decided 2026-11-15: merge them on the rendezvous hash unless that adds a cycle, an unfed reader, or more messages than the second uplink costs. Under adverts it does add cycles: -in 400 churned drone meshes, 95 deliveries had a pull the merge made inside +in 400 churned drone meshes, 18 deliveries had a pull the merge made inside a cycle, all transient. The one traced was a pending cycle: a merged drone's new parent lost its uplink and rejoined through a drone that had trusted the merged drone's advert, before the lost parent's answer arrived. -The merge also added about 1.5% unfed reader hops. Under asking on subscribe, +The merge also added about 1% unfed reader hops. Under asking on subscribe, no merged pull was ever in a cycle. So the race stands. A drone that loses the one uplink leaves every drone @@ -251,7 +262,7 @@ path in each offer. uplink against five, a ring of 48 relays 8 uplinks against 48, tiers 1095 against 1191 (the cheapest), and an edge beside the warm core 24 against 29, at 4.3 to 11 messages per reader change against 2 to 5.4. Under churn, - the trees each case settled on cost 2.8%, 8.8%, and 4.9% over the cheapest + the trees each case settled on cost 2.7%, 9.4%, and 4.9% over the cheapest for the same readers on random meshes, drones, and regions, against path joins' 2.7%, 28.9%, and 5.0%. - Port the growth sweep, and hold its laws, per link kind. Per reader change, @@ -282,7 +293,8 @@ path in each offer. and first frames assume. A first frame matches the path join's everywhere the sweeps measured, but two drones a hop apart (81 ms against 69). Two edges starting at once each keep their path join (9 against 7), the race - again. + again. Random churned mesh 503 too, which the simulator's pick never + settles. - A far pop keeps pulling through a nearer pop after that pop's reader leaves, while the nearer pop's unshared offer plus the warm link stays below the far pop's own long link. That is the cheaper tree, and the From 6908dc42396c7da763683591561747692940b658 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 15:08:40 -0800 Subject: [PATCH 12/19] docs(quest): price the mark's bit on the offer in the select quest's parts table Co-Authored-By: Claude Opus 5.5 --- quest/m3/cache-routing/select.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quest/m3/cache-routing/select.md b/quest/m3/cache-routing/select.md index 91006b4c880..159e5c6e481 100644 --- a/quest/m3/cache-routing/select.md +++ b/quest/m3/cache-routing/select.md @@ -102,7 +102,7 @@ operator gets the same tree by pricing each edge's uplinks equally. | part | on the wire | relay state | bounded by | without it | |---|---|---|---|---| | advert | offer and path on the announce stream, and its withdrawal | one heard per neighbour per demanded Instance | neighbours; the path by the hop limit | path joins: pops 69 against 61, drones in turn 5 uplinks against 1, a ring 48 against 8 | -| the route layer's mark on adverts | none: each link's mark is configured already | none | none | an edge carries a copy between its cores, which no route would: 51 readers in 400 churned regions fed that way, and 1.05 to 1.55 warm messages per one-viewer reader change on an unpeered CDN against 1.0 | +| the route layer's mark on adverts | one bit on the advert, the answer, and the query's answer, as a route's offer carries | the bit with each heard advert and live pull | one per offer | an edge carries a copy between its cores, which no route would: 51 readers in 400 churned regions fed that way, and 1.05 to 1.55 warm messages per one-viewer reader change on an unpeered CDN against 1.0 | | offer on the subscribe answer | a cost on the answer and its updates | one per pull | one per subscription | nothing can price a move | | path in the offer | a list on the advert, answer, and query answer | as the offer | the hop limit | cycle deliveries up 72%, 7%, and 156% on random meshes, drones, and regions; 3 of 1,200 churned meshes unsettled | | query | a request naming an Instance; one answer, offer and path or none | the edge keeps one answer per extra upstream per demanded Instance; the upstream none | upstreams beyond the first | an edge whose cheaper uplink is cold: 29 against 24 | From a5c65b2fd7c85118783699e618fd97b2ebbc0453 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 16:02:43 -0800 Subject: [PATCH 13/19] feat(moq-net): measure a warmth arm whose settled pulls stay, choosing again only on a new route The pick moved a live pull whenever another source got cheaper, so one relay's move could be another's reason to move, and churned random mesh 503 never settled. Under +stay a relay chooses where to pull when its demand starts and once on each query answer from that join; after that it moves only when its parent says it lost its upstream, its pull runs through itself, or the session closes, and a lost pull is replaced outright. +rejoin also chooses again when a reader of its own starts, and +reroute when its route changes, which follows links and claims, never demand. Neither needs the rule against warm signals from a relay on the taker's own path. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/tests/warm.rs | 102 +++++++++++++++++++++++++-- rs/moq-net/src/route/tests/warmth.rs | 60 ++++++++++++++-- 2 files changed, 151 insertions(+), 11 deletions(-) diff --git a/rs/moq-net/src/route/tests/warm.rs b/rs/moq-net/src/route/tests/warm.rs index 761d8bc6a41..d0f7a1a274e 100644 --- a/rs/moq-net/src/route/tests/warm.rs +++ b/rs/moq-net/src/route/tests/warm.rs @@ -103,6 +103,19 @@ pub(super) struct Arm { /// that stops the same loop: a relay never takes a warm signal from a relay named /// on its own pull's path, whose warmth may be this relay's own demand. pub chain: bool, + /// A settled pull stays: a relay chooses where to pull when its demand starts, and + /// once more on each query answer from that join, then moves only when its parent + /// says it lost its upstream, the pull runs through itself, or the session closes. + /// No price a neighbour offers later moves it, so no relay's move is another's + /// reason to move, and no rule about whose warmth it is applies. + pub stay: bool, + /// With [`Arm::stay`], a relay already pulling for others chooses again, once, when + /// a reader of its own starts: an outside event, like a join. + pub rejoin: bool, + /// With [`Arm::stay`], a relay chooses again, once, each time its route changes: + /// the route layer's routes follow links and claims, never demand, so no relay's + /// pull is another's reason to move. + pub reroute: bool, } impl Arm { @@ -124,7 +137,10 @@ impl Arm { let hops = if self.hops { "+hops" } else { "" }; let merge = if self.merge { "+merge" } else { "" }; let chain = if self.chain { "+chain" } else { "" }; - format!("{mode}{hops}{merge}{chain}") + let stay = if self.stay { "+stay" } else { "" }; + let rejoin = if self.rejoin { "+rejoin" } else { "" }; + let reroute = if self.reroute { "+reroute" } else { "" }; + format!("{mode}{hops}{merge}{chain}{stay}{rejoin}{reroute}") } } @@ -250,6 +266,13 @@ struct Carry { told: BTreeMap<(Node, u64), Option>, heard: BTreeMap, advertised: BTreeMap, + /// Upstreams whose query answer arrived since the relay last judged one, with + /// [`Arm::stay`]. + fresh: BTreeSet, + /// A reader of its own started while it already pulled, with [`Arm::rejoin`]. + rejoin: bool, + /// The route it last chose by, with [`Arm::reroute`]. + route: Option<(Node, Metric)>, } impl Carry { @@ -720,7 +743,8 @@ impl WarmSim { .collect(), }; // Without the chain, an ancestor's warmth may be this relay's own demand. - let ancestors: Vec = match self.arm.chain { + // A relay that never moves for price has no reason to fence its own warmth off. + let ancestors: Vec = match self.arm.chain || self.arm.stay { true => Vec::new(), false => self .live(node, cast) @@ -948,11 +972,19 @@ impl WarmSim { // It asks before it pulls, so the first answers can steer the first pull. if let Some(best) = best.filter(|_| carry.unanswered.is_empty()) { let pull = self.subscribe(node, cast, best.via); - self.entry(node, cast).up = Some(pull); + let route = self.route(node, cast); + let carry = self.entry(node, cast); + carry.up = Some(pull); + carry.fresh.clear(); + carry.route = route; } return; }; - let moving = carry.next.as_ref().map(|next| next.via); + if self.arm.stay { + self.stay(node, cast, &up, &options, best); + return; + } + let moving = self.entry(node, cast).next.as_ref().map(|next| next.via); let target = match &up.state { // Nothing flows yet, so a better source replaces it outright. State::Pending => { @@ -1035,6 +1067,62 @@ impl WarmSim { /// Move toward `target` unless already there or on the way, marking a new pull the /// merge asked for. + /// [`Arm::stay`]'s rule for a relay already pulling: a pending pull waits for its + /// answer, failing closed only if its source left; a live one moves only for a + /// query answer from its join that is strictly cheaper, judged once, keeping the old + /// pull until the new one is live; a lost one, or one through itself, is replaced by + /// a fresh choice. + fn stay(&mut self, node: Node, cast: usize, up: &Pull, options: &[Choice], best: Option) { + let hops = self.arm.hops; + let reroute = self.arm.reroute; + let route = self.route(node, cast); + let carry = self.entry(node, cast); + let moving = carry.next.as_ref().map(|next| next.via); + let fresh = std::mem::take(&mut carry.fresh); + let mut rejoin = std::mem::take(&mut carry.rejoin); + if reroute && carry.route != route && up.state != State::Pending { + carry.route = route; + rejoin = true; + } + match &up.state { + State::Pending => { + carry.fresh = fresh; + carry.rejoin = rejoin; + if !options.iter().any(|choice| choice.via == up.via) { + let old = self.entry(node, cast).up.take(); + self.unsubscribe(node, cast, old); + } + } + State::Live(signal) if !(hops && signal.path.contains(&node)) => { + let own = signal.cost + self.price(up.via, node).warm; + let answer = (options.iter()) + .filter(|choice| fresh.contains(&choice.via) && choice.offered.is_some() && choice.cost < own) + .min_by_key(|choice| (choice.cost, Reverse(self.hash(cast, choice.via)), choice.via)); + // A reader starting is a join: the best source now, if strictly cheaper. + let rejoined = best.filter(|best| rejoin && best.via != up.via && best.cost < own); + let target = (answer.map(|choice| choice.via)) + .or(rejoined.map(|best| best.via)) + .or(moving) + .unwrap_or(up.via); + self.toward(node, cast, target, false); + } + // Nothing flows from a lost pull, so a new one replaces it outright, and fails + // closed like any pending pull: two relays that each turn to the other on stale + // adverts would otherwise wait on each other for good. + _ => { + let (old, next) = (carry.up.take(), carry.next.take()); + self.unsubscribe(node, cast, old); + self.unsubscribe(node, cast, next); + if let Some(best) = best { + let pull = self.subscribe(node, cast, best.via); + let carry = self.entry(node, cast); + carry.up = Some(pull); + carry.route = route; + } + } + } + } + fn toward(&mut self, node: Node, cast: usize, target: Node, merged: bool) { let carry = self.entry(node, cast); let up = carry.up.as_ref().map(|pull| pull.via); @@ -1176,6 +1264,7 @@ impl WarmSim { fn recv(&mut self, from: Node, to: Node, cast: usize, msg: Msg) { self.dirty.insert((to, cast)); + let stay = self.arm.stay; // A publisher answers at once. if let Msg::Subscribe { stream, open: true, .. } = msg && !self.relay(to) @@ -1227,6 +1316,9 @@ impl WarmSim { if carry.asking.get(&from) == Some(&stream) && let Some(heard) = carry.asks.get_mut(&from) { + if stay && word.is_some() { + carry.fresh.insert(from); + } *heard = word; carry.unanswered.remove(&from); } @@ -1245,7 +1337,9 @@ impl WarmSim { /// A reader at `node` starts or stops reading broadcast `cast`. pub fn read(&mut self, node: Node, cast: usize, reading: bool) { let round = self.round; + let rejoin = self.arm.rejoin; let carry = self.entry(node, cast); + carry.rejoin |= rejoin && reading && !carry.reader && carry.up.is_some(); carry.reader = reading; carry.since = reading.then_some(round); self.dirty.insert((node, cast)); diff --git a/rs/moq-net/src/route/tests/warmth.rs b/rs/moq-net/src/route/tests/warmth.rs index 00b0d9c7d0d..e5c3a8aba92 100644 --- a/rs/moq-net/src/route/tests/warmth.rs +++ b/rs/moq-net/src/route/tests/warmth.rs @@ -20,6 +20,9 @@ fn arms() -> Vec { hops, merge: false, chain: false, + stay: false, + rejoin: false, + reroute: false, }); } for mode in [ @@ -36,6 +39,9 @@ fn arms() -> Vec { hops, merge, chain: false, + stay: false, + rejoin: false, + reroute: false, }); } } @@ -49,6 +55,9 @@ const fn arm(mode: Mode, merge: bool) -> Arm { hops: true, merge, chain: false, + stay: false, + rejoin: false, + reroute: false, } } @@ -406,20 +415,27 @@ fn a_join_skips_pops_without_demand_and_warm_ones_draw_it() { let edges: BTreeSet<(Node, Node)> = middle.trees[0].edges.iter().copied().collect(); let joined = BTreeSet::from([(4, 0), (0, 1), (1, 2), (2, 3)]); let skip = BTreeSet::from([(4, 0), (0, 1), (0, 2), (0, 3)]); + // A settled pull stays: the middle pop joins the near pop's copy, but the far + // pop, settled on its long link, never moves for the warm pops beside it. + let stayed = BTreeSet::from([(4, 0), (0, 1), (1, 2), (0, 3)]); match arm.mode { Mode::Path => assert_eq!(edges, skip, "{case}"), + _ if arm.stay => assert_eq!(edges, stayed, "{case}"), // One query goes to the cheapest link, which for the middle pop is the near // pop's and for the far pop the middle's. _ => assert_eq!(edges, joined, "{case}"), } - // The near pop's reader leaves and the middle pop returns to its long link. + // The near pop's reader leaves and the middle pop returns to its long link, + // unless its pull stays, when the near pop carries on for it. let near = last(&rows, "near_pop_leaves"); - assert!( + let carriers: &[Node] = if arm.stay { &[1] } else { &[] }; + assert_eq!( near.trees[0].edges.contains(&(0, 2)), + !arm.stay, "{case}: {:?}", near.trees[0].edges ); - assert!(near.carriers[0].is_empty(), "{case}: {:?}", near.carriers[0]); + assert_eq!(near.carriers[0], carriers, "{case}"); } } @@ -490,10 +506,34 @@ const CHAIN: Arm = Arm { ..NOT_DOWN_QUERY }; +/// The pick with settled pulls staying: no move for price after the join. +const STAY: Arm = Arm { + stay: true, + ..NOT_DOWN_QUERY +}; + +/// [`STAY`], choosing again when a reader of its own starts. +const REJOIN: Arm = Arm { rejoin: true, ..STAY }; + +/// [`STAY`], choosing again when its route changes. +const REROUTE: Arm = Arm { reroute: true, ..STAY }; + /// The narrower audiences for useful adverts, which keep them off a CDN's edges. -/// [`NOT_DOWN_QUERY`] is the pick, and [`CHAIN`] the pick with the chain. +/// [`NOT_DOWN_QUERY`] is the pick, [`CHAIN`] the pick with the chain, [`STAY`] the +/// pick whose settled pulls stay, and [`REJOIN`] and [`REROUTE`] that, choosing again +/// on a reader of its own or a new route. fn narrower() -> Vec { - vec![NOT_NEXT, NOT_DOWN, NOT_DOWN_ASK, NOT_DOWN_ASK_UP, NOT_DOWN_QUERY, CHAIN] + vec![ + NOT_NEXT, + NOT_DOWN, + NOT_DOWN_ASK, + NOT_DOWN_ASK_UP, + NOT_DOWN_QUERY, + CHAIN, + STAY, + REJOIN, + REROUTE, + ] } /// The cheaper ways to advertise measured beside the rest: the narrower audiences, @@ -722,8 +762,8 @@ fn an_edge_finds_the_warm_core_beside_its_route() { /// The price of taking no warm signal from a relay on the taker's own path. Relay 1 /// is warm for a reader of its own, and 3 pulls through 1 and 2 at 4. Once the /// direct link 1 to 3 opens, 1's warm copy is 3 away, but 1 is 3's ancestor, so the -/// pick stays: its tree costs 14 where 13 is cheapest, the publisher's link included. The chain, which tells 1 that -/// 2's pull is 3's demand, takes it. +/// pick stays: its tree costs 14 where 13 is cheapest, the publisher's link +/// included. The chain, which tells 1 that 2's pull is 3's demand, takes it. #[test] fn an_ancestor_warm_on_its_own_is_passed_over() { let shape = Shape { @@ -752,6 +792,10 @@ fn an_ancestor_warm_on_its_own_is_passed_over() { }; assert_eq!(tree(NOT_DOWN_QUERY), (14, 13)); assert_eq!(tree(CHAIN), (13, 13)); + // A settled pull stays on 14. The new link moves its route, but the route reaches + // 3 before 1's advert does, so the one choice the move buys finds only the route. + assert_eq!(tree(STAY), (14, 13)); + assert_eq!(tree(REROUTE), (14, 13)); } /// One CSV row of the growth sweeps. @@ -1040,6 +1084,8 @@ fn every_arm_settles_after_churn() { NOT_DOWN_ASK_UP, NOT_DOWN_QUERY, CHAIN, + STAY, + REROUTE, ] { for shape in [random as fn(&mut XorShift) -> Shape, flying, regional] { let out = churn(arm, 0..12, shape); From c9bbde0dbfaba0f2dfb495ae328b5798f437e8df Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 16:14:07 -0800 Subject: [PATCH 14/19] docs(quest): recommend that a settled pull stays in the cache-routing select quest The pick that moved a live pull for price never settles churned random mesh 503. The settled-pull arm chooses at a join, once per query answer, and on a route change, and settles all 6,000 churned meshes at 2,000 per kind. It drifts further from the cheapest tree: +3.4%, +16.2%, and +5.0% on random, drone, and region meshes, against the pick's +2.7%, +9.4%, and +4.9%. The quest now states the Imase and Waxman bound and what it assumes, which rules go, and a redone parts table. The pick stays as the measured alternative. Co-Authored-By: Claude Opus 5.5 --- quest/m3/cache-routing/select.md | 392 ++++++++++++++++--------------- 1 file changed, 197 insertions(+), 195 deletions(-) diff --git a/quest/m3/cache-routing/select.md b/quest/m3/cache-routing/select.md index 159e5c6e481..ccfc5cc4e4b 100644 --- a/quest/m3/cache-routing/select.md +++ b/quest/m3/cache-routing/select.md @@ -1,15 +1,17 @@ -# [XL] Relays pull each broadcast from the cheapest copy +# [XL] Relays pull each broadcast from a warm copy and keep it ## Goal A relay with demand for an Instance advertises it on the announce stream to -each peer or upstream that could take it, an edge queries its other upstreams -when it gains demand, and each relay pulls every demanded broadcast from the -cheapest copy of its Instance, moving an existing pull only when the route it -uses gets worse or ends, or another is strictly cheaper in summed cost. A peer -that sends no warm signal is treated as cold and behaves as it does today. -Relays export how many upstream copies they hold and how many moves a warm -copy caused. +each peer or upstream that could take it, and an edge queries its other +upstreams when it gains demand. When a relay's demand starts, it pulls from +the cheapest copy it has heard of, its route or a warm neighbour, and once +more on each query answer from that join. After that the pull stays: it moves +only when its parent loses its upstream, the pull runs through the relay +itself, the session closes, or the relay's route changes, never because a +neighbour's price did. A peer that sends no warm signal is treated as cold and +behaves as it does today. Relays export how many upstream copies they hold +and what each choice was for. ## Plan @@ -20,21 +22,81 @@ Plan, is replaced. Recommended 2026-11-16 on [moq-dev/moq#8903](https://github.com/moq-dev/moq/pull/8903), awaiting the -maintainer's confirmation at merge: **adverts between peers and toward -upstreams, only where the copy could be taken, never down a link the neighbour -marked upstream, and never up one with a copy that came over one; an edge -instead queries its other upstreams once when it gains demand; the data path -in every offer; no warm signal taken from a relay on the taker's own path; and -the race left standing**. Four parts: adverts, the path, the query, and one -rule. Adverts reuse the route layer's mark: it passes no route from one marked -link to another, so an edge carries nothing between its cores. +maintainer's confirmation at merge: **a settled pull stays.** A relay chooses +where to pull when its demand starts, from its route and the warm copies it +has heard of by then, and once more on each query answer from that join. +After that it chooses again only on an outside event: its parent loses its +upstream, its pull runs through itself, its session closes, or its route +changes. What it hears comes from **adverts between peers and toward +upstreams, only where the copy could be taken, never down a link the +neighbour marked upstream, and never up one with a copy that came over +one**; **a one-shot query** from an edge to its other upstreams when it gains +demand; and **the data path and the mark in every offer**. The race between +simultaneous starts stands. -The maintainer's decision is whether to keep the query, below. Rejected: +A rise in the parent's offer is not an event. A rise is what a sibling's +move or a neighbour's lost reader causes, and the point of the rule is that +no relay's move is another relay's reason to move. A route change is one: +the route layer's routes follow links and claims, never demand. + +So the pull graph changes only on an outside event (a reader starts or +stops, a link or relay fails or returns, a publisher moves), and each relay +makes a bounded number of choices per event. Settling needs no argument about +prices: after the last outside event, choices come only from the route +layer settling (its own proof), from one query answer per upstream, and from +pending pulls failing closed. It is greedy online Steiner without +rearrangement, apart from the moves a route change buys. + +**What the guarantee costs.** Imase and Waxman (Dynamic Steiner Tree +Problem, SIAM J. Discrete Math. 4(3), 1991) bound greedy joins without +rearrangement at O(log k) times the cheapest tree for k terminals, and show no +online rule without rearrangement does better than ½ log2 k. The bound +assumes terminals only arrive, fixed link costs, and each newcomer joining the +nearest node of the whole tree by a shortest path. Here a relay joins the +cheapest copy it has heard of, priced by the copy's whole path rather than +the one link it adds, and readers also leave; for arrivals and departures +together their constant-factor algorithm rearranges O(n^(3/2)) edges over n +requests, which this never does for price. So the bound is a guide, and the +sweeps measure the drift: + +| | the recommendation | the pick it replaces | path joins | +|---|---|---|---| +| settled trees over the cheapest, 400 churned random / drone / region meshes | +3.4% / +16.2% / +5.0% | +2.7% / +9.4% / +4.9% | +2.7% / +28.9% / +5.0% | +| unsettled of 2,000 per kind | 0 / 0 / 0 | 1 / 0 / 0 (random case 503) | 0 / 0 / 0 | +| longest cycle under churn, deliveries | 20 / 18 / 19 | 18 / 39 / 24 | 9 / 14 / 17 | +| unfed reader-hops | 19,008 / 11,691 / 23,097 | 18,000 / 12,381 / 22,477 | 15,657 / 7,723 / 15,731 | +| pops, summed over its five phases (cheapest 59) | 71 | 61 | 69 | +| drones in turn | 75, one uplink | 75, one uplink | 155, five | +| drones one / two hops apart | 41 / 33, at 69 / 62 ms | 33 / 25, at 81 / 69 ms | 41 / 41, at 69 / 62 ms | +| edges, regions, rings, tiers | as the pick | 12, 24, 2904, 1095 | 14, 29, 9624, 1191 | +| warm messages per reader change, unpeered CDN | 1.0 | 1.0 | 0 | + +Its drift is a relay that stays on a detour once the warm copy that drew it +loses its other readers: on pops, the middle pop keeps pulling through the +near pop after the near pop's reader leaves (71 against path joins' 69). +Choosing again when a relay's own reader starts (`+rejoin`) changed nothing +measurable; choosing again on a route change (`+reroute`, part of the +recommendation) took random meshes from +4.4% to +3.4% and regions from ++6.0% to +5.0%. Drone meshes drift most, since their routes rarely change. +No timer is added. + +**The pick it replaces** moved a live pull whenever another source got +strictly cheaper, fenced by a rule that a relay takes no warm signal from a +relay on its own path. It never settles churned random mesh 503, a loop +through a sibling: 6 pulls from 4, which makes 4 warm for 3, so 3 offers 6 +less than 4 does; 6 moves to 3, 4 turns cold for 3, and 6 moves back. Each +fence covered one shape of a relay's own demand coming back as a price, as +the old route rule's fixes did; refusing every offer whose path crosses the +taker's below the publisher's relay unsettled other arms instead. Under the +recommendation the reflection costs at most one wrong choice per event, never +a loop. + +Rejected, besides the pick: - Advertising to every neighbour that could take the copy, down marked links too: it grows with a region's edges, since a warm core tells every edge it could feed. 1.7 to 118 warm messages per one-viewer reader change as edges - per core grow from 3 to 128, against the pick's flat 1.0. + per core grow from 3 to 128, against the recommendation's flat 1.0. - Advertising to every neighbour, which also grows with the neighbours of every relay on the warm chain: 9 to 66 per one-viewer reader change as a full core mesh grows from 4 to 32 cores. @@ -46,73 +108,57 @@ The maintainer's decision is whether to keep the query, below. Rejected: demanded broadcast (32 per core, against none) and it left 1 of 400 churned regions unsettled. - Naming in every subscription the relays whose demand alone it carries, so - an offer leaves them out: a second fix for the loop the rule stops. It adds - a list and an update message on each change (6,904 to 8,485 per 400 - churned meshes), loses a shared drone uplink (41 against 33 with two drones - a hop apart), and left 1 of 2,000 churned random meshes unsettled, as the - rule does, though a different one. + an offer leaves them out: another fence for the pick's loops. It adds a + list and an update message on each change (6,904 to 8,485 per 400 churned + meshes) and still left 1 of 2,000 churned random meshes unsettled. - Asking the k cheapest neighbours on each SUBSCRIBE. It stays bounded per subscribe only because it stops finding the copy: on a full drone mesh past 5 drones, `ask_1` shared no uplink at 64 drones and `ask_2` few. It doubled the time to a first frame and left about twice the readers unfed under churn. On CDN tiers `ask_1` sent 5.7 to 7 warm messages per - one-viewer reader change against the pick's 1.0. + one-viewer reader change against 1.0. - Advertising only once two others want the copy, which lost both cases; never advertising over a link the sender marked upstream, which saves nothing in a core mesh. - Joins along the path only, the cost baseline, which delivers neither case. That PR's simulator, `rs/moq-net/src/route/tests/warm.rs`, runs every rule -below, and its `warmth.rs` holds the shapes, the churn sweep, and the growth -sweep this ports onto production code. +below as `advertise_not_down_query+hops+stay+reroute`, and its `warmth.rs` +holds the shapes, the churn sweep, and the growth sweep this ports onto +production code. ### Decision: the query The query is the one part that only an edge with more than one upstream -uses. Without it the pick is adverts, the path, and the rule, and an unpeered -CDN sends no warm message at all. With equal uplinks the query buys nothing: every edge of -a region ranks the two cores by the same rendezvous hash per broadcast, so -every edge's path join lands on the same core, always; path joins already -built the cheapest tree on every unpeered CDN shape swept. It pays only where an -edge's cheaper uplink is to the core that is cold. - -| shape | tree, with / without the query (cheapest) | warm messages per reader change, with / without | -|---|---|---| -| CDN, 8 cores, 16 edges per core, one viewer | 2236 / 2236 (2236) | 1.00 / 0.00 | -| CDN, 8 cores, 16 edges per core, crowd | 544 / 544 (544) | 1.00 / 0.00 | -| CDN, 32 cores, 3 edges per core, crowd | 256 / 256 (256) | 1.00 / 0.00 | -| tiers, crowd (edges peered) | 1095 / 1095 (1095) | 4.71 / 3.38 | -| tiers, one viewer | 1096 / 1096 (1096) | 5.50 / 3.50 | -| two edges, unequal uplinks, in turn | 12 / 13 (12) | 2.50 / 0.50 | -| two edges, unequal uplinks, at once | 9 / 9 (7) | 3.00 / 1.00 | -| one region, unequal uplinks | 24 / 29 (24) | 2.00 / 0.00 | -| 400 churned regions: settled trees over cheapest; unfed reader-hops | +4.9% / +5.3%; 22,477 / 16,810 | 21,182 / 4,879 in all | - -Recommended: keep the query. Where uplinks are unequal, it saves the second -core's copy over the core mesh, the dearest link a region has. It costs one -request and one answer per extra upstream per first viewer at an edge, holds -nothing on the core, and carries no traffic. Under churn it is dearer: 4.3 -times the warm messages and 34% more unfed reader-hops, much of which is the -simulator delivering one control message per step across the whole mesh, so -every query delays a route or a subscribe somewhere else. Without it an -operator gets the same tree by pricing each edge's uplinks equally. +uses; without it an unpeered CDN sends no warm message at all. With equal +uplinks it buys nothing: every edge of a region ranks the two cores by the +same rendezvous hash per broadcast, so every edge's path join lands on the +same core, always. It pays only where an edge's cheaper uplink is to the core +that is cold: one region with unequal uplinks costs 24 with it and 29 +without, and two edges in turn 12 against 13. It costs one request and one +answer per extra upstream per first viewer at an edge, exactly 1.0 warm +message per reader change on the CDN sweeps, holds nothing on the core, and +carries no traffic. Recommended: keep it. Without it an operator gets the +same tree by pricing each edge's uplinks equally. ### What each part costs | part | on the wire | relay state | bounded by | without it | |---|---|---|---|---| -| advert | offer and path on the announce stream, and its withdrawal | one heard per neighbour per demanded Instance | neighbours; the path by the hop limit | path joins: pops 69 against 61, drones in turn 5 uplinks against 1, a ring 48 against 8 | -| the route layer's mark on adverts | one bit on the advert, the answer, and the query's answer, as a route's offer carries | the bit with each heard advert and live pull | one per offer | an edge carries a copy between its cores, which no route would: 51 readers in 400 churned regions fed that way, and 1.05 to 1.55 warm messages per one-viewer reader change on an unpeered CDN against 1.0 | -| offer on the subscribe answer | a cost on the answer and its updates | one per pull | one per subscription | nothing can price a move | -| path in the offer | a list on the advert, answer, and query answer | as the offer | the hop limit | cycle deliveries up 72%, 7%, and 156% on random meshes, drones, and regions; 3 of 1,200 churned meshes unsettled | -| query | a request naming an Instance; one answer, offer and path or none | the edge keeps one answer per extra upstream per demanded Instance; the upstream none | upstreams beyond the first | an edge whose cheaper uplink is cold: 29 against 24 | -| no warm signal from a relay on the taker's path | none | none, the path of its own pull | none | 1 of 400 churned regions never settles. Its price: a move up the taker's own branch to a relay warm on its own account, 14 against 13 in `an_ancestor_warm_on_its_own_is_passed_over`; under churn the chain, which keeps that move, settled on trees about as dear (+2.6%, +9.6%, +5.2% over cheapest against the rule's +2.7%, +9.4%, +4.9%) | - -The rule replaces a chain in every subscription naming the relays whose -demand alone it carries. The offer's path cannot stand in for the chain on -the sender's side: the demand that comes back is downstream of the sender, -and its path names only relays upstream of it. On the taker's side the path -of its own pull is enough. +| advert | offer, path, and mark on the announce stream, in order with the route messages, and its withdrawal | one heard per neighbour per demanded Instance | neighbours; the path by the hop limit | path joins: drones in turn 5 uplinks against 1, a ring 48 against 8, tiers 1191 against 1095 | +| the route layer's mark on adverts | one bit on the advert, the answer, and the query's answer, as a route's offer carries | the bit with each heard advert and live pull | one per offer | an edge carries a copy between its cores, which no route would: 51 readers in 400 churned regions fed that way under the pick, and 1.05 to 1.55 warm messages per one-viewer reader change on an unpeered CDN against 1.0 | +| offer on the subscribe answer | a cost, path, and mark on the answer, and an update on each change; none when the parent lost its upstream | one per pull | one per subscription | nothing prices a query answer or a new route against the pull, and a lost parent cannot say so | +| path in the offer | a list on the advert, the answer, and the query's answer | as the offer | the hop limit | 2 of 6,000 churned meshes unsettled (random 433, drones 1783), longest cycle 109 deliveries against 27: a pull cycle formed on stale adverts cannot be seen | +| query | a request naming an Instance; one answer, offer, path, and mark or none | the edge keeps one answer per extra upstream per demanded Instance until judged; the upstream none | upstreams beyond the first | an edge whose cheaper uplink is cold: 29 against 24 | +| a settled pull stays | none | the route it chose by | one choice per outside event | the pick: random churned mesh 503 never settles | +| choosing again on a route change | none | as above | route changes, which the route layer bounds | settled trees +4.4% against +3.4% on random meshes, +6.0% against +5.0% on regions | + +What the pick needed and this deletes: the rule against a warm signal from a +relay on the taker's own path, since a relay that never moves for price +cannot be drawn back up its own branch; the rule that a move under way stays +while no dearer; and make-before-break bookkeeping for price moves, kept only +for a query answer judged after the pull went live. A lost pull carries +nothing, so a new one replaces it outright. ### What a relay sends @@ -120,122 +166,83 @@ of its own pull is enough. neighbour N: `O(R→N) = O(P→R) + warm(P, R)` while R has demand besides N, otherwise `O(P→R) + cold(P, R)`, the chain priced unshared. The publisher's relay offers 0. An offer is never below its sender's own cost. -- **The path.** Every offer names the relays the data passes from the - publisher's relay to the sender. Without it, cycle deliveries under churn - rose by 72% on random meshes, 7% on drone meshes, and 156% on regions, 1 - drone mesh and 2 regions of 400 never settled, and the merge below cannot - work. +- **The path and the mark.** Every offer names the relays the data passes + from the publisher's relay to the sender, and says whether it came over a + link its receiver marked upstream, as a route's offer does. - **Adverts.** While R has demand for an Instance besides N, N has not marked R upstream, and R has not both marked N upstream and taken its copy - over a marked link, it tells N its offer and path on the announce stream - when `O(R→N) + cold(R, N)` is below N's route cost for the Instance, or - equals it through a next hop other than R, since a tie prefers a warm copy. - It tells again on each change, and withdraws when the demand, its upstream, - or that test ends. Adverts keep their order with the route layer's - messages on the same announce stream: a withdrawn route never trails an - advert sent after it. Nothing it skips between peers could ever be taken: a + over a marked link, it tells N its offer on the announce stream when + `O(R→N) + cold(R, N)` is below N's route cost for the Instance, or equals it + through a next hop other than R, since a tie prefers a warm copy. It tells + again on each change, and withdraws when the demand, its upstream, or that + test ends. Adverts keep their order with the route layer's messages on the + same announce stream. Nothing it skips between peers could ever be taken: a relay at rest costs no more than its own route, and a tie through R is the path join already. R knows N's route cost from what N announces it, and - computes it when N's route runs through R. Skipping the parent and children - by rule instead left 8 of 400 churned drone meshes unsettled. The mark is - the route layer's own rule, a route from one marked link never passed to - another: without it an edge advertised one core's copy to its other core, + computes it when N's route runs through R. The mark is the route layer's + own rule: without it an edge advertised one core's copy to its other core, which then pulled through the edge, a path no route offers. - **The query.** A relay with more than one upstream sends each upstream besides the one its path join takes one query when it gains demand: one - request, one answer. The upstream answers its warm offer and path, or that - it has none, from what it holds, and keeps nothing: it forwards nothing, - holds no demand for it, and sends no update. The relay does not wait: it - joins along its route at once, moves if an answer is strictly cheaper, and - drops an answer once a pull from that upstream is answered, live or lost: - an answer kept longer went stale and held a move to a core that had lost - its upstream. It is not a - metadata-only subscription: one forwarded by a cold core would hold demand - of a kind across the core mesh for as long as the viewer stays, the - standby the epic keeps opt in and off inside the CDN. An upstream cannot - tell which of its upstreams a dual-homed edge's route uses, since the edge - announces it nothing, so only the edge can find a warm core beside its - route. -- **Answers.** The subscribe answer carries the parent's offer and path to the - subscriber, and an update on each change. A subscriber waits for that answer - before it counts the pull as live. + request, one answer. The upstream answers its warm offer, or that it has + none, from what it holds, and keeps nothing: it forwards nothing, holds no + demand for it, and sends no update. It is not a metadata-only + subscription: one forwarded by a cold core would hold demand of a kind + across the core mesh for as long as the viewer stays, the standby the epic + keeps opt in and off inside the CDN. An upstream cannot tell which of its + upstreams a dual-homed edge's route uses, so only the edge can find a warm + core beside its route. +- **Answers.** The subscribe answer carries the parent's offer, path, and + mark, an update on each change, and none once the parent lost its own + upstream, which it says at once to a new subscriber too. A subscriber waits + for that answer before it counts the pull as live. ### What a relay chooses -- Its own cost is `D(R) = O(P→R) + warm(P, R)`. A candidate costs its offer - plus the link's cold price, and the route costs the route layer's summed - cold cost, so a route never ranks better for being extended. -- It pulls from a neighbour only when that is strictly cheaper than `D(R)`. - It never pulls from a neighbour that pulls through it, nor from one whose - path names it. -- It takes no advert or answer from a relay named on its own live pull's - path; its route still counts. That relay's warmth may be this relay's own - demand coming back: N pulls through X from R, so R has demand besides N - and offers N its warm cost; N moves to R, X's pull ends, R's offer turns - cold, N goes back through X, and so on for good. Without the rule, 1 of - 400 churned regions never settled that way under most warm arms; with it, - none of 2,000 drone meshes or regions and 1 of 2,000 random meshes under - the pick. That one, case 503, loops through a sibling: 6 pulls from 4, - which makes 4 warm for 3, so 3 offers 6 less than 4 does; 6 moves to 3, 4 - turns cold for 3, and 6 moves back. 3's offer crosses 6's path at 4, below - the publisher's relay, but refusing every offer that crosses it left a - region unsettled under five other arms and did not settle more under the - pick, so the rule stays as it is and the case is an acceptance test. - A relay on N's path is upstream of N's parent, - so N gives up only a move up its own branch, which its route still offers - when that is the cheaper way. It costs a move to a relay on N's path that - is warm on its own account, which N cannot tell apart from its own demand: - in `an_ancestor_warm_on_its_own_is_passed_over` N keeps a tree of 14 where - 13 is cheapest, but under churn the trees it settled on cost about what - the chain's did, which keeps such moves. -- A move is make before break: the old pull stays until the new one is - answered live, and a move under way stays while it is no dearer than - staying. Its own subscription shares the old parent, which lowers what - staying costs, and dropping the move for that raises it again: a random - mesh oscillated for good until this rule. -- A pending pull whose source left its options fails closed. Two relays that - asked each other at once on stale adverts otherwise wait on each other for - good. -- Equal candidates fall to a warm copy before the route, then the rendezvous - hash, as `route_order` in `rs/moq-net/src/model/origin.rs` already ranks - equal routes. - -Why no cycle outlasts the messages in flight: a relay's cost is its parent's -offer plus a warm price of at least 1, and no offer is below its sender's -cost, so cost rises strictly along every pull chain. Two relays can never -each prefer the other on current offers, and a cycle needs one stale offer. -Gaining demand lowers offers to others, and the one it lowers to the relay -whose own demand it is, that relay ignores; losing it raises only the offer to -the neighbour that still pulls, which then moves only to something strictly -cheaper. The argument misses what one relay's demand does to a price two -paths share, as case 503 above shows, and the sweep is no proof: all 400 -churned meshes of each kind settled under the pick, cycles lasted at most 39 -deliveries, each while some answer was in flight, and at 2,000 per kind one -random mesh never settled. +- **When its demand starts**, it pulls from the cheapest of its route and + the warm copies it has heard of. A copy costs its offer plus the link's + cold price; the route costs the route layer's summed cold cost. Equal + candidates fall to a warm copy before the route, then the rendezvous hash, + as `route_order` in `rs/moq-net/src/model/origin.rs` already ranks equal + routes. It joins at once, without waiting for its queries. +- **On each query answer from that join**, judged once, it moves to the + answering upstream if that is strictly cheaper than its own cost + `O(P→R) + warm(P, R)`, keeping the old pull until the new one is live. +- **When its route changes**, it chooses once more, as at a join, from what + it has heard by then, moving only to something strictly cheaper. A route + change can reach a relay before the advert that came with it; then the one + choice finds only the route (`an_ancestor_warm_on_its_own_is_passed_over` + keeps 14 where 13 is cheapest). +- **When its parent answers none, or its pull's path names it**, it replaces + the pull outright with a fresh choice. +- **Otherwise it stays**, whatever its parent or anyone else offers later. +- It never pulls from a neighbour that pulls through it, nor from one whose + path names it. A pending pull whose source left its options fails closed: + two relays that each turned to the other on stale adverts otherwise wait on + each other for good. The longest such cycle in 2,000 churned meshes per + kind lasted 27 deliveries on random meshes, 44 on drones, and 19 on + regions; the lost-uplink case shows one for 17 steps. + +Why it settles: after the last outside event, a relay chooses only as the +route layer settles (finitely many route changes, which its own TLA+ model +and sweeps cover), on each query answer from a join (one per extra +upstream), and when a pending pull fails closed or a parent says it lost its +upstream. A choice never follows a price, so no relay's choice makes another +choose, except by ending a pull it fed, which happens once per pull. The +sweep agrees, though it is no proof: 6,000 churned meshes, 2,000 per kind, +all settled. The port model-checks it. ### Simultaneous starts -Two drones reaching the CDN at once each pay their own warm uplink, so -neither offers the other anything cheaper, and both uplinks stay until one -drone's demand ends. The maintainer's rule, decided 2026-11-15: merge them -on the rendezvous hash unless that adds a cycle, an unfed reader, or more -messages than the second uplink costs. Under adverts it does add cycles: -in 400 churned drone meshes, 18 deliveries had a pull the merge made inside -a cycle, all transient. The one traced was a pending cycle: a merged -drone's new parent lost its uplink and rejoined through a drone that had -trusted the merged drone's advert, before the lost parent's answer arrived. -The merge also added about 1% unfed reader hops. Under asking on subscribe, -no merged pull was ever in a cycle. - -So the race stands. A drone that loses the one uplink leaves every drone -it fed taking its own uplink at once: 4 uplinks on the five-drone shape, -1 with the merge. If the maintainer takes the merge instead, it is: among -equal-cost copies a drone can reach, it joins the neighbour with the higher -rendezvous hash whose offer equals its own cost, once that leaves it no -dearer than its unshared cost and every other option. It never joins a -sibling under a parent other than the publisher's relay: leaving that parent -raises the parent's offer to the sibling the merge relied on. It needs the -path in each offer. +Two drones reaching the CDN at once each pay their own uplink, and both stay +until one drone's demand ends; under the recommendation nothing merges them, +since a merge is a move for price. The maintainer's rule, decided 2026-11-15, +was to merge them on the rendezvous hash unless that adds a cycle, an unfed +reader, or more messages than the second uplink costs; under the pick's +adverts it added cycles (18 deliveries through merged pulls in 400 churned +drone meshes) and about 1% unfed reader-hops. So the race stands. A drone +that loses the one uplink leaves every drone it fed choosing again at once: +4 uplinks on the five-drone shape. ### What holds @@ -243,13 +250,16 @@ path in each offer. least 1, summed from the source along the path the data takes. - Extend today's selection rather than adding a second selector. A front already moves to a better route of its own Instance and resplices - (`rs/moq-net/src/model/front.rs`). A front moves only to a copy of its own - Instance. + (`rs/moq-net/src/model/front.rs`); here it does so only on the events + above. A front moves only to a copy of its own Instance. - The rule runs in production code that the fixtures and sweeps drive. The stripped rule lived only in a test simulator that checked it against itself, so a rule that never offered a copy passed. Port the warmth shapes and the churn sweep onto production relays: pops, drones in turn and at - once, the lost uplink, the ring, and the tiers. + once, the lost uplink, the ring, the tiers, and random churned mesh 503. +- Model-check settling with TLA+, as the route layer's handoff is: a few + relays, readers starting and stopping, links failing, and stale adverts, + checking that the pull graph stops changing once the outside events stop. - Bound the warm state a peer can make a relay hold: one advert per neighbour per Instance the relay has a route for, a path no longer than the route layer's hop limit, and adverts for anything else dropped. Adverts are @@ -257,16 +267,13 @@ path in each offer. every link, so a relay hears at most one per neighbour per demanded broadcast. A query leaves nothing on the upstream; the edge keeps at most one answer per upstream beyond the first per demanded broadcast. -- Check the Goal's efficiency against plain path routing on the same shapes. - On the simulator: pops 61 against 69 (cheapest tree 59), drones in turn one - uplink against five, a ring of 48 relays 8 uplinks against 48, tiers 1095 - against 1191 (the cheapest), and an edge beside the warm core 24 against - 29, at 4.3 to 11 messages per reader change against 2 to 5.4. Under churn, - the trees each case settled on cost 2.7%, 9.4%, and 4.9% over the cheapest - for the same readers on random meshes, drones, and regions, against path - joins' 2.7%, 28.9%, and 5.0%. +- Check the Goal's efficiency against plain path routing on the same shapes, + as the table above does: on the simulator, 3.4 to 11 messages per reader + change against 2 to 5.4, and under churn the trees settled on cost 3.4%, + 16.2%, and 5.0% over the cheapest for the same readers on random meshes, + drones, and regions, against path joins' 2.7%, 28.9%, and 5.0%. - Port the growth sweep, and hold its laws, per link kind. Per reader change, - warm messages are flat in the number of demanded broadcasts for every arm. + warm messages are flat in the number of demanded broadcasts. - Between cores, a full peer mesh: flat in cores. A core with demand is never cheaper for another core than that core's own route. - Down a marked link: none. Up it: one query per upstream beyond the @@ -290,17 +297,11 @@ path in each offer. once, whose uplinks the race keeps, and the same sweeps with each link delivering in order at its own pace, rather than one control message per delivery step across the mesh, which is what the simulator's unfed counts - and first frames assume. A first frame matches the path join's everywhere - the sweeps measured, but two drones a hop apart (81 ms against 69). Two - edges starting at once each keep their path join (9 against 7), the race - again. Random churned mesh 503 too, which the simulator's pick never - settles. -- A far pop keeps pulling through a nearer pop after that pop's reader - leaves, while the nearer pop's unshared offer plus the warm link stays - below the far pop's own long link. That is the cheaper tree, and the - hysteresis that keeps it from flapping. -- Export upstream copies held, moves by cause (strictly cheaper, route lost - or worse, parent lost its upstream), and adverts sent and held, through the + and first frames assume. A first frame matched the path join's on every + shape measured. Two edges starting at once each keep their path join (9 + against 7), the race again. +- Export upstream copies held, choices by cause (join, query answer, route + change, parent lost its upstream), and adverts sent and held, through the relay's existing metrics. - `rs/moq-net/AGENTS.md` says a relay behaves the same warm or cold. That covers answers to requests; this changes which neighbour a request goes @@ -313,10 +314,11 @@ then-current wip version with this. Public API: report any relay configuration added, such as a warm cost per link. Wire, in the then-current wip version and all additive: an advert on -the announce stream carrying an Instance's offer and path, and its -withdrawal; the same offer and path on the subscribe answer and its updates; -and a query naming an Instance, answered once with an offer and path or -none. Nothing lite-07 freezes changes. +the announce stream carrying an Instance's offer, path, and mark, and its +withdrawal, in order with the route messages there; the same offer, path, and +mark on the subscribe answer and its updates; and a query naming an Instance, +answered once with an offer, path, and mark or none. Nothing lite-07 freezes +changes. ## Required From 1a03ed20ee936492a775c8de8722b740508b2225 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 17:26:48 -0800 Subject: [PATCH 15/19] feat(moq-net): measure settled pulls without the query, and make the query a later step The warmth simulator gains `advertise_not_down+stay+reroute`. It settles all 6,000 churned meshes and costs +5.4% over the cheapest tree on churned regions, against +5.0% with the query, at a fifth of the warm messages. Simulator fixes: - A relay still waiting on its asks no longer answers a new subscriber lost. - A reader at a relay already fed records its first frame at once. - toward()'s doc comment is back on toward(). select.md: - It recommends settled pulls, shipped first without the query. The query is a later step, with its price and condition. - The recommendation awaits the maintainer's confirmation at merge. - It says the settling argument does not yet bound retries after the last outside event, and requires the model check and a fallback without a timer. Co-Authored-By: Claude Opus 5.5 --- quest/m3/cache-routing/select.md | 196 ++++++++++++++++----------- rs/moq-net/src/route/tests/warm.rs | 24 +++- rs/moq-net/src/route/tests/warmth.rs | 14 +- 3 files changed, 144 insertions(+), 90 deletions(-) diff --git a/quest/m3/cache-routing/select.md b/quest/m3/cache-routing/select.md index ccfc5cc4e4b..3fd0770b9b0 100644 --- a/quest/m3/cache-routing/select.md +++ b/quest/m3/cache-routing/select.md @@ -3,13 +3,13 @@ ## Goal A relay with demand for an Instance advertises it on the announce stream to -each peer or upstream that could take it, and an edge queries its other -upstreams when it gains demand. When a relay's demand starts, it pulls from -the cheapest copy it has heard of, its route or a warm neighbour, and once -more on each query answer from that join. After that the pull stays: it moves -only when its parent loses its upstream, the pull runs through the relay -itself, the session closes, or the relay's route changes, never because a -neighbour's price did. A peer that sends no warm signal is treated as cold and +each peer or upstream that could take it. When a relay's demand starts, it +pulls from the cheapest copy it has heard of, its route or a warm neighbour. +After that the pull stays: it moves only when its parent loses its upstream, +the pull runs through the relay itself, the session closes, or the relay's +route changes, never because a neighbour's price did. A later step lets an +edge query its other upstreams when it gains demand, once the conditions +below hold. A peer that sends no warm signal is treated as cold and behaves as it does today. Relays export how many upstream copies they hold and what each choice was for. @@ -24,28 +24,28 @@ Recommended 2026-11-16 on [moq-dev/moq#8903](https://github.com/moq-dev/moq/pull/8903), awaiting the maintainer's confirmation at merge: **a settled pull stays.** A relay chooses where to pull when its demand starts, from its route and the warm copies it -has heard of by then, and once more on each query answer from that join. -After that it chooses again only on an outside event: its parent loses its -upstream, its pull runs through itself, its session closes, or its route -changes. What it hears comes from **adverts between peers and toward -upstreams, only where the copy could be taken, never down a link the -neighbour marked upstream, and never up one with a copy that came over -one**; **a one-shot query** from an edge to its other upstreams when it gains -demand; and **the data path and the mark in every offer**. The race between -simultaneous starts stands. +has heard of by then. After that it chooses again only on an outside event: +its parent loses its upstream, its pull runs through itself, its session +closes, or its route changes. What it hears comes from **adverts between +peers and toward upstreams, only where the copy could be taken, never down a +link the neighbour marked upstream, and never up one with a copy that came +over one**, and **the data path and the mark in every offer**. The race +between simultaneous starts stands. **A one-shot query** from an edge to its +other upstreams is a later step, with its price and condition below; with +it, a relay also chooses once more on each query answer from its join. +Rejected: the price-moving pick, which moved a live pull whenever another +source got strictly cheaper and never settles churned random mesh 503. A rise in the parent's offer is not an event. A rise is what a sibling's move or a neighbour's lost reader causes, and the point of the rule is that no relay's move is another relay's reason to move. A route change is one: the route layer's routes follow links and claims, never demand. -So the pull graph changes only on an outside event (a reader starts or -stops, a link or relay fails or returns, a publisher moves), and each relay -makes a bounded number of choices per event. Settling needs no argument about -prices: after the last outside event, choices come only from the route -layer settling (its own proof), from one query answer per upstream, and from -pending pulls failing closed. It is greedy online Steiner without -rearrangement, apart from the moves a route change buys. +So no price ever moves a pull: a pull moves on an outside event (a reader +starts or stops, a link or relay fails or returns, a publisher moves), on the +route layer settling after one, or on what those set off. It is greedy online +Steiner without rearrangement, apart from the moves a route change buys. +That it then settles is not yet proven; see below. **What the guarantee costs.** Imase and Waxman (Dynamic Steiner Tree Problem, SIAM J. Discrete Math. 4(3), 1991) bound greedy joins without @@ -59,17 +59,21 @@ together their constant-factor algorithm rearranges O(n^(3/2)) edges over n requests, which this never does for price. So the bound is a guide, and the sweeps measure the drift: -| | the recommendation | the pick it replaces | path joins | -|---|---|---|---| -| settled trees over the cheapest, 400 churned random / drone / region meshes | +3.4% / +16.2% / +5.0% | +2.7% / +9.4% / +4.9% | +2.7% / +28.9% / +5.0% | -| unsettled of 2,000 per kind | 0 / 0 / 0 | 1 / 0 / 0 (random case 503) | 0 / 0 / 0 | -| longest cycle under churn, deliveries | 20 / 18 / 19 | 18 / 39 / 24 | 9 / 14 / 17 | -| unfed reader-hops | 19,008 / 11,691 / 23,097 | 18,000 / 12,381 / 22,477 | 15,657 / 7,723 / 15,731 | -| pops, summed over its five phases (cheapest 59) | 71 | 61 | 69 | -| drones in turn | 75, one uplink | 75, one uplink | 155, five | -| drones one / two hops apart | 41 / 33, at 69 / 62 ms | 33 / 25, at 81 / 69 ms | 41 / 41, at 69 / 62 ms | -| edges, regions, rings, tiers | as the pick | 12, 24, 2904, 1095 | 14, 29, 9624, 1191 | -| warm messages per reader change, unpeered CDN | 1.0 | 1.0 | 0 | +| | settled pulls, first cut | with the query, later | the price-moving pick | path joins | +|---|---|---|---|---| +| settled trees over the cheapest, 400 churned random / drone / region meshes | +3.4% / +16.2% / +5.4% | +3.4% / +16.2% / +5.0% | +2.7% / +9.4% / +4.9% | +2.7% / +28.9% / +5.0% | +| unsettled of 2,000 per kind | 0 / 0 / 0 | 0 / 0 / 0 | 1 / 0 / 0 (random case 503) | 0 / 0 / 0 | +| longest cycle under churn, deliveries | 20 / 18 / 14 | 20 / 18 / 19 | 18 / 39 / 24 | 9 / 14 / 17 | +| unfed reader-hops | 19,008 / 11,691 / 17,458 | 19,008 / 11,691 / 23,097 | 18,000 / 12,381 / 22,477 | 15,657 / 7,723 / 15,731 | +| warm messages, 400 churned regions | 4,653 | 21,014 | 21,182 | 0 | +| pops, summed over its five phases (cheapest 59) | 71 | 71 | 61 | 69 | +| drones in turn | 75, one uplink | 75, one uplink | 75, one uplink | 155, five | +| drones one / two hops apart | 41 / 33, at 69 / 62 ms | as the first cut | 33 / 25, at 81 / 69 ms | 41 / 41, at 69 / 62 ms | +| edges in turn, one region with unequal uplinks, rings, tiers | 13, 29, 2904, 1095 | 12, 24, 2904, 1095 | 12, 24, 2904, 1095 | 14, 29, 9624, 1191 | +| warm messages per reader change, unpeered CDN | 0 | 1.0 | 1.0 | 0 | + +Random and drone meshes have no relay with two upstreams, so the query +changes nothing there. Its drift is a relay that stays on a detour once the warm copy that drew it loses its other readers: on pops, the middle pop keeps pulling through the @@ -80,7 +84,7 @@ recommendation) took random meshes from +4.4% to +3.4% and regions from +6.0% to +5.0%. Drone meshes drift most, since their routes rarely change. No timer is added. -**The pick it replaces** moved a live pull whenever another source got +**The price-moving pick, rejected,** moved a live pull whenever another source got strictly cheaper, fenced by a rule that a relay takes no warm signal from a relay on its own path. It never settles churned random mesh 503, a loop through a sibling: 6 pulls from 4, which makes 4 warm for 3, so 3 offers 6 @@ -91,12 +95,13 @@ taker's below the publisher's relay unsettled other arms instead. Under the recommendation the reflection costs at most one wrong choice per event, never a loop. -Rejected, besides the pick: +Also rejected: - Advertising to every neighbour that could take the copy, down marked links too: it grows with a region's edges, since a warm core tells every edge it could feed. 1.7 to 118 warm messages per one-viewer reader change as edges - per core grow from 3 to 128, against the recommendation's flat 1.0. + per core grow from 3 to 128, against none in the first cut and a flat 1.0 + with the query. - Advertising to every neighbour, which also grows with the neighbours of every relay on the warm chain: 9 to 66 per one-viewer reader change as a full core mesh grows from 4 to 32 cores. @@ -122,24 +127,41 @@ Rejected, besides the pick: nothing in a core mesh. - Joins along the path only, the cost baseline, which delivers neither case. -That PR's simulator, `rs/moq-net/src/route/tests/warm.rs`, runs every rule -below as `advertise_not_down_query+hops+stay+reroute`, and its `warmth.rs` +That PR's simulator, `rs/moq-net/src/route/tests/warm.rs`, runs the first +cut as `advertise_not_down+hops+stay+reroute` and the later step as +`advertise_not_down_query+hops+stay+reroute`, and its `warmth.rs` holds the shapes, the churn sweep, and the growth sweep this ports onto production code. -### Decision: the query - -The query is the one part that only an edge with more than one upstream -uses; without it an unpeered CDN sends no warm message at all. With equal -uplinks it buys nothing: every edge of a region ranks the two cores by the -same rendezvous hash per broadcast, so every edge's path join lands on the -same core, always. It pays only where an edge's cheaper uplink is to the core -that is cold: one region with unequal uplinks costs 24 with it and 29 -without, and two edges in turn 12 against 13. It costs one request and one -answer per extra upstream per first viewer at an edge, exactly 1.0 warm -message per reader change on the CDN sweeps, holds nothing on the core, and -carries no traffic. Recommended: keep it. Without it an operator gets the -same tree by pricing each edge's uplinks equally. +### Decision: the query, a later step + +Recommended: ship adverts, the mark, the path, and settled pulls first, and +add the query as a later step; an automated review on #8903 proposed the +same. The query is the one part that only an edge with more than one +upstream uses. + +- **What it buys:** only where an edge's cheaper uplink is to the core that + is cold. One region with unequal uplinks costs 24 with it and 29 without, + and two edges in turn 12 against 13; churned regions settle +5.0% over the + cheapest with it against +5.4% without (+5.1% against +5.3% at 2,000 per + kind). With equal uplinks it buys nothing: every edge of a region ranks the + two cores by the same rendezvous hash per broadcast, so every edge's path + join lands on the same core. +- **What it costs:** a request and an answer on the wire, and exactly 1.0 + warm message per reader change on an unpeered CDN against none. Under + churn, per 400 regions, 21,014 warm messages against 4,653, 59,924 + messages against 45,254, and 23,097 unfed reader-hops against 17,458, much + of that from the simulator delivering one control message per step across + the mesh. The edge holds one answer per extra upstream per demanded + Instance; the upstream holds nothing and carries no traffic. It also adds a + choice after the join that the settling model must cover. +- **Until then** an operator gets the same tree by pricing each edge's + uplinks equally. +- **Its condition:** add it once the settling model holds with the query in + it, the edge's bound of one answer per extra upstream per demanded + Instance, and none on the upstream, is enforced and tested, and a + deployment has unequal uplinks where the core-mesh copy it saves matters. + Shipping it in the first cut instead is measured above, and settles too. ### What each part costs @@ -149,7 +171,7 @@ same tree by pricing each edge's uplinks equally. | the route layer's mark on adverts | one bit on the advert, the answer, and the query's answer, as a route's offer carries | the bit with each heard advert and live pull | one per offer | an edge carries a copy between its cores, which no route would: 51 readers in 400 churned regions fed that way under the pick, and 1.05 to 1.55 warm messages per one-viewer reader change on an unpeered CDN against 1.0 | | offer on the subscribe answer | a cost, path, and mark on the answer, and an update on each change; none when the parent lost its upstream | one per pull | one per subscription | nothing prices a query answer or a new route against the pull, and a lost parent cannot say so | | path in the offer | a list on the advert, the answer, and the query's answer | as the offer | the hop limit | 2 of 6,000 churned meshes unsettled (random 433, drones 1783), longest cycle 109 deliveries against 27: a pull cycle formed on stale adverts cannot be seen | -| query | a request naming an Instance; one answer, offer, path, and mark or none | the edge keeps one answer per extra upstream per demanded Instance until judged; the upstream none | upstreams beyond the first | an edge whose cheaper uplink is cold: 29 against 24 | +| query, the later step | a request naming an Instance; one answer, offer, path, and mark or none | the edge keeps one answer per extra upstream per demanded Instance until judged; the upstream none | upstreams beyond the first | an edge whose cheaper uplink is cold: 29 against 24 | | a settled pull stays | none | the route it chose by | one choice per outside event | the pick: random churned mesh 503 never settles | | choosing again on a route change | none | as above | route changes, which the route layer bounds | settled trees +4.4% against +3.4% on random meshes, +6.0% against +5.0% on regions | @@ -182,7 +204,7 @@ nothing, so a new one replaces it outright. computes it when N's route runs through R. The mark is the route layer's own rule: without it an edge advertised one core's copy to its other core, which then pulled through the edge, a path no route offers. -- **The query.** A relay with more than one upstream sends each upstream +- **The query, the later step.** A relay with more than one upstream sends each upstream besides the one its path join takes one query when it gains demand: one request, one answer. The upstream answers its warm offer, or that it has none, from what it holds, and keeps nothing: it forwards nothing, holds no @@ -205,7 +227,7 @@ nothing, so a new one replaces it outright. candidates fall to a warm copy before the route, then the rendezvous hash, as `route_order` in `rs/moq-net/src/model/origin.rs` already ranks equal routes. It joins at once, without waiting for its queries. -- **On each query answer from that join**, judged once, it moves to the +- **On each query answer from that join**, once the query lands, judged once, it moves to the answering upstream if that is strictly cheaper than its own cost `O(P→R) + warm(P, R)`, keeping the old pull until the new one is live. - **When its route changes**, it chooses once more, as at a join, from what @@ -220,17 +242,23 @@ nothing, so a new one replaces it outright. path names it. A pending pull whose source left its options fails closed: two relays that each turned to the other on stale adverts otherwise wait on each other for good. The longest such cycle in 2,000 churned meshes per - kind lasted 27 deliveries on random meshes, 44 on drones, and 19 on - regions; the lost-uplink case shows one for 17 steps. - -Why it settles: after the last outside event, a relay chooses only as the -route layer settles (finitely many route changes, which its own TLA+ model -and sweeps cover), on each query answer from a join (one per extra -upstream), and when a pending pull fails closed or a parent says it lost its -upstream. A choice never follows a price, so no relay's choice makes another -choose, except by ending a pull it fed, which happens once per pull. The -sweep agrees, though it is no proof: 6,000 churned meshes, 2,000 per kind, -all settled. The port model-checks it. + kind lasted 27 deliveries on random meshes, 44 on drones, and 30 on + regions (19 with the query); the lost-uplink case shows one for 17 steps. + +Why it should settle, and what is not yet shown: after the last outside +event, a relay chooses only as the route layer settles (finitely many route +changes, which its own TLA+ model and sweeps cover), with the query on each +answer from a join (one per extra upstream), and when a pending pull fails closed +or a parent says it lost its upstream. No choice follows a price. But a +fresh choice after a lost parent or a failed pending pull can pick a warm +copy on a stale advert, give that relay demand and so a join of its own, or +end another relay's pull, so "once per pull" does not bound the number of +pulls after the last outside event (an automated review on #8903). The sweep +agrees with settling, though it is no proof: 6,000 churned meshes, 2,000 per +kind, all settled. The port's model gate below closes it, and if the model +cannot rule out repeated warm retries within one epoch, a relay whose warm +choice is lost or fails closed takes its route until its route or its own +demand next changes; no timer. ### Simultaneous starts @@ -258,8 +286,14 @@ that loses the one uplink leaves every drone it fed choosing again at once: and the churn sweep onto production relays: pops, drones in turn and at once, the lost uplink, the ring, the tiers, and random churned mesh 503. - Model-check settling with TLA+, as the route layer's handoff is: a few - relays, readers starting and stopping, links failing, and stale adverts, - checking that the pull graph stops changing once the outside events stop. + relays, readers starting and stopping, links failing, stale adverts, and + two relays pending on each other with no further route event. Check both + that the pull graph stops changing once the outside events stop and the + updates drain, and that every reader whose relay has a route is then fed. + If the model finds warm retries that repeat within one route and demand + epoch, bound them: a relay whose warm choice is lost or fails closed takes + its route until its route or its own demand next changes. The query is + enabled only once the model holds with it. - Bound the warm state a peer can make a relay hold: one advert per neighbour per Instance the relay has a route for, a path no longer than the route layer's hop limit, and adverts for anything else dropped. Adverts are @@ -270,23 +304,23 @@ that loses the one uplink leaves every drone it fed choosing again at once: - Check the Goal's efficiency against plain path routing on the same shapes, as the table above does: on the simulator, 3.4 to 11 messages per reader change against 2 to 5.4, and under churn the trees settled on cost 3.4%, - 16.2%, and 5.0% over the cheapest for the same readers on random meshes, + 16.2%, and 5.4% over the cheapest (5.0% on regions with the query) for the same readers on random meshes, drones, and regions, against path joins' 2.7%, 28.9%, and 5.0%. - Port the growth sweep, and hold its laws, per link kind. Per reader change, warm messages are flat in the number of demanded broadcasts. - Between cores, a full peer mesh: flat in cores. A core with demand is never cheaper for another core than that core's own route. - - Down a marked link: none. Up it: one query per upstream beyond the - first, with its answer, when an edge gains demand, and no advert of a - copy that came down another. Exactly 1.0 per reader change, one viewer or - a crowd, flat from 4 to 32 cores and 3 to 128 edges per core, up to - 2,080 relays; a core holds nothing at rest. + - Down a marked link: none. Up it: no advert of a copy that came down + another, so none in the first cut. With the query, one per upstream + beyond the first, with its answer, when an edge gains demand: exactly 1.0 + per reader change, one viewer or a crowd, flat from 4 to 32 cores and 3 + to 128 edges per core, up to 2,080 relays; a core holds nothing at rest. - Between edges or drones that peer: n − 1 per first viewer (63 at 64 drones), since every one could take the copy; that is what buys the cheapest tree there (432 against 1,328 to 1,456 for asking and path joins, a crowd of 9 on 16 broadcasts). - At rest the busiest relay of 80 holds 99 query answers for 4,096 - one-viewer broadcasts, against 4,355 metadata-only subscriptions for + With the query, at rest the busiest relay of 80 holds 99 query answers + for 4,096 one-viewer broadcasts, against 4,355 metadata-only subscriptions for `ask_1` and 139,264 adverts sent to every neighbour. - A region's edges do not peer every pair as it grows: they reach each other through the region's cores. Peered, the region is a drone mesh and pays its @@ -300,8 +334,8 @@ that loses the one uplink leaves every drone it fed choosing again at once: and first frames assume. A first frame matched the path join's on every shape measured. Two edges starting at once each keep their path join (9 against 7), the race again. -- Export upstream copies held, choices by cause (join, query answer, route - change, parent lost its upstream), and adverts sent and held, through the +- Export upstream copies held, choices by cause (join, route change, parent + lost its upstream, and query answer once it lands), and adverts sent and held, through the relay's existing metrics. - `rs/moq-net/AGENTS.md` says a relay behaves the same warm or cold. That covers answers to requests; this changes which neighbour a request goes @@ -316,9 +350,9 @@ Public API: report any relay configuration added, such as a warm cost per link. Wire, in the then-current wip version and all additive: an advert on the announce stream carrying an Instance's offer, path, and mark, and its withdrawal, in order with the route messages there; the same offer, path, and -mark on the subscribe answer and its updates; and a query naming an Instance, -answered once with an offer, path, and mark or none. Nothing lite-07 freezes -changes. +mark on the subscribe answer and its updates; and, with the later step, a +query naming an Instance, answered once with an offer, path, and mark or +none. Nothing lite-07 freezes changes. ## Required diff --git a/rs/moq-net/src/route/tests/warm.rs b/rs/moq-net/src/route/tests/warm.rs index d0f7a1a274e..da27e6b70e0 100644 --- a/rs/moq-net/src/route/tests/warm.rs +++ b/rs/moq-net/src/route/tests/warm.rs @@ -1065,13 +1065,12 @@ impl WarmSim { stays.then_some(target.via) } - /// Move toward `target` unless already there or on the way, marking a new pull the - /// merge asked for. /// [`Arm::stay`]'s rule for a relay already pulling: a pending pull waits for its /// answer, failing closed only if its source left; a live one moves only for a - /// query answer from its join that is strictly cheaper, judged once, keeping the old - /// pull until the new one is live; a lost one, or one through itself, is replaced by - /// a fresh choice. + /// query answer from its join, or a choice [`Arm::rejoin`] or [`Arm::reroute`] + /// grants, to something strictly cheaper, judged once, keeping the old pull until + /// the new one is live; a lost one, or one through itself, is replaced by a fresh + /// choice. fn stay(&mut self, node: Node, cast: usize, up: &Pull, options: &[Choice], best: Option) { let hops = self.arm.hops; let reroute = self.arm.reroute; @@ -1123,6 +1122,8 @@ impl WarmSim { } } + /// Move toward `target` unless already there or on the way, marking a new pull the + /// merge asked for. fn toward(&mut self, node: Node, cast: usize, target: Node, merged: bool) { let carry = self.entry(node, cast); let up = carry.up.as_ref().map(|pull| pull.via); @@ -1147,8 +1148,11 @@ impl WarmSim { /// Tell each neighbour pulling through `node` what it offers now. fn answer(&mut self, node: Node, cast: usize) { let carry = self.entry(node, cast).clone(); - // Neither pulling nor trying to: a new subscriber hears so at once. - let stuck = carry.next.is_none() && carry.up.as_ref().is_none_or(|pull| pull.state == State::Lost); + // Neither pulling nor trying to, nor waiting on asks: a new subscriber hears so + // at once. + let stuck = carry.unanswered.is_empty() + && carry.next.is_none() + && carry.up.as_ref().is_none_or(|pull| pull.state == State::Lost); for (&child, streams) in &carry.down { let offer = self.offer(node, cast, child); for &stream in streams { @@ -1342,6 +1346,12 @@ impl WarmSim { carry.rejoin |= rejoin && reading && !carry.reader && carry.up.is_some(); carry.reader = reading; carry.since = reading.then_some(round); + // A reader at a relay already fed has its first frame at once, whether or not + // anything is sent. + if reading && self.fed(node, cast) && self.live(node, cast).is_some() { + self.entry(node, cast).since = None; + self.waits.push(0); + } self.dirty.insert((node, cast)); self.step(); } diff --git a/rs/moq-net/src/route/tests/warmth.rs b/rs/moq-net/src/route/tests/warmth.rs index e5c3a8aba92..ce6626ad0b2 100644 --- a/rs/moq-net/src/route/tests/warmth.rs +++ b/rs/moq-net/src/route/tests/warmth.rs @@ -518,10 +518,18 @@ const REJOIN: Arm = Arm { rejoin: true, ..STAY }; /// [`STAY`], choosing again when its route changes. const REROUTE: Arm = Arm { reroute: true, ..STAY }; +/// [`REROUTE`] without the query: adverts and the path only. +const REROUTE_UNQUERIED: Arm = Arm { + stay: true, + reroute: true, + ..NOT_DOWN +}; + /// The narrower audiences for useful adverts, which keep them off a CDN's edges. /// [`NOT_DOWN_QUERY`] is the pick, [`CHAIN`] the pick with the chain, [`STAY`] the -/// pick whose settled pulls stay, and [`REJOIN`] and [`REROUTE`] that, choosing again -/// on a reader of its own or a new route. +/// pick whose settled pulls stay, [`REJOIN`] and [`REROUTE`] that, choosing again +/// on a reader of its own or a new route, and [`REROUTE_UNQUERIED`] the last without +/// the query. fn narrower() -> Vec { vec![ NOT_NEXT, @@ -533,6 +541,7 @@ fn narrower() -> Vec { STAY, REJOIN, REROUTE, + REROUTE_UNQUERIED, ] } @@ -1086,6 +1095,7 @@ fn every_arm_settles_after_churn() { CHAIN, STAY, REROUTE, + REROUTE_UNQUERIED, ] { for shape in [random as fn(&mut XorShift) -> Shape, flying, regional] { let out = churn(arm, 0..12, shape); From 3be3a25a6b883e46b3156f2d1cd4529b4ffcc519 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 17:33:11 -0800 Subject: [PATCH 16/19] docs(quest): say the advert mark covers every hop of the copy's path in the select quest Co-Authored-By: Claude Opus 5.5 --- quest/m3/cache-routing/select.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/quest/m3/cache-routing/select.md b/quest/m3/cache-routing/select.md index 3fd0770b9b0..0bc12446def 100644 --- a/quest/m3/cache-routing/select.md +++ b/quest/m3/cache-routing/select.md @@ -29,7 +29,7 @@ its parent loses its upstream, its pull runs through itself, its session closes, or its route changes. What it hears comes from **adverts between peers and toward upstreams, only where the copy could be taken, never down a link the neighbour marked upstream, and never up one with a copy that came -over one**, and **the data path and the mark in every offer**. The race +down one anywhere on its path**, and **the data path and the mark in every offer**. The race between simultaneous starts stands. **A one-shot query** from an edge to its other upstreams is a later step, with its price and condition below; with it, a relay also chooses once more on each query answer from its join. @@ -189,11 +189,12 @@ nothing, so a new one replaces it outright. otherwise `O(P→R) + cold(P, R)`, the chain priced unshared. The publisher's relay offers 0. An offer is never below its sender's own cost. - **The path and the mark.** Every offer names the relays the data passes - from the publisher's relay to the sender, and says whether it came over a - link its receiver marked upstream, as a route's offer does. + from the publisher's relay to the sender, and carries the mark: set once + the copy came down a link its receiver marked upstream anywhere on that + path, and kept on every hop after, as a route's down-only flag is. - **Adverts.** While R has demand for an Instance besides N, N has not - marked R upstream, and R has not both marked N upstream and taken its copy - over a marked link, it tells N its offer on the announce stream when + marked R upstream, and R has not both marked N upstream and a copy whose + mark is set, it tells N its offer on the announce stream when `O(R→N) + cold(R, N)` is below N's route cost for the Instance, or equals it through a next hop other than R, since a tie prefers a warm copy. It tells again on each change, and withdraws when the demand, its upstream, or that From 558d6a675bbbca40518a5c36bee53b6caf0a2d16 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 18:21:06 -0800 Subject: [PATCH 17/19] quest: a spike for warm pulls that move and still settle The maintainer picked this PR to merge and asked for a warmth spike with it. The new quest takes the settling model check from the selection quest and looks for a rule where a pull moves to a copy that turns warm later. Co-Authored-By: Claude Fable 5.2 --- quest/m3/cache-routing/select.md | 27 +++---- quest/m3/cache-routing/settle.md | 120 +++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 13 deletions(-) create mode 100644 quest/m3/cache-routing/settle.md diff --git a/quest/m3/cache-routing/select.md b/quest/m3/cache-routing/select.md index 0bc12446def..4d751c8e511 100644 --- a/quest/m3/cache-routing/select.md +++ b/quest/m3/cache-routing/select.md @@ -20,9 +20,12 @@ Rewritten 2026-11-15 (maintainer, /quest-plan on to build, item 14 of the [epic](/quest/m3/cache-routing/README.md)'s earlier Plan, is replaced. -Recommended 2026-11-16 on -[moq-dev/moq#8903](https://github.com/moq-dev/moq/pull/8903), awaiting the -maintainer's confirmation at merge: **a settled pull stays.** A relay chooses +Decided 2026-11-16 (maintainer, merge pick of +[moq-dev/moq#8903](https://github.com/moq-dev/moq/pull/8903), which +recommended it): **a settled pull stays**, unless +[the spike](/quest/m3/cache-routing/settle.md) the maintainer asked for with +that pick finds a rule that moves a pull for price and still settles; this +Plan is then rewritten for that rule. A relay chooses where to pull when its demand starts, from its route and the warm copies it has heard of by then. After that it chooses again only on an outside event: its parent loses its upstream, its pull runs through itself, its session @@ -256,7 +259,7 @@ copy on a stale advert, give that relay demand and so a join of its own, or end another relay's pull, so "once per pull" does not bound the number of pulls after the last outside event (an automated review on #8903). The sweep agrees with settling, though it is no proof: 6,000 churned meshes, 2,000 per -kind, all settled. The port's model gate below closes it, and if the model +kind, all settled. [The spike](/quest/m3/cache-routing/settle.md)'s model closes it, and if the model cannot rule out repeated warm retries within one epoch, a relay whose warm choice is lost or fails closed takes its route until its route or its own demand next changes; no timer. @@ -286,15 +289,12 @@ that loses the one uplink leaves every drone it fed choosing again at once: itself, so a rule that never offered a copy passed. Port the warmth shapes and the churn sweep onto production relays: pops, drones in turn and at once, the lost uplink, the ring, the tiers, and random churned mesh 503. -- Model-check settling with TLA+, as the route layer's handoff is: a few - relays, readers starting and stopping, links failing, stale adverts, and - two relays pending on each other with no further route event. Check both - that the pull graph stops changing once the outside events stop and the - updates drain, and that every reader whose relay has a route is then fed. - If the model finds warm retries that repeat within one route and demand - epoch, bound them: a relay whose warm choice is lost or fails closed takes - its route until its route or its own demand next changes. The query is - enabled only once the model holds with it. +- Port the rule [the spike](/quest/m3/cache-routing/settle.md) model-checked + with TLA+, and keep that model in step with the port. If its model found + warm retries that repeat within one route and demand epoch, the rule it + hands over already bounds them: a relay whose warm choice is lost or fails + closed takes its route until its route or its own demand next changes. + The query is enabled only once the model holds with it. - Bound the warm state a peer can make a relay hold: one advert per neighbour per Instance the relay has a route for, a path no longer than the route layer's hop limit, and adverts for anything else dropped. Adverts are @@ -358,6 +358,7 @@ none. Nothing lite-07 freezes changes. ## Required - [The switch](/quest/m2/multicast/switch.md) - the route layer it ranks on +- [The spike](/quest/m3/cache-routing/settle.md) - the rule to port, model-checked to settle ## Related diff --git a/quest/m3/cache-routing/settle.md b/quest/m3/cache-routing/settle.md new file mode 100644 index 00000000000..cb845fe29ea --- /dev/null +++ b/quest/m3/cache-routing/settle.md @@ -0,0 +1,120 @@ +# [L] Warm pulls move for price and still settle + +## Goal + +The warmth simulator has a rule under which a pull moves to a cheaper warm +copy that turns up later, and a model check shows the rule settles: once +readers, links, and publishers stop changing, every pull stops moving and +every routed reader is fed, under every order of delivery at a small bound. +[The selection quest](/quest/m3/cache-routing/select.md) then names that rule +beside settled pulls, with what each costs and what it would add to the wire. + +If no candidate holds up, the PR says how each one failed, and the model +check runs on settled pulls instead, with its fallback made part of the +rule if the check needs it. + +This is a spike: no production code and no wire change. + +## Plan + +Decided 2026-11-16 (maintainer, in the coordinating session, after asking +what warmth would cost before the cut): spike warmth now. The coordinator +read that as this quest, run beside [the switch](/quest/m2/multicast/switch.md). +Whether any warm wire rides lite-07 is the maintainer's to decide on its +result, so report the wire it would need and do not add it. + +Why a spike: [moq-dev/moq#8903](https://github.com/moq-dev/moq/pull/8903) +measured two rules and neither is the one to ship before the cut. + +- Settled pulls settle in every case swept, but a pull never moves to a pop + that turns warm later. On an unpeered CDN the first cut sends no warm + message, so it is path joins with extra parts; on the pops shape it costs + 71 against path joins' 69. Its settling is argued, not shown. +- The price-moving pick does move (61 on pops, drones +9.4% over the + cheapest against +16.2%), and never settles churned random mesh 503: a + relay's own demand comes back to it as a cheaper price two hops away. + +Every fence tried so far kept one shape of that reflection out. Look for a +rule where a relay's choice cannot depend on its own demand by construction. + +**First candidate: order the readers.** The coordinator's, untested; attack +it before building on it. + +- A relay with a reader of its own for an Instance has a key for it: its + route cost to the Instance, then its Node ID. Also measure a hash of Node + ID and Instance, which does not change with routes. +- A pull carries the lowest key among the readers it feeds: the relay's own, + and each downstream pull's. Adverts and subscribe answers carry it. +- A relay takes a warm copy only from a neighbour whose key is below its + own. Such a copy is fed whatever this relay does, so it costs the link + alone. The relay takes the cheapest of those copies and its route, and + chooses again whenever that set or its own key changes. +- The claim to test: the tree is then the one greedy joins build when + readers arrive in key order, whatever order they really arrived in. A + reader's path depends only on readers with lower keys, so it settles by + induction on the key, and each step is a search for the nearest of a + fixed set, the shape [the route proof](/rs/moq-net/verify/route-algebra/README.md) + covers. Imase and Waxman's O(log k) bound holds for any arrival order, and + here it would hold after departures too, since the tree is rebuilt. +- What it should deliver: the pop that turns warm later draws the pulls + behind it; two drones that start at once merge, the higher key moving (the + maintainer suggested a hash tie-break where one drone reneges); case 503 + settles. +- Where it may break: a relay that only forwards takes the key of the pull + through it, and its key drops when a lower one arrives, so it must choose + again and may move the higher keyed pulls it already feeds. A price that + still reads demand above the first hop, as `O(P→R) + warm(P, R)` does, + would bring the reflection back. Stale adverts and keys in flight. +- What it costs, to measure: moves and unfed reader-hops per reader change, + an advert update when a key changes without the demand starting or + ending, and one key per pull and per heard advert. + +**Other candidates**, if the first fails or to beat it: + +- Price each copy at what the pull adds to the tree. The tree's total cost + is then a potential: a move that lowers a relay's price lowers the tree's + cost by as much, provided the price is exact and no two moves overlap. The + pick failed both. Exact needs the asker's demand left out at every hop, + which naming sole demand tried and still left 1 of 2,000 unsettled; + moves that do not overlap need a lock or an order. +- Count only relays with a reader of their own as warm, which no relay's + choice changes, and order by route cost so two such relays never adopt + each other. It loses coalescing at relays that only forward, which is + every core of a CDN. + +Work in `rs/moq-net/src/route/tests/warm.rs` and `warmth.rs`, as new arms +beside `advertise_not_down+hops+stay+reroute` and the pick, so every table +in [select](/quest/m3/cache-routing/select.md) gains a column. + +Model the chosen rule in TLA+ under `rs/moq-net/verify/`, as the other +models there are laid out, with its nightly lane and skill: + +- A few relays, readers starting and stopping, links failing and returning, + adverts and keys delivered in any order and stale. +- Settling is checked from when the outside events stop, not from when the + queues drain: a retry loop keeps the queues full. +- Both properties: the pulls go quiet, and every reader the route layer + routes is then fed, two relays waiting on each other included. +- Show the check has teeth: the price-moving pick, or the rule with its + order removed, must fail it. + +This takes over the model check [select](/quest/m3/cache-routing/select.md) +asked of the port; move that requirement here and leave select the port of +the model's rule. + +Report in the PR: + +- Each candidate tried and how it failed or held, with the case. +- The tables of [moq-dev/moq#8903](https://github.com/moq-dev/moq/pull/8903) + with the new column: trees over the cheapest, unsettled, longest cycle, + unfed reader-hops, warm messages, pops, drones, tiers, growth per link + kind. +- The wire and relay state the rule would add over the first cut, and what + a peer that does not send it sees. +- A recommendation between it and settled pulls for + [select](/quest/m3/cache-routing/select.md), as a decision under + `## Waiting` for the maintainer. + +## Related + +- [Selection](/quest/m3/cache-routing/select.md) - ports the rule this picks From 38dadc9fb634baacc79df5b4dff61a96ea54d126 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 19:38:05 -0800 Subject: [PATCH 18/19] test(moq-net): a marked route does not tie with an unmarked one The merge pointed ties() at the proof's candidates but kept comparing rank alone, so the warmth simulator could send a request over a marked route the route order now ranks below the one in use. Found by the local review. Co-Authored-By: Claude Fable 5.2 --- quest/m3/cache-routing/select.md | 11 ++++++----- rs/moq-net/src/route/tests/multi.rs | 6 +++--- rs/moq-net/src/route/tests/sticky.rs | 21 +++++++++++++++++++++ 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/quest/m3/cache-routing/select.md b/quest/m3/cache-routing/select.md index 4671e486d60..1e4b00aa112 100644 --- a/quest/m3/cache-routing/select.md +++ b/quest/m3/cache-routing/select.md @@ -67,8 +67,8 @@ sweeps measure the drift: | settled trees over the cheapest, 400 churned random / drone / region meshes | +3.4% / +16.2% / +5.4% | +3.4% / +16.2% / +5.0% | +2.7% / +9.4% / +4.9% | +2.7% / +28.9% / +5.0% | | unsettled of 2,000 per kind | 0 / 0 / 0 | 0 / 0 / 0 | 1 / 0 / 0 (random case 503) | 0 / 0 / 0 | | longest cycle under churn, deliveries | 20 / 18 / 14 | 20 / 18 / 19 | 18 / 39 / 24 | 16 / 14 / 17 | -| unfed reader-hops | 19,008 / 11,691 / 17,520 | 19,008 / 11,691 / 23,080 | 18,000 / 12,381 / 22,446 | 15,700 / 7,725 / 15,715 | -| warm messages, 400 churned regions | 4,642 | 20,993 | 21,138 | 0 | +| unfed reader-hops | 19,008 / 11,691 / 17,470 | 19,008 / 11,691 / 23,089 | 18,000 / 12,381 / 22,472 | 15,700 / 7,725 / 15,735 | +| warm messages, 400 churned regions | 4,655 | 21,020 | 21,151 | 0 | | pops, summed over its five phases (cheapest 59) | 71 | 71 | 61 | 69 | | drones in turn | 75, one uplink | 75, one uplink | 75, one uplink | 155, five | | drones one / two hops apart | 41 / 33, at 69 / 62 ms | as the first cut | 33 / 25, at 81 / 69 ms | 41 / 41, at 69 / 62 ms | @@ -79,7 +79,8 @@ Random and drone meshes have no relay with two upstreams, so the query changes nothing there. The table is the rerun of 2026-11-16, after the route order gained the mark -([moq-dev/moq#8904](https://github.com/moq-dev/moq/pull/8904)). Every +([moq-dev/moq#8904](https://github.com/moq-dev/moq/pull/8904)) and a +request's equal next hops follow it. Every settled tree and unsettled count is as first measured; the churn rows moved by under 1%, apart from path joins' longest cycle on random meshes, 9 before. @@ -157,8 +158,8 @@ upstream uses. join lands on the same core. - **What it costs:** a request and an answer on the wire, and exactly 1.0 warm message per reader change on an unpeered CDN against none. Under - churn, per 400 regions, 20,993 warm messages against 4,642, 59,896 - messages against 45,266, and 23,080 unfed reader-hops against 17,520, much + churn, per 400 regions, 21,020 warm messages against 4,655, 59,921 + messages against 45,280, and 23,089 unfed reader-hops against 17,470, much of that from the simulator delivering one control message per step across the mesh. The edge holds one answer per extra upstream per demanded Instance; the upstream holds nothing and carries no traffic. It also adds a diff --git a/rs/moq-net/src/route/tests/multi.rs b/rs/moq-net/src/route/tests/multi.rs index 23f528bfd19..8d24f8e335f 100644 --- a/rs/moq-net/src/route/tests/multi.rs +++ b/rs/moq-net/src/route/tests/multi.rs @@ -1154,14 +1154,14 @@ impl Router { Some((heard.offer.cost + claim.cost, heard.offer.names(self.me))) } - /// Every neighbour whose route to `origin` ranks with the one in use: the equal - /// next hops a request may take. + /// Every neighbour whose route to `origin` ranks with the one in use, mark + /// included: the equal next hops a request may take. pub fn ties(&self, origin: Id) -> Vec { let Some(best) = self.best.get(&origin) else { return Vec::new(); }; (self.candidates(origin).into_iter()) - .filter(|candidate| candidate.rank() == best.rank()) + .filter(|candidate| candidate.rank() == best.rank() && candidate.offer.marked == best.offer.marked) .map(|candidate| candidate.via) .collect() } diff --git a/rs/moq-net/src/route/tests/sticky.rs b/rs/moq-net/src/route/tests/sticky.rs index 831d1320134..5cff2fc5995 100644 --- a/rs/moq-net/src/route/tests/sticky.rs +++ b/rs/moq-net/src/route/tests/sticky.rs @@ -317,3 +317,24 @@ fn an_equal_route_in_use_never_leaves_a_neighbour_without_one() { assert_eq!(settled(1, 2), (Some(1), true), "the unmarked route first"); assert_eq!(settled(2, 1), (Some(1), true), "the marked route first"); } + +/// A request may take any next hop tied with the route in use, and a marked route +/// does not tie with an unmarked one of the same rank: taking it would carry a copy +/// between two upstreams, which the route in use never does. +#[test] +fn a_marked_route_does_not_tie_with_an_unmarked_one() { + let mut net = MultiSim::new(KEPT, 6); + net.client(5); + net.publish(5, 100); + net.link(5, 0, 1, Tie::Peer); + net.link(0, 1, 1, Tie::Peer); + net.link(0, 2, 1, Tie::Peer); + net.link(3, 2, 1, Tie::Up); + net.link(3, 1, 1, Tie::Peer); + net.claim(5, "live/0", true); + assert!(settle(&mut net, &mut Rng(7))); + let router = net.router(3).expect("relay 3"); + let origin = hop(0, 0); + assert_eq!(router.candidates(origin).len(), 2, "both routes are heard"); + assert_eq!(router.ties(origin), vec![1]); +} From 51e2789a615d0c77ba067d9f6707a0e418785360 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 20:55:10 -0800 Subject: [PATCH 19/19] quest: answer the online reviews on the warmth merge The spike quest says the model check has moved, not that it is to move. The selection quest no longer claims reflection can never loop before the model shows it, and bounds heard adverts by routed Instances as its own rule does. The churn sweep's failed column holds the count alone; the first failure's dump goes to stderr instead of breaking the row. Co-Authored-By: Claude Fable 5.2 --- quest/m3/cache-routing/select.md | 7 ++++--- quest/m3/cache-routing/settle.md | 3 +-- rs/moq-net/src/route/tests/warmth.rs | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/quest/m3/cache-routing/select.md b/quest/m3/cache-routing/select.md index 1e4b00aa112..2a951a8ea44 100644 --- a/quest/m3/cache-routing/select.md +++ b/quest/m3/cache-routing/select.md @@ -101,8 +101,9 @@ less than 4 does; 6 moves to 3, 4 turns cold for 3, and 6 moves back. Each fence covered one shape of a relay's own demand coming back as a price, as the old route rule's fixes did; refusing every offer whose path crosses the taker's below the publisher's relay unsettled other arms instead. Under the -recommendation the reflection costs at most one wrong choice per event, never -a loop. +recommendation no price moves a pull, so the reflection can make a choice +wrong but cannot make one. Whether the choices a lost or failed pull sets +off can repeat is not yet shown, as below. Also rejected: @@ -176,7 +177,7 @@ upstream uses. | part | on the wire | relay state | bounded by | without it | |---|---|---|---|---| -| advert | offer, path, and mark on the announce stream, in order with the route messages, and its withdrawal | one heard per neighbour per demanded Instance | neighbours; the path by the hop limit | path joins: drones in turn 5 uplinks against 1, a ring 48 against 8, tiers 1191 against 1095 | +| advert | offer, path, and mark on the announce stream, in order with the route messages, and its withdrawal | one heard per neighbour per Instance the relay has a route for | neighbours; the path by the hop limit | path joins: drones in turn 5 uplinks against 1, a ring 48 against 8, tiers 1191 against 1095 | | the route layer's mark on adverts | one bit on the advert, the answer, and the query's answer, as a route's offer carries | the bit with each heard advert and live pull | one per offer | an edge carries a copy between its cores, which no route would: 51 readers in 400 churned regions fed that way under the pick, and 1.05 to 1.55 warm messages per one-viewer reader change on an unpeered CDN against 1.0 | | offer on the subscribe answer | a cost, path, and mark on the answer, and an update on each change; none when the parent lost its upstream | one per pull | one per subscription | nothing prices a query answer or a new route against the pull, and a lost parent cannot say so | | path in the offer | a list on the advert, the answer, and the query's answer | as the offer | the hop limit | 2 of 6,000 churned meshes unsettled (random 433, drones 1783), longest cycle 109 deliveries against 27: a pull cycle formed on stale adverts cannot be seen | diff --git a/quest/m3/cache-routing/settle.md b/quest/m3/cache-routing/settle.md index cb845fe29ea..181487fc1c2 100644 --- a/quest/m3/cache-routing/settle.md +++ b/quest/m3/cache-routing/settle.md @@ -99,8 +99,7 @@ models there are laid out, with its nightly lane and skill: order removed, must fail it. This takes over the model check [select](/quest/m3/cache-routing/select.md) -asked of the port; move that requirement here and leave select the port of -the model's rule. +had asked of the port; select now ports the rule this model checks. Report in the PR: diff --git a/rs/moq-net/src/route/tests/warmth.rs b/rs/moq-net/src/route/tests/warmth.rs index ce6626ad0b2..0b39e2eb66c 100644 --- a/rs/moq-net/src/route/tests/warmth.rs +++ b/rs/moq-net/src/route/tests/warmth.rs @@ -1356,12 +1356,12 @@ fn benchmark_warmth_churn() { out.settle, out.tree, out.cheapest, - out.first.map_or("0".into(), |first| format!( - "{}: {}", - out.failed, - first.replace(',', ";") - )), + out.failed, ); + // The first failure's dump spans lines, so it stays out of the row. + if let Some(first) = &out.first { + eprintln!("{name},{}: {first}", arm.name()); + } } } }