From a49dad2df2b121448538f074c36660a63f38b4e7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 21:18:44 -0800 Subject: [PATCH 1/7] chore(quest): claim quest/m3/cache-routing/settle.md Quest-Claim: 306cf96ddbf1e7bd From 46b820d89f19a0d0b073cfb91a6f00b696d34ce6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 23:07:42 -0800 Subject: [PATCH 2/7] spike(moq-net): ranked readers in the warmth simulator, and a settling model Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/tests/warm.rs | 198 ++++++++++++- rs/moq-net/src/route/tests/warmth.rs | 90 +++++- rs/moq-net/verify/warm/Drones.cfg | 24 ++ rs/moq-net/verify/warm/Orderless.cfg | 21 ++ rs/moq-net/verify/warm/Pick.cfg | 21 ++ rs/moq-net/verify/warm/PickDrones.cfg | 21 ++ rs/moq-net/verify/warm/WarmSettle.tla | 404 ++++++++++++++++++++++++++ 7 files changed, 766 insertions(+), 13 deletions(-) create mode 100644 rs/moq-net/verify/warm/Drones.cfg create mode 100644 rs/moq-net/verify/warm/Orderless.cfg create mode 100644 rs/moq-net/verify/warm/Pick.cfg create mode 100644 rs/moq-net/verify/warm/PickDrones.cfg create mode 100644 rs/moq-net/verify/warm/WarmSettle.tla diff --git a/rs/moq-net/src/route/tests/warm.rs b/rs/moq-net/src/route/tests/warm.rs index da27e6b70e0..b5a40c908e3 100644 --- a/rs/moq-net/src/route/tests/warm.rs +++ b/rs/moq-net/src/route/tests/warm.rs @@ -116,8 +116,30 @@ pub(super) struct Arm { /// the route layer's routes follow links and claims, never demand, so no relay's /// pull is another's reason to move. pub reroute: bool, + /// Readers are ranked, and a relay takes a warm copy only from a neighbour feeding + /// a reader ranked below every one it feeds itself: such a copy is there whatever + /// this relay does, so it costs the cold price of the link alone. A relay heads for + /// the cheapest such copy or its route whenever either changes, so a pull follows a + /// copy that turns warm later. + pub order: Option, + /// With [`Arm::order`], a copy costs what it does from the publisher, its offer plus + /// the link, rather than the link alone, and a live pull stays unless another + /// source is strictly cheaper than it, priced at the warm link it flows over. + pub summed: bool, } +/// How [`Arm::order`] ranks a reader, then by its relay. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Order { + /// Its relay's route cost: a relay nearer the publisher ranks first. + Route, + /// The rendezvous hash of its relay and the broadcast, which no route changes. + Hash, +} + +/// A reader's rank under [`Arm::order`]: lower ranks first. +pub(super) type Rank = (u64, Node); + impl Arm { pub fn name(self) -> String { let mode = match self.mode { @@ -140,7 +162,13 @@ impl Arm { let stay = if self.stay { "+stay" } else { "" }; let rejoin = if self.rejoin { "+rejoin" } else { "" }; let reroute = if self.reroute { "+reroute" } else { "" }; - format!("{mode}{hops}{merge}{chain}{stay}{rejoin}{reroute}") + let order = match self.order { + None => "", + Some(Order::Route) => "+order_route", + Some(Order::Hash) => "+order_hash", + }; + let summed = if self.summed { "+summed" } else { "" }; + format!("{mode}{hops}{merge}{chain}{stay}{rejoin}{reroute}{order}{summed}") } } @@ -170,6 +198,9 @@ pub(super) struct Signal { pub cost: Metric, pub path: Vec, pub marked: bool, + /// With [`Arm::order`], the lowest rank among the readers the sender feeds besides + /// its receiver. + pub rank: Option, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -191,6 +222,9 @@ struct Pull { merged: bool, /// The relays whose demand alone it carries, as last told the parent. chain: Vec, + /// With [`Arm::order`], the lowest rank among the readers it feeds, as last told + /// the parent. + rank: Option, } #[derive(Clone, Debug)] @@ -201,6 +235,12 @@ enum Msg { stream: u64, open: bool, chain: Vec, + rank: Option, + }, + /// A change to the lowest rank a payload subscription feeds, with [`Arm::order`]. + Rank { + stream: u64, + rank: Option, }, /// A change to the relays whose demand alone a payload subscription carries. Chain { @@ -255,6 +295,8 @@ struct Carry { answered: BTreeMap<(Node, u64), Option>, /// The relays whose demand alone each subscription through it carries. chains: BTreeMap<(Node, u64), Vec>, + /// The lowest rank each subscription through it feeds, with [`Arm::order`]. + ranks: BTreeMap<(Node, u64), Option>, /// Neighbours it asks, with each one's latest word. asks: BTreeMap>, /// The query it holds open to each. @@ -327,6 +369,8 @@ pub(super) struct Sent { pub adverts: usize, /// Updates to the relays whose demand alone a payload subscription carries. pub chains: usize, + /// Updates to the lowest rank a payload subscription feeds, with [`Arm::order`]. + pub ranks: usize, } impl Sent { @@ -334,9 +378,10 @@ impl Sent { self.subscribes + self.answers + self.warm() } - /// What only a warm signal costs: queries, their answers, adverts, and chain updates. + /// What only a warm signal costs: queries, their answers, adverts, and chain and + /// rank updates. pub fn warm(self) -> usize { - self.asks + self.told + self.adverts + self.chains + self.asks + self.told + self.adverts + self.chains + self.ranks } } @@ -351,6 +396,7 @@ impl std::ops::Add for Sent { told: self.told + more.told, adverts: self.adverts + more.adverts, chains: self.chains + more.chains, + ranks: self.ranks + more.ranks, } } } @@ -366,6 +412,7 @@ impl std::ops::Sub for Sent { told: self.told - was.told, adverts: self.adverts - was.adverts, chains: self.chains - was.chains, + ranks: self.ranks - was.ranks, } } } @@ -617,6 +664,7 @@ impl WarmSim { match &msg { Msg::Subscribe { .. } => self.sent.subscribes += 1, Msg::Chain { .. } => self.sent.chains += 1, + Msg::Rank { .. } => self.sent.ranks += 1, Msg::Answer { .. } => self.sent.answers += 1, Msg::Ask { .. } => self.sent.asks += 1, Msg::Told { .. } => self.sent.told += 1, @@ -626,6 +674,7 @@ impl WarmSim { let stream = match &msg { Msg::Subscribe { stream, .. } | Msg::Chain { stream, .. } + | Msg::Rank { stream, .. } | Msg::Answer { stream, .. } | Msg::Ask { stream, .. } | Msg::Told { stream, .. } => *stream as usize, @@ -691,6 +740,7 @@ impl WarmSim { cost: 0, path: Vec::new(), marked: false, + rank: None, }); } let signal = self.live(node, cast)?; @@ -699,6 +749,9 @@ impl WarmSim { let cost = match (self.arm.mode, carry.others(peer)) { // A join along the route needs no cost, only the confirmation. (Mode::Path, _) => 0, + // The copy is there whoever asks: what it costs reads no relay's demand, so + // no rank's choice moves another's price. + _ if self.arm.order.is_some() => signal.cost + price.warm, (_, true) => signal.cost + price.warm, (_, false) => signal.cost + price.cold, }; @@ -707,7 +760,13 @@ impl WarmSim { false => Vec::new(), }; let marked = signal.marked || self.upstream.contains(&(node, carry.up.as_ref()?.via)); - Some(Signal { cost, path, marked }) + let rank = self.arm.order.and_then(|_| self.rank(node, cast, Some(peer))); + Some(Signal { + cost, + path, + marked, + rank, + }) } /// The warm signal `node` gives `peer`: its offer, while it has demand besides `peer`. @@ -716,6 +775,28 @@ impl WarmSim { self.offer(node, cast, peer) } + /// The rank of a reader at `node`, with [`Arm::order`]. + fn own(&self, node: Node, cast: usize) -> Option { + let order = self.arm.order?; + self.carry(node, cast).filter(|carry| carry.reader)?; + Some(match order { + // A reader whose relay has no route yet ranks last among them. + Order::Route => (self.route(node, cast).map_or(Metric::MAX, |(_, cost)| cost), node), + Order::Hash => (self.hash(cast, node), node), + }) + } + + /// The lowest rank among the readers `node` feeds, its own and each subscription's + /// through it, leaving out `besides`'s. + fn rank(&self, node: Node, cast: usize, besides: Option) -> Option { + let carry = self.carry(node, cast)?; + let below = (carry.down.iter()) + .filter(|&(&child, _)| Some(child) != besides) + .flat_map(|(&child, streams)| streams.iter().map(move |&stream| (child, stream))) + .filter_map(|key| carry.ranks.get(&key).copied().flatten()); + self.own(node, cast).into_iter().chain(below).min() + } + /// Every source `node` could pull `cast` from: its route, and each warm neighbour it /// heard of. Never a neighbour that pulls through it. fn options(&self, node: Node, cast: usize) -> Vec { @@ -744,29 +825,52 @@ impl WarmSim { }; // Without the chain, an ancestor's warmth may be this relay's own demand. // A relay that never moves for price has no reason to fence its own warmth off. - let ancestors: Vec = match self.arm.chain || self.arm.stay { + let ancestors: Vec = match self.arm.chain || self.arm.stay || self.arm.order.is_some() { true => Vec::new(), false => self .live(node, cast) .map_or_else(Vec::new, |signal| signal.path.clone()), }; + // Only a copy that feeds a reader ranked below every one this relay feeds, which + // is there whatever this relay does. Its parent's answer counts as much as an + // advert: a parent need not advertise to a relay already pulling from it. + let mine = self.arm.order.and_then(|_| self.rank(node, cast, None)); + let mut heard = heard; + if self.arm.order.is_some() + && let Some(pull) = &carry.up + && let State::Live(signal) = &pull.state + && !heard.iter().any(|&(via, _)| via == pull.via) + { + heard.push((pull.via, signal)); + } for (via, signal) in heard { if !self.is_open(node, via) || carry.down.contains_key(&via) || ancestors.contains(&via) || (self.arm.hops && signal.path.contains(&node)) + || (self.arm.order.is_some() && !signal.rank.is_some_and(|rank| Some(rank) < mine)) { continue; } out.push(Choice { via, - cost: signal.cost + self.price(via, node).cold, + cost: self.copy(signal, via, node), offered: Some(signal.cost), }); } out } + /// What `signal`'s copy at `via` costs `node`: its offer and the cold link, or with + /// [`Arm::order`] and not [`Arm::summed`] the link alone. + fn copy(&self, signal: &Signal, via: Node, node: Node) -> Metric { + let link = self.price(via, node).cold; + match self.arm.order.is_some() && !self.arm.summed { + true => link, + false => signal.cost + link, + } + } + /// The cheapest of `options`, a warm copy before the route, then the higher hash. fn best<'a>(&self, cast: usize, options: &'a [Choice]) -> Option<&'a Choice> { (options.iter()).min_by_key(|choice| { @@ -788,7 +892,8 @@ impl WarmSim { } self.queries(node, cast); self.pull(node, cast); - self.own(node, cast); + self.chained(node, cast); + self.ranked(node, cast); self.answer(node, cast); self.tell(node, cast); self.advertise(node, cast); @@ -893,10 +998,12 @@ impl WarmSim { true => (self.carry(node, cast)).map_or_else(|| vec![node], |carry| carry.chain(node)), false => Vec::new(), }; + let rank = self.rank(node, cast, Some(via)); let open = Msg::Subscribe { stream, open: true, chain: chain.clone(), + rank, }; self.send(node, via, Lane::Pull, cast, open); Pull { @@ -905,11 +1012,36 @@ impl WarmSim { state: State::Pending, merged: false, chain, + rank, + } + } + + /// Tell each parent when the lowest rank its pull feeds changes, with [`Arm::order`]. + fn ranked(&mut self, node: Node, cast: usize) { + if self.arm.order.is_none() { + return; + } + let carry = self.entry(node, cast); + let pulls: Vec<(Node, u64, Option)> = (carry.up.iter().chain(carry.next.iter())) + .map(|pull| (pull.via, pull.stream, pull.rank)) + .collect(); + for (via, stream, was) in pulls { + let rank = self.rank(node, cast, Some(via)); + if rank == was { + continue; + } + let carry = self.entry(node, cast); + for pull in [&mut carry.up, &mut carry.next].into_iter().flatten() { + if pull.stream == stream { + pull.rank = rank; + } + } + self.send(node, via, Lane::Pull, cast, Msg::Rank { stream, rank }); } } /// Tell each parent when the relays whose demand alone its pull carries change. - fn own(&mut self, node: Node, cast: usize) { + fn chained(&mut self, node: Node, cast: usize) { // A join along the route prices nothing, so it needs no chain. if self.arm.mode == Mode::Path || !self.arm.chain { return; @@ -936,6 +1068,7 @@ impl WarmSim { stream, open: false, chain: Vec::new(), + rank: None, }; self.send(node, pull.via, Lane::Pull, cast, close); } @@ -984,6 +1117,10 @@ impl WarmSim { self.stay(node, cast, &up, &options, best); return; } + if self.arm.order.is_some() { + self.ordered(node, cast, &up, &options, best); + return; + } let moving = self.entry(node, cast).next.as_ref().map(|next| next.via); let target = match &up.state { // Nothing flows yet, so a better source replaces it outright. @@ -1122,6 +1259,40 @@ impl WarmSim { } } + /// [`Arm::order`]'s rule for a relay already pulling, from its route and the copies + /// feeding a lower rank. A live pull heads for the best of them, and with + /// [`Arm::summed`] stays while still among them unless another is strictly cheaper + /// than it, priced at the warm link it already flows over; it keeps flowing until + /// the new one is live. A pending + /// pull is replaced outright by a better one, and a lost one, or one through itself, by + /// a fresh choice. With no source left it lets go, so a neighbour pulling through it + /// that is its only way out chooses again. + fn ordered(&mut self, node: Node, cast: usize, up: &Pull, options: &[Choice], best: Option) { + let hops = self.arm.hops; + match (&up.state, best) { + (State::Live(signal), Some(best)) if !(hops && signal.path.contains(&node)) => { + let own = signal.cost + self.price(up.via, node).warm; + let kept = self.arm.summed && options.iter().any(|choice| choice.via == up.via); + let target = match kept && best.cost >= own { + true => up.via, + false => best.via, + }; + self.toward(node, cast, target, false); + } + (State::Pending, Some(best)) if best.via == up.via => {} + (_, best) => { + let carry = self.entry(node, cast); + let (old, next) = (carry.up.take(), carry.next.take()); + self.unsubscribe(node, cast, old); + self.unsubscribe(node, cast, next); + if let Some(best) = best { + let pull = self.subscribe(node, cast, best.via); + self.entry(node, cast).up = Some(pull); + } + } + } + } + /// Move toward `target` unless already there or on the way, marking a new pull the /// merge asked for. fn toward(&mut self, node: Node, cast: usize, target: Node, merged: bool) { @@ -1214,7 +1385,7 @@ impl WarmSim { /// Whether `audience` lets `node` advertise `signal` to `peer`. fn scoped(&self, audience: Audience, node: Node, cast: usize, peer: Node, signal: &Signal) -> bool { - let price = signal.cost + self.price(node, peer).cold; + let price = self.copy(signal, node, peer); let heard = self .net .router(node) @@ -1282,9 +1453,16 @@ impl WarmSim { stream, open: true, chain, + rank, } => { carry.down.entry(from).or_default().insert(stream); carry.chains.insert((from, stream), chain); + carry.ranks.insert((from, stream), rank); + } + Msg::Rank { stream, rank } => { + if carry.down.get(&from).is_some_and(|streams| streams.contains(&stream)) { + carry.ranks.insert((from, stream), rank); + } } // Only while the subscription is open: it may have closed since. Msg::Chain { stream, chain } => { @@ -1298,6 +1476,7 @@ impl WarmSim { close(&mut carry.down, from, stream); carry.answered.remove(&(from, stream)); carry.chains.remove(&(from, stream)); + carry.ranks.remove(&(from, stream)); } Msg::Answer { stream, offer } => { for pull in [&mut carry.up, &mut carry.next].into_iter().flatten() { @@ -1377,6 +1556,7 @@ impl WarmSim { carry.down.remove(&y); carry.answered.retain(|&(child, _), _| child != y); carry.chains.retain(|&(child, _), _| child != y); + carry.ranks.retain(|&(child, _), _| child != y); carry.asks.remove(&y); carry.unanswered.remove(&y); carry.asking.remove(&y); diff --git a/rs/moq-net/src/route/tests/warmth.rs b/rs/moq-net/src/route/tests/warmth.rs index 0b39e2eb66c..3dddfd74347 100644 --- a/rs/moq-net/src/route/tests/warmth.rs +++ b/rs/moq-net/src/route/tests/warmth.rs @@ -23,6 +23,8 @@ fn arms() -> Vec { stay: false, rejoin: false, reroute: false, + order: None, + summed: false, }); } for mode in [ @@ -42,6 +44,8 @@ fn arms() -> Vec { stay: false, rejoin: false, reroute: false, + order: None, + summed: false, }); } } @@ -58,6 +62,8 @@ const fn arm(mode: Mode, merge: bool) -> Arm { stay: false, rejoin: false, reroute: false, + order: None, + summed: false, } } @@ -418,9 +424,13 @@ fn a_join_skips_pops_without_demand_and_warm_ones_draw_it() { // A settled pull stays: the middle pop joins the near pop's copy, but the far // pop, settled on its long link, never moves for the warm pops beside it. let stayed = BTreeSet::from([(4, 0), (0, 1), (1, 2), (0, 3)]); + // Ranked by hash, the middle pop ranks below the near one, so the near pop takes + // the middle pop's copy instead, at the same cost. + let hashed = BTreeSet::from([(4, 0), (0, 2), (2, 1), (2, 3)]); match arm.mode { Mode::Path => assert_eq!(edges, skip, "{case}"), _ if arm.stay => assert_eq!(edges, stayed, "{case}"), + _ if arm.order == Some(Order::Hash) => assert_eq!(edges, hashed, "{case}"), // One query goes to the cheapest link, which for the middle pop is the near // pop's and for the far pop the middle's. _ => assert_eq!(edges, joined, "{case}"), @@ -525,6 +535,27 @@ const REROUTE_UNQUERIED: Arm = Arm { ..NOT_DOWN }; +/// Adverts and the path, with readers ranked by route cost: a relay takes a warm copy +/// only from a lower rank, and moves whenever the cheapest such copy or its route +/// changes. +const ORDER_ROUTE: Arm = Arm { + order: Some(Order::Route), + ..NOT_DOWN +}; + +/// [`ORDER_ROUTE`], ranked by the rendezvous hash instead. +const ORDER_HASH: Arm = Arm { + order: Some(Order::Hash), + ..NOT_DOWN +}; + +/// [`ORDER_ROUTE`], pricing a copy from the publisher and keeping a live pull unless +/// another source is strictly cheaper. +const ORDER_SUMMED: Arm = Arm { + summed: true, + ..ORDER_ROUTE +}; + /// The narrower audiences for useful adverts, which keep them off a CDN's edges. /// [`NOT_DOWN_QUERY`] is the pick, [`CHAIN`] the pick with the chain, [`STAY`] the /// pick whose settled pulls stay, [`REJOIN`] and [`REROUTE`] that, choosing again @@ -542,6 +573,9 @@ fn narrower() -> Vec { REJOIN, REROUTE, REROUTE_UNQUERIED, + ORDER_ROUTE, + ORDER_HASH, + ORDER_SUMMED, ] } @@ -1096,6 +1130,9 @@ fn every_arm_settles_after_churn() { STAY, REROUTE, REROUTE_UNQUERIED, + ORDER_ROUTE, + ORDER_HASH, + ORDER_SUMMED, ] { for shape in [random as fn(&mut XorShift) -> Shape, flying, regional] { let out = churn(arm, 0..12, shape); @@ -1269,7 +1306,7 @@ fn mean(values: &[usize]) -> f64 { #[ignore = "a benchmark; Nightly runs it from test/route-sweep.sh"] fn benchmark_warmth_cases() { println!( - "case,arm,phases,tree_cost,path_cost,cheapest_cost,uplinks,max_copies,carriers,messages,warm_messages,messages_per_reader_change,warm_per_reader_change,first_frame_ms,cycle_steps,longest_cycle,unfed_reader_rounds,advert_messages,query_messages,chain_messages,right" + "case,arm,phases,tree_cost,path_cost,cheapest_cost,uplinks,max_copies,carriers,messages,warm_messages,messages_per_reader_change,warm_per_reader_change,first_frame_ms,cycle_steps,longest_cycle,unfed_reader_rounds,advert_messages,query_messages,chain_messages,rank_messages,right" ); let ms = |rounds: f64| rounds * HOP.as_secs_f64() * 1000.0; for base in cases().into_iter().map(Some).chain([None]) { @@ -1289,7 +1326,7 @@ fn benchmark_warmth_cases() { let end = rows.last().expect("a phase"); let right = rows.iter().find_map(|row| row.right.clone().err()); println!( - "{},{},{},{cost},{},{cheapest},{},{},{},{},{},{:.2},{:.2},{:.0},{},{},{},{},{},{},{}", + "{},{},{},{cost},{},{cheapest},{},{},{},{},{},{:.2},{:.2},{:.0},{},{},{},{},{},{},{},{}", case.name, arm.name(), rows.len(), @@ -1308,6 +1345,7 @@ fn benchmark_warmth_cases() { sent.adverts, sent.asks + sent.told, sent.chains, + sent.ranks, right.map_or("yes".into(), |err| err.replace(',', ";")), ); } @@ -1320,7 +1358,7 @@ fn benchmark_warmth_cases() { #[ignore = "a benchmark; Nightly runs it from test/route-sweep.sh"] fn benchmark_warmth_churn() { println!( - "mesh,arm,cases,deliveries,cycle_deliveries,starved_deliveries,merge_cycle_deliveries,longest_cycle,unfed_reader_hops,extra_uplink_hops,reader_changes,messages,warm_messages,advert_messages,query_messages,chain_messages,settle_rounds,settled_tree_cost,settled_cheapest_cost,failed" + "mesh,arm,cases,deliveries,cycle_deliveries,starved_deliveries,merge_cycle_deliveries,longest_cycle,unfed_reader_hops,extra_uplink_hops,reader_changes,messages,warm_messages,advert_messages,query_messages,chain_messages,rank_messages,settle_rounds,settled_tree_cost,settled_cheapest_cost,failed" ); let cases: usize = std::env::var("MOQ_WARMTH_CASES").map_or(400, |cases| cases.parse().expect("MOQ_WARMTH_CASES")); // A comma-separated list of arm names to run, for a longer sweep of a few. @@ -1337,7 +1375,7 @@ fn benchmark_warmth_churn() { for arm in arms().into_iter().chain(audiences()).filter(picked) { let out = churn(arm, 0..cases, shape); println!( - "{name},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}", + "{name},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}", arm.name(), out.cases, out.watch.steps, @@ -1353,6 +1391,7 @@ fn benchmark_warmth_churn() { out.sent.adverts, out.sent.asks + out.sent.told, out.sent.chains, + out.sent.ranks, out.settle, out.tree, out.cheapest, @@ -1365,3 +1404,46 @@ fn benchmark_warmth_churn() { } } } + + +// --- Ranked readers --- + +/// Ranked by route cost, a pull follows a pop that turns warm later, and leaves it +/// once its reader goes: every phase settles on the cheapest tree, where a settled +/// pull keeps the far pop on its long link and later on the middle pop's detour. +#[test] +fn a_ranked_pull_follows_a_pop_that_turns_warm_later() { + let rows = run(&pops(), ORDER_ROUTE, &pop_phases(), 1); + settles(&rows, &ORDER_ROUTE.name()); + for row in &rows { + assert_eq!(row.cost(), row.cheapest.iter().sum::(), "{}", row.phase); + } + let stayed: Metric = run(&pops(), REROUTE_UNQUERIED, &pop_phases(), 1).iter().map(Row::cost).sum(); + assert!(rows.iter().map(Row::cost).sum::() < stayed); +} + +/// Drones that start at once each take an uplink before any is warm; ranked, every +/// drone above the lowest then sees its copy and moves, leaving one uplink, where +/// settled pulls keep all four. +#[test] +fn ranked_drones_starting_at_once_keep_one_uplink() { + for arm in [ORDER_ROUTE, ORDER_HASH] { + let rows = run(&drones(4), arm, &drone_phases(4, true), 3); + settles(&rows, &arm.name()); + assert_eq!(rows.last().expect("a phase").uplinks(), 1, "{}", arm.name()); + } + let rows = run(&drones(4), REROUTE_UNQUERIED, &drone_phases(4, true), 3); + assert_eq!(rows.last().expect("a phase").uplinks(), 4); +} + +/// Churned random mesh 503, which the price-moving pick never settles: a relay's own +/// demand comes back to it as a cheaper price two hops away. Ranked, no relay's +/// choice reads a higher rank's demand, and it settles. +#[test] +fn ranked_pulls_settle_random_case_503() { + for arm in [ORDER_ROUTE, ORDER_HASH] { + let out = churn(arm, 503..504, random); + assert_eq!(out.failed, 0, "{}: {:?}", arm.name(), out.first); + } + assert_eq!(churn(NOT_DOWN_QUERY, 503..504, random).failed, 1); +} diff --git a/rs/moq-net/verify/warm/Drones.cfg b/rs/moq-net/verify/warm/Drones.cfg new file mode 100644 index 00000000000..3ad92e2be2d --- /dev/null +++ b/rs/moq-net/verify/warm/Drones.cfg @@ -0,0 +1,24 @@ +\* Three drones meshed over cheap links, each with a costly uplink to the +\* publisher's relay: readers start and stop at each drone three times in all, +\* and one uplink may close and reopen. +CONSTANTS + Shape = "drones" + Flaky = {2} + Readable = {2, 3, 4} + Readers = {} + MaxReads = 3 + MaxCloses = 0 + Rule = "order" + Order = "route" + Summed = FALSE + +SPECIFICATION Spec + +CHECK_DEADLOCK FALSE + +INVARIANTS + TypeOK + +PROPERTIES + PullsQuiet + EventuallyFed diff --git a/rs/moq-net/verify/warm/Orderless.cfg b/rs/moq-net/verify/warm/Orderless.cfg new file mode 100644 index 00000000000..c2c9441e5a8 --- /dev/null +++ b/rs/moq-net/verify/warm/Orderless.cfg @@ -0,0 +1,21 @@ +CONSTANTS + Shape = "drones" + Flaky = {} + Readable = {2, 3, 4} + Readers = {} + MaxReads = 3 + MaxCloses = 0 + Rule = "orderless" + Order = "route" + Summed = FALSE + +SPECIFICATION Spec + +CHECK_DEADLOCK FALSE + +INVARIANTS + TypeOK + +PROPERTIES + PullsQuiet + EventuallyFed diff --git a/rs/moq-net/verify/warm/Pick.cfg b/rs/moq-net/verify/warm/Pick.cfg new file mode 100644 index 00000000000..6a102030ba3 --- /dev/null +++ b/rs/moq-net/verify/warm/Pick.cfg @@ -0,0 +1,21 @@ +CONSTANTS + Shape = "mesh503" + Flaky = {} + Readable = {2, 3, 4} + Readers = {2, 4} + MaxReads = 2 + MaxCloses = 0 + Rule = "pick" + Order = "route" + Summed = FALSE + +SPECIFICATION Spec + +CHECK_DEADLOCK FALSE + +INVARIANTS + TypeOK + +PROPERTIES + PullsQuiet + EventuallyFed diff --git a/rs/moq-net/verify/warm/PickDrones.cfg b/rs/moq-net/verify/warm/PickDrones.cfg new file mode 100644 index 00000000000..031a615c97d --- /dev/null +++ b/rs/moq-net/verify/warm/PickDrones.cfg @@ -0,0 +1,21 @@ +CONSTANTS + Shape = "drones" + Flaky = {} + Readable = {2, 3, 4} + Readers = {} + MaxReads = 3 + MaxCloses = 0 + Rule = "pick" + Order = "route" + Summed = FALSE + +SPECIFICATION Spec + +CHECK_DEADLOCK FALSE + +INVARIANTS + TypeOK + +PROPERTIES + PullsQuiet + EventuallyFed diff --git a/rs/moq-net/verify/warm/WarmSettle.tla b/rs/moq-net/verify/warm/WarmSettle.tla new file mode 100644 index 00000000000..91fd1134243 --- /dev/null +++ b/rs/moq-net/verify/warm/WarmSettle.tla @@ -0,0 +1,404 @@ +------------------------------ MODULE WarmSettle ------------------------------ +(***************************************************************************) +(* Where each relay pulls one broadcast from, under the warmth rules the *) +(* simulator in `rs/moq-net/src/route/tests/warm.rs` measures, and whether *) +(* the pulls settle once readers, links, and the publisher stop changing. *) +(* *) +(* Each relay with demand (a reader, or a neighbour pulling through it) *) +(* pulls through one parent: its route's next hop, or a neighbour with a *) +(* warm copy it heard of. What a relay tells a neighbour over one session *) +(* is a register the neighbour sees late: the rank its pull carries, its *) +(* answer to the neighbour's pull, and its advert. `Deliver` hands the *) +(* newest value over at any moment, so a relay acts on stale words for as *) +(* long as TLC likes, skips values, and hears each neighbour in any order. *) +(* *) +(* `Rule` picks how a relay chooses: *) +(* - "order": readers are ranked, by route cost then relay or by relay *) +(* alone (`Order`). A relay takes a warm copy only from a neighbour *) +(* feeding a reader ranked below every reader it feeds itself, priced *) +(* at the link alone (or from the publisher, `Summed`), and heads for *) +(* the cheapest such copy or its route whenever either changes. *) +(* - "orderless": the same with the rank check removed. *) +(* - "pick": the price-moving pick moq-dev/moq#8903 rejected: a copy is *) +(* priced from the publisher, warm only while its sender has demand *) +(* besides the asker, and a live pull moves to anything strictly *) +(* cheaper, fenced off its own ancestors. *) +(* - "stay": settled pulls, the first cut select.md ports: a relay *) +(* chooses when its demand starts, when its route changes, and when its *) +(* pull is lost or runs through itself, never for a price. *) +(* *) +(* Settling is judged from when the outside events stop (`Read`, `Close`, *) +(* `Reopen`, each bounded), not from when the registers drain: with weak *) +(* fairness on every delivery and every relay's reaction, `PullsQuiet` *) +(* says the pulls eventually stop changing, and `EventuallyFed` that every *) +(* reader whose relay has a route is then fed for good. *) +(* *) +(* Scope: one broadcast and one publisher, which never moves. The route *) +(* layer is an oracle: each relay's route is the shortest over the open *) +(* links at once (`verify-route` and the route proof cover how routes *) +(* settle). The three words a session carries one way arrive together, *) +(* though the subscription, its answer, and the advert ride separate *) +(* streams. No link is marked upstream, no advert is withheld as useless, *) +(* and there is no query. *) +(***************************************************************************) +EXTENDS Integers, Sequences, FiniteSets + +CONSTANTS + Shape, \* the mesh, by name: see Links + Flaky, \* relays whose link to the publisher's relay may close and reopen + Readable, \* relays whose reader may start and stop + Readers, \* relays reading at the start + MaxReads, \* reader starts and stops, in all + MaxCloses, \* link closes and reopens, in all + Rule, \* "order", "orderless", "pick", or "stay" + Order, \* with "order": "route" or "id" + Summed \* with "order": a copy is priced from the publisher, not the link alone + +\* Each shape's links, once each as <> with a < b; the +\* publisher joins relay 1. +Links == + CASE Shape = "drones" -> + \* Three drones meshed over cheap links, each with a costly uplink. + {<<1, 2, 10, 4>>, <<1, 3, 10, 4>>, <<1, 4, 10, 4>>, + <<2, 3, 2, 1>>, <<2, 4, 2, 1>>, <<3, 4, 2, 1>>} + [] Shape = "pops" -> + \* Pops on a line from the publisher's, with long links passing over them. + {<<1, 2, 6, 1>>, <<2, 3, 4, 1>>, <<3, 4, 4, 1>>, <<1, 3, 6, 6>>, <<1, 4, 8, 8>>} + [] Shape = "mesh503" -> + \* Churned random mesh 503's links among its relays 4, 3, 6, and the + \* publisher's 5, as relays 2, 3, 4, and 1. + {<<2, 3, 1, 1>>, <<3, 4, 2, 1>>, <<1, 2, 6, 1>>, <<2, 4, 6, 3>>, <<1, 4, 2, 1>>} + +N == Cardinality(UNION {{l[1], l[2]} : l \in Links}) +Home == 1 +Relays == 1 .. N +None == 0 +Inf == 100 +\* No rank: above every rank a reader has, and no pull at all below every one. +NoRank == 10 * Inf + N + 1 +NoPull == -1 +Ordered == Rule \in {"order", "orderless"} + +Min(S) == CHOOSE x \in S : \A y \in S : x <= y + +Pair(a, b) == {a, b} +AllPairs == {Pair(l[1], l[2]) : l \in Links} +\* Each link's prices, and each relay's neighbours, worked out once. +Prices == [p \in AllPairs |-> LET l == CHOOSE l \in Links : Pair(l[1], l[2]) = p IN <>] +Neighbours == [r \in Relays |-> {n \in Relays : Pair(r, n) \in AllPairs}] +Cold(a, b) == Prices[Pair(a, b)][1] +Warm(a, b) == Prices[Pair(a, b)][2] +Nb(r) == Neighbours[r] +\* Whether a rule reads what a copy costs from the publisher: the link alone needs none. +Costed == ~(Ordered /\ ~Summed) + +InPath(r, path) == \E i \in 1 .. Len(path) : path[i] = r + +NoSig == [cost |-> 0, path |-> <<>>, rank |-> NoRank] +Empty == [pull |-> NoPull, + ans |-> [kind |-> "none", sig |-> NoSig], + adv |-> [on |-> FALSE, sig |-> NoSig]] + +VARIABLES + reader, \* reader[r]: a reader at r + open, \* the open links, each as a pair + route, \* route[r]: r's next hop toward the publisher and its cold cost + up, \* up[r]: the neighbour r pulls through, or None + nxt, \* nxt[r]: a move under way: it replaces up[r] once answered + view, \* view[r][s]: what r last heard over its session to s + chosen, \* chosen[r]: with "stay", the route r last chose by + reads, \* reader starts and stops so far + closes \* link closes and reopens so far + +vars == <> + +(***************************************************************************) +(* The route layer, as an oracle over the open links. *) +(***************************************************************************) +OpenNb(r, o) == {n \in Nb(r) : Pair(r, n) \in o} + +RECURSIVE Dist(_, _) +Dist(k, o) == + IF k = 0 THEN [r \in Relays |-> IF r = Home THEN 0 ELSE Inf] + ELSE LET d == Dist(k - 1, o) IN + [r \in Relays |-> IF r = Home THEN 0 + ELSE Min({d[r]} \cup {Cold(r, n) + d[n] : n \in OpenNb(r, o)})] + +\* Each relay's next hop, the lowest of equal ones, and its cost. +Routes(o) == + LET d == Dist(N, o) IN + [r \in Relays |-> + IF r = Home \/ d[r] >= Inf THEN <> + ELSE <>] + +Hop(r) == route[r][1] +Reach(r) == r = Home \/ route[r][2] < Inf +IsOpen(a, b) == Pair(a, b) \in open + +(***************************************************************************) +(* What a relay knows of its own pull and its neighbours'. *) +(***************************************************************************) +Down(r) == {c \in Nb(r) : IsOpen(r, c) /\ view[r][c].pull # NoPull} +Demand(r) == reader[r] \/ Down(r) # {} +Others(r, c) == reader[r] \/ Down(r) \ {c} # {} + +Own(r) == + IF ~reader[r] THEN NoRank + ELSE IF Order = "id" THEN r + ELSE route[r][2] * (N + 1) + r + +\* The lowest rank among the readers r feeds, leaving out c's. +RankOf(r, c) == + IF ~Ordered THEN 0 + ELSE Min({Own(r)} \cup {view[r][d].pull : d \in Down(r) \ {c}}) + +Ans(r, s) == view[r][s].ans +\* How r's pull through s stands: "live", "self" (its path names r), "lost", or "pending". +State(r, s) == + CASE Ans(r, s).kind = "live" /\ ~InPath(r, Ans(r, s).sig.path) -> "live" + [] Ans(r, s).kind = "live" -> "self" + [] Ans(r, s).kind = "lost" -> "lost" + [] OTHER -> "pending" + +Live(r) == r = Home \/ (up[r] # None /\ State(r, up[r]) = "live") +Sig(r) == IF r = Home THEN NoSig ELSE Ans(r, up[r]).sig + +\* What r offers c, while r is fed. +Offer(r, c) == + LET link == IF r = Home THEN 0 + ELSE IF Ordered \/ Others(r, c) THEN Warm(up[r], r) + ELSE Cold(up[r], r) + IN [cost |-> IF Costed THEN Sig(r).cost + link ELSE 0, + path |-> Append(Sig(r).path, r), + rank |-> IF Ordered THEN RankOf(r, c) ELSE NoRank] + +Stuck(r) == r # Home /\ nxt[r] = None /\ (up[r] = None \/ State(r, up[r]) = "lost") + +Answer(r, c) == + IF c \notin Down(r) THEN Empty.ans + ELSE IF Live(r) THEN [kind |-> "live", sig |-> Offer(r, c)] + ELSE IF Stuck(r) THEN [kind |-> "lost", sig |-> NoSig] + ELSE [kind |-> "wait", sig |-> NoSig] + +Advert(r, c) == + IF Live(r) /\ Others(r, c) THEN [on |-> TRUE, sig |-> Offer(r, c)] + ELSE Empty.adv + +\* What r tells c now: the rank its pull through c carries, its answer, its advert. +Pub(r, c) == + [pull |-> IF up[r] = c \/ nxt[r] = c THEN RankOf(r, c) ELSE NoPull, + ans |-> Answer(r, c), + adv |-> Advert(r, c)] + +(***************************************************************************) +(* What a relay chooses. *) +(***************************************************************************) +\* The warm copies r heard of: adverts, and with "order" its parent's answer. +Heard(r, u) == + {<> : s \in {s \in OpenNb(r, open) : view[r][s].adv.on}} + \cup (IF Ordered /\ u # None /\ State(r, u) = "live" /\ ~view[r][u].adv.on + THEN {<>} ELSE {}) + +CopyCost(r, s, sig) == + IF Ordered /\ ~Summed THEN Cold(s, r) ELSE sig.cost + Cold(s, r) + +\* The pick fences off the relays on its own live pull's path. +Ancestors(r, u) == + IF Rule = "pick" /\ u # None /\ State(r, u) = "live" + THEN {Ans(r, u).sig.path[i] : i \in 1 .. Len(Ans(r, u).sig.path)} + ELSE {} + +Options(r, u) == + LET mine == RankOf(r, None) + routes == IF Hop(r) # None /\ Hop(r) \notin Down(r) + THEN {[via |-> Hop(r), cost |-> route[r][2], copy |-> FALSE]} + ELSE {} + copies == {[via |-> h[1], cost |-> CopyCost(r, h[1], h[2]), copy |-> TRUE] : + h \in {h \in Heard(r, u) : + /\ h[1] \notin Down(r) + /\ h[1] \notin Ancestors(r, u) + /\ ~InPath(r, h[2].path) + /\ Rule = "order" => h[2].rank < mine}} + IN routes \cup copies + +\* The cheaper, a warm copy before the route, then the lower relay. +Better(a, b) == + \/ a.cost < b.cost + \/ a.cost = b.cost /\ a.copy /\ ~b.copy + \/ a.cost = b.cost /\ a.copy = b.copy /\ a.via < b.via +Best(opts) == CHOOSE o \in opts : \A p \in opts : p = o \/ Better(o, p) +Vias(opts) == {o.via : o \in opts} + +\* Head for t, keeping u until the move is answered. +Toward(u, x, t) == + IF t = u THEN <> ELSE IF t = x THEN <> ELSE <> + +\* The pull and move r's rule picks, from the pull u and move x it has once a +\* move that was answered took effect. +Decide(r, u, x) == + LET opts == Options(r, u) + best == IF opts = {} THEN None ELSE Best(opts).via + st == IF u = None THEN "none" ELSE State(r, u) + own == IF st = "live" THEN Ans(r, u).sig.cost + Warm(u, r) ELSE Inf + IN + CASE st = "none" -> <> + [] Ordered -> + (CASE st = "live" /\ best # None -> + IF Summed /\ u \in Vias(opts) /\ Best(opts).cost >= own + THEN Toward(u, x, u) ELSE Toward(u, x, best) + [] st = "pending" /\ best = u -> <> + [] OTHER -> <>) + [] Rule = "pick" -> + (CASE st = "pending" -> + LET cur == {o \in opts : o.via = u} IN + IF best = None THEN <> + ELSE IF best # u /\ (cur = {} \/ \A o \in cur : Best(opts).cost < o.cost) + THEN <> ELSE <> + [] st = "live" -> + LET moving == IF x # None /\ \E o \in opts : o.via = x /\ o.cost <= own + THEN x ELSE None + IN IF best # None /\ Best(opts).cost < own THEN Toward(u, x, best) + ELSE IF moving # None THEN Toward(u, x, moving) + ELSE Toward(u, x, u) + [] OTHER -> Toward(u, x, IF best = None THEN u ELSE best)) + [] OTHER -> \* "stay" + (CASE st = "pending" -> IF u \in Vias(opts) THEN <> ELSE <> + [] st = "live" -> + IF chosen[r] # route[r] /\ best # None /\ best # u /\ Best(opts).cost < own + THEN Toward(u, x, best) + ELSE Toward(u, x, IF x # None THEN x ELSE u) + [] OTHER -> <>) + +\* r's pull and move once it has reacted to what it knows. +Next1(r) == + IF ~Demand(r) THEN <> + ELSE + LET x0 == nxt[r] + done == x0 # None /\ State(r, x0) = "live" + drop == x0 # None /\ State(r, x0) \in {"lost", "self"} + u == IF done THEN x0 ELSE up[r] + x == IF done \/ drop THEN None ELSE x0 + IN Decide(r, u, x) + +\* With "stay", the route a relay chose by: updated whenever it chooses afresh +\* or its route changed while its pull was answered. +Chose(r, pulls) == + IF Rule # "stay" THEN chosen[r] + ELSE IF pulls[1] = None THEN <> + ELSE IF up[r] = None \/ (pulls[1] # up[r] /\ pulls[2] = None) THEN route[r] + ELSE IF State(r, up[r]) # "pending" THEN route[r] + ELSE chosen[r] + +(***************************************************************************) +(* Steps. *) +(***************************************************************************) +Init == + /\ reader = [r \in Relays |-> r \in Readers] + /\ open = AllPairs + /\ route = Routes(AllPairs) + /\ up = [r \in Relays |-> None] + /\ nxt = [r \in Relays |-> None] + /\ view = [r \in Relays |-> [s \in Relays |-> Empty]] + /\ chosen = [r \in Relays |-> <>] + /\ reads = 0 + /\ closes = 0 + +\* A relay reacts to what it knows, when that changes its pull. +React(r) == + /\ r # Home + /\ LET pulls == Next1(r) + c == Chose(r, pulls) + IN /\ <> # <> + /\ up' = [up EXCEPT ![r] = pulls[1]] + /\ nxt' = [nxt EXCEPT ![r] = pulls[2]] + /\ chosen' = [chosen EXCEPT ![r] = c] + /\ UNCHANGED <> + +\* r hears what s tells it now. +Deliver(s, r) == + /\ s \in Nb(r) + /\ IsOpen(s, r) + /\ view[r][s] # Pub(s, r) + /\ view' = [view EXCEPT ![r][s] = Pub(s, r)] + /\ UNCHANGED <> + +Read(r) == + /\ r \in Readable + /\ reads < MaxReads + /\ reads' = reads + 1 + /\ reader' = [reader EXCEPT ![r] = ~@] + /\ UNCHANGED <> + +\* The session over a link closes, and both ends forget what it carried. +Close(a, b) == + /\ a = Home /\ b \in Flaky + /\ IsOpen(a, b) + /\ closes < MaxCloses + /\ closes' = closes + 1 + /\ open' = open \ {Pair(a, b)} + /\ route' = Routes(open') + /\ LET other(r) == IF r = a THEN b ELSE a IN + /\ up' = [r \in Relays |-> + IF r \in {a, b} /\ up[r] = other(r) + THEN (IF nxt[r] = other(r) THEN None ELSE nxt[r]) + ELSE up[r]] + /\ nxt' = [r \in Relays |-> + IF r \in {a, b} /\ (up[r] = other(r) \/ nxt[r] = other(r)) THEN None + ELSE nxt[r]] + /\ view' = [view EXCEPT ![a][b] = Empty, ![b][a] = Empty] + /\ UNCHANGED <> + +Reopen(a, b) == + /\ a = Home /\ b \in Flaky + /\ ~IsOpen(a, b) + /\ closes < MaxCloses + /\ closes' = closes + 1 + /\ open' = open \cup {Pair(a, b)} + /\ route' = Routes(open') + /\ UNCHANGED <> + +Outside == + \/ \E r \in Relays : Read(r) + \/ \E r \in Flaky : Close(Home, r) \/ Reopen(Home, r) + +Next == + \/ Outside + \/ \E r \in Relays : React(r) + \/ \E r \in Relays, s \in Relays : Deliver(s, r) + +Spec == + /\ Init + /\ [][Next]_vars + /\ \A r \in Relays : WF_vars(React(r)) + /\ \A r \in Relays, s \in Relays : WF_vars(Deliver(s, r)) + +(***************************************************************************) +(* Rules. *) +(***************************************************************************) +TypeOK == + /\ reader \in [Relays -> BOOLEAN] + /\ up \in [Relays -> Relays \cup {None}] + /\ nxt \in [Relays -> Relays \cup {None}] + /\ \A r \in Relays : up[r] # None => up[r] \in Nb(r) + /\ \A r \in Relays : nxt[r] # None => nxt[r] \in Nb(r) /\ nxt[r] # up[r] + +\* r's pulls lead to the publisher, each held by its parent and answered live. +RECURSIVE FedFrom(_, _) +FedFrom(r, k) == + IF r = Home THEN TRUE + ELSE IF k = 0 THEN FALSE + ELSE LET p == up[r] IN + /\ p # None + /\ IsOpen(r, p) + /\ State(r, p) = "live" + /\ view[p][r].pull # NoPull + /\ FedFrom(p, k - 1) + +AllFed == \A r \in Relays : reader[r] /\ Reach(r) => FedFrom(r, N) + +\* Once the outside events stop, the pulls stop changing. +PullsQuiet == <>[][UNCHANGED <>]_vars + +\* Once they stop, every reader with a route is fed for good. +EventuallyFed == <>[]AllFed +============================================================================== From 2df4a9041e0171cdf0d0cb5935827abbf181a4a4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 01:53:23 -0800 Subject: [PATCH 3/7] spike(moq-net): ranked readers settle under TLA+, priced from the publisher Advert audiences under ranked readers are priced from the publisher, so a full core mesh's warm messages stay flat. WarmSettle gains configs for ranked readers, settled pulls, and two open findings: the pick, and the ranked rule without its rank. select.md names ranked readers beside settled pulls; the settle quest is done. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-warm/SKILL.md | 63 +++++++ .github/workflows/nightly.yml | 7 +- quest/m3/cache-routing/select.md | 163 ++++++++++++++---- quest/m3/cache-routing/settle.md | 119 ------------- rs/moq-net/src/route/tests/warm.rs | 5 +- rs/moq-net/src/route/tests/warmth.rs | 54 ++++-- rs/moq-net/verify/README.md | 2 +- rs/moq-net/verify/warm/Drones.cfg | 8 +- rs/moq-net/verify/warm/Flap.cfg | 23 +++ rs/moq-net/verify/warm/Hash.cfg | 23 +++ rs/moq-net/verify/warm/Mesh503.cfg | 24 +++ rs/moq-net/verify/warm/Orderless.cfg | 5 + rs/moq-net/verify/warm/Pick.cfg | 9 +- rs/moq-net/verify/warm/Settled.cfg | 22 +++ .../{PickDrones.cfg => SettledDrones.cfg} | 3 +- rs/moq-net/verify/warm/WarmSettle.tla | 50 ++++-- 16 files changed, 385 insertions(+), 195 deletions(-) create mode 100644 .claude/skills/verify-warm/SKILL.md delete mode 100644 quest/m3/cache-routing/settle.md create mode 100644 rs/moq-net/verify/warm/Flap.cfg create mode 100644 rs/moq-net/verify/warm/Hash.cfg create mode 100644 rs/moq-net/verify/warm/Mesh503.cfg create mode 100644 rs/moq-net/verify/warm/Settled.cfg rename rs/moq-net/verify/warm/{PickDrones.cfg => SettledDrones.cfg} (80%) diff --git a/.claude/skills/verify-warm/SKILL.md b/.claude/skills/verify-warm/SKILL.md new file mode 100644 index 00000000000..332c5f1301a --- /dev/null +++ b/.claude/skills/verify-warm/SKILL.md @@ -0,0 +1,63 @@ +--- +name: verify-warm +description: Model-check whether warm pulls settle (`rs/moq-net/verify/warm/WarmSettle.tla`) with TLA+, for the rules the warmth simulator measures (`rs/moq-net/src/route/tests/warm.rs`): ranked readers, settled pulls, and the rejected price-moving pick. Use when a change touches how a relay chooses where to pull a broadcast from (its route or a neighbour's warm copy), what a warm advert or subscribe answer carries (cost, path, rank), which neighbours a relay advertises to, or when a relay chooses again, or when a warmth bug needs every ordering of adverts, answers, and ranks checked rather than a random churn sweep. +--- + +# Verify warm pulls settle + +`rs/moq-net/verify/warm/WarmSettle.tla` models where each relay pulls one broadcast from: its route's next hop, or a neighbour with a warm copy it heard of. What one relay tells a neighbour over a session (the rank its pull carries, its answer to the neighbour's pull, and its advert) is a register the neighbour reads late, so every relay acts on stale words for as long as TLC likes. Readers start and stop and uplinks close and reopen up to a bound; then nothing outside changes. With weak fairness on every delivery and every relay's reaction, TLC checks that the pulls then go quiet and every reader with a route is fed, from every state the bounded outside events reach. + +`Rule` picks the rule: + +- `"order"`: ranked readers, the rule [moq-dev/moq#8986](https://github.com/moq-dev/moq/pull/8986) measured. A reader ranks by its relay's route cost, then its relay (`Order = "route"`), or by relay alone (`"id"`). A relay takes a warm copy only from a neighbour feeding a reader ranked below every reader it feeds itself, priced at the link's cold price alone, and heads for the cheapest such copy or its route whenever either changes. A pull carries the lowest rank it feeds, and an advert or answer the lowest rank its sender feeds besides its receiver. +- `"orderless"`: the same with the rank check removed. +- `"pick"`: the price-moving pick moq-dev/moq#8903 rejected. +- `"stay"`: settled pulls, the first cut `quest/m3/cache-routing/select.md` names. + +## Rules + +| Rule | Holds when | Harm if broken | +| --- | --- | --- | +| `TypeOK` | Every pull and move names a neighbour, a move never names the pull it replaces, and only a pull or move is remembered as answered. | The model, not the rule, is wrong. | +| `PullsQuiet` | Once the outside events stop, the pulls stop changing (`<>[][UNCHANGED <>]_vars`), however the registers keep moving. | Relays move pulls between copies for good, as case 503 did under the pick: copies flap and readers stall on every move. | +| `EventuallyFed` | Once the outside events stop, every reader whose relay has a route is fed for good: its pulls lead to the publisher's relay, each held by its parent and answered live. | Two relays wait on each other, or a reader with a route is never fed. | + +## Run + +```sh +quest gate just rs tla moq-net/verify/warm/WarmSettle.tla -config rs/moq-net/verify/warm/Drones.cfg +``` + +| Config | Rule | Checks | States, time at four workers | +| --- | --- | --- | --- | +| `Drones.cfg` | ranked by route cost | three drones meshed with costly uplinks, readers starting and stopping three times | 164,434, 5 minutes | +| `Hash.cfg` | ranked by relay | the same | 164,434, 5 minutes | +| `Flap.cfg` | ranked by route cost | the drones with two reader changes and one uplink closing and reopening | FLAP_STATES | +| `Mesh503.cfg` | ranked by route cost | churned random mesh 503's six relays and readers, from no pulls | 490,125, 25 minutes | +| `Settled.cfg` | settled pulls | the same | 407,526, 6 minutes | +| `SettledDrones.cfg` | settled pulls | as `Drones.cfg` | 6,852, seconds | +| `Pick.cfg` | the price-moving pick | as `Mesh503.cfg`; fails `PullsQuiet` (open finding) | fails within 2 minutes | +| `Orderless.cfg` | ranked, rank check removed | as `Drones.cfg`; fails `PullsQuiet` (open finding) | fails within 6 minutes | + +Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. Two carry one, and neither is to be fixed. `Pick.cfg` shows the check has teeth: the rule it models is the one moq-dev/moq#8903 rejected for never settling case 503. `Orderless.cfg` shows the rank is what settles the ranked rule: without it, readers at drones 2 and 3 each take the other's warm copy over their uplinks, find their own path in what they now pull, fall back to their uplinks, and repeat for good. + +## Reading a failure + +Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on the simulator: build the shape with `WarmSim::link` and `publish`, start readers with `read`, close links with `close`, and deliver one message at a time with `deliver` in the model's order, under the arm the config's `Rule` names (`ORDER_ROUTE`, `ORDER_HASH`, `REROUTE_UNQUERIED`, `NOT_DOWN_QUERY` in `route/tests/warmth.rs`). `ranked_pulls_settle_random_case_503` replays case 503. + +## Coverage + +Sync commit: the merge of moq-dev/moq#8986. Run `git log -- rs/moq-net/src/route/tests/warm.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. The rule lives in the simulator only; the port in `select.md` moves it into the relay and this table with it. + +| Code | Model | +| --- | --- | +| `rs/moq-net/src/route/tests/warm.rs` `WarmSim::ordered`, `pull`, `toward` | `Decide` (`Ordered`), `Next1`, `Toward` | +| `rs/moq-net/src/route/tests/warm.rs` `WarmSim::options`, `copy`, `best` | `Options`, `Heard`, `CopyCost`, `Best` | +| `rs/moq-net/src/route/tests/warm.rs` `WarmSim::rank`, `own`, `ranked` | `RankOf`, `Own`, `Pub` (`pull`) | +| `rs/moq-net/src/route/tests/warm.rs` `WarmSim::offer`, `warm`, `answer`, `advertise`, `scoped` | `Offer`, `Answer`, `Advert`, `Useful` | +| `rs/moq-net/src/route/tests/warm.rs` `WarmSim::stay` | `Decide` (`"stay"`), `Chose` | +| `rs/moq-net/src/route/tests/warm.rs` `WarmSim::close`, `reopen` | `Close`, `Reopen` | + +## Not covered + +One broadcast and one publisher, which never moves. The route layer is an oracle: each relay's route is the shortest over the open links at once, the lowest relay among equal ones; `verify-route` and the route proof cover how routes settle. The three words one session carries one way arrive together, though the subscription, its answer, and the advert ride separate streams, and a register keeps only its newest value, so a relay may skip values but never reads one out of order on one session. No link is marked upstream, so the mark is left out, and there is no query. A pull replaced by one through the same neighbour is the same pull. Relays never restart. A pass covers four relays with three reader changes, or two and one uplink flap, and case 503's six relays with their readers fixed. diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 8d6192579da..f2272c52e39 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -476,7 +476,8 @@ jobs: matrix: # The largest models get a runner each, as do the route model's two # largest configs (Early about 14M distinct states, LagWide about - # 4.7M); the `route` and `rest` legs take every other config, so a + # 4.7M) and the warm model (its configs take about an hour together); + # the `route` and `rest` legs take every other config, so a # new one is checked without editing this list. sh/gh/nightly.test.py # fails on a path that no longer exists. include: @@ -488,8 +489,10 @@ jobs: paths: rs/moq-net/verify/route/LagWide.cfg - name: route paths: "rs/moq-net/verify/route :!rs/moq-net/verify/route/Early.cfg :!rs/moq-net/verify/route/LagWide.cfg" + - name: warm + paths: rs/moq-net/verify/warm - name: rest - paths: ":!rs/moq-net/verify/resume :!rs/moq-net/verify/route" + paths: ":!rs/moq-net/verify/resume :!rs/moq-net/verify/route :!rs/moq-net/verify/warm" steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/quest/m3/cache-routing/select.md b/quest/m3/cache-routing/select.md index 2a951a8ea44..9b29717eb89 100644 --- a/quest/m3/cache-routing/select.md +++ b/quest/m3/cache-routing/select.md @@ -11,7 +11,9 @@ route changes, never because a neighbour's price did. A later step lets an edge query its other upstreams when it gains demand, once the conditions below hold. A peer that sends no warm signal is treated as cold and behaves as it does today. Relays export how many upstream copies they hold -and what each choice was for. +and what each choice was for. Whether a pull instead moves to a cheaper copy +under ranked readers, which also settles, waits on the maintainer (see +Ranked readers below). ## Plan @@ -22,10 +24,13 @@ Plan, is replaced. Decided 2026-11-16 (maintainer, merge pick of [moq-dev/moq#8903](https://github.com/moq-dev/moq/pull/8903), which -recommended it): **a settled pull stays**, unless -[the spike](/quest/m3/cache-routing/settle.md) the maintainer asked for with -that pick finds a rule that moves a pull for price and still settles; this -Plan is then rewritten for that rule. A relay chooses +recommended it): **a settled pull stays**, unless the spike the maintainer +asked for with that pick finds a rule that moves a pull for price and still +settles. It found one, +[moq-dev/moq#8986](https://github.com/moq-dev/moq/pull/8986): ranked +readers, below. Which of the two this ports waits on the maintainer; until +then the Plan below stays settled pulls, and the section on ranked readers +says what changes with it. A relay chooses where to pull when its demand starts, from its route and the warm copies it has heard of by then. After that it chooses again only on an outside event: its parent loses its upstream, its pull runs through itself, its session @@ -48,7 +53,7 @@ So no price ever moves a pull: a pull moves on an outside event (a reader starts or stops, a link or relay fails or returns, a publisher moves), on the route layer settling after one, or on what those set off. It is greedy online Steiner without rearrangement, apart from the moves a route change buys. -That it then settles is not yet proven; see below. +A TLA+ model shows it settles at small bounds; see below. **What the guarantee costs.** Imase and Waxman (Dynamic Steiner Tree Problem, SIAM J. Discrete Math. 4(3), 1991) bound greedy joins without @@ -62,21 +67,27 @@ together their constant-factor algorithm rearranges O(n^(3/2)) edges over n requests, which this never does for price. So the bound is a guide, and the sweeps measure the drift: -| | settled pulls, first cut | with the query, later | the price-moving pick | path joins | -|---|---|---|---|---| -| settled trees over the cheapest, 400 churned random / drone / region meshes | +3.4% / +16.2% / +5.4% | +3.4% / +16.2% / +5.0% | +2.7% / +9.4% / +4.9% | +2.7% / +28.9% / +5.0% | -| unsettled of 2,000 per kind | 0 / 0 / 0 | 0 / 0 / 0 | 1 / 0 / 0 (random case 503) | 0 / 0 / 0 | -| longest cycle under churn, deliveries | 20 / 18 / 14 | 20 / 18 / 19 | 18 / 39 / 24 | 16 / 14 / 17 | -| unfed reader-hops | 19,008 / 11,691 / 17,470 | 19,008 / 11,691 / 23,089 | 18,000 / 12,381 / 22,472 | 15,700 / 7,725 / 15,735 | -| warm messages, 400 churned regions | 4,655 | 21,020 | 21,151 | 0 | -| pops, summed over its five phases (cheapest 59) | 71 | 71 | 61 | 69 | -| drones in turn | 75, one uplink | 75, one uplink | 75, one uplink | 155, five | -| drones one / two hops apart | 41 / 33, at 69 / 62 ms | as the first cut | 33 / 25, at 81 / 69 ms | 41 / 41, at 69 / 62 ms | -| edges in turn, one region with unequal uplinks, rings, tiers | 13, 29, 2904, 1095 | 12, 24, 2904, 1095 | 12, 24, 2904, 1095 | 14, 29, 9624, 1191 | -| warm messages per reader change, unpeered CDN | 0 | 1.0 | 1.0 | 0 | +| | settled pulls, first cut | with the query, later | ranked readers | the price-moving pick | path joins | +|---|---|---|---|---|---| +| settled trees over the cheapest, 400 churned random / drone / region meshes | +3.4% / +16.2% / +5.4% | +3.4% / +16.2% / +5.0% | +1.2% / +4.9% / +4.4% | +2.7% / +9.4% / +4.9% | +2.7% / +28.9% / +5.0% | +| unsettled of 2,000 per kind | 0 / 0 / 0 | 0 / 0 / 0 | 0 / 0 / 0 | 1 / 0 / 0 (random case 503) | 0 / 0 / 0 | +| longest cycle under churn, deliveries | 20 / 18 / 14 | 20 / 18 / 19 | 19 / 23 / 18 | 18 / 39 / 24 | 16 / 14 / 17 | +| unfed reader-hops | 19,008 / 11,691 / 17,470 | 19,008 / 11,691 / 23,089 | 21,917 / 14,584 / 20,661 | 18,000 / 12,381 / 22,472 | 15,700 / 7,725 / 15,735 | +| warm messages, 400 churned regions | 4,655 | 21,020 | 13,133 | 21,151 | 0 | +| pops, summed over its five phases (cheapest 59) | 71 | 71 | 59 | 61 | 69 | +| drones in turn | 75, one uplink | 75, one uplink | 75, one uplink | 75, one uplink | 155, five | +| drones / ring / edges at once (cheapest 19 / 112 / 7) | 51 / 392 / 9 | as the first cut | 19 / 112 / 8 | 51 / 392 / 9 | 51 / 392 / 9 | +| drones lose the uplink (cheapest 94) | 118, four uplinks, a 17-step cycle | 118, four uplinks, a 17-step cycle | 94, one uplink | 118, four uplinks | 198 | +| drones one / two hops apart | 41 / 33, at 69 / 62 ms | as the first cut | 17 / 17, at 81 / 69 ms | 33 / 25, at 81 / 69 ms | 41 / 41, at 69 / 62 ms | +| edges in turn, one region with unequal uplinks, rings, tiers | 13, 29, 2904, 1095 | 12, 24, 2904, 1095 | 13, 29, 2904, 1095 | 12, 24, 2904, 1095 | 14, 29, 9624, 1191 | +| warm messages per reader change, unpeered CDN, one viewer / crowd | 0 / 0 | 1.0 / 1.0 | 0 / 0.67 | 1.0 / 1.0 | 0 / 0 | +| warm messages per reader change, 64 peered drones, a crowd | 63 | 63 | 161 | 70 | 0 | Random and drone meshes have no relay with two upstreams, so the query -changes nothing there. +changes nothing there. Ranked readers were measured on 2026-11-17 beside a +rerun of the other columns, which matched the table; at 2,000 per kind they +settle +1.4% / +6.3% / +4.6% over the cheapest, against the first cut's ++3.2% / +16.0% / +5.3%. The table is the rerun of 2026-11-16, after the route order gained the mark ([moq-dev/moq#8904](https://github.com/moq-dev/moq/pull/8904)) and a @@ -141,7 +152,86 @@ That PR's simulator, `rs/moq-net/src/route/tests/warm.rs`, runs the first cut as `advertise_not_down+hops+stay+reroute` and the later step as `advertise_not_down_query+hops+stay+reroute`, and its `warmth.rs` holds the shapes, the churn sweep, and the growth sweep this ports onto -production code. +production code. Ranked readers run as `advertise_not_down+hops+order_route`. + +### Ranked readers, waiting on the maintainer + +[moq-dev/moq#8986](https://github.com/moq-dev/moq/pull/8986) spiked a rule +under which a pull moves to a cheaper warm copy that turns up later, and +`rs/moq-net/verify/warm/` (the `verify-warm` skill) model-checks that it +settles. + +- **The rank.** A relay with a reader of its own for an Instance ranks it + by its route cost to the Instance, then its Node ID. A pull carries the + lowest rank among the readers it feeds, its relay's own and each + downstream pull's, and an update when that changes. An advert and a + subscribe answer carry the lowest rank their sender feeds besides their + receiver. +- **The choice.** A relay takes a warm copy only from a neighbour feeding a + reader ranked strictly below every reader the relay feeds itself. That + copy is fed whatever the relay does, so it costs the link's cold price + alone. The relay heads for the cheapest such copy or its route, make + before break, whenever that set or its own rank changes; nothing else + moves it, and no price above the first hop enters the choice. +- **Why it settles.** A relay's choice reads only copies feeding lower + ranks, and what those feed never depends on a higher rank's choice. So + once the outside events stop, the lowest rank's pulls settle, then the + next, by induction on the rank: no relay's demand comes back to it as a + price, the reflection every fence of the pick chased. TLC checks it from + every state the bounded outside events reach, delivering every word late + and in any order, on three drones with three reader changes, two reader + changes and an uplink flap, and churned random mesh 503's six relays. + With the rank check removed it fails on the three drones: two readers' + relays each take the other's copy, find their own path in it, fall back, + and repeat. The pick fails on case 503, as the sweep found. +- **What stays from the first cut.** Adverts, the mark, the path, the + answer's offer, and choosing again on a route change. Adverts still go + only where the copy could be taken, priced from the publisher (offer plus + the link's cold price, against the neighbour's route); pricing that test + at the link alone made a full core mesh's warm messages grow with its + cores (0.9 to 28.6 per reader change from 4 to 32 cores), where priced + from the publisher they stay flat: none for one viewer, 0.67 for a crowd. The path still breaks + a cycle formed on stale adverts: without it no mesh went unsettled, but + the longest cycle on drones grew from 29 to 99 deliveries. +- **What it buys.** The pop that turns warm later draws the pulls behind it + (pops 59, the cheapest, against 71). Drones, rings, and edges that start + at once merge onto one uplink (19 against 51 on drones, 112 against 392 on + the ring): the higher rank moves, the merge the maintainer's rule of + 2026-11-15 asked for, here without the cycles it added under the pick. A drone that loses the uplink + hands it to one other drone, not four, and without the first cut's + 17-step cycle. Churned trees settle at a third of the first cut's drift on + drones and random meshes. +- **What it costs.** On the wire, in the then-current wip version: a rank + on each SUBSCRIBE and an update message when it changes; a rank on each + advert and subscribe answer and its updates. A rank is a route cost and a + Node ID. Relay state: one rank per served subscription, one per heard + advert or answer, and the rank last told on each pull, all bounded as the + adverts are. Messages: 13,133 warm messages per 400 churned regions + against 4,655, rank updates 8,494 of them; 0.67 per crowd reader change on + an unpeered CDN against none; about 2.6 times the first cut's per reader + change between peers (161 against 63 for a crowd on 64 drones), since a + rank change reaches every peer that could take the copy. Readers: moves + stall readers in the simulator, which delivers one control message per + step across the mesh, so unfed reader-hops rise by 15% to 25%, and a + drone one hop from another waits 81 ms for a first frame against 69. +- **A peer that sends no rank** is ranked below every ranked reader: a + relay it pulls from never takes a warm copy for its sake and pulls along + its route, as today. Its adverts are never taken, since nothing says the + copy is fed regardless. +- **Tried and dropped on the way.** Pricing a copy at the parent's summed + offer plus the link, as the first cut does, and keeping the current copy + while no dearer: it reads demand above the first hop, so relays kept + dearer copies (drones +18.4% against +4.9%). Ranking by Node ID alone: it + settles too (`Hash.cfg`, and 0 of 6,000 churned meshes unsettled under an + earlier advert audience), but the near pop never hears a middle pop + ranked above it, so pops keeps the path join's tree. + +Recommended: port ranked readers rather than settled pulls. Both settle in +every sweep and pass the model; ranked readers move toward the cheapest +tree where warmth matters, between peers and at simultaneous starts, and +match the first cut's trees on an unpeered CDN, for one more field on three +messages and one update message. Settled pulls stay the choice if that +wire or the extra moves are not worth a third of the drift. ### Decision: the query, a later step @@ -265,11 +355,11 @@ fresh choice after a lost parent or a failed pending pull can pick a warm copy on a stale advert, give that relay demand and so a join of its own, or end another relay's pull, so "once per pull" does not bound the number of pulls after the last outside event (an automated review on #8903). The sweep -agrees with settling, though it is no proof: 6,000 churned meshes, 2,000 per -kind, all settled. [The spike](/quest/m3/cache-routing/settle.md)'s model closes it, and if the model -cannot rule out repeated warm retries within one epoch, a relay whose warm -choice is lost or fails closed takes its route until its route or its own -demand next changes; no timer. +agrees with settling: 6,000 churned meshes, 2,000 per kind, all settled. +The `verify-warm` model checks it at small bounds: settled pulls go quiet +and feed every routed reader on churned random mesh 503 and on three drones +with three reader changes (`Settled.cfg`, `SettledDrones.cfg`). It found no +warm retries that repeat, so the rule takes no fallback. ### Simultaneous starts @@ -279,7 +369,8 @@ since a merge is a move for price. The maintainer's rule, decided 2026-11-15, was to merge them on the rendezvous hash unless that adds a cycle, an unfed reader, or more messages than the second uplink costs; under the pick's adverts it added cycles (18 deliveries through merged pulls in 400 churned -drone meshes) and about 1% unfed reader-hops. So the race stands. A drone +drone meshes) and about 1% unfed reader-hops. So the race stands under settled +pulls; ranked readers merge them (above). A drone that loses the one uplink leaves every drone it fed choosing again at once: 4 uplinks on the five-drone shape. @@ -296,12 +387,10 @@ that loses the one uplink leaves every drone it fed choosing again at once: itself, so a rule that never offered a copy passed. Port the warmth shapes and the churn sweep onto production relays: pops, drones in turn and at once, the lost uplink, the ring, the tiers, and random churned mesh 503. -- Port the rule [the spike](/quest/m3/cache-routing/settle.md) model-checked - with TLA+, and keep that model in step with the port. If its model found - warm retries that repeat within one route and demand epoch, the rule it - hands over already bounds them: a relay whose warm choice is lost or fails - closed takes its route until its route or its own demand next changes. - The query is enabled only once the model holds with it. +- Port the rule the `verify-warm` model checks (`Rule = "stay"` for settled + pulls, `"order"` for ranked readers), and move its Coverage table from the + simulator onto the port, keeping the model in step with it. The query is + enabled only once the model holds with it. - Bound the warm state a peer can make a relay hold: one advert per neighbour per Instance the relay has a route for, a path no longer than the route layer's hop limit, and adverts for anything else dropped. Adverts are @@ -360,12 +449,18 @@ the announce stream carrying an Instance's offer, path, and mark, and its withdrawal, in order with the route messages there; the same offer, path, and mark on the subscribe answer and its updates; and, with the later step, a query naming an Instance, answered once with an offer, path, and mark or -none. Nothing lite-07 freezes changes. +none. With ranked readers, also a rank on the SUBSCRIBE, the advert, and +the answer, and an update message for a subscription's rank. Nothing +lite-07 freezes changes. ## Required - [The switch](/quest/m2/multicast/switch.md) - the route layer it ranks on -- [The spike](/quest/m3/cache-routing/settle.md) - the rule to port, model-checked to settle + +## Waiting + +- Maintainer: port ranked readers or settled pulls (see Ranked readers, + waiting on the maintainer). Recommended: ranked readers. ## Related diff --git a/quest/m3/cache-routing/settle.md b/quest/m3/cache-routing/settle.md deleted file mode 100644 index 181487fc1c2..00000000000 --- a/quest/m3/cache-routing/settle.md +++ /dev/null @@ -1,119 +0,0 @@ -# [L] Warm pulls move for price and still settle - -## Goal - -The warmth simulator has a rule under which a pull moves to a cheaper warm -copy that turns up later, and a model check shows the rule settles: once -readers, links, and publishers stop changing, every pull stops moving and -every routed reader is fed, under every order of delivery at a small bound. -[The selection quest](/quest/m3/cache-routing/select.md) then names that rule -beside settled pulls, with what each costs and what it would add to the wire. - -If no candidate holds up, the PR says how each one failed, and the model -check runs on settled pulls instead, with its fallback made part of the -rule if the check needs it. - -This is a spike: no production code and no wire change. - -## Plan - -Decided 2026-11-16 (maintainer, in the coordinating session, after asking -what warmth would cost before the cut): spike warmth now. The coordinator -read that as this quest, run beside [the switch](/quest/m2/multicast/switch.md). -Whether any warm wire rides lite-07 is the maintainer's to decide on its -result, so report the wire it would need and do not add it. - -Why a spike: [moq-dev/moq#8903](https://github.com/moq-dev/moq/pull/8903) -measured two rules and neither is the one to ship before the cut. - -- Settled pulls settle in every case swept, but a pull never moves to a pop - that turns warm later. On an unpeered CDN the first cut sends no warm - message, so it is path joins with extra parts; on the pops shape it costs - 71 against path joins' 69. Its settling is argued, not shown. -- The price-moving pick does move (61 on pops, drones +9.4% over the - cheapest against +16.2%), and never settles churned random mesh 503: a - relay's own demand comes back to it as a cheaper price two hops away. - -Every fence tried so far kept one shape of that reflection out. Look for a -rule where a relay's choice cannot depend on its own demand by construction. - -**First candidate: order the readers.** The coordinator's, untested; attack -it before building on it. - -- A relay with a reader of its own for an Instance has a key for it: its - route cost to the Instance, then its Node ID. Also measure a hash of Node - ID and Instance, which does not change with routes. -- A pull carries the lowest key among the readers it feeds: the relay's own, - and each downstream pull's. Adverts and subscribe answers carry it. -- A relay takes a warm copy only from a neighbour whose key is below its - own. Such a copy is fed whatever this relay does, so it costs the link - alone. The relay takes the cheapest of those copies and its route, and - chooses again whenever that set or its own key changes. -- The claim to test: the tree is then the one greedy joins build when - readers arrive in key order, whatever order they really arrived in. A - reader's path depends only on readers with lower keys, so it settles by - induction on the key, and each step is a search for the nearest of a - fixed set, the shape [the route proof](/rs/moq-net/verify/route-algebra/README.md) - covers. Imase and Waxman's O(log k) bound holds for any arrival order, and - here it would hold after departures too, since the tree is rebuilt. -- What it should deliver: the pop that turns warm later draws the pulls - behind it; two drones that start at once merge, the higher key moving (the - maintainer suggested a hash tie-break where one drone reneges); case 503 - settles. -- Where it may break: a relay that only forwards takes the key of the pull - through it, and its key drops when a lower one arrives, so it must choose - again and may move the higher keyed pulls it already feeds. A price that - still reads demand above the first hop, as `O(P→R) + warm(P, R)` does, - would bring the reflection back. Stale adverts and keys in flight. -- What it costs, to measure: moves and unfed reader-hops per reader change, - an advert update when a key changes without the demand starting or - ending, and one key per pull and per heard advert. - -**Other candidates**, if the first fails or to beat it: - -- Price each copy at what the pull adds to the tree. The tree's total cost - is then a potential: a move that lowers a relay's price lowers the tree's - cost by as much, provided the price is exact and no two moves overlap. The - pick failed both. Exact needs the asker's demand left out at every hop, - which naming sole demand tried and still left 1 of 2,000 unsettled; - moves that do not overlap need a lock or an order. -- Count only relays with a reader of their own as warm, which no relay's - choice changes, and order by route cost so two such relays never adopt - each other. It loses coalescing at relays that only forward, which is - every core of a CDN. - -Work in `rs/moq-net/src/route/tests/warm.rs` and `warmth.rs`, as new arms -beside `advertise_not_down+hops+stay+reroute` and the pick, so every table -in [select](/quest/m3/cache-routing/select.md) gains a column. - -Model the chosen rule in TLA+ under `rs/moq-net/verify/`, as the other -models there are laid out, with its nightly lane and skill: - -- A few relays, readers starting and stopping, links failing and returning, - adverts and keys delivered in any order and stale. -- Settling is checked from when the outside events stop, not from when the - queues drain: a retry loop keeps the queues full. -- Both properties: the pulls go quiet, and every reader the route layer - routes is then fed, two relays waiting on each other included. -- Show the check has teeth: the price-moving pick, or the rule with its - order removed, must fail it. - -This takes over the model check [select](/quest/m3/cache-routing/select.md) -had asked of the port; select now ports the rule this model checks. - -Report in the PR: - -- Each candidate tried and how it failed or held, with the case. -- The tables of [moq-dev/moq#8903](https://github.com/moq-dev/moq/pull/8903) - with the new column: trees over the cheapest, unsettled, longest cycle, - unfed reader-hops, warm messages, pops, drones, tiers, growth per link - kind. -- The wire and relay state the rule would add over the first cut, and what - a peer that does not send it sees. -- A recommendation between it and settled pulls for - [select](/quest/m3/cache-routing/select.md), as a decision under - `## Waiting` for the maintainer. - -## Related - -- [Selection](/quest/m3/cache-routing/select.md) - ports the rule this picks diff --git a/rs/moq-net/src/route/tests/warm.rs b/rs/moq-net/src/route/tests/warm.rs index b5a40c908e3..e7f56f6fab2 100644 --- a/rs/moq-net/src/route/tests/warm.rs +++ b/rs/moq-net/src/route/tests/warm.rs @@ -1385,7 +1385,10 @@ impl WarmSim { /// Whether `audience` lets `node` advertise `signal` to `peer`. fn scoped(&self, audience: Audience, node: Node, cast: usize, peer: Node, signal: &Signal) -> bool { - let price = self.copy(signal, node, peer); + // Priced from the publisher, even where the taker prices it at the link alone: a + // copy at the link alone beats nearly any route, so a warm core would tell every + // core it meshes with. A relay already pulling keeps its parent by its answer. + let price = signal.cost + self.price(node, peer).cold; let heard = self .net .router(node) diff --git a/rs/moq-net/src/route/tests/warmth.rs b/rs/moq-net/src/route/tests/warmth.rs index 3dddfd74347..f2a9c7bb237 100644 --- a/rs/moq-net/src/route/tests/warmth.rs +++ b/rs/moq-net/src/route/tests/warmth.rs @@ -424,13 +424,12 @@ fn a_join_skips_pops_without_demand_and_warm_ones_draw_it() { // A settled pull stays: the middle pop joins the near pop's copy, but the far // pop, settled on its long link, never moves for the warm pops beside it. let stayed = BTreeSet::from([(4, 0), (0, 1), (1, 2), (0, 3)]); - // Ranked by hash, the middle pop ranks below the near one, so the near pop takes - // the middle pop's copy instead, at the same cost. - let hashed = BTreeSet::from([(4, 0), (0, 2), (2, 1), (2, 3)]); match arm.mode { + // Ranked by hash, the middle pop ranks below the near one, whose route beats the + // middle pop's copy priced from the publisher, so nothing is advertised to it. Mode::Path => assert_eq!(edges, skip, "{case}"), + _ if arm.order == Some(Order::Hash) => assert_eq!(edges, skip, "{case}"), _ if arm.stay => assert_eq!(edges, stayed, "{case}"), - _ if arm.order == Some(Order::Hash) => assert_eq!(edges, hashed, "{case}"), // One query goes to the cheapest link, which for the middle pop is the near // pop's and for the far pop the middle's. _ => assert_eq!(edges, joined, "{case}"), @@ -549,6 +548,13 @@ const ORDER_HASH: Arm = Arm { ..NOT_DOWN }; +/// [`ORDER_ROUTE`] without the path in its offers: only a rank keeps a relay off a +/// copy that runs through it. +const ORDER_PATHLESS: Arm = Arm { + hops: false, + ..ORDER_ROUTE +}; + /// [`ORDER_ROUTE`], pricing a copy from the publisher and keeping a live pull unless /// another source is strictly cheaper. const ORDER_SUMMED: Arm = Arm { @@ -576,6 +582,7 @@ fn narrower() -> Vec { ORDER_ROUTE, ORDER_HASH, ORDER_SUMMED, + ORDER_PATHLESS, ] } @@ -744,20 +751,36 @@ fn grow(shape: &Shape, readers: &[(Node, usize)], in_turn: bool, arm: Arm, cheap } } -/// A first viewer's adverts on a CDN: told to every neighbour they grow with the -/// core mesh, and told only where they could be taken they do not, since a core -/// with demand is never cheaper for another core than its own route. +/// A first viewer's adverts on a CDN, and a crowd's under ranked readers: told to +/// every neighbour they grow with the core mesh, and told only where they could be +/// taken they do not, since a core with demand is never cheaper for another core +/// than its own route. #[test] fn useful_adverts_do_not_grow_with_the_core_mesh() { - let warm = |arm: Arm, regions: usize| { + let warm = |arm: Arm, regions: usize, crowd: bool| { let shape = cdn(regions, 3, false, 1); - // The first edge of region 1, which the publisher's core is not in. - let grown = grow(&shape, &[(2 * regions + 3, 0)], false, arm, false); + // The first edge of region 1, which the publisher's core is not in, then + // the first edge of region 2. + let readers = [(2 * regions + 3, 0), (2 * regions + 6, 0)]; + let readers = match crowd { + true => &readers[..], + false => &readers[..1], + }; + let grown = grow(&shape, readers, false, arm, false); assert_eq!(grown.right, Ok(()), "{}", arm.name()); grown.sent.warm() }; - assert_eq!(warm(USEFUL, 4), warm(USEFUL, 8)); - assert!(warm(ADVERTISE, 4) < warm(ADVERTISE, 8)); + assert_eq!(warm(USEFUL, 4, false), warm(USEFUL, 8, false)); + assert!(warm(ADVERTISE, 4, false) < warm(ADVERTISE, 8, false)); + // Ranked readers price the audience from the publisher too; at the link alone + // a warm core told every other core. + for crowd in [false, true] { + assert_eq!( + warm(ORDER_ROUTE, 8, crowd), + warm(ORDER_ROUTE, 32, crowd), + "crowd {crowd}" + ); + } } /// Under the pick a warm core tells none of its region's edges, and each edge queries @@ -1133,6 +1156,7 @@ fn every_arm_settles_after_churn() { ORDER_ROUTE, ORDER_HASH, ORDER_SUMMED, + ORDER_PATHLESS, ] { for shape in [random as fn(&mut XorShift) -> Shape, flying, regional] { let out = churn(arm, 0..12, shape); @@ -1405,7 +1429,6 @@ fn benchmark_warmth_churn() { } } - // --- Ranked readers --- /// Ranked by route cost, a pull follows a pop that turns warm later, and leaves it @@ -1418,7 +1441,10 @@ fn a_ranked_pull_follows_a_pop_that_turns_warm_later() { for row in &rows { assert_eq!(row.cost(), row.cheapest.iter().sum::(), "{}", row.phase); } - let stayed: Metric = run(&pops(), REROUTE_UNQUERIED, &pop_phases(), 1).iter().map(Row::cost).sum(); + let stayed: Metric = run(&pops(), REROUTE_UNQUERIED, &pop_phases(), 1) + .iter() + .map(Row::cost) + .sum(); assert!(rows.iter().map(Row::cost).sum::() < stayed); } diff --git a/rs/moq-net/verify/README.md b/rs/moq-net/verify/README.md index bd016626902..5f05da964dd 100644 --- a/rs/moq-net/verify/README.md +++ b/rs/moq-net/verify/README.md @@ -17,4 +17,4 @@ After a change to a covered function, mirror it in the model, rerun every config - Mechanized: the rule converges on every network, from every state, under loss, reordering, and duplication, to one fixed point, with ties broken by one fixed order, provided every node keeps acting and keeps hearing newer messages from each neighbour while the network holds still; the same for claims across trust boundaries; and the floor and the named drains leave that fixed point unchanged. - Tested against the proof: the prototype's rank and extend functions meet its axioms, and 322 of their results are checked by Agda. - On paper and tested, not mechanized: the rule as it runs puts the next hop in use ahead of the shorter path and the neighbour, so its stable state is not unique. It still settles, and every node settles on the rank and mark the fixed order gives it, except where a longer path meets the 32-hop limit; only the next hop varies. -- Not given: loop freedom at every instant, warmth, liveness under endless churn, the chain as Rust records it, and which relays announce under `claimed`. +- Not given: loop freedom at every instant, warmth (`verify-warm` model-checks that warm pulls settle), liveness under endless churn, the chain as Rust records it, and which relays announce under `claimed`. diff --git a/rs/moq-net/verify/warm/Drones.cfg b/rs/moq-net/verify/warm/Drones.cfg index 3ad92e2be2d..39b40399916 100644 --- a/rs/moq-net/verify/warm/Drones.cfg +++ b/rs/moq-net/verify/warm/Drones.cfg @@ -1,9 +1,9 @@ -\* Three drones meshed over cheap links, each with a costly uplink to the -\* publisher's relay: readers start and stop at each drone three times in all, -\* and one uplink may close and reopen. +\* Ranked readers on three drones meshed over cheap links, each with a costly +\* uplink to the publisher's relay: readers start and stop at the drones three +\* times in all. CONSTANTS Shape = "drones" - Flaky = {2} + Flaky = {} Readable = {2, 3, 4} Readers = {} MaxReads = 3 diff --git a/rs/moq-net/verify/warm/Flap.cfg b/rs/moq-net/verify/warm/Flap.cfg new file mode 100644 index 00000000000..cb0b58629b8 --- /dev/null +++ b/rs/moq-net/verify/warm/Flap.cfg @@ -0,0 +1,23 @@ +\* Ranked readers on the drones of Drones.cfg: two reader changes, and the +\* uplink of drone 2 closing and reopening once. +CONSTANTS + Shape = "drones" + Flaky = {2} + Readable = {2, 3, 4} + Readers = {} + MaxReads = 2 + MaxCloses = 1 + Rule = "order" + Order = "route" + Summed = FALSE + +SPECIFICATION Spec + +CHECK_DEADLOCK FALSE + +INVARIANTS + TypeOK + +PROPERTIES + PullsQuiet + EventuallyFed diff --git a/rs/moq-net/verify/warm/Hash.cfg b/rs/moq-net/verify/warm/Hash.cfg new file mode 100644 index 00000000000..d012af86a67 --- /dev/null +++ b/rs/moq-net/verify/warm/Hash.cfg @@ -0,0 +1,23 @@ +\* Ranked readers as in Drones.cfg, ranked by relay alone rather than by route +\* cost first. +CONSTANTS + Shape = "drones" + Flaky = {} + Readable = {2, 3, 4} + Readers = {} + MaxReads = 3 + MaxCloses = 0 + Rule = "order" + Order = "id" + Summed = FALSE + +SPECIFICATION Spec + +CHECK_DEADLOCK FALSE + +INVARIANTS + TypeOK + +PROPERTIES + PullsQuiet + EventuallyFed diff --git a/rs/moq-net/verify/warm/Mesh503.cfg b/rs/moq-net/verify/warm/Mesh503.cfg new file mode 100644 index 00000000000..c275158ac6a --- /dev/null +++ b/rs/moq-net/verify/warm/Mesh503.cfg @@ -0,0 +1,24 @@ +\* Ranked readers on the six relays of random churn case 503, which never +\* settled under the price-moving pick: readers fixed at five of them, from no +\* pulls at all. +CONSTANTS + Shape = "mesh503" + Flaky = {} + Readable = {} + Readers = {1, 3, 4, 5, 6} + MaxReads = 0 + MaxCloses = 0 + Rule = "order" + Order = "route" + Summed = FALSE + +SPECIFICATION Spec + +CHECK_DEADLOCK FALSE + +INVARIANTS + TypeOK + +PROPERTIES + PullsQuiet + EventuallyFed diff --git a/rs/moq-net/verify/warm/Orderless.cfg b/rs/moq-net/verify/warm/Orderless.cfg index c2c9441e5a8..0f5879da4d0 100644 --- a/rs/moq-net/verify/warm/Orderless.cfg +++ b/rs/moq-net/verify/warm/Orderless.cfg @@ -1,3 +1,8 @@ +\* Open finding: ranked readers with the rank check removed never settle on +\* the drones of Drones.cfg. With readers at drones 2 and 3, each takes the +\* other's warm copy over its uplink, sees its own path in what it now pulls, +\* falls back to its uplink, and both repeat for good. It shows the rank is what +\* settles the ranked rule. CONSTANTS Shape = "drones" Flaky = {} diff --git a/rs/moq-net/verify/warm/Pick.cfg b/rs/moq-net/verify/warm/Pick.cfg index 6a102030ba3..4b7feb7eecd 100644 --- a/rs/moq-net/verify/warm/Pick.cfg +++ b/rs/moq-net/verify/warm/Pick.cfg @@ -1,9 +1,12 @@ +\* Open finding: the price-moving pick moq-dev/moq#8903 rejected never settles +\* case 503: one relay moves its pull between two neighbours for good. It shows +\* PullsQuiet has teeth; the rule is not to be fixed, so this stays a finding. CONSTANTS Shape = "mesh503" Flaky = {} - Readable = {2, 3, 4} - Readers = {2, 4} - MaxReads = 2 + Readable = {} + Readers = {1, 3, 4, 5, 6} + MaxReads = 0 MaxCloses = 0 Rule = "pick" Order = "route" diff --git a/rs/moq-net/verify/warm/Settled.cfg b/rs/moq-net/verify/warm/Settled.cfg new file mode 100644 index 00000000000..0343144f44b --- /dev/null +++ b/rs/moq-net/verify/warm/Settled.cfg @@ -0,0 +1,22 @@ +\* Settled pulls, the first cut select.md names, on case 503 as in Mesh503.cfg. +CONSTANTS + Shape = "mesh503" + Flaky = {} + Readable = {} + Readers = {1, 3, 4, 5, 6} + MaxReads = 0 + MaxCloses = 0 + Rule = "stay" + Order = "route" + Summed = FALSE + +SPECIFICATION Spec + +CHECK_DEADLOCK FALSE + +INVARIANTS + TypeOK + +PROPERTIES + PullsQuiet + EventuallyFed diff --git a/rs/moq-net/verify/warm/PickDrones.cfg b/rs/moq-net/verify/warm/SettledDrones.cfg similarity index 80% rename from rs/moq-net/verify/warm/PickDrones.cfg rename to rs/moq-net/verify/warm/SettledDrones.cfg index 031a615c97d..bfe2940648c 100644 --- a/rs/moq-net/verify/warm/PickDrones.cfg +++ b/rs/moq-net/verify/warm/SettledDrones.cfg @@ -1,3 +1,4 @@ +\* Settled pulls on the drones of Drones.cfg. CONSTANTS Shape = "drones" Flaky = {} @@ -5,7 +6,7 @@ CONSTANTS Readers = {} MaxReads = 3 MaxCloses = 0 - Rule = "pick" + Rule = "stay" Order = "route" Summed = FALSE diff --git a/rs/moq-net/verify/warm/WarmSettle.tla b/rs/moq-net/verify/warm/WarmSettle.tla index 91fd1134243..ccc60352ed4 100644 --- a/rs/moq-net/verify/warm/WarmSettle.tla +++ b/rs/moq-net/verify/warm/WarmSettle.tla @@ -38,8 +38,9 @@ (* links at once (`verify-route` and the route proof cover how routes *) (* settle). The three words a session carries one way arrive together, *) (* though the subscription, its answer, and the advert ride separate *) -(* streams. No link is marked upstream, no advert is withheld as useless, *) -(* and there is no query. *) +(* streams. No link is marked upstream, so the mark is left out, and there *) +(* is no query. A relay advertises to a neighbour only where the copy, *) +(* priced from the publisher, beats the neighbour's route (`Useful`). *) (***************************************************************************) EXTENDS Integers, Sequences, FiniteSets @@ -65,9 +66,11 @@ Links == \* Pops on a line from the publisher's, with long links passing over them. {<<1, 2, 6, 1>>, <<2, 3, 4, 1>>, <<3, 4, 4, 1>>, <<1, 3, 6, 6>>, <<1, 4, 8, 8>>} [] Shape = "mesh503" -> - \* Churned random mesh 503's links among its relays 4, 3, 6, and the - \* publisher's 5, as relays 2, 3, 4, and 1. - {<<2, 3, 1, 1>>, <<3, 4, 2, 1>>, <<1, 2, 6, 1>>, <<2, 4, 6, 3>>, <<1, 4, 2, 1>>} + \* Churned random mesh 503 where the pick never settled its second + \* broadcast: its relays 5, 4, 3, 6, 2, and 0 as 1 to 6, over the links + \* open at the end. + {<<1, 2, 6, 1>>, <<2, 3, 1, 1>>, <<2, 4, 6, 3>>, <<3, 4, 2, 1>>, + <<3, 5, 2, 2>>, <<2, 5, 5, 3>>, <<4, 6, 3, 3>>} N == Cardinality(UNION {{l[1], l[2]} : l \in Links}) Home == 1 @@ -89,8 +92,6 @@ Neighbours == [r \in Relays |-> {n \in Relays : Pair(r, n) \in AllPairs}] Cold(a, b) == Prices[Pair(a, b)][1] Warm(a, b) == Prices[Pair(a, b)][2] Nb(r) == Neighbours[r] -\* Whether a rule reads what a copy costs from the publisher: the link alone needs none. -Costed == ~(Ordered /\ ~Summed) InPath(r, path) == \E i \in 1 .. Len(path) : path[i] = r @@ -106,11 +107,12 @@ VARIABLES up, \* up[r]: the neighbour r pulls through, or None nxt, \* nxt[r]: a move under way: it replaces up[r] once answered view, \* view[r][s]: what r last heard over its session to s + ok, \* ok[r]: those of r's pulls answered live once, whose silence is a loss chosen, \* chosen[r]: with "stay", the route r last chose by reads, \* reader starts and stops so far closes \* link closes and reopens so far -vars == <> +vars == <> (***************************************************************************) (* The route layer, as an oracle over the open links. *) @@ -128,7 +130,8 @@ Dist(k, o) == Routes(o) == LET d == Dist(N, o) IN [r \in Relays |-> - IF r = Home \/ d[r] >= Inf THEN <> + IF r = Home THEN <> + ELSE IF d[r] >= Inf THEN <> ELSE <>] Hop(r) == route[r][1] @@ -153,11 +156,13 @@ RankOf(r, c) == ELSE Min({Own(r)} \cup {view[r][d].pull : d \in Down(r) \ {c}}) Ans(r, s) == view[r][s].ans -\* How r's pull through s stands: "live", "self" (its path names r), "lost", or "pending". +\* How r's pull through s stands: "live", "self" (its path names r), "lost", or +\* "pending". A parent answers once it is fed or stuck, and every change after, so +\* one that answered live and then has nothing to offer has lost its upstream. State(r, s) == CASE Ans(r, s).kind = "live" /\ ~InPath(r, Ans(r, s).sig.path) -> "live" [] Ans(r, s).kind = "live" -> "self" - [] Ans(r, s).kind = "lost" -> "lost" + [] Ans(r, s).kind = "lost" \/ s \in ok[r] -> "lost" [] OTHER -> "pending" Live(r) == r = Home \/ (up[r] # None /\ State(r, up[r]) = "live") @@ -168,7 +173,7 @@ Offer(r, c) == LET link == IF r = Home THEN 0 ELSE IF Ordered \/ Others(r, c) THEN Warm(up[r], r) ELSE Cold(up[r], r) - IN [cost |-> IF Costed THEN Sig(r).cost + link ELSE 0, + IN [cost |-> Sig(r).cost + link, path |-> Append(Sig(r).path, r), rank |-> IF Ordered THEN RankOf(r, c) ELSE NoRank] @@ -180,8 +185,14 @@ Answer(r, c) == ELSE IF Stuck(r) THEN [kind |-> "lost", sig |-> NoSig] ELSE [kind |-> "wait", sig |-> NoSig] +\* Whether c could take r's copy, priced from the publisher whatever c prices it at: +\* below c's route, or, when c's route runs through r, below r's own route. +Useful(r, c) == + IF Hop(c) = r THEN Offer(r, c).cost < route[r][2] + ELSE Offer(r, c).cost + Cold(r, c) <= route[c][2] + Advert(r, c) == - IF Live(r) /\ Others(r, c) THEN [on |-> TRUE, sig |-> Offer(r, c)] + IF Live(r) /\ Others(r, c) /\ Useful(r, c) THEN [on |-> TRUE, sig |-> Offer(r, c)] ELSE Empty.adv \* What r tells c now: the rank its pull through c carries, its answer, its advert. @@ -253,7 +264,7 @@ Decide(r, u, x) == LET cur == {o \in opts : o.via = u} IN IF best = None THEN <> ELSE IF best # u /\ (cur = {} \/ \A o \in cur : Best(opts).cost < o.cost) - THEN <> ELSE <> + THEN <> ELSE <> [] st = "live" -> LET moving == IF x # None /\ \E o \in opts : o.via = x /\ o.cost <= own THEN x ELSE None @@ -299,6 +310,7 @@ Init == /\ up = [r \in Relays |-> None] /\ nxt = [r \in Relays |-> None] /\ view = [r \in Relays |-> [s \in Relays |-> Empty]] + /\ ok = [r \in Relays |-> {}] /\ chosen = [r \in Relays |-> <>] /\ reads = 0 /\ closes = 0 @@ -312,6 +324,8 @@ React(r) == /\ up' = [up EXCEPT ![r] = pulls[1]] /\ nxt' = [nxt EXCEPT ![r] = pulls[2]] /\ chosen' = [chosen EXCEPT ![r] = c] + \* A pull replaced outright by one through the same neighbour starts afresh. + /\ ok' = [ok EXCEPT ![r] = {s \in @ : s \in {pulls[1], pulls[2]} /\ State(r, s) # "lost"}] /\ UNCHANGED <> \* r hears what s tells it now. @@ -320,6 +334,8 @@ Deliver(s, r) == /\ IsOpen(s, r) /\ view[r][s] # Pub(s, r) /\ view' = [view EXCEPT ![r][s] = Pub(s, r)] + /\ ok' = [ok EXCEPT ![r] = + IF s \in {up[r], nxt[r]} /\ Pub(s, r).ans.kind = "live" THEN @ \cup {s} ELSE @] /\ UNCHANGED <> Read(r) == @@ -327,7 +343,7 @@ Read(r) == /\ reads < MaxReads /\ reads' = reads + 1 /\ reader' = [reader EXCEPT ![r] = ~@] - /\ UNCHANGED <> + /\ UNCHANGED <> \* The session over a link closes, and both ends forget what it carried. Close(a, b) == @@ -346,6 +362,7 @@ Close(a, b) == IF r \in {a, b} /\ (up[r] = other(r) \/ nxt[r] = other(r)) THEN None ELSE nxt[r]] /\ view' = [view EXCEPT ![a][b] = Empty, ![b][a] = Empty] + /\ ok' = [ok EXCEPT ![a] = @ \ {b}, ![b] = @ \ {a}] /\ UNCHANGED <> Reopen(a, b) == @@ -355,7 +372,7 @@ Reopen(a, b) == /\ closes' = closes + 1 /\ open' = open \cup {Pair(a, b)} /\ route' = Routes(open') - /\ UNCHANGED <> + /\ UNCHANGED <> Outside == \/ \E r \in Relays : Read(r) @@ -381,6 +398,7 @@ TypeOK == /\ nxt \in [Relays -> Relays \cup {None}] /\ \A r \in Relays : up[r] # None => up[r] \in Nb(r) /\ \A r \in Relays : nxt[r] # None => nxt[r] \in Nb(r) /\ nxt[r] # up[r] + /\ \A r \in Relays : ok[r] \subseteq {up[r], nxt[r]} \ {None} \* r's pulls lead to the publisher, each held by its parent and answered live. RECURSIVE FedFrom(_, _) From e48f18a47264afc50fb4dfd65374d59d43a20cda Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 03:51:42 -0800 Subject: [PATCH 4/7] test(moq-net): a flap that finishes, and only the main ranked arm in growth Flap holds two readers fixed while drone 2's uplink closes and reopens: 44,427 states, where a reader change during the flap passed 2M states without finishing. The growth sweep keeps ranked readers' variants out, since they change trees rather than how adverts grow. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-warm/SKILL.md | 4 ++-- quest/m3/cache-routing/select.md | 5 +++-- rs/moq-net/src/route/tests/warmth.rs | 8 +++++++- rs/moq-net/verify/warm/Flap.cfg | 13 +++++++------ test/route-sweep.sh | 2 +- 5 files changed, 20 insertions(+), 12 deletions(-) diff --git a/.claude/skills/verify-warm/SKILL.md b/.claude/skills/verify-warm/SKILL.md index 332c5f1301a..1c601b6a62c 100644 --- a/.claude/skills/verify-warm/SKILL.md +++ b/.claude/skills/verify-warm/SKILL.md @@ -32,7 +32,7 @@ quest gate just rs tla moq-net/verify/warm/WarmSettle.tla -config rs/moq-net/ver | --- | --- | --- | --- | | `Drones.cfg` | ranked by route cost | three drones meshed with costly uplinks, readers starting and stopping three times | 164,434, 5 minutes | | `Hash.cfg` | ranked by relay | the same | 164,434, 5 minutes | -| `Flap.cfg` | ranked by route cost | the drones with two reader changes and one uplink closing and reopening | FLAP_STATES | +| `Flap.cfg` | ranked by route cost | readers fixed at two drones while one's uplink closes and reopens, changing its rank | 44,427, under a minute | | `Mesh503.cfg` | ranked by route cost | churned random mesh 503's six relays and readers, from no pulls | 490,125, 25 minutes | | `Settled.cfg` | settled pulls | the same | 407,526, 6 minutes | | `SettledDrones.cfg` | settled pulls | as `Drones.cfg` | 6,852, seconds | @@ -60,4 +60,4 @@ Sync commit: the merge of moq-dev/moq#8986. Run `git log -- rs/moq-net/src/route ## Not covered -One broadcast and one publisher, which never moves. The route layer is an oracle: each relay's route is the shortest over the open links at once, the lowest relay among equal ones; `verify-route` and the route proof cover how routes settle. The three words one session carries one way arrive together, though the subscription, its answer, and the advert ride separate streams, and a register keeps only its newest value, so a relay may skip values but never reads one out of order on one session. No link is marked upstream, so the mark is left out, and there is no query. A pull replaced by one through the same neighbour is the same pull. Relays never restart. A pass covers four relays with three reader changes, or two and one uplink flap, and case 503's six relays with their readers fixed. +One broadcast and one publisher, which never moves. The route layer is an oracle: each relay's route is the shortest over the open links at once, the lowest relay among equal ones; `verify-route` and the route proof cover how routes settle. The three words one session carries one way arrive together, though the subscription, its answer, and the advert ride separate streams, and a register keeps only its newest value, so a relay may skip values but never reads one out of order on one session. No link is marked upstream, so the mark is left out, and there is no query. A pull replaced by one through the same neighbour is the same pull. Relays never restart. A pass covers four relays with three reader changes, or two fixed readers and one uplink closing and reopening, and case 503's six relays with their readers fixed. A reader change during the flap passed 2M states without finishing, so it is left out. diff --git a/quest/m3/cache-routing/select.md b/quest/m3/cache-routing/select.md index 9b29717eb89..cb431ee394c 100644 --- a/quest/m3/cache-routing/select.md +++ b/quest/m3/cache-routing/select.md @@ -179,8 +179,9 @@ settles. next, by induction on the rank: no relay's demand comes back to it as a price, the reflection every fence of the pick chased. TLC checks it from every state the bounded outside events reach, delivering every word late - and in any order, on three drones with three reader changes, two reader - changes and an uplink flap, and churned random mesh 503's six relays. + and in any order, on three drones with three reader changes, two fixed + readers while an uplink closes and reopens, and churned random mesh 503's + six relays. With the rank check removed it fails on the three drones: two readers' relays each take the other's copy, find their own path in it, fall back, and repeat. The pick fails on case 503, as the sweep found. diff --git a/rs/moq-net/src/route/tests/warmth.rs b/rs/moq-net/src/route/tests/warmth.rs index f2a9c7bb237..b5cf594c1c5 100644 --- a/rs/moq-net/src/route/tests/warmth.rs +++ b/rs/moq-net/src/route/tests/warmth.rs @@ -609,7 +609,13 @@ fn growing() -> Vec { ADVERTISE, USEFUL, ]; - arms.extend(audiences()); + // Of ranked readers only the one measured against the others: its variants change + // trees, not how adverts grow, and each arm adds about a minute to the sweep. + arms.extend( + audiences() + .into_iter() + .filter(|arm| arm.order.is_none() || *arm == ORDER_ROUTE), + ); arms } diff --git a/rs/moq-net/verify/warm/Flap.cfg b/rs/moq-net/verify/warm/Flap.cfg index cb0b58629b8..634da1aca93 100644 --- a/rs/moq-net/verify/warm/Flap.cfg +++ b/rs/moq-net/verify/warm/Flap.cfg @@ -1,12 +1,13 @@ -\* Ranked readers on the drones of Drones.cfg: two reader changes, and the -\* uplink of drone 2 closing and reopening once. +\* Ranked readers at drones 2 and 3 of Drones.cfg, from no pulls, while the +\* uplink of drone 2 closes and reopens: each changes drone 2's route cost and +\* so its rank. CONSTANTS Shape = "drones" Flaky = {2} - Readable = {2, 3, 4} - Readers = {} - MaxReads = 2 - MaxCloses = 1 + Readable = {} + Readers = {2, 3} + MaxReads = 0 + MaxCloses = 2 Rule = "order" Order = "route" Summed = FALSE diff --git a/test/route-sweep.sh b/test/route-sweep.sh index 19b948bb80a..2f1dd669f86 100755 --- a/test/route-sweep.sh +++ b/test/route-sweep.sh @@ -20,5 +20,5 @@ MOQ_ROUTE_CASES=3000 just rs test -P sweep -p moq-net --release --run-ignored on MOQ_BAKEOFF_RELAYS=12 just rs test --profile=sweep -p moq-net --release --run-ignored only -E 'test(/^route::tests::bakeoff::(sweep|benchmark)_/)' # The multicast router: every failure on every shape at 32 relays, every delivery order the gate's searches leave out, what one event costs it, and how its floors grow. just rs test --profile=sweep -p moq-net --release --run-ignored only -E 'test(/^route::tests::multicast(_search|_cost)?::/)' -# Warmth: path joins, asking, and advertising over fixed shapes, 400 churned meshes per arm on three kinds of mesh, and growth over cores, edges per core, drones, and up to 4096 broadcasts, about 10 minutes. +# Warmth: path joins, asking, advertising, and ranked readers over fixed shapes, 400 churned meshes per arm on three kinds of mesh, and growth over cores, edges per core, drones, and up to 4096 broadcasts, about 10 minutes. just rs test --profile=sweep -p moq-net --release --run-ignored only -E 'test(/^route::tests::warmth::benchmark_/)' From 694b0d0847d6a261f7610c9d605ad941aee04385 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 04:21:27 -0800 Subject: [PATCH 5/7] verify(moq-net): the warm model breaks source ties as the simulator does The simulator takes the neighbour with the highest rendezvous hash among equal sources; the model took the lowest relay. Relay IDs stand in for those hashes, so the model now takes the highest. Every config keeps its result. The skill's sync commit is the PR's base. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-warm/SKILL.md | 16 ++++++++-------- quest/m3/cache-routing/select.md | 2 +- rs/moq-net/verify/warm/Orderless.cfg | 7 +++---- rs/moq-net/verify/warm/WarmSettle.tla | 6 ++++-- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.claude/skills/verify-warm/SKILL.md b/.claude/skills/verify-warm/SKILL.md index 1c601b6a62c..3e53613d6d6 100644 --- a/.claude/skills/verify-warm/SKILL.md +++ b/.claude/skills/verify-warm/SKILL.md @@ -30,16 +30,16 @@ quest gate just rs tla moq-net/verify/warm/WarmSettle.tla -config rs/moq-net/ver | Config | Rule | Checks | States, time at four workers | | --- | --- | --- | --- | -| `Drones.cfg` | ranked by route cost | three drones meshed with costly uplinks, readers starting and stopping three times | 164,434, 5 minutes | -| `Hash.cfg` | ranked by relay | the same | 164,434, 5 minutes | +| `Drones.cfg` | ranked by route cost | three drones meshed with costly uplinks, readers starting and stopping three times | 158,198, 5 minutes | +| `Hash.cfg` | ranked by relay | the same | 158,198, 5 minutes | | `Flap.cfg` | ranked by route cost | readers fixed at two drones while one's uplink closes and reopens, changing its rank | 44,427, under a minute | -| `Mesh503.cfg` | ranked by route cost | churned random mesh 503's six relays and readers, from no pulls | 490,125, 25 minutes | +| `Mesh503.cfg` | ranked by route cost | churned random mesh 503's six relays and readers, from no pulls | 490,125, 20 minutes | | `Settled.cfg` | settled pulls | the same | 407,526, 6 minutes | | `SettledDrones.cfg` | settled pulls | as `Drones.cfg` | 6,852, seconds | -| `Pick.cfg` | the price-moving pick | as `Mesh503.cfg`; fails `PullsQuiet` (open finding) | fails within 2 minutes | -| `Orderless.cfg` | ranked, rank check removed | as `Drones.cfg`; fails `PullsQuiet` (open finding) | fails within 6 minutes | +| `Pick.cfg` | the price-moving pick | as `Mesh503.cfg`; fails `PullsQuiet` (open finding) | fails within 6 minutes | +| `Orderless.cfg` | ranked, rank check removed | as `Drones.cfg`; fails `PullsQuiet` (open finding) | fails within 9 minutes | -Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. Two carry one, and neither is to be fixed. `Pick.cfg` shows the check has teeth: the rule it models is the one moq-dev/moq#8903 rejected for never settling case 503. `Orderless.cfg` shows the rank is what settles the ranked rule: without it, readers at drones 2 and 3 each take the other's warm copy over their uplinks, find their own path in what they now pull, fall back to their uplinks, and repeat for good. +Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. Two carry one, and neither is to be fixed. `Pick.cfg` shows the check has teeth: the rule it models is the one moq-dev/moq#8903 rejected for never settling case 503. `Orderless.cfg` shows the rank is what settles the ranked rule: without it, readers at two drones each take the other's warm copy over their uplinks, find their own path in what they now pull, drop it, and repeat for good. ## Reading a failure @@ -47,12 +47,12 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: the merge of moq-dev/moq#8986. Run `git log -- rs/moq-net/src/route/tests/warm.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. The rule lives in the simulator only; the port in `select.md` moves it into the relay and this table with it. +Sync commit: `914382986815bc5bd785c8b88d4be0386f235435`, the base of [moq-dev/moq#8986](https://github.com/moq-dev/moq/pull/8986), which added the ranked rule and this model. Run `git log 9143829868.. -- rs/moq-net/src/route/tests/warm.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. The rule lives in the simulator only; the port in `select.md` moves it into the relay and this table with it. | Code | Model | | --- | --- | | `rs/moq-net/src/route/tests/warm.rs` `WarmSim::ordered`, `pull`, `toward` | `Decide` (`Ordered`), `Next1`, `Toward` | -| `rs/moq-net/src/route/tests/warm.rs` `WarmSim::options`, `copy`, `best` | `Options`, `Heard`, `CopyCost`, `Best` | +| `rs/moq-net/src/route/tests/warm.rs` `WarmSim::options`, `copy`, `best` | `Options`, `Heard`, `CopyCost`, `Best`, `Better` (relay IDs for rendezvous hashes) | | `rs/moq-net/src/route/tests/warm.rs` `WarmSim::rank`, `own`, `ranked` | `RankOf`, `Own`, `Pub` (`pull`) | | `rs/moq-net/src/route/tests/warm.rs` `WarmSim::offer`, `warm`, `answer`, `advertise`, `scoped` | `Offer`, `Answer`, `Advert`, `Useful` | | `rs/moq-net/src/route/tests/warm.rs` `WarmSim::stay` | `Decide` (`"stay"`), `Chose` | diff --git a/quest/m3/cache-routing/select.md b/quest/m3/cache-routing/select.md index cb431ee394c..826609986d2 100644 --- a/quest/m3/cache-routing/select.md +++ b/quest/m3/cache-routing/select.md @@ -183,7 +183,7 @@ settles. readers while an uplink closes and reopens, and churned random mesh 503's six relays. With the rank check removed it fails on the three drones: two readers' - relays each take the other's copy, find their own path in it, fall back, + relays each take the other's copy, find their own path in it, drop it, and repeat. The pick fails on case 503, as the sweep found. - **What stays from the first cut.** Adverts, the mark, the path, the answer's offer, and choosing again on a route change. Adverts still go diff --git a/rs/moq-net/verify/warm/Orderless.cfg b/rs/moq-net/verify/warm/Orderless.cfg index 0f5879da4d0..e607ace148a 100644 --- a/rs/moq-net/verify/warm/Orderless.cfg +++ b/rs/moq-net/verify/warm/Orderless.cfg @@ -1,8 +1,7 @@ \* Open finding: ranked readers with the rank check removed never settle on -\* the drones of Drones.cfg. With readers at drones 2 and 3, each takes the -\* other's warm copy over its uplink, sees its own path in what it now pulls, -\* falls back to its uplink, and both repeat for good. It shows the rank is what -\* settles the ranked rule. +\* the drones of Drones.cfg. With readers at two drones, each takes the other's +\* warm copy over its uplink, sees its own path in what it now pulls, drops it, +\* and both repeat for good. It shows the rank is what settles the ranked rule. CONSTANTS Shape = "drones" Flaky = {} diff --git a/rs/moq-net/verify/warm/WarmSettle.tla b/rs/moq-net/verify/warm/WarmSettle.tla index ccc60352ed4..c533e45fa56 100644 --- a/rs/moq-net/verify/warm/WarmSettle.tla +++ b/rs/moq-net/verify/warm/WarmSettle.tla @@ -232,11 +232,13 @@ Options(r, u) == /\ Rule = "order" => h[2].rank < mine}} IN routes \cup copies -\* The cheaper, a warm copy before the route, then the lower relay. +\* The cheaper, a warm copy before the route, then the higher relay: the +\* simulator's `best` takes the neighbour with the highest rendezvous hash, and +\* relay IDs stand in for those hashes here, as Order = "id" ranks by them. Better(a, b) == \/ a.cost < b.cost \/ a.cost = b.cost /\ a.copy /\ ~b.copy - \/ a.cost = b.cost /\ a.copy = b.copy /\ a.via < b.via + \/ a.cost = b.cost /\ a.copy = b.copy /\ a.via > b.via Best(opts) == CHOOSE o \in opts : \A p \in opts : p = o \/ Better(o, p) Vias(opts) == {o.via : o \in opts} From bedbbd1e4fc29a70eca9d8c50be50aea385feb3c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 04:44:15 -0800 Subject: [PATCH 6/7] quest(cache-routing): select ports ranked readers The maintainer picked ranked readers over settled pulls on 2026-11-17, and warmth's wire for the first wip version after the lite-07 cut. select.md becomes the plan to port the one rule, now Required on the cut, and carries two requirements the spike left open: the model bound that did not finish, and measuring with per-link delivery the unfed reader-hops the simulator's one-message-per-step clock inflates. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-warm/SKILL.md | 4 +- quest/m3/cache-routing/README.md | 12 +- quest/m3/cache-routing/select.md | 637 ++++++++++++---------------- 3 files changed, 293 insertions(+), 360 deletions(-) diff --git a/.claude/skills/verify-warm/SKILL.md b/.claude/skills/verify-warm/SKILL.md index 3e53613d6d6..eb91cb0855f 100644 --- a/.claude/skills/verify-warm/SKILL.md +++ b/.claude/skills/verify-warm/SKILL.md @@ -9,10 +9,10 @@ description: Model-check whether warm pulls settle (`rs/moq-net/verify/warm/Warm `Rule` picks the rule: -- `"order"`: ranked readers, the rule [moq-dev/moq#8986](https://github.com/moq-dev/moq/pull/8986) measured. A reader ranks by its relay's route cost, then its relay (`Order = "route"`), or by relay alone (`"id"`). A relay takes a warm copy only from a neighbour feeding a reader ranked below every reader it feeds itself, priced at the link's cold price alone, and heads for the cheapest such copy or its route whenever either changes. A pull carries the lowest rank it feeds, and an advert or answer the lowest rank its sender feeds besides its receiver. +- `"order"`: ranked readers, the rule [moq-dev/moq#8986](https://github.com/moq-dev/moq/pull/8986) measured and `quest/m3/cache-routing/select.md` ports. A reader ranks by its relay's route cost, then its relay (`Order = "route"`), or by relay alone (`"id"`). A relay takes a warm copy only from a neighbour feeding a reader ranked below every reader it feeds itself, priced at the link's cold price alone, and heads for the cheapest such copy or its route whenever either changes. A pull carries the lowest rank it feeds, and an advert or answer the lowest rank its sender feeds besides its receiver. - `"orderless"`: the same with the rank check removed. - `"pick"`: the price-moving pick moq-dev/moq#8903 rejected. -- `"stay"`: settled pulls, the first cut `quest/m3/cache-routing/select.md` names. +- `"stay"`: settled pulls, which the maintainer rejected for ranked readers on 2026-11-17; kept to show the model covers both. ## Rules diff --git a/quest/m3/cache-routing/README.md b/quest/m3/cache-routing/README.md index e4981209334..41c4e437f98 100644 --- a/quest/m3/cache-routing/README.md +++ b/quest/m3/cache-routing/README.md @@ -33,6 +33,11 @@ a node that says it carries a broadcast). What stands and what goes: - ✅ Warm means demand, not cache contents, and pop skipping and one uplink per group of drones are the two cases that must work (maintainer, 2026-11-15, [epic](/quest/m2/multicast/README.md) item 9). +- ✅ A pull moves to a cheaper warm copy only from a reader ranked below its + own: ranked readers, which a TLA+ model shows settle (maintainer, + 2026-11-17, [moq-dev/moq#8986](https://github.com/moq-dev/moq/pull/8986); + rejected: settled pulls). Its wire lands in the first wip version after + the lite-07 cut. - The non-goal of "one copy across a multi-hop sparse overlay rather than among direct neighbors" is dropped: a mesh is the product this serves. @@ -57,10 +62,9 @@ This README's own work, once its child lands: plus a skip link: a cold broadcast takes the skip, a reader on the skipped relay makes the far relay join it, and that reader leaving sends it back. Two LAN relays with their own uplinks and readers on both pull one copy - over one uplink when the readers arrive in turn. When they arrive at once, - both uplinks stay until one relay's readers leave, unless the maintainer - keeps the tie-break [warmth](https://github.com/moq-dev/moq/pull/8903) measured. The other uplink takes over - when that one is lost. + over one uplink when the readers arrive in turn, and when they arrive at + once, the higher-ranked relay moving onto the other's copy. The other + uplink takes over when that one is lost. - Docs: warm and cold costs in `doc/bin/relay/cluster.md`, and a section on the routing concept page once the [routing guide](/quest/m3/routing-guide.md) has added it. diff --git a/quest/m3/cache-routing/select.md b/quest/m3/cache-routing/select.md index 826609986d2..5773ce45c8d 100644 --- a/quest/m3/cache-routing/select.md +++ b/quest/m3/cache-routing/select.md @@ -1,19 +1,20 @@ -# [XL] Relays pull each broadcast from a warm copy and keep it +# [XL] Relays pull each broadcast from a warm copy, ranked by reader ## Goal A relay with demand for an Instance advertises it on the announce stream to -each peer or upstream that could take it. When a relay's demand starts, it -pulls from the cheapest copy it has heard of, its route or a warm neighbour. -After that the pull stays: it moves only when its parent loses its upstream, -the pull runs through the relay itself, the session closes, or the relay's -route changes, never because a neighbour's price did. A later step lets an -edge query its other upstreams when it gains demand, once the conditions -below hold. A peer that sends no warm signal is treated as cold and -behaves as it does today. Relays export how many upstream copies they hold -and what each choice was for. Whether a pull instead moves to a cheaper copy -under ranked readers, which also settles, waits on the maintainer (see -Ranked readers below). +each peer or upstream that could take it. A relay with a reader of its own +ranks it by its route cost to the Instance, then its Node ID, and every +pull, advert, and subscribe answer carries the lowest rank its sender feeds. +A relay pulls from the cheapest of its route and the warm copies that feed a +reader ranked below every reader it feeds, and moves, make before break, +whenever that set or its own rank changes. Once readers, links, and +publishers stop changing, the pulls settle. A later step lets an edge query +its other upstreams when it gains demand, once the conditions below hold. A +peer that sends no rank or warm signal ranks below every ranked reader and +is treated as cold, behaving as it does today. Relays export how many +upstream copies they hold and what each choice was for. The wire lands in +the first wip version after the lite-07 cut. ## Plan @@ -23,106 +24,124 @@ to build, item 14 of the [epic](/quest/m3/cache-routing/README.md)'s earlier Plan, is replaced. Decided 2026-11-16 (maintainer, merge pick of -[moq-dev/moq#8903](https://github.com/moq-dev/moq/pull/8903), which -recommended it): **a settled pull stays**, unless the spike the maintainer -asked for with that pick finds a rule that moves a pull for price and still -settles. It found one, -[moq-dev/moq#8986](https://github.com/moq-dev/moq/pull/8986): ranked -readers, below. Which of the two this ports waits on the maintainer; until -then the Plan below stays settled pulls, and the section on ranked readers -says what changes with it. A relay chooses -where to pull when its demand starts, from its route and the warm copies it -has heard of by then. After that it chooses again only on an outside event: -its parent loses its upstream, its pull runs through itself, its session -closes, or its route changes. What it hears comes from **adverts between -peers and toward upstreams, only where the copy could be taken, never down a -link the neighbour marked upstream, and never up one with a copy that came -down one anywhere on its path**, and **the data path and the mark in every offer**. The race -between simultaneous starts stands. **A one-shot query** from an edge to its -other upstreams is a later step, with its price and condition below; with -it, a relay also chooses once more on each query answer from its join. -Rejected: the price-moving pick, which moved a live pull whenever another -source got strictly cheaper and never settles churned random mesh 503. - -A rise in the parent's offer is not an event. A rise is what a sibling's -move or a neighbour's lost reader causes, and the point of the rule is that -no relay's move is another relay's reason to move. A route change is one: -the route layer's routes follow links and claims, never demand. - -So no price ever moves a pull: a pull moves on an outside event (a reader -starts or stops, a link or relay fails or returns, a publisher moves), on the -route layer settling after one, or on what those set off. It is greedy online -Steiner without rearrangement, apart from the moves a route change buys. -A TLA+ model shows it settles at small bounds; see below. +[moq-dev/moq#8903](https://github.com/moq-dev/moq/pull/8903)): a settled +pull stays, unless a spike finds a rule that moves a pull for price and +still settles. What a relay hears comes from **adverts between peers and +toward upstreams, only where the copy could be taken, never down a link the +neighbour marked upstream, and never up one with a copy that came down one +anywhere on its path**, and **the data path and the mark in every offer**. +**A one-shot query** from an edge to its other upstreams is a later step, +with its price and condition below. Rejected: the price-moving pick, which +moved a live pull whenever another source got strictly cheaper and never +settles churned random mesh 503. + +Decided 2026-11-17 (maintainer, on +[moq-dev/moq#8986](https://github.com/moq-dev/moq/pull/8986#issuecomment-6948028151), +the spike, which recommended it): ✅ **ranked readers** (rejected: settled +pulls, which give up about two thirds of the drift for a small wire saving). +Adverts, the mark, the path, and the query as a later step stand from +2026-11-16. + +Decided 2026-11-17 (maintainer, same comment): ✅ **warmth ships in the next +wip version, after the lite-07 cut** (rejected: holding lite-07 for the +port; freezing the rank and advert fields in lite-07 before the port has +exercised them). The 2026-11-14 decision stands: lite-07 freezes without a +warm wire. The cut waits on [the switch](/quest/m2/multicast/switch.md), and +this port is the first item after it. + +**Why ranked readers settle.** A relay's choice reads only copies feeding +lower ranks, and what those feed never depends on a higher rank's choice. +Once the outside events stop and the route layer settles (its own TLA+ +model and sweeps cover that), every rank is fixed; then the lowest rank's +pulls settle, then the next, by induction on the rank. No relay's demand +comes back to it as a price, the reflection every fence of the pick chased. +`rs/moq-net/verify/warm/` (the `verify-warm` skill) checks it with TLA+ from +every state the bounded outside events reach, delivering every word late +and in any order: three drones with three reader changes, two fixed readers +while an uplink closes and reopens, and churned random mesh 503's six +relays. With the rank check removed it fails on the three drones: two +readers' relays each take the other's copy, find their own path in it, drop +it, and repeat. The pick fails on case 503, as the sweep found. **What the guarantee costs.** Imase and Waxman (Dynamic Steiner Tree Problem, SIAM J. Discrete Math. 4(3), 1991) bound greedy joins without rearrangement at O(log k) times the cheapest tree for k terminals, and show no online rule without rearrangement does better than ½ log2 k. The bound assumes terminals only arrive, fixed link costs, and each newcomer joining the -nearest node of the whole tree by a shortest path. Here a relay joins the -cheapest copy it has heard of, priced by the copy's whole path rather than -the one link it adds, and readers also leave; for arrivals and departures -together their constant-factor algorithm rearranges O(n^(3/2)) edges over n -requests, which this never does for price. So the bound is a guide, and the -sweeps measure the drift: - -| | settled pulls, first cut | with the query, later | ranked readers | the price-moving pick | path joins | +nearest node of the whole tree by a shortest path. Ranked readers rearrange: +once settled, each relay's pull depends only on lower-ranked readers, as if +readers had arrived in rank order whatever order they really came in, so a +departure re-forms the tree rather than leaving the detour an arrival-only +rule keeps. The bound is still a guide, since a relay joins only copies it +has heard of, priced by the link alone; the sweeps measure the drift: + +| | ranked readers, the port | settled pulls, rejected | with the query, measured under settled pulls | the price-moving pick, rejected | path joins | |---|---|---|---|---|---| -| settled trees over the cheapest, 400 churned random / drone / region meshes | +3.4% / +16.2% / +5.4% | +3.4% / +16.2% / +5.0% | +1.2% / +4.9% / +4.4% | +2.7% / +9.4% / +4.9% | +2.7% / +28.9% / +5.0% | +| settled trees over the cheapest, 400 churned random / drone / region meshes | +1.2% / +4.9% / +4.4% | +3.4% / +16.2% / +5.4% | +3.4% / +16.2% / +5.0% | +2.7% / +9.4% / +4.9% | +2.7% / +28.9% / +5.0% | | unsettled of 2,000 per kind | 0 / 0 / 0 | 0 / 0 / 0 | 0 / 0 / 0 | 1 / 0 / 0 (random case 503) | 0 / 0 / 0 | -| longest cycle under churn, deliveries | 20 / 18 / 14 | 20 / 18 / 19 | 19 / 23 / 18 | 18 / 39 / 24 | 16 / 14 / 17 | -| unfed reader-hops | 19,008 / 11,691 / 17,470 | 19,008 / 11,691 / 23,089 | 21,917 / 14,584 / 20,661 | 18,000 / 12,381 / 22,472 | 15,700 / 7,725 / 15,735 | -| warm messages, 400 churned regions | 4,655 | 21,020 | 13,133 | 21,151 | 0 | -| pops, summed over its five phases (cheapest 59) | 71 | 71 | 59 | 61 | 69 | +| longest cycle under churn, deliveries | 19 / 23 / 18 | 20 / 18 / 14 | 20 / 18 / 19 | 18 / 39 / 24 | 16 / 14 / 17 | +| unfed reader-hops, in the simulator's clock (below) | 21,917 / 14,584 / 20,661 | 19,008 / 11,691 / 17,470 | 19,008 / 11,691 / 23,089 | 18,000 / 12,381 / 22,472 | 15,700 / 7,725 / 15,735 | +| warm messages, 400 churned regions | 13,133 | 4,655 | 21,020 | 21,151 | 0 | +| pops, summed over its five phases (cheapest 59) | 59 | 71 | 71 | 61 | 69 | | drones in turn | 75, one uplink | 75, one uplink | 75, one uplink | 75, one uplink | 155, five | -| drones / ring / edges at once (cheapest 19 / 112 / 7) | 51 / 392 / 9 | as the first cut | 19 / 112 / 8 | 51 / 392 / 9 | 51 / 392 / 9 | -| drones lose the uplink (cheapest 94) | 118, four uplinks, a 17-step cycle | 118, four uplinks, a 17-step cycle | 94, one uplink | 118, four uplinks | 198 | -| drones one / two hops apart | 41 / 33, at 69 / 62 ms | as the first cut | 17 / 17, at 81 / 69 ms | 33 / 25, at 81 / 69 ms | 41 / 41, at 69 / 62 ms | -| edges in turn, one region with unequal uplinks, rings, tiers | 13, 29, 2904, 1095 | 12, 24, 2904, 1095 | 13, 29, 2904, 1095 | 12, 24, 2904, 1095 | 14, 29, 9624, 1191 | -| warm messages per reader change, unpeered CDN, one viewer / crowd | 0 / 0 | 1.0 / 1.0 | 0 / 0.67 | 1.0 / 1.0 | 0 / 0 | -| warm messages per reader change, 64 peered drones, a crowd | 63 | 63 | 161 | 70 | 0 | - +| drones / ring / edges at once (cheapest 19 / 112 / 7) | 19 / 112 / 8 | 51 / 392 / 9 | as settled pulls | 51 / 392 / 9 | 51 / 392 / 9 | +| drones lose the uplink (cheapest 94) | 94, one uplink | 118, four uplinks, a 17-step cycle | 118, four uplinks, a 17-step cycle | 118, four uplinks | 198 | +| drones one / two hops apart | 17 / 17, at 81 / 69 ms | 41 / 33, at 69 / 62 ms | as settled pulls | 33 / 25, at 81 / 69 ms | 41 / 41, at 69 / 62 ms | +| edges in turn, one region with unequal uplinks, rings, tiers | 13, 29, 2904, 1095 | 13, 29, 2904, 1095 | 12, 24, 2904, 1095 | 12, 24, 2904, 1095 | 14, 29, 9624, 1191 | +| warm messages per reader change, unpeered CDN, one viewer / crowd | 0 / 0.67 | 0 / 0 | 1.0 / 1.0 | 1.0 / 1.0 | 0 / 0 | +| warm messages per reader change, 64 peered drones, a crowd | 161 | 63 | 63 | 70 | 0 | + +At 2,000 churned meshes per kind, ranked readers settle +1.4% / +6.3% / ++4.6% over the cheapest, against settled pulls' +3.2% / +16.0% / +5.3%. Random and drone meshes have no relay with two upstreams, so the query -changes nothing there. Ranked readers were measured on 2026-11-17 beside a -rerun of the other columns, which matched the table; at 2,000 per kind they -settle +1.4% / +6.3% / +4.6% over the cheapest, against the first cut's -+3.2% / +16.0% / +5.3%. - -The table is the rerun of 2026-11-16, after the route order gained the mark -([moq-dev/moq#8904](https://github.com/moq-dev/moq/pull/8904)) and a -request's equal next hops follow it. Every -settled tree and unsettled count is as first measured; the churn rows moved -by under 1%, apart from path joins' longest cycle on random meshes, 9 before. - -Its drift is a relay that stays on a detour once the warm copy that drew it -loses its other readers: on pops, the middle pop keeps pulling through the -near pop after the near pop's reader leaves (71 against path joins' 69). -Choosing again when a relay's own reader starts (`+rejoin`) changed nothing -measurable; choosing again on a route change (`+reroute`, part of the -recommendation) took random meshes from +4.4% to +3.4% and regions from -+6.0% to +5.0%. Drone meshes drift most, since their routes rarely change. -No timer is added. - -**The price-moving pick, rejected,** moved a live pull whenever another source got -strictly cheaper, fenced by a rule that a relay takes no warm signal from a -relay on its own path. It never settles churned random mesh 503, a loop -through a sibling: 6 pulls from 4, which makes 4 warm for 3, so 3 offers 6 -less than 4 does; 6 moves to 3, 4 turns cold for 3, and 6 moves back. Each -fence covered one shape of a relay's own demand coming back as a price, as -the old route rule's fixes did; refusing every offer whose path crosses the -taker's below the publisher's relay unsettled other arms instead. Under the -recommendation no price moves a pull, so the reflection can make a choice -wrong but cannot make one. Whether the choices a lost or failed pull sets -off can repeat is not yet shown, as below. +changes nothing there. The settled-pulls, query, pick, and path-join columns +are the rerun of 2026-11-16, after the route order gained the mark +([moq-dev/moq#8904](https://github.com/moq-dev/moq/pull/8904)); ranked +readers were measured on 2026-11-17 beside a rerun that matched them. + +The simulator, `rs/moq-net/src/route/tests/warm.rs`, runs ranked readers as +`advertise_not_down+hops+order_route`, settled pulls as +`advertise_not_down+hops+stay+reroute`, and the query as +`advertise_not_down_query+hops+stay+reroute`; its `warmth.rs` holds the +shapes, the churn sweep, and the growth sweep this ports onto production +code. + +**Settled pulls, rejected.** A relay chose where to pull when its demand +started and chose again only on an outside event: its parent lost its +upstream, its pull ran through itself, its session closed, or its route +changed, never because a price did. It settles too: 0 of 6,000 churned +meshes unsettled, and `Settled.cfg` and `SettledDrones.cfg` pass the same +model. Its drift is a relay that stays on a detour once the warm copy that +drew it loses its other readers (pops 71 against the cheapest 59), and two +relays that start at once each keep their own uplink (drones 51 against 19). +It needs no rank on the wire and sends fewer messages between peers, which +the maintainer judged not worth about two thirds of the drift. + +**The price-moving pick, rejected,** moved a live pull whenever another +source got strictly cheaper, fenced by a rule that a relay takes no warm +signal from a relay on its own path. It never settles churned random mesh +503, a loop through a sibling: 6 pulls from 4, which makes 4 warm for 3, so 3 +offers 6 less than 4 does; 6 moves to 3, 4 turns cold for 3, and 6 moves +back. Each fence covered one shape of a relay's own demand coming back as a +price, as the old route rule's fixes did. The rank removes the cause rather +than a shape of it. Also rejected: +- For ranked readers, pricing a copy at the parent's summed offer plus the + link, and keeping the current copy while no dearer: it reads demand above + the first hop, so relays kept dearer copies (drones +18.4% against +4.9%). +- Ranking by Node ID alone, or a hash of it: it settles too, but the near + pop never hears a middle pop ranked above it, so pops keeps the path + join's tree. +- Pricing the advert audience at the link alone, as the choice prices a + copy: a warm core told every core it meshes with, 0.9 to 28.6 warm + messages per reader change from 4 to 32 cores. - Advertising to every neighbour that could take the copy, down marked links too: it grows with a region's edges, since a warm core tells every edge it could feed. 1.7 to 118 warm messages per one-viewer reader change as edges - per core grow from 3 to 128, against none in the first cut and a flat 1.0 - with the query. + per core grow from 3 to 128, against none with adverts as decided and a + flat 1.0 with the query. - Advertising to every neighbour, which also grows with the neighbours of every relay on the warm chain: 9 to 66 per one-viewer reader change as a full core mesh grows from 4 to 32 cores. @@ -148,232 +167,130 @@ Also rejected: nothing in a core mesh. - Joins along the path only, the cost baseline, which delivers neither case. -That PR's simulator, `rs/moq-net/src/route/tests/warm.rs`, runs the first -cut as `advertise_not_down+hops+stay+reroute` and the later step as -`advertise_not_down_query+hops+stay+reroute`, and its `warmth.rs` -holds the shapes, the churn sweep, and the growth sweep this ports onto -production code. Ranked readers run as `advertise_not_down+hops+order_route`. - -### Ranked readers, waiting on the maintainer +### The query, a later step -[moq-dev/moq#8986](https://github.com/moq-dev/moq/pull/8986) spiked a rule -under which a pull moves to a cheaper warm copy that turns up later, and -`rs/moq-net/verify/warm/` (the `verify-warm` skill) model-checks that it -settles. - -- **The rank.** A relay with a reader of its own for an Instance ranks it - by its route cost to the Instance, then its Node ID. A pull carries the - lowest rank among the readers it feeds, its relay's own and each - downstream pull's, and an update when that changes. An advert and a - subscribe answer carry the lowest rank their sender feeds besides their - receiver. -- **The choice.** A relay takes a warm copy only from a neighbour feeding a - reader ranked strictly below every reader the relay feeds itself. That - copy is fed whatever the relay does, so it costs the link's cold price - alone. The relay heads for the cheapest such copy or its route, make - before break, whenever that set or its own rank changes; nothing else - moves it, and no price above the first hop enters the choice. -- **Why it settles.** A relay's choice reads only copies feeding lower - ranks, and what those feed never depends on a higher rank's choice. So - once the outside events stop, the lowest rank's pulls settle, then the - next, by induction on the rank: no relay's demand comes back to it as a - price, the reflection every fence of the pick chased. TLC checks it from - every state the bounded outside events reach, delivering every word late - and in any order, on three drones with three reader changes, two fixed - readers while an uplink closes and reopens, and churned random mesh 503's - six relays. - With the rank check removed it fails on the three drones: two readers' - relays each take the other's copy, find their own path in it, drop it, - and repeat. The pick fails on case 503, as the sweep found. -- **What stays from the first cut.** Adverts, the mark, the path, the - answer's offer, and choosing again on a route change. Adverts still go - only where the copy could be taken, priced from the publisher (offer plus - the link's cold price, against the neighbour's route); pricing that test - at the link alone made a full core mesh's warm messages grow with its - cores (0.9 to 28.6 per reader change from 4 to 32 cores), where priced - from the publisher they stay flat: none for one viewer, 0.67 for a crowd. The path still breaks - a cycle formed on stale adverts: without it no mesh went unsettled, but - the longest cycle on drones grew from 29 to 99 deliveries. -- **What it buys.** The pop that turns warm later draws the pulls behind it - (pops 59, the cheapest, against 71). Drones, rings, and edges that start - at once merge onto one uplink (19 against 51 on drones, 112 against 392 on - the ring): the higher rank moves, the merge the maintainer's rule of - 2026-11-15 asked for, here without the cycles it added under the pick. A drone that loses the uplink - hands it to one other drone, not four, and without the first cut's - 17-step cycle. Churned trees settle at a third of the first cut's drift on - drones and random meshes. -- **What it costs.** On the wire, in the then-current wip version: a rank - on each SUBSCRIBE and an update message when it changes; a rank on each - advert and subscribe answer and its updates. A rank is a route cost and a - Node ID. Relay state: one rank per served subscription, one per heard - advert or answer, and the rank last told on each pull, all bounded as the - adverts are. Messages: 13,133 warm messages per 400 churned regions - against 4,655, rank updates 8,494 of them; 0.67 per crowd reader change on - an unpeered CDN against none; about 2.6 times the first cut's per reader - change between peers (161 against 63 for a crowd on 64 drones), since a - rank change reaches every peer that could take the copy. Readers: moves - stall readers in the simulator, which delivers one control message per - step across the mesh, so unfed reader-hops rise by 15% to 25%, and a - drone one hop from another waits 81 ms for a first frame against 69. -- **A peer that sends no rank** is ranked below every ranked reader: a - relay it pulls from never takes a warm copy for its sake and pulls along - its route, as today. Its adverts are never taken, since nothing says the - copy is fed regardless. -- **Tried and dropped on the way.** Pricing a copy at the parent's summed - offer plus the link, as the first cut does, and keeping the current copy - while no dearer: it reads demand above the first hop, so relays kept - dearer copies (drones +18.4% against +4.9%). Ranking by Node ID alone: it - settles too (`Hash.cfg`, and 0 of 6,000 churned meshes unsettled under an - earlier advert audience), but the near pop never hears a middle pop - ranked above it, so pops keeps the path join's tree. - -Recommended: port ranked readers rather than settled pulls. Both settle in -every sweep and pass the model; ranked readers move toward the cheapest -tree where warmth matters, between peers and at simultaneous starts, and -match the first cut's trees on an unpeered CDN, for one more field on three -messages and one update message. Settled pulls stay the choice if that -wire or the extra moves are not worth a third of the drift. - -### Decision: the query, a later step - -Recommended: ship adverts, the mark, the path, and settled pulls first, and -add the query as a later step; an automated review on #8903 proposed the -same. The query is the one part that only an edge with more than one -upstream uses. +Decided 2026-11-16 with settled pulls; it stands under ranked readers. The +query is the one part that only an edge with more than one upstream uses. - **What it buys:** only where an edge's cheaper uplink is to the core that - is cold. One region with unequal uplinks costs 24 with it and 29 without, - and two edges in turn 12 against 13; churned regions settle +5.0% over the - cheapest with it against +5.4% without (+5.1% against +5.3% at 2,000 per - kind). With equal uplinks it buys nothing: every edge of a region ranks the - two cores by the same rendezvous hash per broadcast, so every edge's path - join lands on the same core. + is cold. Under settled pulls one region with unequal uplinks cost 24 with + it and 29 without, and two edges in turn 12 against 13; churned regions + settled +5.0% over the cheapest with it against +5.4% without. With equal + uplinks it buys nothing: every edge of a region ranks the two cores by the + same rendezvous hash per broadcast, so every edge's path join lands on the + same core. Ranked readers measured 29 and 13 there too. - **What it costs:** a request and an answer on the wire, and exactly 1.0 - warm message per reader change on an unpeered CDN against none. Under - churn, per 400 regions, 21,020 warm messages against 4,655, 59,921 - messages against 45,280, and 23,089 unfed reader-hops against 17,470, much - of that from the simulator delivering one control message per step across - the mesh. The edge holds one answer per extra upstream per demanded - Instance; the upstream holds nothing and carries no traffic. It also adds a - choice after the join that the settling model must cover. + warm message per reader change on an unpeered CDN against none. The edge + holds one answer per extra upstream per demanded Instance; the upstream + holds nothing and carries no traffic. +- **Under ranked readers** an answer carries the upstream's rank besides the + edge, and joins the edge's options like an advert: taken only from a lower + rank. It was measured only under settled pulls. - **Until then** an operator gets the same tree by pricing each edge's uplinks equally. -- **Its condition:** add it once the settling model holds with the query in - it, the edge's bound of one answer per extra upstream per demanded - Instance, and none on the upstream, is enforced and tested, and a - deployment has unequal uplinks where the core-mesh copy it saves matters. - Shipping it in the first cut instead is measured above, and settles too. +- **Its condition:** add it once the simulator measures it under ranked + readers, the settling model holds with the query in it, the edge's bound of + one answer per extra upstream per demanded Instance, and none on the + upstream, is enforced and tested, and a deployment has unequal uplinks + where the core-mesh copy it saves matters. ### What each part costs | part | on the wire | relay state | bounded by | without it | |---|---|---|---|---| -| advert | offer, path, and mark on the announce stream, in order with the route messages, and its withdrawal | one heard per neighbour per Instance the relay has a route for | neighbours; the path by the hop limit | path joins: drones in turn 5 uplinks against 1, a ring 48 against 8, tiers 1191 against 1095 | -| the route layer's mark on adverts | one bit on the advert, the answer, and the query's answer, as a route's offer carries | the bit with each heard advert and live pull | one per offer | an edge carries a copy between its cores, which no route would: 51 readers in 400 churned regions fed that way under the pick, and 1.05 to 1.55 warm messages per one-viewer reader change on an unpeered CDN against 1.0 | -| offer on the subscribe answer | a cost, path, and mark on the answer, and an update on each change; none when the parent lost its upstream | one per pull | one per subscription | nothing prices a query answer or a new route against the pull, and a lost parent cannot say so | -| path in the offer | a list on the advert, the answer, and the query's answer | as the offer | the hop limit | 2 of 6,000 churned meshes unsettled (random 433, drones 1783), longest cycle 109 deliveries against 27: a pull cycle formed on stale adverts cannot be seen | -| query, the later step | a request naming an Instance; one answer, offer, path, and mark or none | the edge keeps one answer per extra upstream per demanded Instance until judged; the upstream none | upstreams beyond the first | an edge whose cheaper uplink is cold: 29 against 24 | -| a settled pull stays | none | the route it chose by | one choice per outside event | the pick: random churned mesh 503 never settles | -| choosing again on a route change | none | as above | route changes, which the route layer bounds | settled trees +4.4% against +3.4% on random meshes, +6.0% against +5.0% on regions | - -What the pick needed and this deletes: the rule against a warm signal from a -relay on the taker's own path, since a relay that never moves for price -cannot be drawn back up its own branch; the rule that a move under way stays -while no dearer; and make-before-break bookkeeping for price moves, kept only -for a query answer judged after the pull went live. A lost pull carries -nothing, so a new one replaces it outright. +| advert | offer, path, mark, and rank on the announce stream, in order with the route messages, and its withdrawal | one heard per neighbour per Instance the relay has a route for | neighbours; the path by the hop limit | path joins: drones in turn 5 uplinks against 1, a ring 48 against 8, tiers 1191 against 1095 | +| the rank | a route cost and a Node ID on each SUBSCRIBE, advert, and subscribe answer, and an update message for a subscription's rank | one per served subscription and per heard advert or answer, and the rank last told on each pull | as the adverts | the rule never settles (`Orderless.cfg`); without moving for price at all, settled pulls drift +3.4% / +16.2% / +5.4% | +| the route layer's mark on adverts | one bit on the advert, the answer, and the query's answer, as a route's offer carries | the bit with each heard advert and live pull | one per offer | an edge carries a copy between its cores, which no route would: 51 readers in 400 churned regions fed that way under the pick | +| offer on the subscribe answer | a cost, path, mark, and rank on the answer, and an update on each change; none when the parent lost its upstream | one per pull | one per subscription | a relay cannot keep its current parent by its answer once that parent stops advertising to it, and a lost parent cannot say so | +| path in the offer | a list on the advert, the answer, and the query's answer | as the offer | the hop limit | under ranked readers no mesh went unsettled, but a pull cycle formed on stale adverts lasted up to 99 deliveries on drones against 29; under settled pulls 2 of 6,000 churned meshes never settled | +| query, the later step | a request naming an Instance; one answer, offer, path, mark, and rank or none | the edge keeps one answer per extra upstream per demanded Instance until judged; the upstream none | upstreams beyond the first | an edge whose cheaper uplink is cold: 29 against 24 | + +What the pick needed and ranked readers delete: the rule against a warm +signal from a relay on the taker's own path, since a neighbour pulling +through a relay never feeds a rank below that relay's; and the rule that a +move under way stays while no dearer. Make before break stays for every +move of a live pull. A lost pull carries nothing, so a new one replaces it +outright. ### What a relay sends +- **The rank.** A relay with a reader of its own for an Instance ranks it + `(route cost, Node ID)`, the route layer's summed cold cost to the + Instance; it changes with the route. Each SUBSCRIBE carries the lowest rank + among the readers it feeds, the relay's own and each downstream pull's, + and an update message when that changes. A relay with no ranked reader + downstream sends none. - **The offer.** What a relay R with a live upstream through parent P offers - neighbour N: `O(R→N) = O(P→R) + warm(P, R)` while R has demand besides N, - otherwise `O(P→R) + cold(P, R)`, the chain priced unshared. The publisher's - relay offers 0. An offer is never below its sender's own cost. + neighbour N: `O(R→N) = O(P→R) + warm(P, R)`. The publisher's relay offers + 0. The offer prices the advert audience below; the choice prices a copy at + the link alone. - **The path and the mark.** Every offer names the relays the data passes from the publisher's relay to the sender, and carries the mark: set once the copy came down a link its receiver marked upstream anywhere on that path, and kept on every hop after, as a route's down-only flag is. - **Adverts.** While R has demand for an Instance besides N, N has not marked R upstream, and R has not both marked N upstream and a copy whose - mark is set, it tells N its offer on the announce stream when - `O(R→N) + cold(R, N)` is below N's route cost for the Instance, or equals it - through a next hop other than R, since a tie prefers a warm copy. It tells - again on each change, and withdraws when the demand, its upstream, or that - test ends. Adverts keep their order with the route layer's messages on the - same announce stream. Nothing it skips between peers could ever be taken: a - relay at rest costs no more than its own route, and a tie through R is the - path join already. R knows N's route cost from what N announces it, and - computes it when N's route runs through R. The mark is the route layer's - own rule: without it an edge advertised one core's copy to its other core, - which then pulled through the edge, a path no route offers. -- **The query, the later step.** A relay with more than one upstream sends each upstream - besides the one its path join takes one query when it gains demand: one - request, one answer. The upstream answers its warm offer, or that it has - none, from what it holds, and keeps nothing: it forwards nothing, holds no - demand for it, and sends no update. It is not a metadata-only - subscription: one forwarded by a cold core would hold demand of a kind - across the core mesh for as long as the viewer stays, the standby the epic - keeps opt in and off inside the CDN. An upstream cannot tell which of its - upstreams a dual-homed edge's route uses, so only the edge can find a warm - core beside its route. -- **Answers.** The subscribe answer carries the parent's offer, path, and - mark, an update on each change, and none once the parent lost its own - upstream, which it says at once to a new subscriber too. A subscriber waits - for that answer before it counts the pull as live. + mark is set, it tells N its offer and the lowest rank it feeds besides N on + the announce stream when `O(R→N) + cold(R, N)` is below N's route cost for + the Instance, or equals it through a next hop other than R. It tells again + on each change, and withdraws when the demand, its upstream, or that test + ends. Adverts keep their order with the route layer's messages on the same + announce stream. The audience is priced from the publisher, unlike the + choice, so a warm core never tells another core whose own route is + cheaper. R knows N's route cost from what N announces it, and computes it + when N's route runs through R. The mark is the route layer's own rule: + without it an edge advertised one core's copy to its other core, which + then pulled through the edge, a path no route offers. +- **Answers.** The subscribe answer carries the parent's offer, path, mark, + and the lowest rank it feeds besides the subscriber, an update on each + change, and none once the parent lost its own upstream, which it says at + once to a new subscriber too. A subscriber waits for that answer before it + counts the pull as live. +- **The query, the later step.** A relay with more than one upstream sends + each upstream besides the one its path join takes one query when it gains + demand: one request, one answer. The upstream answers its warm offer and + rank, or that it has none, from what it holds, and keeps nothing: it + forwards nothing, holds no demand for it, and sends no update. It is not a + metadata-only subscription: one forwarded by a cold core would hold demand + of a kind across the core mesh for as long as the viewer stays, the + standby the epic keeps opt in and off inside the CDN. ### What a relay chooses -- **When its demand starts**, it pulls from the cheapest of its route and - the warm copies it has heard of. A copy costs its offer plus the link's - cold price; the route costs the route layer's summed cold cost. Equal - candidates fall to a warm copy before the route, then the rendezvous hash, - as `route_order` in `rs/moq-net/src/model/origin.rs` already ranks equal - routes. It joins at once, without waiting for its queries. -- **On each query answer from that join**, once the query lands, judged once, it moves to the - answering upstream if that is strictly cheaper than its own cost - `O(P→R) + warm(P, R)`, keeping the old pull until the new one is live. -- **When its route changes**, it chooses once more, as at a join, from what - it has heard by then, moving only to something strictly cheaper. A route - change can reach a relay before the advert that came with it; then the one - choice finds only the route (`an_ancestor_warm_on_its_own_is_passed_over` - keeps 14 where 13 is cheapest). -- **When its parent answers none, or its pull's path names it**, it replaces - the pull outright with a fresh choice. -- **Otherwise it stays**, whatever its parent or anyone else offers later. -- It never pulls from a neighbour that pulls through it, nor from one whose - path names it. A pending pull whose source left its options fails closed: - two relays that each turned to the other on stale adverts otherwise wait on - each other for good. The longest such cycle in 2,000 churned meshes per - kind lasted 27 deliveries on random meshes, 44 on drones, and 30 on - regions (19 with the query); the lost-uplink case shows one for 17 steps. - -Why it should settle, and what is not yet shown: after the last outside -event, a relay chooses only as the route layer settles (finitely many route -changes, which its own TLA+ model and sweeps cover), with the query on each -answer from a join (one per extra upstream), and when a pending pull fails closed -or a parent says it lost its upstream. No choice follows a price. But a -fresh choice after a lost parent or a failed pending pull can pick a warm -copy on a stale advert, give that relay demand and so a join of its own, or -end another relay's pull, so "once per pull" does not bound the number of -pulls after the last outside event (an automated review on #8903). The sweep -agrees with settling: 6,000 churned meshes, 2,000 per kind, all settled. -The `verify-warm` model checks it at small bounds: settled pulls go quiet -and feed every routed reader on churned random mesh 503 and on three drones -with three reader changes (`Settled.cfg`, `SettledDrones.cfg`). It found no -warm retries that repeat, so the rule takes no fallback. +- **Its rank** is the lowest rank it feeds: its own reader's, and each + subscription it serves. +- **Its options** are its route, priced at the route layer's summed cold + cost, and each warm copy it has heard of, by advert or by its current + parent's answer, whose rank is strictly below its own, priced at the + link's cold price alone: that copy is fed whatever this relay does. A copy + whose path names the relay is never an option. +- **It heads for the cheapest option** whenever its options or its rank + change: at a join, on an advert, answer, or rank update, and on a route + change. Equal options fall to a warm copy before the route, then the + rendezvous hash, as `route_order` in `rs/moq-net/src/model/origin.rs` + already ranks equal routes. A live pull moves make before break: the old + pull stays until the new one is answered live. There is no hysteresis and + no timer. +- **When its parent answers none, or its pull's path names it**, it + replaces the pull outright with a fresh choice. A pending pull whose + target is no longer the best is replaced: two relays that each turned to + the other on stale adverts otherwise wait on each other for good. +- **A peer that sends no rank** ranks below every ranked reader: a relay it + pulls from never takes a warm copy for its sake and pulls along its route, + as today. Its adverts are never taken, since nothing says the copy is fed + regardless. ### Simultaneous starts -Two drones reaching the CDN at once each pay their own uplink, and both stay -until one drone's demand ends; under the recommendation nothing merges them, -since a merge is a move for price. The maintainer's rule, decided 2026-11-15, -was to merge them on the rendezvous hash unless that adds a cycle, an unfed -reader, or more messages than the second uplink costs; under the pick's -adverts it added cycles (18 deliveries through merged pulls in 400 churned -drone meshes) and about 1% unfed reader-hops. So the race stands under settled -pulls; ranked readers merge them (above). A drone -that loses the one uplink leaves every drone it fed choosing again at once: -4 uplinks on the five-drone shape. +Drones, rings, and edges that start at once merge onto one uplink: the +higher rank moves onto the lower rank's copy once it hears the advert (19 +against 51 on drones, 112 against 392 on the ring). That is the merge the +maintainer's rule of 2026-11-15 asked for, which under the pick's adverts +added cycles (18 deliveries through merged pulls in 400 churned drone +meshes); ranked readers merge with no cycle, the same unfed reader-hops (15 +on drones at once), and 64 messages against 32. A drone that loses the +uplink hands it to one other drone, not four (94 against settled pulls' +118). ### What holds @@ -381,87 +298,99 @@ that loses the one uplink leaves every drone it fed choosing again at once: least 1, summed from the source along the path the data takes. - Extend today's selection rather than adding a second selector. A front already moves to a better route of its own Instance and resplices - (`rs/moq-net/src/model/front.rs`); here it does so only on the events - above. A front moves only to a copy of its own Instance. + (`rs/moq-net/src/model/front.rs`); here it does so on the events above. A + front moves only to a copy of its own Instance. - The rule runs in production code that the fixtures and sweeps drive. The stripped rule lived only in a test simulator that checked it against itself, so a rule that never offered a copy passed. Port the warmth shapes and the churn sweep onto production relays: pops, drones in turn and at once, the lost uplink, the ring, the tiers, and random churned mesh 503. -- Port the rule the `verify-warm` model checks (`Rule = "stay"` for settled - pulls, `"order"` for ranked readers), and move its Coverage table from the - simulator onto the port, keeping the model in step with it. The query is - enabled only once the model holds with it. +- Port the rule the `verify-warm` model checks as `Rule = "order"`, move its + Coverage table from the simulator onto the port, and keep the model in + step with it. The query is enabled only once the model holds with it. +- Close the model bound the spike left open: a reader change during an + uplink flap, where a rank changes while readers move. On the three drones + it passed 2M states in about an hour without finishing, so the spike + checks the flap with fixed readers only (`Flap.cfg`). Get it to finish, by + a smaller mesh (two drones and the publisher's relay) or a symmetry + reduction, or say why it cannot and what covers it instead. +- Measure what the moves cost readers before reading the simulator's unfed + figure as production behaviour. Under churn ranked readers leave 15% to + 25% more reader-hops unfed than settled pulls in the simulator, yet readers + already fed lose nothing: their unfed hops are equal or fewer (5,388 + against 5,443 on random meshes, 1,977 against 2,098 on drones), since a + move is make before break. The whole rise is readers waiting for their + first frame (18,307 against 15,033 on random meshes, 13,477 against + 10,273 on drones). Their subscribe and answer wait behind the rank and + advert messages the rule adds, because the simulator delivers one control + message per step across the whole mesh: unfed hops per delivery match + across arms (0.319 against 0.317 on random meshes, 0.174 against 0.160 on + drones, 0.304 against 0.316 on regions). Keeping a pending pull while its + target stayed an option changed nothing (21,809 against 21,917 on random + meshes). The + acceptance tests measure unfed reader-hops and first frames with each link + delivering in order at its own pace, beside path joins. - Bound the warm state a peer can make a relay hold: one advert per neighbour per Instance the relay has a route for, a path no longer than the - route layer's hop limit, and adverts for anything else dropped. Adverts are - per broadcast like the claims the announce stream already carries over - every link, so a relay hears at most one per neighbour per demanded - broadcast. A query leaves nothing on the upstream; the edge keeps at most - one answer per upstream beyond the first per demanded broadcast. + route layer's hop limit, one rank per served subscription and heard + advert, and adverts for anything else dropped. Adverts are per broadcast + like the claims the announce stream already carries over every link, so a + relay hears at most one per neighbour per demanded broadcast. A query + leaves nothing on the upstream; the edge keeps at most one answer per + upstream beyond the first per demanded broadcast. - Check the Goal's efficiency against plain path routing on the same shapes, - as the table above does: on the simulator, 3.4 to 11 messages per reader - change against 2 to 5.4, and under churn the trees settled on cost 3.4%, - 16.2%, and 5.4% over the cheapest (5.0% on regions with the query) for the same readers on random meshes, - drones, and regions, against path joins' 2.7%, 28.9%, and 5.0%. + as the table above does: on the simulator, 4.3 to 13.8 messages per reader + change against 2 to 5.4, apart from the lost uplink, and under churn the + trees settled on cost 1.2%, 4.9%, and 4.4% over the cheapest for the same + readers on random meshes, drones, and regions, against path joins' 2.7%, + 28.9%, and 5.0%. - Port the growth sweep, and hold its laws, per link kind. Per reader change, - warm messages are flat in the number of demanded broadcasts. + warm messages are flat in the number of demanded broadcasts, and trees and + first frames match settled pulls' in every growth row. - Between cores, a full peer mesh: flat in cores. A core with demand is - never cheaper for another core than that core's own route. - - Down a marked link: none. Up it: no advert of a copy that came down - another, so none in the first cut. With the query, one per upstream - beyond the first, with its answer, when an edge gains demand: exactly 1.0 - per reader change, one viewer or a crowd, flat from 4 to 32 cores and 3 - to 128 edges per core, up to 2,080 relays; a core holds nothing at rest. - - Between edges or drones that peer: n − 1 per first viewer (63 at 64 - drones), since every one could take the copy; that is what buys the - cheapest tree there (432 against 1,328 to 1,456 for asking and path - joins, a crowd of 9 on 16 broadcasts). - With the query, at rest the busiest relay of 80 holds 99 query answers - for 4,096 one-viewer broadcasts, against 4,355 metadata-only subscriptions for - `ask_1` and 139,264 adverts sent to every neighbour. + never cheaper for another core than that core's own route, so it never + advertises to one. An unpeered CDN sends none for one viewer and 0.67 per + reader change for a crowd, the rank updates on subscriptions it already + holds, from 4 to 32 cores. + - Down a marked link: no advert. Up it: no advert of a copy that came down + another. With the query, one per upstream beyond the first, with its + answer, when an edge gains demand. + - Between edges or drones that peer: about 2.5 n per crowd reader change + (161 at 64 drones, against settled pulls' 63), since every one could + take the copy and every rank change reaches it; that is what buys the + cheapest tree there. - A region's edges do not peer every pair as it grows: they reach each other - through the region's cores. Peered, the region is a drone mesh and pays its - n − 1 per first viewer (64 at 64 edges per core); the tree it buys there, - 288 against 544 unpeered for a crowd, comes from pricing an edge-to-edge - link at a third of an uplink. -- Port the cases as acceptance tests, two of them new: drones starting at - once, whose uplinks the race keeps, and the same sweeps with each link - delivering in order at its own pace, rather than one control message per - delivery step across the mesh, which is what the simulator's unfed counts - and first frames assume. A first frame matched the path join's on every - shape measured. Two edges starting at once each keep their path join (9 - against 7), the race again. -- Export upstream copies held, choices by cause (join, route change, parent - lost its upstream, and query answer once it lands), and adverts sent and held, through the + through the region's cores. Peered, the region is a drone mesh and pays + its per-peer cost (165 per crowd reader change at 64 edges per core); the + tree it buys there, 288 against 544 unpeered for a crowd, comes from + pricing an edge-to-edge link at a third of an uplink. +- Port the cases as acceptance tests, with drones, rings, and edges starting + at once, which ranked readers merge, and the same sweeps with each link + delivering at its own pace. A first frame matched the path join's on every + shape measured except drones one hop apart (81 ms against 69), where the + second drone first pulls its uplink and then moves onto the first. +- Export upstream copies held, choices by cause (join, advert or answer, + rank change, route change, parent lost its upstream, and query answer once + it lands), moves, and adverts and rank updates sent and held, through the relay's existing metrics. - `rs/moq-net/AGENTS.md` says a relay behaves the same warm or cold. That covers answers to requests; this changes which neighbour a request goes to. Bring a rewording to the maintainer if the two contradict. -Decided 2026-11-14 (maintainer, pick on -[moq-dev/moq#8764](https://github.com/moq-dev/moq/pull/8764)): ✅ lite-07 -freezes without a warm wire, so whatever wire the pick needs lands in the -then-current wip version with this. - Public API: report any relay configuration added, such as a warm cost per -link. Wire, in the then-current wip version and all additive: an advert on -the announce stream carrying an Instance's offer, path, and mark, and its -withdrawal, in order with the route messages there; the same offer, path, and -mark on the subscribe answer and its updates; and, with the later step, a -query naming an Instance, answered once with an offer, path, and mark or -none. With ranked readers, also a rank on the SUBSCRIBE, the advert, and -the answer, and an update message for a subscription's rank. Nothing +link. Wire, in the first wip version after the lite-07 cut and all +additive: an advert on the announce stream carrying an Instance's offer, +path, mark, and rank, and its withdrawal, in order with the route messages +there; a rank on the SUBSCRIBE and a message updating it; the same offer, +path, mark, and rank on the subscribe answer and its updates; and, with the +later step, a query naming an Instance, answered once with an offer, path, +mark, and rank or none. A rank is a route cost and a Node ID. Nothing lite-07 freezes changes. ## Required - [The switch](/quest/m2/multicast/switch.md) - the route layer it ranks on - -## Waiting - -- Maintainer: port ranked readers or settled pulls (see Ranked readers, - waiting on the maintainer). Recommended: ranked readers. +- [Lite-07 finalized](/quest/m2/lite07-finalize.md) - warmth's wire lands in the first wip version after the cut (maintainer, 2026-11-17) ## Related From dd31e33d6c9e195ff866ba9015e891c777f9b4b2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 05:07:00 -0800 Subject: [PATCH 7/7] quest(cache-routing): the port pins a relay an older relay pulls through The plan said a peer with no rank ranks below every reader, while the simulator drops such a pull from a relay's rank, and neither the sweep nor the model ever had an unranked relay in a mesh. The plan now states one rule for the port to build and requires the mix in its sweep and model. It also narrows what the model's one register per neighbour covers and asks the port for independent delivery. The skill's sync commit moves to the PR's base. Co-Authored-By: Claude Fable 5.2 --- .claude/skills/verify-warm/SKILL.md | 2 +- quest/m3/cache-routing/select.md | 39 +++++++++++++++++++++++------ 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/.claude/skills/verify-warm/SKILL.md b/.claude/skills/verify-warm/SKILL.md index eb91cb0855f..06044fb4b90 100644 --- a/.claude/skills/verify-warm/SKILL.md +++ b/.claude/skills/verify-warm/SKILL.md @@ -47,7 +47,7 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `914382986815bc5bd785c8b88d4be0386f235435`, the base of [moq-dev/moq#8986](https://github.com/moq-dev/moq/pull/8986), which added the ranked rule and this model. Run `git log 9143829868.. -- rs/moq-net/src/route/tests/warm.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. The rule lives in the simulator only; the port in `select.md` moves it into the relay and this table with it. +Sync commit: `1399add3c886f1c168f0c6ba3ba55586a04feb6c`, the base of [moq-dev/moq#8986](https://github.com/moq-dev/moq/pull/8986), which added the ranked rule and this model. Run `git log 1399add3c8.. -- rs/moq-net/src/route/tests/warm.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. The rule lives in the simulator only; the port in `select.md` moves it into the relay and this table with it. | Code | Model | | --- | --- | diff --git a/quest/m3/cache-routing/select.md b/quest/m3/cache-routing/select.md index 5773ce45c8d..8a4dffd5e86 100644 --- a/quest/m3/cache-routing/select.md +++ b/quest/m3/cache-routing/select.md @@ -11,8 +11,8 @@ reader ranked below every reader it feeds, and moves, make before break, whenever that set or its own rank changes. Once readers, links, and publishers stop changing, the pulls settle. A later step lets an edge query its other upstreams when it gains demand, once the conditions below hold. A -peer that sends no rank or warm signal ranks below every ranked reader and -is treated as cold, behaving as it does today. Relays export how many +relay that sends no rank or warm signal is treated as cold and behaves as it +does today, and a relay it pulls through takes its route. Relays export how many upstream copies they hold and what each choice was for. The wire lands in the first wip version after the lite-07 cut. @@ -56,8 +56,13 @@ model and sweeps cover that), every rank is fixed; then the lowest rank's pulls settle, then the next, by induction on the rank. No relay's demand comes back to it as a price, the reflection every fence of the pick chased. `rs/moq-net/verify/warm/` (the `verify-warm` skill) checks it with TLA+ from -every state the bounded outside events reach, delivering every word late -and in any order: three drones with three reader changes, two fixed readers +every state the bounded outside events reach, delivering each neighbour's +words late: what one relay tells a neighbour (its pull's rank, its answer, +its advert) arrives together as that neighbour's newest word, any number of +steps stale, and neighbours fall behind independently. It never delivers one +neighbour's new rank beside its old advert or answer, which the wire can, +since they ride separate streams; the port covers that, below. The model +checks three drones with three reader changes, two fixed readers while an uplink closes and reopens, and churned random mesh 503's six relays. With the rank check removed it fails on the three drones: two readers' relays each take the other's copy, find their own path in it, drop @@ -275,10 +280,16 @@ outright. replaces the pull outright with a fresh choice. A pending pull whose target is no longer the best is replaced: two relays that each turned to the other on stale adverts otherwise wait on each other for good. -- **A peer that sends no rank** ranks below every ranked reader: a relay it - pulls from never takes a warm copy for its sake and pulls along its route, - as today. Its adverts are never taken, since nothing says the copy is fed - regardless. +- **A subscription with no rank.** From an endpoint it is a reader of the + relay it reaches, which ranks it as its own. From a relay, one on a version + before the warm wire, it pins the relay it reaches to its route for that + Instance while it lasts: nothing carries a rank or a path through the older + relay, so no rule here can say a warm copy does not loop back through it. + The older relay sends no advert, so its copy is never taken. + The spike never had such a relay in a mesh, and its simulator does not + build this rule: `WarmSim::rank` drops a pull with no rank, so a relay + feeding an older relay and a ranked reader of its own would still take a + warm copy. Build the rule above, and test the mix as below. ### Simultaneous starts @@ -314,6 +325,18 @@ uplink hands it to one other drone, not four (94 against settled pulls' checks the flap with fixed readers only (`Flap.cfg`). Get it to finish, by a smaller mesh (two drones and the publisher's relay) or a symmetry reduction, or say why it cannot and what covers it instead. +- Put a relay that sends no rank, path, or advert in the churn sweep and the + model, beside ranked relays and with a ranked reader on the relay it pulls + through: the mix a rollout with two releases side by side produces. The + rule above pins that relay to its route. Counting the older relay's pull + as a reader of the relay it reaches, as an endpoint's is, would keep warmth + for the pinned relay's own readers; take it only if the sweep and the + model hold with it. +- Cover independent delivery, which the model's one register per neighbour + cannot: a neighbour's new rank arriving before or after its advert and + its answer, a stale advert outliving a rank's withdrawal or replacement + included. Split the register in the model, or pin the cases in the port's + acceptance tests with each stream delivering at its own pace. - Measure what the moves cost readers before reading the simulator's unfed figure as production behaviour. Under churn ranked readers leave 15% to 25% more reader-hops unfed than settled pulls in the simulator, yet readers