From 221c87c968a18f57fc9f26a02a49282aa016688c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 22:33:32 -0800 Subject: [PATCH 01/40] chore(quest): claim quest/m2/cluster-routing/routes/session.md Quest-Claim: e7a08a7c96080d03 From b755d0e9cb57f4e21cc6e42d6c59c7caa5ce9981 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 23:07:23 -0800 Subject: [PATCH 02/40] feat(net): present each route's claims at a stream's whole interest A floor is now one marker per route across the stream's interest. The router presents its per-region view onto it: claims inside keep their path, the most specific covering claim moves onto the interest (moved, not ended, when withdrawn), and the floor certifies only when the view knows the whole interest at one revision. Claims a stream carries uncertified stay in its heard view and show where no certified floor proves them absent; they go offline, never END, when their stream drops them. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/mod.rs | 170 ++++++++++++++++------ rs/moq-net/src/route/tests/handoff.rs | 145 ++++++++++-------- rs/moq-net/src/route/tests/mod.rs | 64 ++++++-- rs/moq-net/src/route/tests/routing.rs | 46 +++++- rs/moq-net/src/route/view.rs | 202 ++++++++++++++++++++------ 5 files changed, 458 insertions(+), 169 deletions(-) diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index 64ee72f116c..07f131aee0b 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -20,6 +20,7 @@ mod view; mod tests; use std::{ + borrow::Cow, cmp::Reverse, collections::{BTreeMap, BTreeSet}, fmt, @@ -30,7 +31,7 @@ use crate::Path; pub use proof::RouteKey; pub(crate) use proof::{EXPORTER_LABEL, Proof, Transcript}; -pub(crate) use view::{View, overlaps, within}; +pub(crate) use view::{View, within}; /// An opaque routing identity, declared by an origin or derived by [`Scope`]. pub(crate) type Id = u64; @@ -141,13 +142,15 @@ pub(crate) enum Reach { pub(crate) enum Out { /// ROUTE_START or ROUTE_UPDATE, ROUTE_HOLD for a held reach, or ROUTE_END for `None`. Route { to: N, id: Id, reach: Option }, - /// A batch of announcement changes for `id`, closed by its floor list, which - /// replaces the last one sent for that route. + /// A batch of announcement changes for `id`, as the neighbor's request presents + /// them, closed by ANNOUNCE_FLOOR: the revision the stream's claims for the route + /// are the origin's at, across the neighbor's whole interest, or `None` when this + /// node does not know them there at one revision. Batch { to: N, id: Id, changes: Vec<(Path, Option)>, - floors: BTreeMap, + floor: Option, }, /// ROUTE_REQUEST, toward the neighbor whose route this node would use. Request { to: N, id: Id, seqno: Seqno }, @@ -240,6 +243,9 @@ struct Peer { draining: bool, /// The prefix the neighbor asked this node for, once it opened a stream. watch: Option, + /// The prefix this node asked the neighbor for, which its floors certify; the + /// node's interest when unset. + asked: Option, } /// An identity this node serves itself. @@ -367,6 +373,7 @@ impl Router { link, draining: false, watch: None, + asked: None, }, ); // Nothing is sent or learned until a stream opens. @@ -416,6 +423,14 @@ impl Router { Ok(()) } + /// This node asked `neighbor` for announcements under `prefix`: the region its + /// floors certify. + pub fn asks(&mut self, neighbor: N, prefix: Path) -> Result<(), Error> { + let peer = self.neighbors.get_mut(&neighbor).ok_or(Error::UnknownNeighbor)?; + peer.asked = Some(prefix); + Ok(()) + } + /// `neighbor` sent GOAWAY: it ranks below every other eligible route while /// its session stays up. pub fn goaway(&mut self, neighbor: N) { @@ -632,15 +647,10 @@ impl Router { } /// A batch of announcement changes for `named` from `from`, closed by its - /// floor list, on the stream answering interest generation `interest`. - pub fn batch( - &mut self, - from: N, - named: Id, - changes: Vec<(Path, Option)>, - floors: BTreeMap, - interest: u32, - ) { + /// floor, on the stream answering interest generation `interest`. A floor + /// certifies the whole region this node asked for at that revision; `None` + /// certifies nothing, so the stream's claims stay only while it carries them. + pub fn batch(&mut self, from: N, named: Id, changes: Vec<(Path, Option)>, floor: Option, interest: u32) { let Some(peer) = self.neighbors.get(&from) else { return; }; @@ -652,6 +662,8 @@ impl Router { return; }; let scope = peer.link.scope.as_ref().map(|scope| scope.regions.clone()); + let region = peer.asked.clone().unwrap_or_else(|| self.watch.clone()); + let floors = floor.map(|rev| BTreeMap::from([(region, rev)])).unwrap_or_default(); let heard = self.heard.entry((id, from)).or_insert_with(|| Heard { raw: View::default(), view: View::default(), @@ -664,7 +676,7 @@ impl Router { } heard.raw.apply(changes, floors); heard.view = match &scope { - Some(regions) => heard.raw.restrict_all(regions), + Some(regions) => heard.raw.present_all(regions), None => heard.raw.clone(), }; let view = heard.view.clone(); @@ -1041,18 +1053,58 @@ impl Router { fn forgot_outside(&mut self, id: Id, kept: &[Path]) { #[cfg(test)] for ((key, claim), count) in self.forgot.iter_mut() { - if *key == id && !kept.iter().any(|region| overlaps(claim, region)) { + if *key == id && !kept.iter().any(|region| within(claim, region)) { *count += 1; } } } - /// The view this node shows and forwards for `id`, having selected `route`. - fn adopted(&self, id: Id, route: Option>) -> Option<&View> { + /// The view this node shows and forwards for `id`, having selected `route`: what + /// it knows certified, and each claim a certifier carries uncertified where no + /// floor it knows covers the claim. + fn adopted(&self, id: Id, route: Option>) -> Option>> { if let Some(origin) = self.origins.get(&id) { - return Some(&origin.view); + return Some(Cow::Borrowed(&origin.view)); + } + if !self.live(id, route) { + return None; + } + let merged = self.views.get(&id); + let mut loose = self.loose(id, route, merged).peekable(); + if loose.peek().is_none() { + return merged.map(Cow::Borrowed); } - self.live(id, route).then(|| self.views.get(&id)).flatten() + let mut view = merged.cloned().unwrap_or_default(); + for (claim, item, _) in loose { + view.items.entry(claim.clone()).or_insert_with(|| item.clone()); + } + Some(Cow::Owned(view)) + } + + /// The uncertified claims of `id` its certifiers carry on streams answering the + /// current interest where `merged` certifies nothing, the selected next hop's + /// first, and whether the carrier is held. + fn loose<'a>( + &'a self, + id: Id, + route: Option>, + merged: Option<&'a View>, + ) -> impl Iterator + 'a { + let mut certifiers = self.certifiers(id); + if let Some(next) = route.and_then(|route| route.next) { + certifiers.sort_by_key(|&n| n != next); + } + let routes = self.rx.get(&id); + certifiers.into_iter().flat_map(move |next| { + let held = routes.is_some_and(|routes| routes[&next].held); + self.heard + .get(&(id, next)) + .filter(|heard| heard.interest == self.interest) + .into_iter() + .flat_map(|heard| heard.view.uncertified()) + .filter(move |(claim, _)| merged.is_none_or(|merged| merged.as_of(claim).is_none())) + .map(move |(claim, item)| (claim, item, held)) + }) } /// Whether this node sends `id` to `neighbor` at all: its own identities to @@ -1135,12 +1187,13 @@ impl Router { // Recording may have tightened the feasibility distance: select again. let route = self.selected(id); let live = self.live(id, route); - let adopted = self.adopted(id, route).cloned(); - // A sender that certifies anything answers every stream, even one whose - // interest none of its regions meets, so the receiver knows it certifies - // nothing there. - let answers = adopted.as_ref().is_some_and(|view| !view.floors.is_empty()); - let mut restricted: Vec<(Path, View)> = Vec::new(); + let adopted = self.adopted(id, route).map(Cow::into_owned); + // A sender that knows anything answers every stream, even one whose interest + // none of its regions meets, so the receiver knows it certifies nothing there. + let answers = adopted + .as_ref() + .is_some_and(|view| !view.floors.is_empty() || !view.items.is_empty()); + let mut presented: Vec<(Path, View)> = Vec::new(); for &to in peers { let peer = &self.neighbors[&to]; @@ -1174,20 +1227,18 @@ impl Router { self.sent_views.remove(&(id, to)); continue; } - // Peers mostly share an interest, so restrict once per interest. - let index = match restricted.iter().position(|(interest, _)| *interest == watch) { + // Peers mostly share an interest, so present once per interest. + let index = match presented.iter().position(|(interest, _)| *interest == watch) { Some(index) => index, None => { - let view = adopted - .as_ref() - .map_or_else(View::default, |view| view.restrict(&watch)); - restricted.push((watch, view)); - restricted.len() - 1 + let view = adopted.as_ref().map_or_else(View::default, |view| view.present(&watch)); + presented.push((watch.clone(), view)); + presented.len() - 1 } }; - let view = &restricted[index].1; + let view = &presented[index].1; let changes = match self.sent_views.get(&(id, to)) { - Some(sent) if sent == view => continue, + Some(sent) if sent.items == view.items && sent.floor(&watch) == view.floor(&watch) => continue, Some(sent) => view.diff(sent), None if !answers => continue, None => view.diff(&View::default()), @@ -1197,7 +1248,7 @@ impl Router { to, id, changes, - floors: view.floors.clone(), + floor: view.floor(&watch), }); } } @@ -1255,23 +1306,39 @@ impl Router { fn show(&mut self, id: Id) { let mut shown = BTreeMap::new(); let route = self.selected(id); + let usable = route.is_some(); + let own = self.origins.contains_key(&id); + let merged = match own { + true => None, + false => self.views.get(&id), + }; + // An uncertified claim is available through a carrier that is not held. + let mut loose: BTreeMap<&Path, bool> = BTreeMap::new(); + if !own && self.live(id, route) { + for (claim, _, held) in self.loose(id, route, merged) { + *loose.entry(claim).or_insert(false) |= !held; + } + } if let Some(view) = self.adopted(id, route) { - let usable = route.is_some(); - let own = self.origins.contains_key(&id); - // A claim is available only through a certifier that is not held: - // another scope's usable route cannot serve it. + // A certified claim is available only through a certifier that is not + // held: another scope's usable route cannot serve it. let serving = match usable && !view.items.is_empty() && !own { true => self.certified(id, false), false => None, }; let current = self.current.get(&id); for (claim, item) in &view.items { - let certified = current.is_none_or(|regions| regions.iter().any(|region| overlaps(claim, region))); + // A claim a stream answering the current interest carries is current. + let certified = loose.contains_key(claim) + || current.is_none_or(|regions| regions.iter().any(|region| within(claim, region))); let available = usable - && serving - .as_ref() - .is_none_or(|regions| regions.iter().any(|region| overlaps(claim, region))); - if overlaps(claim, &self.watch) && certified { + && match loose.get(claim) { + Some(&unheld) => unheld || serving.is_none(), + None => serving + .as_ref() + .is_none_or(|regions| regions.iter().any(|region| within(claim, region))), + }; + if within(claim, &self.watch) && certified { shown.insert( claim.clone(), Shown { @@ -1303,13 +1370,20 @@ impl Router { } } None => { - // A claim no routed neighbor's region covers was let go, not ended. + // A claim no routed neighbor's region covers was let go, not ended, + // and so was one the view never certified: only its stream left. let uncertified = self .certified(id, true) - .is_some_and(|regions| !regions.iter().any(|region| overlaps(claim, region))); - let kind = if !overlaps(claim, &self.watch) { + .is_some_and(|regions| !regions.iter().any(|region| within(claim, region))) + || !own && self.views.get(&id).is_none_or(|view| view.as_of(claim).is_none()); + // A claim presented at a narrower interest moved up to the claim it + // stood for, which is still shown. + let moved = shown + .iter() + .any(|(above, now)| above != claim && within(claim, above) && now.item == was.item); + let kind = if !within(claim, &self.watch) { Kind::Unwatched - } else if self.live(id, route) && !uncertified { + } else if self.live(id, route) && !uncertified && !moved { Kind::End } else { Kind::Offline diff --git a/rs/moq-net/src/route/tests/handoff.rs b/rs/moq-net/src/route/tests/handoff.rs index 4c72ca7a1b6..89ad0107f78 100644 --- a/rs/moq-net/src/route/tests/handoff.rs +++ b/rs/moq-net/src/route/tests/handoff.rs @@ -411,10 +411,11 @@ fn two_scoped_uplinks() -> Sim { sim } -/// A relay forwards a key whose floors are all grant scopes, and its peer -/// moves to an interest no scope meets. The relay still answers the new -/// stream, certifying nothing, so the peer lets go of a claim it kept from the -/// scoped view instead of waiting on an answer that never comes. +/// A relay forwards a key whose floors are all grant scopes: each presents a +/// claim above its scope at the scope itself, and certifies nothing across a +/// peer's whole interest, so the peer shows both presentations uncertified. +/// When the peer moves to an interest no scope meets, each relay still answers +/// the new stream with nothing, so the peer holds no stale stream open. #[test] fn a_disjoint_interest_is_answered_with_nothing() { let (origin, z) = (0, 4); @@ -422,12 +423,23 @@ fn a_disjoint_interest_is_answered_with_nothing() { sim.announce(origin, "live/", 0, None); assert!(sim.run()); let id = qualified("local", 7); - assert_eq!(sim.shows(z, id, "live/"), Some(true)); + assert_eq!(sim.shows(z, id, "live/"), None); + assert_eq!(sim.shows(z, id, "live/a/"), Some(true)); + assert_eq!(sim.shows(z, id, "live/b/"), Some(true)); + assert!( + sim.nodes[z].views().all(|(_, view)| view.items.is_empty()), + "nothing certified" + ); sim.watch(z, "live/c"); assert!(sim.run()); - assert_eq!(sim.shows(z, id, "live/"), None); - assert_eq!(life(&sim, z, "live/"), [Kind::Start, Kind::Offline]); + assert!(sim.shown(z).is_empty()); + assert_eq!(life(&sim, z, "live/a/"), [Kind::Start, Kind::Unwatched]); + let router = &sim.nodes[z]; + assert!( + router.heard.values().all(|heard| heard.interest == router.interest), + "both relays answered" + ); sim.check().unwrap(); } @@ -484,11 +496,12 @@ fn a_lost_scope_keeps_newer_knowledge_where_still_certified() { sim.check().unwrap(); } -/// When one scoped uplink closes, its relay is starved and holds, so the third -/// relay shows that scope unavailable; once a seqno request recovers a route, -/// nothing certifies the lost scope and the third lets it go. Certifiers are -/// feasible neighbors only, so the two never keep the scope alive through each -/// other. +/// When one scoped uplink closes, its relay is starved and holds, and the +/// third relay knows that scope's claims only uncertified, through the starved +/// relay. Once the other relay's newer seqno makes the starved one infeasible, +/// nothing carries the lost scope and the third lets it go, whether or not the +/// starved relay's own requests get through. Certifiers are feasible neighbors +/// only, so the two never keep the scope alive through each other. #[test] fn a_relay_lets_go_of_a_scope_its_uplinks_lost() { let (origin, local, x, z) = (0, 1, 2, 4); @@ -510,13 +523,17 @@ fn a_relay_lets_go_of_a_scope_its_uplinks_lost() { sim.disconnect(x, local); sim.unannounce(origin, "live/a/x"); assert!(sim.run()); - let lost = if request { None } else { Some(false) }; if !request { // Only the ROUTE update reaches z, not the requests. while sim.deliver(x, z) {} assert!(sim.run()); } - assert_eq!(sim.shows(z, id, "live/a/x"), lost, "request {request}"); + assert_eq!(sim.shows(z, id, "live/a/x"), None, "request {request}"); + assert_eq!( + life(&sim, z, "live/a/x").last(), + Some(&Kind::Offline), + "request {request}" + ); assert_eq!(sim.shows(z, id, "live/b/z"), Some(true), "request {request}"); assert_eq!(false_ends(&sim.events), 0, "request {request}"); sim.check().unwrap(); @@ -524,16 +541,27 @@ fn a_relay_lets_go_of_a_scope_its_uplinks_lost() { } /// Every lite-07 claim covers the paths below it, so a claim above an -/// interest is inside it. +/// interest covers it, and a stream answering the interest presents it there. +/// Withdrawing it moves the presentation to the next claim up rather than +/// ending it. #[test] -fn a_claim_above_an_interest_overlaps_it() { +fn a_claim_above_an_interest_is_presented_at_it() { let (origin, client) = (0, 3); let mut sim = lagging("live/a/x"); + sim.announce(origin, "live", 0, None); sim.announce(origin, "live/a", 0, None); sim.announce(origin, "live/b", 0, None); assert!(sim.run()); - assert_eq!(sim.shows(client, 7, "live/a"), Some(true)); - assert_eq!(sim.shows(client, 7, "live/b"), None); + assert_eq!(sim.shows(client, 7, "live/a/x"), Some(true)); + assert_eq!(sim.shown(client).len(), 1); + let below = sim.shown(client)[&(7, p("live/a/x"))].item; + assert_eq!(Some(below), sim.truth(7, "live/a", sim.revision(7))); + + sim.unannounce(origin, "live/a"); + assert!(sim.run()); + let above = sim.shown(client)[&(7, p("live/a/x"))].item; + assert_eq!(Some(above), sim.truth(7, "live", sim.revision(7))); + assert!(!life(&sim, client, "live/a/x").contains(&Kind::End), "moved, not ended"); sim.check().unwrap(); } @@ -560,21 +588,29 @@ fn wildcard_claims_end_per_origin_and_are_certified_below() { sim.announce(o2, "live/", 0, Some(1)); assert!(sim.run()); assert_eq!(sim.shown(client).len(), 2); + assert_eq!( + sim.shows(client, 10, "live/a/"), + Some(true), + "presented at the interest" + ); sim.hold(o1, r2); sim.unannounce(o1, "live/"); assert!(sim.run()); - assert_eq!(sim.shows(client, 10, "live/"), None); + assert_eq!(sim.shows(client, 10, "live/a/"), None); assert_eq!( - sim.shows(client, 11, "live/"), + sim.shows(client, 11, "live/a/"), Some(true), "the other origin's claim stays" ); sim.disconnect(client, r1); assert!(sim.run()); - assert_eq!(sim.shows(client, 10, "live/"), None); - assert_eq!(sim.shows(client, 11, "live/"), Some(true)); + assert_eq!(sim.shows(client, 10, "live/a/"), None); + assert_eq!(sim.shows(client, 11, "live/a/"), Some(true)); + // Each origin's claim starts, and only the withdrawn one ends. + assert_eq!(life(&sim, client, "live/a/"), [Kind::Start, Kind::Start, Kind::End]); + assert_eq!(false_ends(&sim.events), 0); } /// An origin process restarts with a fresh identity: its claims start a new @@ -1042,22 +1078,20 @@ fn sweep_scoped_handoff() { sim.check().unwrap_or_else(|err| panic!("scenario {scenario}: {err}")); let watch = sim.watching(client).clone(); let rev = sim.revision(7); - let expected: BTreeSet = CLAIMS - .into_iter() - .filter(|claim| sim.truth(7, claim, rev).is_some()) - .map(p) - .filter(|claim| { - SCOPES.iter().map(|scope| p(scope)).any(|scope| { - let region = if within(&scope, &watch) { - scope - } else if within(&watch, &scope) { - watch.clone() - } else { - return false; - }; - overlaps(claim, ®ion) - }) + // Each relay presents its scope to the client's interest. + let expected: BTreeSet = SCOPES + .iter() + .map(|scope| p(scope)) + .filter_map(|scope| { + if within(&scope, &watch) { + Some(scope) + } else if within(&watch, &scope) { + Some(watch.clone()) + } else { + None + } }) + .flat_map(|region| sim.presented(7, ®ion, rev).into_keys()) .collect(); let id = qualified("local", 7); let shown: BTreeSet = sim @@ -1212,25 +1246,23 @@ fn settled(sim: &mut Sim, label: &str) { sim.check().unwrap(); let live: Vec = (RELAYS..RELAYS + 2).map(|origin| sim.proven(origin)).collect(); for node in 0..RELAYS + 3 { - let mut expected = BTreeSet::new(); + let mut expected = BTreeMap::new(); for &id in &live { assert!( sim.route(node, id).is_some(), "{label}: node {node} has no route to {id}" ); - for claim in CLAIMS { - if overlaps(&p(claim), sim.watching(node)) && sim.truth(id, claim, sim.revision(id)).is_some() { - expected.insert((id, p(claim))); - } + for (claim, item) in sim.presented(id, sim.watching(node), sim.revision(id)) { + expected.insert((id, claim), item); } } - let shown: BTreeSet<(Id, Path)> = sim.shown(node).keys().cloned().collect(); - assert_eq!(shown, expected, "{label}: node {node}"); + let shown: BTreeSet<&(Id, Path)> = sim.shown(node).keys().collect(); + assert_eq!(shown, expected.keys().collect(), "{label}: node {node}"); for ((id, claim), shown) in sim.shown(node) { assert!(shown.available, "{label}: node {node} {id} {claim}"); assert_eq!( - sim.truth(*id, claim.as_str(), sim.revision(*id)), - Some(shown.item), + expected.get(&(*id, claim.clone())), + Some(&shown.item), "{label}: node {node} {id} {claim}" ); } @@ -1324,14 +1356,13 @@ fn a_restarted_route_sends_a_complete_snapshot() { sim.check().unwrap(); } -/// A claim covering both interests stays shown when a client moves to a -/// disjoint interest: what it knows of that claim is current as of the old -/// interest, which carried every change to it until the switch. A withdrawal -/// still queued on the old stream is discarded with it and arrives on the -/// new stream's snapshot instead: the claim ends once, never revives, and no -/// END is false. +/// A claim covering two disjoint interests is presented at each one's own +/// path, so a client moving between them stops watching the old presentation +/// at once. A withdrawal still queued on the old stream is discarded with it, +/// and the new stream's snapshot never shows the withdrawn claim: nothing +/// revives and no END is false. #[test] -fn a_covering_claim_survives_a_disjoint_interest_until_the_new_snapshot() { +fn a_covering_claim_moves_with_a_disjoint_interest() { let (origin, relay, client) = (0, 1, 2); let mut sim = Sim::new(3); sim.leaf(origin); @@ -1342,19 +1373,19 @@ fn a_covering_claim_survives_a_disjoint_interest_until_the_new_snapshot() { sim.originate(origin, 7, 0); sim.announce(origin, "live", 0, None); assert!(sim.run()); - assert_eq!(sim.shows(client, 7, "live"), Some(true)); + assert_eq!(sim.shows(client, 7, "live/a"), Some(true)); sim.hold(relay, client); sim.unannounce(origin, "live"); assert!(sim.run()); sim.watch(client, "live/b"); assert!(sim.run()); - assert_eq!(sim.shows(client, 7, "live"), Some(true), "lags like any snapshot"); + assert!(sim.shown(client).is_empty(), "nothing answers the new interest yet"); sim.release(relay, client); assert!(sim.run()); - assert_eq!(sim.shows(client, 7, "live"), None); - assert_eq!(life(&sim, client, "live"), [Kind::Start, Kind::End]); + assert!(sim.shown(client).is_empty()); + assert_eq!(life(&sim, client, "live/a"), [Kind::Start, Kind::Unwatched]); assert_eq!(revivals(&sim.events), 0); assert_eq!(false_ends(&sim.events), 0); sim.check().unwrap(); diff --git a/rs/moq-net/src/route/tests/mod.rs b/rs/moq-net/src/route/tests/mod.rs index 7c46f61cfd5..93d8fc9b5c3 100644 --- a/rs/moq-net/src/route/tests/mod.rs +++ b/rs/moq-net/src/route/tests/mod.rs @@ -97,7 +97,7 @@ enum Msg { Batch { id: Id, changes: Vec<(Path, Option)>, - floors: BTreeMap, + floor: Option, interest: u32, }, Request { @@ -387,9 +387,9 @@ impl Sim { Msg::Batch { id, changes, - floors, + floor, interest, - } => router.batch(from, id, changes, floors, interest), + } => router.batch(from, id, changes, floor, interest), Msg::Request { id, seqno } => router.request(from, id, seqno), } self.stats.delivered += 1; @@ -468,12 +468,7 @@ impl Sim { self.stats.route_msgs += 1; (to, Msg::Route { id, reach }) } - Out::Batch { - to, - id, - changes, - floors, - } => { + Out::Batch { to, id, changes, floor } => { self.stats.claim_msgs += 1; self.stats.claim_items += changes.len(); let interest = self.nodes[to].interest(); @@ -482,7 +477,7 @@ impl Sim { Msg::Batch { id, changes, - floors, + floor, interest, }, ) @@ -526,7 +521,7 @@ impl Sim { let origin = self.source(event.id).unwrap_or(event.id); let since = event.item.since; let withdrawn = self - .truth(origin, event.claim.as_str(), self.revision(origin)) + .covering(origin, &event.claim, self.revision(origin)) .is_none_or(|item| item.since != since); self.events.push(SimEvent { node, @@ -618,6 +613,39 @@ impl Sim { .and_then(|change| change.item) } + /// The origin's most specific claim at or above `claim` as of revision `rev`: + /// what a stream answering `claim` presents there. + pub fn covering(&self, id: Id, claim: &Path, rev: Rev) -> Option { + let history = self.history.get(&id)?; + let ancestors: BTreeSet<&Path> = history + .iter() + .map(|change| &change.claim) + .filter(|known| within(claim, known)) + .collect(); + ancestors + .into_iter() + .rev() + .filter_map(|known| self.truth(id, known.as_str(), rev)) + .next() + } + + /// The origin's claims as of revision `rev` as a stream answering `prefix` + /// presents them. + pub fn presented(&self, id: Id, prefix: &Path, rev: Rev) -> BTreeMap { + let claims: BTreeSet<&Path> = (self.history.get(&id).into_iter().flatten()) + .map(|change| &change.claim) + .filter(|claim| within(claim, prefix)) + .collect(); + let mut out: BTreeMap = claims + .into_iter() + .filter_map(|claim| Some((claim.clone(), self.truth(id, claim.as_str(), rev)?))) + .collect(); + if let Some(item) = self.covering(id, prefix, rev) { + out.entry(prefix.clone()).or_insert(item); + } + out + } + pub fn revision(&self, id: Id) -> Rev { self.history .get(&id) @@ -635,10 +663,11 @@ impl Sim { continue; }; checked.insert((node, key)); - let claims: BTreeSet<&Path> = self.history[&id].iter().map(|change| &change.claim).collect(); - if let Some(claim) = view.items.keys().find(|claim| !claims.contains(claim)) { - return Err(format!("node {node} holds {key}'s {claim} with no origin history")); - } + // A floor's region presents the most specific claim covering it. + let claims: BTreeSet<&Path> = (self.history[&id].iter().map(|change| &change.claim)) + .chain(view.items.keys()) + .chain(view.floors.keys()) + .collect(); for claim in claims { let Some(rev) = view.as_of(claim) else { if view.items.contains_key(claim) { @@ -647,7 +676,10 @@ impl Sim { continue; }; let known = view.items.get(claim).copied(); - let truth = self.truth(id, claim.as_str(), rev); + let truth = match view.floors.contains_key(claim) { + true => self.covering(id, claim, rev), + false => self.truth(id, claim.as_str(), rev), + }; if known != truth { return Err(format!( "node {node} holds {key}'s {claim} as {known:?} at revision {rev}, where the origin had {truth:?}" diff --git a/rs/moq-net/src/route/tests/routing.rs b/rs/moq-net/src/route/tests/routing.rs index 01316c3c763..cbb7e9af7b9 100644 --- a/rs/moq-net/src/route/tests/routing.rs +++ b/rs/moq-net/src/route/tests/routing.rs @@ -844,7 +844,6 @@ fn an_overflowing_route_neither_certifies_nor_holds() { down_only: false, })) }; - let floors = |region: &str| BTreeMap::from([(p(region), 1)]); let item = Item { seed: 0, epoch: None, @@ -859,10 +858,11 @@ fn an_overflowing_route_neither_certifies_nor_holds() { let mut router: Router = Router::new(1); router.connect(near, Link::default()).unwrap(); router.connect(far, far_link.clone()).unwrap(); + router.asks(near, p("live/a")).unwrap(); router.route(near, 7, finite(1)); - router.batch(near, 7, vec![(p("live/a/x"), Some(item))], floors("live/a"), 0); + router.batch(near, 7, vec![(p("live/a/x"), Some(item))], Some(1), 0); router.route(far, 7, finite(1)); - router.batch(far, 7, vec![(p("live/b/y"), Some(item))], floors(""), 0); + router.batch(far, 7, vec![(p("live/b/y"), Some(item))], Some(1), 0); assert_eq!(router.selected(7).and_then(|route| route.next), Some(near)); assert_eq!(router.shows(7, &p("live/a/x")), Some(true)); assert_eq!( @@ -874,9 +874,47 @@ fn an_overflowing_route_neither_certifies_nor_holds() { let mut router: Router = Router::new(1); router.connect(far, far_link).unwrap(); router.route(far, 7, finite(1)); - router.batch(far, 7, vec![(p("live/x"), Some(item))], floors(""), 0); + router.batch(far, 7, vec![(p("live/x"), Some(item))], Some(1), 0); router.route(far, 7, Some(Reach::Held)); assert!(router.selected(7).is_none()); assert!(!router.grounded(7), "an unreachable held route recovers nothing"); assert_eq!(router.shows(7, &p("live/x")), None); } + +/// A fresh claim arrives only on a stream that certifies nothing, while an +/// older floor on another stream proves it absent. The merge rule trusts the +/// floor, so the claim stays hidden until the certifying neighbor refreshes +/// past it or lets go: that is the bound on how late an uncertified claim +/// shows. +#[test] +fn a_fresh_uncertified_claim_waits_on_an_older_floor() { + let finite = |metric| { + Some(Reach::Finite(Advert { + seqno: 0, + metric, + down_only: false, + })) + }; + let item = Item { + seed: 0, + epoch: None, + since: 2, + }; + let (certifier, carrier) = (1, 2); + for refresh in [true, false] { + let mut router: Router = Router::new(1); + router.connect(certifier, Link::default()).unwrap(); + router.connect(carrier, Link::default()).unwrap(); + router.route(certifier, 7, finite(1)); + router.batch(certifier, 7, Vec::new(), Some(1), 0); + router.route(carrier, 7, finite(1)); + router.batch(carrier, 7, vec![(p("live/x"), Some(item))], None, 0); + assert_eq!(router.shows(7, &p("live/x")), None, "the older floor hides it"); + + match refresh { + true => router.batch(certifier, 7, vec![(p("live/x"), Some(item))], Some(2), 0), + false => router.disconnect(certifier), + } + assert_eq!(router.shows(7, &p("live/x")), Some(true), "refresh {refresh}"); + } +} diff --git a/rs/moq-net/src/route/view.rs b/rs/moq-net/src/route/view.rs index e11b44a6d88..b57288be05d 100644 --- a/rs/moq-net/src/route/view.rs +++ b/rs/moq-net/src/route/view.rs @@ -11,18 +11,14 @@ pub(crate) fn within(path: &Path, prefix: &Path) -> bool { path.has_prefix(prefix) } -/// Whether a claim on `claim` can concern a path under `region`. -/// -/// Every claim covers its own path and everything below it, so a claim meets a -/// region when either lies within the other. -pub(crate) fn overlaps(claim: &Path, region: &Path) -> bool { - within(claim, region) || within(region, claim) -} - /// What a node knows of one origin's claims. /// -/// Each region's floor certifies that the claims overlapping it are exactly the -/// origin's claims there as of that revision. +/// Each region's floor certifies that the claims within it are exactly the +/// origin's claims there as of that revision, as a stream answering that region +/// presents them: every claim inside the region at its own path, and the most +/// specific one above it at the region itself (see [`View::present`]). A claim +/// within no floor is uncertified: some stream carries it, and nothing proves it +/// current. #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct View { pub floors: BTreeMap, @@ -39,32 +35,70 @@ impl Default for View { } impl View { - /// The view as a stream to a peer interested in `prefix` carries it. + /// The view as a stream answering `prefix` presents it. /// - /// A region meeting the interest is narrowed to whichever of the two lies - /// within the other, so every region a stream carries lies within its - /// interest. - pub fn restrict(&self, prefix: &Path) -> Self { - let mut floors: BTreeMap = BTreeMap::new(); - for (region, &floor) in &self.floors { - let narrowed = if within(region, prefix) { - region - } else if within(prefix, region) { - prefix - } else { - continue; - }; - let entry = floors.entry(narrowed.clone()).or_insert(floor); - *entry = (*entry).max(floor); + /// Every claim inside the prefix keeps its path, and of the claims above it only + /// the most specific remains, presented at the prefix itself: a receiver can tell + /// apart no claims that cover the whole prefix alike. A region meeting the prefix + /// is narrowed to whichever of the two lies within the other. + /// + /// The newest floor over the prefix certifies what covers it there: a claim + /// between the two, else the one that floor presents at its own region, else + /// none. A claim above that floor is older knowledge, or uncertified, so it is + /// not presented. + pub fn present(&self, prefix: &Path) -> Self { + let mut out = Self { + floors: narrow(&self.floors, prefix), + items: BTreeMap::new(), + }; + let mut covering: Option<(&Path, &A)> = None; + for (claim, item) in &self.items { + if within(claim, prefix) { + out.items.insert(claim.clone(), item.clone()); + } + if within(prefix, claim) && covering.is_none_or(|(best, _)| claim.len() > best.len()) { + covering = Some((claim, item)); + } } - // A claim stays only while a kept floor still certifies it. - let items = self - .items - .iter() - .filter(|(claim, _)| floors.keys().any(|region| overlaps(claim, region))) - .map(|(claim, item)| (claim.clone(), item.clone())) - .collect(); - Self { floors, items } + if let Some(&floor) = out.floors.get(prefix) { + let certifier = self + .floors + .iter() + .filter(|&(region, &rev)| rev == floor && within(prefix, region)) + .map(|(region, _)| region) + .max_by_key(|region| region.len()); + if certifier.is_some_and(|certifier| covering.is_some_and(|(claim, _)| !within(claim, certifier))) { + covering = None; + } + } + if let Some((_, item)) = covering { + out.items.entry(prefix.clone()).or_insert_with(|| item.clone()); + } + out + } + + /// The view presented to the union of `regions`. + pub fn present_all(&self, regions: &[Path]) -> Self { + let mut out = Self::default(); + for region in regions { + let presented = self.present(region); + out.items.extend(presented.items); + for (region, floor) in presented.floors { + let entry = out.floors.entry(region).or_insert(floor); + *entry = (*entry).max(floor); + } + } + out + } + + /// What the view certifies within `prefix`: the view as presented there, less + /// the claims no kept floor certifies. + pub fn restrict(&self, prefix: &Path) -> Self { + let mut out = self.present(prefix); + let floors = &out.floors; + out.items + .retain(|claim, _| floors.keys().any(|region| within(claim, region))); + out } /// The view restricted to the union of `regions`. @@ -76,21 +110,40 @@ impl View { out } - /// The newest revision this view knows `claim` at, if any region covers it. + /// The newest revision this view knows `claim` at, if a region certifies it. pub fn as_of(&self, claim: &Path) -> Option { self.floors .iter() - .filter(|(region, _)| overlaps(claim, region)) + .filter(|(region, _)| within(claim, region)) .map(|(_, floor)| *floor) .max() } + /// The claims no floor certifies. + pub fn uncertified(&self) -> impl Iterator { + self.items.iter().filter(|(claim, _)| self.as_of(claim).is_none()) + } + + /// The single revision the view certifies all of `prefix` at, if one floor + /// covers it and nothing within it is known at another revision. + pub fn floor(&self, prefix: &Path) -> Option { + match self.floors.iter().next() { + Some((region, &rev)) if self.floors.len() == 1 && region == prefix => Some(rev), + _ => None, + } + } + /// Take `other`'s state for every claim it knows at a newer revision, then /// raise the floors and drop any region a wider one at least as new covers. + /// An uncertified claim of `other` is known at no revision, so it is not taken. pub fn merge(&mut self, other: &Self) { let claims: BTreeSet = self.items.keys().chain(other.items.keys()).cloned().collect(); for claim in claims { - if other.as_of(&claim) > self.as_of(&claim) { + let (theirs, ours) = (other.as_of(&claim), self.as_of(&claim)); + // At one revision, a claim exactly on a wider region outranks the claim + // above it that a floor on this path presented here. + let exact = theirs == ours && self.floors.contains_key(&claim) && !other.floors.contains_key(&claim); + if theirs > ours || exact { match other.items.get(&claim) { Some(item) => self.items.insert(claim, item.clone()), None => self.items.remove(&claim), @@ -123,7 +176,7 @@ impl View { if Some(floor) > newest && !kept.floors.contains_key(region) { kept.floors.insert(region.clone(), floor); for (claim, item) in &self.items { - if overlaps(claim, region) { + if within(claim, region) { kept.items.insert(claim.clone(), item.clone()); } } @@ -150,8 +203,7 @@ impl View { out } - /// Apply a batch: its changes, then its floor list, which replaces the - /// last one. + /// Apply a batch: its changes, then its floors, which replace the last. pub fn apply(&mut self, changes: Vec<(Path, Option)>, floors: BTreeMap) { for (claim, item) in changes { match item { @@ -163,6 +215,24 @@ impl View { } } +/// `floors` narrowed to `prefix`: a region meeting it becomes whichever of the two +/// lies within the other, keeping the newest floor where two narrow alike. +fn narrow(floors: &BTreeMap, prefix: &Path) -> BTreeMap { + let mut out: BTreeMap = BTreeMap::new(); + for (region, &floor) in floors { + let narrowed = if within(region, prefix) { + region + } else if within(prefix, region) { + prefix + } else { + continue; + }; + let entry = out.entry(narrowed.clone()).or_insert(floor); + *entry = (*entry).max(floor); + } + out +} + #[cfg(test)] mod tests { use super::*; @@ -179,10 +249,54 @@ mod tests { } #[test] - fn restrict_narrows_regions_to_the_interest() { - let v = view(&[("", 5)], &[("live/a", 1), ("live", 2), ("other/x", 3)]); - let r = v.restrict(&p("live/a")); - assert_eq!(r, view(&[("live/a", 5)], &[("live/a", 1), ("live", 2)])); + fn present_narrows_regions_and_moves_the_covering_claim_onto_the_interest() { + let v = view(&[("", 5)], &[("live/a/x", 1), ("live", 2), ("other/x", 3)]); + let r = v.present(&p("live/a")); + assert_eq!(r, view(&[("live/a", 5)], &[("live/a/x", 1), ("live/a", 2)])); + assert_eq!(r.floor(&p("live/a")), Some(5)); + } + + #[test] + fn present_keeps_only_the_most_specific_covering_claim() { + let v = view(&[("", 5)], &[("", 9), ("live", 10), ("live/a", 1)]); + assert_eq!(v.present(&p("live/a/x")), view(&[("live/a/x", 5)], &[("live/a/x", 1)])); + // A claim at the interest itself is the most specific there is. + let v = view(&[("", 5)], &[("live", 10), ("live/a", 1)]); + assert_eq!(v.present(&p("live/a")), view(&[("live/a", 5)], &[("live/a", 1)])); + } + + #[test] + fn restrict_moves_a_certified_covering_claim_and_drops_an_uncertified_one() { + let v = view(&[("", 5)], &[("live/a/x", 1), ("live", 2)]); + assert_eq!( + v.restrict(&p("live/a")), + view(&[("live/a", 5)], &[("live/a", 2), ("live/a/x", 1)]) + ); + // The floor proves no claim covers its region, so the claim above is stale. + let v = view(&[("live/a", 5)], &[("live/a/x", 1), ("live", 2)]); + assert_eq!(v.restrict(&p("live/a/y")), view(&[("live/a/y", 5)], &[])); + } + + #[test] + fn a_wider_floor_at_one_revision_replaces_a_claim_moved_onto_a_narrower_one() { + let mut a = view(&[("live/b/z", 5)], &[("live/b/z", 1)]); + a.merge(&view(&[("live/b", 5)], &[("live/b", 1)])); + assert_eq!(a, view(&[("live/b", 5)], &[("live/b", 1)])); + } + + #[test] + fn a_floor_certifies_only_what_lies_within_it() { + let v = view(&[("live/a", 5)], &[("live/a", 1), ("live/a/x", 2), ("live", 3)]); + assert_eq!(v.as_of(&p("live/a")), Some(5)); + assert_eq!(v.as_of(&p("live/a/x")), Some(5)); + assert_eq!(v.as_of(&p("live")), None); + assert_eq!( + v.uncertified().map(|(claim, _)| claim.as_str()).collect::>(), + ["live"] + ); + // Two floors certify no single revision. + let v = view(&[("live", 5), ("live/a", 6)], &[]); + assert_eq!(v.floor(&p("live")), None); } #[test] From 62250733f3226ed2469bfafc3b771e32a0ed32e0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 23:25:21 -0800 Subject: [PATCH 03/40] feat(net): one route node per origin, fed the routes it serves itself The origin keeps a route layer node its lite-07 sessions will share, created on first use with a driver task that announces every route the origin serves itself. A route the route layer selected names its origin node, which a reader can filter on and which ranks routes by the rendezvous hash of the path and that node. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/auth.rs | 41 ++++ rs/moq-net/src/model/mod.rs | 1 + rs/moq-net/src/model/origin.rs | 163 ++++++++++++- rs/moq-net/src/route/mod.rs | 14 +- rs/moq-net/src/route/node.rs | 411 +++++++++++++++++++++++++++++++++ 5 files changed, 623 insertions(+), 7 deletions(-) create mode 100644 rs/moq-net/src/route/node.rs diff --git a/rs/moq-net/src/auth.rs b/rs/moq-net/src/auth.rs index 35b6955bf14..3cd7c038974 100644 --- a/rs/moq-net/src/auth.rs +++ b/rs/moq-net/src/auth.rs @@ -543,6 +543,36 @@ impl Permit { self.limit.as_ref().is_none_or(|limit| limit.matches(path)) } + /// Whether this allows every path at or beneath `prefix`, judged pattern by + /// pattern, so a `false` may be spurious but a `true` never is. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn covers_subtree(&self, prefix: &str) -> bool { + let Ok(subtree) = moq_pattern::Pattern::subtree(prefix) else { + return false; + }; + let covers = |patterns: &Option| patterns.as_ref().is_none_or(|patterns| patterns.contains(&subtree)); + covers(&self.granted) && covers(&self.limit) + } + + /// The prefixes this confines a peer to, or `None` when it allows every path: + /// the heads of the grant's patterns, narrowed by the limit's. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn regions(&self) -> Option> { + if self.covers_subtree("") { + return None; + } + let heads = |patterns: &Option| patterns.as_ref().map(crate::model::interest_prefixes); + let regions = match (heads(&self.granted), heads(&self.limit)) { + (Some(granted), Some(limit)) => granted + .iter() + .flat_map(|a| limit.iter().filter_map(move |b| narrower(a, b))) + .collect(), + (Some(regions), None) | (None, Some(regions)) => regions, + (None, None) => return None, + }; + Some(regions) + } + /// Whether this allows everything `other` does, judged part by part, so a `false` /// may be spurious but a `true` never is. pub(crate) fn covers(&self, other: &Self) -> bool { @@ -555,6 +585,17 @@ impl Permit { } } +/// Whichever of two prefixes lies within the other, or `None` when they are disjoint. +fn narrower(a: &crate::Path, b: &crate::Path) -> Option { + if a.has_prefix(b) { + Some(a.clone()) + } else if b.has_prefix(a) { + Some(b.clone()) + } else { + None + } +} + /// The tokens this side presents and the grant they earn, across every session built on it. /// /// A session made without one creates its own, returned by `Session::auth`, which ends diff --git a/rs/moq-net/src/model/mod.rs b/rs/moq-net/src/model/mod.rs index b010421a7b5..d80baef7c0f 100644 --- a/rs/moq-net/src/model/mod.rs +++ b/rs/moq-net/src/model/mod.rs @@ -56,6 +56,7 @@ pub use timed::Timed; pub mod origin { pub use super::follow::Following; pub(crate) use super::origin_impl::HiddenWithheld; + pub(crate) use super::origin_impl::NodeFilter; #[cfg(feature = "async")] pub(crate) use super::origin_impl::HiddenWithholding; #[cfg(feature = "async")] diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 7949c6ef0e0..ccedcb9f912 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -517,6 +517,12 @@ pub struct RouteInfo { /// Where the route entered this origin; see [`Self::source`]. Never forwarded. pub(crate) source: Source, + + /// The origin node that serves the route, for a route the lite-07 route layer + /// selected: a request resolved through it names that node to the next hop. + /// `None` for a route this node serves itself, published here or learned from a + /// peer that names no node. + pub(crate) node: Option, } impl Default for RouteInfo { @@ -529,6 +535,7 @@ impl Default for RouteInfo { auth: None, via: Hop::UNKNOWN, source: Source::Local, + node: None, } } } @@ -612,6 +619,18 @@ impl RouteInfo { self } + /// The origin node a route-layer route serves; see [`Self::node`]. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn with_node(mut self, node: Option) -> Self { + self.node = node; + self + } + + /// The origin node serving this route, when the route layer selected it. + pub(crate) fn node(&self) -> Option { + self.node + } + /// Whether this route passed through an anonymous hop. /// /// True when the chain holds a 0 anywhere, including Lite03 hop-count @@ -794,9 +813,11 @@ fn route_order(path: &Path, entry: &RouteEntry) -> RouteOrder { entry.cost, !entry.local, entry.hops.len(), - match entry.link == Link::Local && entry.prefix == *path { - true => (0, 0), - false => ( + match (entry.node, entry.link == Link::Local && entry.prefix == *path) { + // Every node that knows the claim ranks its origin node alike. + (Some(node), _) => (fnv_key(path.as_str(), node.to_le_bytes()), 0), + (None, true) => (0, 0), + (None, false) => ( instance_key(path.as_str(), &entry.instance), chain_key(path.as_str(), &entry.hops), ), @@ -808,9 +829,9 @@ fn route_order(path: &Path, entry: &RouteEntry) -> RouteOrder { /// The key [`route_order`] ranks an entry by: lower wins. type RouteOrder = (Reverse, bool, Cost, bool, usize, (u64, u64), Reverse); -/// The `(hops, cost, source, instance, epoch, auth)` metadata an announce cursor delivers +/// The `(hops, cost, source, instance, epoch, auth, node)` metadata an announce cursor delivers /// alongside a prefix. -type RouteMeta = (Hops, Cost, Source, Instance, u64, Option); +type RouteMeta = (Hops, Cost, Source, Instance, u64, Option, Option); /// The earlier of two optional deadlines. fn earliest(a: Option, b: Option) -> Option { @@ -1289,6 +1310,7 @@ impl OriginConsumerState { instance: Some(meta.3), epoch: meta.4, auth: meta.5, + node: meta.6, }, }; let event = match kind { @@ -1328,6 +1350,8 @@ struct RouteEntry { local: bool, /// The link the entry arrived on, from the handle that inserted it. link: Link, + /// The origin node serving the route; see [`RouteInfo::node`]. + node: Option, /// The queue requests under this route are served from, when the announcer /// serves content on demand (a prefix [`Route`]). `None` for an advertise-only /// announcement ([`Producer::announce`]) and for a local broadcast. @@ -1402,18 +1426,20 @@ impl RouteEntry { self.instance.clone(), self.epoch, self.auth.clone(), + self.node, ) } /// Whether [`Self::meta`] would equal `meta`, without building it. fn delivers(&self, meta: &RouteMeta) -> bool { - let (hops, cost, source, instance, epoch, auth) = meta; + let (hops, cost, source, instance, epoch, auth, node) = meta; *hops == self.hops && *cost == self.cost && *source == self.entered() && *instance == self.instance && *epoch == self.epoch && *auth == self.auth + && *node == self.node } /// Whether a request for `path` can be served through this entry. A served @@ -1520,6 +1546,31 @@ struct Horizon { /// forwards them ([`Consumer::forwardable`]). Selection then falls back to a /// shorter route instead of withdrawing a prefix an in-cap path still reaches. max_hops: Option, + /// Which origin nodes' routes this reader sees ([`Consumer::with_node`]). + node: NodeFilter, +} + +/// Which origin nodes' routes a reader sees: a request naming a node resolves only +/// through that node's routes. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub(crate) enum NodeFilter { + /// Every route. + #[default] + Any, + /// Only the routes this node serves itself. + Own, + /// Only the routes the route layer selected toward this origin node. + Is(u64), +} + +impl NodeFilter { + fn admits(self, node: Option) -> bool { + match self { + Self::Any => true, + Self::Own => node.is_none(), + Self::Is(id) => node == Some(id), + } + } } impl Horizon { @@ -1528,6 +1579,7 @@ impl Horizon { self.admits_link(entry.link) && entry.visible_to(self.exclude) && self.max_hops.is_none_or(|max| entry.hops.len() < max.get()) + && self.node.admits(entry.node) } /// Whether routes arriving on `link` may be observed through this horizon. @@ -2297,6 +2349,25 @@ impl Producer { self } + /// The rank of the link this handle is given to; see [`Self::with_preference`]. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn preference(&self) -> Preference { + self.preference + } + + /// The link this handle's routes arrive on; see [`Self::with_link`]. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn link(&self) -> Link { + self.link + } + + /// The route layer node every lite-07 session on this origin shares; see + /// [`Consumer::route_node`]. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn route_node(&self) -> crate::route::Node { + self.consume().route_node() + } + /// This origin's construction config. pub fn config(&self) -> Config { let shared = self.shared.lock(); @@ -2885,6 +2956,7 @@ impl Announcing { via, local: self.local, link: self.link, + node: route.node, server: serving.server.clone(), source: serving.source.clone(), advertised: serving.advertised, @@ -3329,6 +3401,7 @@ impl AnnounceProducer { entry.cost = route.cost; entry.auth = route.auth.clone(); entry.via = route.via; + entry.node = route.node; entry.advertised = true; let claim = entry.claim.clone(); if advertising { @@ -3380,6 +3453,7 @@ impl AnnounceProducer { auth: entry.auth.clone(), via: entry.via, source: entry.entered(), + node: entry.node, }) } @@ -3478,6 +3552,38 @@ impl Drop for AnnounceProducer { /// Drives origin lifecycle work and cache expiration with caller-supplied time. /// +/// Feeds a route layer node the routes this node serves itself, best per prefix, +/// for as long as the origin lives. +struct OwnClaims { + routes: Routes, + node: crate::route::Node, +} + +impl kio::Task for OwnClaims { + type Output = (); + + fn poll(&mut self, waiter: &kio::Waiter) -> Poll<()> { + while let Some(event) = ready!(self.routes.poll_next(waiter)) { + match event { + RouteEvent::Start(announce) + | RouteEvent::Update(announce) + | RouteEvent::Restart(announce) + | RouteEvent::Takeover { announce, .. } => { + let route = announce.route; + let claim = route.instance.map(|instance| crate::route::Claim { + instance, + epoch: route.epoch, + cost: route.cost.value(), + }); + self.node.own(announce.prefix, claim); + } + RouteEvent::End(announce) => self.node.own(announce.prefix, None), + } + } + Poll::Ready(()) + } +} + /// Returned by [`Producer::new`]. Poll on external activity or at the deadline /// it returns, supplying nondecreasing instants. RouteInfo changes, track serving, linger, /// failover, and teardown run here; the route table and announce cursors update @@ -5748,6 +5854,10 @@ struct OriginState { // The sessions withholding this origin's hidden routes from their peer. hidden_withheld: HiddenWithheld, + + // The route layer node this origin's lite-07 sessions share; see + // [`Consumer::route_node`]. + route_node: Option, } impl OriginState { @@ -5766,6 +5876,7 @@ impl OriginState { listeners: Vec::new(), next_listener: 0, hidden_withheld: HiddenWithheld::default(), + route_node: None, } } @@ -6951,6 +7062,46 @@ impl Consumer { self } + /// The route layer node every lite-07 session on this origin shares, created on + /// first use along with the task that feeds it the routes this node serves + /// itself: everything in the table the route layer did not select. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn route_node(&self) -> crate::route::Node { + let mut shared = self.shared.lock(); + if let Some(node) = &shared.route_node { + return node.clone(); + } + let node = crate::route::Node::random(); + shared.route_node = Some(node.clone()); + drop(shared); + // The whole table, whatever this handle's scope. + let own = Self { + scope: OriginScope::default(), + root: Path::default(), + horizon: Horizon { + node: NodeFilter::Own, + ..Horizon::default() + }, + covering: false, + ..self.clone() + } + .with_hidden(true); + self.tasks.push(OwnClaims { + routes: own.routes(), + node: node.clone(), + }); + node + } + + /// A clone that sees only the routes `node` admits: a request naming an origin + /// node resolves only through that node's routes, and the route layer's own + /// claims are the routes this node serves itself. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn with_node(mut self, node: NodeFilter) -> Self { + self.horizon.node = node; + self + } + /// A clone whose announcements skip the routes this origin cannot forward: a /// chain already at [`Config::max_hops`] has no room for our hop. Sessions whose /// wire carries the hop path apply this to the routes they advertise. diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index 07f131aee0b..6365df7e286 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -13,6 +13,7 @@ #![cfg_attr(not(test), allow(dead_code, reason = "the lite-07 session does not drive it yet"))] +mod node; mod proof; mod view; @@ -29,6 +30,7 @@ use std::{ use crate::Path; +pub(crate) use node::{Claim, Node}; pub use proof::RouteKey; pub(crate) use proof::{EXPORTER_LABEL, Proof, Transcript}; pub(crate) use view::{View, within}; @@ -298,6 +300,8 @@ pub(crate) struct Router { heard: BTreeMap<(Id, N), Heard>, sent_views: BTreeMap<(Id, N), View>, shown: BTreeMap<(Id, Path), Shown>, + /// The identities recomputed since [`Self::changed`] last took them. + changed: BTreeSet, /// Mints a fresh identity on rotation. rng: u64, /// Per claim this node has shown, how many times it forgot that claim. @@ -340,6 +344,7 @@ impl Router { heard: BTreeMap::new(), sent_views: BTreeMap::new(), shown: BTreeMap::new(), + changed: BTreeSet::new(), rng: seed | 1, #[cfg(test)] forgot: BTreeMap::new(), @@ -350,6 +355,12 @@ impl Router { } } + /// Take the identities recomputed since the last call: their selection or + /// shown claims may have changed. + pub fn changed(&mut self) -> BTreeSet { + std::mem::take(&mut self.changed) + } + /// Take the messages queued for neighbors, in order. pub fn drain(&mut self) -> Vec> { std::mem::take(&mut self.out) @@ -922,7 +933,7 @@ impl Router { /// The identity this node names for `id` in a request to `next`: the one /// `next` advertised, if this node qualified it. - fn name(&self, next: N, id: Id) -> Id { + pub fn name(&self, next: N, id: Id) -> Id { self.filed.get(&(id, next)).copied().unwrap_or(id) } @@ -1154,6 +1165,7 @@ impl Router { { self.touched += ids.len(); } + self.changed.extend(&ids); for id in ids { self.prune(id); self.advertise(id, &peers); diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs new file mode 100644 index 00000000000..b151608c50c --- /dev/null +++ b/rs/moq-net/src/route/node.rs @@ -0,0 +1,411 @@ +//! One origin node's router, shared by every lite-07 announce stream on its origin. +//! +//! Each announce stream is its own neighbor key, one per direction: a stream this +//! node asked for (it receives routes and claims, and sends ROUTE_REQUEST) and a +//! stream it answers (it sends routes and claims, and receives ROUTE_REQUEST and +//! the ROUTE_END echo). Keying by stream lets a replacement stream open before +//! the old one closes, so no route drops in between. +//! +//! The node does no I/O. A stream's session feeds it what the peer sent and takes +//! [`Output`]s from its queue: wire messages to send, and the shown claims it +//! serves into the origin as the next hop of their selected route. A claim moves +//! to another stream's session when its identity's selected route does, and the +//! old session lets go only once the new one serves it, so the origin always +//! holds a route for it. + +use std::{ + collections::{BTreeMap, BTreeSet, HashMap}, + sync::{Arc, Mutex, MutexGuard}, +}; + +use crate::{Instance, Path}; + +use rand::RngExt; + +use super::{Error, Id, Link, Metric, Out, QUALIFIED, Reach, Rev, Router, Seqno}; + +/// One announce stream in one direction. +pub(crate) type Key = u64; + +/// A claim's attributes on the route layer. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Claim { + pub instance: Instance, + pub epoch: u64, + /// The origin's Announce Cost. + pub cost: u64, +} + +/// A shown claim as the origin serves it through the selected route. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Served { + pub claim: Claim, + /// The selected route's metric: a path through the claim costs the Announce + /// Cost plus this. + pub metric: Metric, + /// Whether a usable route serves it; a held one leaves it shown but unavailable. + pub available: bool, +} + +/// What a stream's session acts on, in order. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum Output { + /// Send ROUTE_START, ROUTE_UPDATE, ROUTE_HOLD, or for `None` ROUTE_END. + Route { id: Id, reach: Option }, + /// Send these claim changes for the route, closed by ANNOUNCE_FLOOR. + Batch { + id: Id, + changes: Vec<(Path, Option)>, + floor: Option, + }, + /// Send ROUTE_REQUEST, naming the identity as the peer advertised it. + Request { id: Id, seqno: Seqno }, + /// Echo the peer's ROUTE_END for the identity it named `id`. + Echo { id: Id }, + /// Serve `claim` of `id` into the origin through this stream, or update how it + /// is served. A request forwarded through it names `named`, the identity as + /// the peer advertised it. Confirm with [`Stream::serving`]. + Serve { + id: Id, + named: Id, + claim: Path, + served: Served, + }, + /// Stop serving `claim` of `id` through this stream. + Unserve { id: Id, claim: Path }, +} + +/// The node every lite-07 session of one origin shares. +#[derive(Clone)] +pub(crate) struct Node { + state: Arc>, +} + +impl std::fmt::Debug for Node { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Node").finish_non_exhaustive() + } +} + +struct State { + router: Router, + next_key: Key, + queues: HashMap>, + /// Each shown claim of another identity, and the streams serving it. + homes: BTreeMap<(Id, Path), Home>, +} + +/// Where one shown claim is served. +#[derive(Default)] +struct Home { + /// The stream it is served through, and what it was last sent. + key: Option<(Key, Served)>, + /// Streams that served it before and let go once `key` confirms. + leaving: BTreeSet, + /// Whether `key` confirmed it serves the claim. + confirmed: bool, +} + +impl Node { + /// A node declaring a fresh identity, `seed` keying its rotation. + pub fn new(id: Id, seed: u64) -> Self { + debug_assert!(id != 0 && id & QUALIFIED == 0, "a declared identity is non-zero and unqualified"); + let mut router = Router::new(seed); + router.originate(id, 0); + Self { + state: Arc::new(Mutex::new(State { + router, + next_key: 1, + queues: HashMap::new(), + homes: BTreeMap::new(), + })), + } + } + + /// A node with a random identity. + pub fn random() -> Self { + let mut rng = kio::clock::rng(); + let id = rng.random_range(1..QUALIFIED); + Self::new(id, rng.random()) + } + + fn lock(&self) -> MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(|poison| poison.into_inner()) + } + + /// The identity this node declares, which a rotation replaces. + pub fn declared(&self) -> Id { + self.lock().router.declared().expect("a node always declares an identity") + } + + /// This node's own claim at `path` changed: `None` withdraws it. + pub fn own(&self, path: Path, claim: Option) { + let mut state = self.lock(); + let changed = match claim { + Some(claim) => { + let current = state.router.declared().and_then(|id| state.router.view(id)?.items.get(&path).cloned()); + current.as_ref() != Some(&claim) && state.router.announce(path, claim).is_ok() + } + None => state.router.unannounce(path).is_ok(), + }; + if changed { + state.flush(); + } + } + + /// Register a stream this node asked the peer for under `prefix`. + pub fn receive(&self, link: Link, prefix: Path) -> Result { + self.register(link, |router, key| router.asks(key, prefix)) + } + + /// Register a stream answering the peer's request for `prefix`. + pub fn answer(&self, link: Link, prefix: Path) -> Result { + self.register(link, |router, key| router.peer_interest(key, prefix)) + } + + fn register( + &self, + link: Link, + open: impl FnOnce(&mut Router, Key) -> Result<(), Error>, + ) -> Result { + let mut state = self.lock(); + let key = state.next_key; + state.next_key += 1; + state.router.connect(key, link)?; + let queue = kio::Queue::new(); + state.queues.insert(key, queue.clone()); + if let Err(err) = open(&mut state.router, key) { + state.router.disconnect(key); + state.queues.remove(&key); + return Err(err); + } + // The node's own set of sessions changed. + let _ = state.router.bump(); + state.flush(); + Ok(Stream { + node: self.clone(), + key, + queue, + }) + } + + #[cfg(test)] + pub(crate) fn shown(&self) -> BTreeMap<(Id, Path), (Claim, bool)> { + (self.lock().router.shown().iter()) + .map(|(key, shown)| (key.clone(), (shown.item.clone(), shown.available))) + .collect() + } +} + +/// One announce stream's key on the node. Dropping it disconnects the stream. +pub(crate) struct Stream { + node: Node, + key: Key, + queue: kio::Queue, +} + +impl std::fmt::Debug for Stream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Stream").field("key", &self.key).finish_non_exhaustive() + } +} + +impl Stream { + /// The next output for this stream's session. + pub fn poll_output(&self, waiter: &kio::Waiter) -> std::task::Poll { + match self.queue.poll_pop(waiter) { + std::task::Poll::Ready(Ok(output)) => std::task::Poll::Ready(output), + // The queue closes only once the stream dropped. + _ => std::task::Poll::Pending, + } + } + + /// The next output already queued, without waiting. + pub fn try_output(&self) -> Option { + self.queue.try_pop().ok().flatten() + } + + fn with(&self, f: impl FnOnce(&mut Router, Key) -> R) -> R { + let mut state = self.node.lock(); + let out = f(&mut state.router, self.key); + state.flush(); + out + } + + /// ROUTE_START, ROUTE_UPDATE, ROUTE_HOLD (`Held`), or ROUTE_END (`None`) for + /// the identity the peer names `named`. + pub fn route(&self, named: Id, reach: Option) { + self.with(|router, key| router.route(key, named, reach)); + } + + /// A batch of claim changes for the route the peer names `named`, closed by + /// its floor. + pub fn batch(&self, named: Id, changes: Vec<(Path, Option)>, floor: Option) { + self.with(|router, key| { + let interest = router.interest(); + router.batch(key, named, changes, floor, interest) + }); + } + + /// The peer sent ROUTE_REQUEST for `id`, as this node advertised it. + pub fn request(&self, id: Id, seqno: Seqno) { + self.with(|router, key| router.request(key, id, seqno)); + } + + /// The peer echoed this node's ROUTE_END for `id`. + pub fn echo(&self, id: Id) { + self.with(|router, key| router.echo(key, id)); + } + + /// The peer sent GOAWAY: its routes rank below every other. + pub fn goaway(&self) { + self.with(|router, key| router.goaway(key)); + } + + /// This stream's session now serves `claim` of `id`, so wherever it was served + /// before lets go. + pub fn serving(&self, id: Id, claim: &Path) { + let mut state = self.node.lock(); + let State { homes, queues, .. } = &mut *state; + let Some(home) = homes.get_mut(&(id, claim.clone())) else { + return; + }; + if home.key.as_ref().is_none_or(|(key, _)| *key != self.key) || home.confirmed { + return; + } + home.confirmed = true; + for key in std::mem::take(&mut home.leaving) { + push(queues, key, Output::Unserve { id, claim: claim.clone() }); + } + } +} + +impl Drop for Stream { + fn drop(&mut self) { + self.queue.close(); + let mut state = self.node.lock(); + state.queues.remove(&self.key); + state.router.disconnect(self.key); + for home in state.homes.values_mut() { + home.leaving.remove(&self.key); + if home.key.as_ref().is_some_and(|(key, _)| *key == self.key) { + // Served nowhere until the router moves it: nothing is left here to wait for. + home.key = None; + home.confirmed = false; + } + } + let _ = state.router.bump(); + state.flush(); + } +} + +fn push(queues: &HashMap>, key: Key, output: Output) { + if let Some(queue) = queues.get(&key) { + let _ = queue.try_push(output); + } +} + +impl State { + /// Route the router's messages and shown claims to the streams. + fn flush(&mut self) { + for out in self.router.drain() { + let (key, output) = match out { + Out::Route { to, id, reach } => (to, Output::Route { id, reach }), + Out::Batch { to, id, changes, floor } => (to, Output::Batch { id, changes, floor }), + Out::Request { to, id, seqno } => (to, Output::Request { id, seqno }), + Out::Echo { to, id } => (to, Output::Echo { id }), + }; + push(&self.queues, key, output); + } + // The shown claims below are read whole; the events only say which changed. + self.router.events(); + let changed = self.router.changed(); + let declared = self.router.declared(); + for id in changed { + if Some(id) != declared { + self.home(id); + } + } + } + + /// Serve each shown claim of `id` through the stream its selected route + /// goes through, keeping it where it is while no route is selected. + fn home(&mut self, id: Id) { + let selected = self.router.selected(id); + let claims: Vec<(Path, Claim, bool)> = (self.router.shown().range((id, Path::empty())..)) + .take_while(|((shown, _), _)| *shown == id) + .map(|((_, claim), shown)| (claim.clone(), shown.item.clone(), shown.available)) + .collect(); + let mut gone: BTreeSet = self + .homes + .range((id, Path::empty())..) + .take_while(|((home, _), _)| *home == id) + .map(|((_, claim), _)| claim.clone()) + .collect(); + for (claim, item, available) in claims { + gone.remove(&claim); + let home = self.homes.entry((id, claim.clone())).or_default(); + let current = home.key.as_ref().map(|(key, served)| (*key, served.metric)); + let Some((key, metric)) = selected + .and_then(|route| Some((route.next?, route.metric))) + .or(current) + else { + continue; + }; + let served = Served { + claim: item, + metric, + available, + }; + match &home.key { + Some((at, sent)) if *at == key && *sent == served => continue, + Some((at, _)) if *at == key => {} + Some((at, _)) => { + if home.confirmed { + home.leaving.insert(*at); + } + home.confirmed = false; + } + None => home.confirmed = false, + } + home.key = Some((key, served.clone())); + let named = self.router.name(key, id); + push( + &self.queues, + key, + Output::Serve { + id, + named, + claim, + served, + }, + ); + } + for claim in gone { + let Some(home) = self.homes.remove(&(id, claim.clone())) else { + continue; + }; + for key in home.key.map(|(key, _)| key).into_iter().chain(home.leaving) { + push(&self.queues, key, Output::Unserve { id, claim: claim.clone() }); + } + } + // Nothing left to recover: let the identity go. + let shows = (self.router.shown().range((id, Path::empty())..).next()).is_some_and(|((shown, _), _)| *shown == id); + if !self.router.holds(id) && !shows { + self.router.collect(id); + } + } +} + +/// The lite-06 hop a router identity travels as, so a relay translating its routes +/// for a lite-06 peer recognizes them coming back. +pub(crate) fn hop(id: Id) -> crate::Hop { + // FNV-1a over the identity, folded into the 53 bits a hop id keeps. + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for byte in id.to_le_bytes() { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x0100_0000_01b3); + } + let id = (hash & ((1 << 53) - 1)).max(1); + crate::Hop::new(id).expect("a non-zero hop id") +} From 97419faf5f090d5830d6fb093672fa10416dee3b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 23:26:22 -0800 Subject: [PATCH 04/40] test(net): the route node serves claims through their route's stream Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/node.rs | 136 +++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index b151608c50c..93918d36f48 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -409,3 +409,139 @@ pub(crate) fn hop(id: Id) -> crate::Hop { let id = (hash & ((1 << 53) - 1)).max(1); crate::Hop::new(id).expect("a non-zero hop id") } + +#[cfg(test)] +mod tests { + use super::*; + use crate::route::Advert; + + fn p(path: &'static str) -> Path { + Path::from_static(path) + } + + fn claim(instance: &Instance, cost: u64) -> Claim { + Claim { + instance: instance.clone(), + epoch: 0, + cost, + } + } + + fn outputs(stream: &Stream) -> Vec { + std::iter::from_fn(|| stream.try_output()).collect() + } + + /// Deliver everything `from` would send on the wire to the stream receiving it, + /// returning what else it queued. + fn pump(from: &Stream, to: &Stream) -> Vec { + let mut rest = Vec::new(); + for output in outputs(from) { + match output { + Output::Route { id, reach } => to.route(id, reach), + Output::Batch { id, changes, floor } => to.batch(id, changes, floor), + other => rest.push(other), + } + } + rest + } + + /// The serve commands among a stream's outputs. + fn served(outputs: Vec) -> Vec { + (outputs.into_iter()) + .filter(|output| matches!(output, Output::Serve { .. } | Output::Unserve { .. })) + .collect() + } + + #[test] + fn a_claim_is_served_through_the_stream_its_route_arrived_on() { + let instance = Instance::random(); + let (origin, relay) = (Node::new(1, 1), Node::new(2, 2)); + let answer = origin.answer(Link::default(), Path::empty()).unwrap(); + let receive = relay.receive(Link::default(), Path::empty()).unwrap(); + origin.own(p("live/x"), Some(claim(&instance, 5))); + pump(&answer, &receive); + assert_eq!( + served(outputs(&receive)), + [Output::Serve { + id: 1, + named: 1, + claim: p("live/x"), + served: Served { + claim: claim(&instance, 5), + metric: 1, + available: true, + }, + }] + ); + + origin.own(p("live/x"), None); + pump(&answer, &receive); + assert_eq!( + served(outputs(&receive)), + [Output::Unserve { + id: 1, + claim: p("live/x") + }] + ); + } + + #[test] + fn a_claim_moves_to_a_cheaper_stream_once_that_one_serves_it() { + let instance = Instance::random(); + let origin = Node::new(1, 1); + let relay = Node::new(2, 2); + let far = relay + .receive( + Link { + cost: 3, + ..Link::default() + }, + Path::empty(), + ) + .unwrap(); + let far_answer = origin.answer(Link::default(), Path::empty()).unwrap(); + origin.own(p("live/x"), Some(claim(&instance, 0))); + pump(&far_answer, &far); + assert!(matches!(served(outputs(&far))[..], [Output::Serve { .. }])); + far.serving(1, &p("live/x")); + + let near = relay.receive(Link::default(), Path::empty()).unwrap(); + let near_answer = origin.answer(Link::default(), Path::empty()).unwrap(); + pump(&far_answer, &far); + pump(&near_answer, &near); + assert!(matches!(&served(outputs(&near))[..], [Output::Serve { served, .. }] if served.metric == 1)); + assert!(served(outputs(&far)).is_empty(), "the old stream serves until the new one does"); + near.serving(1, &p("live/x")); + assert_eq!( + served(outputs(&far)), + [Output::Unserve { + id: 1, + claim: p("live/x") + }] + ); + } + + #[test] + fn a_held_route_keeps_serving_where_it_was_but_unavailable() { + let instance = Instance::random(); + let relay = Node::new(2, 2); + let receive = relay.receive(Link::default(), Path::empty()).unwrap(); + let advert = Advert { + seqno: 0, + metric: 0, + down_only: false, + }; + receive.route(1, Some(Reach::Finite(advert))); + receive.batch(1, vec![(p("live/x"), Some(claim(&instance, 0)))], Some(1)); + assert!(matches!(served(outputs(&receive))[..], [Output::Serve { .. }])); + receive.serving(1, &p("live/x")); + + receive.route(1, Some(Reach::Held)); + assert!(matches!( + &served(outputs(&receive))[..], + [Output::Serve { served, .. }] if !served.available + )); + receive.route(1, None); + assert!(matches!(served(outputs(&receive))[..], [Output::Unserve { .. }])); + } +} From 30a653f15874161d994828360993aef61d370b2d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 23:53:59 -0800 Subject: [PATCH 05/40] feat(relay)!: remove cluster.id and refuse zero-cost links Every relay now takes a random Hop ID on each start: `--cluster-id`, `MOQ_CLUSTER_ID`, and `[cluster] id` are gone and refused as unknown settings, matching the route layer's per-process node identity. A cluster link costs at least 1, the price of an unpriced link. A peer priced at 0 (`?cost=0`, or an object `cost` or `egress` of 0) is a configuration error, as is a LAN-advertised `cluster.node` whose cost is 0 or malformed. Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/cluster.md | 19 ++- doc/bin/relay/config.md | 1 - rs/moq-cli/MIGRATE.md | 10 ++ rs/moq-cli/src/args.rs | 4 +- rs/moq-relay/MIGRATE.md | 14 +++ rs/moq-relay/benches/session.rs | 1 - rs/moq-relay/src/cache.rs | 6 +- rs/moq-relay/src/cluster.rs | 109 +++++++----------- rs/moq-relay/src/config.rs | 20 +--- rs/moq-relay/src/settings.rs | 3 - .../tests/integration/cluster_unknown.rs | 10 +- rs/moq-relay/tests/integration/embed.rs | 8 +- .../tests/integration/released_cli.rs | 1 - .../tests/integration/upstream_links.rs | 53 +++++---- 14 files changed, 122 insertions(+), 137 deletions(-) diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index c0f6fc071f2..e1bead6ae27 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -153,9 +153,10 @@ connect = [{ url = "tls://core-a.internal:4443/", upstream = true }] ## Link costs Add `?cost=N` to a peer URL to route by price instead of hop count. An unpriced -link costs 1, which reproduces plain hop counting. Each relay adds the price of -the link an announcement arrived on before forwarding it, so a route's cost is -the sum of what it crossed. +link costs 1, which reproduces plain hop counting, and 1 is also the minimum: a +cost of 0 is refused. Each relay adds the price of the link an announcement +arrived on before forwarding it, so a route's cost is the sum of what it +crossed. Routing prefers the longest covering prefix, then a fully identified hop list over one holding an anonymous hop (0), then the lowest cost, then the shortest @@ -167,7 +168,7 @@ restarts takes over from its lingering old session at once. ```toml [cluster] -connect = ["https://sibling.same-dc/?cost=0", "https://us-east.example.com/?cost=10"] +connect = ["https://sibling.same-dc/?cost=1", "https://us-east.example.com/?cost=10"] ``` The same policy reads as an object, the only form that accepts `token` and @@ -176,7 +177,7 @@ The same policy reads as an object, the only form that accepts `token` and ```toml [cluster] connect = [ - { url = "https://sibling.same-dc/", cost = 0 }, + { url = "https://sibling.same-dc/", cost = 1 }, { url = "https://us-east.example.com/", cost = 10, token = "PEER_JWT" }, ] ``` @@ -253,11 +254,9 @@ decides which relays receive the cluster token. ## Identity Each relay has a Hop ID: the value it adds to a route's hop list for loop -detection and shortest-path routing. It is random on every start, which is fine -for loop detection but makes a restarted relay look like a new node. Set -`cluster.id` to a stable non-zero 64-bit integer to pin it: below 2^53 if older -browser clients decode it, and below 2^62 to keep loop detection over moq-lite-06 -and older links, which carry a larger id as 0. +detection and shortest-path routing. It is random on every start, so a +restarted relay is a new node to the rest of the mesh, and there is no setting +to pin it. ## Failure detection diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index 20b76f970c7..25afa723fda 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -305,7 +305,6 @@ See [Authentication](/bin/relay/auth). connect = ["https://us-east.example.com/?cost=10"] # Peers to dial. ?cost prices the link, or use {url, cost, egress, token, upstream} objects. node = "https://us-west.example.com/" # This relay's own URL. token = "cluster.jwt" # JWT for dials without an inline ?jwt=. -id = 12345 # Stable Hop ID across restarts. route_key = "route.key" # Route key; relays sharing the file fail over as one to scoped peers. max_hops = 32 # Longest accepted/forwarded hop path (1 to 32). diff --git a/rs/moq-cli/MIGRATE.md b/rs/moq-cli/MIGRATE.md index e29b62468e9..3070c7399d2 100644 --- a/rs/moq-cli/MIGRATE.md +++ b/rs/moq-cli/MIGRATE.md @@ -4,6 +4,16 @@ How to update your invocation and config for each breaking change, newest first. ## Unreleased +### `--cluster-id` is removed + +Remove `--cluster-id` and `MOQ_CLUSTER_ID`: `moq` refuses both as unknown settings. +The Hop ID is random on every run and nothing pins it. + +### Zero-cost cluster links are refused + +A `--cluster-connect` or `--cluster-api-url` peer priced at 0 (`?cost=0`, or an object `cost` or `egress` of 0) is refused, as is a `--cluster-node` carrying `?cost=0` under `--cluster-lan`. +A link costs at least 1, the price of an unpriced link: use `?cost=1` for a same-datacenter sibling. + ### The iroh transport is removed ([#7345](https://github.com/moq-dev/moq/pull/7345)) The `iroh` Cargo feature and the `--iroh-*` flags are gone, and a `MOQ_IROH_*` variable stops startup like any other unknown `MOQ_*` variable. diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index 3888d4bccdb..b366ff3357c 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -555,7 +555,7 @@ impl MoqSide { /// The cluster this process publishes and subscribes on, with the driver that /// must run for as long as it serves. Built once; the origin is its origin, whose - /// Hop ID is random per run unless `--cluster-id` names the node. + /// Hop ID is random per run. pub fn cluster(&self) -> anyhow::Result<(moq_relay::cluster::Cluster, moq_relay::cluster::Driver)> { moq_relay::cluster::Cluster::new(moq_relay::cluster::Options::new(self.cluster.clone())) } @@ -2682,7 +2682,7 @@ mod tests { ("--cluster-api-url", "https://api.example/peers", "--cluster-api-url"), ("--cluster-node", "https://self.example", "--cluster-node"), ("--cluster-token", "cluster.jwt", "--cluster-token"), - ("--cluster-id", "1", "--cluster-id"), + ("--cluster-max-hops", "8", "--cluster-max-hops"), ("--cluster-tier", "internal", "--cluster-tier"), ] { let cli = Invocation::try_parse_from(["moq", flag, value, "auth", "generate"]).unwrap(); diff --git a/rs/moq-relay/MIGRATE.md b/rs/moq-relay/MIGRATE.md index 6f06755853d..3823718c8af 100644 --- a/rs/moq-relay/MIGRATE.md +++ b/rs/moq-relay/MIGRATE.md @@ -5,6 +5,20 @@ Rolling a cluster across a release also follows the [upgrade guide](https://moq. ## Unreleased +### `cluster.id` is removed + +Remove `--cluster-id`, `MOQ_CLUSTER_ID`, and `[cluster] id`; in Rust, drop `cluster::Config::id`. +A relay that still sets one stops at startup: the flag, variable, and key are refused as unknown settings. +Each relay picks a random Hop ID on every start and nothing pins it, so a restarted relay is a new node to the mesh. +`Cluster::id()` still reports the one in use. + +### Zero-cost cluster links are refused + +A link costs at least 1, the price of an unpriced link, since routing needs every link to cost something. +A peer priced at 0 (`?cost=0` on its URL, or an object `cost` or `egress` of 0) stops startup in `cluster.connect` and rejects a `cluster.api.url` update, which keeps the last good list. +With LAN discovery on, a `cluster.node` whose `?cost=` is 0 or malformed stops startup, and a LAN peer advertising one is skipped. +Price a same-datacenter sibling at 1, and raise the other links' costs if it must stay relatively cheaper. + ### Shutdown reports deadline cuts separately from end reports `shutdown::Observer::drained()` now returns `shutdown::Drained` instead of `()`. diff --git a/rs/moq-relay/benches/session.rs b/rs/moq-relay/benches/session.rs index bfa7b429a41..c8a7e0923de 100644 --- a/rs/moq-relay/benches/session.rs +++ b/rs/moq-relay/benches/session.rs @@ -76,7 +76,6 @@ impl Backend for Relay { let cache = config.init().unwrap(); let pool = cache.pool.clone(); let mut options = cluster::Options::default(); - options.config.id = Some(id); // The shared sweeps include more announcements than a default ordinary client. options.session.announces = options.peer.announces; let mut stats = stats::Config::default(); diff --git a/rs/moq-relay/src/cache.rs b/rs/moq-relay/src/cache.rs index 2cf25f2ad5c..281a101fd4e 100644 --- a/rs/moq-relay/src/cache.rs +++ b/rs/moq-relay/src/cache.rs @@ -337,12 +337,12 @@ mod tests { async fn setup_failure_stops_governor() { let before = spawned(); - // `--cluster-id 0` is rejected, so the cache is dropped before it is attached. + // `--cluster-max-hops 0` is rejected, so the cache is dropped before it is attached. let cluster = crate::cluster::Config { - id: Some(0), + max_hops: Some(0), ..Default::default() }; - assert!(attach(&governed(), cluster).is_err(), "cluster id 0 is rejected"); + assert!(attach(&governed(), cluster).is_err(), "a hop limit of 0 is rejected"); settle().await; assert_eq!(spawned(), before, "a failed setup should not strand the governor"); diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 86edbb4178f..7e6ff25831e 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -287,6 +287,10 @@ impl DialTarget { "cluster peer sets token alongside a URL ?jwt=; use one or the other" ); } + anyhow::ensure!( + peer.cost != Some(0) && peer.egress != Some(0), + "cluster peer sets a cost or egress of 0; a link costs at least 1, the price of an unpriced link" + ); let cost = peer.cost.or(take_cost(&mut url)?); // An unpriced link costs 1 (see `moq_net::Client::with_cost`), so `egress` // alone is symmetric only at that price. @@ -722,23 +726,6 @@ impl Drop for Held { #[serde(default, deny_unknown_fields)] #[non_exhaustive] pub struct Config { - /// Fixed origin (hop) id for this relay, identifying it in the hop chains - /// carried on each broadcast for loop detection and shortest-path routing. - /// - /// Unset (the default) picks a fresh random id on every start. Set it to give - /// a node a stable identity across restarts. Must be non-zero; 0 errors at - /// startup. Keep it below 2^53 for compatibility with older `@moq/lite` JS - /// clients, which decode hop ids as a `u53` and reject anything larger. A - /// moq-lite-06 or older link carries an id at or above 2^62 as 0, losing loop - /// detection through this relay on that link. - #[usage( - name = "cluster-id", - long = "cluster-id", - env = "MOQ_CLUSTER_ID", - setting = "cluster.id" - )] - pub id: Option, - /// Route key file: the Ed25519 key this relay proves to every moq-lite-07 peer, /// which files the routes this relay sends under it when this relay's grant is scoped. /// @@ -770,11 +757,12 @@ pub struct Config { /// /// A `?cost=N` query param (or object `cost`) prices the link (moq-lite-06+): /// every announcement crossing it adds `N` to its route cost, so routing - /// prefers cheap paths over short ones. Use `0` for a same-datacenter sibling - /// and something large for a metered backbone; an unpriced link costs 1, - /// which reproduces plain hop counting. The param is consumed by this relay, - /// not sent to the peer. Object `egress` defaults to `cost`; anything else is - /// rejected until asymmetric routing lands. + /// prefers cheap paths over short ones. The minimum and the price of an + /// unpriced link is 1, which reproduces plain hop counting: use it for a + /// same-datacenter sibling and something large for a metered backbone. A + /// cost of 0 is refused. The param is consumed by this relay, not sent to the + /// peer. Object `egress` defaults to `cost`; anything else is rejected until + /// asymmetric routing lands. #[usage( name = "cluster-connect", long = "cluster-connect", @@ -1379,8 +1367,9 @@ impl Cluster { /// and stops publishing stats. A stats publisher panic ends the driver like /// its other work. Needs no Tokio runtime. /// - /// Errors if `config.id` is set to 0. An unset id picks a fresh random origin. - /// Errors too if the `peer.quic` receive window does not fit a QUIC varint, + /// The origin takes a fresh random Hop ID on every call. + /// + /// Errors if the `peer.quic` receive window does not fit a QUIC varint, /// `config.tier` is not a valid stats tier, or the stats config is invalid. pub fn new(options: Options) -> anyhow::Result<(Self, Driver)> { let Options { @@ -1390,10 +1379,6 @@ impl Cluster { peer, stats, } = options; - let id = match config.id { - Some(id) => Hop::new(id).context("--cluster-id must be non-zero")?, - None => Hop::random(), - }; let deprecated = config.deprecated(); anyhow::ensure!(deprecated.is_empty(), "{deprecated}"); // Reject a repeated static identity with conflicting policy up front, @@ -1406,7 +1391,7 @@ impl Cluster { crate::configured_tier(config.tier.clone()).context("invalid --cluster-tier")?; peer.quic.validate()?; let mut work = kio::Tasks::new(); - let mut origin_config = origin::Config::new(id); + let mut origin_config = origin::Config::new(Hop::random()); origin_config.route_key = Some(match &config.route_key { Some(path) => (&moq_crypto::sign::SecretKey::load_or_generate(path) .with_context(|| format!("--cluster-route-key {}", path.display()))?) @@ -1432,7 +1417,7 @@ impl Cluster { work.push(Child::new("stats publisher", publisher.run())); let stats = stats_producer.registry().clone(); let nodes = crate::nodes::Nodes::default(); - tracing::info!(hop_id = %origin.hop(), configured = config.id.is_some(), "cluster initialized"); + tracing::info!(hop_id = %origin.hop(), "cluster initialized"); let cluster = Cluster { session_limits: session.resolve(), peer_limits: peer.resolve(), @@ -2316,6 +2301,8 @@ async fn lan_discovery( Only `?cost=` may be advertised; pass credentials with `--cluster-token` instead.", published.join(", ") ); + // Every LAN peer skips an advertised cost that is malformed or 0, so refuse it here. + take_cost(&mut url.clone()).context("invalid --cluster-node")?; config = config.with_node(url); } if let Some(secret) = secret { @@ -2330,7 +2317,9 @@ async fn lan_discovery( /// The param is dial-side configuration, not something the peer reads off the /// URL (it rides SETUP instead), so it is stripped before connecting. An /// unparseable value is an error rather than a silent default: a mispriced link -/// skews routing for every broadcast crossing it. +/// skews routing for every broadcast crossing it. So is 0: route selection needs +/// every link to cost something, or a route could match its own feasibility +/// distance. fn take_cost(url: &mut Url) -> anyhow::Result> { let Some(value) = url .query_pairs() @@ -2341,6 +2330,10 @@ fn take_cost(url: &mut Url) -> anyhow::Result> { let cost: u64 = value .parse() .with_context(|| format!("invalid cost {value:?} on cluster peer URL"))?; + anyhow::ensure!( + cost > 0, + "cost 0 on cluster peer URL; a link costs at least 1, the price of an unpriced link" + ); let remaining: Vec<(String, String)> = url .query_pairs() @@ -2770,8 +2763,8 @@ mod tests { /// silent default. #[test] fn cost_param_is_consumed() { - let mut url = Url::parse("https://peer.example/?jwt=abc&cost=0").unwrap(); - assert_eq!(take_cost(&mut url).unwrap(), Some(0)); + let mut url = Url::parse("https://peer.example/?jwt=abc&cost=1").unwrap(); + assert_eq!(take_cost(&mut url).unwrap(), Some(1)); assert_eq!(url.as_str(), "https://peer.example/?jwt=abc"); let mut url = Url::parse("https://peer.example/").unwrap(); @@ -3356,18 +3349,6 @@ mod tests { assert!(commands.is_empty(), "an invalid list must not reach the dial map"); } - #[tokio::test] - async fn cluster_id_sets_origin() { - for id in [42, u64::MAX] { - let cluster = new_cluster(Config { - id: Some(id), - ..Default::default() - }) - .expect("valid id"); - assert_eq!(cluster.origin.hop().id(), id); - } - } - /// Cache settings land on the one origin serving and stats share. A handle /// cloned at construction stays on that origin. #[tokio::test] @@ -3380,22 +3361,16 @@ mod tests { let cache = cache.init().expect("cache"); let pool = cache.pool.clone(); - let cluster = Cluster::spawn( - Options::new(Config { - id: Some(42), - ..Default::default() - }) - .with_cache(cache) - .with_stats(crate::stats::Config { + let cluster = Cluster::spawn(Options::new(Config::default()).with_cache(cache).with_stats( + crate::stats::Config { enabled: true, node: Some("test".to_string()), ..Default::default() - }), - ) + }, + )) .expect("cluster"); let origin = cluster.origin.clone(); - assert_eq!(origin.hop().id(), 42); assert_eq!(origin.config().cache_duration, duration); assert_eq!(origin.config().pool.expiry(), Some(duration)); @@ -3423,17 +3398,21 @@ mod tests { .unwrap(); } - /// The reserved `cluster.id` 0 is rejected, since it cannot be excluded for loop - /// detection. + /// A link costs at least 1, so a static peer priced at 0 stops startup, in the + /// URL or the object form. #[test] - fn cluster_id_zero_errors() { - let err = new_cluster(Config { - id: Some(0), - ..Default::default() - }) - .err() - .expect("should error"); - assert!(format!("{err}").contains("--cluster-id"), "got: {err}"); + fn zero_cost_peer_errors() { + for peer in [ + Peer::new("https://peer.example/?cost=0"), + Peer::new("https://peer.example/").with_cost(0), + Peer::new("https://peer.example/").with_egress(0), + ] { + let config = Config { + connect: vec![peer.clone()], + ..Default::default() + }; + assert!(new_cluster(config).is_err(), "{peer:?} was accepted"); + } } #[tokio::test] diff --git a/rs/moq-relay/src/config.rs b/rs/moq-relay/src/config.rs index df0c538b970..33cf5f1ee56 100644 --- a/rs/moq-relay/src/config.rs +++ b/rs/moq-relay/src/config.rs @@ -1136,12 +1136,13 @@ ws = false /// An embedder can merge its relay fragment without parsing the binary CLI again. #[test] fn merge_into_relay_fragment() { - if !test_env::Env::without(&["MOQ_CLUSTER_ID"]).isolate() { + if !test_env::Env::without(&["MOQ_CLUSTER_MAX_HOPS"]).isolate() { return; } let (parsed, cli) = - Cli::parse_from_with_settings(&[std::ffi::OsStr::new("--cluster-id"), std::ffi::OsStr::new("9")]).unwrap(); - let file = toml::from_str::("[cluster]\nid = 7\n").unwrap(); + Cli::parse_from_with_settings(&[std::ffi::OsStr::new("--cluster-max-hops"), std::ffi::OsStr::new("9")]) + .unwrap(); + let file = toml::from_str::("[cluster]\nmax_hops = 7\n").unwrap(); let source = moq_tokio::cli::FileSource { path: std::path::Path::new("relay.toml"), value: &file, @@ -1150,8 +1151,8 @@ ws = false config .merge_into(&cli, &usage_rs::config::EnvLayer::from_process(), Some(source)) .unwrap(); - assert_eq!(config.cluster.id, Some(9)); - assert_eq!(config.source("cluster.id"), Some("--cluster-id")); + assert_eq!(config.cluster.max_hops, Some(9)); + assert_eq!(config.source("cluster.max_hops"), Some("--cluster-max-hops")); } /// One source a merged setting is read from, and what it must end up as. @@ -1174,7 +1175,6 @@ ws = false "MOQ_CACHE_DURATION", "MOQ_CONNECT_TIMEOUT", "MOQ_CLUSTER_TIER", - "MOQ_CLUSTER_ID", "MOQ_CLUSTER_MAX_HOPS", "MOQ_QUIC_RECEIVE_WINDOW", "MOQ_CONNECT_TLS_SYSTEM_ROOTS", @@ -1288,14 +1288,6 @@ ws = false check: |config| assert_eq!(config.cluster.tier.as_deref(), Some("region")), source: "tier", }, - Case { - name: "optional-int-from-file", - toml: "[cluster]\nid = 12345\n", - cli: &[], - key: "cluster.id", - check: |config| assert_eq!(config.cluster.id, Some(12345)), - source: "id", - }, Case { name: "hop-limit-from-file", toml: "[cluster]\nmax_hops = 8\n", diff --git a/rs/moq-relay/src/settings.rs b/rs/moq-relay/src/settings.rs index 3b798f655e9..7ab93dc6d04 100644 --- a/rs/moq-relay/src/settings.rs +++ b/rs/moq-relay/src/settings.rs @@ -71,9 +71,6 @@ struct Runtime { #[derive(usage_rs::Config)] #[usage(prefix = "cluster")] struct Cluster { - #[usage(env = "MOQ_CLUSTER_ID", cli("--cluster-id"))] - id: Option, - #[usage(env = "MOQ_CLUSTER_ROUTE_KEY", cli("--cluster-route-key"))] route_key: Option, diff --git a/rs/moq-relay/tests/integration/cluster_unknown.rs b/rs/moq-relay/tests/integration/cluster_unknown.rs index 813e6f068f2..1045ab7944f 100644 --- a/rs/moq-relay/tests/integration/cluster_unknown.rs +++ b/rs/moq-relay/tests/integration/cluster_unknown.rs @@ -28,7 +28,7 @@ struct Node { task: tokio::task::JoinHandle<()>, } -async fn spawn_relay(id: u64, connect: Vec, cluster_version: Option) -> Node { +async fn spawn_relay(connect: Vec, cluster_version: Option) -> Node { let _ = rind_rustls::default_provider().unwrap().install_default(); let mut config = Config::default(); @@ -38,7 +38,6 @@ async fn spawn_relay(id: u64, connect: Vec, cluster_version: Option) -> Self { - let a = spawn_relay(11, vec![], cluster_version).await; - let b = spawn_relay(12, vec![format!("tcp://127.0.0.1:{}", a.port)], cluster_version).await; + let a = spawn_relay(vec![], cluster_version).await; + let b = spawn_relay(vec![format!("tcp://127.0.0.1:{}", a.port)], cluster_version).await; let c = spawn_relay( - 13, vec![ format!("tcp://127.0.0.1:{}", a.port), format!("tcp://127.0.0.1:{}", b.port), @@ -118,7 +116,7 @@ impl Mesh { cluster_version, ) .await; - let d = spawn_relay(14, vec![format!("tcp://127.0.0.1:{}", c.port)], cluster_version).await; + let d = spawn_relay(vec![format!("tcp://127.0.0.1:{}", c.port)], cluster_version).await; // One client for every poll: building one loads the system trust roots. let http = reqwest::Client::new(); for (node, dialed) in [(&b, 1), (&c, 2), (&d, 1)] { diff --git a/rs/moq-relay/tests/integration/embed.rs b/rs/moq-relay/tests/integration/embed.rs index 98580bc7f07..88d65297604 100644 --- a/rs/moq-relay/tests/integration/embed.rs +++ b/rs/moq-relay/tests/integration/embed.rs @@ -557,19 +557,19 @@ struct EmbeddedConfig { fn embedded_cli_merges_only_relay_settings() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("relay.toml"); - std::fs::write(&path, "[cluster]\nid = 7\n[stats]\nenabled = false\n").unwrap(); + std::fs::write(&path, "[cluster]\nmax_hops = 7\n[stats]\nenabled = false\n").unwrap(); let (mut parsed, cli) = EmbeddedConfig::parse_from_with_settings(&[ path.as_os_str(), std::ffi::OsStr::new("--worker-name"), std::ffi::OsStr::new("recorder"), - std::ffi::OsStr::new("--cluster-id"), + std::ffi::OsStr::new("--cluster-max-hops"), std::ffi::OsStr::new("9"), ]) .expect("parse embedding CLI"); parsed.relay.merge(&cli).unwrap(); assert_eq!(parsed.worker_name.as_deref(), Some("recorder")); - assert_eq!(parsed.relay.cluster.id, Some(9)); - assert_eq!(parsed.relay.source("cluster.id"), Some("--cluster-id")); + assert_eq!(parsed.relay.cluster.max_hops, Some(9)); + assert_eq!(parsed.relay.source("cluster.max_hops"), Some("--cluster-max-hops")); assert!(!parsed.relay.stats.enabled); assert!(parsed.relay.source("stats.enabled").unwrap().contains("enabled")); // The explicit file value equals the runtime default but still has file provenance. diff --git a/rs/moq-relay/tests/integration/released_cli.rs b/rs/moq-relay/tests/integration/released_cli.rs index 38a9707554b..262f9aa3acd 100644 --- a/rs/moq-relay/tests/integration/released_cli.rs +++ b/rs/moq-relay/tests/integration/released_cli.rs @@ -54,7 +54,6 @@ const RELEASED: &[(&str, Option<&str>)] = &[ ("client-version", Some("MOQ_CLIENT_VERSION")), ("cluster-connect", Some("MOQ_CLUSTER_CONNECT")), ("cluster-connect-api", Some("MOQ_CLUSTER_CONNECT_API")), - ("cluster-id", Some("MOQ_CLUSTER_ID")), ("cluster-node", Some("MOQ_CLUSTER_NODE")), ("cluster-tier", Some("MOQ_CLUSTER_TIER")), ("cluster-token", Some("MOQ_CLUSTER_TOKEN")), diff --git a/rs/moq-relay/tests/integration/upstream_links.rs b/rs/moq-relay/tests/integration/upstream_links.rs index 71b97ec486a..8f640d90c89 100644 --- a/rs/moq-relay/tests/integration/upstream_links.rs +++ b/rs/moq-relay/tests/integration/upstream_links.rs @@ -52,9 +52,10 @@ where .expect("test thread panicked"); } -/// One relay: its origin, listen port, and every task it runs. Dropping it -/// aborts them all, closing its sessions like a crash. +/// One relay: its Hop ID, origin, listen port, and every task it runs. Dropping +/// it aborts them all, closing its sessions like a crash. struct Node { + id: u64, origin: origin::Producer, port: u16, _tasks: tokio::task::JoinSet<()>, @@ -82,7 +83,7 @@ fn grant(path: &str) -> Grant { const LITE_07: &str = "moq-lite-07-wip"; -async fn node(id: u64, connect: Vec) -> Node { +async fn node(connect: Vec) -> Node { let _ = rind_rustls::default_provider().unwrap().install_default(); let mut tasks = tokio::task::JoinSet::new(); @@ -106,9 +107,9 @@ async fn node(id: u64, connect: Vec) -> Node { client.version = vec![LITE_07.parse().expect("lite-07")]; let mut config = cluster::Config::default(); - config.id = Some(id); config.connect = connect; let cluster = crate::spawn_cluster(cluster::Options::new(config)).with_connect(client, Default::default()); + let id = cluster.id(); let (auth, mut admissions) = auth::Auth::embedded(format!("node-{id}")); tasks.spawn(async move { @@ -135,6 +136,7 @@ async fn node(id: u64, connect: Vec) -> Node { }); Node { + id, origin: cluster.origin.clone(), port, _tasks: tasks, @@ -241,12 +243,6 @@ fn has(routes: &BTreeMap>, paths: &[&str]) -> bool { paths.iter().all(|path| routes.contains_key(*path)) } -const R: u64 = 1; -const C1: u64 = 11; -const C2: u64 = 12; -const E1: u64 = 21; -const E2: u64 = 22; - /// Two regions: two edges whose links to two cores are upstream, and one relay /// that also serves clients. Both edges send a path to the same core, an edge's /// broadcast reaches the far relay, and losing a core moves only its paths. @@ -256,13 +252,15 @@ fn tiered_regions() { } async fn tiered_regions_inner() { - let r = node(R, vec![]).await; - let c1 = node(C1, vec![r.dial(false, false)]).await; - let c2 = node(C2, vec![r.dial(false, false)]).await; - let e1 = node(E1, vec![c1.dial(true, false), c2.dial(true, false)]).await; - let e2 = node(E2, vec![c1.dial(true, false), c2.dial(true, false)]).await; - - let paths: Vec = (0..8).map(|i| format!("live/{i}")).collect(); + let r = node(vec![]).await; + let c1 = node(vec![r.dial(false, false)]).await; + let c2 = node(vec![r.dial(false, false)]).await; + let e1 = node(vec![c1.dial(true, false), c2.dial(true, false)]).await; + let e2 = node(vec![c1.dial(true, false), c2.dial(true, false)]).await; + + // Each path's core is a hash over the random Hop IDs, so enough paths that both + // cores get some: all 32 on one core is a 2^-31 chance. + let paths: Vec = (0..32).map(|i| format!("live/{i}")).collect(); let paths: Vec<&str> = paths.iter().map(String::as_str).collect(); let _published: Vec<_> = paths.iter().map(|path| publish(&r.origin, path)).collect(); routes_when("both cores learn the far region", &c1.origin, |routes| { @@ -298,21 +296,22 @@ async fn tiered_regions_inner() { // Losing a core moves only its paths. let on = |core_id: u64| -> Vec<&str> { paths.iter().copied().filter(|p| core(&at_e1, p) == core_id).collect() }; - let (on_c1, on_c2) = (on(C1), on(C2)); + let (on_c1, on_c2) = (on(c1.id), on(c2.id)); assert!( !on_c1.is_empty() && !on_c2.is_empty(), "paths all on one core: {at_e1:?}" ); + let lost = c1.id; drop(c1); let after = routes_when("edge 1 drops the lost core", &e1.origin, |routes| { - has(routes, &paths) && routes.values().all(|hops| !hops.contains(&C1)) + has(routes, &paths) && routes.values().all(|hops| !hops.contains(&lost)) }) .await; for path in &on_c2 { assert_eq!(after[*path], at_e1[*path], "{path} moved off the surviving core"); } for path in &on_c1 { - assert_eq!(after[*path], [R, C2], "{path} did not fail over"); + assert_eq!(after[*path], [r.id, c2.id], "{path} did not fail over"); } } @@ -324,11 +323,11 @@ fn edges_never_transit_between_cores() { } async fn edges_never_transit_between_cores_inner() { - let c1 = node(C1, vec![]).await; - let e1 = node(E1, vec![c1.dial(true, false)]).await; + let c1 = node(vec![]).await; + let e1 = node(vec![c1.dial(true, false)]).await; // The core dials this edge, which still treats it as upstream. - let c2 = node(C2, vec![e1.dial(false, true)]).await; - let e2 = node(E2, vec![c1.dial(true, false), c2.dial(true, false)]).await; + let c2 = node(vec![e1.dial(false, true)]).await; + let e2 = node(vec![c1.dial(true, false), c2.dial(true, false)]).await; let _core = publish(&c1.origin, "live/c1"); routes_when("edge 1 learns core 1", &e1.origin, |routes| has(routes, &["live/c1"])).await; @@ -358,9 +357,9 @@ fn uplink_transits_both_ways() { } async fn uplink_transits_both_ways_inner() { - let cdn = node(31, vec![]).await; - let uplink = node(32, vec![cdn.dial(true, false)]).await; - let mesh = node(33, vec![uplink.dial(false, false)]).await; + let cdn = node(vec![]).await; + let uplink = node(vec![cdn.dial(true, false)]).await; + let mesh = node(vec![uplink.dial(false, false)]).await; let _mesh = publish(&mesh.origin, "mesh/cam"); let _cdn = publish(&cdn.origin, "cdn/cam"); From 8059e163e41020904abd5c1a0f847ce3bf43c669 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 00:34:58 -0800 Subject: [PATCH 06/40] feat(net)!: lite-07 names a claim's route instead of its hop chain ANNOUNCE_START and ANNOUNCE_TAKEOVER carry the Route ID, ANNOUNCE_UPDATE only the Announce Cost, and ANNOUNCE_RESTART no hops; hop bases are gone. ANNOUNCE_OK has no body and SETUP no Hop parameter on lite-07, a Cost of 0 is refused, and SUBSCRIBE, FETCH, TRACK, TRACKS, and DATAGRAMS carry the Node ID. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/benches/announce.rs | 6 +- rs/moq-net/src/coding/writer.rs | 1 + rs/moq-net/src/fuzz.rs | 29 +- rs/moq-net/src/lite/announce.rs | 352 +++++++++++++------- rs/moq-net/src/lite/compress.rs | 525 ++++++++---------------------- rs/moq-net/src/lite/datagram.rs | 9 + rs/moq-net/src/lite/fetch.rs | 18 + rs/moq-net/src/lite/instance.rs | 17 + rs/moq-net/src/lite/publisher.rs | 54 ++- rs/moq-net/src/lite/setup.rs | 98 ++++-- rs/moq-net/src/lite/subscribe.rs | 18 + rs/moq-net/src/lite/subscriber.rs | 36 +- rs/moq-net/src/lite/track.rs | 29 ++ rs/moq-net/src/lite/tracks.rs | 16 +- rs/moq-net/src/lite/version.rs | 14 +- rs/moq-net/src/path/mod.rs | 1 + rs/moq-net/src/test_interop.rs | 1 + 17 files changed, 639 insertions(+), 585 deletions(-) diff --git a/rs/moq-net/benches/announce.rs b/rs/moq-net/benches/announce.rs index c30a7bbe082..f567100d78b 100644 --- a/rs/moq-net/benches/announce.rs +++ b/rs/moq-net/benches/announce.rs @@ -9,8 +9,10 @@ //! //! - health: `/private/channel_N/stream-health-` from a few origins behind //! a shared pair of relays, the shape that motivated compression. -//! - unique: random single-segment paths and random two-hop chains, where nothing -//! is shared and every START pays the four zero base/keep bytes. +//! - unique: random single-segment paths and random two-hop chains, where no path +//! is shared and every START pays the two zero base/keep bytes. +//! +//! The hop chains travel only on lite-06; lite-07 names each claim's route instead. //! //! The byte counts print once before the timings. //! diff --git a/rs/moq-net/src/coding/writer.rs b/rs/moq-net/src/coding/writer.rs index 01bb04c8da2..71591b52578 100644 --- a/rs/moq-net/src/coding/writer.rs +++ b/rs/moq-net/src/coding/writer.rs @@ -509,6 +509,7 @@ mod tests { .buffer(&lite::Track { broadcast: crate::Path::from_static("test"), instance: None, + node: 0, track: "video".into(), }) .unwrap(); diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index bc08a7261ef..a3c8d81b274 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -359,6 +359,9 @@ pub fn lite_wire(data: &[u8]) -> bool { } /// One announcement on an announce stream, resolved: what the application sees. +/// +/// The hop chain travels only on lite-06: lite-07 names the claim's route instead, so +/// its writer drops the chain and its reader decodes an empty one. #[derive(Clone, Debug, PartialEq, Eq)] pub enum Announced { /// ANNOUNCE_START: a route's suffix and wire hop chain, taking the next id. @@ -397,19 +400,20 @@ impl AnnounceWriter { pub fn write(&mut self, announced: &Announced, data: &mut Vec) { let msg = match announced { Announced::Start(suffix, hops) => { - let (_, suffix, hops) = self.encoder.start(suffix.clone(), hops.clone()); + let (_, suffix) = self.encoder.start(suffix.clone()); // A fixed instance, so the bytes stay pinned and match what `js/net` writes. lite::AnnounceBroadcast::Active { + route: 0, instance: self.version.has_instance().then(|| "a".parse().unwrap()), epoch: 0, suffix, - hops, + hops: self.chain(hops), cost: Default::default(), } } Announced::Update(id, hops) => lite::AnnounceBroadcast::Update { id: *id, - hops: self.encoder.update(*id, hops.clone()), + hops: self.chain(hops), cost: Default::default(), }, Announced::End(id) => { @@ -420,6 +424,14 @@ impl AnnounceWriter { msg.encode(&mut Encoder::new(data, self.version.into()), self.version) .expect("could not encode an announcement"); } + + /// The hop chain as this version carries it: none on lite-07. + fn chain(&self, hops: &Hops) -> Hops { + match self.version.has_routes() { + true => Hops::default(), + false => hops.clone(), + } + } } /// Mark `track` as a copy received from a remote publisher, as a subscriber session does, for @@ -574,12 +586,10 @@ pub fn decode_announces(data: &[u8], compress: bool) -> Vec { let mut resolved = Vec::new(); while let Ok(msg) = lite::AnnounceBroadcast::decode(&mut data, version) { let announced = match msg { - lite::AnnounceBroadcast::Active { suffix, hops, .. } => decoder - .start(suffix, hops) - .map(|(suffix, hops)| Announced::Start(suffix, hops)), - lite::AnnounceBroadcast::Update { id, hops, .. } => { - decoder.update(id, hops).map(|(_, hops)| Announced::Update(id, hops)) + lite::AnnounceBroadcast::Active { suffix, hops, .. } => { + decoder.start(suffix).map(|suffix| Announced::Start(suffix, hops)) } + lite::AnnounceBroadcast::Update { id, hops, .. } => decoder.get(id).map(|_| Announced::Update(id, hops)), lite::AnnounceBroadcast::EndedId { id } => decoder.end(id).map(|_| Announced::End(id)), _ => continue, }; @@ -682,6 +692,7 @@ impl LiteSample { Self::Subscribe => lite::Subscribe { order: crate::track::Order::Descending, instance: None, + node: 0, id: 3, broadcast: Path::from_static("room/alice"), track: "video".into(), @@ -880,6 +891,7 @@ impl Default for Messages { lite_subscribe: lite::Subscribe { order: crate::track::Order::Descending, instance: None, + node: 0, id: 7, broadcast: Path::from_static("room/alice"), track: "video".into(), @@ -1062,6 +1074,7 @@ pub fn encode_subscribe(broadcast: &Path, track: &str, ietf: bool, out: &mut Vec id: 0, broadcast: broadcast.clone(), instance: None, + node: 0, track: track.into(), priority: 0, max_delay: std::time::Duration::ZERO, diff --git a/rs/moq-net/src/lite/announce.rs b/rs/moq-net/src/lite/announce.rs index 8e9d42770b4..baa005bbb97 100644 --- a/rs/moq-net/src/lite/announce.rs +++ b/rs/moq-net/src/lite/announce.rs @@ -59,20 +59,23 @@ pub fn update_supported(version: Version) -> bool { /// repeating the path. Older versions send a single `ANNOUNCE_BROADCAST` /// message that retracts by path (`Ended`). /// -/// The path and hop chain are as they appear on the wire: on lite-07 they may name a -/// base announcement, resolved against the stream's history by -/// [`AnnounceDecoder`](super::AnnounceDecoder). +/// The path is as it appears on the wire: on lite-07 it may name a base announcement, +/// resolved against the stream's history by [`AnnounceDecoder`](super::AnnounceDecoder). +/// +/// Lite-07 names the route each claim travels under in place of a hop chain, so `hops` +/// must be empty there and `route` must be 0 on every earlier version. #[derive(Clone, Debug, PartialEq, Eq)] pub enum AnnounceBroadcast { /// ANNOUNCE_START (lite-06) / active (older): a broadcast is now available. - /// Carries the path suffix, (lite-07+) the instance and epoch it serves, the hop - /// chain, and (lite-06+) the static route cost, and assigns the next announce id. - /// The instance and epoch change only with a `Restart`. + /// Carries the path suffix, (lite-07+) the Route ID and the instance and epoch it + /// serves, (before lite-07) the hop chain, and (lite-06+) the cost, and assigns the + /// next announce id. The instance and epoch change only with a `Restart`. Active { suffix: PathRef, + route: u64, instance: Option, epoch: u64, - hops: HopsRef, + hops: Hops, cost: Cost, }, /// Pre-lite-06: a broadcast is no longer available, retracted by path. @@ -81,9 +84,8 @@ pub enum AnnounceBroadcast { /// announce id. The id is retired; referencing it again is a protocol violation. EndedId { id: u64 }, /// ANNOUNCE_UPDATE (lite-06+): atomically replace the metadata of the announcement - /// with this id (e.g. a new hop chain after a relay failover, or a route whose cost - /// moved). The id stays live. - Update { id: u64, hops: HopsRef, cost: Cost }, + /// with this id: its cost, and before lite-07 its hop chain. The id stays live. + Update { id: u64, hops: Hops, cost: Cost }, /// ANNOUNCE_RESTART (lite-07+): the announcement with this id now serves the /// instance and epoch given. Another instance is another stream, so whatever the /// receiver resolved under its prefix is stale; a higher epoch of the same one @@ -92,7 +94,6 @@ pub enum AnnounceBroadcast { id: u64, instance: Option, epoch: u64, - hops: HopsRef, cost: Cost, }, /// ANNOUNCE_TAKEOVER (lite-07+): an `Active` that continues what the live @@ -101,9 +102,9 @@ pub enum AnnounceBroadcast { Takeover { predecessor: u64, suffix: PathRef, + route: u64, instance: Option, epoch: u64, - hops: HopsRef, cost: Cost, }, /// An unknown lite-06+ announce type. The length-prefixed body was skipped so @@ -216,55 +217,37 @@ impl<'a> Decode<'a, Version> for PathRef { } } -/// A hop chain as it travels: `literal` leading hops, followed by the last `keep` -/// entries of a base announcement's chain. -/// -/// `base` counts back like [`PathRef::base`], independently of it. Only lite-07 -/// carries a base; every other version is `literal` alone. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct HopsRef { - pub base: u64, - pub literal: Hops, - pub keep: u64, +/// Encode the Route ID a claim travels under, which only lite-07 carries. +fn encode_route_id(w: &mut Encoder<'_>, version: Version, route: u64) -> Result<(), EncodeError> { + match version.has_routes() { + true => w.varint(route), + false if route != 0 => Err(EncodeError::InvalidState), + false => Ok(()), + } } -impl HopsRef { - /// The whole chain, with no base. - pub fn literal(literal: Hops) -> Self { - Self { - base: 0, - literal, - keep: 0, - } +/// Decode the Route ID a claim travels under, 0 on a version without one. +fn decode_route_id(r: &mut Decoder<'_>, version: Version) -> Result { + match version.has_routes() { + true => r.varint(), + false => Ok(0), } } -impl Encode for HopsRef { - fn encode(&self, w: &mut Encoder<'_>, version: Version) -> Result<(), EncodeError> { - if !version.has_announce_compression() { - if self.base != 0 || self.keep != 0 { - return Err(EncodeError::Version); - } - return self.literal.encode(w, version); - } - w.varint(self.base)?; - self.literal.encode(w, version)?; - w.varint(self.keep) +/// Encode a lite-06+ announcement's hop chain, which lite-07 replaces with its Route ID. +fn encode_chain(w: &mut Encoder<'_>, version: Version, hops: &Hops) -> Result<(), EncodeError> { + match version.has_routes() { + true if !hops.is_empty() => Err(EncodeError::InvalidState), + true => Ok(()), + false => hops.encode(w, version), } } -impl<'a> Decode<'a, Version> for HopsRef { - fn decode(buf: &mut Decoder<'a>, version: Version) -> Result { - if !version.has_announce_compression() { - return Ok(Self::literal(Hops::decode(buf, version)?)); - } - let base = buf.varint()?; - let literal = Hops::decode(buf, version)?; - let keep = buf.varint()?; - if base == 0 && keep != 0 { - return Err(DecodeError::InvalidValue); - } - Ok(Self { base, literal, keep }) +/// Decode a lite-06+ announcement's hop chain, empty on lite-07. +fn decode_chain(r: &mut Decoder<'_>, version: Version) -> Result { + match version.has_routes() { + true => Ok(Hops::default()), + false => Hops::decode(r, version), } } @@ -337,46 +320,46 @@ impl Encode for AnnounceBroadcast { match self { Self::Active { suffix, + route, instance, epoch, hops, cost, } => { suffix.encode(w, version)?; + encode_route_id(w, version, *route)?; super::instance::encode_route(w, version, instance.as_ref(), *epoch)?; - hops.encode(w, version)?; + encode_chain(w, version, hops)?; cost.encode(w, version)?; } Self::EndedId { id } => w.varint(*id)?, Self::Update { id, hops, cost } => { w.varint(*id)?; - hops.encode(w, version)?; + encode_chain(w, version, hops)?; cost.encode(w, version)?; } Self::Restart { id, instance, epoch, - hops, cost, } => { w.varint(*id)?; super::instance::encode_route(w, version, instance.as_ref(), *epoch)?; - hops.encode(w, version)?; cost.encode(w, version)?; } Self::Takeover { predecessor, suffix, + route, instance, epoch, - hops, cost, } => { w.varint(*predecessor)?; suffix.encode(w, version)?; + encode_route_id(w, version, *route)?; super::instance::encode_route(w, version, instance.as_ref(), *epoch)?; - hops.encode(w, version)?; cost.encode(w, version)?; } Self::Synced => {} @@ -407,14 +390,19 @@ impl Encode for AnnounceBroadcast { let prefix = w.prefix_varint(); match self { // The cost is a lite-06 addition, so it is simply not on the wire here. - Self::Active { suffix, hops, .. } => { - // Bases are a lite-07 addition; PathRef and HopsRef refuse one here. - if suffix.base != 0 || hops.base != 0 { + Self::Active { + suffix, route, hops, .. + } => { + // Bases are a lite-07 addition; PathRef refuses one here. + if suffix.base != 0 { return Err(EncodeError::Version); } + if *route != 0 { + return Err(EncodeError::InvalidState); + } AnnounceStatus::Active.encode(w, version)?; suffix.rest.encode(w, version)?; - encode_hops(w, version, &hops.literal)?; + encode_hops(w, version, hops)?; } Self::Ended { suffix, hops } => { AnnounceStatus::Ended.encode(w, version)?; @@ -456,12 +444,14 @@ impl<'a> Decode<'a, Version> for AnnounceBroadcast { Ok(match typ { ANNOUNCE_START => { let suffix = PathRef::decode(&mut body, version)?; + let route = decode_route_id(&mut body, version)?; let (instance, epoch) = super::instance::decode_route(&mut body, version)?; Self::Active { suffix, + route, instance, epoch, - hops: HopsRef::decode(&mut body, version)?, + hops: decode_chain(&mut body, version)?, cost: Cost::decode(&mut body, version)?, } } @@ -469,7 +459,7 @@ impl<'a> Decode<'a, Version> for AnnounceBroadcast { ANNOUNCE_END => Self::EndedId { id: body.varint()? }, ANNOUNCE_UPDATE => Self::Update { id: body.varint()?, - hops: HopsRef::decode(&mut body, version)?, + hops: decode_chain(&mut body, version)?, cost: Cost::decode(&mut body, version)?, }, ANNOUNCE_RESTART if version.has_announce_restart() => { @@ -479,20 +469,20 @@ impl<'a> Decode<'a, Version> for AnnounceBroadcast { id, instance, epoch, - hops: HopsRef::decode(&mut body, version)?, cost: Cost::decode(&mut body, version)?, } } ANNOUNCE_TAKEOVER if version.has_announce_takeover() => { let predecessor = body.varint()?; let suffix = PathRef::decode(&mut body, version)?; + let route = decode_route_id(&mut body, version)?; let (instance, epoch) = super::instance::decode_route(&mut body, version)?; Self::Takeover { predecessor, suffix, + route, instance, epoch, - hops: HopsRef::decode(&mut body, version)?, cost: Cost::decode(&mut body, version)?, } } @@ -561,9 +551,10 @@ impl AnnounceBroadcast { Ok(match status { AnnounceStatus::Active => Self::Active { suffix: PathRef::literal(suffix), + route: 0, instance: None, epoch: 0, - hops: HopsRef::literal(hops), + hops, cost: Cost::UNKNOWN, }, AnnounceStatus::Ended => Self::Ended { suffix, hops }, @@ -574,9 +565,10 @@ impl AnnounceBroadcast { // invalid value there. AnnounceStatus::Update if update_supported(version) => Self::Active { suffix: PathRef::literal(suffix), + route: 0, instance: None, epoch: 0, - hops: HopsRef::literal(hops), + hops, cost: Cost::UNKNOWN, }, AnnounceStatus::Update => return Err(DecodeError::InvalidValue), @@ -817,8 +809,10 @@ impl<'a> Message<'a> for AnnounceInit { /// /// `origin` is the responder's session origin id. In Lite05 the publisher no /// longer stamps it onto each Announce's hop chain; the subscriber appends it on -/// receipt instead. Lite05/06 carry `active`, the number of starts immediately -/// following; lite07 marks the initial set with SYNCED instead. +/// receipt instead. Lite05/06 carry it and `active`, the number of starts immediately +/// following. Lite07 has an empty body: its claims name their route instead of a hop +/// chain, and SYNCED marks the initial set, so neither field is sent and both decode +/// as unknown. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct AnnounceOk { pub origin: Hop, @@ -831,9 +825,15 @@ impl<'a> Message<'a> for AnnounceOk { if !version.has_announce_ok() { return Err(DecodeError::Version); } + if version.has_routes() { + return Ok(Self { + origin: Hop::UNKNOWN, + active: 0, + }); + } let origin = Hop::decode(r, version)?; - let active = if version.has_synced() { 0 } else { r.varint()? }; + let active = r.varint()?; Ok(Self { origin, active }) } @@ -841,12 +841,12 @@ impl<'a> Message<'a> for AnnounceOk { if !version.has_announce_ok() { return Err(EncodeError::Version); } + if version.has_routes() { + return Ok(()); + } self.origin.encode(w, version)?; - if !version.has_synced() { - w.varint(self.active)?; - } - Ok(()) + w.varint(self.active) } } @@ -880,8 +880,10 @@ mod tests { assert!(matches!(decode(&wire, usize::MAX), Err(DecodeError::InvalidValue))); } + /// Lite-07's ANNOUNCE_OK is empty: SYNCED replaces the count, and routes replace the + /// Hop ID. Lite-06 keeps both. #[test] - fn synced_replaces_only_lite07s_count() { + fn announce_ok_is_empty_on_lite07() { for version in [Version::Lite06, Version::Lite07] { let mut bytes = Vec::new(); AnnounceOk { @@ -895,10 +897,25 @@ mod tests { if version == Version::Lite06 { vec![2, 1, 7] } else { - vec![1, 1] + vec![0] } ); } + assert_eq!( + crate::coding::decode_buf(&mut &[0u8][..], Version::Lite07, |r, v| AnnounceOk::decode(r, v)).unwrap(), + AnnounceOk { + origin: Hop::UNKNOWN, + active: 0, + } + ); + // A body where the draft has none is malformed. + assert!( + crate::coding::decode_buf(&mut &[1u8, 1][..], Version::Lite07, |r, v| AnnounceOk::decode(r, v)).is_err() + ); + } + + #[test] + fn synced_marks_lite07s_initial_set() { let mut bytes = Vec::new(); AnnounceBroadcast::Synced .encode(&mut Encoder::new(&mut bytes, Version::Lite07.into()), Version::Lite07) @@ -918,10 +935,11 @@ mod tests { // Encode a normal Active, then flip its status byte (1 -> 2). let mut buf = Vec::new(); AnnounceBroadcast::Active { + route: 0, instance: None, epoch: 0, suffix: PathRef::literal(Path::from_static("foo/bar")), - hops: HopsRef::default(), + hops: Hops::default(), cost: Cost::default(), } .encode(&mut Encoder::new(&mut buf, version.into()), version) @@ -1010,10 +1028,11 @@ mod tests { let mut hops = Hops::new(); hops.push(Hop::new(7).unwrap()).unwrap(); let msg = AnnounceBroadcast::Active { + route: 0, instance: None, epoch: 0, suffix: PathRef::literal(Path::from_static("room/cam")), - hops: HopsRef::literal(hops.clone()), + hops: hops.clone(), cost: Cost::UNKNOWN, }; assert_eq!(broadcast_round_trip(&msg, Version::Lite05), msg); @@ -1034,10 +1053,11 @@ mod tests { let cost = Cost::new(12); let active = AnnounceBroadcast::Active { + route: 0, instance: None, epoch: 0, suffix: PathRef::literal(Path::from_static("room/cam")), - hops: HopsRef::literal(hops.clone()), + hops: hops.clone(), cost, }; assert_eq!(broadcast_round_trip(&active, Version::Lite06), active); @@ -1045,22 +1065,18 @@ mod tests { let ended = AnnounceBroadcast::EndedId { id: 3 }; assert_eq!(broadcast_round_trip(&ended, Version::Lite06), ended); - let update = AnnounceBroadcast::Update { - id: 3, - hops: HopsRef::literal(hops), - cost, - }; + let update = AnnounceBroadcast::Update { id: 3, hops, cost }; assert_eq!(broadcast_round_trip(&update, Version::Lite06), update); } - // Lite07 carries both bases as they travel; the codec resolves nothing. + // Lite07 carries the path base as it travels, and the route in place of a hop chain; + // the codec resolves nothing. #[test] fn announce_broadcast_round_trip_on_lite07() { - let mut hops = Hops::new(); - hops.push(Hop::new(7).unwrap()).unwrap(); let cost = Cost::new(12); let active = AnnounceBroadcast::Active { + route: 4, instance: Some("a".parse().unwrap()), epoch: 5, suffix: PathRef { @@ -1068,22 +1084,14 @@ mod tests { keep: 3, rest: Path::from_static("cam"), }, - hops: HopsRef { - base: 1, - literal: hops.clone(), - keep: 2, - }, + hops: Hops::default(), cost, }; assert_eq!(broadcast_round_trip(&active, Version::Lite07), active); let update = AnnounceBroadcast::Update { id: 3, - hops: HopsRef { - base: 4, - literal: hops.clone(), - keep: 1, - }, + hops: Hops::default(), cost, }; assert_eq!(broadcast_round_trip(&update, Version::Lite07), update); @@ -1093,11 +1101,6 @@ mod tests { id: 3, instance: Some(Instance::random()), epoch, - hops: HopsRef { - base: 4, - literal: hops.clone(), - keep: 1, - }, cost, }; assert_eq!(broadcast_round_trip(&restart, Version::Lite07), restart); @@ -1109,28 +1112,127 @@ mod tests { keep: 1, rest: Path::from_static("p"), }, + route: 0, instance: Some(Instance::random()), epoch, - hops: HopsRef { - base: 1, - literal: hops.clone(), - keep: 1, - }, cost, }; assert_eq!(broadcast_round_trip(&takeover, Version::Lite07), takeover); } } + /// The draft's field order, byte for byte: ANNOUNCE_START is the path base, keep, and + /// rest, the Route ID, the instance and epoch, and the cost; ANNOUNCE_UPDATE is the + /// Announce ID and the cost; ANNOUNCE_TAKEOVER leads with its predecessor. + #[test] + fn lite07_announce_layout() { + let encode = |msg: AnnounceBroadcast| { + let mut buf = Vec::new(); + msg.encode(&mut Encoder::new(&mut buf, Version::Lite07.into()), Version::Lite07) + .unwrap(); + buf + }; + let suffix = PathRef { + base: 1, + keep: 2, + rest: Path::from_static("c"), + }; + assert_eq!( + encode(AnnounceBroadcast::Active { + suffix: suffix.clone(), + route: 3, + instance: Some("a".parse().unwrap()), + epoch: 5, + hops: Hops::default(), + cost: Cost::new(6), + }), + [0, 9, 1, 2, 1, b'c', 3, 1, b'a', 5, 6] + ); + assert_eq!( + encode(AnnounceBroadcast::Update { + id: 7, + hops: Hops::default(), + cost: Cost::new(6), + }), + [2, 2, 7, 6] + ); + assert_eq!( + encode(AnnounceBroadcast::Restart { + id: 7, + instance: Some("a".parse().unwrap()), + epoch: 5, + cost: Cost::new(6), + }), + [3, 5, 7, 1, b'a', 5, 6] + ); + assert_eq!( + encode(AnnounceBroadcast::Takeover { + predecessor: 9, + suffix, + route: 3, + instance: Some("a".parse().unwrap()), + epoch: 5, + cost: Cost::new(6), + }), + [5, 10, 9, 1, 2, 1, b'c', 3, 1, b'a', 5, 6] + ); + } + + /// Lite-07 names a claim's route instead of its hop chain, so a chain is refused there. + #[test] + fn lite07_refuses_a_hop_chain() { + let mut hops = Hops::new(); + hops.push(Hop::new(7).unwrap()).unwrap(); + let refused = [ + AnnounceBroadcast::Active { + route: 0, + instance: Some("a".parse().unwrap()), + epoch: 0, + suffix: PathRef::literal(Path::from_static("cam")), + hops: hops.clone(), + cost: Cost::default(), + }, + AnnounceBroadcast::Update { + id: 0, + hops, + cost: Cost::default(), + }, + ]; + for msg in refused { + let mut buf = Vec::new(); + let res = msg.encode(&mut Encoder::new(&mut buf, Version::Lite07.into()), Version::Lite07); + assert!(matches!(res, Err(EncodeError::InvalidState)), "{msg:?}"); + } + } + + /// Only lite-07 carries a Route ID: a non-zero one has nowhere to go earlier, and an + /// earlier start decodes as route 0. + #[test] + fn a_route_id_needs_lite07() { + let msg = AnnounceBroadcast::Active { + route: 1, + instance: None, + epoch: 0, + suffix: PathRef::literal(Path::from_static("cam")), + hops: Hops::default(), + cost: Cost::default(), + }; + for version in [Version::Lite05, Version::Lite06] { + let mut buf = Vec::new(); + let res = msg.encode(&mut Encoder::new(&mut buf, version.into()), version); + assert!(matches!(res, Err(EncodeError::InvalidState)), "{version:?}"); + } + } + // Only lite-07 carries ANNOUNCE_TAKEOVER: older versions send a start instead. #[test] fn announce_takeover_needs_lite07() { let takeover = AnnounceBroadcast::Takeover { predecessor: 0, suffix: PathRef::literal(Path::from_static("p")), + route: 0, instance: Some(Instance::random()), epoch: 0, - hops: HopsRef::default(), cost: Cost::default(), }; assert!(matches!( @@ -1150,7 +1252,6 @@ mod tests { id: 1, instance: Some("a".parse().unwrap()), epoch: 0, - hops: HopsRef::default(), cost: Cost::default(), }; let mut buf = Vec::new(); @@ -1181,22 +1282,21 @@ mod tests { // A keep copies from a base, so one without a base is malformed. #[test] fn a_keep_without_a_base_is_rejected() { - for (path_keep, hop_keep) in [(1u8, 0u8), (0, 1)] { - // Path base, path keep, empty rest, instance "a", epoch, hop base, no hops, hop keep, cost. - let body = [0, path_keep, 0, 1, b'a', 0, 0, 0, hop_keep, 0]; - let mut buf = vec![ANNOUNCE_START as u8, body.len() as u8]; - buf.extend_from_slice(&body); - assert!(matches!( - crate::coding::decode_buf(&mut &buf[..], Version::Lite07, |r, v| AnnounceBroadcast::decode(r, v)), - Err(DecodeError::InvalidValue) - )); - } + // Path base, path keep, empty rest, route, instance "a", epoch, cost. + let body = [0, 1, 0, 0, 1, b'a', 0, 0]; + let mut buf = vec![ANNOUNCE_START as u8, body.len() as u8]; + buf.extend_from_slice(&body); + assert!(matches!( + crate::coding::decode_buf(&mut &buf[..], Version::Lite07, |r, v| AnnounceBroadcast::decode(r, v)), + Err(DecodeError::InvalidValue) + )); } // Only lite-07 has room for a base. #[test] fn a_base_needs_lite07() { let msg = AnnounceBroadcast::Active { + route: 0, instance: None, epoch: 0, suffix: PathRef { @@ -1204,7 +1304,7 @@ mod tests { keep: 1, rest: Path::from_static("cam"), }, - hops: HopsRef::default(), + hops: Hops::default(), cost: Cost::default(), }; for version in [Version::Lite05, Version::Lite06] { @@ -1228,7 +1328,7 @@ mod tests { assert!(matches!( AnnounceBroadcast::Update { id: 1, - hops: HopsRef::default(), + hops: Hops::default(), cost: Cost::default() } .encode(&mut Encoder::new(&mut buf, Version::Lite05.into()), Version::Lite05), @@ -1251,20 +1351,22 @@ mod tests { #[test] fn route_cost_is_dropped_before_lite06() { let msg = AnnounceBroadcast::Active { + route: 0, instance: None, epoch: 0, suffix: PathRef::literal(Path::from_static("room/cam")), - hops: HopsRef::default(), + hops: Hops::default(), cost: Cost::new(9), }; let got = broadcast_round_trip(&msg, Version::Lite05); assert_eq!( got, AnnounceBroadcast::Active { + route: 0, instance: None, epoch: 0, suffix: PathRef::literal(Path::from_static("room/cam")), - hops: HopsRef::default(), + hops: Hops::default(), cost: Cost::UNKNOWN, } ); diff --git a/rs/moq-net/src/lite/compress.rs b/rs/moq-net/src/lite/compress.rs index 1ab2bfeb1ea..3c015d9c2d3 100644 --- a/rs/moq-net/src/lite/compress.rs +++ b/rs/moq-net/src/lite/compress.rs @@ -1,57 +1,49 @@ -//! Lite-07 announce compression: an ANNOUNCE_START may copy the head of a live -//! announcement's path, and an ANNOUNCE_START or ANNOUNCE_UPDATE the tail of a live -//! announcement's hop chain, naming that base by its distance back from the stream's -//! next unassigned Announce ID. +//! Lite-07 announce compression: an ANNOUNCE_START or ANNOUNCE_TAKEOVER may copy the +//! head of a live announcement's path, naming that base by its distance back from the +//! stream's next unassigned Announce ID. //! //! The codec in `announce.rs` stays stateless and carries bases as they travel. The //! decoder here resolves them against the stream's live announcements, which the //! subscriber has to track anyway, and the encoder picks them. -use std::{ - borrow::Borrow, - collections::{BTreeSet, HashMap}, - hash::Hash, -}; +use std::collections::{BTreeSet, HashMap}; use crate::{ - Error, Hop, Hops, Path, + Error, Path, coding::{Form, varint}, }; -use super::{HopsRef, PathRef, Version}; - -/// A live announcement, as the peer holds it: its wire suffix and wire hop chain -/// (the one excluding ANNOUNCE_OK's Hop ID). -#[derive(Clone, Debug)] -struct Entry { - suffix: Path, - hops: Hops, -} +use super::{PathRef, Version}; -/// The live announcements on one stream by Announce ID, which is all a base can name. +/// The live announcements on one stream by Announce ID, each with its wire suffix, +/// which is all a base can name. #[derive(Debug, Default)] struct Live { next: u64, - entries: HashMap, + suffixes: HashMap, } impl Live { + /// The suffix of the live announcement with this id. + fn suffix(&self, id: u64) -> Result<&Path, Error> { + self.suffixes.get(&id).ok_or(Error::ProtocolViolation) + } + /// The live announcement `distance` back from the next id, or `None` for 0. - fn base(&self, distance: u64) -> Result, Error> { + fn base(&self, distance: u64) -> Result, Error> { if distance == 0 { return Ok(None); } let id = self.next.checked_sub(distance).ok_or(Error::ProtocolViolation)?; - let entry = self.entries.get(&id).ok_or(Error::ProtocolViolation)?; - Ok(Some(entry)) + self.suffix(id).map(Some) } - fn resolve_path(&self, path: PathRef) -> Result { + fn resolve(&self, path: PathRef) -> Result { let Some(base) = self.base(path.base)? else { return Ok(path.rest); }; let keep = usize::try_from(path.keep).map_err(|_| Error::ProtocolViolation)?; - let head = head(&base.suffix, keep).ok_or(Error::ProtocolViolation)?; + let head = head(base, keep).ok_or(Error::ProtocolViolation)?; let suffix = match (head.is_empty(), path.rest.is_empty()) { (true, _) => Ok(path.rest), (false, true) => Path::new(head), @@ -63,20 +55,6 @@ impl Live { } Ok(suffix) } - - fn resolve_hops(&self, hops: HopsRef) -> Result { - let Some(base) = self.base(hops.base)? else { - return Ok(hops.literal); - }; - let keep = usize::try_from(hops.keep).map_err(|_| Error::ProtocolViolation)?; - let tail = base.hops.as_slice(); - let start = tail.len().checked_sub(keep).ok_or(Error::ProtocolViolation)?; - let mut out = hops.literal; - for hop in &tail[start..] { - out.push(*hop).map_err(|_| Error::ProtocolViolation)?; - } - Ok(out) - } } /// The first `keep` segments of `path`, or `None` if it has fewer. @@ -100,84 +78,56 @@ pub(crate) struct AnnounceDecoder { } impl AnnounceDecoder { - /// An ANNOUNCE_START: resolve it and assign it the next id. - pub fn start(&mut self, suffix: PathRef, hops: HopsRef) -> Result<(Path, Hops), Error> { - let suffix = self.live.resolve_path(suffix)?; - let hops = self.live.resolve_hops(hops)?; + /// An ANNOUNCE_START: resolve its suffix and assign it the next id. + pub fn start(&mut self, suffix: PathRef) -> Result { + let suffix = self.live.resolve(suffix)?; let id = self.live.next; self.live.next += 1; - self.live.entries.insert( - id, - Entry { - suffix: suffix.clone(), - hops: hops.clone(), - }, - ); - Ok((suffix, hops)) + self.live.suffixes.insert(id, suffix.clone()); + Ok(suffix) } /// An ANNOUNCE_TAKEOVER: resolve it like a start against the live `predecessor`, - /// returning the predecessor's suffix too. One at the predecessor's own suffix - /// retires the predecessor, since a suffix holds one advertisement. - pub fn takeover(&mut self, predecessor: u64, suffix: PathRef, hops: HopsRef) -> Result<(Path, Path, Hops), Error> { - let previous = self - .live - .entries - .get(&predecessor) - .ok_or(Error::ProtocolViolation)? - .suffix - .clone(); - let (suffix, hops) = self.start(suffix, hops)?; + /// returning the predecessor's suffix, then its own. One at the predecessor's own + /// suffix retires the predecessor, since a suffix holds one advertisement. + pub fn takeover(&mut self, predecessor: u64, suffix: PathRef) -> Result<(Path, Path), Error> { + let previous = self.live.suffix(predecessor)?.clone(); + let suffix = self.start(suffix)?; if suffix == previous { - self.live.entries.remove(&predecessor); + self.live.suffixes.remove(&predecessor); } - Ok((previous, suffix, hops)) + Ok((previous, suffix)) } - /// An ANNOUNCE_UPDATE: resolve its chain before replacing the old one, which it - /// may itself be based on. - pub fn update(&mut self, id: u64, hops: HopsRef) -> Result<(Path, Hops), Error> { - let hops = self.live.resolve_hops(hops)?; - let entry = self.live.entries.get_mut(&id).ok_or(Error::ProtocolViolation)?; - entry.hops = hops.clone(); - Ok((entry.suffix.clone(), hops)) + /// The suffix of the live id an ANNOUNCE_UPDATE or ANNOUNCE_RESTART names, which + /// stays live. + pub fn get(&self, id: u64) -> Result { + self.live.suffix(id).cloned() } /// An ANNOUNCE_END: retire the id, returning its suffix. pub fn end(&mut self, id: u64) -> Result { - let entry = self.live.entries.remove(&id).ok_or(Error::ProtocolViolation)?; - Ok(entry.suffix) + self.live.suffixes.remove(&id).ok_or(Error::ProtocolViolation) } } -/// The live ids under a run of keys: path segments from the head, or hop ids from -/// the tail. Every node holds the ids of the entries beneath it, so the entry sharing -/// the longest run with a query is found by walking the query down once, at a cost -/// that grows with its length rather than with the number of entries. -#[derive(Debug)] -struct Trie { - children: HashMap>, +/// The live ids under a run of leading path segments. Every node holds the ids of the +/// entries beneath it, so the entry sharing the longest head with a query is found by +/// walking the query down once, at a cost that grows with its length rather than with +/// the number of entries. +#[derive(Debug, Default)] +struct Trie { + children: HashMap, } #[derive(Debug)] -struct Node { +struct Node { ids: BTreeSet, - children: HashMap>, + children: HashMap, } -impl Default for Trie { - fn default() -> Self { - Self { - children: HashMap::new(), - } - } -} - -impl Trie { - fn insert<'a, Q>(&mut self, keys: impl Iterator, id: u64) - where - Q: ?Sized + ToOwned + 'a, - { +impl Trie { + fn insert<'a>(&mut self, keys: impl Iterator, id: u64) { let mut children = &mut self.children; for key in keys { let node = children.entry(key.to_owned()).or_insert_with(|| Node { @@ -189,19 +139,11 @@ impl Trie { } } - fn remove<'a, Q>(&mut self, keys: impl Iterator, id: u64) - where - Q: ?Sized + Hash + Eq + 'a, - K: Borrow, - { + fn remove<'a>(&mut self, keys: impl Iterator, id: u64) { Self::remove_from(&mut self.children, keys, id); } - fn remove_from<'a, Q>(children: &mut HashMap>, mut keys: impl Iterator, id: u64) - where - Q: ?Sized + Hash + Eq + 'a, - K: Borrow, - { + fn remove_from<'a>(children: &mut HashMap, mut keys: impl Iterator, id: u64) { let Some(key) = keys.next() else { return; }; @@ -219,11 +161,7 @@ impl Trie { /// How many leading keys the query shares with its closest entry, and that entry's /// id, preferring the newest (the smallest distance to encode). - fn longest<'a, Q>(&self, keys: impl Iterator) -> Option<(usize, u64)> - where - Q: ?Sized + Hash + Eq + 'a, - K: Borrow, - { + fn longest<'a>(&self, keys: impl Iterator) -> Option<(usize, u64)> { let mut children = &self.children; let mut best = None; for (depth, key) in keys.enumerate() { @@ -242,20 +180,13 @@ impl Trie { /// versions that have them. Only lite-07 keeps the live set, since nothing earlier /// can name a base. /// -/// One trie over the live paths by segment, and one over the live hop chains from the -/// last hop back, find the longest shared head and tail. A base is used only when it -/// encodes smaller than the literal. +/// A trie over the live paths by segment finds the longest shared head. A base is +/// used only when it encodes smaller than the literal. #[derive(Debug)] pub(crate) struct AnnounceEncoder { version: Version, live: Live, - heads: Trie, - tails: Trie, -} - -/// A hop chain from the last hop back. -fn reversed(hops: &Hops) -> impl Iterator { - hops.as_slice().iter().rev() + heads: Trie, } impl AnnounceEncoder { @@ -264,60 +195,39 @@ impl AnnounceEncoder { version, live: Live::default(), heads: Trie::default(), - tails: Trie::default(), } } - /// An ANNOUNCE_START: its id (on versions that assign one) and its wire form. - pub fn start(&mut self, suffix: Path, hops: Hops) -> (Option, PathRef, HopsRef) { + /// An ANNOUNCE_START: its id (on versions that assign one) and its wire suffix. + pub fn start(&mut self, suffix: Path) -> (Option, PathRef) { if !self.version.has_announce_id() { - return (None, PathRef::literal(suffix), HopsRef::literal(hops)); + return (None, PathRef::literal(suffix)); } let id = self.live.next; self.live.next += 1; if !self.version.has_announce_compression() { - return (Some(id), PathRef::literal(suffix), HopsRef::literal(hops)); + return (Some(id), PathRef::literal(suffix)); } // Distances count back from the id this START takes. let path = self.path_ref(&suffix, id); - let chain = self.hops_ref(&hops, id); self.heads.insert(suffix.parts(), id); - self.tails.insert(reversed(&hops), id); - self.live.entries.insert(id, Entry { suffix, hops }); - (Some(id), path, chain) + self.live.suffixes.insert(id, suffix); + (Some(id), path) } /// An ANNOUNCE_TAKEOVER of the live `predecessor`: like [`Self::start`], retiring /// the predecessor when it holds the same suffix. - pub fn takeover(&mut self, predecessor: u64, suffix: Path, hops: Hops) -> (u64, PathRef, HopsRef) { - let replaces = self - .live - .entries - .get(&predecessor) - .is_some_and(|entry| entry.suffix == suffix); - let (id, path, chain) = self.start(suffix, hops); + pub fn takeover(&mut self, predecessor: u64, suffix: Path) -> (u64, PathRef) { + let replaces = self.live.suffixes.get(&predecessor) == Some(&suffix); + let (id, path) = self.start(suffix); let id = id.expect("a takeover is only sent on a version with announce ids"); if replaces { self.end(predecessor); } - (id, path, chain) - } - - /// An ANNOUNCE_UPDATE for a live `id`: its wire hop chain. - pub fn update(&mut self, id: u64, hops: Hops) -> HopsRef { - if !self.version.has_announce_compression() { - return HopsRef::literal(hops); - } - // The decoder resolves against the chain this replaces, so it is a valid base. - let chain = self.hops_ref(&hops, self.live.next); - let entry = self.live.entries.get_mut(&id).expect("update for a live id"); - self.tails.remove(reversed(&entry.hops), id); - self.tails.insert(reversed(&hops), id); - entry.hops = hops; - chain + (id, path) } /// An ANNOUNCE_END for a live `id`. @@ -325,9 +235,8 @@ impl AnnounceEncoder { if !self.version.has_announce_compression() { return; } - let entry = self.live.entries.remove(&id).expect("end for a live id"); - self.heads.remove(entry.suffix.parts(), id); - self.tails.remove(reversed(&entry.hops), id); + let suffix = self.live.suffixes.remove(&id).expect("end for a live id"); + self.heads.remove(suffix.parts(), id); } /// The wire form of `suffix`, sized from its parts so only the winner is built. @@ -350,35 +259,6 @@ impl AnnounceEncoder { false => literal(), } } - - /// The wire form of `hops`, sized from its parts so only the winner is built. - fn hops_ref(&self, hops: &Hops, next: u64) -> HopsRef { - let form: Form = self.version.into(); - let literal = || HopsRef::literal(hops.clone()); - let Some((keep, id)) = self.tails.longest(reversed(hops)) else { - return literal(); - }; - let head = &hops.as_slice()[..hops.len() - keep]; - - let base = next - id; - let keep = keep as u64; - let chain = |hops: &[Hop]| { - varint_size(hops.len() as u64, form) - + hops - .iter() - .map(|hop| varint_size(hop.wire_id(form), form)) - .sum::() - }; - let size = varint_size(base, form) + chain(head) + varint_size(keep, form); - match size < varint_size(0, form) * 2 + chain(hops.as_slice()) { - true => HopsRef { - base, - literal: Hops::try_from(head.to_vec()).expect("a prefix of a valid chain is valid"), - keep, - }, - false => literal(), - } - } } /// The bytes `value` takes as a varint in `form`. @@ -399,10 +279,6 @@ mod tests { const VERSION: Version = Version::Lite07; - fn hops(ids: &[u64]) -> Hops { - Hops::try_from(ids.iter().map(|id| Hop::from_wire(*id)).collect::>()).unwrap() - } - fn path(base: u64, keep: u64, rest: &str) -> PathRef { PathRef { base, @@ -411,85 +287,52 @@ mod tests { } } - fn chain(base: u64, literal: &[u64], keep: u64) -> HopsRef { - HopsRef { - base, - literal: hops(literal), - keep, - } - } - - fn literal(suffix: &str, ids: &[u64]) -> (PathRef, HopsRef) { - (path(0, 0, suffix), chain(0, ids, 0)) + fn literal(suffix: &str) -> PathRef { + path(0, 0, suffix) } #[test] - fn resolves_a_path_head_and_hop_tail() { + fn resolves_a_path_head() { let mut decoder = AnnounceDecoder::default(); - let (p, h) = literal("pid/private/channel_1/health-1", &[1, 2, 3]); - decoder.start(p, h).unwrap(); + decoder.start(literal("pid/private/channel_1/health-1")).unwrap(); - // Three leading segments of the latest START, and its last two hops. - let (suffix, chain) = decoder.start(path(1, 3, "health-2"), chain(1, &[7], 2)).unwrap(); + // Three leading segments of the latest START. + let suffix = decoder.start(path(1, 3, "health-2")).unwrap(); assert_eq!(suffix.as_str(), "pid/private/channel_1/health-2"); - assert_eq!(chain, hops(&[7, 2, 3])); } #[test] - fn keeps_the_whole_path_and_chain() { + fn keeps_the_whole_path() { let mut decoder = AnnounceDecoder::default(); - let (p, h) = literal("a/b", &[1, 2]); - decoder.start(p, h).unwrap(); - // A full keep with an empty rest names a route above the base's: not a - // duplicate, since the rest extends it. - let (suffix, chain) = decoder.start(path(1, 2, "c"), chain(1, &[], 2)).unwrap(); + decoder.start(literal("a/b")).unwrap(); + // A full keep with a rest names a route beneath the base's: not a duplicate, + // since the rest extends it. + let suffix = decoder.start(path(1, 2, "c")).unwrap(); + assert_eq!(suffix.as_str(), "a/b/c"); + // And a full keep with an empty rest copies it whole. + let suffix = decoder.start(path(1, 3, "")).unwrap(); assert_eq!(suffix.as_str(), "a/b/c"); - assert_eq!(chain, hops(&[1, 2])); - // An update copying its own full chain. - let (suffix, chain) = decoder.update(0, chain_ref(2, 2)).unwrap(); - assert_eq!(suffix.as_str(), "a/b"); - assert_eq!(chain, hops(&[1, 2])); - } - - fn chain_ref(base: u64, keep: u64) -> HopsRef { - chain(base, &[], keep) } #[test] fn a_new_stream_has_no_base() { let mut decoder = AnnounceDecoder::default(); - assert!(matches!( - decoder.start(path(1, 0, "a"), chain(0, &[], 0)), - Err(Error::ProtocolViolation) - )); - let mut decoder = AnnounceDecoder::default(); - assert!(matches!( - decoder.start(path(0, 0, "a"), chain(1, &[], 0)), - Err(Error::ProtocolViolation) - )); + assert!(matches!(decoder.start(path(1, 0, "a")), Err(Error::ProtocolViolation))); } #[test] fn rejects_a_retired_or_unassigned_base() { let mut decoder = AnnounceDecoder::default(); - let (p, h) = literal("a", &[1]); - decoder.start(p, h).unwrap(); - let (p, h) = literal("b", &[2]); - decoder.start(p, h).unwrap(); + decoder.start(literal("a")).unwrap(); + decoder.start(literal("b")).unwrap(); decoder.end(0).unwrap(); // Id 0 is two back, and retired. - assert!(matches!( - decoder.start(path(2, 1, "x"), chain(0, &[], 0)), - Err(Error::ProtocolViolation) - )); + assert!(matches!(decoder.start(path(2, 1, "x")), Err(Error::ProtocolViolation))); // Three back was never assigned. - assert!(matches!( - decoder.start(path(3, 0, "x"), chain(0, &[], 0)), - Err(Error::ProtocolViolation) - )); + assert!(matches!(decoder.start(path(3, 0, "x")), Err(Error::ProtocolViolation))); // Id 1 is still live, so reusing it works after the END. - let (suffix, _) = decoder.start(path(1, 1, "x"), chain(1, &[], 1)).unwrap(); + let suffix = decoder.start(path(1, 1, "x")).unwrap(); assert_eq!(suffix.as_str(), "b/x"); } @@ -498,27 +341,22 @@ mod tests { #[test] fn takeover_assigns_an_id_and_retires_one_in_place() { let mut decoder = AnnounceDecoder::default(); - let (p, h) = literal("pool", &[1]); - decoder.start(p, h).unwrap(); + decoder.start(literal("pool")).unwrap(); - let (previous, suffix, chain) = decoder.takeover(0, path(1, 1, "p"), chain_ref(1, 1)).unwrap(); + let (previous, suffix) = decoder.takeover(0, path(1, 1, "p")).unwrap(); assert_eq!((previous.as_str(), suffix.as_str()), ("pool", "pool/p")); - assert_eq!(chain, hops(&[1])); - decoder.update(0, chain_ref(0, 0)).unwrap(); + assert_eq!(decoder.get(0).unwrap().as_str(), "pool"); // Id 2 at id 1's own suffix, copied from id 1 itself, which then retires. - let (previous, suffix, _) = decoder.takeover(1, path(1, 2, ""), chain_ref(1, 1)).unwrap(); + let (previous, suffix) = decoder.takeover(1, path(1, 2, "")).unwrap(); assert_eq!((previous.as_str(), suffix.as_str()), ("pool/p", "pool/p")); + assert!(matches!(decoder.get(1), Err(Error::ProtocolViolation))); assert!(matches!( - decoder.update(1, chain_ref(0, 0)), - Err(Error::ProtocolViolation) - )); - assert!(matches!( - decoder.takeover(1, path(0, 0, "pool/p/q"), chain_ref(0, 0)), + decoder.takeover(1, literal("pool/p/q")), Err(Error::ProtocolViolation) )); assert!(matches!( - decoder.takeover(9, path(0, 0, "pool/p/q"), chain_ref(0, 0)), + decoder.takeover(9, literal("pool/p/q")), Err(Error::ProtocolViolation) )); assert_eq!(decoder.end(2).unwrap().as_str(), "pool/p"); @@ -528,54 +366,17 @@ mod tests { #[test] fn rejects_a_keep_longer_than_the_base() { let mut decoder = AnnounceDecoder::default(); - let (p, h) = literal("a/b", &[1]); - decoder.start(p, h).unwrap(); - assert!(matches!( - decoder.start(path(1, 3, "x"), chain(0, &[], 0)), - Err(Error::ProtocolViolation) - )); - assert!(matches!( - decoder.update(0, chain_ref(1, 2)), - Err(Error::ProtocolViolation) - )); - } - - #[test] - fn rejects_a_resolved_chain_breaking_the_hop_rules() { - let mut decoder = AnnounceDecoder::default(); - let (p, h) = literal("a", &[1, 2]); - decoder.start(p, h).unwrap(); - // A literal hop the kept tail repeats. - assert!(matches!( - decoder.start(path(0, 0, "b"), chain(1, &[2], 1)), - Err(Error::ProtocolViolation) - )); - - // Anonymous hops may repeat across the two halves. - let (p, h) = literal("c", &[0]); - decoder.start(p, h).unwrap(); - let (_, resolved) = decoder.start(path(0, 0, "d"), chain(1, &[0], 1)).unwrap(); - assert_eq!(resolved, hops(&[0, 0])); - - // And the resolved chain is still capped. - // MAX_HOPS entries. - let full: Vec = (1..=32).collect(); - let (p, h) = literal("e", &full); - decoder.start(p, h).unwrap(); - assert!(matches!( - decoder.start(path(0, 0, "f"), chain(1, &[100], 32)), - Err(Error::ProtocolViolation) - )); + decoder.start(literal("a/b")).unwrap(); + assert!(matches!(decoder.start(path(1, 3, "x")), Err(Error::ProtocolViolation))); } #[test] fn rejects_a_resolved_path_too_deep() { let mut decoder = AnnounceDecoder::default(); let deep = vec!["x"; Path::MAX_PARTS].join("/"); - let (p, h) = literal(&deep, &[]); - decoder.start(p, h).unwrap(); + decoder.start(literal(&deep)).unwrap(); assert!(matches!( - decoder.start(path(1, Path::MAX_PARTS as u64, "y"), chain(0, &[], 0)), + decoder.start(path(1, Path::MAX_PARTS as u64, "y")), Err(Error::ProtocolViolation) )); } @@ -584,26 +385,20 @@ mod tests { fn rejects_a_resolved_path_too_long() { let mut decoder = AnnounceDecoder::default(); let head = "x".repeat(Path::MAX_BYTES - 2); - let (p, h) = literal(&head, &[]); - decoder.start(p, h).unwrap(); + decoder.start(literal(&head)).unwrap(); // `head/y` is exactly the limit, and one byte more is refused. - let (suffix, _) = decoder.start(path(1, 1, "y"), chain(0, &[], 0)).unwrap(); + let suffix = decoder.start(path(1, 1, "y")).unwrap(); assert_eq!(suffix.len(), Path::MAX_BYTES); - assert!(matches!( - decoder.start(path(1, 1, "yz"), chain(0, &[], 0)), - Err(Error::ProtocolViolation) - )); + assert!(matches!(decoder.start(path(1, 1, "yz")), Err(Error::ProtocolViolation))); } - /// Encode a message with `encoder`, decode it with `decoder`, and check that the - /// resolved values match the input. - fn start(encoder: &mut AnnounceEncoder, decoder: &mut AnnounceDecoder, suffix: &str, ids: &[u64]) -> (u64, usize) { - let (id, wire_path, wire_hops) = encoder.start(Path::new(suffix).unwrap(), hops(ids)); - let size = wire_path.encode_bytes(VERSION).unwrap().len() + wire_hops.encode_bytes(VERSION).unwrap().len(); - let (got_path, got_hops) = decoder.start(wire_path, wire_hops).unwrap(); - assert_eq!(got_path.as_str(), suffix); - assert_eq!(got_hops, hops(ids)); + /// Encode a START with `encoder`, decode it with `decoder`, and check that the + /// resolved suffix matches the input. Returns its id and wire size. + fn start(encoder: &mut AnnounceEncoder, decoder: &mut AnnounceDecoder, suffix: &str) -> (u64, usize) { + let (id, wire) = encoder.start(Path::new(suffix).unwrap()); + let size = wire.encode_bytes(VERSION).unwrap().len(); + assert_eq!(decoder.start(wire).unwrap().as_str(), suffix); (id.unwrap(), size) } @@ -612,22 +407,17 @@ mod tests { fn encoder_takeover_decodes_to_its_input() { let mut encoder = AnnounceEncoder::new(VERSION); let mut decoder = AnnounceDecoder::default(); - let (claim, _) = start(&mut encoder, &mut decoder, "pool", &[1, 2]); + let (claim, _) = start(&mut encoder, &mut decoder, "pool"); let mut predecessor = claim; - for (suffix, ids) in [ - ("pool/p", &[3, 1, 2][..]), - ("pool/p", &[4, 1, 2]), - ("pool/p/q", &[4, 1, 2]), - ] { - let (id, wire_path, wire_hops) = encoder.takeover(predecessor, Path::new(suffix).unwrap(), hops(ids)); - let (_, got_path, got_hops) = decoder.takeover(predecessor, wire_path, wire_hops).unwrap(); - assert_eq!(got_path.as_str(), suffix); - assert_eq!(got_hops, hops(ids)); + for suffix in ["pool/p", "pool/p", "pool/p/q"] { + let (id, wire) = encoder.takeover(predecessor, Path::new(suffix).unwrap()); + let (_, got) = decoder.takeover(predecessor, wire).unwrap(); + assert_eq!(got.as_str(), suffix); predecessor = id; } // Both sides retired the id replaced in place and kept the rest. - let (id, wire_path, wire_hops) = encoder.start(Path::new("other").unwrap(), hops(&[1, 2])); - decoder.start(wire_path, wire_hops).unwrap(); + let (id, wire) = encoder.start(Path::new("other").unwrap()); + decoder.start(wire).unwrap(); assert_eq!(id, Some(4)); assert!(matches!(decoder.end(1), Err(Error::ProtocolViolation))); decoder.end(claim).unwrap(); @@ -640,13 +430,10 @@ mod tests { let mut encoder = AnnounceEncoder::new(VERSION); let mut decoder = AnnounceDecoder::default(); - // Several origins behind a shared pair of relays. - let relays = [1u64 << 40, 1u64 << 41]; let mut ids = Vec::new(); for n in 0..40u64 { - let origin = 1000 + n % 4; let suffix = format!("pid{}/private/channel_{}/stream-health-{}", n % 4, n % 7, n); - let (id, _) = start(&mut encoder, &mut decoder, &suffix, &[origin, relays[0], relays[1]]); + let (id, _) = start(&mut encoder, &mut decoder, &suffix); ids.push(id); // Retire every third one so bases come and go. if n % 3 == 0 { @@ -656,103 +443,57 @@ mod tests { } } - // Updates resolve against the chain they replace. - for id in ids.iter().copied().take(5) { - let chain = hops(&[2000, relays[1]]); - let wire = encoder.update(id, chain.clone()); - let (_, got) = decoder.update(id, wire).unwrap(); - assert_eq!(got, chain); - } - - // And a later START still matches what the encoder remembers. - start( - &mut encoder, - &mut decoder, - "pid0/private/channel_0/x", - &[1000, relays[0], relays[1]], - ); + // A later START still matches what the encoder remembers. + start(&mut encoder, &mut decoder, "pid0/private/channel_0/x"); } #[test] - fn encoder_picks_the_longest_shared_head_and_tail() { + fn encoder_picks_the_longest_shared_head() { let mut encoder = AnnounceEncoder::new(VERSION); let mut decoder = AnnounceDecoder::default(); - start(&mut encoder, &mut decoder, "a/b/c", &[1u64 << 50, 1u64 << 51]); + start(&mut encoder, &mut decoder, "a/b/c"); // Shares a byte prefix with `a/b/d`, but no second segment. - start(&mut encoder, &mut decoder, "a/b-z", &[3]); - - let (_, wire_path, wire_hops) = encoder.start(Path::from_static("a/b/d"), hops(&[5, 1u64 << 51])); - assert_eq!(wire_path, path(2, 2, "d")); - assert_eq!(wire_hops, chain(2, &[5], 1)); - } + start(&mut encoder, &mut decoder, "a/b-z"); - /// Hop 100 is 2 bytes as a QUIC varint and 1 byte as leading-ones. Sharing that - /// single hop from 128 announcements back costs 4 bytes in both forms (the base - /// is 2 bytes either way). The literal chain is 5 bytes in QUIC and 4 in - /// leading-ones, so QUIC would compress and leading-ones ties. A tie stays literal. - #[test] - fn a_tie_stays_literal_where_hop_100_sizes_differ() { - let quic = Form::Quic; - let leading = Form::LeadingOnes; - assert_eq!(varint_size(100, quic), 2); - assert_eq!(varint_size(100, leading), 1); - - let base = 128; - let compressed = |form| varint_size(base, form) + varint_size(0, form) + varint_size(1, form); - let literal = |form| varint_size(0, form) * 2 + varint_size(1, form) + varint_size(100, form); - assert!(compressed(quic) < literal(quic)); - assert_eq!(compressed(leading), literal(leading)); - - let mut encoder = AnnounceEncoder::new(VERSION); - let mut decoder = AnnounceDecoder::default(); - start(&mut encoder, &mut decoder, "origin", &[100]); - for n in 1..128 { - let suffix = format!("f{n}"); - start(&mut encoder, &mut decoder, &suffix, &[1]); - } - let (_, wire_path, wire_hops) = encoder.start(Path::from_static("again"), hops(&[100])); - assert_eq!(wire_hops, HopsRef::literal(hops(&[100]))); - let (got_path, got_hops) = decoder.start(wire_path, wire_hops).unwrap(); - assert_eq!(got_path.as_str(), "again"); - assert_eq!(got_hops, hops(&[100])); + let (_, wire) = encoder.start(Path::from_static("a/b/d")); + assert_eq!(wire, path(2, 2, "d")); } #[test] fn encoder_sends_literal_when_nothing_is_shared() { let mut encoder = AnnounceEncoder::new(VERSION); let mut decoder = AnnounceDecoder::default(); - start(&mut encoder, &mut decoder, "a", &[1]); - let (_, wire_path, wire_hops) = encoder.start(Path::from_static("b"), hops(&[2])); - assert_eq!(wire_path, PathRef::literal(Path::from_static("b"))); - assert_eq!(wire_hops, HopsRef::literal(hops(&[2]))); + start(&mut encoder, &mut decoder, "a"); + let (_, wire) = encoder.start(Path::from_static("b")); + assert_eq!(wire, PathRef::literal(Path::from_static("b"))); } #[test] fn lite06_assigns_ids_without_bases() { let mut encoder = AnnounceEncoder::new(Version::Lite06); - let (id, _, _) = encoder.start(Path::from_static("a/b"), hops(&[1])); + let (id, _) = encoder.start(Path::from_static("a/b")); assert_eq!(id, Some(0)); - let (id, wire_path, wire_hops) = encoder.start(Path::from_static("a/c"), hops(&[1])); + let (id, wire) = encoder.start(Path::from_static("a/c")); assert_eq!(id, Some(1)); - assert_eq!(wire_path, PathRef::literal(Path::from_static("a/c"))); - assert_eq!(wire_hops, HopsRef::literal(hops(&[1]))); + assert_eq!(wire, PathRef::literal(Path::from_static("a/c"))); encoder.end(0); - assert_eq!(encoder.update(1, hops(&[1])), HopsRef::literal(hops(&[1]))); } /// A compressed stream pinned byte for byte: `js/net/src/lite/announce.test.ts` /// decodes the same hex, so a codec change on either side breaks both. - const GOLDEN: &str = "001800000a726f6f6d2f612f63616d01610000029111a2220000000f0102036d69630161000101b33301000208000101c044440100010101000e020101620161000201c055550100"; + const GOLDEN: &str = + "001200000a726f6f6d2f612f63616d0001610000000b0102036d69630001610000020200000101010009020101620001610000"; #[cfg(feature = "async")] fn golden() -> Vec { use crate::fuzz::Announced; + // Lite-07 names a route instead of a hop chain, so none travels. vec![ - Announced::Start(Path::from_static("room/a/cam"), hops(&[0x1111, 0x2222])), - Announced::Start(Path::from_static("room/a/mic"), hops(&[0x3333, 0x2222])), - Announced::Update(0, hops(&[0x4444, 0x2222])), + Announced::Start(Path::from_static("room/a/cam"), crate::Hops::new()), + Announced::Start(Path::from_static("room/a/mic"), crate::Hops::new()), + Announced::Update(0, crate::Hops::new()), Announced::End(1), - Announced::Start(Path::from_static("room/b"), hops(&[0x5555, 0x2222])), + Announced::Start(Path::from_static("room/b"), crate::Hops::new()), ] } @@ -767,7 +508,7 @@ mod tests { /// The literal lite-07 stream `js/net` writes for the same announcements, which /// never picks a base. - const JS_LITERAL: &str = "001800000a726f6f6d2f612f63616d01610000029111a2220000001800000a726f6f6d2f612f6d69630161000002b333a2220000020a000002c04444a22200000101010015000006726f6f6d2f620161000002c05555a2220000"; + const JS_LITERAL: &str = "001200000a726f6f6d2f612f63616d0001610000001200000a726f6f6d2f612f6d6963000161000002020000010101000e000006726f6f6d2f620001610000"; #[test] #[cfg(feature = "async")] diff --git a/rs/moq-net/src/lite/datagram.rs b/rs/moq-net/src/lite/datagram.rs index d5fd1bb0e71..349f6c3d870 100644 --- a/rs/moq-net/src/lite/datagram.rs +++ b/rs/moq-net/src/lite/datagram.rs @@ -117,6 +117,9 @@ pub struct Datagrams<'a> { pub broadcast: Path, /// The instance the subscriber expects; see [`crate::origin::Route::instance`]. pub instance: Option, + /// The Node ID of the origin that serves the request, or 0 for whichever the + /// publisher selects. + pub node: u64, pub track: Cow<'a, str>, /// The subscriber's priority. pub priority: i8, @@ -129,6 +132,7 @@ impl Datagrams<'_> { id: self.id, broadcast: self.broadcast, instance: self.instance, + node: self.node, track: Cow::Owned(self.track.into_owned()), priority: self.priority, } @@ -145,12 +149,14 @@ impl<'a> Message<'a> for Datagrams<'a> { let id = r.varint()?; let broadcast = Path::decode(r, version)?; let instance = super::instance::decode_pin(r, version)?; + let node = super::instance::decode_node(r, version)?; let track = Cow::Borrowed(r.str()?); let priority = super::priority::from_wire(r.u8()?); Ok(Self { id, broadcast, instance, + node, track, priority, }) @@ -163,6 +169,7 @@ impl<'a> Message<'a> for Datagrams<'a> { w.varint(self.id)?; self.broadcast.encode(w, version)?; super::instance::encode_pin(w, version, self.instance.as_ref())?; + super::instance::encode_node(w, version, self.node)?; w.string(&self.track)?; w.u8(super::priority::to_wire(self.priority)); Ok(()) @@ -362,6 +369,7 @@ mod test { id: 7, broadcast: Path::from_static("room/alice"), instance: None, + node: 0x7a3f, track: "audio".into(), priority: -3, }; @@ -369,6 +377,7 @@ mod test { let (decoded, _) = Datagrams::decode_slice(&buf, Version::Lite07).unwrap(); assert_eq!(decoded.id, 7); assert_eq!(decoded.broadcast, request.broadcast); + assert_eq!(decoded.node, 0x7a3f); assert_eq!(decoded.track, "audio"); assert_eq!(decoded.priority, -3); assert!(matches!( diff --git a/rs/moq-net/src/lite/fetch.rs b/rs/moq-net/src/lite/fetch.rs index 575136726a6..288655be66a 100644 --- a/rs/moq-net/src/lite/fetch.rs +++ b/rs/moq-net/src/lite/fetch.rs @@ -16,6 +16,9 @@ pub struct Fetch<'a> { /// The instance the subscriber expects; see [`crate::origin::RouteInfo::instance`]. /// Lite07+ only. pub instance: Option, + /// The Node ID of the origin that serves the request, or 0 for whichever the + /// publisher selects. Lite07+ only. + pub node: u64, pub track: Cow<'a, str>, pub priority: i8, /// Reader freshness budget. Lite07+ only; omission uses the publisher's max age alone. @@ -34,6 +37,7 @@ impl Fetch<'_> { Fetch { broadcast: self.broadcast, instance: self.instance, + node: self.node, track: Cow::Owned(self.track.into_owned()), priority: self.priority, max_delay: self.max_delay, @@ -56,6 +60,7 @@ impl<'a> Message<'a> for Fetch<'a> { let broadcast = Path::decode(r, version)?; let instance = super::instance::decode_pin(r, version)?; + let node = super::instance::decode_node(r, version)?; let track = Cow::Borrowed(r.str()?); let priority = super::priority::from_wire(r.u8()?); let max_delay = match version.has_fetch_max_delay() { @@ -76,6 +81,7 @@ impl<'a> Message<'a> for Fetch<'a> { Ok(Self { broadcast, instance, + node, track, priority, max_delay, @@ -95,6 +101,7 @@ impl<'a> Message<'a> for Fetch<'a> { self.broadcast.encode(w, version)?; super::instance::encode_pin(w, version, self.instance.as_ref())?; + super::instance::encode_node(w, version, self.node)?; w.string(&self.track)?; w.u8(super::priority::to_wire(self.priority)); match version.has_fetch_max_delay() { @@ -128,6 +135,7 @@ mod test { fn fetch_sample() -> Fetch<'static> { Fetch { instance: None, + node: 0, broadcast: Path::from_static("room"), track: Cow::Borrowed("video"), priority: 3, @@ -171,6 +179,16 @@ mod test { .into_owned() } + /// Only lite-07 names the origin node; older versions decode as letting the + /// publisher select. + #[test] + fn fetch_node_roundtrips_only_on_lite07() { + let mut msg = fetch_sample(); + msg.node = 0x7a3f; + assert_eq!(fetch_roundtrip(Version::Lite07, &msg).node, 0x7a3f); + assert_eq!(fetch_roundtrip(Version::Lite06, &msg).node, 0); + } + #[test] fn fetch_roundtrips() { for version in [Version::Lite03, Version::Lite04, Version::Lite05] { diff --git a/rs/moq-net/src/lite/instance.rs b/rs/moq-net/src/lite/instance.rs index 9d46a3179f0..336e2357c73 100644 --- a/rs/moq-net/src/lite/instance.rs +++ b/rs/moq-net/src/lite/instance.rs @@ -29,6 +29,23 @@ pub(super) fn decode_pin(r: &mut Decoder<'_>, version: Version) -> Result { } /// Retained state, summed over the node, for budgets and leak checks. +#[cfg(test)] #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub(crate) struct Retained { pub items: usize, @@ -473,6 +479,7 @@ impl Router { self.update(ids); } + #[cfg(test)] /// Ask every neighbor for announcements under `prefix` instead, returning /// the new interest generation. Each neighbor answers on a new stream, and a /// batch from an older one is discarded. @@ -537,11 +544,13 @@ impl Router { .map(|(id, _)| *id) } + #[cfg(test)] /// The declared identity's seqno. pub fn seqno(&self) -> Option { self.origins.values().find(|origin| origin.declared).map(|o| o.seqno) } + #[cfg(test)] /// The declared identity's claim revision. pub fn revision(&self) -> Option { let origin = self.origins.values().find(|origin| origin.declared)?; @@ -849,16 +858,19 @@ impl Router { .collect() } + #[cfg(test)] /// The route this node would offer an upstream link for `id`. pub fn export(&self, id: Id) -> Option> { self.selected(id).filter(|route| !route.down_only) } + #[cfg(test)] /// Whether this node holds a reachable route to `id` but none is feasible. pub fn starved(&self, id: Id) -> bool { self.selected(id).is_none() && self.infeasible(id).next().is_some() } + #[cfg(test)] /// Whether this node, with no usable route to `id`, holds a held route it /// could recover through. pub fn grounded(&self, id: Id) -> bool { @@ -870,21 +882,25 @@ impl Router { self.rx.contains_key(&id) } + #[cfg(test)] /// Whether this node keeps a feasibility distance for `id`. pub fn remembers(&self, id: Id) -> bool { self.fd.contains_key(&id) } + #[cfg(test)] /// How many identities this node holds routes to. pub fn identities(&self) -> usize { self.rx.len() } + #[cfg(test)] /// How many qualified keys this node remembers the forwarded name of. pub fn aliases(&self) -> usize { self.filed.len() } + #[cfg(test)] /// How many region floors this node keeps for `id`. pub fn floors(&self, id: Id) -> usize { self.views.get(&id).map_or(0, |view| view.floors.len()) @@ -918,11 +934,13 @@ impl Router { &self.shown } + #[cfg(test)] /// Whether this node shows `claim` from `id`, and whether it is available. pub fn shows(&self, id: Id, claim: &Path) -> Option { self.shown.get(&(id, claim.clone())).map(|shown| shown.available) } + #[cfg(test)] pub fn retained(&self) -> Retained { let mut out = Retained::default(); for (id, view) in &self.views { diff --git a/rs/moq-net/src/route/proof.rs b/rs/moq-net/src/route/proof.rs index 7bc6fec6772..bdba760f977 100644 --- a/rs/moq-net/src/route/proof.rs +++ b/rs/moq-net/src/route/proof.rs @@ -76,6 +76,7 @@ pub(crate) struct Proof { impl Proof { /// Its encoded size; any other is malformed. + #[cfg(test)] pub const LEN: usize = KEY_LEN + SIGNATURE_LEN; pub fn to_bytes(self) -> Vec { diff --git a/rs/moq-net/tests/integration/support/harness.rs b/rs/moq-net/tests/integration/support/harness.rs index baac0218ad5..2fb68611a8d 100644 --- a/rs/moq-net/tests/integration/support/harness.rs +++ b/rs/moq-net/tests/integration/support/harness.rs @@ -186,7 +186,7 @@ pub async fn connect_mock(opts: MockConnectOptions) -> MockPair { client = client.with_implementation(implementation); } if let Some(publish) = &opts.client_publish { - client = client.with_publisher(&publish.clone().with_proxy(opts.proxy)); + client = client.with_publisher(publish.clone().with_proxy(opts.proxy)); } let client_discovery = opts .client_subscribe @@ -202,7 +202,7 @@ pub async fn connect_mock(opts: MockConnectOptions) -> MockPair { .with_versions(opts.version.into()) .with_extensions(opts.server_extensions); if let Some(publish) = &opts.server_publish { - server = server.with_publisher(&publish.clone().with_proxy(opts.proxy)); + server = server.with_publisher(publish.clone().with_proxy(opts.proxy)); } if let Some(subscribe) = opts.server_subscribe { server = server.with_subscriber(subscribe); From 917341a336ca2f7966da8e3a3f584d6ead0410b0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 02:41:46 -0800 Subject: [PATCH 13/40] fix(net): drop a lite-06 route carrying this node's route identity A relay translating a moq-lite-07 claim for a moq-lite-06 peer starts the chain with the claiming node's route identity hop. When that chain came back to the claiming node over lite-06, nothing in it named the node's own origin hop, so it was filed as one of the node's own claims again and kept the broadcast circling after its publisher left. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/subscriber.rs | 4 ++ rs/moq-net/src/model/origin.rs | 9 ++++ rs/moq-net/tests/integration/mesh_withdraw.rs | 44 +++++++++++++++++++ 3 files changed, 57 insertions(+) diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 18e5fecb62d..b932e1df108 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -865,6 +865,10 @@ impl Subscriber { } None => hops.contains(&self.self_origin), }; + // A claim of this node's that a moq-lite-07 relay translated for a lite-06 peer + // starts with its route identity's hop instead of this origin's. Filed again as + // an own claim, it would keep itself alive once its source left. + let reflected = reflected || self.origin.route_hop().is_some_and(|hop| hops.contains(&hop)); if reflected { tracing::debug!(route = %self.log_path(path), "dropping reflected announce"); return None; diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index bef12aef14b..9902ca030f9 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -2388,6 +2388,15 @@ impl Producer { self.consume().route_node() } + /// The lite-06 hop this node's route identity travels as, once a moq-lite-07 + /// session created the route node: a relay translating this node's claims for a + /// lite-06 peer starts their chain with it. `None` while no session has, which + /// leaves the node uncreated. + pub(crate) fn route_hop(&self) -> Option { + let node = self.shared.lock().route_node.clone()?; + Some(crate::route::node::hop(node.declared())) + } + /// This origin's construction config. pub fn config(&self) -> Config { let shared = self.shared.lock(); diff --git a/rs/moq-net/tests/integration/mesh_withdraw.rs b/rs/moq-net/tests/integration/mesh_withdraw.rs index 21bc17b1fcc..2c6a1be509b 100644 --- a/rs/moq-net/tests/integration/mesh_withdraw.rs +++ b/rs/moq-net/tests/integration/mesh_withdraw.rs @@ -635,3 +635,47 @@ async fn withdrawal_counts(capped: bool) -> (usize, usize, usize) { } (hunted, revived, extra) } + +/// A relay translating a moq-lite-07 claim for a lite-06 peer starts its chain with +/// the claiming node's route identity, not that node's hop, so the claim coming +/// back to that node over lite-06 is a reflection all the same. Filed as one of its +/// own claims again, the broadcast would outlive its publisher, carried around the +/// ring by itself. +#[kio::test] +async fn mixed_version_ring_withdraw_retracts() { + let lite07: Version = "moq-lite-07-wip".parse().unwrap(); + let lite06: Version = "moq-lite-06".parse().unwrap(); + let a = produce_origin(1); + let b = produce_origin(2); + let c = produce_origin(3); + let _ab = peer(lite07, &a, &b).await; + let _bc = peer(lite06, &b, &c).await; + // C publishes to A but hears nothing from it directly, so its only route to A's + // broadcast is B's translation. + let mut options = MockConnectOptions::new(lite06); + options.client_publish = Some(c.clone().with_link(origin::Link::Peer).consume().with_hidden(true)); + options.server_subscribe = Some(a.clone().with_link(origin::Link::Peer)); + let _ca = connect_mock(options).await; + + let mut at_a = watch(a.consume().routes()); + let mut at_c = watch(c.consume().routes()); + let broadcast = a.route("room/x", origin::RouteOptions::hidden()).unwrap(); + broadcast.announce().unwrap(); + kio::clock::sim::idle().await; + assert_eq!(drain(&mut at_a).get("room/x"), Some(&vec![Kind::Start])); + assert_eq!(drain(&mut at_c).get("room/x"), Some(&vec![Kind::Start])); + + drop(broadcast); + // A reflected claim circles the ring at one instant forever, so a regression never + // settles here and fails on the runner's timeout instead. + kio::clock::sim::idle().await; + for (relay, watched) in [("a", &mut at_a), ("c", &mut at_c)] { + let updates = drain(watched); + let kinds = updates.get("room/x"); + assert_eq!( + kinds.and_then(|kinds| kinds.last()), + Some(&Kind::End), + "{relay}: {kinds:?}" + ); + } +} From 28bb2b91f55ed66182a90bf2fae819508cbc3b34 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 02:41:51 -0800 Subject: [PATCH 14/40] docs: lite-07 routes, proxy opt-in, and lite-06 translation Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/cluster.md | 21 +++++++++++++++++---- doc/lib/rs/moq-net.md | 1 + rs/moq-net/MIGRATE.md | 10 ++++++++++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index e1bead6ae27..11beeb67d9f 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -13,8 +13,20 @@ upstream bandwidth. Each broadcast carries the list of relays it passed through. That hop list catches loops and picks the shortest route, and every relay breaks ties the same way so the cluster converges instead of flapping. Both wire protocols -carry it: natively on moq-lite, and via the [cluster extension](/draft/moq-cluster) -on moq-transport. +carry it: natively on moq-lite up to moq-lite-06, and via the +[cluster extension](/draft/moq-cluster) on moq-transport. + +moq-lite-07-wip carries no hop list. Each publishing node declares a random +routing identity, and relays run a distance-vector route to it, loop-free by +Babel's feasibility condition, so every claim of one origin shares that route. +The first relay a request reaches picks the origin node, by a hash of the path +and the node's identity that every relay computes alike, and names it on the +request so later hops follow their own route to that node. A relay running both +versions translates: a moq-lite-06 peer's routes enter the route layer as the +relay's own claims, and lite-07 routes reach a moq-lite-06 peer as a hop list +that starts at their origin node. A relay drops a moq-lite-06 route whose hop +list names its own routing identity, since that is one of its claims coming +back. A track from a moq-transport publisher keeps its Object IDs, object properties, and END\_OF\_GROUP status objects across a moq-lite-07 link, since relays @@ -94,8 +106,9 @@ max_hops = 6 ``` The same setting is `--cluster-max-hops` or `MOQ_CLUSTER_MAX_HOPS`. It covers -moq-lite-03 through moq-lite-07-wip and moq-transport drafts 18 through 22 when -the MoQ Cluster extension is negotiated. A forwarded path counts this relay's +moq-lite-03 through moq-lite-06 and moq-transport drafts 18 through 22 when +the MoQ Cluster extension is negotiated. moq-lite-07-wip routes carry no hop +path, so it does not bound them. A forwarded path counts this relay's own hop even when its peer appends it on receipt. Older versions and peers without hop paths hide their depth, which this setting cannot bound. diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index 90c8bc15e34..bc265aba3b2 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -20,6 +20,7 @@ What you use it for, beyond what the concept page already describes: - **Rank several CDNs.** Dial each with its own subscribe handle, `origin.clone().with_preference(origin::Preference::new(rank))` (higher wins; 0 is neutral and a negative rank demotes), all on one origin. Requests use the preferred link that has a route and fail over to the next; see [multiple CDNs](/concept/moq-lite#multiple-cdns). - **Request one instance, or follow them all.** `consumer.request(path, None)` resolves the instance serving a path now, and fails `Unroutable` only once discovery has synced with no route for it. `consumer.follow(path, None)` hands out each publisher instance serving a path from `available()`: the first once announced, then each one that takes over, `None` once the path goes offline, and whatever serves it next, like JS `follow(path)`. `consumer.announced()` lists paths only as present or absent. A broadcast already resolved stays on the old instance until dropped or its route goes. See [identity and recovery](/concept/identity) to choose between following and holding one instance. - **Discovery follows what you watch.** A client session asks the relay for announcements only under the prefixes its subscribe origin's readers watch, with `announced()`, `request`, or `follow`, stops asking about a prefix 2s after nothing at or beneath it is watched, and asks again on reconnect. A broadcast resolved through a discovered route keeps its path watched until dropped. `request` owns discovery of its path, so no separate watch is needed. A server session, or a client fed into an origin handle marked `with_link(origin::Link::Peer)` or `Link::Upstream`, asks for everything it may, so a relay discovers everything. An HTTP gateway (the moq-hls `Server`, the moq-rtc WHEP router) watches its whole scope while it lives, so the paths its callers name never open a request each. moq-lite-03/04 sessions, whose announcements mark no initial set, still ask for everything. +- **Relay by opting into proxying.** On moq-lite-07 a session sends its peer only the routes to its origin's own claims, so an application never becomes transit for anyone else. A relay built on the library marks the handle it publishes with `origin.clone().with_proxy(true)`, or one consumer with `consumer.with_proxy(true)`, to carry what it learned from other peers too; `moq-relay` does so for every session. Older versions advertise everything either way. - **Dial and accept sockets.** `socket::Endpoint::new(&origin, prefix, id)` announces a hidden endpoint at `/.socket/`, used only to route dials. A session speaks one of two protocols, each with its own dial and accept. `accept_tcp()` and `accept_track()` yield an `Incoming` dial to `accept` or `reject` (dropping it rejects). `dial_tcp(peer)` and an accepted `tcp` dial yield a `socket::TcpStream`, a `group::Producer` and `group::Consumer` pair carrying one reliable, ordered byte pipe each way (finish is FIN, abort is RST, and `ready` paces the writer to the reader); `dial_track(peer, info)` and an accepted `track` dial yield a `socket::TrackStream`, this side's `track::Producer` and a `track::Subscriber` of the peer's half, subscribed reliably and in order until the application updates it, one message per group. A dial resolves once the peer accepts, and an accept checks that the claimed dialer really dialed this endpoint, waiting for its endpoint while the dial stays open. Every hop needs moq-lite-07 or moq-transport: while the origin has an older moq-lite client session whose publish half doesn't opt into hidden broadcasts (`with_hidden(true)`), dials and accepts fail with `Unsupported`. A dial to a peer behind such a hop waits, as for a peer not online yet. Session tracks are never announced or listed. See [Sockets](/concept/socket). - **One track per name.** Concurrent subscriptions share one request and producer. The application owns sequence numbers: the broadcast keeps nothing of an ended track, so a producer that replaces it starts at 0. A name always means the same content, so the replacement continues past its predecessor's groups with `create_group` at an explicit sequence. Readers of an aborted track resume onto the replacement when they read through an origin request, whose route always names an instance; otherwise their subscription ends with the error. - **Authored tracks default to a zero `max_age`.** `track::Info::default()` sets `max_age` to `Some(Duration::ZERO)`, the live edge only; a publisher that wants readers to reach history sets a longer bound, or `with_max_age(None)` for no publisher limit. Media tracks from `hang::container::track_info` declare `hang::container::MAX_AGE` (30 seconds). `max_age` is how long a superseded group stays fresh, never the newest group; it frees no bytes, which the cache pool's expiry reclaims. Received metadata keeps absence as no limit, and an origin's `cache_duration` can only cap it lower. diff --git a/rs/moq-net/MIGRATE.md b/rs/moq-net/MIGRATE.md index 13e37ae6884..6bb0cd3d322 100644 --- a/rs/moq-net/MIGRATE.md +++ b/rs/moq-net/MIGRATE.md @@ -4,6 +4,16 @@ How to update your code for each breaking change, newest first. ## Unreleased +### moq-lite-07 routes through the route layer ([#8023](https://github.com/moq-dev/moq/pull/8023)) + +moq-lite-07-wip (opt-in) replaces the hop list with routes: each origin node declares a random routing identity, every session on an origin feeds one router, and a request names the origin node the first hop picked. Older versions are unchanged. On moq-lite-07: + +- A relay built on the library must opt into proxying, or its peers are sent only the routes to its origin's own claims: publish `origin.clone().with_proxy(true)`, or mark one consumer with `consumer.with_proxy(true)`. `moq-relay` does this for every session; an application publishing only its own broadcasts leaves it off. +- A link cost of 0 is refused at SETUP: `Client::with_cost(0)` fails the session. Use 1, the price of an unpriced link. +- A route carries no hop chain. `RouteInfo::hops()` on a route learned over moq-lite-07 names only its origin node, and `origin::Config::max_hops` does not bound it: routes are loop-free by feasibility instead. +- ANNOUNCE_TAKEOVER is never sent. A job answering a claim at the claim's own path arrives as a restart, and one beneath it as a start, until [announce takeover](https://github.com/moq-dev/moq/blob/main/quest/m2/announce-takeover.md) carries the claim's instance onto the job. +- A pool's paths spread by a hash of the path and each origin node's routing identity, which every relay computes alike, rather than by publisher instance, so every relay picks the same origin for a path. + ### Socket readers start at the beginning New `track` socket sessions subscribe from group 0, and `tcp` sessions join their From 6662e4d6609402aae06755c266f5545777fb8e31 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 02:59:32 -0800 Subject: [PATCH 15/40] test(net): a lost next hop restarts an epochless route, and lite-07 carries on One publisher reaches a relay through two neighbors and the preferred one goes away. Wires that mint an instance per announcement restart the route onto the standby's; lite-07 carries the publisher's instance and already serves the claim through both, so it continues with nothing announced. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/tests/integration/instance.rs | 74 ++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/rs/moq-net/tests/integration/instance.rs b/rs/moq-net/tests/integration/instance.rs index f48942da2c6..e1063c3c107 100644 --- a/rs/moq-net/tests/integration/instance.rs +++ b/rs/moq-net/tests/integration/instance.rs @@ -100,3 +100,77 @@ async fn moq_transport_mints_per_announcement() { job_reannounced(version).await; } } + +/// One publisher reaches a relay through two neighbors, and the one the relay prefers +/// goes away. lite-07 carries the publisher's instance through both, so the relay keeps +/// serving the same stream through the other. Every other wire mints one per announcement, so changing +/// the next hop, with the same origin and no epoch, restarts the route rather than +/// updating it in place. +async fn next_hop_lost(version: &str) { + let version: Version = version.parse().unwrap(); + let publisher = produce_origin(1); + let preferred = produce_origin(2); + let standby = produce_origin(3); + let relay = produce_origin(4); + let _job = publisher + .route("job", moq_net::origin::RouteOptions::default()) + .unwrap(); + let _to_preferred = link(version, &publisher, &preferred).await; + let _to_standby = link(version, &publisher, &standby).await; + let mut announced = relay.consume().routes(); + let first_hop = link( + version, + &preferred, + &relay.clone().with_preference(origin::Preference::new(1)), + ) + .await; + let first = started(&mut announced).await; + let _second_hop = link(version, &standby, &relay).await; + if let Ok(event) = kio::clock::timeout(Duration::from_secs(1), announced.next()).await { + panic!("{version}: the standby route changed the preferred one: {event:?}"); + } + + first_hop.server.abort(moq_net::Error::Cancel); + first_hop.client.abort(moq_net::Error::Cancel); + let event = kio::clock::timeout(Duration::from_secs(10), announced.next()).await; + match version.to_string().as_str() { + // The relay already serves the claim through every feasible route, so the + // standby carries on with nothing to announce. + "moq-lite-07-wip" => { + if let Ok(event) = event { + panic!("{version}: expected the same stream to continue, got {event:?}"); + } + kio::clock::timeout(Duration::from_secs(10), relay.consume().request("job", None)) + .await + .expect("request hung") + .expect("the standby serves the claim"); + } + _ => match event.expect("announce hung").expect("announce ended") { + origin::RouteEvent::Restart(announce) => { + assert_ne!( + announce.route.instance(), + &first, + "{version}: spliced onto the old stream" + ) + } + other => panic!("{version}: expected a restart, got {other:?}"), + }, + } +} + +#[kio::test] +async fn next_hop_lost_lite07() { + next_hop_lost("moq-lite-07-wip").await; +} + +#[kio::test] +async fn next_hop_lost_restarts_older_lite() { + for version in ["moq-lite-06", "moq-lite-03"] { + next_hop_lost(version).await; + } +} + +#[kio::test] +async fn next_hop_lost_restarts_moq_transport() { + next_hop_lost("moq-transport-22").await; +} From 92cc680fd6bb47f428031ad771926c657165a0c5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 03:02:59 -0800 Subject: [PATCH 16/40] test(net): a lite-07 route and its claims count against the announce cap Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/tests/integration/request_caps.rs | 28 ++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/rs/moq-net/tests/integration/request_caps.rs b/rs/moq-net/tests/integration/request_caps.rs index 6e3928fc3fa..00817bddcd1 100644 --- a/rs/moq-net/tests/integration/request_caps.rs +++ b/rs/moq-net/tests/integration/request_caps.rs @@ -184,3 +184,31 @@ async fn an_announce_past_the_cap_closes_the_session() { ); } } + +/// On moq-lite-07 the peer's route takes a slot as well as each claim under it, so +/// whatever a peer makes the route layer hold stays under the cap: here its route and +/// first claim fill it, and the second claim closes the session. +#[kio::test] +async fn a_lite07_claim_past_the_cap_closes_the_session() { + let version: Version = "moq-lite-07-wip".parse().unwrap(); + let client = produce_origin(1); + let server = produce_origin(2); + let a = client.route("a", moq_net::origin::RouteOptions::hidden()).unwrap(); + let b = client.route("b", moq_net::origin::RouteOptions::hidden()).unwrap(); + a.announce().unwrap(); + b.announce().unwrap(); + + let mut options = MockConnectOptions::new(version); + options.client_publish = Some(client.consume()); + options.server_subscribe = Some(server.clone()); + options.server_limits = Some(limits(2)); + let pair = connect_mock(options).await; + + let err = kio::clock::timeout(TIMEOUT, pair.client.closed()) + .await + .expect("the session stayed open"); + assert!( + matches!(err, moq_net::Error::Session(moq_net::SessionError::TooManyRequests)), + "closed with {err}" + ); +} From db9b5e331d6cdcc27a72ca7b76e611c7728ffbe9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 03:02:59 -0800 Subject: [PATCH 17/40] chore(quest): narrow routes/session.md to bounded per-stream state Co-Authored-By: Claude Opus 5.5 --- quest/m2/cluster-routing/routes/session.md | 184 +++++---------------- 1 file changed, 44 insertions(+), 140 deletions(-) diff --git a/quest/m2/cluster-routing/routes/session.md b/quest/m2/cluster-routing/routes/session.md index 53002f8076c..cd06f1bd925 100644 --- a/quest/m2/cluster-routing/routes/session.md +++ b/quest/m2/cluster-routing/routes/session.md @@ -1,150 +1,54 @@ -# [XL] Route session integration +# [L] Route session integration: bounded state ## Goal -One `Router` per origin node, fed by every lite-07 session, drives that -node's origin and every announce stream it answers. Its shown claims and -selected routes become the origin's routes, its outbox becomes each -stream's ROUTE and ANNOUNCE messages, and the first hop pins each request -to the identity it selected. A lite-06 peer is translated at the relay. A -peer can make the router hold only bounded state. The configuration -changes the contract decided ship with it. - -lite-07's live wire switches to the route layer that -`drafts/draft-lcurley-moq-lite.md` specifies, in Rust: ANNOUNCE_START, ANNOUNCE_UPDATE, ANNOUNCE_RESTART, and -ANNOUNCE_TAKEOVER name a Route ID and carry the Announce Cost in place of -the Hop ID list (`HopsRef`, `Hop Base`/`Hop Keep`, and `cluster.max_hops` -go from lite-07); ANNOUNCE_OK loses its Hop ID and SETUP its Hop parameter; -TRACK, TRACKS, SUBSCRIBE, FETCH, and DATAGRAMS carry the Node ID; a Cost -parameter of 0 is refused; the subscriber sends ROUTE_REQUEST and echoes -ROUTE_END. The new messages' codecs already exist (`AnnounceBroadcast`'s -route variants and `AnnounceReply`); until this lands the subscriber -refuses route messages. A hop-less ANNOUNCE cannot run before the router -drives sessions, which is why the switch lands here. +The route layer drives every lite-07 session +([#8023](https://github.com/moq-dev/moq/pull/8023)): one `route::Node` per +origin, Route IDs on the wire, requests pinned to the first hop's origin node, +lite-06 translated at the relay. What remains: the state a node keeps per +stream it answers is bounded by the origins that stream reads, not the whole +node table, and the public API drops what the route layer replaced. ## Plan - Scope route state to interested origins, not the CDN's whole node table. - Send a route before any announcement referencing it. The epic's handoff contract - defines held routes, letting go, and collection. -- ✅ Decided 2026-10-15 (maintainer): retract and re-file when a session's - grant union crosses the scoped boundary in either direction (a `**` token - added, or its expiry). Every forwarded route the session delivered is - retracted, and its current advertisements are re-filed under the new key, - so no route, feasibility distance, or alias stays under the old one. - Expiry cannot be refused, so refusing the transition was not an option. -- Only the first hop selects the origin. It keeps `route_order`'s shape over - the origin nodes announcing the path: longest prefix, newest epoch (within - one instance only, per [Instance and epoch](https://github.com/moq-dev/moq/pull/7489), - decided 2026-10-30), the link's priority (`origin::Producer::with_preference`), seed plus route - metric, then a rendezvous hash of the path and the identity, and the newest - connection only for an otherwise identical route. The hash drops the hop - chain, which differs per node, so every node agrees on a pool member - (`pool_ties_agree_only_on_a_path_and_identity_hash`). The request then names - that identity, and every later hop follows its own route to it, so - forwarding needs a route but not the claim. -- Reselecting the origin at every hop loops although each origin's routes are - loop-free (RFC 8966 s2.7): a ring of same-epoch origins whose relays - prioritize different links, and nested wildcards where a relay has not - heard the specific claim, both cycle past requester exclusion; joining - another relay's subscription by path or instance deadlocks; and without an - instance a later hop splices another publisher under the requester's - announcement with no Restart (`local_priorities_loop_path_only_requests`, - `overlapping_wildcards_loop_where_specific_claims_are_unheard`, - `aggregation_cycles_starve_without_a_forwarding_constraint`, - `reselection_without_an_instance_splices_another_publisher`, - `sweep_forwarding_progress`). A relay therefore joins a forwarded request - only to a subscription following its route to the same identity. When that - identity becomes unreachable the forwarded request ends, and the first hop - re-resolves, resuming only on a matching epoch. A request carrying a cost - bound instead also never loops, but stalls where a transit hop has not - heard the claim and still splices epochless sources. -- Mixed versions: a lite-06 peer keeps today's path vector, translated at the - relay that speaks both, for the rollout window only. -- Interest changes: key the router by announce stream, not session. The old - stream stays open, its routes counting and its batches discarded, until - the new stream has answered, so no route drops in between. A stream is - just another neighbor key to the router. -- A batch carries changes against the last batch on its stream. Restricting - a view to an interest is still linear in that origin's claims per change; - make it incremental here. -- Decided 2026-11-05 (maintainer, /quest-plan, the epic's Plan items 10 to - 12): each stream's floor is a whole-interest marker. Project the router's - per-region view onto it, certifying nothing unless the view knows the - route's claims across the stream's interest at one revision; present per - route only the most specific claim covering the interest, moved rather - than ended when it is withdrawn; and certify absence only for what the - stream's `Hidden` filter shows. Rerun the handoff sweeps through that - projection: a scope a relay lost now goes offline downstream only on - ROUTE_END, the accepted cost, so update the scoped fixtures' expectations - to it, while no sweep may revive a known withdrawal or report a false END. - Also cover a fresh claim that arrives only on an uncertified stream while - an older certified floor on another stream proves it absent - ([review](https://github.com/moq-dev/moq/pull/7764#discussion_r4485081975)): - the merge rule hides it until the certifying peer refreshes or lets go, - and the test pins that bound. -- Bound what a peer makes the router hold (routes, claims, views per - session) by design, as the session caps announcements today. -- Public API: the route-change surface on `broadcast::Route` and its - bindings will likely change, `Hops` and saturating `Cost` go, zero-cost - links are refused (update `?cost=0` examples and report the breaking - configuration change), a per-connection `proxy` preference is added, and - `cluster.id`/`--cluster-id` is removed rather than kept as an unused label - (decided 2026-10-13). `Preference` continues to rank feasible transit - routes; report it. -- Open, for the implementer to settle and record: how much of today's route - trie, fronts, and `route_order` survives intact. - -Decided 2026-10-27 (maintainer): the route layer deletes the epochless instance -machinery in `rs/moq-net/src/model/origin.rs`: `Instance::Route(generation)` -and its renewals, `restarted_since`, the held-restart Release for epochless -routes, and Selection retirement, leaving Follow's same-epoch -Update/Restart split as the only takeover rule. Once each origin is its own -ANNOUNCE and the first hop pins the request to its identity, the first hop's -route is the instance, so the per-hop renewal has nothing left to protect. -An epochless change of serving route stays a Restart, including the -same-origin next-hop change with no epoch (test it), and the routing -contract's epochless fixtures keep passing. Minting an epoch for every route -instead was tried and rejected -([moq-dev/moq#6948](https://github.com/moq-dev/moq/pull/6948)): each relay -mints at its own time, so mint order outranks cost, preference, and the -local-broadcast tie and relays disagree on a pool path's winner, while -ranking a minted epoch last keeps the renewal it was meant to delete. - -Moved 2026-10-30 (maintainer, the [route epic](/quest/m1/api-audit/route/README.md)): -[Instance and epoch](https://github.com/moq-dev/moq/pull/7489) now deletes that -epochless machinery, so the route layer no longer does. A session mints -an `instance` (`~` plus random bits) for any route that -arrives without one, and newest-wins compares epochs only within one -instance, so mint order no longer outranks cost or preference. The rest of -the 2026-10-27 decision stands. The broadcast epoch fields here become that -publisher instance and its per-instance epoch, in place of the UUIDv7 -epoch. The route layer's own origin-node identity, with its seqno and -revision, stays separate ("keep administrative identity and broadcast epoch -separate", in the epic). - -Decided 2026-11-05 (maintainer, /quest-plan): a scoped peer's identities -are keyed by a route key the peer proves in its lite-07 SETUP: the session's -`PeerSetup::route_key` once its SETUP arrives, and `None` for a peer that proved -none. Relays set theirs with `--cluster-route-key`. Every identity a scoped, proven peer sends, its own included, is qualified -under its proven key, so relays sharing a key file fail over as one. A peer that proves no -key gets a per-session key. `Scope::qualify` still takes the key from the -caller, so this quest does not wait on the proof. Options shown: - -- ✅ A route key the peer proves once per session in SETUP, before the lite-07 cut - (2026-11-05, maintainer, /quest-plan; supersedes the per-announce proof option) -- A route label carried in the grant (the earlier recommendation), with the - proven key added after the cut -- A per-session key now -- A key the relay operator configures per scoped remote - -Research behind it: proving a neighbor's own key cannot remove qualification, -since a public key is public and a scoped peer could still forward a victim's -key with an inflated seqno; origin-signed routes still leave cost lies and -blackholing, at the price of per-update signatures and expiry refreshes. Keyed -by a proven key, every remaining attack stays inside the forger's grant, with -no grant field, no issuer-coordinated labels, and support for token-less -admission. + Today the router advertises every route to every watching stream, because + the epic's handoff contract (`routes/README.md`, "a sender that certifies + anything for a route answers every stream") has a sender answer even a + stream whose interest none of the route's regions meets. Scoping needs + that sentence amended (see Waiting). + What a peer makes the router hold is already capped: each route and claim + it sends takes one of the session's announce slots + (`a_lite07_claim_past_the_cap_closes_the_session`). What this bounds is + the state this node keeps per stream it answers. +- Present incrementally. `View::present` walks every claim of a route per + change and per interest (`Router::advertise`), so a large origin costs + linear work per update. Do it with the scoping change, which reshapes the + same per-stream presentation. +- Public API: `Hops`, `RouteInfo::hops`, and the saturating `Cost` stay while + lite-06 and the moq-transport cluster extension still carry hop chains. + Remove them, or hide them behind those wires, when the rollout window + closes (see Waiting). + +## Waiting + +- Maintainer: amend the handoff contract's "a sender that certifies anything + for a route answers every stream" (`routes/README.md`), or keep it. It is a + decided contract, so an agent does not change it. Options: **send a route + only while it meets the stream's interest (recommended)**: a route starts on + a stream before its first claim in or covering the interest, and when its + last such claim goes it sends a floor certifying the withdrawal, then ends + the route, so withdrawals still END and per-stream state is bounded by the + origins a peer reads, at a ROUTE_END round trip when an origin leaves an + interest; or **keep answering every stream** and bound only by per-peer + caps, simpler but carrying the whole node table on every stream. Done when + the README says which. +- Maintainer: decide when `Hops`, `RouteInfo::hops`, and the saturating + `Cost` leave the public API. Every older wire still fills them, so an agent + cannot drop them without picking the rollout's end. Options: **when lite-06 + retires (recommended)**, since removing them now only moves them behind a + private type; or **now**, keeping a private chain type for the older wires. + Done when the Plan item above names the point. ## Related From 2c9c3964c1f6b44c4d605a06804c88a3da484340 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 03:07:27 -0800 Subject: [PATCH 18/40] chore(net): allow route_hop unused without the async session Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/origin.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index c6bc7fe435b..a3d52a3b704 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -2392,6 +2392,7 @@ impl Producer { /// session created the route node: a relay translating this node's claims for a /// lite-06 peer starts their chain with it. `None` while no session has, which /// leaves the node uncreated. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn route_hop(&self) -> Option { let node = self.shared.lock().route_node.clone()?; Some(crate::route::node::hop(node.declared())) From a627b584b67d4cbf1a5b8393bb2c32fa1e540c2f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 08:48:40 -0800 Subject: [PATCH 19/40] feat(net): lite-07 carries ANNOUNCE_TAKEOVER on Route IDs A route-layer claim records the claim it took over and the instance that one served. The origin's own takeovers feed it, a relay keeps it from the ANNOUNCE_TAKEOVER it received, and an answering stream sends the claim as a takeover of its predecessor when the peer held that predecessor, under the same route, before the batch, and every claim it holds at that path serves the same instance. The receiver forwards the claim with its predecessor's route, and in place keeps the predecessor's sources. The legacy publisher and subscriber takeover paths were unreachable once lite-07 routed, so they go. Co-Authored-By: Claude Opus 5.5 --- quest/m2/announce-takeover.md | 6 +- rs/moq-net/MIGRATE.md | 1 - rs/moq-net/src/lite/publisher.rs | 55 +----- rs/moq-net/src/lite/publisher/routed.rs | 143 +++++++++++---- rs/moq-net/src/lite/subscriber.rs | 165 ++---------------- rs/moq-net/src/lite/subscriber/routed.rs | 118 ++++++++++++- rs/moq-net/src/model/origin.rs | 28 +-- rs/moq-net/src/route/node.rs | 38 ++-- .../tests/integration/announce_takeover.rs | 27 +-- 9 files changed, 295 insertions(+), 286 deletions(-) diff --git a/quest/m2/announce-takeover.md b/quest/m2/announce-takeover.md index 710e5ae1799..5323a2d5111 100644 --- a/quest/m2/announce-takeover.md +++ b/quest/m2/announce-takeover.md @@ -44,8 +44,10 @@ instance rule already covers the claim case and needs no wire message, so this quest removes the predecessor machinery and grows to [XL]. - **Removing ANNOUNCE_TAKEOVER.** Delete the 0x5 message and its codec in - `rs/moq-net/src/lite/announce.rs`, `compress.rs`, `publisher.rs`, - `subscriber.rs` and `replay.rs`; the predecessor tracking in + `rs/moq-net/src/lite/announce.rs`, `compress.rs`, `publisher/routed.rs`, + `subscriber/routed.rs` and `replay.rs`; the route layer's claim lineage + (`Claim::continues` in `rs/moq-net/src/route/node.rs`, fed by `OwnClaims` in + `origin.rs`); the predecessor tracking in `rs/moq-net/src/model/origin.rs` and `front.rs` (the `seen` stamps in `OriginConsumerState` and `PendingFront::via`); relay forwarding in `rs/moq-relay`; and `rs/moq-net/tests/integration/announce_takeover.rs`. diff --git a/rs/moq-net/MIGRATE.md b/rs/moq-net/MIGRATE.md index b5c634ff3ab..61f04b2cb1d 100644 --- a/rs/moq-net/MIGRATE.md +++ b/rs/moq-net/MIGRATE.md @@ -11,7 +11,6 @@ moq-lite-07-wip (opt-in) replaces the hop list with routes: each origin node dec - A relay built on the library must opt into proxying, or its peers are sent only the routes to its origin's own claims: publish `origin.clone().with_proxy(true)`, or mark one consumer with `consumer.with_proxy(true)`. `moq-relay` does this for every session; an application publishing only its own broadcasts leaves it off. - A link cost of 0 is refused at SETUP: `Client::with_cost(0)` fails the session. Use 1, the price of an unpriced link. - A route carries no hop chain. `RouteInfo::hops()` on a route learned over moq-lite-07 names only its origin node, and `origin::Config::max_hops` does not bound it: routes are loop-free by feasibility instead. -- ANNOUNCE_TAKEOVER is never sent. A job answering a claim at the claim's own path arrives as a restart, and one beneath it as a start, until [announce takeover](https://github.com/moq-dev/moq/blob/main/quest/m2/announce-takeover.md) carries the claim's instance onto the job. - A pool's paths spread by a hash of the path and each origin node's routing identity, which every relay computes alike, rather than by publisher instance, so every relay picks the same origin for a path. ### `track::Consumer::unmetered` reads outside any stats session diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 09cbdc8dbc0..7b5d610bff0 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -1371,43 +1371,6 @@ impl AnnounceRun { Ok(()) } - /// Advertise `suffix` as continuing the peer's live `predecessor` (suffix and id): - /// ANNOUNCE_TAKEOVER, which retires the predecessor when it held `suffix`. - #[expect(clippy::too_many_arguments, reason = "private plumbing mirroring start and restart")] - fn takeover( - &mut self, - stream: &mut Stream, - predecessor: (crate::Path, u64), - suffix: crate::Path, - hops: Hops, - cost: crate::origin::Cost, - route: Served, - absolute: &crate::Path, - ) -> Result<(), Error> { - tracing::debug!(route = %absolute, predecessor = %predecessor.0, "takeover"); - let (id, wire) = self.encoder.takeover(predecessor.1, suffix.clone()); - self.live.insert( - suffix, - Advertised { - id: Some(id), - hops, - cost, - served: route.clone(), - }, - ); - let (instance, epoch) = route; - stream.writer.buffer(&lite::AnnounceBroadcast::Takeover { - predecessor: predecessor.1, - suffix: wire, - route: 0, - instance, - epoch, - cost, - })?; - self.sent(stream)?; - Ok(()) - } - /// Replace the peer's advertisement for `suffix` with another instance or epoch: /// ANNOUNCE_RESTART on lite-07, an end and a fresh start before it. fn restart( @@ -1662,26 +1625,10 @@ impl AnnounceRun { continue; } - // A takeover names its predecessor's id, so only one the peer holds. Elsewhere it + // Only lite-07 carries a takeover, and it answers from the route layer. Here one // is another instance, in place or not. - let taken = predecessor - .as_ref() - .filter(|_| self.version.has_announce_takeover()) - .filter(|predecessor| **predecessor == suffix || !self.live.contains_key(&suffix)) - .and_then(|predecessor| Some((predecessor.clone(), self.live.get(predecessor)?.id?))); let restart = restart || predecessor.as_ref() == Some(&suffix); match self.outgoing(&update.route, &absolute) { - Some((hops, cost)) if let Some((predecessor, id)) = taken => { - self.takeover( - stream, - (predecessor, id), - suffix, - hops, - cost, - served(&update.route), - &absolute, - )?; - } Some((hops, cost)) if restart && self.live.contains_key(&suffix) => { self.restart(stream, suffix, hops, cost, served(&update.route), &absolute)?; } diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index 74ed7ad79de..865c15b9b23 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -7,7 +7,7 @@ //! peer cannot tell what was left out from what is gone. use std::{ - collections::{BTreeMap, HashMap}, + collections::{BTreeMap, BTreeSet, HashMap}, task::{Poll, ready}, }; @@ -278,8 +278,9 @@ impl Answer { paths.push(path); } sent.floor = floor; + let mut started = BTreeSet::new(); for path in paths { - self.sync(stream, id, path)?; + self.sync(stream, id, path, &mut started)?; } self.floor(stream, id) } @@ -316,47 +317,87 @@ impl Answer { } /// Bring the peer's claim at `path` for the route to `id` in line with the node's. - fn sync(&mut self, stream: &mut Stream, id: route::Id, path: Path) -> Result<(), Error> { + /// + /// `started` collects the claims this pass started: a claim takes over only one the + /// peer held before, so a snapshot replays no takeover. + fn sync( + &mut self, + stream: &mut Stream, + id: route::Id, + path: Path, + started: &mut BTreeSet, + ) -> Result<(), Error> { let sent = self.routes.get(&id).expect("only a held route syncs"); let wanted = sent.claims.get(&path).filter(|_| self.shows(&path)).cloned(); let wire = sent.wire; + let taken = (wanted.as_ref()) + .and_then(|claim| self.continued(id, &path, claim)) + .filter(|(taken, _)| !started.contains(taken)); let key = (id, path); match (self.live.get_mut(&key), wanted) { (None, Some(claim)) => { - tracing::debug!(route = %key.1, node = id, "announce"); let suffix = key .1 .strip_prefix(&self.prefix) .expect("shown claims are under the prefix"); - let (announce, suffix) = self.encoder.start(suffix); - let announce = announce.expect("lite-07 assigns announce ids"); - stream.writer.buffer(&Msg::Active { - suffix, - route: wire, - instance: Some(claim.instance.clone()), - epoch: claim.epoch, - hops: crate::Hops::new(), - cost: origin::Cost::new(claim.cost), - })?; + let announce = match taken { + Some((taken, predecessor)) => { + tracing::debug!(route = %key.1, node = id, predecessor = %taken, "takeover"); + self.takeover(stream, wire, predecessor, suffix, &claim)? + } + None => { + tracing::debug!(route = %key.1, node = id, "announce"); + let (announce, suffix) = self.encoder.start(suffix); + stream.writer.buffer(&Msg::Active { + suffix, + route: wire, + instance: Some(claim.instance.clone()), + epoch: claim.epoch, + hops: crate::Hops::new(), + cost: origin::Cost::new(claim.cost), + })?; + announce.expect("lite-07 assigns announce ids") + } + }; + started.insert(key.1.clone()); self.live.insert(key, (announce, claim)); } (Some((announce, held)), Some(claim)) => { - if held.instance != claim.instance || held.epoch != claim.epoch { - tracing::debug!(route = %key.1, "restart"); - stream.writer.buffer(&Msg::Restart { - id: *announce, - instance: Some(claim.instance.clone()), - epoch: claim.epoch, - cost: origin::Cost::new(claim.cost), - })?; - } else if held.cost != claim.cost { - stream.writer.buffer(&Msg::Update { - id: *announce, - hops: crate::Hops::new(), - cost: origin::Cost::new(claim.cost), - })?; - } - *held = claim; + let announce = *announce; + let restart = held.instance != claim.instance || held.epoch != claim.epoch; + let reprice = held.cost != claim.cost; + let replacement = match taken { + // In place: a new Announce ID replaces the one it continues. + Some((taken, predecessor)) if taken == key.1 => { + tracing::debug!(route = %key.1, node = id, "takeover"); + let suffix = key + .1 + .strip_prefix(&self.prefix) + .expect("held claims are under the prefix"); + self.takeover(stream, wire, predecessor, suffix, &claim)? + } + _ if restart => { + tracing::debug!(route = %key.1, "restart"); + stream.writer.buffer(&Msg::Restart { + id: announce, + instance: Some(claim.instance.clone()), + epoch: claim.epoch, + cost: origin::Cost::new(claim.cost), + })?; + announce + } + _ => { + if reprice { + stream.writer.buffer(&Msg::Update { + id: announce, + hops: crate::Hops::new(), + cost: origin::Cost::new(claim.cost), + })?; + } + announce + } + }; + self.live.insert(key, (replacement, claim)); } (Some(_), None) => { let (announce, _) = self.live.remove(&key).expect("matched above"); @@ -368,6 +409,45 @@ impl Answer { Ok(()) } + /// Send `claim` at `suffix` under the route `wire` as an ANNOUNCE_TAKEOVER of the + /// peer's live `predecessor`, returning its Announce ID. + fn takeover( + &mut self, + stream: &mut Stream, + wire: u64, + predecessor: u64, + suffix: Path, + claim: &node::Claim, + ) -> Result { + let (announce, suffix) = self.encoder.takeover(predecessor, suffix); + stream.writer.buffer(&Msg::Takeover { + predecessor, + suffix, + route: wire, + instance: Some(claim.instance.clone()), + epoch: claim.epoch, + cost: origin::Cost::new(claim.cost), + })?; + Ok(announce) + } + + /// The claim the peer holds that `claim` at `path` takes over, and its Announce ID. + /// + /// Only the claim it continues, under the same route at the instance it continued, + /// and only while every claim the peer holds at that path serves the same instance: + /// otherwise the peer cannot tell which one it resolved through. + fn continued(&self, id: route::Id, path: &Path, claim: &node::Claim) -> Option<(Path, u64)> { + let (taken, instance) = claim.continues.as_ref()?; + if !path.has_prefix(taken) { + return None; + } + let (predecessor, held) = self.live.get(&(id, taken.clone()))?; + let alike = self.routes.keys().all(|other| { + (self.live.get(&(*other, taken.clone()))).is_none_or(|(_, other)| other.instance == *instance) + }); + (held.instance == *instance && alike).then(|| (taken.clone(), *predecessor)) + } + /// The grant changed: end what it no longer covers, start what it now does, and /// floor every route again, which certifies only while the grant covers the prefix. fn refilter(&mut self, stream: &mut Stream) -> Result<(), Error> { @@ -375,8 +455,9 @@ impl Answer { ids.sort_unstable(); for id in ids { let paths: Vec = self.routes[&id].claims.keys().cloned().collect(); + let mut started = BTreeSet::new(); for path in paths { - self.sync(stream, id, path)?; + self.sync(stream, id, path, &mut started)?; } if self.routes[&id].floored { self.floor(stream, id)?; diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index b932e1df108..39931d853b1 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -599,35 +599,10 @@ impl Subscriber { &mut run.announced, )?; } - lite::AnnounceBroadcast::Takeover { - predecessor, - suffix, - instance, - epoch, - cost, - .. - } => { - // Resolved like a start against the live predecessor, which it retires when - // it takes the predecessor's own suffix. An unknown or retired predecessor is - // a protocol violation. - let (previous, suffix) = run.decoder.takeover(predecessor, suffix)?; - let previous = prefix.join(&previous)?; - let path = prefix.join(&suffix)?; - // Lite-07 carries no hop chain; its claims name their route instead. - self.takeover_announce( - previous, - path, - instance.map(|instance| (instance, epoch)), - crate::Hops::new(), - cost, - run.link_cost, - run.responder_origin, - &mut run.announced, - )?; - } lite::AnnounceBroadcast::Skipped | lite::AnnounceBroadcast::Synced => {} - // Only lite-07 carries the route layer, which `routed` handles above. - lite::AnnounceBroadcast::RouteStart { .. } + // Only lite-07 carries the route layer and takeovers, which `routed` handles above. + lite::AnnounceBroadcast::Takeover { .. } + | lite::AnnounceBroadcast::RouteStart { .. } | lite::AnnounceBroadcast::RouteUpdate { .. } | lite::AnnounceBroadcast::RouteHold { .. } | lite::AnnounceBroadcast::RouteEnd { .. } @@ -683,56 +658,7 @@ impl Subscriber { // outside our scope, so don't serve it. Reflections are already // filtered above. let route = self.announced_route(&path, hops, cost, link_cost, responder_origin, announced); - Ok(announced.offer(self, path, route, None)) - } - - /// Handle an ANNOUNCE_TAKEOVER: a new advertisement at `path` continuing what the peer's - /// live advertisement at `previous`, at or above it, served beneath it. - /// - /// At `previous` itself it replaces that advertisement like a restart; elsewhere it - /// starts one, and the predecessor stays. Either way the new route enters the origin - /// before anything leaves, taking over from the route the predecessor attached, if it - /// did, so what resolved through that one beneath `path` carries on. - #[expect( - clippy::too_many_arguments, - reason = "private plumbing; a single-use argument struct would only rename the parameters" - )] - fn takeover_announce( - &mut self, - previous: Path, - path: Path, - // See `start_announce`. - instance: Option<(crate::Instance, u64)>, - hops: crate::Hops, - cost: crate::origin::Cost, - link_cost: u64, - responder_origin: Option, - announced: &mut Announced, - ) -> Result { - // The continued route lies beneath its predecessor. - if !path.has_prefix(&previous) { - return Err(Error::ProtocolViolation); - } - if path == previous { - announced.restart(&path, instance); - } else { - // Like a start: one current advertisement per prefix. - if announced.contains(&path) { - return Err(Error::ProtocolViolation); - } - announced.reserve(path.clone(), instance)?; - } - - let Some(hops) = self.chain(&path, hops, responder_origin) else { - if path == previous { - announced.declined(path); - } - return Ok(false); - }; - - tracing::debug!(route = %self.log_path(&path), predecessor = %self.log_path(&previous), "takeover"); - let route = self.announced_route(&path, hops, cost, link_cost, responder_origin, announced); - Ok(announced.offer(self, path, route, Some(&previous))) + Ok(announced.offer(self, path, route)) } /// The route to announce for a prefix this peer advertised, charging our @@ -818,7 +744,7 @@ impl Subscriber { return Ok(true); } - Ok(announced.offer(self, path, metadata, None)) + Ok(announced.offer(self, path, metadata)) } /// Handle an ANNOUNCE_RESTART: a live advertisement serves another instance, or a @@ -849,7 +775,7 @@ impl Subscriber { tracing::debug!(route = %self.log_path(&path), hops = hops.len(), "restart"); let route = self.announced_route(&path, hops, cost, link_cost, responder_origin, announced); // Held to the limit like a start: a restart outside it is withheld, dropping the old route. - Ok(announced.offer(self, path, route, None)) + Ok(announced.offer(self, path, route)) } /// The full chain of a new or replacing advertisement, or `None` when it is @@ -7735,7 +7661,7 @@ mod tests { assert_eq!(hops, vec![crate::Hop::UNKNOWN]); } - fn restart_subscriber(session: SinkSession) -> (Subscriber, crate::origin::Consumer) { + pub(super) fn restart_subscriber(session: SinkSession) -> (Subscriber, crate::origin::Consumer) { let origin = origin::Config::new(crate::Hop::new(1).unwrap()).produce(); let consumer = origin.consume(); let subscriber = Subscriber::new(SubscriberConfig { @@ -7753,53 +7679,6 @@ mod tests { (subscriber, consumer) } - /// A takeover in place keeps the sources the predecessor minted open: they serve the - /// instance it continues. A restart in place closes them. - #[kio::test] - async fn a_takeover_in_place_keeps_the_predecessors_sources() { - for takeover in [true, false] { - let (mut subscriber, _consumer) = restart_subscriber(SinkSession::new(Default::default())); - let mut announced = Announced::default(); - let path = Path::from_static("room/host"); - let hop = Some(crate::Hop::new(7).unwrap()); - subscriber - .start_announce( - path.clone(), - None, - crate::Hops::new(), - crate::origin::Cost::UNKNOWN, - 0, - hop, - &mut announced, - ) - .unwrap(); - let minted = announced.attached(&path).unwrap().live.consume(); - - match takeover { - true => subscriber.takeover_announce( - path.clone(), - path.clone(), - None, - crate::Hops::new(), - crate::origin::Cost::UNKNOWN, - 0, - hop, - &mut announced, - ), - false => subscriber.restart_announce( - path.clone(), - crate::Hops::new(), - crate::origin::Cost::UNKNOWN, - 0, - hop, - &mut announced, - ), - } - .unwrap(); - assert_eq!(minted.is_closed(), !takeover, "takeover: {takeover}"); - } - } - /// A restart re-prices the announced route in place: consumers observe another /// active update with the new metadata rather than a retract-and-announce. #[kio::test] @@ -8778,15 +8657,8 @@ impl Announced { /// Put the peer's route into the origin, so paths under it resolve through this /// session on demand, or hold it back when the session's limit does not cover it. - /// The route takes over from the one attached at `predecessor`, if any. Returns - /// whether it attached. - fn offer( - &mut self, - subscriber: &Subscriber, - path: Path, - route: crate::origin::RouteInfo, - predecessor: Option<&Path>, - ) -> bool { + /// Returns whether it attached. + fn offer(&mut self, subscriber: &Subscriber, path: Path, route: crate::origin::RouteInfo) -> bool { if !subscriber .auth .within_limit(crate::auth::Direction::Subscribe, path.as_str()) @@ -8795,23 +8667,10 @@ impl Announced { self.routes.insert(path, Held::Withheld(route)); return false; } - let in_place = predecessor == Some(&path); // An error means the prefix is outside our origin's scope, so don't serve it. - let predecessor = predecessor.and_then(|previous| match self.routes.get(previous)? { - Held::Attached(attached) => Some(&attached.dynamic), - _ => None, - }); - match subscriber.origin.forward(&path, route.clone(), predecessor) { + match subscriber.origin.forward(&path, route.clone(), None) { Ok(dynamic) => { - // In place, the successor continues the instance the predecessor's sources - // serve: they live on with it rather than closing with the advertisement. - let live = in_place - .then(|| self.attached(&path).map(|previous| std::mem::take(&mut previous.live))) - .flatten(); - self.attach(path.clone(), route, dynamic); - if let (Some(live), Some(attached)) = (live, self.attached(&path)) { - attached.live = live; - } + self.attach(path, route, dynamic); true } Err(_) => { @@ -8930,7 +8789,7 @@ impl Announced { if subscriber.going_away.is_set() { route.cost = crate::origin::Cost::DRAIN; } - self.offer(subscriber, path, route, None); + self.offer(subscriber, path, route); } _ => unreachable!("only attached and withheld routes change"), } diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 6a084efa3d1..8a006e25058 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -205,14 +205,20 @@ impl Routed { epoch, cost, } => { - let (previous, suffix) = decoder.takeover(predecessor, suffix)?; - let (previous, path) = (self.path(prefix, &previous)?, self.path(prefix, &suffix)?); - // Kept compiling only: lite-07's ANNOUNCE_TAKEOVER is on its way out, so a - // claim taking over its predecessor's own path simply replaces it. - if previous == path { + let (_, suffix) = decoder.takeover(predecessor, suffix)?; + let path = self.path(prefix, &suffix)?; + // The predecessor is a live claim of the same route covering the new one. + let taken = self.claims.get(&predecessor).ok_or(Error::ProtocolViolation)?; + if taken.route != route || !path.has_prefix(&taken.path) { + return Err(Error::ProtocolViolation); + } + let continues = (taken.path.clone(), taken.claim.instance.clone()); + // At its own path it replaces the predecessor. + if continues.0 == path { self.end(predecessor)?; } - let claim = claim(instance, epoch, cost)?; + let mut claim = claim(instance, epoch, cost)?; + claim.continues = Some(continues); self.start(decoder.last(), route, path, claim)?; } Msg::Update { id, cost, .. } => { @@ -403,12 +409,29 @@ impl Routed { return; } } + // The route this one takes over, as this stream serves it at the instance it + // continues, so what resolved through that one beneath the claim carries on. + let taken = (served.claim.continues.as_ref()) + .and_then(|(taken, instance)| { + let entry = self.served.get_mut(&(id, taken.clone()))?; + (entry.route.route.instance.as_ref() == Some(instance)).then_some((taken, entry)) + }) + .map(|(taken, entry)| match *taken == key.1 { + // In place, its sources serve what the new one continues: they live on with + // it rather than closing with the old route. + true => (&entry.route.dynamic, Some(std::mem::take(&mut entry.route.live))), + false => (&entry.route.dynamic, None), + }); // A new instance or epoch replaces the old route, attached before it goes. tracing::debug!(route = %subscriber.log_path(&path), node = id, "announce"); - match subscriber.origin.forward(&path, route.clone(), None) { + let (predecessor, live) = taken.unzip(); + match subscriber.origin.forward(&path, route.clone(), predecessor) { Ok(dynamic) => { let wake = crate::util::QueueWake::new(key.clone(), self.ready.clone()); - let entry = AnnouncedRoute::new(route, dynamic, wake); + let mut entry = AnnouncedRoute::new(route, dynamic, wake); + if let Some(live) = live.flatten() { + entry.live = live; + } entry.waker.wake_by_ref(); self.served.insert(key.clone(), ServedClaim { named, route: entry }); self.stream.serving(id, &key.1); @@ -457,6 +480,7 @@ fn claim(instance: Option, epoch: u64, cost: crate::origin::Cos instance: instance.ok_or(Error::ProtocolViolation)?, epoch, cost: cost.value(), + continues: None, }) } @@ -483,3 +507,81 @@ fn scoped(link: &route::Link, regions: &Option>, key: [u8; 32]) -> rou ..link.clone() } } + +#[cfg(test)] +mod tests { + use crate::lite::{AnnounceBroadcast as Msg, AnnounceDecoder, PathRef, test_transport::SinkSession}; + use crate::{Instance, Path, origin::Cost, route::Advert}; + + /// A takeover in place keeps the sources the predecessor minted open: they serve the + /// instance it continues. A restart in place closes them. + #[kio::test] + async fn a_takeover_in_place_keeps_the_predecessors_sources() { + for takeover in [true, false] { + let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + let prefix = Path::empty(); + let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let mut decoder = AnnounceDecoder::default(); + let path = Path::from_static("room/host"); + let advert = Advert { + seqno: 0, + metric: 0, + down_only: false, + }; + let start = Msg::Active { + suffix: PathRef::literal(path.clone()), + route: 0, + instance: Some(Instance::random()), + epoch: 0, + hops: Default::default(), + cost: Cost::new(0), + }; + for msg in [Msg::RouteStart { node: 7, advert }, start] { + routed.receive(&prefix, msg, &mut decoder).unwrap(); + } + routed + .receive( + &prefix, + Msg::Floor { + id: 0, + revision: Some(0), + }, + &mut decoder, + ) + .unwrap(); + routed.outputs(&subscriber); + let minted = routed.served[&(7, path.clone())].route.live.consume(); + + let instance = Some(Instance::random()); + let replace = match takeover { + true => Msg::Takeover { + predecessor: 0, + suffix: PathRef::literal(path.clone()), + route: 0, + instance, + epoch: 0, + cost: Cost::new(0), + }, + false => Msg::Restart { + id: 0, + instance, + epoch: 0, + cost: Cost::new(0), + }, + }; + routed.receive(&prefix, replace, &mut decoder).unwrap(); + routed + .receive( + &prefix, + Msg::Floor { + id: 0, + revision: Some(1), + }, + &mut decoder, + ) + .unwrap(); + routed.outputs(&subscriber); + assert_eq!(minted.is_closed(), !takeover, "takeover: {takeover}"); + } + } +} diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index c4f1b275266..ebb0ed615d7 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -3614,21 +3614,21 @@ impl OwnClaims { } fn apply(&self, event: RouteEvent) { - match event { - RouteEvent::Start(announce) - | RouteEvent::Update(announce) - | RouteEvent::Restart(announce) - | RouteEvent::Takeover { announce, .. } => { - let route = announce.route; - let claim = route.instance.map(|instance| crate::route::node::Claim { - instance, - epoch: route.epoch, - cost: route.cost.value(), - }); - self.node.own(announce.prefix, claim); + let (announce, predecessor) = match event { + RouteEvent::Start(announce) | RouteEvent::Update(announce) | RouteEvent::Restart(announce) => { + (announce, None) } - RouteEvent::End(announce) => self.node.own(announce.prefix, None), - } + RouteEvent::Takeover { announce, predecessor } => (announce, Some(predecessor)), + RouteEvent::End(announce) => return self.node.own(announce.prefix, None, None), + }; + let route = announce.route; + let claim = route.instance.map(|instance| crate::route::node::Claim { + instance, + epoch: route.epoch, + cost: route.cost.value(), + continues: None, + }); + self.node.own(announce.prefix, claim, predecessor.as_ref()); } } diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index e2799902c9f..3509f01cbb8 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -35,6 +35,11 @@ pub(crate) struct Claim { pub epoch: u64, /// The origin's Announce Cost. pub cost: u64, + /// The claim of the same identity this one took over, at or above its path, and the + /// instance that claim served: what was resolved through it beneath this path is + /// this claim's broadcast. Kept while the instance stays, and sent as + /// ANNOUNCE_TAKEOVER to a peer still holding that claim. + pub continues: Option<(Path, Instance)>, } /// A shown claim as the origin serves it through the selected route. @@ -170,16 +175,24 @@ impl Node { .expect("a node always declares an identity") } - /// This node's own claim at `path` changed: `None` withdraws it. - pub fn own(&self, path: Path, claim: Option) { + /// This node's own claim at `path` changed: `None` withdraws it. A claim taking over + /// the own claim at `predecessor` continues the instance that one serves now. + pub fn own(&self, path: Path, claim: Option, predecessor: Option<&Path>) { let mut state = self.lock(); let changed = match claim { - Some(claim) => { - let current = state - .router - .declared() - .and_then(|id| state.router.view(id)?.items.get(&path).cloned()); - current.as_ref() != Some(&claim) && state.router.announce(path, claim).is_ok() + Some(mut claim) => { + let view = state.router.declared().and_then(|id| state.router.view(id)); + let current = view.and_then(|view| view.items.get(&path)); + claim.continues = match predecessor { + Some(predecessor) => (view.and_then(|view| view.items.get(predecessor))) + .map(|taken| (predecessor.clone(), taken.instance.clone())), + // A re-price or a newer epoch of the same instance continues what it did. + None => current + .filter(|current| current.instance == claim.instance) + .and_then(|current| current.continues.clone()), + }; + let changed = current != Some(&claim); + changed && state.router.announce(path, claim).is_ok() } None => state.router.unannounce(path).is_ok(), }; @@ -470,6 +483,7 @@ mod tests { instance: instance.clone(), epoch: 0, cost, + continues: None, } } @@ -504,7 +518,7 @@ mod tests { let (origin, relay) = (Node::new(1, 1), Node::new(2, 2)); let answer = origin.answer(Link::default(), Path::empty()).unwrap(); let receive = relay.receive(Link::default(), Path::empty()).unwrap(); - origin.own(p("live/x"), Some(claim(&instance, 5))); + origin.own(p("live/x"), Some(claim(&instance, 5)), None); pump(&answer, &receive); assert_eq!( served(outputs(&receive)), @@ -520,7 +534,7 @@ mod tests { }] ); - origin.own(p("live/x"), None); + origin.own(p("live/x"), None, None); pump(&answer, &receive); assert_eq!( served(outputs(&receive)), @@ -548,7 +562,7 @@ mod tests { let far_answer = origin.answer(Link::default(), Path::empty()).unwrap(); let near = relay.receive(Link::default(), Path::empty()).unwrap(); let near_answer = origin.answer(Link::default(), Path::empty()).unwrap(); - origin.own(p("live/x"), Some(claim(&instance, 0))); + origin.own(p("live/x"), Some(claim(&instance, 0)), None); pump(&far_answer, &far); pump(&near_answer, &near); let metric = |outputs: Vec| match &served(outputs)[..] { @@ -581,7 +595,7 @@ mod tests { let first_answer = origin.answer(Link::default(), Path::empty()).unwrap(); let second = relay.receive(Link::default(), Path::empty()).unwrap(); let second_answer = origin.answer(Link::default(), Path::empty()).unwrap(); - origin.own(p("live/x"), Some(claim(&instance, 0))); + origin.own(p("live/x"), Some(claim(&instance, 0)), None); pump(&first_answer, &first); pump(&second_answer, &second); outputs(&first); diff --git a/rs/moq-net/tests/integration/announce_takeover.rs b/rs/moq-net/tests/integration/announce_takeover.rs index d0a55fa3a49..90cdb661d58 100644 --- a/rs/moq-net/tests/integration/announce_takeover.rs +++ b/rs/moq-net/tests/integration/announce_takeover.rs @@ -112,9 +112,7 @@ fn lite07(version: Version) -> bool { } /// `W -> R -> E`: the edge reads through `W`'s claim on its first subscription, and each -/// hop announces the answer. Lite-07 routes the answer as another of `W`'s claims, a -/// restart at the claim's own path and a start beneath it, and never sends -/// ANNOUNCE_TAKEOVER, which quest/m2/announce-takeover.md removes. +/// hop announces the answer as a takeover of the claim on lite-07, a start before it. async fn chain(version: &str, claim: &str) { let version: Version = version.parse().unwrap(); let worker = Worker::new(1, claim, "w"); @@ -139,14 +137,18 @@ async fn chain(version: &str, claim: &str) { assert_eq!(watch(&edge, "pool/p").await, "w", "{version}"); assert_eq!(worker.outputs.borrow().len(), 1, "{version}"); - let answered = match (lite07(version), claim == "pool/p") { + let answered = if !lite07(version) { // An ordinary announcement, or one replacing the claim at its own path. - (false, true) => vec![ - ("end", "pool/p".to_string(), None), - ("start", "pool/p".to_string(), None), - ], - (true, true) => vec![("restart", "pool/p".to_string(), None)], - (_, false) => vec![("start", "pool/p".to_string(), None)], + if claim == "pool/p" { + vec![ + ("end", "pool/p".to_string(), None), + ("start", "pool/p".to_string(), None), + ] + } else { + vec![("start", "pool/p".to_string(), None)] + } + } else { + vec![("takeover", "pool/p".to_string(), Some(claim.to_string()))] }; assert_eq!(drain(&mut at_relay).await, answered, "{version}: at the relay"); assert_eq!(drain(&mut at_edge).await, answered, "{version}: at the edge"); @@ -252,7 +254,10 @@ async fn an_older_downstream_gets_a_start() { drain(&mut at_edge).await; assert_eq!(watch(&relay, "pool/p").await, "w"); - assert_eq!(drain(&mut at_relay).await, [("start", "pool/p".to_string(), None)]); + assert_eq!( + drain(&mut at_relay).await, + [("takeover", "pool/p".to_string(), Some("pool".to_string()))] + ); assert_eq!(drain(&mut at_edge).await, [("start", "pool/p".to_string(), None)]); assert_eq!(watch(&edge, "pool/p").await, "w"); assert_eq!(worker.outputs.borrow().len(), 1, "the edge asked the worker again"); From b9f3078ddd66b5c09398b9107cf3dcb0d9970865 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 08:55:29 -0800 Subject: [PATCH 20/40] test: skip native JS lite-07 lanes until js/net speaks the route layer js/net still speaks lite-07 from before the route layer, so the in-tree lanes running it on lite-07-wip fail against this checkout's relay. Native JS dials its defaults in the interop matrix, and every lane that would run it on lite-07 records a skip naming quest/m2/cluster-routing/routes/js.md, which restores them. The same goes for lite_tracks_interop and the lite-07 legs of the max-age and relay-fill JS suites. The routes epic records the maintainer's 2026-11-06 picks: ANNOUNCE_TAKEOVER ported onto Route IDs, and a route sent on a stream only while it meets the stream's interest. session.md keeps the scoping work it implements. Co-Authored-By: Claude Opus 5.5 --- quest/m2/cluster-routing/routes/README.md | 15 ++++-- quest/m2/cluster-routing/routes/js.md | 12 +++++ quest/m2/cluster-routing/routes/session.md | 23 +++------ rs/moq-net/src/test_interop.rs | 2 +- rs/moq-tokio/tests/integration/max_age.rs | 4 +- rs/moq-tokio/tests/integration/relay_fill.rs | 4 +- test/interop/README.md | 6 +++ test/interop/interop.sh | 52 ++++++++++++++++++-- test/interop/socket.sh | 5 ++ test/justfile | 2 +- 10 files changed, 95 insertions(+), 30 deletions(-) diff --git a/quest/m2/cluster-routing/routes/README.md b/quest/m2/cluster-routing/routes/README.md index f11c48a24d6..76724476ea5 100644 --- a/quest/m2/cluster-routing/routes/README.md +++ b/quest/m2/cluster-routing/routes/README.md @@ -20,13 +20,14 @@ claim whose withdrawal is already known. 5. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **Floor regions have no order, and a repeated region is a violation.** A sorted order would need Rust's `Path` ordering reproduced exactly in JS. Decoding into a map rejects duplicates at the same cost. Superseded the same day by item 10, which removes regions. 6. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **The Hop setup parameter and ANNOUNCE_OK's Hop ID are dropped from lite-07.** Their only roles were path-vector exclusion and the implicit trailing hop. ANNOUNCE_OK reserves no body for a route-key proof: [moq-dev/moq#7937](https://github.com/moq-dev/moq/pull/7937), accepted the same day, proves the route key once per session in SETUP. The live hop-based codec still switches in session integration, as choice 1 requires. 7. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **An echo naming a Route ID that was never ended, or was already echoed, is a violation at the publisher.** The publisher can check this from its live and unacknowledged ids alone, with no growing state. -8. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **ANNOUNCE_TAKEOVER keeps its shape, with a Route ID in place of hops, and its predecessor must be under the same route.** [quest/m2/announce-takeover.md](https://github.com/moq-dev/moq/blob/main/quest/m2/announce-takeover.md) deletes the message, so this keeps the change minimal. +8. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **ANNOUNCE_TAKEOVER keeps its shape, with a Route ID in place of hops, and its predecessor must be under the same route.** [quest/m2/announce-takeover.md](https://github.com/moq-dev/moq/blob/main/quest/m2/announce-takeover.md) deletes the message, so this keeps the change minimal. Decided 2026-11-06 (maintainer, /quest-complete on [moq-dev/moq#8023](https://github.com/moq-dev/moq/pull/8023)), asked whether session integration ports the message or lets lite-07 answer a claim as a restart or start until announce-takeover lands: ✅ **port it**, so a job answering a claim stays a takeover across relays. Rejected: accepting the restart and start. 9. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **Unknown announce types stay skipped by length on lite-06+**, as before. An unknown reply type is refused. 10. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **ANNOUNCE_FLOOR is a marker, not a region list:** `{Route ID, Revision}`. It closes a batch and certifies the stream's claims for that route across the whole requested prefix as the origin's at that Revision. A publisher that cannot vouch for the whole requested prefix at one Revision still closes the batch, with a floor that certifies nothing, encoded per item 13. Regions, the replacing list, and the 65,535-byte overflow go away. Accepted cost: a relay with partial knowledge of an origin certifies nothing for that route, so a receiver lets its stale claims go only on ROUTE_END. Rejected: one region per message; a multipart atomic list. 11. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **Clamped claims collapse to the best one.** When several of a route's claims cover a request's prefix (`live` at cost 10 and `live/a` at cost 1, request `live/a/x`), the publisher presents only the most specific, and when it is withdrawn moves the presented claim to the next covering one with ANNOUNCE_UPDATE or ANNOUNCE_RESTART instead of ending it. No wire change; one live claim per prefix per route holds, and a receiver merges claims at the prefix each stream presents them. Rejected: carrying each claim's ancestor depth. 12. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **A floor certifies only what its own request shows.** A stream whose request did not set `Hidden` certifies nothing about hidden claims. No new wire field. Rejected: isolating differently filtered views. 13. ✅ Decided 2026-11-05 (maintainer): **ANNOUNCE_FLOOR encodes `Revision + 1`, with 0 meaning the floor certifies nothing,** the same offset as `Max Age`. Rust decodes it to `Option`. It replaces a Revision of 2^64-1 as the marker: a huge sentinel is a bad pattern and costs 9 bytes as a lite-07 varint. Rejected: a separate message; a boolean field. 14. ✅ Decided 2026-11-05 (maintainer: "use 0 as a sentinel, or make a new message type"; the agent picked the message): **A held route is ROUTE_HOLD (0xB), `{Route ID}`,** replacing a ROUTE_UPDATE with a Metric of 2^64-1. A metric of 0 is real (the origin's own route), so 0 cannot mark held, and offsetting every metric by one would still leave a held update's seqno and down-only bit meaningless and refused. The message carries only what a hold means, item 4's rule goes away, and a metric is now only bounded by overflow, so the wire refuses no metric. The 2^64-1 Seqno and Revision bounds stay: they are bounds, not markers, leaving room for a ROUTE_REQUEST's successor and for the floor's offset. +15. ✅ Decided 2026-11-06 (maintainer, /quest-complete on [moq-dev/moq#8023](https://github.com/moq-dev/moq/pull/8023)), asked whether to amend the handoff contract's "a sender that certifies anything for a route answers every stream" or keep it: ✅ **send a route on a stream only while it meets the stream's interest**, with a floor certifying the last withdrawal before the route ends. Withdrawals still END, and what a node keeps per stream it answers is bounded by the origins that stream reads, at a ROUTE_END round trip when an origin leaves an interest. Rejected: keep answering every stream and bound only by per-peer caps, which is simpler but carries the whole node table on every stream. [Route session integration](/quest/m2/cluster-routing/routes/session.md) implements it. The route layer's core landed in @@ -330,10 +331,14 @@ loss never reports an END: delivered (`a_lost_scope_keeps_newer_knowledge_where_still_certified`). Otherwise a wildcard above both was last learned through the lost floor and is unknown at the kept floors' revision, so the view is rebuilt from the - neighbours' last views, a collection of that region. A sender that - certifies anything for a route answers every stream, even one whose - interest none of its regions meets, so its peer learns it certifies nothing - there rather than waiting (`a_disjoint_interest_is_answered_with_nothing`). + neighbours' last views, a collection of that region. A sender sends a + route on a stream only while the route meets the stream's interest (Plan + item 15): it starts the route before its first claim in or covering the + interest, and when the last such claim goes it sends a floor certifying the + withdrawal, then ends the route. A peer therefore holds no route, and + waits on no certifier, for an origin its interest does not meet. The router + still answers every stream until session integration scopes it + (`a_disjoint_interest_is_answered_with_nothing`). A claim is available only through a certifier that is not held; another scope's usable route cannot serve it. A certifier that has not answered yet counts as serving, as for a new next hop whose announcements trail its diff --git a/quest/m2/cluster-routing/routes/js.md b/quest/m2/cluster-routing/routes/js.md index 0426fdc7758..eb052a55830 100644 --- a/quest/m2/cluster-routing/routes/js.md +++ b/quest/m2/cluster-routing/routes/js.md @@ -18,6 +18,18 @@ Decided 2026-10-27 (maintainer), from the cheapest one; fix it with the route-selection rework here. - Feed the availability API from [Announcement availability](/quest/m2/cluster-routing/routes/availability.md). - Rust/JS interop covers the route messages, the floor marker, and the echo. +- Restore the native JS lite-07 lanes. Decided 2026-11-06 (maintainer, + /quest-complete on [moq-dev/moq#8023](https://github.com/moq-dev/moq/pull/8023)), + asked how to handle the in-tree lanes running JS on lite-07-wip once the + Rust side moved to the route wire: ✅ **skip them until this quest lands**, + so the nightly report stays meaningful. Rejected: leave them red; hold + #8023 for this quest. Every skip names this quest (`grep -rn routes/js.md + test rs`): `JS_LITE07_SKIP` in `test/interop/interop.sh` and `socket.sh` + (native JS grant printing, claims, track replacement, sockets, the lite-07 + compress and drop legs, the compressed catalog's lite-07 reader, and + `--versions moq-lite-07-wip` with JS clients), `lite_tracks_interop` in + `test/justfile`, and the lite-07 legs of `max_age_relay_javascript` and + `relay_fill_javascript`. ## Required diff --git a/quest/m2/cluster-routing/routes/session.md b/quest/m2/cluster-routing/routes/session.md index cd06f1bd925..9afc6653e1c 100644 --- a/quest/m2/cluster-routing/routes/session.md +++ b/quest/m2/cluster-routing/routes/session.md @@ -12,11 +12,13 @@ node table, and the public API drops what the route layer replaced. ## Plan - Scope route state to interested origins, not the CDN's whole node table. - Today the router advertises every route to every watching stream, because - the epic's handoff contract (`routes/README.md`, "a sender that certifies - anything for a route answers every stream") has a sender answer even a - stream whose interest none of the route's regions meets. Scoping needs - that sentence amended (see Waiting). + Today the router advertises every route to every watching stream. The + epic's handoff contract now has a sender send a route on a stream only + while it meets the stream's interest, with a floor certifying the last + withdrawal before the route ends (`routes/README.md` Plan item 15, decided + 2026-11-06): start the route before its first claim in or covering the + interest, and end it after the floor that certifies its last such claim's + withdrawal. Update `a_disjoint_interest_is_answered_with_nothing` to match. What a peer makes the router hold is already capped: each route and claim it sends takes one of the session's announce slots (`a_lite07_claim_past_the_cap_closes_the_session`). What this bounds is @@ -32,17 +34,6 @@ node table, and the public API drops what the route layer replaced. ## Waiting -- Maintainer: amend the handoff contract's "a sender that certifies anything - for a route answers every stream" (`routes/README.md`), or keep it. It is a - decided contract, so an agent does not change it. Options: **send a route - only while it meets the stream's interest (recommended)**: a route starts on - a stream before its first claim in or covering the interest, and when its - last such claim goes it sends a floor certifying the withdrawal, then ends - the route, so withdrawals still END and per-stream state is bounded by the - origins a peer reads, at a ROUTE_END round trip when an origin leaves an - interest; or **keep answering every stream** and bound only by per-peer - caps, simpler but carrying the whole node table on every stream. Done when - the README says which. - Maintainer: decide when `Hops`, `RouteInfo::hops`, and the saturating `Cost` leave the public API. Every older wire still fills them, so an agent cannot drop them without picking the rollout's end. Options: **when lite-06 diff --git a/rs/moq-net/src/test_interop.rs b/rs/moq-net/src/test_interop.rs index cee73c582a5..80446166634 100644 --- a/rs/moq-net/src/test_interop.rs +++ b/rs/moq-net/src/test_interop.rs @@ -554,7 +554,7 @@ fn lite_auth_unsupported_interop() { /// with a listing Rust decodes, and JS decodes Rust's listing events and re-encodes them byte for /// byte. #[test] -#[ignore = "requires Bun; run by just test interop"] +#[ignore = "JS encodes TRACKS without a Node ID until quest/m2/cluster-routing/routes/js.md, which restores it to just test interop"] fn lite_tracks_interop() { use crate::coding::Decode; diff --git a/rs/moq-tokio/tests/integration/max_age.rs b/rs/moq-tokio/tests/integration/max_age.rs index fda969668c1..c4866d01c86 100644 --- a/rs/moq-tokio/tests/integration/max_age.rs +++ b/rs/moq-tokio/tests/integration/max_age.rs @@ -141,7 +141,9 @@ async fn max_age_relay_rust() { #[tokio::test] #[ignore = "requires Bun; run by just test max-age in the interop workflow"] async fn max_age_relay_javascript() { - for version in ["moq-lite-07-wip", "moq-transport-18", "moq-transport-22"] { + // moq-lite-07-wip returns once js/net speaks its route layer + // (quest/m2/cluster-routing/routes/js.md). + for version in ["moq-transport-18", "moq-transport-22"] { for publish in [false, true] { kio::clock::timeout(TIMEOUT, relay(version.parse().unwrap(), Some(publish))) .await diff --git a/rs/moq-tokio/tests/integration/relay_fill.rs b/rs/moq-tokio/tests/integration/relay_fill.rs index 693e4cbb61b..01c35216814 100644 --- a/rs/moq-tokio/tests/integration/relay_fill.rs +++ b/rs/moq-tokio/tests/integration/relay_fill.rs @@ -275,7 +275,9 @@ async fn relay(version: Version) -> anyhow::Result<()> { #[tokio::test] #[ignore = "requires Bun; run by just test relay-fill in the interop workflow"] async fn relay_fill_javascript() { - for version in ["moq-lite-05", "moq-lite-06", "moq-lite-07-wip"] { + // moq-lite-07-wip returns once js/net speaks its route layer + // (quest/m2/cluster-routing/routes/js.md). + for version in ["moq-lite-05", "moq-lite-06"] { kio::clock::timeout(TIMEOUT, relay(version.parse().unwrap())) .await .unwrap() diff --git a/test/interop/README.md b/test/interop/README.md index 1939304a3b6..40f68e764c6 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -20,6 +20,12 @@ and Bun over WebSocket and WebTransport. Fresh readers request or follow the claim's own path and child paths. A reader-first case checks that an absent request ends while its follower waits for the publisher to arrive. +Until [`js/net` speaks lite-07's route layer](../../quest/m2/cluster-routing/routes/js.md), +native JS dials its defaults in the matrix, leaving its grant unchecked, and every +lane that would run it on lite-07 records a skip naming that quest: claims, track +replacement, sockets, and the lite-07 compress and drop legs. The compressed catalog +runs without its lite-07 reader, and `--versions moq-lite-07-wip` refuses JS clients. + Every publisher also sends an Opus tone, one sine per channel, and every subscriber decodes it. Python and C++ send a mono 440 Hz tone; the rest send 440 Hz left and 660 Hz right, so a channel swap fails too. The Rust CLI imports it from ffmpeg in diff --git a/test/interop/interop.sh b/test/interop/interop.sh index a2e22c223c5..60039316ce6 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -276,6 +276,26 @@ needs_js() { needs js || needs js-native || needs js-native-node || needs js-native-bun } +# js/net speaks the lite-07 wire from before the route layer until +# quest/m2/cluster-routing/routes/js.md ports it and removes this: native JS dials its +# defaults in the matrix, and a lane running it on moq-lite-07-wip is skipped. +JS_LITE07_SKIP="JS speaks pre-route lite-07 until quest/m2/cluster-routing/routes/js.md" + +# js_lite07 : true if the lane runs native JS on moq-lite-07-wip. +js_lite07() { + local version="$1" client + [[ "$version" == moq-lite-07-wip ]] || return 1 + for client in "${@:2}"; do [[ "$client" != js-native-* ]] || return 0; done + return 1 +} + +for version in "${VERSION_LIST[@]}"; do + if [[ "$version" == moq-lite-07-wip ]] && needs_js; then + echo "error: $JS_LITE07_SKIP; pin moq-lite-07-wip without JS clients" >&2 + exit 2 + fi +done + # Released-wire compatibility (compat.sh) pins one version and may run released # binaries that predate AUTH, so it runs anonymous: it compares the wire, and the # default matrix covers auth. @@ -285,11 +305,12 @@ COMPAT=0 # The clients that print the grant they received as an `auth granted` line. The # Python, Go, and C++ clients read it through moq-ffi; moq-c's frozen ABI and # GStreamer expose none, and the browser's shared connection keeps its session private. -# Grants are checked on $AUTH_VERSION, so only in the `auto` pass. +# Native JS prints one too, but dials its defaults until it speaks lite-07's route +# layer (see JS_LITE07_SKIP). Grants are checked on $AUTH_VERSION, so only in the `auto` pass. prints_grant() { [[ "$COMPAT" -eq 0 && "$VERSION" == auto ]] || return 1 case "$1" in - rust | python | go | cpp | js-native-node | js-native-bun) return 0 ;; + rust | python | go | cpp) return 0 ;; *) return 1 ;; esac } @@ -348,8 +369,9 @@ DROP_VERSIONS=(moq-lite-05 moq-lite-06 moq-lite-07-wip) # The lite drafts the compressed-track lanes cross: lite-07 carries the flag, lite-06 gets inflated bytes. COMPRESS_VERSIONS=(moq-lite-07-wip moq-lite-06) # The compressed catalog's concurrent readers: lite-07 carries the flag, and lite-06 and a -# moq-transport reader that does not offer FLATE share the relay's one inflated copy. -CATALOG_VERSIONS=(moq-lite-07-wip moq-lite-06 moq-transport-22) +# moq-transport reader that does not offer FLATE share the relay's one inflated copy. The +# lite-07 reader returns with quest/m2/cluster-routing/routes/js.md (see JS_LITE07_SKIP). +CATALOG_VERSIONS=(moq-lite-06 moq-transport-22) # Every publisher in this checkout sends a tone per audio channel, and every subscriber decodes # AUDIO_MS of it for tone.py (the browser: its page) to check against the publisher's list. This @@ -1904,6 +1926,11 @@ run_drop_pair() { local name="drop-${version}-${pub}-${sub}" local fifo="$LOGS/tail-ack" broadcast="$name-$$-$RANDOM" local cell=(case=drop publisher="$pub" subscriber="$sub" version="$version" log="$LOGS/$name-sub.log") + if js_lite07 "$version" "$pub" "$sub"; then + echo " SKIP drop $version $pub -> $sub" + record skip "${cell[@]}" detail="$JS_LITE07_SKIP" + return + fi if is_broken "$pub" || is_broken "$sub"; then echo " FAIL drop $version $pub -> $sub (client unavailable)" record fail "${cell[@]}" detail="client unavailable" @@ -1975,6 +2002,11 @@ run_tail_pair() { fi local cell=(case="$kind" publisher="$pub" subscriber="$sub" log="$LOGS/$name-sub.log") [[ -z "${4:-}" ]] || cell+=(version="$version") + if js_lite07 "$version" "$pub" "$sub"; then + echo " SKIP $label $pub -> $sub" + record skip "${cell[@]}" detail="$JS_LITE07_SKIP" + return + fi if is_broken "$pub" || is_broken "$sub"; then echo " FAIL $label $pub -> $sub (client unavailable)" record fail "${cell[@]}" detail="client unavailable" @@ -2320,6 +2352,11 @@ run_claims_case() { local prefix="claims-$1-$2-$$-$RANDOM" command=(bun) local cell=(case="claims $transport" publisher="$runtime" subscriber="$runtime" version=moq-lite-07-wip log="$LOGS/$name.log") [[ "$runtime" != js-native-node ]] || command=(node --import tsx) + if js_lite07 moq-lite-07-wip "$runtime"; then + echo " SKIP claims $transport $runtime" + record skip "${cell[@]}" detail="$JS_LITE07_SKIP" + return + fi if is_broken "$runtime"; then record fail "${cell[@]}" detail="client unavailable" overall=1 @@ -2377,6 +2414,11 @@ run_replacement_case() { local runtime="$1" transport="$2" name="replace-$1-$2" started=$SECONDS local prefix="replace-$1-$2-$$-$RANDOM" local cell=(case="track replacement $transport" publisher="$runtime" subscriber="$runtime" version=moq-lite-07-wip log="$LOGS/$name.log") + if js_lite07 moq-lite-07-wip "$runtime"; then + echo " SKIP track replacement $transport $runtime" + record skip "${cell[@]}" detail="$JS_LITE07_SKIP" + return + fi if is_broken "$runtime"; then record fail "${cell[@]}" detail="client unavailable" overall=1 @@ -2461,7 +2503,7 @@ run_catalog_fanout() { got=$(relay_log_since "$from" | sed -n 's/.*conn{id=[0-9]* .*negotiated version=\([^ ]*\).*/\1/p' | sort | tr '\n' ' ') if [[ -z "$why" ]] && ! grep -qx 'catalog acknowledged' "$LOGS/$name-pub.log"; then why="publisher did not acknowledge" - elif [[ -z "$why" && "$got" != "moq-lite-06 moq-lite-07-wip moq-lite-07-wip moq-transport-22 " ]]; then + elif [[ -z "$why" && "$got" != "moq-lite-06 moq-lite-07-wip moq-transport-22 " ]]; then why="relay connections negotiated ${got:-nothing}" elif [[ -z "$why" && "$((after - before))" != "$bytes" ]]; then why="relay inflated $((after - before)) bytes, not the catalog's ${bytes:-?} once" diff --git a/test/interop/socket.sh b/test/interop/socket.sh index 42fd013e36c..f28f3ab351b 100644 --- a/test/interop/socket.sh +++ b/test/interop/socket.sh @@ -25,6 +25,11 @@ run_socket_pair() { local name="socket-$protocol-$dialer-$acceptor-$version" prefix="socket-$protocol-$dialer-$acceptor-$version-$$-$RANDOM" local events="$LOGS/socket-events" ack="$LOGS/socket-ack" local cell=(case="socket $protocol" publisher="$dialer" subscriber="$acceptor" version="$version" log="$LOGS/$name-dial.log") + if js_lite07 "$version" "$dialer" "$acceptor"; then + echo " SKIP socket $protocol $dialer -> $acceptor $version" + record skip "${cell[@]}" detail="$JS_LITE07_SKIP" + return + fi if is_broken "$dialer" || is_broken "$acceptor"; then record fail "${cell[@]}" detail="client unavailable" overall=1 diff --git a/test/justfile b/test/justfile index bd9db4d79e4..b2d851bcc7e 100644 --- a/test/justfile +++ b/test/justfile @@ -64,6 +64,7 @@ harness: # Finite track tails run too; `--tail` selects just the CLI FETCH close and Rust/Node/Bun tail checks. # `--versions ietf` reruns them with every client pinned to each even moq-transport draft. # Both runs first check the Rust/JS wire and session interop suites, in both directions. +# `lite_tracks_interop` rejoins them with quest/m2/cluster-routing/routes/js.md. [positional-arguments] interop *args: cd .. && bun install --frozen-lockfile @@ -75,7 +76,6 @@ interop *args: ietf_range_updates_interop \ padding_interop \ sign_interop \ - tracks_interop \ varint_interop \ --run-ignored all ./interop/interop.sh "$@" From e896ae79bca05e8f3503da845edc9b802d69338d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 09:52:44 -0800 Subject: [PATCH 21/40] fix(net): lite-07 receive admits each claim by the limit, bounds pending, serves per claim - A lite-07 stream hands the node only the peer's claims the session's limit admits, not every claim within its regions. A limit narrower than its regions certifies nothing, so a floor never certifies a claim it left out absent. A permit change that admits a different set files the stream again, as a scope change does. - Claim changes between floors are kept as their net change to the committed claims, so what a stream holds is bounded by its claims, not by the messages it read. - The node serves a claim only through the streams that can carry it: within what this node asked the neighbor for and within its scope. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/auth.rs | 19 +++ rs/moq-net/src/lite/subscriber/routed.rs | 192 +++++++++++++++++++++-- rs/moq-net/src/route/mod.rs | 18 ++- rs/moq-net/src/route/node.rs | 36 ++++- 4 files changed, 247 insertions(+), 18 deletions(-) diff --git a/rs/moq-net/src/auth.rs b/rs/moq-net/src/auth.rs index 8419aa4dd64..f2fbca84b73 100644 --- a/rs/moq-net/src/auth.rs +++ b/rs/moq-net/src/auth.rs @@ -527,6 +527,15 @@ pub(crate) struct Permit { } impl Permit { + /// A permit limiting the peer to `limit`, with no grant of its own. + #[cfg(test)] + pub(crate) fn limited(limit: Patterns) -> Self { + Self { + granted: None, + limit: Some(limit), + } + } + /// Whether both allow `path`. pub(crate) fn matches(&self, path: &str) -> bool { self.granted(path) && self.within_limit(path) @@ -554,6 +563,16 @@ impl Permit { covers(&self.granted) && covers(&self.limit) } + /// Whether the limit alone allows every path at or beneath `prefix`, judged + /// pattern by pattern, so a `false` may be spurious but a `true` never is. + #[cfg(feature = "async")] + pub(crate) fn within_limit_subtree(&self, prefix: &str) -> bool { + let Ok(subtree) = moq_pattern::Pattern::subtree(prefix) else { + return false; + }; + self.limit.as_ref().is_none_or(|limit| limit.contains(&subtree)) + } + /// The prefixes this confines the peer to, or `None` when it confines nothing: the /// heads of the scope's patterns, which grants this side issued. What the peer /// granted this side limits only what this side does, so a client never scopes the diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 8a006e25058..b804c10cd84 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -24,13 +24,25 @@ struct WireRoute { reach: route::Reach, /// The Announce IDs of its live claims. claims: BTreeSet, - /// Claim changes since its last ANNOUNCE_FLOOR. - pending: Vec<(Path, Option)>, + /// How its claims differ from `committed`, by path, so what the stream keeps + /// between floors is bounded by its claims rather than the messages it read. + pending: BTreeMap>, /// Its claims and floor as of that ANNOUNCE_FLOOR, replayed when the stream /// files again. `None` until the first. committed: Option<(BTreeMap, Option)>, } +impl WireRoute { + /// The claim at `path` is now `claim`, pending the next floor. + fn change(&mut self, path: Path, claim: Option) { + let committed = self.committed.as_ref().and_then(|(claims, _)| claims.get(&path)); + match committed == claim.as_ref() { + true => self.pending.remove(&path), + false => self.pending.insert(path, claim), + }; + } +} + /// One live claim the peer sent, by its Announce ID. struct WireClaim { route: u64, @@ -54,6 +66,9 @@ pub(super) struct Routed { link: route::Link, /// The scope's regions it filed under, or `None` unscoped. regions: Option>, + /// The permit it filed under. The node holds only the peer's claims its limit + /// admits; the regions alone can be wider. + permit: crate::auth::Permit, /// The peer's SETUP, whose proven route key files a scoped peer's identities. peer_setup: crate::lite::PeerSetup, /// Files them instead while the peer proved none: this session's own. @@ -113,6 +128,7 @@ impl Routed { stream, link, regions, + permit: permit.clone(), peer_setup, session_key, prefix, @@ -153,7 +169,7 @@ impl Routed { node, reach, claims: BTreeSet::new(), - pending: Vec::new(), + pending: BTreeMap::new(), committed: None, }, ); @@ -173,6 +189,7 @@ impl Routed { self.replies.push(AnnounceReply::RouteEnd { id }); } Msg::Floor { id, revision } => { + let certifies = self.certifies(); let route = self.routes.get_mut(&id).ok_or(Error::ProtocolViolation)?; let changes = std::mem::take(&mut route.pending); let (committed, floor) = route.committed.get_or_insert_default(); @@ -183,6 +200,11 @@ impl Routed { }; } *floor = revision; + // The node never held a claim the limit leaves out, so it neither starts nor ends. + let changes = (changes.into_iter()) + .filter(|(path, _)| admits(&self.root, &self.permit, path)) + .collect(); + let revision = revision.filter(|_| certifies); self.stream.batch(route.node, changes, revision); } Msg::Active { @@ -275,7 +297,7 @@ impl Routed { } self.held.push(self.slots.acquire()?); wire.claims.insert(id); - wire.pending.push((path.clone(), Some(claim.clone()))); + wire.change(path.clone(), Some(claim.clone())); self.claims.insert(id, WireClaim { route, path, claim }); Ok(()) } @@ -286,14 +308,14 @@ impl Routed { self.held.pop(); let route = self.routes.get_mut(&claim.route).ok_or(Error::ProtocolViolation)?; route.claims.remove(&id); - route.pending.push((claim.path, None)); + route.change(claim.path, None); Ok(()) } fn pending(&mut self, id: u64, change: (Path, Option)) -> Result<(), Error> { let route = self.claims.get(&id).ok_or(Error::ProtocolViolation)?.route; let route = self.routes.get_mut(&route).ok_or(Error::ProtocolViolation)?; - route.pending.push(change); + route.change(change.0, change.1); Ok(()) } @@ -310,12 +332,18 @@ impl Routed { self.outputs(subscriber); } - /// The session's permit changed: a scope the stream no longer matches files it - /// again, the peer's routes and claims replayed under a new key before the old - /// one lets go. + /// The session's permit changed: a scope the stream no longer matches, or a + /// limit admitting a different set of the peer's claims, files it again, the + /// peer's routes and the claims the limit admits replayed before the old stream + /// lets go. The peer still holds each claim either way. pub(super) fn limit(&mut self, subscriber: &Subscriber, permit: &crate::auth::Permit) -> Result<(), Error> { let regions = regions(&self.root, permit); - if regions == self.regions { + let admitted = (self.routes.values()) + .filter_map(|route| route.committed.as_ref()) + .flat_map(|(claims, _)| claims.keys()) + .any(|path| admits(&self.root, &self.permit, path) != admits(&self.root, permit, path)); + self.permit = permit.clone(); + if regions == self.regions && !admitted { return Ok(()); } let key = self.peer_setup.route_key().unwrap_or(self.session_key); @@ -332,11 +360,14 @@ impl Routed { let route = &self.routes[id]; stream.route(route.node, Some(route.reach)); } + let certifies = self.certifies(); for id in &ids { let route = &self.routes[id]; if let Some((claims, floor)) = &route.committed { - let changes = claims.iter().map(|(path, claim)| (path.clone(), Some(claim.clone()))); - stream.batch(route.node, changes.collect(), *floor); + let changes = (claims.iter()) + .filter(|(path, _)| admits(&self.root, &self.permit, path)) + .map(|(path, claim)| (path.clone(), Some(claim.clone()))); + stream.batch(route.node, changes.collect(), floor.filter(|_| certifies)); } } if self.drained { @@ -352,6 +383,26 @@ impl Routed { Ok(()) } + /// Whether the peer's floors may certify what reaches the node: the limit admits + /// all of the requested prefix within the scope's regions. A narrower limit leaves + /// claims out that a floor would certify absent, so the claims it admits stay only + /// while the peer carries them. + fn certifies(&self) -> bool { + let Some(regions) = &self.regions else { + return true; + }; + (regions.iter()) + .filter_map(|region| match () { + _ if self.prefix.has_prefix(region) => Some(&self.prefix), + _ if region.has_prefix(&self.prefix) => Some(region), + _ => None, + }) + .all(|covered| { + (covered.strip_prefix(&self.root)) + .is_some_and(|relative| self.permit.within_limit_subtree(relative.as_str())) + }) + } + /// Act on everything the node queued for the stream: serve the claims it hands /// over, and queue the replies it asks for. pub(super) fn outputs(&mut self, subscriber: &Subscriber) { @@ -492,6 +543,12 @@ fn hops(id: route::Id) -> crate::Hops { hops } +/// Whether `permit`'s limit admits the peer's claim at the absolute `path`. +fn admits(root: &Path, permit: &crate::auth::Permit, path: &Path) -> bool { + path.strip_prefix(root) + .is_some_and(|relative| permit.within_limit(relative.as_str())) +} + /// The absolute regions a permit confines the peer's claims to. fn regions(root: &Path, permit: &crate::auth::Permit) -> Option> { let regions = permit.regions()?; @@ -511,7 +568,116 @@ fn scoped(link: &route::Link, regions: &Option>, key: [u8; 32]) -> rou #[cfg(test)] mod tests { use crate::lite::{AnnounceBroadcast as Msg, AnnounceDecoder, PathRef, test_transport::SinkSession}; - use crate::{Instance, Path, origin::Cost, route::Advert}; + use crate::{Instance, Path, auth::Permit, origin::Cost, route::Advert}; + + const ADVERT: Advert = Advert { + seqno: 0, + metric: 0, + down_only: false, + }; + + fn start(path: &'static str) -> Msg { + Msg::Active { + suffix: PathRef::literal(Path::from_static(path)), + route: 0, + instance: Some(Instance::random()), + epoch: 0, + hops: Default::default(), + cost: Cost::new(0), + } + } + + fn limited(pattern: &str) -> Permit { + Permit::limited([pattern.parse::().unwrap()].into_iter().collect()) + } + + /// The claims the stream serves into the origin, by path. + fn served(routed: &super::Routed) -> Vec { + let mut served: Vec = routed.served.keys().map(|(_, path)| path.to_string()).collect(); + served.sort(); + served + } + + /// A limit narrower than its regions still leaves out the claims it does not + /// admit, and a change that keeps the regions brings them in and out again. + #[kio::test] + async fn the_limit_admits_each_claim() { + let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + let prefix = Path::empty(); + let public = limited("live/*/public/**"); + let everything = limited("live/**"); + assert_eq!(public.regions(), everything.regions(), "the limits share their regions"); + + let mut routed = super::Routed::new(&subscriber, &prefix, 1, &public).unwrap(); + let mut decoder = AnnounceDecoder::default(); + let floor = Msg::Floor { + id: 0, + revision: Some(0), + }; + let route = Msg::RouteStart { + node: 7, + advert: ADVERT, + }; + for msg in [route, start("live/a/public/x"), start("live/a/private/y"), floor] { + routed.receive(&prefix, msg, &mut decoder).unwrap(); + } + routed.outputs(&subscriber); + assert_eq!(served(&routed), ["live/a/public/x"]); + + routed.limit(&subscriber, &everything).unwrap(); + assert_eq!(served(&routed), ["live/a/private/y", "live/a/public/x"]); + + routed.limit(&subscriber, &public).unwrap(); + assert_eq!(served(&routed), ["live/a/public/x"]); + } + + /// What the stream keeps between floors is the net change to its committed claims, + /// however many messages the peer sends without one. + #[kio::test] + async fn pending_changes_net_out() { + let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + let prefix = Path::empty(); + let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let mut decoder = AnnounceDecoder::default(); + let floor = Msg::Floor { + id: 0, + revision: Some(0), + }; + let route = Msg::RouteStart { + node: 7, + advert: ADVERT, + }; + for msg in [route, start("room/host"), floor] { + routed.receive(&prefix, msg, &mut decoder).unwrap(); + } + let pending = |routed: &super::Routed| routed.routes[&0].pending.len(); + assert_eq!(pending(&routed), 0); + + for cost in 1..1000 { + let update = Msg::Update { + id: 0, + hops: Default::default(), + cost: Cost::new(cost), + }; + routed.receive(&prefix, update, &mut decoder).unwrap(); + } + assert_eq!(pending(&routed), 1, "updates to one claim"); + + let back = Msg::Update { + id: 0, + hops: Default::default(), + cost: Cost::new(0), + }; + routed.receive(&prefix, back, &mut decoder).unwrap(); + assert_eq!(pending(&routed), 0, "an update back to the committed claim"); + + for _ in 0..100 { + routed.receive(&prefix, start("room/guest"), &mut decoder).unwrap(); + let id = decoder.last(); + routed.receive(&prefix, Msg::EndedId { id }, &mut decoder).unwrap(); + } + assert_eq!(pending(&routed), 0, "claims started and ended between floors"); + } /// A takeover in place keeps the sources the predecessor minted open: they serve the /// instance it continues. A restart in place closes them. diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index c9b3bb12fcc..89efac0beb1 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -271,6 +271,16 @@ struct Peer { asked: Option, } +impl Peer { + /// Whether a request for a claim at `claim` may go through this neighbor: the + /// claim lies within what this node asked it for and within its scope. A route + /// reaches an origin node, not each of its claims. + fn carries(&self, claim: &Path) -> bool { + self.asked.as_ref().is_none_or(|asked| within(claim, asked)) + && (self.link.scope.as_ref()).is_none_or(|scope| scope.regions.iter().any(|region| within(claim, region))) + } +} + /// An identity this node serves itself. struct Origin { seqno: Seqno, @@ -846,14 +856,16 @@ impl Router { best.map(|(_, selected)| selected) } - /// Every usable feasible route to `id` through a neighbor, with this node's - /// metric through each. Any of them forwards without a loop. - pub fn routes(&self, id: Id) -> Vec<(N, Metric)> { + /// Every usable feasible route to `id` through a neighbor that can carry + /// `claim`, with this node's metric through each. Any of them forwards without a + /// loop. + pub fn routes(&self, id: Id, claim: &Path) -> Vec<(N, Metric)> { if self.origins.contains_key(&id) { return Vec::new(); } (self.rx.get(&id).into_iter().flatten()) .filter(|(_, entry)| !entry.held && self.feasible(id, entry.advert)) + .filter(|(next, _)| self.neighbors[*next].carries(claim)) .filter_map(|(&next, entry)| Some((next, add(entry.advert.metric, self.neighbors[&next].link.cost)?))) .collect() } diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 3509f01cbb8..e45f4d8d411 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -374,9 +374,9 @@ impl State { } /// Serve each shown claim of `id` through every stream with a usable feasible - /// route to it, keeping it where it is, unavailable, while there is none. + /// route to it that can carry the claim, keeping it where it is, unavailable, while + /// there is none. fn home(&mut self, id: Id) { - let routes = self.router.routes(id); let claims: Vec<(Path, Claim, bool)> = (self.router.shown().range((id, Path::empty())..)) .take_while(|((shown, _), _)| *shown == id) .map(|((_, claim), shown)| (claim.clone(), shown.item.clone(), shown.available)) @@ -387,6 +387,7 @@ impl State { .collect(); for (claim, item, available) in claims { gone.remove(&claim); + let routes = self.router.routes(id, &claim); let home = self.homes.entry((id, claim.clone())).or_default(); let wanted: Vec<(Key, Metric, bool)> = match routes.is_empty() { true => (home.keys.iter()) @@ -586,6 +587,37 @@ mod tests { assert!(served(outputs(&near)).is_empty(), "the remaining route serves on"); } + /// Two streams reach one origin node for different prefixes: each claim is served + /// only through the streams that asked for it, however cheap the other. + #[test] + fn a_claim_is_served_only_through_streams_that_carry_it() { + let instance = Instance::random(); + let origin = Node::new(1, 1); + let relay = Node::new(2, 2); + let a = relay.receive(Link::default(), p("live/a")).unwrap(); + let a_answer = origin.answer(Link::default(), p("live/a")).unwrap(); + let costly = Link { + cost: 3, + ..Link::default() + }; + let b = relay.receive(costly, p("live/b")).unwrap(); + let b_answer = origin.answer(Link::default(), p("live/b")).unwrap(); + origin.own(p("live/a/x"), Some(claim(&instance, 0)), None); + origin.own(p("live/b/y"), Some(claim(&instance, 0)), None); + pump(&a_answer, &a); + pump(&b_answer, &b); + let claims = |outputs: Vec| -> Vec { + (served(outputs).into_iter()) + .filter_map(|output| match output { + Output::Serve { claim, .. } => Some(claim), + _ => None, + }) + .collect() + }; + assert_eq!(claims(outputs(&a)), [p("live/a/x")]); + assert_eq!(claims(outputs(&b)), [p("live/b/y")]); + } + #[test] fn a_route_lost_with_its_session_leaves_the_others_serving() { let instance = Instance::random(); From 0e32a180cc4dab24d46a4bf1dbe2b6facfb708dd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 10:02:41 -0800 Subject: [PATCH 22/40] fix(net): coalesce a lite-07 stream's unread route outputs per identity A stream whose reader stalls kept every route change and batch the node queued. Its outbox now holds one entry per identity: an end first, then the last start and hold, and one batch with each claim's last change and the last floor, starts before ends as the router orders them. A claim the reader was never sent leaves nothing when it ends, so what a stalled stream holds is bounded by its claims. A route started and ended unread reaches the reader as an end alone, which it acks at once since the peer never held the route. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/publisher/routed.rs | 16 +- rs/moq-net/src/route/node.rs | 230 ++++++++++++++++++++++-- 2 files changed, 226 insertions(+), 20 deletions(-) diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index 865c15b9b23..e3331f9da30 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -116,7 +116,9 @@ impl Answer { let answer = node .answer(self.link.clone(), self.prefix.clone()) .map_err(|_| Error::ProtocolViolation)?; - while let Some(output) = answer.try_output() { + // Held before its first outputs, so an end it acks at once reaches the node. + self.stream = Some(answer); + while let Some(output) = self.stream.as_ref().and_then(node::Stream::try_output) { self.output(stream, output)?; } // Every route the peer holds has a floor before SYNCED; one the node has @@ -132,7 +134,6 @@ impl Answer { } } stream.writer.buffer(&Msg::Synced)?; - self.stream = Some(answer); self.phase = Phase::Running; } @@ -253,8 +254,15 @@ impl Answer { } self.echoes.insert(sent.wire, id); } - // Held only follows a route the peer holds, and an end of one it does not is moot. - (Some(route::Reach::Held) | None, None) => {} + // The node coalesced the route's start away: the peer never held it, so nothing + // of it lingers, and the end is acked at once. + (None, None) => { + if let Some(answer) = &self.stream { + answer.echo(id); + } + } + // Held only follows a route the peer holds. + (Some(route::Reach::Held), None) => {} } Ok(()) } diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index e45f4d8d411..9367e4c956c 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -15,8 +15,9 @@ //! only once another serves it, so the origin always holds a route for it. use std::{ - collections::{BTreeMap, BTreeSet, HashMap}, + collections::{BTreeMap, BTreeSet, HashMap, VecDeque}, sync::{Arc, Mutex, MutexGuard}, + task::{Poll, ready}, }; use crate::{Instance, Path}; @@ -53,7 +54,8 @@ pub(crate) struct Served { pub available: bool, } -/// What a stream's session acts on, in order. +/// What a stream's session acts on, in order. One identity's route changes and +/// batches coalesce while the session leaves them unread. #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) enum Output { /// Send ROUTE_START, ROUTE_UPDATE, ROUTE_HOLD, or for `None` ROUTE_END. @@ -93,10 +95,134 @@ impl std::fmt::Debug for Node { } } +/// Each stream's outputs, by its key. +type Queues = HashMap>; + +/// What one stream has yet to read. Route changes and claim batches coalesce per +/// identity, and a claim the reader was never sent leaves nothing when it ends, so a +/// stream whose reader stalls holds one entry per identity, bounded by its claims, +/// however much the node changes meanwhile. +#[derive(Default)] +struct Outbox { + /// Identities with route changes or claims pending, in the order they first queued. + order: VecDeque, + pending: HashMap, + /// Outputs to read in order: the other kinds, and a pending entry once taken. + ready: VecDeque, + /// The claims the reader was sent for each identity. + delivered: HashMap>, +} + +/// One identity's route changes and claim batches since the reader last took it. +#[derive(Default)] +struct Pending { + /// A ROUTE_END goes first, retiring every claim sent before. + ended: bool, + /// The last finite reach since, which starts the route if the reader holds none. + finite: Option, + /// The route was held after that. + held: bool, + /// The batches since, merged: each claim's last change, and the last floor. + batch: Option<(BTreeMap>, Option)>, +} + +impl Outbox { + fn is_empty(&self) -> bool { + self.ready.is_empty() && self.order.is_empty() + } + + /// The pending entry for `id`, queued behind the others when new. + fn entry<'a>(order: &mut VecDeque, pending: &'a mut HashMap, id: Id) -> &'a mut Pending { + pending.entry(id).or_insert_with(|| { + order.push_back(id); + Pending::default() + }) + } + + fn push(&mut self, output: Output) { + match output { + Output::Route { id, reach } => { + let entry = Self::entry(&mut self.order, &mut self.pending, id); + match reach { + None => { + *entry = Pending { + ended: true, + ..Pending::default() + } + } + Some(Reach::Held) => entry.held = true, + Some(finite) => { + entry.finite = Some(finite); + entry.held = false; + } + } + } + Output::Batch { id, changes, floor } => { + let entry = Self::entry(&mut self.order, &mut self.pending, id); + let sent = self.delivered.get(&id); + let ended = entry.ended; + let (merged, last) = entry.batch.get_or_insert_default(); + for (path, claim) in changes { + // The reader holds nothing here once the entry reaches it. + let unsent = ended || sent.is_none_or(|sent| !sent.contains(&path)); + match claim { + None if unsent => merged.remove(&path), + claim => merged.insert(path, claim), + }; + } + *last = floor; + } + other => self.ready.push_back(other), + } + } + + fn pop(&mut self) -> Option { + if self.ready.is_empty() { + let id = self.order.pop_front()?; + let pending = self.pending.remove(&id).expect("every queued identity is pending"); + self.take(id, pending); + } + self.ready.pop_front() + } + + /// Queue one identity's pending entry to read, as the router would have sent it. + fn take(&mut self, id: Id, pending: Pending) { + if pending.ended { + self.delivered.remove(&id); + self.ready.push_back(Output::Route { id, reach: None }); + } + if let Some(reach) = pending.finite { + self.ready.push_back(Output::Route { id, reach: Some(reach) }); + } + if pending.held { + self.ready.push_back(Output::Route { + id, + reach: Some(Reach::Held), + }); + } + if let Some((merged, floor)) = pending.batch { + let sent = self.delivered.entry(id).or_default(); + // As the router orders a batch: what starts or changes, then what ends. + let (ends, starts): (Vec<_>, Vec<_>) = merged.into_iter().partition(|(_, claim)| claim.is_none()); + for (path, _) in &starts { + sent.insert(path.clone()); + } + for (path, _) in &ends { + sent.remove(path); + } + if sent.is_empty() { + self.delivered.remove(&id); + } + let changes = starts.into_iter().chain(ends).collect(); + self.ready.push_back(Output::Batch { id, changes, floor }); + } + } +} + struct State { router: Router, next_key: Key, - queues: HashMap>, + queues: Queues, /// Each shown claim of another identity, and the streams serving it. homes: BTreeMap<(Id, Path), Home>, } @@ -117,7 +243,7 @@ struct Slot { impl Home { /// Unserve every leaving stream once a wanted one serves the claim. - fn release(&mut self, queues: &HashMap>, id: Id, claim: &Path) { + fn release(&mut self, queues: &Queues, id: Id, claim: &Path) { if !self.keys.values().any(|slot| !slot.leaving && slot.confirmed) { return; } @@ -220,7 +346,7 @@ impl Node { let key = state.next_key; state.next_key += 1; state.router.connect(key, link)?; - let queue = kio::Queue::new(); + let queue = kio::Shared::default(); state.queues.insert(key, queue.clone()); if let Err(err) = open(&mut state.router, key) { state.router.disconnect(key); @@ -237,6 +363,11 @@ impl Node { }) } + #[cfg(test)] + pub(crate) fn retained(&self) -> super::Retained { + self.lock().router.retained() + } + #[cfg(test)] pub(crate) fn shown(&self) -> BTreeMap<(Id, Path), (Claim, bool)> { (self.lock().router.shown().iter()) @@ -249,7 +380,7 @@ impl Node { pub(crate) struct Stream { node: Node, key: Key, - queue: kio::Queue, + queue: kio::Shared, } impl std::fmt::Debug for Stream { @@ -260,17 +391,21 @@ impl std::fmt::Debug for Stream { impl Stream { /// The next output for this stream's session. - pub fn poll_output(&self, waiter: &kio::Waiter) -> std::task::Poll { - match self.queue.poll_pop(waiter) { - std::task::Poll::Ready(Ok(output)) => std::task::Poll::Ready(output), - // The queue closes only once the stream dropped. - _ => std::task::Poll::Pending, - } + pub fn poll_output(&self, waiter: &kio::Waiter) -> Poll { + let ready = |outbox: &kio::Ref<'_, Outbox>| match outbox.is_empty() { + true => Poll::Pending, + false => Poll::Ready(()), + }; + let mut outbox = ready!(self.queue.poll(waiter, ready)); + Poll::Ready(outbox.pop().expect("an outbox holding something pops it")) } /// The next output already queued, without waiting. pub fn try_output(&self) -> Option { - self.queue.try_pop().ok().flatten() + if self.queue.read().is_empty() { + return None; + } + self.queue.lock().pop() } fn with(&self, f: impl FnOnce(&mut Router, Key) -> R) -> R { @@ -331,7 +466,6 @@ impl Stream { impl Drop for Stream { fn drop(&mut self) { - self.queue.close(); let mut state = self.node.lock(); state.queues.remove(&self.key); state.router.disconnect(self.key); @@ -344,9 +478,9 @@ impl Drop for Stream { } } -fn push(queues: &HashMap>, key: Key, output: Output) { +fn push(queues: &Queues, key: Key, output: Output) { if let Some(queue) = queues.get(&key) { - let _ = queue.try_push(output); + queue.lock().push(output); } } @@ -618,6 +752,70 @@ mod tests { assert_eq!(claims(outputs(&b)), [p("live/b/y")]); } + /// A reader that stops reading holds one entry for the route, however often its + /// claims change: the last change to each, and none for a claim it was never sent. + #[test] + fn a_stalled_reader_holds_the_net_change() { + let instance = Instance::random(); + let origin = Node::new(1, 1); + let answer = origin.answer(Link::default(), Path::empty()).unwrap(); + outputs(&answer); + for cost in 0..1000 { + origin.own(p("live/x"), Some(claim(&instance, cost)), None); + origin.own(p("live/churn"), Some(claim(&instance, cost)), None); + origin.own(p("live/churn"), None, None); + } + let read = outputs(&answer); + let batch = |read: &[Output]| match read { + [.., Output::Batch { changes, .. }] => changes.clone(), + _ => panic!("no batch: {read:?}"), + }; + assert!(read.len() <= 2, "{read:?}"); + assert_eq!(batch(&read), [(p("live/x"), Some(claim(&instance, 999)))]); + + // Once sent, a claim's end is kept, behind what started. + for cost in 0..1000 { + origin.own(p("live/y"), Some(claim(&instance, cost)), None); + } + origin.own(p("live/x"), None, None); + let read = outputs(&answer); + assert_eq!(read.len(), 1, "{read:?}"); + assert_eq!( + batch(&read), + [(p("live/y"), Some(claim(&instance, 999))), (p("live/x"), None)] + ); + } + + /// A route that starts and ends while the reader stalls reaches it as an end alone, + /// which the reader acks at once since it never held the route. + #[test] + fn a_route_started_and_ended_unread_is_an_end() { + let instance = Instance::random(); + let origin = Node::new(1, 1); + let relay = Node::new(2, 2); + let receive = relay.receive(Link::default(), Path::empty()).unwrap(); + let answer = origin.answer(Link::default(), Path::empty()).unwrap(); + origin.own(p("live/x"), Some(claim(&instance, 0)), None); + pump(&answer, &receive); + let edge = relay + .answer( + Link { + proxy: true, + ..Link::default() + }, + Path::empty(), + ) + .unwrap(); + drop(receive); + let read: Vec = (outputs(&edge).into_iter()) + .filter(|output| matches!(output, Output::Route { id: 1, .. } | Output::Batch { id: 1, .. })) + .collect(); + assert_eq!(read, [Output::Route { id: 1, reach: None }]); + assert_eq!(relay.retained().echoes, 1, "the end awaits its echo"); + edge.echo(1); + assert_eq!(relay.retained().echoes, 0); + } + #[test] fn a_route_lost_with_its_session_leaves_the_others_serving() { let instance = Instance::random(); From 2329bb19170885172f81d889528a0ac3c08c3998 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 10:03:03 -0800 Subject: [PATCH 23/40] chore: leave MIGRATE.md to the release cut Main now has the release cut write migration notes; this PR carries them in its Impact section instead. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/MIGRATE.md | 10 ---------- rs/moq-net/MIGRATE.md | 9 --------- rs/moq-relay/MIGRATE.md | 14 -------------- 3 files changed, 33 deletions(-) diff --git a/rs/moq-cli/MIGRATE.md b/rs/moq-cli/MIGRATE.md index 3070c7399d2..e29b62468e9 100644 --- a/rs/moq-cli/MIGRATE.md +++ b/rs/moq-cli/MIGRATE.md @@ -4,16 +4,6 @@ How to update your invocation and config for each breaking change, newest first. ## Unreleased -### `--cluster-id` is removed - -Remove `--cluster-id` and `MOQ_CLUSTER_ID`: `moq` refuses both as unknown settings. -The Hop ID is random on every run and nothing pins it. - -### Zero-cost cluster links are refused - -A `--cluster-connect` or `--cluster-api-url` peer priced at 0 (`?cost=0`, or an object `cost` or `egress` of 0) is refused, as is a `--cluster-node` carrying `?cost=0` under `--cluster-lan`. -A link costs at least 1, the price of an unpriced link: use `?cost=1` for a same-datacenter sibling. - ### The iroh transport is removed ([#7345](https://github.com/moq-dev/moq/pull/7345)) The `iroh` Cargo feature and the `--iroh-*` flags are gone, and a `MOQ_IROH_*` variable stops startup like any other unknown `MOQ_*` variable. diff --git a/rs/moq-net/MIGRATE.md b/rs/moq-net/MIGRATE.md index 3d7ff6a09ef..c9a8ccd87ec 100644 --- a/rs/moq-net/MIGRATE.md +++ b/rs/moq-net/MIGRATE.md @@ -4,15 +4,6 @@ How to update your code for each breaking change, newest first. ## Unreleased -### moq-lite-07 routes through the route layer ([#8023](https://github.com/moq-dev/moq/pull/8023)) - -moq-lite-07-wip (opt-in) replaces the hop list with routes: each origin node declares a random routing identity, every session on an origin feeds one router, and a request names the origin node the first hop picked. Older versions are unchanged. On moq-lite-07: - -- A relay built on the library must opt into proxying, or its peers are sent only the routes to its origin's own claims: publish `origin.clone().with_proxy(true)`, or mark one consumer with `consumer.with_proxy(true)`. `moq-relay` does this for every session; an application publishing only its own broadcasts leaves it off. -- A link cost of 0 is refused at SETUP: `Client::with_cost(0)` fails the session. Use 1, the price of an unpriced link. -- A route carries no hop chain. `RouteInfo::hops()` on a route learned over moq-lite-07 names only its origin node, and `origin::Config::max_hops` does not bound it: routes are loop-free by feasibility instead. -- A pool's paths spread by a hash of the path and each origin node's routing identity, which every relay computes alike, rather than by publisher instance, so every relay picks the same origin for a path. - ### Snapshots split connected sessions into idle and active `stats::Snapshot::activity()` returns each tier's `stats::Activity`: connected sessions with diff --git a/rs/moq-relay/MIGRATE.md b/rs/moq-relay/MIGRATE.md index 3823718c8af..6f06755853d 100644 --- a/rs/moq-relay/MIGRATE.md +++ b/rs/moq-relay/MIGRATE.md @@ -5,20 +5,6 @@ Rolling a cluster across a release also follows the [upgrade guide](https://moq. ## Unreleased -### `cluster.id` is removed - -Remove `--cluster-id`, `MOQ_CLUSTER_ID`, and `[cluster] id`; in Rust, drop `cluster::Config::id`. -A relay that still sets one stops at startup: the flag, variable, and key are refused as unknown settings. -Each relay picks a random Hop ID on every start and nothing pins it, so a restarted relay is a new node to the mesh. -`Cluster::id()` still reports the one in use. - -### Zero-cost cluster links are refused - -A link costs at least 1, the price of an unpriced link, since routing needs every link to cost something. -A peer priced at 0 (`?cost=0` on its URL, or an object `cost` or `egress` of 0) stops startup in `cluster.connect` and rejects a `cluster.api.url` update, which keeps the last good list. -With LAN discovery on, a `cluster.node` whose `?cost=` is 0 or malformed stops startup, and a LAN peer advertising one is skipped. -Price a same-datacenter sibling at 1, and raise the other links' costs if it must stay relatively cheaper. - ### Shutdown reports deadline cuts separately from end reports `shutdown::Observer::drained()` now returns `shutdown::Drained` instead of `()`. From 6746f6e1d8067ab272fbe354d6fba20ed7b686d8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 10:08:03 -0800 Subject: [PATCH 24/40] refactor(net): name the outbox's merged changes for clippy Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/node.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 9367e4c956c..9e5d00083e3 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -123,9 +123,12 @@ struct Pending { /// The route was held after that. held: bool, /// The batches since, merged: each claim's last change, and the last floor. - batch: Option<(BTreeMap>, Option)>, + batch: Option<(Changes, Option)>, } +/// Each claim's last change, by path. +type Changes = BTreeMap>; + impl Outbox { fn is_empty(&self) -> bool { self.ready.is_empty() && self.order.is_empty() From 2d6790dbb2b8b172cade4e288f2839162c97eba2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 10:21:02 -0800 Subject: [PATCH 25/40] fix(net): ack routes ended unread at once, so a stalled outbox holds only held routes The outbox now knows which routes its reader holds. A ROUTE_END of a route the reader never read, a start or a restart since its last read, cancels what is pending and is acked to the router at once, as the reader would: no peer holds it. So churning fresh identities leaves nothing behind, and an end, restart, and end of a route the reader holds reaches it as one end with one echo owed. The answer no longer acks an unread end itself; the node does. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/publisher/routed.rs | 16 +- rs/moq-net/src/route/node.rs | 206 ++++++++++++++++-------- 2 files changed, 141 insertions(+), 81 deletions(-) diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index e3331f9da30..865c15b9b23 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -116,9 +116,7 @@ impl Answer { let answer = node .answer(self.link.clone(), self.prefix.clone()) .map_err(|_| Error::ProtocolViolation)?; - // Held before its first outputs, so an end it acks at once reaches the node. - self.stream = Some(answer); - while let Some(output) = self.stream.as_ref().and_then(node::Stream::try_output) { + while let Some(output) = answer.try_output() { self.output(stream, output)?; } // Every route the peer holds has a floor before SYNCED; one the node has @@ -134,6 +132,7 @@ impl Answer { } } stream.writer.buffer(&Msg::Synced)?; + self.stream = Some(answer); self.phase = Phase::Running; } @@ -254,15 +253,8 @@ impl Answer { } self.echoes.insert(sent.wire, id); } - // The node coalesced the route's start away: the peer never held it, so nothing - // of it lingers, and the end is acked at once. - (None, None) => { - if let Some(answer) = &self.stream { - answer.echo(id); - } - } - // Held only follows a route the peer holds. - (Some(route::Reach::Held), None) => {} + // Held only follows a route the peer holds, and an end of one it does not is moot. + (Some(route::Reach::Held) | None, None) => {} } Ok(()) } diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 9e5d00083e3..fd0e3e130e0 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -99,24 +99,26 @@ impl std::fmt::Debug for Node { type Queues = HashMap>; /// What one stream has yet to read. Route changes and claim batches coalesce per -/// identity, and a claim the reader was never sent leaves nothing when it ends, so a -/// stream whose reader stalls holds one entry per identity, bounded by its claims, -/// however much the node changes meanwhile. +/// identity, and what the reader was never sent leaves nothing when it ends: a route +/// started and ended unread is acked to the router at once, as the reader would. A +/// stream whose reader stalls holds one entry per route it holds or will start, +/// bounded by its claims, however much the node changes meanwhile. #[derive(Default)] struct Outbox { - /// Identities with route changes or claims pending, in the order they first queued. - order: VecDeque, - pending: HashMap, + /// Identities with route changes or claims pending, by when they first queued. + order: BTreeMap, + next: u64, + pending: HashMap, /// Outputs to read in order: the other kinds, and a pending entry once taken. ready: VecDeque, - /// The claims the reader was sent for each identity. + /// Each route the reader holds, and the claims it was sent under it. delivered: HashMap>, } /// One identity's route changes and claim batches since the reader last took it. #[derive(Default)] struct Pending { - /// A ROUTE_END goes first, retiring every claim sent before. + /// A ROUTE_END goes first, ending the route the reader holds and its claims. ended: bool, /// The last finite reach since, which starts the route if the reader holds none. finite: Option, @@ -135,35 +137,57 @@ impl Outbox { } /// The pending entry for `id`, queued behind the others when new. - fn entry<'a>(order: &mut VecDeque, pending: &'a mut HashMap, id: Id) -> &'a mut Pending { - pending.entry(id).or_insert_with(|| { - order.push_back(id); - Pending::default() - }) + fn entry(&mut self, id: Id) -> &mut Pending { + let (order, next) = (&mut self.order, &mut self.next); + let (_, entry) = self.pending.entry(id).or_insert_with(|| { + let seq = *next; + *next += 1; + order.insert(seq, id); + (seq, Pending::default()) + }); + entry } - fn push(&mut self, output: Output) { + /// Queue `output`. Returns the identity of a ROUTE_END the reader will never read, + /// since it never read the route it ends, for the caller to ack in its place. + fn push(&mut self, output: Output) -> Option { match output { - Output::Route { id, reach } => { - let entry = Self::entry(&mut self.order, &mut self.pending, id); - match reach { - None => { - *entry = Pending { - ended: true, - ..Pending::default() - } - } - Some(Reach::Held) => entry.held = true, - Some(finite) => { - entry.finite = Some(finite); - entry.held = false; - } + Output::Route { id, reach: None } => { + let holds = self.delivered.contains_key(&id); + let entry = self.entry(id); + // The reader holds the route once it reads what is pending: an end of + // that keeps any end before, an end of a start it never read is acked. + let ended = entry.ended; + let acked = !holds || ended; + *entry = Pending { + ended: holds, + ..Pending::default() + }; + if !holds && let Some((seq, _)) = self.pending.remove(&id) { + self.order.remove(&seq); } + acked.then_some(id) + } + Output::Route { + id, + reach: Some(Reach::Held), + } => { + self.entry(id).held = true; + None + } + Output::Route { + id, + reach: Some(finite), + } => { + let entry = self.entry(id); + entry.finite = Some(finite); + entry.held = false; + None } Output::Batch { id, changes, floor } => { - let entry = Self::entry(&mut self.order, &mut self.pending, id); + let ended = self.entry(id).ended; + let (_, entry) = self.pending.get_mut(&id).expect("just queued"); let sent = self.delivered.get(&id); - let ended = entry.ended; let (merged, last) = entry.batch.get_or_insert_default(); for (path, claim) in changes { // The reader holds nothing here once the entry reaches it. @@ -174,15 +198,19 @@ impl Outbox { }; } *last = floor; + None + } + other => { + self.ready.push_back(other); + None } - other => self.ready.push_back(other), } } fn pop(&mut self) -> Option { if self.ready.is_empty() { - let id = self.order.pop_front()?; - let pending = self.pending.remove(&id).expect("every queued identity is pending"); + let (_, id) = self.order.pop_first()?; + let (_, pending) = self.pending.remove(&id).expect("every queued identity is pending"); self.take(id, pending); } self.ready.pop_front() @@ -195,6 +223,7 @@ impl Outbox { self.ready.push_back(Output::Route { id, reach: None }); } if let Some(reach) = pending.finite { + self.delivered.entry(id).or_default(); self.ready.push_back(Output::Route { id, reach: Some(reach) }); } if pending.held { @@ -204,17 +233,15 @@ impl Outbox { }); } if let Some((merged, floor)) = pending.batch { - let sent = self.delivered.entry(id).or_default(); // As the router orders a batch: what starts or changes, then what ends. let (ends, starts): (Vec<_>, Vec<_>) = merged.into_iter().partition(|(_, claim)| claim.is_none()); - for (path, _) in &starts { - sent.insert(path.clone()); - } - for (path, _) in &ends { - sent.remove(path); - } - if sent.is_empty() { - self.delivered.remove(&id); + if let Some(sent) = self.delivered.get_mut(&id) { + for (path, _) in &starts { + sent.insert(path.clone()); + } + for (path, _) in &ends { + sent.remove(path); + } } let changes = starts.into_iter().chain(ends).collect(); self.ready.push_back(Output::Batch { id, changes, floor }); @@ -481,23 +508,35 @@ impl Drop for Stream { } } -fn push(queues: &Queues, key: Key, output: Output) { - if let Some(queue) = queues.get(&key) { - queue.lock().push(output); - } +/// Queue `output` for the stream `key`. Returns the identity of a ROUTE_END its +/// reader will never read, whose echo the router is owed. +fn push(queues: &Queues, key: Key, output: Output) -> Option { + queues.get(&key).and_then(|queue| queue.lock().push(output)) } impl State { /// Route the router's messages and shown claims to the streams. fn flush(&mut self) { - for out in self.router.drain() { - let (key, output) = match out { - Out::Route { to, id, reach } => (to, Output::Route { id, reach }), - Out::Batch { to, id, changes, floor } => (to, Output::Batch { id, changes, floor }), - Out::Request { to, id, seqno } => (to, Output::Request { id, seqno }), - Out::Echo { to, id } => (to, Output::Echo { id }), - }; - push(&self.queues, key, output); + loop { + let mut acked = Vec::new(); + for out in self.router.drain() { + let (key, output) = match out { + Out::Route { to, id, reach } => (to, Output::Route { id, reach }), + Out::Batch { to, id, changes, floor } => (to, Output::Batch { id, changes, floor }), + Out::Request { to, id, seqno } => (to, Output::Request { id, seqno }), + Out::Echo { to, id } => (to, Output::Echo { id }), + }; + if let Some(id) = push(&self.queues, key, output) { + acked.push((key, id)); + } + } + if acked.is_empty() { + break; + } + // A ROUTE_END coalesced away before its reader read the route: no peer holds it. + for (key, id) in acked { + self.router.echo(key, id); + } } // The shown claims below are read whole; the events only say which changed. self.router.events(); @@ -789,17 +828,13 @@ mod tests { ); } - /// A route that starts and ends while the reader stalls reaches it as an end alone, - /// which the reader acks at once since it never held the route. + /// Routes that start and end while the reader stalls leave nothing for it, and the + /// router is acked at once as the reader would: churning fresh identities holds + /// nothing. A route the reader holds ends once, however often it restarted unread. #[test] - fn a_route_started_and_ended_unread_is_an_end() { + fn routes_ended_unread_leave_nothing() { let instance = Instance::random(); - let origin = Node::new(1, 1); let relay = Node::new(2, 2); - let receive = relay.receive(Link::default(), Path::empty()).unwrap(); - let answer = origin.answer(Link::default(), Path::empty()).unwrap(); - origin.own(p("live/x"), Some(claim(&instance, 0)), None); - pump(&answer, &receive); let edge = relay .answer( Link { @@ -809,13 +844,46 @@ mod tests { Path::empty(), ) .unwrap(); - drop(receive); - let read: Vec = (outputs(&edge).into_iter()) - .filter(|output| matches!(output, Output::Route { id: 1, .. } | Output::Batch { id: 1, .. })) - .collect(); - assert_eq!(read, [Output::Route { id: 1, reach: None }]); - assert_eq!(relay.retained().echoes, 1, "the end awaits its echo"); - edge.echo(1); + outputs(&edge); + let advert = |seqno| { + Some(Reach::Finite(Advert { + seqno, + metric: 0, + down_only: false, + })) + }; + for origin in 10..110 { + let receive = relay.receive(Link::default(), Path::empty()).unwrap(); + receive.route(origin, advert(0)); + receive.batch(origin, vec![(p("live/x"), Some(claim(&instance, 0)))], Some(1)); + drop(receive); + } + let of = |read: Vec, of: fn(Id) -> bool| -> Vec { + (read.into_iter()) + .filter(|output| matches!(output, Output::Route { id, .. } | Output::Batch { id, .. } if of(*id))) + .collect() + }; + let read = of(outputs(&edge), |id| id >= 10); + assert_eq!(read, [], "the churned routes"); + assert_eq!(relay.retained().echoes, 0); + + // The reader holds a route, which ends, restarts, and ends again unread. + let receive = relay.receive(Link::default(), Path::empty()).unwrap(); + receive.route(7, advert(0)); + receive.batch(7, vec![(p("live/x"), Some(claim(&instance, 0)))], Some(1)); + assert_eq!(of(outputs(&edge), |id| id == 7).len(), 2, "the route and its batch"); + receive.route(7, None); + receive.route(7, advert(1)); + receive.batch(7, vec![(p("live/x"), Some(claim(&instance, 0)))], Some(2)); + receive.route(7, None); + let read = of(outputs(&edge), |id| id == 7); + assert_eq!(read, [Output::Route { id: 7, reach: None }]); + assert_eq!( + relay.retained().echoes, + 1, + "only the end the reader reads awaits its echo" + ); + edge.echo(7); assert_eq!(relay.retained().echoes, 0); } From 554fae60f1e907c3c8d2e4b5d430e16e7591e1f9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 10:41:13 -0800 Subject: [PATCH 26/40] fix(net): present a handle's mounts on lite-07 A lite-07 answer read only the requested prefix from the route node, so a mounted handle's target never reached a lite-07 reader, as it does in process and on lite-06. The answer now reads each mount beneath its prefix through a node stream of its own, at the target, and names what it presents under the mount point on the same routes. What the origin holds beneath a mount point stays hidden. A route stands while any of its streams carries it; one that stops early ends what it presented and is acked at once. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/publisher/routed.rs | 190 ++++++++++++++++++------ rs/moq-net/src/model/mod.rs | 2 + rs/moq-net/src/model/origin.rs | 58 +++++++- rs/moq-net/tests/integration/main.rs | 1 + rs/moq-net/tests/integration/mount.rs | 172 +++++++++++++++++++++ 5 files changed, 376 insertions(+), 47 deletions(-) create mode 100644 rs/moq-net/tests/integration/mount.rs diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index 865c15b9b23..cde695a2ae3 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -5,6 +5,9 @@ //! hidden claims unless the peer asked for them, and anything this handle's scope or //! the session's grant leaves out, in which case a floor certifies nothing, since the //! peer cannot tell what was left out from what is gone. +//! +//! A handle's mounts beneath the prefix are read through a node stream each, at the +//! target, and what they present is named under the mount point on the same routes. use std::{ collections::{BTreeMap, BTreeSet, HashMap}, @@ -24,7 +27,10 @@ use crate::{ struct Sent { /// Its Route ID on the stream. wire: u64, - /// Every claim the node sent for it, before filtering. + /// Each source's last reach for it, by index: the route stands while one has one, + /// and the first one's is the peer's. + reach: BTreeMap, + /// Every claim the node sent for it, named as the handle names it, before filtering. claims: BTreeMap, /// The node's floor for those claims. floor: Option, @@ -32,6 +38,20 @@ struct Sent { floored: bool, } +impl Sent { + fn reach(&self) -> Option { + self.reach.values().next().copied() + } +} + +/// A node stream the answer reads. +struct Source { + stream: node::Stream, + /// The mount it reads through, which names what it presents. `None` for the + /// requested prefix's own. + mount: Option, +} + enum Phase { /// Waiting for the session's grant before the first message. Init, @@ -50,12 +70,17 @@ pub(super) struct Answer { prefix: Path, hidden: bool, link: route::Link, - stream: Option, + /// The prefix's own node stream, then one per mount beneath it. Empty until the + /// grant is known, and once withdrawing. + sources: Vec, + /// The source read first next, so a busy one does not hold the others. + turn: usize, encoder: lite::AnnounceEncoder, routes: HashMap, next_route: u64, - /// Route IDs ended and not yet echoed, and the origin node each reached. - echoes: HashMap, + /// Route IDs ended and not yet echoed, the origin node each reached, and the + /// source that ended it. + echoes: HashMap, /// The claims the peer holds: their Announce ID and attributes. live: HashMap<(route::Id, Path), (u64, node::Claim)>, gate: crate::auth::Gate, @@ -82,7 +107,8 @@ impl Answer { origin, hidden, link, - stream: None, + sources: Vec::new(), + turn: 0, encoder: lite::AnnounceEncoder::new(Version::Lite07), routes: HashMap::new(), next_route: 0, @@ -113,11 +139,19 @@ impl Answer { active: 0, })?; let node = self.origin.route_node(); - let answer = node - .answer(self.link.clone(), self.prefix.clone()) - .map_err(|_| Error::ProtocolViolation)?; - while let Some(output) = answer.try_output() { - self.output(stream, output)?; + let mounts = self.origin.mounted(&self.prefix); + let interests = std::iter::once((self.prefix.clone(), None)) + .chain(mounts.into_iter().map(|mount| (mount.interest.clone(), Some(mount)))); + for (interest, mount) in interests { + let stream = node + .answer(self.link.clone(), interest) + .map_err(|_| Error::ProtocolViolation)?; + self.sources.push(Source { stream, mount }); + } + for index in 0..self.sources.len() { + while let Some(output) = self.sources[index].stream.try_output() { + self.output(stream, index, output)?; + } } // Every route the peer holds has a floor before SYNCED; one the node has // not certified yet certifies nothing. @@ -132,7 +166,6 @@ impl Answer { } } stream.writer.buffer(&Msg::Synced)?; - self.stream = Some(answer); self.phase = Phase::Running; } @@ -152,7 +185,7 @@ impl Answer { if withdrawing { // Let go of the node first, so nothing more is queued, then end every route. - self.stream = None; + self.sources.clear(); for (_, sent) in std::mem::take(&mut self.routes) { stream.writer.buffer(&Msg::RouteEnd { id: sent.wire })?; } @@ -182,9 +215,16 @@ impl Answer { Poll::Pending => {} } - let answer = self.stream.as_ref().expect("running streams hold the node"); - if let Poll::Ready(output) = answer.poll_output(waiter) { - self.output(stream, output)?; + let count = self.sources.len(); + let ready = (0..count).map(|offset| (self.turn + offset) % count).find_map(|index| { + match self.sources[index].stream.poll_output(waiter) { + Poll::Ready(output) => Some((index, output)), + Poll::Pending => None, + } + }); + if let Some((index, output)) = ready { + self.turn = index + 1; + self.output(stream, index, output)?; continue; } return Poll::Pending; @@ -193,22 +233,36 @@ impl Answer { /// A ROUTE_REQUEST, or the echo of a ROUTE_END. fn reply(&mut self, reply: lite::AnnounceReply) -> Result<(), Error> { - let answer = self.stream.as_ref().expect("running streams hold the node"); match reply { - lite::AnnounceReply::RouteRequest { node, seqno } => answer.request(node, seqno), + lite::AnnounceReply::RouteRequest { node, seqno } => { + // Through the first source carrying the route, or the prefix's own. + let index = (self.routes.get(&node)) + .and_then(|sent| sent.reach.keys().next().copied()) + .unwrap_or(0); + if let Some(source) = self.sources.get(index) { + source.stream.request(node, seqno); + } + } lite::AnnounceReply::RouteEnd { id } => { // The echo names the Route ID; the node knows the route by its origin node. - let node = self.echoes.remove(&id).ok_or(Error::ProtocolViolation)?; - answer.echo(node); + let (node, index) = self.echoes.remove(&id).ok_or(Error::ProtocolViolation)?; + if let Some(source) = self.sources.get(index) { + source.stream.echo(node); + } } } Ok(()) } - fn output(&mut self, stream: &mut Stream, output: node::Output) -> Result<(), Error> { + fn output( + &mut self, + stream: &mut Stream, + source: usize, + output: node::Output, + ) -> Result<(), Error> { match output { - node::Output::Route { id, reach } => self.route(stream, id, reach), - node::Output::Batch { id, changes, floor } => self.batch(stream, id, changes, floor), + node::Output::Route { id, reach } => self.route(stream, source, id, reach), + node::Output::Batch { id, changes, floor } => self.batch(stream, source, id, changes, floor), // Only a stream this node asked for requests, serves, or echoes. node::Output::Request { .. } | node::Output::Echo { .. } @@ -220,29 +274,34 @@ impl Answer { fn route( &mut self, stream: &mut Stream, + source: usize, id: route::Id, reach: Option, ) -> Result<(), Error> { - match (reach, self.routes.get(&id)) { - (Some(route::Reach::Finite(advert)), Some(sent)) => { - stream.writer.buffer(&Msg::RouteUpdate { id: sent.wire, advert })?; - } - (Some(route::Reach::Finite(advert)), None) => { + let Some(sent) = self.routes.get_mut(&id) else { + // Held only follows a route the peer holds, and an end of one it does not is moot. + if let Some(reach @ route::Reach::Finite(advert)) = reach { let wire = self.next_route; self.next_route += 1; stream.writer.buffer(&Msg::RouteStart { node: id, advert })?; - self.routes.insert( - id, - Sent { - wire, - claims: BTreeMap::new(), - floor: None, - floored: false, - }, - ); + let sent = Sent { + wire, + reach: BTreeMap::from([(source, reach)]), + claims: BTreeMap::new(), + floor: None, + floored: false, + }; + self.routes.insert(id, sent); + } + return Ok(()); + }; + let before = sent.reach(); + match reach { + Some(reach) => { + sent.reach.insert(source, reach); } - (Some(route::Reach::Held), Some(sent)) => stream.writer.buffer(&Msg::RouteHold { id: sent.wire })?, - (None, Some(_)) => { + None if sent.reach.remove(&source).is_none() => return Ok(()), + None if sent.reach.is_empty() => { let sent = self.routes.remove(&id).expect("matched above"); stream.writer.buffer(&Msg::RouteEnd { id: sent.wire })?; // The ROUTE_END retires every claim under the route. @@ -251,17 +310,47 @@ impl Answer { self.encoder.end(announce); } } - self.echoes.insert(sent.wire, id); + self.echoes.insert(sent.wire, (id, source)); + return Ok(()); + } + None => { + // Another source still carries the route, so the peer keeps it: the claims + // this one presented end, and its end is acked at once. + let paths: Vec = sent.claims.keys().cloned().collect(); + let paths: Vec = (paths.into_iter()).filter(|path| self.source(path) == source).collect(); + self.sources[source].stream.echo(id); + let sent = self.routes.get_mut(&id).expect("matched above"); + for path in &paths { + sent.claims.remove(path); + } + let mut started = BTreeSet::new(); + for path in paths { + self.sync(stream, id, path, &mut started)?; + } } - // Held only follows a route the peer holds, and an end of one it does not is moot. - (Some(route::Reach::Held) | None, None) => {} + } + let sent = &self.routes[&id]; + match sent.reach() { + after if after == before => {} + Some(route::Reach::Finite(advert)) => stream.writer.buffer(&Msg::RouteUpdate { id: sent.wire, advert })?, + Some(route::Reach::Held) => stream.writer.buffer(&Msg::RouteHold { id: sent.wire })?, + None => unreachable!("a route stands while a source carries it"), } Ok(()) } + /// The index of the source presenting the claim at the handle-side `path`: the + /// mount it lies beneath, or the prefix's own. + fn source(&self, path: &Path) -> usize { + (self.sources.iter()) + .position(|source| source.mount.as_ref().is_some_and(|mount| mount.holds(path))) + .unwrap_or(0) + } + fn batch( &mut self, stream: &mut Stream, + source: usize, id: route::Id, changes: Vec<(Path, Option)>, floor: Option, @@ -269,10 +358,27 @@ impl Answer { let Some(sent) = self.routes.get_mut(&id) else { return Ok(()); }; + let mount = self.sources.get(source).and_then(|source| source.mount.as_ref()); let mut paths = Vec::with_capacity(changes.len()); for (path, claim) in changes { + // Named as the handle names it: beneath the mount point for a mount, and never + // a claim the origin holds beneath a mount for the prefix's own. + let path = match mount { + Some(mount) => match mount.name(&path) { + Some(named) => named, + None => continue, + }, + None if self.origin.shadows(&path) => continue, + None => path, + }; match claim { - Some(claim) => sent.claims.insert(path.clone(), claim), + Some(mut claim) => { + if let Some(mount) = mount { + claim.continues = (claim.continues.take()) + .and_then(|(taken, instance)| Some((mount.name(&taken)?, instance))); + } + sent.claims.insert(path.clone(), claim) + } None => sent.claims.remove(&path), }; paths.push(path); diff --git a/rs/moq-net/src/model/mod.rs b/rs/moq-net/src/model/mod.rs index 975f59ad0ed..01b84877195 100644 --- a/rs/moq-net/src/model/mod.rs +++ b/rs/moq-net/src/model/mod.rs @@ -62,6 +62,8 @@ pub mod origin { // private: capability gap pub(crate) use super::origin_impl::Lookup; #[cfg(feature = "async")] + pub(crate) use super::origin_impl::Mounted; + #[cfg(feature = "async")] pub(crate) use super::origin_impl::NodeFilter; pub use super::origin_impl::{ Config, Consumer, Cost, Driver, HopLimit, Link, Preference, Producer, Request, Requesting, Route, RouteInfo, diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index ebb0ed615d7..c3d2a518071 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -1920,6 +1920,31 @@ struct Mount { target: Path, } +/// A mount as a reader of a prefix sees it through a handle: where the origin holds +/// what it presents, and how the handle names that. +#[derive(Clone, Debug)] +#[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] +pub(crate) struct Mounted { + mount: Mount, + /// The absolute, origin-side prefix to read: the target, or beneath it for a + /// reader's prefix beneath the mount point. + pub interest: Path, +} + +#[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] +impl Mounted { + /// How the handle names a claim at the absolute, origin-side `path`, which lies at + /// or beneath the interest: at or beneath the mount point. + pub fn name(&self, path: &Path) -> Option { + self.mount.name(path) + } + + /// Whether the absolute, handle-side `path` lies at or beneath the mount point. + pub fn holds(&self, path: &Path) -> bool { + path.has_prefix(&self.mount.at) + } +} + impl Mount { /// Where the handle-side absolute `path` resolves, when it is under this mount /// and the result fits the wire limits. @@ -7186,17 +7211,40 @@ impl Consumer { self.horizon.upstream } - /// Whether this handle reaches a claim at the absolute `path` as the origin keys - /// it: a path at or beneath it may be in its scope, as a covering reader sees, and - /// it is not under a mount, which resolves elsewhere, or an excluded subtree. + /// Whether this handle reaches a claim at the absolute `path` as it names it: a + /// path at or beneath it may be in its scope, as a covering reader sees, and it is + /// not in an excluded subtree. Beneath a mount, `path` names the target's claim + /// there, never one the origin holds at `path` itself (see [`Self::mounted`]). #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn reaches(&self, path: &Path) -> bool { let Ok(subtree) = Pattern::subtree(path.as_str()) else { return false; }; - self.scope.mount(path).is_none() - && self.scope.allowed.overlaps(&subtree) + self.scope.allowed.overlaps(&subtree) && !self.scope.excluded.iter().any(|excluded| path.has_prefix(excluded)) + && self.scope.mount(path).is_none_or(|mount| mount.resolve(path).is_some()) + } + + /// Whether the absolute `path` lies beneath one of this handle's mounts, where the + /// origin's own claims at that path are hidden from it. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn shadows(&self, path: &Path) -> bool { + self.scope.mount(path).is_some() + } + + /// The mounts a reader of the absolute `prefix` sees through this handle: each one + /// at or beneath it, or the one it lies beneath, unless excluded. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn mounted(&self, prefix: &Path) -> Vec { + (self.scope.mounts.iter()) + .filter(|mount| self.scope.holes(Some(mount)).is_some()) + .filter_map(|mount| { + Some(Mounted { + interest: mount.translate_head(prefix)?, + mount: mount.clone(), + }) + }) + .collect() } /// Whether this handle reaches every path at or beneath the absolute `prefix`, diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index 78b2f3a2b22..50416d0c00c 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -43,6 +43,7 @@ mod late_lower_group; mod lazy_discovery; mod mesh_withdraw; mod model; +mod mount; mod multi_cdn; mod partial_group; mod quiet_catalog; diff --git a/rs/moq-net/tests/integration/mount.rs b/rs/moq-net/tests/integration/mount.rs new file mode 100644 index 00000000000..6967b6ef412 --- /dev/null +++ b/rs/moq-net/tests/integration/mount.rs @@ -0,0 +1,172 @@ +#![cfg(feature = "async")] + +//! A session answered through a mounted handle presents the target's broadcasts under +//! the mount point on every version, hides what the origin holds at the mount point +//! itself, and a reader there reaches the target. + +use crate::support; + +use std::time::Duration; + +use moq_net::{Hop, Pattern, Patterns, Timestamp, Version, origin, track}; +use support::harness::{MockConnectOptions, MockPair, connect_mock}; + +const TEST_TIMEOUT: Duration = Duration::from_secs(600); + +fn produce_origin(hop: u64) -> origin::Producer { + let (producer, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); + support::harness::spawn(driver); + producer +} + +/// Publish a broadcast at `path` whose `video` track carries the path. +fn publish(origin: &origin::Producer, path: &str) -> (origin::Route, track::Producer) { + let route = origin.route(path, origin::RouteOptions::default()).unwrap(); + let track = route + .broadcast() + .create_track( + "video", + track::Info::default().with_timescale(moq_net::Timescale::MILLI), + ) + .unwrap(); + let mut group = track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, path.as_bytes().to_vec()).unwrap(); + group.finish().unwrap(); + (route, track) +} + +const PUBLISHED: [&str; 4] = [ + "tenant-a/public/cam", + "tenant-a/private/mic", + "tenant-b/own", + "tenant-b/shared/forged", +]; + +/// Tenant B's handle rooted at `root`, with tenant A's public broadcasts mounted at +/// `tenant-b/shared`. +fn tenant(origin: &origin::Producer, root: &str) -> origin::Consumer { + origin + .mount("tenant-b/shared", "tenant-a/public") + .unwrap() + .scope(root, &Patterns::from(Pattern::all())) + .unwrap() + .consume() + .with_proxy(true) +} + +/// Have `to` read `from` through a session. +async fn link(version: Version, from: origin::Consumer, to: &origin::Producer) -> MockPair { + let mut options = MockConnectOptions::new(version); + options.server_publish = Some(from); + options.client_subscribe = Some(to.clone()); + connect_mock(options).await +} + +/// Every announce event already delivered, as `(started, prefix)`. +async fn drain(routes: &mut origin::Routes) -> Vec<(bool, String)> { + let mut events = Vec::new(); + while let Ok(event) = kio::clock::timeout(Duration::from_secs(1), routes.next()).await { + match event.expect("announce ended") { + origin::RouteEvent::Start(announce) => events.push((true, announce.prefix.to_string())), + origin::RouteEvent::End(announce) => events.push((false, announce.prefix.to_string())), + _ => {} + } + } + events.sort(); + events +} + +/// The first frame of `video` at `path` through `origin`. +async fn watch(origin: &origin::Producer, path: &str) -> String { + let broadcast = origin.consume().request(path, None).await.unwrap(); + let mut subscription = broadcast.track("video").unwrap().subscribe(None).await.unwrap(); + let mut group = kio::clock::timeout(Duration::from_secs(10), subscription.recv_group()) + .await + .expect("subscription hung") + .expect("the subscription was reset") + .expect("a group"); + let frame = group.read_frame().await.unwrap().expect("a frame"); + String::from_utf8(frame.payload.to_vec()).unwrap() +} + +fn started(paths: &[&str]) -> Vec<(bool, String)> { + paths.iter().map(|path| (true, path.to_string())).collect() +} + +fn ended(paths: &[&str]) -> Vec<(bool, String)> { + paths.iter().map(|path| (false, path.to_string())).collect() +} + +async fn mounted(version: &str) { + let version: Version = version.parse().unwrap(); + let source = produce_origin(1); + let viewer = produce_origin(2); + let mut published: Vec<_> = PUBLISHED.into_iter().map(|path| publish(&source, path)).collect(); + let _pair = link(version, tenant(&source, "tenant-b"), &viewer).await; + kio::clock::sleep(Duration::from_millis(500)).await; + + let mut routes = viewer.consume().routes(); + assert_eq!(drain(&mut routes).await, started(&["own", "shared/cam"]), "{version}"); + assert_eq!(watch(&viewer, "shared/cam").await, "tenant-a/public/cam", "{version}"); + + // What leaves the target leaves the mount. + drop(published.remove(0)); + assert_eq!(drain(&mut routes).await, ended(&["shared/cam"]), "{version}"); +} + +/// A session rooted inside the mount reads only the target. +async fn rooted_inside(version: &str) { + let version: Version = version.parse().unwrap(); + let source = produce_origin(1); + let viewer = produce_origin(2); + let _published: Vec<_> = PUBLISHED.into_iter().map(|path| publish(&source, path)).collect(); + let _pair = link(version, tenant(&source, "tenant-b/shared"), &viewer).await; + kio::clock::sleep(Duration::from_millis(500)).await; + + let mut routes = viewer.consume().routes(); + assert_eq!(drain(&mut routes).await, started(&["cam"]), "{version}"); + assert_eq!(watch(&viewer, "cam").await, "tenant-a/public/cam", "{version}"); +} + +/// `W -> R -> V`: the relay learns the target from upstream and presents it under the +/// mount, and lets it go with the upstream. +async fn relayed(version: &str) { + let version: Version = version.parse().unwrap(); + let worker = produce_origin(1); + let relay = produce_origin(2); + let viewer = produce_origin(3); + let _published: Vec<_> = PUBLISHED.into_iter().map(|path| publish(&worker, path)).collect(); + let upstream = link(version, worker.consume(), &relay).await; + let _downstream = link(version, tenant(&relay, "tenant-b"), &viewer).await; + kio::clock::sleep(Duration::from_millis(500)).await; + + let mut routes = viewer.consume().routes(); + assert_eq!(drain(&mut routes).await, started(&["own", "shared/cam"]), "{version}"); + assert_eq!(watch(&viewer, "shared/cam").await, "tenant-a/public/cam", "{version}"); + + drop(upstream); + kio::clock::sleep(Duration::from_millis(500)).await; + assert_eq!(drain(&mut routes).await, ended(&["own", "shared/cam"]), "{version}"); +} + +macro_rules! mounted_tests { + ($($name:ident: $test:ident, $version:literal,)*) => { + $( + #[kio::test] + async fn $name() { + kio::clock::timeout(TEST_TIMEOUT, $test($version)) + .await + .expect("timed out"); + } + )* + }; +} + +mounted_tests! { + mounted_lite06: mounted, "moq-lite-06", + mounted_lite07: mounted, "moq-lite-07-wip", + rooted_inside_lite06: rooted_inside, "moq-lite-06", + rooted_inside_lite07: rooted_inside, "moq-lite-07-wip", + relayed_lite06: relayed, "moq-lite-06", + relayed_lite07: relayed, "moq-lite-07-wip", +} From 8e03f17f70a36c13bc8a6b142725f95e8dbeb122 Mon Sep 17 00:00:00 2001 From: "pullfrog[bot]" <226033991+pullfrog[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:22:13 +0000 Subject: [PATCH 27/40] test: retarget the drill mutations and allow the gated lite-07 interop test --- sh/gh/test-inventory.allow | 2 + .../relay-withdraws-lost-publisher.patch | 38 +++++++++++++------ .../subscriber-leaks-broadcasts.patch | 22 +++++++---- 3 files changed, 43 insertions(+), 19 deletions(-) diff --git a/sh/gh/test-inventory.allow b/sh/gh/test-inventory.allow index 45e9fe71542..4f421f76ee0 100644 --- a/sh/gh/test-inventory.allow +++ b/sh/gh/test-inventory.allow @@ -40,3 +40,5 @@ ignore moq-video::frame::tests::d3d11_resize_matches_cpu /quest/m5/video-hardwar ignore moq-video::render::export::dx12::tests::exports_an_nv12_texture /quest/m5/video-hardware.md ignore moq-video::render::export::dx12::tests::media_foundation_encodes_the_export_in_place /quest/m5/video-hardware.md +# Need another implementation to speak the route wire; the quest ports it. +ignore moq-net::test_interop::lite_tracks_interop /quest/m2/cluster-routing/routes/js.md diff --git a/test/drill/mutations/relay-withdraws-lost-publisher.patch b/test/drill/mutations/relay-withdraws-lost-publisher.patch index 4dbf9ee93d5..b325d14d5e7 100644 --- a/test/drill/mutations/relay-withdraws-lost-publisher.patch +++ b/test/drill/mutations/relay-withdraws-lost-publisher.patch @@ -9,16 +9,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs -@@ -7730,7 +7730,7 @@ impl Announced { - let in_place = predecessor == Some(&path); - // An error means the prefix is outside our origin's scope, so don't serve it. - let predecessor = predecessor.and_then(|previous| match self.routes.get(previous)? { -- Held::Attached(attached) => Some(&attached.dynamic), -+ Held::Attached(attached) => Some(&*attached.dynamic), - _ => None, - }); - match subscriber.origin.forward(&path, route.clone(), predecessor) { -@@ -7819,7 +7819,7 @@ impl Announced { +@@ -8818,7 +8818,7 @@ }; self.held.pop(); if let Held::Attached(entry) = held { @@ -27,7 +18,16 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. } } -@@ -7921,7 +7921,8 @@ struct AnnouncedRoute { +@@ -8902,7 +8902,7 @@ + let validity = if subscriber.version.has_instance() { + Default::default() + } else { +- crate::model::broadcast::RequestValidity::for_source(&entry.dynamic, request.path()) ++ crate::model::broadcast::RequestValidity::for_source(&*entry.dynamic, request.path()) + }; + let source = subscriber.origin.create_source(path.clone(), validity); + // The handler exists before the requester sees the source, so a track it +@@ -8930,7 +8930,8 @@ /// without recomputing the chain. route: crate::origin::RouteInfo, /// Dropping it retracts the route and rejects its queued requests. @@ -37,7 +37,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. /// Closes as the route goes, which closes every source it minted. live: kio::Producer<()>, /// Whether the GOAWAY drain already re-priced this route. -@@ -7941,7 +7942,7 @@ impl AnnouncedRoute { +@@ -8950,7 +8951,7 @@ ) -> Self { Self { route, @@ -46,3 +46,17 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. live: Default::default(), drained: false, waker: std::task::Waker::from(wake.clone()), +diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs +--- a/rs/moq-net/src/lite/subscriber/routed.rs ++++ b/rs/moq-net/src/lite/subscriber/routed.rs +@@ -470,8 +470,8 @@ + .map(|(taken, entry)| match *taken == key.1 { + // In place, its sources serve what the new one continues: they live on with + // it rather than closing with the old route. +- true => (&entry.route.dynamic, Some(std::mem::take(&mut entry.route.live))), +- false => (&entry.route.dynamic, None), ++ true => (&*entry.route.dynamic, Some(std::mem::take(&mut entry.route.live))), ++ false => (&*entry.route.dynamic, None), + }); + // A new instance or epoch replaces the old route, attached before it goes. + tracing::debug!(route = %subscriber.log_path(&path), node = id, "announce"); diff --git a/test/drill/mutations/subscriber-leaks-broadcasts.patch b/test/drill/mutations/subscriber-leaks-broadcasts.patch index c484dbf1dc4..8738695e886 100644 --- a/test/drill/mutations/subscriber-leaks-broadcasts.patch +++ b/test/drill/mutations/subscriber-leaks-broadcasts.patch @@ -9,10 +9,9 @@ # session is gone, so a cancelled reader's broadcast parks forever instead of # closing. Leaking only one is not a leak: the other still closes the front. diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs -index adbc9aeec1..e4edd44fcf 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs -@@ -1903,7 +1903,8 @@ struct PrefixRun { +@@ -2042,7 +2042,8 @@ /// it comes from the connect config or the peer's SETUP, neither of which /// changes for the life of the session. link_cost: u64, @@ -22,7 +21,7 @@ index adbc9aeec1..e4edd44fcf 100644 // Lite06+: announce ids. Each received `active` implicitly assigns the next // per-stream ordinal; `ended`/`restart` reference it instead of repeating the // path, and lite-07 bases name it too. Tracked even for announces we drop -@@ -1991,7 +1992,10 @@ impl AnnouncePrefix { +@@ -2145,7 +2146,10 @@ let mut run: PrefixRun = PrefixRun { responder_origin, link_cost, @@ -34,22 +33,31 @@ index adbc9aeec1..e4edd44fcf 100644 decoder: lite::AnnounceDecoder::default(), auth_epoch: 0, initial, -@@ -2123,7 +2127,8 @@ struct SourceServe { - subscriber: Subscriber, +@@ -2301,7 +2305,8 @@ path: Path, instance: Option, + node: u64, - source: crate::model::broadcast::SourceGuard, + // MUTATION: never dropped, so a session that ends keeps every source it minted alive. + source: std::mem::ManuallyDrop, dynamic: crate::broadcast::Requests, route: kio::Consumer<()>, // A dedicated close-watch handle, since each pending operation needs its own. -@@ -2269,7 +2274,7 @@ impl SourceServe { - subscriber, +@@ -2451,7 +2456,7 @@ path: minted.path, instance: minted.instance, + node: minted.node, - source: minted.source, + source: std::mem::ManuallyDrop::new(minted.source), datagram_requests: minted.dynamic.datagrams(), dynamic: minted.dynamic, route: minted.route, +@@ -2560,7 +2565,7 @@ + let serve = TrackServe { + validity: self.dynamic.validity().clone(), + route: Some(self.route.clone()), +- source: Some((*self.source).clone()), ++ source: Some((**self.source).clone()), + subscriber: self.subscriber.clone(), + path: self.path.clone(), + instance: self.instance.clone(), From 9c5b2b9d14a5955ef25fa5e59d78b1280390c34c Mon Sep 17 00:00:00 2001 From: quest fixer Date: Fri, 6 Nov 2026 12:56:34 -0800 Subject: [PATCH 28/40] test(net): supply default node in legacy request helpers --- rs/moq-net/src/lite/publisher.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index b65205fe34f..d45217b3a5b 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -9597,6 +9597,7 @@ mod tests { fn legacy_track() -> lite::Track<'static> { lite::Track { instance: None, + node: 0, broadcast: crate::Path::from_static("room"), track: "events".into(), } @@ -9606,6 +9607,7 @@ mod tests { lite::Subscribe { order: crate::track::Order::Descending, instance: None, + node: 0, id: 0, broadcast: crate::Path::from_static("room"), track: "events".into(), From e067fe5ccf3bf08338c5feafb17ffc0bdd2369fe Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 16:07:44 -0800 Subject: [PATCH 29/40] fix(net): build the route node without the fuzz feature, retarget a drill mutation The route node minted lite-06 hops through Hop::new, which only test and fuzz builds compile, so the plain library build failed. The relay-withdraws-lost-publisher mutation patched the removed takeover branch and now also unwraps the route where the routed subscriber consumes it. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/node.rs | 2 +- .../relay-withdraws-lost-publisher.patch | 37 +++++++++++-------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index fd0e3e130e0..d7db156c0a8 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -643,7 +643,7 @@ pub(crate) fn hop(id: Id) -> crate::Hop { hash = hash.wrapping_mul(0x0100_0000_01b3); } let id = (hash & ((1 << 53) - 1)).max(1); - crate::Hop::new(id).expect("a non-zero hop id") + crate::Hop::from(std::num::NonZeroU64::new(id).expect("a non-zero hop id")) } #[cfg(test)] diff --git a/test/drill/mutations/relay-withdraws-lost-publisher.patch b/test/drill/mutations/relay-withdraws-lost-publisher.patch index b325d14d5e7..66e1b0ba07d 100644 --- a/test/drill/mutations/relay-withdraws-lost-publisher.patch +++ b/test/drill/mutations/relay-withdraws-lost-publisher.patch @@ -9,7 +9,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs -@@ -8818,7 +8818,7 @@ +@@ -8817,7 +8817,7 @@ impl Announced { }; self.held.pop(); if let Held::Attached(entry) = held { @@ -18,7 +18,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. } } -@@ -8902,7 +8902,7 @@ +@@ -8901,7 +8901,7 @@ fn mint_sources(subscriber: &Subscriber, entry let validity = if subscriber.version.has_instance() { Default::default() } else { @@ -27,7 +27,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. }; let source = subscriber.origin.create_source(path.clone(), validity); // The handler exists before the requester sees the source, so a track it -@@ -8930,7 +8930,8 @@ +@@ -8929,7 +8929,8 @@ struct AnnouncedRoute { /// without recomputing the chain. route: crate::origin::RouteInfo, /// Dropping it retracts the route and rejects its queued requests. @@ -37,7 +37,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. /// Closes as the route goes, which closes every source it minted. live: kio::Producer<()>, /// Whether the GOAWAY drain already re-priced this route. -@@ -8950,7 +8951,7 @@ +@@ -8949,7 +8950,7 @@ impl AnnouncedRoute { ) -> Self { Self { route, @@ -49,14 +49,21 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs -@@ -470,8 +470,8 @@ - .map(|(taken, entry)| match *taken == key.1 { - // In place, its sources serve what the new one continues: they live on with - // it rather than closing with the old route. -- true => (&entry.route.dynamic, Some(std::mem::take(&mut entry.route.live))), -- false => (&entry.route.dynamic, None), -+ true => (&*entry.route.dynamic, Some(std::mem::take(&mut entry.route.live))), -+ false => (&*entry.route.dynamic, None), - }); - // A new instance or epoch replaces the old route, attached before it goes. - tracing::debug!(route = %subscriber.log_path(&path), node = id, "announce"); +@@ -354,7 +354,7 @@ impl Routed { + self.previous = previous; + self.outputs(subscriber); + for (_, served) in std::mem::take(&mut self.previous) { +- served.route.dynamic.withdrawn(); ++ std::mem::ManuallyDrop::into_inner(served.route.dynamic).withdrawn(); + } + Ok(()) + } +@@ -398,7 +398,7 @@ impl Routed { + } => self.serve(subscriber, id, named, claim, served), + node::Output::Unserve { id, claim } => { + if let Some(served) = self.served.remove(&(id, claim)) { +- served.route.dynamic.withdrawn(); ++ std::mem::ManuallyDrop::into_inner(served.route.dynamic).withdrawn(); + } + } + // The ROUTE_END echo goes out as the peer's ROUTE_END is read, and a stream From 140af504807d36a630cb9a320e797b68582d9c39 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 18:19:57 -0800 Subject: [PATCH 30/40] fix(net): lite-07 receive streams honor their limit, hold echo slots, coalesce serves A permit with a new limit files the stream again with the new regions before replaying, so its floors certify against the scope it now has. A claim is served only through a session whose limit admits it, even when its regions reach it through another session proving the same key. An ended route keeps its announce slot until its ROUTE_END echo is flushed. The node coalesces serving changes per claim, and a claim left with no usable route keeps the name its peer gave the identity. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/auth.rs | 6 + rs/moq-net/src/lite/subscriber/routed.rs | 135 ++++++++++++++++++--- rs/moq-net/src/route/node.rs | 147 +++++++++++++++++++++-- 3 files changed, 260 insertions(+), 28 deletions(-) diff --git a/rs/moq-net/src/auth.rs b/rs/moq-net/src/auth.rs index 1a412ad0dd8..8623a8acc55 100644 --- a/rs/moq-net/src/auth.rs +++ b/rs/moq-net/src/auth.rs @@ -588,6 +588,12 @@ impl Permit { Some(crate::model::interest_prefixes(limit)) } + /// Whether both set the same limit on the peer, whatever either was granted. + #[cfg(feature = "async")] + pub(crate) fn same_limit(&self, other: &Self) -> bool { + self.limit == other.limit + } + /// Whether this allows everything `other` does, judged part by part, so a `false` /// may be spurious but a `true` never is. pub(crate) fn covers(&self, other: &Self) -> bool { diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index b804c10cd84..8c3740ca4db 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -82,6 +82,9 @@ pub(super) struct Routed { /// The session's announce cap: one slot per live route and claim. slots: crate::session::Slots, held: Vec, + /// The slots of routes the peer ended, kept until their ROUTE_END echo is flushed, + /// so a peer leaving its replies unread holds its routes' slots. + echoing: Vec, served: HashMap<(route::Id, Path), ServedClaim>, /// What the stream served before it filed again, while the new one takes it up. previous: HashMap<(route::Id, Path), ServedClaim>, @@ -138,6 +141,7 @@ impl Routed { claims: HashMap::new(), slots: subscriber.announces.clone(), held: Vec::new(), + echoing: Vec::new(), served: HashMap::new(), previous: HashMap::new(), ready: Default::default(), @@ -184,7 +188,7 @@ impl Routed { self.claims.remove(&claim); self.held.pop(); } - self.held.pop(); + self.echoing.extend(self.held.pop()); self.stream.route(route.node, None); self.replies.push(AnnounceReply::RouteEnd { id }); } @@ -332,24 +336,21 @@ impl Routed { self.outputs(subscriber); } - /// The session's permit changed: a scope the stream no longer matches, or a - /// limit admitting a different set of the peer's claims, files it again, the - /// peer's routes and the claims the limit admits replayed before the old stream - /// lets go. The peer still holds each claim either way. + /// The session's permit changed. A new limit files the stream again, the peer's + /// routes and the claims the limit admits replayed before the old stream lets go: + /// it may change the scope, the claims admitted, the claims the node may serve + /// here, and what the floors certify. The peer still holds each claim either way. pub(super) fn limit(&mut self, subscriber: &Subscriber, permit: &crate::auth::Permit) -> Result<(), Error> { - let regions = regions(&self.root, permit); - let admitted = (self.routes.values()) - .filter_map(|route| route.committed.as_ref()) - .flat_map(|(claims, _)| claims.keys()) - .any(|path| admits(&self.root, &self.permit, path) != admits(&self.root, permit, path)); + let same = permit.same_limit(&self.permit); self.permit = permit.clone(); - if regions == self.regions && !admitted { + if same { return Ok(()); } + self.regions = regions(&self.root, permit); let key = self.peer_setup.route_key().unwrap_or(self.session_key); let stream = self .node - .receive(scoped(&self.link, ®ions, key), self.prefix.clone()) + .receive(scoped(&self.link, &self.regions, key), self.prefix.clone()) .map_err(refused)?; // What the old stream served stays in the origin until the new one serves it // again, or lets go after. @@ -373,7 +374,6 @@ impl Routed { if self.drained { stream.goaway(); } - self.regions = regions; drop(std::mem::replace(&mut self.stream, stream)); self.previous = previous; self.outputs(subscriber); @@ -437,6 +437,13 @@ impl Routed { // Outside the session's root, so nothing a request could name here. return; }; + // A route reaches the peer's origin node, but the scope's regions can be wider + // than its limit: a claim another session of the same peer carries is served + // here only if this session may carry it too. Left unconfirmed, the node keeps + // it where the limit admits it. + if !self.permit.within_limit(path.as_str()) { + return; + } let cost = match self.drained || !served.available { true => crate::origin::Cost::DRAIN, false => crate::origin::Cost::new(served.claim.cost.saturating_add(served.metric)), @@ -507,8 +514,10 @@ impl Routed { for reply in self.replies.drain(..) { writer.buffer(&reply)?; } - if let Poll::Ready(Err(err)) = writer.poll_flush(waiter) { - return Err(err); + match writer.poll_flush(waiter) { + Poll::Ready(Ok(())) => self.echoing.clear(), + Poll::Ready(Err(err)) => return Err(err), + Poll::Pending => {} } while let Poll::Ready(Ok(key)) = self.ready.poll_pop(waiter) { let Some(entry) = self.served.get_mut(&key) else { @@ -631,6 +640,102 @@ mod tests { assert_eq!(served(&routed), ["live/a/public/x"]); } + /// An unscoped stream narrowed to a limit that leaves a claim out certifies nothing, + /// so widening it again brings that claim back at the same revision. + #[kio::test] + async fn a_new_scope_certifies_against_its_own_regions() { + let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + let prefix = Path::empty(); + let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let mut decoder = AnnounceDecoder::default(); + let floor = Msg::Floor { + id: 0, + revision: Some(0), + }; + let route = Msg::RouteStart { + node: 7, + advert: ADVERT, + }; + for msg in [route, start("live/a/public/x"), start("live/a/private/y"), floor] { + routed.receive(&prefix, msg, &mut decoder).unwrap(); + } + routed.outputs(&subscriber); + assert_eq!(served(&routed), ["live/a/private/y", "live/a/public/x"]); + + routed.limit(&subscriber, &limited("live/*/public/**")).unwrap(); + assert_eq!(served(&routed), ["live/a/public/x"]); + + routed.limit(&subscriber, &limited("live/**")).unwrap(); + assert_eq!(served(&routed), ["live/a/private/y", "live/a/public/x"]); + } + + /// Two sessions proving one route key share the peer's routes, but a session serves + /// only the claims its own limit admits, even where its regions reach the rest. + #[kio::test] + async fn a_claim_is_served_only_where_the_limit_admits_it() { + let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + let prefix = Path::empty(); + let everything = limited("live/**"); + let public = limited("live/*/public/**"); + let mut wide = super::Routed::new(&subscriber, &prefix, 1, &everything).unwrap(); + let mut narrow = super::Routed::new(&subscriber, &prefix, 1, &everything).unwrap(); + narrow.session_key = wide.session_key; + narrow.limit(&subscriber, &public).unwrap(); + + for routed in [&mut wide, &mut narrow] { + let mut decoder = AnnounceDecoder::default(); + let floor = Msg::Floor { + id: 0, + revision: Some(0), + }; + let route = Msg::RouteStart { + node: 7, + advert: ADVERT, + }; + for msg in [route, start("live/a/public/x"), start("live/a/private/y"), floor] { + routed.receive(&prefix, msg, &mut decoder).unwrap(); + } + } + wide.outputs(&subscriber); + narrow.outputs(&subscriber); + assert_eq!(served(&wide), ["live/a/private/y", "live/a/public/x"]); + assert_eq!(served(&narrow), ["live/a/public/x"]); + } + + /// A route the peer ended keeps its announce slot until its ROUTE_END echo is + /// flushed, so a peer leaving its replies unread cannot churn routes past the cap. + #[kio::test] + async fn an_ended_route_holds_its_slot_until_its_echo_is_flushed() { + let (mut subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + subscriber.announces = crate::session::Slots::new(2); + let prefix = Path::empty(); + let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let mut decoder = AnnounceDecoder::default(); + let budget = kio::Producer::new(0); + let log = crate::lite::test_transport::Log::default(); + let mut writer = crate::coding::Writer::new( + crate::lite::test_transport::SinkSend::stepped(log, budget.consume()), + crate::lite::Version::Lite07, + ); + let waiter = kio::Waiter::noop(); + + for (id, node) in [(0, 7), (1, 8)] { + let route = Msg::RouteStart { node, advert: ADVERT }; + routed.receive(&prefix, route, &mut decoder).unwrap(); + routed.receive(&prefix, Msg::RouteEnd { id }, &mut decoder).unwrap(); + routed.poll(&subscriber, &mut writer, &waiter).unwrap(); + } + let route = Msg::RouteStart { + node: 9, + advert: ADVERT, + }; + assert!(routed.receive(&prefix, route, &mut decoder).is_err(), "the unflushed echoes hold the cap"); + + *budget.write().ok().expect("the budget is open") = usize::MAX; + routed.poll(&subscriber, &mut writer, &waiter).unwrap(); + assert_eq!(subscriber.announces.available(), 2); + } + /// What the stream keeps between floors is the net change to its committed claims, /// however many messages the peer sends without one. #[kio::test] diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index d7db156c0a8..46ef403cb6c 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -100,9 +100,10 @@ type Queues = HashMap>; /// What one stream has yet to read. Route changes and claim batches coalesce per /// identity, and what the reader was never sent leaves nothing when it ends: a route -/// started and ended unread is acked to the router at once, as the reader would. A -/// stream whose reader stalls holds one entry per route it holds or will start, -/// bounded by its claims, however much the node changes meanwhile. +/// started and ended unread is acked to the router at once, as the reader would. +/// Serving changes coalesce per claim. A stream whose reader stalls holds one entry +/// per route it holds or will start and per claim it serves, however much the node +/// changes meanwhile. #[derive(Default)] struct Outbox { /// Identities with route changes or claims pending, by when they first queued. @@ -111,6 +112,12 @@ struct Outbox { pending: HashMap, /// Outputs to read in order: the other kinds, and a pending entry once taken. ready: VecDeque, + /// Each claim's last serving change, read after the rest: its identity as the peer + /// names it and how it is served, or `None` to stop serving it. Keyed by when it + /// last changed, so a takeover's successor still reads before its predecessor goes. + serves: BTreeMap)>, + /// Where each claim's last change sits in `serves`. + serving: HashMap<(Id, Path), u64>, /// Each route the reader holds, and the claims it was sent under it. delivered: HashMap>, } @@ -133,7 +140,7 @@ type Changes = BTreeMap>; impl Outbox { fn is_empty(&self) -> bool { - self.ready.is_empty() && self.order.is_empty() + self.ready.is_empty() && self.order.is_empty() && self.serves.is_empty() } /// The pending entry for `id`, queued behind the others when new. @@ -200,6 +207,21 @@ impl Outbox { *last = floor; None } + // Each serving change replaces the last: the session acts on where the claim + // ends up, in place when it keeps its instance. + Output::Serve { + id, + named, + claim, + served, + } => { + self.serve(id, claim, Some((named, served))); + None + } + Output::Unserve { id, claim } => { + self.serve(id, claim, None); + None + } other => { self.ready.push_back(other); None @@ -209,13 +231,39 @@ impl Outbox { fn pop(&mut self) -> Option { if self.ready.is_empty() { - let (_, id) = self.order.pop_first()?; - let (_, pending) = self.pending.remove(&id).expect("every queued identity is pending"); - self.take(id, pending); + match self.order.pop_first() { + Some((_, id)) => { + let (_, pending) = self.pending.remove(&id).expect("every queued identity is pending"); + self.take(id, pending); + } + None => { + let (_, (id, claim, serve)) = self.serves.pop_first()?; + self.serving.remove(&(id, claim.clone())); + return Some(match serve { + Some((named, served)) => Output::Serve { + id, + named, + claim, + served, + }, + None => Output::Unserve { id, claim }, + }); + } + } } self.ready.pop_front() } + /// Replace the claim's queued serving change with `serve`, queued last. + fn serve(&mut self, id: Id, claim: Path, serve: Option<(Id, Served)>) { + let seq = self.next; + self.next += 1; + if let Some(old) = self.serving.insert((id, claim.clone()), seq) { + self.serves.remove(&old); + } + self.serves.insert(seq, (id, claim, serve)); + } + /// Queue one identity's pending entry to read, as the router would have sent it. fn take(&mut self, id: Id, pending: Pending) { if pending.ended { @@ -266,6 +314,8 @@ struct Home { struct Slot { served: Served, + /// The identity as the stream's peer names it, kept while no route to it is left. + named: Id, confirmed: bool, /// No longer a route to the claim's identity: let go once a wanted stream confirms. leaving: bool, @@ -565,40 +615,45 @@ impl State { gone.remove(&claim); let routes = self.router.routes(id, &claim); let home = self.homes.entry((id, claim.clone())).or_default(); - let wanted: Vec<(Key, Metric, bool)> = match routes.is_empty() { + // With no route left, the router no longer knows the name a stream's peer + // gave the identity, so it keeps the one it was served under. + let wanted: Vec<(Key, Metric, bool, Id)> = match routes.is_empty() { true => (home.keys.iter()) - .map(|(key, slot)| (*key, slot.served.metric, false)) + .map(|(key, slot)| (*key, slot.served.metric, false, slot.named)) + .collect(), + false => (routes.iter()) + .map(|&(key, metric)| (key, metric, available, self.router.name(key, id))) .collect(), - false => routes.iter().map(|&(key, metric)| (key, metric, available)).collect(), }; for slot in home.keys.values_mut() { slot.leaving = true; } - for (key, metric, available) in wanted { + for (key, metric, available, named) in wanted { let served = Served { claim: item.clone(), metric, available, }; match home.keys.get_mut(&key) { - Some(slot) if slot.served == served => { + Some(slot) if slot.served == served && slot.named == named => { slot.leaving = false; continue; } Some(slot) => { slot.served = served.clone(); + slot.named = named; slot.leaving = false; } None => { let slot = Slot { served: served.clone(), + named, confirmed: false, leaving: false, }; home.keys.insert(key, slot); } } - let named = self.router.name(key, id); let serve = Output::Serve { id, named, @@ -794,6 +849,72 @@ mod tests { assert_eq!(claims(outputs(&b)), [p("live/b/y")]); } + /// A claim of a scoped peer left with no usable route stays where it is served, + /// under the name that peer gave its identity, so the requests it serves meanwhile + /// still name it as the peer does. + #[test] + fn a_claim_without_a_route_keeps_the_peers_name() { + let instance = Instance::random(); + let origin = Node::new(1, 1); + let relay = Node::new(2, 2); + let scoped = || Link { + scope: Some(super::super::Scope { + regions: vec![Path::empty()], + qualify: super::super::qualifier([7; 32]), + }), + ..Link::default() + }; + let first = relay.receive(scoped(), Path::empty()).unwrap(); + let second = relay.receive(scoped(), Path::empty()).unwrap(); + let first_answer = origin.answer(Link::default(), Path::empty()).unwrap(); + let second_answer = origin.answer(Link::default(), Path::empty()).unwrap(); + origin.own(p("live/x"), Some(claim(&instance, 0)), None); + pump(&first_answer, &first); + pump(&second_answer, &second); + second.route(1, Some(Reach::Held)); + let names = |outputs: Vec| -> Vec<(Id, bool)> { + (served(outputs).into_iter()) + .filter_map(|output| match output { + Output::Serve { named, served, .. } => Some((named, served.available)), + _ => None, + }) + .collect() + }; + assert_eq!(names(outputs(&first)), [(1, true)]); + + // The second still holds a route it may recover through. + first.route(1, None); + assert_eq!(names(outputs(&first)), [(1, false)]); + } + + /// A stream whose reader stalls holds one serving change per claim, however often + /// the claim changes through another stream meanwhile. + #[test] + fn a_stalled_stream_holds_one_serve_per_claim() { + let instance = Instance::random(); + let origin = Node::new(1, 1); + let relay = Node::new(2, 2); + let fast = relay.receive(Link::default(), Path::empty()).unwrap(); + let stalled = relay.receive(Link::default(), Path::empty()).unwrap(); + let fast_answer = origin.answer(Link::default(), Path::empty()).unwrap(); + let stalled_answer = origin.answer(Link::default(), Path::empty()).unwrap(); + origin.own(p("live/x"), Some(claim(&instance, 0)), None); + pump(&fast_answer, &fast); + pump(&stalled_answer, &stalled); + for cost in 1..1000 { + origin.own(p("live/x"), Some(claim(&instance, cost)), None); + pump(&fast_answer, &fast); + outputs(&fast); + } + pump(&stalled_answer, &stalled); + let read = served(outputs(&stalled)); + assert_eq!(read.len(), 1, "{read:?}"); + assert!( + matches!(&read[0], Output::Serve { served, .. } if served.claim.cost == 999), + "{read:?}" + ); + } + /// A reader that stops reading holds one entry for the route, however often its /// claims change: the last change to each, and none for a claim it was never sent. #[test] From f2e03117c2adc659cf0313f22bc06f381d2ed7c9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 18:28:51 -0800 Subject: [PATCH 31/40] fix(net): keep upstream-learned claims from lite-07 upstreams, answer nothing past the root A relay's own claims include routes it learned from an upstream over an older version; an upstream answer now leaves those out and certifies no floor for them, as the lite-06 horizon does. A lite-07 prefix too long to lie beneath the handle's root answers nothing instead of the root. Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/cluster.md | 6 + rs/moq-net-tests/tests/integration/restart.rs | 9 +- rs/moq-net/src/lite/publisher.rs | 31 +++++ rs/moq-net/src/lite/publisher/routed.rs | 52 ++++---- rs/moq-net/src/lite/subscriber/routed.rs | 1 + rs/moq-net/src/model/origin.rs | 117 ++++++++++++++---- rs/moq-net/src/route/node.rs | 56 +++------ rs/moq-net/tests/integration/mesh_withdraw.rs | 37 ++++++ 8 files changed, 223 insertions(+), 86 deletions(-) diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index 11beeb67d9f..a08f00c2984 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -179,6 +179,12 @@ the same one for a given path. A publisher connected straight to the relay is not hashed at its own path: its newest session wins, so a publisher that restarts takes over from its lingering old session at once. +moq-lite-07-wip carries no hop list, so both hop-list steps drop out: the first +relay ranks the origin nodes announcing a path the same way otherwise, and +remaining ties hash the path with each node's routing identity. Later hops +follow their own route to the node it picked. Routes learned over an older +version rank as above. + ```toml [cluster] connect = ["https://sibling.same-dc/?cost=1", "https://us-east.example.com/?cost=10"] diff --git a/rs/moq-net-tests/tests/integration/restart.rs b/rs/moq-net-tests/tests/integration/restart.rs index 3a5a97e1e5f..3dcd2cfffd3 100644 --- a/rs/moq-net-tests/tests/integration/restart.rs +++ b/rs/moq-net-tests/tests/integration/restart.rs @@ -679,11 +679,10 @@ impl Pool { } } -/// Wake the worker for `path` through the claim and subscribe. Every wire carries the -/// answer as an ordinary announcement, which resets the first subscription through the -/// claim, so wake the worker first and subscribe through the exact route. Lite-07's -/// takeover is gone until quest/m2/announce-takeover.md carries the claim's instance onto -/// the exact route. +/// Wake the worker for `path` through the claim and subscribe. Wake the worker first and +/// subscribe through the exact route, so the test reads the same on every wire, whether +/// it carries the answer as a takeover or as an ordinary announcement that resets the +/// first subscription through the claim. async fn claimed(origin: &origin::Producer, path: &str) -> (broadcast::Consumer, track::Subscriber) { let cold = origin.consume().request(path, None).await.unwrap(); drop(cold.track("video").unwrap().subscribe(None).await); diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 91551958e83..c3399d98c2e 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -3636,6 +3636,37 @@ mod announce_test { ); } + /// A lite-07 prefix too long to lie beneath the handle's root names nothing there, + /// so the answer announces nothing rather than what the root holds. + #[test] + fn a_lite07_prefix_past_the_root_announces_nothing() { + let origin = Hop::new(1).unwrap().produce(); + let root = "r".repeat(100); + let _route = origin.announce(format!("{root}/x"), origin::RouteInfo::default()).unwrap(); + let consumer = origin + .consume() + .scope(root.as_str(), &crate::Patterns::from(crate::Pattern::all())) + .unwrap(); + let prefix = crate::Path::new(&"p".repeat(crate::Path::MAX_BYTES - 50)).unwrap(); + let mut answer = super::routed::Answer::new(consumer, prefix, true, crate::auth::Link::new(false)); + let log = Log::default(); + let mut wire = Wire { + writes: log.writes.clone(), + cursor: 0, + version: Version::Lite07, + }; + let mut stream = Stream:: { + writer: Writer::new( + crate::transport::boxed::SendStream::new(SinkSend::new(log)), + Version::Lite07, + ), + reader: Reader::new(crate::transport::boxed::RecvStream::new(PendingRecv), Version::Lite07), + }; + assert!(answer.poll(&mut stream, false, &kio::Waiter::noop()).is_pending()); + wire.take_ok(); + assert_eq!(wire.take_announces(), [lite::AnnounceBroadcast::Synced]); + } + #[test] fn synced_survives_a_removed_initial_route_and_live_churn() { let origin = Hop::new(1).unwrap().produce(); diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index cde695a2ae3..58dbe852dbc 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -2,9 +2,9 @@ //! //! The node sends the peer its routes and, closed by an ANNOUNCE_FLOOR, each route's //! claims under the requested prefix. The stream filters what it puts on the wire: -//! hidden claims unless the peer asked for them, and anything this handle's scope or -//! the session's grant leaves out, in which case a floor certifies nothing, since the -//! peer cannot tell what was left out from what is gone. +//! hidden claims unless the peer asked for them, and anything this handle's scope, the +//! session's grant, or the upstream horizon leaves out, in which case a floor certifies +//! nothing, since the peer cannot tell what was left out from what is gone. //! //! A handle's mounts beneath the prefix are read through a node stream each, at the //! target, and what they present is named under the mount point on the same routes. @@ -66,8 +66,9 @@ enum Phase { /// Answers one lite-07 announce stream. pub(super) struct Answer { origin: origin::Consumer, - /// The requested prefix, absolute. - prefix: Path, + /// The requested prefix, absolute. `None` when it is too long to lie beneath the + /// handle's root: nothing is there, so the stream announces nothing. + prefix: Option, hidden: bool, link: route::Link, /// The prefix's own node stream, then one per mount beneath it. Empty until the @@ -100,7 +101,7 @@ impl Answer { proxy: origin.proxy(), scope: None, }; - let absolute = origin.root().join(&prefix).unwrap_or_else(|_| origin.root().clone()); + let absolute = origin.root().join(&prefix).ok(); Self { gate: crate::auth::Gate::new(auth, prefix, crate::auth::Direction::Publish), prefix: absolute, @@ -139,8 +140,9 @@ impl Answer { active: 0, })?; let node = self.origin.route_node(); - let mounts = self.origin.mounted(&self.prefix); - let interests = std::iter::once((self.prefix.clone(), None)) + let mounts = (self.prefix.as_ref()).map_or_else(Vec::new, |prefix| self.origin.mounted(prefix)); + let interests = (self.prefix.iter()) + .map(|prefix| (prefix.clone(), None)) .chain(mounts.into_iter().map(|mount| (mount.interest.clone(), Some(mount)))); for (interest, mount) in interests { let stream = node @@ -394,9 +396,13 @@ impl Answer { /// Send the ANNOUNCE_FLOOR closing a batch for the route to `id`. fn floor(&mut self, stream: &mut Stream, id: route::Id) -> Result<(), Error> { let certified = self.certified(); + let upstream = self.link.upstream; let sent = self.routes.get_mut(&id).expect("only a held route has a batch"); sent.floored = true; - let revision = sent.floor.filter(|_| certified); + // Like the grant, the upstream horizon leaves claims out that a floor would + // certify absent. + let hides = upstream && sent.claims.values().any(|claim| claim.upstream); + let revision = sent.floor.filter(|_| certified && !hides); stream.writer.buffer(&Msg::Floor { id: sent.wire, revision, @@ -407,13 +413,21 @@ impl Answer { /// scope or the session's grant leaves out lies beneath it. A hidden claim left out /// does not count, since the peer did not ask to see it. fn certified(&self) -> bool { - let relative = self.prefix.strip_prefix(self.origin.root()).unwrap_or_default(); - self.origin.covers(&self.prefix) && self.permit.covers_subtree(relative.as_str()) + let Some(prefix) = &self.prefix else { + return false; + }; + let relative = prefix.strip_prefix(self.origin.root()).unwrap_or_default(); + self.origin.covers(prefix) && self.permit.covers_subtree(relative.as_str()) + } + + /// The absolute `path` as the peer names it, under its requested prefix. + fn suffix(&self, path: &Path) -> Option { + path.strip_prefix(self.prefix.as_ref()?) } /// Whether the peer is shown the claim at the absolute `path`. fn shows(&self, path: &Path) -> bool { - let Some(suffix) = path.strip_prefix(&self.prefix) else { + let Some(suffix) = self.suffix(path) else { return false; }; let Some(relative) = path.strip_prefix(self.origin.root()) else { @@ -434,7 +448,9 @@ impl Answer { started: &mut BTreeSet, ) -> Result<(), Error> { let sent = self.routes.get(&id).expect("only a held route syncs"); - let wanted = sent.claims.get(&path).filter(|_| self.shows(&path)).cloned(); + let wanted = (sent.claims.get(&path)) + .filter(|claim| self.shows(&path) && !(self.link.upstream && claim.upstream)) + .cloned(); let wire = sent.wire; let taken = (wanted.as_ref()) .and_then(|claim| self.continued(id, &path, claim)) @@ -442,10 +458,7 @@ impl Answer { let key = (id, path); match (self.live.get_mut(&key), wanted) { (None, Some(claim)) => { - let suffix = key - .1 - .strip_prefix(&self.prefix) - .expect("shown claims are under the prefix"); + let suffix = self.suffix(&key.1).expect("shown claims are under the prefix"); let announce = match taken { Some((taken, predecessor)) => { tracing::debug!(route = %key.1, node = id, predecessor = %taken, "takeover"); @@ -476,10 +489,7 @@ impl Answer { // In place: a new Announce ID replaces the one it continues. Some((taken, predecessor)) if taken == key.1 => { tracing::debug!(route = %key.1, node = id, "takeover"); - let suffix = key - .1 - .strip_prefix(&self.prefix) - .expect("held claims are under the prefix"); + let suffix = self.suffix(&key.1).expect("held claims are under the prefix"); self.takeover(stream, wire, predecessor, suffix, &claim)? } _ if restart => { diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 8c3740ca4db..28db4a8d62f 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -541,6 +541,7 @@ fn claim(instance: Option, epoch: u64, cost: crate::origin::Cos epoch, cost: cost.value(), continues: None, + upstream: false, }) } diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 414d0c3f11b..fd702db6140 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -3594,6 +3594,12 @@ impl Drop for AnnounceProducer { /// for as long as the origin lives. struct OwnClaims { routes: Routes, + /// The same routes, less those learned on an upstream link. + downstream: Routes, + /// The claim fed at each prefix. + best: HashMap, + /// The instance at each prefix that a route not learned upstream serves. + local: HashMap, node: crate::route::Node, } @@ -3601,40 +3607,99 @@ impl kio::Task for OwnClaims { type Output = (); fn poll(&mut self, waiter: &kio::Waiter) -> Poll<()> { - while let Some(event) = ready!(self.routes.poll_next(waiter)) { - self.apply(event); + loop { + if let Poll::Ready(event) = self.downstream.poll_next(waiter) { + match event { + Some(event) => self.local(event), + None => return Poll::Ready(()), + } + continue; + } + match ready!(self.routes.poll_next(waiter)) { + Some(event) => self.apply(event), + None => return Poll::Ready(()), + } } - Poll::Ready(()) } } impl OwnClaims { /// Feed the node the routes the table already holds, so a stream answered before /// the driver next runs sees them. - fn new(routes: Routes, node: crate::route::Node) -> Self { - let mut own = Self { routes, node }; + fn new(routes: Routes, downstream: Routes, node: crate::route::Node) -> Self { + let mut own = Self { + routes, + downstream, + best: HashMap::new(), + local: HashMap::new(), + node, + }; + while let Some(event) = own.downstream.try_next() { + own.local(event); + } while let Some(event) = own.routes.try_next() { own.apply(event); } own } - fn apply(&self, event: RouteEvent) { + fn apply(&mut self, event: RouteEvent) { let (announce, predecessor) = match event { RouteEvent::Start(announce) | RouteEvent::Update(announce) | RouteEvent::Restart(announce) => { (announce, None) } RouteEvent::Takeover { announce, predecessor } => (announce, Some(predecessor)), - RouteEvent::End(announce) => return self.node.own(announce.prefix, None, None), + RouteEvent::End(announce) => { + self.best.remove(&announce.prefix); + return self.node.own(announce.prefix, None, None); + } }; let route = announce.route; - let claim = route.instance.map(|instance| crate::route::node::Claim { + let Some(instance) = route.instance else { + return; + }; + let claim = crate::route::node::Claim { instance, epoch: route.epoch, cost: route.cost.value(), continues: None, - }); - self.node.own(announce.prefix, claim, predecessor.as_ref()); + upstream: false, + }; + self.best.insert(announce.prefix.clone(), claim); + self.feed(announce.prefix, predecessor.as_ref()); + } + + /// The route not learned upstream changed at a prefix: whether the claim there may + /// go to an upstream peer follows it. + fn local(&mut self, event: RouteEvent) { + match event { + RouteEvent::End(announce) => { + self.local.remove(&announce.prefix); + self.feed(announce.prefix, None); + } + RouteEvent::Start(announce) + | RouteEvent::Update(announce) + | RouteEvent::Restart(announce) + | RouteEvent::Takeover { announce, .. } => { + if let Some(instance) = announce.route.instance { + self.local.insert(announce.prefix.clone(), instance); + } + self.feed(announce.prefix, None); + } + } + } + + /// Feed the node the claim at `prefix`, marked as learned upstream unless a route + /// not learned upstream serves the same instance there. + fn feed(&self, prefix: Path, predecessor: Option<&Path>) { + let Some(claim) = self.best.get(&prefix) else { + return; + }; + let claim = crate::route::node::Claim { + upstream: self.local.get(&prefix) != Some(&claim.instance), + ..claim.clone() + }; + self.node.own(prefix, Some(claim), predecessor); } } @@ -7147,20 +7212,24 @@ impl Consumer { shared.route_node = Some(node.clone()); drop(shared); // The whole table, whatever this handle's scope. - let own = Self { - scope: OriginScope::default(), - root: Path::default(), - horizon: Horizon { - node: NodeFilter::Own, - ..Horizon::default() - }, - covering: false, - ..self.clone() - } - .with_hidden(true) - // Feeding the node is no reason to ask any peer about anything. - .serving(); - self.tasks.push(OwnClaims::new(own.routes(), node.clone())); + let own = |upstream| { + Self { + scope: OriginScope::default(), + root: Path::default(), + horizon: Horizon { + node: NodeFilter::Own, + upstream, + ..Horizon::default() + }, + covering: false, + ..self.clone() + } + .with_hidden(true) + // Feeding the node is no reason to ask any peer about anything. + .serving() + }; + let claims = OwnClaims::new(own(false).routes(), own(true).routes(), node.clone()); + self.tasks.push(claims); node } diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 46ef403cb6c..4a4a288d0f3 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -41,6 +41,10 @@ pub(crate) struct Claim { /// this claim's broadcast. Kept while the instance stays, and sent as /// ANNOUNCE_TAKEOVER to a peer still holding that claim. pub continues: Option<(Path, Instance)>, + /// This node's own claim, served through a route learned from an upstream peer: it + /// is never sent to another upstream, so this node never carries traffic between + /// two. Always unset on a claim off the wire. + pub upstream: bool, } /// A shown claim as the origin serves it through the selected route. @@ -112,10 +116,9 @@ struct Outbox { pending: HashMap, /// Outputs to read in order: the other kinds, and a pending entry once taken. ready: VecDeque, - /// Each claim's last serving change, read after the rest: its identity as the peer - /// names it and how it is served, or `None` to stop serving it. Keyed by when it - /// last changed, so a takeover's successor still reads before its predecessor goes. - serves: BTreeMap)>, + /// Each claim's last Serve or Unserve, read after the rest. Keyed by when it last + /// changed, so a takeover's successor still reads before its predecessor goes. + serves: BTreeMap, /// Where each claim's last change sits in `serves`. serving: HashMap<(Id, Path), u64>, /// Each route the reader holds, and the claims it was sent under it. @@ -209,17 +212,13 @@ impl Outbox { } // Each serving change replaces the last: the session acts on where the claim // ends up, in place when it keeps its instance. - Output::Serve { - id, - named, - claim, - served, - } => { - self.serve(id, claim, Some((named, served))); - None - } - Output::Unserve { id, claim } => { - self.serve(id, claim, None); + Output::Serve { id, ref claim, .. } | Output::Unserve { id, ref claim } => { + let seq = self.next; + self.next += 1; + if let Some(old) = self.serving.insert((id, claim.clone()), seq) { + self.serves.remove(&old); + } + self.serves.insert(seq, output); None } other => { @@ -237,33 +236,17 @@ impl Outbox { self.take(id, pending); } None => { - let (_, (id, claim, serve)) = self.serves.pop_first()?; - self.serving.remove(&(id, claim.clone())); - return Some(match serve { - Some((named, served)) => Output::Serve { - id, - named, - claim, - served, - }, - None => Output::Unserve { id, claim }, - }); + let (_, output) = self.serves.pop_first()?; + if let Output::Serve { id, claim, .. } | Output::Unserve { id, claim } = &output { + self.serving.remove(&(*id, claim.clone())); + } + return Some(output); } } } self.ready.pop_front() } - /// Replace the claim's queued serving change with `serve`, queued last. - fn serve(&mut self, id: Id, claim: Path, serve: Option<(Id, Served)>) { - let seq = self.next; - self.next += 1; - if let Some(old) = self.serving.insert((id, claim.clone()), seq) { - self.serves.remove(&old); - } - self.serves.insert(seq, (id, claim, serve)); - } - /// Queue one identity's pending entry to read, as the router would have sent it. fn take(&mut self, id: Id, pending: Pending) { if pending.ended { @@ -716,6 +699,7 @@ mod tests { epoch: 0, cost, continues: None, + upstream: false, } } diff --git a/rs/moq-net/tests/integration/mesh_withdraw.rs b/rs/moq-net/tests/integration/mesh_withdraw.rs index 035d0e4959f..f02e8d392a3 100644 --- a/rs/moq-net/tests/integration/mesh_withdraw.rs +++ b/rs/moq-net/tests/integration/mesh_withdraw.rs @@ -679,3 +679,40 @@ async fn mixed_version_ring_withdraw_retracts() { ); } } + +/// A relay never carries traffic between two upstreams, across versions too: a +/// broadcast it learned from a moq-lite-06 upstream enters the route layer as one of +/// its own claims, yet never reaches a moq-lite-07 upstream, while one published at +/// the relay does. +#[kio::test] +async fn an_upstream_learned_claim_never_reaches_another_upstream() { + let lite07: Version = "moq-lite-07-wip".parse().unwrap(); + let lite06: Version = "moq-lite-06".parse().unwrap(); + let first = produce_origin(1); + let relay = produce_origin(2); + let second = produce_origin(3); + let upstream = relay.clone().with_link(origin::Link::Upstream); + let mut pairs = Vec::new(); + for (version, core) in [(lite06, &first), (lite07, &second)] { + let core = core.clone().with_link(origin::Link::Peer); + let mut options = MockConnectOptions::new(version); + options.client_publish = Some(upstream.consume().with_hidden(true)); + options.client_subscribe = Some(upstream.clone()); + options.server_publish = Some(core.consume().with_hidden(true)); + options.server_subscribe = Some(core); + pairs.push(connect_mock(options).await); + } + + let mut at_relay = watch(relay.consume().routes()); + let mut at_second = watch(second.consume().routes()); + let learned = first.route("room/learned", origin::RouteOptions::hidden()).unwrap(); + learned.announce().unwrap(); + let local = relay.route("room/local", origin::RouteOptions::hidden()).unwrap(); + local.announce().unwrap(); + kio::clock::sim::idle().await; + let relayed = drain(&mut at_relay); + assert!(relayed.contains_key("room/learned"), "{relayed:?}"); + let seen = drain(&mut at_second); + assert_eq!(seen.get("room/local"), Some(&vec![Kind::Start]), "{seen:?}"); + assert!(!seen.contains_key("room/learned"), "{seen:?}"); +} From 7e0897d2495c9ad46acea5bc2dfa8dbe6b9532a8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 18:33:52 -0800 Subject: [PATCH 32/40] style(net): format the new lite-07 tests Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/publisher.rs | 4 +++- rs/moq-net/src/lite/subscriber/routed.rs | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index c3399d98c2e..78892904425 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -3642,7 +3642,9 @@ mod announce_test { fn a_lite07_prefix_past_the_root_announces_nothing() { let origin = Hop::new(1).unwrap().produce(); let root = "r".repeat(100); - let _route = origin.announce(format!("{root}/x"), origin::RouteInfo::default()).unwrap(); + let _route = origin + .announce(format!("{root}/x"), origin::RouteInfo::default()) + .unwrap(); let consumer = origin .consume() .scope(root.as_str(), &crate::Patterns::from(crate::Pattern::all())) diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 28db4a8d62f..e31ac3fd245 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -730,7 +730,10 @@ mod tests { node: 9, advert: ADVERT, }; - assert!(routed.receive(&prefix, route, &mut decoder).is_err(), "the unflushed echoes hold the cap"); + assert!( + routed.receive(&prefix, route, &mut decoder).is_err(), + "the unflushed echoes hold the cap" + ); *budget.write().ok().expect("the budget is open") = usize::MAX; routed.poll(&subscriber, &mut writer, &waiter).unwrap(); From d5ed89fd6045c6e0bcf726519083ccb831a10d2d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 18:41:13 -0800 Subject: [PATCH 33/40] test(drill): the lost-publisher mutation compiles against the routed subscriber The ported patch dropped the deref where the routed subscriber passes the predecessor's route to the origin, so the mutated tree failed to build. Co-Authored-By: Claude Opus 5.5 --- .../relay-withdraws-lost-publisher.patch | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/test/drill/mutations/relay-withdraws-lost-publisher.patch b/test/drill/mutations/relay-withdraws-lost-publisher.patch index 66e1b0ba07d..109e8096040 100644 --- a/test/drill/mutations/relay-withdraws-lost-publisher.patch +++ b/test/drill/mutations/relay-withdraws-lost-publisher.patch @@ -49,7 +49,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs -@@ -354,7 +354,7 @@ impl Routed { +@@ -378,7 +378,7 @@ impl Routed { self.previous = previous; self.outputs(subscriber); for (_, served) in std::mem::take(&mut self.previous) { @@ -58,7 +58,7 @@ diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subs } Ok(()) } -@@ -398,7 +398,7 @@ impl Routed { +@@ -422,7 +422,7 @@ impl Routed { } => self.serve(subscriber, id, named, claim, served), node::Output::Unserve { id, claim } => { if let Some(served) = self.served.remove(&(id, claim)) { @@ -67,3 +67,14 @@ diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subs } } // The ROUTE_END echo goes out as the peer's ROUTE_END is read, and a stream +@@ -477,8 +477,8 @@ impl Routed { + .map(|(taken, entry)| match *taken == key.1 { + // In place, its sources serve what the new one continues: they live on with + // it rather than closing with the old route. +- true => (&entry.route.dynamic, Some(std::mem::take(&mut entry.route.live))), +- false => (&entry.route.dynamic, None), ++ true => (&*entry.route.dynamic, Some(std::mem::take(&mut entry.route.live))), ++ false => (&*entry.route.dynamic, None), + }); + // A new instance or epoch replaces the old route, attached before it goes. + tracing::debug!(route = %subscriber.log_path(&path), node = id, "announce"); From f51f5292bec7f044cfb7d437acfb79b512080f20 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 18:57:03 -0800 Subject: [PATCH 34/40] fix(net): a coalesced Serve keeps its place behind a takeover Replacing a queued Serve moved it to the back, behind its predecessor's Unserve, so a takeover repriced before a stalled reader drained lost the predecessor it continues. A Serve replacing a Serve now keeps its place; an Unserve, or a Serve after one, still goes last. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/node.rs | 49 +++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 4a4a288d0f3..c9e1756fe59 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -116,8 +116,10 @@ struct Outbox { pending: HashMap, /// Outputs to read in order: the other kinds, and a pending entry once taken. ready: VecDeque, - /// Each claim's last Serve or Unserve, read after the rest. Keyed by when it last - /// changed, so a takeover's successor still reads before its predecessor goes. + /// Each claim's last Serve or Unserve, read after the rest, in queue order. A + /// takeover queues its successor's Serve before its predecessor's Unserve, and the + /// session needs the predecessor still served to carry it on: so a Serve replacing + /// a queued Serve keeps its place, while an Unserve, or a Serve after one, goes last. serves: BTreeMap, /// Where each claim's last change sits in `serves`. serving: HashMap<(Id, Path), u64>, @@ -213,9 +215,17 @@ impl Outbox { // Each serving change replaces the last: the session acts on where the claim // ends up, in place when it keeps its instance. Output::Serve { id, ref claim, .. } | Output::Unserve { id, ref claim } => { + let key = (id, claim.clone()); + if let Some(&seq) = self.serving.get(&key) + && matches!(output, Output::Serve { .. }) + && matches!(self.serves.get(&seq), Some(Output::Serve { .. })) + { + self.serves.insert(seq, output); + return None; + } let seq = self.next; self.next += 1; - if let Some(old) = self.serving.insert((id, claim.clone()), seq) { + if let Some(old) = self.serving.insert(key, seq) { self.serves.remove(&old); } self.serves.insert(seq, output); @@ -833,6 +843,39 @@ mod tests { assert_eq!(claims(outputs(&b)), [p("live/b/y")]); } + /// A stalled stream reads a takeover's successor before its predecessor goes, even + /// once the successor changed again meanwhile, so the session still finds the + /// predecessor to carry on. + #[test] + fn a_stalled_takeover_reads_its_successor_first() { + let (old, new) = (Instance::random(), Instance::random()); + let origin = Node::new(1, 1); + let relay = Node::new(2, 2); + let fast = relay.receive(Link::default(), Path::empty()).unwrap(); + let stalled = relay.receive(Link::default(), Path::empty()).unwrap(); + let fast_answer = origin.answer(Link::default(), Path::empty()).unwrap(); + let stalled_answer = origin.answer(Link::default(), Path::empty()).unwrap(); + origin.own(p("live"), Some(claim(&old, 0)), None); + pump(&fast_answer, &fast); + pump(&stalled_answer, &stalled); + outputs(&fast); + outputs(&stalled); + + origin.own(p("live/x"), Some(claim(&new, 0)), Some(&p("live"))); + origin.own(p("live"), None, None); + pump(&fast_answer, &fast); + origin.own(p("live/x"), Some(claim(&new, 5)), None); + pump(&fast_answer, &fast); + let read: Vec<(Path, Option)> = (served(outputs(&stalled)).into_iter()) + .map(|output| match output { + Output::Serve { claim, served, .. } => (claim, Some(served.claim.cost)), + Output::Unserve { claim, .. } => (claim, None), + other => panic!("not a serving change: {other:?}"), + }) + .collect(); + assert_eq!(read, [(p("live/x"), Some(5)), (p("live"), None)]); + } + /// A claim of a scoped peer left with no usable route stays where it is served, /// under the name that peer gave its identity, so the requests it serves meanwhile /// still name it as the peer does. From 1f5d0533683b6eed9212299e3d6cc86bf4998d12 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 20:09:03 -0800 Subject: [PATCH 35/40] fix(relay): a removed cluster ID names its removal at startup --cluster-id, MOQ_CLUSTER_ID, and [cluster] id stay as hidden settings reported through deprecated(), so a config that still sets one stops saying the Hop ID is random rather than refusing an unknown setting. The moq-net guide marks the proxying claim as lite-07-wip and opt-in. Co-Authored-By: Claude Opus 5.5 --- doc/lib/rs/moq-net.md | 2 +- rs/moq-relay/src/cluster.rs | 38 +++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index e655a8d6508..1449c8678ca 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -21,7 +21,7 @@ What you use it for, beyond what the concept page already describes: - **Rank several CDNs.** Dial each with its own subscribe handle, `origin.clone().with_preference(origin::Preference::new(rank))` (higher wins; 0 is neutral and a negative rank demotes), all on one origin. Requests use the preferred link that has a route and fail over to the next; see [multiple CDNs](/concept/moq-lite#multiple-cdns). - **Request one instance, or follow them all.** `consumer.request(path, None)` resolves the instance serving a path now, and fails `Unroutable` only once discovery has synced with no route for it. `consumer.follow(path, None)` hands out each publisher instance serving a path from `available()`: the first once announced, then each one that takes over, `None` once the path goes offline, and whatever serves it next, like JS `follow(path)`. `consumer.announced()` lists paths only as present or absent. A broadcast already resolved stays on the old instance until dropped or its route goes. See [identity and recovery](/concept/identity) to choose between following and holding one instance. - **Discovery follows what you watch.** A client session asks the relay for announcements only under the prefixes its subscribe origin's readers watch, with `announced()`, `request`, or `follow`, stops asking about a prefix 2s after nothing at or beneath it is watched, and asks again on reconnect. A broadcast resolved through a discovered route keeps its path watched until dropped. `request` owns discovery of its path, so no separate watch is needed. A server session, or a client fed into an origin handle marked `with_link(origin::Link::Peer)` or `Link::Upstream`, asks for everything it may, so a relay discovers everything. An HTTP gateway (the moq-hls `Server`, the moq-rtc WHEP router) watches its whole scope while it lives, so the paths its callers name never open a request each. moq-lite-03/04 sessions, whose announcements mark no initial set, still ask for everything. -- **Relay by opting into proxying.** On moq-lite-07 a session sends its peer only the routes to its origin's own claims, so an application never becomes transit for anyone else. A relay built on the library marks the handle it publishes with `origin.clone().with_proxy(true)`, or one consumer with `consumer.with_proxy(true)`, to carry what it learned from other peers too; `moq-relay` does so for every session. Older versions advertise everything either way. +- **Relay by opting into proxying.** On moq-lite-07 (`moq-lite-07-wip`, opt-in) a session sends its peer only the routes to its origin's own claims, so an application never becomes transit for anyone else. A relay built on the library marks the handle it publishes with `origin.clone().with_proxy(true)`, or one consumer with `consumer.with_proxy(true)`, to carry what it learned from other peers too; `moq-relay` does so for every session. Older versions advertise everything either way. - **Dial and accept sockets.** `socket::Endpoint::new(&origin, prefix, id)` announces a hidden endpoint at `/.socket/`, used only to route dials. A session speaks one of two protocols, each with its own dial and accept. `accept_tcp()` and `accept_track()` yield an `Incoming` dial to `accept` or `reject` (dropping it rejects). `dial_tcp(peer)` and an accepted `tcp` dial yield a `socket::TcpStream`, a `group::Producer` and `group::Consumer` pair carrying one reliable, ordered byte pipe each way (finish is FIN, abort is RST, and `ready` paces the writer to the reader); `dial_track(peer, info)` and an accepted `track` dial yield a `socket::TrackStream`, this side's `track::Producer` and a `track::Subscriber` of the peer's half, subscribed reliably and in order until the application updates it, one message per group. A dial resolves once the peer accepts, and an accept checks that the claimed dialer really dialed this endpoint, waiting for its endpoint while the dial stays open. Every hop needs moq-lite-07 or moq-transport: while the origin has an older moq-lite client session whose publish half doesn't opt into hidden broadcasts (`with_hidden(true)`), dials and accepts fail with `Unsupported`. A dial to a peer behind such a hop waits, as for a peer not online yet. Session tracks are never announced or listed. See [Sockets](/concept/socket). - **One track per name.** Concurrent subscriptions share one request and producer. The application owns sequence numbers: the broadcast keeps nothing of an ended track, so a producer that replaces it starts at 0. A name always means the same content, so the replacement continues past its predecessor's groups with `create_group` at an explicit sequence. Readers of an aborted track resume onto the replacement when they read through an origin request, whose route always names an instance; otherwise their subscription ends with the error. - **Authored tracks default to a zero `max_age`.** `track::Info::default()` sets `max_age` to `Some(Duration::ZERO)`, the live edge only; a publisher that wants readers to reach history sets a longer bound, or `with_max_age(None)` for no publisher limit. Media tracks from `hang::container::track_info` declare `hang::container::MAX_AGE` (30 seconds). `max_age` is how long a superseded group stays fresh, never the newest group; it frees no bytes, which the cache pool's expiry reclaims. Received metadata keeps absence as no limit, and an origin's `cache_duration` can only cap it lower. diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 12594c1198f..d47179d427c 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -794,6 +794,12 @@ pub struct Config { #[usage(skip)] #[serde(rename = "connect_api_tls_root", skip_serializing)] pub(crate) removed_api_root: Option, + #[usage(long = "cluster-id", env = "MOQ_CLUSTER_ID", hide = true)] + #[serde(skip)] + pub(crate) removed_id: Option, + #[usage(skip)] + #[serde(rename = "id", skip_serializing)] + pub(crate) removed_id_toml: Option, /// This relay's own externally-reachable URL (identity). Sent to /// `--cluster-api-url` as a `?node=` query param so the endpoint can return @@ -862,6 +868,13 @@ impl Config { if self.removed_api_root.is_some() { found.toml("[cluster] connect_api_tls_root", "[cluster.api.tls] root", None); } + const ID: &str = "the Hop ID is random on every start and nothing pins it"; + if self.removed_id.is_some() { + found.removed("--cluster-id", Some("MOQ_CLUSTER_ID"), ID); + } + if self.removed_id_toml.is_some() { + found.toml("[cluster] id", "removed", Some(ID)); + } if self.connect.iter().any(|peer| is_legacy_peer(peer.url())) { found.changed( @@ -3765,6 +3778,31 @@ mod tests { dialed.release("never-dialed", DialSource::Mdns, &mut no_dial); } + /// A removed cluster ID still parses, from the flag, the variable, or the file, so + /// the process can say why it stops instead of refusing an unknown setting. + #[test] + fn a_removed_cluster_id_is_reported() { + let flagged = Config { + removed_id: Some("7".into()), + ..Default::default() + }; + let reported = flagged.deprecated().to_string(); + assert!( + reported.contains("--cluster-id / MOQ_CLUSTER_ID -> removed"), + "{reported}" + ); + assert!(new_cluster(flagged).is_err()); + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cluster-id.toml"); + std::fs::write(&path, "[cluster]\nid = 7\n").unwrap(); + let args = vec![std::ffi::OsString::from("moq-relay"), std::ffi::OsString::from(&path)]; + let Err(err) = RelayConfig::parse_and_merge(args) else { + panic!("a removed cluster ID loaded"); + }; + assert!(format!("{err:#}").contains("[cluster] id -> removed"), "{err:#}"); + } + #[test] fn cluster_connect_api_survives_toml_merge() { let toml = "[cluster.api]\nurl = \"https://api.example.com/cluster/connect\"\ntls.root = [\"api.pem\"]\n"; From 70771b3cd64cbef909cd4f150b3a353bfaf117b6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 20:43:35 -0800 Subject: [PATCH 36/40] fix(net): a node that does not proxy keeps lite-06-learned claims to itself Routes learned over an older version enter the route layer as this node's own claims. An own claim whose route crossed another node is now marked learned, and an answer to a peer this node does not proxy for leaves it out and certifies no floor for it, so an application is never transit between a lite-06 and a lite-07 peer. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/publisher/routed.rs | 23 ++++++++---- rs/moq-net/src/lite/subscriber/routed.rs | 1 + rs/moq-net/src/model/origin.rs | 2 ++ rs/moq-net/src/route/node.rs | 5 +++ rs/moq-net/tests/integration/mesh_withdraw.rs | 35 +++++++++++++++++++ 5 files changed, 59 insertions(+), 7 deletions(-) diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index 58dbe852dbc..fe02bec9cb9 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -3,8 +3,9 @@ //! The node sends the peer its routes and, closed by an ANNOUNCE_FLOOR, each route's //! claims under the requested prefix. The stream filters what it puts on the wire: //! hidden claims unless the peer asked for them, and anything this handle's scope, the -//! session's grant, or the upstream horizon leaves out, in which case a floor certifies -//! nothing, since the peer cannot tell what was left out from what is gone. +//! session's grant, the upstream horizon, or the proxy preference leaves out, in which +//! case a floor certifies nothing, since the peer cannot tell what was left out from +//! what is gone. //! //! A handle's mounts beneath the prefix are read through a node stream each, at the //! target, and what they present is named under the mount point on the same routes. @@ -63,6 +64,13 @@ enum Phase { Closing, } +/// Whether a stream on `link` carries `claim` at all: never an upstream-learned claim +/// to an upstream, nor one learned from another node to a peer this node does not +/// proxy for. +fn sends(link: &route::Link, claim: &node::Claim) -> bool { + !(link.upstream && claim.upstream) && (link.proxy || !claim.learned) +} + /// Answers one lite-07 announce stream. pub(super) struct Answer { origin: origin::Consumer, @@ -396,12 +404,13 @@ impl Answer { /// Send the ANNOUNCE_FLOOR closing a batch for the route to `id`. fn floor(&mut self, stream: &mut Stream, id: route::Id) -> Result<(), Error> { let certified = self.certified(); - let upstream = self.link.upstream; + // Like the grant, the upstream horizon and the proxy preference leave claims out + // that a floor would certify absent. + let hides = (self.routes.get(&id).into_iter()) + .flat_map(|sent| sent.claims.values()) + .any(|claim| !sends(&self.link, claim)); let sent = self.routes.get_mut(&id).expect("only a held route has a batch"); sent.floored = true; - // Like the grant, the upstream horizon leaves claims out that a floor would - // certify absent. - let hides = upstream && sent.claims.values().any(|claim| claim.upstream); let revision = sent.floor.filter(|_| certified && !hides); stream.writer.buffer(&Msg::Floor { id: sent.wire, @@ -449,7 +458,7 @@ impl Answer { ) -> Result<(), Error> { let sent = self.routes.get(&id).expect("only a held route syncs"); let wanted = (sent.claims.get(&path)) - .filter(|claim| self.shows(&path) && !(self.link.upstream && claim.upstream)) + .filter(|claim| self.shows(&path) && sends(&self.link, claim)) .cloned(); let wire = sent.wire; let taken = (wanted.as_ref()) diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index e31ac3fd245..31732dc7ddd 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -542,6 +542,7 @@ fn claim(instance: Option, epoch: u64, cost: crate::origin::Cos cost: cost.value(), continues: None, upstream: false, + learned: false, }) } diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index fd702db6140..7c9749d5f10 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -3664,6 +3664,8 @@ impl OwnClaims { cost: route.cost.value(), continues: None, upstream: false, + // Only a route published here has crossed no node. + learned: !route.hops.is_empty(), }; self.best.insert(announce.prefix.clone(), claim); self.feed(announce.prefix, predecessor.as_ref()); diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index c9e1756fe59..66f62a45940 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -45,6 +45,10 @@ pub(crate) struct Claim { /// is never sent to another upstream, so this node never carries traffic between /// two. Always unset on a claim off the wire. pub upstream: bool, + /// This node's own claim, served through a route another node announced over an + /// older version: sent only to a peer this node proxies for, so a node that does + /// not proxy is never transit. Always unset on a claim off the wire. + pub learned: bool, } /// A shown claim as the origin serves it through the selected route. @@ -710,6 +714,7 @@ mod tests { cost, continues: None, upstream: false, + learned: false, } } diff --git a/rs/moq-net/tests/integration/mesh_withdraw.rs b/rs/moq-net/tests/integration/mesh_withdraw.rs index f02e8d392a3..ffdbb0cb14b 100644 --- a/rs/moq-net/tests/integration/mesh_withdraw.rs +++ b/rs/moq-net/tests/integration/mesh_withdraw.rs @@ -716,3 +716,38 @@ async fn an_upstream_learned_claim_never_reaches_another_upstream() { assert_eq!(seen.get("room/local"), Some(&vec![Kind::Start]), "{seen:?}"); assert!(!seen.contains_key("room/learned"), "{seen:?}"); } + +/// A node that does not proxy is never transit, across versions too: a broadcast it +/// learned over moq-lite-06 enters the route layer as one of its own claims, yet never +/// reaches a moq-lite-07 peer, while one published there does. +#[kio::test] +async fn a_node_that_does_not_proxy_keeps_what_it_learned_over_lite06() { + let lite07: Version = "moq-lite-07-wip".parse().unwrap(); + let lite06: Version = "moq-lite-06".parse().unwrap(); + let first = produce_origin(1); + let app = produce_origin(2); + let second = produce_origin(3); + let mut pairs = Vec::new(); + for (version, relay) in [(lite06, &first), (lite07, &second)] { + let mut options = MockConnectOptions::new(version); + options.client_publish = Some(app.consume()); + options.client_subscribe = Some(app.clone()); + options.server_publish = Some(relay.consume().with_hidden(true)); + options.server_subscribe = Some(relay.clone().with_link(origin::Link::Peer)); + options.proxy = false; + pairs.push(connect_mock(options).await); + } + + let mut at_app = watch(app.consume().routes()); + let mut at_second = watch(second.consume().routes()); + let learned = first.route("room/learned", origin::RouteOptions::hidden()).unwrap(); + learned.announce().unwrap(); + let local = app.route("room/local", origin::RouteOptions::hidden()).unwrap(); + local.announce().unwrap(); + kio::clock::sim::idle().await; + let seen_by_app = drain(&mut at_app); + assert!(seen_by_app.contains_key("room/learned"), "{seen_by_app:?}"); + let seen = drain(&mut at_second); + assert_eq!(seen.get("room/local"), Some(&vec![Kind::Start]), "{seen:?}"); + assert!(!seen.contains_key("room/learned"), "{seen:?}"); +} From b02bdff9f86596376b9c85b379f9bf7dd84895f5 Mon Sep 17 00:00:00 2001 From: quest fixer Date: Sat, 7 Nov 2026 01:02:28 -0800 Subject: [PATCH 37/40] fix(net): initialize routing node in base setup tests --- rs/moq-net/src/lite/subscriber.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 59ab1d3fc95..5a7b7041282 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -6462,6 +6462,7 @@ mod tests { subscriber, path: Path::from_static("room/host"), instance: None, + node: 0, name: "audio".to_string(), }; @@ -6559,6 +6560,7 @@ mod tests { subscriber, path: Path::from_static("room/host"), instance: None, + node: 0, name: "audio".to_string(), }; @@ -6621,6 +6623,7 @@ mod tests { subscriber, path: Path::from_static("room/host"), instance: None, + node: 0, name: "audio".to_string(), }; From b327b8eaa432c83dce01eb3f2aabc40366f67c5e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 07:24:46 -0800 Subject: [PATCH 38/40] test(interop): skip the browser route key lane until JS speaks the routed lite-07 The lane main added in #8116 dials the relay from a browser on moq-lite-07-wip, which js/net still speaks from before the route layer. It records a skip naming routes/js.md, like the other JS lite-07 lanes. Co-Authored-By: Claude Opus 5.5 --- quest/m2/cluster-routing/routes/js.md | 4 ++-- test/interop/README.md | 9 ++++++--- test/interop/interop.sh | 5 +++++ 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/quest/m2/cluster-routing/routes/js.md b/quest/m2/cluster-routing/routes/js.md index eb052a55830..2c5126b740e 100644 --- a/quest/m2/cluster-routing/routes/js.md +++ b/quest/m2/cluster-routing/routes/js.md @@ -25,8 +25,8 @@ the cheapest one; fix it with the route-selection rework here. so the nightly report stays meaningful. Rejected: leave them red; hold #8023 for this quest. Every skip names this quest (`grep -rn routes/js.md test rs`): `JS_LITE07_SKIP` in `test/interop/interop.sh` and `socket.sh` - (native JS grant printing, claims, track replacement, sockets, the lite-07 - compress and drop legs, the compressed catalog's lite-07 reader, and + (native JS grant printing, claims, track replacement, sockets, the browser + route key proof, the lite-07 compress and drop legs, the compressed catalog's lite-07 reader, and `--versions moq-lite-07-wip` with JS clients), `lite_tracks_interop` in `test/justfile`, and the lite-07 legs of `max_age_relay_javascript` and `relay_fill_javascript`. diff --git a/test/interop/README.md b/test/interop/README.md index 0382fe8437f..e91c5817136 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -23,8 +23,9 @@ request ends while its follower waits for the publisher to arrive. Until [`js/net` speaks lite-07's route layer](../../quest/m2/cluster-routing/routes/js.md), native JS dials its defaults in the matrix, leaving its grant unchecked, and every lane that would run it on lite-07 records a skip naming that quest: claims, track -replacement, sockets, and the lite-07 compress and drop legs. The compressed catalog -runs without its lite-07 reader, and `--versions moq-lite-07-wip` refuses JS clients. +replacement, sockets, the browser route key proof, and the lite-07 compress and +drop legs. The compressed catalog runs without its lite-07 reader, and +`--versions moq-lite-07-wip` refuses JS clients. Every publisher also sends an Opus tone, one sine per channel, and every subscriber decodes it. Python and C++ send a mono 440 Hz tone; the rest send 440 Hz left and @@ -243,7 +244,9 @@ a varint rather than the draft's 64 bits, so `@moq/net` sends no proof from Firefox, and the tab must connect unproven. A build that cannot prove yet skips, or fails under `CI_STRICT`, which release PRs set, so the lite-07 cut waits for both browsers to prove. A build the table says cannot prove but that does fails, -so the table follows the browsers. +so the table follows the browsers. Until +[routes/js.md](../../quest/m2/cluster-routing/routes/js.md) ports `js/net` to the +route layer, the lane records a skip naming it. ## moq-transport diff --git a/test/interop/interop.sh b/test/interop/interop.sh index 62d775e2d5f..520f3541633 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -2368,6 +2368,11 @@ run_matrix() { local engine status for engine in "${ROUTE_KEY_ENGINES[@]}"; do log="$HARNESS_RUN/route-key-$engine.log" + if [[ -n "$JS_LITE07_SKIP" ]]; then + echo " SKIP route key $engine" + record skip case="route key $engine" subscriber=js version=moq-lite-07-wip detail="$JS_LITE07_SKIP" + continue + fi if is_broken js; then echo " FAIL route key $engine (browser client unavailable)" record fail case="route key $engine" subscriber=js version=moq-lite-07-wip detail="browser client unavailable" From 0b70a604ff3f0ee1e5f14940e4ead01decc15bba Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 11:31:45 -0800 Subject: [PATCH 39/40] fix(net): bound the lite-07 ROUTE_REQUEST backlog by identity A downstream peer raising its ROUTE_REQUEST seqno for a route through a stream whose peer leaves its replies unread queued one request per raise, and the session buffered every reply into the writer while its flush was pending, so relay memory grew without bound. The node now keeps each stream's requests per identity, the highest seqno subsuming lower ones, and the session takes them, and its ROUTE_END echoes, only once the writer has flushed. A stalled writer holds one request per identity the peer advertised, which the announce cap bounds. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/publisher/routed.rs | 7 +- rs/moq-net/src/lite/subscriber/routed.rs | 96 ++++++++++++++++++++---- rs/moq-net/src/route/node.rs | 25 +++++- 3 files changed, 107 insertions(+), 21 deletions(-) diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index fe02bec9cb9..6ed94319d1e 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -273,11 +273,8 @@ impl Answer { match output { node::Output::Route { id, reach } => self.route(stream, source, id, reach), node::Output::Batch { id, changes, floor } => self.batch(stream, source, id, changes, floor), - // Only a stream this node asked for requests, serves, or echoes. - node::Output::Request { .. } - | node::Output::Echo { .. } - | node::Output::Serve { .. } - | node::Output::Unserve { .. } => Ok(()), + // Only a stream this node asked for serves or echoes. + node::Output::Echo { .. } | node::Output::Serve { .. } | node::Output::Unserve { .. } => Ok(()), } } diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 31732dc7ddd..76f8d20358b 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -85,13 +85,13 @@ pub(super) struct Routed { /// The slots of routes the peer ended, kept until their ROUTE_END echo is flushed, /// so a peer leaving its replies unread holds its routes' slots. echoing: Vec, + /// The ROUTE_END echoes not yet written, each holding a slot in `echoing`. + ends: Vec, served: HashMap<(route::Id, Path), ServedClaim>, /// What the stream served before it filed again, while the new one takes it up. previous: HashMap<(route::Id, Path), ServedClaim>, /// Served claims whose request queue woke since the last pass. ready: kio::Queue<(route::Id, Path)>, - /// Messages for the peer, written by the stream's task. - replies: Vec, /// The peer sent a GOAWAY: every claim served here ranks last. drained: bool, } @@ -142,10 +142,10 @@ impl Routed { slots: subscriber.announces.clone(), held: Vec::new(), echoing: Vec::new(), + ends: Vec::new(), served: HashMap::new(), previous: HashMap::new(), ready: Default::default(), - replies: Vec::new(), drained: false, }) } @@ -190,7 +190,7 @@ impl Routed { } self.echoing.extend(self.held.pop()); self.stream.route(route.node, None); - self.replies.push(AnnounceReply::RouteEnd { id }); + self.ends.push(id); } Msg::Floor { id, revision } => { let certifies = self.certifies(); @@ -404,7 +404,7 @@ impl Routed { } /// Act on everything the node queued for the stream: serve the claims it hands - /// over, and queue the replies it asks for. + /// over. pub(super) fn outputs(&mut self, subscriber: &Subscriber) { while let Some(output) = self.stream.try_output() { self.output(subscriber, output); @@ -413,7 +413,6 @@ impl Routed { fn output(&mut self, subscriber: &Subscriber, output: node::Output) { match output { - node::Output::Request { id, seqno } => self.replies.push(AnnounceReply::RouteRequest { node: id, seqno }), node::Output::Serve { id, named, @@ -511,13 +510,29 @@ impl Routed { while let Poll::Ready(output) = self.stream.poll_output(waiter) { self.output(subscriber, output); } - for reply in self.replies.drain(..) { - writer.buffer(&reply)?; - } - match writer.poll_flush(waiter) { - Poll::Ready(Ok(())) => self.echoing.clear(), - Poll::Ready(Err(err)) => return Err(err), - Poll::Pending => {} + // Replies are buffered only once the last ones flushed, so a peer leaving them + // unread holds its echoes' slots and one request per identity, nothing more. + loop { + match writer.poll_flush(waiter) { + Poll::Ready(Ok(())) => {} + Poll::Ready(Err(err)) => return Err(err), + Poll::Pending => break, + } + // The echoes buffered before went out: keep a slot per echo still queued. + self.echoing.truncate(self.ends.len()); + let requests = match self.stream.poll_requests(waiter) { + Poll::Ready(requests) => requests, + Poll::Pending => Default::default(), + }; + if self.ends.is_empty() && requests.is_empty() { + break; + } + for id in self.ends.drain(..) { + writer.buffer(&AnnounceReply::RouteEnd { id })?; + } + for (node, seqno) in requests { + writer.buffer(&AnnounceReply::RouteRequest { node, seqno })?; + } } while let Poll::Ready(Ok(key)) = self.ready.poll_pop(waiter) { let Some(entry) = self.served.get_mut(&key) else { @@ -741,6 +756,61 @@ mod tests { assert_eq!(subscriber.announces.available(), 2); } + /// A downstream peer raising its ROUTE_REQUEST seqno while this stream's peer + /// leaves its replies unread holds one request for the identity, sent once the + /// writer flushes and naming the highest seqno. + #[kio::test] + async fn a_stalled_writer_holds_one_request_per_identity() { + use crate::coding::Decode; + + let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + let prefix = Path::empty(); + let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let mut decoder = AnnounceDecoder::default(); + let budget = kio::Producer::new(0); + let log = crate::lite::test_transport::Log::default(); + let mut writer = crate::coding::Writer::new( + crate::lite::test_transport::SinkSend::stepped(log.clone(), budget.consume()), + crate::lite::Version::Lite07, + ); + let waiter = kio::Waiter::noop(); + let route = Msg::RouteStart { + node: 7, + advert: ADVERT, + }; + routed.receive(&prefix, route, &mut decoder).unwrap(); + let downstream = (subscriber.origin.route_node()) + .answer( + crate::route::Link { + proxy: true, + ..Default::default() + }, + Path::empty(), + ) + .unwrap(); + + for seqno in 1..=1000 { + downstream.request(7, seqno); + routed.poll(&subscriber, &mut writer, &waiter).unwrap(); + } + + *budget.write().ok().expect("the budget is open") = usize::MAX; + routed.poll(&subscriber, &mut writer, &waiter).unwrap(); + let writes = log.writes.lock().unwrap().clone(); + let mut buf = writes.as_slice(); + let mut requests = Vec::new(); + while !buf.is_empty() { + let (reply, size) = crate::lite::AnnounceReply::decode_slice(buf, crate::lite::Version::Lite07).unwrap(); + buf = &buf[size..]; + requests.push(reply); + } + // The first went out before the writer stalled; the rest merged behind it. + assert_eq!( + requests, + [1, 1000].map(|seqno| crate::lite::AnnounceReply::RouteRequest { node: 7, seqno }) + ); + } + /// What the stream keeps between floors is the net change to its committed claims, /// however many messages the peer sends without one. #[kio::test] diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 66f62a45940..5cb25080859 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -74,8 +74,6 @@ pub(crate) enum Output { changes: Vec<(Path, Option)>, floor: Option, }, - /// Send ROUTE_REQUEST, naming the identity as the peer advertised it. - Request { id: Id, seqno: Seqno }, /// Echo the peer's ROUTE_END for the identity it named `id`. Echo { id: Id }, /// Serve `claim` of `id` into the origin through this stream, or update how it @@ -129,6 +127,9 @@ struct Outbox { serving: HashMap<(Id, Path), u64>, /// Each route the reader holds, and the claims it was sent under it. delivered: HashMap>, + /// ROUTE_REQUESTs to send, by the identity as the peer advertised it. A higher + /// seqno subsumes a lower one, so a writer that stalls holds one per identity. + requests: BTreeMap, } /// One identity's route changes and claim batches since the reader last took it. @@ -485,6 +486,17 @@ impl Stream { self.queue.lock().pop() } + /// The ROUTE_REQUESTs queued since the last call, each identity's highest seqno. + /// Take them only once the writer has room, so they keep coalescing meanwhile. + pub fn poll_requests(&self, waiter: &kio::Waiter) -> Poll> { + let ready = |outbox: &kio::Ref<'_, Outbox>| match outbox.requests.is_empty() { + true => Poll::Pending, + false => Poll::Ready(()), + }; + let mut outbox = ready!(self.queue.poll(waiter, ready)); + Poll::Ready(std::mem::take(&mut outbox.requests)) + } + fn with(&self, f: impl FnOnce(&mut Router, Key) -> R) -> R { let mut state = self.node.lock(); let out = f(&mut state.router, self.key); @@ -570,7 +582,14 @@ impl State { let (key, output) = match out { Out::Route { to, id, reach } => (to, Output::Route { id, reach }), Out::Batch { to, id, changes, floor } => (to, Output::Batch { id, changes, floor }), - Out::Request { to, id, seqno } => (to, Output::Request { id, seqno }), + Out::Request { to, id, seqno } => { + if let Some(queue) = self.queues.get(&to) { + let mut outbox = queue.lock(); + let want = outbox.requests.entry(id).or_insert(seqno); + *want = (*want).max(seqno); + } + continue; + } Out::Echo { to, id } => (to, Output::Echo { id }), }; if let Some(id) = push(&self.queues, key, output) { From 2a7e718e033d119b7693891c6878f2cbf60cad85 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 11:46:17 -0800 Subject: [PATCH 40/40] fix(net): a lite-07 answer gives route outputs a turn beside replies A peer sending ROUTE_REQUEST without pause kept the answer loop reading replies, so it never served the node's outputs and never yielded the task. Each pass now takes one reply and one output, and the loop yields on a cooperative budget like the publisher's other loops. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/publisher.rs | 66 +++++++++++++++++++++++++ rs/moq-net/src/lite/publisher/routed.rs | 14 ++++-- 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 5ef3e6c7b78..b4f12a640dc 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -3671,6 +3671,72 @@ mod announce_test { assert_eq!(wire.take_announces(), [lite::AnnounceBroadcast::Synced]); } + /// A lite-07 peer that never stops sending ROUTE_REQUEST neither holds the task + /// nor keeps a new claim from being announced. + #[kio::test] + async fn a_lite07_reply_flood_yields_and_announces() { + /// A peer whose replies never run out. + struct Flood(bytes::Bytes); + impl crate::transport::poll::Chunks for Flood { + type Error = SinkError; + fn poll_read_chunk( + &mut self, + _: &kio::Waiter, + _: usize, + ) -> std::task::Poll, SinkError>> { + std::task::Poll::Ready(Ok(Some(self.0.clone()))) + } + fn stop(&mut self, _: u32) {} + } + + let origin = Hop::new(1).unwrap().produce(); + let consumer = origin.consume(); + let mut answer = + super::routed::Answer::new(consumer, crate::Path::empty(), true, crate::auth::Link::new(false)); + let request = lite::AnnounceReply::RouteRequest { node: 7, seqno: 1 }; + let flood = Flood(crate::coding::Encode::encode_bytes(&request, Version::Lite07).unwrap()); + let log = Log::default(); + let mut wire = Wire { + writes: log.writes.clone(), + cursor: 0, + version: Version::Lite07, + }; + let mut stream = Stream:: { + writer: Writer::new( + crate::transport::boxed::SendStream::new(SinkSend::new(log)), + Version::Lite07, + ), + reader: Reader::new( + crate::transport::boxed::RecvStream::new(crate::transport::poll::Buffered::new(flood)), + Version::Lite07, + ), + }; + let waiter = kio::Waiter::noop(); + assert!( + answer.poll(&mut stream, false, &waiter).is_pending(), + "the flood yields" + ); + wire.take_ok(); + let announces = wire.take_announces(); + assert_eq!( + announces.last(), + Some(&lite::AnnounceBroadcast::Synced), + "{announces:?}" + ); + + let _route = origin.announce("x", origin::RouteInfo::default()).unwrap(); + // The origin feeds its node from a task. + kio::clock::sim::yield_now().await; + assert!(answer.poll(&mut stream, false, &waiter).is_pending()); + let announces = wire.take_announces(); + assert!( + announces + .iter() + .any(|msg| matches!(msg, lite::AnnounceBroadcast::Active { .. })), + "{announces:?}" + ); + } + #[test] fn synced_survives_a_removed_initial_route_and_live_churn() { let origin = Hop::new(1).unwrap().produce(); diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index 6ed94319d1e..fb4310ec671 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -96,6 +96,8 @@ pub(super) struct Answer { epoch: u64, permit: crate::auth::Permit, phase: Phase, + /// Passes before the stream yields, so a peer flooding replies cannot hold the task. + budget: kio::coop::Budget, } impl Answer { @@ -126,6 +128,7 @@ impl Answer { epoch: 0, permit: Default::default(), phase: Phase::Init, + budget: kio::coop::Budget::new(32), } } @@ -180,6 +183,7 @@ impl Answer { } loop { + ready!(self.budget.poll_yield(waiter)); // Deliver what is buffered before taking more work. ready!(stream.writer.poll_flush(waiter))?; @@ -214,10 +218,12 @@ impl Answer { continue; } + // One reply and one output per pass, so neither direction starves the other. + let mut progress = false; match stream.reader.poll_decode_maybe::(waiter) { Poll::Ready(Ok(Some(reply))) => { self.reply(reply)?; - continue; + progress = true; } // The peer closed its side: it wants nothing more. Poll::Ready(Ok(None)) => return Poll::Ready(Ok(())), @@ -235,9 +241,11 @@ impl Answer { if let Some((index, output)) = ready { self.turn = index + 1; self.output(stream, index, output)?; - continue; + progress = true; + } + if !progress { + return Poll::Pending; } - return Poll::Pending; } }