From 8e50f5f10cb0b35ceb4aff0b91b3a7545c7f0796 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 13:38:12 -0800 Subject: [PATCH 01/25] chore(quest): claim quest/m2/cluster-routes/acceptance.md Quest-Claim: 791427cae5fd1709 From 3c7c7ca5e89c2bff53b97071434d84b14ee973cc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 14:02:24 -0800 Subject: [PATCH 02/25] fix(moq-net): lite-07 routes skip the lite-06 churn mitigations, and a let-go claim leaves every copy at once The update hold and the retraction quarantine now hold only routes learned over a hop chain. A claim the route node lets go of, ended or lost, retires every copy the origin holds under one lock, so a withdrawal no longer steps through backup copies. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/subscriber/routed.rs | 26 +++-- rs/moq-net/src/model/follow.rs | 15 ++- rs/moq-net/src/model/origin.rs | 103 ++++++++++++++++-- rs/moq-net/src/route/node.rs | 40 +++++-- rs/moq-net/src/route/node/tests.rs | 28 +++-- rs/moq-net/tests/integration/mesh_withdraw.rs | 61 ++++++----- rs/moq-net/tests/integration/path_routes.rs | 10 +- .../tests/integration/route_acceptance.rs | 86 +++++++++++---- rs/moq-net/tests/integration/route_cluster.rs | 12 +- .../tests/integration/route_failover.rs | 4 +- test/interop/clients/js/media.ts | 5 +- 11 files changed, 282 insertions(+), 108 deletions(-) diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index e8a04f18fd8..df4d48d2071 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -137,11 +137,18 @@ type ServedKey = (NodeId, Path, Instance); struct Left(crate::origin::Claim); impl node::Orphan for Left { - fn release(self: Box, ended: bool) { - // Dropped, a claim the node let go of goes offline rather than ending. - if ended { - self.0.withdrawn(); - } + fn release(self: Box, gone: node::Gone) { + release(self.0, gone); + } +} + +/// Let go of a copy of a claim the origin holds, for the reason `gone` gives. +fn release(claim: crate::origin::Claim, gone: node::Gone) { + match gone { + // Dropped, it goes offline, while another copy serves. + node::Gone::Moved => drop(claim), + node::Gone::Lost => claim.retire(false), + node::Gone::Ended => claim.retire(true), } } @@ -767,13 +774,10 @@ impl Routed { origin, path, instance, - ended, + gone, } => { - // Dropped, a claim the node let go of goes offline rather than ending. - if let Some(served) = self.served.remove(&(origin, path, instance)) - && ended - { - served.route.dynamic.withdrawn(); + if let Some(served) = self.served.remove(&(origin, path, instance)) { + release(served.route.dynamic, gone); } } } diff --git a/rs/moq-net/src/model/follow.rs b/rs/moq-net/src/model/follow.rs index 066f62d6d2b..4b94e4b49d9 100644 --- a/rs/moq-net/src/model/follow.rs +++ b/rs/moq-net/src/model/follow.rs @@ -458,7 +458,7 @@ mod tests { origin::{Cost, RouteInfo, RouteOptions}, }; - /// The follower's next event, which must arrive within the origin's update hold. + /// The follower's next event, which must arrive within a second. async fn followed(follow: &mut Follow) -> (&'static str, String) { let event = kio::clock::timeout(Duration::from_secs(1), follow.next()) .await @@ -488,7 +488,7 @@ mod tests { } } - /// The follower stays quiet past the origin's update hold. + /// The follower stays quiet for a second. async fn quiet(follow: &mut Follow) { if let Ok(event) = kio::clock::timeout(Duration::from_secs(1), follow.next()).await { panic!("expected nothing, got {event:?}"); @@ -648,7 +648,7 @@ mod tests { ); } - /// The next change handed out, which must arrive within the origin's update hold. + /// The next change handed out, which must arrive within a second. async fn changed(announced: &mut Following) -> Option { kio::clock::timeout(Duration::from_secs(1), announced.available()) .await @@ -656,7 +656,7 @@ mod tests { .expect("the wait failed") } - /// The next instance handed out, which must arrive within the origin's update hold. + /// The next instance handed out, which must arrive within a second. async fn available(announced: &mut Following) -> broadcast::Consumer { changed(announced).await.expect("an end, not an instance") } @@ -668,10 +668,9 @@ mod tests { } } - /// Nothing more is handed out past the origin's update hold. + /// Nothing more is handed out for a second. async fn none(announced: &mut Following) { - let hold = super::super::origin_impl::lite06_churn::DEFAULT_UPDATE_HOLD * 2; - if let Ok(result) = kio::clock::timeout(hold, announced.available()).await { + if let Ok(result) = kio::clock::timeout(Duration::from_secs(1), announced.available()).await { panic!( "expected nothing, got {:?}", result.map(|broadcast| broadcast.map(|broadcast| broadcast.info().instance.clone())) @@ -746,7 +745,7 @@ mod tests { let (_arrival, _publish) = futures::join!(available(&mut announced), async { let _unanswered = requested(&dynamic).await; drop(dynamic); - kio::clock::sleep(super::super::origin_impl::lite06_churn::DEFAULT_UPDATE_HOLD * 2).await; + kio::clock::sleep(Duration::from_secs(1)).await; origin.route("live", RouteOptions::default()).unwrap() }); none(&mut announced).await; diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 8959392598f..01d2e5a83f6 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -134,8 +134,9 @@ pub struct Config { /// leaving each track's own window in force. pub cache_duration: Option, - /// How long an announce cursor holds a changed route before delivering it: a - /// [`lite06_churn`] mitigation. + /// How long an announce cursor holds a changed route that arrived over a hop chain + /// before delivering it: a [`lite06_churn`] mitigation. A moq-lite-07 route and one + /// published here never wait. /// /// A new route, a removed one, and another instance are delivered at once; only an /// update waits, and a newer change during the wait replaces it. When a publisher @@ -157,7 +158,9 @@ pub struct Config { /// through a lite-06 mesh's hop-chain paths, which go with lite-06. They are the update /// hold and the retraction quarantine ([`Config::update_hold`]) and the same-clock dedup /// of a retraction's queue entry ([`Routes::quarantine`]). The cursor every version -/// shares applies them; each site names this module, so deleting it finds them all. +/// shares applies them, to the routes [`hunts`](lite06_churn::hunts) picks out alone, so +/// a reader of a lite-06 mesh keeps them whatever version it speaks; each site names this +/// module, so deleting it finds them all. pub(super) mod lite06_churn { use std::time::Duration; @@ -168,6 +171,14 @@ pub(super) mod lite06_churn { /// [`Config::update_hold`](super::Config::update_hold). Path hunting after a /// withdrawal settles within it on random meshes with links of up to 250 ms. pub(super) const RETRACT_QUARANTINE: Duration = Duration::from_secs(2); + + /// Whether a route arrived over a hop chain (moq-lite-03 to 06, or moq-transport), + /// whose withdrawals hunt through stale paths: the only routes these mitigations + /// hold. A moq-lite-07 route names the node claiming it, whose withdrawal is final, + /// and one published here crossed no hop. + pub(super) fn hunts(hops: &super::Hops, node: Option) -> bool { + node.is_none() && !hops.is_empty() + } } use lite06_churn::{DEFAULT_UPDATE_HOLD, RETRACT_QUARANTINE}; @@ -1013,7 +1024,8 @@ impl Quarantine { /// first hop, instance and epoch, ranked strictly worse. A republish ranks the same /// and a restart carries a new instance or epoch, so neither waits. fn holds(&self, meta: &RouteMeta) -> bool { - self.publisher.is_some_and(|hop| meta.0.iter().next() == Some(&hop)) + lite06_churn::hunts(&meta.0, meta.6) + && self.publisher.is_some_and(|hop| meta.0.iter().next() == Some(&hop)) && meta.3 == self.instance.0 && meta.4 == self.instance.1 && (meta.1, meta.0.len()) > self.rank @@ -1298,7 +1310,9 @@ impl OriginConsumerState { } let pending = self.pending.get_mut(prefix).unwrap(); let wake = match pending.update.as_ref().unwrap() { - PendingUpdate::Announce(_) if self.delivered.contains(prefix) => { + PendingUpdate::Announce((meta, _)) + if self.delivered.contains(prefix) && lite06_churn::hunts(&meta.0, meta.6) => + { // The clock reads the driver's last advance, which may be stale: anchor // the hold on the first advance after the update arrived, by waking the // driver. @@ -1456,6 +1470,9 @@ struct RouteEntry { /// The route was derived from that peer's advertisement, so it serves nobody /// until the peer announces again. See [`Claim::withdrawn`]. stale: bool, + /// The route layer let go of the claim this entry follows through another of its + /// copies, whose own retraction is on its way. See [`Claim::retire`]. + retired: bool, /// [`prefix_claim`] of [`Self::prefix`], built once at announce time. /// /// The announce sync evaluates a route's claim once per (cursor, route) pair, @@ -1489,7 +1506,7 @@ impl RouteEntry { /// Whether cursors see the entry and requests resolve through it. fn live(&self) -> bool { - self.advertised && !self.stale + self.advertised && !self.stale && !self.retired } fn is_anonymous(&self) -> bool { @@ -3077,6 +3094,7 @@ impl Announcing { source: serving.source.clone(), advertised: serving.advertised, stale, + retired: false, claim: claim.clone(), reach: reach(&self.scope, claim), lends: serving.lends, @@ -3568,6 +3586,15 @@ impl Claim { self.state.announcement.retract(Retract::Withdrawn); } + /// The route layer let go of the claim this follows: every copy of it the table holds, + /// each of the same node, instance and epoch, goes with this one under one lock, rather + /// than readers seeing the rest in turn as their sessions retract them. They end with + /// `ended`, else go offline. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn retire(self, ended: bool) { + self.state.announcement.retract(Retract::Retired { ended }); + } + /// Retire a route its announcer ended without a hop chain to stale, such as a /// moq-transport PUBLISH_NAMESPACE_DONE: readers see it end, not go offline. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] @@ -3596,6 +3623,9 @@ enum Retract { Ended, /// A peer withdrew it, which also stales the routes derived through that peer. Withdrawn, + /// The route layer let go of the claim it follows: see [`Claim::retire`]. Ended, it + /// stales as [`Self::Withdrawn`] does. + Retired { ended: bool }, } /// The write half of an advertisement: a live claim that paths under a @@ -3816,11 +3846,14 @@ impl AnnounceProducer { continue; }; servers.extend(entry.server.clone()); + if let Retract::Retired { .. } = how { + shared.retire_copies(&entry); + } // A peer remains reachable while any of its sessions advertises this // prefix. Remove this session's route before testing the remaining // claims, under the same lock, so overlapping withdrawals cannot // invalidate a newer advertisement or miss the last withdrawal. - if how == Retract::Withdrawn + if matches!(how, Retract::Withdrawn | Retract::Retired { ended: true }) && let Some(&peer) = entry.hops.iter().last() && peer != Hop::UNKNOWN && !shared @@ -3832,8 +3865,8 @@ impl AnnounceProducer { shared.restale(prefix); } let gone = match how { - Retract::Lost => Gone::Offline, - Retract::Ended | Retract::Withdrawn => Gone::End, + Retract::Lost | Retract::Retired { ended: false } => Gone::Offline, + Retract::Ended | Retract::Withdrawn | Retract::Retired { ended: true } => Gone::End, }; shared.sync_route(&entry.prefix, &entry.claim, gone); if entry.advertised @@ -6479,6 +6512,19 @@ impl OriginState { .is_some_and(|peers| hops.iter().any(|hop| peers.contains(hop))) } + /// Retire every other copy at its prefix of the claim `entry` follows: the same + /// forwarding node, instance and epoch. See [`Claim::retire`]. + fn retire_copies(&mut self, entry: &RouteEntry) { + let Some(node) = entry.node else { + return; + }; + for other in self.routes.at_mut(&entry.prefix) { + if other.node == Some(node) && other.instance == entry.instance && other.epoch == entry.epoch { + other.retired = true; + } + } + } + /// Recompute which entries at `prefix` are stale after its withdrawals changed. fn restale(&mut self, prefix: &Path) { let peers = self.withdrawn.get(prefix); @@ -8581,7 +8627,7 @@ impl Routes { else { return; }; - if self.update_hold.is_zero() { + if self.update_hold.is_zero() || !lite06_churn::hunts(&update.route.hops, update.route.node) { return; } // An anonymous route names no publisher, so its quarantine holds nothing. @@ -13225,6 +13271,43 @@ mod tests { assert!(producer.shared.lock().withdrawn.is_empty()); } + /// The route layer letting go of one copy of a lite-07 claim takes every copy of the + /// same node, instance and epoch with it: a cursor sees the claim end or go offline, + /// never a move onto another copy first. Another node's claim of the instance stays. + #[kio::test] + async fn retiring_a_claim_retires_every_copy() { + for ended in [true, false] { + let producer = origin(1).produce(); + let peer = producer.clone().with_link(Link::Peer); + let mut announced = producer.consume().routes(); + let copy = |node: u64, cost: u64| { + RouteInfo::new() + .with_instance(instance(), 0) + .with_hops(hops(&[node])) + .with_cost(cost) + .with_node(Some(node_id(node))) + }; + let near = peer.forward("room", copy(7, 1)).unwrap(); + let _far = peer.forward("room", copy(7, 2)).unwrap(); + announced.expect_next_active("room").await; + announced.assert_next_wait(); + near.retire(ended); + match ended { + true => announced.expect_next_ended("room").await, + false => announced.expect_next_offline("room").await, + } + announced.assert_next_wait(); + + let near = peer.forward("room", copy(7, 1)).unwrap(); + let _other = peer.forward("room", copy(8, 3)).unwrap(); + announced.expect_next_active("room").await; + announced.assert_next_wait(); + near.retire(ended); + let other = announced.expect_next_active("room").await; + assert_eq!(other.node, Some(node_id(8)), "ended: {ended}"); + } + } + /// Overlapping sessions carrying one instance are independent claims, regardless of /// announcement order, and falling back from one to the other is the same stream. #[kio::test] diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index e8786976456..ffbb91bf679 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -115,21 +115,32 @@ pub(crate) enum Output { instance: Instance, served: Served, }, - /// Stop serving it. `ended` when its publisher withdrew it, as far as this node - /// knows; otherwise the node let go of it here. + /// Stop serving it, for the reason `gone` gives. Unserve { origin: NodeId, path: Path, instance: Instance, - ended: bool, + gone: Gone, }, } +/// Why a session stops serving a claim. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Gone { + /// Another session serves it now. + Moved, + /// The node let go of it here: every copy the origin holds goes, offline. + Lost, + /// Its publisher withdrew it, as far as this node knows: every copy the origin holds + /// ends. + Ended, +} + /// A served claim's place in the origin, which outlives the session that served it until /// another session serves the claim or the reader lets it go. pub(crate) trait Orphan: Send { - /// Let go of it: `ended` when the claim's publisher withdrew it. - fn release(self: Box, ended: bool); + /// Let go of it, for the reason `gone` gives. + fn release(self: Box, gone: Gone); } /// Where a request a peer sent goes; see [`Node::poll_forward`]. @@ -191,7 +202,7 @@ struct State { /// The broadcasts each session has a slot for. serving: HashMap>, /// Orphans to release once the lock is. - released: Vec<(Box, bool)>, + released: Vec<(Box, Gone)>, /// Whether each of the origin's own claims is filed as having crossed a marked link. own: BTreeMap, } @@ -247,8 +258,8 @@ impl Drop for Locked<'_> { let released = std::mem::take(&mut state.released); drop(state); // An orphan's release reaches into the origin, which may be feeding this node. - for (orphan, ended) in released { - orphan.release(ended); + for (orphan, gone) in released { + orphan.release(gone); } } } @@ -613,7 +624,7 @@ impl Stream { let key = (origin, self.key); match (state.lanes.get_mut(broadcast)).and_then(|slots| slots.get_mut(&key)) { Some(slot) => slot.orphan = Some(orphan), - None => state.released.push((orphan, false)), + None => state.released.push((orphan, Gone::Moved)), } } } @@ -828,6 +839,13 @@ impl State { if shown && !served { return; } + // Unshown, every slot leaves at once, so each tells its copy all of them go: the + // origin then drops them together rather than showing the rest in turn. + let gone = match (shown, ended) { + (_, true) => Gone::Ended, + (false, false) => Gone::Lost, + (true, false) => Gone::Moved, + }; let leaving: Vec = (slots.iter()) .filter(|(_, slot)| slot.leaving) .map(|(key, _)| *key) @@ -843,7 +861,7 @@ impl State { } } match slot.orphan { - Some(orphan) => self.released.push((orphan, ended)), + Some(orphan) => self.released.push((orphan, gone)), None => { let (path, instance) = broadcast.clone(); outputs.push(( @@ -852,7 +870,7 @@ impl State { origin, path, instance, - ended, + gone, }, )); } diff --git a/rs/moq-net/src/route/node/tests.rs b/rs/moq-net/src/route/node/tests.rs index bf09c543c49..6325d51ffe8 100644 --- a/rs/moq-net/src/route/node/tests.rs +++ b/rs/moq-net/src/route/node/tests.rs @@ -113,13 +113,20 @@ struct Left(Arc); impl Left { const HELD: u8 = 0; - const RELEASED: u8 = 1; - const ENDED: u8 = 2; + /// Another session serves the claim. + const MOVED: u8 = 1; + /// The node let go of the claim, and every copy with it. + const LOST: u8 = 2; + const ENDED: u8 = 3; } impl Orphan for Left { - fn release(self: Box, ended: bool) { - let how = if ended { Self::ENDED } else { Self::RELEASED }; + fn release(self: Box, gone: Gone) { + let how = match gone { + Gone::Moved => Self::MOVED, + Gone::Lost => Self::LOST, + Gone::Ended => Self::ENDED, + }; self.0.store(how, Ordering::SeqCst); } } @@ -211,7 +218,7 @@ fn only_the_closest_newest_copy_is_served() { assert_eq!(confirm(&second), 1); let serves = serves(&first); assert!( - matches!(serves.as_slice(), [Output::Unserve { ended: false, .. }]), + matches!(serves.as_slice(), [Output::Unserve { gone: Gone::Moved, .. }]), "{serves:?}" ); } @@ -376,7 +383,8 @@ fn an_unavailable_routed_claim_takes_no_request() { } /// A session that closes leaves what it served as an orphan, which the node holds while -/// the reader still shows the claim and releases, not ended, once the debounce runs out. +/// the reader still shows the claim and releases as lost, every copy with it, once the +/// debounce runs out. #[test] fn an_orphan_is_released_when_the_reader_lets_go() { let (node, clock) = node(); @@ -394,7 +402,7 @@ fn an_orphan_is_released_when_the_reader_lets_go() { ); expire(&node, &clock); - assert_eq!(left.load(Ordering::SeqCst), Left::RELEASED); + assert_eq!(left.load(Ordering::SeqCst), Left::LOST); } /// A session that returns before the debounce runs out takes the claim over: the orphan @@ -414,7 +422,7 @@ fn an_orphan_is_released_once_another_session_serves_the_claim() { announce(&second, &room, live(0)); assert_eq!(left.load(Ordering::SeqCst), Left::HELD, "nothing serves the claim yet"); assert_eq!(confirm(&second), 1); - assert_eq!(left.load(Ordering::SeqCst), Left::RELEASED); + assert_eq!(left.load(Ordering::SeqCst), Left::MOVED); } /// The origin's own copy of a claim never waits to be served, so a session refusing the @@ -435,7 +443,7 @@ fn an_orphan_is_released_once_every_session_refuses_the_claim() { announce(&second, &room, live(0)); assert_eq!(left.load(Ordering::SeqCst), Left::HELD); second.serving(node.me(), &room, false); - assert_eq!(left.load(Ordering::SeqCst), Left::RELEASED); + assert_eq!(left.load(Ordering::SeqCst), Left::MOVED); } /// A claim its publisher withdraws stops being served as ended, so the origin withdraws @@ -450,7 +458,7 @@ fn a_withdrawn_claim_is_unserved_as_ended() { announce(&publisher, &room, Change::Ended); let serves = serves(&publisher); assert!( - matches!(serves.as_slice(), [Output::Unserve { ended: true, .. }]), + matches!(serves.as_slice(), [Output::Unserve { gone: Gone::Ended, .. }]), "{serves:?}" ); } diff --git a/rs/moq-net/tests/integration/mesh_withdraw.rs b/rs/moq-net/tests/integration/mesh_withdraw.rs index 38c66c0384e..6224f517899 100644 --- a/rs/moq-net/tests/integration/mesh_withdraw.rs +++ b/rs/moq-net/tests/integration/mesh_withdraw.rs @@ -168,11 +168,11 @@ pub(super) const PRODUCTION_LATENCY_MS: &[u64] = &[ ]; /// One withdrawal on a partial mesh retracts everywhere once. A relay two hops -/// from the publisher's hears only that its neighbor withdrew, and must not -/// fall back to, and re-advertise, another path derived from the same -/// announcement. Holding route updates (`origin::Config::with_update_hold`) lets the -/// withdrawal remove those paths first; with link latency and no hold, each -/// withdrawal here costs tens of thousands of announcements. +/// from the publisher's must not fall back to, and re-advertise, another path to +/// the withdrawn claim. Over hop chains, with link latency and no update hold, each +/// withdrawal here cost tens of thousands of announcements. Lite-07 holds nothing: +/// the claim's own withdrawal is final, and a relay holding a copy of it through +/// each neighbour drops every copy at once. #[kio::test] async fn partial_mesh_withdraw_retracts_once() { let version: Version = "moq-lite-07-wip".parse().unwrap(); @@ -218,8 +218,8 @@ const fn ms(millis: u64) -> Duration { Duration::from_millis(millis) } -/// Eight relays where, without the retraction quarantine, relays 1 and 5 -/// retract a broadcast published on relay 7 and then announce it again. +/// Eight relays where, over hop chains without the retraction quarantine, relays 1 +/// and 5 retract a broadcast published on relay 7 and then announce it again. #[rustfmt::skip] const STALE_PATHS: &[Link] = &[ ((0, 1), ms(35)), ((1, 2), ms(90)), ((0, 3), ms(129)), ((2, 4), ms(140)), ((1, 5), ms(137)), ((4, 6), ms(148)), @@ -227,9 +227,10 @@ const STALE_PATHS: &[Link] = &[ ]; /// A relay that already retracted a withdrawn broadcast must not announce it -/// again. Its neighbors, two or more hops from the publisher's relay, still hunt -/// through stale paths; one that falls back to a path not through this relay -/// offers it as a new route after the relay forgot the withdrawal. +/// again. Over hop chains its neighbors, two or more hops from the publisher's +/// relay, still hunt through stale paths, and one that falls back to a path not +/// through this relay offers it as a new route after the relay forgot the +/// withdrawal. On lite-07 the claim's withdrawal is final, so none offers one. #[kio::test] async fn retracted_relay_ignores_a_stale_path() { let version: Version = "moq-lite-07-wip".parse().unwrap(); @@ -354,9 +355,10 @@ impl Cluster { } /// #4349: a publisher on one relay cleanly closes a broadcast, and a client -/// watching from another relay sees one retraction and no re-announce. On the -/// release before the retraction quarantine (#5376), this client saw it -/// retracted, announced, and retracted again. See [`slow_ring_retracts_once`]. +/// watching from another relay sees one retraction and no re-announce, with no +/// update hold or retraction quarantine, which lite-07 routes skip. Lite-06 still +/// re-announces: see [`cross_node_close_reannounces_on_lite06`] and +/// [`slow_ring_retracts_once`]. #[kio::test] async fn cross_node_close_retracts_once_lite07() { cross_node_close_retracts_once("moq-lite-07-wip").await; @@ -383,22 +385,20 @@ async fn cross_node_close_retracts_once(version: &str) { } } -/// The positive control for [`cross_node_close_retracts_once`]: without the -/// update hold and the quarantine, the client sees the broadcast come back -/// before it ends. +/// The lite-06 residual of [`cross_node_close_retracts_once`]: the client sees the +/// broadcast come back before it ends, with the update hold and the quarantine or +/// without. Each lite-06 relay mints its own instance for what a neighbor +/// announces, so falling back onto a stale path is a restart, which the hold does +/// not absorb, under an instance the quarantine does not recognize. #[kio::test] -async fn cross_node_close_without_hold_reannounces() { - let mut cluster = Cluster::new( - "moq-lite-06", - STALE_PATHS, - origin::Config::default().with_update_hold(Duration::ZERO), - 7, - 1, - ) - .await; - let (broadcast, _) = cluster.publish("room/a").await; - let (client, _) = cluster.close("room/a", broadcast).await; - assert_eq!(client, [Kind::End, Kind::Start, Kind::End]); +async fn cross_node_close_reannounces_on_lite06() { + for config in [origin::Config::default(), origin::Config::default().with_update_hold(Duration::ZERO)] { + let hold = config.update_hold(); + let mut cluster = Cluster::new("moq-lite-06", STALE_PATHS, config, 7, 1).await; + let (broadcast, _) = cluster.publish("room/a").await; + let (client, _) = cluster.close("room/a", broadcast).await; + assert_eq!(client, [Kind::End, Kind::Start, Kind::End], "hold {hold:?}"); + } } /// Relay 3 hears a broadcast published on relay 0 first over the longer, faster @@ -505,6 +505,11 @@ async fn mesh_retracts_once() { /// On a lite-06 mesh each relay mints its own instance for what a neighbor /// announces, so the move to the late, shorter path is another instance: the /// watching client sees the live broadcast retracted and announced again. +/// +/// Lite-06 keeps it. Its wire carries no instance, so a relay cannot tell a move +/// between neighbors from its publisher restarting, and keeping the instance +/// across the move would give a restarted broadcast its old identity. A mesh on +/// lite-07 updates in place: [`late_better_path_updates_in_place`]. #[kio::test] async fn residual_late_better_path_restarts_on_lite06() { let (published, closed) = late_better_path("moq-lite-06", "moq-lite-06").await; diff --git a/rs/moq-net/tests/integration/path_routes.rs b/rs/moq-net/tests/integration/path_routes.rs index f717f92404b..085d8377c84 100644 --- a/rs/moq-net/tests/integration/path_routes.rs +++ b/rs/moq-net/tests/integration/path_routes.rs @@ -210,8 +210,10 @@ async fn a_flapping_link_keeps_a_dual_homed_publisher_routed() { } /// `X` is `P`'s only way out and neighbours `R1`, `R2` and `R3`, which mesh with each other. -/// `X` dies: each relay hears its own session close and its neighbours' DEADs, and none -/// moves onto another's route through `X` on the way. Each withdraws the claim once. +/// `X` dies: each relay hears its own session close and its neighbours' DEADs. One that +/// hears its own first may move onto a neighbour's route through `X`, which nothing yet +/// says is dead, only until that neighbour's DEAD lands, here in the same instant. Each +/// withdraws the claim once. #[kio::test] async fn a_dead_relay_reaches_every_neighbour_as_withdrawals() { let version = lite07(); @@ -238,8 +240,8 @@ async fn a_dead_relay_reaches_every_neighbour_as_withdrawals() { for (i, watch) in watched.iter_mut().enumerate() { let seen = watch.seen(start); assert!( - !seen.iter().any(|seen| seen.available && !withdrawn(seen)), - "R{}: no move onto a route through X: {seen:?}", + seen.iter().all(|seen| !seen.available || seen.at.is_zero()), + "R{}: on a route through X only until its DEAD lands: {seen:?}", i + 1 ); assert_eq!( diff --git a/rs/moq-net/tests/integration/route_acceptance.rs b/rs/moq-net/tests/integration/route_acceptance.rs index af0399a1921..ff8dee2b575 100644 --- a/rs/moq-net/tests/integration/route_acceptance.rs +++ b/rs/moq-net/tests/integration/route_acceptance.rs @@ -216,8 +216,7 @@ async fn ring() -> ( /// [`ring`]: the link from `R1` to `R2` closes. `R2` preferred the path through `R1`, of /// cost 3, and its backup runs the other way round the ring, of cost 4: it moves there at -/// once (within the origin's update hold), the broadcast available throughout. Every -/// other session stays up. +/// once, the broadcast available throughout. Every other session stays up. #[kio::test] async fn a_ring_moves_at_once_to_a_backup_avoiding_the_far_end() { let (_p, _live, relays, links, _p_r0) = ring().await; @@ -237,8 +236,10 @@ async fn a_ring_moves_at_once_to_a_backup_avoiding_the_far_end() { /// [`ring`]: the link from `R0` to `R1` closes. Every backup runs the other way round the /// ring, through `R0`, the closed link's far end, on sessions the DEAD does not name: `R2` -/// moves onto its backup, of cost 4, and `R1` onto `R2`'s, of cost 5, both at once and -/// available throughout. Every session of the backup stays up. +/// moves onto its backup, of cost 4, available throughout, and `R1` onto `R2`'s, of cost +/// 5, both at once. `R2` routed through `R1`, so it offered `R1` nothing until it moved: +/// `R1` shows the claim unavailable until `R2`'s route lands, here in the same instant. +/// Every session of the backup stays up. #[kio::test] async fn a_ring_moves_at_once_to_a_backup_through_the_far_end() { let (_p, _live, relays, links, _p_r0) = ring().await; @@ -248,13 +249,15 @@ async fn a_ring_moves_at_once_to_a_backup_through_the_far_end() { cut(&links[0]); quiesce().await; - for (name, watch, cost) in [("R2", &mut near, 4), ("R1", &mut far, 5)] { + for (name, watch, cost, waits) in [("R2", &mut near, 4, false), ("R1", &mut far, 5, true)] { let seen = watch.seen(start); + assert!(!seen.iter().any(withdrawn), "{name} stays shown: {seen:?}"); assert!( - seen.iter().all(|seen| seen.available && !withdrawn(seen)), - "{name} stays available: {seen:?}" + seen.iter().all(|seen| seen.available || (waits && seen.at.is_zero())), + "{name} is unavailable only while its backup is on its way: {seen:?}" ); let moved = seen.last().expect("moved"); + assert!(moved.available, "{name} ends available: {seen:?}"); assert!(moved.cost == cost, "{name} ends on its backup: {seen:?}"); assert!(moved.at < DEBOUNCE, "{name} moves at once: {seen:?}"); } @@ -265,9 +268,9 @@ async fn a_ring_moves_at_once_to_a_backup_through_the_far_end() { /// `A` and `B` each reach `P` directly and peer with each other, over a link lagging by /// [`LAG`]. `P` dies: each relay's only route left is the other's direct route, which the -/// other's own close made stale. Each moves onto it until the other's DEAD ends it one lag -/// after the cut, shorter than the origin's update hold, so neither shows the move. Both -/// show the claim unavailable and let go a debounce after that DEAD. +/// other's own close made stale. Each moves onto it at once, as the route rule must while +/// nothing says it is stale, until the other's DEAD ends it one lag after the cut. Both +/// then show the claim unavailable and let go a debounce after that DEAD, once. #[kio::test] async fn two_relays_routing_through_each_other_both_let_go() { const LAG: Duration = Duration::from_millis(200); @@ -284,7 +287,12 @@ async fn two_relays_routing_through_each_other_both_let_go() { quiesce().await; for (name, watch) in ["A", "B"].into_iter().zip(&mut watched) { let seen = watch.seen(start); - assert!(!seen.iter().any(|seen| seen.available), "{name}: {seen:?}"); + assert!( + seen.iter().all(|seen| !seen.available || seen.at.is_zero()), + "{name}: on the stale route only until its DEAD lands: {seen:?}" + ); + let unavailable = seen.iter().find(|seen| !seen.available).expect("unavailable"); + assert_eq!(unavailable.at, LAG, "{name}: unavailable once the DEAD lands: {seen:?}"); assert_eq!( seen.iter().filter(|seen| withdrawn(seen)).count(), 1, @@ -296,6 +304,40 @@ async fn two_relays_routing_through_each_other_both_let_go() { } } +/// `P` feeds `A`, which neighbours `B`. `P`'s link to `A` closes: no route reaches the claim, +/// so `A` lets it go a debounce later. Just after, `P` links to `B`, and `A` shows the claim +/// again the moment `B`'s route lands, at its higher cost. A lite-07 claim's own withdrawal is +/// final for the routes it had, so nothing holds a later one back the way a hop chain's +/// retraction quarantine would. +#[kio::test] +async fn a_claim_let_go_returns_at_once_on_a_worse_route() { + let version = lite07(); + let [p, a, b] = [1, 2, 3].map(produce_origin); + let _live = publish(&p, "live"); + let p_a = peer(version, &p, &a).await; + let _a_b = peer(version, &a, &b).await; + quiesce().await; + + let mut watch = record(&a, "live").await; + let start = kio::clock::now(); + cut(&p_a); + kio::clock::sleep(DEBOUNCE).await; + land().await; + let _p_b = peer(version, &p, &b).await; + let linked = kio::clock::now().saturating_duration_since(start); + land().await; + let seen = watch.seen(start); + let [lost, gone, back] = seen.as_slice() else { + panic!("let go, then back: {seen:?}"); + }; + assert!(!lost.available && lost.at.is_zero(), "{seen:?}"); + assert!(withdrawn(gone) && gone.at == DEBOUNCE, "{seen:?}"); + assert!( + back.kind == support::Kind::Start && back.available && back.cost == 2 && back.at == linked, + "back at once, through B: {seen:?}" + ); +} + /// How far apart [`a_vanished_relay_is_noticed_a_keep_alive_apart`]'s neighbours notice. const KEEP_ALIVE: Duration = Duration::from_secs(3); @@ -304,9 +346,8 @@ const KEEP_ALIVE: Duration = Duration::from_secs(3); /// neighbours notice one at a time, `R1` at once, `R2` half a [`KEEP_ALIVE`] later, `R3` a /// whole one later, as keep-alives spaced that far apart would. /// -/// Measured, each move shown 300 ms later through the origin's update hold: nothing keeps -/// a relay off a route through `X` that a neighbour still offers, so `R1` moves onto -/// `R2`'s at once and onto `R3`'s when `R2` notices, as `R2` does. `R3` notices at 3 s, +/// Measured: nothing keeps a relay off a route through `X` that a neighbour still offers, +/// so `R1` moves onto `R2`'s at once and onto `R3`'s when `R2` notices, as `R2` does. `R3` notices at 3 s, /// and all three show the claim unavailable then and go offline a debounce later, once /// each: the broadcast is never withdrawn early, but readers on `R1` and `R2` see it /// available while nothing can serve it, and a request made then follows a route into @@ -376,8 +417,8 @@ async fn a_vanished_relay_is_noticed_a_keep_alive_apart() { .map(|seen| seen.at.as_millis()) .collect() }; - assert_eq!(moves(&seen[0]), [300, 1_800], "R1: {:?}", seen[0]); - assert_eq!(moves(&seen[1]), [1_800], "R2: {:?}", seen[1]); + assert_eq!(moves(&seen[0]), [0, 1_500], "R1: {:?}", seen[0]); + assert_eq!(moves(&seen[1]), [1_500], "R2: {:?}", seen[1]); assert_eq!(moves(&seen[2]), [], "R3: {:?}", seen[2]); } @@ -447,7 +488,9 @@ async fn a_dispute_wheel_settles() { /// cost it had before. /// /// `B` and `C` move onto the backup as soon as the cut's news arrives, staying available, -/// and back as soon as the primary returns. `A` is left with the backup through `B`. +/// and back as soon as the primary returns. `A` is left with the backup through `B`, which +/// routed through `A` and so offered it nothing until it moved: `A` shows the claim +/// unavailable until then, here in the same instant. #[kio::test] async fn a_primary_that_fails_and_returns_takes_every_relay_back() { for back in [DEBOUNCE / 2, HOLD + DEBOUNCE] { @@ -504,11 +547,16 @@ async fn a_primary_that_fails_and_returns_takes_every_relay_back() { ); } assert!( - seen[0].iter().all(|seen| seen.available && !withdrawn(seen)), + seen[0].iter().all(|seen| !withdrawn(seen) && (seen.available || seen.at.is_zero())), "A after {back:?}: {:?}", seen[0] ); - assert_eq!(seen[0][0].cost, 5, "A after {back:?}: onto the backup: {:?}", seen[0]); + let backup = seen[0].iter().find(|seen| seen.available).expect("moved"); + assert!( + backup.cost == 5 && backup.at.is_zero(), + "A after {back:?}: onto the backup at once: {:?}", + seen[0] + ); } } diff --git a/rs/moq-net/tests/integration/route_cluster.rs b/rs/moq-net/tests/integration/route_cluster.rs index 228362a5f19..df7ea4483bd 100644 --- a/rs/moq-net/tests/integration/route_cluster.rs +++ b/rs/moq-net/tests/integration/route_cluster.rs @@ -463,8 +463,10 @@ impl Lan { /// publisher only over the CDN's own expensive link, never through the LAN. /// /// At the cut `lan[0]` moves onto the route through the CDN at once: its reader's -/// announcement stays, available. The closed session still ends the subscription it -/// carried, so a fresh one reads on through the CDN. +/// announcement stays. `cdn[0]` reached the local broadcast through `lan[0]`, so it offered +/// it nothing until the cut, and `lan[0]` shows it unavailable until that route lands, here +/// in the same instant. The closed session still ends the subscription it carried, so a +/// fresh one reads on through the CDN. #[kio::test] async fn a_partitioned_lan_reaches_itself_through_the_cdn_and_heals() { let mut lan = Lan::new().await; @@ -498,9 +500,13 @@ async fn a_partitioned_lan_reaches_itself_through_the_cdn_and_heals() { let partitioned = island.seen(); // The local publisher's push, `lan[1]`'s uplink, the CDN's link, and `lan[0]`'s uplink. let through_cdn = 1 + 1 + CDN_INTERNAL + 1; - let [moved] = partitioned.as_slice() else { + let [lost, moved] = partitioned.as_slice() else { panic!("the island moved once: {partitioned:?}"); }; + assert!( + lost.kind == Kind::Update && !lost.available && lost.instance == instance && lost.at == cut_at, + "unavailable until the CDN's route lands: {partitioned:?}" + ); assert!( moved.kind == Kind::Update && moved.available diff --git a/rs/moq-net/tests/integration/route_failover.rs b/rs/moq-net/tests/integration/route_failover.rs index cc2fcbcc9f7..4665dbcdf78 100644 --- a/rs/moq-net/tests/integration/route_failover.rs +++ b/rs/moq-net/tests/integration/route_failover.rs @@ -363,8 +363,8 @@ fn cut(pair: &MockPair) { pair.client.abort(Error::Cancel); } -/// How long the moves in [`next_hop_moves_keep_a_live_reader`] take to land: past the -/// origin's update hold, and the new next hop's upstream latency both ways. +/// How long the moves in [`next_hop_moves_keep_a_live_reader`] take to land: past the new +/// next hop's upstream latency both ways. const MOVE: u64 = 75; /// `P` publishes `live` to `A` and `B`, and the relay `R` pulls both, preferring `A`. A diff --git a/test/interop/clients/js/media.ts b/test/interop/clients/js/media.ts index 18d0cc8ef09..0afb8c6254d 100644 --- a/test/interop/clients/js/media.ts +++ b/test/interop/clients/js/media.ts @@ -200,8 +200,9 @@ const REPLACEMENT_DELAY_MS = 50; /** * How long a following player may take to present a replacement or fallback once the new route is - * ready or the old one withdrawn. Routing: the relay holds a route change for its update hold - * (moq-net's `origin::DEFAULT_UPDATE_HOLD`; the JS origin holds none), and the announcement plus + * ready or the old one withdrawn. Routing: the relay holds a route change that arrived over a hop + * chain (lite-06 and older, or moq-transport) for its update hold (moq-net's + * `origin::DEFAULT_UPDATE_HOLD`; lite-07 and the JS origin hold none), and the announcement plus * the catalog and track subscriptions take five loopback round trips, each charged at QUIC's 1 ms * timer granularity. Presentation: a keyframe interval of codec cadence, one frame to decode, and * the player's delay. Observation: one page sample and one driver poll. From d07578292b73ea54d7f8299b93394514d5215214 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 14:10:23 -0800 Subject: [PATCH 03/25] test(moq-net): pin a grant too wide for one ANNOUNCE_REQUEST, and point Node ID 0 at interop The acceptance quest keeps the scoped subscriber's fix, which waits on a maintainer decision between a wire and an API change. Co-Authored-By: Claude Opus 5.5 --- quest/m2/cluster-routes/README.md | 6 +- quest/m2/cluster-routes/acceptance.md | 87 +++++++------------ quest/m2/multicast/README.md | 4 +- rs/moq-net/tests/integration/auth.rs | 76 ++++++++++++++++ rs/moq-net/tests/integration/route_cluster.rs | 4 +- test/interop/README.md | 5 +- 6 files changed, 118 insertions(+), 64 deletions(-) diff --git a/quest/m2/cluster-routes/README.md b/quest/m2/cluster-routes/README.md index 47e75f5c81f..f51ab5daa96 100644 --- a/quest/m2/cluster-routes/README.md +++ b/quest/m2/cluster-routes/README.md @@ -42,9 +42,9 @@ cursor hold was the earlier mitigation. Main also contains retraction quarantine and [moq-dev/moq#5391](https://github.com/moq-dev/moq/pull/5391)'s same-clock queue dedup. #5375 measured residual stale revivals, so neither longer holds -nor one passing mesh establish correctness. Retire the mitigations only -after the incident fixtures and failover tests demonstrate their -replacement, as [acceptance](/quest/m2/cluster-routes/acceptance.md) does. +nor one passing mesh establish correctness. All three now hold only routes +learned over a hop chain, and go with lite-06: lite-07 routes skip them, +proven by the incident fixtures and failover tests on the route layer. ## Related diff --git a/quest/m2/cluster-routes/acceptance.md b/quest/m2/cluster-routes/acceptance.md index 849fd746ae8..5b4b32c0675 100644 --- a/quest/m2/cluster-routes/acceptance.md +++ b/quest/m2/cluster-routes/acceptance.md @@ -2,22 +2,12 @@ ## Goal -The route layer is proven on the implementation, and the mitigations its -fixtures replace are retired: [moq-dev/moq#5375](https://github.com/moq-dev/moq/pull/5375)'s -retraction quarantine, [moq-dev/moq#5391](https://github.com/moq-dev/moq/pull/5391)'s -same-clock queue dedup, and the 300 ms update hold, all marked `lite06_churn` in -`rs/moq-net/src/model/origin.rs`. +A lite-07 subscriber whose grant permits more disjoint regions than one +ANNOUNCE_REQUEST can name either discovers every region or says it cannot: +no reader sees a region it never asked for as synced and empty. ## Plan -Before the lite-07 freeze, exercise a scoped subscriber whose permitted, disjoint prefixes -cannot fit one ANNOUNCE_REQUEST and cannot be widened within the grant. In -`lite/subscriber/routed.rs`, `asks` can discard a prefix while `release` treats an -empty set of matching replies as synchronized. Publish inside the last permitted region -and verify either complete discovery or a surfaced inability to answer. An empty, -synchronized snapshot for an unasked region is incorrect. The audit traced these branches -in [moq-dev/moq#8987](https://github.com/moq-dev/moq/pull/8987); confirm the behavior on real sessions and fix it if reproduced. - Decided 2026-11-14 (maintainer, /quest-complete on [moq-dev/moq#8768](https://github.com/moq-dev/moq/pull/8768)): ✅ stays in m2 and gates the cut, per lite07-finalize's 2026-10-13 rule that routes' @@ -25,49 +15,34 @@ tier and partition tests run before the cut (rejected: moving it to m3 and fixing late routing flaws in lite-08). A flaw found here is fixed on lite-07-wip before it freezes. -The fixtures landed in [moq-dev/moq#8832](https://github.com/moq-dev/moq/pull/8832), -and the reference model is deleted. On real sessions with time mocked: the -topology and prior-art fixtures (`tests/integration/route_acceptance.rs`), -failover liveness and the live-graph tests of -[moq-dev/moq#4644](https://github.com/moq-dev/moq/pull/4644) (`route_failover.rs`), -the tiered cluster, partition, and scoped-interest checks (`route_cluster.rs`), -and mount continuity (`mount.rs`). -[moq-dev/moq#8987](https://github.com/moq-dev/moq/pull/8987) put lite-07 on -`route::relay::Router` and rewrote their assertions for it; the old -router's forwarding and operation-count fixtures went with it, and -`route/tests/multicast*.rs` covers the new one. The route fan-out quest -those pinned was deleted -with the old rule, its one surviving cost moved to -[route benches](/quest/m3/route-benches.md). - -Decided 2026-11-16 (maintainer, /quest-plan of the pre-cut audit): the -switch makes `origin::Config::update_hold` crate-private, since nothing -outside moq-net sets it, and puts whatever of the three mitigations survives -here under one named lite-06 marker. So retiring them below changes no -public API, and one that a lite-06 mesh keeps is already marked temporary. -[moq-dev/moq#9053](https://github.com/moq-dev/moq/pull/9053) did both: the -three are marked by the `lite06_churn` module, and still run on every -version, so gating any that a lite-06 mesh keeps to the lite-06 session -path is part of retiring them below. - -What remains: - -- Retire the quarantine, the dedup, and the hold, now that the fixtures - run on the route layer that replaces them. The quarantine - is what turns the flaws into a 2 s gap: a relay that shows a held claim - offline announces it again only after `RETRACT_QUARANTINE`. The hold and - the quarantine sit on the announce cursor, which every version shares, so - check what a lite-06 mesh loses without them - (`mesh_withdraw::cross_node_close_without_hold_reannounces` is its - positive control) before deleting them; if a lite-06 mesh still needs - them, retire them for lite-07 routes only. -- `mesh_withdraw::residual_late_better_path_restarts_on_lite06` pins a - lite-06 mesh where each relay mints its own instance, so a client sees - `Start, End, Start` while the broadcast stays live. Turn it into an update, - or record why a lite-06 mesh keeps it. -- A request naming Node ID 0 on a lite-07 session: Rust clients always name - a node, so only publisher unit tests cover a relay answering 0. Cover it - where `js/net` sends it, in `just test interop`. +[moq-dev/moq#9112](https://github.com/moq-dev/moq/pull/9112) confined the +`lite06_churn` mitigations to routes learned over a hop chain, made a claim +the route node lets go of leave every copy at once, recorded why lite-06 +keeps its `Start, End, Start` residual, and pointed the Node ID 0 coverage +at the native JS claims lanes of `just test interop`. What remains: + +`auth::residual_lite_07_a_grant_too_wide_to_ask_syncs_without_its_last_region` +pins the flaw on real sessions. A relay grants its client 18 disjoint +regions over two tokens; each token's grant fits its own AUTH message, but +the union does not fit the one ANNOUNCE_REQUEST a subscriber's whole set +must (`lite::request_fits`, which the publisher enforces). `coarsen` in +`lite/subscriber/routed.rs` then drops the last region, and `release` treats +a want no asked prefix overlaps as answered: a whole-tree reader syncs +without that region's broadcast, and a reader of the region alone syncs +empty. Fix it per the decision under Waiting, and turn the residual into +the passing case. + +## Waiting + +- Maintainer: how a subscriber handles a grant too wide for one + ANNOUNCE_REQUEST. (A, recommended) Wire: a subscriber's set may pass one + message when each prefix past it is a region the publisher granted, so the + publisher's own grant bounds what it holds and the client discovers + everything; changes the lite-07-wip draft and both publishers, `rs/moq-net` + and `js/net`. (B) API: a reader of an unasked region learns the session + cannot answer it, a public change in Rust, JS, and the bindings. (C) + Neither: a reader overlapping an unasked region never syncs, a hang in + place of a wrong answer. Done once the choice is recorded in Plan. ## Related diff --git a/quest/m2/multicast/README.md b/quest/m2/multicast/README.md index 0196142d0c8..7d3b0a17666 100644 --- a/quest/m2/multicast/README.md +++ b/quest/m2/multicast/README.md @@ -33,8 +33,8 @@ Its proof is `rs/moq-net/verify/route-algebra/`, and the quests written against the old wire were retargeted or deleted. The lite-07 cut still waits on this epic's last child, [the uncovered cases](/quest/m2/multicast/accept.md), and on -[route layer acceptance](/quest/m2/cluster-routes/acceptance.md) retiring -the lite-06 mitigations. After the cut come warmth's wire +[route layer acceptance](/quest/m2/cluster-routes/acceptance.md)'s +scoped subscriber whose grant is too wide to ask for. After the cut come warmth's wire ([cache-aware routing](/quest/m3/cache-routing/README.md)), [route benches](/quest/m3/route-benches.md), [the proof binding](/quest/m3/route-proof-binding.md), diff --git a/rs/moq-net/tests/integration/auth.rs b/rs/moq-net/tests/integration/auth.rs index d593ef5eae1..6da56bb6509 100644 --- a/rs/moq-net/tests/integration/auth.rs +++ b/rs/moq-net/tests/integration/auth.rs @@ -2444,3 +2444,79 @@ async fn added(handle: &auth::Handle, token: impl Into) -> moq_net credential.answered().await?; Ok(credential) } + +/// A relay grants its client 18 disjoint regions over two tokens, half each, and publishes +/// a broadcast in each; none widens within the grant. Each token's grant fits its own +/// message, but their union's prefixes cannot all fit the one ANNOUNCE_REQUEST a +/// subscriber's whole set must, so the client asks for all but the last region. A reader +/// of the whole tree syncs without that region's broadcast, and a reader of the region +/// alone syncs empty, though the region was never asked for. +/// +/// Pinned until the maintainer settles the fix in the acceptance quest +/// (`quest/m2/cluster-routes/acceptance.md`): the client either discovers every region, +/// which takes a wire change to how much a subscriber may ask, or says it cannot, which +/// takes an API change to how a reader learns it. +#[kio::test] +async fn residual_lite_07_a_grant_too_wide_to_ask_syncs_without_its_last_region() { + within(async { + let relay = produce_origin(1); + let regions: Vec = (0..18).map(|i| format!("r{i:02}/{}", "x".repeat(3800))).collect(); + let _published: Vec = (regions.iter()) + .map(|region| { + let route = relay + .route(format!("{region}/live"), origin::RouteOptions::hidden()) + .unwrap(); + route.announce().unwrap(); + route + }) + .collect(); + let subscribe = |regions: &[String]| { + let subtrees = regions.iter().map(|region| Pattern::subtree(region).unwrap()); + grant_of(Patterns::default(), subtrees.collect()) + }; + let client = produce_origin(9); + let mut pair = connect(Options { + server_publish: Some(relay.clone()), + client_subscribe: Some(client.clone()), + server_requests: true, + ..Default::default() + }) + .await; + let (first, second) = (subscribe(®ions[..9]), subscribe(®ions[9..])); + let _issued = serve(pair.requests.take().unwrap(), move |token| match token { + b"" => Some(first.clone()), + b"t1" => Some(second.clone()), + _ => None, + }); + let _t1 = added(&pair.client.auth(), "t1").await.expect("t1 granted"); + let whole = subscribe(®ions).subscribe; + wait_for(pair.client.auth().grant(), |grant| { + grant.as_ref().is_some_and(|grant| grant.subscribe == whole) + }) + .await; + + let last = regions.last().unwrap(); + let mut everything = client.consume().routes(); + let mut region = (client.consume().scope(last, &Patterns::from(Pattern::all()))) + .unwrap() + .routes(); + kio::clock::sleep(Duration::from_secs(1)).await; + let mut found = std::collections::BTreeSet::new(); + while let Some(event) = everything.try_next() { + found.insert(event.announce.prefix.to_string()); + } + assert!(everything.poll_synced(&kio::Waiter::noop()).is_ready()); + let missing: Vec<&String> = (regions.iter()) + .filter(|region| !found.contains(&format!("{region}/live"))) + .collect(); + assert_eq!(missing, [last], "synced without the last region"); + + assert!(region.try_next().is_none(), "the last region was asked for"); + assert!( + region.poll_synced(&kio::Waiter::noop()).is_ready(), + "the last region's reader syncs empty" + ); + }) + .await + .expect("timed out"); +} diff --git a/rs/moq-net/tests/integration/route_cluster.rs b/rs/moq-net/tests/integration/route_cluster.rs index df7ea4483bd..44b2d21a061 100644 --- a/rs/moq-net/tests/integration/route_cluster.rs +++ b/rs/moq-net/tests/integration/route_cluster.rs @@ -1006,8 +1006,8 @@ impl Origin { /// over its own session. A client lists only `live`. It holds `L`'s route and never `V`'s: /// `V` churning costs the client no byte. A request outside that interest still resolves /// through the relay. A Rust client's request discovers its own path, so it names `V`'s -/// node; Node ID 0, the publisher selecting, comes only from other implementations and -/// is covered at the publisher (`an_answered_track_stream_holds_the_track_until_the_requester_closes`). +/// node; Node ID 0, the publisher selecting, comes only from other implementations: the +/// native JS claims lanes of `just test interop` cover a relay answering it. #[kio::test] async fn a_client_watching_one_prefix_holds_only_the_origins_meeting_it() { let relay = produce_origin(1); diff --git a/test/interop/README.md b/test/interop/README.md index 543e1a9d357..b5d407eb079 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -18,7 +18,10 @@ WebCodecs output painted to a canvas and drives the player's pause/resume contro The native JS tail lanes also resolve on-demand claims on lite-07, under Node and Bun over WebSocket and WebTransport. Fresh readers request or follow the claim's own path and child paths. A reader-first case checks that an absent -request ends while its follower waits for the publisher to arrive. +request ends while its follower waits for the publisher to arrive. A browser forwards +nothing and knows no node, so every request `@moq/net` sends names Node ID 0, leaving the +relay to pick the claim's node: these lanes are where a relay answers that end to end, +since a Rust client always names one. Every publisher also sends an Opus tone, one sine per channel, and every subscriber decodes it. Python and C++ send a mono 440 Hz tone; the rest send 440 Hz left and From 366afa82eb02d268ba7308e91ef1c1c9e53a04dc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 14:22:55 -0800 Subject: [PATCH 04/25] test(moq-net): retarget the lost-publisher drill at retire, and widen the follow gap's margin Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/follow.rs | 3 +- rs/moq-net/tests/integration/mesh_withdraw.rs | 5 ++- .../tests/integration/route_acceptance.rs | 4 +- .../relay-withdraws-lost-publisher.patch | 39 ++++++++++++------- 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/rs/moq-net/src/model/follow.rs b/rs/moq-net/src/model/follow.rs index 4b94e4b49d9..a2379c0ae53 100644 --- a/rs/moq-net/src/model/follow.rs +++ b/rs/moq-net/src/model/follow.rs @@ -745,7 +745,8 @@ mod tests { let (_arrival, _publish) = futures::join!(available(&mut announced), async { let _unanswered = requested(&dynamic).await; drop(dynamic); - kio::clock::sleep(Duration::from_secs(1)).await; + // A gap, well inside the second `available` waits. + kio::clock::sleep(Duration::from_millis(600)).await; origin.route("live", RouteOptions::default()).unwrap() }); none(&mut announced).await; diff --git a/rs/moq-net/tests/integration/mesh_withdraw.rs b/rs/moq-net/tests/integration/mesh_withdraw.rs index 6224f517899..120b38cb8c2 100644 --- a/rs/moq-net/tests/integration/mesh_withdraw.rs +++ b/rs/moq-net/tests/integration/mesh_withdraw.rs @@ -392,7 +392,10 @@ async fn cross_node_close_retracts_once(version: &str) { /// not absorb, under an instance the quarantine does not recognize. #[kio::test] async fn cross_node_close_reannounces_on_lite06() { - for config in [origin::Config::default(), origin::Config::default().with_update_hold(Duration::ZERO)] { + for config in [ + origin::Config::default(), + origin::Config::default().with_update_hold(Duration::ZERO), + ] { let hold = config.update_hold(); let mut cluster = Cluster::new("moq-lite-06", STALE_PATHS, config, 7, 1).await; let (broadcast, _) = cluster.publish("room/a").await; diff --git a/rs/moq-net/tests/integration/route_acceptance.rs b/rs/moq-net/tests/integration/route_acceptance.rs index ff8dee2b575..9d2ae958001 100644 --- a/rs/moq-net/tests/integration/route_acceptance.rs +++ b/rs/moq-net/tests/integration/route_acceptance.rs @@ -547,7 +547,9 @@ async fn a_primary_that_fails_and_returns_takes_every_relay_back() { ); } assert!( - seen[0].iter().all(|seen| !withdrawn(seen) && (seen.available || seen.at.is_zero())), + seen[0] + .iter() + .all(|seen| !withdrawn(seen) && (seen.available || seen.at.is_zero())), "A after {back:?}: {:?}", seen[0] ); diff --git a/test/drill/mutations/relay-withdraws-lost-publisher.patch b/test/drill/mutations/relay-withdraws-lost-publisher.patch index 3274fda1547..b307a3091f1 100644 --- a/test/drill/mutations/relay-withdraws-lost-publisher.patch +++ b/test/drill/mutations/relay-withdraws-lost-publisher.patch @@ -4,12 +4,14 @@ # Removes the relay's withdrawal of a publisher it lost. Each announced route # holds an origin handle whose drop retracts it; wrapping that in `ManuallyDrop` # means a route outlives the session that announced it, so a crashed publisher's -# name stays announced with nothing behind it. Explicit withdrawals still consume -# the handle, so only the implicit retraction on session loss is removed. +# name stays announced with nothing behind it, and a copy the route node lets go +# of without its publisher ending it is leaked rather than retired. Explicit +# withdrawals still consume the handle, so only the implicit retraction on +# session loss is removed. diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs -@@ -11273,7 +11273,7 @@ impl Announced { +@@ -10418,7 +10418,7 @@ impl Announced { }; self.held.pop(); if let Held::Attached(entry) = held { @@ -18,7 +20,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. } } -@@ -11359,7 +11359,7 @@ fn mint_sources( +@@ -10504,7 +10504,7 @@ fn mint_sources( let validity = if subscriber.version.has_instance() { Default::default() } else { @@ -27,7 +29,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. }; let source = subscriber.origin.create_source(path.clone(), validity); // The handler exists before the requester sees the source, so a track it -@@ -11387,7 +11387,8 @@ struct AnnouncedRoute { +@@ -10532,7 +10532,8 @@ struct AnnouncedRoute { /// without recomputing the chain. route: crate::origin::RouteInfo, /// Dropping it retracts the route and rejects its queued requests. @@ -37,7 +39,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. /// Closes as the route goes, which closes every source it minted. live: kio::Producer<()>, /// Queues this route on its prefix's ready set when its request queue wakes. -@@ -11405,7 +11406,7 @@ impl AnnouncedRoute { +@@ -10550,7 +10551,7 @@ impl AnnouncedRoute { ) -> Self { Self { route, @@ -49,7 +51,18 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs -@@ -291,7 +291,7 @@ impl Routed { +@@ -146,8 +146,8 @@ impl node::Orphan for Left { + fn release(claim: crate::origin::Claim, gone: node::Gone) { + match gone { + // Dropped, it goes offline, while another copy serves. +- node::Gone::Moved => drop(claim), +- node::Gone::Lost => claim.retire(false), ++ // MUTATION: a copy let go of without its publisher ending it stays announced. ++ node::Gone::Moved | node::Gone::Lost => std::mem::forget(claim), + node::Gone::Ended => claim.retire(true), + } + } +@@ -418,7 +417,7 @@ impl Routed { for ((origin, path, instance), mut served) in self.served.drain() { let unavailable = served.route.route.clone().with_available(false); served.route.update(unavailable); @@ -58,12 +71,12 @@ diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subs self.stream.orphan(origin, &(path, instance), Box::new(left)); } } -@@ -641,7 +641,7 @@ impl Routed { - if let Some(served) = self.served.remove(&(origin, path, instance)) - && ended - { -- served.route.dynamic.withdrawn(); -+ std::mem::ManuallyDrop::into_inner(served.route.dynamic).withdrawn(); +@@ -777,7 +776,7 @@ impl Routed { + gone, + } => { + if let Some(served) = self.served.remove(&(origin, path, instance)) { +- release(served.route.dynamic, gone); ++ release(std::mem::ManuallyDrop::into_inner(served.route.dynamic), gone); } } } From b67e47c2368cedb0f9818d05a1869464fc6e642a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 14:45:56 -0800 Subject: [PATCH 05/25] fix(moq-net): a late retirement spares the copies a later showing of the claim serves Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/subscriber/routed.rs | 1 + rs/moq-net/src/model/origin.rs | 50 +++++++++++++++++++++--- rs/moq-net/src/route/node.rs | 17 ++++++++ rs/moq-net/src/route/node/tests.rs | 45 +++++++++++++++++++++ 4 files changed, 108 insertions(+), 5 deletions(-) diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index df4d48d2071..2bc9556dd56 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -821,6 +821,7 @@ impl Routed { priority: Some(served.priority), // A receiver refuses a claim above its path by more than the path's segments. above: u8::try_from(served.attr.above).unwrap_or(u8::MAX), + lifecycle: served.lifecycle, }; // A claim no route reaches is unavailable: it stays shown, and serves nothing. let route = crate::origin::RouteInfo::new() diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 01d2e5a83f6..df4d4cd7c9e 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -605,6 +605,10 @@ pub(crate) struct Lane { /// exactly there. The longer prefix wins, so a claim at 0 outranks every broader one. /// A path has at most 32 segments, so a byte holds it, and the route entry stays small. pub above: u8, + /// Which showing of the claim the route layer serves: a new one each time the claim + /// is shown again after every copy of it left, so a copy retired late never retires + /// the copies a later showing serves. See [`Claim::retire`]. + pub lifecycle: u64, } impl Default for Lane { @@ -613,6 +617,7 @@ impl Default for Lane { primary: true, priority: None, above: 0, + lifecycle: 0, } } } @@ -3587,9 +3592,9 @@ impl Claim { } /// The route layer let go of the claim this follows: every copy of it the table holds, - /// each of the same node, instance and epoch, goes with this one under one lock, rather - /// than readers seeing the rest in turn as their sessions retract them. They end with - /// `ended`, else go offline. + /// each of the same node, instance, epoch and [`Lane::lifecycle`], goes with this one + /// under one lock, rather than readers seeing the rest in turn as their sessions + /// retract them. They end with `ended`, else go offline. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn retire(self, ended: bool) { self.state.announcement.retract(Retract::Retired { ended }); @@ -6513,13 +6518,18 @@ impl OriginState { } /// Retire every other copy at its prefix of the claim `entry` follows: the same - /// forwarding node, instance and epoch. See [`Claim::retire`]. + /// forwarding node, instance, epoch and showing ([`Lane::lifecycle`]). See + /// [`Claim::retire`]. fn retire_copies(&mut self, entry: &RouteEntry) { let Some(node) = entry.node else { return; }; for other in self.routes.at_mut(&entry.prefix) { - if other.node == Some(node) && other.instance == entry.instance && other.epoch == entry.epoch { + if other.node == Some(node) + && other.instance == entry.instance + && other.epoch == entry.epoch + && other.lane.lifecycle == entry.lane.lifecycle + { other.retired = true; } } @@ -13308,6 +13318,36 @@ mod tests { } } + /// A copy whose session releases it only after the route layer showed the claim again + /// through another session retires its own showing, not the new one. + #[kio::test] + async fn a_late_retirement_spares_a_later_showing() { + for ended in [true, false] { + let producer = origin(1).produce(); + let peer = producer.clone().with_link(Link::Peer); + let mut announced = producer.consume().routes(); + let copy = |cost: u64, lifecycle: u64| { + RouteInfo::new() + .with_instance(instance(), 0) + .with_hops(hops(&[7])) + .with_cost(cost) + .with_node(Some(node_id(7))) + .with_lane(Lane { + lifecycle, + ..Lane::default() + }) + }; + let old = peer.forward("room", copy(1, 1)).unwrap(); + announced.expect_next_active("room").await; + announced.assert_next_wait(); + let _new = peer.forward("room", copy(2, 2)).unwrap(); + old.retire(ended); + let new = announced.expect_next_active("room").await; + assert_eq!(new.cost, Cost::from(2), "ended: {ended}"); + announced.assert_next_wait(); + } + } + /// Overlapping sessions carrying one instance are independent claims, regardless of /// announcement order, and falling back from one to the other is the same stream. #[kio::test] diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index ffbb91bf679..c47deeee30b 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -102,6 +102,10 @@ pub(crate) struct Served { /// The node a request through the session names, or `None` when the peer selects: /// it gave this node the claim itself. pub named: Option, + /// Which showing of the claim this is: every slot of it shares one, and a claim + /// shown again after every slot left gets a new one, so a session still releasing + /// the old showing's copy never takes the new one's with it. + pub lifecycle: u64, } /// A change to what a session serves into the origin. @@ -192,6 +196,8 @@ struct State { router: Router, clock: Deadlines, next_key: Key, + /// The next [`Served::lifecycle`]. + next_lifecycle: u64, queues: HashMap>, /// Each session's own key, by the tag every stream of the session shares. tags: HashMap, @@ -296,6 +302,7 @@ impl Node { router, clock, next_key: LOCAL_MARKED + 1, + next_lifecycle: 0, queues: HashMap::new(), tags: HashMap::new(), mounts: HashMap::new(), @@ -752,6 +759,15 @@ impl State { let (path, instance) = broadcast; let shown = self.router.shown().contains_key(&(instance.clone(), path.clone())); let mut wanted: BTreeMap = BTreeMap::new(); + // A slot still held keeps its showing; none, and this is a new one. + let held = (self.lanes.get(broadcast)).and_then(|slots| slots.values().next()); + let lifecycle = match held { + Some(slot) => slot.served.lifecycle, + None => { + self.next_lifecycle += 1; + self.next_lifecycle + } + }; if shown { for way in self.router.ways(broadcast) { let (session, named) = match way.next { @@ -772,6 +788,7 @@ impl State { chain: way.claim.chain, marked: way.claim.marked, named, + lifecycle, }; wanted.insert((way.origin, session), served); } diff --git a/rs/moq-net/src/route/node/tests.rs b/rs/moq-net/src/route/node/tests.rs index 6325d51ffe8..a8b0f6e9228 100644 --- a/rs/moq-net/src/route/node/tests.rs +++ b/rs/moq-net/src/route/node/tests.rs @@ -425,6 +425,51 @@ fn an_orphan_is_released_once_another_session_serves_the_claim() { assert_eq!(left.load(Ordering::SeqCst), Left::MOVED); } +/// A claim another session takes over keeps its showing, and one shown again after the +/// node let go of it gets a new one, so the origin never retires the new copies with the +/// old. +#[test] +fn a_claim_shown_again_is_a_new_showing() { + let lifecycle = |stream: &Stream| { + let serves = serves(stream); + let [ + Output::Serve { + origin, + path, + instance, + served, + }, + ] = serves.as_slice() + else { + panic!("one claim to serve: {serves:?}"); + }; + stream.serving(*origin, &(path.clone(), instance.clone()), true); + served.lifecycle + }; + let (node, clock) = node(); + let room = cast("room"); + let first = publisher(&node, 1); + announce(&first, &room, live(0)); + let shown = lifecycle(&first); + let left = Arc::new(AtomicU8::new(Left::HELD)); + first.orphan(node.me(), &room, Box::new(Left(left.clone()))); + drop(first); + + let second = publisher(&node, 2); + announce(&second, &room, live(0)); + assert_eq!(lifecycle(&second), shown, "taken over, not shown again"); + assert_eq!(left.load(Ordering::SeqCst), Left::MOVED); + let left = Arc::new(AtomicU8::new(Left::HELD)); + second.orphan(node.me(), &room, Box::new(Left(left.clone()))); + drop(second); + expire(&node, &clock); + assert_eq!(left.load(Ordering::SeqCst), Left::LOST); + + let third = publisher(&node, 3); + announce(&third, &room, live(0)); + assert_ne!(lifecycle(&third), shown, "shown again after the node let go"); +} + /// The origin's own copy of a claim never waits to be served, so a session refusing the /// claim still releases the orphan another session left. #[test] From 61d8e350c6eee95db4830b536226c7867f672231 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 15:05:18 -0800 Subject: [PATCH 06/25] fix(moq-net): retiring a claim takes its copies at an older epoch too Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/origin.rs | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index df4d4cd7c9e..304d4c2fe65 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -3592,7 +3592,7 @@ impl Claim { } /// The route layer let go of the claim this follows: every copy of it the table holds, - /// each of the same node, instance, epoch and [`Lane::lifecycle`], goes with this one + /// each of the same node, instance and [`Lane::lifecycle`] at any epoch, goes with this one /// under one lock, rather than readers seeing the rest in turn as their sessions /// retract them. They end with `ended`, else go offline. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] @@ -6518,8 +6518,9 @@ impl OriginState { } /// Retire every other copy at its prefix of the claim `entry` follows: the same - /// forwarding node, instance, epoch and showing ([`Lane::lifecycle`]). See - /// [`Claim::retire`]. + /// forwarding node, instance and showing ([`Lane::lifecycle`]), at any epoch, since a + /// copy whose session has yet to take a newer epoch would otherwise be left as the + /// fallback. See [`Claim::retire`]. fn retire_copies(&mut self, entry: &RouteEntry) { let Some(node) = entry.node else { return; @@ -6527,7 +6528,6 @@ impl OriginState { for other in self.routes.at_mut(&entry.prefix) { if other.node == Some(node) && other.instance == entry.instance - && other.epoch == entry.epoch && other.lane.lifecycle == entry.lane.lifecycle { other.retired = true; @@ -13282,8 +13282,9 @@ mod tests { } /// The route layer letting go of one copy of a lite-07 claim takes every copy of the - /// same node, instance and epoch with it: a cursor sees the claim end or go offline, - /// never a move onto another copy first. Another node's claim of the instance stays. + /// same node, instance and showing with it, a copy still at an older epoch too: a + /// cursor sees the claim end or go offline, never a move onto another copy first. + /// Another node's claim of the instance stays. #[kio::test] async fn retiring_a_claim_retires_every_copy() { for ended in [true, false] { @@ -13308,6 +13309,19 @@ mod tests { } announced.assert_next_wait(); + // A session that has yet to take the newer epoch still holds the older one. + let newer = |node: u64, cost: u64| copy(node, cost).with_instance(instance(), 1); + let near = peer.forward("room", newer(7, 1)).unwrap(); + let _stale = peer.forward("room", copy(7, 2)).unwrap(); + announced.expect_next_active("room").await; + announced.assert_next_wait(); + near.retire(ended); + match ended { + true => announced.expect_next_ended("room").await, + false => announced.expect_next_offline("room").await, + } + announced.assert_next_wait(); + let near = peer.forward("room", copy(7, 1)).unwrap(); let _other = peer.forward("room", copy(8, 3)).unwrap(); announced.expect_next_active("room").await; From 1595e40cecefa430cb4f54275334014c66788f52 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 15:24:00 -0800 Subject: [PATCH 07/25] fix(moq-net): a copy ended while its claim stays shown is withdrawn alone Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/subscriber/routed.rs | 1 + rs/moq-net/src/route/node.rs | 10 ++++++++-- rs/moq-net/src/route/node/tests.rs | 2 ++ .../mutations/relay-withdraws-lost-publisher.patch | 2 +- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 2bc9556dd56..38bf73338d6 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -149,6 +149,7 @@ fn release(claim: crate::origin::Claim, gone: node::Gone) { node::Gone::Moved => drop(claim), node::Gone::Lost => claim.retire(false), node::Gone::Ended => claim.retire(true), + node::Gone::Withdrawn => claim.withdrawn(), } } diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index c47deeee30b..3f516b9e2f0 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -138,6 +138,9 @@ pub(crate) enum Gone { /// Its publisher withdrew it, as far as this node knows: every copy the origin holds /// ends. Ended, + /// Its publisher withdrew this copy, and the reader shows the claim again at once: + /// this copy alone ends, and the copies still serving it stay. + Withdrawn, } /// A served claim's place in the origin, which outlives the session that served it until @@ -857,10 +860,13 @@ impl State { return; } // Unshown, every slot leaves at once, so each tells its copy all of them go: the - // origin then drops them together rather than showing the rest in turn. + // origin then drops them together rather than showing the rest in turn. Shown, the + // staying slots share the leaving ones' showing, so a leaving slot speaks only for + // its own copy. let gone = match (shown, ended) { - (_, true) => Gone::Ended, + (false, true) => Gone::Ended, (false, false) => Gone::Lost, + (true, true) => Gone::Withdrawn, (true, false) => Gone::Moved, }; let leaving: Vec = (slots.iter()) diff --git a/rs/moq-net/src/route/node/tests.rs b/rs/moq-net/src/route/node/tests.rs index a8b0f6e9228..baea85da78a 100644 --- a/rs/moq-net/src/route/node/tests.rs +++ b/rs/moq-net/src/route/node/tests.rs @@ -118,6 +118,7 @@ impl Left { /// The node let go of the claim, and every copy with it. const LOST: u8 = 2; const ENDED: u8 = 3; + const WITHDRAWN: u8 = 4; } impl Orphan for Left { @@ -126,6 +127,7 @@ impl Orphan for Left { Gone::Moved => Self::MOVED, Gone::Lost => Self::LOST, Gone::Ended => Self::ENDED, + Gone::Withdrawn => Self::WITHDRAWN, }; self.0.store(how, Ordering::SeqCst); } diff --git a/test/drill/mutations/relay-withdraws-lost-publisher.patch b/test/drill/mutations/relay-withdraws-lost-publisher.patch index b307a3091f1..f3974ee046b 100644 --- a/test/drill/mutations/relay-withdraws-lost-publisher.patch +++ b/test/drill/mutations/relay-withdraws-lost-publisher.patch @@ -60,8 +60,8 @@ diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subs + // MUTATION: a copy let go of without its publisher ending it stays announced. + node::Gone::Moved | node::Gone::Lost => std::mem::forget(claim), node::Gone::Ended => claim.retire(true), + node::Gone::Withdrawn => claim.withdrawn(), } - } @@ -418,7 +417,7 @@ impl Routed { for ((origin, path, instance), mut served) in self.served.drain() { let unavailable = served.route.route.clone().with_available(false); From 16fed2fefb66e8d05b84e0e5e28e05b121f38dd4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 15:48:58 -0800 Subject: [PATCH 08/25] fix(moq-net): a live copy of the same claim keeps its claimant's hop from going stale Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/origin.rs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 304d4c2fe65..22aa2f76283 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -3857,14 +3857,19 @@ impl AnnounceProducer { // A peer remains reachable while any of its sessions advertises this // prefix. Remove this session's route before testing the remaining // claims, under the same lock, so overlapping withdrawals cannot - // invalidate a newer advertisement or miss the last withdrawal. + // invalidate a newer advertisement or miss the last withdrawal. A live copy + // of the same lite-07 claim, such as one a later showing serves over another + // chain, says its claimant still advertises it too. + let same_claim = |other: &RouteEntry| { + entry.node.is_some() && other.node == entry.node && other.instance == entry.instance + }; if matches!(how, Retract::Withdrawn | Retract::Retired { ended: true }) && let Some(&peer) = entry.hops.iter().last() && peer != Hop::UNKNOWN && !shared .routes .at(prefix) - .any(|other| other.live() && other.hops.iter().last() == Some(&peer)) + .any(|other| other.live() && (other.hops.iter().last() == Some(&peer) || same_claim(other))) { shared.withdrawn.entry(prefix.clone()).or_default().insert(peer); shared.restale(prefix); @@ -13333,17 +13338,18 @@ mod tests { } /// A copy whose session releases it only after the route layer showed the claim again - /// through another session retires its own showing, not the new one. + /// through another session retires its own showing, not the new one, even when the new + /// one's chain passes through the old one's last hop. #[kio::test] async fn a_late_retirement_spares_a_later_showing() { for ended in [true, false] { let producer = origin(1).produce(); let peer = producer.clone().with_link(Link::Peer); let mut announced = producer.consume().routes(); - let copy = |cost: u64, lifecycle: u64| { + let copy = |chain: &[u64], cost: u64, lifecycle: u64| { RouteInfo::new() .with_instance(instance(), 0) - .with_hops(hops(&[7])) + .with_hops(hops(chain)) .with_cost(cost) .with_node(Some(node_id(7))) .with_lane(Lane { @@ -13351,10 +13357,10 @@ mod tests { ..Lane::default() }) }; - let old = peer.forward("room", copy(1, 1)).unwrap(); + let old = peer.forward("room", copy(&[7], 1, 1)).unwrap(); announced.expect_next_active("room").await; announced.assert_next_wait(); - let _new = peer.forward("room", copy(2, 2)).unwrap(); + let _new = peer.forward("room", copy(&[7, 9], 2, 2)).unwrap(); old.retire(ended); let new = announced.expect_next_active("room").await; assert_eq!(new.cost, Cost::from(2), "ended: {ended}"); From 66c4be19d09b6b81701a16f3c54f3adf91ab74aa Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 16:37:32 -0800 Subject: [PATCH 09/25] fix(moq-net): the route mesh harness lets go of a claim as a session does Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/fuzz/mesh.rs | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/rs/moq-net/src/fuzz/mesh.rs b/rs/moq-net/src/fuzz/mesh.rs index 6ef28b0e1ff..2292461830e 100644 --- a/rs/moq-net/src/fuzz/mesh.rs +++ b/rs/moq-net/src/fuzz/mesh.rs @@ -74,11 +74,19 @@ struct Upstream { struct Left(crate::origin::Claim, Arc); impl node::Orphan for Left { - fn release(self: Box, ended: bool) { + fn release(self: Box, gone: node::Gone) { self.1.fetch_add(1, Ordering::Relaxed); - if ended { - self.0.withdrawn(); - } + release(self.0, gone); + } +} + +/// Let go of a served claim as a session does, for the reason `gone` gives. +fn release(claim: crate::origin::Claim, gone: node::Gone) { + match gone { + node::Gone::Moved => drop(claim), + node::Gone::Lost => claim.retire(false), + node::Gone::Ended => claim.retire(true), + node::Gone::Withdrawn => claim.withdrawn(), } } @@ -376,6 +384,7 @@ impl RouteMesh { primary: served.primary, priority: Some(served.priority), above: u8::try_from(served.attr.above).unwrap_or(u8::MAX), + lifecycle: served.lifecycle, }; let route = crate::origin::RouteInfo::new() .with_instance(instance.clone(), served.attr.epoch) @@ -403,12 +412,10 @@ impl RouteMesh { origin: node, path, instance, - ended, + gone, } => { - if let Some(held) = up.served.remove(&(node, path, instance)) - && ended - { - held.claim.withdrawn(); + if let Some(held) = up.served.remove(&(node, path, instance)) { + release(held.claim, gone); } } } From 3469e58f9fd2ec42bdeea6617a8277c7b971acc2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 16:42:59 -0800 Subject: [PATCH 10/25] quest: weigh option D for a grant too wide to ask for, and recommend it Co-Authored-By: Claude Opus 5.5 --- quest/m2/cluster-routes/acceptance.md | 75 ++++++++++++++++++++++++--- 1 file changed, 67 insertions(+), 8 deletions(-) diff --git a/quest/m2/cluster-routes/acceptance.md b/quest/m2/cluster-routes/acceptance.md index 5b4b32c0675..03b6fa3d3f4 100644 --- a/quest/m2/cluster-routes/acceptance.md +++ b/quest/m2/cluster-routes/acceptance.md @@ -32,17 +32,76 @@ without that region's broadcast, and a reader of the region alone syncs empty. Fix it per the decision under Waiting, and turn the residual into the passing case. +What the publishers do today with an ask wider than the grant (traced in +code, then run on a server enforcing `r00/a/**` and `r01/b/**` with the +subscriber forced to ask for the root): + +- The ask is legal. The draft already says a prefix outside the + subscriber's scope "lists nothing and is SYNCED at once" + (`doc/draft/moq-lite.md`, Announce). Confining the ask to the grant is + only `coarsen`'s choice, from + [moq-dev/moq#8987](https://github.com/moq-dev/moq/pull/8987). Neither + publisher checks an ask against the grant: both filter each claim by its + path (`Answer::shows` in `lite/publisher/routed.rs`, `#changes` in + `js/net/src/lite/answer.ts`), and `Answer::poll_permit` re-filters every + claim the node told as the grant changes. The root ask got ANNOUNCE_OK, + SYNCED, both granted broadcasts, nothing outside the grant, and no close. +- A claim above a granted region is lost. A publisher presents a broader + claim at the requested prefix itself (`relay.rs`, "presented at the + prefix itself"). Asked for a region, a service claim at `r00` arrives at + `r00/a`, which the grant covers. Asked for the root, it would arrive at + `r00`, which the grant does not cover, so the filter drops it. The reader + syncs without a claim that serves its region. +- `js/net`'s subscriber already asks for the root once its readers' set + does not fit (`#wanted` in `js/net/src/lite/routed.ts`), ignoring the + grant. So a confined JS client with that many readers has this gap + against either publisher today. + ## Waiting - Maintainer: how a subscriber handles a grant too wide for one - ANNOUNCE_REQUEST. (A, recommended) Wire: a subscriber's set may pass one - message when each prefix past it is a region the publisher granted, so the - publisher's own grant bounds what it holds and the client discovers - everything; changes the lite-07-wip draft and both publishers, `rs/moq-net` - and `js/net`. (B) API: a reader of an unasked region learns the session - cannot answer it, a public change in Rust, JS, and the bindings. (C) - Neither: a reader overlapping an unasked region never syncs, a hang in - place of a wrong answer. Done once the choice is recorded in Plan. + ANNOUNCE_REQUEST. Recommended: D. + - (D) Ask wider, answer the granted part. `coarsen` falls back to the + regions' longest shared ancestor, up to the root, and never drops a + region. Draft: a requested prefix wider than the subscriber's scope is + answered as if each granted region inside it were asked: a claim above a + region is presented at that region, and SYNCED follows every region's + initial set. Add one ordering rule: a publisher answers with a grant + change before the ANNOUNCE_OK of the next edit it reads after the + AUTH_OK, so a subscriber whose grant widens under a wide ask can send an + empty edit and hold new readers until its ANNOUNCE_OK. `rs/moq-net`: + the publisher reads each granted region inside an ask through its own + node key, as it reads mounts, re-narrowed on each grant change. That + also keeps the publisher from reading ungranted subtrees, which bounds + its work and leaves SYNCED's timing blind to what it hides. The + subscriber's `coarsen` widens instead of dropping. `js/net`: the answer + presents a claim above a granted region at the region; the subscriber + already falls back to the root, and gains the widening sync. No message + field changes and no public API changes. It also fixes JS's existing + root fallback. + - (A) Exception to the size rule. A subscriber's set may pass one message + when each prefix past it is a region the publisher granted. Draft: the + exception, plus how both ends derive a grant's regions from its patterns + (`interest_prefixes`, wildcards included), plus what happens when a + narrowing (an AUTH_OK replacing the grant, an AUTH_ERROR, or a lapse) + crosses an edit already sent. The acceptor judges a request when it + reads it, so that edit names a region no longer granted, and as written + the publisher must close the session. A narrowing between edits leaves + the set past the limit until the subscriber's next edit removes the + region. `rs/moq-net`: the publisher checks each extra prefix against its + grant, and `coarsen` keeps every granted region. `js/net`: the same + check, and its subscriber, which knows nothing of the grant today, has + to ask by the grant's regions. + - (B) API: a reader of an unasked region learns the session cannot answer + it, a public change in Rust, JS, and the bindings. + - (C) Neither: a reader overlapping an unasked region never syncs, a hang + in place of a wrong answer. + + D over A: D keeps the one-message rule whole, never names a region on the + wire, so a narrowing cannot turn an honest edit into a violation, and + needs no shared rule for deriving regions. It costs a presentation rule + and the publisher's narrowing, which JS's root fallback needs anyway. Done + once the choice is recorded in Plan. ## Related From 861f189532989ed9e2f0d96a622538c36e3d519d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 17:10:03 -0800 Subject: [PATCH 11/25] quest: a wide grant is asked wider and answered in part Maintainer decision on moq-dev/moq#9112: option D. Co-Authored-By: Claude Fable 5.2 --- quest/m2/cluster-routes/acceptance.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/quest/m2/cluster-routes/acceptance.md b/quest/m2/cluster-routes/acceptance.md index 03b6fa3d3f4..d8bc7207ef4 100644 --- a/quest/m2/cluster-routes/acceptance.md +++ b/quest/m2/cluster-routes/acceptance.md @@ -57,10 +57,14 @@ subscriber forced to ask for the root): grant. So a confined JS client with that many readers has this gap against either publisher today. -## Waiting +Decided 2026-11-19 (maintainer, in the coordinating session, on +[moq-dev/moq#9112](https://github.com/moq-dev/moq/pull/9112)): ✅ D, ask +wider and answer the granted part (rejected: A, the size exception for +granted regions; B, the public API change; C, leaving the reader to hang). +It changes the lite-07-wip draft and both publishers, so it lands before +the cut. The options as they were put: -- Maintainer: how a subscriber handles a grant too wide for one - ANNOUNCE_REQUEST. Recommended: D. +- How a subscriber handles a grant too wide for one ANNOUNCE_REQUEST. - (D) Ask wider, answer the granted part. `coarsen` falls back to the regions' longest shared ancestor, up to the root, and never drops a region. Draft: a requested prefix wider than the subscriber's scope is From 9abc43e578bf50e2a53577b0ffb42f5f53d2198f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 21:26:52 -0800 Subject: [PATCH 12/25] fix(moq-net): a copy of an ended claim another grant brought never shows Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/relay.rs | 224 +++++++++++++++++++++++++++++++++- 1 file changed, 220 insertions(+), 4 deletions(-) diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index a92faf34b37..9e291d8e4ac 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -40,7 +40,11 @@ //! identity. A claim's chain names each relay it entered or left a routing domain by, the //! announcer, then each grant's exporter and adopter, and a node takes no claim whose //! chain, with it appended, names a relay twice or grows past the longest path: that claim -//! left a relay's domain and came back. +//! left a relay's domain and came back. Floors order one claimant's claims, not the copies +//! two grants announced again under two claimants, so a node that hears a claim's publisher +//! ended it holds back every copy of it standing then, the same announcer, first in the +//! chain, at no newer epoch, until each copy's own end arrives: no copy shows a moment +//! between the two ends. //! //! Each neighbour asks for a set of prefixes edited in place, as under the path rule: a //! routing neighbour forwards for the whole domain, so it asks for the whole tree, and @@ -413,6 +417,35 @@ pub(crate) fn precedence(item: &A) -> (u64, Reverse) { (item.above(), Reverse(item.epoch())) } +/// The relay that first announced `claim`, which `origin` claims: the first its chain +/// names, or `origin`'s relay for a claim that crossed no grant. +fn announcer(claim: &Claim, origin: NodeId) -> Id { + claim.chain.first().copied().unwrap_or(origin.relay) +} + +/// The copies of an ended claim that stood when its end arrived: past a grant, each was +/// announced again under another claimant, so its end comes later by its own way. Each +/// is held back until its holder ends it; a holder ends its copy before it claims again, +/// as a claimant's floors and a neighbour's stream order them, so a claim announced +/// again is never held back. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Ended { + /// The newest epoch ended. + epoch: u64, + /// The claimants whose claim is a copy. + origins: BTreeSet, + /// The neighbours whose given claim is a copy. + peers: BTreeSet, +} + +/// Who holds a copy of a claim at a node: a claimant it knows through a route, or a +/// neighbour that gave it across a grant. +#[derive(Clone, Copy, Debug)] +enum Holder { + Origin(NodeId), + Peer(N), +} + /// A broadcast: a path, and the Instance a claim there serves. pub(crate) type Broadcast = (Path, I); @@ -724,6 +757,9 @@ pub(crate) struct Router { revs: BTreeMap, /// Per broadcast, the claims each endpoint or scoped neighbour gave this node. givers: BTreeMap, BTreeMap>>, + /// Per broadcast, each announcer whose claim its publisher ended, with the copies of it + /// that other ways brought and still stand ([`Self::stale_copy`]). + withdrawn: BTreeMap, BTreeMap>>, /// The neighbour behind each of this node's own claims. sources: BTreeMap, N>, /// The forwarding nodes claiming each broadcast, by path. @@ -828,6 +864,7 @@ impl Router { known: BTreeMap::new(), revs: BTreeMap::new(), givers: BTreeMap::new(), + withdrawn: BTreeMap::new(), sources: BTreeMap::new(), claimants: BTreeMap::new(), ranked: BTreeMap::new(), @@ -1963,6 +2000,7 @@ impl Router { let mut claimed = Vec::new(); let mut unclaimed = Vec::new(); let mut lost = Vec::new(); + let mut ended = Vec::new(); for (broadcast, change) in &changes { match change { Change::Live(claim) => { @@ -1974,6 +2012,9 @@ impl Router { Change::Ended | Change::Lost => { if let Some(claim) = known.items.remove(broadcast) { unclaimed.push(broadcast.clone()); + if *change == Change::Ended { + ended.push((broadcast.clone(), announcer(&claim, origin), claim.item.epoch())); + } // `restate` keeps its own claim's source. if *change == Change::Lost && !own { lost.push((broadcast.clone(), claim)); @@ -2001,6 +2042,97 @@ impl Router { let changes = changes.clone(); self.out.push((peer, Msg::Claims { origin, changes, floor })); } + // The late end of a copy already held back says nothing new. + let ended: Vec<(Broadcast, Id, u64)> = (ended.into_iter()) + .filter(|(broadcast, announcer, _)| { + let held = self.withdrawn.get(broadcast).and_then(|held| held.get(announcer)); + !held.is_some_and(|ended| ended.origins.contains(&origin)) + }) + .collect(); + // A claimant that let go of its copy holds none. + for (broadcast, change) in &changes { + if !matches!(change, Change::Live(_)) { + self.unhold(broadcast, |ended| _ = ended.origins.remove(&origin)); + } + } + // Its publisher ended the claim: every copy of it standing now is stale, and this + // node's own, taken across a grant from one, ends with it. + for (broadcast, announcer, epoch) in ended { + if self.withdraw(&broadcast, announcer, epoch) && self.givers.contains_key(&broadcast) { + self.restate(broadcast.clone(), Change::Ended); + } + } + } + + /// Hold back each copy of `broadcast` standing now of `announcer`'s claim at no newer + /// than `epoch`, which its publisher ended. Returns whether any stands. + fn withdraw(&mut self, broadcast: &Broadcast, announcer: Id, epoch: u64) -> bool { + let copy = |by: Id, claim: &Claim| by == announcer && claim.item.epoch() <= epoch; + let (path, instance) = broadcast; + let origins: BTreeSet = (self.claimants.get(path).and_then(|claims| claims.get(instance))) + .into_iter() + .flatten() + .filter(|origin| { + let claim = self.known.get(origin).and_then(|known| known.items.get(broadcast)); + claim.is_some_and(|claim| copy(self::announcer(claim, **origin), claim)) + }) + .copied() + .collect(); + let peers: BTreeSet = (self.givers.get(broadcast).into_iter().flatten()) + .filter(|(_, claim)| claim.chain.first().is_some_and(|by| copy(*by, claim))) + .map(|(peer, _)| *peer) + .collect(); + self.count(origins.len() + peers.len()); + if origins.is_empty() && peers.is_empty() { + return false; + } + let held = self.withdrawn.entry(broadcast.clone()).or_default(); + let ended = held.entry(announcer).or_insert_with(|| Ended { + epoch, + origins: BTreeSet::new(), + peers: BTreeSet::new(), + }); + ended.epoch = ended.epoch.max(epoch); + ended.origins.extend(origins); + ended.peers.extend(peers); + self.dirty.insert(broadcast.clone()); + true + } + + /// A holder let go of its copy of an ended claim of `broadcast`, as `release` says: + /// forget each ended claim no copy of stands any more. + fn unhold(&mut self, broadcast: &Broadcast, release: impl Fn(&mut Ended)) { + let Some(held) = self.withdrawn.get_mut(broadcast) else { + return; + }; + held.retain(|_, ended| { + release(ended); + !(ended.origins.is_empty() && ended.peers.is_empty()) + }); + if held.is_empty() { + self.withdrawn.remove(broadcast); + } + } + + /// Whether `claim` of `broadcast`, held by `holder`, is a copy of a claim its publisher + /// ended that stood when the end arrived ([`Ended`]): it would show a moment before its + /// own end arrives by its own way. + fn stale_copy(&self, broadcast: &Broadcast, claim: &Claim, holder: Holder) -> bool { + let Some(held) = self.withdrawn.get(broadcast) else { + return false; + }; + let by = match holder { + Holder::Origin(origin) => Some(announcer(claim, origin)), + Holder::Peer(_) => claim.chain.first().copied(), + }; + let Some(ended) = by.and_then(|by| held.get(&by)) else { + return false; + }; + let holds = match holder { + Holder::Origin(origin) => ended.origins.contains(&origin), + Holder::Peer(peer) => ended.peers.contains(&peer), + }; + holds && claim.item.epoch() <= ended.epoch } fn claim(&mut self, broadcast: &Broadcast, origin: NodeId) { @@ -2171,6 +2303,7 @@ impl Router { self.givers.remove(broadcast); } } + self.unhold(broadcast, |ended| _ = ended.peers.remove(&peer)); } /// Announce the cheapest claim any neighbour gave this node for `broadcast`, or @@ -2178,6 +2311,7 @@ impl Router { /// the marked identity, and moves between the two as its source does. fn restate(&mut self, broadcast: Broadcast, cause: Change) { let cheapest = (self.givers.get(&broadcast).into_iter().flatten()) + .filter(|(peer, claim)| !self.stale_copy(&broadcast, claim, Holder::Peer(**peer))) .map(|(&peer, claim)| { self.visit(); // A claim from a neighbour that sent GOAWAY is draining from here on. @@ -2295,7 +2429,7 @@ impl Router { (origins.into_iter().flatten()).filter_map(move |&origin| { self.visit(); let claim = self.known.get(&origin)?.items.get(&broadcast)?; - Some((origin, claim)) + (!self.stale_copy(&broadcast, claim, Holder::Origin(origin))).then_some((origin, claim)) }) } @@ -2319,7 +2453,10 @@ impl Router { if self.is_self(origin) { for (peer, given) in self.givers.get(broadcast).into_iter().flatten() { self.visit(); - if precedence(&given.item) != precedence(&claim.item) || given.unavailable { + if precedence(&given.item) != precedence(&claim.item) + || given.unavailable + || self.stale_copy(broadcast, given, Holder::Peer(*peer)) + { continue; } let mut given = given.clone(); @@ -2406,7 +2543,9 @@ impl Router { self.visit(); let (path, instance) = broadcast; let closest = self.closest(broadcast).map(|(_, claim)| claim.item.clone()); - let claimed = (self.claimants.get(path)).is_some_and(|claims| claims.contains_key(instance)); + // A copy of a claim its publisher ended claims nothing. + let claimed = (self.claimants.get(path)).is_some_and(|claims| claims.contains_key(instance)) + && self.claims_of(broadcast).next().is_some(); let shown = match claimed { true => { self.unwait(broadcast); @@ -2797,6 +2936,7 @@ impl Router { known, revs, givers: _, + withdrawn: _, sources, claimants: _, ranked: _, @@ -3692,6 +3832,82 @@ mod tests { } } + /// Two grants announce one publisher's claim again under two claimants, so the two + /// copies reach a node as different claims. Once its publisher ends the claim the + /// node hears of first, the other copy, the same announcer at no newer epoch, is never + /// shown, nor this node's own taken from it; a newer epoch from that announcer is. + #[test] + fn a_copy_of_an_ended_claim_is_never_shown() { + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Routing(node(2))); + open(&mut router, 4, Role::Routing(node(4))); + open(&mut router, 5, Role::Scoped(vec![p(ROOT)])); + router.recv(2, route(2, 2, 1)).unwrap(); + router.recv(4, route(4, 4, 2)).unwrap(); + let copy = |chain: Vec, epoch: u64| { + let Change::Live(claim) = chained(chain) else { + unreachable!("a live claim"); + }; + Change::Live(Claim { + item: run(epoch), + ..claim + }) + }; + let broadcast = (p("live"), 7); + let claims = |origin: Id, change: Change, floor: Rev| Msg::Claims { + origin: node(origin), + changes: vec![(broadcast.clone(), change)], + floor, + }; + router.recv(2, claims(2, copy(vec![9, 2], 0), 1)).unwrap(); + router.recv(4, claims(4, copy(vec![9, 4], 0), 1)).unwrap(); + let given = vec![(broadcast.clone(), copy(vec![9, 8], 0))]; + router.recv(5, Msg::Announce { changes: given }).unwrap(); + let shown = |router: &Router| router.shown().contains_key(&(7, p("live"))); + assert!(shown(&router)); + + router.recv(2, claims(2, Change::Ended, 2)).unwrap(); + assert!(!shown(&router), "the copies are of the ended claim"); + assert!(router.own().1.is_empty(), "nor is the node's own taken from one"); + + router.recv(4, claims(4, copy(vec![9, 4], 1), 2)).unwrap(); + assert!(shown(&router), "a newer epoch is a new claim"); + } + + /// A publisher that withdraws its claim and announces it again at the same epoch, as + /// `origin::Route::withdraw` and `announce` do, is shown at once: only the copies that + /// stood when the end arrived are held back. + #[test] + fn a_claim_announced_again_at_the_same_epoch_shows_at_once() { + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Routing(node(2))); + open(&mut router, 3, Role::Scoped(vec![p(ROOT)])); + router.recv(2, route(2, 2, 1)).unwrap(); + let broadcast = (p("live"), 7); + let given = |change: Change| Msg::Announce { + changes: vec![(broadcast.clone(), change)], + }; + let claims = |change: Change, floor: Rev| Msg::Claims { + origin: node(2), + changes: vec![(broadcast.clone(), change)], + floor, + }; + router.recv(3, given(chained(vec![9]))).unwrap(); + router.recv(2, claims(chained(vec![9, 2]), 1)).unwrap(); + let own = |router: &Router| router.own().1.contains_key(&broadcast); + assert!(own(&router)); + + router.recv(3, given(Change::Ended)).unwrap(); + assert!( + !router.shown().contains_key(&(7, p("live"))), + "node 2's copy is of the ended claim" + ); + router.recv(3, given(chained(vec![9]))).unwrap(); + assert!(own(&router), "announced again, the claim is no copy"); + router.recv(2, claims(Change::Ended, 2)).unwrap(); + assert!(own(&router)); + } + /// A relay never tells a claim whose chain its own Relay would carry to the longest /// path, which no node could send. #[test] From 354d1e1d846c9e62ff9d9e3a614841e9bfaa37b4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 22:35:21 -0800 Subject: [PATCH 13/25] fix(moq-net): a held copy goes with its holder's route, and a newer end is never mistaken for an old copy's Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/relay.rs | 88 ++++++++++++++++++++++++++++++++--- 1 file changed, 82 insertions(+), 6 deletions(-) diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index 9e291d8e4ac..6dc39bd8f44 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -1536,6 +1536,7 @@ impl Router { self.count(known.items.len()); for (broadcast, claim) in known.items { self.unclaim(&broadcast, origin); + self.unhold(&broadcast, |ended| _ = ended.origins.remove(&origin)); self.dirty.insert(broadcast.clone()); self.drop_claim(broadcast, origin, claim, None); } @@ -2044,15 +2045,23 @@ impl Router { } // The late end of a copy already held back says nothing new. let ended: Vec<(Broadcast, Id, u64)> = (ended.into_iter()) - .filter(|(broadcast, announcer, _)| { + .filter(|(broadcast, announcer, epoch)| { let held = self.withdrawn.get(broadcast).and_then(|held| held.get(announcer)); - !held.is_some_and(|ended| ended.origins.contains(&origin)) + !held.is_some_and(|ended| ended.origins.contains(&origin) && *epoch <= ended.epoch) }) .collect(); - // A claimant that let go of its copy holds none. + // A claimant that let go of its copy, or claims a newer epoch, holds none. for (broadcast, change) in &changes { - if !matches!(change, Change::Live(_)) { - self.unhold(broadcast, |ended| _ = ended.origins.remove(&origin)); + match change { + Change::Live(claim) => { + let epoch = claim.item.epoch(); + self.unhold(broadcast, |ended| { + if epoch > ended.epoch { + ended.origins.remove(&origin); + } + }); + } + _ => self.unhold(broadcast, |ended| _ = ended.origins.remove(&origin)), } } // Its publisher ended the claim: every copy of it standing now is stale, and this @@ -2282,7 +2291,14 @@ impl Router { match change { Change::Live(claim) => { peer.given.insert(broadcast.clone(), claim.clone()); + let epoch = claim.item.epoch(); self.givers.entry(broadcast.clone()).or_default().insert(from, claim); + // A neighbour that gives a newer epoch holds no copy of an older end. + self.unhold(&broadcast, |ended| { + if epoch > ended.epoch { + ended.peers.remove(&from); + } + }); self.restate(broadcast, Change::Lost); } cause => { @@ -2936,7 +2952,7 @@ impl Router { known, revs, givers: _, - withdrawn: _, + withdrawn, sources, claimants: _, ranked: _, @@ -2969,6 +2985,7 @@ impl Router { (me, forwards, debounce, sessions, peers, routing, fresh, heard, unnamed), (dead, closed, draining, stale, best, marks, known, revs), (sources, ended, dirty, waiting, since, gone, reader.shown(), out), + withdrawn, ); super::digest(hasher, &state); } @@ -3037,6 +3054,31 @@ impl Router { assert_eq!(self.tx_hops, tx_hops, "sent hops"); assert_eq!(self.sent_to, sent_to, "sent routes"); assert_eq!(self.givers, givers, "given claims"); + // Each holder of an ended claim's copy still holds one. + for (broadcast, held) in &self.withdrawn { + assert!(!held.is_empty(), "an empty withdrawal"); + for (announcer, ended) in held { + assert!( + !(ended.origins.is_empty() && ended.peers.is_empty()), + "a withdrawal no copy holds" + ); + let copy = |by: Id, claim: &Claim| by == *announcer && claim.item.epoch() <= ended.epoch; + for origin in &ended.origins { + let claim = self.known.get(origin).and_then(|known| known.items.get(broadcast)); + assert!( + claim.is_some_and(|claim| copy(self::announcer(claim, *origin), claim)), + "held by {origin:?}" + ); + } + for peer in &ended.peers { + let claim = givers.get(broadcast).and_then(|claims| claims.get(peer)); + assert!( + claim.is_some_and(|claim| claim.chain.first().is_some_and(|by| copy(*by, claim))), + "held by {peer:?}" + ); + } + } + } assert_eq!(self.askers, askers, "askers"); let mut claimants: BTreeMap>> = BTreeMap::new(); for (origin, known) in &self.known { @@ -3872,6 +3914,40 @@ mod tests { router.recv(4, claims(4, copy(vec![9, 4], 1), 2)).unwrap(); assert!(shown(&router), "a newer epoch is a new claim"); + + // That newer claim ends too: node 4 held a copy of the older end, which says + // nothing of the newer one, and node 2's copy of it is held back. + router.recv(2, claims(2, copy(vec![9, 2], 1), 3)).unwrap(); + router.recv(4, claims(4, Change::Ended, 3)).unwrap(); + assert!(!shown(&router), "node 2's copy is of the newer ended claim"); + } + + /// A claimant whose last route goes holds no copy of an ended claim any more, so one + /// it claims again once it returns, at the same epoch, shows. + #[test] + fn a_holder_that_returns_claims_again() { + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Routing(node(2))); + open(&mut router, 4, Role::Routing(node(4))); + router.recv(2, route(2, 2, 1)).unwrap(); + router.recv(4, route(4, 4, 2)).unwrap(); + let broadcast = (p("live"), 7); + let claims = |origin: Id, change: Change, floor: Rev| Msg::Claims { + origin: node(origin), + changes: vec![(broadcast.clone(), change)], + floor, + }; + router.recv(2, claims(2, chained(vec![9, 2]), 1)).unwrap(); + router.recv(4, claims(4, chained(vec![9, 4]), 1)).unwrap(); + router.recv(2, claims(2, Change::Ended, 2)).unwrap(); + let shown = |router: &Router| router.shown().contains_key(&(7, p("live"))); + assert!(!shown(&router), "node 4's copy is of the ended claim"); + + router.disconnect(4); + open(&mut router, 4, Role::Routing(node(4))); + router.recv(4, route(4, 4, 2)).unwrap(); + router.recv(4, claims(4, chained(vec![9, 4]), 1)).unwrap(); + assert!(shown(&router), "claimed again after it let go"); } /// A publisher that withdraws its claim and announces it again at the same epoch, as From b5e608f7c0ec418873cd12dcec27361f01f233ac Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 23:33:16 -0800 Subject: [PATCH 14/25] fix(moq-net): a holder that moves to another announcer's claim holds no copy Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/relay.rs | 38 ++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index 6dc39bd8f44..5e51c52434f 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -1536,7 +1536,7 @@ impl Router { self.count(known.items.len()); for (broadcast, claim) in known.items { self.unclaim(&broadcast, origin); - self.unhold(&broadcast, |ended| _ = ended.origins.remove(&origin)); + self.unhold(&broadcast, |_, ended| _ = ended.origins.remove(&origin)); self.dirty.insert(broadcast.clone()); self.drop_claim(broadcast, origin, claim, None); } @@ -2050,18 +2050,19 @@ impl Router { !held.is_some_and(|ended| ended.origins.contains(&origin) && *epoch <= ended.epoch) }) .collect(); - // A claimant that let go of its copy, or claims a newer epoch, holds none. + // A claimant that let go of its copy, or now claims another announcer's claim or a + // newer epoch, holds none. for (broadcast, change) in &changes { match change { Change::Live(claim) => { - let epoch = claim.item.epoch(); - self.unhold(broadcast, |ended| { - if epoch > ended.epoch { + let (by, epoch) = (announcer(claim, origin), claim.item.epoch()); + self.unhold(broadcast, |announcer, ended| { + if by != announcer || epoch > ended.epoch { ended.origins.remove(&origin); } }); } - _ => self.unhold(broadcast, |ended| _ = ended.origins.remove(&origin)), + _ => self.unhold(broadcast, |_, ended| _ = ended.origins.remove(&origin)), } } // Its publisher ended the claim: every copy of it standing now is stale, and this @@ -2110,12 +2111,12 @@ impl Router { /// A holder let go of its copy of an ended claim of `broadcast`, as `release` says: /// forget each ended claim no copy of stands any more. - fn unhold(&mut self, broadcast: &Broadcast, release: impl Fn(&mut Ended)) { + fn unhold(&mut self, broadcast: &Broadcast, release: impl Fn(Id, &mut Ended)) { let Some(held) = self.withdrawn.get_mut(broadcast) else { return; }; - held.retain(|_, ended| { - release(ended); + held.retain(|announcer, ended| { + release(*announcer, ended); !(ended.origins.is_empty() && ended.peers.is_empty()) }); if held.is_empty() { @@ -2291,11 +2292,12 @@ impl Router { match change { Change::Live(claim) => { peer.given.insert(broadcast.clone(), claim.clone()); - let epoch = claim.item.epoch(); + let (by, epoch) = (claim.chain.first().copied(), claim.item.epoch()); self.givers.entry(broadcast.clone()).or_default().insert(from, claim); - // A neighbour that gives a newer epoch holds no copy of an older end. - self.unhold(&broadcast, |ended| { - if epoch > ended.epoch { + // A neighbour that gives another announcer's claim, or a newer epoch, holds + // no copy of an older end. + self.unhold(&broadcast, |announcer, ended| { + if by != Some(announcer) || epoch > ended.epoch { ended.peers.remove(&from); } }); @@ -2319,7 +2321,7 @@ impl Router { self.givers.remove(broadcast); } } - self.unhold(broadcast, |ended| _ = ended.peers.remove(&peer)); + self.unhold(broadcast, |_, ended| _ = ended.peers.remove(&peer)); } /// Announce the cheapest claim any neighbour gave this node for `broadcast`, or @@ -3943,6 +3945,14 @@ mod tests { let shown = |router: &Router| router.shown().contains_key(&(7, p("live"))); assert!(!shown(&router), "node 4's copy is of the ended claim"); + // Moving to another announcer's claim and back at the same epoch: no copy any more. + router.recv(4, claims(4, chained(vec![10, 4]), 2)).unwrap(); + router.recv(4, claims(4, chained(vec![9, 4]), 3)).unwrap(); + assert!(shown(&router), "node 4 let go of its copy"); + router.recv(2, claims(2, chained(vec![9, 2]), 3)).unwrap(); + router.recv(2, claims(2, Change::Ended, 4)).unwrap(); + assert!(!shown(&router), "node 4 holds a copy again"); + router.disconnect(4); open(&mut router, 4, Role::Routing(node(4))); router.recv(4, route(4, 4, 2)).unwrap(); From 554aa756fc720474c00dc3739265fad7d50cf7bc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 23:47:30 -0800 Subject: [PATCH 15/25] fix(moq-net): an ended claim holds back only its own copies, and a lagging copy's skipped revisions release its holds A record of an ended claim now keys on its announcer and its distance above the path, so ending an exact claim leaves the same announcer's broader claim presented there. A copy that lagged and overtook skips revisions that can hide an end and the same claim announced again, so it releases every hold of its origin. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/relay.rs | 153 +++++++++++++++++++++++++++------- 1 file changed, 124 insertions(+), 29 deletions(-) diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index 5e51c52434f..b4bd9165830 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -423,6 +423,21 @@ fn announcer(claim: &Claim, origin: NodeId) -> Id { claim.chain.first().copied().unwrap_or(origin.relay) } +/// Which claim a copy is of: its [`announcer`], and how far above the broadcast's path +/// the claim sits ([`Item::above`]). One announcer can stand at a path with an exact +/// claim and a broader one presented there, and ending one leaves the other. +type Of = (Id, u64); + +/// What `claim`, which `origin` claims, is a copy of. +fn of(claim: &Claim, origin: NodeId) -> Of { + (announcer(claim, origin), claim.item.above()) +} + +/// What `claim`, which a neighbour gave, is a copy of: none for one with no chain. +fn given_of(claim: &Claim) -> Option { + claim.chain.first().map(|by| (*by, claim.item.above())) +} + /// The copies of an ended claim that stood when its end arrived: past a grant, each was /// announced again under another claimant, so its end comes later by its own way. Each /// is held back until its holder ends it; a holder ends its copy before it claims again, @@ -757,9 +772,9 @@ pub(crate) struct Router { revs: BTreeMap, /// Per broadcast, the claims each endpoint or scoped neighbour gave this node. givers: BTreeMap, BTreeMap>>, - /// Per broadcast, each announcer whose claim its publisher ended, with the copies of it - /// that other ways brought and still stand ([`Self::stale_copy`]). - withdrawn: BTreeMap, BTreeMap>>, + /// Per broadcast, each claim ([`Of`]) its publisher ended, with the copies of it that + /// other ways brought and still stand ([`Self::stale_copy`]). + withdrawn: BTreeMap, BTreeMap>>, /// The neighbour behind each of this node's own claims. sources: BTreeMap, N>, /// The forwarding nodes claiming each broadcast, by path. @@ -1938,6 +1953,7 @@ impl Router { return Ok(()); } let mut visited = 0; + let mut lagged = false; let changes = match was == known.floor || mutated!(self, Lagging) { // The copy matched this node's at the same revision, so the message's changes // bring it to the new one. @@ -1964,6 +1980,7 @@ impl Router { }), ); visited = heard.items.len() + known.items.len(); + lagged = true; diff } }; @@ -1971,6 +1988,11 @@ impl Router { let known = self.known.get_mut(&origin).expect("known"); known.floor = known.floor.max(floor); self.publish(origin, changes); + // The revisions a lagging copy skipped can hide an end and the same claim announced + // again, which differ in nothing: `origin` holds no copy it held before them. + if lagged { + self.unhold_origin(origin); + } Ok(()) } @@ -2014,7 +2036,7 @@ impl Router { if let Some(claim) = known.items.remove(broadcast) { unclaimed.push(broadcast.clone()); if *change == Change::Ended { - ended.push((broadcast.clone(), announcer(&claim, origin), claim.item.epoch())); + ended.push((broadcast.clone(), of(&claim, origin), claim.item.epoch())); } // `restate` keeps its own claim's source. if *change == Change::Lost && !own { @@ -2044,9 +2066,9 @@ impl Router { self.out.push((peer, Msg::Claims { origin, changes, floor })); } // The late end of a copy already held back says nothing new. - let ended: Vec<(Broadcast, Id, u64)> = (ended.into_iter()) - .filter(|(broadcast, announcer, epoch)| { - let held = self.withdrawn.get(broadcast).and_then(|held| held.get(announcer)); + let ended: Vec<(Broadcast, Of, u64)> = (ended.into_iter()) + .filter(|(broadcast, of, epoch)| { + let held = self.withdrawn.get(broadcast).and_then(|held| held.get(of)); !held.is_some_and(|ended| ended.origins.contains(&origin) && *epoch <= ended.epoch) }) .collect(); @@ -2055,9 +2077,9 @@ impl Router { for (broadcast, change) in &changes { match change { Change::Live(claim) => { - let (by, epoch) = (announcer(claim, origin), claim.item.epoch()); - self.unhold(broadcast, |announcer, ended| { - if by != announcer || epoch > ended.epoch { + let (by, epoch) = (of(claim, origin), claim.item.epoch()); + self.unhold(broadcast, |of, ended| { + if by != of || epoch > ended.epoch { ended.origins.remove(&origin); } }); @@ -2067,8 +2089,8 @@ impl Router { } // Its publisher ended the claim: every copy of it standing now is stale, and this // node's own, taken across a grant from one, ends with it. - for (broadcast, announcer, epoch) in ended { - if self.withdraw(&broadcast, announcer, epoch) && self.givers.contains_key(&broadcast) { + for (broadcast, of, epoch) in ended { + if self.withdraw(&broadcast, of, epoch) && self.givers.contains_key(&broadcast) { self.restate(broadcast.clone(), Change::Ended); } } @@ -2076,20 +2098,20 @@ impl Router { /// Hold back each copy of `broadcast` standing now of `announcer`'s claim at no newer /// than `epoch`, which its publisher ended. Returns whether any stands. - fn withdraw(&mut self, broadcast: &Broadcast, announcer: Id, epoch: u64) -> bool { - let copy = |by: Id, claim: &Claim| by == announcer && claim.item.epoch() <= epoch; + fn withdraw(&mut self, broadcast: &Broadcast, ended: Of, epoch: u64) -> bool { + let copy = |by: Of, claim: &Claim| by == ended && claim.item.epoch() <= epoch; let (path, instance) = broadcast; let origins: BTreeSet = (self.claimants.get(path).and_then(|claims| claims.get(instance))) .into_iter() .flatten() .filter(|origin| { let claim = self.known.get(origin).and_then(|known| known.items.get(broadcast)); - claim.is_some_and(|claim| copy(self::announcer(claim, **origin), claim)) + claim.is_some_and(|claim| copy(of(claim, **origin), claim)) }) .copied() .collect(); let peers: BTreeSet = (self.givers.get(broadcast).into_iter().flatten()) - .filter(|(_, claim)| claim.chain.first().is_some_and(|by| copy(*by, claim))) + .filter(|(_, claim)| given_of(claim).is_some_and(|by| copy(by, claim))) .map(|(peer, _)| *peer) .collect(); self.count(origins.len() + peers.len()); @@ -2097,7 +2119,7 @@ impl Router { return false; } let held = self.withdrawn.entry(broadcast.clone()).or_default(); - let ended = held.entry(announcer).or_insert_with(|| Ended { + let ended = held.entry(ended).or_insert_with(|| Ended { epoch, origins: BTreeSet::new(), peers: BTreeSet::new(), @@ -2111,12 +2133,12 @@ impl Router { /// A holder let go of its copy of an ended claim of `broadcast`, as `release` says: /// forget each ended claim no copy of stands any more. - fn unhold(&mut self, broadcast: &Broadcast, release: impl Fn(Id, &mut Ended)) { + fn unhold(&mut self, broadcast: &Broadcast, release: impl Fn(Of, &mut Ended)) { let Some(held) = self.withdrawn.get_mut(broadcast) else { return; }; - held.retain(|announcer, ended| { - release(*announcer, ended); + held.retain(|of, ended| { + release(*of, ended); !(ended.origins.is_empty() && ended.peers.is_empty()) }); if held.is_empty() { @@ -2124,6 +2146,19 @@ impl Router { } } + /// `origin` holds no copy of any ended claim. + fn unhold_origin(&mut self, origin: NodeId) { + let held: Vec> = (self.withdrawn.iter()) + .filter(|(_, held)| held.values().any(|ended| ended.origins.contains(&origin))) + .map(|(broadcast, _)| broadcast.clone()) + .collect(); + self.count(held.len()); + for broadcast in held { + self.unhold(&broadcast, |_, ended| _ = ended.origins.remove(&origin)); + self.dirty.insert(broadcast); + } + } + /// Whether `claim` of `broadcast`, held by `holder`, is a copy of a claim its publisher /// ended that stood when the end arrived ([`Ended`]): it would show a moment before its /// own end arrives by its own way. @@ -2132,8 +2167,8 @@ impl Router { return false; }; let by = match holder { - Holder::Origin(origin) => Some(announcer(claim, origin)), - Holder::Peer(_) => claim.chain.first().copied(), + Holder::Origin(origin) => Some(of(claim, origin)), + Holder::Peer(_) => given_of(claim), }; let Some(ended) = by.and_then(|by| held.get(&by)) else { return false; @@ -2292,12 +2327,12 @@ impl Router { match change { Change::Live(claim) => { peer.given.insert(broadcast.clone(), claim.clone()); - let (by, epoch) = (claim.chain.first().copied(), claim.item.epoch()); + let (by, epoch) = (given_of(&claim), claim.item.epoch()); self.givers.entry(broadcast.clone()).or_default().insert(from, claim); // A neighbour that gives another announcer's claim, or a newer epoch, holds // no copy of an older end. - self.unhold(&broadcast, |announcer, ended| { - if by != Some(announcer) || epoch > ended.epoch { + self.unhold(&broadcast, |of, ended| { + if by != Some(of) || epoch > ended.epoch { ended.peers.remove(&from); } }); @@ -3059,23 +3094,23 @@ impl Router { // Each holder of an ended claim's copy still holds one. for (broadcast, held) in &self.withdrawn { assert!(!held.is_empty(), "an empty withdrawal"); - for (announcer, ended) in held { + for (ended_of, ended) in held { assert!( !(ended.origins.is_empty() && ended.peers.is_empty()), "a withdrawal no copy holds" ); - let copy = |by: Id, claim: &Claim| by == *announcer && claim.item.epoch() <= ended.epoch; + let copy = |by: Of, claim: &Claim| by == *ended_of && claim.item.epoch() <= ended.epoch; for origin in &ended.origins { let claim = self.known.get(origin).and_then(|known| known.items.get(broadcast)); assert!( - claim.is_some_and(|claim| copy(self::announcer(claim, *origin), claim)), + claim.is_some_and(|claim| copy(of(claim, *origin), claim)), "held by {origin:?}" ); } for peer in &ended.peers { let claim = givers.get(broadcast).and_then(|claims| claims.get(peer)); assert!( - claim.is_some_and(|claim| claim.chain.first().is_some_and(|by| copy(*by, claim))), + claim.is_some_and(|claim| given_of(claim).is_some_and(|by| copy(by, claim))), "held by {peer:?}" ); } @@ -3960,6 +3995,66 @@ mod tests { assert!(shown(&router), "claimed again after it let go"); } + /// A claimant's copy that a newer route's first claims overtake holds no copy any more: + /// the revisions it skipped can hide an end and the same claim announced again, which + /// the claims alone cannot tell from no change at all. + #[test] + fn a_holder_overtaken_by_a_newer_route_claims_again() { + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Routing(node(2))); + open(&mut router, 4, Role::Routing(node(4))); + open(&mut router, 5, Role::Routing(node(5))); + router.recv(2, route(2, 2, 1)).unwrap(); + router.recv(4, route(4, 4, 2)).unwrap(); + router.recv(5, route(4, 5, 3)).unwrap(); + let broadcast = (p("live"), 7); + let claims = |origin: Id, change: Change, floor: Rev| Msg::Claims { + origin: node(origin), + changes: vec![(broadcast.clone(), change)], + floor, + }; + router.recv(2, claims(2, chained(vec![9, 2]), 1)).unwrap(); + router.recv(4, claims(4, chained(vec![9, 4]), 1)).unwrap(); + router.recv(2, claims(2, Change::Ended, 2)).unwrap(); + let shown = |router: &Router| router.shown().contains_key(&(7, p("live"))); + assert!(!shown(&router), "node 4's copy is of the ended claim"); + + // Node 4 ended its copy at 2 and claimed it again at 3; the route through 5 brings 3 + // before the route through 4 brings 2. + router.recv(5, claims(4, chained(vec![9, 4]), 3)).unwrap(); + assert!(shown(&router), "node 4 may have claimed it again"); + } + + /// Ending an announcer's exact claim leaves its broader claim presented at the same + /// path: that one is no copy of what ended. + #[test] + fn ending_an_exact_claim_leaves_a_broader_one() { + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Routing(node(2))); + open(&mut router, 4, Role::Routing(node(4))); + router.recv(2, route(2, 2, 1)).unwrap(); + router.recv(4, route(4, 4, 2)).unwrap(); + let broadcast = (p("live"), 7); + let claims = |origin: Id, change: Change, floor: Rev| Msg::Claims { + origin: node(origin), + changes: vec![(broadcast.clone(), change)], + floor, + }; + let Change::Live(broader) = chained(vec![9, 4]) else { + unreachable!("a live claim"); + }; + let broader = Claim { + item: Run { epoch: 0, above: 1 }, + ..broader + }; + router.recv(2, claims(2, chained(vec![9, 2]), 1)).unwrap(); + router.recv(4, claims(4, Change::Live(broader), 1)).unwrap(); + router.recv(2, claims(2, Change::Ended, 2)).unwrap(); + let shown = router.shown(); + let shown = shown.get(&(7, p("live"))).expect("the broader claim shows"); + assert_eq!(shown.item.above, 1); + } + /// A publisher that withdraws its claim and announces it again at the same epoch, as /// `origin::Route::withdraw` and `announce` do, is shown at once: only the copies that /// stood when the end arrived are held back. From 87c853ee3a1db410e73fcb7aeac67b9e0e8e2e68 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 00:35:29 -0800 Subject: [PATCH 16/25] fix(moq-net): a lagging copy releases holds only past two skipped revisions One message changes each claim at most once, so an end and the same claim announced again take two revisions: a lagging copy one revision ahead still shows a held copy unchanged. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/relay.rs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index b4bd9165830..073133be3f8 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -1980,7 +1980,8 @@ impl Router { }), ); visited = heard.items.len() + known.items.len(); - lagged = true; + // An end and the same claim announced again take two revisions. + lagged = floor.saturating_sub(known.floor) >= 2; diff } }; @@ -1988,8 +1989,8 @@ impl Router { let known = self.known.get_mut(&origin).expect("known"); known.floor = known.floor.max(floor); self.publish(origin, changes); - // The revisions a lagging copy skipped can hide an end and the same claim announced - // again, which differ in nothing: `origin` holds no copy it held before them. + // Two or more revisions a lagging copy skipped can hide an end and the same claim + // announced again, which differ in nothing: `origin` holds no copy it held before. if lagged { self.unhold_origin(origin); } @@ -4019,9 +4020,16 @@ mod tests { let shown = |router: &Router| router.shown().contains_key(&(7, p("live"))); assert!(!shown(&router), "node 4's copy is of the ended claim"); - // Node 4 ended its copy at 2 and claimed it again at 3; the route through 5 brings 3 - // before the route through 4 brings 2. - router.recv(5, claims(4, chained(vec![9, 4]), 3)).unwrap(); + // One skipped revision changes each claim once at most: an unchanged copy is still + // the one held. + router.recv(5, claims(4, chained(vec![9, 4]), 2)).unwrap(); + assert!(!shown(&router), "one revision cannot end and claim again"); + + // Node 4 ended its copy at 3 and claimed it again at 4; the route through 6 brings 4 + // before the route through 4 brings 3. + open(&mut router, 6, Role::Routing(node(6))); + router.recv(6, route(4, 6, 4)).unwrap(); + router.recv(6, claims(4, chained(vec![9, 4]), 4)).unwrap(); assert!(shown(&router), "node 4 may have claimed it again"); } From 6bc49fd5d2e8976b965706998d121059d792ee47 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 01:12:12 -0800 Subject: [PATCH 17/25] fix(moq-net): an ended claim holds back its copies at every path they are presented A scoped neighbour can be given a claim lowered to a deeper path, so a copy of an ended claim can stand below it. The publisher's end now holds back each copy at or below the claim's own prefix, lowered by as many segments, and ends it there. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/relay.rs | 135 +++++++++++++++++++++++++--------- 1 file changed, 99 insertions(+), 36 deletions(-) diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index 073133be3f8..d2146e620dc 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -42,9 +42,9 @@ //! chain, with it appended, names a relay twice or grows past the longest path: that claim //! left a relay's domain and came back. Floors order one claimant's claims, not the copies //! two grants announced again under two claimants, so a node that hears a claim's publisher -//! ended it holds back every copy of it standing then, the same announcer, first in the -//! chain, at no newer epoch, until each copy's own end arrives: no copy shows a moment -//! between the two ends. +//! ended it holds back every copy of it standing then (the same announcer, first in the +//! chain, from the same prefix, at no newer epoch, at whatever path it is presented) +//! until each copy's own end arrives: no copy shows a moment between the two ends. //! //! Each neighbour asks for a set of prefixes edited in place, as under the path rule: a //! routing neighbour forwards for the whole domain, so it asks for the whole tree, and @@ -2091,45 +2091,67 @@ impl Router { // Its publisher ended the claim: every copy of it standing now is stale, and this // node's own, taken across a grant from one, ends with it. for (broadcast, of, epoch) in ended { - if self.withdraw(&broadcast, of, epoch) && self.givers.contains_key(&broadcast) { - self.restate(broadcast.clone(), Change::Ended); + for held in self.withdraw(&broadcast, of, epoch) { + if self.givers.contains_key(&held) { + self.restate(held, Change::Ended); + } } } } - /// Hold back each copy of `broadcast` standing now of `announcer`'s claim at no newer - /// than `epoch`, which its publisher ended. Returns whether any stands. - fn withdraw(&mut self, broadcast: &Broadcast, ended: Of, epoch: u64) -> bool { - let copy = |by: Of, claim: &Claim| by == ended && claim.item.epoch() <= epoch; + /// Hold back each copy standing now of the claim of `broadcast` its publisher ended, + /// `ended`, at no newer than `epoch`, at every path a copy is presented at: at or below + /// the claim's own prefix, lowered by as many segments. Returns the broadcasts where + /// any stands. + fn withdraw(&mut self, broadcast: &Broadcast, ended: Of, epoch: u64) -> Vec> { let (path, instance) = broadcast; - let origins: BTreeSet = (self.claimants.get(path).and_then(|claims| claims.get(instance))) - .into_iter() - .flatten() - .filter(|origin| { - let claim = self.known.get(origin).and_then(|known| known.items.get(broadcast)); - claim.is_some_and(|claim| copy(of(claim, **origin), claim)) - }) - .copied() - .collect(); - let peers: BTreeSet = (self.givers.get(broadcast).into_iter().flatten()) - .filter(|(_, claim)| given_of(claim).is_some_and(|by| copy(by, claim))) - .map(|(peer, _)| *peer) - .collect(); - self.count(origins.len() + peers.len()); - if origins.is_empty() && peers.is_empty() { - return false; + let (announcer, above) = ended; + let depth = path.parts().count(); + let Some(base) = usize::try_from(above).ok().and_then(|above| depth.checked_sub(above)) else { + return Vec::new(); + }; + let own = path.head(base).expect("a head no deeper than the path"); + // A copy a neighbour gives is this node's own claim there, unless already held. + let mut paths: BTreeSet = below(&self.claimants, &own).map(|(path, _)| path.clone()).collect(); + paths.extend([own, path.clone()]); + self.count(paths.len()); + let mut held = Vec::new(); + for at in paths { + let of = (announcer, (at.parts().count() - base) as u64); + let copy = |by: Of, claim: &Claim| by == of && claim.item.epoch() <= epoch; + let here = (at, instance.clone()); + let origins: BTreeSet = (self.claimants.get(&here.0).and_then(|claims| claims.get(instance))) + .into_iter() + .flatten() + .filter(|origin| { + let claim = self.known.get(origin).and_then(|known| known.items.get(&here)); + claim.is_some_and(|claim| copy(self::of(claim, **origin), claim)) + }) + .copied() + .collect(); + let peers: BTreeSet = (self.givers.get(&here).into_iter().flatten()) + .filter(|(_, claim)| given_of(claim).is_some_and(|by| copy(by, claim))) + .map(|(peer, _)| *peer) + .collect(); + self.count(origins.len() + peers.len()); + if origins.is_empty() && peers.is_empty() { + continue; + } + let record = self.withdrawn.entry(here.clone()).or_default(); + let record = record.entry(of).or_insert_with(|| Ended { + epoch, + origins: BTreeSet::new(), + peers: BTreeSet::new(), + }); + record.epoch = record.epoch.max(epoch); + record.origins.extend(origins); + record.peers.extend(peers); + // Its publisher ended it, wherever a copy was presented. + self.ended.insert(here.clone()); + self.dirty.insert(here.clone()); + held.push(here); } - let held = self.withdrawn.entry(broadcast.clone()).or_default(); - let ended = held.entry(ended).or_insert_with(|| Ended { - epoch, - origins: BTreeSet::new(), - peers: BTreeSet::new(), - }); - ended.epoch = ended.epoch.max(epoch); - ended.origins.extend(origins); - ended.peers.extend(peers); - self.dirty.insert(broadcast.clone()); - true + held } /// A holder let go of its copy of an ended claim of `broadcast`, as `release` says: @@ -4063,6 +4085,47 @@ mod tests { assert_eq!(shown.item.above, 1); } + /// A copy of an ended claim presented at a deeper path, lowered by as many segments, + /// is held back there too, while another claim of the announcer at that path is not. + #[test] + fn a_copy_presented_below_an_ended_claim_is_never_shown() { + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Routing(node(2))); + open(&mut router, 4, Role::Routing(node(4))); + router.recv(2, route(2, 2, 1)).unwrap(); + router.recv(4, route(4, 4, 2)).unwrap(); + let at = |path: &str, origin: Id, above: u64, floor: Rev| { + let Change::Live(claim) = chained(vec![9, origin]) else { + unreachable!("a live claim"); + }; + Msg::Claims { + origin: node(origin), + changes: vec![( + (p(path), 7), + Change::Live(Claim { + item: Run { epoch: 0, above }, + ..claim + }), + )], + floor, + } + }; + router.recv(2, at("live", 2, 0, 1)).unwrap(); + router.recv(4, at("live/hd", 4, 1, 1)).unwrap(); + router.recv(4, at("live/hd/a", 4, 0, 2)).unwrap(); + let shown = |router: &Router, path: &str| router.shown().contains_key(&(7, p(path))); + assert!(shown(&router, "live/hd")); + + let ended = Msg::Claims { + origin: node(2), + changes: vec![((p("live"), 7), Change::Ended)], + floor: 2, + }; + router.recv(2, ended).unwrap(); + assert!(!shown(&router, "live/hd"), "node 4's copy is of the ended claim"); + assert!(shown(&router, "live/hd/a"), "node 4's own claim there is no copy"); + } + /// A publisher that withdraws its claim and announces it again at the same epoch, as /// `origin::Route::withdraw` and `announce` do, is shown at once: only the copies that /// stood when the end arrived are held back. From 6192adbdd2124e70029733becfcbfca7424ed681 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 02:16:07 -0800 Subject: [PATCH 18/25] fix(moq-net): a reader below an ended claim ends at once only if it shows a copy of it A claim lost beside an ended one, in the same message, waits out the debounce at its own path, while the ended claim's copy there stays held back. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/relay.rs | 42 +++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index d2146e620dc..458d2ad32b8 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -2146,8 +2146,12 @@ impl Router { record.epoch = record.epoch.max(epoch); record.origins.extend(origins); record.peers.extend(peers); - // Its publisher ended it, wherever a copy was presented. - self.ended.insert(here.clone()); + // The reader there ends at once only if it shows a copy of the ended claim: one + // showing another claim that went too, such as an exact one lost, waits out the + // debounce. + if (self.reader.get(instance.clone(), &here.0)).is_some_and(|shown| shown.item.above() == of.1) { + self.ended.insert(here.clone()); + } self.dirty.insert(here.clone()); held.push(here); } @@ -4126,6 +4130,40 @@ mod tests { assert!(shown(&router, "live/hd/a"), "node 4's own claim there is no copy"); } + /// A reader below an ended claim that shows another claim there, lost in the same + /// message, shows it unavailable for the debounce rather than ending it, while the + /// ended claim's copy there stays held back. + #[test] + fn a_claim_lost_beside_an_ended_one_waits_out_the_debounce() { + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Routing(node(2))); + open(&mut router, 4, Role::Routing(node(4))); + router.recv(2, route(2, 2, 1)).unwrap(); + router.recv(4, route(4, 4, 2)).unwrap(); + let live = |path: &str, chain: Vec, above: u64| { + let Change::Live(claim) = chained(chain) else { + unreachable!("a live claim"); + }; + let item = Run { epoch: 0, above }; + ((p(path), 7), Change::Live(Claim { item, ..claim })) + }; + let claims = |origin: Id, changes, floor: Rev| Msg::Claims { + origin: node(origin), + changes, + floor, + }; + let changes = vec![live("live", vec![9, 2], 0), live("live/hd", vec![8, 2], 0)]; + router.recv(2, claims(2, changes, 1)).unwrap(); + router.recv(4, claims(4, vec![live("live/hd", vec![9, 4], 1)], 1)).unwrap(); + let shown = |router: &Router| router.shown().get(&(7, p("live/hd"))).cloned(); + assert_eq!(shown(&router).map(|shown| shown.item.above), Some(0)); + + let changes = vec![((p("live"), 7), Change::Ended), ((p("live/hd"), 7), Change::Lost)]; + router.recv(2, claims(2, changes, 2)).unwrap(); + let shown = shown(&router).expect("the lost claim shows for the debounce"); + assert_eq!((shown.item.above, shown.available), (0, false)); + } + /// A publisher that withdraws its claim and announces it again at the same epoch, as /// `origin::Route::withdraw` and `announce` do, is shown at once: only the copies that /// stood when the end arrived are held back. From 88d91784e1b596437fc67175624ee050c76caf9b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 02:22:04 -0800 Subject: [PATCH 19/25] style(moq-net): format the mixed-batch test Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/relay.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index 458d2ad32b8..4d6e6cefb51 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -4154,7 +4154,9 @@ mod tests { }; let changes = vec![live("live", vec![9, 2], 0), live("live/hd", vec![8, 2], 0)]; router.recv(2, claims(2, changes, 1)).unwrap(); - router.recv(4, claims(4, vec![live("live/hd", vec![9, 4], 1)], 1)).unwrap(); + router + .recv(4, claims(4, vec![live("live/hd", vec![9, 4], 1)], 1)) + .unwrap(); let shown = |router: &Router| router.shown().get(&(7, p("live/hd"))).cloned(); assert_eq!(shown(&router).map(|shown| shown.item.above), Some(0)); From 36f832c2126516388e65e281fa8e161666535cfd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 03:29:00 -0800 Subject: [PATCH 20/25] fix(moq-net): a reader below an ended claim keeps a lost claim's cause by what the message lost The shown item carries no announcer, so matching its distance mistook another announcer's lost claim at the copy's distance for the copy. A held path is marked ended unless the same message lost a claim there. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/relay.rs | 58 ++++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index 4d6e6cefb51..d45618fde2c 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -2090,8 +2090,12 @@ impl Router { } // Its publisher ended the claim: every copy of it standing now is stale, and this // node's own, taken across a grant from one, ends with it. + let lost: BTreeSet> = (changes.iter()) + .filter(|(_, change)| *change == Change::Lost) + .map(|(broadcast, _)| broadcast.clone()) + .collect(); for (broadcast, of, epoch) in ended { - for held in self.withdraw(&broadcast, of, epoch) { + for held in self.withdraw(&broadcast, of, epoch, &lost) { if self.givers.contains_key(&held) { self.restate(held, Change::Ended); } @@ -2102,8 +2106,14 @@ impl Router { /// Hold back each copy standing now of the claim of `broadcast` its publisher ended, /// `ended`, at no newer than `epoch`, at every path a copy is presented at: at or below /// the claim's own prefix, lowered by as many segments. Returns the broadcasts where - /// any stands. - fn withdraw(&mut self, broadcast: &Broadcast, ended: Of, epoch: u64) -> Vec> { + /// any stands. `lost` are the broadcasts the same message lost a claim of. + fn withdraw( + &mut self, + broadcast: &Broadcast, + ended: Of, + epoch: u64, + lost: &BTreeSet>, + ) -> Vec> { let (path, instance) = broadcast; let (announcer, above) = ended; let depth = path.parts().count(); @@ -2146,10 +2156,9 @@ impl Router { record.epoch = record.epoch.max(epoch); record.origins.extend(origins); record.peers.extend(peers); - // The reader there ends at once only if it shows a copy of the ended claim: one - // showing another claim that went too, such as an exact one lost, waits out the - // debounce. - if (self.reader.get(instance.clone(), &here.0)).is_some_and(|shown| shown.item.above() == of.1) { + // The reader there showed the copy, a claim that stands, or one that went with + // this end: one lost waits out the debounce. + if !lost.contains(&here) { self.ended.insert(here.clone()); } self.dirty.insert(here.clone()); @@ -4166,6 +4175,41 @@ mod tests { assert_eq!((shown.item.above, shown.available), (0, false)); } + /// As [`Self::a_claim_lost_beside_an_ended_one_waits_out_the_debounce`], with the lost + /// claim another announcer's at the copy's own distance: still lost, not ended. + #[test] + fn a_claim_lost_beside_an_ended_one_at_the_same_distance_waits_out_the_debounce() { + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Routing(node(2))); + open(&mut router, 4, Role::Routing(node(4))); + router.recv(2, route(2, 2, 1)).unwrap(); + router.recv(4, route(4, 4, 2)).unwrap(); + let live = |path: &str, chain: Vec, above: u64| { + let Change::Live(claim) = chained(chain) else { + unreachable!("a live claim"); + }; + let item = Run { epoch: 0, above }; + ((p(path), 7), Change::Live(Claim { item, ..claim })) + }; + let claims = |origin: Id, changes, floor: Rev| Msg::Claims { + origin: node(origin), + changes, + floor, + }; + let changes = vec![live("live", vec![9, 2], 0), live("live/hd", vec![8, 2], 1)]; + router.recv(2, claims(2, changes, 1)).unwrap(); + router + .recv(4, claims(4, vec![live("live/hd", vec![9, 4], 1)], 1)) + .unwrap(); + assert!(router.shown().contains_key(&(7, p("live/hd")))); + + let changes = vec![((p("live"), 7), Change::Ended), ((p("live/hd"), 7), Change::Lost)]; + router.recv(2, claims(2, changes, 2)).unwrap(); + let shown = router.shown().get(&(7, p("live/hd"))).cloned(); + let shown = shown.expect("the lost claim shows for the debounce"); + assert!(!shown.available); + } + /// A publisher that withdraws its claim and announces it again at the same epoch, as /// `origin::Route::withdraw` and `announce` do, is shown at once: only the copies that /// stood when the end arrived are held back. From 3baea325d04d1fe589f0ee3afcaaa06bfb81e3b7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 04:01:17 -0800 Subject: [PATCH 21/25] fix(moq-net): a copy of an ended claim lost in the same message ends with it A lost claim keeps the reader's debounce only if it was no copy of the claim the message ended: the publisher's end covers every copy. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/relay.rs | 59 ++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index d45618fde2c..48f5547d4eb 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -2025,6 +2025,8 @@ impl Router { let mut unclaimed = Vec::new(); let mut lost = Vec::new(); let mut ended = Vec::new(); + // What each claim this message lost was a copy of, and at which epoch. + let mut gone = Vec::new(); for (broadcast, change) in &changes { match change { Change::Live(claim) => { @@ -2036,8 +2038,9 @@ impl Router { Change::Ended | Change::Lost => { if let Some(claim) = known.items.remove(broadcast) { unclaimed.push(broadcast.clone()); - if *change == Change::Ended { - ended.push((broadcast.clone(), of(&claim, origin), claim.item.epoch())); + match change { + Change::Ended => ended.push((broadcast.clone(), of(&claim, origin), claim.item.epoch())), + _ => gone.push((broadcast.clone(), of(&claim, origin), claim.item.epoch())), } // `restate` keeps its own claim's source. if *change == Change::Lost && !own { @@ -2090,12 +2093,8 @@ impl Router { } // Its publisher ended the claim: every copy of it standing now is stale, and this // node's own, taken across a grant from one, ends with it. - let lost: BTreeSet> = (changes.iter()) - .filter(|(_, change)| *change == Change::Lost) - .map(|(broadcast, _)| broadcast.clone()) - .collect(); for (broadcast, of, epoch) in ended { - for held in self.withdraw(&broadcast, of, epoch, &lost) { + for held in self.withdraw(&broadcast, of, epoch, &gone) { if self.givers.contains_key(&held) { self.restate(held, Change::Ended); } @@ -2106,13 +2105,14 @@ impl Router { /// Hold back each copy standing now of the claim of `broadcast` its publisher ended, /// `ended`, at no newer than `epoch`, at every path a copy is presented at: at or below /// the claim's own prefix, lowered by as many segments. Returns the broadcasts where - /// any stands. `lost` are the broadcasts the same message lost a claim of. + /// any stands. `lost` are the claims the same message lost, each with what it was a + /// copy of and its epoch. fn withdraw( &mut self, broadcast: &Broadcast, ended: Of, epoch: u64, - lost: &BTreeSet>, + lost: &[(Broadcast, Of, u64)], ) -> Vec> { let (path, instance) = broadcast; let (announcer, above) = ended; @@ -2156,9 +2156,11 @@ impl Router { record.epoch = record.epoch.max(epoch); record.origins.extend(origins); record.peers.extend(peers); - // The reader there showed the copy, a claim that stands, or one that went with - // this end: one lost waits out the debounce. - if !lost.contains(&here) { + // The reader there showed a copy, a claim that stands, or one that went with this + // end: a lost claim that was no copy of the ended one waits out the debounce. + let other = + |(at, by, at_epoch): &(Broadcast, Of, u64)| *at == here && !(*by == of && *at_epoch <= epoch); + if !lost.iter().any(other) { self.ended.insert(here.clone()); } self.dirty.insert(here.clone()); @@ -4210,6 +4212,39 @@ mod tests { assert!(!shown.available); } + /// A copy of an ended claim that the same message loses ends with it: the publisher's + /// end covers every copy. + #[test] + fn a_copy_lost_beside_its_ended_claim_ends() { + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Routing(node(2))); + open(&mut router, 4, Role::Routing(node(4))); + router.recv(2, route(2, 2, 1)).unwrap(); + router.recv(4, route(4, 4, 2)).unwrap(); + let live = |path: &str, chain: Vec, above: u64| { + let Change::Live(claim) = chained(chain) else { + unreachable!("a live claim"); + }; + let item = Run { epoch: 0, above }; + ((p(path), 7), Change::Live(Claim { item, ..claim })) + }; + let claims = |origin: Id, changes, floor: Rev| Msg::Claims { + origin: node(origin), + changes, + floor, + }; + let changes = vec![live("live", vec![9, 2], 0), live("live/hd", vec![9, 2], 1)]; + router.recv(2, claims(2, changes, 1)).unwrap(); + router + .recv(4, claims(4, vec![live("live/hd", vec![9, 4], 1)], 1)) + .unwrap(); + assert!(router.shown().contains_key(&(7, p("live/hd")))); + + let changes = vec![((p("live"), 7), Change::Ended), ((p("live/hd"), 7), Change::Lost)]; + router.recv(2, claims(2, changes, 2)).unwrap(); + assert!(!router.shown().contains_key(&(7, p("live/hd"))), "ended, not lost"); + } + /// A publisher that withdraws its claim and announces it again at the same epoch, as /// `origin::Route::withdraw` and `announce` do, is shown at once: only the copies that /// stood when the end arrived are held back. From 68a90f482427e1ef5680cb4f1547178e7f6750f0 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 05:31:41 -0800 Subject: [PATCH 22/25] fix(moq-net): a copy lost beside its ended claim ends with no other holder Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/relay.rs | 49 +++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index 1dcf6a99701..4def7f7a08a 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -2106,7 +2106,7 @@ impl Router { /// `ended`, at no newer than `epoch`, at every path a copy is presented at: at or below /// the claim's own prefix, lowered by as many segments. Returns the broadcasts where /// any stands. `lost` are the claims the same message lost, each with what it was a - /// copy of and its epoch. + /// copy of and its epoch: a copy among them ends with the claim, held back or not. fn withdraw( &mut self, broadcast: &Broadcast, @@ -2123,6 +2123,9 @@ impl Router { let own = path.head(base).expect("a head no deeper than the path"); // A copy a neighbour gives is this node's own claim there, unless already held. let mut paths: BTreeSet = below(&self.claimants, &own).map(|(path, _)| path.clone()).collect(); + // A copy this message lost has no claimant left to find it by. + let lost_at = (lost.iter()).filter(|((at, lost), ..)| lost == instance && super::prefix::within(at, &own)); + paths.extend(lost_at.map(|((at, _), ..)| at.clone())); paths.extend([own, path.clone()]); self.count(paths.len()); let mut held = Vec::new(); @@ -2130,6 +2133,12 @@ impl Router { let of = (announcer, (at.parts().count() - base) as u64); let copy = |by: Of, claim: &Claim| by == of && claim.item.epoch() <= epoch; let here = (at, instance.clone()); + // The reader there showed a copy, a claim that stands, or one that went with this + // end: a lost claim that was no copy of the ended one waits out the debounce. + let other = + |(at, by, at_epoch): &(Broadcast, Of, u64)| *at == here && !(*by == of && *at_epoch <= epoch); + let lost_copy = + |(at, by, at_epoch): &(Broadcast, Of, u64)| *at == here && *by == of && *at_epoch <= epoch; let origins: BTreeSet = (self.claimants.get(&here.0).and_then(|claims| claims.get(instance))) .into_iter() .flatten() @@ -2145,6 +2154,11 @@ impl Router { .collect(); self.count(origins.len() + peers.len()); if origins.is_empty() && peers.is_empty() { + // Nothing left to hold back, but the copy lost here ends with its claim. + if lost.iter().any(lost_copy) && !lost.iter().any(other) { + self.ended.insert(here.clone()); + self.dirty.insert(here); + } continue; } let record = self.withdrawn.entry(here.clone()).or_default(); @@ -2156,10 +2170,6 @@ impl Router { record.epoch = record.epoch.max(epoch); record.origins.extend(origins); record.peers.extend(peers); - // The reader there showed a copy, a claim that stands, or one that went with this - // end: a lost claim that was no copy of the ended one waits out the debounce. - let other = - |(at, by, at_epoch): &(Broadcast, Of, u64)| *at == here && !(*by == of && *at_epoch <= epoch); if !lost.iter().any(other) { self.ended.insert(here.clone()); } @@ -4245,6 +4255,35 @@ mod tests { assert!(!router.shown().contains_key(&(7, p("live/hd"))), "ended, not lost"); } + /// As [`Self::a_copy_lost_beside_its_ended_claim_ends`], with no other holder of the + /// copy: nothing is left to hold back, and the reader still ends. + #[test] + fn a_copy_lost_beside_its_ended_claim_ends_with_no_other_holder() { + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Routing(node(2))); + router.recv(2, route(2, 2, 1)).unwrap(); + let live = |path: &str, above: u64| { + let Change::Live(claim) = chained(vec![9, 2]) else { + unreachable!("a live claim"); + }; + let item = Run { epoch: 0, above }; + ((p(path), 7), Change::Live(Claim { item, ..claim })) + }; + let claims = |changes, floor: Rev| Msg::Claims { + origin: node(2), + changes, + floor, + }; + router + .recv(2, claims(vec![live("live", 0), live("live/hd", 1)], 1)) + .unwrap(); + assert!(router.shown().contains_key(&(7, p("live/hd")))); + + let changes = vec![((p("live"), 7), Change::Ended), ((p("live/hd"), 7), Change::Lost)]; + router.recv(2, claims(changes, 2)).unwrap(); + assert!(!router.shown().contains_key(&(7, p("live/hd"))), "ended, not lost"); + } + /// A publisher that withdraws its claim and announces it again at the same epoch, as /// `origin::Route::withdraw` and `announce` do, is shown at once: only the copies that /// stood when the end arrived are held back. From 4c2921bf8a9f90939dfe64976af0264330ad69b5 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 07:13:20 -0800 Subject: [PATCH 23/25] fix(moq-net): a relay passing skipped revisions on as one lets its neighbours let go of the holder too Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/relay.rs | 44 +++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index 4def7f7a08a..962cab2d41e 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -1953,7 +1953,9 @@ impl Router { return Ok(()); } let mut visited = 0; - let mut lagged = false; + // Two or more revisions skipped, here or by a relay that passed them on as one, can + // hide an end and the same claim announced again, which differ in nothing. + let lagged = floor.saturating_sub(known.floor) >= 2; let changes = match was == known.floor || mutated!(self, Lagging) { // The copy matched this node's at the same revision, so the message's changes // bring it to the new one. @@ -1980,8 +1982,6 @@ impl Router { }), ); visited = heard.items.len() + known.items.len(); - // An end and the same claim announced again take two revisions. - lagged = floor.saturating_sub(known.floor) >= 2; diff } }; @@ -1989,8 +1989,7 @@ impl Router { let known = self.known.get_mut(&origin).expect("known"); known.floor = known.floor.max(floor); self.publish(origin, changes); - // Two or more revisions a lagging copy skipped can hide an end and the same claim - // announced again, which differ in nothing: `origin` holds no copy it held before. + // `origin` holds no copy it held before. if lagged { self.unhold_origin(origin); } @@ -4080,6 +4079,41 @@ mod tests { assert!(shown(&router), "node 4 may have claimed it again"); } + /// A relay that let go of a holder it was overtaken on passes the skipped revisions on + /// as one: a neighbour whose copy was in step lets go of that holder too, rather than + /// hold back for good a claim the relay shows. + #[test] + fn a_holder_overtaken_upstream_claims_again_downstream() { + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Routing(node(2))); + open(&mut router, 4, Role::Routing(node(4))); + router.recv(2, route(2, 2, 1)).unwrap(); + router.recv(4, route(4, 4, 2)).unwrap(); + let live = (p("live"), 7); + let claims = |origin: Id, changes, floor: Rev| Msg::Claims { + origin: node(origin), + changes, + floor, + }; + router + .recv(2, claims(2, vec![(live.clone(), chained(vec![9, 2]))], 1)) + .unwrap(); + router + .recv(4, claims(4, vec![(live.clone(), chained(vec![9, 4]))], 1)) + .unwrap(); + router + .recv(2, claims(2, vec![(live.clone(), Change::Ended)], 2)) + .unwrap(); + let shown = |router: &Router| router.shown().contains_key(&(7, p("live"))); + assert!(!shown(&router), "node 4's copy is of the ended claim"); + + // The neighbour skipped node 4's end at 2 and its claim again at 3, and passes on + // what differs at 4: only another broadcast. + let other = vec![((p("other"), 7), chained(vec![8, 4]))]; + router.recv(4, claims(4, other, 4)).unwrap(); + assert!(shown(&router), "node 4 may have claimed it again"); + } + /// Ending an announcer's exact claim leaves its broader claim presented at the same /// path: that one is no copy of what ended. #[test] From 01b27f5e408074f86c67485584d5814fd81849bd Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 07:29:00 -0800 Subject: [PATCH 24/25] fix(moq-net): an ended claim holds back copies only where its own presentation reaches A scoped neighbour's claim may name another announcer with a prefix above its grant; ending it held back that announcer's copies anywhere under the prefix. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/relay.rs | 52 +++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index 962cab2d41e..cab48dcb2f2 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -2103,7 +2103,7 @@ impl Router { /// Hold back each copy standing now of the claim of `broadcast` its publisher ended, /// `ended`, at no newer than `epoch`, at every path a copy is presented at: at or below - /// the claim's own prefix, lowered by as many segments. Returns the broadcasts where + /// the ended one's path, lowered by as many segments. Returns the broadcasts where /// any stands. `lost` are the claims the same message lost, each with what it was a /// copy of and its epoch: a copy among them ends with the claim, held back or not. fn withdraw( @@ -2119,13 +2119,15 @@ impl Router { let Some(base) = usize::try_from(above).ok().and_then(|above| depth.checked_sub(above)) else { return Vec::new(); }; - let own = path.head(base).expect("a head no deeper than the path"); + // The end reaches only where its own presentation does: a claimant, such as a scoped + // neighbour's adopter, may present a claim whose prefix lies outside what it may + // claim, so the prefix alone would let it end copies anywhere under it. // A copy a neighbour gives is this node's own claim there, unless already held. - let mut paths: BTreeSet = below(&self.claimants, &own).map(|(path, _)| path.clone()).collect(); + let mut paths: BTreeSet = below(&self.claimants, path).map(|(path, _)| path.clone()).collect(); // A copy this message lost has no claimant left to find it by. - let lost_at = (lost.iter()).filter(|((at, lost), ..)| lost == instance && super::prefix::within(at, &own)); + let lost_at = (lost.iter()).filter(|((at, lost), ..)| lost == instance && super::prefix::within(at, path)); paths.extend(lost_at.map(|((at, _), ..)| at.clone())); - paths.extend([own, path.clone()]); + paths.insert(path.clone()); self.count(paths.len()); let mut held = Vec::new(); for at in paths { @@ -4079,6 +4081,46 @@ mod tests { assert!(shown(&router), "node 4 may have claimed it again"); } + /// An end reaches only where its own presentation does: a scoped neighbour that names + /// another announcer, with a prefix above its grant, and ends that claim, holds back no + /// copy outside the grant. + #[test] + fn an_end_reaches_no_copy_outside_its_presentation() { + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Routing(node(2))); + open(&mut router, 3, Role::Scoped(vec![p("a/x")])); + router.recv(2, route(2, 2, 1)).unwrap(); + let at = |chain: Vec, above: u64| { + let Change::Live(claim) = chained(chain) else { + unreachable!("a live claim"); + }; + Change::Live(Claim { + item: Run { epoch: 0, above }, + ..claim + }) + }; + // Announcer 9's claim at `a`, presented at `a/v`. + let changes = vec![((p("a/v"), 7), at(vec![9, 2], 1))]; + router + .recv( + 2, + Msg::Claims { + origin: node(2), + changes, + floor: 1, + }, + ) + .unwrap(); + assert!(router.shown().contains_key(&(7, p("a/v")))); + + let given = |change| Msg::Announce { + changes: vec![((p("a/x/y"), 7), change)], + }; + router.recv(3, given(at(vec![9], 2))).unwrap(); + router.recv(3, given(Change::Ended)).unwrap(); + assert!(router.shown().contains_key(&(7, p("a/v"))), "outside the grant"); + } + /// A relay that let go of a holder it was overtaken on passes the skipped revisions on /// as one: a neighbour whose copy was in step lets go of that holder too, rather than /// hold back for good a claim the relay shows. From 004a4e72d78830e465803f8462d961584bbbc068 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 08:28:26 -0800 Subject: [PATCH 25/25] quest: moq-relay's handoff tests hold under load Co-Authored-By: Claude Opus 5.5 --- quest/m0/test-flakes-3/handoff-ready-load.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 quest/m0/test-flakes-3/handoff-ready-load.md diff --git a/quest/m0/test-flakes-3/handoff-ready-load.md b/quest/m0/test-flakes-3/handoff-ready-load.md new file mode 100644 index 00000000000..0e050d45ea6 --- /dev/null +++ b/quest/m0/test-flakes-3/handoff-ready-load.md @@ -0,0 +1,11 @@ +# [S] moq-relay's handoff tests hold under load + +## Goal + +`moq-relay`'s `handoff::` integration tests pass however loaded the machine is. + +## Plan + +Three of them (`a_killed_relay_s_clients_are_reset_by_the_one_it_replaced`, `a_restart_starts_the_new_relay_beside_the_old`, `a_relay_no_successor_takes_over_from_drains`) failed under a full `just check` on 2026-11-19, each with `no READY reached the service manager: Elapsed` after the test's 10s wall-clock `TIMEOUT`, on a branch that touched no moq-relay code ([moq-dev/moq#9112](https://github.com/moq-dev/moq/pull/9112)). +Run alone, all seven pass in 1.2s. +Reproduce under load first, then make the READY wait follow an event, such as the relay process exiting, instead of a deadline.