From 08a56ba9e7f317bce80f5ec908971dc3c22670d4 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:39:04 -0800 Subject: [PATCH 1/2] chore(quest): claim quest/m2/cluster-routes/dead-instance-flip.md Quest-Claim: c7892a280d15f22f From 45ad829b2f2bfedbdf6e711fab16e01745843990 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 00:57:46 -0800 Subject: [PATCH 2/2] fix(moq-net): a dead instance held unavailable never outranks a live one The origin ranked the route layer's closest way (Lane::primary) ahead of availability, so while a route change moved a live instance's closest way to another session, a restarted publisher's dead instance, held unavailable, won the path back for a moment: a reader saw C -> B -> C. The closest way now ranks below availability, where the doc already put it, and the lite-07 draft ranks an Unavailable claim with a draining one, as JS already did. Co-Authored-By: Claude Opus 5.5 --- doc/concept/identity.md | 9 ++- drafts/draft-lcurley-moq-lite.md | 4 +- quest/m2/cluster-routes/dead-instance-flip.md | 13 ----- rs/moq-net/src/model/origin.rs | 56 +++++++++++++++++-- 4 files changed, 58 insertions(+), 24 deletions(-) delete mode 100644 quest/m2/cluster-routes/dead-instance-flip.md diff --git a/doc/concept/identity.md b/doc/concept/identity.md index c1e958adf06..760ffcf4d69 100644 --- a/doc/concept/identity.md +++ b/doc/concept/identity.md @@ -72,9 +72,12 @@ A route resolving a path is chosen in this order: 2. Within one instance, only its highest epoch is eligible, even over a cheaper route. Readers continue across an epoch change without a seam, because the timeline continues. -3. Among eligible routes: a route from a peer that has not sent GOAWAY - (a draining peer's routes rank below every other, whatever their preference - or cost, and serve only as a last resort), then the highest priority, which on +3. Among eligible routes: a route that still reaches its publisher, from a + peer that has not sent GOAWAY (an unavailable route, held while the network + may recover a path, and a draining peer's routes rank below every other, + whatever their priority or cost, and serve only as a last resort, so a + restarted publisher's lingering instance never takes the path back), then + the highest priority, which on moq-lite 07 is the lowest of any link along the route's path, then an identified chain over an anonymous one, then the lowest cost, then a broadcast published on diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 022454bad74..bb1a05dd58e 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -557,7 +557,7 @@ The winning tier is the whole answer: a refusal from it never falls through to a Nor is a refusal retried at another claim of the same prefix. Within the tier, a claim is ineligible while another claim of the same prefix carries its Instance at a higher Epoch, however cheap it is and whether or not a route reaches it, so a request fails with UNROUTABLE while the newest Epoch is unavailable. -Among the eligible claims, the receiver SHOULD prefer one that is not draining, then the highest priority, then the lowest cost, the claim's cost plus the cost of the route to the node announcing it, then the lowest Rendezvous Hash, and then the most recently received, so a reconnecting publisher is not outranked by the stale session it replaced. +Among the eligible claims, the receiver SHOULD prefer one that is neither draining nor `Unavailable`, then the highest priority, then the lowest cost, the claim's cost plus the cost of the route to the node announcing it, then the lowest Rendezvous Hash, and then the most recently received, so a reconnecting publisher is not outranked by the stale session it replaced. The Rendezvous Hash is the 64-bit FNV-1a hash, with the standard offset basis `0xCBF29CE484222325` and prime, of the requested path's UTF-8 bytes followed by the claim's Instance. Every node that knows the same claims ranks them the same, so equal-cost Instances of one prefix share its paths and every relay sends one path to the same Instance. When choosing which claim to advertise for a prefix, the requested path is the prefix itself. @@ -2299,7 +2299,7 @@ A stream that FINs without a GROUP_END carries a closed Group, which may have he - Replaced Warm and Cold Route Cost with one cost, which each link adds to. - The Cost parameter must be at least 1. - Added `Above` to ANNOUNCE_START and ANNOUNCE_UPDATE: how many segments broader than the requested prefix a presented claim is. A claim presented at the requested prefix from above otherwise looked exactly like a claim at the prefix, so a broadcast nested under another's path could lose to the broader claim covering it, depending on the order a relay listed them in. Receivers take a claim to cover its path less its Above. -- Replaced the shortest-path, Hop ID, Instance Hash, and Chain Hash tie-breaks with the claim's drain and priority, its cost, and a Rendezvous Hash of the requested path and the claim's Instance, a standard FNV-1a. +- Replaced the shortest-path, Hop ID, Instance Hash, and Chain Hash tie-breaks with the claim's drain or unavailability and priority, its cost, and a Rendezvous Hash of the requested path and the claim's Instance, a standard FNV-1a. - Removed the standby-ordering deployment requirements on Route Cost seeds and bounds. ### Tracks {#changelog-07-tracks} diff --git a/quest/m2/cluster-routes/dead-instance-flip.md b/quest/m2/cluster-routes/dead-instance-flip.md deleted file mode 100644 index c5a997ad402..00000000000 --- a/quest/m2/cluster-routes/dead-instance-flip.md +++ /dev/null @@ -1,13 +0,0 @@ -# [M] A dead instance's stale path never wins a lite-07 route - -## Goal - -A reader of a path whose route flips C → B → C never sees B when B is an instance already known dead, whenever it polls, so the route-sweep invariants hold with no window for a step back to a dead instance. - -## Plan - -Found by [moq-dev/moq#9108](https://github.com/moq-dev/moq/pull/9108), which fixed the Nightly `mesh_sweep` failures: when the route flipped C → B → C before the reader polled, the reader got a second restart to C. #9108 merges a restart back to the instance the reader already shows into an Update, but a reader that polls mid-flip can still see B, so the sweep can still flake. - -The flip became reachable with the lite-07 route rule in [moq-dev/moq#8987](https://github.com/moq-dev/moq/pull/8987). Decide whether that rule should stop a dead instance's stale path from winning at all, which removes the flip, rather than hiding it at the reader. Recommended: stop it at the rule if the route model (`/verify-failover`, `/verify-warm`) shows no case where the stale path must win; check the draft's route section for the same. - -Public API: none. Wire: possibly the lite-07-wip route rule; update the draft in the same PR if so. diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 8959392598f..250435c00c4 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -906,8 +906,8 @@ impl<'a> Epochs<'a> { /// neither draining (see [`RouteInfo::draining`]) nor unavailable (see /// [`RouteInfo::available`]), so a peer that sent a GOAWAY or a held route ranks below /// every other eligible route and serves only as a last resort. Then the route layer's -/// selected route ahead of its alternates. Then the higher link [`Priority`], so -/// metrics from different operators are never compared. Then an identified chain (no 0) outranks an anonymous one regardless +/// closest way to a broadcast ahead of its others ([`Lane::primary`]). Then the higher +/// link [`Priority`], so metrics from different operators are never compared. Then an identified chain (no 0) outranks an anonymous one regardless /// of cost, then the cheapest cost, then a broadcast published on this origin (it serves /// what is here, not a claim that has to ask), then the shortest hop chain, then a hash /// of the instance and `path`, so every node converges on the same winner, then, among @@ -936,9 +936,12 @@ fn route_order(path: &Path, entry: &RouteEntry) -> RouteOrder { // ranks below a broader one: a stream presents a broader claim at the request, // where it ties an exact claim for the path. entry.lane.above, - // The route layer's closest claim ahead of another node's of the same broadcast. - !entry.lane.primary, entry.draining || !entry.available, + // The route layer's closest way ahead of its others to the same broadcast. Below + // availability, since the route layer marks every way to a broadcast available or + // none: a live instance's other way outranks a dead one held unavailable while a + // route change moves its closest way to another session. + !entry.lane.primary, // A lite-07 route ranks by the lowest priority of any link on its path. Reverse(match entry.lane.priority { Some(priority) => entry.priority.min(Priority::new(priority)), @@ -1643,8 +1646,8 @@ struct Horizon { /// through that node's routes. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub(crate) enum NodeFilter { - /// Every route. Of the route layer's, the primary ones rank first (see [`Lane`]), so - /// a wider covering route serves only while no narrower one does. + /// Every route. Of the route layer's, the most specific claim ranks first and then each + /// broadcast's closest way (see [`Lane`]). #[default] Any, /// Only the routes this node serves itself. @@ -10998,6 +11001,47 @@ mod tests { ); } + /// A restarted publisher's dead instance, held unavailable, never wins back the path from + /// its live replacement while the replacement's closest way moves to another session: + /// the old way is demoted before the session taking over serves the new one. + #[kio::test] + #[cfg(feature = "async")] + async fn a_dead_instance_never_outranks_a_live_one_between_its_ways() { + let producer = origin(1).produce(); + let consumer = producer.consume(); + let mut announced = consumer.routes(); + let route = |instance: &crate::Instance, node, primary, available, cost| { + RouteInfo::new() + .with_instance(instance.clone(), 0) + .with_hops(hops(&[node])) + .with_node(Some(node_id(node))) + .with_lane(Lane { + primary, + priority: Some(0), + ..Lane::default() + }) + .with_available(available) + .with_cost(cost) + }; + let (dead, live) = (crate::Instance::random(), crate::Instance::random()); + + // The dead instance is cheaper, and still the closest way to its own claim. + let _dead = producer + .forward("room/alice", route(&dead, 10, true, false, 1)) + .unwrap(); + let way = producer.forward("room/alice", route(&live, 11, true, true, 5)).unwrap(); + assert_eq!(announced.expect_next_active("room/alice").await.instance(), &live); + + way.update(route(&live, 11, false, true, 5)).unwrap(); + assert!( + announced.try_next().is_none(), + "the live instance's demoted way keeps the path" + ); + let _pending = consumer.request("room/alice", None); + let request = queued(&way).await; + assert_eq!(request.path().as_str(), "room/alice"); + } + /// A draining route ranks below every other eligible one, even on a preferred link /// against a demoted one, whatever the costs say. #[kio::test]