From 486b3dda26aa976a7aee3712079a75769c051882 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:37:39 -0800 Subject: [PATCH 1/7] chore(quest): claim quest/m2/multicast/detour-resume.md Quest-Claim: 1ff1517e3623daae From 4de79a967bea8ef5b62ac43c24c07988750555b1 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:04:54 -0800 Subject: [PATCH 2/7] fix(moq-net): a reader resumes on a detour learned after its session closed A closed source no longer excludes a lite-07 claim's route: the front asks the claim again, and the request waits until another session serves the claim or its debounce runs out. A track nothing resumes still ends with the error that ended its copy, not the later Unroutable. Co-Authored-By: Claude Opus 5.5 --- quest/m2/multicast/README.md | 10 +- quest/m2/multicast/detour-resume.md | 66 --------- rs/moq-net/src/model/front.rs | 136 +++++++++++++++++- rs/moq-net/src/model/origin.rs | 1 + .../tests/integration/catalog_handover.rs | 16 +-- .../tests/integration/route_audit/contest.rs | 15 +- .../tests/integration/route_audit/failures.rs | 21 ++- .../tests/integration/route_audit/mod.rs | 12 -- rs/moq-net/tests/integration/route_cluster.rs | 13 +- 9 files changed, 155 insertions(+), 135 deletions(-) delete mode 100644 quest/m2/multicast/detour-resume.md diff --git a/quest/m2/multicast/README.md b/quest/m2/multicast/README.md index c3a17568a5d..49e7b082a6c 100644 --- a/quest/m2/multicast/README.md +++ b/quest/m2/multicast/README.md @@ -33,10 +33,12 @@ Its proof is `rs/moq-net/verify/route-algebra/`, and the quests written against the old wire were retargeted or deleted. [moq-dev/moq#9111](https://github.com/moq-dev/moq/pull/9111) ran the cases no -test covered on real sessions. The lite-07 cut still waits on this epic's -children, the two flaws it found, -[a reader resuming on a detour learned after its session closed](/quest/m2/multicast/detour-resume.md) -and [a route moving behind one next hop resending a group](/quest/m2/multicast/resend.md), +test covered on real sessions, and +[moq-dev/moq#9154](https://github.com/moq-dev/moq/pull/9154) fixed the first +flaw it found: a front now rides through a lost claim's debounce, so a reader +resumes on a detour learned after its session closed. The lite-07 cut still +waits on this epic's children, the other flaw, +[a route moving behind one next hop resending a group](/quest/m2/multicast/resend.md), whose fix changes the wire, [the mesh replay](/quest/m2/multicast/mesh-replay.md), and [draining dialed links](/quest/m2/multicast/drain-dialed-links.md), and on diff --git a/quest/m2/multicast/detour-resume.md b/quest/m2/multicast/detour-resume.md deleted file mode 100644 index 62bd19c84ec..00000000000 --- a/quest/m2/multicast/detour-resume.md +++ /dev/null @@ -1,66 +0,0 @@ -# [M] A reader resumes on a detour learned after its session closed - -## Goal - -A relay whose serving session closes before a detour to the same Instance -reaches it keeps its front, and its readers resume on the detour once it -arrives, every group once. Today the front ends at the close: the reader -restarts onto another Instance serving the path, or, when none does, reads -nothing more for good, even though the same Instance is back on the detour a -round trip later. - -`route_audit::failures::a_relay_resumes_on_a_detour_learned_after_the_cut` -and the readers `detour_lost` pins in `rs/moq-net/tests/integration/route_audit/` -read on seamlessly, and `detour_lost` is gone. - -## Plan - -Decided 2026-11-19 (coordinator, under the maintainer's pick that day to -merge on the coordinator's recommendation): ✅ kept, under this epic, so the -lite-07 cut waits on it (rejected: dropping it; m3). A reader that reads -nothing more for good after one link cut is the worst failure the audit -found, and it needs only a ring of four. - -Found by [the uncovered cases](https://github.com/moq-dev/moq/pull/9111). It -happens wherever every neighbour's route to the announcing relay ran through -the relay that lost its session, so none was offered back to it: a ring of -four or more, or any mesh where the relay's neighbours reach the publisher -through it. - -Root cause: `source_closed` excludes the closed source's route id -(`rs/moq-net/src/model/front.rs`, `self.excluded.insert(candidate.route)`). -A lite-07 claim keeps one table entry while the route layer moves it to -another next hop, so this excludes the very entry the detour revives. With no -other route at that instant, `pick` finds no candidate, and the front ends -for good (`select` in `rs/moq-net/src/model/origin.rs`, then -`end(Error::Unroutable)` in `front.rs`). A following application is then -stuck: `Following` hands out a closed broadcast again only once its Instance -stops serving the path (`rs/moq-net/src/model/follow.rs`, `moved`), and the -Instance never stops: it is back on the detour. - -Recommended: a front rides through its claim's debounce. A closed source no -longer excludes a route-layer entry (`entry.node.is_some()`), or `select` -waits rather than ends while an excluded entry of the resolved Instance is -still shown unavailable. A track that nothing resumes still ends with the -session's error, not a later `Unroutable`. Rerun the `verify-failover` model, -since this changes the front lifecycle it covers. - -An experiment with the first form (about 8 lines in `pick`) made every audit -case seamless and changed 7 existing tests: two read on where they expected -an end (`route_cluster::a_partitioned_lan_reaches_itself_through_the_cdn_and_heals` -and `catalog_handover::lite_07::reconnect_with_viewer`), and five ended a -debounce later with `Unroutable` instead of at once with the session's error -(`subscription_end_integrity::a_session_death_ends_the_track_with_its_error`, -`subscription_end_integrity::a_session_close_cancels_the_track_from_either_side`, -`track_tail::a_session_death_before_the_tail_aborts_the_track`, and both -`multi_cdn::transport_*_secondary_restarts`). The first two are the fix -working. The other five keep their error once the session's error is -carried to the end. - -Rejected: letting `Following` hand out the same Instance again once its -broadcast ended while the Instance still serves the path, which removes the -permanent stall but restarts the reader at the live edge, skipping groups; -and accepting it, which leaves a reader stuck on a one-round-trip gap, -against requirement 2 of [the epic](/quest/m2/multicast/README.md). - -Not a wire change. diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index 788b7176ff9..93a8a69214f 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -45,6 +45,10 @@ pub(super) struct Candidate { /// a carried instance says the answer is the same stream. A minted one names the /// announcement, and the route could answer with whatever stream now wins upstream. pub carried: bool, + /// Whether the route is a lite-07 claim. A session that closes leaves its claims in + /// the table, unavailable, until another session serves them or the claim's debounce + /// runs out, and a request through one waits for either. + pub claim: bool, } /// Why an upstream request through a route did not produce a source. @@ -518,8 +522,11 @@ impl Front { } // A standing route can outlive the source it produced. Asking it again // would re-request the broadcast that just ended; another route may still - // resume it. - self.excluded.insert(candidate.route); + // resume it. A claim is asked again: the request waits until another session + // serves the claim, which the front then resumes on, or the claim goes. + if !candidate.claim { + self.excluded.insert(candidate.route); + } self.serving = None; self.serving_closing = false; actions.push(Action::Detach { source, drain: true }); @@ -654,6 +661,14 @@ impl Front { return; } if !spliced { + // The source closed under the copy and nothing serves yet: what the copy died + // of is what ends the track, unless a route the front waits on resumes it. + if let Err(err) = result + && serving.is_none() + && track.ended.is_none() + { + track.lost.get_or_insert(err); + } return; } } @@ -956,6 +971,7 @@ mod tests { route, local: false, carried: true, + claim: false, } } @@ -964,6 +980,7 @@ mod tests { route, local: true, carried: true, + claim: false, } } @@ -1226,6 +1243,115 @@ mod tests { ); } + fn claim(route: u64) -> Candidate { + Candidate { + claim: true, + ..remote(route) + } + } + + /// A front whose claim's session closed under `video` before its copy failed, and that + /// asks the claim again, which waits until another session serves it or the claim goes. + fn orphaned() -> Front { + let mut front = serving(claim(1), 100); + assert_actions( + front.step(Event::SourceClosed { source: 100 }), + &[ + Action::Detach { + source: 100, + drain: true, + }, + Action::Reselect, + ], + ); + assert!(front.excluded_routes().is_empty()); + assert_actions( + front.step(Event::Selected { + best: Some(claim(1)), + serving_closing: false, + }), + &[Action::Request { route: 1 }], + ); + assert_actions( + front.step(Event::TrackEnded { + track: name("video"), + source: 100, + closing: false, + result: Err(Error::Session(crate::SessionError::Cancel)), + delivered: true, + }), + &[], + ); + // The claim went, whether a session took it over or it ran out its debounce. + assert_actions( + front.step(Event::Resolved { + route: 1, + result: Err(Refusal { + err: Error::Unroutable, + standing: false, + }), + }), + &[Action::Reselect], + ); + front + } + + /// The claim comes back on another session, a detour learned after the close: the + /// front resumes on it rather than end. + #[test] + fn a_closed_claim_resumes_on_the_session_that_serves_it_next() { + let mut front = orphaned(); + assert_actions( + front.step(Event::Selected { + best: Some(claim(2)), + serving_closing: false, + }), + &[Action::Request { route: 2 }], + ); + assert_actions( + front.step(Event::Resolved { + route: 2, + result: Ok(200), + }), + &[Action::Query { + track: name("video"), + source: 200, + }], + ); + assert_actions( + front.step(Event::TrackInfo { + track: name("video"), + source: 200, + closing: false, + result: Ok(info()), + retry: None, + }), + &[Action::Splice { + track: name("video"), + source: 200, + }], + ); + assert!(front.lost("video").is_none()); + } + + /// Nothing serves the claim again: the front ends, and its track with what ended the + /// copy, the session's error, not the later `Unroutable`. + #[test] + fn a_closed_claim_nothing_resumes_ends_with_the_sessions_error() { + let mut front = orphaned(); + assert_actions( + front.step(Event::Selected { + best: None, + serving_closing: false, + }), + &[Action::End { err: Error::Unroutable }], + ); + assert_eq!( + front.lost("video").map(|err| err.to_string()), + Some(Error::Session(crate::SessionError::Cancel).to_string()) + ); + } + fn refuse(front: &mut Front, route: u64) -> Vec { front.step(Event::Resolved { route, @@ -1594,6 +1720,7 @@ mod tests { route: 1, local, carried, + claim: false, }; let abort = [Action::Abort { track: name("video"), @@ -1846,6 +1973,7 @@ mod tests { route: 1, local, carried: false, + claim: false, }, 100, ); @@ -2725,8 +2853,9 @@ mod tests { best: Some(remote(1)), serving_closing: false, }, + // A claim, which its source closing leaves to be asked again. Event::Selected { - best: Some(remote(2)), + best: Some(claim(2)), serving_closing: false, }, Event::Selected { @@ -2749,6 +2878,7 @@ mod tests { }), }, Event::SourceClosed { source: 100 }, + Event::SourceClosed { source: 200 }, Event::TrackAssigned { track: name("v"), now: t0, diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 8959392598f..b2447f35fd7 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -4780,6 +4780,7 @@ impl FrontServe { route: entry.id, local: entry.local, carried: !entry.instance.is_minted(), + claim: entry.node.is_some(), }); // No candidate irreversibly ends this front. Leave the cache under the same lock // a lookup takes, before a route can return and a newcomer join the broadcast the diff --git a/rs/moq-net/tests/integration/catalog_handover.rs b/rs/moq-net/tests/integration/catalog_handover.rs index 2147dfd0166..82ab11c973b 100644 --- a/rs/moq-net/tests/integration/catalog_handover.rs +++ b/rs/moq-net/tests/integration/catalog_handover.rs @@ -68,8 +68,8 @@ enum Trigger { Unannounce, /// `R` gains a direct route to `P` that beats `A`. BetterRoute, - /// The session `A -> R` dies with no standby, then reconnects. The broadcast closes - /// with its last route, so a viewer's subscription ends rather than resuming. + /// The session `A -> R` dies with no standby, then reconnects within the claim's + /// debounce: the claim comes back on the new session, and a viewer resumes on it. Reconnect, } @@ -249,15 +249,6 @@ impl Reader { assert_eq!(self.take(expected.len(), who).await, expected, "{who}"); } - /// The subscription fails or ends, rather than hanging or delivering more. - async fn expect_end(&mut self, who: &str) { - match kio::clock::timeout(READ_TIMEOUT, self.rx.next()).await { - Ok(None | Some(Err(_))) => {} - Ok(Some(Ok(frame))) => panic!("{who}: delivered {frame:?} past its route"), - Err(_) => panic!("{who}: hung instead of ending"), - } - } - /// Nothing more has arrived: no duplicate frame or group trails. fn expect_quiet(&mut self, who: &str) { if let Ok(extra) = self.rx.try_recv() { @@ -301,9 +292,6 @@ async fn handover(version: &str, trigger: Trigger, holder: Holder) { }; topology.trigger(trigger).await; - if let (Trigger::Reconnect, Some(mut ended)) = (trigger, viewer.take()) { - ended.expect_end(&format!("{tag}: viewer")).await; - } topology.write("delta2"); if let Some(viewer) = &mut viewer { diff --git a/rs/moq-net/tests/integration/route_audit/contest.rs b/rs/moq-net/tests/integration/route_audit/contest.rs index fc44d30f45e..c40c7da872c 100644 --- a/rs/moq-net/tests/integration/route_audit/contest.rs +++ b/rs/moq-net/tests/integration/route_audit/contest.rs @@ -3,7 +3,7 @@ //! one Instance to the other mid-stream. moq-transport and lite-06 publishers carry no //! Instance, so each relay that takes their announcement mints one, and the same holds. -use super::{Feed, Join, Mesh, Source, detour_lost, readers, reports, ring, shown}; +use super::{Feed, Join, Mesh, Source, readers, reports, ring, shown}; use crate::path_routes::{lite07, quiesce}; use moq_net::Version; @@ -91,11 +91,6 @@ async fn opposite_claims_each_serve_their_nearest() { let reports = reports(&mut readers, since, "opposite claims, R0-R1 cut"); for (i, reader) in readers.iter_mut().enumerate() { let report = &reports[i]; - if i == 1 { - // The session `R1` read over closed, and its detour arrives a round trip later. - assert!(detour_lost(report, true), "R1: {report:?}"); - continue; - } assert!(report.seamless(), "R{i}: {report:?}"); if let (Some(before), Some(now)) = (near[i], after[i]) && before != now @@ -142,11 +137,6 @@ async fn unversioned(versions: [Version; 2]) { let cut = reports(&mut readers, since, &format!("{versions:?}, R0-R1 cut")); for (i, reader) in readers.iter_mut().enumerate() { let report = &cut[i]; - if i == 1 { - // The session `R1` read over closed, and its detour arrives a round trip later. - assert!(detour_lost(report, true), "{versions:?}: R1: {report:?}"); - continue; - } assert!(report.seamless(), "{versions:?}: R{i}: {report:?}"); assert_eq!(reader.tag().as_deref(), Some(nearest[i]), "{versions:?}: R{i} reads on"); } @@ -161,8 +151,7 @@ async fn unversioned(versions: [Version; 2]) { let report = &lost[i]; assert!(report.clean(), "{versions:?}: R{i}: {report:?}"); assert_eq!(reader.tag().as_deref(), Some("B"), "{versions:?}: R{i} ends on B"); - // `R1` already restarted onto `B` when its detour was lost. - let restarts = usize::from(nearest[i] == "A" && i != 1); + let restarts = usize::from(nearest[i] == "A"); assert_eq!(report.runs.len(), 1 + restarts, "{versions:?}: R{i}: {report:?}"); } } diff --git a/rs/moq-net/tests/integration/route_audit/failures.rs b/rs/moq-net/tests/integration/route_audit/failures.rs index 34fe4f7a113..2b5f3e5249b 100644 --- a/rs/moq-net/tests/integration/route_audit/failures.rs +++ b/rs/moq-net/tests/integration/route_audit/failures.rs @@ -2,7 +2,7 @@ //! its neighbours notice seconds apart, failures at one instant on a mesh larger than any //! exhaustive check, and no group crossing a link twice for one subscription. -use super::{Feed, Join, Mesh, Source, detour_lost, measured, readers, reports, ring, shown}; +use super::{Feed, Join, Mesh, Source, measured, readers, reports, ring, shown}; use crate::path_routes::{cut, lite07, quiesce}; use crate::support::Rng; @@ -10,10 +10,9 @@ use std::collections::BTreeSet; use std::time::Duration; /// A ring of `count` relays with a publisher on `R0`, the link from `R1` to `R2` priced -/// `r1_r2`. The link from `R0` to `R1` closes, and `R1` resumes the way round the ring. -/// Unless `held`, every other route to `R0` ran through `R1`, so `R1` held none at the cut: -/// its detour arrives a round trip later, once `R2` hears the closed session named. -async fn detour(count: usize, r1_r2: u64, held: bool) { +/// `r1_r2`. The link from `R0` to `R1` closes, and every reader, `R1`'s included, reads on +/// the way round the ring, every group once. +async fn detour(count: usize, r1_r2: u64) { let mut mesh = Mesh::new(count); for i in 0..count { let cost = if i == 1 { r1_r2 } else { 1 }; @@ -30,10 +29,6 @@ async fn detour(count: usize, r1_r2: u64, held: bool) { quiesce().await; let case = format!("ring of {count}, R1-R2 at {r1_r2}, R0-R1 cut"); for (i, report) in reports(&mut readers, since, &case).iter().enumerate() { - if i == 1 && !held { - assert!(detour_lost(report, false), "{case}: R1: {report:?}"); - continue; - } assert!(report.seamless(), "{case}: R{i}: {report:?}"); } assert_eq!(mesh.doubled(), [], "ring of {count}, R1-R2 at {r1_r2}"); @@ -44,15 +39,17 @@ async fn detour(count: usize, r1_r2: u64, held: bool) { #[kio::test] async fn a_relay_resumes_on_a_detour_it_already_holds() { for (count, r1_r2) in [(3, 1), (4, 3), (5, 4)] { - detour(count, r1_r2, true).await; + detour(count, r1_r2).await; } } -/// `R1` holds no other route to `R0` when its session closes. +/// `R1` holds no other route to `R0` when its session closes: every other one ran through +/// `R1`, so its detour arrives a round trip later, once `R2` hears the closed session named. +/// `R1`'s front rides through the claim's debounce until it does. #[kio::test] async fn a_relay_resumes_on_a_detour_learned_after_the_cut() { for count in [4, 5, 6] { - detour(count, 1, false).await; + detour(count, 1).await; } } diff --git a/rs/moq-net/tests/integration/route_audit/mod.rs b/rs/moq-net/tests/integration/route_audit/mod.rs index d7cd920b515..9137b1dcd26 100644 --- a/rs/moq-net/tests/integration/route_audit/mod.rs +++ b/rs/moq-net/tests/integration/route_audit/mod.rs @@ -439,18 +439,6 @@ impl Reader { } } -/// Pinned, not endorsed, until [the detour fix](/quest/m2/multicast/detour-resume.md): a relay -/// whose serving session closes before the detour to the same Instance reaches it, one -/// round trip later, ends its front. Its reader restarts cleanly onto another Instance -/// when one serves the path (`other`), and otherwise reads nothing more. The fix makes -/// each such reader seamless, and each case calling this changes with it. -pub(super) fn detour_lost(report: &Report, other: bool) -> bool { - match other { - true => report.clean() && report.runs.len() == 2, - false => report.after == 0, - } -} - /// What a reader saw across a failure. #[derive(Clone, Debug, Default)] pub(super) struct Report { diff --git a/rs/moq-net/tests/integration/route_cluster.rs b/rs/moq-net/tests/integration/route_cluster.rs index 228362a5f19..3ade39a7b51 100644 --- a/rs/moq-net/tests/integration/route_cluster.rs +++ b/rs/moq-net/tests/integration/route_cluster.rs @@ -463,8 +463,8 @@ impl Lan { /// publisher only over the CDN's own expensive link, never through the LAN. /// /// At the cut `lan[0]` moves onto the route through the CDN at once: its reader's -/// announcement stays, available. The closed session still ends the subscription it -/// carried, so a fresh one reads on through the CDN. +/// announcement stays, available, and the subscription the closed session carried reads on +/// through the CDN. #[kio::test] async fn a_partitioned_lan_reaches_itself_through_the_cdn_and_heals() { let mut lan = Lan::new().await; @@ -509,15 +509,6 @@ async fn a_partitioned_lan_reaches_itself_through_the_cdn_and_heals() { && moved.at.saturating_duration_since(cut_at) < DEBOUNCE, "through the CDN at once: {partitioned:?}" ); - // The closed session ends the subscription it carried, and a fresh one joins at the - // latest group, the one already read. - let ended = kio::clock::timeout(Duration::from_secs(1), rx.next()).await; - assert!( - matches!(ended, Ok(Some((_, Err(_))))), - "the closed session ended the track: {ended:?}" - ); - let (_broadcast, mut rx) = subscribe(&lan.lan[0], "local").await; - read_group(&mut rx, "O", 0).await; local.group(1); read_group(&mut rx, "O", 1).await; From 8a275b578e25c8a151d155aeb826e6c0df6fb452 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 00:13:44 -0800 Subject: [PATCH 3/7] fix(moq-net): a closed claim's next session refusing ends the track with its verdict The error a copy died of when its source closed is reported only if the front ends with nothing resuming the track, never in place of the next source's refusal, such as a copy framing the track differently. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/front.rs | 41 +++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index 93a8a69214f..61f53bb2419 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -201,6 +201,10 @@ struct Track { /// re-splice is awaited. A refusal of that re-splice, or the front ending first, reports /// this instead: the source closing is what ended the track. lost: Option, + /// What the copy died of when its source closed under it with nothing serving the + /// track. A front ending before another source splices it reports this, but unlike + /// [`Self::lost`] a refusal does not: the next source's answer is its own verdict. + closed: Option, /// Whether anyone reads the track: a copy is only spliced in for a reader. wanted: bool, /// A detached source whose copy still feeds the track until the serving source's @@ -303,7 +307,8 @@ impl Front { /// Why `track`'s serving copy died, while its re-splice is still awaited. A track the /// front ends meanwhile ends with this rather than the front's end. pub(super) fn lost(&self, track: &str) -> Option { - self.tracks.get(track)?.lost.clone() + let track = self.tracks.get(track)?; + track.lost.clone().or_else(|| track.closed.clone()) } /// Whether the front is over. @@ -346,6 +351,7 @@ impl Front { refused: HashSet::new(), refusal: None, lost: None, + closed: None, wanted: false, ended: None, draining: None, @@ -613,6 +619,7 @@ impl Front { Ok(()) => { track.draining = None; track.lost = None; + track.closed = None; track.stranded = None; track.state = TrackState::Spliced { source }; actions.push(Action::Splice { @@ -667,7 +674,7 @@ impl Front { && serving.is_none() && track.ended.is_none() { - track.lost.get_or_insert(err); + track.closed.get_or_insert(err); } return; } @@ -916,6 +923,7 @@ impl Front { TrackState::Idle | TrackState::Querying { .. } => { track.state = TrackState::Parked { since: now }; track.lost = None; + track.closed = None; // An ended track has nothing to drop. if track.ended.is_none() { actions.push(Action::Park { track: name }); @@ -1334,6 +1342,35 @@ mod tests { assert!(front.lost("video").is_none()); } + /// The session that serves the claim next frames the track differently: its verdict + /// ends the track, not the closed session's error. + #[test] + fn a_closed_claims_next_session_refusing_ends_with_its_verdict() { + let mut front = orphaned(); + front.step(Event::Selected { + best: Some(claim(2)), + serving_closing: false, + }); + front.step(Event::Resolved { + route: 2, + result: Ok(200), + }); + assert_actions( + front.step(Event::TrackInfo { + track: name("video"), + source: 200, + closing: false, + result: Ok(info().with_compression(track::Compression::Deflate)), + retry: None, + }), + &[Action::Abort { + track: name("video"), + err: Error::Unsupported, + retry: None, + }], + ); + } + /// Nothing serves the claim again: the front ends, and its track with what ended the /// copy, the session's error, not the later `Unroutable`. #[test] From 4faec8a67550c4aec0e7112ac6e6400d3624ff93 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 01:09:52 -0800 Subject: [PATCH 4/7] quest: moq-tokio's fleet drain test holds under load Co-Authored-By: Claude Opus 5.5 --- .../m0/test-flakes-3/fleet-drain-handover.md | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 quest/m0/test-flakes-3/fleet-drain-handover.md diff --git a/quest/m0/test-flakes-3/fleet-drain-handover.md b/quest/m0/test-flakes-3/fleet-drain-handover.md new file mode 100644 index 00000000000..b93689ca8f2 --- /dev/null +++ b/quest/m0/test-flakes-3/fleet-drain-handover.md @@ -0,0 +1,23 @@ +# [S] moq-tokio's fleet drain test holds under load + +## Goal + +`moq-tokio`'s `connection::tests::a_fleet_drain_redials_through_a_fresh_resolve` +passes however loaded the machine is. + +## Plan + +It failed once under a full `just check` on 2026-11-19 +([moq-dev/moq#9154](https://github.com/moq-dev/moq/pull/9154)): the +subscription ended with `Session(GoawayTimeout)` instead of reading `g1`. +Alone and under 96 parallel copies it passed 816 of 816. + +The test runs on real TCP and the wall clock: the drained session closes at a +500ms handover cap, and the test assumes the client dials the second relay and +moves the track onto it within that. Cutting the cap to 1ms reproduces the +failure every time, on `main` and on #9154 alike, so a loaded machine that +takes longer than 500ms to dial loses the race. + +Recommended: drive the handover cap off a paused or manual clock (see this +epic's rules), or assert on the second session serving the track before the +cap may run out, rather than raising the cap. From 6904f1fde5b1f1b680c94babd6846b544cb418fc Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 02:48:19 -0800 Subject: [PATCH 5/7] fix(moq-net): a front waits on a closed claim rather than ask it again Asking a claim's route again after its source closed could mint a fresh source from the closing session, whose refusal then aborted the track. The route stays excluded, and a selection that finds nothing waits while an excluded claim is still in the table, until another session serves it or its debounce runs out. Co-Authored-By: Claude Opus 5.5 --- .../m0/test-flakes-3/fleet-drain-handover.md | 23 -------- rs/moq-net/src/model/front.rs | 53 +++++++++++-------- rs/moq-net/src/model/origin.rs | 9 ++-- 3 files changed, 35 insertions(+), 50 deletions(-) delete mode 100644 quest/m0/test-flakes-3/fleet-drain-handover.md diff --git a/quest/m0/test-flakes-3/fleet-drain-handover.md b/quest/m0/test-flakes-3/fleet-drain-handover.md deleted file mode 100644 index b93689ca8f2..00000000000 --- a/quest/m0/test-flakes-3/fleet-drain-handover.md +++ /dev/null @@ -1,23 +0,0 @@ -# [S] moq-tokio's fleet drain test holds under load - -## Goal - -`moq-tokio`'s `connection::tests::a_fleet_drain_redials_through_a_fresh_resolve` -passes however loaded the machine is. - -## Plan - -It failed once under a full `just check` on 2026-11-19 -([moq-dev/moq#9154](https://github.com/moq-dev/moq/pull/9154)): the -subscription ended with `Session(GoawayTimeout)` instead of reading `g1`. -Alone and under 96 parallel copies it passed 816 of 816. - -The test runs on real TCP and the wall clock: the drained session closes at a -500ms handover cap, and the test assumes the client dials the second relay and -moves the track onto it within that. Cutting the cap to 1ms reproduces the -failure every time, on `main` and on #9154 alike, so a loaded machine that -takes longer than 500ms to dial loses the race. - -Recommended: drive the handover cap off a paused or manual clock (see this -epic's rules), or assert on the second session serving the track before the -cap may run out, rather than raising the cap. diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index 61f53bb2419..f8971055f67 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -45,9 +45,8 @@ pub(super) struct Candidate { /// a carried instance says the answer is the same stream. A minted one names the /// announcement, and the route could answer with whatever stream now wins upstream. pub carried: bool, - /// Whether the route is a lite-07 claim. A session that closes leaves its claims in - /// the table, unavailable, until another session serves them or the claim's debounce - /// runs out, and a request through one waits for either. + /// Whether the route is a lite-07 claim, which the route layer keeps in the table + /// through its debounce once its session closes; see [`Front::waiting`]. pub claim: bool, } @@ -244,6 +243,8 @@ pub(super) struct Front { upstream: Option, /// Routes excluded from selection: their source ended while still advertised. excluded: HashSet, + /// The excluded routes that are claims, while the table still shows them. + orphans: HashSet, /// Whether the parked requesters were resolved (the first source attached). resolved: bool, tracks: BTreeMap, Track>, @@ -272,6 +273,7 @@ impl Front { serving_closing: false, upstream: None, excluded: HashSet::new(), + orphans: HashSet::new(), resolved: false, tracks: BTreeMap::new(), parked: BTreeSet::new(), @@ -292,6 +294,16 @@ impl Front { /// Forget excluded routes that left the table (a reconnect is a fresh entry). pub(super) fn retain_routes(&mut self, standing: impl Fn(u64) -> bool) { self.excluded.retain(|route| standing(*route)); + self.orphans.retain(|route| self.excluded.contains(route)); + } + + /// Whether a selection that finds no route leaves the front waiting rather than ending. + /// A claim whose source closed with its session stays in the table, unavailable, until + /// another session serves it, a detour that may be learned only a round trip later and + /// always arrives as another route, or until its debounce runs out. Its own route is + /// never asked again: it would re-request the broadcast that just ended. + pub(super) fn waiting(&self) -> bool { + !self.orphans.is_empty() } /// The route the attached source came through, if any. @@ -443,6 +455,8 @@ impl Front { self.upstream = Some(candidate); actions.push(Action::Request { route: candidate.route }); } + // Nothing serves the path yet, but a claim it lost may still come back on a detour. + None if self.waiting() => self.upstream = None, // Nothing serves the path: the front is over, even with a live source. Its // route left the table and nothing replaced it, so it serves nobody new; // a later announcement gets a fresh front. @@ -528,10 +542,10 @@ impl Front { } // A standing route can outlive the source it produced. Asking it again // would re-request the broadcast that just ended; another route may still - // resume it. A claim is asked again: the request waits until another session - // serves the claim, which the front then resumes on, or the claim goes. - if !candidate.claim { - self.excluded.insert(candidate.route); + // resume it. + self.excluded.insert(candidate.route); + if candidate.claim { + self.orphans.insert(candidate.route); } self.serving = None; self.serving_closing = false; @@ -1258,8 +1272,8 @@ mod tests { } } - /// A front whose claim's session closed under `video` before its copy failed, and that - /// asks the claim again, which waits until another session serves it or the claim goes. + /// A front whose claim's session closed under `video` before its copy failed, with no + /// other route yet: it waits on the claim the table still shows, never asking it again. fn orphaned() -> Front { let mut front = serving(claim(1), 100); assert_actions( @@ -1272,13 +1286,14 @@ mod tests { Action::Reselect, ], ); - assert!(front.excluded_routes().is_empty()); + assert_eq!(front.excluded_routes(), &HashSet::from([1])); + assert!(front.waiting()); assert_actions( front.step(Event::Selected { - best: Some(claim(1)), + best: None, serving_closing: false, }), - &[Action::Request { route: 1 }], + &[], ); assert_actions( front.step(Event::TrackEnded { @@ -1290,17 +1305,6 @@ mod tests { }), &[], ); - // The claim went, whether a session took it over or it ran out its debounce. - assert_actions( - front.step(Event::Resolved { - route: 1, - result: Err(Refusal { - err: Error::Unroutable, - standing: false, - }), - }), - &[Action::Reselect], - ); front } @@ -1376,6 +1380,9 @@ mod tests { #[test] fn a_closed_claim_nothing_resumes_ends_with_the_sessions_error() { let mut front = orphaned(); + // The claim's debounce ran out: it left the table. + front.retain_routes(|_| false); + assert!(!front.waiting()); assert_actions( front.step(Event::Selected { best: None, diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index b2447f35fd7..4333699744e 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -4782,10 +4782,11 @@ impl FrontServe { carried: !entry.instance.is_minted(), claim: entry.node.is_some(), }); - // No candidate irreversibly ends this front. Leave the cache under the same lock - // a lookup takes, before a route can return and a newcomer join the broadcast the - // end actions are about to close. Existing requesters keep their original verdict. - if best.is_none() { + // No candidate irreversibly ends this front, unless it waits on a claim. Leave the + // cache under the same lock a lookup takes, before a route can return and a newcomer + // join the broadcast the end actions are about to close. Existing requesters keep + // their original verdict. + if best.is_none() && !self.front.waiting() { table .fronts .remove_if(&(self.path.clone(), self.horizon, self.pin.clone()), |front| { From 86f39ee5ccd98c4a2885207825a8ce5176035a72 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 05:16:21 -0800 Subject: [PATCH 6/7] fix(moq-net): a waiting front's late copy and a route's refusal keep their verdicts A copy the closed source answered after its close, dying while the front waits on its claim, now waits for the detour instead of aborting the track. A route that serves the claim next refusing the broadcast ends every track with that refusal, not the closed session's error. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/front.rs | 88 ++++++++++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index f8971055f67..c7bf27a239e 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -490,8 +490,12 @@ impl Front { // The winning route's answer is final, even while another source serves: // asking a sibling or a shorter prefix instead would turn one refusal into // a request per candidate. + // A closed source's error gives way to it too: the route's answer is its own verdict. Err(Refusal { err, standing: true }) => { self.upstream = None; + for track in self.tracks.values_mut() { + track.closed = None; + } self.end(err, actions); } } @@ -656,6 +660,8 @@ impl Front { ) { let strands = self.strands(); let serving = self.serving.map(|(serving, _)| serving); + // Nothing serves or is being asked, but a claim the front lost may still resume it. + let waiting = serving.is_none() && self.upstream.is_none() && self.waiting(); let Some(track) = self.tracks.get_mut(&name) else { return; }; @@ -721,6 +727,13 @@ impl Front { track.lost = Some(err); self.fail_over(name, actions); } + // A copy the closed source answered after its close died while the front waits on + // the claim it served: as for a draining copy, what it died of ends the track + // unless a session that serves the claim next resumes it. + Err(err) if delivered && waiting => { + track.state = TrackState::Idle; + track.closed.get_or_insert(err); + } // The route lost the content, which another route of the instance continues: // wait for one rather than ask this route again, which can only refuse. A copy // drained from a replaced source leaves the track to the serving one. @@ -1396,6 +1409,79 @@ mod tests { ); } + /// The route that serves the claim next refuses the broadcast: every track ends with + /// that refusal, as its requesters do, not the closed session's error. + #[test] + fn a_closed_claims_next_route_refusing_ends_with_its_verdict() { + let mut front = orphaned(); + front.step(Event::Selected { + best: Some(claim(2)), + serving_closing: false, + }); + assert_actions(refuse(&mut front, 2), &[Action::End { err: Error::NotFound }]); + assert!(front.lost("video").is_none()); + } + + /// The closed source answers a query it was asked before its close, and the copy it + /// splices then dies while the front waits on its claim: the track waits too, and the + /// session that serves the claim next resumes it. + #[test] + fn a_copy_spliced_after_its_claim_closed_dying_waits_for_the_detour() { + let mut front = querying(claim(1), 100); + front.step(Event::SourceClosed { source: 100 }); + assert_actions( + front.step(Event::Selected { + best: None, + serving_closing: false, + }), + &[], + ); + assert_actions( + front.step(Event::TrackInfo { + track: name("video"), + source: 100, + closing: true, + result: Ok(info()), + retry: None, + }), + &[Action::Splice { + track: name("video"), + source: 100, + }], + ); + assert_actions( + front.step(Event::TrackEnded { + track: name("video"), + source: 100, + closing: true, + result: Err(Error::Session(crate::SessionError::Cancel)), + delivered: true, + }), + &[], + ); + assert_eq!( + front.lost("video").map(|err| err.to_string()), + Some(Error::Session(crate::SessionError::Cancel).to_string()) + ); + assert_actions( + front.step(Event::Selected { + best: Some(claim(2)), + serving_closing: false, + }), + &[Action::Request { route: 2 }], + ); + assert_actions( + front.step(Event::Resolved { + route: 2, + result: Ok(200), + }), + &[Action::Query { + track: name("video"), + source: 200, + }], + ); + } + fn refuse(front: &mut Front, route: u64) -> Vec { front.step(Event::Resolved { route, @@ -2897,7 +2983,7 @@ mod tests { best: Some(remote(1)), serving_closing: false, }, - // A claim, which its source closing leaves to be asked again. + // A claim, which its source closing leaves excluded but waited on. Event::Selected { best: Some(claim(2)), serving_closing: false, From 221abc5e05f140c6f95c6db86785ef593ed23431 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 06:53:49 -0800 Subject: [PATCH 7/7] fix(moq-net): a closed source's late copy dying mid-switch leaves the route its verdict A copy the closed source answered after its close, dying while the next route is asked, recorded its session error as a serving copy's loss. That route refusing the broadcast, or its source refusing the track, then ended the track with the closed session's error instead of its own verdict. It is now kept as what the closed source's copy died of, as while the front waits on its claim. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/front.rs | 89 ++++++++++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index c7bf27a239e..25c07ea91c3 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -666,6 +666,8 @@ impl Front { return; }; let spliced = track.state == (TrackState::Spliced { source }); + // A copy the closed source answered after its close: neither served nor drained. + let late = serving != Some(source) && track.draining != Some(source); // A track finishing on a copy other than the serving source's asks that source for // one anew, since finishing lets go of a query in flight: the group reads still // holding the track continue from it, as after a later source change. @@ -715,7 +717,8 @@ impl Front { // route a switch already under way moves to, whatever the error, since the // route it leaves may be what failed. A copy still read from a source that // closed during the switch has no serving source to judge the instance by, so - // the route the switch asks speaks for it. + // the route the switch asks speaks for it. What that copy died of is no + // serving copy's loss: the route's answer is its own verdict, as while waiting. Err(err) if delivered && (err.transient() || self.upstream.is_some()) @@ -724,7 +727,11 @@ impl Front { .is_some_and(|candidate| candidate.carried) => { track.state = TrackState::Idle; - track.lost = Some(err); + if late { + track.closed.get_or_insert(err); + } else { + track.lost = Some(err); + } self.fail_over(name, actions); } // A copy the closed source answered after its close died while the front waits on @@ -1482,6 +1489,84 @@ mod tests { ); } + /// A front whose claim closed, asking the route that serves it next, with a copy the + /// closed source answered after its close spliced in and then dead. + fn late_copy_died_while_asking() -> Front { + let mut front = querying(claim(1), 100); + front.step(Event::SourceClosed { source: 100 }); + assert_actions( + front.step(Event::Selected { + best: Some(claim(2)), + serving_closing: false, + }), + &[Action::Request { route: 2 }], + ); + assert_actions( + front.step(Event::TrackInfo { + track: name("video"), + source: 100, + closing: true, + result: Ok(info()), + retry: None, + }), + &[Action::Splice { + track: name("video"), + source: 100, + }], + ); + assert_actions( + front.step(Event::TrackEnded { + track: name("video"), + source: 100, + closing: true, + result: Err(Error::Session(crate::SessionError::Cancel)), + delivered: true, + }), + &[], + ); + front + } + + /// The late copy dies while the next route is asked, and that route refuses the + /// broadcast: the track ends with the refusal, not the closed session's error. + #[test] + fn a_late_copy_dying_while_asking_yields_to_a_broadcast_refusal() { + let mut front = late_copy_died_while_asking(); + assert_actions(refuse(&mut front, 2), &[Action::End { err: Error::NotFound }]); + assert!(front.lost("video").is_none()); + } + + /// The late copy dies while the next route is asked, whose source then refuses the + /// track: the track aborts with that refusal, not the closed session's error. + #[test] + fn a_late_copy_dying_while_asking_yields_to_a_track_refusal() { + let mut front = late_copy_died_while_asking(); + assert_actions( + front.step(Event::Resolved { + route: 2, + result: Ok(200), + }), + &[Action::Query { + track: name("video"), + source: 200, + }], + ); + assert_actions( + front.step(Event::TrackInfo { + track: name("video"), + source: 200, + closing: false, + result: Err(Error::Unsupported), + retry: None, + }), + &[Action::Abort { + track: name("video"), + err: Error::Unsupported, + retry: None, + }], + ); + } + fn refuse(front: &mut Front, route: u64) -> Vec { front.step(Event::Resolved { route,