From bbc5ade7abcbb1c22701d83afa300ace9e3571d5 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 19:37:32 -0800 Subject: [PATCH 1/8] chore(quest): claim quest/m2/multicast/splice-stall.md Quest-Claim: 32d28bc0e9267bc3 From 4973b74ad7b1ca69f426cb59cfa16b7a19c4c566 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 21:27:47 -0800 Subject: [PATCH 2/8] fix(moq-net): a copy lost with its session holds the track for the next route A received source's session fails the copies it carries before it closes their broadcast. The front read the dying source as open, asked it again, and took the session's end as the track's verdict, so a relay told its readers the path was unroutable while it still showed the broadcast. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/error.rs | 8 +++ rs/moq-net/src/model/front.rs | 117 ++++++++++++++++++++++++++++++++- rs/moq-net/src/model/origin.rs | 67 +++++++++++++++++++ 3 files changed, 191 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/error.rs b/rs/moq-net/src/error.rs index a697914e9bd..1f40fbfae02 100644 --- a/rs/moq-net/src/error.rs +++ b/rs/moq-net/src/error.rs @@ -597,6 +597,14 @@ impl Error { pub(crate) fn transient(&self) -> bool { self.retry() == Retry::Same } + + /// Whether the session that carried the failed request ended: the route it came through + /// is gone, which a relay passes on as `Unroutable`. A session fails the copies it carries + /// before it closes the broadcasts they came from, so this says the source is closing + /// before its broadcast does. + pub(crate) fn session_ended(&self) -> bool { + matches!(self, Self::Session(_) | Self::Transport(_) | Self::IdleTimeout) + } } /// What a caller does after a request fails, as [`Error::retry`] answers it. diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index 25c07ea91c3..faa9ff42402 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -583,6 +583,7 @@ impl Front { result: Result)>, actions: &mut Vec, ) { + let closing = self.closing(source, closing, result.as_ref().err().map(|(err, _)| err)); let serving = self.serving.is_some_and(|(serving, _)| serving == source); let strands = self.strands(); let Some(track) = self.tracks.get_mut(&name) else { @@ -658,6 +659,7 @@ impl Front { delivered: bool, actions: &mut Vec, ) { + let closing = self.closing(source, closing, result.as_ref().err()); let strands = self.strands(); let serving = self.serving.map(|(serving, _)| serving); // Nothing serves or is being asked, but a claim the front lost may still resume it. @@ -732,7 +734,10 @@ impl Front { } else { track.lost = Some(err); } - self.fail_over(name, actions); + // A closing source is asked nothing more: its close brings the next route. + if !closing { + self.fail_over(name, actions); + } } // A copy the closed source answered after its close died while the front waits on // the claim it served: as for a draining copy, what it died of ends the track @@ -867,6 +872,18 @@ impl Front { actions.push(Action::Query { track: name, source }); } + /// Whether `source` is closing, as the driver read from its broadcast or as `err`, what it + /// just failed with, says first: a received source is its session's, which fails the + /// copies it carries before it closes their broadcasts. A local source's error is its + /// publisher's word, and its broadcast need not close. + fn closing(&self, source: u64, closing: bool, err: Option<&Error>) -> bool { + closing + || (err.is_some_and(Error::session_ended) + && self + .serving + .is_some_and(|(serving, candidate)| serving == source && !candidate.local)) + } + /// Whether an unroutable end from the serving source judges its route rather than the /// track, so the track waits for the table to move. Only a received route can be stale: /// a local publisher's word is the track's verdict, and no news about it is in flight. A @@ -1715,6 +1732,104 @@ mod tests { ); } + /// A received source's session fails the copies it carries before it closes their + /// broadcast. A copy lost with its session's end is that source closing, even while its + /// broadcast reads as open and the table still shows its route: the track asks it nothing + /// more and takes no answer of its as a verdict, but waits for the next route. + #[test] + fn a_copy_lost_with_its_session_waits_for_the_next_route() { + let mut front = serving(remote(1), 100); + assert_actions( + front.step(Event::TrackEnded { + track: name("video"), + source: 100, + closing: false, + result: Err(Error::IdleTimeout), + delivered: true, + }), + &[], + ); + // The table moving asks again, and the session's end answers. + assert_actions( + front.step(Event::Selected { + best: Some(remote(1)), + serving_closing: false, + }), + &[Action::Query { + track: name("video"), + source: 100, + }], + ); + assert_actions( + front.step(Event::TrackInfo { + track: name("video"), + source: 100, + closing: false, + result: Err(Error::IdleTimeout), + retry: None, + }), + &[], + ); + assert_actions( + front.step(Event::SourceClosed { source: 100 }), + &[ + Action::Detach { + source: 100, + drain: true, + }, + Action::Reselect, + ], + ); + assert_actions( + front.step(Event::Selected { + best: Some(remote(2)), + serving_closing: false, + }), + &[Action::Request { route: 2 }], + ); + assert_actions( + front.step(Event::Resolved { + route: 2, + result: Ok(200), + }), + &[Action::Query { + track: name("video"), + source: 200, + }], + ); + assert!(!front.ended()); + + // A local source's error is its publisher's word, whose broadcast need not close: the + // track asks again, and the answer is a verdict. + let mut front = serving(local(1), 100); + let query = [Action::Query { + track: name("video"), + source: 100, + }]; + let lost = Event::TrackEnded { + track: name("video"), + source: 100, + closing: false, + result: Err(Error::IdleTimeout), + delivered: true, + }; + assert_actions(front.step(lost), &query); + assert_actions( + front.step(Event::TrackInfo { + track: name("video"), + source: 100, + closing: false, + result: Err(Error::IdleTimeout), + retry: None, + }), + &[Action::Abort { + track: name("video"), + err: Error::IdleTimeout, + retry: None, + }], + ); + } + /// A copy reset before delivering, seen only once its source began closing, is no /// refusal, yet ends the track with the reset rather than `Unroutable` when nothing /// replaces the source, or when its replacement refuses. The driver reports it as the copy's end once spliced, or as the answer to diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 38230904420..42bbb3b593f 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -16608,6 +16608,73 @@ mod tests { drop(upstream); } + /// A session ending aborts the copies it carries before it closes the broadcasts they + /// came from. A copy failing with its session's end is that source closing, whatever its + /// broadcast says yet: the track waits for the next route, rather than ask the dying + /// source again and take its answer as the track's verdict. A relay ending it there would + /// tell its own readers the path is unroutable while it still shows the broadcast. + #[kio::test] + #[cfg(feature = "async")] + async fn a_copy_lost_with_its_session_waits_for_the_next_route() { + for err in [ + Error::IdleTimeout, + Error::Transport("reset".into()), + Error::Session(crate::SessionError::Cancel), + ] { + let (producer, server, upstream, old, mut subscription) = remote_front().await; + // A longer way to the same instance, which wins once the first route's source closes. + let detour = producer + .forward( + "room", + RouteInfo::new().with_instance(instance(), 0).with_hops(hops(&[20, 30])), + ) + .unwrap(); + // The dying session refuses whatever it is asked. + let mut asked = upstream.requests(); + old.abort(err.clone()).unwrap(); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + let again = asked.requested_track().now_or_never(); + let asked_again = again.is_some(); + if let Some(request) = again { + request.expect("request").reject(err.clone()); + } + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + if let Poll::Ready(Err(ended)) = subscription.poll_recv_group(&kio::Waiter::noop()) { + panic!("{err}: the track ended with its route's loss: {ended}"); + } + assert!(!asked_again, "{err}: asked the dying source again"); + + // The session finishes closing, and the detour takes over. + drop(asked); + upstream.close(); + let replacement = broadcast::Info::new().produce(); + let mut handler = replacement.requests(); + queued(&detour).await.serve(&replacement); + let request = kio::clock::timeout(Duration::from_secs(1), handler.requested_track()) + .await + .expect("the front asked the detour") + .expect("request"); + assert_eq!(request.name(), "video"); + let resumed = request.accept(crate::track::Info::timed()); + // The same content: the reader already has the first group. + for payload in [b"before", b"after!"] { + let mut group = resumed.append_next().unwrap(); + group.write_frame(group.key_time(), payload.as_ref()).unwrap(); + group.finish().unwrap(); + } + let mut group = next_group(&mut subscription) + .await + .unwrap_or_else(|err| panic!("{err}: the track ended: {err}")) + .expect("a group from the detour"); + assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"after!"); + drop(server); + } + } + /// The reader leaves, parking the track. A fetch of a whole group the parked copy /// caches answers at once, without the front serving the copy again; an open group /// still waits on the front, and is served once the copy is back. From f0b98c09ebb83aee494fb87d5946574ea6038594 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 21:55:41 -0800 Subject: [PATCH 3/8] verify(failover): a front holds a track while a dead relay's route still shows Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-failover/SKILL.md | 5 +-- rs/moq-net/verify/failover/FailoverChain.tla | 34 ++++++++++++++------ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/.claude/skills/verify-failover/SKILL.md b/.claude/skills/verify-failover/SKILL.md index 670b6928ac1..43a932f47dd 100644 --- a/.claude/skills/verify-failover/SKILL.md +++ b/.claude/skills/verify-failover/SKILL.md @@ -27,7 +27,7 @@ quest gate just rs tla moq-net/verify/failover/FailoverChain.tla -config rs/moq- quest gate just rs tla moq-net/verify/failover/FailoverChain.tla -config rs/moq-net/verify/failover/Mesh.cfg -simulate num=200000 -depth 100 ``` -The first checks every rule with `E` failing over between `A` and `B`, one death, and one switch per relay, with a client that stays, so a deadlock is a playback hang: about 3.8M states. The second lets the client leave at any point, so a deadlock is a relay still holding a copy after it left; leaving is always possible there, which is why it cannot stand in for the first: about 4.1M states. Each takes about three minutes and passes at the sync commit. The third adds `B`'s route through `A`, so the route also moves at the middle hop, after `B`'s track ended too (`Action::Reroute`). It is too large to exhaust (past 139M distinct states at depth 38 after an hour, its queue still growing), so its `Simulate:` line samples 800k random behaviors in about 7 minutes, and Nightly samples new ones each run. An exhaustive run passed every state through depth 37. Before the reroute, a sample that size missed the depth-31 hang that BFS found after about 48M states, so a passing sample is weak evidence: check a change at the middle hop by running it without the `-simulate` arguments for as long as it takes. +The first checks every rule with `E` failing over between `A` and `B`, one death, and one switch per relay, with a client that stays, so a deadlock is a playback hang: about 6.0M states. The second lets the client leave at any point, so a deadlock is a relay still holding a copy after it left; leaving is always possible there, which is why it cannot stand in for the first: about 6.6M states. Each takes about three minutes and passes at the sync commit. The third adds `B`'s route through `A`, so the route also moves at the middle hop, after `B`'s track ended too (`Action::Reroute`). It is too large to exhaust (past 139M distinct states at depth 38 after an hour, its queue still growing), so its `Simulate:` line samples 800k random behaviors in about 7 minutes, and Nightly samples new ones each run. An exhaustive run passed every state through depth 37. Before the reroute, a sample that size missed the depth-31 hang that BFS found after about 48M states, so a passing sample is weak evidence: check a change at the middle hop by running it without the `-simulate` arguments for as long as it takes. ## Reading a failure @@ -35,7 +35,7 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `61f89568394638553a519d7d8cfc0019385dd3a0`, plus [moq-dev/moq#9048](https://github.com/moq-dev/moq/pull/9048), which mirrored the code since 88572c583e and kept a finished track a read holds past its linger, and [moq-dev/moq#9155](https://github.com/moq-dev/moq/pull/9155), which resumes the newest group past the frames held and lends them to the new copy (`Resume`, `Lend`). Run `git log 61f8956839.. -- rs/moq-net/src/model/resume.rs rs/moq-net/src/model/front.rs rs/moq-net/src/model/origin.rs rs/moq-net/src/model/track.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. +Sync commit: `61f89568394638553a519d7d8cfc0019385dd3a0`, plus [moq-dev/moq#9048](https://github.com/moq-dev/moq/pull/9048), which mirrored the code since 88572c583e and kept a finished track a read holds past its linger, and [moq-dev/moq#9155](https://github.com/moq-dev/moq/pull/9155), which resumes the newest group past the frames held and lends them to the new copy (`Resume`, `Lend`), and the splice-stall fix, which holds a track while a dead relay's route still shows, since a session fails its copies before it closes their sources (`Shows`, `Withdraw`). Run `git log 61f8956839.. -- rs/moq-net/src/model/resume.rs rs/moq-net/src/model/front.rs rs/moq-net/src/model/origin.rs rs/moq-net/src/model/track.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. | Code | Model | | --- | --- | @@ -47,6 +47,7 @@ Sync commit: `61f89568394638553a519d7d8cfc0019385dd3a0`, plus [moq-dev/moq#9048] | `rs/moq-net/src/model/resume.rs` `Recover::poll_fetch`, `Fetching`; `rs/moq-net/src/model/track.rs` `fetch_group` | `Ask` | | `rs/moq-net/src/model/front.rs` `track_ended`, `Action::Finish`; `rs/moq-net/src/model/origin.rs` `TrackIo::end` | `FrontFinish`, `Whole` | | `rs/moq-net/src/model/front.rs` failover and its unroutable end, `Action::Reroute` | `Switch`, `FrontFail` | +| `rs/moq-net/src/model/front.rs` `Front::closing`; `rs/moq-net/src/error.rs` `Error::session_ended` | `Shows`, `Withdraw`, `FrontFail` | | `rs/moq-net/src/model/resume.rs` `Route::answering`, `Phase::Finished` | `Answering`, `LetGo` | | `rs/moq-net/src/model/front.rs` `Action::Forget`; `rs/moq-net/src/model/track.rs` `TrackWeak::is_held`, `TrackWeak::poll_held_changed` | `Forget`, `Reads` | diff --git a/rs/moq-net/verify/failover/FailoverChain.tla b/rs/moq-net/verify/failover/FailoverChain.tla index dd562ca87d6..e2049ee5298 100644 --- a/rs/moq-net/verify/failover/FailoverChain.tla +++ b/rs/moq-net/verify/failover/FailoverChain.tla @@ -69,7 +69,7 @@ Max(S) == CHOOSE x \in S : \A y \in S : y <= x VARIABLES org, \* the origin's track: frames written per group, and whether it ended - alive, \* which relays are up + alive, \* each relay "up", "shown" (dead, its route still in the table), or "down" deaths, \* relays that died cp, \* cp[e]: the copy at e[1] of the track e[2] serves, and e[2]'s reader for it front, \* front[r]: the route r serves from, how its track ended, the route it @@ -173,7 +173,12 @@ Answering(u) == IF front[u].end = "ok" /\ Live(<>) THEN <> ELSE Serving(u) -Routable(w) == IF w = "O" THEN TRUE ELSE alive[w] /\ front[w].end # "err" +Routable(w) == IF w = "O" THEN TRUE ELSE alive[w] = "up" /\ front[w].end # "err" +\* A route still in the table: a dead relay's route stays until its session's close is +\* seen (a claim's debounce). Its copies have failed and nothing new is asked through +\* it, but a front holds a track rather than end it while it shows: a session fails the +\* copies it carries before it closes their sources, and that loss judges no track. +Shows(w) == Routable(w) \/ (w # "O" /\ alive[w] = "shown") \* A complete copy (`TrackState::is_complete` once every group below the end \* finished): its upstream ended it, and every group it asked holds every frame. @@ -215,7 +220,7 @@ Exhausted(e, g) == SrcFr(e[2], cp[e].src[g], g) <= cp[e].fr[g] Init == /\ org = [fr |-> [g \in Groups |-> -1], fin |-> FALSE] - /\ alive = [r \in Relays |-> TRUE] + /\ alive = [r \in Relays |-> "up"] /\ deaths = 0 \* The client subscribed through E, E through A, and A to the origin. /\ cp = [e \in Edges |-> @@ -314,7 +319,7 @@ Abort(e, g) == /\ SrcDead(u, cp[e].src[g], g) /\ Holders(u, e, g) = {} /\ \/ front[u].end = "err" - \/ front[u].end = "ok" /\ ~Live(Serving(u)) /\ \A w \in Up(u) : ~Routable(w) + \/ front[u].end = "ok" /\ ~Live(Serving(u)) /\ \A w \in Up(u) : ~Shows(w) \/ cp[e].src[g] = front[u].cur /\ Live(Serving(u)) /\ cp' = [cp EXCEPT ![e].bad = @ \cup {g}] /\ UNCHANGED <> @@ -363,12 +368,13 @@ FrontFinish(u) == /\ front' = [front EXCEPT ![u].end = "ok", ![u].last = front[u].cur] /\ UNCHANGED <> -\* No route is left: the front fails the track, and every reader with it. +\* No route is left, not even one still shown: the front fails the track, and every +\* reader with it. FrontFail(u) == /\ u \in Relays /\ front[u].end = "open" /\ ~Live(Serving(u)) \/ ~Routable(front[u].cur) - /\ \A w \in Up(u) : ~Routable(w) + /\ \A w \in Up(u) : ~Shows(w) /\ front' = [front EXCEPT ![u].end = "err"] /\ cp' = [e \in Edges |-> IF e[2] = u /\ Live(e) THEN [cp[e] EXCEPT !.st = "failed"] ELSE cp[e]] @@ -426,12 +432,13 @@ Forget(u) == /\ cp' = [e \in Edges |-> IF e[1] = u THEN Gone ELSE cp[e]] /\ UNCHANGED <> -\* A relay dies: copies downstream of it fail, and its own vanish. +\* A relay dies: copies downstream of it fail, and its own vanish. Its route still +\* shows until `Withdraw`. Die(r) == /\ r \in Mortal - /\ alive[r] + /\ alive[r] = "up" /\ deaths < MaxDeaths - /\ alive' = [alive EXCEPT ![r] = FALSE] + /\ alive' = [alive EXCEPT ![r] = "shown"] /\ deaths' = deaths + 1 /\ cp' = [e \in Edges |-> IF e[1] = r THEN Gone @@ -439,6 +446,12 @@ Die(r) == ELSE cp[e]] /\ UNCHANGED <> +\* A dead relay's route leaves the table. +Withdraw(r) == + /\ alive[r] = "shown" + /\ alive' = [alive EXCEPT ![r] = "down"] + /\ UNCHANGED <> + \* The client unsubscribes. Unsubscribe == /\ Leave @@ -469,7 +482,7 @@ Next == \/ \E w \in Up(u) : Switch(u, w) \/ LetGo(u, w) \/ \E g \in Groups : Ask(u, g) \/ \E e \in Edges : EndOk(e) - \/ \E r \in Mortal : Die(r) + \/ \E r \in Mortal : Die(r) \/ Withdraw(r) \/ Unsubscribe \/ Done @@ -499,5 +512,6 @@ TypeOK == /\ cp[e].bad \subseteq Groups /\ \A g \in Groups : cp[e].fr[g] \in -1 .. F /\ cp[e].lent[g] \in 0 .. F /\ \A r \in Relays : front[r].end \in {"open", "ok", "err"} + /\ \A r \in Relays : alive[r] \in {"up", "shown", "down"} ============================================================================= From 2fe8ca77f3d6e9a2a52286912f5ae53cb4ef1d4d Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 23:21:24 -0800 Subject: [PATCH 4/8] quest: splice-stall keeps what the copy-before-close fix did not settle; a front logs each step Co-Authored-By: Claude Opus 5.5 --- quest/m2/multicast/splice-stall.md | 113 +++++++++++++---------------- rs/moq-net/src/model/origin.rs | 8 +- 2 files changed, 58 insertions(+), 63 deletions(-) diff --git a/quest/m2/multicast/splice-stall.md b/quest/m2/multicast/splice-stall.md index f48499d04a6..e13ec101af9 100644 --- a/quest/m2/multicast/splice-stall.md +++ b/quest/m2/multicast/splice-stall.md @@ -19,70 +19,59 @@ or in the nightly ring meshes, once `[[known]]` no longer names this quest in Found by [the mesh replay](/test/mesh-replay/README.md). On main after [roll-echo-stall](https://github.com/moq-dev/moq/pull/9174) and [reopen-starve](https://github.com/moq-dev/moq/pull/9179), 19 of 20 -`reopen.toml` runs (ring4, 32 broadcasts, debug build) passed. The failing run -starved 2 reader paths. After -[resend](https://github.com/moq-dev/moq/pull/9155) landed, 2 of 3 passed. The -third shows the same strand: `a`'s reader got Unroutable for `c/5` as `a`'s -links reopened, then read nothing for 73s, across a re-follow of the path. -That run's 128 paths starved at the kill came while the host ran at 100% CPU, -so they are load, not this bug. - -What that run's logs show. `b`'s reader followed `d/0` and `d/3` through `a`. -The `a`-`b` link paused, and its session idle-timed out 10s later. At that -moment, relay `b` reset the reader's live subscriptions for exactly those two -paths with Unroutable (`subscribe error ... err=unroutable` in the reader's -log). The reader's follow logged no end and stayed online. Relay `b` never -withdrew or updated `d/0`, so the reader's front, which strands a track -refused as unroutable until its table moves (`model/front.rs` -`track_ended` and `refuse`, `Track::stranded`, `unstrand`), never asked again. -The other 30 paths at `b` switched to the copy through `c`. That copy handed -out about one group every 5s until the `a`-`b` link returned, then fed every -reader again. - -Reopen-starve's first plan named this strand: a refused track waits for -its table to move. #9179 removed the refusal between relays (a request follows -a claim only through a neighbour that holds it). The strand remains one hop -down, between a relay and its client. - -Open, and why it could not be settled yet: - -- Why relay `b` ended those two tracks Unroutable. The run had info logs only. - Candidates: `Front::selected` finding no candidate (#9179's `holds` filter - leaves none while a neighbour's copy of the claims lags its route) with no - orphaned claim to wait on, so the front ends `Unroutable`. Or the publisher's - node filter refusing. Reruns with debug logs on `lite::publisher`, - `lite::subscriber`, and `model` failed only on hosts saturated by other - work (100% CPU, relays at 5-30% of a core). There the measuring reader - itself starved on loopback, so those runs show nothing about this bug. -- A mock-time reproduction. A ring4 integration test on `peer_with_latency` - links passed: 8 to 32 broadcasts from `d`, readers on `a`, `b`, and `c` - over their own sessions, one link cut or paused. It also passed with the - cut swept from 0 to 1.5s after the readers start. The window needs timing - the mocked clock does not hit. - -Recommended fix, pending the cause: a hop must not answer a live subscription -with Unroutable while it keeps showing that broadcast available. Either the -front waits for a way as it does for an orphaned claim (`Front::waiting`), or -the hop updates its announcement so the downstream's table moves. Fixing only -the client to re-ask on a quiet table would need a timer and could loop on a -standing refusal. Rerun `/verify-failover`, since either changes the front -lifecycle it covers. - -Next steps: capture one failure with debug logs on a host that is not -saturated, and record per-relay CPU with it to tell load apart from logic. -Find which path in relay `b` produced the Unroutable. Write the test at the -lowest layer that shows it, likely a front unit test in `model/front.rs`. -Then fix it and rerun `reopen.toml` at least eight times. +`reopen.toml` runs (ring4, 32 broadcasts, debug build) passed. In the failing +run, `b`'s reader followed `d/0` and `d/3` through `a`. When the `a`-`b` +session idle-timed out, relay `b` reset the reader's live subscriptions for +exactly those two paths with Unroutable, and never withdrew or updated `d/0`. +The reader's front strands a track refused as unroutable until its table moves +(`model/front.rs` `refuse`, `Track::stranded`, `unstrand`), so it never asked +again. The other 30 paths switched to the copy through `c`. That run had info +logs only. + +Decided 2026-11-20 by the maintainer: + +- ✅ The hop holds (rejected: the reader re-asks on a timer, which adds a + backstop timer and can loop on a refusal that stands). A hop never answers a + live subscription with Unroutable while it still shows that broadcast + available. Its front either waits for a route, as it already does for an + orphaned claim (`Front::waiting`), or changes its announcement so the reader + asks again. +- ✅ The lite-07 cut waits on this fix. + +What [moq-dev/moq#9219](https://github.com/moq-dev/moq/pull/9219) did: + +- Closed one way a hop broke that rule. A lite session fails the copies it + carries before it closes the broadcasts they came from. A front that saw a + copy die with its session's end (`IdleTimeout`, `Session`, `Transport`) from a + source whose broadcast still read as open asked that source again and took + its answer as the track's verdict. Relayed, that end reads as Unroutable. + The front now counts a received serving source as closing on such an error + (`Front::closing`, `Error::session_ended`) and holds the track for the next + route. `/verify-failover` models the window: a dead relay's route still + shows until it leaves the table (`Shows`, `Withdraw`). +- Added a debug line for every front step (`moq_net::model`), so a failing run + shows how a front ended a track. + +That race was not confirmed as this failure. On an unloaded host, 59 base runs +and 24 fixed runs passed, and no reader got Unroutable during the single link +reopen in any of them. Holding the drain open 2s after the transport closed +did not make the base fail either. Traced relays show the source closing +before its copies end. Debug runs on the CI host at load 70 or more starve +readers by load alone: sessions to the detours idle-time out too. + +Next steps: capture one failure with +`RUST_LOG=info,moq_net::model=debug`, on a host under real but not saturating +load (relays below a core each), and record per-relay CPU with it. Rerun +`reopen.toml` on a debug build until a reader gets Unroutable during the +single link reopen; that alone marks the bug, starving or not. In the failing +relay's log, find the front step that ended that path. Then write the test at +the lowest layer that shows it, fix it as decided, rerun `/verify-failover`, +and rerun the replay at least 20 times before dropping the `[[known]]` entries +in `test/mesh-replay/script.toml`. Pauses are not part of this quest. A debug build pauses about as long on -lite-06 as on lite-07 under the same load: link 13-14s, relay reopen 24-35s, -kill 25-30s. A release build pauses the same on both versions: link 11-12s, -relay reopen 17-19s, kill 20-23s. The release 0.17.17 binary pauses 10s, 16s, -and 19s at a lower load. - -The lite-07 cut waits on it, as decided for reopen-starve on 2026-11-19 -(under the maintainer's delegation for lite-07 routing calls): it is a -starved reader after a link flap or a crash, which a deployment sees daily. +lite-06 as on lite-07 under the same load, and a release build the same on +both versions. Not a wire change. diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 42bbb3b593f..05f02c46b75 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -5063,7 +5063,13 @@ impl kio::Task for FrontServe { Event::TrackInfo { track, source, .. } => Some((track.clone(), *source)), _ => None, }; - for action in self.front.step(event) { + // Every transition, for telling how a front ended a track in a run that failed. + let traced = tracing::enabled!(tracing::Level::DEBUG).then(|| format!("{event:?}")); + let actions = self.front.step(event); + if let Some(event) = traced { + tracing::debug!(path = %self.path, %event, ?actions, "front step"); + } + for action in actions { match action { Action::Reselect => { let event = self.select(); From 4af6837350d9a5fd72d36ae449f705b311b0ae38 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:20:02 -0800 Subject: [PATCH 5/8] fix(moq-net): a lite session closes its sources before it fails their tracks A front reads whether a source is closing from its broadcast, and takes a failure from a source still open as the track's verdict. The lite subscriber could fail a copy, or a query waiting on TRACK_INFO, with the session's end before it closed the source, so a relay ended a track it could resume on the next route and told its readers Unroutable. Close the source first, from the session's state rather than the error (a stream's own reset can carry the same error): each run that fails a track while the session is ending (`TrackServe::close_if_ending`), and the driver's abort before it ends every subscription (`SubscriberDriver::abort`). This replaces the error-based `Front::closing` and `Error::session_ended`. Co-Authored-By: Claude Opus 5.5 --- quest/m2/multicast/splice-stall.md | 25 +++--- rs/moq-net/src/error.rs | 8 -- rs/moq-net/src/lite/subscriber.rs | 99 ++++++++++++++++++++++++ rs/moq-net/src/model/front.rs | 117 +---------------------------- rs/moq-net/src/model/origin.rs | 67 ----------------- 5 files changed, 114 insertions(+), 202 deletions(-) diff --git a/quest/m2/multicast/splice-stall.md b/quest/m2/multicast/splice-stall.md index e13ec101af9..d4fe1f5d0aa 100644 --- a/quest/m2/multicast/splice-stall.md +++ b/quest/m2/multicast/splice-stall.md @@ -40,23 +40,26 @@ Decided 2026-11-20 by the maintainer: What [moq-dev/moq#9219](https://github.com/moq-dev/moq/pull/9219) did: -- Closed one way a hop broke that rule. A lite session fails the copies it - carries before it closes the broadcasts they came from. A front that saw a - copy die with its session's end (`IdleTimeout`, `Session`, `Transport`) from a - source whose broadcast still read as open asked that source again and took - its answer as the track's verdict. Relayed, that end reads as Unroutable. - The front now counts a received serving source as closing on such an error - (`Front::closing`, `Error::session_ended`) and holds the track for the next - route. `/verify-failover` models the window: a dead relay's route still - shows until it leaves the table (`Shows`, `Withdraw`). +- Closed one way a hop broke that rule. A front reads whether a source is + closing from its broadcast, and takes a failure from a source still open as + the track's verdict. A lite session could fail the copies it carries, or a + query waiting on TRACK_INFO, before it closed the broadcasts they came from: + a front that saw a copy die that way asked the dying source again and ended + the track with its answer, which a relay passes on as Unroutable. The lite + subscriber now closes a source before failing its tracks with the session's + end (`TrackServe::close_if_ending`, `SubscriberDriver::abort`), so the front + holds the track for the next route. It reads the session's state, not the + error, since a stream's own reset can carry the same error. The IETF + subscriber is unchanged. `/verify-failover` models the window: a dead relay's + route still shows until it leaves the table (`Shows`, `Withdraw`). - Added a debug line for every front step (`moq_net::model`), so a failing run shows how a front ended a track. That race was not confirmed as this failure. On an unloaded host, 59 base runs and 24 fixed runs passed, and no reader got Unroutable during the single link reopen in any of them. Holding the drain open 2s after the transport closed -did not make the base fail either. Traced relays show the source closing -before its copies end. Debug runs on the CI host at load 70 or more starve +did not make the base fail either. Traced relays showed the source closing +before its copies end, the order the fix now guarantees. Debug runs on the CI host at load 70 or more starve readers by load alone: sessions to the detours idle-time out too. Next steps: capture one failure with diff --git a/rs/moq-net/src/error.rs b/rs/moq-net/src/error.rs index 1f40fbfae02..a697914e9bd 100644 --- a/rs/moq-net/src/error.rs +++ b/rs/moq-net/src/error.rs @@ -597,14 +597,6 @@ impl Error { pub(crate) fn transient(&self) -> bool { self.retry() == Retry::Same } - - /// Whether the session that carried the failed request ended: the route it came through - /// is gone, which a relay passes on as `Unroutable`. A session fails the copies it carries - /// before it closes the broadcasts they came from, so this says the source is closing - /// before its broadcast does. - pub(crate) fn session_ended(&self) -> bool { - matches!(self, Self::Session(_) | Self::Transport(_) | Self::IdleTimeout) - } } /// What a caller does after a request fails, as [`Error::retry`] answers it. diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index a681670af07..360f882bd3d 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -1021,6 +1021,11 @@ impl SubscriberDriver { // Before the subscribe map, so a TRACK_INFO request dropped with this driver // rejects with `err` rather than `Dropped`. self.subscriber.note_end(err); + // Before the copies they serve, so each front reads what its copy fails with as its + // source going, not as the track's verdict. + for source in self.sources.iter() { + source.source.close(); + } self.cleanup.abort(err); } @@ -7043,6 +7048,84 @@ mod tests { assert!(track < fetch, "a FETCH opened its stream before TRACK"); } + /// The front reads whether a source is closing from its broadcast, and takes a failure from a + /// source still open as the track's verdict. So a run that fails its copy, or the query + /// still waiting on TRACK_INFO, with the session's end closes the source first: the front + /// then waits for the next route rather than end a track its relay still shows. + #[kio::test] + async fn a_track_failed_with_its_session_closes_its_source_first() { + use crate::transport::poll::Session as _; + // Lite04 serves at once; Lite05 waits on TRACK_INFO. + for version in [Version::Lite04, Version::Lite05] { + let (session, peer) = crate::transport::memory::pair(Default::default()); + let subscriber = pipelining(crate::driver::Transport::new(session.clone()), version); + let source = crate::broadcast::Info::new().produce(); + let consumer = source.consume(); + let mut dynamic = source.requests(); + let mut waiting = consumer.track("video").unwrap().subscribe(None); + assert!((&mut waiting).now_or_never().is_none(), "waiting on the publisher"); + let request = dynamic.requested_track().now_or_never().unwrap().expect("request"); + let mut run = TrackServeRun::new( + TrackServe { + validity: Default::default(), + route: None, + source: Some(source.clone()), + subscriber, + path: Path::from_static("room"), + instance: None, + node: None, + name: "video".to_string(), + }, + request, + ); + assert!(kio::Task::poll(&mut run, &kio::Waiter::noop()).is_pending()); + assert!(!consumer.is_closed(), "{version:?}: closed before the session ended"); + + peer.close(crate::SessionError::Cancel.to_code(), "gone"); + for _ in 0..4 { + let _ = kio::Task::poll(&mut run, &kio::Waiter::noop()); + } + let ended = match (&mut waiting).now_or_never().expect("the track ended") { + Ok(mut copy) => copy.recv_group().now_or_never().expect("the copy ended").err(), + Err(err) => Some(err), + }; + assert!(ended.is_some(), "{version:?}: the track ended cleanly"); + assert!( + consumer.is_closed(), + "{version:?}: failed the track before closing its source" + ); + } + } + + /// The session's end aborts every subscription still open, so it closes every source first, + /// for the same reason a run does. + #[kio::test] + async fn the_session_end_closes_its_sources_before_its_subscriptions() { + let (session, _peer) = crate::transport::memory::pair(Default::default()); + let subscriber = pipelining(crate::driver::Transport::new(session), Version::Lite07); + let source = crate::broadcast::Info::new().produce(); + let consumer = source.consume(); + let route = kio::Producer::new(()); + let _ = subscriber.sources.try_push(MintedSource { + path: Path::from_static("room"), + instance: None, + node: None, + dynamic: source.requests(), + source: crate::model::broadcast::SourceGuard::new(source), + route: route.consume(), + }); + let mut driver = SubscriberDriver::new(subscriber, Vec::new(), false, route_stream::Publish::idle()); + let _ = driver.poll(&kio::Waiter::noop()); + assert_eq!(driver.sources.len(), 1, "the driver took the source"); + assert!(!consumer.is_closed()); + driver.abort(&Error::IdleTimeout); + assert!( + consumer.is_closed(), + "aborted its subscriptions before closing its sources" + ); + drop(route); + } + /// A lite-07 source of `room` over `session`, a consumer to list it through, and the /// route that keeps it served. fn listed_source( @@ -10671,6 +10754,18 @@ struct TrackServe { } impl TrackServe { + /// Close the source before failing a copy or a request with the session's end. The front + /// reads whether a source is closing from its broadcast, and takes a failure from a source + /// still open as the track's verdict: a session that failed its copies before closing their + /// sources would end tracks its relay could resume on the next route. + fn close_if_ending(&self) { + if let Some(source) = &self.source + && self.subscriber.session.poll_ending(&kio::Waiter::noop()).is_ready() + { + source.close(); + } + } + /// Watches whether the session still lets us receive this broadcast from the peer: /// our grant, and the ceiling on what the peer may publish. fn gate(&self) -> crate::auth::Gate { @@ -11364,6 +11459,7 @@ impl FinishedServe { // The groups it already holds stay readable, its clean end too once settled. if let Poll::Ready(err) = self.closed.poll_closed(waiter) { if let Some(serving) = self.serving.take() { + serve.close_if_ending(); let _ = serving.abort(Error::from_transport(err)); } return Poll::Ready(()); @@ -11759,6 +11855,7 @@ impl TrackServeRun { if let Some(pipelined) = pipelined { pipelined.abort(&self.serve, &err); } + self.serve.close_if_ending(); request.reject(err); return Poll::Ready(()); } @@ -11808,6 +11905,7 @@ impl TrackServeRun { if let Some(datagrams) = &mut self.datagrams { datagrams.end(Err(err.clone())); } + self.serve.close_if_ending(); let _ = serve_loop.serving.abort(err); } } @@ -11916,6 +12014,7 @@ impl Drop for TrackServeRun { if let Some(pipelined) = pipelined.take() { pipelined.abort(&self.serve, &err); } + self.serve.close_if_ending(); request.reject(err); } } diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index faa9ff42402..25c07ea91c3 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -583,7 +583,6 @@ impl Front { result: Result)>, actions: &mut Vec, ) { - let closing = self.closing(source, closing, result.as_ref().err().map(|(err, _)| err)); let serving = self.serving.is_some_and(|(serving, _)| serving == source); let strands = self.strands(); let Some(track) = self.tracks.get_mut(&name) else { @@ -659,7 +658,6 @@ impl Front { delivered: bool, actions: &mut Vec, ) { - let closing = self.closing(source, closing, result.as_ref().err()); let strands = self.strands(); let serving = self.serving.map(|(serving, _)| serving); // Nothing serves or is being asked, but a claim the front lost may still resume it. @@ -734,10 +732,7 @@ impl Front { } else { track.lost = Some(err); } - // A closing source is asked nothing more: its close brings the next route. - if !closing { - self.fail_over(name, actions); - } + self.fail_over(name, actions); } // A copy the closed source answered after its close died while the front waits on // the claim it served: as for a draining copy, what it died of ends the track @@ -872,18 +867,6 @@ impl Front { actions.push(Action::Query { track: name, source }); } - /// Whether `source` is closing, as the driver read from its broadcast or as `err`, what it - /// just failed with, says first: a received source is its session's, which fails the - /// copies it carries before it closes their broadcasts. A local source's error is its - /// publisher's word, and its broadcast need not close. - fn closing(&self, source: u64, closing: bool, err: Option<&Error>) -> bool { - closing - || (err.is_some_and(Error::session_ended) - && self - .serving - .is_some_and(|(serving, candidate)| serving == source && !candidate.local)) - } - /// Whether an unroutable end from the serving source judges its route rather than the /// track, so the track waits for the table to move. Only a received route can be stale: /// a local publisher's word is the track's verdict, and no news about it is in flight. A @@ -1732,104 +1715,6 @@ mod tests { ); } - /// A received source's session fails the copies it carries before it closes their - /// broadcast. A copy lost with its session's end is that source closing, even while its - /// broadcast reads as open and the table still shows its route: the track asks it nothing - /// more and takes no answer of its as a verdict, but waits for the next route. - #[test] - fn a_copy_lost_with_its_session_waits_for_the_next_route() { - let mut front = serving(remote(1), 100); - assert_actions( - front.step(Event::TrackEnded { - track: name("video"), - source: 100, - closing: false, - result: Err(Error::IdleTimeout), - delivered: true, - }), - &[], - ); - // The table moving asks again, and the session's end answers. - assert_actions( - front.step(Event::Selected { - best: Some(remote(1)), - serving_closing: false, - }), - &[Action::Query { - track: name("video"), - source: 100, - }], - ); - assert_actions( - front.step(Event::TrackInfo { - track: name("video"), - source: 100, - closing: false, - result: Err(Error::IdleTimeout), - retry: None, - }), - &[], - ); - assert_actions( - front.step(Event::SourceClosed { source: 100 }), - &[ - Action::Detach { - source: 100, - drain: true, - }, - Action::Reselect, - ], - ); - assert_actions( - front.step(Event::Selected { - best: Some(remote(2)), - serving_closing: false, - }), - &[Action::Request { route: 2 }], - ); - assert_actions( - front.step(Event::Resolved { - route: 2, - result: Ok(200), - }), - &[Action::Query { - track: name("video"), - source: 200, - }], - ); - assert!(!front.ended()); - - // A local source's error is its publisher's word, whose broadcast need not close: the - // track asks again, and the answer is a verdict. - let mut front = serving(local(1), 100); - let query = [Action::Query { - track: name("video"), - source: 100, - }]; - let lost = Event::TrackEnded { - track: name("video"), - source: 100, - closing: false, - result: Err(Error::IdleTimeout), - delivered: true, - }; - assert_actions(front.step(lost), &query); - assert_actions( - front.step(Event::TrackInfo { - track: name("video"), - source: 100, - closing: false, - result: Err(Error::IdleTimeout), - retry: None, - }), - &[Action::Abort { - track: name("video"), - err: Error::IdleTimeout, - retry: None, - }], - ); - } - /// A copy reset before delivering, seen only once its source began closing, is no /// refusal, yet ends the track with the reset rather than `Unroutable` when nothing /// replaces the source, or when its replacement refuses. The driver reports it as the copy's end once spliced, or as the answer to diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 05f02c46b75..9e7c8d397b5 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -16614,73 +16614,6 @@ mod tests { drop(upstream); } - /// A session ending aborts the copies it carries before it closes the broadcasts they - /// came from. A copy failing with its session's end is that source closing, whatever its - /// broadcast says yet: the track waits for the next route, rather than ask the dying - /// source again and take its answer as the track's verdict. A relay ending it there would - /// tell its own readers the path is unroutable while it still shows the broadcast. - #[kio::test] - #[cfg(feature = "async")] - async fn a_copy_lost_with_its_session_waits_for_the_next_route() { - for err in [ - Error::IdleTimeout, - Error::Transport("reset".into()), - Error::Session(crate::SessionError::Cancel), - ] { - let (producer, server, upstream, old, mut subscription) = remote_front().await; - // A longer way to the same instance, which wins once the first route's source closes. - let detour = producer - .forward( - "room", - RouteInfo::new().with_instance(instance(), 0).with_hops(hops(&[20, 30])), - ) - .unwrap(); - // The dying session refuses whatever it is asked. - let mut asked = upstream.requests(); - old.abort(err.clone()).unwrap(); - for _ in 0..10 { - kio::clock::sim::yield_now().await; - } - let again = asked.requested_track().now_or_never(); - let asked_again = again.is_some(); - if let Some(request) = again { - request.expect("request").reject(err.clone()); - } - for _ in 0..10 { - kio::clock::sim::yield_now().await; - } - if let Poll::Ready(Err(ended)) = subscription.poll_recv_group(&kio::Waiter::noop()) { - panic!("{err}: the track ended with its route's loss: {ended}"); - } - assert!(!asked_again, "{err}: asked the dying source again"); - - // The session finishes closing, and the detour takes over. - drop(asked); - upstream.close(); - let replacement = broadcast::Info::new().produce(); - let mut handler = replacement.requests(); - queued(&detour).await.serve(&replacement); - let request = kio::clock::timeout(Duration::from_secs(1), handler.requested_track()) - .await - .expect("the front asked the detour") - .expect("request"); - assert_eq!(request.name(), "video"); - let resumed = request.accept(crate::track::Info::timed()); - // The same content: the reader already has the first group. - for payload in [b"before", b"after!"] { - let mut group = resumed.append_next().unwrap(); - group.write_frame(group.key_time(), payload.as_ref()).unwrap(); - group.finish().unwrap(); - } - let mut group = next_group(&mut subscription) - .await - .unwrap_or_else(|err| panic!("{err}: the track ended: {err}")) - .expect("a group from the detour"); - assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"after!"); - drop(server); - } - } - /// The reader leaves, parking the track. A fetch of a whole group the parked copy /// caches answers at once, without the front serving the copy again; an open group /// still waits on the front, and is served once the copy is back. From c0cbf44f8b37193034f0087d6ed4dc06bc5bb764 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:35:07 -0800 Subject: [PATCH 6/8] verify(failover): two deaths reach FrontFail, and the sync commit moves to the base With one death E always kept a route, so FrontFail and the Shows hold never ran. Deaths.cfg lets A and B both die. NoSkipWhileHeld and SettledIsWhole now exempt a finished track whose every route left, as their comments already said; with one death the clause always holds. The sync commit moves to 1a34abe437 after an audit found no unmirrored covered change since 61f8956839, and the coverage row names the lite subscriber's closes. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-failover/SKILL.md | 11 ++++++----- rs/moq-net/verify/failover/Deaths.cfg | 19 +++++++++++++++++++ rs/moq-net/verify/failover/FailoverChain.tla | 7 ++++--- 3 files changed, 29 insertions(+), 8 deletions(-) create mode 100644 rs/moq-net/verify/failover/Deaths.cfg diff --git a/.claude/skills/verify-failover/SKILL.md b/.claude/skills/verify-failover/SKILL.md index 43a932f47dd..51d00690667 100644 --- a/.claude/skills/verify-failover/SKILL.md +++ b/.claude/skills/verify-failover/SKILL.md @@ -16,7 +16,7 @@ Each rule is an INVARIANT, except `Settles`, which is TLC's deadlock check: ever | `CleanEndIsComplete` | The client's track ends cleanly only with every group handed out. | The viewer loses media silently at the end. | | `CleanEndIsDeclared` | The client's track ends cleanly only after the origin ended it. | A failover reads as the broadcast ending. | | `NoSkipWhileHeld` | A group the client started fails only once `E` has no route left. | A group is cut short while another path holds the rest. | -| `SettledIsWhole` | A finished read holds every frame of every group. | A group ends short of its last frame. | +| `SettledIsWhole` | A finished read holds every frame of every group, unless `E` has no route left. | A group ends short of its last frame. | | `Settles` (deadlock) | Every path ends with the read over, or, once the client left, with no relay holding a copy. | Playback hangs, or a relay keeps its upstream subscribed after the viewer left. | ## Run @@ -24,10 +24,11 @@ Each rule is an INVARIANT, except `Settles`, which is TLC's deadlock check: ever ```sh quest gate just rs tla moq-net/verify/failover/FailoverChain.tla quest gate just rs tla moq-net/verify/failover/FailoverChain.tla -config rs/moq-net/verify/failover/Leave.cfg +quest gate just rs tla moq-net/verify/failover/FailoverChain.tla -config rs/moq-net/verify/failover/Deaths.cfg quest gate just rs tla moq-net/verify/failover/FailoverChain.tla -config rs/moq-net/verify/failover/Mesh.cfg -simulate num=200000 -depth 100 ``` -The first checks every rule with `E` failing over between `A` and `B`, one death, and one switch per relay, with a client that stays, so a deadlock is a playback hang: about 6.0M states. The second lets the client leave at any point, so a deadlock is a relay still holding a copy after it left; leaving is always possible there, which is why it cannot stand in for the first: about 6.6M states. Each takes about three minutes and passes at the sync commit. The third adds `B`'s route through `A`, so the route also moves at the middle hop, after `B`'s track ended too (`Action::Reroute`). It is too large to exhaust (past 139M distinct states at depth 38 after an hour, its queue still growing), so its `Simulate:` line samples 800k random behaviors in about 7 minutes, and Nightly samples new ones each run. An exhaustive run passed every state through depth 37. Before the reroute, a sample that size missed the depth-31 hang that BFS found after about 48M states, so a passing sample is weak evidence: check a change at the middle hop by running it without the `-simulate` arguments for as long as it takes. +The first checks every rule with `E` failing over between `A` and `B`, one death, and one switch per relay, with a client that stays, so a deadlock is a playback hang: about 6.0M states. The second lets the client leave at any point, so a deadlock is a relay still holding a copy after it left; leaving is always possible there, which is why it cannot stand in for the first: about 6.6M states. With one death `E` always has a route, so the third lets both `A` and `B` die: only there does `E`'s front end the track (`FrontFail`), once no dead relay's route still shows (`Withdraw`): about 9.2M states. Each takes three to four minutes and passes at the sync commit. The fourth adds `B`'s route through `A`, so the route also moves at the middle hop, after `B`'s track ended too (`Action::Reroute`). It is too large to exhaust (past 139M distinct states at depth 38 after an hour, its queue still growing), so its `Simulate:` line samples 800k random behaviors in about 7 minutes, and Nightly samples new ones each run. An exhaustive run passed every state through depth 37. Before the reroute, a sample that size missed the depth-31 hang that BFS found after about 48M states, so a passing sample is weak evidence: check a change at the middle hop by running it without the `-simulate` arguments for as long as it takes. ## Reading a failure @@ -35,7 +36,7 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `61f89568394638553a519d7d8cfc0019385dd3a0`, plus [moq-dev/moq#9048](https://github.com/moq-dev/moq/pull/9048), which mirrored the code since 88572c583e and kept a finished track a read holds past its linger, and [moq-dev/moq#9155](https://github.com/moq-dev/moq/pull/9155), which resumes the newest group past the frames held and lends them to the new copy (`Resume`, `Lend`), and the splice-stall fix, which holds a track while a dead relay's route still shows, since a session fails its copies before it closes their sources (`Shows`, `Withdraw`). Run `git log 61f8956839.. -- rs/moq-net/src/model/resume.rs rs/moq-net/src/model/front.rs rs/moq-net/src/model/origin.rs rs/moq-net/src/model/track.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. +Sync commit: `1a34abe43717e1815108de792e04c86a44a971bc`, plus the splice-stall fix, which holds a track while a dead relay's route still shows, since a session closes its sources before it fails their copies (`Shows`, `Withdraw`). Run `git log 1a34abe437.. -- rs/moq-net/src/model/resume.rs rs/moq-net/src/model/front.rs rs/moq-net/src/model/origin.rs rs/moq-net/src/model/track.rs rs/moq-net/src/lite/subscriber.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. | Code | Model | | --- | --- | @@ -47,10 +48,10 @@ Sync commit: `61f89568394638553a519d7d8cfc0019385dd3a0`, plus [moq-dev/moq#9048] | `rs/moq-net/src/model/resume.rs` `Recover::poll_fetch`, `Fetching`; `rs/moq-net/src/model/track.rs` `fetch_group` | `Ask` | | `rs/moq-net/src/model/front.rs` `track_ended`, `Action::Finish`; `rs/moq-net/src/model/origin.rs` `TrackIo::end` | `FrontFinish`, `Whole` | | `rs/moq-net/src/model/front.rs` failover and its unroutable end, `Action::Reroute` | `Switch`, `FrontFail` | -| `rs/moq-net/src/model/front.rs` `Front::closing`; `rs/moq-net/src/error.rs` `Error::session_ended` | `Shows`, `Withdraw`, `FrontFail` | +| `rs/moq-net/src/lite/subscriber.rs` `TrackServe::close_if_ending`, `SubscriberDriver::abort` | `Shows`, `Withdraw`, `FrontFail` | | `rs/moq-net/src/model/resume.rs` `Route::answering`, `Phase::Finished` | `Answering`, `LetGo` | | `rs/moq-net/src/model/front.rs` `Action::Forget`; `rs/moq-net/src/model/track.rs` `TrackWeak::is_held`, `TrackWeak::poll_held_changed` | `Forget`, `Reads` | ## Not covered -Routing is abstracted to any live upstream winning, so which route wins and announce order are the router's own tests' (`rs/moq-net/src/route/tests`); how a reader picks among its copies, bounds replaced copies, and orders groups is `verify-resume`'s; drops, skips, and re-offers on a hop are `verify-subscribe`'s. The client's track fails only through `FrontFail`, so whether one dead relay can end playback is that step's guard, not a rule. Every hop has an hour's budget, so nothing goes stale or expires. No source refuses the track, so a finished track's reroute never concludes (`Action::Conclude`). Time is abstracted, so the linger may run out at any point once nothing reads the track. The origin writes each group whole and never fails, `E` never dies, a dead relay never returns, and a relay's cache is dropped when it parks. A fetched group holds its head from frame 0 (`lend_head`), and compressed groups are left out. The model lends only at a switch, so the query a front opens on a route before switching to it (`Action::Query`, `follow_demand`) is the unit test `a_query_resuming_a_new_copy_lends_it_the_head`, and a loan never runs out of cache budget. A pass covers two groups of two frames, one death, one switch per relay, and one client. +Routing is abstracted to any live upstream winning, so which route wins and announce order are the router's own tests' (`rs/moq-net/src/route/tests`); how a reader picks among its copies, bounds replaced copies, and orders groups is `verify-resume`'s; drops, skips, and re-offers on a hop are `verify-subscribe`'s. The client's track fails only through `FrontFail`, so whether a dead relay whose route still shows can end playback is that step's guard, not a rule. Every hop has an hour's budget, so nothing goes stale or expires. No source refuses the track, so a finished track's reroute never concludes (`Action::Conclude`). Time is abstracted, so the linger may run out at any point once nothing reads the track. The origin writes each group whole and never fails, `E` never dies, a dead relay never returns, and a relay's cache is dropped when it parks. A fetched group holds its head from frame 0 (`lend_head`), and compressed groups are left out. The model lends only at a switch, so the query a front opens on a route before switching to it (`Action::Query`, `follow_demand`) is the unit test `a_query_resuming_a_new_copy_lends_it_the_head`, and a loan never runs out of cache budget. A pass covers two groups of two frames, one death (two in `Deaths.cfg`), one switch per relay, and one client. diff --git a/rs/moq-net/verify/failover/Deaths.cfg b/rs/moq-net/verify/failover/Deaths.cfg new file mode 100644 index 00000000000..170241101d4 --- /dev/null +++ b/rs/moq-net/verify/failover/Deaths.cfg @@ -0,0 +1,19 @@ +\* FailoverChain.cfg with both A and B mortal. Only here can E lose every route, +\* so only here does E's front end the track (`FrontFail`), and only once no +\* dead relay's route still shows: a front holds the track until `Withdraw`. +CONSTANTS + G = 2 + F = 2 + MaxDeaths = 2 + MaxSwitches = 1 + Mesh = FALSE + Leave = FALSE + +SPECIFICATION Spec + +INVARIANTS + TypeOK + CleanEndIsComplete + CleanEndIsDeclared + NoSkipWhileHeld + SettledIsWhole diff --git a/rs/moq-net/verify/failover/FailoverChain.tla b/rs/moq-net/verify/failover/FailoverChain.tla index e2049ee5298..a1968378e1b 100644 --- a/rs/moq-net/verify/failover/FailoverChain.tla +++ b/rs/moq-net/verify/failover/FailoverChain.tla @@ -498,12 +498,13 @@ CleanEndIsComplete == Live(CE) /\ cp[CE].fin => \A g \in Groups : cp[CE].fr[g] > \* The client's track ends cleanly only after the origin ended it. CleanEndIsDeclared == cp[CE].fin => org.fin -\* A group the client started fails only once no route is left to continue it. -NoSkipWhileHeld == cp[CE].bad # {} => front["E"].end = "err" +\* A group the client started fails only once no route is left to continue it: E's +\* track failed, or it finished and no route of E's still shows. +NoSkipWhileHeld == cp[CE].bad # {} => front["E"].end = "err" \/ \A w \in Up("E") : ~Shows(w) \* A finished read holds every frame of every group, unless E lost every route. SettledIsWhole == - (Finished /\ ~left /\ cp[CE].st = "live") => + (Finished /\ ~left /\ cp[CE].st = "live" /\ \E w \in Up("E") : Shows(w)) => \A g \in Groups : cp[CE].fr[g] = F TypeOK == From 8c13e5a93f0c2d98b41b8dd6f0eaf5b38eb25315 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:51:14 -0800 Subject: [PATCH 7/8] verify(failover): the sync commit moves to the merged base #9202, #9217 and #9130 change no covered behavior the model represents: announce scope, subscriber scalars the first sync mirrors, and ordered reads. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-failover/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.claude/skills/verify-failover/SKILL.md b/.claude/skills/verify-failover/SKILL.md index 51d00690667..1e0dfbc9520 100644 --- a/.claude/skills/verify-failover/SKILL.md +++ b/.claude/skills/verify-failover/SKILL.md @@ -36,7 +36,7 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `1a34abe43717e1815108de792e04c86a44a971bc`, plus the splice-stall fix, which holds a track while a dead relay's route still shows, since a session closes its sources before it fails their copies (`Shows`, `Withdraw`). Run `git log 1a34abe437.. -- rs/moq-net/src/model/resume.rs rs/moq-net/src/model/front.rs rs/moq-net/src/model/origin.rs rs/moq-net/src/model/track.rs rs/moq-net/src/lite/subscriber.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. +Sync commit: `46059266ec9e149c37147577bc40ca3a7cea556a`, plus the splice-stall fix, which holds a track while a dead relay's route still shows, since a session closes its sources before it fails their copies (`Shows`, `Withdraw`). Run `git log 46059266ec.. -- rs/moq-net/src/model/resume.rs rs/moq-net/src/model/front.rs rs/moq-net/src/model/origin.rs rs/moq-net/src/model/track.rs rs/moq-net/src/lite/subscriber.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. | Code | Model | | --- | --- | From 49cd263e8469d740e9407fa97c51abf4c6ab8d51 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 20 Nov 2026 01:05:50 -0800 Subject: [PATCH 8/8] fix(moq-net): a front reads whether a source is closing after its copy's outcome FrontStep::Info read `closing` before polling the copy, so a teardown closing the source between the two reads could hand the front a copy failure with `closing` false, which it takes as the source's own verdict. The subscriber closes a source before it fails that source's copies, so reading `closing` after the outcome always sees the close. The quest note now counts the runs of the fix that landed, not the one it replaced. Co-Authored-By: Claude Opus 5.5 --- quest/m2/multicast/splice-stall.md | 5 +++-- rs/moq-net/src/model/origin.rs | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/quest/m2/multicast/splice-stall.md b/quest/m2/multicast/splice-stall.md index d4fe1f5d0aa..f09ac739601 100644 --- a/quest/m2/multicast/splice-stall.md +++ b/quest/m2/multicast/splice-stall.md @@ -56,8 +56,9 @@ What [moq-dev/moq#9219](https://github.com/moq-dev/moq/pull/9219) did: shows how a front ended a track. That race was not confirmed as this failure. On an unloaded host, 59 base runs -and 24 fixed runs passed, and no reader got Unroutable during the single link -reopen in any of them. Holding the drain open 2s after the transport closed +passed, as did 21 release runs of the fix that landed (and 24 debug runs of an +earlier, error-based fix it replaced), and no reader got Unroutable during the +single link reopen in any of them. Holding the drain open 2s after the transport closed did not make the base fail either. Traced relays showed the source closing before its copies end, the order the fix now guarantees. Debug runs on the CI host at load 70 or more starve readers by load alone: sessions to the detours idle-time out too. diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index b3f0a274bf5..2db491f070e 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -5543,7 +5543,6 @@ impl kio::Task for FrontServe { } FrontStep::SourceClosed(source) => Event::SourceClosed { source }, FrontStep::Info(name, source, result) => { - let closing = self.sources.get(&source).is_some_and(|s| s.is_closing()); let Some(io) = self.tracks.get_mut(&name) else { continue }; let Some(asked) = io.query.take() else { continue }; let retry = asked.query.retry(); @@ -5557,6 +5556,9 @@ impl kio::Task for FrontServe { }, Err(err) => Err(err), }; + // Read after the copy's outcome: a subscriber closes a source before it fails + // that source's copies, so a failure seen above is already reflected here. + let closing = self.sources.get(&source).is_some_and(|s| s.is_closing()); // Staged only while the track has a reader, or a read holds the finished // track: without one the machine will not splice, and a held copy would // keep the source subscribed.