From e4a928e4948d3d99eb948925b6a5c9b75b7e9c7c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 05:15:19 -0800 Subject: [PATCH 1/3] chore(quest): claim quest/m1/reconnect-probe-redirect.md Quest-Claim: 613be5f867589eb8 From 0754259ff8c71571536baddc08030bc21e2eb6e9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 06:37:03 -0800 Subject: [PATCH 2/3] fix(moq-net): follow a GOAWAY redirect from a draining fallback session When a probe of the preferred transport gets through, the fallback session drains as the predecessor. Its GOAWAY watcher now moves with it, and while no replacement serves, an accepted redirect becomes the only target and drops a dial still headed for the leaving node. js/net's Reload mirrors it. The reconnect model gains FollowPred; RedirectFollowed moves into Handover.cfg and RestartWait.cfg, and RedirectLost.cfg is deleted. Also rewords the Backoff doc to what the redirect cycle check does. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-reconnect/SKILL.md | 9 +-- js/net/src/connection/reload.test.ts | 69 +++++++++++++++++ js/net/src/connection/reload.ts | 36 ++++++++- quest/m1/reconnect-probe-redirect.md | 33 -------- rs/moq-net/src/reconnect.rs | 81 ++++++++++++++++++-- rs/moq-net/src/reconnect/tests.rs | 40 ++++++---- rs/moq-net/verify/reconnect/Handover.cfg | 2 +- rs/moq-net/verify/reconnect/Handover.tla | 42 +++++++++- rs/moq-net/verify/reconnect/RedirectLost.cfg | 15 ---- rs/moq-net/verify/reconnect/RestartWait.cfg | 1 + 10 files changed, 248 insertions(+), 80 deletions(-) delete mode 100644 quest/m1/reconnect-probe-redirect.md delete mode 100644 rs/moq-net/verify/reconnect/RedirectLost.cfg diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index 0e9ca3c98cf..71f92286208 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -26,16 +26,13 @@ Each rule is an INVARIANT; the harm says why it matters. All but `NoOrphan`, `Re ```sh quest gate just rs tla moq-net/verify/reconnect/Handover.tla quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/verify/reconnect/RestartWait.cfg -quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/verify/reconnect/RedirectLost.cfg ``` -The first checks every rule but the one with an open finding, over four sessions: about 1.7M distinct states, under a minute. The second checks the same rules under `Restart::Wait` over three sessions: about 290k states. Both pass at the sync commit. The last fails at the sync commit, on its finding below, within seconds. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. +The first checks every rule over four sessions: about 1.4M distinct states, under a minute. The second checks the same rules under `Restart::Wait` over three sessions: about 300k states. Both pass at the sync commit. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. ## Open findings -### A redirect after a probe got through is lost - -`RedirectLost.cfg` fails. On a fallback, `probed` moves the live session into the predecessor and drops its `goaway::Consumer` with the `Live` it came from, so nothing reads that session's GOAWAY anymore. A redirect its peer sends while the handshake over the preferred transport runs is never followed: when that handshake fails, the next walk dials the address the peer is leaving. `residual_a_redirect_after_a_probe_got_through_is_lost` in `reconnect/tests.rs` replays it. Once fixed, flip that test, move `RedirectFollowed` into `Handover.cfg`, and delete `RedirectLost.cfg`. +None. ## Reading a failure @@ -47,7 +44,7 @@ Sync commit: `4486b7adc108649bb3e7924554394ee30fa0e5f2`. Run `git log 4486b7adc1 | Code | Model | | --- | --- | -| `rs/moq-net/src/reconnect.rs` `Reconnect::poll` (queue, predecessor, phases) | `Pop`, `Retire`, `IdleClose`, `Dial`, `Abandon`, `BackoffEnds`, `LiveEnds`, `ProbeAsk`, `ProbeAbandon` | +| `rs/moq-net/src/reconnect.rs` `Reconnect::poll` (queue, predecessor, phases), `follow` | `Pop`, `Retire`, `IdleClose`, `FollowPred`, `Dial`, `Abandon`, `BackoffEnds`, `LiveEnds`, `ProbeAsk`, `ProbeAbandon` | | `rs/moq-net/src/reconnect.rs` `established`, and `walk_failed` ending the first walk | `Established`, `Failed`, `Abandon` | | `rs/moq-net/src/reconnect.rs` `connected`, `fallback`, `failed`, `probed` | `Connected`, `Failed`, `ProbeFails`, `ProbeGetsThrough` | | `rs/moq-net/src/reconnect.rs` `end`, `closed`, `done`, `Predecessor` | `EndGoaway`, `EndClosed`, `Done`, `Retire`, `IdleClose` | diff --git a/js/net/src/connection/reload.test.ts b/js/net/src/connection/reload.test.ts index 0439262c224..e62ded16805 100644 --- a/js/net/src/connection/reload.test.ts +++ b/js/net/src/connection/reload.test.ts @@ -1670,6 +1670,75 @@ test("a WebSocket fallback returns to WebTransport once it is reachable", async } }); +// A redirect the fallback's peer sends while it drains, after a probe got through but its +// handshake failed, still decides where the replacement goes: the redial after the backoff goes +// where the peer named, not back to the node that is leaving. +test("a redirect from the draining WebSocket fallback is followed", async () => { + using clock = manual(); + const originalWebTransport = globalThis.WebTransport; + const originalStream = (globalThis as { WebSocketStream?: unknown }).WebSocketStream; + const url = new URL("https://fallback-redirect.example.com/"); + const elsewhere = new URL("https://fallback-redirect.example.com:4444/"); + + let blocked = true; + const dialed: string[] = []; + const servers: Promise[] = []; + let fallback: WebTransport | undefined; + globalThis.WebTransport = function StubWebTransport(to: string | URL) { + if (blocked) { + const refused = Promise.reject(new Error("UDP blocked")); + refused.catch(() => {}); + return { ready: refused, closed: refused, close() {} }; + } + dialed.push(String(to)); + // The leaving node answers the probe, but no MoQ version is in common with it. + if (new URL(to).port !== elsewhere.port) return createMockTransportPair("h3").client; + const pair = createMockTransportPair(Lite.ALPN_06); + const server = accept(pair.server, new URL(to)); + server.catch(() => {}); + servers.push(server); + return pair.client; + } as unknown as typeof WebTransport; + (globalThis as { WebSocketStream?: unknown }).WebSocketStream = linkedWebSocketStream((socket, protocol) => { + const transport = Qmux.accept(socket, { protocol }) as unknown as WebTransport; + fallback = transport; + servers.push(transport.ready.then(() => accept(transport, url))); + }); + + const warned = spyOn(console, "warn").mockImplementation(() => {}); + const reload = new Reload({ url, versions: [Lite.ALPN_06], delay: { initial: Duration.millis(5_000) } }); + try { + for (let i = 0; i < 10 && reload.established.peek()?.transport !== "websocket"; i++) { + await clock.advance(Duration.millis(100)); + } + const server = servers[0]; + if (!server || !fallback) throw new Error("no fallback session"); + + blocked = false; + // The probe gets through, the fallback tells its peer it is leaving, and the handshake + // over the probe fails, so the loop backs off with the fallback still serving. + for (let i = 0; i < 20 && dialed.length === 0; i++) await clock.advance(Duration.millis(1_000)); + expect(await (await server).draining).toEqual({ uri: "" }); + await clock.advance(Duration.millis(10)); + expect(reload.established.peek()?.transport).toBe("websocket"); + + const stream = await Stream.open(fallback, { version: Lite.Version.DRAFT_06 }); + await stream.writer.u53(Lite.StreamId.Goaway); + await new Lite.Goaway(elsewhere.href).encode(stream.writer, Lite.Version.DRAFT_06); + for (let i = 0; i < 100 && reload.established.peek()?.transport !== "webtransport"; i++) { + await clock.advance(Duration.millis(100)); + } + expect(reload.established.peek()?.transport).toBe("webtransport"); + expect(dialed.map((to) => new URL(to).port)).toEqual(["", elsewhere.port]); + } finally { + reload.close(); + warned.mockRestore(); + for (const server of servers) (await server.catch(() => undefined))?.abort(); + globalThis.WebTransport = originalWebTransport; + (globalThis as { WebSocketStream?: unknown }).WebSocketStream = originalStream; + } +}); + // A WebSocket win is provisional: once the session it carried ends, the redial gives // WebTransport its head start again, since the outage that lost the race may be over. test("a WebSocket session ending restores WebTransport's head start", async () => { diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index 7cd890847d1..5b2357f2bb3 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -664,6 +664,7 @@ export class Reload { // backoff already paces it, so its replacement dials at once. A fallback that // served past the healthy bar still earns the healthy reset. if (reached) { + this.#follow(connection, at, pinnedTransport(connection.transport, this.#pinned())); if (now().since(connected).ge(this.#pacing().healthy)) { this.#delay = undefined; this.#deadline = undefined; @@ -766,10 +767,8 @@ export class Reload { * redirect is refused, which ends the sequence rather than redialing. */ #migrate(connection: Session, dialing: URL, drain: Drain): boolean { - const hashes = this.webtransport?.serverCertificateHashes?.length ?? 0; - const configured = hashes > 0 || this.webtransport?.serverCertificate !== undefined; // The pin is a WebTransport option. A WebSocket that won the race never used it. - const pinned = pinnedTransport(connection.transport, configured); + const pinned = pinnedTransport(connection.transport, this.#pinned()); let next: URL | undefined; try { @@ -803,6 +802,37 @@ export class Reload { return true; } + /** Whether a certificate pin is configured for WebTransport. */ + #pinned(): boolean { + const hashes = this.webtransport?.serverCertificateHashes?.length ?? 0; + return hashes > 0 || this.webtransport?.serverCertificate !== undefined; + } + + /** + * Follow a redirect the peer of `connection`, a fallback session we left for a probe that + * got through, sends while it drains and nothing has replaced it yet: the next dial goes + * where the peer named, and a dial still headed for the leaving peer starts over. A restart + * in place or a refused redirect is ignored, since we are already leaving. No loop scoring: + * only a probe that got through leads here, and the probe's backoff paces those. + */ + #follow(connection: Session, dialing: URL, pinned: boolean): void { + void connection.draining.then((drain) => { + if (this.#draining?.serving() !== connection || this.established.peek() !== connection) return; + let next: URL | undefined; + try { + next = target(this.goaway.redirect ?? "same-host", drain.uri, dialing, pinned); + } catch { + return; + } + if (!next) return; + console.info("draining fallback peer redirected; following"); + this.#redirect.set(next); + this.#redirectHref = this.#sequenceHref; + // A backoff still paces the next dial, which now goes to the new target. + if (this.#dialing) this.#tick.update((prev) => prev + 1); + }); + } + /** * End a session this loop is done with. Its groups in flight finish within the close * deadline, unless the page is going away: nothing would run to finish them, and an abort diff --git a/quest/m1/reconnect-probe-redirect.md b/quest/m1/reconnect-probe-redirect.md deleted file mode 100644 index b29b0b6a882..00000000000 --- a/quest/m1/reconnect-probe-redirect.md +++ /dev/null @@ -1,33 +0,0 @@ -# [S] A GOAWAY to a draining fallback session is followed - -## Goal - -When a WebSocket fallback session drains because a QUIC probe got through, -and its peer then sends a GOAWAY naming another node, the client's next dial -goes to that node, not back to the one that is leaving. - -## Plan - -Found 2026-11-10 by the reconnect model ([moq-dev/moq#8447](https://github.com/moq-dev/moq/pull/8447)): -`rs/moq-net/verify/reconnect/RedirectLost.cfg` breaks `RedirectFollowed`. -`probed()` in `rs/moq-net/src/reconnect.rs` drops the fallback session's -GOAWAY watcher along with its `Live` state, so a redirect that arrives while -it drains is lost; if the QUIC handshake then fails, the policy redials the -leaving node until the handover window ends, then reports offline. -`residual_a_redirect_after_a_probe_got_through_is_lost` in -`reconnect/tests.rs` replays it. - -Recommended fix: keep the GOAWAY watcher on the predecessor, and while no -replacement serves, act on an accepted redirect the way `end()` does. Flip the -residual test, move `RedirectFollowed` into `Handover.cfg`, delete -`RedirectLost.cfg`, and update the `verify-reconnect` skill. - -Also fix the `Backoff` doc in the same file: it promises a GOAWAY back to an -endpoint that already redirected us is never followed at once, but the -`Redirects` cycle check catches a loop only after a lap (A to B, B to C, C to B -is followed once). Reword the doc to what the check does; no code change. - -Caused by: [moq-dev/moq#8330](https://github.com/moq-dev/moq/pull/8330), which -added the fallback probe. - -Public API: none. Wire: none. diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs index ef837f59027..5a3c57e95ec 100644 --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs @@ -64,10 +64,11 @@ const PROBE_MAX: Duration = Duration::from_secs(300); /// peer sends one to each session until it goes, and each of those serves, so it /// restarts the timeout instead. /// -/// A GOAWAY off a healthy session migrates at once, unless it sends the connection back -/// to an endpoint that already redirected it elsewhere and the session lived less than -/// `max`. Two peers redirecting to each other thus wait out this backoff and count -/// against the timeout. +/// A GOAWAY off a healthy session migrates at once, unless the run of redirects, among +/// sessions that each lived less than `max`, has looped back through an endpoint. The loop +/// check keeps one endpoint, so it catches a cycle within a few laps rather than at the +/// first revisit: A to B, B to C, C to B is followed once before it is caught. Peers +/// redirecting in a loop then wait out this backoff and count against the timeout. #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[non_exhaustive] pub struct Backoff { @@ -324,7 +325,7 @@ pub enum Action { /// [`Attempt::client`]. Dial(Attempt), /// Drop the dial or probe in flight: it ran out its deadline, which already counted as - /// its failure. + /// its failure, or the predecessor's peer redirected while it headed for that peer. Abandon, /// The [`predecessor`](Reconnect::predecessor) serves while a replacement is dialed: the /// live session got a GOAWAY and now drains as it, or the replacement closed while it @@ -558,6 +559,11 @@ impl + From> Reco continue; } } + // Until a replacement serves, a redirect the predecessor's peer sends decides where + // the next dial goes. + if !live && let Some(action) = self.follow(waiter) { + return action; + } let handover = self.predecessor.as_ref().map(|predecessor| predecessor.deadline); match &mut self.phase { @@ -727,10 +733,20 @@ impl + From> Reco if let Some(older) = self.predecessor.take() { older.retire(); } + // Its peer may still redirect us while it drains, which `follow` acts on. A fixed + // target never moves, so it needs no watch. + let redirect = old + .goaway + .filter(|_| !self.targets[index].fixed()) + .map(|goaway| Redirected { + goaway, + url: self.targets[index].url().clone(), + }); self.predecessor = Some(Predecessor { deadline: now.saturating_add(self.config.goaway.handover), session: old.session, closing: false, + redirect, }); // Only now, once the fallback stops taking new work: a probe that never gets through // leaves the serving session's tokens and their grants alone. @@ -883,6 +899,51 @@ impl + From> Reco self.back_off(now, self.config.backoff.timeout, false, "reconnecting after backoff"); } + /// Act on a GOAWAY the predecessor's peer sent after a probe moved us off it, the way + /// [`end`](Self::end) would have: an accepted redirect becomes the only target, and a dial + /// still headed for the leaving peer is dropped. Anything else leaves the targets alone, + /// since the policy is already leaving that session. + fn follow(&mut self, waiter: &kio::Waiter) -> Option> { + let predecessor = self.predecessor.as_mut()?; + let redirect = predecessor.redirect.as_ref()?; + let Poll::Ready(msg) = redirect.goaway.poll(waiter) else { + return None; + }; + let redirect = predecessor.redirect.take()?; + let target = match msg.map(|msg| { + self.config + .goaway + .redirect + .target(msg.uri(), &redirect.url, self.config.pinned) + }) { + Ok(Ok(Some(target))) => target, + // A closed session, a restart in place, or a refused redirect: nothing to follow. + _ => return None, + }; + tracing::info!(peer = %Shown(&target), "draining fallback peer redirected; following"); + // No loop scoring here: only a probe that got through leads here, and the probe's + // backoff paces those. The session this dial lands on is scored when it ends. + let redirected = std::mem::replace(&mut self.targets, vec![A::from(target)]); + self.configured.get_or_insert(redirected); + // A walk of the replaced targets gives way to one of the new target. + if matches!(self.phase, Phase::Dialing(_) | Phase::Next { .. }) { + self.walk = Walk::default(); + } + match self.phase { + Phase::Dialing(_) => { + self.phase = Phase::Start; + Some(Action::Abandon) + } + // An index into the replaced targets. + Phase::Next { .. } => { + self.phase = Phase::Start; + None + } + // A backoff still paces the next dial, which now goes to the new target. + _ => None, + } + } + /// Score how a live session ended and decide what follows. fn end(&mut self, now: Instant, ended: Ended) { let Phase::Live(live) = std::mem::replace(&mut self.phase, Phase::Start) else { @@ -964,6 +1025,8 @@ impl + From> Reco deadline: now.saturating_add(self.config.goaway.handover(msg.timeout())), session: live.session, closing: false, + // Its GOAWAY was just acted on. + redirect: None, }); self.queue.push_back(Action::Migrating); @@ -1153,6 +1216,14 @@ struct Predecessor { deadline: Instant, /// Whether [`Self::poll_idle`] started the graceful close. closing: bool, + /// Watches for a redirect from the peer, set when a probe moved us off this session. + redirect: Option, +} + +/// A drained fallback session's GOAWAY, and the URL a redirect in it is judged against. +struct Redirected { + goaway: goaway::Consumer, + url: Url, } impl Predecessor { diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index 26787a7591b..c08da1a6807 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -912,12 +912,11 @@ async fn a_peer_goaway_wins_over_a_probe() { ); } -/// Pins an open finding (`verify-reconnect`, `RedirectFollowed`): once a probe got -/// through, the fallback session drains as the predecessor and nothing watches its peer's -/// GOAWAY anymore, so a redirect the peer sends while the handshake runs is never read and -/// the redial goes back to the node that is leaving. +/// Once a probe got through, the fallback session drains as the predecessor, and a +/// redirect its peer sends while the handshake runs still decides where we go: the +/// handshake to the leaving node is dropped and the dial goes where the peer named. #[kio::test] -async fn residual_a_redirect_after_a_probe_got_through_is_lost() { +async fn a_redirect_after_a_probe_got_through_is_followed() { let (mut machine, fallback_server) = on_fallback(Config::default()).await; probe(&mut machine).await; assert!(machine.probed(kio::clock::now(), Ok(()))); @@ -928,19 +927,34 @@ async fn residual_a_redirect_after_a_probe_got_through_is_lost() { .drain() .send(goaway::Goaway::redirect(elsewhere.to_string())) .expect("the fallback peer sends a GOAWAY"); - kio::clock::sim::idle().await; - // The handshake over the preferred transport fails, so the policy dials again. + assert!(matches!(next(&mut machine).await, Action::Abandon)); + let Action::Dial(attempt) = next(&mut machine).await else { + panic!("expected a dial"); + }; + assert_eq!(attempt.target(), &elsewhere); + assert!(machine.predecessor().is_some(), "the redirected session still drains"); +} + +/// A redirect from the draining fallback's peer during the backoff after a failed +/// handshake keeps the wait, and the redial goes where the peer named. +#[kio::test] +async fn a_redirect_during_the_backoff_after_a_probe_is_followed() { + let (mut machine, fallback_server) = on_fallback(Config::default()).await; + probe(&mut machine).await; + assert!(machine.probed(kio::clock::now(), Ok(()))); + assert!(matches!(next(&mut machine).await, Action::Migrating)); machine.failed(kio::clock::now(), TestError::Dial, Verdict::Retry); assert!(matches!(next(&mut machine).await, Action::Retrying(_))); + + let elsewhere = url("https://peer:2/"); + fallback_server + .drain() + .send(goaway::Goaway::redirect(elsewhere.to_string())) + .expect("the fallback peer sends a GOAWAY"); let Action::Dial(attempt) = next(&mut machine).await else { panic!("expected a redial"); }; - assert_eq!( - attempt.target(), - &url("https://peer:1/"), - "the redial followed the redirect: flip this test" - ); - assert!(machine.predecessor().is_some(), "the redirected session still drains"); + assert_eq!(attempt.target(), &elsewhere); } /// A client and server session where the client holds a subscription the server serves, diff --git a/rs/moq-net/verify/reconnect/Handover.cfg b/rs/moq-net/verify/reconnect/Handover.cfg index 7a0638a8a0d..ea4c0c89aa3 100644 --- a/rs/moq-net/verify/reconnect/Handover.cfg +++ b/rs/moq-net/verify/reconnect/Handover.cfg @@ -1,5 +1,4 @@ \* Two endpoints a GOAWAY may move between, one it may not, and four sessions. -\* RedirectFollowed lives in RedirectLost.cfg while its finding is open. CONSTANTS Endpoints = {"a", "b"} Start = "a" @@ -17,3 +16,4 @@ INVARIANTS NoOrphan PresenceBalanced FirstWalkFailsFast + RedirectFollowed diff --git a/rs/moq-net/verify/reconnect/Handover.tla b/rs/moq-net/verify/reconnect/Handover.tla index 3a4ebf35512..0f9ad8f38dd 100644 --- a/rs/moq-net/verify/reconnect/Handover.tla +++ b/rs/moq-net/verify/reconnect/Handover.tla @@ -68,10 +68,11 @@ vars == <> \* "abort" for `Session::abort`, "drain" for a `start_close` that finished), \* whether a `start_close` began, the GOAWAY it got, its age, whether it \* carries no subscription or fetch, whether the policy acted on its GOAWAY, -\* and whether we sent our own. +\* whether we sent our own, and whether the policy watches its GOAWAY as the +\* predecessor (`Predecessor::redirect`, set when a probe moved us off it). Fresh(e) == [ep |-> e, open |-> TRUE, by |-> "", closing |-> FALSE, goaway |-> NoGoaway, age |-> 0, idle |-> FALSE, heeded |-> FALSE, - sent |-> FALSE] + sent |-> FALSE, watched |-> FALSE] Unused == [Fresh(Start) EXCEPT !.open = FALSE] Timers == {"dial", "backoff", "probe", "handover"} @@ -222,10 +223,40 @@ Misdial(e) == /\ ~sess[s].heeded /\ sess[s].goaway # e +\* `Reconnect::follow`, which runs before the phase while no replacement +\* serves: the predecessor's peer sent a GOAWAY. +PredGoaway == + /\ phase \notin {"live", "done"} + /\ pred # None + /\ sess[pred].watched + /\ sess[pred].goaway # NoGoaway + +\* An accepted redirect becomes the target and drops a dial still headed for +\* the leaving peer (`Action::Abandon`); anything else is only read. +FollowPred == + /\ queue = <<>> + /\ ~PredDue + /\ PredGoaway + /\ LET uri == sess[pred].goaway IN + IF uri \in Endpoints + THEN /\ sess' = [sess EXCEPT ![pred].watched = FALSE, ![pred].heeded = TRUE] + /\ target' = uri + /\ IF phase = "backoff" + THEN UNCHANGED <> + ELSE /\ phase' = "start" + /\ UNCHANGED timer + /\ dial' = IF phase = "dialing" THEN "none" ELSE dial + /\ probing' = IF phase = "dialing" THEN FALSE ELSE probing + ELSE /\ sess' = [sess EXCEPT ![pred].watched = FALSE] + /\ UNCHANGED <> + /\ UNCHANGED <> + \* `Phase::Next`: ask for a dial. Dial == /\ queue = <<>> /\ ~PredDue + /\ ~PredGoaway /\ phase = "start" /\ phase' = "dialing" /\ timer' = [timer EXCEPT !["dial"] = FALSE] @@ -241,6 +272,7 @@ Dial == Abandon == /\ queue = <<>> /\ ~PredDue + /\ ~PredGoaway /\ phase = "dialing" /\ timer["dial"] /\ dial' = "none" @@ -261,6 +293,7 @@ Abandon == BackoffEnds == /\ queue = <<>> /\ ~PredDue + /\ ~PredGoaway /\ phase = "backoff" /\ timer["backoff"] /\ phase' = "start" @@ -348,7 +381,7 @@ ProbeAbandon == /\ UNCHANGED <> -Policy == Pop \/ Retire \/ IdleClose \/ Dial \/ Abandon \/ BackoffEnds +Policy == Pop \/ Retire \/ IdleClose \/ FollowPred \/ Dial \/ Abandon \/ BackoffEnds \/ LiveEnds \/ ProbeAsk \/ ProbeAbandon (***************************************************************************) @@ -428,7 +461,8 @@ ProbeGetsThrough == \/ /\ ~Goaway(live) /\ phase' = "dialing" /\ timer' = [timer EXCEPT !["dial"] = FALSE, !["handover"] = FALSE] - /\ sess' = [Abort(sess, pred) EXCEPT ![live].sent = TRUE] + \* Its watcher moves with it, unless `Restart::Wait` dropped it. + /\ sess' = [Abort(sess, pred) EXCEPT ![live].sent = TRUE, ![live].watched = watch] /\ pred' = live /\ live' = None /\ probe' = "none" diff --git a/rs/moq-net/verify/reconnect/RedirectLost.cfg b/rs/moq-net/verify/reconnect/RedirectLost.cfg deleted file mode 100644 index 6ce1c02840c..00000000000 --- a/rs/moq-net/verify/reconnect/RedirectLost.cfg +++ /dev/null @@ -1,15 +0,0 @@ -\* Open finding: a GOAWAY after a probe got through is lost, so -\* RedirectFollowed fails. Once fixed, move the rule into Handover.cfg and -\* delete this file. -CONSTANTS - Endpoints = {"a", "b"} - Start = "a" - Refused = "far" - Restart = "redial" - MaxSessions = 3 - -SPECIFICATION Spec -VIEW View - -INVARIANTS - RedirectFollowed diff --git a/rs/moq-net/verify/reconnect/RestartWait.cfg b/rs/moq-net/verify/reconnect/RestartWait.cfg index ffe2e47a78d..da9611b396e 100644 --- a/rs/moq-net/verify/reconnect/RestartWait.cfg +++ b/rs/moq-net/verify/reconnect/RestartWait.cfg @@ -17,3 +17,4 @@ INVARIANTS NoOrphan PresenceBalanced FirstWalkFailsFast + RedirectFollowed From dfeaaa82cdcb002d0188a8f0bd73080f9e549353 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 06:44:32 -0800 Subject: [PATCH 3/3] fix(js/net): hold a fallback's redirect while a replacement serves Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-reconnect/SKILL.md | 2 +- js/net/src/connection/reload.test.ts | 72 ++++++++++++++++++++++++ js/net/src/connection/reload.ts | 37 +++++++++--- 3 files changed, 101 insertions(+), 10 deletions(-) diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index 71f92286208..ef1b52da40c 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -40,7 +40,7 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `4486b7adc108649bb3e7924554394ee30fa0e5f2`. Run `git log 4486b7adc1.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. +Sync commit: `bd617cc1eebc93e0c76a362cc83f2f3852c50244`. Run `git log bd617cc1ee.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. | Code | Model | | --- | --- | diff --git a/js/net/src/connection/reload.test.ts b/js/net/src/connection/reload.test.ts index e62ded16805..b334f0f9749 100644 --- a/js/net/src/connection/reload.test.ts +++ b/js/net/src/connection/reload.test.ts @@ -1739,6 +1739,78 @@ test("a redirect from the draining WebSocket fallback is followed", async () => } }); +// A redirect the draining fallback's peer sends while the probe's replacement serves is kept: +// once that replacement closes and the fallback serves again, the redial goes where the peer +// named. +test("a redirect from the draining WebSocket fallback outlives a replacement that closes", async () => { + using clock = manual(); + const originalWebTransport = globalThis.WebTransport; + const originalStream = (globalThis as { WebSocketStream?: unknown }).WebSocketStream; + const url = new URL("https://fallback-held.example.com/"); + const elsewhere = new URL("https://fallback-held.example.com:4444/"); + + let blocked = true; + const dialed: string[] = []; + const replacements: Promise[] = []; + const servers: Promise[] = []; + let fallback: WebTransport | undefined; + globalThis.WebTransport = function StubWebTransport(to: string | URL) { + if (blocked) { + const refused = Promise.reject(new Error("UDP blocked")); + refused.catch(() => {}); + return { ready: refused, closed: refused, close() {} }; + } + dialed.push(String(to)); + const pair = createMockTransportPair(Lite.ALPN_06); + const server = accept(pair.server, new URL(to)); + server.catch(() => {}); + servers.push(server); + replacements.push(server); + return pair.client; + } as unknown as typeof WebTransport; + (globalThis as { WebSocketStream?: unknown }).WebSocketStream = linkedWebSocketStream((socket, protocol) => { + const transport = Qmux.accept(socket, { protocol }) as unknown as WebTransport; + fallback = transport; + servers.push(transport.ready.then(() => accept(transport, url))); + }); + + const warned = spyOn(console, "warn").mockImplementation(() => {}); + const reload = new Reload({ + url, + versions: [Lite.ALPN_06], + delay: { initial: Duration.millis(1_000) }, + goaway: { handover: Duration.millis(60_000) }, + }); + try { + for (let i = 0; i < 10 && reload.established.peek()?.transport !== "websocket"; i++) { + await clock.advance(Duration.millis(100)); + } + if (!fallback) throw new Error("no fallback session"); + + blocked = false; + for (let i = 0; i < 20 && reload.established.peek()?.transport !== "webtransport"; i++) { + await clock.advance(Duration.millis(1_000)); + } + expect(reload.established.peek()?.transport).toBe("webtransport"); + + const stream = await Stream.open(fallback, { version: Lite.Version.DRAFT_06 }); + await stream.writer.u53(Lite.StreamId.Goaway); + await new Lite.Goaway(elsewhere.href).encode(stream.writer, Lite.Version.DRAFT_06); + await clock.advance(Duration.millis(10)); + + // The replacement closes, so the fallback serves again until the redial. + (await replacements[0])?.abort(); + for (let i = 0; i < 100 && dialed.length < 2; i++) await clock.advance(Duration.millis(100)); + expect(dialed.map((to) => new URL(to).port)).toEqual(["", elsewhere.port]); + } finally { + reload.close(); + warned.mockRestore(); + for (const server of servers) (await server.catch(() => undefined))?.abort(); + globalThis.WebTransport = originalWebTransport; + (globalThis as { WebSocketStream?: unknown }).WebSocketStream = originalStream; + } +}); + // A WebSocket win is provisional: once the session it carried ends, the redial gives // WebTransport its head start again, since the outage that lost the race may be over. test("a WebSocket session ending restores WebTransport's head start", async () => { diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index 5b2357f2bb3..52406fe6928 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -338,6 +338,8 @@ export class Reload { // A session the peer sent GOAWAY on, serving its groups in flight while the replacement // dials. Retired when it closes, at its handover cap, or when the sequence ends. #draining: Draining | undefined; + // A redirect from the draining fallback's peer, held while a replacement serves. + #pending: { connection: Session; next: URL } | undefined; // Sessions this loop ended that are still draining, each until its close settles. #ending = new Map>(); @@ -646,6 +648,7 @@ export class Reload { current = undefined; // A predecessor still draining serves again, so this is no outage. if (this.established.peek() === connection) this.#serve(this.#draining?.serving()); + this.#applyPending(); this.#retry(effect, connected, ended ?? undefined); return; } @@ -810,14 +813,13 @@ export class Reload { /** * Follow a redirect the peer of `connection`, a fallback session we left for a probe that - * got through, sends while it drains and nothing has replaced it yet: the next dial goes - * where the peer named, and a dial still headed for the leaving peer starts over. A restart - * in place or a refused redirect is ignored, since we are already leaving. No loop scoring: + * got through, sends while it drains: whenever nothing else serves, the next dial goes where + * the peer named, and a dial still headed for the leaving peer starts over. A restart in + * place or a refused redirect is ignored, since we are already leaving. No loop scoring: * only a probe that got through leads here, and the probe's backoff paces those. */ #follow(connection: Session, dialing: URL, pinned: boolean): void { void connection.draining.then((drain) => { - if (this.#draining?.serving() !== connection || this.established.peek() !== connection) return; let next: URL | undefined; try { next = target(this.goaway.redirect ?? "same-host", drain.uri, dialing, pinned); @@ -825,14 +827,31 @@ export class Reload { return; } if (!next) return; - console.info("draining fallback peer redirected; following"); - this.#redirect.set(next); - this.#redirectHref = this.#sequenceHref; - // A backoff still paces the next dial, which now goes to the new target. - if (this.#dialing) this.#tick.update((prev) => prev + 1); + this.#pending = { connection, next }; + this.#applyPending(); }); } + /** + * Apply the pending redirect once its session is the one serving, which a replacement that + * closes hands back to; drop it once that session retires. + */ + #applyPending(): void { + const pending = this.#pending; + if (!pending) return; + if (this.#draining?.serving() !== pending.connection) { + this.#pending = undefined; + return; + } + if (this.established.peek() !== pending.connection) return; + this.#pending = undefined; + console.info("draining fallback peer redirected; following"); + this.#redirect.set(pending.next); + this.#redirectHref = this.#sequenceHref; + // A backoff still paces the next dial, which now goes to the new target. + if (this.#dialing) this.#tick.update((prev) => prev + 1); + } + /** * End a session this loop is done with. Its groups in flight finish within the close * deadline, unless the page is going away: nothing would run to finish them, and an abort