diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index 0e9ca3c98cf..ef1b52da40c 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -26,16 +26,13 @@ Each rule is an INVARIANT; the harm says why it matters. All but `NoOrphan`, `Re ```sh quest gate just rs tla moq-net/verify/reconnect/Handover.tla quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/verify/reconnect/RestartWait.cfg -quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/verify/reconnect/RedirectLost.cfg ``` -The first checks every rule but the one with an open finding, over four sessions: about 1.7M distinct states, under a minute. The second checks the same rules under `Restart::Wait` over three sessions: about 290k states. Both pass at the sync commit. The last fails at the sync commit, on its finding below, within seconds. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. +The first checks every rule over four sessions: about 1.4M distinct states, under a minute. The second checks the same rules under `Restart::Wait` over three sessions: about 300k states. Both pass at the sync commit. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. ## Open findings -### A redirect after a probe got through is lost - -`RedirectLost.cfg` fails. On a fallback, `probed` moves the live session into the predecessor and drops its `goaway::Consumer` with the `Live` it came from, so nothing reads that session's GOAWAY anymore. A redirect its peer sends while the handshake over the preferred transport runs is never followed: when that handshake fails, the next walk dials the address the peer is leaving. `residual_a_redirect_after_a_probe_got_through_is_lost` in `reconnect/tests.rs` replays it. Once fixed, flip that test, move `RedirectFollowed` into `Handover.cfg`, and delete `RedirectLost.cfg`. +None. ## Reading a failure @@ -43,11 +40,11 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `4486b7adc108649bb3e7924554394ee30fa0e5f2`. Run `git log 4486b7adc1.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. +Sync commit: `bd617cc1eebc93e0c76a362cc83f2f3852c50244`. Run `git log bd617cc1ee.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. | Code | Model | | --- | --- | -| `rs/moq-net/src/reconnect.rs` `Reconnect::poll` (queue, predecessor, phases) | `Pop`, `Retire`, `IdleClose`, `Dial`, `Abandon`, `BackoffEnds`, `LiveEnds`, `ProbeAsk`, `ProbeAbandon` | +| `rs/moq-net/src/reconnect.rs` `Reconnect::poll` (queue, predecessor, phases), `follow` | `Pop`, `Retire`, `IdleClose`, `FollowPred`, `Dial`, `Abandon`, `BackoffEnds`, `LiveEnds`, `ProbeAsk`, `ProbeAbandon` | | `rs/moq-net/src/reconnect.rs` `established`, and `walk_failed` ending the first walk | `Established`, `Failed`, `Abandon` | | `rs/moq-net/src/reconnect.rs` `connected`, `fallback`, `failed`, `probed` | `Connected`, `Failed`, `ProbeFails`, `ProbeGetsThrough` | | `rs/moq-net/src/reconnect.rs` `end`, `closed`, `done`, `Predecessor` | `EndGoaway`, `EndClosed`, `Done`, `Retire`, `IdleClose` | diff --git a/js/net/src/connection/reload.test.ts b/js/net/src/connection/reload.test.ts index 0439262c224..b334f0f9749 100644 --- a/js/net/src/connection/reload.test.ts +++ b/js/net/src/connection/reload.test.ts @@ -1670,6 +1670,147 @@ test("a WebSocket fallback returns to WebTransport once it is reachable", async } }); +// A redirect the fallback's peer sends while it drains, after a probe got through but its +// handshake failed, still decides where the replacement goes: the redial after the backoff goes +// where the peer named, not back to the node that is leaving. +test("a redirect from the draining WebSocket fallback is followed", async () => { + using clock = manual(); + const originalWebTransport = globalThis.WebTransport; + const originalStream = (globalThis as { WebSocketStream?: unknown }).WebSocketStream; + const url = new URL("https://fallback-redirect.example.com/"); + const elsewhere = new URL("https://fallback-redirect.example.com:4444/"); + + let blocked = true; + const dialed: string[] = []; + const servers: Promise[] = []; + let fallback: WebTransport | undefined; + globalThis.WebTransport = function StubWebTransport(to: string | URL) { + if (blocked) { + const refused = Promise.reject(new Error("UDP blocked")); + refused.catch(() => {}); + return { ready: refused, closed: refused, close() {} }; + } + dialed.push(String(to)); + // The leaving node answers the probe, but no MoQ version is in common with it. + if (new URL(to).port !== elsewhere.port) return createMockTransportPair("h3").client; + const pair = createMockTransportPair(Lite.ALPN_06); + const server = accept(pair.server, new URL(to)); + server.catch(() => {}); + servers.push(server); + return pair.client; + } as unknown as typeof WebTransport; + (globalThis as { WebSocketStream?: unknown }).WebSocketStream = linkedWebSocketStream((socket, protocol) => { + const transport = Qmux.accept(socket, { protocol }) as unknown as WebTransport; + fallback = transport; + servers.push(transport.ready.then(() => accept(transport, url))); + }); + + const warned = spyOn(console, "warn").mockImplementation(() => {}); + const reload = new Reload({ url, versions: [Lite.ALPN_06], delay: { initial: Duration.millis(5_000) } }); + try { + for (let i = 0; i < 10 && reload.established.peek()?.transport !== "websocket"; i++) { + await clock.advance(Duration.millis(100)); + } + const server = servers[0]; + if (!server || !fallback) throw new Error("no fallback session"); + + blocked = false; + // The probe gets through, the fallback tells its peer it is leaving, and the handshake + // over the probe fails, so the loop backs off with the fallback still serving. + for (let i = 0; i < 20 && dialed.length === 0; i++) await clock.advance(Duration.millis(1_000)); + expect(await (await server).draining).toEqual({ uri: "" }); + await clock.advance(Duration.millis(10)); + expect(reload.established.peek()?.transport).toBe("websocket"); + + const stream = await Stream.open(fallback, { version: Lite.Version.DRAFT_06 }); + await stream.writer.u53(Lite.StreamId.Goaway); + await new Lite.Goaway(elsewhere.href).encode(stream.writer, Lite.Version.DRAFT_06); + for (let i = 0; i < 100 && reload.established.peek()?.transport !== "webtransport"; i++) { + await clock.advance(Duration.millis(100)); + } + expect(reload.established.peek()?.transport).toBe("webtransport"); + expect(dialed.map((to) => new URL(to).port)).toEqual(["", elsewhere.port]); + } finally { + reload.close(); + warned.mockRestore(); + for (const server of servers) (await server.catch(() => undefined))?.abort(); + globalThis.WebTransport = originalWebTransport; + (globalThis as { WebSocketStream?: unknown }).WebSocketStream = originalStream; + } +}); + +// A redirect the draining fallback's peer sends while the probe's replacement serves is kept: +// once that replacement closes and the fallback serves again, the redial goes where the peer +// named. +test("a redirect from the draining WebSocket fallback outlives a replacement that closes", async () => { + using clock = manual(); + const originalWebTransport = globalThis.WebTransport; + const originalStream = (globalThis as { WebSocketStream?: unknown }).WebSocketStream; + const url = new URL("https://fallback-held.example.com/"); + const elsewhere = new URL("https://fallback-held.example.com:4444/"); + + let blocked = true; + const dialed: string[] = []; + const replacements: Promise[] = []; + const servers: Promise[] = []; + let fallback: WebTransport | undefined; + globalThis.WebTransport = function StubWebTransport(to: string | URL) { + if (blocked) { + const refused = Promise.reject(new Error("UDP blocked")); + refused.catch(() => {}); + return { ready: refused, closed: refused, close() {} }; + } + dialed.push(String(to)); + const pair = createMockTransportPair(Lite.ALPN_06); + const server = accept(pair.server, new URL(to)); + server.catch(() => {}); + servers.push(server); + replacements.push(server); + return pair.client; + } as unknown as typeof WebTransport; + (globalThis as { WebSocketStream?: unknown }).WebSocketStream = linkedWebSocketStream((socket, protocol) => { + const transport = Qmux.accept(socket, { protocol }) as unknown as WebTransport; + fallback = transport; + servers.push(transport.ready.then(() => accept(transport, url))); + }); + + const warned = spyOn(console, "warn").mockImplementation(() => {}); + const reload = new Reload({ + url, + versions: [Lite.ALPN_06], + delay: { initial: Duration.millis(1_000) }, + goaway: { handover: Duration.millis(60_000) }, + }); + try { + for (let i = 0; i < 10 && reload.established.peek()?.transport !== "websocket"; i++) { + await clock.advance(Duration.millis(100)); + } + if (!fallback) throw new Error("no fallback session"); + + blocked = false; + for (let i = 0; i < 20 && reload.established.peek()?.transport !== "webtransport"; i++) { + await clock.advance(Duration.millis(1_000)); + } + expect(reload.established.peek()?.transport).toBe("webtransport"); + + const stream = await Stream.open(fallback, { version: Lite.Version.DRAFT_06 }); + await stream.writer.u53(Lite.StreamId.Goaway); + await new Lite.Goaway(elsewhere.href).encode(stream.writer, Lite.Version.DRAFT_06); + await clock.advance(Duration.millis(10)); + + // The replacement closes, so the fallback serves again until the redial. + (await replacements[0])?.abort(); + for (let i = 0; i < 100 && dialed.length < 2; i++) await clock.advance(Duration.millis(100)); + expect(dialed.map((to) => new URL(to).port)).toEqual(["", elsewhere.port]); + } finally { + reload.close(); + warned.mockRestore(); + for (const server of servers) (await server.catch(() => undefined))?.abort(); + globalThis.WebTransport = originalWebTransport; + (globalThis as { WebSocketStream?: unknown }).WebSocketStream = originalStream; + } +}); + // A WebSocket win is provisional: once the session it carried ends, the redial gives // WebTransport its head start again, since the outage that lost the race may be over. test("a WebSocket session ending restores WebTransport's head start", async () => { diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index 7cd890847d1..52406fe6928 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -338,6 +338,8 @@ export class Reload { // A session the peer sent GOAWAY on, serving its groups in flight while the replacement // dials. Retired when it closes, at its handover cap, or when the sequence ends. #draining: Draining | undefined; + // A redirect from the draining fallback's peer, held while a replacement serves. + #pending: { connection: Session; next: URL } | undefined; // Sessions this loop ended that are still draining, each until its close settles. #ending = new Map>(); @@ -646,6 +648,7 @@ export class Reload { current = undefined; // A predecessor still draining serves again, so this is no outage. if (this.established.peek() === connection) this.#serve(this.#draining?.serving()); + this.#applyPending(); this.#retry(effect, connected, ended ?? undefined); return; } @@ -664,6 +667,7 @@ export class Reload { // backoff already paces it, so its replacement dials at once. A fallback that // served past the healthy bar still earns the healthy reset. if (reached) { + this.#follow(connection, at, pinnedTransport(connection.transport, this.#pinned())); if (now().since(connected).ge(this.#pacing().healthy)) { this.#delay = undefined; this.#deadline = undefined; @@ -766,10 +770,8 @@ export class Reload { * redirect is refused, which ends the sequence rather than redialing. */ #migrate(connection: Session, dialing: URL, drain: Drain): boolean { - const hashes = this.webtransport?.serverCertificateHashes?.length ?? 0; - const configured = hashes > 0 || this.webtransport?.serverCertificate !== undefined; // The pin is a WebTransport option. A WebSocket that won the race never used it. - const pinned = pinnedTransport(connection.transport, configured); + const pinned = pinnedTransport(connection.transport, this.#pinned()); let next: URL | undefined; try { @@ -803,6 +805,53 @@ export class Reload { return true; } + /** Whether a certificate pin is configured for WebTransport. */ + #pinned(): boolean { + const hashes = this.webtransport?.serverCertificateHashes?.length ?? 0; + return hashes > 0 || this.webtransport?.serverCertificate !== undefined; + } + + /** + * Follow a redirect the peer of `connection`, a fallback session we left for a probe that + * got through, sends while it drains: whenever nothing else serves, the next dial goes where + * the peer named, and a dial still headed for the leaving peer starts over. A restart in + * place or a refused redirect is ignored, since we are already leaving. No loop scoring: + * only a probe that got through leads here, and the probe's backoff paces those. + */ + #follow(connection: Session, dialing: URL, pinned: boolean): void { + void connection.draining.then((drain) => { + let next: URL | undefined; + try { + next = target(this.goaway.redirect ?? "same-host", drain.uri, dialing, pinned); + } catch { + return; + } + if (!next) return; + this.#pending = { connection, next }; + this.#applyPending(); + }); + } + + /** + * Apply the pending redirect once its session is the one serving, which a replacement that + * closes hands back to; drop it once that session retires. + */ + #applyPending(): void { + const pending = this.#pending; + if (!pending) return; + if (this.#draining?.serving() !== pending.connection) { + this.#pending = undefined; + return; + } + if (this.established.peek() !== pending.connection) return; + this.#pending = undefined; + console.info("draining fallback peer redirected; following"); + this.#redirect.set(pending.next); + this.#redirectHref = this.#sequenceHref; + // A backoff still paces the next dial, which now goes to the new target. + if (this.#dialing) this.#tick.update((prev) => prev + 1); + } + /** * End a session this loop is done with. Its groups in flight finish within the close * deadline, unless the page is going away: nothing would run to finish them, and an abort diff --git a/quest/m1/reconnect-probe-redirect.md b/quest/m1/reconnect-probe-redirect.md deleted file mode 100644 index b29b0b6a882..00000000000 --- a/quest/m1/reconnect-probe-redirect.md +++ /dev/null @@ -1,33 +0,0 @@ -# [S] A GOAWAY to a draining fallback session is followed - -## Goal - -When a WebSocket fallback session drains because a QUIC probe got through, -and its peer then sends a GOAWAY naming another node, the client's next dial -goes to that node, not back to the one that is leaving. - -## Plan - -Found 2026-11-10 by the reconnect model ([moq-dev/moq#8447](https://github.com/moq-dev/moq/pull/8447)): -`rs/moq-net/verify/reconnect/RedirectLost.cfg` breaks `RedirectFollowed`. -`probed()` in `rs/moq-net/src/reconnect.rs` drops the fallback session's -GOAWAY watcher along with its `Live` state, so a redirect that arrives while -it drains is lost; if the QUIC handshake then fails, the policy redials the -leaving node until the handover window ends, then reports offline. -`residual_a_redirect_after_a_probe_got_through_is_lost` in -`reconnect/tests.rs` replays it. - -Recommended fix: keep the GOAWAY watcher on the predecessor, and while no -replacement serves, act on an accepted redirect the way `end()` does. Flip the -residual test, move `RedirectFollowed` into `Handover.cfg`, delete -`RedirectLost.cfg`, and update the `verify-reconnect` skill. - -Also fix the `Backoff` doc in the same file: it promises a GOAWAY back to an -endpoint that already redirected us is never followed at once, but the -`Redirects` cycle check catches a loop only after a lap (A to B, B to C, C to B -is followed once). Reword the doc to what the check does; no code change. - -Caused by: [moq-dev/moq#8330](https://github.com/moq-dev/moq/pull/8330), which -added the fallback probe. - -Public API: none. Wire: none. diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs index ef837f59027..5a3c57e95ec 100644 --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs @@ -64,10 +64,11 @@ const PROBE_MAX: Duration = Duration::from_secs(300); /// peer sends one to each session until it goes, and each of those serves, so it /// restarts the timeout instead. /// -/// A GOAWAY off a healthy session migrates at once, unless it sends the connection back -/// to an endpoint that already redirected it elsewhere and the session lived less than -/// `max`. Two peers redirecting to each other thus wait out this backoff and count -/// against the timeout. +/// A GOAWAY off a healthy session migrates at once, unless the run of redirects, among +/// sessions that each lived less than `max`, has looped back through an endpoint. The loop +/// check keeps one endpoint, so it catches a cycle within a few laps rather than at the +/// first revisit: A to B, B to C, C to B is followed once before it is caught. Peers +/// redirecting in a loop then wait out this backoff and count against the timeout. #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[non_exhaustive] pub struct Backoff { @@ -324,7 +325,7 @@ pub enum Action { /// [`Attempt::client`]. Dial(Attempt), /// Drop the dial or probe in flight: it ran out its deadline, which already counted as - /// its failure. + /// its failure, or the predecessor's peer redirected while it headed for that peer. Abandon, /// The [`predecessor`](Reconnect::predecessor) serves while a replacement is dialed: the /// live session got a GOAWAY and now drains as it, or the replacement closed while it @@ -558,6 +559,11 @@ impl + From> Reco continue; } } + // Until a replacement serves, a redirect the predecessor's peer sends decides where + // the next dial goes. + if !live && let Some(action) = self.follow(waiter) { + return action; + } let handover = self.predecessor.as_ref().map(|predecessor| predecessor.deadline); match &mut self.phase { @@ -727,10 +733,20 @@ impl + From> Reco if let Some(older) = self.predecessor.take() { older.retire(); } + // Its peer may still redirect us while it drains, which `follow` acts on. A fixed + // target never moves, so it needs no watch. + let redirect = old + .goaway + .filter(|_| !self.targets[index].fixed()) + .map(|goaway| Redirected { + goaway, + url: self.targets[index].url().clone(), + }); self.predecessor = Some(Predecessor { deadline: now.saturating_add(self.config.goaway.handover), session: old.session, closing: false, + redirect, }); // Only now, once the fallback stops taking new work: a probe that never gets through // leaves the serving session's tokens and their grants alone. @@ -883,6 +899,51 @@ impl + From> Reco self.back_off(now, self.config.backoff.timeout, false, "reconnecting after backoff"); } + /// Act on a GOAWAY the predecessor's peer sent after a probe moved us off it, the way + /// [`end`](Self::end) would have: an accepted redirect becomes the only target, and a dial + /// still headed for the leaving peer is dropped. Anything else leaves the targets alone, + /// since the policy is already leaving that session. + fn follow(&mut self, waiter: &kio::Waiter) -> Option> { + let predecessor = self.predecessor.as_mut()?; + let redirect = predecessor.redirect.as_ref()?; + let Poll::Ready(msg) = redirect.goaway.poll(waiter) else { + return None; + }; + let redirect = predecessor.redirect.take()?; + let target = match msg.map(|msg| { + self.config + .goaway + .redirect + .target(msg.uri(), &redirect.url, self.config.pinned) + }) { + Ok(Ok(Some(target))) => target, + // A closed session, a restart in place, or a refused redirect: nothing to follow. + _ => return None, + }; + tracing::info!(peer = %Shown(&target), "draining fallback peer redirected; following"); + // No loop scoring here: only a probe that got through leads here, and the probe's + // backoff paces those. The session this dial lands on is scored when it ends. + let redirected = std::mem::replace(&mut self.targets, vec![A::from(target)]); + self.configured.get_or_insert(redirected); + // A walk of the replaced targets gives way to one of the new target. + if matches!(self.phase, Phase::Dialing(_) | Phase::Next { .. }) { + self.walk = Walk::default(); + } + match self.phase { + Phase::Dialing(_) => { + self.phase = Phase::Start; + Some(Action::Abandon) + } + // An index into the replaced targets. + Phase::Next { .. } => { + self.phase = Phase::Start; + None + } + // A backoff still paces the next dial, which now goes to the new target. + _ => None, + } + } + /// Score how a live session ended and decide what follows. fn end(&mut self, now: Instant, ended: Ended) { let Phase::Live(live) = std::mem::replace(&mut self.phase, Phase::Start) else { @@ -964,6 +1025,8 @@ impl + From> Reco deadline: now.saturating_add(self.config.goaway.handover(msg.timeout())), session: live.session, closing: false, + // Its GOAWAY was just acted on. + redirect: None, }); self.queue.push_back(Action::Migrating); @@ -1153,6 +1216,14 @@ struct Predecessor { deadline: Instant, /// Whether [`Self::poll_idle`] started the graceful close. closing: bool, + /// Watches for a redirect from the peer, set when a probe moved us off this session. + redirect: Option, +} + +/// A drained fallback session's GOAWAY, and the URL a redirect in it is judged against. +struct Redirected { + goaway: goaway::Consumer, + url: Url, } impl Predecessor { diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index 26787a7591b..c08da1a6807 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -912,12 +912,11 @@ async fn a_peer_goaway_wins_over_a_probe() { ); } -/// Pins an open finding (`verify-reconnect`, `RedirectFollowed`): once a probe got -/// through, the fallback session drains as the predecessor and nothing watches its peer's -/// GOAWAY anymore, so a redirect the peer sends while the handshake runs is never read and -/// the redial goes back to the node that is leaving. +/// Once a probe got through, the fallback session drains as the predecessor, and a +/// redirect its peer sends while the handshake runs still decides where we go: the +/// handshake to the leaving node is dropped and the dial goes where the peer named. #[kio::test] -async fn residual_a_redirect_after_a_probe_got_through_is_lost() { +async fn a_redirect_after_a_probe_got_through_is_followed() { let (mut machine, fallback_server) = on_fallback(Config::default()).await; probe(&mut machine).await; assert!(machine.probed(kio::clock::now(), Ok(()))); @@ -928,19 +927,34 @@ async fn residual_a_redirect_after_a_probe_got_through_is_lost() { .drain() .send(goaway::Goaway::redirect(elsewhere.to_string())) .expect("the fallback peer sends a GOAWAY"); - kio::clock::sim::idle().await; - // The handshake over the preferred transport fails, so the policy dials again. + assert!(matches!(next(&mut machine).await, Action::Abandon)); + let Action::Dial(attempt) = next(&mut machine).await else { + panic!("expected a dial"); + }; + assert_eq!(attempt.target(), &elsewhere); + assert!(machine.predecessor().is_some(), "the redirected session still drains"); +} + +/// A redirect from the draining fallback's peer during the backoff after a failed +/// handshake keeps the wait, and the redial goes where the peer named. +#[kio::test] +async fn a_redirect_during_the_backoff_after_a_probe_is_followed() { + let (mut machine, fallback_server) = on_fallback(Config::default()).await; + probe(&mut machine).await; + assert!(machine.probed(kio::clock::now(), Ok(()))); + assert!(matches!(next(&mut machine).await, Action::Migrating)); machine.failed(kio::clock::now(), TestError::Dial, Verdict::Retry); assert!(matches!(next(&mut machine).await, Action::Retrying(_))); + + let elsewhere = url("https://peer:2/"); + fallback_server + .drain() + .send(goaway::Goaway::redirect(elsewhere.to_string())) + .expect("the fallback peer sends a GOAWAY"); let Action::Dial(attempt) = next(&mut machine).await else { panic!("expected a redial"); }; - assert_eq!( - attempt.target(), - &url("https://peer:1/"), - "the redial followed the redirect: flip this test" - ); - assert!(machine.predecessor().is_some(), "the redirected session still drains"); + assert_eq!(attempt.target(), &elsewhere); } /// A client and server session where the client holds a subscription the server serves, diff --git a/rs/moq-net/verify/reconnect/Handover.cfg b/rs/moq-net/verify/reconnect/Handover.cfg index 7a0638a8a0d..ea4c0c89aa3 100644 --- a/rs/moq-net/verify/reconnect/Handover.cfg +++ b/rs/moq-net/verify/reconnect/Handover.cfg @@ -1,5 +1,4 @@ \* Two endpoints a GOAWAY may move between, one it may not, and four sessions. -\* RedirectFollowed lives in RedirectLost.cfg while its finding is open. CONSTANTS Endpoints = {"a", "b"} Start = "a" @@ -17,3 +16,4 @@ INVARIANTS NoOrphan PresenceBalanced FirstWalkFailsFast + RedirectFollowed diff --git a/rs/moq-net/verify/reconnect/Handover.tla b/rs/moq-net/verify/reconnect/Handover.tla index 3a4ebf35512..0f9ad8f38dd 100644 --- a/rs/moq-net/verify/reconnect/Handover.tla +++ b/rs/moq-net/verify/reconnect/Handover.tla @@ -68,10 +68,11 @@ vars == <> \* "abort" for `Session::abort`, "drain" for a `start_close` that finished), \* whether a `start_close` began, the GOAWAY it got, its age, whether it \* carries no subscription or fetch, whether the policy acted on its GOAWAY, -\* and whether we sent our own. +\* whether we sent our own, and whether the policy watches its GOAWAY as the +\* predecessor (`Predecessor::redirect`, set when a probe moved us off it). Fresh(e) == [ep |-> e, open |-> TRUE, by |-> "", closing |-> FALSE, goaway |-> NoGoaway, age |-> 0, idle |-> FALSE, heeded |-> FALSE, - sent |-> FALSE] + sent |-> FALSE, watched |-> FALSE] Unused == [Fresh(Start) EXCEPT !.open = FALSE] Timers == {"dial", "backoff", "probe", "handover"} @@ -222,10 +223,40 @@ Misdial(e) == /\ ~sess[s].heeded /\ sess[s].goaway # e +\* `Reconnect::follow`, which runs before the phase while no replacement +\* serves: the predecessor's peer sent a GOAWAY. +PredGoaway == + /\ phase \notin {"live", "done"} + /\ pred # None + /\ sess[pred].watched + /\ sess[pred].goaway # NoGoaway + +\* An accepted redirect becomes the target and drops a dial still headed for +\* the leaving peer (`Action::Abandon`); anything else is only read. +FollowPred == + /\ queue = <<>> + /\ ~PredDue + /\ PredGoaway + /\ LET uri == sess[pred].goaway IN + IF uri \in Endpoints + THEN /\ sess' = [sess EXCEPT ![pred].watched = FALSE, ![pred].heeded = TRUE] + /\ target' = uri + /\ IF phase = "backoff" + THEN UNCHANGED <> + ELSE /\ phase' = "start" + /\ UNCHANGED timer + /\ dial' = IF phase = "dialing" THEN "none" ELSE dial + /\ probing' = IF phase = "dialing" THEN FALSE ELSE probing + ELSE /\ sess' = [sess EXCEPT ![pred].watched = FALSE] + /\ UNCHANGED <> + /\ UNCHANGED <> + \* `Phase::Next`: ask for a dial. Dial == /\ queue = <<>> /\ ~PredDue + /\ ~PredGoaway /\ phase = "start" /\ phase' = "dialing" /\ timer' = [timer EXCEPT !["dial"] = FALSE] @@ -241,6 +272,7 @@ Dial == Abandon == /\ queue = <<>> /\ ~PredDue + /\ ~PredGoaway /\ phase = "dialing" /\ timer["dial"] /\ dial' = "none" @@ -261,6 +293,7 @@ Abandon == BackoffEnds == /\ queue = <<>> /\ ~PredDue + /\ ~PredGoaway /\ phase = "backoff" /\ timer["backoff"] /\ phase' = "start" @@ -348,7 +381,7 @@ ProbeAbandon == /\ UNCHANGED <> -Policy == Pop \/ Retire \/ IdleClose \/ Dial \/ Abandon \/ BackoffEnds +Policy == Pop \/ Retire \/ IdleClose \/ FollowPred \/ Dial \/ Abandon \/ BackoffEnds \/ LiveEnds \/ ProbeAsk \/ ProbeAbandon (***************************************************************************) @@ -428,7 +461,8 @@ ProbeGetsThrough == \/ /\ ~Goaway(live) /\ phase' = "dialing" /\ timer' = [timer EXCEPT !["dial"] = FALSE, !["handover"] = FALSE] - /\ sess' = [Abort(sess, pred) EXCEPT ![live].sent = TRUE] + \* Its watcher moves with it, unless `Restart::Wait` dropped it. + /\ sess' = [Abort(sess, pred) EXCEPT ![live].sent = TRUE, ![live].watched = watch] /\ pred' = live /\ live' = None /\ probe' = "none" diff --git a/rs/moq-net/verify/reconnect/RedirectLost.cfg b/rs/moq-net/verify/reconnect/RedirectLost.cfg deleted file mode 100644 index 6ce1c02840c..00000000000 --- a/rs/moq-net/verify/reconnect/RedirectLost.cfg +++ /dev/null @@ -1,15 +0,0 @@ -\* Open finding: a GOAWAY after a probe got through is lost, so -\* RedirectFollowed fails. Once fixed, move the rule into Handover.cfg and -\* delete this file. -CONSTANTS - Endpoints = {"a", "b"} - Start = "a" - Refused = "far" - Restart = "redial" - MaxSessions = 3 - -SPECIFICATION Spec -VIEW View - -INVARIANTS - RedirectFollowed diff --git a/rs/moq-net/verify/reconnect/RestartWait.cfg b/rs/moq-net/verify/reconnect/RestartWait.cfg index ffe2e47a78d..da9611b396e 100644 --- a/rs/moq-net/verify/reconnect/RestartWait.cfg +++ b/rs/moq-net/verify/reconnect/RestartWait.cfg @@ -17,3 +17,4 @@ INVARIANTS NoOrphan PresenceBalanced FirstWalkFailsFast + RedirectFollowed