From 238981d6d4d6ad7c7b25cf15d09e5da820a31f37 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 12 Nov 2026 15:23:28 -0800 Subject: [PATCH 01/12] chore(quest): claim quest/m3/reconnect-predecessor-status.md Quest-Claim: ac5f58eb6cb84c5c From a1b7449af63399d13e475a6adff9c7acd1f40472 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 12 Nov 2026 16:49:08 -0800 Subject: [PATCH 02/12] fix(reconnect): a dropped replacement migrates back to the serving predecessor Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/reconnect.rs | 34 +++++-- rs/moq-net/src/reconnect/tests.rs | 65 ++++++++++-- rs/moq-tokio/src/connection.rs | 163 +++++++++++++++++++++++++----- 3 files changed, 219 insertions(+), 43 deletions(-) diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs index 1b13b7ca336..883a24bf524 100644 --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs @@ -344,15 +344,16 @@ pub enum Action { /// Drop the dial or probe in flight: it ran out its deadline, which already counted as /// its failure. Abandon, - /// The live session got a GOAWAY and now drains as the [`predecessor`](Reconnect::predecessor) - /// while a replacement is dialed. + /// The [`predecessor`](Reconnect::predecessor) serves while a replacement is dialed: the + /// live session got a GOAWAY and now drains as it, or the replacement closed while it + /// still drains. Migrating, - /// The predecessor closed or overstayed its handover. + /// The predecessor closed, overstayed its handover, or gave way to a newer one. Retired, /// Nothing serves anymore: the live session ended with this error, or the predecessor /// retired between sessions. A redial follows. Offline(Option), - /// Why the policy is waiting to redial: a walk failed or a session ended too soon. + /// Why the policy is waiting to redial: a walk failed, or a session ended. Retrying(E), /// Poll again by this instant, or on a wake. Wait(Option), @@ -740,9 +741,7 @@ impl + From> Reco now.saturating_duration_since(old.since), &Pacing::new(&self.config.backoff), ); - if let Some(older) = self.predecessor.take() { - older.retire(); - } + self.retire(); self.predecessor = Some(Predecessor { deadline: now.saturating_add(self.config.goaway.handover), session: old.session, @@ -975,9 +974,7 @@ impl + From> Reco // serving until it closes or overstays, so its routes stay attached and live tracks // splice onto the replacement at a group boundary. tracing::info!(peer = %Shown(&url), "upstream GOAWAY; migrating"); - if let Some(old) = self.predecessor.take() { - old.retire(); - } + self.retire(); self.predecessor = Some(Predecessor { deadline: now.saturating_add(self.config.goaway.handover(msg.timeout())), session: live.session, @@ -1029,7 +1026,14 @@ impl + From> Reco return self.done(Err(err)); } - self.queue.push_back(Action::Offline(Some(err.clone()))); + // A predecessor still draining serves again, so this is no outage: the caller moves + // back onto it. One closing gracefully already carries nothing and takes no new work. + if self.predecessor.as_ref().is_some_and(|predecessor| !predecessor.closing) { + self.queue.push_back(Action::Migrating); + self.queue.push_back(Action::Retrying(err.clone())); + } else { + self.queue.push_back(Action::Offline(Some(err.clone()))); + } if healthy { // The window was reset, so a one-off drop reconnects promptly. tracing::warn!(peer = %Shown(url), "session closed, reconnecting"); @@ -1077,6 +1081,14 @@ impl + From> Reco .into() } + /// Close the predecessor ahead of its handover, for a newer one to replace it. + fn retire(&mut self) { + if let Some(older) = self.predecessor.take() { + older.retire(); + self.queue.push_back(Action::Retired); + } + } + fn done(&mut self, result: Result<(), E>) { self.predecessor = None; self.phase = Phase::Done; diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index 2d55c7fabf2..2962cddeb61 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -987,11 +987,11 @@ async fn busy_handshake() -> (Session, Session, Vec>) { ) } -/// Pins an open finding (`verify-reconnect`, `OfflineMeansNothingServes`): a replacement -/// that drops while the predecessor still drains reports [`Action::Offline`], whose -/// promise is that nothing serves, though the predecessor still carries a subscription. +/// A replacement that drops while the predecessor still serves hands back to it as +/// [`Action::Migrating`], never [`Action::Offline`]: nothing is down, and the predecessor +/// still carries a subscription. Only its retirement then takes the connection offline. #[kio::test] -async fn residual_a_dropped_replacement_reports_offline_while_the_predecessor_serves() { +async fn a_dropped_replacement_migrates_back_to_the_serving_predecessor() { let a = url("tcp://peer:1/"); let b = url("tcp://peer:2/"); let mut machine: Reconnect = Reconnect::new(Config::default(), vec![a], kio::clock::now()).unwrap(); @@ -1011,16 +1011,63 @@ async fn residual_a_dropped_replacement_reports_offline_while_the_predecessor_se drop(replacement_server); let action = next(&mut machine).await; - assert!( - matches!(action, Action::Offline(Some(_))), - "the replacement's drop no longer reports offline: flip this test ({action:?})" - ); - let (predecessor, _) = machine.predecessor().expect("the predecessor still drains"); + assert!(matches!(action, Action::Migrating), "{action:?}"); + let action = next(&mut machine).await; + assert!(matches!(action, Action::Retrying(TestError::Net(_))), "{action:?}"); + assert!(machine.session().is_none(), "the replacement is gone"); + let (predecessor, deadline) = machine.predecessor().expect("the predecessor still drains"); assert!(predecessor.poll_closed(&kio::Waiter::noop()).is_pending()); assert!( predecessor.poll_idle(&kio::Waiter::noop()).is_pending(), "it still serves" ); + + // The redial waits out its backoff while the predecessor serves, and its handover still + // bounds it: once that ends, nothing serves. + assert!(matches!(next(&mut machine).await, Action::Dial(_))); + let action = next(&mut machine).await; + assert!(matches!(action, Action::Retired), "{action:?}"); + assert!(kio::clock::now() >= deadline); + let action = next(&mut machine).await; + assert!(matches!(action, Action::Offline(None)), "{action:?}"); +} + +/// A replacement that drops while the predecessor closes gracefully, carrying nothing, +/// leaves nothing serving: it reports [`Action::Offline`]. +#[kio::test] +async fn a_dropped_replacement_goes_offline_over_a_closing_predecessor() { + let a = url("tcp://peer:1/"); + let mut machine: Reconnect = + Reconnect::new(Config::default(), vec![a.clone()], kio::clock::now()).unwrap(); + let mut run = Run::default(); + drive(&mut machine, &[(a, Peer::Serve)], &mut run, |run| !run.accepts.is_empty()).await; + + kio::clock::sleep(Duration::from_secs(2)).await; + run.servers[0].drain().send(goaway::Goaway::new()).unwrap(); + assert!(matches!(next(&mut machine).await, Action::Migrating)); + assert!(matches!(next(&mut machine).await, Action::Dial(_))); + let (replacement, replacement_server) = handshake().await; + machine.connected(kio::clock::now(), replacement); + // The idle predecessor starts its graceful close under the replacement. + assert!(matches!( + machine.poll(kio::clock::now(), &kio::Waiter::noop()), + Action::Wait(_) + )); + drop(replacement_server); + + // Whether the graceful close finishes first only orders `Retired` against the offline. + let mut actions = Vec::new(); + loop { + match next(&mut machine).await { + Action::Dial(_) => break, + Action::Retired => actions.push("retired"), + Action::Offline(Some(_)) => actions.push("offline"), + Action::Retrying(_) => actions.push("retrying"), + action => panic!("unexpected {action:?}"), + } + } + actions.sort(); + assert_eq!(actions, ["offline", "retired"]); } /// A target that never reaches the preferred transport keeps its fallback session and diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index 7bd7c4f9895..4567341f5b7 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -445,7 +445,14 @@ struct Migrating { /// The policy's draining predecessor and its handover deadline, so [`Connection::close`] drains it /// too without overstaying that window. Boxed: only a migration holds one, and only briefly. -type Predecessor = Box<(moq_net::Session, kio::clock::Instant)>; +type Predecessor = Box; + +/// A session draining after a GOAWAY, which serves again if its replacement drops first. +struct Draining { + session: moq_net::Session, + deadline: kio::clock::Instant, + transport: crate::Transport, +} impl Lifecycle { /// What [`Connection::status`] reports: `None` before the first connect and once closed. @@ -544,29 +551,53 @@ impl Shared { } } - /// The session in [`State`] keeps serving while its replacement comes up, so - /// only the status moves. - fn migrating(&self) { - if let Ok(mut state) = self.state.write() { - state.revision += 1; - if let Lifecycle::Live(live) = &mut state.lifecycle { - live.migrating = Some(Migrating::default()); + /// The policy's predecessor, `session` draining until `deadline`, serves while a + /// replacement comes up. + /// + /// With no predecessor mirrored yet, the live session got the GOAWAY: it is that + /// predecessor and keeps serving, so only the status moves. With one, which the policy + /// would have retired first if it were older, the replacement serving on top of it closed: + /// the handle serves on the predecessor again rather than on a closed session. + fn migrating(&self, session: &moq_net::Session, deadline: kio::clock::Instant) { + let Ok(mut state) = self.state.write() else { + return; + }; + state.revision += 1; + // Only a live session migrates: a GOAWAY or a probe needs one, and so does a + // replacement closing on top of a predecessor. + let Lifecycle::Live(live) = &mut state.lifecycle else { + return; + }; + live.migrating = Some(Migrating::default()); + match &live.predecessor { + Some(predecessor) => { + live.session = predecessor.session.clone(); + live.transport = predecessor.transport; + // The closed replacement's estimates are gone, and the predecessor's no + // longer forward, so freeze on its latest like any handover does. + let _ = self.send_bw.set(session.send_bandwidth().and_then(|bw| bw.peek())); + let _ = self.recv_bw.set(session.recv_bandwidth().and_then(|bw| bw.peek())); + } + None => { + live.predecessor = Some(Box::new(Draining { + session: session.clone(), + deadline, + transport: live.transport, + })); } } } - /// Mirror the policy's draining predecessor, for [`Connection::close`]. - fn predecessor(&self, predecessor: Option<(&moq_net::Session, kio::clock::Instant)>) { + /// The policy's predecessor retired, so [`Connection::close`] no longer drains it. + fn retired(&self) { // A write wakes every watcher, so skip one that changes nothing. - if predecessor.is_none() && self.state.read().lifecycle.predecessor().is_none() { + if self.state.read().lifecycle.predecessor().is_none() { return; } - let predecessor = predecessor.map(|(session, deadline)| Box::new((session.clone(), deadline))); if let Ok(mut state) = self.state.write() { match &mut state.lifecycle { - Lifecycle::Live(live) => live.predecessor = predecessor, - Lifecycle::Offline { predecessor: slot, .. } => *slot = predecessor, - // Only a session that connected can drain, and nothing drains once the loop ends. + Lifecycle::Live(live) => live.predecessor = None, + Lifecycle::Offline { predecessor, .. } => *predecessor = None, Lifecycle::Connecting { .. } | Lifecycle::Closed(_) => {} } } @@ -1029,7 +1060,8 @@ impl Connection { *closed = Some(moq_net::Error::Cancel); let state = self.state.read(); let session = state.lifecycle.live().map(|live| live.session.clone()); - (session, state.lifecycle.predecessor().cloned()) + let predecessor = state.lifecycle.predecessor(); + (session, predecessor.map(|predecessor| (predecessor.session.clone(), predecessor.deadline))) }; self.task.handle.abort(); let session = async move { @@ -1039,7 +1071,7 @@ impl Connection { } }; let predecessor = async move { - let Some((session, deadline)) = predecessor.map(|predecessor| *predecessor) else { + let Some((session, deadline)) = predecessor else { return Ok(()); }; let abort = session.clone(); @@ -1188,12 +1220,13 @@ impl Connection { #[cfg(all(feature = "websocket", feature = "quic"))] left_fallback(&mut probe, &mut fallback); forward = None; - // After the predecessor is published, so a close woken by this status - // finds it and honors its handover deadline. - shared.predecessor(policy.predecessor()); - shared.migrating(); + // One write publishes the predecessor with this status, so a close woken by + // it finds the predecessor and honors its handover deadline. + if let Some((session, deadline)) = policy.predecessor() { + shared.migrating(session, deadline); + } } - Action::Retired => shared.predecessor(None), + Action::Retired => shared.retired(), Action::Offline(failure) => { #[cfg(all(feature = "websocket", feature = "quic"))] left_fallback(&mut probe, &mut fallback); @@ -1225,7 +1258,7 @@ impl Connection { Action::Done(res) => { #[cfg(all(feature = "websocket", feature = "quic"))] left_fallback(&mut probe, &mut fallback); - shared.predecessor(None); + shared.retired(); return res; } Action::Wait(_) => unreachable!("handled while polling"), @@ -2726,6 +2759,90 @@ mod tests { ); } + /// A replacement that drops while the drained session still serves leaves the handle + /// on that session, reporting Migrating rather than a Disconnected that would claim + /// nothing serves. Its subscription keeps delivering through the handover. + #[cfg(feature = "tcp")] + #[tokio::test] + async fn a_dropped_replacement_hands_back_to_the_draining_session() { + const WAIT: Duration = Duration::from_secs(10); + + let origin = crate::origin::spawn(); + let broadcast = origin.route("cam", moq_net::origin::RouteOptions::default()).unwrap(); + let track = broadcast.broadcast().create_track("video", timed()).unwrap(); + let (addr_a, mut accepted_a, _task_a) = serve(origin.clone()).await; + let (addr_b, mut accepted_b, _task_b) = serve(origin.clone()).await; + + // Unique to this test: the table is process-wide. + const HOST: &str = "dropped-replacement.test"; + crate::resolve::hosts::point(HOST, [addr_a]); + + let subscriber = crate::origin::spawn(); + let mut config = crate::connect::Config::default(); + // Long enough that only the replacement's drop can change what serves. + config.goaway.handover = Some((WAIT * 3).into()); + config.backoff.initial = Some((FAST).into()); + config.backoff.healthy = Some((FAST).into()); + let client = config + .init(Default::default()) + .unwrap() + .with_subscriber(subscriber.clone()); + let connection = client.connect(url(&format!("tcp://{HOST}:1/"))); + let monitor = connection.monitor(); + let session_a = kio::clock::timeout(WAIT, accepted_a.recv()).await.unwrap().unwrap(); + + let cam = kio::clock::timeout(WAIT, subscriber.consume().follow("cam", None).unwrap().available()) + .await + .unwrap() + .unwrap() + .unwrap(); + let mut sub = cam.track("video").unwrap().subscribe(None).await.unwrap(); + let mut group = track.append_group().unwrap(); + group.write_frame(moq_net::Timestamp::ZERO, b"g0".as_ref()).unwrap(); + group.finish().unwrap(); + let g0 = kio::clock::timeout(WAIT, sub.recv_group()).await.unwrap().unwrap().unwrap(); + assert_eq!(g0.sequence, 0); + + crate::resolve::hosts::point(HOST, [addr_b]); + session_a.drain().send(moq_net::goaway::Goaway::new()).unwrap(); + let session_b = kio::clock::timeout(WAIT, accepted_b.recv()).await.unwrap().unwrap(); + let mut state = monitor.state(); + while !matches!(state.phase, Phase::Connected { .. }) || state.connects < 2 { + state = kio::clock::timeout(WAIT, monitor.state_changed(state.revision)) + .await + .unwrap() + .unwrap(); + } + + session_b.abort(moq_net::Error::Cancel); + let state = kio::clock::timeout(WAIT, monitor.state_changed(state.revision)) + .await + .unwrap() + .unwrap(); + assert!( + matches!(state.phase, Phase::Migrating { transport: crate::Transport::Tcp, .. }), + "{:?}", + state.phase + ); + assert!( + matches!(connection.poll_established(&kio::Waiter::noop()), Poll::Ready(Ok(()))), + "the handle dropped the serving session" + ); + assert!( + connection + .draining() + .is_some_and(|goaway| matches!(goaway.poll(&kio::Waiter::noop()), Poll::Ready(Ok(_)))), + "the handle serves on a session other than the drained one" + ); + assert!(monitor.stats().is_some()); + + let mut group = track.append_group().unwrap(); + group.write_frame(moq_net::Timestamp::ZERO, b"g1".as_ref()).unwrap(); + group.finish().unwrap(); + let g1 = kio::clock::timeout(WAIT, sub.recv_group()).await.unwrap().unwrap().unwrap(); + assert_eq!(g1.sequence, 1, "the drained session stopped delivering"); + } + /// Under [`Restart::Wait`] an empty-URI GOAWAY changes nothing while the peer keeps /// the session: no migration and no redial. Its close then reconnects like any drop. #[cfg(feature = "tcp")] From 097be547fc86bcf416625bb723907c518e0e1fa9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 12 Nov 2026 17:13:43 -0800 Subject: [PATCH 03/12] fix(reconnect): a dropped replacement hands back to the draining predecessor The policy reports Migrating, not Offline, when the replacement closes while an open predecessor still drains, and moq-tokio's Connection serves on that predecessor again. js/net's Reload does the same. Session gains identity (PartialEq) so Shared can match the policy's predecessor. The Handover model now checks OfflineMeansNothingServes in every config. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-reconnect/SKILL.md | 11 ++--- js/net/src/connection/migrate.test.ts | 44 ++++++++++++++++++- js/net/src/connection/reload.ts | 8 +++- quest/m3/reconnect-predecessor-status.md | 32 -------------- rs/moq-net/src/reconnect.rs | 18 +++----- rs/moq-net/src/session.rs | 9 ++++ rs/moq-net/verify/reconnect/Handover.cfg | 4 +- rs/moq-net/verify/reconnect/Handover.tla | 29 ++++++++---- .../verify/reconnect/OfflineWhileServing.cfg | 15 ------- rs/moq-net/verify/reconnect/RestartWait.cfg | 1 + rs/moq-tokio/src/connection.rs | 9 ++-- 11 files changed, 95 insertions(+), 85 deletions(-) delete mode 100644 quest/m3/reconnect-predecessor-status.md delete mode 100644 rs/moq-net/verify/reconnect/OfflineWhileServing.cfg diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index 4637c7c5772..b272c72c021 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -15,7 +15,7 @@ Each rule is an INVARIANT; the harm says why it matters. All but `NoOrphan`, `Re | --- | --- | --- | | `NoStaleSession` | The session a `Connection` reports as live is never one the policy aborted or started closing. | An app opens subscriptions on a session that is going away. | | `StatusTruthful` | Connected is on the policy's live session, Migrating on its predecessor while nothing else is live, Closed exactly when the policy is done, and the predecessor `Connection::close` drains is the policy's. | Status or `close` acts on the wrong session, or a closed loop looks alive. | -| `OfflineMeansNothingServes` | Disconnected is reported only when no predecessor still serves (`Action::Offline`: "nothing serves anymore"). | An app waiting on `established` stalls, and the UI says disconnected, while a session still carries its media. | +| `OfflineMeansNothingServes` | Disconnected is reported only when no predecessor still serves (`Action::Offline`: "nothing serves anymore"); one closing gracefully carries nothing and takes no new work. | An app waiting on `established` stalls, and the UI says disconnected, while a session still carries its media. | | `NoOrphan` | Until the policy is done, every open session is the live one or the predecessor. | A transport and the peer's state stay open with nothing able to reach them. | | `PresenceBalanced` | `sessions_started - sessions_ended` is 1 while connected and 0 otherwise. | Presence counts drift on every migration. | | `FirstWalkFailsFast` | Nothing backs off before a session has been established: a failed first walk ends the policy (a refusal of since-replaced tokens is the exception, and tokens are not modeled). | A setup mistake (an untrusted certificate, no version in common) retries forever instead of surfacing. | @@ -27,10 +27,9 @@ Each rule is an INVARIANT; the harm says why it matters. All but `NoOrphan`, `Re quest gate just rs tla moq-net/verify/reconnect/Handover.tla quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/verify/reconnect/RestartWait.cfg quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/verify/reconnect/RedirectLost.cfg -quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/verify/reconnect/OfflineWhileServing.cfg ``` -The first checks every rule but the two with open findings, over four sessions: about 1.2M distinct states, under a minute. The second checks the same rules under `Restart::Wait` over three sessions: about 240k states. Both pass at the sync commit. The last two fail at the sync commit, each on its finding below, within seconds. `just test tla rs/moq-net/verify/reconnect` runs every config here, as Nightly does, and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. +The first checks every rule but the one with an open finding, over four sessions: about 1.2M distinct states, under a minute. The second checks the same rules under `Restart::Wait` over three sessions: about 240k states. Both pass at the sync commit. The last fails at the sync commit, on its finding below, within seconds. `just test tla rs/moq-net/verify/reconnect` runs every config here, as Nightly does, and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. ## Open findings @@ -38,10 +37,6 @@ The first checks every rule but the two with open findings, over four sessions: `RedirectLost.cfg` fails. On a fallback, `probed` moves the live session into the predecessor and drops its `goaway::Consumer` with the `Live` it came from, so nothing reads that session's GOAWAY anymore. A redirect its peer sends while the handshake over the preferred transport runs is never followed: when that handshake fails, the next walk dials the address the peer is leaving. `residual_a_redirect_after_a_probe_got_through_is_lost` in `reconnect/tests.rs` replays it. Once fixed, flip that test, move `RedirectFollowed` into `Handover.cfg`, and delete `RedirectLost.cfg`. -### A replacement dropping mid-handover reports Disconnected - -`OfflineWhileServing.cfg` fails. When the replacement closes while the predecessor still drains, `closed` queues `Action::Offline`, whose doc promises nothing serves, and `Shared::offline` reports Disconnected and stops handing out a session, though the predecessor still carries its subscriptions until it retires. `residual_a_dropped_replacement_reports_offline_while_the_predecessor_serves` replays it on the policy. Once fixed, flip that test, move `OfflineMeansNothingServes` into `Handover.cfg`, and delete `OfflineWhileServing.cfg`. - ## Reading a failure Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on the policy with the helpers in `reconnect/tests.rs` (`next`, `handshake`, `on_fallback`, `probe`, `connected`, `failed`, `probed`, a server's `drain().send(..)`), one call per step, on the simulated clock. A path through `Shared` needs `moq-tokio`'s connection tests instead. @@ -58,7 +53,7 @@ Sync commit: `103052ed074a87f00269ecb0d639905482ecfb6f`. Run `git log 103052ed07 | `rs/moq-net/src/reconnect.rs` `end`, `closed`, `done`, `Predecessor` | `EndGoaway`, `EndClosed`, `Done`, `Retire`, `IdleClose` | | `rs/moq-net/src/reconnect.rs` `Restart::Wait` | `LiveEnds` with `Restart = "wait"` (`RestartWait.cfg`) | | `rs/moq-net/src/reconnect/redirect.rs` `Redirect::target` | `Refused` and `Endpoints` | -| `rs/moq-tokio/src/connection.rs` `run`, `Shared::connected`, `migrating`, `predecessor`, `offline`, `end` | `Connected`, `Apply`, `OnMigrating`, `OnRetired`, `OnOffline`, `OnDone` | +| `rs/moq-tokio/src/connection.rs` `run`, `Shared::connected`, `migrating`, `retired`, `offline`, `end` | `Connected`, `Apply`, `OnMigrating`, `OnRetired`, `OnOffline`, `OnDone` | ## Not covered diff --git a/js/net/src/connection/migrate.test.ts b/js/net/src/connection/migrate.test.ts index 539d5c6e209..f8a7b86ec19 100644 --- a/js/net/src/connection/migrate.test.ts +++ b/js/net/src/connection/migrate.test.ts @@ -127,6 +127,44 @@ test("an empty GOAWAY migrates the follower to the replacement session", async ( } }); +test("a replacement that closes mid-handover hands back to the draining session", async () => { + const clock = new Manual(); + const installed = clock.install(); + const { dials } = fleet(); + const reload = new Reload({ + url, + websocket: { enabled: false }, + goaway: { handover }, + delay: { initial: Duration.millis(1) }, + }); + + try { + await waitUntil(() => reload.established.peek() !== undefined); + const first = reload.established.peek(); + const drained = dials[0]; + if (!drained) throw new Error("no first dial"); + await goaway(drained.server, ""); + await elapse(clock, () => reload.established.peek() !== first && reload.established.peek() !== undefined); + const replacement = dials[1]; + if (!replacement) throw new Error("no replacement dial"); + + replacement.server.close(); + await elapse(clock, () => replacement.closed); + await elapse(clock, () => reload.established.peek() === first || reload.established.peek() === undefined); + // Nothing is down: the drained session still serves until its handover ends. + expect(reload.established.peek()).toBe(first); + expect(reload.status.peek()).toBe("connected"); + expect(drained.closed).toBe(false); + + // Its handover still bounds it; then a redial serves. + await elapse(clock, () => drained.closed); + expect(reload.established.peek()).not.toBe(first); + } finally { + reload.close(); + installed.close(); + } +}); + test("a GOAWAY without a timeout hands over at the configured cap", async () => { const clock = new Manual(); const installed = clock.install(); @@ -265,16 +303,18 @@ test("residual: a redirected connection redials only its dead target", async () try { await waitUntil(() => reload.status.peek() === "connected"); + const first = reload.established.peek(); await goaway(dials[0]?.server as WebTransport, moved.href); await elapse(clock, () => dials.length === 2 && reload.status.peek() === "connected"); expect(dials[1]?.url).toBe(moved.href); - // The relay it was sent to dies and stays down. + // The relay it was sent to dies and stays down. The drained session serves until its + // handover ends, and the redials miss it. dead = true; const before = attempts.length; dials[1]?.server.close(); await elapse(clock, () => attempts.length >= before + 4); - expect(reload.status.peek()).not.toBe("connected"); + expect(reload.established.peek()).toBe(first); expect(attempts.slice(before).every((href) => href === moved.href)).toBe(true); } finally { reload.close(); diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index 723386d9efd..977a97357ea 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -663,7 +663,8 @@ export class Reload { console.warn(`connection closed, reconnecting: ${ended ? reason(ended) : "no error"}`); this.#redirects.reset(); current = undefined; - if (this.established.peek() === connection) this.#serve(undefined); + // A predecessor still draining serves again, so this is no outage. + if (this.established.peek() === connection) this.#serve(this.#draining?.serving()); this.#retry(effect, connected, ended ?? undefined); return; } @@ -1084,6 +1085,11 @@ class Draining { }); } + /** The old session while it still serves, before it retires. */ + serving(): Session | undefined { + return this.#retired ? undefined : this.#connection; + } + /** Stop serving from the old session, returning it the first time for the caller to end. */ retire(): Session | undefined { if (this.#retired) return undefined; diff --git a/quest/m3/reconnect-predecessor-status.md b/quest/m3/reconnect-predecessor-status.md deleted file mode 100644 index 37766167bf4..00000000000 --- a/quest/m3/reconnect-predecessor-status.md +++ /dev/null @@ -1,32 +0,0 @@ -# [M] A connection reports Migrating while its predecessor still serves - -## Goal - -When a GOAWAY's replacement session connects and then drops while the old -session still drains, the handle reports Migrating and exposes the old -session, not Disconnected with no session. Disconnected means nothing serves. - -## Plan - -Found 2026-11-10 by the reconnect model ([moq-dev/moq#8447](https://github.com/moq-dev/moq/pull/8447)): -`rs/moq-net/verify/reconnect/OfflineWhileServing.cfg` breaks -`OfflineMeansNothingServes`. `closed()` in `rs/moq-net/src/reconnect.rs` -queues `Action::Offline`, whose doc says nothing serves anymore, while the -predecessor still carries live subscriptions until its handover ends; the -handle reports Disconnected, and `established()` and `session()` return -nothing, so code waiting on them stalls. -`residual_a_dropped_replacement_reports_offline_while_the_predecessor_serves` -in `reconnect/tests.rs` replays it. - -Decided 2026-11-10 (maintainer, /quest-plan): ✅ fix the behaviour: report -Migrating, with `Shared`'s live session swapped to the predecessor until it -retires (rejected: rewording `Action::Offline` and `Lifecycle` to allow it). -Mirror the status in js/net's `Reload` if it shares the shape. Flip the -residual test, move the rule into `Handover.cfg`, delete -`OfflineWhileServing.cfg`, and update the `verify-reconnect` skill. - -Caused by: [moq-dev/moq#7904](https://github.com/moq-dev/moq/pull/7904), which -made the connect policy a poll machine with this action queue. - -Public API: none, unless the fix changes what `Lifecycle` exposes. -Wire: none. diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs index 883a24bf524..9a5d3b61e90 100644 --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs @@ -348,7 +348,7 @@ pub enum Action { /// live session got a GOAWAY and now drains as it, or the replacement closed while it /// still drains. Migrating, - /// The predecessor closed, overstayed its handover, or gave way to a newer one. + /// The predecessor closed or overstayed its handover. Retired, /// Nothing serves anymore: the live session ended with this error, or the predecessor /// retired between sessions. A redial follows. @@ -741,7 +741,9 @@ impl + From> Reco now.saturating_duration_since(old.since), &Pacing::new(&self.config.backoff), ); - self.retire(); + if let Some(older) = self.predecessor.take() { + older.retire(); + } self.predecessor = Some(Predecessor { deadline: now.saturating_add(self.config.goaway.handover), session: old.session, @@ -974,7 +976,9 @@ impl + From> Reco // serving until it closes or overstays, so its routes stay attached and live tracks // splice onto the replacement at a group boundary. tracing::info!(peer = %Shown(&url), "upstream GOAWAY; migrating"); - self.retire(); + if let Some(old) = self.predecessor.take() { + old.retire(); + } self.predecessor = Some(Predecessor { deadline: now.saturating_add(self.config.goaway.handover(msg.timeout())), session: live.session, @@ -1081,14 +1085,6 @@ impl + From> Reco .into() } - /// Close the predecessor ahead of its handover, for a newer one to replace it. - fn retire(&mut self) { - if let Some(older) = self.predecessor.take() { - older.retire(); - self.queue.push_back(Action::Retired); - } - } - fn done(&mut self, result: Result<(), E>) { self.predecessor = None; self.phase = Phase::Done; diff --git a/rs/moq-net/src/session.rs b/rs/moq-net/src/session.rs index a856f4b6394..6f5b2c384c9 100644 --- a/rs/moq-net/src/session.rs +++ b/rs/moq-net/src/session.rs @@ -214,6 +214,15 @@ pub struct Session { datagrams_unrouted: Option>, } +/// Two handles are equal when they are clones of the same session. +impl PartialEq for Session { + fn eq(&self, other: &Self) -> bool { + self.close.same_channel(&other.close) + } +} + +impl Eq for Session {} + impl Session { /// Returns the negotiated protocol version. pub fn version(&self) -> Version { diff --git a/rs/moq-net/verify/reconnect/Handover.cfg b/rs/moq-net/verify/reconnect/Handover.cfg index 7b1d6902c65..7a0638a8a0d 100644 --- a/rs/moq-net/verify/reconnect/Handover.cfg +++ b/rs/moq-net/verify/reconnect/Handover.cfg @@ -1,6 +1,5 @@ \* Two endpoints a GOAWAY may move between, one it may not, and four sessions. -\* RedirectFollowed and OfflineMeansNothingServes live in RedirectLost.cfg and -\* OfflineWhileServing.cfg while their findings are open. +\* RedirectFollowed lives in RedirectLost.cfg while its finding is open. CONSTANTS Endpoints = {"a", "b"} Start = "a" @@ -14,6 +13,7 @@ VIEW View INVARIANTS NoStaleSession StatusTruthful + OfflineMeansNothingServes NoOrphan PresenceBalanced FirstWalkFailsFast diff --git a/rs/moq-net/verify/reconnect/Handover.tla b/rs/moq-net/verify/reconnect/Handover.tla index 7a299d55e14..7ec8cca2d3a 100644 --- a/rs/moq-net/verify/reconnect/Handover.tla +++ b/rs/moq-net/verify/reconnect/Handover.tla @@ -111,13 +111,21 @@ Abort(sx, s) == IF s # None /\ sx[s].open \* `Shared::predecessor`. Mirror(p) == spred' = IF lc \in {"live", "offline"} THEN p ELSE spred -\* `Action::Migrating`: publish the predecessor, then `Shared::migrating`. -\* Leaving the fallback drops the probe in flight. +\* `Action::Migrating`: `Shared::migrating` with the policy's predecessor. The +\* mirrored one means the replacement on top of it closed, and the handle +\* serves on it again; any other is the live session, which got the GOAWAY, +\* and is mirrored as it. Leaving the fallback drops the probe in flight. OnMigrating(p) == - /\ Mirror(p) - /\ migrating' = IF lc = "live" THEN TRUE ELSE migrating + /\ IF lc = "live" /\ p # None + THEN /\ migrating' = TRUE + /\ IF spred = p + THEN /\ lcSess' = p + /\ UNCHANGED spred + ELSE /\ spred' = p + /\ UNCHANGED lcSess + ELSE UNCHANGED <> /\ probing' = FALSE - /\ UNCHANGED <> + /\ UNCHANGED <> OnRetired == /\ Mirror(None) @@ -277,13 +285,15 @@ EndGoaway(s, uri) == /\ Emit(<<"Migrating">>, s) /\ UNCHANGED watch -\* `Reconnect::closed` for a close a retry may repair. +\* `Reconnect::closed` for a close a retry may repair. A predecessor that is +\* not closing gracefully serves again, so the policy migrates back onto it; +\* the `Retrying` that follows changes no status, so it is left out. EndClosed == /\ live' = None /\ phase' = "backoff" /\ timer' = [timer EXCEPT !["backoff"] = FALSE] /\ probe' = "none" - /\ Emit(<<"Offline">>, pred) + /\ Emit(IF pred # None /\ ~sess[pred].closing THEN <<"Migrating">> ELSE <<"Offline">>, pred) /\ UNCHANGED <> \* `Phase::Live`: a GOAWAY is checked before the close. @@ -531,9 +541,10 @@ StatusTruthful == /\ (lc = "closed") = (phase = "done") /\ lc # "closed" => spred = pred -\* `Action::Offline`: nothing serves anymore. +\* `Action::Offline`: nothing serves anymore. A predecessor closing gracefully +\* already carries nothing and takes no new work. OfflineMeansNothingServes == - Parked /\ lc = "offline" => ~Open(pred) + Parked /\ lc = "offline" => ~Open(pred) \/ sess[pred].closing \* Every session the policy let go of is closed: one left open holds a \* transport and the peer's state with nothing able to reach it. diff --git a/rs/moq-net/verify/reconnect/OfflineWhileServing.cfg b/rs/moq-net/verify/reconnect/OfflineWhileServing.cfg deleted file mode 100644 index a36b9efa1df..00000000000 --- a/rs/moq-net/verify/reconnect/OfflineWhileServing.cfg +++ /dev/null @@ -1,15 +0,0 @@ -\* Open finding: a replacement dropping mid-handover reports Disconnected, so -\* OfflineMeansNothingServes fails. Once fixed, move the rule into -\* Handover.cfg and delete this file. -CONSTANTS - Endpoints = {"a", "b"} - Start = "a" - Refused = "far" - Restart = "redial" - MaxSessions = 3 - -SPECIFICATION Spec -VIEW View - -INVARIANTS - OfflineMeansNothingServes diff --git a/rs/moq-net/verify/reconnect/RestartWait.cfg b/rs/moq-net/verify/reconnect/RestartWait.cfg index 4b02f1049bd..ffe2e47a78d 100644 --- a/rs/moq-net/verify/reconnect/RestartWait.cfg +++ b/rs/moq-net/verify/reconnect/RestartWait.cfg @@ -13,6 +13,7 @@ VIEW View INVARIANTS NoStaleSession StatusTruthful + OfflineMeansNothingServes NoOrphan PresenceBalanced FirstWalkFailsFast diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index 4567341f5b7..7516e98841b 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -554,10 +554,9 @@ impl Shared { /// The policy's predecessor, `session` draining until `deadline`, serves while a /// replacement comes up. /// - /// With no predecessor mirrored yet, the live session got the GOAWAY: it is that - /// predecessor and keeps serving, so only the status moves. With one, which the policy - /// would have retired first if it were older, the replacement serving on top of it closed: - /// the handle serves on the predecessor again rather than on a closed session. + /// Either the live session got the GOAWAY and is that predecessor, so only the status + /// moves, or the replacement serving on top of the mirrored predecessor closed, and the + /// handle serves on the predecessor again rather than on a closed session. fn migrating(&self, session: &moq_net::Session, deadline: kio::clock::Instant) { let Ok(mut state) = self.state.write() else { return; @@ -569,7 +568,7 @@ impl Shared { return; }; live.migrating = Some(Migrating::default()); - match &live.predecessor { + match live.predecessor.as_ref().filter(|predecessor| predecessor.session == *session) { Some(predecessor) => { live.session = predecessor.session.clone(); live.transport = predecessor.transport; From 0de136feda7f53ec388d525e63bd3b89b6237745 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 12 Nov 2026 17:13:48 -0800 Subject: [PATCH 04/12] docs(verify-reconnect): move the sync commit to the predecessor fix Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-reconnect/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index b272c72c021..af95c974315 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -43,7 +43,7 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `103052ed074a87f00269ecb0d639905482ecfb6f`. Run `git log 103052ed07.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. +Sync commit: `097be547fc86bcf416625bb723907c518e0e1fa9`. Run `git log 097be547fc.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. | Code | Model | | --- | --- | From b4b7588fd5bf6f8c27f3005a4779420ea5dc60c5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 12 Nov 2026 17:37:21 -0800 Subject: [PATCH 05/12] style: rustfmt Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/reconnect.rs | 6 +++++- rs/moq-net/src/reconnect/tests.rs | 5 ++++- rs/moq-tokio/src/connection.rs | 31 ++++++++++++++++++++++++++----- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs index 9a5d3b61e90..4bb2ac17483 100644 --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs @@ -1032,7 +1032,11 @@ impl + From> Reco // A predecessor still draining serves again, so this is no outage: the caller moves // back onto it. One closing gracefully already carries nothing and takes no new work. - if self.predecessor.as_ref().is_some_and(|predecessor| !predecessor.closing) { + if self + .predecessor + .as_ref() + .is_some_and(|predecessor| !predecessor.closing) + { self.queue.push_back(Action::Migrating); self.queue.push_back(Action::Retrying(err.clone())); } else { diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index 2962cddeb61..f83eac69b67 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -1040,7 +1040,10 @@ async fn a_dropped_replacement_goes_offline_over_a_closing_predecessor() { let mut machine: Reconnect = Reconnect::new(Config::default(), vec![a.clone()], kio::clock::now()).unwrap(); let mut run = Run::default(); - drive(&mut machine, &[(a, Peer::Serve)], &mut run, |run| !run.accepts.is_empty()).await; + drive(&mut machine, &[(a, Peer::Serve)], &mut run, |run| { + !run.accepts.is_empty() + }) + .await; kio::clock::sleep(Duration::from_secs(2)).await; run.servers[0].drain().send(goaway::Goaway::new()).unwrap(); diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index 7516e98841b..cb6e11d23e6 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -568,7 +568,11 @@ impl Shared { return; }; live.migrating = Some(Migrating::default()); - match live.predecessor.as_ref().filter(|predecessor| predecessor.session == *session) { + match live + .predecessor + .as_ref() + .filter(|predecessor| predecessor.session == *session) + { Some(predecessor) => { live.session = predecessor.session.clone(); live.transport = predecessor.transport; @@ -1060,7 +1064,10 @@ impl Connection { let state = self.state.read(); let session = state.lifecycle.live().map(|live| live.session.clone()); let predecessor = state.lifecycle.predecessor(); - (session, predecessor.map(|predecessor| (predecessor.session.clone(), predecessor.deadline))) + ( + session, + predecessor.map(|predecessor| (predecessor.session.clone(), predecessor.deadline)), + ) }; self.task.handle.abort(); let session = async move { @@ -2799,7 +2806,11 @@ mod tests { let mut group = track.append_group().unwrap(); group.write_frame(moq_net::Timestamp::ZERO, b"g0".as_ref()).unwrap(); group.finish().unwrap(); - let g0 = kio::clock::timeout(WAIT, sub.recv_group()).await.unwrap().unwrap().unwrap(); + let g0 = kio::clock::timeout(WAIT, sub.recv_group()) + .await + .unwrap() + .unwrap() + .unwrap(); assert_eq!(g0.sequence, 0); crate::resolve::hosts::point(HOST, [addr_b]); @@ -2819,7 +2830,13 @@ mod tests { .unwrap() .unwrap(); assert!( - matches!(state.phase, Phase::Migrating { transport: crate::Transport::Tcp, .. }), + matches!( + state.phase, + Phase::Migrating { + transport: crate::Transport::Tcp, + .. + } + ), "{:?}", state.phase ); @@ -2838,7 +2855,11 @@ mod tests { let mut group = track.append_group().unwrap(); group.write_frame(moq_net::Timestamp::ZERO, b"g1".as_ref()).unwrap(); group.finish().unwrap(); - let g1 = kio::clock::timeout(WAIT, sub.recv_group()).await.unwrap().unwrap().unwrap(); + let g1 = kio::clock::timeout(WAIT, sub.recv_group()) + .await + .unwrap() + .unwrap() + .unwrap(); assert_eq!(g1.sequence, 1, "the drained session stopped delivering"); } From b350d724c742b77b511af235ec99ef2a161471c1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 12 Nov 2026 17:37:49 -0800 Subject: [PATCH 06/12] quest: the io_uring memlock startup test holds on current kernels Co-Authored-By: Claude Opus 5.5 --- quest/m0/uring-memlock-test.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 quest/m0/uring-memlock-test.md diff --git a/quest/m0/uring-memlock-test.md b/quest/m0/uring-memlock-test.md new file mode 100644 index 00000000000..1a3f71884ac --- /dev/null +++ b/quest/m0/uring-memlock-test.md @@ -0,0 +1,25 @@ +# [S] The io_uring memlock startup test holds on current kernels + +## Goal + +`moq-relay`'s `runtime_uring::uring_startup_reports_the_memlock_limit` +passes on every host the gate runs on, or is rewritten or deleted so it +pins what the relay actually guarantees. + +## Plan + +Found 2026-11-12 by a `just check` on a Fedora host running Linux 7.3, +unrelated to the change under test: the test sets `RLIMIT_MEMLOCK` to 0 +and expects `ready.wait().await.expect_err("a worker needs locked memory")` +(`rs/moq-relay/tests/integration/runtime_uring.rs`), but the io_uring worker +starts anyway. One re-run with `--features io-uring` failed the same way, so +it is deterministic on that host, not a flake. A likely cause, unconfirmed: +newer kernels charge io_uring rings to the memory cgroup rather than +`RLIMIT_MEMLOCK`, so a ring may no longer need locked memory at all. Find which ring setup still charges +the limit, if any, and decide from that: keep the error path and make the +test provoke it in a way the kernel enforces, or drop the expectation if the +relay no longer needs locked memory. + +## Hosts + +- linux - io_uring From aba0eec596f4b70530c4ae60b0e18616e5165b4b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 12 Nov 2026 17:42:21 -0800 Subject: [PATCH 07/12] docs(verify-reconnect): anchor the sync commit to the PR base Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-reconnect/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index af95c974315..58119547813 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -43,7 +43,7 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `097be547fc86bcf416625bb723907c518e0e1fa9`. Run `git log 097be547fc.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. +Sync commit: `c681b7a095bf2e1ce41bc405acdaa7993c5fe217`. Run `git log c681b7a095.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. | Code | Model | | --- | --- | From 6ca927145961d5724f769d79c005767818bafe16 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 12 Nov 2026 22:59:11 -0800 Subject: [PATCH 08/12] quest: drop the uring memlock follow-up, kept by #8546 Co-Authored-By: Claude Opus 5.5 --- quest/m0/uring-memlock-test.md | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 quest/m0/uring-memlock-test.md diff --git a/quest/m0/uring-memlock-test.md b/quest/m0/uring-memlock-test.md deleted file mode 100644 index 1a3f71884ac..00000000000 --- a/quest/m0/uring-memlock-test.md +++ /dev/null @@ -1,25 +0,0 @@ -# [S] The io_uring memlock startup test holds on current kernels - -## Goal - -`moq-relay`'s `runtime_uring::uring_startup_reports_the_memlock_limit` -passes on every host the gate runs on, or is rewritten or deleted so it -pins what the relay actually guarantees. - -## Plan - -Found 2026-11-12 by a `just check` on a Fedora host running Linux 7.3, -unrelated to the change under test: the test sets `RLIMIT_MEMLOCK` to 0 -and expects `ready.wait().await.expect_err("a worker needs locked memory")` -(`rs/moq-relay/tests/integration/runtime_uring.rs`), but the io_uring worker -starts anyway. One re-run with `--features io-uring` failed the same way, so -it is deterministic on that host, not a flake. A likely cause, unconfirmed: -newer kernels charge io_uring rings to the memory cgroup rather than -`RLIMIT_MEMLOCK`, so a ring may no longer need locked memory at all. Find which ring setup still charges -the limit, if any, and decide from that: keep the error path and make the -test provoke it in a way the kernel enforces, or drop the expectation if the -relay no longer needs locked memory. - -## Hosts - -- linux - io_uring From be5db2ebc4c6f1963a7786c6e66b01793d18d77e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 10:56:33 -0800 Subject: [PATCH 09/12] docs(verify-reconnect): state counts after the merge with main Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-reconnect/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index 29eef319093..49f77829672 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -29,7 +29,7 @@ quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/ quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/verify/reconnect/RedirectLost.cfg ``` -The first checks every rule but the one with an open finding, over four sessions: about 1.2M distinct states, under a minute. The second checks the same rules under `Restart::Wait` over three sessions: about 240k states. Both pass at the sync commit. The last fails at the sync commit, on its finding below, within seconds. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. +The first checks every rule but the one with an open finding, over four sessions: about 1.7M distinct states, under a minute. The second checks the same rules under `Restart::Wait` over three sessions: about 300k states. Both pass at the sync commit. The last fails at the sync commit, on its finding below, within seconds. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. ## Open findings From 28cc1b0977bad27e18b9c22f1a018b489590eae5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 11:01:42 -0800 Subject: [PATCH 10/12] docs(verify-reconnect): move the sync commit to the merged base Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-reconnect/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index 49f77829672..3ffd09ddc4f 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -43,7 +43,7 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `c681b7a095bf2e1ce41bc405acdaa7993c5fe217`. Run `git log c681b7a095.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. +Sync commit: `4486b7adc108649bb3e7924554394ee30fa0e5f2`. Run `git log 4486b7adc1.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. | Code | Model | | --- | --- | From 04f3977d967976577b133da230670f8662e9f275 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 11:13:34 -0800 Subject: [PATCH 11/12] test(reconnect): reach a closing predecessor deterministically; no redial races the hand-back checks Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/reconnect/tests.rs | 22 +++++++++++----------- rs/moq-tokio/src/connection.rs | 6 ++++++ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index 4b9ac240d9a..a94e3550c4e 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -1063,27 +1063,27 @@ async fn a_dropped_replacement_goes_offline_over_a_closing_predecessor() { assert!(matches!(next(&mut machine).await, Action::Migrating)); assert!(matches!(next(&mut machine).await, Action::Dial(_))); let (replacement, replacement_server) = handshake().await; - machine.connected(kio::clock::now(), replacement); - // The idle predecessor starts its graceful close under the replacement. - assert!(matches!( - machine.poll(kio::clock::now(), &kio::Waiter::noop()), - Action::Wait(_) - )); + machine.connected(kio::clock::now(), replacement.clone()); + // The replacement is already closed when the next poll starts the idle predecessor's + // graceful close, so its close is seen over a predecessor that is closing, not retired. drop(replacement_server); + replacement.closed().await; - // Whether the graceful close finishes first only orders `Retired` against the offline. + let action = next(&mut machine).await; + assert!(matches!(action, Action::Offline(Some(_))), "{action:?}"); + let (_, deadline) = machine.predecessor().expect("the predecessor is still closing"); + // The graceful close then ends it well before its handover would. let mut actions = Vec::new(); loop { match next(&mut machine).await { Action::Dial(_) => break, Action::Retired => actions.push("retired"), - Action::Offline(Some(_)) => actions.push("offline"), - Action::Retrying(_) => actions.push("retrying"), + Action::Offline(None) => actions.push("offline"), action => panic!("unexpected {action:?}"), } } - actions.sort(); - assert_eq!(actions, ["offline", "retired"]); + assert_eq!(actions, ["retired", "offline"]); + assert!(kio::clock::now() < deadline); } /// A target that never reaches the preferred transport keeps its fallback session and diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index cbe1d823b84..3a211b43bcf 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -2784,6 +2784,7 @@ mod tests { #[cfg(feature = "tcp")] #[tokio::test] async fn a_dropped_replacement_hands_back_to_the_draining_session() { + kio::clock::host_time(); // real TCP sockets const WAIT: Duration = Duration::from_secs(10); let origin = crate::origin::spawn(); @@ -2837,6 +2838,11 @@ mod tests { .unwrap(); } + // The redial after B's drop fails, so no new session can replace the handed-back one + // before the checks read it; a failed redial still reports Migrating. + let dead = dead().await; + let dead = dead.socket_addrs(|| None).unwrap(); + crate::resolve::hosts::point(HOST, dead); session_b.abort(moq_net::Error::Cancel); let state = kio::clock::timeout(WAIT, monitor.state_changed(state.revision)) .await From e20d64a4f06ab54196c686ba06b1b9aa8b4e040b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 16:04:20 -0800 Subject: [PATCH 12/12] docs(verify-reconnect): restate RestartWait's state count after the merge --- .claude/skills/verify-reconnect/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index 340fd2d8e33..0e9ca3c98cf 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -29,7 +29,7 @@ quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/ quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/verify/reconnect/RedirectLost.cfg ``` -The first checks every rule but the one with an open finding, over four sessions: about 1.7M distinct states, under a minute. The second checks the same rules under `Restart::Wait` over three sessions: about 300k states. Both pass at the sync commit. The last fails at the sync commit, on its finding below, within seconds. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. +The first checks every rule but the one with an open finding, over four sessions: about 1.7M distinct states, under a minute. The second checks the same rules under `Restart::Wait` over three sessions: about 290k states. Both pass at the sync commit. The last fails at the sync commit, on its finding below, within seconds. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. ## Open findings