From 2b14753e809a8033b391d0f5823a58f1e86ed729 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 06:07:26 -0800 Subject: [PATCH 1/3] chore(quest): claim quest/m0/test-flakes-3/fleet-drain-redial.md Quest-Claim: 7dee9941d3773cc7 From 59d9769a0cab8d8c968ca23e4a112f988634e996 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 06:28:50 -0800 Subject: [PATCH 2/3] fix(moq-tokio): the fleet drain test holds its handover cap on a manual clock The test raced the drained session's 500 ms real-time handover cap against the redial, backoff, and delivery of g1. Over moq-lite-06 the subscription never moves to the replacement (each session's announcement mints its own instance), so g1 only arrived while the drained session still served; on a loaded host the cap closed it first and recv_group saw GoawayTimeout. Install a kio Manual clock: the sockets stay real, but the cap passes only when the test advances it, after g1 is read. The test now also asserts the drained session closes exactly at the cap and not before. Co-Authored-By: Claude Opus 5.5 --- nix/Cargo.nix | 5 +- quest/m0/test-flakes-3/fleet-drain-redial.md | 19 ----- rs/moq-tokio/Cargo.toml | 2 +- rs/moq-tokio/src/connection.rs | 80 ++++++++++---------- 4 files changed, 44 insertions(+), 62 deletions(-) delete mode 100644 quest/m0/test-flakes-3/fleet-drain-redial.md diff --git a/nix/Cargo.nix b/nix/Cargo.nix index 35df0c49d89..6a8437139d3 100644 --- a/nix/Cargo.nix +++ b/nix/Cargo.nix @@ -22133,7 +22133,10 @@ rec { { name = "kio"; packageId = "kio"; - features = [ "sim" ]; + features = [ + "manual" + "sim" + ]; } { name = "moq-congestion"; diff --git a/quest/m0/test-flakes-3/fleet-drain-redial.md b/quest/m0/test-flakes-3/fleet-drain-redial.md deleted file mode 100644 index 602498f52da..00000000000 --- a/quest/m0/test-flakes-3/fleet-drain-redial.md +++ /dev/null @@ -1,19 +0,0 @@ -# [S] A fleet drain's redial delivers the next group under load - -## Goal - -`moq-tokio` `connection::tests::a_fleet_drain_redials_through_a_fresh_resolve` -passes under a loaded `just check`. - -## Plan - -Seen once on 2026-11-18 during `just check` for -[moq-dev/moq#9140](https://github.com/moq-dev/moq/pull/9140), which does not -touch moq-tokio or GOAWAY: the subscriber's `recv_group` for `g1` failed with -`Session(GoawayTimeout)` (`rs/moq-tokio/src/connection.rs`, the `g1` wait). It -passed 5 of 5 alone and 40 of 40 under `--stress-count 40 --test-threads 16`, -taking 0.5 to 3 s alone, so it rides on wall-clock time. - -Find why the drained session's deadline can end the subscription before the -redial to B carries `g1`, and drive the timers with a manual clock or assert on -the handover event instead of real time, per the epic's rules. diff --git a/rs/moq-tokio/Cargo.toml b/rs/moq-tokio/Cargo.toml index 6eb71f6dbd8..57caa1b4d6b 100644 --- a/rs/moq-tokio/Cargo.toml +++ b/rs/moq-tokio/Cargo.toml @@ -102,7 +102,7 @@ criterion = { package = "bench-harness", path = "../bench-harness" } # Use the channel type behind `mdns_sd::Receiver` so the mDNS readiness gate can # be driven with synthetic daemon events. flume = "0.12" -kio = { path = "../kio", features = ["sim"] } +kio = { path = "../kio", features = ["manual", "sim"] } # The integration tests build congestion controllers directly, including the candidates the # real-socket comparisons run. moq-congestion = { path = "../moq-congestion", features = ["lab"] } diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index 269baca83b3..5993ab5c032 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -2805,13 +2805,19 @@ mod tests { /// The fleet drain: a relay withdrawn from DNS sends an empty-URI GOAWAY with a /// deadline, and the client lands on a healthy relay by resolving the configured /// name again. The drained relay still accepts, so a cached resolve would land - /// right back on it. The live track hands over at a group boundary, and the old - /// session closes at our handover cap, well before the peer's own deadline. + /// right back on it. The drained session keeps delivering the live track until it + /// closes at our handover cap, well before the peer's own deadline. + /// + /// The sockets are real but the clock is manual, so the cap passes only when the + /// test moves it: a slow host cannot close the drained session before it delivers. + /// Over moq-lite-06 the subscription stays on the drained session, since each + /// session's announcement serves an instance of its own. #[tokio::test] async fn a_fleet_drain_redials_through_a_fresh_resolve() { - const WAIT: Duration = Duration::from_secs(10); + use futures::FutureExt; const HANDOVER: Duration = Duration::from_millis(500); const DEADLINE: Duration = Duration::from_secs(30); + let clock = kio::clock::Manual::new().install(&tokio::runtime::Handle::current()); // Two relays of one fleet, serving the same live broadcast. let origin = crate::origin::spawn(); @@ -2827,8 +2833,6 @@ mod tests { let subscriber = crate::origin::spawn(); let mut config = crate::connect::Config::default(); config.goaway.handover = Some((HANDOVER).into()); - // A session younger than the healthy bar counts as redirected immediately and - // waits out a backoff, which this test is not about. config.backoff.initial = Some((FAST).into()); config.backoff.healthy = Some((FAST).into()); let client = config @@ -2838,32 +2842,30 @@ mod tests { let url: Url = format!("tcp://{HOST}:1/").parse().unwrap(); let _connection = client.connect(url); - let session_a = kio::clock::timeout(WAIT, accepted_a.recv()).await.unwrap().unwrap(); + let session_a = accepted_a.recv().await.unwrap(); - let consumer = subscriber.consume(); - let cam = kio::clock::timeout(WAIT, consumer.follow("cam", None).unwrap().available()) - .await + let cam = subscriber + .consume() + .follow("cam", None) .unwrap() + .available() + .await .unwrap() .unwrap(); let mut sub = cam.track("video").unwrap().subscribe(None).await.unwrap(); - let mut group = track - .append_group( - moq_net::Timestamp::from_millis(track.largest().map_or(0, |largest| largest.group + 1)).unwrap(), - ) - .unwrap(); + let mut group = track.append_group(moq_net::Timestamp::from_millis(0).unwrap()).unwrap(); group - .write_frame(moq_net::Timestamp::from_millis(group.start).unwrap(), b"g0".as_ref()) + .write_frame(moq_net::Timestamp::from_millis(0).unwrap(), b"g0".as_ref()) .unwrap(); group.finish().unwrap(); - let g0 = kio::clock::timeout(WAIT, sub.recv_group()) - .await - .unwrap() - .unwrap() - .unwrap(); + let g0 = sub.recv_group().await.unwrap().unwrap(); assert_eq!(g0.start, 0); + // A session younger than the healthy bar counts as redirected immediately and + // waits out a backoff, which this test is not about. + clock.advance(FAST); + // Withdraw A from DNS, then drain it. crate::resolve::hosts::point(HOST, [addr_b]); let drained = kio::clock::now(); @@ -2871,34 +2873,30 @@ mod tests { .goaway(moq_net::goaway::Goaway::new().with_timeout(Some(DEADLINE))) .unwrap(); - let _session_b = kio::clock::timeout(WAIT, accepted_b.recv()) + let _session_b = accepted_b + .recv() .await - .expect("never redialed through the fresh resolve") - .unwrap(); + .expect("never redialed through the fresh resolve"); - let mut group = track - .append_group( - moq_net::Timestamp::from_millis(track.largest().map_or(0, |largest| largest.group + 1)).unwrap(), - ) - .unwrap(); + let mut group = track.append_group(moq_net::Timestamp::from_millis(1).unwrap()).unwrap(); group - .write_frame(moq_net::Timestamp::from_millis(group.start).unwrap(), b"g1".as_ref()) + .write_frame(moq_net::Timestamp::from_millis(1).unwrap(), b"g1".as_ref()) .unwrap(); group.finish().unwrap(); - let mut g1 = kio::clock::timeout(WAIT, sub.recv_group()) - .await - .unwrap() - .unwrap() - .unwrap(); - assert_eq!(g1.start, 1, "delivery resumes at the next group after the swap"); + let mut g1 = sub.recv_group().await.unwrap().unwrap(); + assert_eq!(g1.start, 1, "delivery continues at the next group during the handover"); assert_eq!(g1.read_frame().await.unwrap().unwrap().payload[..], b"g1"[..]); - - kio::clock::timeout(WAIT, session_a.closed()) - .await - .expect("the drained session never closed"); assert!( - drained.elapsed() < DEADLINE, - "the old session outlived the handover cap and waited for the peer's deadline" + session_a.closed().now_or_never().is_none(), + "the drained session closed before its handover cap while still serving" + ); + + clock.advance(HANDOVER); + session_a.closed().await; + assert_eq!( + drained.elapsed(), + HANDOVER, + "the old session closed at the handover cap, not the peer's deadline" ); assert!( accepted_a.try_recv().is_err(), From 9ebd86b9913489bd2117c8c57b5c07044ec740ff Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 06:29:17 -0800 Subject: [PATCH 3/3] chore(quest): follow-up for lite-06 subscriptions across a GOAWAY handover Co-Authored-By: Claude Opus 5.5 --- quest/m3/lite-06-handover-subscriptions.md | 40 ++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 quest/m3/lite-06-handover-subscriptions.md diff --git a/quest/m3/lite-06-handover-subscriptions.md b/quest/m3/lite-06-handover-subscriptions.md new file mode 100644 index 00000000000..0380128ae73 --- /dev/null +++ b/quest/m3/lite-06-handover-subscriptions.md @@ -0,0 +1,40 @@ +# [M] A moq-lite-06 subscription survives a GOAWAY handover + +## Goal + +A client on moq-lite-06 (today's default) keeps a live subscription through +a GOAWAY handover: once the replacement session serves the broadcast, the +track continues from it instead of ending when the drained session closes. + +## Plan + +Found by [moq-dev/moq#9176](https://github.com/moq-dev/moq/pull/9176) while +fixing `moq-tokio`'s `a_fleet_drain_redials_through_a_fresh_resolve`. Over +lite-06 the wire carries no instance, so the subscriber's origin mints one +per announcement. The front serving the track is pinned to the drained +session's instance (`pick` in `rs/moq-net/src/model/origin.rs`), so it +never selects the replacement's route. When the drained session closes, the +front ends `Unroutable` and the reader sees `GoawayTimeout`, even with the +replacement up and announcing. Over lite-07, which carries the instance, the +same test's track continues on the replacement. The comments in +`rs/moq-net/src/reconnect.rs` (the GOAWAY handling in `ended`, and the +module doc on the fallback probe) say live tracks splice onto the replacement, +which holds only for lite-07. + +A possible fix: the connect policy knows the replacement is a handover of the +same peer, so its announcements of a path the predecessor serves could inherit +the predecessor's instance. Check first whether a same-path announcement from a +redial of the same URL is safe to read as one instance (a fleet name can land +on a relay serving a restarted publisher). + +## Waiting + +- Maintainer: fix it for lite-06, or only correct the `reconnect.rs` comments + and leave lite-06 subscribers to re-subscribe until lite-07 is the default. + Recommended: correct the comments now and fix it, since a relay drain ends + every live subscription of a lite-06 client today. An agent can't decide this: it + depends on how soon lite-07 becomes the default. + +## Related + +- [GOAWAY migration follow-ups](/quest/m3/goaway-migration-followups.md) - closes an idle predecessor once the replacement has synced