diff --git a/nix/Cargo.nix b/nix/Cargo.nix index 35df0c49d8..6a8437139d 100644 --- a/nix/Cargo.nix +++ b/nix/Cargo.nix @@ -22133,7 +22133,10 @@ rec { { name = "kio"; packageId = "kio"; - features = [ "sim" ]; + features = [ + "manual" + "sim" + ]; } { name = "moq-congestion"; diff --git a/quest/m0/test-flakes-3/fleet-drain-redial.md b/quest/m0/test-flakes-3/fleet-drain-redial.md deleted file mode 100644 index 602498f52d..0000000000 --- a/quest/m0/test-flakes-3/fleet-drain-redial.md +++ /dev/null @@ -1,19 +0,0 @@ -# [S] A fleet drain's redial delivers the next group under load - -## Goal - -`moq-tokio` `connection::tests::a_fleet_drain_redials_through_a_fresh_resolve` -passes under a loaded `just check`. - -## Plan - -Seen once on 2026-11-18 during `just check` for -[moq-dev/moq#9140](https://github.com/moq-dev/moq/pull/9140), which does not -touch moq-tokio or GOAWAY: the subscriber's `recv_group` for `g1` failed with -`Session(GoawayTimeout)` (`rs/moq-tokio/src/connection.rs`, the `g1` wait). It -passed 5 of 5 alone and 40 of 40 under `--stress-count 40 --test-threads 16`, -taking 0.5 to 3 s alone, so it rides on wall-clock time. - -Find why the drained session's deadline can end the subscription before the -redial to B carries `g1`, and drive the timers with a manual clock or assert on -the handover event instead of real time, per the epic's rules. diff --git a/quest/m3/lite-06-handover-subscriptions.md b/quest/m3/lite-06-handover-subscriptions.md new file mode 100644 index 0000000000..0380128ae7 --- /dev/null +++ b/quest/m3/lite-06-handover-subscriptions.md @@ -0,0 +1,40 @@ +# [M] A moq-lite-06 subscription survives a GOAWAY handover + +## Goal + +A client on moq-lite-06 (today's default) keeps a live subscription through +a GOAWAY handover: once the replacement session serves the broadcast, the +track continues from it instead of ending when the drained session closes. + +## Plan + +Found by [moq-dev/moq#9176](https://github.com/moq-dev/moq/pull/9176) while +fixing `moq-tokio`'s `a_fleet_drain_redials_through_a_fresh_resolve`. Over +lite-06 the wire carries no instance, so the subscriber's origin mints one +per announcement. The front serving the track is pinned to the drained +session's instance (`pick` in `rs/moq-net/src/model/origin.rs`), so it +never selects the replacement's route. When the drained session closes, the +front ends `Unroutable` and the reader sees `GoawayTimeout`, even with the +replacement up and announcing. Over lite-07, which carries the instance, the +same test's track continues on the replacement. The comments in +`rs/moq-net/src/reconnect.rs` (the GOAWAY handling in `ended`, and the +module doc on the fallback probe) say live tracks splice onto the replacement, +which holds only for lite-07. + +A possible fix: the connect policy knows the replacement is a handover of the +same peer, so its announcements of a path the predecessor serves could inherit +the predecessor's instance. Check first whether a same-path announcement from a +redial of the same URL is safe to read as one instance (a fleet name can land +on a relay serving a restarted publisher). + +## Waiting + +- Maintainer: fix it for lite-06, or only correct the `reconnect.rs` comments + and leave lite-06 subscribers to re-subscribe until lite-07 is the default. + Recommended: correct the comments now and fix it, since a relay drain ends + every live subscription of a lite-06 client today. An agent can't decide this: it + depends on how soon lite-07 becomes the default. + +## Related + +- [GOAWAY migration follow-ups](/quest/m3/goaway-migration-followups.md) - closes an idle predecessor once the replacement has synced diff --git a/rs/moq-tokio/Cargo.toml b/rs/moq-tokio/Cargo.toml index 6eb71f6dbd..57caa1b4d6 100644 --- a/rs/moq-tokio/Cargo.toml +++ b/rs/moq-tokio/Cargo.toml @@ -102,7 +102,7 @@ criterion = { package = "bench-harness", path = "../bench-harness" } # Use the channel type behind `mdns_sd::Receiver` so the mDNS readiness gate can # be driven with synthetic daemon events. flume = "0.12" -kio = { path = "../kio", features = ["sim"] } +kio = { path = "../kio", features = ["manual", "sim"] } # The integration tests build congestion controllers directly, including the candidates the # real-socket comparisons run. moq-congestion = { path = "../moq-congestion", features = ["lab"] } diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index 269baca83b..5993ab5c03 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -2805,13 +2805,19 @@ mod tests { /// The fleet drain: a relay withdrawn from DNS sends an empty-URI GOAWAY with a /// deadline, and the client lands on a healthy relay by resolving the configured /// name again. The drained relay still accepts, so a cached resolve would land - /// right back on it. The live track hands over at a group boundary, and the old - /// session closes at our handover cap, well before the peer's own deadline. + /// right back on it. The drained session keeps delivering the live track until it + /// closes at our handover cap, well before the peer's own deadline. + /// + /// The sockets are real but the clock is manual, so the cap passes only when the + /// test moves it: a slow host cannot close the drained session before it delivers. + /// Over moq-lite-06 the subscription stays on the drained session, since each + /// session's announcement serves an instance of its own. #[tokio::test] async fn a_fleet_drain_redials_through_a_fresh_resolve() { - const WAIT: Duration = Duration::from_secs(10); + use futures::FutureExt; const HANDOVER: Duration = Duration::from_millis(500); const DEADLINE: Duration = Duration::from_secs(30); + let clock = kio::clock::Manual::new().install(&tokio::runtime::Handle::current()); // Two relays of one fleet, serving the same live broadcast. let origin = crate::origin::spawn(); @@ -2827,8 +2833,6 @@ mod tests { let subscriber = crate::origin::spawn(); let mut config = crate::connect::Config::default(); config.goaway.handover = Some((HANDOVER).into()); - // A session younger than the healthy bar counts as redirected immediately and - // waits out a backoff, which this test is not about. config.backoff.initial = Some((FAST).into()); config.backoff.healthy = Some((FAST).into()); let client = config @@ -2838,32 +2842,30 @@ mod tests { let url: Url = format!("tcp://{HOST}:1/").parse().unwrap(); let _connection = client.connect(url); - let session_a = kio::clock::timeout(WAIT, accepted_a.recv()).await.unwrap().unwrap(); + let session_a = accepted_a.recv().await.unwrap(); - let consumer = subscriber.consume(); - let cam = kio::clock::timeout(WAIT, consumer.follow("cam", None).unwrap().available()) - .await + let cam = subscriber + .consume() + .follow("cam", None) .unwrap() + .available() + .await .unwrap() .unwrap(); let mut sub = cam.track("video").unwrap().subscribe(None).await.unwrap(); - let mut group = track - .append_group( - moq_net::Timestamp::from_millis(track.largest().map_or(0, |largest| largest.group + 1)).unwrap(), - ) - .unwrap(); + let mut group = track.append_group(moq_net::Timestamp::from_millis(0).unwrap()).unwrap(); group - .write_frame(moq_net::Timestamp::from_millis(group.start).unwrap(), b"g0".as_ref()) + .write_frame(moq_net::Timestamp::from_millis(0).unwrap(), b"g0".as_ref()) .unwrap(); group.finish().unwrap(); - let g0 = kio::clock::timeout(WAIT, sub.recv_group()) - .await - .unwrap() - .unwrap() - .unwrap(); + let g0 = sub.recv_group().await.unwrap().unwrap(); assert_eq!(g0.start, 0); + // A session younger than the healthy bar counts as redirected immediately and + // waits out a backoff, which this test is not about. + clock.advance(FAST); + // Withdraw A from DNS, then drain it. crate::resolve::hosts::point(HOST, [addr_b]); let drained = kio::clock::now(); @@ -2871,34 +2873,30 @@ mod tests { .goaway(moq_net::goaway::Goaway::new().with_timeout(Some(DEADLINE))) .unwrap(); - let _session_b = kio::clock::timeout(WAIT, accepted_b.recv()) + let _session_b = accepted_b + .recv() .await - .expect("never redialed through the fresh resolve") - .unwrap(); + .expect("never redialed through the fresh resolve"); - let mut group = track - .append_group( - moq_net::Timestamp::from_millis(track.largest().map_or(0, |largest| largest.group + 1)).unwrap(), - ) - .unwrap(); + let mut group = track.append_group(moq_net::Timestamp::from_millis(1).unwrap()).unwrap(); group - .write_frame(moq_net::Timestamp::from_millis(group.start).unwrap(), b"g1".as_ref()) + .write_frame(moq_net::Timestamp::from_millis(1).unwrap(), b"g1".as_ref()) .unwrap(); group.finish().unwrap(); - let mut g1 = kio::clock::timeout(WAIT, sub.recv_group()) - .await - .unwrap() - .unwrap() - .unwrap(); - assert_eq!(g1.start, 1, "delivery resumes at the next group after the swap"); + let mut g1 = sub.recv_group().await.unwrap().unwrap(); + assert_eq!(g1.start, 1, "delivery continues at the next group during the handover"); assert_eq!(g1.read_frame().await.unwrap().unwrap().payload[..], b"g1"[..]); - - kio::clock::timeout(WAIT, session_a.closed()) - .await - .expect("the drained session never closed"); assert!( - drained.elapsed() < DEADLINE, - "the old session outlived the handover cap and waited for the peer's deadline" + session_a.closed().now_or_never().is_none(), + "the drained session closed before its handover cap while still serving" + ); + + clock.advance(HANDOVER); + session_a.closed().await; + assert_eq!( + drained.elapsed(), + HANDOVER, + "the old session closed at the handover cap, not the peer's deadline" ); assert!( accepted_a.try_recv().is_err(),