diff --git a/quest/m1/README.md b/quest/m1/README.md index fb528778308..226ab149711 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -83,7 +83,6 @@ lands before the cut. - [A/V clock](/quest/m1/av-clock.md) - the audio playhead drives Sync.reference while audio plays, through per-track sync handles - [Publish channel count](/quest/m1/publish-audio-channel-count.md) - forcing a channel count on an Audio.Capture stops costing the subscriber gaps of silence, and the media fixture's cold-start audio-tone flake is fixed at its cause - [Relay signal install](/quest/m1/relay-signal-install.md) - a shutdown signal after the relay starts serving drains it, without leaking handlers into an embedder whose startup failed -- [Rust redirect loops](/quest/m1/tokio-redirect-loop.md) - moq-tokio escalates its backoff when GOAWAY redirects bounce between peers, as JS does - [JS redirect keys strip userinfo](/quest/m1/js-reload-userinfo.md) - js/net keys a reconnect endpoint without URL userinfo, matching Rust - [Container reads drop backfill](/quest/m1/read-backfill.md) - the JS container consumer drops groups below its read cursor like Rust - [Rust request credit](/quest/m1/rs-request-credit.md) - a moq-net request fails at once without stream credit instead of waiting diff --git a/quest/m1/js-reload-userinfo.md b/quest/m1/js-reload-userinfo.md index 266be1810ce..d78f90a9c67 100644 --- a/quest/m1/js-reload-userinfo.md +++ b/quest/m1/js-reload-userinfo.md @@ -14,7 +14,3 @@ userinfo; match it exactly, and test both a userinfo URL and a redirect loop that differs only in userinfo. Public API: none. Wire: none. - -## Required - -- [Rust redirect loops](/quest/m1/tokio-redirect-loop.md) - lands the Rust rule this mirrors diff --git a/quest/m1/tokio-redirect-loop.md b/quest/m1/tokio-redirect-loop.md deleted file mode 100644 index 944da5adef0..00000000000 --- a/quest/m1/tokio-redirect-loop.md +++ /dev/null @@ -1,41 +0,0 @@ -# [S] moq-tokio escalates its backoff on GOAWAY redirect loops - -## Goal - -When two relays redirect a Rust client to each other after each session -outlives `backoff.initial`, moq-tokio's reconnect loop escalates its delay up -to `backoff.max` and gives up at `backoff.timeout` with a redirect-loop error, -instead of bouncing between them with no delay forever. A one-way move and a -same-place GOAWAY (an empty URI, a restart) still migrate at once. - -## Plan - -Found 2026-10-20 checking GOAWAY follow-ups of -[#5998](https://github.com/moq-dev/moq/pull/5998). -[#5959](https://github.com/moq-dev/moq/pull/5959) fixes this shape in JS -(`js/net/src/connection/reload.ts`) and lists Rust as unchecked. In -`rs/moq-tokio/src/connection.rs`, a GOAWAY off a session that lived at least -`backoff.initial` continues with no delay, because `Retry::settle` has -already reset the delay, start and last error for that session; only an -immediate redirect escalates. - -- Mirror #5959's rule once it lands: while migrations are chained, remember - the endpoints that redirected the client elsewhere, and route a redirect - back to one of them through the escalating retry without the - healthy-session reset. Clear the set when a session ends without a GOAWAY - or a new sequence starts. Use the same error name as JS. -- Regression tests on a mocked clock, as #5959's: mutual A/B redirects are - bounded and end at `backoff.timeout`; a one-way move and same-place - GOAWAYs redial at once. - -Decided 2026-10-20 (maintainer): no quest reissues individual requests a -GOAWAY interrupts. Same-epoch subscriptions already resume through route -swaps, namespaces re-advertise on the new session, and epochless requests -end, as instances are never stitched. - -Public API: none. Wire: none. Behavior: a redirect loop now backs off and -can end the connection with an error. - -## Related - -- [#5959](https://github.com/moq-dev/moq/pull/5959) - the JS fix this mirrors diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index 66b3ecd4dcb..ead3335a02c 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -58,6 +58,11 @@ fn settles_walk(err: &Error) -> bool { /// A GOAWAY naming no new URI is no failure: a restarting peer sends one to each session until /// it goes, and each of those /// serves, so it restarts the timeout instead. +/// +/// A GOAWAY off a session that outlived `initial` migrates at once, unless it sends the +/// connection back to an endpoint that already redirected it elsewhere and the session lived +/// less than `max`. Two peers redirecting to each other thus wait out this backoff and count +/// against the timeout. #[derive(Clone, Debug, usage_rs::Args, serde::Serialize, serde::Deserialize)] #[usage(unknown_flags = "error", args_override_self = false)] #[serde(default, deny_unknown_fields)] @@ -119,9 +124,9 @@ pub struct Backoff { /// [`crate::connect::Config::timeout`](crate::connect::Config::timeout), so a handshake /// slower than this still lands. Giving up can therefore take up to one whole walk longer. /// - /// Resets after a stable connection (one that outlives the initial backoff), so a flapping - /// session that reconnects then immediately drops still counts toward the timeout. Set to - /// `Some(Duration::ZERO)` for unlimited retries. `None` takes the default. When this config was + /// Resets after a stable connection (one that outlives the initial backoff) outside a redirect + /// loop, so a flapping session that reconnects then immediately drops still counts toward the + /// timeout. Set to `Some(Duration::ZERO)` for unlimited retries. `None` takes the default. When this config was /// parsed from the command line, a `--backoff-timeout` given there wins over this field. /// /// An `Option` because zero is a real setting: a command-line parse leaves this field at its @@ -850,6 +855,8 @@ impl Connection { let mut addrs = addrs; // An old session kept alive after a GOAWAY so its in-flight groups finish. let mut draining: Option = None; + // The current run of migrations, to tell a redirect loop from a healthy handover. + let mut redirects = Redirects::default(); loop { if !timeout.is_zero() && retry.start.elapsed() >= timeout { @@ -878,9 +885,7 @@ impl Connection { shared.retrying(err); } - // A session that stayed up past the initial backoff is healthy; one that - // ended sooner counts as a failed attempt however it ended. - let healthy = retry.settle(connected.elapsed(), initial); + let tenure = connected.elapsed(); // The connected target owns the policy, including in one-shot mode. A // refused redirect is terminal: the peer is leaving and named somewhere we @@ -909,6 +914,28 @@ impl Connection { return Ok(()); } + // Two peers that each outlive the initial backoff can still bounce us between + // them forever. Returning to an endpoint that already sent us elsewhere in this + // run of migrations skips the healthy-session reset, so the backoff escalates and + // the retry window still applies. A one-way rolling restart never revisits one, + // and a GOAWAY back to the same place is a restart, not a move, so it never + // counts. A session that outlived the capped delay already redials no faster + // than the backoff would allow, so it starts a new run: rolling restarts days + // apart never add up to a loop. + if matches!(ended, Ended::Closed(_)) || tenure >= pacing.max { + redirects.reset(); + } + // Only an accepted redirect is a move: an empty or ignored one reconnects in place. + let looped = assigned.as_ref().is_some_and(|target| { + let from = endpoint(&url); + let to = endpoint(target); + to != from && redirects.depart(from, &to) + }); + + // A session that stayed up past the initial backoff is healthy; one that + // ended sooner, or carried on a redirect loop, counts as a failed attempt. + let healthy = !looped && retry.settle(tenure, initial); + if let Ended::Goaway(msg) = &ended { // An accepted redirect is an assignment: keep dialing it from here on, and // only it. The peer named exactly one place to go, which retires @@ -941,8 +968,9 @@ impl Connection { continue; } - // Drained almost immediately. The backoff still escalates, so the redials - // cannot hammer the peer, and the old session serves across the sleep. + // Drained almost immediately, or looped back. The backoff still escalates, so + // the redials cannot hammer the peer, and the old session serves across the + // sleep. // // An empty URI is "reconnect to me": the peer is restarting and drains // each new session until it goes, which is no failure, since every one @@ -950,7 +978,12 @@ impl Connection { // peers bouncing us between them eventually give up. let restarting = msg.uri().is_empty(); if !restarting { - let err = Error::Reconnect("peer redirected immediately".to_string()); + let reason = if looped { + "redirect loop" + } else { + "peer redirected immediately" + }; + let err = Error::Reconnect(reason.to_string()); shared.retrying(&err); retry.last_error = Some(err); } @@ -960,7 +993,11 @@ impl Connection { let Some(wait) = retry_wait(retry.delay, retry.start, window) else { return Err(timeout_error(timeout, retry.last_error.as_ref())); }; - tracing::warn!(peer = %Endpoint(&url), ?wait, "peer drained the session immediately; retrying after backoff"); + if looped { + tracing::warn!(peer = %Endpoint(&url), ?wait, "peer redirected back in a loop; retrying after backoff"); + } else { + tracing::warn!(peer = %Endpoint(&url), ?wait, "peer drained the session immediately; retrying after backoff"); + } // Keep the handover bounded across the sleep: nothing else polls the // predecessor while the loop is between connections. sleep_draining(wait, &mut draining, shared).await; @@ -1002,8 +1039,7 @@ impl Connection { // resets, or redirects us straight back out). Treat it as a failed // connection: keep the reason so the give-up timeout reports a real cause, // and fall through to the shared backoff sleep below so repeated flaps - // escalate instead of spinning the CPU. This is what bounds a redirect - // loop between two peers. + // escalate instead of spinning the CPU. let err = match ended { Ended::Closed(Err(err)) => Some(err), Ended::Closed(Ok(())) => None, @@ -1324,6 +1360,62 @@ fn retry_wait(delay: Duration, retry_start: tokio::time::Instant, timeout: Durat Some(wait.min(timeout.checked_sub(retry_start.elapsed())?)) } +/// Where `url` dials, without its userinfo, query, or fragment, where a peer may mint fresh +/// credentials on every redirect. Two URLs that differ only there reach the same endpoint. +fn endpoint(url: &Url) -> Url { + let mut bare = url.clone(); + // Only fails for a URL that cannot carry userinfo, which then has none to strip. + let _ = bare.set_username(""); + let _ = bare.set_password(None); + bare.set_query(None); + bare.set_fragment(None); + bare +} + +/// A run of GOAWAY redirects, and whether it has revisited an endpoint. +/// +/// The peer picks every hop, so remembering them all would grow without bound. Brent's cycle +/// detection instead keeps one departed endpoint, replaced at doubling intervals, and catches a +/// cycle of any length within a few laps of it. Once caught, the run stays a loop until reset. +struct Redirects { + mark: Option, + span: u64, + hops: u64, + looped: bool, +} + +impl Default for Redirects { + fn default() -> Self { + Self { + mark: None, + span: 1, + hops: 0, + looped: false, + } + } +} + +impl Redirects { + /// Record leaving `from` for a different endpoint `to`, and return whether this run loops. + fn depart(&mut self, from: Url, to: &Url) -> bool { + if self.mark.as_ref() == Some(to) { + self.looped = true; + } + self.hops += 1; + if self.hops == self.span { + self.mark = Some(from); + self.span = self.span.saturating_mul(2); + self.hops = 0; + } + self.looped + } + + /// Start a new run. + fn reset(&mut self) { + *self = Self::default(); + } +} + /// Why a session stopped being the live one. enum Ended { /// The transport closed. @@ -2735,6 +2827,55 @@ mod tests { "a cached resolve redialed the drained relay" ); } + + #[test] + fn redirect_loop_detection_catches_any_cycle_in_constant_memory() { + for tail in 0..4 { + for cycle in 2..=8 { + // `tail` one-way moves, then `cycle` endpoints redirecting around a ring. + let at = |hop: usize| { + if hop < tail { + url(&format!("tcp://tail{hop}/")) + } else { + url(&format!("tcp://ring{}/", (hop - tail) % cycle)) + } + }; + let mut redirects = Redirects::default(); + let mut caught = None; + for hop in 0..4 * (tail + cycle) { + let looped = redirects.depart(at(hop), &at(hop + 1)); + match caught { + Some(_) => assert!(looped, "a caught loop stays caught"), + None if looped => caught = Some(hop), + None => {} + } + } + let caught = caught.unwrap_or_else(|| panic!("tail {tail}, cycle {cycle}: never caught")); + // Never before the run first returns to an endpoint it left. + assert!( + caught + 1 >= tail + cycle, + "tail {tail}, cycle {cycle}: caught at {caught}" + ); + } + } + + let mut redirects = Redirects::default(); + for hop in 0..1000 { + let from = url(&format!("tcp://a{hop}/")); + let to = url(&format!("tcp://a{}/", hop + 1)); + assert!(!redirects.depart(from, &to), "a one-way run never loops"); + } + } + + #[test] + fn an_endpoint_ignores_the_credentials() { + assert_eq!( + endpoint(&url("tcp://token-a:secret@relay:1/path?jwt=a#x")), + endpoint(&url("tcp://token-b@relay:1/path?jwt=b")) + ); + // The path is part of the request target, so it names a different endpoint. + assert_ne!(endpoint(&url("tcp://relay:1/a")), endpoint(&url("tcp://relay:1/b"))); + } } #[cfg(test)] diff --git a/rs/moq-tokio/tests/integration/reconnect.rs b/rs/moq-tokio/tests/integration/reconnect.rs index ab60b026ed2..e7e61f2242b 100644 --- a/rs/moq-tokio/tests/integration/reconnect.rs +++ b/rs/moq-tokio/tests/integration/reconnect.rs @@ -427,3 +427,181 @@ async fn rides_out_a_drain(backoff: moq_tokio::Backoff, drained: usize) { _ = tokio::time::sleep(Duration::from_secs(30)) => panic!("never landed after the drain"), } } + +/// How long each session on a bouncing fleet lives before its relay sends the GOAWAY. Twice +/// the 50ms initial backoff these tests use, so every session counts as healthy. +const LIFE: Duration = Duration::from_millis(100); + +/// Send every session `sessions` yields the GOAWAY `uri(n)` once it has lived [`LIFE`], `n` +/// counting its sessions, and report each accept's time on `log`. Each session is held until +/// the client leaves it, as a draining relay does. +fn bounce( + mut sessions: tokio::sync::mpsc::UnboundedReceiver, + log: tokio::sync::mpsc::UnboundedSender, + uri: impl Fn(usize) -> String + Send + 'static, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + for n in 0.. { + let Some(session) = sessions.recv().await else { return }; + let _ = log.send(tokio::time::Instant::now()); + let uri = uri(n); + tokio::spawn(async move { + tokio::time::sleep(LIFE).await; + let _ = session.drain().send(moq_net::goaway::Goaway::redirect(uri)); + session.closed().await; + }); + } + }) +} + +/// Two relays redirecting every session to each other once it has lived [`LIFE`], minting a +/// fresh credential in the query each time when `rotate` is set. Returns the first relay's +/// URL, a receiver of every accept's time across both, and their tasks. +async fn ping_pong( + rotate: bool, +) -> ( + url::Url, + tokio::sync::mpsc::UnboundedReceiver, + Vec>, +) { + let (port_a, sessions_a, task_a) = spawn_server().await; + let (port_b, sessions_b, task_b) = spawn_server().await; + let (log, accepts) = tokio::sync::mpsc::unbounded_channel(); + let to = move |port: u16| { + move |n: usize| match rotate { + true => format!("tcp://127.0.0.1:{port}/?jwt={n}"), + false => format!("tcp://127.0.0.1:{port}/"), + } + }; + let tasks = vec![ + task_a, + task_b, + bounce(sessions_a, log.clone(), to(port_b)), + bounce(sessions_b, log, to(port_a)), + ]; + let url = format!("tcp://127.0.0.1:{port_a}/").parse().expect("parse url"); + (url, accepts, tasks) +} + +/// A backoff starting at 50ms that doubles up to `max`, giving up after `timeout`. +fn bounced_backoff(max: Duration, timeout: Duration) -> moq_tokio::Backoff { + let mut backoff = moq_tokio::Backoff::default(); + backoff.initial = Some(Duration::from_millis(50)); + backoff.multiplier = 2; + backoff.max = Some(max); + backoff.timeout = Some(timeout); + backoff +} + +/// How long the client took to redial after each of `hops` GOAWAYs: the time between two +/// accepts, less the [`LIFE`] the first one served before its GOAWAY. +async fn redial_waits( + accepts: &mut tokio::sync::mpsc::UnboundedReceiver, + hops: usize, +) -> Vec { + let mut last = accepts.recv().await.expect("never connected"); + let mut waits = Vec::new(); + for hop in 0..hops { + let next = tokio::time::timeout(Duration::from_secs(10), accepts.recv()) + .await + .unwrap_or_else(|_| panic!("hop {hop}: never redialed")) + .expect("the relays stopped"); + waits.push(next.saturating_duration_since(last).saturating_sub(LIFE)); + last = next; + } + waits +} + +/// Two relays that each outlive the initial backoff can still bounce the client between them +/// forever. The first move is a healthy migration; every return after it is a loop that waits +/// out the jittered backoff, at least half of a delay escalating to the cap. The redirects are +/// still honoured: a loop is something to slow down, not a failure. +#[tokio::test] +async fn two_peers_redirecting_to_each_other_escalate_the_backoff() { + for rotate in [false, true] { + let (url, mut accepts, _tasks) = ping_pong(rotate).await; + let backoff = bounced_backoff(Duration::from_millis(400), Duration::ZERO); + let _connection = client(backoff).connect(url); + + let delays = [0, 50, 100, 200, 400, 400].map(Duration::from_millis); + let waits = redial_waits(&mut accepts, delays.len()).await; + for (hop, (wait, delay)) in waits.into_iter().zip(delays).enumerate() { + assert!( + wait >= delay / 2, + "rotate {rotate}, hop {hop}: waited {wait:?} against a {delay:?} backoff" + ); + } + } +} + +/// The retry window still ends a redirect loop. Without the loop guard it never would: every +/// hop is healthy and restarts the window. +#[tokio::test] +async fn a_redirect_loop_stops_at_the_retry_window() { + let (url, mut accepts, _tasks) = ping_pong(false).await; + let backoff = bounced_backoff(Duration::from_millis(400), Duration::from_millis(300)); + let connection = client(backoff).connect(url); + let closed = connection.closed(); + tokio::pin!(closed); + for hop in 0.. { + tokio::select! { + res = &mut closed => { + let err = res.expect_err("the loop ended without an error"); + assert!( + matches!(&err, moq_tokio::Error::Reconnect(msg) if msg.contains("redirect loop")), + "{err}" + ); + return; + } + accept = accepts.recv() => { + accept.expect("the relays stopped"); + assert!(hop < 10, "the loop never gave up"); + } + } + } +} + +/// Run `hops` migrations of a connection whose retry window is shorter than a session's +/// life. A healthy migration restarts the window, while any hop scored as a loop spends it +/// and ends the connection, so this fails on a backoff the hops should never take. +async fn migrates_without_backing_off( + connection: moq_tokio::Connection, + accepts: &mut tokio::sync::mpsc::UnboundedReceiver, + hops: usize, +) { + tokio::select! { + _ = redial_waits(accepts, hops) => {} + res = connection.closed() => panic!("a migration backed off: {res:?}"), + } +} + +/// A loop no faster than the capped delay already redials as slowly as the backoff would, so +/// each hop starts a new run and migrates at once. +#[tokio::test] +async fn a_loop_no_faster_than_the_capped_delay_migrates_without_backing_off() { + let (url, mut accepts, _tasks) = ping_pong(false).await; + // The cap is the initial delay, well under each session's life. + let backoff = bounced_backoff(Duration::from_millis(50), Duration::from_millis(60)); + migrates_without_backing_off(client(backoff).connect(url), &mut accepts, 6).await; +} + +/// A one-way rolling restart never revisits a relay, and a GOAWAY back to the same place, +/// named or empty, is a restart rather than a move. None of them backs off. +#[tokio::test] +async fn a_one_way_rolling_restart_migrates_without_backing_off() { + let (port_a, sessions_a, _task_a) = spawn_server().await; + let (port_b, sessions_b, _task_b) = spawn_server().await; + let (port_c, sessions_c, _task_c) = spawn_server().await; + let (log, mut accepts) = tokio::sync::mpsc::unbounded_channel(); + let _a = bounce(sessions_a, log.clone(), move |_| format!("tcp://127.0.0.1:{port_b}/")); + let _b = bounce(sessions_b, log.clone(), move |_| format!("tcp://127.0.0.1:{port_c}/")); + // The last relay restarts in place, alternating a named URI and an empty one. + let _c = bounce(sessions_c, log, move |n| match n % 2 { + 0 => format!("tcp://127.0.0.1:{port_c}/"), + _ => String::new(), + }); + + let backoff = bounced_backoff(Duration::from_millis(400), Duration::from_millis(60)); + let url: url::Url = format!("tcp://127.0.0.1:{port_a}/").parse().expect("parse url"); + migrates_without_backing_off(client(backoff).connect(url), &mut accepts, 6).await; +}