From 8894f6a1c2e2f3440d1fbaccf4b335a1567b4a5b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 11 Oct 2026 20:58:43 -0700 Subject: [PATCH 1/2] fix(moq-tokio): a restarting peer's GOAWAYs do not exhaust the retry window A draining relay keeps accepting for its whole drain and sends each new session an empty-URI GOAWAY at once. The reconnect loop scored every one as a failed attempt, so with the default 10s window and a 10s drain a client gave up just as the relay finished restarting. An empty URI asks the client back to the same place, and each of those sessions serves until the client leaves it, so none is a failure: the window restarts instead. The backoff still escalates, and a GOAWAY that names a URI still counts, so a redirect loop between two peers gives up. Co-Authored-By: Claude Opus 5.5 --- rs/moq-relay/tests/shutdown_signal.rs | 86 +++++++++++++++++++++++++++ rs/moq-tokio/src/connection.rs | 24 +++++--- rs/moq-tokio/tests/reconnect.rs | 59 ++++++++++++++++++ 3 files changed, 162 insertions(+), 7 deletions(-) diff --git a/rs/moq-relay/tests/shutdown_signal.rs b/rs/moq-relay/tests/shutdown_signal.rs index 1b25078cf41..b295cee1899 100644 --- a/rs/moq-relay/tests/shutdown_signal.rs +++ b/rs/moq-relay/tests/shutdown_signal.rs @@ -20,6 +20,9 @@ //! The drain ends as soon as every session has left rather than waiting out the //! window, which a stop-time budget depends on. //! +//! A reconnecting client rides out a restart on the same address: the GOAWAYs +//! the drain sends its redials must not exhaust its retry window. +//! //! A raised signal reaches every listener in the process, and a relay exits at //! once on its second one, so the tests that raise SIGINT hold [`SIGINT`]: //! `cargo test` (the Nix check phase) runs them as threads of one process. @@ -88,6 +91,11 @@ fn a_trigger_before_run_keeps_the_deadline() { run_test(a_trigger_before_run_keeps_the_deadline_inner); } +#[test] +fn a_reconnecting_client_rides_out_a_restart() { + run_test(a_reconnecting_client_rides_out_a_restart_inner); +} + async fn sigint_drains_sessions_before_exiting_inner() { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); @@ -314,6 +322,84 @@ async fn a_trigger_before_run_keeps_the_deadline_inner() { ); } +/// A restart on the same address, with a reconnecting publisher whose retry window is a third of +/// the drain. The drain sends each of its redials an empty-URI GOAWAY until the window closes. +/// Every one of those sessions served, so none counts against the retry window, and the +/// publisher is still there to land on the relay that takes the address over. +async fn a_reconnecting_client_rides_out_a_restart_inner() { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let relay = Relay::load(relay_config()) + .await + .expect("load relay") + .with_signals(false); + let addr = relay.tcp_addr().expect("TCP listener bound"); + let trigger = relay.shutdown_trigger().clone(); + let run = tokio::spawn(relay.run()); + let url: url::Url = format!("tcp://{addr}/").parse().expect("parse url"); + + let origin = moq_tokio::origin::spawn(); + let broadcast = origin.create_broadcast("cam").expect("create broadcast"); + broadcast.announce(Default::default()).expect("announce"); + let mut publisher = moq_tokio::connect::Config::default(); + // Redials paced well inside the drain, under a window that ends long before it. + publisher.backoff.initial = Duration::from_millis(200); + publisher.backoff.max = Duration::from_millis(200); + publisher.backoff.timeout = Some(DRAIN_TIMEOUT / 3); + let publisher = publisher + .init(Default::default()) + .expect("client init") + .with_publisher(&origin) + .connect(url.clone()) + .established() + .await + .expect("connect"); + + // Holds the drain open for its whole window, as the rest of a busy relay's sessions would. + let straggler = straggler(addr.port()).await; + + trigger.start(); + tokio::time::timeout(Duration::from_secs(15), run) + .await + .expect("relay never exited after the drain window") + .expect("relay task panicked") + .expect("relay exited with an error"); + drop(straggler); + + let mut config = relay_config(); + config.listen.tcp.bind = Some(addr); + let relay = Relay::load(config) + .await + .expect("restart on the same address") + .with_signals(false); + let trigger = relay.shutdown_trigger().clone(); + let run = tokio::spawn(relay.run()); + + let subscriber = moq_tokio::origin::spawn(); + let consumer = subscriber.consume(); + let observer = client(Vec::new()) + .with_subscriber(subscriber) + .connect(url) + .established() + .await + .expect("connect"); + tokio::select! { + res = consumer.routed_broadcast("cam") => { + res.expect("the publisher's broadcast never resolved"); + } + res = publisher.closed() => panic!("the publisher gave up during the restart: {res:?}"), + _ = tokio::time::sleep(Duration::from_secs(15)) => panic!("the publisher never reached the new relay"), + } + + drop((publisher, observer)); + trigger.start(); + tokio::time::timeout(Duration::from_secs(15), run) + .await + .expect("relay never exited after its sessions left") + .expect("relay task panicked") + .expect("relay exited with an error"); +} + /// A one-shot client offering `version` (every version when empty): a /// reconnecting one would migrate on the GOAWAY and hide whether the relay /// closed the original session. diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index 68cef08755d..950f14cdc42 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -50,7 +50,9 @@ fn settles_walk(err: &Error) -> bool { /// /// The delays carry jitter, so a fleet /// knocked offline together does not reconnect in lockstep. The timeout bounds every failure; -/// a settled rejection or invalid local configuration stops retrying. +/// a settled rejection or invalid local configuration stops retrying. A GOAWAY naming no new URI +/// is no failure: a restarting peer sends one to each session until it goes, and each of those +/// serves, so it restarts the timeout instead. #[derive(Clone, Debug, usage::Args, serde::Serialize, serde::Deserialize)] #[usage(unknown_flags = "error", args_override_self = false)] #[serde(default, deny_unknown_fields)] @@ -971,15 +973,23 @@ impl Connection { continue; } - // Redirected almost immediately. Still follow it, but score it as a - // failed attempt so two peers bouncing us between them escalate - // through backoff and eventually give up. The old session serves - // across the sleep, so the redirect loop costs time, not data. - retry.last_error = Some(Error::Reconnect("peer redirected immediately".to_string())); + // Drained almost immediately. The backoff still escalates, so the redials + // cannot hammer the peer, and the old session serves across the sleep. + if msg.uri().is_empty() { + // "Reconnect to me": the peer is restarting and drains each new + // session until it goes, which is no failure, since every one + // serves. The window restarts here, so it counts only the time + // after the peer stops answering. + retry.start = tokio::time::Instant::now(); + } else { + // Redirected: score it as a failed attempt, so two peers bouncing us + // between them eventually give up. + retry.last_error = Some(Error::Reconnect("peer redirected immediately".to_string())); + } let Some(wait) = retry_wait(retry.delay, retry.start, timeout) else { return Err(timeout_error(timeout, retry.last_error.as_ref())); }; - tracing::warn!(peer = %Endpoint(&url), ?wait, "peer redirected immediately; retrying after backoff"); + tracing::warn!(peer = %Endpoint(&url), ?wait, "peer drained the session immediately; retrying after backoff"); // Keep the handover bounded across the sleep: nothing else polls the // predecessor while the loop is between connections. sleep_draining(wait, &mut draining, shared).await; diff --git a/rs/moq-tokio/tests/reconnect.rs b/rs/moq-tokio/tests/reconnect.rs index b0766962058..0f1d874c9a7 100644 --- a/rs/moq-tokio/tests/reconnect.rs +++ b/rs/moq-tokio/tests/reconnect.rs @@ -325,3 +325,62 @@ async fn an_empty_goaway_preserves_configured_fallbacks() { .expect("configured fallback was discarded") .expect("server B stopped accepting"); } + +/// A graceful restart: the server keeps accepting while it drains, and sends every new session +/// an empty-URI GOAWAY at once, for longer than the retry window. Each of those sessions serves +/// until the client moves on, so none is a failure, and the client is still there to land on +/// the first session the server lets stay. +#[tokio::test] +async fn a_draining_peer_does_not_exhaust_the_retry_window() { + // Each redial waits at least half of its 200ms delay, so the drained sessions alone span + // more than twice the window. A GOAWAY arriving inside the 200ms health bar is what makes a + // session count as drained at once rather than as a healthy session that later migrated. + const DRAINED: usize = 12; + let mut config = moq_tokio::connect::Config::default(); + config.backoff.initial = Duration::from_millis(200); + config.backoff.max = Duration::from_millis(200); + config.backoff.timeout = Some(Duration::from_millis(500)); + let client = config.init(Default::default()).expect("failed to init client"); + + let mut server = moq_tokio::listen::Config::default(); + server.tcp.bind = Some("127.0.0.1:0".parse().expect("parse addr")); + let mut server = server + .init(Default::default()) + .expect("init server") + .listen() + .await + .expect("bind tcp listener"); + let port = server.tcp_local_addr().expect("tcp listener bound").port(); + + let (served_tx, mut served) = tokio::sync::mpsc::unbounded_channel(); + let _task = tokio::spawn(async move { + let origin = moq_tokio::origin::spawn(); + // Held, as a draining server does: each one closes when the client leaves it. + let mut drained = Vec::new(); + while let Some(request) = server.accept().await { + let Ok(session) = request.with_publisher(&origin).ok().await else { + continue; + }; + if drained.len() < DRAINED { + session + .drain() + .send(moq_net::goaway::Goaway::new().with_timeout(Duration::from_secs(30))) + .expect("send goaway"); + drained.push(session); + } else { + let _ = served_tx.send(session); + } + } + }); + + let url: url::Url = format!("tcp://127.0.0.1:{port}/").parse().expect("parse url"); + let connection = client.connect(url); + + tokio::select! { + session = served.recv() => { + session.expect("server stopped accepting"); + } + res = connection.closed() => panic!("gave up during the drain: {res:?}"), + _ = tokio::time::sleep(Duration::from_secs(30)) => panic!("never landed after the drain"), + } +} From 668cd249abdf76444011b2632d3559596ccf7e0f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 12 Oct 2026 18:29:02 -0700 Subject: [PATCH 2/2] fix(moq-tokio): restart the retry window after a restart's pacing sleep The window restarted before the sleep, which retry_wait capped at the window. A backoff delay longer than the window spent all of it, and the loop gave up without redialing the restarting peer. Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/connection.rs | 26 ++++++++++++++---------- rs/moq-tokio/tests/reconnect.rs | 35 ++++++++++++++++++++++++--------- 2 files changed, 42 insertions(+), 19 deletions(-) diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index 950f14cdc42..88e59188ab0 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -975,18 +975,19 @@ impl Connection { // Drained almost immediately. The backoff still escalates, so the redials // cannot hammer the peer, and the old session serves across the sleep. - if msg.uri().is_empty() { - // "Reconnect to me": the peer is restarting and drains each new - // session until it goes, which is no failure, since every one - // serves. The window restarts here, so it counts only the time - // after the peer stops answering. - retry.start = tokio::time::Instant::now(); - } else { - // Redirected: score it as a failed attempt, so two peers bouncing us - // between them eventually give up. + // + // An empty URI is "reconnect to me": the peer is restarting and drains + // each new session until it goes, which is no failure, since every one + // serves. A named URI is a redirect, scored as a failed attempt so two + // peers bouncing us between them eventually give up. + let restarting = msg.uri().is_empty(); + if !restarting { retry.last_error = Some(Error::Reconnect("peer redirected immediately".to_string())); } - let Some(wait) = retry_wait(retry.delay, retry.start, timeout) else { + // A restart's pacing is not capped by the window: a delay longer than the + // window would otherwise spend it all and end the loop without a redial. + let window = if restarting { Duration::ZERO } else { timeout }; + let Some(wait) = retry_wait(retry.delay, retry.start, window) else { return Err(timeout_error(timeout, retry.last_error.as_ref())); }; tracing::warn!(peer = %Endpoint(&url), ?wait, "peer drained the session immediately; retrying after backoff"); @@ -994,6 +995,11 @@ impl Connection { // predecessor while the loop is between connections. sleep_draining(wait, &mut draining, shared).await; retry.delay = pacing.next(retry.delay); + if restarting { + // After the sleep, so the window counts only the time after the + // peer stops answering. + retry.start = tokio::time::Instant::now(); + } continue; } diff --git a/rs/moq-tokio/tests/reconnect.rs b/rs/moq-tokio/tests/reconnect.rs index 0f1d874c9a7..51a96c48343 100644 --- a/rs/moq-tokio/tests/reconnect.rs +++ b/rs/moq-tokio/tests/reconnect.rs @@ -335,12 +335,29 @@ async fn a_draining_peer_does_not_exhaust_the_retry_window() { // Each redial waits at least half of its 200ms delay, so the drained sessions alone span // more than twice the window. A GOAWAY arriving inside the 200ms health bar is what makes a // session count as drained at once rather than as a healthy session that later migrated. - const DRAINED: usize = 12; - let mut config = moq_tokio::connect::Config::default(); - config.backoff.initial = Duration::from_millis(200); - config.backoff.max = Duration::from_millis(200); - config.backoff.timeout = Some(Duration::from_millis(500)); - let client = config.init(Default::default()).expect("failed to init client"); + let mut backoff = moq_tokio::Backoff::default(); + backoff.initial = Duration::from_millis(200); + backoff.max = Duration::from_millis(200); + backoff.timeout = Some(Duration::from_millis(500)); + rides_out_a_drain(backoff, 12).await; +} + +/// A restart's pacing sleep can be longer than the retry window. It must not spend the window, +/// or the loop would give up right after the sleep without redialing the restarting peer. +#[tokio::test] +async fn a_drain_backoff_longer_than_the_retry_window_still_redials() { + // From the second drained session on, the delay is 400ms against a 100ms window. + let mut backoff = moq_tokio::Backoff::default(); + backoff.initial = Duration::from_millis(200); + backoff.max = Duration::from_millis(400); + backoff.timeout = Some(Duration::from_millis(100)); + rides_out_a_drain(backoff, 4).await; +} + +/// Connect with `backoff` to a server that sends its first `drained` sessions an empty-URI +/// GOAWAY at once, and assert the client is still there to land on the session after them. +async fn rides_out_a_drain(backoff: moq_tokio::Backoff, drained: usize) { + let client = client(backoff); let mut server = moq_tokio::listen::Config::default(); server.tcp.bind = Some("127.0.0.1:0".parse().expect("parse addr")); @@ -356,17 +373,17 @@ async fn a_draining_peer_does_not_exhaust_the_retry_window() { let _task = tokio::spawn(async move { let origin = moq_tokio::origin::spawn(); // Held, as a draining server does: each one closes when the client leaves it. - let mut drained = Vec::new(); + let mut held = Vec::new(); while let Some(request) = server.accept().await { let Ok(session) = request.with_publisher(&origin).ok().await else { continue; }; - if drained.len() < DRAINED { + if held.len() < drained { session .drain() .send(moq_net::goaway::Goaway::new().with_timeout(Duration::from_secs(30))) .expect("send goaway"); - drained.push(session); + held.push(session); } else { let _ = served_tx.send(session); }