diff --git a/rs/moq-relay/tests/shutdown_signal.rs b/rs/moq-relay/tests/shutdown_signal.rs index 1b25078cf41..b295cee1899 100644 --- a/rs/moq-relay/tests/shutdown_signal.rs +++ b/rs/moq-relay/tests/shutdown_signal.rs @@ -20,6 +20,9 @@ //! The drain ends as soon as every session has left rather than waiting out the //! window, which a stop-time budget depends on. //! +//! A reconnecting client rides out a restart on the same address: the GOAWAYs +//! the drain sends its redials must not exhaust its retry window. +//! //! A raised signal reaches every listener in the process, and a relay exits at //! once on its second one, so the tests that raise SIGINT hold [`SIGINT`]: //! `cargo test` (the Nix check phase) runs them as threads of one process. @@ -88,6 +91,11 @@ fn a_trigger_before_run_keeps_the_deadline() { run_test(a_trigger_before_run_keeps_the_deadline_inner); } +#[test] +fn a_reconnecting_client_rides_out_a_restart() { + run_test(a_reconnecting_client_rides_out_a_restart_inner); +} + async fn sigint_drains_sessions_before_exiting_inner() { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); @@ -314,6 +322,84 @@ async fn a_trigger_before_run_keeps_the_deadline_inner() { ); } +/// A restart on the same address, with a reconnecting publisher whose retry window is a third of +/// the drain. The drain sends each of its redials an empty-URI GOAWAY until the window closes. +/// Every one of those sessions served, so none counts against the retry window, and the +/// publisher is still there to land on the relay that takes the address over. +async fn a_reconnecting_client_rides_out_a_restart_inner() { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let relay = Relay::load(relay_config()) + .await + .expect("load relay") + .with_signals(false); + let addr = relay.tcp_addr().expect("TCP listener bound"); + let trigger = relay.shutdown_trigger().clone(); + let run = tokio::spawn(relay.run()); + let url: url::Url = format!("tcp://{addr}/").parse().expect("parse url"); + + let origin = moq_tokio::origin::spawn(); + let broadcast = origin.create_broadcast("cam").expect("create broadcast"); + broadcast.announce(Default::default()).expect("announce"); + let mut publisher = moq_tokio::connect::Config::default(); + // Redials paced well inside the drain, under a window that ends long before it. + publisher.backoff.initial = Duration::from_millis(200); + publisher.backoff.max = Duration::from_millis(200); + publisher.backoff.timeout = Some(DRAIN_TIMEOUT / 3); + let publisher = publisher + .init(Default::default()) + .expect("client init") + .with_publisher(&origin) + .connect(url.clone()) + .established() + .await + .expect("connect"); + + // Holds the drain open for its whole window, as the rest of a busy relay's sessions would. + let straggler = straggler(addr.port()).await; + + trigger.start(); + tokio::time::timeout(Duration::from_secs(15), run) + .await + .expect("relay never exited after the drain window") + .expect("relay task panicked") + .expect("relay exited with an error"); + drop(straggler); + + let mut config = relay_config(); + config.listen.tcp.bind = Some(addr); + let relay = Relay::load(config) + .await + .expect("restart on the same address") + .with_signals(false); + let trigger = relay.shutdown_trigger().clone(); + let run = tokio::spawn(relay.run()); + + let subscriber = moq_tokio::origin::spawn(); + let consumer = subscriber.consume(); + let observer = client(Vec::new()) + .with_subscriber(subscriber) + .connect(url) + .established() + .await + .expect("connect"); + tokio::select! { + res = consumer.routed_broadcast("cam") => { + res.expect("the publisher's broadcast never resolved"); + } + res = publisher.closed() => panic!("the publisher gave up during the restart: {res:?}"), + _ = tokio::time::sleep(Duration::from_secs(15)) => panic!("the publisher never reached the new relay"), + } + + drop((publisher, observer)); + trigger.start(); + tokio::time::timeout(Duration::from_secs(15), run) + .await + .expect("relay never exited after its sessions left") + .expect("relay task panicked") + .expect("relay exited with an error"); +} + /// A one-shot client offering `version` (every version when empty): a /// reconnecting one would migrate on the GOAWAY and hide whether the relay /// closed the original session. diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index 68cef08755d..88e59188ab0 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -50,7 +50,9 @@ fn settles_walk(err: &Error) -> bool { /// /// The delays carry jitter, so a fleet /// knocked offline together does not reconnect in lockstep. The timeout bounds every failure; -/// a settled rejection or invalid local configuration stops retrying. +/// a settled rejection or invalid local configuration stops retrying. A GOAWAY naming no new URI +/// is no failure: a restarting peer sends one to each session until it goes, and each of those +/// serves, so it restarts the timeout instead. #[derive(Clone, Debug, usage::Args, serde::Serialize, serde::Deserialize)] #[usage(unknown_flags = "error", args_override_self = false)] #[serde(default, deny_unknown_fields)] @@ -971,19 +973,33 @@ impl Connection { continue; } - // Redirected almost immediately. Still follow it, but score it as a - // failed attempt so two peers bouncing us between them escalate - // through backoff and eventually give up. The old session serves - // across the sleep, so the redirect loop costs time, not data. - retry.last_error = Some(Error::Reconnect("peer redirected immediately".to_string())); - let Some(wait) = retry_wait(retry.delay, retry.start, timeout) else { + // Drained almost immediately. The backoff still escalates, so the redials + // cannot hammer the peer, and the old session serves across the sleep. + // + // An empty URI is "reconnect to me": the peer is restarting and drains + // each new session until it goes, which is no failure, since every one + // serves. A named URI is a redirect, scored as a failed attempt so two + // peers bouncing us between them eventually give up. + let restarting = msg.uri().is_empty(); + if !restarting { + retry.last_error = Some(Error::Reconnect("peer redirected immediately".to_string())); + } + // A restart's pacing is not capped by the window: a delay longer than the + // window would otherwise spend it all and end the loop without a redial. + let window = if restarting { Duration::ZERO } else { timeout }; + let Some(wait) = retry_wait(retry.delay, retry.start, window) else { return Err(timeout_error(timeout, retry.last_error.as_ref())); }; - tracing::warn!(peer = %Endpoint(&url), ?wait, "peer redirected immediately; retrying after backoff"); + tracing::warn!(peer = %Endpoint(&url), ?wait, "peer drained the session immediately; retrying after backoff"); // Keep the handover bounded across the sleep: nothing else polls the // predecessor while the loop is between connections. sleep_draining(wait, &mut draining, shared).await; retry.delay = pacing.next(retry.delay); + if restarting { + // After the sleep, so the window counts only the time after the + // peer stops answering. + retry.start = tokio::time::Instant::now(); + } continue; } diff --git a/rs/moq-tokio/tests/reconnect.rs b/rs/moq-tokio/tests/reconnect.rs index b0766962058..51a96c48343 100644 --- a/rs/moq-tokio/tests/reconnect.rs +++ b/rs/moq-tokio/tests/reconnect.rs @@ -325,3 +325,79 @@ async fn an_empty_goaway_preserves_configured_fallbacks() { .expect("configured fallback was discarded") .expect("server B stopped accepting"); } + +/// A graceful restart: the server keeps accepting while it drains, and sends every new session +/// an empty-URI GOAWAY at once, for longer than the retry window. Each of those sessions serves +/// until the client moves on, so none is a failure, and the client is still there to land on +/// the first session the server lets stay. +#[tokio::test] +async fn a_draining_peer_does_not_exhaust_the_retry_window() { + // Each redial waits at least half of its 200ms delay, so the drained sessions alone span + // more than twice the window. A GOAWAY arriving inside the 200ms health bar is what makes a + // session count as drained at once rather than as a healthy session that later migrated. + let mut backoff = moq_tokio::Backoff::default(); + backoff.initial = Duration::from_millis(200); + backoff.max = Duration::from_millis(200); + backoff.timeout = Some(Duration::from_millis(500)); + rides_out_a_drain(backoff, 12).await; +} + +/// A restart's pacing sleep can be longer than the retry window. It must not spend the window, +/// or the loop would give up right after the sleep without redialing the restarting peer. +#[tokio::test] +async fn a_drain_backoff_longer_than_the_retry_window_still_redials() { + // From the second drained session on, the delay is 400ms against a 100ms window. + let mut backoff = moq_tokio::Backoff::default(); + backoff.initial = Duration::from_millis(200); + backoff.max = Duration::from_millis(400); + backoff.timeout = Some(Duration::from_millis(100)); + rides_out_a_drain(backoff, 4).await; +} + +/// Connect with `backoff` to a server that sends its first `drained` sessions an empty-URI +/// GOAWAY at once, and assert the client is still there to land on the session after them. +async fn rides_out_a_drain(backoff: moq_tokio::Backoff, drained: usize) { + let client = client(backoff); + + let mut server = moq_tokio::listen::Config::default(); + server.tcp.bind = Some("127.0.0.1:0".parse().expect("parse addr")); + let mut server = server + .init(Default::default()) + .expect("init server") + .listen() + .await + .expect("bind tcp listener"); + let port = server.tcp_local_addr().expect("tcp listener bound").port(); + + let (served_tx, mut served) = tokio::sync::mpsc::unbounded_channel(); + let _task = tokio::spawn(async move { + let origin = moq_tokio::origin::spawn(); + // Held, as a draining server does: each one closes when the client leaves it. + let mut held = Vec::new(); + while let Some(request) = server.accept().await { + let Ok(session) = request.with_publisher(&origin).ok().await else { + continue; + }; + if held.len() < drained { + session + .drain() + .send(moq_net::goaway::Goaway::new().with_timeout(Duration::from_secs(30))) + .expect("send goaway"); + held.push(session); + } else { + let _ = served_tx.send(session); + } + } + }); + + let url: url::Url = format!("tcp://127.0.0.1:{port}/").parse().expect("parse url"); + let connection = client.connect(url); + + tokio::select! { + session = served.recv() => { + session.expect("server stopped accepting"); + } + res = connection.closed() => panic!("gave up during the drain: {res:?}"), + _ = tokio::time::sleep(Duration::from_secs(30)) => panic!("never landed after the drain"), + } +}