Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 86 additions & 0 deletions rs/moq-relay/tests/shutdown_signal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@
//! The drain ends as soon as every session has left rather than waiting out the
//! window, which a stop-time budget depends on.
//!
//! A reconnecting client rides out a restart on the same address: the GOAWAYs
//! the drain sends its redials must not exhaust its retry window.
//!
//! A raised signal reaches every listener in the process, and a relay exits at
//! once on its second one, so the tests that raise SIGINT hold [`SIGINT`]:
//! `cargo test` (the Nix check phase) runs them as threads of one process.
Expand Down Expand Up @@ -88,6 +91,11 @@ fn a_trigger_before_run_keeps_the_deadline() {
run_test(a_trigger_before_run_keeps_the_deadline_inner);
}

#[test]
fn a_reconnecting_client_rides_out_a_restart() {
run_test(a_reconnecting_client_rides_out_a_restart_inner);
}

async fn sigint_drains_sessions_before_exiting_inner() {
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();

Expand Down Expand Up @@ -314,6 +322,84 @@ async fn a_trigger_before_run_keeps_the_deadline_inner() {
);
}

/// A restart on the same address, with a reconnecting publisher whose retry window is a third of
/// the drain. The drain sends each of its redials an empty-URI GOAWAY until the window closes.
/// Every one of those sessions served, so none counts against the retry window, and the
/// publisher is still there to land on the relay that takes the address over.
async fn a_reconnecting_client_rides_out_a_restart_inner() {
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();

let relay = Relay::load(relay_config())
.await
.expect("load relay")
.with_signals(false);
let addr = relay.tcp_addr().expect("TCP listener bound");
let trigger = relay.shutdown_trigger().clone();
let run = tokio::spawn(relay.run());
let url: url::Url = format!("tcp://{addr}/").parse().expect("parse url");

let origin = moq_tokio::origin::spawn();
let broadcast = origin.create_broadcast("cam").expect("create broadcast");
broadcast.announce(Default::default()).expect("announce");
let mut publisher = moq_tokio::connect::Config::default();
// Redials paced well inside the drain, under a window that ends long before it.
publisher.backoff.initial = Duration::from_millis(200);
publisher.backoff.max = Duration::from_millis(200);
publisher.backoff.timeout = Some(DRAIN_TIMEOUT / 3);
let publisher = publisher
.init(Default::default())
.expect("client init")
.with_publisher(&origin)
.connect(url.clone())
.established()
.await
.expect("connect");

// Holds the drain open for its whole window, as the rest of a busy relay's sessions would.
let straggler = straggler(addr.port()).await;

trigger.start();
tokio::time::timeout(Duration::from_secs(15), run)
.await
.expect("relay never exited after the drain window")
.expect("relay task panicked")
.expect("relay exited with an error");
drop(straggler);

let mut config = relay_config();
config.listen.tcp.bind = Some(addr);
let relay = Relay::load(config)
.await
.expect("restart on the same address")
.with_signals(false);
let trigger = relay.shutdown_trigger().clone();
let run = tokio::spawn(relay.run());

let subscriber = moq_tokio::origin::spawn();
let consumer = subscriber.consume();
let observer = client(Vec::new())
.with_subscriber(subscriber)
.connect(url)
.established()
.await
.expect("connect");
tokio::select! {
res = consumer.routed_broadcast("cam") => {
res.expect("the publisher's broadcast never resolved");
}
res = publisher.closed() => panic!("the publisher gave up during the restart: {res:?}"),
_ = tokio::time::sleep(Duration::from_secs(15)) => panic!("the publisher never reached the new relay"),
}

drop((publisher, observer));
trigger.start();
tokio::time::timeout(Duration::from_secs(15), run)
.await
.expect("relay never exited after its sessions left")
.expect("relay task panicked")
.expect("relay exited with an error");
}

/// A one-shot client offering `version` (every version when empty): a
/// reconnecting one would migrate on the GOAWAY and hide whether the relay
/// closed the original session.
Expand Down
32 changes: 24 additions & 8 deletions rs/moq-tokio/src/connection.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,9 @@ fn settles_walk(err: &Error) -> bool {
///
/// The delays carry jitter, so a fleet
/// knocked offline together does not reconnect in lockstep. The timeout bounds every failure;
/// a settled rejection or invalid local configuration stops retrying.
/// a settled rejection or invalid local configuration stops retrying. A GOAWAY naming no new URI
/// is no failure: a restarting peer sends one to each session until it goes, and each of those
/// serves, so it restarts the timeout instead.
#[derive(Clone, Debug, usage::Args, serde::Serialize, serde::Deserialize)]
#[usage(unknown_flags = "error", args_override_self = false)]
#[serde(default, deny_unknown_fields)]
Expand Down Expand Up @@ -971,19 +973,33 @@ impl Connection {
continue;
}

// Redirected almost immediately. Still follow it, but score it as a
// failed attempt so two peers bouncing us between them escalate
// through backoff and eventually give up. The old session serves
// across the sleep, so the redirect loop costs time, not data.
retry.last_error = Some(Error::Reconnect("peer redirected immediately".to_string()));
let Some(wait) = retry_wait(retry.delay, retry.start, timeout) else {
// Drained almost immediately. The backoff still escalates, so the redials
// cannot hammer the peer, and the old session serves across the sleep.
//
// An empty URI is "reconnect to me": the peer is restarting and drains
// each new session until it goes, which is no failure, since every one
// serves. A named URI is a redirect, scored as a failed attempt so two
// peers bouncing us between them eventually give up.
let restarting = msg.uri().is_empty();
if !restarting {
retry.last_error = Some(Error::Reconnect("peer redirected immediately".to_string()));
}
// A restart's pacing is not capped by the window: a delay longer than the
// window would otherwise spend it all and end the loop without a redial.
let window = if restarting { Duration::ZERO } else { timeout };
let Some(wait) = retry_wait(retry.delay, retry.start, window) else {
return Err(timeout_error(timeout, retry.last_error.as_ref()));
};
tracing::warn!(peer = %Endpoint(&url), ?wait, "peer redirected immediately; retrying after backoff");
tracing::warn!(peer = %Endpoint(&url), ?wait, "peer drained the session immediately; retrying after backoff");
// Keep the handover bounded across the sleep: nothing else polls the
// predecessor while the loop is between connections.
sleep_draining(wait, &mut draining, shared).await;
retry.delay = pacing.next(retry.delay);
if restarting {
// After the sleep, so the window counts only the time after the
// peer stops answering.
retry.start = tokio::time::Instant::now();
}
continue;
}

Expand Down
76 changes: 76 additions & 0 deletions rs/moq-tokio/tests/reconnect.rs
Original file line number Diff line number Diff line change
Expand Up @@ -325,3 +325,79 @@ async fn an_empty_goaway_preserves_configured_fallbacks() {
.expect("configured fallback was discarded")
.expect("server B stopped accepting");
}

/// A graceful restart: the server keeps accepting while it drains, and sends every new session
/// an empty-URI GOAWAY at once, for longer than the retry window. Each of those sessions serves
/// until the client moves on, so none is a failure, and the client is still there to land on
/// the first session the server lets stay.
#[tokio::test]
async fn a_draining_peer_does_not_exhaust_the_retry_window() {
// Each redial waits at least half of its 200ms delay, so the drained sessions alone span
// more than twice the window. A GOAWAY arriving inside the 200ms health bar is what makes a
// session count as drained at once rather than as a healthy session that later migrated.
let mut backoff = moq_tokio::Backoff::default();
backoff.initial = Duration::from_millis(200);
backoff.max = Duration::from_millis(200);
backoff.timeout = Some(Duration::from_millis(500));
rides_out_a_drain(backoff, 12).await;
}

/// A restart's pacing sleep can be longer than the retry window. It must not spend the window,
/// or the loop would give up right after the sleep without redialing the restarting peer.
#[tokio::test]
async fn a_drain_backoff_longer_than_the_retry_window_still_redials() {
// From the second drained session on, the delay is 400ms against a 100ms window.
let mut backoff = moq_tokio::Backoff::default();
backoff.initial = Duration::from_millis(200);
backoff.max = Duration::from_millis(400);
backoff.timeout = Some(Duration::from_millis(100));
rides_out_a_drain(backoff, 4).await;
}

/// Connect with `backoff` to a server that sends its first `drained` sessions an empty-URI
/// GOAWAY at once, and assert the client is still there to land on the session after them.
async fn rides_out_a_drain(backoff: moq_tokio::Backoff, drained: usize) {
let client = client(backoff);

let mut server = moq_tokio::listen::Config::default();
server.tcp.bind = Some("127.0.0.1:0".parse().expect("parse addr"));
let mut server = server
.init(Default::default())
.expect("init server")
.listen()
.await
.expect("bind tcp listener");
let port = server.tcp_local_addr().expect("tcp listener bound").port();

let (served_tx, mut served) = tokio::sync::mpsc::unbounded_channel();
let _task = tokio::spawn(async move {
let origin = moq_tokio::origin::spawn();
// Held, as a draining server does: each one closes when the client leaves it.
let mut held = Vec::new();
while let Some(request) = server.accept().await {
let Ok(session) = request.with_publisher(&origin).ok().await else {
continue;
};
if held.len() < drained {
session
.drain()
.send(moq_net::goaway::Goaway::new().with_timeout(Duration::from_secs(30)))
.expect("send goaway");
held.push(session);
} else {
let _ = served_tx.send(session);
}
}
});

let url: url::Url = format!("tcp://127.0.0.1:{port}/").parse().expect("parse url");
let connection = client.connect(url);

tokio::select! {
session = served.recv() => {
session.expect("server stopped accepting");
}
res = connection.closed() => panic!("gave up during the drain: {res:?}"),
_ = tokio::time::sleep(Duration::from_secs(30)) => panic!("never landed after the drain"),
}
}
Loading