Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion quest/m1/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,6 @@ lands before the cut.
- [A/V clock](/quest/m1/av-clock.md) - the audio playhead drives Sync.reference while audio plays, through per-track sync handles
- [Publish channel count](/quest/m1/publish-audio-channel-count.md) - forcing a channel count on an Audio.Capture stops costing the subscriber gaps of silence, and the media fixture's cold-start audio-tone flake is fixed at its cause
- [Relay signal install](/quest/m1/relay-signal-install.md) - a shutdown signal after the relay starts serving drains it, without leaking handlers into an embedder whose startup failed
- [Rust redirect loops](/quest/m1/tokio-redirect-loop.md) - moq-tokio escalates its backoff when GOAWAY redirects bounce between peers, as JS does
- [JS redirect keys strip userinfo](/quest/m1/js-reload-userinfo.md) - js/net keys a reconnect endpoint without URL userinfo, matching Rust
- [Container reads drop backfill](/quest/m1/read-backfill.md) - the JS container consumer drops groups below its read cursor like Rust
- [Rust request credit](/quest/m1/rs-request-credit.md) - a moq-net request fails at once without stream credit instead of waiting
Expand Down
4 changes: 0 additions & 4 deletions quest/m1/js-reload-userinfo.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,3 @@ userinfo; match it exactly, and test both a userinfo URL and a redirect loop
that differs only in userinfo.

Public API: none. Wire: none.

## Required

- [Rust redirect loops](/quest/m1/tokio-redirect-loop.md) - lands the Rust rule this mirrors
41 changes: 0 additions & 41 deletions quest/m1/tokio-redirect-loop.md

This file was deleted.

165 changes: 153 additions & 12 deletions rs/moq-tokio/src/connection.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,11 @@ fn settles_walk(err: &Error) -> bool {
/// A GOAWAY naming no new URI is no failure: a restarting peer sends one to each session until
/// it goes, and each of those
/// serves, so it restarts the timeout instead.
///
/// A GOAWAY off a session that outlived `initial` migrates at once, unless it sends the
/// connection back to an endpoint that already redirected it elsewhere and the session lived
/// less than `max`. Two peers redirecting to each other thus wait out this backoff and count
/// against the timeout.
#[derive(Clone, Debug, usage_rs::Args, serde::Serialize, serde::Deserialize)]
#[usage(unknown_flags = "error", args_override_self = false)]
#[serde(default, deny_unknown_fields)]
Expand Down Expand Up @@ -119,9 +124,9 @@ pub struct Backoff {
/// [`crate::connect::Config::timeout`](crate::connect::Config::timeout), so a handshake
/// slower than this still lands. Giving up can therefore take up to one whole walk longer.
///
/// Resets after a stable connection (one that outlives the initial backoff), so a flapping
/// session that reconnects then immediately drops still counts toward the timeout. Set to
/// `Some(Duration::ZERO)` for unlimited retries. `None` takes the default. When this config was
/// Resets after a stable connection (one that outlives the initial backoff) outside a redirect
/// loop, so a flapping session that reconnects then immediately drops still counts toward the
/// timeout. Set to `Some(Duration::ZERO)` for unlimited retries. `None` takes the default. When this config was
/// parsed from the command line, a `--backoff-timeout` given there wins over this field.
///
/// An `Option` because zero is a real setting: a command-line parse leaves this field at its
Expand Down Expand Up @@ -850,6 +855,8 @@ impl Connection {
let mut addrs = addrs;
// An old session kept alive after a GOAWAY so its in-flight groups finish.
let mut draining: Option<Draining> = None;
// The current run of migrations, to tell a redirect loop from a healthy handover.
let mut redirects = Redirects::default();

loop {
if !timeout.is_zero() && retry.start.elapsed() >= timeout {
Expand Down Expand Up @@ -878,9 +885,7 @@ impl Connection {
shared.retrying(err);
}

// A session that stayed up past the initial backoff is healthy; one that
// ended sooner counts as a failed attempt however it ended.
let healthy = retry.settle(connected.elapsed(), initial);
let tenure = connected.elapsed();

// The connected target owns the policy, including in one-shot mode. A
// refused redirect is terminal: the peer is leaving and named somewhere we
Expand Down Expand Up @@ -909,6 +914,28 @@ impl Connection {
return Ok(());
}

// Two peers that each outlive the initial backoff can still bounce us between
// them forever. Returning to an endpoint that already sent us elsewhere in this
// run of migrations skips the healthy-session reset, so the backoff escalates and
// the retry window still applies. A one-way rolling restart never revisits one,
// and a GOAWAY back to the same place is a restart, not a move, so it never
// counts. A session that outlived the capped delay already redials no faster
// than the backoff would allow, so it starts a new run: rolling restarts days
// apart never add up to a loop.
if matches!(ended, Ended::Closed(_)) || tenure >= pacing.max {
redirects.reset();
}
// Only an accepted redirect is a move: an empty or ignored one reconnects in place.
let looped = assigned.as_ref().is_some_and(|target| {
let from = endpoint(&url);
let to = endpoint(target);
to != from && redirects.depart(from, &to)
});

// A session that stayed up past the initial backoff is healthy; one that
// ended sooner, or carried on a redirect loop, counts as a failed attempt.
let healthy = !looped && retry.settle(tenure, initial);

if let Ended::Goaway(msg) = &ended {
// An accepted redirect is an assignment: keep dialing it from here on, and
// only it. The peer named exactly one place to go, which retires
Expand Down Expand Up @@ -941,16 +968,22 @@ impl Connection {
continue;
}

// Drained almost immediately. The backoff still escalates, so the redials
// cannot hammer the peer, and the old session serves across the sleep.
// Drained almost immediately, or looped back. The backoff still escalates, so
// the redials cannot hammer the peer, and the old session serves across the
// sleep.
//
// An empty URI is "reconnect to me": the peer is restarting and drains
// each new session until it goes, which is no failure, since every one
// serves. A named URI is a redirect, scored as a failed attempt so two
// peers bouncing us between them eventually give up.
let restarting = msg.uri().is_empty();
if !restarting {
let err = Error::Reconnect("peer redirected immediately".to_string());
let reason = if looped {
"redirect loop"
} else {
"peer redirected immediately"
};
let err = Error::Reconnect(reason.to_string());
shared.retrying(&err);
retry.last_error = Some(err);
}
Expand All @@ -960,7 +993,11 @@ impl Connection {
let Some(wait) = retry_wait(retry.delay, retry.start, window) else {
return Err(timeout_error(timeout, retry.last_error.as_ref()));
};
tracing::warn!(peer = %Endpoint(&url), ?wait, "peer drained the session immediately; retrying after backoff");
if looped {
tracing::warn!(peer = %Endpoint(&url), ?wait, "peer redirected back in a loop; retrying after backoff");
} else {
tracing::warn!(peer = %Endpoint(&url), ?wait, "peer drained the session immediately; retrying after backoff");
}
// Keep the handover bounded across the sleep: nothing else polls the
// predecessor while the loop is between connections.
sleep_draining(wait, &mut draining, shared).await;
Expand Down Expand Up @@ -1002,8 +1039,7 @@ impl Connection {
// resets, or redirects us straight back out). Treat it as a failed
// connection: keep the reason so the give-up timeout reports a real cause,
// and fall through to the shared backoff sleep below so repeated flaps
// escalate instead of spinning the CPU. This is what bounds a redirect
// loop between two peers.
// escalate instead of spinning the CPU.
let err = match ended {
Ended::Closed(Err(err)) => Some(err),
Ended::Closed(Ok(())) => None,
Expand Down Expand Up @@ -1324,6 +1360,62 @@ fn retry_wait(delay: Duration, retry_start: tokio::time::Instant, timeout: Durat
Some(wait.min(timeout.checked_sub(retry_start.elapsed())?))
}

/// Where `url` dials, without its userinfo, query, or fragment, where a peer may mint fresh
/// credentials on every redirect. Two URLs that differ only there reach the same endpoint.
fn endpoint(url: &Url) -> Url {
let mut bare = url.clone();
// Only fails for a URL that cannot carry userinfo, which then has none to strip.
let _ = bare.set_username("");
let _ = bare.set_password(None);
bare.set_query(None);
bare.set_fragment(None);
bare
Comment on lines +1365 to +1372

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Strip non-dial credentials from redirect endpoint keys

For host-based transports, this retains URL userinfo and path even though userinfo is ignored by dialing and paths may carry rotating credentials. A bouncing pair can therefore return redirects such as tcp://token-N@peer/ on every healthy session; each key remains unique, Redirects::depart never detects the revisit, and the retry timeout keeps resetting, recreating the unbounded redirect loop this change is intended to stop. Build the comparison key from the actual transport endpoint (scheme, host, and effective port), preserving the path only for hostless transports such as Unix sockets.

AGENTS.md reference: AGENTS.md:L18-L18

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Userinfo: fixed in 259e604. It never reaches a dial, so endpoint now strips it along with the query and fragment, and the test covers it.

Path: kept, as in JS (moq-dev/moq#6115). For host-based transports the path is the request target (the WebTransport request, or the SETUP path for moqt/moql/tcp/tls), so relay/a and relay/b are different requests rather than credentials. A hostile peer that wants to dodge detection can already rotate ports under the default same-host policy, so no key catches every evasion. What bounds such a peer is that each hop needs a session outliving backoff.initial. Loop detection is there for fleets that bounce clients by accident, not as a bound on hostile peers.

(Written by Claude Opus 5.5)

}

/// A run of GOAWAY redirects, and whether it has revisited an endpoint.
///
/// The peer picks every hop, so remembering them all would grow without bound. Brent's cycle
/// detection instead keeps one departed endpoint, replaced at doubling intervals, and catches a
/// cycle of any length within a few laps of it. Once caught, the run stays a loop until reset.
struct Redirects {
mark: Option<Url>,
span: u64,
hops: u64,
looped: bool,
}

impl Default for Redirects {
fn default() -> Self {
Self {
mark: None,
span: 1,
hops: 0,
looped: false,
}
}
}

impl Redirects {
/// Record leaving `from` for a different endpoint `to`, and return whether this run loops.
fn depart(&mut self, from: Url, to: &Url) -> bool {
if self.mark.as_ref() == Some(to) {
self.looped = true;
}
self.hops += 1;
if self.hops == self.span {
self.mark = Some(from);
self.span = self.span.saturating_mul(2);
self.hops = 0;
}
self.looped
}

/// Start a new run.
fn reset(&mut self) {
*self = Self::default();
}
}

/// Why a session stopped being the live one.
enum Ended {
/// The transport closed.
Expand Down Expand Up @@ -2735,6 +2827,55 @@ mod tests {
"a cached resolve redialed the drained relay"
);
}

#[test]
fn redirect_loop_detection_catches_any_cycle_in_constant_memory() {
for tail in 0..4 {
for cycle in 2..=8 {
// `tail` one-way moves, then `cycle` endpoints redirecting around a ring.
let at = |hop: usize| {
if hop < tail {
url(&format!("tcp://tail{hop}/"))
} else {
url(&format!("tcp://ring{}/", (hop - tail) % cycle))
}
};
let mut redirects = Redirects::default();
let mut caught = None;
for hop in 0..4 * (tail + cycle) {
let looped = redirects.depart(at(hop), &at(hop + 1));
match caught {
Some(_) => assert!(looped, "a caught loop stays caught"),
None if looped => caught = Some(hop),
None => {}
}
}
let caught = caught.unwrap_or_else(|| panic!("tail {tail}, cycle {cycle}: never caught"));
// Never before the run first returns to an endpoint it left.
assert!(
caught + 1 >= tail + cycle,
"tail {tail}, cycle {cycle}: caught at {caught}"
);
}
}

let mut redirects = Redirects::default();
for hop in 0..1000 {
let from = url(&format!("tcp://a{hop}/"));
let to = url(&format!("tcp://a{}/", hop + 1));
assert!(!redirects.depart(from, &to), "a one-way run never loops");
}
}

#[test]
fn an_endpoint_ignores_the_credentials() {
assert_eq!(
endpoint(&url("tcp://token-a:secret@relay:1/path?jwt=a#x")),
endpoint(&url("tcp://token-b@relay:1/path?jwt=b"))
);
// The path is part of the request target, so it names a different endpoint.
assert_ne!(endpoint(&url("tcp://relay:1/a")), endpoint(&url("tcp://relay:1/b")));
}
}

#[cfg(test)]
Expand Down
Loading
Loading