From 5c3ad00c677bf140547892d786284a6735e65f5c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 04:33:57 -0800 Subject: [PATCH 1/3] chore(quest): claim quest/m1/quic-handshake-loss.md Quest-Claim: b40fd7b6dd8752cd From 8c5ef1d94c4de307831b3eab6d88296715629b86 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 06:09:28 -0800 Subject: [PATCH 2/3] fix(moq-quic): the server finishes handshakes under heavy loss handshake_timeout defaults to the 30 s idle timeout, so a client whose probes back off past 10 s under loss is not abandoned. A server still on the client's initial CID switches to sequence 1 only, instead of jumping to whichever NEW_CONNECTION_ID arrives first and retiring the CIDs it skipped while they are still in flight, which quiche's client fails on. Co-Authored-By: Claude Opus 5.5 --- quest/m1/quic-handshake-loss.md | 50 ---------- quest/m1/relay-accept-deadline.md | 29 ++++++ quest/m4/quic-handshake-speedup.md | 41 ++++++++ rs/moq-quic/README.md | 3 +- rs/moq-quic/src/cid_queue.rs | 36 +++++++ rs/moq-quic/src/config/transport/mod.rs | 7 +- rs/moq-quic/src/connection/mod.rs | 13 ++- rs/moq-quic/src/tests/mod.rs | 2 +- rs/moq-quic/src/tests/sim.rs | 110 ++++++++++++++++++++- rs/moq-uring/tests/integration/endpoint.rs | 2 +- test/quic-interop/cases.toml | 37 +------ 11 files changed, 231 insertions(+), 99 deletions(-) delete mode 100644 quest/m1/quic-handshake-loss.md create mode 100644 quest/m1/relay-accept-deadline.md create mode 100644 quest/m4/quic-handshake-speedup.md diff --git a/quest/m1/quic-handshake-loss.md b/quest/m1/quic-handshake-loss.md deleted file mode 100644 index 3e39f477a34..00000000000 --- a/quest/m1/quic-handshake-loss.md +++ /dev/null @@ -1,50 +0,0 @@ -# [S] moq-quic's server finishes handshakes under heavy loss - -## Goal - -A moq-quic server completes a handshake that a mature client keeps -retrying under heavy loss or corruption, so a relay does not shed clients on -bad networks that other servers serve. The QUIC Interop Runner's -`handshakeloss` and `handshakecorruption` cases pass with quinn's, -quiche's, and moq-quic's own clients against moq-quic's server, and their -`known` entries in `test/quic-interop/cases.toml` are gone. - -## Plan - -Found by [the QUIC Interop Runner quest](https://github.com/moq-dev/moq/pull/8708), -which saw two causes in its first matrix: - -- **The handshake inactivity bound is shorter than a client's backoff.** - `handshake_timeout` (`rs/moq-quic/src/config/transport/mod.rs`, 10 s, as - msquic's `HandshakeIdleTimeoutMs`) abandons a handshake that has heard - nothing for 10 s. Under the runner's 30% loss, quinn's client backed its - probes off to gaps of 13.2 s and 12.9 s, and the server logged - "handshake timed out" three times in one run. The self pair hits it too: - on a later run, both moq-quic endpoints gave up 10 s after their last - progress. Decided 2026-11-14 (maintainer, /quest-complete): the bound - becomes the idle timeout, 30 s, since a half-open handshake already costs - no more than an idle connection (rejected: keep 10 s, leave it open, drop - the quest). -- **The server burns connection IDs it never used.** On a reordered - NEW_CONNECTION_ID (sequence 4 before 1 to 3), a server still on sequence 0 - switches at once (`update_rem_cid` where `active_seq() == 0` in - `rs/moq-quic/src/connection/mod.rs`, inherited from quinn), jumping to 4 - and retiring 0 to 3, then retiring 3, 2, and 1 again as they arrive. That - is legal but spends the client's spare IDs, and quiche's client then - fails with `InvalidState` and closes with INTERNAL_ERROR. Switch to the - lowest unused sequence, and retire only the one left. -- Cover both on the seeded sim under handshake loss, then rerun - `just test quic-interop --peers quinn,quiche --cases handshakeloss,handshakecorruption --roles server` - until it is stable, and drop the `known` entries. - -The public matrix on shows quinn's and quiche's -clients failing these cases against several other servers too, so part of -the flakiness is theirs. A cell that still fails after both fixes needs its -cause shown in the pcap before it gets a `known` entry back. - -Public API: none, unless the bound's default changes, which is a documented -behavior change of `handshake_timeout`. Wire: none. - -## Related - -- [Public interop listing](/quest/m4/quic-interop-listing.md) - these cells are public there diff --git a/quest/m1/relay-accept-deadline.md b/quest/m1/relay-accept-deadline.md new file mode 100644 index 00000000000..a75094458e5 --- /dev/null +++ b/quest/m1/relay-accept-deadline.md @@ -0,0 +1,29 @@ +# [S] A relay's accept deadline outlasts a client's handshake backoff + +## Goal + +A relay keeps a client on a bad network long enough to finish its QUIC +handshake, as moq-quic's own handshake timeout now does, so `--listen-timeout` +does not shed the clients that timeout serves. + +## Plan + +Found by [the handshake loss quest](https://github.com/moq-dev/moq/pull/8824), +which raised moq-quic's `handshake_timeout` to 30 s, an inactivity bound. +`moq_net::server::DEFAULT_TIMEOUT` (10 s), which `moq-tokio`'s +`listen.timeout` and so the relay's `--listen-timeout` default to, is a total +deadline from the connection's arrival through the SETUP reply, and it covers +the QUIC handshake too. Under the QUIC Interop Runner's 30% loss a client's +probe gaps alone reach 13 s, so this deadline still drops it. + +Recommendation: raise the default to 30 s, the same as `moq-tokio`'s connect +timeout on the client side, and update `doc/bin/relay/config.md`, +`doc/bin/cli.md`, and `doc/lib/rs/moq-net.md`. Each half-open session then +holds its slot up to three times longer, which admission already bounds. + +Public API: the default of `moq_net::Server::with_timeout` and `--listen-timeout`, +a behavior change. Wire: none. + +## Waiting + +- Maintainer: raise the default accept deadline from 10 s to 30 s, or keep 10 s and accept that a relay sheds clients under heavy loss. Changing an admission default trades slowloris exposure for reach, which is the maintainer's policy call. Recommendation: raise it to 30 s. diff --git a/quest/m4/quic-handshake-speedup.md b/quest/m4/quic-handshake-speedup.md new file mode 100644 index 00000000000..a113c8df135 --- /dev/null +++ b/quest/m4/quic-handshake-speedup.md @@ -0,0 +1,41 @@ +# [M] moq-quic re-sends a lost ServerHello before its PTO + +## Goal + +Under heavy loss or corruption, a moq-quic handshake whose ServerHello was +lost recovers within a round trip of the evidence, instead of waiting out +PTOs backed off past 16 s. The QUIC Interop Runner's `handshakecorruption` +case passes for the moq-quic self pair, and its `known` entry in +`test/quic-interop/cases.toml` is gone. + +## Plan + +Found by [the handshake loss quest](https://github.com/moq-dev/moq/pull/8824), +which fixed the handshake timeout and the server's CID switch. The self +pair still failed `handshakecorruption` in one of nine runs afterwards. In the +failing connection's pcap, the server's Handshake flight reached the client +before the ServerHello did, so the client discarded it; its own probes were +PINGs (its ClientHello was already acknowledged), which the server answered +with bare ACKs; and once the ServerHello arrived, the client's anti-deadlock +probe and the server's next Handshake retransmission were both 8 to 16 s out. +The server heard nothing for 30 s and gave up. + +RFC 9002 section 6.2.3 allows sending unacknowledged CRYPTO data before the PTO, +a limited number of times per connection: + +- A client that receives Handshake packets before it has Handshake keys + probes in Initial at once. +- A server that receives an ack-eliciting client Initial while its own + Initial CRYPTO is unacknowledged re-sends that CRYPTO at once, within + the anti-amplification limit. + +Cover it on the seeded sim (`rs/moq-quic/src/tests/sim.rs`, +`handshake_under_loss`) with corruption, and rerun +`just test quic-interop --peers moq-quic --cases handshakeloss,handshakecorruption --roles server` +until it is stable. + +Public API: none. Wire: none. + +## Related + +- [Public interop listing](/quest/m4/quic-interop-listing.md) - this cell is public there diff --git a/rs/moq-quic/README.md b/rs/moq-quic/README.md index 25535325dc6..e4292323b49 100644 --- a/rs/moq-quic/README.md +++ b/rs/moq-quic/README.md @@ -77,8 +77,9 @@ Changes on top of the upstream commit, besides the renames: - Discarding Initial or Handshake keys resets the PTO backoff, as RFC 9002 A.11 does, so a client's Initial backoff no longer delays the retransmission of a lost Finished. Also [moq-dev/noq#32](https://github.com/moq-dev/noq/pull/32); not offered upstream yet. - **Probes on the peer heard again**: a 1-RTT probe that times out with nothing heard from the peer since the packet it probes for went out lets the next packet heard send the next probe at once, rather than leave a path back from an outage idle for the rest of the backoff, which still doubles with each probe (RFC 9002 section 6.2.1), up to 2^16 times its base. A packet acknowledging anything new already resets the backoff; this covers one that does not, as after an outage that began with everything acknowledged. A peer heard while the probes still go unanswered says nothing of their path, so it leaves the backoff alone. Not offered upstream yet. - **Keep-alive stays enabled**: zero uses the derived interval, and a connection with no idle timeout uses the default idle timeout to derive it. `keep_alive_interval` returns a duration. Not offered upstream: MoQ sessions keep quiet connections and their NAT bindings alive. -- `transport::Config::handshake_timeout` (default 10s, as msquic's `HandshakeIdleTimeoutMs`) governs the idle timer until the handshake completes, and the negotiated idle timeout takes over once established. A 2s idle timeout otherwise expires 3x the initial PTO in, just before the second Initial probe, leaving a client two flights. A server also drops an Initial left waiting for `accept` past it, and a deadline too far out to represent never fires instead of panicking. Also [moq-dev/noq#34](https://github.com/moq-dev/noq/pull/34), whose third change (the handshake idle timeout picks noq's idle-based PTO interval cap) has nothing to apply to here, since quinn's PTO backoff has no such cap. +- `transport::Config::handshake_timeout` (default 30s, the idle timeout's default, so a client's probes backed off past 10s under heavy loss still land) governs the idle timer until the handshake completes, and the negotiated idle timeout takes over once established. A 2s idle timeout otherwise expires 3x the initial PTO in, just before the second Initial probe, leaving a client two flights. A server also drops an Initial left waiting for `accept` past it, and a deadline too far out to represent never fires instead of panicking. Also [moq-dev/noq#34](https://github.com/moq-dev/noq/pull/34), whose third change (the handshake idle timeout picks noq's idle-based PTO interval cap) has nothing to apply to here, since quinn's PTO backoff has no such cap. quinn's `handshake_timeout` test in its async layer expects the handshake to end at the idle timeout, so the driver's copy sets both timeouts. +- **In-order CID switch**: a server still on the client's initial CID moves to sequence 1 once it arrives, not to whichever NEW\_CONNECTION\_ID arrives first, since jumping retires the sequences it skipped while they are still in flight, and quiche's client then fails with `InvalidState` re-sending one it was told to retire. - **Time on [`kio::clock`](../kio)**: the crate's `Instant` is `kio::clock::Instant`, so one clock steps every timer, tokio's paused clock and the simulator included. qlog stamps events relative to `QlogConfig::start_time` on that clock rather than the streamer's own `std::time::Instant`. Address-validation tokens and their log read `kio::clock::wall()` instead of quinn's `TimeSource`, for portable 1970-2554 wall times; the simulator steps wall time with its clock, but paused tokio does not, so token expiry there follows the host's wall clock. - **Send groups** (`streams/schedule.rs`, `Connection::send_groups`): streams outside every group go first, in quinn's order; groups are served strictly by priority, with equal priorities sharing bytes by deficit round robin. A stream's priority is an `i64` ordering it only within its group, and `Streams::open` takes a `send::Options { group, priority }` to start it there, like W3C WebTransport's send options. This replaces quinn's single pending-stream heap, so a port touching `PendingStreamsQueue` has to be redone against the schedule. moq-tokio's `quic::driver` and moq-uring's driver wrap a group in a shared `send::Group` handle that closes on drop. - **Scheduled datagrams** (`Datagrams::send`): a datagram is queued by its `send::Options`, in a send group (or ungrouped) at an `i64` priority, and scheduled with the group's streams, winning a tie with a stream of equal priority and spending the group's byte credit, instead of quinn's datagram-first FIFO in `populate_packet`. Sending never blocks: `datagram_send_buffer_size` is shared, and an overflow evicts the oldest datagram of the lowest-priority group. `SendDatagramError::Blocked` and `Event::DatagramsUnblocked` are gone, and a port touching the `// DATAGRAM` loop has to be redone against `StreamsState::write_frames`. diff --git a/rs/moq-quic/src/cid_queue.rs b/rs/moq-quic/src/cid_queue.rs index 4a6c85fba7f..f4655e655e2 100644 --- a/rs/moq-quic/src/cid_queue.rs +++ b/rs/moq-quic/src/cid_queue.rs @@ -94,6 +94,20 @@ impl CidQueue { Some((cid_data.1.unwrap(), orig_offset..self.offset)) } + /// Switch to the CID right after the active one, if known, returning its reset token and + /// the retired active sequence number + pub(crate) fn next_in_order(&mut self) -> Option<(ResetToken, Range)> { + let next = (self.cursor + 1) % Self::LEN; + let (_, token) = self.buffer[next]?; + self.buffer[self.cursor] = None; + self.cursor = next; + self.offset += 1; + Some(( + token.expect("non-initial CID missing reset token"), + self.offset - 1..self.offset, + )) + } + /// Iterate CIDs in CidQueue that are not `None`, including the active CID fn iter(&self) -> impl Iterator + '_ { (0..Self::LEN).filter_map(move |step| { @@ -178,6 +192,28 @@ mod tests { assert!(q.next().is_none()); } + #[test] + fn next_in_order_waits_for_the_gap() { + let mut q = CidQueue::new(initial_cid()); + q.insert(cid(4, 0)).unwrap(); + q.insert(cid(2, 0)).unwrap(); + assert!(q.next_in_order().is_none(), "sequence 1 is still in flight"); + assert_eq!(q.active_seq(), 0); + + q.insert(cid(1, 0)).unwrap(); + assert_eq!(q.next_in_order().unwrap().1, 0..1); + assert_eq!(q.active_seq(), 1); + assert_eq!(q.next_in_order().unwrap().1, 1..2); + assert!(q.next_in_order().is_none(), "sequence 3 is still in flight"); + + // The CIDs that arrived early are still there to use. + q.insert(cid(3, 0)).unwrap(); + assert_eq!(q.next_in_order().unwrap().1, 2..3); + assert_eq!(q.next_in_order().unwrap().1, 3..4); + assert_eq!(q.active_seq(), 4); + assert!(q.next_in_order().is_none()); + } + #[test] fn wrap() { let mut q = CidQueue::new(initial_cid()); diff --git a/rs/moq-quic/src/config/transport/mod.rs b/rs/moq-quic/src/config/transport/mod.rs index a710c0eab49..d9f5064b8c8 100644 --- a/rs/moq-quic/src/config/transport/mod.rs +++ b/rs/moq-quic/src/config/transport/mod.rs @@ -63,8 +63,9 @@ pub struct Config { /// Used in place of [`idle_timeout`](Self::idle_timeout) until the handshake completes, so a /// short idle timeout meant to detect a dead peer quickly does not also cut a handshake off /// after a single probe. It applies whether the idle timeout is shorter, longer, or disabled, - /// and whatever idle timeout the peer advertises. Defaults to 10 seconds, as msquic's - /// `HandshakeIdleTimeoutMs`. + /// and whatever idle timeout the peer advertises. Defaults to 30 seconds, the default idle + /// timeout: under heavy loss a client backs its probes off past 10 seconds, and a half-open + /// handshake costs no more than an idle connection. pub handshake_timeout: Duration, /// Period of inactivity before sending a keep-alive packet, `None` (the default) to derive it @@ -286,7 +287,7 @@ impl Default for Config { Self { limits: Limits::default(), idle_timeout: Some(DEFAULT_IDLE_TIMEOUT), - handshake_timeout: Duration::from_secs(10), + handshake_timeout: DEFAULT_IDLE_TIMEOUT, keep_alive: None, send_fairness: true, diff --git a/rs/moq-quic/src/connection/mod.rs b/rs/moq-quic/src/connection/mod.rs index 74c369297f0..920f5fc11c1 100644 --- a/rs/moq-quic/src/connection/mod.rs +++ b/rs/moq-quic/src/connection/mod.rs @@ -3312,10 +3312,15 @@ impl Connection { } }; - if self.side.is_server() && self.rem_cids.active_seq() == 0 { - // We're a server still using the initial remote CID for the client, so - // let's switch immediately to enable clientside stateless resets. - self.update_rem_cid(); + // A server still on the client's initial CID switches at once, to enable + // client-side stateless resets, but only to sequence 1: jumping to a later + // one would retire the CIDs before it, still in flight, without using them. + if self.side.is_server() + && self.rem_cids.active_seq() == 0 + && let Some((reset_token, retired)) = self.rem_cids.next_in_order() + { + self.spaces[SpaceId::Data].pending.retire_cids.extend(retired); + self.set_reset_token(reset_token); } } Frame::NewToken(NewToken { token }) => { diff --git a/rs/moq-quic/src/tests/mod.rs b/rs/moq-quic/src/tests/mod.rs index 56ccd2df982..f662575b152 100644 --- a/rs/moq-quic/src/tests/mod.rs +++ b/rs/moq-quic/src/tests/mod.rs @@ -1194,7 +1194,7 @@ fn deferred_accept_uses_handshake_idle_timeout() { pair.time = start.saturating_add(Duration::from_secs(3)); assert!(pair.server.try_accept(delayed, pair.time).is_ok()); - pair.time = start.saturating_add(Duration::from_secs(10)); + pair.time = start.saturating_add(transport::Config::default().handshake_timeout); assert_matches!(pair.server.try_accept(late, pair.time), Err(ConnectionError::TimedOut)); } diff --git a/rs/moq-quic/src/tests/sim.rs b/rs/moq-quic/src/tests/sim.rs index c7bc9df6879..cd366a56f68 100644 --- a/rs/moq-quic/src/tests/sim.rs +++ b/rs/moq-quic/src/tests/sim.rs @@ -72,8 +72,113 @@ fn priority_on_a_clean_path() { }); } +/// A handshake under up to the QUIC Interop Runner's 30% loss either completes or ends only +/// after a side heard nothing for the whole handshake timeout, and the server leaves the client's +/// initial CID for its next in order, retiring nothing else +#[test] +fn handshake_under_loss() { + let link = (0.0..=0.3, probability(0.05), probability(0.1), 0u64..=20_000).prop_map( + |(loss, duplicate, reorder, reorder_delay)| { + let mut link = Link::default(); + link.loss = loss; + link.duplicate = duplicate; + link.reorder = reorder; + link.reorder_delay = Duration::from_micros(reorder_delay); + link + }, + ); + let cases = (any::(), 0u64..=50_000, link.clone(), link); + sweep(cases, |(seed, latency, to_server, to_client)| { + handshake(seed, Duration::from_micros(latency), to_server, to_client) + }); +} + +fn handshake(seed: u64, latency: Duration, to_server: Link, to_client: Link) -> TestCaseResult { + entropy::reseed(seed); + let bound = transport::Config::default().handshake_timeout; + let mut pair = Pair::new(Default::default(), server_config()); + pair.latency = latency; + pair.to_server = to_server; + pair.to_client = to_client; + let start = pair.time; + let client_ch = pair.begin_connect(client_config()); + + // When each side last processed a packet from the other, to tell a give-up from a dead + // network: an arrival it cannot decrypt yet is no sign of life + let mut heard = [vec![start], vec![]]; + let mut frames = [String::new(), String::new()]; + let server_ch = loop { + let now = pair.time; + let stepped = pair.step(); + let server_ch = pair.server.connections.keys().next().copied(); + for (index, ch) in [Some(client_ch), server_ch].into_iter().enumerate() { + let Some(ch) = ch else { continue }; + let conn = match index { + 0 => pair.client_conn_mut(ch), + _ => pair.server_conn_mut(ch), + }; + let received = format!("{:?}", conn.stats().frame_rx); + if received != frames[index] { + frames[index] = received; + heard[index].push(now); + } + let Some(error) = conn.error().cloned() else { continue }; + prop_assert!( + matches!(error, ConnectionError::HandshakeTimeout), + "side {index} failed: {error}" + ); + heard[index].push(now); + let silence = heard[index].windows(2).map(|w| w[1] - w[0]).max().unwrap_or_default(); + prop_assert!(silence >= bound, "side {index} gave up after {silence:?} of silence"); + return Ok(()); + } + if let Some(server_ch) = server_ch + && is_confirmed(pair.client_conn_mut(client_ch)) + && is_connected(pair.server_conn_mut(server_ch)) + { + break server_ch; + } + prop_assert!(pair.time - start < DEADLINE, "stalled for {DEADLINE:?}"); + if !stepped { + // Only an idle timer or an arrival is left + let next = [ + pair.client.inbound.front().map(|x| x.0), + pair.server.inbound.front().map(|x| x.0), + pair.client.next_wakeup(), + pair.server.next_wakeup(), + ] + .into_iter() + .flatten() + .min(); + let Some(next) = next else { + return Err(TestCaseError::fail("deadlocked mid-handshake")); + }; + pair.time = next; + } + }; + + // Heal the path so every NEW_CONNECTION_ID and RETIRE_CONNECTION_ID lands + for link in [&mut pair.to_server, &mut pair.to_client] { + link.loss = 0.0; + link.reorder = 0.0; + } + pair.drive(); + prop_assert_eq!(pair.server_conn_mut(server_ch).active_rem_cid_seq(), 1); + prop_assert_eq!(pair.client_conn_mut(client_ch).active_local_cid_seq().0, 1); + Ok(()) +} + +fn probability(max: f64) -> impl Strategy + Clone { + prop_oneof![Just(0.0), 0.0..max] +} + /// Runs the cases `MOQ_QUIC_SEED` and `MOQ_QUIC_CASES` pick, then `property` on each outcome fn check(cases: impl Strategy, property: impl Fn(&Outcome) -> TestCaseResult) { + sweep(cases, |case| property(&run(&case)?)); +} + +/// Runs `test` on the cases `MOQ_QUIC_SEED` and `MOQ_QUIC_CASES` pick +fn sweep(cases: S, test: impl Fn(S::Value) -> TestCaseResult) { let setting = |name: &str, fallback: u64| std::env::var(name).map_or(fallback, |value| value.parse().expect(name)); let seed = setting("MOQ_QUIC_SEED", 0x6d6f71); let count = setting("MOQ_QUIC_CASES", 64) as u32; @@ -84,7 +189,7 @@ fn check(cases: impl Strategy, property: impl Fn(&Outcome) -> Test failure_persistence: None, ..Default::default() }); - runner.run(&cases, |case| property(&run(&case)?)).unwrap(); + runner.run(&cases, test).unwrap(); } /// Long enough for BBR3 to cycle through ProbeBW, whose timing is random too @@ -158,7 +263,6 @@ fn case( /// A direction of the path, which loses and reorders datagrams only when `lossy` fn link(lossy: bool) -> impl Strategy + Clone { - let probability = |max: f64| prop_oneof![Just(0.0), 0.0..max]; let unless_clean = move |max: f64| match lossy { true => probability(max).boxed(), false => Just(0.0).boxed(), @@ -412,7 +516,7 @@ fn run(case: &Case) -> Result { client_config.transport = transport; let (client_ch, server_ch) = pair.connect_with(client_config); - // Workloads start connected; deterministic handshake tests own flight loss. + // Workloads start connected; `handshake_under_loss` owns flight loss. pair.to_server = case.to_server.clone(); pair.to_client = case.to_client.clone(); let start = pair.time; diff --git a/rs/moq-uring/tests/integration/endpoint.rs b/rs/moq-uring/tests/integration/endpoint.rs index f48c4c836e1..a1485ee8f1b 100644 --- a/rs/moq-uring/tests/integration/endpoint.rs +++ b/rs/moq-uring/tests/integration/endpoint.rs @@ -533,7 +533,7 @@ fn an_unroutable_dial_fails_alone() { let unroutable = std::net::SocketAddr::new(std::net::Ipv4Addr::LOCALHOST.to_ipv6_mapped().into(), port); worker .block_on(async move { - // Well inside the 10s handshake timeout a swallowed error waits for. + // Well inside the 30s handshake timeout a swallowed error waits for. match within( &handle, "the unroutable dial to fail", diff --git a/test/quic-interop/cases.toml b/test/quic-interop/cases.toml index 512d239ea0f..dff86b2e64c 100644 --- a/test/quic-interop/cases.toml +++ b/test/quic-interop/cases.toml @@ -125,44 +125,9 @@ why = "moq-quic's client never moves to a server's preferred address (quest/m4/q [[known]] client = "moq-quic" server = "moq-quic" -case = "handshakeloss" -result = "failed" -why = "Either side abandons a handshake after 10 s without hearing from the other, shorter than a retransmission gap under 30% loss (quest/m1/quic-handshake-loss.md)." - -[[known]] -client = "moq-quic" -server = "moq-quic" -case = "handshakecorruption" -result = "failed" -why = "As handshakeloss (quest/m1/quic-handshake-loss.md)." - -[[known]] -client = "quinn" -server = "moq-quic" -case = "handshakeloss" -result = "failed" -why = "moq-quic's server abandons a handshake after 10 s without hearing from the client, shorter than quinn's retransmission gaps under 30% loss (quest/m1/quic-handshake-loss.md)." - -[[known]] -client = "quinn" -server = "moq-quic" -case = "handshakecorruption" -result = "failed" -why = "As handshakeloss (quest/m1/quic-handshake-loss.md)." - -[[known]] -client = "quiche" -server = "moq-quic" -case = "handshakeloss" -result = "failed" -why = "quiche's client fails with InvalidState after moq-quic's server retires the connection IDs it skipped (quest/m1/quic-handshake-loss.md)." - -[[known]] -client = "quiche" -server = "moq-quic" case = "handshakecorruption" result = "failed" -why = "As handshakeloss (quest/m1/quic-handshake-loss.md)." +why = "About one run in ten: the server's Handshake flight lands before the client has the ServerHello, and both sides then probe only on a PTO backed off past 16 s, so a side hears nothing for the 30 s handshake timeout (quest/m4/quic-handshake-speedup.md)." [[known]] client = "moq-quic" From b086a93aa98a6b98f631164888a88b7238a011d6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 11:37:50 -0800 Subject: [PATCH 3/3] fix(moq-quic): answer review on the handshake loss PR - The sim property checks the silence right before a side gave up, not the largest gap anywhere. - moq-uring's transport config forwards `handshake_timeout`, so its black-hole and backlog tests set 1 s instead of waiting out 30 s. - The relay accept deadline quest records the maintainer's 30 s decision and states what the deadline covers and what admission bounds. Co-Authored-By: Claude Opus 5.5 --- quest/m1/relay-accept-deadline.md | 37 ++++++++++++++-------- rs/moq-quic/src/tests/sim.rs | 8 ++--- rs/moq-uring/benches/unreachable_dials.rs | 2 +- rs/moq-uring/src/quic/driver/mod.rs | 1 + rs/moq-uring/src/quic/handshake.rs | 6 ++-- rs/moq-uring/src/quic/transport.rs | 8 +++++ rs/moq-uring/tests/integration/endpoint.rs | 15 +++++---- 7 files changed, 49 insertions(+), 28 deletions(-) diff --git a/quest/m1/relay-accept-deadline.md b/quest/m1/relay-accept-deadline.md index a75094458e5..765cc7cfe41 100644 --- a/quest/m1/relay-accept-deadline.md +++ b/quest/m1/relay-accept-deadline.md @@ -3,8 +3,8 @@ ## Goal A relay keeps a client on a bad network long enough to finish its QUIC -handshake, as moq-quic's own handshake timeout now does, so `--listen-timeout` -does not shed the clients that timeout serves. +handshake under the QUIC Interop Runner's 30% loss, so `--listen-timeout` +does not shed the clients moq-quic's own handshake timeout now keeps. ## Plan @@ -12,18 +12,27 @@ Found by [the handshake loss quest](https://github.com/moq-dev/moq/pull/8824), which raised moq-quic's `handshake_timeout` to 30 s, an inactivity bound. `moq_net::server::DEFAULT_TIMEOUT` (10 s), which `moq-tokio`'s `listen.timeout` and so the relay's `--listen-timeout` default to, is a total -deadline from the connection's arrival through the SETUP reply, and it covers -the QUIC handshake too. Under the QUIC Interop Runner's 30% loss a client's -probe gaps alone reach 13 s, so this deadline still drops it. - -Recommendation: raise the default to 30 s, the same as `moq-tokio`'s connect -timeout on the client side, and update `doc/bin/relay/config.md`, -`doc/bin/cli.md`, and `doc/lib/rs/moq-net.md`. Each half-open session then -holds its slot up to three times longer, which admission already bounds. +deadline from the connection's arrival. It covers the QUIC handshake, the +client's SETUP, and the caller's admission, and for moq-lite 01/02, whose +server SETUP waits on `ok()`, the reply too; every other version sends its +reply from the started session, outside the deadline. Under the QUIC Interop +Runner's 30% loss a client's probe gaps alone reach 13 s, so this deadline +still drops it. + +Decided 2026-11-15 (maintainer, /quest-complete): raise the default to 30 s +(rejected: keep 10 s), the same as `moq-tokio`'s connect timeout on the client +side, and update `doc/bin/relay/config.md`, `doc/bin/cli.md`, and +`doc/lib/rs/moq-net.md`. + +A total deadline is not an inactivity bound: a client that keeps making +progress but needs more than 30 s overall, such as one configured with a +handshake timeout above 30 s and hitting several long gaps, is still shed. +That is the intended trade: past 30 s the client's own connect timeout gives +up too. + +Nothing else bounds a half-open session: `Admission` only counts active +connections for draining and caps none, so this deadline is the whole bound on +how long each one holds its slot, now up to three times longer. Public API: the default of `moq_net::Server::with_timeout` and `--listen-timeout`, a behavior change. Wire: none. - -## Waiting - -- Maintainer: raise the default accept deadline from 10 s to 30 s, or keep 10 s and accept that a relay sheds clients under heavy loss. Changing an admission default trades slowloris exposure for reach, which is the maintainer's policy call. Recommendation: raise it to 30 s. diff --git a/rs/moq-quic/src/tests/sim.rs b/rs/moq-quic/src/tests/sim.rs index cd366a56f68..4b9a37bd75e 100644 --- a/rs/moq-quic/src/tests/sim.rs +++ b/rs/moq-quic/src/tests/sim.rs @@ -105,7 +105,7 @@ fn handshake(seed: u64, latency: Duration, to_server: Link, to_client: Link) -> // When each side last processed a packet from the other, to tell a give-up from a dead // network: an arrival it cannot decrypt yet is no sign of life - let mut heard = [vec![start], vec![]]; + let mut heard = [start; 2]; let mut frames = [String::new(), String::new()]; let server_ch = loop { let now = pair.time; @@ -120,15 +120,15 @@ fn handshake(seed: u64, latency: Duration, to_server: Link, to_client: Link) -> let received = format!("{:?}", conn.stats().frame_rx); if received != frames[index] { frames[index] = received; - heard[index].push(now); + heard[index] = now; } let Some(error) = conn.error().cloned() else { continue }; prop_assert!( matches!(error, ConnectionError::HandshakeTimeout), "side {index} failed: {error}" ); - heard[index].push(now); - let silence = heard[index].windows(2).map(|w| w[1] - w[0]).max().unwrap_or_default(); + // The silence right before it gave up, not any earlier one + let silence = now - heard[index]; prop_assert!(silence >= bound, "side {index} gave up after {silence:?} of silence"); return Ok(()); } diff --git a/rs/moq-uring/benches/unreachable_dials.rs b/rs/moq-uring/benches/unreachable_dials.rs index d118d3dd85c..26d4c2e1f6d 100644 --- a/rs/moq-uring/benches/unreachable_dials.rs +++ b/rs/moq-uring/benches/unreachable_dials.rs @@ -64,7 +64,7 @@ mod linux { pub fn benchmark(c: &mut Criterion) { let mut group = c.benchmark_group("unreachable_dials"); - // The background dials end at the 10s handshake timeout, so each point + // The background dials end at the 30s handshake timeout, so each point // has to finish well inside it. group .sample_size(10) diff --git a/rs/moq-uring/src/quic/driver/mod.rs b/rs/moq-uring/src/quic/driver/mod.rs index f07d3066fd5..67cde5f075c 100644 --- a/rs/moq-uring/src/quic/driver/mod.rs +++ b/rs/moq-uring/src/quic/driver/mod.rs @@ -152,6 +152,7 @@ fn with_qlog(transport: &mut moq_quic::transport::Config, config: &transport::Co fn transport_config(config: &transport::Config) -> Result { let mut transport = moq_quic::transport::Config::default(); transport.idle_timeout = Some(config.idle_timeout); + transport.handshake_timeout = config.handshake_timeout; transport.keep_alive = config.keep_alive; // Our defaults where the caller set none, rather than moq-quic's; the rate caps pass through. transport.limits = config.limits; diff --git a/rs/moq-uring/src/quic/handshake.rs b/rs/moq-uring/src/quic/handshake.rs index 28cdf598968..45dac4ba9bd 100644 --- a/rs/moq-uring/src/quic/handshake.rs +++ b/rs/moq-uring/src/quic/handshake.rs @@ -15,11 +15,11 @@ use kio::clock::Instant; pub struct Config { /// How long an incoming connection has from its first packet to finish its /// whole handshake, or `None` (the default) for no deadline. Without one, - /// only inactivity ends a stalled handshake: moq-quic's fixed 10 s - /// handshake idle timeout until QUIC is established, which - /// [`transport::Config::idle_timeout`] does not change, then + /// only inactivity ends a stalled handshake: + /// [`transport::Config::handshake_timeout`] until QUIC is established, then /// [`transport::Config::idle_timeout`] through the layers on top. /// + /// [`transport::Config::handshake_timeout`]: super::transport::Config::handshake_timeout /// [`transport::Config::idle_timeout`]: super::transport::Config::idle_timeout pub timeout: Option, } diff --git a/rs/moq-uring/src/quic/transport.rs b/rs/moq-uring/src/quic/transport.rs index 8e62eb400f8..8990d802e57 100644 --- a/rs/moq-uring/src/quic/transport.rs +++ b/rs/moq-uring/src/quic/transport.rs @@ -16,6 +16,12 @@ pub struct Config { /// Close the connection after this long without activity. Must fit a QUIC /// varint in milliseconds. pub idle_timeout: std::time::Duration, + /// The inactivity bound until the handshake completes, in place of + /// [`idle_timeout`](Self::idle_timeout), so a short idle timeout does not + /// cut off a handshake whose probes back off under loss. Defaults to + /// moq-quic's 30 seconds. See + /// [`moq_quic::transport::Config::handshake_timeout`]. + pub handshake_timeout: std::time::Duration, /// Flow-control windows and stream counts. Each `None` takes its default: /// /// - 1024 streams of each kind a peer may have open at once. MoQ opens a @@ -58,6 +64,7 @@ impl Default for Config { fn default() -> Self { Self { idle_timeout: std::time::Duration::from_secs(10), + handshake_timeout: moq_quic::transport::Config::default().handshake_timeout, limits: Limits::default(), congestion: Congestion::Delay.into(), keep_alive: None, @@ -74,6 +81,7 @@ impl std::fmt::Debug for Config { let mut debug = f.debug_struct("Config"); debug .field("idle_timeout", &self.idle_timeout) + .field("handshake_timeout", &self.handshake_timeout) .field("limits", &self.limits) .field("congestion", &format_args!("..")) .field("keep_alive", &self.keep_alive) diff --git a/rs/moq-uring/tests/integration/endpoint.rs b/rs/moq-uring/tests/integration/endpoint.rs index a1485ee8f1b..86f61e43543 100644 --- a/rs/moq-uring/tests/integration/endpoint.rs +++ b/rs/moq-uring/tests/integration/endpoint.rs @@ -486,9 +486,10 @@ fn an_unanswered_dial_times_out() { let client_sock = handle .udp(UdpSocket::bind("127.0.0.1:0").expect("bind"), udp::Config::default()) .expect("client socket"); - // The dial ends at moq-quic's handshake idle timeout (10s), not the idle timeout. + // The dial ends at the handshake timeout, not the idle timeout. let hole_addr = hole.local_addr().expect("addr"); - let dial = dial_config(); + let mut dial = dial_config(); + dial.transport.handshake_timeout = Duration::from_secs(1); let result = worker .block_on(async move { quic::client::connect(client_sock, &dial, hole_addr, "localhost").await }) @@ -584,9 +585,9 @@ fn the_backlog_bounds_pending_handshakes() { let client_sock = handle .udp(UdpSocket::bind("127.0.0.1:0").expect("bind"), udp::Config::default()) .expect("client socket"); - // The server never answers, so the dial ends at moq-quic's handshake idle - // timeout (10s). - let dial = dial_config(); + // The server never answers, so the dial ends at the handshake timeout. + let mut dial = dial_config(); + dial.transport.handshake_timeout = Duration::from_secs(1); let result = worker .block_on(async move { quic::client::connect(client_sock, &dial, server, "localhost").await }) @@ -622,7 +623,9 @@ fn the_backlog_bounds_queued_connections() { let second_sock = handle .udp(UdpSocket::bind("127.0.0.1:0").expect("bind"), udp::Config::default()) .expect("second client socket"); - let second_config = dial_config(); + // The server never answers, so the dial ends at the handshake timeout. + let mut second_config = dial_config(); + second_config.transport.handshake_timeout = Duration::from_secs(1); let second = worker .block_on(async { quic::client::connect(second_sock, &second_config, server, "localhost").await }) .expect("worker");