From 92f788b20b913d1180fe5d74aa4a2b65f45af042 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 16:15:13 -0700 Subject: [PATCH 1/3] quest(m0): claim platform-windows-drill From 234431b1c0df12762a756771b9d03c33d8621383 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 16:23:20 -0700 Subject: [PATCH 2/3] fix(shaper): keep forwarding after a peer's port goes away Windows reports an ICMP port unreachable as ConnectionReset on the next recv_from, even on an unconnected UDP socket. The shaper treated that as fatal: once the relay restart drill killed its relay, the next keep-alive the shaper forwarded drew the error, the flow's reply task failed, and the whole shaper stopped, so the clients never reconnected after the retarget. Skip ConnectionReset on both receive paths, as the QUIC driver already does, and make the in-memory test socket report it the way Windows does so the unit tests catch a regression on every platform. Co-Authored-By: Claude Opus 5.5 --- quest/m0/ci/README.md | 1 - quest/m0/ci/platform-queue.md | 1 - quest/m0/ci/platform-windows-drill.md | 19 ------- rs/moq-shaper/src/lib.rs | 78 ++++++++++++++++++++++++++- rs/moq-shaper/src/mem.rs | 23 +++++--- 5 files changed, 93 insertions(+), 29 deletions(-) delete mode 100644 quest/m0/ci/platform-windows-drill.md diff --git a/quest/m0/ci/README.md b/quest/m0/ci/README.md index 775a9978946..f13bff53540 100644 --- a/quest/m0/ci/README.md +++ b/quest/m0/ci/README.md @@ -65,7 +65,6 @@ The historical samples locate candidates; they are not controlled benchmarks. - [Pin Quest's gate](/quest/m0/ci/quest-gate-pin.md) - make shared command admission available - [Share local check capacity](/quest/m0/ci/local-gate.md) - gate expensive phases and bound internal fanout - [Dart FFI build](/quest/m0/ci/dart-ffi-build.md) - the Dart check builds release `moq-ffi` once instead of twice -- [Windows restart drill](/quest/m0/ci/platform-windows-drill.md) - the relay restart drill passes on Windows in the merge queue - [macOS zero-RTT test](/quest/m0/ci/platform-macos-zero-rtt.md) - the moq-tokio 0-RTT driver test passes on macOS in the merge queue - [Require platform gates](/quest/m0/ci/platform-queue.md) - activate the consolidated platform checks in the main and release rulesets after the workflow lands - [mbx on R2](/quest/m0/ci/mbx-r2.md) - Rust builds on every OS read an R2 cache that only `main` writes, escaping the 10 GB Actions cache diff --git a/quest/m0/ci/platform-queue.md b/quest/m0/ci/platform-queue.md index 8e81d044d64..88fd2b0135a 100644 --- a/quest/m0/ci/platform-queue.md +++ b/quest/m0/ci/platform-queue.md @@ -35,7 +35,6 @@ the implementation PR. ## Required -- [Windows restart drill](/quest/m0/ci/platform-windows-drill.md) - a required Platform gate would block every merge while it fails - [macOS zero-RTT test](/quest/m0/ci/platform-macos-zero-rtt.md) - a required Platform gate would block every merge while it fails ## Related diff --git a/quest/m0/ci/platform-windows-drill.md b/quest/m0/ci/platform-windows-drill.md deleted file mode 100644 index 3700662a2ac..00000000000 --- a/quest/m0/ci/platform-windows-drill.md +++ /dev/null @@ -1,19 +0,0 @@ -# [S] Windows relay restart drill passes in the merge queue - -## Goal - -`moq-relay::drills relay_killed_mid_group_aborts_then_resumes` (both the -`loopback` and `impaired` lanes) passes on the Windows Platform job, so -Platform can become a required merge-queue gate. - -## Plan - -- It fails on every merge-queue Platform run that executes Windows, back to - at least 2026-10-16 05:41 UTC (runs 38857400879 through 38931667318). Both - lanes panic in `expect_status` at `rs/moq-relay/tests/drills.rs:661`: - "no status change within 10s, wanted Connected". Linux and macOS pass. -- Find when it started (bisect the merge-queue history) and the root cause: - why the reconnect loop never reports `Connected` after the relay restarts on - Windows. Fix the cause; do not raise the timeout or skip the test on Windows. -- Land a regression check that fails without the fix on Windows. The drill - itself is that check if it runs in the queue. diff --git a/rs/moq-shaper/src/lib.rs b/rs/moq-shaper/src/lib.rs index a2c838fcdd2..16468f0f19b 100644 --- a/rs/moq-shaper/src/lib.rs +++ b/rs/moq-shaper/src/lib.rs @@ -701,7 +701,7 @@ async fn run( loop { let (size, from) = tokio::select! { - res = listen.recv_from(&mut buf) => res.context("receive from a client")?, + res = recv(&listen, &mut buf) => res.context("receive from a client")?, Some(res) = tasks.join_next() => { res.context("flow task panicked")??; continue; @@ -817,7 +817,7 @@ async fn reply( ) -> anyhow::Result<()> { let mut buf = vec![0u8; u16::MAX as usize]; loop { - let (size, from) = socket.recv_from(&mut buf).await.context("receive from the target")?; + let (size, from) = recv(&socket, &mut buf).await.context("receive from the target")?; // Anything else reaching this ephemeral port is not part of the path. if from != *target.lock().expect("target poisoned") || is_cut(&cuts) { continue; @@ -828,6 +828,19 @@ async fn reply( } } +/// Receive the next datagram, skipping the port unreachable an earlier send +/// drew. Windows reports one as `ConnectionReset` even on an unconnected +/// socket, but a peer that went away (a killed relay, a closed client) is the +/// path losing datagrams, not the shaper failing. +async fn recv(socket: &UdpSocket, buf: &mut [u8]) -> std::io::Result<(usize, SocketAddr)> { + loop { + match socket.recv_from(buf).await { + Err(err) if err.kind() == std::io::ErrorKind::ConnectionReset => continue, + res => return res, + } + } +} + /// Send each treated datagram at its departure time, after its batch if the /// link has one. async fn deliver( @@ -1388,6 +1401,67 @@ mod tests { assert!(shaper.retarget(ipv6).is_err(), "retargeted where no flow can reach"); } + #[tokio::test(start_paused = true)] + async fn a_closed_target_loses_datagrams_until_a_retarget() { + let dead = UdpSocket::bind(LOCALHOST).await.unwrap(); + let next = UdpSocket::bind(LOCALHOST).await.unwrap(); + let shaper = Shaper::bind(Config { + bind: LOCALHOST, + target: dead.local_addr().unwrap(), + seed: 1, + up: Profile::default(), + down: Profile::default(), + }) + .await + .unwrap(); + let client = UdpSocket::bind(LOCALHOST).await.unwrap(); + client.connect(shaper.addr()).await.unwrap(); + let mut buf = [0u8; 8]; + + client.send(b"before").await.unwrap(); + dead.recv_from(&mut buf).await.unwrap(); + + // The flow sends to the closed port, drawing an error on its next receive. + drop(dead); + client.send(b"lost").await.unwrap(); + while shaper.stats().up.packets < 2 { + tokio::time::sleep(Duration::from_millis(1)).await; + } + + shaper.retarget(next.local_addr().unwrap()).unwrap(); + client.send(b"after").await.unwrap(); + let (size, flow) = tokio::time::timeout(Duration::from_secs(1), next.recv_from(&mut buf)) + .await + .expect("the retarget never forwarded") + .unwrap(); + assert_eq!(&buf[..size], b"after"); + next.send_to(b"reply", flow).await.unwrap(); + let size = tokio::time::timeout(Duration::from_secs(1), client.recv(&mut buf)) + .await + .expect("the reply never arrived") + .unwrap(); + assert_eq!(&buf[..size], b"reply"); + shaper.verify().unwrap(); + } + + #[tokio::test(start_paused = true)] + async fn a_closed_client_leaves_the_others_forwarding() { + let (shaper, client) = setup(1, Profile::default(), Profile::default()).await; + + // The echo returns to the closed port, drawing an error on the listening socket. + let gone = UdpSocket::bind(LOCALHOST).await.unwrap(); + gone.connect(shaper.addr()).await.unwrap(); + gone.send(b"gone").await.unwrap(); + drop(gone); + while shaper.stats().down.packets < 1 { + tokio::time::sleep(Duration::from_millis(1)).await; + } + + let got = round_trip(&client, 10).await; + assert_eq!(got, (0..10).collect::>()); + shaper.verify().unwrap(); + } + #[tokio::test(start_paused = true)] async fn an_undelayed_datagram_waits_for_no_timer() { let (_shaper, client) = setup(1, Profile::default(), Profile::default()).await; diff --git a/rs/moq-shaper/src/mem.rs b/rs/moq-shaper/src/mem.rs index bf7e8964364..f3a34901ca2 100644 --- a/rs/moq-shaper/src/mem.rs +++ b/rs/moq-shaper/src/mem.rs @@ -18,14 +18,17 @@ use tokio::sync::mpsc; type Datagram = (SocketAddr, Vec); +/// What a receive returns next: a datagram, or the error an earlier send drew. +type Arrival = io::Result; + /// Every bound socket by port, which is the whole address on this network. -static PORTS: LazyLock>>> = LazyLock::new(Default::default); +static PORTS: LazyLock>>> = LazyLock::new(Default::default); static NEXT_PORT: AtomicU16 = AtomicU16::new(1); pub struct UdpSocket { addr: SocketAddr, peer: OnceLock, - queue: tokio::sync::Mutex>, + queue: tokio::sync::Mutex>, } impl UdpSocket { @@ -54,17 +57,25 @@ impl UdpSocket { self.peer.set(peer).map_err(|_| io::ErrorKind::AlreadyExists.into()) } - /// Like UDP, a datagram to a port nobody bound is lost without an error. + /// A datagram to a port nobody bound is lost, and the sender's next receive + /// fails with `ConnectionReset`: Windows reports the port unreachable that + /// way even on an unconnected socket, where Linux and macOS stay silent. pub async fn send_to(&self, buf: &[u8], dest: SocketAddr) -> io::Result { - if let Some(port) = PORTS.lock().unwrap().get(&dest.port()) { - let _ = port.send((self.addr, buf.to_vec())); + let ports = PORTS.lock().unwrap(); + match ports.get(&dest.port()) { + Some(port) => { + let _ = port.send(Ok((self.addr, buf.to_vec()))); + } + None => { + let _ = ports[&self.addr.port()].send(Err(io::ErrorKind::ConnectionReset.into())); + } } Ok(buf.len()) } pub async fn recv_from(&self, buf: &mut [u8]) -> io::Result<(usize, SocketAddr)> { // The port's sender lives until this socket drops, so the queue never ends. - let (from, datagram) = self.queue.lock().await.recv().await.expect("bound port"); + let (from, datagram) = self.queue.lock().await.recv().await.expect("bound port")?; let size = datagram.len().min(buf.len()); buf[..size].copy_from_slice(&datagram[..size]); Ok((size, from)) From 465378e13d3fecda8c8c8d5f5a9de83803e59c36 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 19:00:08 -0700 Subject: [PATCH 3/3] quest(m0): drop the Windows drill exception from nightly-green This PR fixes the drill and deletes its quest, so the link no longer resolves. Co-Authored-By: Claude Opus 5.5 --- quest/m0/ci/nightly-green.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/quest/m0/ci/nightly-green.md b/quest/m0/ci/nightly-green.md index dcce9e2b979..eb6eee93958 100644 --- a/quest/m0/ci/nightly-green.md +++ b/quest/m0/ci/nightly-green.md @@ -3,9 +3,8 @@ ## Goal Every Nightly workload that failed on 2026-10-16 (run 38897397428, commit -b4eab0db1) passes on the next scheduled run, except the ones other quests own: -the Linux release builds ([Release builds link again](/quest/m0/release-link.md)) -and the Windows relay drill ([Windows restart drill](/quest/m0/ci/platform-windows-drill.md)). +b4eab0db1) passes on the next scheduled run, except the Linux release builds, +which [Release builds link again](/quest/m0/release-link.md) owns. ## Plan