diff --git a/quest/m0/ci/README.md b/quest/m0/ci/README.md index 44eaf6d2a7b..e2758b6e381 100644 --- a/quest/m0/ci/README.md +++ b/quest/m0/ci/README.md @@ -64,7 +64,6 @@ The historical samples locate candidates; they are not controlled benchmarks. - [Pin Quest's gate](/quest/m0/ci/quest-gate-pin.md) - make shared command admission available - [Share local check capacity](/quest/m0/ci/local-gate.md) - gate expensive phases and bound internal fanout -- [Windows restart drill](/quest/m0/ci/platform-windows-drill.md) - the relay restart drill passes on Windows in the merge queue - [Require platform gates](/quest/m0/ci/platform-queue.md) - activate the consolidated platform checks in the main and release rulesets after the workflow lands - [mbx on R2](/quest/m0/ci/mbx-r2.md) - Rust builds on every OS read an R2 cache that only `main` writes, escaping the 10 GB Actions cache - [Nix cache](/quest/m0/ci/nix-cache.md) - `nix.moq.dev` on R2 replaces Cachix for CI and releases diff --git a/quest/m0/ci/nightly-green.md b/quest/m0/ci/nightly-green.md index dcce9e2b979..eb6eee93958 100644 --- a/quest/m0/ci/nightly-green.md +++ b/quest/m0/ci/nightly-green.md @@ -3,9 +3,8 @@ ## Goal Every Nightly workload that failed on 2026-10-16 (run 38897397428, commit -b4eab0db1) passes on the next scheduled run, except the ones other quests own: -the Linux release builds ([Release builds link again](/quest/m0/release-link.md)) -and the Windows relay drill ([Windows restart drill](/quest/m0/ci/platform-windows-drill.md)). +b4eab0db1) passes on the next scheduled run, except the Linux release builds, +which [Release builds link again](/quest/m0/release-link.md) owns. ## Plan diff --git a/quest/m0/ci/platform-queue.md b/quest/m0/ci/platform-queue.md index 5b4520faac0..1a1444bda39 100644 --- a/quest/m0/ci/platform-queue.md +++ b/quest/m0/ci/platform-queue.md @@ -33,10 +33,6 @@ the implementation PR. - Nightly and releases share the five-job macOS cap. Measure overlap with queue runs before reserving capacity or reducing the queue build limit. -## Required - -- [Windows restart drill](/quest/m0/ci/platform-windows-drill.md) - a required Platform gate would block every merge while it fails - ## Related - [mbx on R2](/quest/m0/ci/mbx-r2.md) - gives platform jobs a Rust cache diff --git a/quest/m0/ci/platform-windows-drill.md b/quest/m0/ci/platform-windows-drill.md deleted file mode 100644 index 3700662a2ac..00000000000 --- a/quest/m0/ci/platform-windows-drill.md +++ /dev/null @@ -1,19 +0,0 @@ -# [S] Windows relay restart drill passes in the merge queue - -## Goal - -`moq-relay::drills relay_killed_mid_group_aborts_then_resumes` (both the -`loopback` and `impaired` lanes) passes on the Windows Platform job, so -Platform can become a required merge-queue gate. - -## Plan - -- It fails on every merge-queue Platform run that executes Windows, back to - at least 2026-10-16 05:41 UTC (runs 38857400879 through 38931667318). Both - lanes panic in `expect_status` at `rs/moq-relay/tests/drills.rs:661`: - "no status change within 10s, wanted Connected". Linux and macOS pass. -- Find when it started (bisect the merge-queue history) and the root cause: - why the reconnect loop never reports `Connected` after the relay restarts on - Windows. Fix the cause; do not raise the timeout or skip the test on Windows. -- Land a regression check that fails without the fix on Windows. The drill - itself is that check if it runs in the queue. diff --git a/rs/moq-shaper/src/lib.rs b/rs/moq-shaper/src/lib.rs index a2c838fcdd2..16468f0f19b 100644 --- a/rs/moq-shaper/src/lib.rs +++ b/rs/moq-shaper/src/lib.rs @@ -701,7 +701,7 @@ async fn run( loop { let (size, from) = tokio::select! { - res = listen.recv_from(&mut buf) => res.context("receive from a client")?, + res = recv(&listen, &mut buf) => res.context("receive from a client")?, Some(res) = tasks.join_next() => { res.context("flow task panicked")??; continue; @@ -817,7 +817,7 @@ async fn reply( ) -> anyhow::Result<()> { let mut buf = vec![0u8; u16::MAX as usize]; loop { - let (size, from) = socket.recv_from(&mut buf).await.context("receive from the target")?; + let (size, from) = recv(&socket, &mut buf).await.context("receive from the target")?; // Anything else reaching this ephemeral port is not part of the path. if from != *target.lock().expect("target poisoned") || is_cut(&cuts) { continue; @@ -828,6 +828,19 @@ async fn reply( } } +/// Receive the next datagram, skipping the port unreachable an earlier send +/// drew. Windows reports one as `ConnectionReset` even on an unconnected +/// socket, but a peer that went away (a killed relay, a closed client) is the +/// path losing datagrams, not the shaper failing. +async fn recv(socket: &UdpSocket, buf: &mut [u8]) -> std::io::Result<(usize, SocketAddr)> { + loop { + match socket.recv_from(buf).await { + Err(err) if err.kind() == std::io::ErrorKind::ConnectionReset => continue, + res => return res, + } + } +} + /// Send each treated datagram at its departure time, after its batch if the /// link has one. async fn deliver( @@ -1388,6 +1401,67 @@ mod tests { assert!(shaper.retarget(ipv6).is_err(), "retargeted where no flow can reach"); } + #[tokio::test(start_paused = true)] + async fn a_closed_target_loses_datagrams_until_a_retarget() { + let dead = UdpSocket::bind(LOCALHOST).await.unwrap(); + let next = UdpSocket::bind(LOCALHOST).await.unwrap(); + let shaper = Shaper::bind(Config { + bind: LOCALHOST, + target: dead.local_addr().unwrap(), + seed: 1, + up: Profile::default(), + down: Profile::default(), + }) + .await + .unwrap(); + let client = UdpSocket::bind(LOCALHOST).await.unwrap(); + client.connect(shaper.addr()).await.unwrap(); + let mut buf = [0u8; 8]; + + client.send(b"before").await.unwrap(); + dead.recv_from(&mut buf).await.unwrap(); + + // The flow sends to the closed port, drawing an error on its next receive. + drop(dead); + client.send(b"lost").await.unwrap(); + while shaper.stats().up.packets < 2 { + tokio::time::sleep(Duration::from_millis(1)).await; + } + + shaper.retarget(next.local_addr().unwrap()).unwrap(); + client.send(b"after").await.unwrap(); + let (size, flow) = tokio::time::timeout(Duration::from_secs(1), next.recv_from(&mut buf)) + .await + .expect("the retarget never forwarded") + .unwrap(); + assert_eq!(&buf[..size], b"after"); + next.send_to(b"reply", flow).await.unwrap(); + let size = tokio::time::timeout(Duration::from_secs(1), client.recv(&mut buf)) + .await + .expect("the reply never arrived") + .unwrap(); + assert_eq!(&buf[..size], b"reply"); + shaper.verify().unwrap(); + } + + #[tokio::test(start_paused = true)] + async fn a_closed_client_leaves_the_others_forwarding() { + let (shaper, client) = setup(1, Profile::default(), Profile::default()).await; + + // The echo returns to the closed port, drawing an error on the listening socket. + let gone = UdpSocket::bind(LOCALHOST).await.unwrap(); + gone.connect(shaper.addr()).await.unwrap(); + gone.send(b"gone").await.unwrap(); + drop(gone); + while shaper.stats().down.packets < 1 { + tokio::time::sleep(Duration::from_millis(1)).await; + } + + let got = round_trip(&client, 10).await; + assert_eq!(got, (0..10).collect::>()); + shaper.verify().unwrap(); + } + #[tokio::test(start_paused = true)] async fn an_undelayed_datagram_waits_for_no_timer() { let (_shaper, client) = setup(1, Profile::default(), Profile::default()).await; diff --git a/rs/moq-shaper/src/mem.rs b/rs/moq-shaper/src/mem.rs index bf7e8964364..f3a34901ca2 100644 --- a/rs/moq-shaper/src/mem.rs +++ b/rs/moq-shaper/src/mem.rs @@ -18,14 +18,17 @@ use tokio::sync::mpsc; type Datagram = (SocketAddr, Vec); +/// What a receive returns next: a datagram, or the error an earlier send drew. +type Arrival = io::Result; + /// Every bound socket by port, which is the whole address on this network. -static PORTS: LazyLock>>> = LazyLock::new(Default::default); +static PORTS: LazyLock>>> = LazyLock::new(Default::default); static NEXT_PORT: AtomicU16 = AtomicU16::new(1); pub struct UdpSocket { addr: SocketAddr, peer: OnceLock, - queue: tokio::sync::Mutex>, + queue: tokio::sync::Mutex>, } impl UdpSocket { @@ -54,17 +57,25 @@ impl UdpSocket { self.peer.set(peer).map_err(|_| io::ErrorKind::AlreadyExists.into()) } - /// Like UDP, a datagram to a port nobody bound is lost without an error. + /// A datagram to a port nobody bound is lost, and the sender's next receive + /// fails with `ConnectionReset`: Windows reports the port unreachable that + /// way even on an unconnected socket, where Linux and macOS stay silent. pub async fn send_to(&self, buf: &[u8], dest: SocketAddr) -> io::Result { - if let Some(port) = PORTS.lock().unwrap().get(&dest.port()) { - let _ = port.send((self.addr, buf.to_vec())); + let ports = PORTS.lock().unwrap(); + match ports.get(&dest.port()) { + Some(port) => { + let _ = port.send(Ok((self.addr, buf.to_vec()))); + } + None => { + let _ = ports[&self.addr.port()].send(Err(io::ErrorKind::ConnectionReset.into())); + } } Ok(buf.len()) } pub async fn recv_from(&self, buf: &mut [u8]) -> io::Result<(usize, SocketAddr)> { // The port's sender lives until this socket drops, so the queue never ends. - let (from, datagram) = self.queue.lock().await.recv().await.expect("bound port"); + let (from, datagram) = self.queue.lock().await.recv().await.expect("bound port")?; let size = datagram.len().min(buf.len()); buf[..size].copy_from_slice(&datagram[..size]); Ok((size, from))