From dcacb0961deb9f30bc342f222adde20504a464b1 Mon Sep 17 00:00:00 2001 From: Zac Shenker Date: Tue, 20 Oct 2026 09:27:50 -0600 Subject: [PATCH 1/4] fix(quic): resolve SendStream::stopped after a local reset The driver woke stopped() waiters only on StreamEvent::Finished or StreamEvent::Stopped. A local reset produces neither, and moq-quic frees a reset stream without an event, so a stopped() future pending across reset() or reset_at() never resolved, and its entry in the connection's stopped map stayed until the connection closed. A stopped() call after the reset also waited until the reset was acknowledged. reset() and reset_at() now flag and wake the stream's waiters and drop the entry. stopped() joins the waiters when called, not on its first poll, so a reset in between reaches it, and answers at once when the stream is already resolved or reset. The future resolves with Ok(None), as a fresh call already did once the reset stream was freed. Co-Authored-By: Claude Opus 5.5 --- rs/moq-quic/README.md | 1 + rs/moq-tokio/src/quic/driver/connection.rs | 36 ++++++++-- rs/moq-tokio/src/quic/driver/send_stream.rs | 38 ++++++++-- rs/moq-tokio/src/quic/driver/tests.rs | 80 ++++++++++++++++++++- 4 files changed, 140 insertions(+), 15 deletions(-) diff --git a/rs/moq-quic/README.md b/rs/moq-quic/README.md index a61c8dd2503..3c91154f14d 100644 --- a/rs/moq-quic/README.md +++ b/rs/moq-quic/README.md @@ -90,6 +90,7 @@ Changes on top of the upstream commit, besides the renames: - **Stream credit on stop**: `RecvStream::stop` on a stream whose final size is already known (finished or reset) frees it and queues `MAX_STREAMS` at once, as reading the reset does, instead of leaving the credit until the next packet arrives. Not offered upstream yet; quinn `main` has the same gap. - **Drop resets** (`quic::driver::SendStream`): a send stream dropped before `finish` is reset with code 0, or with the peer's `STOP_SENDING` code, instead of quinn's implicit `finish`, so a cancelled writer never ends a truncated stream with a FIN. This matches moq-uring's driver and moq-net's transport contract. `quic::web` resets after the WebTransport header with the mapped code. quinn's `stream_stopped_2` finishes its stream before dropping it. Not offered upstream: it reverses a documented quinn behaviour. - **Reset secret** (`EndpointConfig::reset_secret`): derives the stateless reset key from an operator's secret with HKDF-SHA256, and keys a `HashedConnectionIdGenerator` from it too, since a restarted endpoint drops packets for connection IDs it can't validate before it would reset them. It also lowers `min_reset_interval` to 1ms, so a restart resets thousands of connections in seconds. Both drivers take one, so a restarted server resets its predecessor's connections. Not offered upstream: it only combines quinn's `reset_key` and `HashedConnectionIdGenerator::from_key`. +- **Stopped after a local reset**: moq-tokio's driver resolves `SendStream::stopped` with `None` once the stream is reset locally, as it already did once the reset was acknowledged and the stream freed. No stream event reports a local reset, so quinn leaves a pending `stopped` waiting, and its waker registered, until the connection closes. Not offered upstream yet; quinn 0.11 has the same gap. - **Connection limits** (`Limits`, `Connection::set_limits`, and the drivers' `set_limits`): one validated call changes a live connection's stream limit and windows, refusing values a peer would reject and changing nothing then. moq-tokio and moq-uring re-export the type, so both runtimes check limits the same way. Not offered upstream: it only bundles quinn's existing setters. Recheck the BBR3 changes when cherry-picking quinn#2481 updates, or if it merges. diff --git a/rs/moq-tokio/src/quic/driver/connection.rs b/rs/moq-tokio/src/quic/driver/connection.rs index 5ceccbb4d35..aa552623f60 100644 --- a/rs/moq-tokio/src/quic/driver/connection.rs +++ b/rs/moq-tokio/src/quic/driver/connection.rs @@ -7,7 +7,7 @@ use std::{ pin::Pin, sync::{ Arc, - atomic::{AtomicUsize, Ordering}, + atomic::{AtomicBool, AtomicUsize, Ordering}, }, task::{Context, Poll, Waker, ready}, }; @@ -642,6 +642,12 @@ impl Connection { self.0.state.lock("stats").inner.stats() } + /// How many streams have `SendStream::stopped` waiters registered, to check none leak + #[cfg(test)] + pub(crate) fn stopped_waiters(&self) -> usize { + self.0.state.lock("stopped_waiters").stopped.len() + } + /// The wire counters the driver updates as it sends and receives. #[cfg(feature = "net")] pub(crate) fn wire(&self) -> moq_net::transport::Wire { @@ -1092,7 +1098,7 @@ pub(crate) struct State { endpoint_events: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, pub(crate) blocked_writers: FxHashMap, pub(crate) blocked_readers: FxHashMap, - pub(crate) stopped: FxHashMap>, + pub(crate) stopped: FxHashMap>, /// Always set to Some before the connection becomes drained pub(crate) error: Option, sender: Pin>, @@ -1372,6 +1378,14 @@ impl State { self.close(0u32.into(), Bytes::new(), shared); } + /// Resolves the stream's `stopped` waiters on a local reset, which no stream event reports + pub(crate) fn stopped_by_reset(&mut self, id: StreamId) { + if let Some(waiters) = self.stopped.remove(&id) { + waiters.reset.store(true, Ordering::Relaxed); + waiters.notify.notify_waiters(); + } + } + pub(crate) fn check_0rtt(&self) -> Result<(), ()> { if self.inner.is_handshaking() || self.inner.accepted_0rtt() @@ -1411,16 +1425,24 @@ fn wake_all(wakers: &mut FxHashMap) { wakers.drain().for_each(|(_, waker)| waker.wake()) } -fn wake_stream_notify(stream_id: StreamId, wakers: &mut FxHashMap>) { - if let Some(notify) = wakers.remove(&stream_id) { - notify.notify_waiters() +/// The `SendStream::stopped` futures waiting on one stream +#[derive(Debug, Default)] +pub(crate) struct StopWaiters { + pub(crate) notify: Notify, + /// The stream was reset locally, so it will neither finish nor be stopped by the peer + pub(crate) reset: AtomicBool, +} + +fn wake_stream_notify(stream_id: StreamId, wakers: &mut FxHashMap>) { + if let Some(waiters) = wakers.remove(&stream_id) { + waiters.notify.notify_waiters() } } -fn wake_all_notify(wakers: &mut FxHashMap>) { +fn wake_all_notify(wakers: &mut FxHashMap>) { wakers .drain() - .for_each(|(_, notify)| notify.notify_waiters()) + .for_each(|(_, waiters)| waiters.notify.notify_waiters()) } /// Errors that can arise when sending a datagram diff --git a/rs/moq-tokio/src/quic/driver/send_stream.rs b/rs/moq-tokio/src/quic/driver/send_stream.rs index 9fb64e16a19..2211eea87fb 100644 --- a/rs/moq-tokio/src/quic/driver/send_stream.rs +++ b/rs/moq-tokio/src/quic/driver/send_stream.rs @@ -2,6 +2,7 @@ use std::{ future::{Future, poll_fn}, io, pin::{Pin, pin}, + sync::atomic::Ordering, task::{Context, Poll}, }; @@ -39,6 +40,8 @@ pub struct SendStream { finished: bool, /// The offset and code a drop before `finish` resets at drop_reset: (u64, VarInt), + /// `reset` or `reset_at` succeeded, so `stopped` has nothing left to wait for + reset: bool, } impl SendStream { @@ -49,6 +52,7 @@ impl SendStream { is_0rtt, finished: false, drop_reset: (0, VarInt::from_u32(0)), + reset: false, } } @@ -217,6 +221,8 @@ impl SendStream { return Ok(()); } conn.inner.send_stream(self.stream).reset(error_code)?; + self.reset = true; + conn.stopped_by_reset(self.stream); conn.wake(); Ok(()) } @@ -238,6 +244,8 @@ impl SendStream { conn.inner .send_stream(self.stream) .reset_at(offset, error_code)?; + self.reset = true; + conn.stopped_by_reset(self.stream); conn.wake(); Ok(()) } @@ -276,8 +284,9 @@ impl SendStream { /// /// Yields `Some` with the stop error code if the peer stops the stream. Yields `None` if the /// local side [`finish()`](Self::finish)es the stream and then the peer acknowledges receipt - /// of all stream data (although not necessarily the processing of it), after which the peer - /// closing the stream is no longer meaningful. + /// of all stream data (although not necessarily the processing of it), or once the local side + /// resets it with [`reset()`](Self::reset) or [`reset_at()`](Self::reset_at), after which the + /// peer closing the stream is no longer meaningful. /// /// For a variety of reasons, the peer may not send acknowledgements immediately upon receiving /// data. As such, relying on `stopped` to know when the peer has read a stream to completion @@ -291,23 +300,40 @@ impl SendStream { let conn = self.conn.clone(); let stream = self.stream; let is_0rtt = self.is_0rtt; + // Join the waiters now rather than on the first poll, so a reset in between, which no + // stream event reports, still reaches this future. + let waiters = { + let mut conn = conn.state.lock("SendStream::stopped"); + match send_stream_stopped(&mut conn, stream, is_0rtt) { + Some(output) => Err(output), + None if self.reset => Err(Ok(None)), + None => Ok(conn.stopped.entry(stream).or_default().clone()), + } + }; async move { + let mut waiters = match waiters { + Ok(waiters) => waiters, + Err(output) => return output, + }; loop { // The `Notify::notified` future needs to be created while the lock is being held, // otherwise a wakeup could be missed if triggered inbetween releasing the lock // and creating the future. // The lock may only be held in a block without `await`s, otherwise the future // becomes `!Send`. `Notify::notified` is lifetime-bound to `Notify`, therefore - // we need to declare `notify` outside of the block, and initialize it inside. - let notify; + // we need to declare `waiters` outside of the block, and update it inside. { let mut conn = conn.state.lock("SendStream::stopped"); if let Some(output) = send_stream_stopped(&mut conn, stream, is_0rtt) { return output; } + if waiters.reset.load(Ordering::Relaxed) { + return Ok(None); + } - notify = conn.stopped.entry(stream).or_default().clone(); - notify.notified() + // A wakeup that resolved nothing removed the waiters; join the current ones + waiters = conn.stopped.entry(stream).or_default().clone(); + waiters.notify.notified() } .await } diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index 96aac54e64b..857c2d0abea 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -44,8 +44,8 @@ use tracing::{error_span, info}; use tracing_subscriber::EnvFilter; use super::{ - ClientConfig, Credit, Endpoint, EndpointConfig, ReadError, ReadToEndError, RecvStream, - SendStream, TransportConfig, VarInt, + ClientConfig, Connection, Credit, Endpoint, EndpointConfig, ReadError, ReadToEndError, + RecvStream, SendStream, TransportConfig, VarInt, }; #[test] @@ -1780,6 +1780,82 @@ async fn finished_send_stream_survives_drop() { assert_eq!(data, b"complete"); } +/// Connects `endpoint` to itself, returning the client and server sides +async fn connect_self(endpoint: &Endpoint) -> (Connection, Connection) { + tokio::try_join!( + endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap(), + async { endpoint.accept().await.unwrap().await } + ) + .unwrap() +} + +/// A local reset resolves `stopped`, though no stream event reports it +#[tokio::test(start_paused = true)] +async fn stream_reset_resolves_stopped() { + let _guard = subscribe(); + let endpoint = endpoint(); + let (conn, _server) = connect_self(&endpoint).await; + let mut send = conn.open_uni(Default::default()).await.unwrap(); + send.write_all(b"hello").await.unwrap(); + send.reset(7u32.into()).unwrap(); + let stopped = timeout(Duration::from_secs(10), send.stopped()).await; + assert_eq!(stopped, Ok(Ok(None))); +} + +#[tokio::test(start_paused = true)] +async fn stream_reset_at_resolves_stopped() { + let _guard = subscribe(); + let endpoint = endpoint(); + let (conn, _server) = connect_self(&endpoint).await; + let mut send = conn.open_uni(Default::default()).await.unwrap(); + send.write_all(b"hello").await.unwrap(); + send.reset_at(2, 7u32.into()).unwrap(); + let stopped = timeout(Duration::from_secs(10), send.stopped()).await; + assert_eq!(stopped, Ok(Ok(None))); +} + +/// `stopped` futures made before a reset resolve on it, whether or not they were polled yet +#[tokio::test(start_paused = true)] +async fn stream_stopped_pending_across_reset() { + let _guard = subscribe(); + let endpoint = endpoint(); + let (conn, _server) = connect_self(&endpoint).await; + let mut send = conn.open_uni(Default::default()).await.unwrap(); + send.write_all(b"hello").await.unwrap(); + let unpolled = send.stopped(); + let polled = tokio::spawn(send.stopped()); + tokio::task::yield_now().await; + assert!(!polled.is_finished()); + send.reset(7u32.into()).unwrap(); + let polled = timeout(Duration::from_secs(10), polled).await; + assert_eq!(polled.expect("polled never resolved").unwrap(), Ok(None)); + let unpolled = timeout(Duration::from_secs(10), unpolled).await; + assert_eq!(unpolled, Ok(Ok(None))); +} + +/// A reset releases the stream's `stopped` waiters, even one dropped while pending +#[tokio::test(start_paused = true)] +async fn stream_reset_releases_stopped() { + let _guard = subscribe(); + let endpoint = endpoint(); + let (conn, _server) = connect_self(&endpoint).await; + let mut send = conn.open_uni(Default::default()).await.unwrap(); + send.write_all(b"hello").await.unwrap(); + { + let mut stopped = pin!(send.stopped()); + let poll = std::future::poll_fn(|cx| Poll::Ready(stopped.as_mut().poll(cx))).await; + assert!(poll.is_pending()); + } + assert_eq!(conn.stopped_waiters(), 1); + send.reset(7u32.into()).unwrap(); + assert_eq!(conn.stopped_waiters(), 0); + // Answered at once, without registering a waiter for a stream that will never report + drop(send.stopped()); + assert_eq!(conn.stopped_waiters(), 0); +} + #[tokio::test] async fn stream_drop_removes_blocked_reader() { let _guard = subscribe(); From d2d1f16cdac6ac03b493dfcd7425551476f1cdd8 Mon Sep 17 00:00:00 2001 From: Zac Shenker Date: Tue, 20 Oct 2026 09:30:21 -0600 Subject: [PATCH 2/4] fix(quic): resolve stopped() on the drop reset A send stream dropped before finish() is now reset, which, like reset(), produces no stream event. Wake the stream's stopped() waiters there too, so a stopped() future that outlives its dropped stream resolves. Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/quic/driver/send_stream.rs | 5 +++-- rs/moq-tokio/src/quic/driver/tests.rs | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/rs/moq-tokio/src/quic/driver/send_stream.rs b/rs/moq-tokio/src/quic/driver/send_stream.rs index 2211eea87fb..ae0ee90d052 100644 --- a/rs/moq-tokio/src/quic/driver/send_stream.rs +++ b/rs/moq-tokio/src/quic/driver/send_stream.rs @@ -285,8 +285,8 @@ impl SendStream { /// Yields `Some` with the stop error code if the peer stops the stream. Yields `None` if the /// local side [`finish()`](Self::finish)es the stream and then the peer acknowledges receipt /// of all stream data (although not necessarily the processing of it), or once the local side - /// resets it with [`reset()`](Self::reset) or [`reset_at()`](Self::reset_at), after which the - /// peer closing the stream is no longer meaningful. + /// resets it with [`reset()`](Self::reset) or [`reset_at()`](Self::reset_at), or by dropping + /// it unfinished, after which the peer closing the stream is no longer meaningful. /// /// For a variety of reasons, the peer may not send acknowledgements immediately upon receiving /// data. As such, relying on `stopped` to know when the peer has read a stream to completion @@ -428,6 +428,7 @@ impl Drop for SendStream { } }; if reset.is_ok() { + conn.stopped_by_reset(self.stream); conn.wake(); } } diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index 857c2d0abea..0942c37c49e 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -1835,6 +1835,21 @@ async fn stream_stopped_pending_across_reset() { assert_eq!(unpolled, Ok(Ok(None))); } +/// A `stopped` future outliving its stream resolves on the drop reset +#[tokio::test(start_paused = true)] +async fn dropped_stream_resolves_stopped() { + let _guard = subscribe(); + let endpoint = endpoint(); + let (conn, _server) = connect_self(&endpoint).await; + let mut send = conn.open_uni(Default::default()).await.unwrap(); + send.write_all(b"truncated").await.unwrap(); + let stopped = send.stopped(); + drop(send); + let stopped = timeout(Duration::from_secs(10), stopped).await; + assert_eq!(stopped, Ok(Ok(None))); + assert_eq!(conn.stopped_waiters(), 0); +} + /// A reset releases the stream's `stopped` waiters, even one dropped while pending #[tokio::test(start_paused = true)] async fn stream_reset_releases_stopped() { From aba1bd66d2e8512f33104d026165edc7fd061779 Mon Sep 17 00:00:00 2001 From: Zac Shenker Date: Wed, 21 Oct 2026 15:30:15 -0600 Subject: [PATCH 3/4] fix(quic): a STOP_SENDING after a local reset no longer wins stopped() moq-quic records a peer's stop code even after the stream was reset locally, so stopped() could answer Some(code) after a successful reset. A reset now records whether the peer had already stopped the stream, and stopped() answers from that record first: an earlier stop still yields its code, a later one is ignored. The socket tests run on the wall clock. Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/quic/driver/connection.rs | 16 +++--- rs/moq-tokio/src/quic/driver/send_stream.rs | 57 ++++++++++++--------- rs/moq-tokio/src/quic/driver/tests.rs | 56 ++++++++++++++++++-- 3 files changed, 92 insertions(+), 37 deletions(-) diff --git a/rs/moq-tokio/src/quic/driver/connection.rs b/rs/moq-tokio/src/quic/driver/connection.rs index aa552623f60..90f3ae12d75 100644 --- a/rs/moq-tokio/src/quic/driver/connection.rs +++ b/rs/moq-tokio/src/quic/driver/connection.rs @@ -6,8 +6,8 @@ use std::{ net::{IpAddr, SocketAddr}, pin::Pin, sync::{ - Arc, - atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, OnceLock, + atomic::{AtomicUsize, Ordering}, }, task::{Context, Poll, Waker, ready}, }; @@ -1378,10 +1378,11 @@ impl State { self.close(0u32.into(), Bytes::new(), shared); } - /// Resolves the stream's `stopped` waiters on a local reset, which no stream event reports - pub(crate) fn stopped_by_reset(&mut self, id: StreamId) { + /// Resolves the stream's `stopped` waiters on a local reset, which no stream event reports, + /// with the peer's STOP_SENDING code if it arrived before the reset + pub(crate) fn stopped_by_reset(&mut self, id: StreamId, stopped: Option) { if let Some(waiters) = self.stopped.remove(&id) { - waiters.reset.store(true, Ordering::Relaxed); + let _ = waiters.reset.set(stopped); waiters.notify.notify_waiters(); } } @@ -1429,8 +1430,9 @@ fn wake_all(wakers: &mut FxHashMap) { #[derive(Debug, Default)] pub(crate) struct StopWaiters { pub(crate) notify: Notify, - /// The stream was reset locally, so it will neither finish nor be stopped by the peer - pub(crate) reset: AtomicBool, + /// The stream was reset locally, with the peer's STOP_SENDING code if it arrived first. A + /// later STOP_SENDING no longer counts. + pub(crate) reset: OnceLock>, } fn wake_stream_notify(stream_id: StreamId, wakers: &mut FxHashMap>) { diff --git a/rs/moq-tokio/src/quic/driver/send_stream.rs b/rs/moq-tokio/src/quic/driver/send_stream.rs index ae0ee90d052..94661e8dfe8 100644 --- a/rs/moq-tokio/src/quic/driver/send_stream.rs +++ b/rs/moq-tokio/src/quic/driver/send_stream.rs @@ -2,7 +2,6 @@ use std::{ future::{Future, poll_fn}, io, pin::{Pin, pin}, - sync::atomic::Ordering, task::{Context, Poll}, }; @@ -40,8 +39,9 @@ pub struct SendStream { finished: bool, /// The offset and code a drop before `finish` resets at drop_reset: (u64, VarInt), - /// `reset` or `reset_at` succeeded, so `stopped` has nothing left to wait for - reset: bool, + /// `reset` or `reset_at` succeeded, with the peer's STOP_SENDING code if it arrived first, so + /// `stopped` has nothing left to wait for + reset: Option>, } impl SendStream { @@ -52,7 +52,7 @@ impl SendStream { is_0rtt, finished: false, drop_reset: (0, VarInt::from_u32(0)), - reset: false, + reset: None, } } @@ -220,9 +220,11 @@ impl SendStream { if self.is_0rtt && conn.check_0rtt().is_err() { return Ok(()); } - conn.inner.send_stream(self.stream).reset(error_code)?; - self.reset = true; - conn.stopped_by_reset(self.stream); + let mut stream = conn.inner.send_stream(self.stream); + let stopped = stream.stopped().ok().flatten(); + stream.reset(error_code)?; + self.reset = Some(stopped); + conn.stopped_by_reset(self.stream, stopped); conn.wake(); Ok(()) } @@ -241,11 +243,11 @@ impl SendStream { if self.is_0rtt && conn.check_0rtt().is_err() { return Ok(()); } - conn.inner - .send_stream(self.stream) - .reset_at(offset, error_code)?; - self.reset = true; - conn.stopped_by_reset(self.stream); + let mut stream = conn.inner.send_stream(self.stream); + let stopped = stream.stopped().ok().flatten(); + stream.reset_at(offset, error_code)?; + self.reset = Some(stopped); + conn.stopped_by_reset(self.stream, stopped); conn.wake(); Ok(()) } @@ -286,7 +288,8 @@ impl SendStream { /// local side [`finish()`](Self::finish)es the stream and then the peer acknowledges receipt /// of all stream data (although not necessarily the processing of it), or once the local side /// resets it with [`reset()`](Self::reset) or [`reset_at()`](Self::reset_at), or by dropping - /// it unfinished, after which the peer closing the stream is no longer meaningful. + /// it unfinished, after which the peer closing the stream is no longer meaningful. A peer's + /// stop code still wins over a local reset if it arrived first, but not if it arrives after. /// /// For a variety of reasons, the peer may not send acknowledgements immediately upon receiving /// data. As such, relying on `stopped` to know when the peer has read a stream to completion @@ -304,10 +307,13 @@ impl SendStream { // stream event reports, still reaches this future. let waiters = { let mut conn = conn.state.lock("SendStream::stopped"); - match send_stream_stopped(&mut conn, stream, is_0rtt) { - Some(output) => Err(output), - None if self.reset => Err(Ok(None)), - None => Ok(conn.stopped.entry(stream).or_default().clone()), + // A reset settles it: a STOP_SENDING after it is recorded but no longer counts. + if let Some(stopped) = self.reset { + Err(Ok(stopped)) + } else if let Some(output) = send_stream_stopped(&mut conn, stream, is_0rtt) { + Err(output) + } else { + Ok(conn.stopped.entry(stream).or_default().clone()) } }; async move { @@ -324,12 +330,12 @@ impl SendStream { // we need to declare `waiters` outside of the block, and update it inside. { let mut conn = conn.state.lock("SendStream::stopped"); + if let Some(&stopped) = waiters.reset.get() { + return Ok(stopped); + } if let Some(output) = send_stream_stopped(&mut conn, stream, is_0rtt) { return output; } - if waiters.reset.load(Ordering::Relaxed) { - return Ok(None); - } // A wakeup that resolved nothing removed the waiters; join the current ones waiters = conn.stopped.entry(stream).or_default().clone(); @@ -416,19 +422,20 @@ impl Drop for SendStream { } // Fails harmlessly when already reset. let mut stream = conn.inner.send_stream(self.stream); - let reset = match stream.stopped() { + let stopped = stream.stopped().ok().flatten(); + let reset = match stopped { // Echo the code, as RFC 9000 section 3.5 asks, even after `finish`: the peer reads // nothing more, so retransmitting the rest is wasted. - Ok(Some(code)) => stream.reset(code), - _ if self.finished => return, - _ => { + Some(code) => stream.reset(code), + None if self.finished => return, + None => { let (offset, code) = self.drop_reset; let offset = offset.min(stream.offset()); stream.reset_at(offset, code) } }; if reset.is_ok() { - conn.stopped_by_reset(self.stream); + conn.stopped_by_reset(self.stream, stopped); conn.wake(); } } diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index 0942c37c49e..0a2741a8826 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -1791,8 +1791,54 @@ async fn connect_self(endpoint: &Endpoint) -> (Connection, Connection) { .unwrap() } +/// Wait until the peer's STOP_SENDING has been processed +async fn stop_sending_received(conn: &Connection) { + timeout(Duration::from_secs(10), async { + while conn.stats().frame_rx.stop_sending == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("no STOP_SENDING arrived"); +} + +/// A peer's stop code that arrived before a local reset still wins +#[tokio::test] +async fn stream_stop_before_reset_wins() { + let _guard = subscribe(); + let endpoint = endpoint(); + let (conn, server) = connect_self(&endpoint).await; + let mut send = conn.open_uni(Default::default()).await.unwrap(); + send.write_all(b"hello").await.unwrap(); + let mut recv = server.accept_uni().await.unwrap(); + recv.stop(5u32.into()).unwrap(); + stop_sending_received(&conn).await; + + send.reset(7u32.into()).unwrap(); + let stopped = timeout(Duration::from_secs(10), send.stopped()).await; + assert_eq!(stopped, Ok(Ok(Some(5u32.into())))); +} + +/// A STOP_SENDING that arrives after a local reset no longer counts +#[tokio::test] +async fn stream_stop_after_reset_is_ignored() { + let _guard = subscribe(); + let endpoint = endpoint(); + let (conn, server) = connect_self(&endpoint).await; + let mut send = conn.open_uni(Default::default()).await.unwrap(); + send.write_all(b"hello").await.unwrap(); + let mut recv = server.accept_uni().await.unwrap(); + // Neither side runs in between, so the peer stops a stream it hasn't seen reset yet. + send.reset(7u32.into()).unwrap(); + recv.stop(5u32.into()).unwrap(); + stop_sending_received(&conn).await; + + let stopped = timeout(Duration::from_secs(10), send.stopped()).await; + assert_eq!(stopped, Ok(Ok(None))); +} + /// A local reset resolves `stopped`, though no stream event reports it -#[tokio::test(start_paused = true)] +#[tokio::test] async fn stream_reset_resolves_stopped() { let _guard = subscribe(); let endpoint = endpoint(); @@ -1804,7 +1850,7 @@ async fn stream_reset_resolves_stopped() { assert_eq!(stopped, Ok(Ok(None))); } -#[tokio::test(start_paused = true)] +#[tokio::test] async fn stream_reset_at_resolves_stopped() { let _guard = subscribe(); let endpoint = endpoint(); @@ -1817,7 +1863,7 @@ async fn stream_reset_at_resolves_stopped() { } /// `stopped` futures made before a reset resolve on it, whether or not they were polled yet -#[tokio::test(start_paused = true)] +#[tokio::test] async fn stream_stopped_pending_across_reset() { let _guard = subscribe(); let endpoint = endpoint(); @@ -1836,7 +1882,7 @@ async fn stream_stopped_pending_across_reset() { } /// A `stopped` future outliving its stream resolves on the drop reset -#[tokio::test(start_paused = true)] +#[tokio::test] async fn dropped_stream_resolves_stopped() { let _guard = subscribe(); let endpoint = endpoint(); @@ -1851,7 +1897,7 @@ async fn dropped_stream_resolves_stopped() { } /// A reset releases the stream's `stopped` waiters, even one dropped while pending -#[tokio::test(start_paused = true)] +#[tokio::test] async fn stream_reset_releases_stopped() { let _guard = subscribe(); let endpoint = endpoint(); From 2dc536f453d77598c0b896745b0020428d935d95 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 17:01:44 -0700 Subject: [PATCH 4/4] fix(quic): keep a STOP_SENDING code for waiters woken before a reset frees the stream A Stopped event detached the waiters before a later reset could record into them, so a waiter that polled after the reset was acknowledged and the stream freed saw Ok(None) instead of the peer's code. The waiters now record the outcome of every event that resolves them. Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/quic/driver/connection.rs | 32 ++++++++++----------- rs/moq-tokio/src/quic/driver/send_stream.rs | 8 +++--- rs/moq-tokio/src/quic/driver/tests.rs | 26 +++++++++++++++++ 3 files changed, 46 insertions(+), 20 deletions(-) diff --git a/rs/moq-tokio/src/quic/driver/connection.rs b/rs/moq-tokio/src/quic/driver/connection.rs index 90f3ae12d75..3570f9b3b04 100644 --- a/rs/moq-tokio/src/quic/driver/connection.rs +++ b/rs/moq-tokio/src/quic/driver/connection.rs @@ -642,6 +642,13 @@ impl Connection { self.0.state.lock("stats").inner.stats() } + /// Whether the connection has freed the send stream's state + #[cfg(test)] + pub(crate) fn send_stream_freed(&self, id: StreamId) -> bool { + let mut state = self.0.state.lock("send_stream_freed"); + state.inner.send_stream(id).stopped().is_err() + } + /// How many streams have `SendStream::stopped` waiters registered, to check none leak #[cfg(test)] pub(crate) fn stopped_waiters(&self) -> usize { @@ -1285,9 +1292,9 @@ impl State { // Might mean any number of streams are ready, so we wake up everyone shared.stream_budget_available[dir as usize].notify_waiters(); } - Stream(StreamEvent::Finished { id }) => wake_stream_notify(id, &mut self.stopped), - Stream(StreamEvent::Stopped { id, .. }) => { - wake_stream_notify(id, &mut self.stopped); + Stream(StreamEvent::Finished { id }) => self.resolve_stopped(id, None), + Stream(StreamEvent::Stopped { id, error_code }) => { + self.resolve_stopped(id, Some(error_code)); wake_stream(id, &mut self.blocked_writers); } PathUpdated => { @@ -1378,11 +1385,10 @@ impl State { self.close(0u32.into(), Bytes::new(), shared); } - /// Resolves the stream's `stopped` waiters on a local reset, which no stream event reports, - /// with the peer's STOP_SENDING code if it arrived before the reset - pub(crate) fn stopped_by_reset(&mut self, id: StreamId, stopped: Option) { + /// Resolves the stream's `stopped` waiters with the peer's STOP_SENDING code, or `None` + pub(crate) fn resolve_stopped(&mut self, id: StreamId, outcome: Option) { if let Some(waiters) = self.stopped.remove(&id) { - let _ = waiters.reset.set(stopped); + let _ = waiters.outcome.set(outcome); waiters.notify.notify_waiters(); } } @@ -1430,15 +1436,9 @@ fn wake_all(wakers: &mut FxHashMap) { #[derive(Debug, Default)] pub(crate) struct StopWaiters { pub(crate) notify: Notify, - /// The stream was reset locally, with the peer's STOP_SENDING code if it arrived first. A - /// later STOP_SENDING no longer counts. - pub(crate) reset: OnceLock>, -} - -fn wake_stream_notify(stream_id: StreamId, wakers: &mut FxHashMap>) { - if let Some(waiters) = wakers.remove(&stream_id) { - waiters.notify.notify_waiters() - } + /// What a STOP_SENDING, a delivered finish, or a local reset settled, kept here because the + /// stream may be freed before a woken waiter polls. Once set, nothing later counts. + pub(crate) outcome: OnceLock>, } fn wake_all_notify(wakers: &mut FxHashMap>) { diff --git a/rs/moq-tokio/src/quic/driver/send_stream.rs b/rs/moq-tokio/src/quic/driver/send_stream.rs index 94661e8dfe8..9ab8a717f94 100644 --- a/rs/moq-tokio/src/quic/driver/send_stream.rs +++ b/rs/moq-tokio/src/quic/driver/send_stream.rs @@ -224,7 +224,7 @@ impl SendStream { let stopped = stream.stopped().ok().flatten(); stream.reset(error_code)?; self.reset = Some(stopped); - conn.stopped_by_reset(self.stream, stopped); + conn.resolve_stopped(self.stream, stopped); conn.wake(); Ok(()) } @@ -247,7 +247,7 @@ impl SendStream { let stopped = stream.stopped().ok().flatten(); stream.reset_at(offset, error_code)?; self.reset = Some(stopped); - conn.stopped_by_reset(self.stream, stopped); + conn.resolve_stopped(self.stream, stopped); conn.wake(); Ok(()) } @@ -330,7 +330,7 @@ impl SendStream { // we need to declare `waiters` outside of the block, and update it inside. { let mut conn = conn.state.lock("SendStream::stopped"); - if let Some(&stopped) = waiters.reset.get() { + if let Some(&stopped) = waiters.outcome.get() { return Ok(stopped); } if let Some(output) = send_stream_stopped(&mut conn, stream, is_0rtt) { @@ -435,7 +435,7 @@ impl Drop for SendStream { } }; if reset.is_ok() { - conn.stopped_by_reset(self.stream, stopped); + conn.resolve_stopped(self.stream, stopped); conn.wake(); } } diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index 0a2741a8826..c99392e4202 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -1837,6 +1837,32 @@ async fn stream_stop_after_reset_is_ignored() { assert_eq!(stopped, Ok(Ok(None))); } +/// A stop code that arrived first still wins once the reset stream is freed before the waiter polls +#[tokio::test] +async fn stream_stop_survives_freed_reset() { + let _guard = subscribe(); + let endpoint = endpoint(); + let (conn, server) = connect_self(&endpoint).await; + let mut send = conn.open_uni(Default::default()).await.unwrap(); + send.write_all(b"hello").await.unwrap(); + let stopped = send.stopped(); + let mut recv = server.accept_uni().await.unwrap(); + recv.stop(5u32.into()).unwrap(); + stop_sending_received(&conn).await; + + send.reset(7u32.into()).unwrap(); + let id = send.id(); + timeout(Duration::from_secs(10), async { + while !conn.send_stream_freed(id) { + tokio::task::yield_now().await; + } + }) + .await + .expect("the reset stream was never freed"); + let stopped = timeout(Duration::from_secs(10), stopped).await; + assert_eq!(stopped, Ok(Ok(Some(5u32.into())))); +} + /// A local reset resolves `stopped`, though no stream event reports it #[tokio::test] async fn stream_reset_resolves_stopped() {