From ffb2cc6135b30806b007bb7483e1752394a2d7d7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 22:59:29 -0800 Subject: [PATCH 1/5] chore(quest): claim quest/a0/qmux-socket-teardown.md Quest-Claim: dbb39d277cf89072 From e3fdec4d3650b8035e66c4995b37fce059abd3a1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 23:07:39 -0800 Subject: [PATCH 2/5] fix(qmux): release socket stats during teardown Share socket stats with terminal cleanup so retained Session clones no longer retain duplicated descriptors. Serialize reads with cleanup and cover TCP and upgraded WebSocket termination. Co-Authored-By: GPT-6.1 Sol --- quest/a0/README.md | 1 - quest/a0/qmux-socket-teardown.md | 19 ------------------- quest/a1/qmux-rate-filter.md | 1 - 3 files changed, 21 deletions(-) delete mode 100644 quest/a0/qmux-socket-teardown.md diff --git a/quest/a0/README.md b/quest/a0/README.md index 26ce85f5..07161d4c 100644 --- a/quest/a0/README.md +++ b/quest/a0/README.md @@ -12,7 +12,6 @@ The Required list is ordered by priority. Independent children may proceed concu - [Quinn final Session drop](/quest/a0/quinn-session-drop.md) - close the connection and release the capsule reader - [Quiche lifecycle](/quest/a0/quiche/README.md) - deterministic harness and retirement handling -- [QMux socket teardown](/quest/a0/qmux-socket-teardown.md) - release retained stats descriptors - [WebSocket handshake timeout](/quest/a0/qmux-ws-handshake-timeout.md) - close sessions whose peer never sends parameters - [QMux initial stream-count bound](/quest/a0/qmux-max-streams-param.md) - refuse initial_max_streams parameters above 2^60 - [QMux stream-state checks](/quest/a0/qmux-stream-state.md) - refuse MAX_STREAM_DATA and STOP_SENDING for unopened local streams diff --git a/quest/a0/qmux-socket-teardown.md b/quest/a0/qmux-socket-teardown.md deleted file mode 100644 index 2bd9d81f..00000000 --- a/quest/a0/qmux-socket-teardown.md +++ /dev/null @@ -1,19 +0,0 @@ -# [M] Release socket-stat handles at transport teardown - -## Goal - -QMux transport termination releases its duplicated stats descriptor even when the application retains a Session clone. - -## Plan - -`TcpStats` duplicates the socket descriptor and `Session` currently retains it independently of the reader/writer tasks. Move that ownership into shared terminal-state cleanup so normal close, peer failure and blocked-writer termination release it exactly once without racing stats reads. - -Approved behavior: socket-derived metrics become unavailable after teardown; other retained session counters remain available. Do not retain a final socket snapshot. Cover TCP and upgraded WebSocket paths, and verify peer-observable EOF or descriptor release while holding a Session clone. Preserve stats while the transport is active. Check PRs #412 and #413 before editing overlapping close paths; do not automatically backport or publish. - -## Closes - -- [#383](https://github.com/moq-dev/web-transport/issues/383) - the implementation completes this report - -## Related - -- [QMux send-rate filter](/quest/a1/qmux-rate-filter.md) - adds state to the same `TcpStats` diff --git a/quest/a1/qmux-rate-filter.md b/quest/a1/qmux-rate-filter.md index b16f041a..f8bc3906 100644 --- a/quest/a1/qmux-rate-filter.md +++ b/quest/a1/qmux-rate-filter.md @@ -26,4 +26,3 @@ Decision prompts (✅ chosen): ## Related - [#460](https://github.com/moq-dev/web-transport/issues/460) - closed by the 0.5 backport -- [QMux socket teardown](/quest/a0/qmux-socket-teardown.md) - edits the same `TcpStats` ownership From 708d069e9cec3f42d242dfab8499daace29b6072 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 23:07:56 -0800 Subject: [PATCH 3/5] fix(qmux): share socket stats with terminal cleanup Serialize live stats reads with teardown and release the shared handle independently of retained Session clones. Cover normal close, peer EOF, blocked writes and upgraded WebSocket failure without sleeps. Co-Authored-By: GPT-6.1 Sol --- rs/qmux/README.md | 7 + rs/qmux/src/session.rs | 16 ++- rs/qmux/src/socket.rs | 8 +- rs/qmux/tests/socket_teardown.rs | 221 +++++++++++++++++++++++++++++++ 4 files changed, 244 insertions(+), 8 deletions(-) create mode 100644 rs/qmux/tests/socket_teardown.rs diff --git a/rs/qmux/README.md b/rs/qmux/README.md index 57baf91f..ad0aabc8 100644 --- a/rs/qmux/README.md +++ b/rs/qmux/README.md @@ -68,6 +68,13 @@ grow a response queue by sending more pings; inbound stream processing continues while the writer is blocked. Pending responses are discarded during teardown, and a close frame ends the writer's output. +## Socket statistics + +Sessions release their socket statistics handle during teardown, even if the +application retains a `Session` clone. Later `stats()` calls omit socket-derived +metrics; QMux's own RTT estimate and stream counters remain available. A caller +that retains its own `SharedSocketStats` copy keeps that handle alive. + ## License Licensed under either of [Apache License, Version 2.0](LICENSE-APACHE) or [MIT license](LICENSE-MIT) at your option. diff --git a/rs/qmux/src/session.rs b/rs/qmux/src/session.rs index 7b9827a9..8e3b51c8 100644 --- a/rs/qmux/src/session.rs +++ b/rs/qmux/src/session.rs @@ -260,8 +260,9 @@ pub struct Session { // The transport's own view of the connection, when it has one. Preferred over // `rtt`: the kernel's estimate needs no round trip of ours, so it is available - // from the first moment rather than one probe interval in. - socket_stats: Option, + // from the first moment rather than one probe interval in. Shared with terminal + // cleanup so retained Session clones cannot keep the socket descriptor alive. + socket_stats: Arc>>, // Closes the connection when the last `Session` clone drops. Never read. _guard: Arc, @@ -2203,8 +2204,8 @@ impl Session { // the wire, pulling the outbound queues in priority order and sharing the // stream maps + scalars above with the reader (no message-passing handoff). // Read before the split: the halves go to separate tasks, and the stats - // source belongs to neither. - let socket_stats = transport.socket_stats(); + // source is released by terminal cleanup, independently of Session clones. + let socket_stats = Arc::new(Mutex::new(transport.socket_stats())); let (writer_half, reader_half) = transport.split(); let mut writer = WriterState { writer: writer_half, @@ -2323,6 +2324,7 @@ impl Session { tokio::spawn(timer.run()); } + let teardown_socket_stats = socket_stats.clone(); tokio::spawn(async move { let err = backend.run().await.err().unwrap_or(Error::Closed); // If we tore down because of a protocol/transport violation *we* @@ -2352,6 +2354,7 @@ impl Session { backend.conn_recv_credit.close(); backend.outbound.close(); backend.streams.lock().unwrap().close(); + teardown_socket_stats.lock().unwrap().take(); // Recorded last, so a `closed()` waiter sees credits and streams closed. // The reader often fails (EOF, WebSocket Close) right after an // APPLICATION_CLOSE, so keep the reason recorded first. @@ -2395,7 +2398,6 @@ impl generic::Session for Session { type Error = Error; fn stats(&self) -> impl generic::Stats { - let socket = self.socket_stats.as_ref(); let streams = if self.config.version.is_qmux() { let state = self.streams.lock().unwrap(); let closed = self.closed.borrow(); @@ -2422,6 +2424,10 @@ impl generic::Session for Session { } else { None }; + // Hold the lock through the reads so teardown cannot drop the descriptor + // during a read, and a reader cannot retain it after cleanup completes. + let socket = self.socket_stats.lock().unwrap(); + let socket = socket.as_ref(); SessionStats { streams, // Fall back to our own QX_PING measurement only when the transport diff --git a/rs/qmux/src/socket.rs b/rs/qmux/src/socket.rs index 615b1448..9d9cbd16 100644 --- a/rs/qmux/src/socket.rs +++ b/rs/qmux/src/socket.rs @@ -14,7 +14,8 @@ use std::time::Duration; /// /// Implementations read through to the socket on each call rather than caching, so /// a caller polling this sees the transport's current view. Every metric is -/// optional: `None` means this transport doesn't track it. +/// optional: `None` means this transport doesn't track it. Sessions release their +/// statistics handle during teardown; subsequent session stats omit these metrics. pub trait SocketStats: Send + Sync + 'static { /// Smoothed round-trip time, if the transport tracks one. fn rtt(&self) -> Option { @@ -43,9 +44,10 @@ mod unix { /// `TCP_CONNECTION_INFO` (macOS). /// /// Holds its own duplicate of the file descriptor, so it stays readable for as - /// long as the session needs it no matter what the transport does with the + /// long as this handle lives no matter what the transport does with the /// original. Construct it from the accepted socket and hand it to the transport - /// with `with_socket_stats`. + /// with `with_socket_stats`. The session releases its handle during teardown; + /// any copies retained by the caller keep the descriptor alive. /// /// Reports nothing for a non-TCP socket (a Unix socket, say): the `getsockopt` /// simply fails and every metric reads `None`. diff --git a/rs/qmux/tests/socket_teardown.rs b/rs/qmux/tests/socket_teardown.rs new file mode 100644 index 00000000..62509406 --- /dev/null +++ b/rs/qmux/tests/socket_teardown.rs @@ -0,0 +1,221 @@ +//! Retained session handles must not retain the socket statistics descriptor. +#![cfg(all(unix, feature = "tcp"))] + +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::Duration; + +use qmux::{Config, Session, SharedSocketStats, SocketStats, TcpStats, Version}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, ReadBuf}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::sync::{oneshot, Notify}; +use web_transport_trait::{SendStream as _, Session as _, Stats as _}; + +// Fields drop in declaration order: the notification follows descriptor release. +struct TrackedStats { + tcp: TcpStats, + _released: oneshot::Sender<()>, +} + +impl SocketStats for TrackedStats { + fn rtt(&self) -> Option { + self.tcp.rtt() + } + + fn estimated_send_rate(&self) -> Option { + self.tcp.estimated_send_rate() + } +} + +fn tracked(socket: &TcpStream) -> (SharedSocketStats, oneshot::Receiver<()>) { + let (released, done) = oneshot::channel(); + ( + Arc::new(TrackedStats { + tcp: TcpStats::new(socket).unwrap(), + _released: released, + }), + done, + ) +} + +async fn sockets() -> (TcpStream, TcpStream) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let (client, server) = tokio::join!( + TcpStream::connect(listener.local_addr().unwrap()), + listener.accept(), + ); + (client.unwrap(), server.unwrap().0) +} + +async fn released(done: oneshot::Receiver<()>, session: &Session) { + tokio::time::timeout(Duration::from_secs(2), done) + .await + .expect("retained Session kept the socket stats descriptor alive") + .expect_err("the sender only signals by dropping"); + assert_eq!(session.stats().rtt(), None); + assert_eq!(session.stats().estimated_send_rate(), None); +} + +fn active(session: &Session) { + #[cfg(any(target_os = "linux", target_os = "macos"))] + assert!(session.stats().rtt().is_some()); + // Stream counters remain available independently of socket stats. + assert!(session.stats().streams().is_some()); +} + +#[tokio::test] +async fn tcp_close_releases_stats_with_retained_clones() { + let (client, server) = sockets().await; + let (client_stats, client_done) = tracked(&client); + let (server_stats, server_done) = tracked(&server); + let mut config = Config::new(Version::QMux02); + // Isolate socket RTT from the QMux estimate, which survives teardown. + config.rtt_interval = Duration::ZERO; + let (client, server) = tokio::join!( + Session::connect( + qmux::transport::Stream::new(client, config.version, config.max_record_size) + .with_socket_stats(client_stats), + config.clone(), + ), + Session::accept( + qmux::transport::Stream::new(server, config.version, config.max_record_size) + .with_socket_stats(server_stats), + config.clone(), + ), + ); + let client = client.unwrap(); + let server = server.unwrap(); + active(&client); + active(&server); + let retained = client.clone(); + client.close(0, "done"); + drop(client); + released(client_done, &retained).await; + released(server_done, &server).await; + assert!(retained.stats().streams().is_some()); + assert!(server.stats().streams().is_some()); +} + +#[tokio::test] +async fn tcp_peer_eof_releases_stats_with_retained_clone() { + let (client, mut peer) = sockets().await; + let (stats, done) = tracked(&client); + let config = Config::new(Version::WebTransport); + let client = Session::connect( + qmux::transport::Stream::new(client, config.version, config.max_record_size) + .with_socket_stats(stats), + config, + ) + .await + .unwrap(); + let retained = client.clone(); + drop(client); + peer.shutdown().await.unwrap(); + released(done, &retained).await; + assert!(matches!(retained.closed().await, qmux::Error::Io(err) + if err.kind() == std::io::ErrorKind::UnexpectedEof)); +} + +// Gate actual socket writes so cancellation skips graceful shutdown. Without +// releasing TcpStats, dropping the transport halves alone cannot deliver EOF. +struct GatedSocket { + socket: TcpStream, + entered: Arc, +} + +impl AsyncRead for GatedSocket { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut self.socket).poll_read(cx, buf) + } +} + +impl AsyncWrite for GatedSocket { + fn poll_write( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + _buf: &[u8], + ) -> Poll> { + self.entered.notify_one(); + Poll::Pending + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.socket).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.socket).poll_shutdown(cx) + } +} + +#[tokio::test] +async fn tcp_blocked_writer_releases_stats_and_delivers_eof() { + let (socket, mut peer) = sockets().await; + let (stats, done) = tracked(&socket); + let entered = Arc::new(Notify::new()); + let config = Config::new(Version::WebTransport); + let client = Session::connect( + qmux::transport::Stream::new( + GatedSocket { + socket, + entered: entered.clone(), + }, + config.version, + config.max_record_size, + ) + .with_socket_stats(stats), + config, + ) + .await + .unwrap(); + let mut send = client.open_uni().await.unwrap(); + send.write(b"blocked").await.unwrap(); + tokio::time::timeout(Duration::from_secs(2), entered.notified()) + .await + .expect("writer never blocked"); + let retained = client.clone(); + client.close(0, "done"); + drop(client); + released(done, &retained).await; + let mut bytes = Vec::new(); + tokio::time::timeout(Duration::from_secs(2), peer.read_to_end(&mut bytes)) + .await + .expect("peer never observed EOF while Session was retained") + .unwrap(); +} + +#[cfg(feature = "ws")] +#[tokio::test] +async fn upgraded_websocket_releases_stats_with_retained_clone() { + use qmux::ws::{tokio_tungstenite, Upgraded}; + + for local_close in [false, true] { + let (client, peer) = sockets().await; + let (stats, done) = tracked(&client); + let (client, peer) = tokio::join!( + tokio_tungstenite::client_async("ws://localhost/", client), + tokio_tungstenite::accept_async(peer), + ); + let client = Upgraded::new(client.unwrap().0) + .with_socket_stats(stats) + .connect(); + let peer = peer.unwrap(); + #[cfg(any(target_os = "linux", target_os = "macos"))] + assert!(client.stats().rtt().is_some()); + let retained = client.clone(); + drop(client); + if local_close { + retained.close(0, "done"); + released(done, &retained).await; + } else { + // An abrupt failure of the upgraded socket must release the duplicate too. + drop(peer); + released(done, &retained).await; + } + } +} From bd122548a2d56a93b5472b538144b8ac04753d83 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 23:45:20 -0800 Subject: [PATCH 4/5] fix(qmux): release stats across every reader exit path Cancel blocked accept delivery on closure, release shared stats from a task-owned drop guard, recover poisoned provider locks, and hide socket metrics immediately after the close signal. Cover each review finding with a regression and preserve retained QMux RTT. Co-Authored-By: GPT-6.1 Sol --- rs/qmux/README.md | 6 +- rs/qmux/src/session.rs | 260 ++++++++++++++++++++++++++++++++++++----- rs/qmux/src/socket.rs | 2 +- 3 files changed, 238 insertions(+), 30 deletions(-) diff --git a/rs/qmux/README.md b/rs/qmux/README.md index ad0aabc8..3e119156 100644 --- a/rs/qmux/README.md +++ b/rs/qmux/README.md @@ -71,8 +71,10 @@ and a close frame ends the writer's output. ## Socket statistics Sessions release their socket statistics handle during teardown, even if the -application retains a `Session` clone. Later `stats()` calls omit socket-derived -metrics; QMux's own RTT estimate and stream counters remain available. A caller +application retains a `Session` clone. Once closure is signalled, `stats()` calls +omit socket-derived metrics; QMux's own RTT estimate and stream counters remain +available. `closed()` reports the terminal reason without waiting for asynchronous +transport cleanup and descriptor release to finish. A caller that retains its own `SharedSocketStats` copy keeps that handle alive. ## License diff --git a/rs/qmux/src/session.rs b/rs/qmux/src/session.rs index 8e3b51c8..bd14d336 100644 --- a/rs/qmux/src/session.rs +++ b/rs/qmux/src/session.rs @@ -186,7 +186,26 @@ impl Drop for SessionGuard { } } +/// Releases the transport stats even if the reader task is cancelled or unwinds. +struct SocketStatsGuard(Arc>>); + +impl Drop for SocketStatsGuard { + fn drop(&mut self) { + // A custom stats provider may have panicked during a read. Its poisoned + // lock must not prevent descriptor release or the rest of teardown. + let stats = self + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + drop(stats); + } +} + /// A multiplexed session over a reliable transport. +/// +/// Closing signals the terminal reason immediately; `closed()` does not wait for +/// asynchronous transport cleanup to finish. #[derive(Clone)] pub struct Session { is_server: bool, @@ -1428,32 +1447,24 @@ impl SessionState { async fn run(&mut self) -> Result<(), Error> { let mut closed = self.closed.subscribe(); - loop { - // The idle timeout and keep-alive ping are owned by the timer task, - // which reads the `last_recv_at` we publish below. Keeping the deadline - // off this select is what stops application backpressure — parking in - // `recv_frame`'s `accept_*.send().await` — from starving the deadline - // and firing a spurious idle close on re-entry. - tokio::select! { - biased; - result = self.reader.recv() => { - let data = result?; - // Publish receive progress for the timer's idle deadline. + // Race the whole loop, including delivery to bounded accept queues, + // against closure so a retained Session cannot strand terminal cleanup. + tokio::select! { + biased; + result = async { + loop { + let data = self.reader.recv().await?; self.last_recv_at.store( millis_since(self.base, tokio::time::Instant::now()), Ordering::Release, ); if self.config.version.uses_records() { - // Record-framed drafts: data is a record containing one or more frames for frame in Frame::decode_record(data)? { self.recv_frame(frame).await?; } } else { - // draft-00 §5.2: `max_frame_size` bounds the whole frame, - // header included, and may only be raised from its initial - // value — which we never do, so the initial value is what a - // peer may send. Each message is one frame here. The legacy - // binding predates the parameter and bounds nothing. + // Draft-00 bounds the whole frame; the legacy binding + // predates max_frame_size and has no such bound. if self.config.version == Version::QMux00 && data.len() > MAX_FRAME_SIZE { return Err(Error::FrameTooLarge); } @@ -1462,9 +1473,9 @@ impl SessionState { } } } - _ = async { closed.wait_for(|err| err.is_some()).await.ok(); } => { - return Err(closed.borrow().clone().unwrap_or(Error::Closed)) - } + } => result, + _ = async { closed.wait_for(|err| err.is_some()).await.ok(); } => { + Err(closed.borrow().clone().unwrap_or(Error::Closed)) } } } @@ -2324,7 +2335,7 @@ impl Session { tokio::spawn(timer.run()); } - let teardown_socket_stats = socket_stats.clone(); + let teardown_socket_stats = SocketStatsGuard(socket_stats.clone()); tokio::spawn(async move { let err = backend.run().await.err().unwrap_or(Error::Closed); // If we tore down because of a protocol/transport violation *we* @@ -2354,10 +2365,10 @@ impl Session { backend.conn_recv_credit.close(); backend.outbound.close(); backend.streams.lock().unwrap().close(); - teardown_socket_stats.lock().unwrap().take(); - // Recorded last, so a `closed()` waiter sees credits and streams closed. - // The reader often fails (EOF, WebSocket Close) right after an - // APPLICATION_CLOSE, so keep the reason recorded first. + drop(teardown_socket_stats); + // Publish the backend's reason after cleanup, preserving an earlier + // close signal such as APPLICATION_CLOSE rather than replacing it + // with the transport's subsequent EOF or WebSocket Close. note_closed(&backend.closed, err); }); @@ -2426,8 +2437,15 @@ impl generic::Session for Session { }; // Hold the lock through the reads so teardown cannot drop the descriptor // during a read, and a reader cannot retain it after cleanup completes. - let socket = self.socket_stats.lock().unwrap(); - let socket = socket.as_ref(); + let socket = self + .socket_stats + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let socket = if self.closed.borrow().is_none() { + socket.as_ref() + } else { + None + }; SessionStats { streams, // Fall back to our own QX_PING measurement only when the transport @@ -5470,3 +5488,191 @@ mod credit_update_tests { } } } + +#[cfg(test)] +mod socket_stats_tests { + use super::*; + use std::time::Duration; + use tokio::sync::oneshot; + use web_transport_trait::{Session as _, Stats as _}; + + struct Stats { + panic: AtomicBool, + _released: oneshot::Sender<()>, + } + + impl crate::SocketStats for Stats { + fn rtt(&self) -> Option { + assert!(!self.panic.swap(false, Ordering::Relaxed), "stats panic"); + Some(Duration::from_millis(1)) + } + } + + struct TestTransport { + stats: crate::SharedSocketStats, + incoming: mpsc::UnboundedReceiver, + received: mpsc::UnboundedSender<()>, + failed: oneshot::Receiver<()>, + } + + struct TestReader { + incoming: mpsc::UnboundedReceiver, + received: mpsc::UnboundedSender<()>, + } + + struct TestWriter(oneshot::Receiver<()>); + + impl Transport for TestTransport { + type Reader = TestReader; + type Writer = TestWriter; + + fn socket_stats(&self) -> Option { + Some(self.stats.clone()) + } + + fn split(self) -> (Self::Writer, Self::Reader) { + ( + TestWriter(self.failed), + TestReader { + incoming: self.incoming, + received: self.received, + }, + ) + } + } + + impl Reader for TestReader { + async fn recv(&mut self) -> Result { + let data = self.incoming.recv().await.ok_or(Error::Closed)?; + self.received.send(()).unwrap(); + Ok(data) + } + } + + impl Writer for TestWriter { + async fn send(&mut self, _: Bytes) -> Result<(), Error> { + Ok(()) + } + async fn close(&mut self) -> Result<(), Error> { + Ok(()) + } + async fn maintain(&mut self) -> Result<(), Error> { + let _ = (&mut self.0).await; + Err(Error::Closed) + } + } + + struct Fixture { + session: Session, + incoming: mpsc::UnboundedSender, + received: mpsc::UnboundedReceiver<()>, + fail: oneshot::Sender<()>, + released: oneshot::Receiver<()>, + } + + fn fixture(panic: bool) -> Fixture { + let (released, done) = oneshot::channel(); + let stats = Arc::new(Stats { + panic: AtomicBool::new(panic), + _released: released, + }); + let (incoming, reader) = mpsc::unbounded_channel(); + let (received, received_rx) = mpsc::unbounded_channel(); + let (fail, failed) = oneshot::channel(); + let session = Session::new( + TestTransport { + stats, + incoming: reader, + received, + failed, + }, + false, + Config::new(Version::WebTransport), + ); + Fixture { + session, + incoming, + received: received_rx, + fail, + released: done, + } + } + + async fn released(done: oneshot::Receiver<()>) { + tokio::time::timeout(Duration::from_secs(1), done) + .await + .expect("stats retained after termination") + .expect_err("stats signal by dropping"); + } + + #[tokio::test(start_paused = true)] + async fn full_accept_queue_does_not_retain_stats_on_close_or_writer_failure() { + for dir in [StreamDir::Uni, StreamDir::Bi] { + for writer_failure in [false, true] { + let mut f = fixture(false); + for index in 0..=1024 { + let frame = Frame::Stream(Stream { + id: StreamId::new(index, dir, true), + offset: 0, + data: Bytes::new(), + fin: false, + }); + f.incoming + .send(frame.encode(Version::WebTransport).unwrap()) + .unwrap(); + // On this single-threaded runtime the reader processes the + // frame until it yields before this notification is observed. + f.received.recv().await.unwrap(); + } + if writer_failure { + f.fail.send(()).unwrap(); + } else { + f.session.close(0, "done"); + } + released(f.released).await; + assert_eq!(f.session.stats().rtt(), None); + } + } + } + + #[tokio::test(start_paused = true)] + async fn panicking_stats_do_not_poison_future_reads_or_teardown() { + let f = fixture(true); + assert!( + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| f.session.stats())).is_err() + ); + assert_eq!(f.session.stats().rtt(), Some(Duration::from_millis(1))); + drop(f.incoming); + released(f.released).await; + assert!(matches!(f.session.closed().await, Error::Closed)); + assert_eq!(f.session.stats().rtt(), None); + } + + #[test] + fn runtime_shutdown_releases_stats_with_retained_session() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let mut f = runtime.block_on(async { fixture(false) }); + drop(runtime); + assert!(matches!( + f.released.try_recv(), + Err(oneshot::error::TryRecvError::Closed) + )); + assert_eq!(f.session.stats().rtt(), None); + } + + #[tokio::test(start_paused = true)] + async fn close_hides_socket_metrics_and_preserves_qmux_rtt() { + let f = fixture(false); + let sent = tokio::time::Instant::now(); + f.session.rtt.sent(0, sent); + f.session.rtt.received(0, sent + Duration::from_millis(2)); + assert_eq!(f.session.stats().rtt(), Some(Duration::from_millis(1))); + f.session.close(0, "done"); + f.session.closed().await; + assert_eq!(f.session.stats().rtt(), Some(Duration::from_millis(2))); + released(f.released).await; + } +} diff --git a/rs/qmux/src/socket.rs b/rs/qmux/src/socket.rs index 9d9cbd16..610284a6 100644 --- a/rs/qmux/src/socket.rs +++ b/rs/qmux/src/socket.rs @@ -15,7 +15,7 @@ use std::time::Duration; /// Implementations read through to the socket on each call rather than caching, so /// a caller polling this sees the transport's current view. Every metric is /// optional: `None` means this transport doesn't track it. Sessions release their -/// statistics handle during teardown; subsequent session stats omit these metrics. +/// statistics handle during teardown and omit these metrics once closure is signalled. pub trait SocketStats: Send + Sync + 'static { /// Smoothed round-trip time, if the transport tracks one. fn rtt(&self) -> Option { From df7ad60962005cd72e90d336513f889cb72d0909 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 00:02:36 -0800 Subject: [PATCH 5/5] fix(qmux): preserve received streams while releasing stats Release the stats guard on closure without cancelling the pinned reader future. Preserve queued payloads and FINs, share accept queue capacity with tests, prove backpressure, and cover release during a blocked transport close. Co-Authored-By: GPT-6.1 Sol --- rs/qmux/src/session.rs | 140 +++++++++++++++++++++++++++++++---------- 1 file changed, 107 insertions(+), 33 deletions(-) diff --git a/rs/qmux/src/session.rs b/rs/qmux/src/session.rs index bd14d336..1af3c9f9 100644 --- a/rs/qmux/src/session.rs +++ b/rs/qmux/src/session.rs @@ -28,6 +28,9 @@ use crate::shared::SharedRecv; /// applying backpressure to the whole session. const DATAGRAM_RECV_BUFFER: usize = 1024; +/// Maximum peer-initiated streams awaiting application acceptance per direction. +const ACCEPT_QUEUE_CAPACITY: usize = 1024; + /// How many outbound datagrams to buffer before dropping. The writer pulls from /// this lane; when it stalls on transport backpressure it stops pulling, the lane /// fills, and `send_datagram` drops on a full lane. Kept small so shedding tracks @@ -1447,24 +1450,32 @@ impl SessionState { async fn run(&mut self) -> Result<(), Error> { let mut closed = self.closed.subscribe(); - // Race the whole loop, including delivery to bounded accept queues, - // against closure so a retained Session cannot strand terminal cleanup. - tokio::select! { - biased; - result = async { - loop { - let data = self.reader.recv().await?; + loop { + // The idle timeout and keep-alive ping are owned by the timer task, + // which reads the `last_recv_at` we publish below. Keeping the deadline + // off this select is what stops application backpressure — parking in + // `recv_frame`'s `accept_*.send().await` — from starving the deadline + // and firing a spurious idle close on re-entry. + tokio::select! { + biased; + result = self.reader.recv() => { + let data = result?; + // Publish receive progress for the timer's idle deadline. self.last_recv_at.store( millis_since(self.base, tokio::time::Instant::now()), Ordering::Release, ); if self.config.version.uses_records() { + // Record-framed drafts: data is a record containing one or more frames for frame in Frame::decode_record(data)? { self.recv_frame(frame).await?; } } else { - // Draft-00 bounds the whole frame; the legacy binding - // predates max_frame_size and has no such bound. + // draft-00 §5.2: `max_frame_size` bounds the whole frame, + // header included, and may only be raised from its initial + // value — which we never do, so the initial value is what a + // peer may send. Each message is one frame here. The legacy + // binding predates the parameter and bounds nothing. if self.config.version == Version::QMux00 && data.len() > MAX_FRAME_SIZE { return Err(Error::FrameTooLarge); } @@ -1473,9 +1484,9 @@ impl SessionState { } } } - } => result, - _ = async { closed.wait_for(|err| err.is_some()).await.ok(); } => { - Err(closed.borrow().clone().unwrap_or(Error::Closed)) + _ = async { closed.wait_for(|err| err.is_some()).await.ok(); } => { + return Err(closed.borrow().clone().unwrap_or(Error::Closed)) + } } } } @@ -2157,8 +2168,8 @@ impl Session { let version = config.version; let our_params = config.to_transport_params(); - let (accept_bi_tx, accept_bi_rx) = mpsc::channel(1024); - let (accept_uni_tx, accept_uni_rx) = mpsc::channel(1024); + let (accept_bi_tx, accept_bi_rx) = mpsc::channel(ACCEPT_QUEUE_CAPACITY); + let (accept_uni_tx, accept_uni_rx) = mpsc::channel(ACCEPT_QUEUE_CAPACITY); let outbound = PriorityQueue::new(8); // Control lane (lossless): RESET/STOP/CLOSE, window updates, ping requests, @@ -2337,7 +2348,25 @@ impl Session { let teardown_socket_stats = SocketStatsGuard(socket_stats.clone()); tokio::spawn(async move { - let err = backend.run().await.err().unwrap_or(Error::Closed); + let mut closed = backend.closed.subscribe(); + let err = { + let run = backend.run(); + tokio::pin!(run); + // Release stats on closure even when the reader is parked on an + // accept queue. Keep polling the same future afterwards so streams + // and frames already received can still reach the application. + let result = tokio::select! { + result = &mut run => Some(result), + _ = async { closed.wait_for(|err| err.is_some()).await.ok(); } => None, + }; + drop(teardown_socket_stats); + match result { + Some(result) => result, + None => run.await, + } + } + .err() + .unwrap_or(Error::Closed); // If we tore down because of a protocol/transport violation *we* // detected, tell the peer with a CONNECTION_CLOSE (0x1c) so their // session rejects too, rather than seeing a bare drop. Enqueue on the @@ -2365,7 +2394,6 @@ impl Session { backend.conn_recv_credit.close(); backend.outbound.close(); backend.streams.lock().unwrap().close(); - drop(teardown_socket_stats); // Publish the backend's reason after cleanup, preserving an earlier // close signal such as APPLICATION_CLOSE rather than replacing it // with the transport's subsequent EOF or WebSocket Close. @@ -5494,7 +5522,7 @@ mod socket_stats_tests { use super::*; use std::time::Duration; use tokio::sync::oneshot; - use web_transport_trait::{Session as _, Stats as _}; + use web_transport_trait::{RecvStream as _, Session as _, Stats as _}; struct Stats { panic: AtomicBool, @@ -5513,6 +5541,7 @@ mod socket_stats_tests { incoming: mpsc::UnboundedReceiver, received: mpsc::UnboundedSender<()>, failed: oneshot::Receiver<()>, + closing: oneshot::Sender<()>, } struct TestReader { @@ -5520,7 +5549,10 @@ mod socket_stats_tests { received: mpsc::UnboundedSender<()>, } - struct TestWriter(oneshot::Receiver<()>); + struct TestWriter { + failed: oneshot::Receiver<()>, + closing: Option>, + } impl Transport for TestTransport { type Reader = TestReader; @@ -5532,7 +5564,10 @@ mod socket_stats_tests { fn split(self) -> (Self::Writer, Self::Reader) { ( - TestWriter(self.failed), + TestWriter { + failed: self.failed, + closing: Some(self.closing), + }, TestReader { incoming: self.incoming, received: self.received, @@ -5554,10 +5589,11 @@ mod socket_stats_tests { Ok(()) } async fn close(&mut self) -> Result<(), Error> { - Ok(()) + self.closing.take().unwrap().send(()).ok(); + std::future::pending().await } async fn maintain(&mut self) -> Result<(), Error> { - let _ = (&mut self.0).await; + let _ = (&mut self.failed).await; Err(Error::Closed) } } @@ -5568,6 +5604,7 @@ mod socket_stats_tests { received: mpsc::UnboundedReceiver<()>, fail: oneshot::Sender<()>, released: oneshot::Receiver<()>, + closing: oneshot::Receiver<()>, } fn fixture(panic: bool) -> Fixture { @@ -5579,12 +5616,14 @@ mod socket_stats_tests { let (incoming, reader) = mpsc::unbounded_channel(); let (received, received_rx) = mpsc::unbounded_channel(); let (fail, failed) = oneshot::channel(); + let (closing, close_started) = oneshot::channel(); let session = Session::new( TestTransport { stats, incoming: reader, received, failed, + closing, }, false, Config::new(Version::WebTransport), @@ -5595,6 +5634,7 @@ mod socket_stats_tests { received: received_rx, fail, released: done, + closing: close_started, } } @@ -5606,24 +5646,35 @@ mod socket_stats_tests { } #[tokio::test(start_paused = true)] - async fn full_accept_queue_does_not_retain_stats_on_close_or_writer_failure() { + async fn full_accept_queue_releases_stats_without_losing_received_streams() { for dir in [StreamDir::Uni, StreamDir::Bi] { for writer_failure in [false, true] { let mut f = fixture(false); - for index in 0..=1024 { - let frame = Frame::Stream(Stream { - id: StreamId::new(index, dir, true), + let frame = |index| { + Frame::Stream(Stream { + id: StreamId::new(index as u64, dir, true), offset: 0, - data: Bytes::new(), - fin: false, - }); - f.incoming - .send(frame.encode(Version::WebTransport).unwrap()) + data: Bytes::from_static(b"received"), + fin: true, + }) + .encode(Version::WebTransport) + .unwrap() + }; + for index in 0..=ACCEPT_QUEUE_CAPACITY { + f.incoming.send(frame(index)).unwrap(); + tokio::time::timeout(Duration::from_secs(1), f.received.recv()) + .await + .expect("reader stopped before accept queue filled") .unwrap(); - // On this single-threaded runtime the reader processes the - // frame until it yields before this notification is observed. - f.received.recv().await.unwrap(); } + // A further frame cannot reach recv() while delivery of the + // preceding stream is parked on the full accept queue. + f.incoming.send(frame(ACCEPT_QUEUE_CAPACITY + 1)).unwrap(); + assert!( + tokio::time::timeout(Duration::from_secs(1), f.received.recv()) + .await + .is_err() + ); if writer_failure { f.fail.send(()).unwrap(); } else { @@ -5631,10 +5682,33 @@ mod socket_stats_tests { } released(f.released).await; assert_eq!(f.session.stats().rtt(), None); + tokio::time::timeout(Duration::from_secs(1), async { + for _ in 0..ACCEPT_QUEUE_CAPACITY + 2 { + let mut recv = match dir { + StreamDir::Uni => f.session.accept_uni().await.unwrap(), + StreamDir::Bi => f.session.accept_bi().await.unwrap().1, + }; + assert_eq!(recv.read_all().await.unwrap(), b"received"[..]); + } + }) + .await + .expect("received streams did not drain after closure"); } } } + #[tokio::test(start_paused = true)] + async fn blocked_transport_close_does_not_retain_socket_stats() { + let f = fixture(false); + f.session.close(0, "done"); + tokio::time::timeout(Duration::from_secs(1), f.closing) + .await + .expect("writer did not enter close") + .unwrap(); + released(f.released).await; + assert_eq!(f.session.stats().rtt(), None); + } + #[tokio::test(start_paused = true)] async fn panicking_stats_do_not_poison_future_reads_or_teardown() { let f = fixture(true);