diff --git a/quest/a0/README.md b/quest/a0/README.md index 10d6d8e1..7c5e3708 100644 --- a/quest/a0/README.md +++ b/quest/a0/README.md @@ -12,6 +12,5 @@ The Required list is ordered by priority. Independent children may proceed concu - [Quinn final Session drop](/quest/a0/quinn-session-drop.md) - close the connection and release the capsule reader - [Quiche lifecycle](/quest/a0/quiche/README.md) - deterministic harness and retirement handling -- [QMux socket teardown](/quest/a0/qmux-socket-teardown.md) - release retained stats descriptors - [QMux initial stream-count bound](/quest/a0/qmux-max-streams-param.md) - refuse initial_max_streams parameters above 2^60 - [Node session and reader lifetime](/quest/a0/node-session-lifetime.md) - `closed` rejects on an abnormal end, and an abandoned reader releases its stream diff --git a/quest/a0/qmux-socket-teardown.md b/quest/a0/qmux-socket-teardown.md deleted file mode 100644 index 8758b8a7..00000000 --- a/quest/a0/qmux-socket-teardown.md +++ /dev/null @@ -1,15 +0,0 @@ -# [M] Release socket-stat handles at transport teardown - -## Goal - -QMux transport termination releases its duplicated stats descriptor even when the application retains a Session clone. - -## Plan - -`TcpStats` duplicates the socket descriptor and `Session` currently retains it independently of the reader/writer tasks. Move that ownership into shared terminal-state cleanup so normal close, peer failure and blocked-writer termination release it exactly once without racing stats reads. - -Approved behavior: socket-derived metrics become unavailable after teardown; other retained session counters remain available. Do not retain a final socket snapshot. Cover TCP and upgraded WebSocket paths, and verify peer-observable EOF or descriptor release while holding a Session clone. Preserve stats while the transport is active. Check PRs #412 and #413 before editing overlapping close paths; do not automatically backport or publish. - -## Closes - -- [#383](https://github.com/moq-dev/web-transport/issues/383) - the implementation completes this report diff --git a/rs/qmux/README.md b/rs/qmux/README.md index f42f36c8..1a6d921f 100644 --- a/rs/qmux/README.md +++ b/rs/qmux/README.md @@ -75,6 +75,15 @@ grow a response queue by sending more pings; inbound stream processing continues while the writer is blocked. Pending responses are discarded during teardown, and a close frame ends the writer's output. +## Socket statistics + +Sessions release their socket statistics handle during teardown, even if the +application retains a `Session` clone. Once closure is signalled, `stats()` calls +omit socket-derived metrics; QMux's own RTT estimate and stream counters remain +available. `closed()` reports the terminal reason without waiting for asynchronous +transport cleanup and descriptor release to finish. A caller +that retains its own `SharedSocketStats` copy keeps that handle alive. + ## Stream control validation MAX_STREAM_DATA and STOP_SENDING close the session when they name a receive-only diff --git a/rs/qmux/src/session.rs b/rs/qmux/src/session.rs index 543b3314..43b4396d 100644 --- a/rs/qmux/src/session.rs +++ b/rs/qmux/src/session.rs @@ -28,6 +28,9 @@ use crate::shared::SharedRecv; /// applying backpressure to the whole session. const DATAGRAM_RECV_BUFFER: usize = 1024; +/// Maximum peer-initiated streams awaiting application acceptance per direction. +const ACCEPT_QUEUE_CAPACITY: usize = 1024; + /// How many outbound datagrams to buffer before dropping. The writer pulls from /// this lane; when it stalls on transport backpressure it stops pulling, the lane /// fills, and `send_datagram` drops on a full lane. Kept small so shedding tracks @@ -186,7 +189,26 @@ impl Drop for SessionGuard { } } +/// Releases the transport stats even if the reader task is cancelled or unwinds. +struct SocketStatsGuard(Arc>>); + +impl Drop for SocketStatsGuard { + fn drop(&mut self) { + // A custom stats provider may have panicked during a read. Its poisoned + // lock must not prevent descriptor release or the rest of teardown. + let stats = self + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + drop(stats); + } +} + /// A multiplexed session over a reliable transport. +/// +/// Closing signals the terminal reason immediately; `closed()` does not wait for +/// asynchronous transport cleanup to finish. #[derive(Clone)] pub struct Session { is_server: bool, @@ -260,8 +282,9 @@ pub struct Session { // The transport's own view of the connection, when it has one. Preferred over // `rtt`: the kernel's estimate needs no round trip of ours, so it is available - // from the first moment rather than one probe interval in. - socket_stats: Option, + // from the first moment rather than one probe interval in. Shared with terminal + // cleanup so retained Session clones cannot keep the socket descriptor alive. + socket_stats: Arc>>, // Closes the connection when the last `Session` clone drops. Never read. _guard: Arc, @@ -2138,8 +2161,8 @@ impl Session { let version = config.version; let our_params = config.to_transport_params(); - let (accept_bi_tx, accept_bi_rx) = mpsc::channel(1024); - let (accept_uni_tx, accept_uni_rx) = mpsc::channel(1024); + let (accept_bi_tx, accept_bi_rx) = mpsc::channel(ACCEPT_QUEUE_CAPACITY); + let (accept_uni_tx, accept_uni_rx) = mpsc::channel(ACCEPT_QUEUE_CAPACITY); let outbound = PriorityQueue::new(8); // Control lane (lossless): RESET/STOP/CLOSE, window updates, ping requests, @@ -2196,8 +2219,8 @@ impl Session { // the wire, pulling the outbound queues in priority order and sharing the // stream maps + scalars above with the reader (no message-passing handoff). // Read before the split: the halves go to separate tasks, and the stats - // source belongs to neither. - let socket_stats = transport.socket_stats(); + // source is released by terminal cleanup, independently of Session clones. + let socket_stats = Arc::new(Mutex::new(transport.socket_stats())); let (writer_half, reader_half) = transport.split(); let mut writer = WriterState { writer: writer_half, @@ -2323,7 +2346,9 @@ impl Session { None }; let mut established = established_rx.clone(); + let teardown_socket_stats = SocketStatsGuard(socket_stats.clone()); tokio::spawn(async move { + let mut closed = backend.closed.subscribe(); let handshake = async { if let Some(deadline) = handshake_deadline { if tokio::time::timeout_at(deadline, established.wait_for(|&done| done)) @@ -2335,12 +2360,29 @@ impl Session { } std::future::pending().await }; + let run = async { + let run = backend.run(); + tokio::pin!(run); + // Release stats on closure even when the reader is parked on an + // accept queue. Keep polling the same future afterwards so streams + // and frames already received can still reach the application. + let result = tokio::select! { + result = &mut run => Some(result), + _ = async { closed.wait_for(|err| err.is_some()).await.ok(); } => None, + }; + drop(teardown_socket_stats); + match result { + Some(result) => result, + None => run.await, + } + }; // The deadline starts at construction and remains pollable even if // the reader is parked on application backpressure. Already-readable - // peer input wins over an expired deadline. + // peer input wins over an expired deadline. Dropping `run` on + // timeout also drops the stats guard. let err = tokio::select! { biased; - result = backend.run() => result.err().unwrap_or(Error::Closed), + result = run => result.err().unwrap_or(Error::Closed), err = handshake => err, }; // If we tore down because of a protocol/transport violation *we* @@ -2370,9 +2412,9 @@ impl Session { backend.conn_recv_credit.close(); backend.outbound.close(); backend.streams.lock().unwrap().close(); - // Recorded last, so a `closed()` waiter sees credits and streams closed. - // The reader often fails (EOF, WebSocket Close) right after an - // APPLICATION_CLOSE, so keep the reason recorded first. + // Publish the backend's reason after cleanup, preserving an earlier + // close signal such as APPLICATION_CLOSE rather than replacing it + // with the transport's subsequent EOF or WebSocket Close. note_closed(&backend.closed, err); }); @@ -2413,7 +2455,6 @@ impl generic::Session for Session { type Error = Error; fn stats(&self) -> impl generic::Stats { - let socket = self.socket_stats.as_ref(); let streams = if self.config.version.is_qmux() { let state = self.streams.lock().unwrap(); let closed = self.closed.borrow(); @@ -2440,6 +2481,17 @@ impl generic::Session for Session { } else { None }; + // Hold the lock through the reads so teardown cannot drop the descriptor + // during a read, and a reader cannot retain it after cleanup completes. + let socket = self + .socket_stats + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let socket = if self.closed.borrow().is_none() { + socket.as_ref() + } else { + None + }; SessionStats { streams, // Fall back to our own QX_PING measurement only when the transport @@ -5781,3 +5833,237 @@ mod credit_update_tests { } } } + +#[cfg(test)] +mod socket_stats_tests { + use super::*; + use std::time::Duration; + use tokio::sync::oneshot; + use web_transport_trait::{RecvStream as _, Session as _, Stats as _}; + + struct Stats { + panic: AtomicBool, + _released: oneshot::Sender<()>, + } + + impl crate::SocketStats for Stats { + fn rtt(&self) -> Option { + assert!(!self.panic.swap(false, Ordering::Relaxed), "stats panic"); + Some(Duration::from_millis(1)) + } + } + + struct TestTransport { + stats: crate::SharedSocketStats, + incoming: mpsc::UnboundedReceiver, + received: mpsc::UnboundedSender<()>, + failed: oneshot::Receiver<()>, + closing: oneshot::Sender<()>, + } + + struct TestReader { + incoming: mpsc::UnboundedReceiver, + received: mpsc::UnboundedSender<()>, + } + + struct TestWriter { + failed: oneshot::Receiver<()>, + closing: Option>, + } + + impl Transport for TestTransport { + type Reader = TestReader; + type Writer = TestWriter; + + fn socket_stats(&self) -> Option { + Some(self.stats.clone()) + } + + fn split(self) -> (Self::Writer, Self::Reader) { + ( + TestWriter { + failed: self.failed, + closing: Some(self.closing), + }, + TestReader { + incoming: self.incoming, + received: self.received, + }, + ) + } + } + + impl Reader for TestReader { + async fn recv(&mut self) -> Result { + let data = self.incoming.recv().await.ok_or(Error::Closed)?; + self.received.send(()).unwrap(); + Ok(data) + } + } + + impl Writer for TestWriter { + async fn send(&mut self, _: Bytes) -> Result<(), Error> { + Ok(()) + } + async fn close(&mut self) -> Result<(), Error> { + self.closing.take().unwrap().send(()).ok(); + std::future::pending().await + } + async fn maintain(&mut self) -> Result<(), Error> { + let _ = (&mut self.failed).await; + Err(Error::Closed) + } + } + + struct Fixture { + session: Session, + incoming: mpsc::UnboundedSender, + received: mpsc::UnboundedReceiver<()>, + fail: oneshot::Sender<()>, + released: oneshot::Receiver<()>, + closing: oneshot::Receiver<()>, + } + + fn fixture(panic: bool) -> Fixture { + let (released, done) = oneshot::channel(); + let stats = Arc::new(Stats { + panic: AtomicBool::new(panic), + _released: released, + }); + let (incoming, reader) = mpsc::unbounded_channel(); + let (received, received_rx) = mpsc::unbounded_channel(); + let (fail, failed) = oneshot::channel(); + let (closing, close_started) = oneshot::channel(); + let session = Session::new( + TestTransport { + stats, + incoming: reader, + received, + failed, + closing, + }, + false, + Config::new(Version::WebTransport), + ); + Fixture { + session, + incoming, + received: received_rx, + fail, + released: done, + closing: close_started, + } + } + + async fn released(done: oneshot::Receiver<()>) { + tokio::time::timeout(Duration::from_secs(1), done) + .await + .expect("stats retained after termination") + .expect_err("stats signal by dropping"); + } + + #[tokio::test(start_paused = true)] + async fn full_accept_queue_releases_stats_without_losing_received_streams() { + for dir in [StreamDir::Uni, StreamDir::Bi] { + for writer_failure in [false, true] { + let mut f = fixture(false); + let frame = |index| { + Frame::Stream(Stream { + id: StreamId::new(index as u64, dir, true), + offset: 0, + data: Bytes::from_static(b"received"), + fin: true, + }) + .encode(Version::WebTransport) + .unwrap() + }; + for index in 0..=ACCEPT_QUEUE_CAPACITY { + f.incoming.send(frame(index)).unwrap(); + tokio::time::timeout(Duration::from_secs(1), f.received.recv()) + .await + .expect("reader stopped before accept queue filled") + .unwrap(); + } + // A further frame cannot reach recv() while delivery of the + // preceding stream is parked on the full accept queue. + f.incoming.send(frame(ACCEPT_QUEUE_CAPACITY + 1)).unwrap(); + assert!( + tokio::time::timeout(Duration::from_secs(1), f.received.recv()) + .await + .is_err() + ); + if writer_failure { + f.fail.send(()).unwrap(); + } else { + f.session.close(0, "done"); + } + released(f.released).await; + assert_eq!(f.session.stats().rtt(), None); + tokio::time::timeout(Duration::from_secs(1), async { + for _ in 0..ACCEPT_QUEUE_CAPACITY + 2 { + let mut recv = match dir { + StreamDir::Uni => f.session.accept_uni().await.unwrap(), + StreamDir::Bi => f.session.accept_bi().await.unwrap().1, + }; + assert_eq!(recv.read_all().await.unwrap(), b"received"[..]); + } + }) + .await + .expect("received streams did not drain after closure"); + } + } + } + + #[tokio::test(start_paused = true)] + async fn blocked_transport_close_does_not_retain_socket_stats() { + let f = fixture(false); + f.session.close(0, "done"); + tokio::time::timeout(Duration::from_secs(1), f.closing) + .await + .expect("writer did not enter close") + .unwrap(); + released(f.released).await; + assert_eq!(f.session.stats().rtt(), None); + } + + #[tokio::test(start_paused = true)] + async fn panicking_stats_do_not_poison_future_reads_or_teardown() { + let f = fixture(true); + assert!( + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| f.session.stats())).is_err() + ); + assert_eq!(f.session.stats().rtt(), Some(Duration::from_millis(1))); + drop(f.incoming); + released(f.released).await; + assert!(matches!(f.session.closed().await, Error::Closed)); + assert_eq!(f.session.stats().rtt(), None); + } + + #[test] + fn runtime_shutdown_releases_stats_with_retained_session() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let mut f = runtime.block_on(async { fixture(false) }); + drop(runtime); + assert!(matches!( + f.released.try_recv(), + Err(oneshot::error::TryRecvError::Closed) + )); + assert_eq!(f.session.stats().rtt(), None); + } + + #[tokio::test(start_paused = true)] + async fn close_hides_socket_metrics_and_preserves_qmux_rtt() { + let f = fixture(false); + let sent = tokio::time::Instant::now(); + f.session.rtt.sent(0, sent); + f.session.rtt.received(0, sent + Duration::from_millis(2)); + assert_eq!(f.session.stats().rtt(), Some(Duration::from_millis(1))); + f.session.close(0, "done"); + f.session.closed().await; + assert_eq!(f.session.stats().rtt(), Some(Duration::from_millis(2))); + released(f.released).await; + } +} diff --git a/rs/qmux/src/socket.rs b/rs/qmux/src/socket.rs index 615b1448..610284a6 100644 --- a/rs/qmux/src/socket.rs +++ b/rs/qmux/src/socket.rs @@ -14,7 +14,8 @@ use std::time::Duration; /// /// Implementations read through to the socket on each call rather than caching, so /// a caller polling this sees the transport's current view. Every metric is -/// optional: `None` means this transport doesn't track it. +/// optional: `None` means this transport doesn't track it. Sessions release their +/// statistics handle during teardown and omit these metrics once closure is signalled. pub trait SocketStats: Send + Sync + 'static { /// Smoothed round-trip time, if the transport tracks one. fn rtt(&self) -> Option { @@ -43,9 +44,10 @@ mod unix { /// `TCP_CONNECTION_INFO` (macOS). /// /// Holds its own duplicate of the file descriptor, so it stays readable for as - /// long as the session needs it no matter what the transport does with the + /// long as this handle lives no matter what the transport does with the /// original. Construct it from the accepted socket and hand it to the transport - /// with `with_socket_stats`. + /// with `with_socket_stats`. The session releases its handle during teardown; + /// any copies retained by the caller keep the descriptor alive. /// /// Reports nothing for a non-TCP socket (a Unix socket, say): the `getsockopt` /// simply fails and every metric reads `None`. diff --git a/rs/qmux/tests/socket_teardown.rs b/rs/qmux/tests/socket_teardown.rs new file mode 100644 index 00000000..62509406 --- /dev/null +++ b/rs/qmux/tests/socket_teardown.rs @@ -0,0 +1,221 @@ +//! Retained session handles must not retain the socket statistics descriptor. +#![cfg(all(unix, feature = "tcp"))] + +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::Duration; + +use qmux::{Config, Session, SharedSocketStats, SocketStats, TcpStats, Version}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, ReadBuf}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::sync::{oneshot, Notify}; +use web_transport_trait::{SendStream as _, Session as _, Stats as _}; + +// Fields drop in declaration order: the notification follows descriptor release. +struct TrackedStats { + tcp: TcpStats, + _released: oneshot::Sender<()>, +} + +impl SocketStats for TrackedStats { + fn rtt(&self) -> Option { + self.tcp.rtt() + } + + fn estimated_send_rate(&self) -> Option { + self.tcp.estimated_send_rate() + } +} + +fn tracked(socket: &TcpStream) -> (SharedSocketStats, oneshot::Receiver<()>) { + let (released, done) = oneshot::channel(); + ( + Arc::new(TrackedStats { + tcp: TcpStats::new(socket).unwrap(), + _released: released, + }), + done, + ) +} + +async fn sockets() -> (TcpStream, TcpStream) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let (client, server) = tokio::join!( + TcpStream::connect(listener.local_addr().unwrap()), + listener.accept(), + ); + (client.unwrap(), server.unwrap().0) +} + +async fn released(done: oneshot::Receiver<()>, session: &Session) { + tokio::time::timeout(Duration::from_secs(2), done) + .await + .expect("retained Session kept the socket stats descriptor alive") + .expect_err("the sender only signals by dropping"); + assert_eq!(session.stats().rtt(), None); + assert_eq!(session.stats().estimated_send_rate(), None); +} + +fn active(session: &Session) { + #[cfg(any(target_os = "linux", target_os = "macos"))] + assert!(session.stats().rtt().is_some()); + // Stream counters remain available independently of socket stats. + assert!(session.stats().streams().is_some()); +} + +#[tokio::test] +async fn tcp_close_releases_stats_with_retained_clones() { + let (client, server) = sockets().await; + let (client_stats, client_done) = tracked(&client); + let (server_stats, server_done) = tracked(&server); + let mut config = Config::new(Version::QMux02); + // Isolate socket RTT from the QMux estimate, which survives teardown. + config.rtt_interval = Duration::ZERO; + let (client, server) = tokio::join!( + Session::connect( + qmux::transport::Stream::new(client, config.version, config.max_record_size) + .with_socket_stats(client_stats), + config.clone(), + ), + Session::accept( + qmux::transport::Stream::new(server, config.version, config.max_record_size) + .with_socket_stats(server_stats), + config.clone(), + ), + ); + let client = client.unwrap(); + let server = server.unwrap(); + active(&client); + active(&server); + let retained = client.clone(); + client.close(0, "done"); + drop(client); + released(client_done, &retained).await; + released(server_done, &server).await; + assert!(retained.stats().streams().is_some()); + assert!(server.stats().streams().is_some()); +} + +#[tokio::test] +async fn tcp_peer_eof_releases_stats_with_retained_clone() { + let (client, mut peer) = sockets().await; + let (stats, done) = tracked(&client); + let config = Config::new(Version::WebTransport); + let client = Session::connect( + qmux::transport::Stream::new(client, config.version, config.max_record_size) + .with_socket_stats(stats), + config, + ) + .await + .unwrap(); + let retained = client.clone(); + drop(client); + peer.shutdown().await.unwrap(); + released(done, &retained).await; + assert!(matches!(retained.closed().await, qmux::Error::Io(err) + if err.kind() == std::io::ErrorKind::UnexpectedEof)); +} + +// Gate actual socket writes so cancellation skips graceful shutdown. Without +// releasing TcpStats, dropping the transport halves alone cannot deliver EOF. +struct GatedSocket { + socket: TcpStream, + entered: Arc, +} + +impl AsyncRead for GatedSocket { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut self.socket).poll_read(cx, buf) + } +} + +impl AsyncWrite for GatedSocket { + fn poll_write( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + _buf: &[u8], + ) -> Poll> { + self.entered.notify_one(); + Poll::Pending + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.socket).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.socket).poll_shutdown(cx) + } +} + +#[tokio::test] +async fn tcp_blocked_writer_releases_stats_and_delivers_eof() { + let (socket, mut peer) = sockets().await; + let (stats, done) = tracked(&socket); + let entered = Arc::new(Notify::new()); + let config = Config::new(Version::WebTransport); + let client = Session::connect( + qmux::transport::Stream::new( + GatedSocket { + socket, + entered: entered.clone(), + }, + config.version, + config.max_record_size, + ) + .with_socket_stats(stats), + config, + ) + .await + .unwrap(); + let mut send = client.open_uni().await.unwrap(); + send.write(b"blocked").await.unwrap(); + tokio::time::timeout(Duration::from_secs(2), entered.notified()) + .await + .expect("writer never blocked"); + let retained = client.clone(); + client.close(0, "done"); + drop(client); + released(done, &retained).await; + let mut bytes = Vec::new(); + tokio::time::timeout(Duration::from_secs(2), peer.read_to_end(&mut bytes)) + .await + .expect("peer never observed EOF while Session was retained") + .unwrap(); +} + +#[cfg(feature = "ws")] +#[tokio::test] +async fn upgraded_websocket_releases_stats_with_retained_clone() { + use qmux::ws::{tokio_tungstenite, Upgraded}; + + for local_close in [false, true] { + let (client, peer) = sockets().await; + let (stats, done) = tracked(&client); + let (client, peer) = tokio::join!( + tokio_tungstenite::client_async("ws://localhost/", client), + tokio_tungstenite::accept_async(peer), + ); + let client = Upgraded::new(client.unwrap().0) + .with_socket_stats(stats) + .connect(); + let peer = peer.unwrap(); + #[cfg(any(target_os = "linux", target_os = "macos"))] + assert!(client.stats().rtt().is_some()); + let retained = client.clone(); + drop(client); + if local_close { + retained.close(0, "done"); + released(done, &retained).await; + } else { + // An abrupt failure of the upgraded socket must release the duplicate too. + drop(peer); + released(done, &retained).await; + } + } +}