From 32e4d7eed7bef85dade218e2bfac89efad1c04b7 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:04:41 -0800 Subject: [PATCH 1/3] chore(quest): claim quest/m1/webtransport-close-reason.md Quest-Claim: 55aca416798ca2c2 From 006e3d8f2b454beafd83c177ee11f2ac76053cd7 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:23:53 -0800 Subject: [PATCH 2/3] fix(tokio,uring)!: a WebTransport stream reset by a closed session reads its close reason A closing session now resets and stops its released streams with WT_SESSION_GONE (draft-ietf-webtrans-http3 4.4), and a reader that sees WT_SESSION_GONE before the CLOSE_WEBTRANSPORT_SESSION capsule waits for the capsule or the connection close, so a handshake refusal reports the peer's code and reason instead of a reset of 0. Error::SessionGone is removed: it can no longer surface. Co-Authored-By: Claude Opus 5.5 --- quest/m1/webtransport-close-reason.md | 36 ----------- rs/moq-tokio/src/quic/web/error.rs | 89 +++++++++++++++++++-------- rs/moq-tokio/src/quic/web/recv.rs | 80 ++++++++++++++++-------- rs/moq-tokio/src/quic/web/send.rs | 70 ++++++++++++--------- rs/moq-tokio/src/quic/web/session.rs | 47 ++++++++++++++ rs/moq-uring/src/quic/web.rs | 30 ++++++--- rs/moq-uring/src/quic/web/error.rs | 6 +- rs/moq-uring/src/quic/web/recv.rs | 80 ++++++++++++++++-------- rs/moq-uring/src/quic/web/send.rs | 72 ++++++++++++---------- 9 files changed, 321 insertions(+), 189 deletions(-) delete mode 100644 quest/m1/webtransport-close-reason.md diff --git a/quest/m1/webtransport-close-reason.md b/quest/m1/webtransport-close-reason.md deleted file mode 100644 index ed980b71553..00000000000 --- a/quest/m1/webtransport-close-reason.md +++ /dev/null @@ -1,36 +0,0 @@ -# [S] A WebTransport client always reads the peer's close reason - -## Goal - -When a peer closes a WebTransport session, the client's error carries the -peer's close code and reason, never `cancelled`, whichever arrives first: a -stream reset or the CLOSE_WEBTRANSPORT_SESSION capsule. That covers every -handshake refusal, such as a version mismatch or a refused token, in -`moq-tokio` and `moq-uring`. - -## Plan - -Follow-up of [moq-dev/moq#8478](https://github.com/moq-dev/moq/pull/8478): -the relay closes the session with the version code (0x15), but its SETUP -stream's reset (code 0) reaches the client before the capsule, so the client -reports `cancelled`. Closing the session before releasing its streams won the -race only sometimes, so ordering on the sender is not the fix. - -Recommended shape (draft-ietf-webtrans-http3 §4.4): a closing session resets -and stops its open streams with WT_SESSION_GONE, and a reader that sees -WT_SESSION_GONE before the capsule waits for it, ending on the connection's -close if none comes. Reading WT_SESSION_GONE already exists -(`rs/moq-tokio/src/quic/web/error.rs`, `session_gone`), but it reports -`Error::SessionGone` until the capsule lands, and nothing sends it. The same -code lives in `rs/moq-uring/src/quic/web/`; keep the two in step, and check -whether `js/net`'s native WebTransport path has the same race. - -Test the race both ways on the paused clock: the reset first and the capsule -first both yield the peer's code and reason. - -Decided 2026-11-11 (maintainer, follow-up planning in /quest-complete): ✅ m1 -(rejected: m3, drop): a non-breaking fix that every handshake refusal's -diagnosis needs. - -Public API: none. Wire: WT_SESSION_GONE on stream resets, as the WebTransport -draft specifies. diff --git a/rs/moq-tokio/src/quic/web/error.rs b/rs/moq-tokio/src/quic/web/error.rs index 40f2dce8c26..4ef40a26219 100644 --- a/rs/moq-tokio/src/quic/web/error.rs +++ b/rs/moq-tokio/src/quic/web/error.rs @@ -1,4 +1,4 @@ -use std::sync::OnceLock; +use std::{sync::Mutex, task::Poll}; /// Why a WebTransport (or raw QUIC) session failed: its handshake, its connection, or a close. #[derive(thiserror::Error, Debug, Clone)] @@ -12,10 +12,6 @@ pub enum Error { #[error("closed: code={0} reason={1}")] Closed(u32, String), - /// The peer tore down the session's streams before its close reason arrived. - #[error("session gone")] - SessionGone, - /// A stream named a different session. #[error("unknown session")] UnknownSession, @@ -97,22 +93,37 @@ const WT_SESSION_GONE: u64 = 0x170d7b68; pub(crate) struct CloseReason { // Raw QUIC carries the application code as is; HTTP/3 maps it into its own code space. raw: bool, - reason: OnceLock, + reason: Mutex, +} + +#[derive(Debug, Default)] +struct Reason { + err: Option, + /// Readers that saw WT_SESSION_GONE before the reason was recorded. + waiters: kio::WaiterList, } impl CloseReason { pub(crate) fn new(raw: bool) -> Self { Self { raw, - reason: OnceLock::new(), + reason: Mutex::default(), } } - /// The QUIC code to reset or stop a stream with. + fn get(&self) -> Option { + self.reason.lock().unwrap().err.clone() + } + + /// The QUIC code to reset or stop a stream with: WT_SESSION_GONE once the session closed, + /// which tells the peer to read the session's close reason instead (draft-ietf-webtrans-http3 §4.4). pub(crate) fn encode_stream_code(&self, code: u32) -> crate::quic::driver::VarInt { if self.raw { return code.into(); } + if self.reason.lock().unwrap().err.is_some() { + return crate::quic::driver::VarInt::from_u64(WT_SESSION_GONE).unwrap(); + } crate::quic::driver::VarInt::try_from(web_transport_proto::error_to_http3(code)).unwrap() } @@ -128,22 +139,42 @@ impl CloseReason { } } - /// The session's close reason when the peer reset or stopped a stream with + /// Poll the session's close reason when the peer reset or stopped a stream with /// WT_SESSION_GONE, which tears down the streams of a closed session /// (draft-ietf-webtrans-http3 §4.4), or None for any other code. /// - /// The reset can beat the close capsule, so until it lands the code and reason are - /// unknown: [`Error::SessionGone`], never a guessed clean close. - pub(crate) fn session_gone(&self, code: crate::quic::driver::VarInt) -> Option { + /// The reset can beat the close capsule, so this is pending until the reason is recorded, + /// which the capsule or the connection's close always does. + pub(crate) fn poll_session_gone( + &self, + waiter: &kio::Waiter, + code: crate::quic::driver::VarInt, + ) -> Option> { if self.raw || code.into_inner() != WT_SESSION_GONE { return None; } - Some(self.reason.get().cloned().unwrap_or(Error::SessionGone)) + let mut reason = self.reason.lock().unwrap(); + Some(match &reason.err { + Some(err) => Poll::Ready(err.clone()), + None => { + reason.waiters.register(waiter); + Poll::Pending + } + }) } /// Record the close reason, returning false if one was already recorded. pub(crate) fn set(&self, err: Error) -> bool { - self.reason.set(err).is_ok() + let mut waiters = { + let mut reason = self.reason.lock().unwrap(); + if reason.err.is_some() { + return false; + } + reason.err = Some(err); + reason.waiters.take() + }; + waiters.wake(); + true } /// Replace an error caused by the connection closing with the recorded reason, or @@ -155,12 +186,12 @@ impl CloseReason { Error::Connection(conn) | Error::Read(recv::Error::Connection(conn)) | Error::Write(send::Error::Connection(conn)) => Some(conn), - Error::Closed(..) | Error::SessionGone | Error::Http3 { .. } => None, + Error::Closed(..) | Error::Http3 { .. } => None, _ => return err, }; - if let Some(reason) = self.reason.get() { - return reason.clone(); + if let Some(reason) = self.get() { + return reason; } let Some(connection::Error::ApplicationClosed(close)) = conn else { @@ -276,22 +307,32 @@ mod tests { assert_eq!(close.decode_stream_code(invalid), None); } - /// WT_SESSION_GONE reports the session's close, before and after its capsule arrives. + /// WT_SESSION_GONE waits for the session's close reason, and a closed session sends it. #[test] - fn session_gone_reads_as_the_close() { + fn session_gone_waits_for_the_close() { let gone = crate::quic::driver::VarInt::from_u64(WT_SESSION_GONE).unwrap(); let close = CloseReason::new(false); - assert!(matches!(close.session_gone(gone), Some(Error::SessionGone))); + struct Flag(std::sync::atomic::AtomicBool); + impl std::task::Wake for Flag { + fn wake(self: std::sync::Arc) { + self.0.store(true, std::sync::atomic::Ordering::SeqCst); + } + } + let flag = std::sync::Arc::new(Flag(Default::default())); + let waiter = kio::Waiter::new(flag.clone().into()); + assert!(matches!(close.poll_session_gone(&waiter, gone), Some(Poll::Pending))); + assert!(close.poll_session_gone(&waiter, close.encode_stream_code(7)).is_none()); close.set(Error::Closed(7, "bye".into())); + assert!(flag.0.load(std::sync::atomic::Ordering::SeqCst)); assert!(matches!( - close.session_gone(gone), - Some(Error::Closed(7, reason)) if reason == "bye" + close.poll_session_gone(&waiter, gone), + Some(Poll::Ready(Error::Closed(7, reason))) if reason == "bye" )); - assert!(close.session_gone(close.encode_stream_code(7)).is_none()); + assert_eq!(close.encode_stream_code(7), gone); // Raw QUIC has no HTTP/3 code space, so the same number is an application code. - assert!(CloseReason::new(true).session_gone(gone).is_none()); + assert!(CloseReason::new(true).poll_session_gone(&waiter, gone).is_none()); } /// A peer close maps into WebTransport code space, or is HTTP/3's own failure when its diff --git a/rs/moq-tokio/src/quic/web/recv.rs b/rs/moq-tokio/src/quic/web/recv.rs index 272e01657b2..1ed5ab57255 100644 --- a/rs/moq-tokio/src/quic/web/recv.rs +++ b/rs/moq-tokio/src/quic/web/recv.rs @@ -14,7 +14,7 @@ use crate::quic::web::CloseReason; /// A stream that can be used to receive bytes. See [`crate::quic::driver::recv::Stream`]. /// -/// Dropping it before the end stops it with code 0. +/// Dropping it before the end stops it with code 0, or WT_SESSION_GONE once the session closed. #[derive(Debug)] pub struct Stream { /// Taken when moq-net stops through its mutable poll trait. @@ -24,9 +24,7 @@ pub struct Stream { } impl Stream { - pub(crate) fn new(mut stream: recv::Stream, close: Arc) -> Self { - // The driver's raw 0 is outside the HTTP/3 range, so a WebTransport peer can't decode it. - stream.set_drop_stop(close.encode_stream_code(0)); + pub(crate) fn new(stream: recv::Stream, close: Arc) -> Self { Self { id: stream.id(), inner: Some(stream), @@ -42,7 +40,11 @@ impl Stream { pub fn poll_lease(&mut self, waiter: &kio::Waiter, want: usize) -> Poll, Error>> { let close = &self.close; match &mut self.inner { - Some(inner) => inner.poll_lease(waiter, want).map_err(|e| map_error(close, e)), + Some(inner) => match inner.poll_lease(waiter, want) { + Poll::Ready(Ok(lease)) => Poll::Ready(Ok(lease)), + Poll::Ready(Err(e)) => poll_error(close, waiter, e).map(Err), + Poll::Pending => Poll::Pending, + }, None => Poll::Ready(Err(Error::ClosedStream)), } } @@ -57,8 +59,8 @@ impl Stream { }; match ready!(inner.poll_received_reset(waiter)) { Ok(None) => Poll::Pending, - Ok(Some(code)) => Poll::Ready(map_error(close, recv::Error::Reset(code))), - Err(err) => Poll::Ready(map_error(close, err)), + Ok(Some(code)) => poll_error(close, waiter, recv::Error::Reset(code)), + Err(err) => poll_error(close, waiter, err), } } @@ -92,41 +94,50 @@ impl Stream { }; match ready!(inner.poll_closed(waiter)) { Ok(None) => Poll::Ready(Ok(())), - Ok(Some(code)) => Poll::Ready(Err(map_error(close, recv::Error::Reset(code)))), - Err(err) => Poll::Ready(Err(map_error(close, err))), + Ok(Some(code)) => poll_error(close, waiter, recv::Error::Reset(code)).map(Err), + Err(err) => poll_error(close, waiter, err).map(Err), } } /// Poll for some bytes without retaining the destination buffer. pub fn poll_read(&mut self, waiter: &kio::Waiter, mut buf: &mut [u8]) -> Poll, Error>> { let close = self.close.clone(); - self.inner()? - .poll_read_buf(waiter, &mut buf) - .map_err(|err| map_error(&close, err)) + match ready!(self.inner()?.poll_read_buf(waiter, &mut buf)) { + Ok(n) => Poll::Ready(Ok(n)), + Err(err) => poll_error(&close, waiter, err).map(Err), + } } /// Read some data into `buf` and return the amount read. See [`recv::Stream::read`]. pub async fn read(&mut self, buf: &mut impl BufMut) -> Result, Error> { - let result = self.inner()?.read(buf).await; - result.map_err(|e| map_error(&self.close, e)) + match self.inner()?.read(buf).await { + Ok(res) => Ok(res), + Err(e) => Err(error(&self.close, e).await), + } } /// Fill the entire buffer with data. See [`recv::Stream::read_exact`]. pub async fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), Error> { - let result = self.inner()?.read_exact(buf).await; - result.map_err(|e| map_error(&self.close, e)) + match self.inner()?.read_exact(buf).await { + Ok(res) => Ok(res), + Err(e) => Err(error(&self.close, e).await), + } } /// Read a chunk of data from the stream. See [`recv::Stream::read_chunk`]. pub async fn read_chunk(&mut self, max: usize) -> Result, Error> { - let result = self.inner()?.read_chunk(max).await; - result.map_err(|e| map_error(&self.close, e)) + match self.inner()?.read_chunk(max).await { + Ok(res) => Ok(res), + Err(e) => Err(error(&self.close, e).await), + } } /// Read until the end of the stream or the limit is hit. See [`recv::Stream::read_to_end`]. pub async fn read_to_end(&mut self, limit: usize) -> Result, Error> { - let result = self.inner()?.read_to_end(limit).await; - result.map_err(|e| map_error(&self.close, e)) + match self.inner()?.read_to_end(limit).await { + Ok(res) => Ok(res), + Err(e) => Err(error(&self.close, e).await), + } } /// Block until the stream has been reset and return the error code. See @@ -140,11 +151,11 @@ impl Stream { }; match inner.received_reset().await { Ok(None) => Ok(None), - Ok(Some(code)) => match map_error(&self.close, recv::Error::Reset(code)) { + Ok(Some(code)) => match error(&self.close, recv::Error::Reset(code)).await { Error::Reset(code) => Ok(Some(code)), err => Err(err), }, - Err(err) => Err(map_error(&self.close, err)), + Err(err) => Err(error(&self.close, err).await), } } @@ -162,13 +173,13 @@ impl Stream { } /// Decode the peer's reset code, and report a session error, or a reset for a session -/// that is gone, as the session's close reason. -fn map_error(close: &CloseReason, e: recv::Error) -> Error { - match e { +/// that is gone, as the session's close reason, pending until that reason is known. +fn poll_error(close: &CloseReason, waiter: &kio::Waiter, e: recv::Error) -> Poll { + Poll::Ready(match e { recv::Error::Reset(code) => match close.decode_stream_code(code) { Some(code) => Error::Reset(code), - None => match close.session_gone(code) { - Some(err) => Error::Session(err), + None => match close.poll_session_gone(waiter, code) { + Some(err) => return err.map(Error::Session), None => Error::InvalidReset(code), }, }, @@ -176,6 +187,21 @@ fn map_error(close: &CloseReason, e: recv::Error) -> Error { Error::Session(e) => Error::Session(close.map(e)), e => e, }, + }) +} + +/// [`poll_error`] as a future. +async fn error(close: &CloseReason, e: recv::Error) -> Error { + kio::wait(|waiter| poll_error(close, waiter, e.clone())).await +} + +impl Drop for Stream { + fn drop(&mut self) { + // Chosen at drop, since the session may have closed since the stream opened. The + // driver's raw 0 is outside the HTTP/3 range, so a WebTransport peer can't decode it. + if let Some(inner) = &mut self.inner { + inner.set_drop_stop(self.close.encode_stream_code(0)); + } } } diff --git a/rs/moq-tokio/src/quic/web/send.rs b/rs/moq-tokio/src/quic/web/send.rs index 5cb22b48842..b1f68a5c243 100644 --- a/rs/moq-tokio/src/quic/web/send.rs +++ b/rs/moq-tokio/src/quic/web/send.rs @@ -17,7 +17,8 @@ use crate::quic::web::CloseReason; /// /// A stream this side opened writes its WebTransport header ahead of its first data, so opening /// one waits only for the peer's stream credit. Dropping it before [`finish`](Self::finish) -/// resets it with code 0, still delivering the header once written. +/// resets it with code 0, or WT_SESSION_GONE once the session closed, still delivering the header +/// once written. /// /// This wrapper is mainly needed for error codes, which is unfortunate. /// WebTransport uses u32 error codes and they're mapped in a reserved HTTP/3 error space. @@ -49,8 +50,7 @@ enum State { impl Stream { /// Wrap `stream`, writing `header` ahead of its data: a WebTransport stream this side /// opened, or empty for one without a header. - pub(crate) fn new(mut stream: send::Stream, close: Arc, header: Bytes) -> Self { - stream.set_drop_reset(None, close.encode_stream_code(0)); + pub(crate) fn new(stream: send::Stream, close: Arc, header: Bytes) -> Self { let mut this = Self { id: stream.id(), state: State::Open(stream), @@ -75,11 +75,10 @@ impl Stream { }; while self.unwritten.has_remaining() { if let Err(err) = std::task::ready!(stream.poll_write_buf(waiter, &mut self.unwritten)) { - return Poll::Ready(Err(self.map_error(err))); + return poll_error(&self.close, waiter, err).map(Err); } } let offset = stream.offset(); - stream.set_drop_reset(Some(offset), self.close.encode_stream_code(0)); self.header = Some(offset); Poll::Ready(Ok(())) } @@ -87,25 +86,9 @@ impl Stream { /// Write from `buf` after the header, returning the size written. fn poll_write_buf(&mut self, waiter: &kio::Waiter, buf: &mut impl Buf) -> Poll> { std::task::ready!(self.poll_header(waiter))?; - let result = std::task::ready!(self.open()?.poll_write_buf(waiter, buf)); - Poll::Ready(result.map_err(|e| self.map_error(e))) - } - - /// Decode the peer's stop code, and report a session error, or a stop for a session - /// that is gone, as the session's close reason. - fn map_error(&self, e: send::Error) -> Error { - match e { - send::Error::Stopped(code) => match self.close.decode_stream_code(code) { - Some(code) => Error::Stopped(code), - None => match self.close.session_gone(code) { - Some(err) => Error::Session(err), - None => Error::InvalidStopped(code), - }, - }, - e => match Error::from(e) { - Error::Session(e) => Error::Session(self.close.map(e)), - e => e, - }, + match std::task::ready!(self.open()?.poll_write_buf(waiter, buf)) { + Ok(n) => Poll::Ready(Ok(n)), + Err(e) => poll_error(&self.close, waiter, e).map(Err), } } @@ -164,14 +147,14 @@ impl Stream { State::Reset(result) => Poll::Ready(result.clone()), State::Unsent => return Poll::Ready(Err(Error::Unsent)), }; - result.map(|result| match result { - Ok(None) => Ok(None), - Ok(Some(code)) => match self.map_error(send::Error::Stopped(code)) { + match std::task::ready!(result) { + Ok(None) => Poll::Ready(Ok(None)), + Ok(Some(code)) => poll_error(&self.close, waiter, send::Error::Stopped(code)).map(|err| match err { Error::Stopped(code) => Ok(Some(code)), err => Err(err), - }, - Err(err) => Err(self.map_error(err)), - }) + }), + Err(err) => poll_error(&self.close, waiter, err).map(Err), + } } /// Finish sending, retaining a handle to acknowledgements and peer cancellation. @@ -269,6 +252,33 @@ impl Stream { } } +/// Decode the peer's stop code, and report a session error, or a stop for a session that is +/// gone, as the session's close reason, pending until that reason is known. +fn poll_error(close: &CloseReason, waiter: &kio::Waiter, e: send::Error) -> Poll { + Poll::Ready(match e { + send::Error::Stopped(code) => match close.decode_stream_code(code) { + Some(code) => Error::Stopped(code), + None => match close.poll_session_gone(waiter, code) { + Some(err) => return err.map(Error::Session), + None => Error::InvalidStopped(code), + }, + }, + e => match Error::from(e) { + Error::Session(e) => Error::Session(close.map(e)), + e => e, + }, + }) +} + +impl Drop for Stream { + fn drop(&mut self) { + // Chosen at drop, since the session may have closed since the stream opened. + if let State::Open(stream) = &mut self.state { + stream.set_drop_reset(self.header, self.close.encode_stream_code(0)); + } + } +} + impl tokio::io::AsyncWrite for Stream { fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, mut buf: &[u8]) -> Poll> { let this = self.get_mut(); diff --git a/rs/moq-tokio/src/quic/web/session.rs b/rs/moq-tokio/src/quic/web/session.rs index f4e66e472ba..d07961ea7a4 100644 --- a/rs/moq-tokio/src/quic/web/session.rs +++ b/rs/moq-tokio/src/quic/web/session.rs @@ -1711,6 +1711,53 @@ mod tests { assert!(matches!(bi, crate::quic::web::send::Error::Stopped(0)), "{bi:?}"); } + /// Streams released after a close are reset and stopped with WT_SESSION_GONE, so the peer + /// reads the close's code and reason from them, whether the reset or the capsule lands first. + #[tokio::test] + async fn streams_after_close_carry_the_close_reason() { + let (client_conn, server_conn, _endpoints) = pair().await; + let (client, server) = tokio::join!( + Session::connect(client_conn, url::Url::parse("https://localhost/").unwrap()), + async { crate::quic::web::Request::accept(server_conn).await.unwrap().ok().await }, + ); + let (client, server) = (client.unwrap(), server.unwrap()); + + let mut send = server.open_uni(Default::default()).await.unwrap(); + send.write_all(&b"setup"[..]).await.unwrap(); + let mut client_send = client.open_uni(Default::default()).await.unwrap(); + client_send.write_all(&b"setup"[..]).await.unwrap(); + let mut recv = client.accept_uni().await.unwrap(); + let mut server_recv = server.accept_uni().await.unwrap(); + server_recv.read_chunk(usize::MAX).await.unwrap(); + + server.close(21, "version"); + drop((send, server_recv)); + + let read = async { + let read = loop { + match recv.read_chunk(usize::MAX).await { + Ok(Some(_)) => {} + res => break res, + } + }; + let write = loop { + if let Err(err) = client_send.write(&mut &b"more"[..]).await { + break err; + } + }; + (read, write) + }; + let (read, write) = kio::clock::timeout(Duration::from_secs(10), read).await.unwrap(); + assert!( + matches!(&read, Err(crate::quic::web::recv::Error::Session(Error::Closed(21, reason))) if reason == "version"), + "{read:?}" + ); + assert!( + matches!(&write, crate::quic::web::send::Error::Session(Error::Closed(21, reason)) if reason == "version"), + "{write:?}" + ); + } + /// A stream that names another session is dropped, and the session's own stream after it /// still arrives. #[tokio::test] diff --git a/rs/moq-uring/src/quic/web.rs b/rs/moq-uring/src/quic/web.rs index 0575e903dca..88bab615d6f 100644 --- a/rs/moq-uring/src/quic/web.rs +++ b/rs/moq-uring/src/quic/web.rs @@ -907,22 +907,31 @@ impl Handles { } impl Handles { + /// The QUIC code to reset or stop a stream with: WT_SESSION_GONE once the session closed, + /// which tells the peer to read the session's close reason instead (draft-ietf-webtrans-http3 §4.4). fn encode_stream_code(&self, code: u32) -> VarInt { - if self.web.is_some() { - http3_code(proto::error_to_http3(code)) - } else { - code.into() + match &self.web { + Some(web) if web.closed.borrow().is_some() => http3_code(WT_SESSION_GONE), + Some(_) => http3_code(proto::error_to_http3(code)), + None => code.into(), } } fn decode_stream_code(&self, code: VarInt) -> Option { decode_stream_code(self.web.is_none(), code) } - fn session_gone(&self, code: VarInt) -> Option { - self.web - .as_ref() - .filter(|_| code.into_inner() == 0x170d7b68) - .map(|web| web.closed.borrow().clone().unwrap_or(Error::SessionGone)) + /// Poll the session's close reason when the peer reset or stopped a stream with + /// WT_SESSION_GONE, which tears down the streams of a closed session + /// (draft-ietf-webtrans-http3 §4.4), or None for any other code. + /// + /// The reset can beat the close capsule, so this waits for it, or for the connection's + /// close, which reading the capsule always ends with. + fn poll_session_gone(&self, waiter: &kio::Waiter, code: VarInt) -> Option> { + let web = self.web.as_ref().filter(|_| code.into_inner() == WT_SESSION_GONE)?; + if let Some(reason) = web.closed.borrow().clone() { + return Some(Poll::Ready(reason)); + } + Some(self.conn.poll_closed(waiter).map(|err| self.map(err.into()))) } /// Replace an error caused by the connection closing with the recorded close reason, or /// decode the peer's close code when none was recorded. @@ -943,6 +952,9 @@ impl Drop for Handles { } } +/// The HTTP/3 code a closed session's streams are reset and stopped with. +const WT_SESSION_GONE: u64 = 0x170d7b68; + /// An HTTP/3 code as a QUIC varint. fn http3_code(code: u64) -> VarInt { VarInt::from_u64(code).expect("HTTP/3 codes are varints") diff --git a/rs/moq-uring/src/quic/web/error.rs b/rs/moq-uring/src/quic/web/error.rs index 5b676d50a27..de834e98027 100644 --- a/rs/moq-uring/src/quic/web/error.rs +++ b/rs/moq-uring/src/quic/web/error.rs @@ -12,10 +12,6 @@ pub enum Error { #[error("closed: code={0} reason={1}")] Closed(u32, String), - /// The peer tore down the session's streams before its close reason arrived. - #[error("session gone")] - SessionGone, - /// HTTP/3 closed the connection with a code of its own, one that names no WebTransport /// code (`H3_NO_ERROR`, say), so not a code the peer's application chose. #[error("http/3 closed: code={code} reason={reason:?}")] @@ -106,7 +102,7 @@ impl Error { /// Whether the session's recorded close reason should replace this error: the connection /// closing caused it, or it is a close itself. pub(super) fn is_close(&self) -> bool { - self.connection().is_some() || matches!(self, Self::Closed(..) | Self::SessionGone | Self::Http3 { .. }) + self.connection().is_some() || matches!(self, Self::Closed(..) | Self::Http3 { .. }) } } diff --git a/rs/moq-uring/src/quic/web/recv.rs b/rs/moq-uring/src/quic/web/recv.rs index 90993dfb850..3179e9bdc62 100644 --- a/rs/moq-uring/src/quic/web/recv.rs +++ b/rs/moq-uring/src/quic/web/recv.rs @@ -12,7 +12,7 @@ use crate::quic::driver::recv; /// A stream that can be used to receive bytes. See [`crate::quic::recv::Stream`]. /// -/// Dropping it before the end stops it with code 0. +/// Dropping it before the end stops it with code 0, or WT_SESSION_GONE once the session closed. #[derive(Debug)] pub struct Stream { /// Taken when moq-net stops through its mutable poll trait. @@ -22,9 +22,7 @@ pub struct Stream { } impl Stream { - pub(super) fn new(mut stream: recv::Stream, close: &Rc) -> Self { - // The driver's raw 0 is outside the HTTP/3 range, so a WebTransport peer can't decode it. - stream.set_drop_stop(close.encode_stream_code(0)); + pub(super) fn new(stream: recv::Stream, close: &Rc) -> Self { Self { id: stream.id(), inner: Some(stream), @@ -40,7 +38,11 @@ impl Stream { pub fn poll_lease(&mut self, waiter: &kio::Waiter, want: usize) -> Poll, Error>> { let close = &self.close; match &mut self.inner { - Some(inner) => inner.poll_lease(waiter, want).map_err(|e| map_error(close, e)), + Some(inner) => match inner.poll_lease(waiter, want) { + Poll::Ready(Ok(lease)) => Poll::Ready(Ok(lease)), + Poll::Ready(Err(e)) => poll_error(close, waiter, e).map(Err), + Poll::Pending => Poll::Pending, + }, None => Poll::Ready(Err(Error::ClosedStream)), } } @@ -55,8 +57,8 @@ impl Stream { }; match ready!(inner.poll_received_reset(waiter)) { Ok(None) => Poll::Pending, - Ok(Some(code)) => Poll::Ready(map_error(close, recv::Error::Reset(code))), - Err(err) => Poll::Ready(map_error(close, err)), + Ok(Some(code)) => poll_error(close, waiter, recv::Error::Reset(code)), + Err(err) => poll_error(close, waiter, err), } } @@ -90,41 +92,50 @@ impl Stream { }; match ready!(inner.poll_closed(waiter)) { Ok(None) => Poll::Ready(Ok(())), - Ok(Some(code)) => Poll::Ready(Err(map_error(close, recv::Error::Reset(code)))), - Err(err) => Poll::Ready(Err(map_error(close, err))), + Ok(Some(code)) => poll_error(close, waiter, recv::Error::Reset(code)).map(Err), + Err(err) => poll_error(close, waiter, err).map(Err), } } /// Poll for some bytes without retaining the destination buffer. pub fn poll_read(&mut self, waiter: &kio::Waiter, mut buf: &mut [u8]) -> Poll, Error>> { let close = self.close.clone(); - self.inner()? - .poll_read_buf(waiter, &mut buf) - .map_err(|err| map_error(&close, err)) + match ready!(self.inner()?.poll_read_buf(waiter, &mut buf)) { + Ok(n) => Poll::Ready(Ok(n)), + Err(err) => poll_error(&close, waiter, err).map(Err), + } } /// Read some data into `buf` and return the amount read. See [`recv::Stream::read`]. pub async fn read(&mut self, buf: &mut impl BufMut) -> Result, Error> { - let result = self.inner()?.read(buf).await; - result.map_err(|e| map_error(&self.close, e)) + match self.inner()?.read(buf).await { + Ok(res) => Ok(res), + Err(e) => Err(error(&self.close, e).await), + } } /// Fill the entire buffer with data. See [`recv::Stream::read_exact`]. pub async fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), Error> { - let result = self.inner()?.read_exact(buf).await; - result.map_err(|e| map_error(&self.close, e)) + match self.inner()?.read_exact(buf).await { + Ok(res) => Ok(res), + Err(e) => Err(error(&self.close, e).await), + } } /// Read a chunk of data from the stream. See [`recv::Stream::read_chunk`]. pub async fn read_chunk(&mut self, max: usize) -> Result, Error> { - let result = self.inner()?.read_chunk(max).await; - result.map_err(|e| map_error(&self.close, e)) + match self.inner()?.read_chunk(max).await { + Ok(res) => Ok(res), + Err(e) => Err(error(&self.close, e).await), + } } /// Read until the end of the stream or the limit is hit. See [`recv::Stream::read_to_end`]. pub async fn read_to_end(&mut self, limit: usize) -> Result, Error> { - let result = self.inner()?.read_to_end(limit).await; - result.map_err(|e| map_error(&self.close, e)) + match self.inner()?.read_to_end(limit).await { + Ok(res) => Ok(res), + Err(e) => Err(error(&self.close, e).await), + } } /// Block until the stream has been reset and return the error code. See @@ -138,11 +149,11 @@ impl Stream { }; match inner.received_reset().await { Ok(None) => Ok(None), - Ok(Some(code)) => match map_error(&self.close, recv::Error::Reset(code)) { + Ok(Some(code)) => match error(&self.close, recv::Error::Reset(code)).await { Error::Reset(code) => Ok(Some(code)), err => Err(err), }, - Err(err) => Err(map_error(&self.close, err)), + Err(err) => Err(error(&self.close, err).await), } } @@ -160,13 +171,13 @@ impl Stream { } /// Decode the peer's reset code, and report a session error, or a reset for a session -/// that is gone, as the session's close reason. -fn map_error(close: &Handles, e: recv::Error) -> Error { - match e { +/// that is gone, as the session's close reason, pending until that reason is known. +fn poll_error(close: &Handles, waiter: &kio::Waiter, e: recv::Error) -> Poll { + Poll::Ready(match e { recv::Error::Reset(code) => match close.decode_stream_code(code) { Some(code) => Error::Reset(code), - None => match close.session_gone(code) { - Some(err) => Error::Session(err), + None => match close.poll_session_gone(waiter, code) { + Some(err) => return err.map(Error::Session), None => Error::InvalidReset(code), }, }, @@ -174,6 +185,21 @@ fn map_error(close: &Handles, e: recv::Error) -> Error { Error::Session(e) => Error::Session(close.map(e)), e => e, }, + }) +} + +/// [`poll_error`] as a future. +async fn error(close: &Handles, e: recv::Error) -> Error { + kio::wait(|waiter| poll_error(close, waiter, e.clone())).await +} + +impl Drop for Stream { + fn drop(&mut self) { + // Chosen at drop, since the session may have closed since the stream opened. The + // driver's raw 0 is outside the HTTP/3 range, so a WebTransport peer can't decode it. + if let Some(inner) = &mut self.inner { + inner.set_drop_stop(self.close.encode_stream_code(0)); + } } } diff --git a/rs/moq-uring/src/quic/web/send.rs b/rs/moq-uring/src/quic/web/send.rs index 02acf02598c..0bd8ec20f17 100644 --- a/rs/moq-uring/src/quic/web/send.rs +++ b/rs/moq-uring/src/quic/web/send.rs @@ -15,7 +15,8 @@ use crate::quic::driver::send; /// /// A stream this side opened writes its WebTransport header ahead of its first data, so opening /// one waits only for the peer's stream credit. Dropping it before [`finish`](Self::finish) -/// resets it with code 0, still delivering the header once written. +/// resets it with code 0, or WT_SESSION_GONE once the session closed, still delivering the header +/// once written. /// /// This wrapper is mainly needed for error codes, which is unfortunate. /// WebTransport uses u32 error codes and they're mapped in a reserved HTTP/3 error space. @@ -45,8 +46,7 @@ enum State { impl Stream { /// Wrap `stream`, writing `header` ahead of its data: a WebTransport stream this side /// opened, or empty for one without a header. - pub(super) fn new(mut stream: send::Stream, close: &Rc, header: Bytes) -> Self { - stream.set_drop_reset(None, close.encode_stream_code(0)); + pub(super) fn new(stream: send::Stream, close: &Rc, header: Bytes) -> Self { let mut this = Self { id: stream.id(), state: State::Open(stream), @@ -60,24 +60,6 @@ impl Stream { this } - /// Decode the peer's stop code, and report a session error, or a stop for a session - /// that is gone, as the session's close reason. - fn map_error(&self, e: send::Error) -> Error { - match e { - send::Error::Stopped(code) => match self.close.decode_stream_code(code) { - Some(code) => Error::Stopped(code), - None => match self.close.session_gone(code) { - Some(err) => Error::Session(err), - None => Error::InvalidStopped(code), - }, - }, - e => match Error::from(e) { - Error::Session(e) => Error::Session(self.close.map(e)), - e => e, - }, - } - } - fn open(&mut self) -> Result<&mut send::Stream, Error> { match &mut self.state { State::Open(stream) => Ok(stream), @@ -127,14 +109,14 @@ impl Stream { State::Reset(result) => Poll::Ready(result.clone()), State::Unsent => return Poll::Ready(Err(Error::Unsent)), }; - result.map(|result| match result { - Ok(None) => Ok(None), - Ok(Some(code)) => match self.map_error(send::Error::Stopped(code)) { + match std::task::ready!(result) { + Ok(None) => Poll::Ready(Ok(None)), + Ok(Some(code)) => poll_error(&self.close, waiter, send::Error::Stopped(code)).map(|err| match err { Error::Stopped(code) => Ok(Some(code)), err => Err(err), - }, - Err(err) => Err(self.map_error(err)), - }) + }), + Err(err) => poll_error(&self.close, waiter, err).map(Err), + } } /// Finish sending, retaining a handle to acknowledgements and peer cancellation. @@ -164,11 +146,10 @@ impl Stream { }; while self.unwritten.has_remaining() { if let Err(err) = std::task::ready!(stream.poll_write_buf(waiter, &mut self.unwritten)) { - return Poll::Ready(Err(self.map_error(err))); + return poll_error(&self.close, waiter, err).map(Err); } } let offset = stream.offset(); - stream.set_drop_reset(Some(offset), self.close.encode_stream_code(0)); self.header = Some(offset); Poll::Ready(Ok(())) } @@ -176,8 +157,10 @@ impl Stream { /// Write from `buf` after the header, returning the size written. fn poll_write_buf(&mut self, waiter: &kio::Waiter, buf: &mut impl Buf) -> Poll> { std::task::ready!(self.poll_header(waiter))?; - let result = std::task::ready!(self.open()?.poll_write_buf(waiter, buf)); - Poll::Ready(result.map_err(|e| self.map_error(e))) + match std::task::ready!(self.open()?.poll_write_buf(waiter, buf)) { + Ok(n) => Poll::Ready(Ok(n)), + Err(e) => poll_error(&self.close, waiter, e).map(Err), + } } /// Write from `buf`, returning the size written. See [`send::Stream::write`]. @@ -258,6 +241,33 @@ impl Stream { } } +/// Decode the peer's stop code, and report a session error, or a stop for a session that is +/// gone, as the session's close reason, pending until that reason is known. +fn poll_error(close: &Handles, waiter: &kio::Waiter, e: send::Error) -> Poll { + Poll::Ready(match e { + send::Error::Stopped(code) => match close.decode_stream_code(code) { + Some(code) => Error::Stopped(code), + None => match close.poll_session_gone(waiter, code) { + Some(err) => return err.map(Error::Session), + None => Error::InvalidStopped(code), + }, + }, + e => match Error::from(e) { + Error::Session(e) => Error::Session(close.map(e)), + e => e, + }, + }) +} + +impl Drop for Stream { + fn drop(&mut self) { + // Chosen at drop, since the session may have closed since the stream opened. + if let State::Open(stream) = &mut self.state { + stream.set_drop_reset(self.header, self.close.encode_stream_code(0)); + } + } +} + impl moq_net::transport::poll::SendStream for Stream { type Error = Error; type Group = send::Group; From ec66ff3102d9b87be34d69eca73e1735f7ac6005 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:49:43 -0800 Subject: [PATCH 3/3] fix(tokio,uring): a stream reset stays the answer to every later read Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/quic/driver/recv.rs | 3 ++- rs/moq-tokio/src/quic/driver/tests.rs | 14 ++++++++++++++ rs/moq-uring/src/quic/driver/recv.rs | 3 ++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/rs/moq-tokio/src/quic/driver/recv.rs b/rs/moq-tokio/src/quic/driver/recv.rs index 686aba505e5..d99771cd11b 100644 --- a/rs/moq-tokio/src/quic/driver/recv.rs +++ b/rs/moq-tokio/src/quic/driver/recv.rs @@ -309,7 +309,8 @@ impl Reader { { use moq_quic::recv::Error::{Blocked, Reset}; - if self.all_data_read { + // A reset stays the answer to every later read, never a clean end. + if self.all_data_read && self.reset.is_none() { return Poll::Ready(Ok(None)); } diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index bfab5349f3b..83796a27974 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -2313,6 +2313,20 @@ async fn reads_after_a_received_reset_fail_with_it() { let mut recv = server.accept_uni().await.unwrap(); send.reset(VarInt::from_u32(7)); + // A read that hit the reset reads it again, never a clean end. + let mut buf = Vec::with_capacity(8); + loop { + match timeout(Duration::from_secs(10), recv.read(&mut buf)).await.unwrap() { + Ok(Some(_)) => {} + res => { + assert!(matches!(res, Err(recv::Error::Reset(_))), "{res:?}"); + break; + } + } + } + let read = recv.read(&mut Vec::with_capacity(8)).await; + assert!(matches!(read, Err(recv::Error::Reset(_))), "{read:?}"); + let reset = timeout(Duration::from_secs(10), recv.received_reset()).await.unwrap(); assert_eq!(reset.unwrap(), Some(VarInt::from_u32(7))); let lease = recv.lease(1).await.err(); diff --git a/rs/moq-uring/src/quic/driver/recv.rs b/rs/moq-uring/src/quic/driver/recv.rs index ecb4f1e2a2b..7cd73e2326e 100644 --- a/rs/moq-uring/src/quic/driver/recv.rs +++ b/rs/moq-uring/src/quic/driver/recv.rs @@ -257,7 +257,8 @@ impl Stream { { use moq_quic::recv::Error::{Blocked, ConnectionLost, Reset}; - if self.all_data_read { + // A reset stays the answer to every later read, never a clean end. + if self.all_data_read && self.reset.is_none() { return Poll::Ready(Ok(None)); }