diff --git a/quest/m3/qos/lag-histogram.md b/quest/m3/qos/lag-histogram.md index ae454e80d5d..f56cafa2110 100644 --- a/quest/m3/qos/lag-histogram.md +++ b/quest/m3/qos/lag-histogram.md @@ -45,8 +45,10 @@ Decided 2026-10-26 (maintainer, on #6791), each with its reason: `Billing` mints an offset whenever a write ends a frame and retires them oldest first, so the frontier rides those offsets rather than minting its own. -- **No hook, no fields.** A subscription whose backend reports `poll_acked` - unsupported (qmux, the browser transport) keeps no frontier and adds +- **No hook, no fields.** A subscription whose backend cannot see + acknowledgments (`poll_acked` answers `Unsupported`, as the browser + transport does, or `Sent`, as qmux does once its link took the bytes, which + says nothing of when the peer got them) keeps no frontier and adds nothing to `lag` or `dropped`. A node serving only such viewers leaves both absent. Document this in `doc/concept/stats.md`. - **The lag edge is the newest group's.** On the branch, `Produced.newest` is diff --git a/quest/m4/quic-ack-hook.md b/quest/m4/quic-ack-hook.md index 01d4977d31c..16b8abed465 100644 --- a/quest/m4/quic-ack-hook.md +++ b/quest/m4/quic-ack-hook.md @@ -16,10 +16,14 @@ Those methods, `poll_acked` on `send::Stream` and `send::Finished` and `Finished::offset`, forward to `moq-quic`'s public `send::Stream::poll_acked`. They fail with the peer's stop code (`send::Error::Stopped`) or the connection's loss (`send::Error::Connection`); a reset consumes the handle, so -no handle ever waits past one. qmux and the browser transport stay on the -trait's default `Acked::Unsupported` (decided 2026-10-22 and 2026-10-26, -maintainer): writing into TCP is not delivery, and the browser's -`WebTransportSendStream.getStats()` is unimplemented in shipping Chrome. +no handle ever waits past one. qmux answers `Acked::Sent` once its link +takes the bytes, which billing counts and acknowledgment consumers ignore: +writing into TCP is still not delivery (decided 2026-10-22, maintainer; +`Sent` added 2026-11-17 in +[moq-dev/moq#8930](https://github.com/moq-dev/moq/pull/8930)). The browser +transport stays on the trait's default `Acked::Unsupported` (decided +2026-10-26): its `WebTransportSendStream.getStats()` is unimplemented in +shipping Chrome. Decided 2026-11-15 (maintainer, /quest-complete): ✅ keep only the trap tests, on the crate-private methods, and drop the public driver API this quest diff --git a/rs/moq-net/src/coding/writer.rs b/rs/moq-net/src/coding/writer.rs index 9236294e349..00eb4112199 100644 --- a/rs/moq-net/src/coding/writer.rs +++ b/rs/moq-net/src/coding/writer.rs @@ -339,8 +339,9 @@ impl Writer { } } -/// A stream's payload billed as the transport acknowledges it, through offsets the writer asks -/// [`poll_acked`](crate::transport::poll::SendStream::poll_acked) about. +/// A stream's payload billed as the transport acknowledges it, or a reliable link takes it, +/// through offsets the writer asks [`poll_acked`](crate::transport::poll::SendStream::poll_acked) +/// about. struct Billing { delivery: crate::stats::Delivery, ack: Ack, @@ -414,7 +415,8 @@ impl Billing { while let Some((offset, through)) = asked.front() { match stream.poll_acked(waiter, offset) { Poll::Pending => break, - Poll::Ready(Ok(Acked::Delivered)) => { + // Bytes a reliable link took bill too: a reset can no longer discard them. + Poll::Ready(Ok(Acked::Delivered | Acked::Sent)) => { self.delivery.deliver(*through); asked.pop_front(); } @@ -630,6 +632,7 @@ mod tests { /// A stream that takes every write, acknowledging a prefix the test moves, or that cannot /// see acknowledgments. It counts the polls waiting on acknowledgment, which stay pending. + /// With `sent`, it answers for the prefix as a reliable link that took it. #[derive(Clone, Default)] struct Acking(std::sync::Arc>); @@ -638,6 +641,7 @@ mod tests { written: u64, acked: u64, unsupported: bool, + sent: bool, reset: bool, waits: usize, } @@ -649,6 +653,12 @@ mod tests { stream } + fn sent() -> Self { + let stream = Self::default(); + stream.0.lock().unwrap().sent = true; + stream + } + fn ack(&self, through: u64) { self.0.lock().unwrap().acked = through; } @@ -675,6 +685,7 @@ mod tests { let mut state = self.0.lock().unwrap(); match () { _ if state.unsupported => Poll::Ready(Ok(crate::transport::poll::Acked::Unsupported)), + _ if *offset <= state.acked && state.sent => Poll::Ready(Ok(crate::transport::poll::Acked::Sent)), _ if *offset <= state.acked => Poll::Ready(Ok(crate::transport::poll::Acked::Delivered)), _ if state.reset => Poll::Ready(Err(Stopped(0))), _ => { @@ -738,10 +749,15 @@ mod tests { /// A frame bills once the transport acknowledges an offset past it, not as the stream takes /// it: each write that ends a frame asks about its own offset, so a frame acknowledged before /// a reset still bills, and nothing the reset discarded does. A park with nothing else to ask - /// about asks about a frame still being written, billing its acknowledged bytes. + /// about asks about a frame still being written, billing its acknowledged bytes. Bytes a + /// reliable link took bill the same way. #[test] fn egress_bills_acknowledged_payload() { - let stream = Acking::default(); + egress_bills_payload(Acking::default()); + egress_bills_payload(Acking::sent()); + } + + fn egress_bills_payload(stream: Acking) { let (mut writer, registry) = billing(&stream); let waiter = &kio::Waiter::noop(); diff --git a/rs/moq-net/src/stats.rs b/rs/moq-net/src/stats.rs index 7ab2b6611ca..44749547046 100644 --- a/rs/moq-net/src/stats.rs +++ b/rs/moq-net/src/stats.rs @@ -53,7 +53,8 @@ //! * `bytes` / `frames` / `groups`: cumulative payload counters bumped as //! groups/frames are read (egress) or written (ingress) in the model. A //! session's egress `bytes` / `frames` count only payload the peer -//! acknowledged (see [`SendStream::poll_acked`](crate::transport::poll::SendStream::poll_acked)), +//! acknowledged, or that a reliable link such as qmux's took (see +//! [`SendStream::poll_acked`](crate::transport::poll::SendStream::poll_acked)), //! never what a reset or a lost session discarded: a frame's bytes once an //! offset past them is acknowledged, and the frame with its last byte. A //! transport that cannot see acknowledgments counts the bytes the stream diff --git a/rs/moq-net/src/transport.rs b/rs/moq-net/src/transport.rs index f5840be552d..0fcb402f474 100644 --- a/rs/moq-net/src/transport.rs +++ b/rs/moq-net/src/transport.rs @@ -706,6 +706,10 @@ pub mod poll { Delivered, /// The transport cannot see acknowledgments, so delivery is unknown, never assumed. Unsupported, + /// The transport handed every byte below the offset to a reliable link, such as qmux's + /// TCP, which delivers it unless the link fails. Not an acknowledgment: it says nothing of + /// when the peer got it, but a reset can no longer discard it. + Sent, } /// An outgoing stream with partial writes, priorities, and reset codes. @@ -727,6 +731,7 @@ pub mod poll { /// /// Errors once those bytes can never be acknowledged: the stream was reset by either /// side or the session closed. Bytes already acknowledged stay so after a reset or stop. + /// A transport over a reliable link answers [`Acked::Sent`] once the link took them. /// A transport that cannot see acknowledgments is ready with [`Acked::Unsupported`] /// immediately, the default, on every stream, so a caller can decide once per session. An /// offset minted by another stream may panic. diff --git a/rs/qmux/src/session.rs b/rs/qmux/src/session.rs index a1d6b4ac267..836dbe5c250 100644 --- a/rs/qmux/src/session.rs +++ b/rs/qmux/src/session.rs @@ -527,10 +527,26 @@ impl SendStream { impl poll::SendStream for SendStream { type Error = Error; type Group = SendGroup; - // Writing into TCP is not delivery, so acknowledgments stay unsupported. - type Offset = (); + type Offset = send::Offset; - fn offset(&self) -> Self::Offset {} + fn offset(&self) -> send::Offset { + self.inner.lock().conn.send_stream(self.id).offset() + } + + /// [`Acked::Sent`](poll::Acked::Sent) once the transport took the bytes, which is not the + /// peer acknowledging them. A reset or stop keeps the prefix that went out before it. + fn poll_acked(&self, waiter: &kio::Waiter, offset: &send::Offset) -> Poll> { + let mut state = self.inner.lock(); + let mut stream = state.conn.send_stream(self.id); + if u64::from(*offset) <= stream.acked_len() { + return Poll::Ready(Ok(poll::Acked::Sent)); + } + match ready!(stream.poll_acked(waiter, *offset)) { + Ok(()) => Poll::Ready(Ok(poll::Acked::Sent)), + Err(send::Error::ConnectionLost) => Poll::Ready(Err(state.lost())), + Err(_) => Poll::Ready(Err(Error::StreamClosed)), + } + } fn poll_write(&mut self, waiter: &kio::Waiter, mut buf: &[u8]) -> Poll> { self.poll_write_buf(waiter, &mut buf) diff --git a/rs/qmux/src/session/tests.rs b/rs/qmux/src/session/tests.rs index 1d10e2c08c6..09c78a6b13d 100644 --- a/rs/qmux/src/session/tests.rs +++ b/rs/qmux/src/session/tests.rs @@ -177,6 +177,49 @@ async fn read_exact(recv: &mut RecvStream, want: usize) -> Result .await } +/// What `send` answers about the bytes below `offset`, without waiting. +fn acked(send: &SendStream, offset: &moq_quic::send::Offset) -> Poll> { + send.poll_acked(&kio::Waiter::noop(), offset) +} + +/// A stream's bytes count as sent once the transport takes them, which a stalled link holds +/// back. A reset discards what the link still holds, so only the prefix that went out stays sent, +/// and egress never bills the rest. +#[tokio::test(start_paused = true)] +async fn bytes_count_as_sent_once_the_link_takes_them() { + for &version in Version::ALL { + let (client, server, up, _down) = link(); + let (client, server) = establish(client, server, config(version), config(version)).await; + + let mut send = client.open_uni(SendOptions::default()).await.unwrap(); + let first = payload(1000, 5); + write_all(&mut send, first.clone()).await; + let mut recv = server.accept_uni().await.unwrap(); + assert_eq!(read_exact(&mut recv, first.len()).await.unwrap(), first, "{version:?}"); + let went_out = send.offset(); + assert!( + matches!(acked(&send, &went_out), Poll::Ready(Ok(poll::Acked::Sent))), + "{version:?}: the link took it" + ); + + up.send(Flow::Stall).unwrap(); + // More than the pipe holds, within the peer's stream window. + write_all(&mut send, payload(2 * PIPE, 6)).await; + let held = send.offset(); + assert!(acked(&send, &held).is_pending(), "{version:?}: the link holds the rest"); + + send.reset(1); + assert!( + matches!(acked(&send, &went_out), Poll::Ready(Ok(poll::Acked::Sent))), + "{version:?}: what went out stays sent" + ); + assert!( + matches!(acked(&send, &held), Poll::Ready(Err(_))), + "{version:?}: the reset discarded the rest" + ); + } +} + /// Leased bytes keep their credit, so incomplete messages fill the receive window rather than /// memory outside it. Once no reader can make progress, the lowest priority holder is evicted, /// and the credit it held lets the others carry on.