From 913bffe0c795494ecba3a37f696f206f380f7836 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 22:26:04 -0800 Subject: [PATCH 1/6] fix(quic): a paused stream holds only its stream window Stream credit ran from the bytes a lease moved out of the assembler, not the bytes its reader took, so a reader that leased and then stopped taking kept granting the peer more, up to the connection window. moq-net's ingest pauses a stream exactly that way to pace a slow reader, assuming it holds at most its stream window, so a paced stream filled the connection window and the stuck window evicted it: a bulk transfer through a relay ended Evicted. Co-Authored-By: Claude Opus 5.5 --- rs/moq-quic/README.md | 2 +- rs/moq-quic/src/connection/streams/lease.rs | 63 ++++++++--- rs/moq-quic/src/connection/streams/recv.rs | 7 +- rs/moq-quic/src/connection/streams/state.rs | 10 +- rs/moq-quic/src/tests/mod.rs | 70 +++++++------ rs/moq-relay/tests/integration/quic_sim.rs | 110 ++++++++++++++++++++ rs/moq-tokio/src/quic/driver/recv.rs | 5 +- rs/moq-tokio/src/quic/driver/tests.rs | 5 +- 8 files changed, 220 insertions(+), 52 deletions(-) diff --git a/rs/moq-quic/README.md b/rs/moq-quic/README.md index 81a64319ab5..6fbd191bbe3 100644 --- a/rs/moq-quic/README.md +++ b/rs/moq-quic/README.md @@ -88,7 +88,7 @@ Changes on top of the upstream commit, besides the renames: - **Fuzzing** runs through the `fuzz` module behind a `fuzz` feature, with one-line shims in [`fuzz/`](fuzz/README.md): upstream's `packet`, `params`, `streamid`, and `streams` targets, plus `frame` and a connection op sequence. Upstream's `cfg(fuzzing)` `fuzzing` module and `arbitrary` feature are gone, so its fuzz patches port by hand. - **Draining endpoint event**: `EndpointEvent::is_draining` fires once per connection when it stops being active, before the drained event, so the driver's `Endpoint::wait_all_draining` can end a shutdown without sitting out every closing period. - **Late stream limits**: peer transport parameters that raise a stream limit after an open was refused wake `Streams::poll_open`, as `MAX_STREAMS` does. A server accepted before the rest of a multi-packet ClientHello (a post-quantum key share is enough) otherwise leaves a 0.5-RTT `open_uni` parked until the idle timeout. Not offered upstream yet. -- **Receive leases** (`streams/lease.rs`, `recv::Stream::lease`), building on [#5675](https://github.com/moq-dev/moq/pull/5675)'s held credit: a lease reads a stream's ordered bytes out of the assembler but keeps their connection credit, counted in `StreamsState::held`, until the reader `advance`s or `take`s past them and `release`s, while stream credit advances as usual. A lease copies the bytes it reads out of their datagram buffers (`Leased::extend`), so a byte of credit never pins a receive batch, whether its reader waits, parks, or takes it. The lease lives in the stream's `Recv`, which a leased stream keeps, with its stream ID, until the application stops it, even past the FIN. When the peer has spent the connection window and no reader can make progress, the stream ranked lowest by `set_priority` (the newest on a tie) is evicted: its bytes and credit are freed, the stream is stopped without `STOP_SENDING`, and its next lease fails with `recv::Error::Evicted`; the application's `stop` then tells the peer and frees the stream once its final size is known. Leased streams that may hold credit are kept in `StreamsState::ranks`, in eviction order, so an eviction costs a logarithm rather than a scan of every stream. `MAX_DATA` goes out for any increase once the peer is within an eighth of the window (`max_data_due`), since what leases release may never add up to the usual threshold. Streams with data their reader has not looked at hold eviction off; they are kept in `StreamsState::unlooked`, so the stuck check stays constant time however many readers wait. `ConnectionStats::credit` reports the leased and available bytes, the streams evicted, and the peak credit spent. A `recv::Lease` owns the chunks it took, so a reader holds no connection borrow while it parses; `restore` or `release` hands it back, and an eviction pass waits while one is out. A port touching `Chunks` or `add_read_credits` has to keep `held` balanced. +- **Receive leases** (`streams/lease.rs`, `recv::Stream::lease`), building on [#5675](https://github.com/moq-dev/moq/pull/5675)'s held credit: a lease reads a stream's ordered bytes out of the assembler but keeps their connection credit, counted in `StreamsState::held`, until the reader `advance`s or `take`s past them and `release`s. Stream credit runs from the bytes the reader took rather than leased (`Leased::untaken`), widened to a waiting reader's `want`, so a reader that stops taking holds its stream window and no more. A lease copies the bytes it reads out of their datagram buffers (`Leased::extend`), so a byte of credit never pins a receive batch, whether its reader waits, parks, or takes it. The lease lives in the stream's `Recv`, which a leased stream keeps, with its stream ID, until the application stops it, even past the FIN. When the peer has spent the connection window and no reader can make progress, the stream ranked lowest by `set_priority` (the newest on a tie) is evicted: its bytes and credit are freed, the stream is stopped without `STOP_SENDING`, and its next lease fails with `recv::Error::Evicted`; the application's `stop` then tells the peer and frees the stream once its final size is known. Leased streams that may hold credit are kept in `StreamsState::ranks`, in eviction order, so an eviction costs a logarithm rather than a scan of every stream. `MAX_DATA` goes out for any increase once the peer is within an eighth of the window (`max_data_due`), since what leases release may never add up to the usual threshold. Streams with data their reader has not looked at hold eviction off; they are kept in `StreamsState::unlooked`, so the stuck check stays constant time however many readers wait. `ConnectionStats::credit` reports the leased and available bytes, the streams evicted, and the peak credit spent. A `recv::Lease` owns the chunks it took, so a reader holds no connection borrow while it parses; `restore` or `release` hands it back, and an eviction pass waits while one is out. A port touching `Chunks` or `add_read_credits` has to keep `held` balanced. - **Owned datagrams**: a received datagram is copied out of its packet when queued, since `datagram_receive_buffer_size` charges only its payload and a slice keeps the whole receive batch alive. quinn has the same gap; not offered upstream yet. - **Stream credit on stop**: `recv::Stream::stop` on a stream whose final size is already known (finished or reset) frees it and queues `MAX_STREAMS` at once, as reading the reset does, instead of leaving the credit until the next packet arrives. Not offered upstream yet; quinn `main` has the same gap. - **Drop resets** (`quic::driver::send::Stream`): a send stream dropped before `finish` is reset with code 0, or with the peer's `STOP_SENDING` code, instead of quinn's implicit `finish`, so a cancelled writer never ends a truncated stream with a FIN. This matches moq-uring's driver and moq-net's transport contract. `quic::web` resets after the WebTransport header with the mapped code. quinn's `stream_stopped_2` finishes its stream before dropping it. Not offered upstream: it reverses a documented quinn behaviour. diff --git a/rs/moq-quic/src/connection/streams/lease.rs b/rs/moq-quic/src/connection/streams/lease.rs index 72df8afa3c5..59263a1f418 100644 --- a/rs/moq-quic/src/connection/streams/lease.rs +++ b/rs/moq-quic/src/connection/streams/lease.rs @@ -19,11 +19,16 @@ use crate::connection::waits::Wait; /// They keep their connection flow control credit until released, so the connection receive /// window bounds what every lease on the connection holds together. Handed out as a [`Lease`] /// by [`Stream::lease()`]. +/// +/// Bytes the reader has yet to take keep their stream credit too (see [`Self::untaken`]), so a +/// reader that stops leasing holds at most its stream window, however much it leased before. #[derive(Debug, Default)] pub(super) struct Leased { chunks: VecDeque, /// Bytes in `chunks`. len: u64, + /// Bytes out in a [`Lease`], until it is handed back. + lent: u64, /// Bytes whose connection credit is withheld: `len`, plus bytes taken or advanced past but /// not yet released. held: u64, @@ -100,6 +105,18 @@ impl Leased { chunk } + /// Bytes moved out of the receive buffer that the reader has yet to take, here or out in a + /// [`Lease`]. They have not been read, so they hold back the stream's credit as unread + /// buffered bytes do. + pub(super) fn untaken(&self) -> u64 { + self.len + self.lent + } + + /// The `len` a reader waits for, or zero. + pub(super) fn wanted(&self) -> u64 { + self.want.unwrap_or(0) + } + /// Return the credit of every byte read out of a [`Lease`]. fn release(&mut self) -> u64 { let freed = self.held - self.len; @@ -185,6 +202,7 @@ impl Leased { /// Discard everything, returning the credit held. pub(super) fn discard(&mut self) -> u64 { self.len = 0; + self.lent = 0; self.chunks.clear(); self.want = None; self.ready = false; @@ -308,6 +326,7 @@ impl Stream<'_> { let lease = self.state.lease_mut(self.id).expect("filled above"); if lease.len >= want || lease.is_fin() { lease.out = true; + lease.lent = lease.len; let out = Lease { chunks: std::mem::take(&mut lease.chunks), len: std::mem::take(&mut lease.len) as usize, @@ -325,11 +344,13 @@ impl Stream<'_> { } None => { lease.want = Some(want); + // A unit larger than the stream window needs room for the rest of it. + let credit = self.queue_max_stream_data(); // It may be waiting behind a gap, which holds eviction off until filled. self.state.unlooked.insert(self.id); // An eviction always has a reader to wake, even with no `MAX_DATA` due yet. let evicted = self.unstick(); - (Err(Error::Blocked), transmit || evicted) + (Err(Error::Blocked), transmit || credit || evicted) } } } @@ -369,19 +390,19 @@ impl Stream<'_> { /// They wait for a later [`Self::release()`], as the start of a message read only in part /// does. pub fn restore(&mut self, lease: Lease) { - self.put_back(lease); + let credit = self.put_back(lease); // Only a stuck pass that waited reruns: a reader handing back what it looked at is not // stuck itself, and one that is waits for more with its next lease. let transmit = self.state.stuck_waiting && self.unstick(); - self.transmit(transmit); + self.transmit(credit || transmit); } /// Hand a lease back, returning the connection credit of every byte read from it or from an /// earlier one, once the units they belong to are complete pub fn release(&mut self, lease: Lease) { - self.put_back(lease); + let credit = self.put_back(lease); let transmit = self.release_credit(); - self.transmit(transmit); + self.transmit(credit || transmit); } /// Wake the driver when a frame is due @@ -410,7 +431,12 @@ impl Stream<'_> { pub(crate) fn peek(&mut self, want: usize) -> (Result<(), Error>, bool) { let (lease, transmit) = self.lease_inner(want); self.transmit(transmit); - (lease.map(|lease| self.put_back(lease)), transmit) + ( + lease.map(|lease| { + self.put_back(lease); + }), + transmit, + ) } /// Read up to `max` leased bytes out, keeping their credit, as reading a [`Lease`] does. @@ -441,21 +467,25 @@ impl Stream<'_> { transmit } - /// Put the bytes `lease` did not read back in front of the stream's lease. + /// Put the bytes `lease` did not read back in front of the stream's lease, returning whether + /// what the reader took grew the stream's credit enough to send. /// /// Dropped instead once the stream was stopped, which discarded its lease and its credit. - fn put_back(&mut self, lease: Lease) { + fn put_back(&mut self, lease: Lease) -> bool { let Some(recv) = self.state.recv_mut(self.id) else { - return; + return false; }; let state = &mut recv.lease; if !std::mem::take(&mut state.out) { - return; + return false; } // An out lease holds eviction off, and nothing fills a lease while it is out. debug_assert!(recv.reader != Reader::Evicted && state.chunks.is_empty()); state.chunks = lease.chunks; state.len = lease.len as u64; + state.lent = 0; + // What the reader took returns its stream credit. + self.queue_max_stream_data() } /// Rank this stream for eviction: when the connection window is stuck, the stream with the @@ -496,11 +526,20 @@ impl Stream<'_> { None => None, }; } - let (_, transmit) = recv.max_stream_data(self.state.stream_receive_window); + Ok(self.queue_max_stream_data()) + } + + /// Queue a `MAX_STREAM_DATA` if the stream's credit grew enough to send, returning whether. + fn queue_max_stream_data(&mut self) -> bool { + let window = self.state.stream_receive_window; + let Some(recv) = self.state.recv_mut(self.id) else { + return false; + }; + let (_, transmit) = recv.max_stream_data(window); if transmit { self.pending.max_stream_data.insert(self.id); } - Ok(transmit) + transmit } /// Return `bytes` of withheld connection credit to the peer. diff --git a/rs/moq-quic/src/connection/streams/recv.rs b/rs/moq-quic/src/connection/streams/recv.rs index 10c6c4a85ee..060e67a1e87 100644 --- a/rs/moq-quic/src/connection/streams/recv.rs +++ b/rs/moq-quic/src/connection/streams/recv.rs @@ -276,8 +276,13 @@ impl Recv { /// `false` the new window should only be transmitted if a previous transmission /// had failed. pub(super) fn max_stream_data(&mut self, stream_receive_window: u64) -> (u64, bool) { + // Credit runs from what the reader took, not what it leased: a reader that stops taking + // holds its window and no more. One waiting on a unit larger than the window gets room + // for the unit, which the connection window bounds. + let taken = self.assembler.bytes_read() - self.lease.untaken(); + let window = stream_receive_window.max(self.lease.wanted()); // A window shrunk since the last announcement cannot take back credit already granted. - let max_stream_data = (self.assembler.bytes_read() + stream_receive_window).max(self.sent_max_stream_data); + let max_stream_data = (taken + window).max(self.sent_max_stream_data); // Only announce a window update if it's significant enough // to make it worthwhile sending a MAX_STREAM_DATA frame. diff --git a/rs/moq-quic/src/connection/streams/state.rs b/rs/moq-quic/src/connection/streams/state.rs index 874c6c2d04a..742d1d2ce9e 100644 --- a/rs/moq-quic/src/connection/streams/state.rs +++ b/rs/moq-quic/src/connection/streams/state.rs @@ -4785,8 +4785,14 @@ mod tests { (Err(recv::Error::TooLarge), false) ); assert!(pending.max_stream_data.is_empty()); - assert_eq!(recv(&mut client, &mut pending, id).peek(0), (Ok(()), true)); - assert!(pending.max_stream_data.contains(&id)); + let mut stream = recv(&mut client, &mut pending, id); + let mut lease = stream.lease(0).unwrap(); + lease.advance(64); + stream.release(lease); + assert!( + pending.max_stream_data.contains(&id), + "taking the bytes returns their credit" + ); let id = StreamId::new(Side::Server, Dir::Uni, 1); data(&mut client, id, 0, 10, false); diff --git a/rs/moq-quic/src/tests/mod.rs b/rs/moq-quic/src/tests/mod.rs index 3dafbdea129..97de51346c0 100644 --- a/rs/moq-quic/src/tests/mod.rs +++ b/rs/moq-quic/src/tests/mod.rs @@ -7022,17 +7022,6 @@ fn leased(pair: &mut Pair, server_ch: ConnectionHandle) -> usize { pair.server_conn_mut(server_ch).stats().credit.leased as usize } -/// Lease what has arrived on `s`, read `n` bytes of it, and release their credit, returning -/// whether the peer finished the stream right after the leased bytes. -fn consume(pair: &mut Pair, server_ch: ConnectionHandle, s: StreamId, n: usize) -> bool { - let mut recv = pair.server_recv(server_ch, s); - let mut lease = recv.lease(0).unwrap(); - let fin = lease.is_fin(); - lease.advance(n); - recv.release(lease); - fin -} - /// Write what the client can of `msg` up to `until`, finishing at its end, and lease what the /// server can, without consuming, until neither makes progress. Returns the bytes newly leased. fn pump_leased( @@ -7071,20 +7060,18 @@ fn pump_leased( } } -/// Leases keep replenishing a stream window much smaller than the connection's, while the -/// connection window stays spent until the bytes are consumed. A consume below the usual -/// `MAX_DATA` threshold still unblocks the peer once it nears its limit, without relying on its -/// DATA_BLOCKED. -/// -/// The peer never spends the whole window: a lease held then would be evicted, since nothing -/// could free it. +/// A reader that leases without taking holds one stream window, however large the +/// connection's: stream credit returns as bytes are taken, and connection credit once they are +/// released. So a reader that stops taking, as moq-net's ingest does to pace a slow reader, +/// never spends the connection window, which would get its stream evicted. #[test] -fn lease_spends_connection_window_only() { +fn leases_hold_one_stream_window() { const WINDOW: usize = 4000; + const STREAM: usize = 1000; let _guard = subscribe(); let mut transport = transport::Config::default(); transport.limits.receive_window = NonZeroU64::new(WINDOW as u64); - transport.limits.stream_receive_window = NonZeroU64::new(1000); + transport.limits.stream_receive_window = NonZeroU64::new(STREAM as u64); let mut pair = Pair::new( Default::default(), server::Config { @@ -7099,23 +7086,42 @@ fn lease_spends_connection_window_only() { .open(Dir::Uni, Default::default()) .unwrap(); let msg = vec![0xAB; WINDOW + 50]; + let all = msg.len(); let mut sent = 0; - // Three stream windows' worth, all leased and none consumed. - assert_eq!(pump_leased(&mut pair, chs, s, (&msg, 3000), &mut sent), 3000); + // Leased and never taken: one stream window. + assert_eq!(pump_leased(&mut pair, chs, s, (&msg, all), &mut sent), STREAM); let stats = pair.server_conn_mut(server_ch).stats().credit; - assert_eq!((stats.leased, stats.available), (3000, 1000)); + assert_eq!( + (stats.leased, stats.available), + (STREAM as u64, (WINDOW - STREAM) as u64) + ); - // Too little to announce while the peer has room, so the peer blocks at the window, and the - // frame that spends the last of its room is what announces it. - assert!(!consume(&mut pair, server_ch, s, 100)); - let all = msg.len(); - assert_eq!(pump_leased(&mut pair, chs, s, (&msg, all), &mut sent), 1050); - assert_eq!(sent, msg.len()); - assert_eq!(leased(&mut pair, server_ch), WINDOW + 50 - 100); + // Taken but restored, as the start of a message is, keeps the connection credit and + // returns the stream credit. + let mut recv = pair.server_recv(server_ch, s); + let mut lease = recv.lease(0).unwrap(); + lease.advance(500); + recv.restore(lease); + assert_eq!(pump_leased(&mut pair, chs, s, (&msg, all), &mut sent), 500); + let stats = pair.server_conn_mut(server_ch).stats().credit; + assert_eq!((stats.leased, stats.available), (1500, (WINDOW - 1500) as u64)); - let all = leased(&mut pair, server_ch); - assert!(consume(&mut pair, server_ch, s, all), "finished after the leased bytes"); + // Taking and releasing everything delivers the rest of the message. + let mut read = 500; + for _ in 0..all { + let mut recv = pair.server_recv(server_ch, s); + let mut lease = recv.lease(0).unwrap(); + let (len, fin) = (lease.remaining(), lease.is_fin()); + lease.advance(len); + recv.release(lease); + read += len; + if fin { + break; + } + pump_leased(&mut pair, chs, s, (&msg, all), &mut sent); + } + assert_eq!((sent, read), (all, all)); assert_eq!(leased(&mut pair, server_ch), 0); pair.server_recv(server_ch, s).stop(0u32.into()).unwrap(); assert_eq!(pair.server_recv(server_ch, s).lease(0).err(), Some(recv::Error::Closed)); diff --git a/rs/moq-relay/tests/integration/quic_sim.rs b/rs/moq-relay/tests/integration/quic_sim.rs index b0da89ecf9b..73a64e2685e 100644 --- a/rs/moq-relay/tests/integration/quic_sim.rs +++ b/rs/moq-relay/tests/integration/quic_sim.rs @@ -100,8 +100,20 @@ struct Relay { /// Serve `cluster` on `addr` through `auth`: each session the server accepts runs a relay /// [`Connection`] on its own simulator task. async fn serve(network: &Network, addr: SocketAddr, cluster: &cluster::Cluster, auth: &auth::Auth) -> Relay { + serve_versions(network, addr, cluster, auth, Vec::new()).await +} + +/// [`serve`], accepting only `versions`, or every default one when empty. +async fn serve_versions( + network: &Network, + addr: SocketAddr, + cluster: &cluster::Cluster, + auth: &auth::Auth, + versions: Vec, +) -> Relay { let _ = rind_rustls::default_provider().unwrap().install_default(); let mut config = moq_tokio::server::Config::default(); + config.listen.version = versions; config.listen.bind = Some(moq_tokio::listen::Bind::Addr(addr)); config.listen.tls.generate = vec!["localhost".into()]; config.sim = Some(network.host(addr.ip())); @@ -221,3 +233,101 @@ fn lossy(network: &Network, a: SocketAddr, b: SocketAddr) { network.path(a, b, |p| *p = path.clone()); network.path(b, a, |p| *p = path); } + +/// The bytes a socket session carries in one group: past the 32 MB live tail, so a hop that +/// sheds rather than paces loses some +const BULK: usize = 40 * 1024 * 1024; +/// The size of each frame of [`BULK`] +const CHUNK: usize = 8 * 1024; +/// The only moq-lite that carries hidden broadcasts, which socket endpoints need at every hop +const LITE_07: &str = "moq-lite-07-wip"; + +/// A socket session's group carries a bulk transfer through a relay without loss: the writer +/// waits on `ready` before each frame, and the reader drains at its own pace, slower than +/// the writer, so every hop has to pace its ingest to that reader rather than shed. +/// +/// A paused stream must hold no more than its stream window, or the connection window fills +/// with bytes nobody takes and the transport evicts the stream. +#[test] +fn a_socket_session_carries_a_bulk_transfer_losslessly() { + memory::run(|| async { + let network = Network::new(0); + let auth = memory::public(&["**"], &[], &[]); + let cluster = memory::start(memory::cluster(), moq_tokio::connect::Config::default()).await; + let lite = vec![LITE_07.parse().unwrap()]; + let relay = serve_versions(&network, SocketAddr::new(ip(1), PORT), &cluster, &auth, lite).await; + + let (writer, _writing) = endpoint(&network, PUBLISHER, &relay, "writer"); + let (reader, _reading) = endpoint(&network, SUBSCRIBER, &relay, "reader"); + + let dial = reader.dial_track(writer.id(), None).unwrap(); + let accept = async { writer.accept_track().await?.accept(None).await }; + let (dialed, accepted) = kio::clock::timeout(TIMEOUT, async { futures::join!(dial, accept) }) + .await + .expect("connect timeout"); + let (dialed, accepted) = (dialed.expect("dial"), accepted.expect("accept")); + + let write = async { + let mut group = accepted.producer.append_group().unwrap(); + let info = moq_net::frame::Info { + size: CHUNK as u64, + timestamp: None, + }; + for index in 0..BULK / CHUNK { + group.ready(&info).await; + group.write_frame(None, vec![index as u8; CHUNK]).unwrap(); + } + group.finish().unwrap(); + }; + let read = async { + let mut ordered = dialed.consumer.ordered(); + let mut group = ordered.next_group().await.unwrap().expect("a group"); + let mut read = 0; + while let Some(frame) = group + .read_frame() + .await + .unwrap_or_else(|err| panic!("after {read} bytes: {err}")) + { + let index = read / CHUNK; + assert_eq!(frame.payload.len(), CHUNK, "frame {index} is cut short"); + assert!( + frame.payload.iter().all(|&byte| byte == index as u8), + "frame {index} is corrupted" + ); + read += CHUNK; + // A consumer draining at about 64 MB/s, slower than the writer. + kio::clock::sleep(Duration::from_micros(CHUNK as u64 / 64)).await; + } + read + }; + let read = kio::clock::timeout(Duration::from_secs(600), async { futures::join!(write, read).1 }) + .await + .expect("the transfer stalled"); + assert_eq!(read, BULK); + }); +} + +/// A socket endpoint `id` on the host at `ip`, publishing and subscribing through `relay`. +fn endpoint( + network: &Network, + ip: IpAddr, + relay: &Relay, + id: &str, +) -> (moq_net::socket::Endpoint, moq_tokio::Connection) { + let origin = moq_tokio::origin::spawn(); + let mut connect = moq_tokio::connect::Config::default(); + connect.bind = Some(SocketAddr::new(ip, CLIENT_PORT)); + connect.tls.fingerprint = relay.fingerprints.clone(); + // Every hop between two socket endpoints carries hidden broadcasts. + connect.version = vec![LITE_07.parse().unwrap()]; + let mut config = moq_tokio::client::Config::default(); + config.connect = connect; + config.sim = Some(network.host(ip)); + let connection = config + .init() + .expect("init client") + .with_publisher(&origin) + .with_subscriber(origin.clone()) + .connect(url(relay)); + (moq_net::socket::Endpoint::new(&origin, "room", id).unwrap(), connection) +} diff --git a/rs/moq-tokio/src/quic/driver/recv.rs b/rs/moq-tokio/src/quic/driver/recv.rs index d99771cd11b..96c6455ac47 100644 --- a/rs/moq-tokio/src/quic/driver/recv.rs +++ b/rs/moq-tokio/src/quic/driver/recv.rs @@ -162,8 +162,9 @@ impl Stream { /// /// The lease owns the bytes and holds no lock, but they keep their connection flow control /// credit until read through the lease as a [`Buf`] and then [`Lease::release`]d; bytes not - /// read are leased again next time. Stream credit still advances, so a stream window smaller - /// than a message keeps the message flowing. + /// read are leased again next time. Stream credit returns as bytes are read through a lease, + /// so a reader that stops reading holds its stream window, and one waiting for a message + /// larger than the window gets room for the message. /// /// When the peer has spent the connection window and no reader can make progress, the /// stream ranked lowest by [`Self::set_priority`] is evicted: its buffered bytes and their diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index 83796a27974..641836b015f 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -2148,8 +2148,9 @@ async fn dropped_connection_cleans_up() { endpoint.wait_idle().await; } -/// Leases keep the connection window spent until consumed, while a stream window far smaller -/// than it keeps replenishing; a lease outlives its stream. +/// Leases keep the connection window spent until consumed, while a reader waiting for more than +/// a stream window far smaller than it gets room for what it waits on; a lease outlives its +/// stream. #[tokio::test] async fn leases_spend_the_connection_window() { const WINDOW: usize = 32 * 1024; From 929466226596e439849d85571f740ce6826891d4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 22:46:24 -0800 Subject: [PATCH 2/6] fix(quic): send a waiting reader the stream credit it needs Co-Authored-By: Claude Opus 5.5 --- rs/moq-quic/src/connection/streams/recv.rs | 5 +- rs/moq-quic/src/tests/mod.rs | 61 ++++++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/rs/moq-quic/src/connection/streams/recv.rs b/rs/moq-quic/src/connection/streams/recv.rs index 060e67a1e87..dba05078d18 100644 --- a/rs/moq-quic/src/connection/streams/recv.rs +++ b/rs/moq-quic/src/connection/streams/recv.rs @@ -291,8 +291,11 @@ impl Recv { // less updates. A fixed size would also work - but it would need to be // smaller than `stream_receive_window` in order to make sure the stream // does not get stuck. + // A reader waiting on more than the peer may send gets it however small: nothing it + // takes in the meantime would add to it. let diff = max_stream_data - self.sent_max_stream_data; - let transmit = self.can_send_flow_control() && diff >= (stream_receive_window / 8); + let short = self.sent_max_stream_data < taken + self.lease.wanted(); + let transmit = self.can_send_flow_control() && (diff >= (stream_receive_window / 8) || short); (max_stream_data, transmit) } diff --git a/rs/moq-quic/src/tests/mod.rs b/rs/moq-quic/src/tests/mod.rs index 97de51346c0..56ccd2df982 100644 --- a/rs/moq-quic/src/tests/mod.rs +++ b/rs/moq-quic/src/tests/mod.rs @@ -7127,6 +7127,67 @@ fn leases_hold_one_stream_window() { assert_eq!(pair.server_recv(server_ch, s).lease(0).err(), Some(recv::Error::Closed)); } +/// Lease `want` bytes of `s` once they arrive, driving the pair until they do, and hand them +/// back unread. +fn await_unit( + pair: &mut Pair, + (client_ch, server_ch): (ConnectionHandle, ConnectionHandle), + s: StreamId, + msg: &[u8], + sent: &mut usize, + want: usize, +) { + for _ in 0..msg.len() { + if let Ok(n) = pair.client_send(client_ch, s).write(&mut &msg[*sent..]) { + *sent += n; + } + pair.drive(); + let mut recv = pair.server_recv(server_ch, s); + match recv.lease(want) { + Ok(lease) => { + assert!(lease.remaining() >= want); + recv.restore(lease); + return; + } + Err(err) => assert_eq!(err, recv::Error::Blocked), + } + } + panic!("{want} bytes never arrived: {sent} sent"); +} + +/// A reader waiting for a unit larger than its stream window gets room for the unit, and after +/// taking a few bytes, room for the few more its next unit needs, however far below the usual +/// update threshold. +#[test] +fn a_waiting_reader_gets_room_for_its_unit() { + const STREAM: usize = 1000; + const UNIT: usize = 3000; + let _guard = subscribe(); + let mut transport = transport::Config::default(); + transport.limits.receive_window = NonZeroU64::new(4000); + transport.limits.stream_receive_window = NonZeroU64::new(STREAM as u64); + let mut pair = Pair::new( + Default::default(), + server::Config { + transport, + ..server_config() + }, + ); + let chs = pair.connect(); + let s = pair.client_streams(chs.0).open(Dir::Uni, Default::default()).unwrap(); + let msg = vec![0xAB; UNIT + 10]; + let mut sent = 0; + + await_unit(&mut pair, chs, s, &msg, &mut sent, UNIT); + + let mut recv = pair.server_recv(chs.1, s); + let mut lease = recv.lease(0).unwrap(); + lease.advance(10); + recv.release(lease); + await_unit(&mut pair, chs, s, &msg, &mut sent, UNIT); + assert_eq!(sent, msg.len()); +} + #[test] fn sender_window_is_not_starvation() { let _guard = subscribe(); From e70d90fec9af0f9195d60c0277c181c3a74be188 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 23:56:24 -0800 Subject: [PATCH 3/6] fix(net): a paused stream reserves what it took of an unsealed segment Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/coding/reader.rs | 76 ++++++++++++++++++--- rs/moq-net/src/transport.rs | 5 +- rs/moq-net/src/transport/boxed.rs | 60 +++++++++------- rs/moq-quic/README.md | 2 +- rs/moq-quic/src/connection/streams/lease.rs | 3 +- rs/moq-quic/src/connection/streams/recv.rs | 4 +- rs/moq-relay/tests/integration/quic_sim.rs | 4 +- rs/moq-tokio/src/quic/driver/recv.rs | 4 +- 8 files changed, 116 insertions(+), 42 deletions(-) diff --git a/rs/moq-net/src/coding/reader.rs b/rs/moq-net/src/coding/reader.rs index 376bd959393..faa70232c1b 100644 --- a/rs/moq-net/src/coding/reader.rs +++ b/rs/moq-net/src/coding/reader.rs @@ -26,6 +26,8 @@ pub struct Reader { want: usize, /// A contiguous copy of a message that spans transport chunks, empty between decodes. scratch: Vec, + /// Bytes taken into a frame's unsealed segment, whose receive credit returns once it seals. + unreleased: u64, stats: crate::stats::Session, } @@ -50,9 +52,10 @@ pub(crate) trait Pause { crate::model::pace::AHEAD } - /// Reserve the credit this stream may hold while paused, returning whether it may pause. - /// A stream outside any session always may. - fn pause(&mut self) -> bool { + /// Reserve the credit this stream may hold while paused, its stream window and the `taken` + /// bytes it read whose credit it has yet to return, returning whether it may pause. A + /// stream outside any session always may. + fn pause(&mut self, _taken: u64) -> bool { true } @@ -65,8 +68,8 @@ impl Pause for crate::transport::boxed::RecvStream { crate::transport::boxed::RecvStream::window(self) } - fn pause(&mut self) -> bool { - crate::transport::boxed::RecvStream::pause(self) + fn pause(&mut self, taken: u64) -> bool { + crate::transport::boxed::RecvStream::pause(self, taken) } fn resume(&mut self) { @@ -97,6 +100,7 @@ impl Reader { version, want: 0, scratch: Vec::new(), + unreleased: 0, stats: Default::default(), } } @@ -396,7 +400,8 @@ impl Reader { Poll::Ready(Err(err)) => break Err(err), Poll::Pending => Poll::Pending, }; - let mut lease = match lease { + // Moved, so the lease's borrow of the reader ends with it and `unreleased` can count. + let mut lease = match { lease } { Poll::Pending => { if owed > 0 { frame.notify(); @@ -430,8 +435,10 @@ impl Reader { if complete || chunk.len() == frame.unsealed() { // Every byte of the segment is in hand, so its credit returns. lease.release(); + self.unreleased = 0; } else { drop(lease); + self.unreleased += chunk.len() as u64; } if !chunk.is_empty() { @@ -488,7 +495,7 @@ impl Reader { self.stream.resume(); Poll::Ready(Ok(())) } - Poll::Pending if self.stream.pause() => { + Poll::Pending if self.stream.pause(self.unreleased) => { let err = ready!(self.stream.poll_reset(waiter)); Poll::Ready(Err(self.version.transport_error(err))) } @@ -557,6 +564,7 @@ impl Reader { version, want: self.want, scratch: self.scratch, + unreleased: self.unreleased, stats: self.stats, } } @@ -1145,11 +1153,27 @@ mod tests { chunks: VecDeque, held: usize, released: usize, + /// The pacing window, or the cap when zero. + window: u64, + /// The taken bytes each pause reserved for. + paused: Vec, } struct CreditedLease<'a>(&'a mut Credited); - impl Pause for Credited {} + impl Pause for Credited { + fn window(&self) -> u64 { + match self.window { + 0 => crate::model::pace::AHEAD, + window => window, + } + } + + fn pause(&mut self, taken: u64) -> bool { + self.paused.push(taken); + true + } + } impl RecvStream for Credited { type Error = crate::lite::test_transport::SinkError; @@ -1237,4 +1261,40 @@ mod tests { } assert_eq!(read, data); } + + /// A stream paused partway through a segment still holds the credit of what it took of + /// it, so the pause reserves that along with the stream window. + #[test] + fn a_pause_mid_segment_reserves_what_it_took() { + use crate::frame::SEGMENT; + const CHUNK: usize = 100_000; + + // An idle reader behind one committed frame, so the writer is paced past the window. + let mut group = crate::group::Info { sequence: 0 }.produce(); + let _idle = group.consume(); + group + .write_frame(crate::Timestamp::ZERO, Bytes::from_static(b"x")) + .unwrap(); + let mut frame = group + .create_frame(crate::frame::Info { + size: 2 * SEGMENT as u64, + timestamp: Some(crate::Timestamp::ZERO), + }) + .unwrap(); + + let stream = Credited { + window: SEGMENT as u64 / 2, + ..Default::default() + }; + let mut reader = Reader::new(stream, crate::lite::Version::Lite05); + for _ in 0..SEGMENT / CHUNK { + reader.stream.chunks.push_back(Bytes::from(vec![0u8; CHUNK])); + } + let noop = &kio::Waiter::noop(); + assert!(reader.poll_read_frame(noop, &mut frame).is_pending(), "paced mid-frame"); + + let stream = &reader.stream; + assert!(stream.held > 0 && stream.held < SEGMENT, "{} bytes held", stream.held); + assert_eq!(stream.paused, [stream.held as u64], "the pause reserves what it holds"); + } } diff --git a/rs/moq-net/src/transport.rs b/rs/moq-net/src/transport.rs index aed519661f0..abbe968b3c9 100644 --- a/rs/moq-net/src/transport.rs +++ b/rs/moq-net/src/transport.rs @@ -145,8 +145,9 @@ impl Metrics for Untracked {} /// /// A session's ingest stops reading a group's stream while the group's fastest reader is far /// behind, so flow control pushes back to the publisher. Such a paused stream keeps up to -/// `stream` bytes counted against the shared `connection` window, so the session pauses only -/// as many streams as fit in half of it; the rest keep reading. See +/// `stream` bytes it has not read, plus what it read of a frame's unsealed segment, counted +/// against the shared `connection` window, so the session pauses only as many streams as fit +/// in half of it; the rest keep reading. See /// [`poll::Session::receive_window`]. #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[non_exhaustive] diff --git a/rs/moq-net/src/transport/boxed.rs b/rs/moq-net/src/transport/boxed.rs index 110d0df768f..ab6c556b3ba 100644 --- a/rs/moq-net/src/transport/boxed.rs +++ b/rs/moq-net/src/transport/boxed.rs @@ -149,15 +149,16 @@ pub(super) struct Shared { } impl Shared { - /// Reserve the credit of one more paused stream, unless it would push the paused total - /// over half the connection window, returning how much it reserved. Without a connection - /// window a paused stream holds only its own credit, so it reserves nothing. - fn reserve(&self) -> Option { + /// Reserve the credit of one more paused stream, its stream window and the `taken` bytes it + /// read without returning their credit, unless it would push the paused total over half the + /// connection window, returning how much it reserved. Without a connection window a paused + /// stream holds only its own credit, so it reserves nothing. + fn reserve(&self, taken: u64) -> Option { let window = self.session.receive_window()?; let Some(connection) = window.connection else { return Some(0); }; - let charge = window.stream.min(connection.get()); + let charge = window.stream.saturating_add(taken).min(connection.get()); let cap = connection.get() / 2; self.paused .fetch_update(Ordering::AcqRel, Ordering::Acquire, |paused| { @@ -465,11 +466,12 @@ impl RecvStream { self.0.window() } - /// Reserve the receive credit this stream may hold while ingest stops reading it, returning - /// whether it may stop: not once the session's paused streams hold half its connection - /// window, nor on a transport that reports no window. - pub(crate) fn pause(&mut self) -> bool { - self.0.pause() + /// Reserve the receive credit this stream may hold while ingest stops reading it, its stream + /// window and the `taken` bytes read without returning their credit, returning whether it + /// may stop: not once the session's paused streams hold half its connection window, nor on + /// a transport that reports no window. + pub(crate) fn pause(&mut self, taken: u64) -> bool { + self.0.pause(taken) } /// Return the credit [`Self::pause`] reserved, as ingest reads the stream again. @@ -612,7 +614,7 @@ trait ErasedRecv: Send + Sync { fn stop(&mut self, code: u32); fn poll_reset(&mut self, waiter: &kio::Waiter) -> Poll; fn window(&self) -> u64; - fn pause(&mut self) -> bool; + fn pause(&mut self, taken: u64) -> bool; fn resume(&mut self); } @@ -826,11 +828,11 @@ impl ErasedRecv for Budgeted { }) } - fn pause(&mut self) -> bool { + fn pause(&mut self, taken: u64) -> bool { if self.held > 0 { return true; } - let Some(held) = self.session.upgrade().and_then(|shared| shared.reserve()) else { + let Some(held) = self.session.upgrade().and_then(|shared| shared.reserve(taken)) else { return false; }; self.held = held; @@ -1086,8 +1088,9 @@ mod test { recv } - /// Each paused stream reserves its stream window, and the session pauses no more than - /// fit in half the connection window; a stream resuming or dropped returns its share. + /// Each paused stream reserves its stream window and what it took of an unsealed segment, + /// and the session pauses no more than fit in half the connection window; a stream + /// resuming or dropped returns its share. #[test] fn paused_streams_hold_at_most_half_the_connection_window() { let gate = kio::Producer::new(true); @@ -1096,22 +1099,31 @@ mod test { .with_receive_window(ReceiveWindow::new(std::num::NonZeroU64::new(100), 20)), ); let mut streams: Vec<_> = (0..3).map(|_| accept(&session)).collect(); - assert!(streams[0].pause()); - assert!(streams[0].pause(), "a paused stream reserves once"); - assert!(streams[1].pause()); - assert!(!streams[2].pause(), "a third would hold 60 of 100"); + assert!(streams[0].pause(0)); + assert!(streams[0].pause(0), "a paused stream reserves once"); + assert!(streams[1].pause(0)); + assert!(!streams[2].pause(0), "a third would hold 60 of 100"); streams[0].resume(); - assert!(streams[2].pause(), "a resumed stream returned its share"); + assert!(streams[2].pause(0), "a resumed stream returned its share"); drop(streams.remove(1)); - assert!(accept(&session).pause(), "so did a dropped one"); + assert!(accept(&session).pause(0), "so did a dropped one"); + + // A stream partway through a segment also holds the credit of what it took of it. + let session = Session::new( + SinkSession::accepted_bi(gate.consume()) + .with_receive_window(ReceiveWindow::new(std::num::NonZeroU64::new(100), 20)), + ); + let mut first = accept(&session); + assert!(first.pause(25), "holds 45 of 100"); + assert!(!accept(&session).pause(0), "a second would hold 65 of 100"); // Without a connection window nothing is shared, so every stream may pause. let session = Session::new( SinkSession::accepted_bi(gate.consume()).with_receive_window(ReceiveWindow::new(None, u64::MAX)), ); let mut streams: Vec<_> = (0..3).map(|_| accept(&session)).collect(); - assert!(streams.iter_mut().all(|stream| stream.pause())); + assert!(streams.iter_mut().all(|stream| stream.pause(0))); } /// A stream paces at the credit its session grants it, so a hop holds about what its @@ -1263,7 +1275,7 @@ mod test { fn an_unbounded_stream_never_pauses() { let gate = kio::Producer::new(true); let session = Session::new(SinkSession::accepted_bi(gate.consume())); - assert!(!accept(&session).pause()); + assert!(!accept(&session).pause(0)); let session = Session::new( SinkSession::accepted_bi(gate.consume()) @@ -1271,6 +1283,6 @@ mod test { ); let mut stream = accept(&session); drop(session); - assert!(!stream.pause()); + assert!(!stream.pause(0)); } } diff --git a/rs/moq-quic/README.md b/rs/moq-quic/README.md index 6fbd191bbe3..abf37f604e2 100644 --- a/rs/moq-quic/README.md +++ b/rs/moq-quic/README.md @@ -88,7 +88,7 @@ Changes on top of the upstream commit, besides the renames: - **Fuzzing** runs through the `fuzz` module behind a `fuzz` feature, with one-line shims in [`fuzz/`](fuzz/README.md): upstream's `packet`, `params`, `streamid`, and `streams` targets, plus `frame` and a connection op sequence. Upstream's `cfg(fuzzing)` `fuzzing` module and `arbitrary` feature are gone, so its fuzz patches port by hand. - **Draining endpoint event**: `EndpointEvent::is_draining` fires once per connection when it stops being active, before the drained event, so the driver's `Endpoint::wait_all_draining` can end a shutdown without sitting out every closing period. - **Late stream limits**: peer transport parameters that raise a stream limit after an open was refused wake `Streams::poll_open`, as `MAX_STREAMS` does. A server accepted before the rest of a multi-packet ClientHello (a post-quantum key share is enough) otherwise leaves a 0.5-RTT `open_uni` parked until the idle timeout. Not offered upstream yet. -- **Receive leases** (`streams/lease.rs`, `recv::Stream::lease`), building on [#5675](https://github.com/moq-dev/moq/pull/5675)'s held credit: a lease reads a stream's ordered bytes out of the assembler but keeps their connection credit, counted in `StreamsState::held`, until the reader `advance`s or `take`s past them and `release`s. Stream credit runs from the bytes the reader took rather than leased (`Leased::untaken`), widened to a waiting reader's `want`, so a reader that stops taking holds its stream window and no more. A lease copies the bytes it reads out of their datagram buffers (`Leased::extend`), so a byte of credit never pins a receive batch, whether its reader waits, parks, or takes it. The lease lives in the stream's `Recv`, which a leased stream keeps, with its stream ID, until the application stops it, even past the FIN. When the peer has spent the connection window and no reader can make progress, the stream ranked lowest by `set_priority` (the newest on a tie) is evicted: its bytes and credit are freed, the stream is stopped without `STOP_SENDING`, and its next lease fails with `recv::Error::Evicted`; the application's `stop` then tells the peer and frees the stream once its final size is known. Leased streams that may hold credit are kept in `StreamsState::ranks`, in eviction order, so an eviction costs a logarithm rather than a scan of every stream. `MAX_DATA` goes out for any increase once the peer is within an eighth of the window (`max_data_due`), since what leases release may never add up to the usual threshold. Streams with data their reader has not looked at hold eviction off; they are kept in `StreamsState::unlooked`, so the stuck check stays constant time however many readers wait. `ConnectionStats::credit` reports the leased and available bytes, the streams evicted, and the peak credit spent. A `recv::Lease` owns the chunks it took, so a reader holds no connection borrow while it parses; `restore` or `release` hands it back, and an eviction pass waits while one is out. A port touching `Chunks` or `add_read_credits` has to keep `held` balanced. +- **Receive leases** (`streams/lease.rs`, `recv::Stream::lease`), building on [#5675](https://github.com/moq-dev/moq/pull/5675)'s held credit: a lease reads a stream's ordered bytes out of the assembler but keeps their connection credit, counted in `StreamsState::held`, until the reader `advance`s or `take`s past them and `release`s. Stream credit runs from the bytes the reader took rather than leased (`Leased::untaken`), widened to a waiting reader's `want`, so a reader that stops taking holds at most its stream window of bytes it has not taken, besides those it took and has not released. A lease copies the bytes it reads out of their datagram buffers (`Leased::extend`), so a byte of credit never pins a receive batch, whether its reader waits, parks, or takes it. The lease lives in the stream's `Recv`, which a leased stream keeps, with its stream ID, until the application stops it, even past the FIN. When the peer has spent the connection window and no reader can make progress, the stream ranked lowest by `set_priority` (the newest on a tie) is evicted: its bytes and credit are freed, the stream is stopped without `STOP_SENDING`, and its next lease fails with `recv::Error::Evicted`; the application's `stop` then tells the peer and frees the stream once its final size is known. Leased streams that may hold credit are kept in `StreamsState::ranks`, in eviction order, so an eviction costs a logarithm rather than a scan of every stream. `MAX_DATA` goes out for any increase once the peer is within an eighth of the window (`max_data_due`), since what leases release may never add up to the usual threshold. Streams with data their reader has not looked at hold eviction off; they are kept in `StreamsState::unlooked`, so the stuck check stays constant time however many readers wait. `ConnectionStats::credit` reports the leased and available bytes, the streams evicted, and the peak credit spent. A `recv::Lease` owns the chunks it took, so a reader holds no connection borrow while it parses; `restore` or `release` hands it back, and an eviction pass waits while one is out. A port touching `Chunks` or `add_read_credits` has to keep `held` balanced. - **Owned datagrams**: a received datagram is copied out of its packet when queued, since `datagram_receive_buffer_size` charges only its payload and a slice keeps the whole receive batch alive. quinn has the same gap; not offered upstream yet. - **Stream credit on stop**: `recv::Stream::stop` on a stream whose final size is already known (finished or reset) frees it and queues `MAX_STREAMS` at once, as reading the reset does, instead of leaving the credit until the next packet arrives. Not offered upstream yet; quinn `main` has the same gap. - **Drop resets** (`quic::driver::send::Stream`): a send stream dropped before `finish` is reset with code 0, or with the peer's `STOP_SENDING` code, instead of quinn's implicit `finish`, so a cancelled writer never ends a truncated stream with a FIN. This matches moq-uring's driver and moq-net's transport contract. `quic::web` resets after the WebTransport header with the mapped code. quinn's `stream_stopped_2` finishes its stream before dropping it. Not offered upstream: it reverses a documented quinn behaviour. diff --git a/rs/moq-quic/src/connection/streams/lease.rs b/rs/moq-quic/src/connection/streams/lease.rs index 59263a1f418..1815c412374 100644 --- a/rs/moq-quic/src/connection/streams/lease.rs +++ b/rs/moq-quic/src/connection/streams/lease.rs @@ -21,7 +21,8 @@ use crate::connection::waits::Wait; /// by [`Stream::lease()`]. /// /// Bytes the reader has yet to take keep their stream credit too (see [`Self::untaken`]), so a -/// reader that stops leasing holds at most its stream window, however much it leased before. +/// reader that stops leasing holds at most its stream window of them, however much it leased +/// before, besides what it took and has not released. #[derive(Debug, Default)] pub(super) struct Leased { chunks: VecDeque, diff --git a/rs/moq-quic/src/connection/streams/recv.rs b/rs/moq-quic/src/connection/streams/recv.rs index dba05078d18..b46d409c7fe 100644 --- a/rs/moq-quic/src/connection/streams/recv.rs +++ b/rs/moq-quic/src/connection/streams/recv.rs @@ -277,8 +277,8 @@ impl Recv { /// had failed. pub(super) fn max_stream_data(&mut self, stream_receive_window: u64) -> (u64, bool) { // Credit runs from what the reader took, not what it leased: a reader that stops taking - // holds its window and no more. One waiting on a unit larger than the window gets room - // for the unit, which the connection window bounds. + // holds at most its window of bytes it has not taken. One waiting on a unit larger than + // the window gets room for the unit, which the connection window bounds. let taken = self.assembler.bytes_read() - self.lease.untaken(); let window = stream_receive_window.max(self.lease.wanted()); // A window shrunk since the last announcement cannot take back credit already granted. diff --git a/rs/moq-relay/tests/integration/quic_sim.rs b/rs/moq-relay/tests/integration/quic_sim.rs index 73a64e2685e..c2b3cf14fe2 100644 --- a/rs/moq-relay/tests/integration/quic_sim.rs +++ b/rs/moq-relay/tests/integration/quic_sim.rs @@ -246,8 +246,8 @@ const LITE_07: &str = "moq-lite-07-wip"; /// waits on `ready` before each frame, and the reader drains at its own pace, slower than /// the writer, so every hop has to pace its ingest to that reader rather than shed. /// -/// A paused stream must hold no more than its stream window, or the connection window fills -/// with bytes nobody takes and the transport evicts the stream. +/// A paused stream must hold no more than the credit its session reserves for it, or the +/// connection window fills with bytes nobody takes and the transport evicts the stream. #[test] fn a_socket_session_carries_a_bulk_transfer_losslessly() { memory::run(|| async { diff --git a/rs/moq-tokio/src/quic/driver/recv.rs b/rs/moq-tokio/src/quic/driver/recv.rs index 96c6455ac47..aa722aa69fa 100644 --- a/rs/moq-tokio/src/quic/driver/recv.rs +++ b/rs/moq-tokio/src/quic/driver/recv.rs @@ -163,8 +163,8 @@ impl Stream { /// The lease owns the bytes and holds no lock, but they keep their connection flow control /// credit until read through the lease as a [`Buf`] and then [`Lease::release`]d; bytes not /// read are leased again next time. Stream credit returns as bytes are read through a lease, - /// so a reader that stops reading holds its stream window, and one waiting for a message - /// larger than the window gets room for the message. + /// so a reader that stops reading holds at most its stream window of unread bytes, and one + /// waiting for a message larger than the window gets room for the message. /// /// When the peer has spent the connection window and no reader can make progress, the /// stream ranked lowest by [`Self::set_priority`] is evicted: its buffered bytes and their From 3e5aff9dda9a81cb945bfd594927167d2beba939 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 23:58:32 -0800 Subject: [PATCH 4/6] refactor(net): lease the frame's stream without borrowing the reader Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/coding/reader.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/rs/moq-net/src/coding/reader.rs b/rs/moq-net/src/coding/reader.rs index faa70232c1b..c01ebef81b5 100644 --- a/rs/moq-net/src/coding/reader.rs +++ b/rs/moq-net/src/coding/reader.rs @@ -396,12 +396,17 @@ impl Reader { // A writer too far ahead of the group's fastest reader leaves the rest in the // stream, so flow control pushes back upstream. let lease = match self.poll_pace(waiter, frame.poll_ready(waiter, self.stream.window())) { - Poll::Ready(Ok(())) => self.poll_lease(waiter, frame.remaining().min(1)), + // The stream alone, so `unreleased` can count while the lease is out. + Poll::Ready(Ok(())) => { + let version = &self.version; + self.stream + .poll_lease(waiter, frame.remaining().min(1)) + .map_err(|err| version.transport_error(err)) + } Poll::Ready(Err(err)) => break Err(err), Poll::Pending => Poll::Pending, }; - // Moved, so the lease's borrow of the reader ends with it and `unreleased` can count. - let mut lease = match { lease } { + let mut lease = match lease { Poll::Pending => { if owed > 0 { frame.notify(); From 31a380509b1baad22979dbba60ea2ee0cc31e19c Mon Sep 17 00:00:00 2001 From: quest fixer Date: Sun, 15 Nov 2026 02:29:00 -0800 Subject: [PATCH 5/6] fix(net): adapt receive-credit test to base model APIs --- rs/moq-net/src/coding/reader.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/src/coding/reader.rs b/rs/moq-net/src/coding/reader.rs index 155f6d9ae67..3d06c29b81b 100644 --- a/rs/moq-net/src/coding/reader.rs +++ b/rs/moq-net/src/coding/reader.rs @@ -1275,7 +1275,7 @@ mod tests { const CHUNK: usize = 100_000; // An idle reader behind one committed frame, so the writer is paced past the window. - let mut group = crate::group::Info { sequence: 0 }.produce(); + let mut group = crate::group::Info { start: 0 }.produce(); let _idle = group.consume(); group .write_frame(crate::Timestamp::ZERO, Bytes::from_static(b"x")) @@ -1283,7 +1283,7 @@ mod tests { let mut frame = group .create_frame(crate::frame::Info { size: 2 * SEGMENT as u64, - timestamp: Some(crate::Timestamp::ZERO), + timestamp: crate::Timestamp::ZERO, }) .unwrap(); From 2d68e883076b1496713cfefbbfb2cd82f9d0c281 Mon Sep 17 00:00:00 2001 From: quest fixer Date: Sun, 15 Nov 2026 02:33:07 -0800 Subject: [PATCH 6/6] fix(relay): adapt bulk-transfer test to base group APIs --- rs/moq-relay/tests/integration/quic_sim.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/rs/moq-relay/tests/integration/quic_sim.rs b/rs/moq-relay/tests/integration/quic_sim.rs index 2345fa38e64..cf5bf439243 100644 --- a/rs/moq-relay/tests/integration/quic_sim.rs +++ b/rs/moq-relay/tests/integration/quic_sim.rs @@ -268,16 +268,18 @@ fn a_socket_session_carries_a_bulk_transfer_losslessly() { let (dialed, accepted) = (dialed.expect("dial"), accepted.expect("accept")); let write = async { - let mut group = accepted.producer.append_group().unwrap(); + let mut group = accepted.producer.append_group(0).unwrap(); let info = moq_net::frame::Info { size: CHUNK as u64, - timestamp: None, + timestamp: moq_net::Timestamp::unscaled(0), }; for index in 0..BULK / CHUNK { group.ready(&info).await; - group.write_frame(None, vec![index as u8; CHUNK]).unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(0), vec![index as u8; CHUNK]) + .unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); }; let read = async { let mut ordered = dialed.consumer.ordered();