From f0bf12d6a508a7efe8dc81df68964591c743f201 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 27 Oct 2026 22:25:40 -0700 Subject: [PATCH 1/4] chore(quest): claim quest/m1/ietf-request-cancel.md Quest-Claim: cfb3a4e274143dda From daf16d96ecbfc270ca7a0e20209fef4909fe5ff7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 27 Oct 2026 23:02:40 -0700 Subject: [PATCH 2/4] fix(net): end moq-transport requests stalled on send credit Every request write in the IETF subscriber awaited its first message before watching for cancellation, so a peer withholding send credit pinned a cancelled SUBSCRIBE, FETCH, TRACK_STATUS, or SUBSCRIBE_NAMESPACE. Each request now buffers its message and flushes it inside the wait that already watches its terminals, as SUBSCRIBE_TRACKS did. Established REQUEST_UPDATEs and held pauses flush the same way. A request cancelled before its message fully left resets its stream: a FIN would hand the peer a truncated message. `Writer::close` resets in that case, and the cancel paths that wait for the peer's half skip the wait. Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 1 - quest/m1/ietf-request-cancel.md | 29 -- quest/m2/ietf-request-lease.md | 5 +- rs/moq-net/src/coding/writer.rs | 37 +++ rs/moq-net/src/ietf/subscriber.rs | 363 ++++++++++++++++------- rs/moq-net/src/ietf/subscriber/tracks.rs | 55 +++- rs/moq-net/src/lite/test_transport.rs | 20 +- 7 files changed, 349 insertions(+), 161 deletions(-) delete mode 100644 quest/m1/ietf-request-cancel.md diff --git a/quest/m1/README.md b/quest/m1/README.md index f0c572d128b..86e371bc760 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -116,7 +116,6 @@ surface and zero-copy texture import, land before the cut. - [Address groups by timestamp](/quest/m1/time-addressing/README.md) - group start timestamps replace sequence numbers in the moq-net API and lite-07 wire, while old subscribers, old publishers, and published JS clients keep working - [Release audit findings](/quest/m1/release-audit/README.md) - resolve newly identified resource, parser, playback, and interoperability risks before the cut - [Pre-cut audit findings](/quest/m1/cut-audit/README.md) - fix the crashes, hostile-input costs, and correctness bugs the 2026-10-24 audit of main found -- [Cancelled moq-transport requests stop](/quest/m1/ietf-request-cancel.md) - a cancelled request releases its state even when the peer withholds credit - [Cluster routing](/quest/m1/cluster-routing/README.md) - prove shared routing contracts, implement per-origin routes, and connect outbound local clusters - [Routing guide](/quest/m1/routing-guide.md) - concepts and practical outbound federation setup - [moq-c at parity](/quest/m1/c/README.md) - the hand-written `moq-c` ships at the cut with every m1 binding feature moq-ffi gains diff --git a/quest/m1/ietf-request-cancel.md b/quest/m1/ietf-request-cancel.md deleted file mode 100644 index 07d95043c8b..00000000000 --- a/quest/m1/ietf-request-cancel.md +++ /dev/null @@ -1,29 +0,0 @@ -# [M] A cancelled moq-transport request stops writing - -## Goal - -Cancelling a moq-transport SUBSCRIBE, FETCH, TRACK_STATUS, or namespace -request releases its state at once, even when the peer's send window holds -back the request's first message. - -## Plan - -Every request write in `rs/moq-net/src/ietf/subscriber.rs` awaits its first -`encode_message` without watching for cancellation, so a peer that withholds -flow-control credit pins a cancelled request. -[moq-dev/moq#6409](https://github.com/moq-dev/moq/pull/6409) already refuses -a request at once when stream credit is exhausted, but an admitted header -still waits for send capacity, and -[moq-dev/moq#6530](https://github.com/moq-dev/moq/pull/6530) landed the Rust -lazy discovery that rewrote these writes. What remains is the awaited first -encode under send-window pressure: watch for cancellation there and reset the -partially written request stream. - -Drafts 14 and 15 shared one ordered control stream, so a partial message -could not be abandoned; every remaining draft gives each request its own -stream, so only the reset path is needed. - -Test with a peer that never grants send credit, on drafts 18 and 22. - -Public API: none. Wire: none. - diff --git a/quest/m2/ietf-request-lease.md b/quest/m2/ietf-request-lease.md index 09505b8926a..4e00869b3c7 100644 --- a/quest/m2/ietf-request-lease.md +++ b/quest/m2/ietf-request-lease.md @@ -26,8 +26,8 @@ one terminal at one await: [#6237](https://github.com/moq-dev/moq/pull/6237), [#6409](https://github.com/moq-dev/moq/pull/6409), [#6433](https://github.com/moq-dev/moq/pull/6433), -[#6643](https://github.com/moq-dev/moq/pull/6643), plus the open -[cancel quest](/quest/m1/ietf-request-cancel.md) and the +[#6643](https://github.com/moq-dev/moq/pull/6643), plus the +[cancel fix](https://github.com/moq-dev/moq/pull/7016) and the [withdraw fix](https://github.com/moq-dev/moq/pull/6964). Each request is a long `async fn` whose every `kio::wait` closure re-lists the right subset of about 46 terminal checks; the two files hold 227 `pin!` sites @@ -58,4 +58,3 @@ Public API: none. Wire: none. ## Required - [moq-transport 18 and 22 only](/quest/m1/ietf-drop/README.md) - deletes about 2,950 production lines and the adapter first -- [Cancelled moq-transport requests stop](/quest/m1/ietf-request-cancel.md) - lands the last point fix this replaces diff --git a/rs/moq-net/src/coding/writer.rs b/rs/moq-net/src/coding/writer.rs index b7d08673cc7..866fc1897a5 100644 --- a/rs/moq-net/src/coding/writer.rs +++ b/rs/moq-net/src/coding/writer.rs @@ -152,6 +152,11 @@ impl Writer { std::future::poll_fn(|cx| self.poll_write_all(cx, buf)).await } + /// Whether every buffered byte has hit the stream, so a FIN would not cut a message short. + pub fn is_flushed(&self) -> bool { + self.buffer.is_empty() + } + /// Mark the stream as finished. /// /// Only valid once the buffer has flushed; the callers that finish @@ -184,7 +189,15 @@ impl Writer { /// discards it, so the [`Drop`] fallback below can throw away bytes the peer never read. /// Consuming the writer is what removes that fallback, and waiting for the acknowledgement /// is what makes the bytes safe. + /// + /// A message still partly buffered resets the stream with [`Error::Cancel`] instead: a FIN + /// would hand the peer a truncated message, and flushing the rest could wait forever on + /// credit the peer withholds. pub async fn close(mut self) -> Result<(), Error> { + if !self.is_flushed() { + self.abort(&Error::Cancel); + return Err(Error::Cancel); + } let Some(mut stream) = self.stream.take() else { return Ok(()); }; @@ -366,6 +379,30 @@ mod tests { assert_eq!(log.priorities(), (0u8..=255).collect::>()); } + /// A message the stream has not taken whole cannot be finished, so closing resets rather + /// than hand the peer a truncated message. + #[moq_clock::test] + async fn closing_an_unflushed_message_resets() { + let log = Log::default(); + let budget = kio::Producer::new(0); + let mut writer = Writer::new( + SinkSend::stepped(log.clone(), budget.consume()), + crate::lite::Version::Lite05, + ); + writer.buffer_varint(5).unwrap(); + let mut cx = std::task::Context::from_waker(Waker::noop()); + assert!(writer.poll_flush(&mut cx).is_pending()); + assert!(!writer.is_flushed()); + + assert!(matches!(writer.close().await, Err(Error::Cancel))); + assert_eq!(log.resets(), vec![StreamError::Cancel.to_code()]); + assert!( + !log.trail() + .iter() + .any(|(_, sent)| matches!(sent, crate::lite::test_transport::Sent::Finish)) + ); + } + /// Writes some bytes, then errors: the shape of a partial encode. #[derive(Debug)] struct Poison; diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 54cebefe420..c2403689bf7 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -916,8 +916,8 @@ struct Held { epoch: Option, } -/// A joining request may be abandoned before it takes an ID. Once committed, its header -/// must still reach the peer, which otherwise remembers the ID it never saw. +/// A joining request may be abandoned before it takes an ID. Once committed, the session +/// owns ending it on the wire. struct JoiningTask { work: Option>>, state: kio::Shared, @@ -950,8 +950,9 @@ impl Drop for JoiningTask { state.committed }; if committed && let Some(work) = self.work.take() { - // The session owns the remaining send, including after the subscription - // ends. It closes the request immediately after delivering its header. + // The session owns cancelling the request, including after the subscription + // ends: closing it once its header is out, or resetting it while the peer's + // send window still holds the header back. self.tasks.push(async move { let _ = work.await; }); @@ -1422,9 +1423,11 @@ impl Subscriber { namespace: prefix.clone(), hidden, }; - stream.writer.encode_message(&msg).await?; + // Flushed while waiting for the answer, so `live` closing still ends a request the + // peer's send window holds back. + stream.writer.buffer_message(ietf::SubscribeNamespace::ID, &msg)?; - tracing::debug!(%prefix, "subscribe_namespace sent"); + tracing::debug!(%prefix, "subscribe_namespace started"); // Read response let Some(framed) = self.next_namespace_message(request, live).await? else { @@ -1481,6 +1484,7 @@ impl Subscriber { return Poll::Ready(Ok(None)); } let mut cx = waiter.context(); + ready!(stream.writer.poll_flush(&mut cx))?; messages .poll_next(&mut stream.reader, &mut cx) .map(|next| next.map(Some)) @@ -1506,6 +1510,13 @@ impl Subscriber { if request_id.is_none() { return; } + // The request never reached the peer whole, so it holds nothing to wait on. + if !writer.is_flushed() { + tracing::debug!(%prefix, "subscribe_namespace abandoned before it was sent"); + reader.abort(&Error::Cancel); + writer.abort(&Error::Cancel); + return; + } tracing::debug!(%prefix, "unsubscribe_namespace"); match super::request_stream::fin_cancels(self.version) { @@ -3076,6 +3087,8 @@ impl Subscriber { let order = aggregate.map(|s| s.order).unwrap_or_default(); // Without an Established subscription, a relay includes the request it holds. let new_group = target.new_group(); + // Buffered only: the wait for the answer flushes it, so the local terminals still end a + // request the peer's send window holds back. let written = match &held { // Resume the held subscription: its group order is the one it was made with, // since no update can change it. Its groups wait on the answer from here. @@ -3090,37 +3103,32 @@ impl Subscriber { // Only a track declaring DYNAMIC_GROUPS takes one on an update. new_group: new_group.filter(|_| held.properties.dynamic_groups), }; - self.write_forward(&mut stream.writer, &forward).await - } - None => { - async { - stream - .writer - .encode_message(&ietf::Subscribe { - max_age, - new_group, - request_id, - track_namespace: broadcast_path.clone(), - track_name: track_name.into(), - subscriber_priority: super::priority::to_wire(priority), - group_order: order.into(), - filter: join.filter, - fill: join.fill, - // A resumed copy already knows the track, from the answer that accepted it. - properties_wanted: matches!(target, Target::Request(_)), - forward: true, - rendezvous: false, - range_filters: false, - authorization_token: None, - epoch: self.epoch.clone(), - }) - .await - } - .await + self.buffer_forward(&mut stream.writer, &forward) } + None => stream.writer.buffer_message( + ietf::Subscribe::ID, + &ietf::Subscribe { + max_age, + new_group, + request_id, + track_namespace: broadcast_path.clone(), + track_name: track_name.into(), + subscriber_priority: super::priority::to_wire(priority), + group_order: order.into(), + filter: join.filter, + fill: join.fill, + // A resumed copy already knows the track, from the answer that accepted it. + properties_wanted: matches!(target, Target::Request(_)), + forward: true, + rendezvous: false, + range_filters: false, + authorization_token: None, + epoch: self.epoch.clone(), + }, + ), }; if let Err(err) = written { - tracing::debug!(%err, "failed to write subscribe"); + tracing::debug!(%err, "failed to encode subscribe"); self.remove_subscribe(request_id); self.entry_ended(&key, request_id, Err(err.clone())); target.fail(err); @@ -3161,7 +3169,10 @@ impl Subscriber { // A local terminal ended the wait, so the peer still holds the request to cancel. let mut unanswered = false; let setup = { - let mut response = std::pin::pin!(self.read_answer(&mut stream, &mut messages, held.as_mut())); + let mut response = std::pin::pin!(async { + std::future::poll_fn(|cx| stream.writer.poll_flush(cx)).await?; + self.read_answer(&mut stream, &mut messages, held.as_mut()).await + }); loop { let setup = kio::wait(|waiter| { let response = waiter.poll_future(response.as_mut()); @@ -3253,7 +3264,7 @@ impl Subscriber { owed: held.owed + 1, copy: resumed.as_ref().map(|idle| idle.track.clone()), }; - self.pause(key, upstream).await; + self.pause(key, upstream); return resumed; } // The publisher may already be serving before it answers. A session abort @@ -3444,6 +3455,8 @@ impl Subscriber { Fill(track::Fill), Update(Update), Invalid(Error), + /// An update failed to go out. + Failed(Error), } // The NEW_GROUP_REQUEST last sent, the SUBSCRIBE's included. It stays outstanding @@ -3496,6 +3509,11 @@ impl Subscriber { if gate.poll_denied(waiter).is_ready() { return Poll::Ready(End::Revoked); } + // An update the peer's send window holds back goes out from here, so it never + // holds back the ends below. + if let Poll::Ready(Err(err)) = stream.writer.poll_flush(&mut waiter.context()) { + return Poll::Ready(End::Failed(err)); + } if !fetch_done && let Some(fut) = fetching.as_mut() && let Poll::Ready(res) = fut.poll(waiter) @@ -3659,7 +3677,7 @@ impl Subscriber { sent_bounds = wanted_bounds.clone().expect("validated update bounds"); } let (asked, new_group, asked_age) = (update.priority, update.new_group, update.max_age); - if let Err(err) = self.write_update(&mut stream.writer, request_id, update).await { + if let Err(err) = self.buffer_update(&mut stream.writer, request_id, update) { tracing::debug!(%err, "failed to update the subscription"); ended = Some(Err(err.clone())); let _ = track.clone().abort(err); @@ -3679,6 +3697,12 @@ impl Subscriber { } unanswered.fetch_add(1, Ordering::Relaxed); } + End::Failed(err) => { + tracing::debug!(%err, "failed to update the subscription"); + ended = Some(Err(err.clone())); + let _ = track.clone().abort(err); + break false; + } End::Revoked => { tracing::info!(broadcast = %crate::path::Rooted(self.origin.root(), broadcast_path), track = %track_name, "subscription no longer authorized"); let _ = track.clone().abort(Error::Unauthorized); @@ -3771,7 +3795,7 @@ impl Subscriber { } }; - // Release an unallocated join now, or let the session finish its committed header. + // Release an unallocated join now, or let the session cancel its committed request. drop(fetching); if let Some(end) = ended { @@ -3788,7 +3812,7 @@ impl Subscriber { owed: unanswered.load(Ordering::Relaxed), copy: Some(track.clone()), }; - self.pause(key, upstream).await; + self.pause(key, upstream); return Some(Idle { demand: track.demand(), track, @@ -3990,9 +4014,9 @@ impl Subscriber { } } - /// Update an Established subscription: REQUEST_UPDATE carrying only what changed, under a - /// Request ID of its own. - async fn write_update( + /// Buffer the update to an Established subscription, for its wait to flush: + /// REQUEST_UPDATE carrying only what changed, under a Request ID of its own. + fn buffer_update( &self, writer: &mut Writer, subscription: RequestId, @@ -4013,8 +4037,9 @@ impl Subscriber { } state.fetches.insert(request_id, FillRequest::Update(subscription)); } - writer - .encode_message(&ietf::SubscribeUpdate { + writer.buffer_message( + ietf::SubscribeUpdate::ID, + &ietf::SubscribeUpdate { max_age: update.max_age, request_id, subscriber_priority: update.priority.map(super::priority::to_wire), @@ -4023,8 +4048,8 @@ impl Subscriber { authorization_token: None, new_group: update.new_group, fill: update.fill, - }) - .await + }, + ) } /// Tell the publisher to stop serving a subscription we are walking away from. @@ -4047,7 +4072,7 @@ impl Subscriber { // Finishing alone would leave the writer's Drop free to RESET_STREAM, and a stream // that has sent its FIN is still retransmitting: the reset would discard what we // already wrote. Closing consumes the writer, removing that fallback, and waits for - // the acknowledgement. + // the acknowledgement. A request the peer's send window held back resets instead. if let Err(err) = writer.close().await { tracing::debug!(%err, "failed to close the subscribe stream"); } @@ -4094,33 +4119,27 @@ impl Subscriber { state.fetches.insert(fetch_id, FillRequest::Joining(subscribe_id)); } } - if let Err(err) = async { - stream - .writer - .encode_message(&ietf::Fetch { - request_id: fetch_id, - subscriber_priority: super::priority::to_wire( - track.subscription().map(|s| s.priority).unwrap_or(0), - ), - group_order: GroupOrder::Ascending, - fetch_type: FetchType::RelativeJoining { - subscriber_request_id: subscribe_id, - group_offset: 0, - }, - range_filters: false, - fill_timeout: false, - properties_wanted: true, - // Served by the subscription it joins, which named the instance. - epoch: None, - }) - .await?; - Ok::<(), Error>(()) - } - .await - { - tracing::debug!(%err, "failed to write joining FETCH"); - return Err(err); - } + // Flushed while waiting for the answer, so a cancellation still ends a request the + // peer's send window holds back. + stream.writer.buffer_message( + ietf::Fetch::ID, + &ietf::Fetch { + request_id: fetch_id, + subscriber_priority: super::priority::to_wire( + track.subscription().map(|s| s.priority).unwrap_or(0), + ), + group_order: GroupOrder::Ascending, + fetch_type: FetchType::RelativeJoining { + subscriber_request_id: subscribe_id, + group_offset: 0, + }, + range_filters: false, + fill_timeout: false, + properties_wanted: true, + // Served by the subscription it joins, which named the instance. + epoch: None, + }, + )?; this.finish_joining_fetch(stream, progress).await } .await; @@ -4167,7 +4186,10 @@ impl Subscriber { let mut messages = super::request_stream::Messages::new(self.session.clone(), self.control.client()); let res = async { let answer = { - let mut response = std::pin::pin!(self.read_fetch_response(&mut stream, &mut messages)); + let mut response = std::pin::pin!(async { + std::future::poll_fn(|cx| stream.writer.poll_flush(cx)).await?; + self.read_fetch_response(&mut stream, &mut messages).await + }); kio::wait(|waiter| { if cancelled(waiter).is_ready() { return Poll::Ready(None); @@ -4264,19 +4286,18 @@ impl Subscriber { stream = stream.with_stats(self.origin.stats()); let request_id = self.control.next_request_id(); - let written = async { - stream - .writer - .encode_message(&ietf::TrackStatus { - request_id, - track_namespace: broadcast_path.clone(), - track_name: track_name.into(), - properties_wanted: true, - epoch: self.epoch.clone(), - }) - .await - } - .await; + // Flushed while waiting for the answer, so the terminals below still end a request + // the peer's send window holds back. + let written = stream.writer.buffer_message( + ietf::TrackStatus::ID, + &ietf::TrackStatus { + request_id, + track_namespace: broadcast_path.clone(), + track_name: track_name.into(), + properties_wanted: true, + epoch: self.epoch.clone(), + }, + ); if let Err(err) = written { request.reject(err); return None; @@ -4287,7 +4308,10 @@ impl Subscriber { let mut demand = request.demand(); let mut closed = self.session.clone(); let response = { - let mut response = std::pin::pin!(self.read_track_status_response(&mut stream)); + let mut response = std::pin::pin!(async { + std::future::poll_fn(|cx| stream.writer.poll_flush(cx)).await?; + self.read_track_status_response(&mut stream).await + }); loop { let res = kio::wait(|waiter| { let response = waiter.poll_future(response.as_mut()); @@ -5389,30 +5413,31 @@ impl Subscriber { end: ietf::Location { group: last, object: 0 }, }, }; - let res = async { - stream - .writer - .encode_message(&ietf::Fetch { - request_id: fetch_id, - subscriber_priority: super::priority::to_wire(fill.priority()), - group_order: GroupOrder::Ascending, - fetch_type, - range_filters: false, - fill_timeout: false, - // The copy learned the track before any fetch, from SUBSCRIBE_OK or - // TRACK_STATUS_OK, so a FETCH_OK repeating it is waste. - properties_wanted: false, - epoch: self.epoch.clone(), - }) - .await?; - Ok::<_, Error>(()) - } - .await; + // Flushed while waiting for the answer, so the terminals below still end a request the + // peer's send window holds back. + let res = stream.writer.buffer_message( + ietf::Fetch::ID, + &ietf::Fetch { + request_id: fetch_id, + subscriber_priority: super::priority::to_wire(fill.priority()), + group_order: GroupOrder::Ascending, + fetch_type, + range_filters: false, + fill_timeout: false, + // The copy learned the track before any fetch, from SUBSCRIBE_OK or + // TRACK_STATUS_OK, so a FETCH_OK repeating it is waste. + properties_wanted: false, + epoch: self.epoch.clone(), + }, + ); let mut messages = super::request_stream::Messages::new(self.session.clone(), self.control.client()); let res = match res { Err(err) => SetupResponse::Response(Err(err)), Ok(()) => { - let mut response = std::pin::pin!(self.read_fetch_response(&mut stream, &mut messages)); + let mut response = std::pin::pin!(async { + std::future::poll_fn(|cx| stream.writer.poll_flush(cx)).await?; + self.read_fetch_response(&mut stream, &mut messages).await + }); kio::wait(|waiter| { let response = waiter.poll_future(response.as_mut()); if let Poll::Ready(Err(err)) = response { @@ -6321,6 +6346,58 @@ mod tests { assert_eq!(occurrences(&log, b"rootns"), 0, "asked the peer for our local root"); } + /// A SUBSCRIBE_NAMESPACE the peer's send window holds back part-way still ends as soon as + /// `live` closes, resetting the stream: a FIN would hand the peer a truncated request, + /// and there is no answer to wait for. + #[moq_clock::test] + async fn a_namespace_request_stalled_on_credit_ends_when_cancelled() { + for version in [Version::Draft18, Version::Draft22] { + let session = crate::lite::test_transport::ScriptedSession::new(Vec::new()).with_send_window(1); + let log = session.log.clone(); + let (tasks, _task_set) = crate::util::TaskSet::new(); + let mut subscriber = Subscriber::new( + crate::time::sim(), + crate::driver::Transport::new(session.clone()), + crate::origin::Config::new(crate::Hop::new(1).unwrap()).produce(), + Control::new(false), + None, + peer::PeerSetup::declared(peer::Peer::default()), + crate::Hop::new(1).unwrap(), + None, + version, + tasks, + Default::default(), + ); + + let (cancel, live) = live(); + let stream = Stream::open(&mut crate::driver::Transport::new(session.clone()), version) + .await + .unwrap(); + let mut run = std::pin::pin!(subscriber.run_subscribe_namespace( + stream, + Path::from_static("room"), + crate::model::Snapshot::landed(), + live + )); + assert!(futures::poll!(run.as_mut()).is_pending()); + assert_eq!(log.writes.lock().unwrap().len(), 1, "{version}: stalled mid-message"); + + drop(cancel); + let Poll::Ready(res) = futures::poll!(run.as_mut()) else { + panic!("{version}: the cancelled request stayed alive"); + }; + assert!(matches!(res, Ok(namespaces::Ended::Local)), "{version}: {res:?}"); + assert_eq!(log.resets(), vec![crate::ietf::error::CANCELLED], "{version}"); + assert_eq!(log.stops(), vec![crate::ietf::error::CANCELLED], "{version}"); + assert!( + !log.trail() + .iter() + .any(|(_, sent)| matches!(sent, crate::lite::test_transport::Sent::Finish)), + "{version}: finished a truncated request" + ); + } + } + /// The peer's REQUEST_OK followed by one NAMESPACE, framed exactly as /// `run_subscribe_namespace` reads it -- built with the crate's own writer so the /// framing can't drift from the encoder under test. @@ -7500,9 +7577,13 @@ mod tests { #[derive(Clone, Copy, Debug)] enum SetupWait { SubscribeOpen, + /// The request's first message, stalled part-way on send credit. + SubscribeWrite, StatusOpen, + StatusWrite, StatusResponse, FetchOpen, + FetchWrite, FetchResponse, } @@ -7521,9 +7602,15 @@ mod tests { stage, SetupWait::SubscribeOpen | SetupWait::StatusOpen | SetupWait::FetchOpen ); + let writing = matches!( + stage, + SetupWait::SubscribeWrite | SetupWait::StatusWrite | SetupWait::FetchWrite + ); let open = kio::Producer::new(false); let session = if opening { crate::lite::test_transport::ScriptedSession::gated_open(Vec::new(), open.consume()) + } else if writing { + crate::lite::test_transport::ScriptedSession::new(Vec::new()).with_send_window(1) } else { crate::lite::test_transport::ScriptedSession::new(Vec::new()).with_unacked_fin() }; @@ -7565,7 +7652,7 @@ mod tests { let mut group = None; let mut accepted = None; let work: std::pin::Pin>> = match stage { - SetupWait::SubscribeOpen => Box::pin(async { + SetupWait::SubscribeOpen | SetupWait::SubscribeWrite => Box::pin(async { assert!( subscriber .subscribe_once( @@ -7579,7 +7666,7 @@ mod tests { .is_none() ); }), - SetupWait::StatusOpen | SetupWait::StatusResponse => Box::pin(async { + SetupWait::StatusOpen | SetupWait::StatusWrite | SetupWait::StatusResponse => Box::pin(async { assert!( subscriber .track_status(&Path::from_static("broadcast"), "video", request, &mut gate) @@ -7608,6 +7695,13 @@ mod tests { if opening { assert!(log.writes.lock().unwrap().is_empty()); } + if writing { + assert_eq!( + log.writes.lock().unwrap().len(), + 1, + "{stage:?}/{version}: stalled mid-message" + ); + } let expected = match cause { Invalidated::Retired => { drop(route); @@ -7650,6 +7744,25 @@ mod tests { if opening { assert!(done, "{stage:?}/{version}/{cause:?}: cancelled wait stayed alive"); assert!(log.writes.lock().unwrap().is_empty()); + } else if writing { + assert!(done, "{stage:?}/{version}/{cause:?}: cancelled write stayed alive"); + // A FIN would hand the peer a truncated message. + assert_eq!( + log.resets(), + vec![crate::ietf::error::CANCELLED], + "{stage:?}/{version}/{cause:?}" + ); + assert!( + !log.trail() + .iter() + .any(|(_, sent)| matches!(sent, crate::lite::test_transport::Sent::Finish)), + "{stage:?}/{version}/{cause:?}: finished a truncated request" + ); + assert_eq!( + log.stops(), + vec![crate::ietf::error::CANCELLED], + "{stage:?}/{version}/{cause:?}" + ); } else { assert_eq!( log.stops(), @@ -7678,9 +7791,12 @@ mod tests { } setup_wait_test!(invalid_subscribe_open_is_refused, SubscribeOpen, [Draft22]); + setup_wait_test!(invalid_subscribe_write_is_refused, SubscribeWrite, [Draft18, Draft22]); setup_wait_test!(invalid_status_open_is_refused, StatusOpen, [Draft22]); + setup_wait_test!(invalid_status_write_is_refused, StatusWrite, [Draft18, Draft22]); setup_wait_test!(invalid_status_response_is_refused, StatusResponse, [Draft22]); setup_wait_test!(invalid_fetch_open_is_refused, FetchOpen, [Draft22]); + setup_wait_test!(invalid_fetch_write_is_refused, FetchWrite, [Draft18, Draft22]); setup_wait_test!(invalid_fetch_response_is_refused, FetchResponse, [Draft22]); /// The control messages that actually reached the wire, by type id. @@ -15059,6 +15175,31 @@ mod joining_fetch_tests { } } + /// A committed joining FETCH the peer's send window holds back part-way is reset once + /// its subscription lets go, rather than waiting for credit to finish the header. + #[moq_clock::test] + async fn a_joining_fetch_stalled_on_credit_resets_when_cancelled() { + for version in JOINING_DRAFTS { + let session = ScriptedSession::new(Vec::new()).with_send_window(1); + let log = session.log.clone(); + let (subscriber, track, mut tasks) = pending_join(crate::driver::Transport::new(session.clone()), version); + let mut joining = subscriber.start_joining_fetch(RequestId(99), &track).unwrap(); + assert!(joining.poll(&kio::Waiter::noop()).is_pending()); + assert_eq!(log.writes.lock().unwrap().len(), 1, "{version}: stalled mid-message"); + + // The session's tasks now own the request. + drop(joining); + let _ = tasks.poll(&kio::Waiter::noop()); + assert_eq!(log.resets(), vec![crate::ietf::error::CANCELLED], "{version}"); + assert!( + !log.trail() + .iter() + .any(|(_, sent)| matches!(sent, crate::lite::test_transport::Sent::Finish)), + "{version}: finished a truncated request" + ); + } + } + /// Abandoning a blocked open leaves the next sequential ID available to unrelated work. #[moq_clock::test] async fn cancelling_a_joining_fetch_open_spends_no_id() { diff --git a/rs/moq-net/src/ietf/subscriber/tracks.rs b/rs/moq-net/src/ietf/subscriber/tracks.rs index f2d5ac2b8ad..a5c61885e05 100644 --- a/rs/moq-net/src/ietf/subscriber/tracks.rs +++ b/rs/moq-net/src/ietf/subscriber/tracks.rs @@ -510,6 +510,12 @@ impl Subscriber { // made stay ours. tracing::debug!(prefix = %crate::path::Rooted(self.origin.root(), prefix), "subscribe_tracks cancelled"); let Stream { mut writer, mut reader } = stream; + // The request never reached the peer whole, so it holds nothing to wait on. + if !writer.is_flushed() { + reader.abort(&Error::Cancel); + writer.abort(&Error::Cancel); + return Ok(()); + } match request_stream::fin_cancels(self.version) { true => { let _ = writer.finish(); @@ -632,9 +638,11 @@ impl Subscriber { ); let mut messages = Messages::new(self.session.clone(), self.control.client()); let accepted = async { - stream - .writer - .encode_message(&ietf::Subscribe { + // Flushed while waiting for the answer, so the entry going still ends a request + // the peer's send window holds back. + stream.writer.buffer_message( + ietf::Subscribe::ID, + &ietf::Subscribe { max_age: None, new_group: None, request_id, @@ -651,9 +659,12 @@ impl Subscriber { range_filters: false, authorization_token: None, epoch: self.epoch.clone(), - }) - .await?; - let mut response = std::pin::pin!(self.read_subscribe_response(&mut stream, &mut messages)); + }, + )?; + let mut response = std::pin::pin!(async { + std::future::poll_fn(|cx| stream.writer.poll_flush(cx)).await?; + self.read_subscribe_response(&mut stream, &mut messages).await + }); let answered = kio::wait(|waiter| { if gone(waiter) { return Poll::Ready(None); @@ -905,6 +916,9 @@ impl Subscriber { if session.poll_closed(&mut cx).is_ready() { return Poll::Ready(Step::Closed); } + if let Poll::Ready(Err(err)) = upstream.stream.writer.poll_flush(&mut cx) { + return Poll::Ready(Step::Message(Err(err))); + } if let Poll::Ready(handoff) = handoff.poll(waiter, |handoff| match handoff.gone || handoff.wanted { true => Poll::Ready(()), false => Poll::Pending, @@ -967,7 +981,13 @@ impl Subscriber { }; tracing::debug!(broadcast = %key.0, track = %key.1, ?end, "held subscription ended"); self.remove_subscribe(upstream.request_id); - let _ = upstream.stream.writer.finish(); + // A pause still held back would go out truncated behind a FIN. + match upstream.stream.writer.is_flushed() { + true => { + let _ = upstream.stream.writer.finish(); + } + false => upstream.stream.writer.abort(&Error::Cancel), + } // A failed upstream fails the copy, as it would while forwarding. TRACK_ENDED leaves it // idle: the end is past groups it never saw, so a returning reader's SUBSCRIBE learns // where, as for a copy whose subscription was cancelled. @@ -1077,14 +1097,15 @@ impl Subscriber { }); } - /// Pause a subscription nobody reads that is its track's entry, and hold it again. - pub(super) async fn pause(&self, key: TrackKey, mut upstream: Upstream) { + /// Pause a subscription nobody reads that is its track's entry, and hold it again. The + /// held watch flushes the pause, so the entry going still ends it under send pressure. + pub(super) fn pause(&self, key: TrackKey, mut upstream: Upstream) { self.park_subscribe(upstream.request_id); let pause = Forward { forward: false, ..Default::default() }; - match self.write_forward(&mut upstream.stream.writer, &pause).await { + match self.buffer_forward(&mut upstream.stream.writer, &pause) { Ok(()) => { upstream.owed += 1; tracing::debug!(broadcast = %key.0, track = %key.1, "subscription paused"); @@ -1127,15 +1148,17 @@ impl Subscriber { } } - /// Set a held subscription's Forward State with REQUEST_UPDATE. - pub(super) async fn write_forward( + /// Buffer the REQUEST_UPDATE setting a held subscription's Forward State, for the + /// caller to flush. + pub(super) fn buffer_forward( &self, writer: &mut Writer, forward: &Forward, ) -> Result<(), Error> { let request_id = self.control.next_request_id(); - writer - .encode_message(&ietf::SubscribeUpdate { + writer.buffer_message( + ietf::SubscribeUpdate::ID, + &ietf::SubscribeUpdate { max_age: forward.max_age, request_id, subscriber_priority: forward.priority.map(super::super::priority::to_wire), @@ -1144,8 +1167,8 @@ impl Subscriber { authorization_token: None, new_group: forward.new_group, fill: forward.fill, - }) - .await + }, + ) } /// Check the answer to a REQUEST_UPDATE a held subscription still owes. It carries diff --git a/rs/moq-net/src/lite/test_transport.rs b/rs/moq-net/src/lite/test_transport.rs index fa3c4f92aaf..43927be79c2 100644 --- a/rs/moq-net/src/lite/test_transport.rs +++ b/rs/moq-net/src/lite/test_transport.rs @@ -965,6 +965,8 @@ pub struct ScriptedSession { datagram_size: usize, /// Set by [`Self::with_groups`]. groups: bool, + /// Set by [`Self::with_send_window`]. + send_window: Option, } impl ScriptedSession { @@ -1003,9 +1005,17 @@ impl ScriptedSession { unacked_unis: Default::default(), datagram_size: 0, groups: false, + send_window: None, } } + /// Let each opened bidi stream send only its first `bytes`, then park its writes for good, + /// like a peer that never grants more send credit. + pub fn with_send_window(mut self, bytes: usize) -> Self { + self.send_window = Some(bytes); + self + } + /// Open send groups, as native QUIC does, recording them in the log. pub fn with_groups(mut self) -> Self { self.groups = true; @@ -1164,7 +1174,15 @@ impl poll::Session for ScriptedSession { Some(queue) => Arc::new(Mutex::new(queue.lock().unwrap().pop_front().unwrap_or_default())), None => self.script.clone(), }; - let mut send = SinkSend::new(self.log.clone()).opened(options); + let mut send = match self.send_window { + // The gate's producer is gone, so the writes past the prefix park forever. + Some(window) => SinkSend { + prefix: window, + ..SinkSend::gated(self.log.clone(), kio::Producer::new(false).consume()) + }, + None => SinkSend::new(self.log.clone()), + } + .opened(options); send.unacked_fin = self.unacked_fin; Poll::Ready(Ok(( send, From 55e9adf17c1ef8c4b2adb0258fb42fff67217f41 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 27 Oct 2026 23:30:39 -0700 Subject: [PATCH 3/4] fix(net): reset a subscription whose publisher ends behind a stalled update PUBLISH_DONE can end an Established subscription while a REQUEST_UPDATE the peer's send window held back is still buffered. The cleanup FINed the stream regardless, tripping Writer::finish's unflushed assertion and, in release builds, handing the peer a truncated update. Reset instead, as the held watcher does. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/ietf/subscriber.rs | 122 +++++++++++++++++++++++++++++- 1 file changed, 120 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index c2403689bf7..4e0e9ad0e9e 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -3826,8 +3826,14 @@ impl Subscriber { let aborted = self.remove_subscribe(request_id).is_none(); if !idle { - // The publisher already ended the request, so a FIN is all we owe it. - stream.writer.finish().ok(); + // The publisher already ended the request, so a FIN is all we owe it, unless an + // update its send window held back would go out truncated behind it. + match stream.writer.is_flushed() { + true => { + stream.writer.finish().ok(); + } + false => stream.writer.abort(&Error::Cancel), + } // A reader may still miss history the publisher holds, such as a group whose // fetch was in flight as the track ended, so a finished copy keeps serving fills. return (finished && !aborted).then(|| Idle { @@ -7845,6 +7851,118 @@ mod tests { } } + /// Run a draft-22 subscription to Established over a peer whose request stream takes only + /// `window` bytes. Returns the session, the running task, the track, and its first reader. + async fn update_under_window( + window: Option, + ) -> ( + crate::lite::test_transport::ScriptedSession, + moq_clock::sim::JoinHandle<()>, + crate::track::Consumer, + kio::Pending, + ) { + let version = Version::Draft22; + let subscribe_ok = message( + ietf::SubscribeOk::ID, + &ietf::SubscribeOk { + track_alias: 7, + largest: None, + properties: Default::default(), + }, + version, + ); + let mut session = crate::lite::test_transport::ScriptedSession::new(subscribe_ok); + if let Some(window) = window { + session = session.with_send_window(window); + } + let (tasks, task_set) = crate::util::TaskSet::new(); + std::mem::forget(task_set); + let mut subscriber = Subscriber::new( + crate::time::sim(), + crate::driver::Transport::new(session.clone()), + crate::origin::Config::new(crate::Hop::new(1).unwrap()).produce(), + Control::new(false), + None, + peer::PeerSetup::declared(peer::Peer::default()), + crate::Hop::new(1).unwrap(), + None, + version, + tasks, + Default::default(), + ); + let producer = crate::broadcast::Info::default().produce(); + let mut dynamic = producer.requests(); + let track = producer.consume().track("video").unwrap(); + let first = track.subscribe(None); + let request = dynamic.requested_track().await.expect("no track requested"); + let serving = moq_clock::sim::spawn(async move { + let _producer = producer; + subscriber + .run_subscribe(Path::from_static("broadcast"), dynamic, None, request) + .await; + }); + settle().await; + (session, serving, track, first) + } + + /// A REQUEST_UPDATE the peer's send window holds back part-way is still buffered when + /// PUBLISH_DONE ends the subscription, so the stream resets rather than FIN a truncated + /// update. + #[moq_clock::test] + async fn a_stalled_update_resets_when_the_publisher_ends() { + // The SUBSCRIBE alone, measured over an open window. + let subscribe = { + let (session, serving, _track, _first) = update_under_window(None).await; + serving.abort(); + session.log.writes.lock().unwrap().len() + }; + + let (session, serving, track, _first) = update_under_window(Some(subscribe + 1)).await; + let _raised = track + .subscribe(track::Subscription { + priority: 5, + ..Default::default() + }) + .await + .unwrap(); + settle().await; + assert_eq!( + session.log.writes.lock().unwrap().len(), + subscribe + 1, + "the update stalled mid-message" + ); + + session.push(&message( + ietf::PublishDone::ID, + &ietf::PublishDone { + status_code: 0x2, + stream_count: 0, + reason_phrase: "done".into(), + }, + Version::Draft22, + )); + // The script push wakes nothing, so a subscription change re-polls the loop. + let _again = track + .subscribe(track::Subscription { + priority: 7, + ..Default::default() + }) + .await + .unwrap(); + // Past the grace PUBLISH_DONE waits out for its data streams; the copy then lingers. + moq_clock::sleep(crate::ledger::GRACE * 2).await; + assert_eq!(session.log.resets(), vec![crate::ietf::error::CANCELLED]); + assert!( + !session + .log + .trail() + .iter() + .any(|(_, sent)| matches!(sent, crate::lite::test_transport::Sent::Finish)), + "finished a truncated update" + ); + serving.abort(); + } + /// When `conflict` is set, an alias-7 binding for the same full track name is seeded /// first, so the subscription under test loses the race to bind it. async fn cancel_a_subscription_inner(version: Version, conflict: bool) -> crate::lite::test_transport::Log { From d0145f2d6d756a504957bcfbb452675fb38018f0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 01:12:18 -0700 Subject: [PATCH 4/4] fix(net): hand off a held subscription only once its writes flush Under withheld send credit, each resume a reader abandoned appended another REQUEST_UPDATE pair to the held stream's buffer, growing it with subscriber churn. The held watcher now hands the subscription over only once its writer has flushed, bounding the buffer to one resume and its pause, and the resumer stops waiting for that handoff once nobody wants the track. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/ietf/subscriber.rs | 59 ++++++++++++++------ rs/moq-net/src/ietf/subscriber/tracks.rs | 69 +++++++++++++++++++----- 2 files changed, 101 insertions(+), 27 deletions(-) diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 4e0e9ad0e9e..72550e30149 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -2940,7 +2940,7 @@ impl Subscriber { &mut self, broadcast_path: &Path, track_name: &str, - target: Target, + mut target: Target, group_fetches: &mut TaskSet, gate: &mut crate::auth::Gate, ) -> Option { @@ -2979,20 +2979,49 @@ impl Subscriber { let adopted = match &self.want(&key) { None => None, Some(handoff) => { - let adopted = kio::wait(|waiter| { - if gate.poll_denied(waiter).is_ready() { - return Poll::Ready(Err(Error::Unauthorized)); - } - validity.check(waiter)?; - Self::poll_handoff(handoff, waiter).map(Ok) - }) - .await; - match adopted { - Ok(adopted) => adopted, - Err(err) => { - self.unwant(key, handoff); - target.fail(err); - return None; + // The handoff waits for the held stream to flush, which a peer withholding + // credit can stall, so nobody wanting the track any more ends the wait too. + enum Abandon { + Failed(Error), + Unused, + } + loop { + let adopt = kio::wait(|waiter| { + if gate.poll_denied(waiter).is_ready() { + return Poll::Ready(Err(Abandon::Failed(Error::Unauthorized))); + } + if let Err(err) = validity.check(waiter) { + return Poll::Ready(Err(Abandon::Failed(err))); + } + if let Poll::Ready(adopted) = Self::poll_handoff(handoff, waiter) { + return Poll::Ready(Ok(adopted)); + } + let unwanted = match &mut target { + Target::Request(request) => crate::util::poll_unwanted(&mut request.demand(), waiter), + Target::Resume(idle) => crate::util::poll_unwanted(&mut idle.demand, waiter), + }; + unwanted.map(|_| Err(Abandon::Unused)) + }) + .await; + match adopt { + Ok(adopted) => break adopted, + Err(Abandon::Failed(err)) => { + self.unwant(key, handoff); + target.fail(err); + return None; + } + // Nobody wants the track: a resumed copy lingers again, a request is refused. + Err(Abandon::Unused) => match target { + Target::Resume(idle) => { + self.unwant(key, handoff); + return Some(idle); + } + Target::Request(ref request) if request.reject_unused(Error::Cancel) => { + self.unwant(key, handoff); + return None; + } + Target::Request(_) => continue, + }, } } } diff --git a/rs/moq-net/src/ietf/subscriber/tracks.rs b/rs/moq-net/src/ietf/subscriber/tracks.rs index a5c61885e05..5c53f18fdad 100644 --- a/rs/moq-net/src/ietf/subscriber/tracks.rs +++ b/rs/moq-net/src/ietf/subscriber/tracks.rs @@ -916,13 +916,18 @@ impl Subscriber { if session.poll_closed(&mut cx).is_ready() { return Poll::Ready(Step::Closed); } - if let Poll::Ready(Err(err)) = upstream.stream.writer.poll_flush(&mut cx) { - return Poll::Ready(Step::Message(Err(err))); - } - if let Poll::Ready(handoff) = handoff.poll(waiter, |handoff| match handoff.gone || handoff.wanted { - true => Poll::Ready(()), - false => Poll::Pending, - }) { + // A held subscription is handed over only once flushed, so resumes and pauses + // can't pile up behind a peer withholding credit. + let flushed = match upstream.stream.writer.poll_flush(&mut cx) { + Poll::Ready(Err(err)) => return Poll::Ready(Step::Message(Err(err))), + Poll::Ready(Ok(())) => true, + Poll::Pending => false, + }; + if let Poll::Ready(handoff) = + handoff.poll(waiter, |handoff| match handoff.gone || (handoff.wanted && flushed) { + true => Poll::Ready(()), + false => Poll::Pending, + }) { return Poll::Ready(match handoff.gone { true => Step::Gone, false => Step::Wanted, @@ -1279,7 +1284,11 @@ mod tests { } fn harness(version: Version) -> Harness { - let (session, peer) = memory::pair(memory::Config::default()); + harness_with(version, memory::Config::default()) + } + + fn harness_with(version: Version, config: memory::Config) -> Harness { + let (session, peer) = memory::pair(config); let (tasks, mut task_set) = TaskSet::new(); let driver = moq_clock::sim::spawn(async move { kio::wait(|waiter| task_set.poll(waiter)).await }); let subscriber = Subscriber::new( @@ -1320,9 +1329,10 @@ mod tests { .expect("the subscriber sent nothing") } + /// One message in one write, so a peer holding one message per stream still delivers it. async fn write<'m, M: ietf::Message<'m> + std::fmt::Debug>(stream: &mut PeerStream, id: u64, msg: &M) { - stream.writer.varint(id).await.unwrap(); - stream.writer.encode(msg).await.unwrap(); + stream.writer.buffer_message(id, msg).unwrap(); + std::future::poll_fn(|cx| stream.writer.poll_flush(cx)).await.unwrap(); settle().await; } @@ -1440,8 +1450,7 @@ mod tests { let mut stream = Stream::open(&mut crate::driver::Transport::new(self.peer.clone()), self.version) .await .unwrap(); - stream.writer.varint(ietf::Publish::ID).await.unwrap(); - stream.writer.encode(&publish).await.unwrap(); + stream.writer.encode_message(&publish).await.unwrap(); let mut ours = Stream::accept(&mut self.subscriber.session.clone(), self.version) .await @@ -1460,6 +1469,42 @@ mod tests { } } + /// Readers coming and going while the peer withholds credit on a held track never pile + /// updates up behind it: the held subscription is handed over only once its pause is out, + /// and a reader leaving meanwhile stops waiting for it. + #[moq_clock::test] + async fn churn_under_withheld_credit_buffers_one_resume_and_pause() { + for version in [Version::Draft18, Version::Draft22] { + // One message in flight per stream, so a peer that stops reading withholds credit. + let config = memory::Config { + stream_buffer: 0, + ..Default::default() + }; + let mut h = harness_with(version, config); + let _listing = h.establish().await; + let mut video = h.publish("video", 1).await; + + for _ in 0..3 { + let (track, reader, serving) = h.read_track("video").await; + settle().await; + drop(reader); + drop(track); + moq_clock::timeout(TIMEOUT, serving) + .await + .unwrap_or_else(|_| panic!("{version}: the abandoned resume stayed alive")) + .unwrap(); + } + + let mut updates = Vec::new(); + while let Ok(Ok(id)) = moq_clock::timeout(TIMEOUT, video.reader.varint()).await { + let body: ietf::Body = video.reader.decode().await.unwrap(); + assert_eq!(id, ietf::SubscribeUpdate::ID, "{version}"); + updates.push(body.message::(version).unwrap().forward); + } + assert_eq!(updates, [Some(true), Some(false)], "{version}"); + } + } + /// A paused PUBLISH of `name` in `namespace`, aliased by its request ID. fn offered<'a>(namespace: &'a str, name: &'a str, request_id: u64) -> ietf::Publish<'a> { ietf::Publish {