diff --git a/quest/m1/README.md b/quest/m1/README.md index dc4f950225f..7f36b6ebf3d 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -128,7 +128,6 @@ removed). - [Release audit findings](/quest/m1/release-audit/README.md) - resolve newly identified resource, parser, playback, and interoperability risks before the cut - [Pre-cut audit findings](/quest/m1/cut-audit/README.md) - fix the crashes, hostile-input costs, and correctness bugs the 2026-10-24 audit of main found - [Security audit findings](/quest/m1/security-audit/README.md) - fix the verified relay, importer, moq-c, browser, and CI findings of the 2026-10-27 external security audit -- [Cancelled moq-transport requests stop](/quest/m1/ietf-request-cancel.md) - a cancelled request releases its state even when the peer withholds credit - [Withdrawn advertisements stop](/quest/m1/ietf-withdraw-credit.md) - a withdrawn moq-transport namespace releases its state even when the peer withholds credit - [Cluster routing](/quest/m1/cluster-routing/README.md) - prove shared routing contracts, implement per-origin routes, and connect outbound local clusters - [Routing guide](/quest/m1/routing-guide.md) - concepts and practical outbound federation setup diff --git a/quest/m1/ietf-request-cancel.md b/quest/m1/ietf-request-cancel.md deleted file mode 100644 index 07d95043c8b..00000000000 --- a/quest/m1/ietf-request-cancel.md +++ /dev/null @@ -1,29 +0,0 @@ -# [M] A cancelled moq-transport request stops writing - -## Goal - -Cancelling a moq-transport SUBSCRIBE, FETCH, TRACK_STATUS, or namespace -request releases its state at once, even when the peer's send window holds -back the request's first message. - -## Plan - -Every request write in `rs/moq-net/src/ietf/subscriber.rs` awaits its first -`encode_message` without watching for cancellation, so a peer that withholds -flow-control credit pins a cancelled request. -[moq-dev/moq#6409](https://github.com/moq-dev/moq/pull/6409) already refuses -a request at once when stream credit is exhausted, but an admitted header -still waits for send capacity, and -[moq-dev/moq#6530](https://github.com/moq-dev/moq/pull/6530) landed the Rust -lazy discovery that rewrote these writes. What remains is the awaited first -encode under send-window pressure: watch for cancellation there and reset the -partially written request stream. - -Drafts 14 and 15 shared one ordered control stream, so a partial message -could not be abandoned; every remaining draft gives each request its own -stream, so only the reset path is needed. - -Test with a peer that never grants send credit, on drafts 18 and 22. - -Public API: none. Wire: none. - diff --git a/quest/m2/ietf-request-lease.md b/quest/m2/ietf-request-lease.md index 09505b8926a..4e00869b3c7 100644 --- a/quest/m2/ietf-request-lease.md +++ b/quest/m2/ietf-request-lease.md @@ -26,8 +26,8 @@ one terminal at one await: [#6237](https://github.com/moq-dev/moq/pull/6237), [#6409](https://github.com/moq-dev/moq/pull/6409), [#6433](https://github.com/moq-dev/moq/pull/6433), -[#6643](https://github.com/moq-dev/moq/pull/6643), plus the open -[cancel quest](/quest/m1/ietf-request-cancel.md) and the +[#6643](https://github.com/moq-dev/moq/pull/6643), plus the +[cancel fix](https://github.com/moq-dev/moq/pull/7016) and the [withdraw fix](https://github.com/moq-dev/moq/pull/6964). Each request is a long `async fn` whose every `kio::wait` closure re-lists the right subset of about 46 terminal checks; the two files hold 227 `pin!` sites @@ -58,4 +58,3 @@ Public API: none. Wire: none. ## Required - [moq-transport 18 and 22 only](/quest/m1/ietf-drop/README.md) - deletes about 2,950 production lines and the adapter first -- [Cancelled moq-transport requests stop](/quest/m1/ietf-request-cancel.md) - lands the last point fix this replaces diff --git a/rs/moq-net/src/coding/writer.rs b/rs/moq-net/src/coding/writer.rs index ef4e06e53cc..ca94aef652c 100644 --- a/rs/moq-net/src/coding/writer.rs +++ b/rs/moq-net/src/coding/writer.rs @@ -152,6 +152,11 @@ impl Writer { kio::wait(|waiter| self.poll_write_all(waiter, buf)).await } + /// Whether every buffered byte has hit the stream, so a FIN would not cut a message short. + pub fn is_flushed(&self) -> bool { + self.buffer.is_empty() + } + /// Mark the stream as finished. /// /// Only valid once the buffer has flushed; the callers that finish @@ -184,7 +189,15 @@ impl Writer { /// discards it, so the [`Drop`] fallback below can throw away bytes the peer never read. /// Consuming the writer is what removes that fallback, and waiting for the acknowledgement /// is what makes the bytes safe. + /// + /// A message still partly buffered resets the stream with [`Error::Cancel`] instead: a FIN + /// would hand the peer a truncated message, and flushing the rest could wait forever on + /// credit the peer withholds. pub async fn close(mut self) -> Result<(), Error> { + if !self.is_flushed() { + self.abort(&Error::Cancel); + return Err(Error::Cancel); + } let Some(mut stream) = self.stream.take() else { return Ok(()); }; @@ -365,6 +378,29 @@ mod tests { assert_eq!(log.priorities(), (0u8..=255).collect::>()); } + /// A message the stream has not taken whole cannot be finished, so closing resets rather + /// than hand the peer a truncated message. + #[moq_clock::test] + async fn closing_an_unflushed_message_resets() { + let log = Log::default(); + let budget = kio::Producer::new(0); + let mut writer = Writer::new( + SinkSend::stepped(log.clone(), budget.consume()), + crate::lite::Version::Lite05, + ); + writer.buffer_varint(5).unwrap(); + assert!(writer.poll_flush(&kio::Waiter::noop()).is_pending()); + assert!(!writer.is_flushed()); + + assert!(matches!(writer.close().await, Err(Error::Cancel))); + assert_eq!(log.resets(), vec![StreamError::Cancel.to_code()]); + assert!( + !log.trail() + .iter() + .any(|(_, sent)| matches!(sent, crate::lite::test_transport::Sent::Finish)) + ); + } + /// Writes some bytes, then errors: the shape of a partial encode. #[derive(Debug)] struct Poison; diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index ed6fa325a39..5c4c0782044 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -916,8 +916,8 @@ struct Held { epoch: Option, } -/// A joining request may be abandoned before it takes an ID. Once committed, its header -/// must still reach the peer, which otherwise remembers the ID it never saw. +/// A joining request may be abandoned before it takes an ID. Once committed, the session +/// owns ending it on the wire. struct JoiningTask { work: Option>>, state: kio::Shared, @@ -950,8 +950,9 @@ impl Drop for JoiningTask { state.committed }; if committed && let Some(work) = self.work.take() { - // The session owns the remaining send, including after the subscription - // ends. It closes the request immediately after delivering its header. + // The session owns cancelling the request, including after the subscription + // ends: closing it once its header is out, or resetting it while the peer's + // send window still holds the header back. self.tasks.push(async move { let _ = work.await; }); @@ -1422,9 +1423,11 @@ impl Subscriber { namespace: prefix.clone(), hidden, }; - stream.writer.encode_message(&msg).await?; + // Flushed while waiting for the answer, so `live` closing still ends a request the + // peer's send window holds back. + stream.writer.buffer_message(ietf::SubscribeNamespace::ID, &msg)?; - tracing::debug!(%prefix, "subscribe_namespace sent"); + tracing::debug!(%prefix, "subscribe_namespace started"); // Read response let Some(framed) = self.next_namespace_message(request, live).await? else { @@ -1480,6 +1483,7 @@ impl Subscriber { if live.poll_closed(waiter).is_ready() { return Poll::Ready(Ok(None)); } + ready!(stream.writer.poll_flush(waiter))?; messages .poll_next(&mut stream.reader, waiter) .map(|next| next.map(Some)) @@ -1505,6 +1509,13 @@ impl Subscriber { if request_id.is_none() { return; } + // The request never reached the peer whole, so it holds nothing to wait on. + if !writer.is_flushed() { + tracing::debug!(%prefix, "subscribe_namespace abandoned before it was sent"); + reader.abort(&Error::Cancel); + writer.abort(&Error::Cancel); + return; + } tracing::debug!(%prefix, "unsubscribe_namespace"); match super::request_stream::fin_cancels(self.version) { @@ -2922,7 +2933,7 @@ impl Subscriber { &mut self, broadcast_path: &Path, track_name: &str, - target: Target, + mut target: Target, group_fetches: &mut TaskSet, gate: &mut crate::auth::Gate, ) -> Option { @@ -2961,20 +2972,49 @@ impl Subscriber { let adopted = match &self.want(&key) { None => None, Some(handoff) => { - let adopted = kio::wait(|waiter| { - if gate.poll_denied(waiter).is_ready() { - return Poll::Ready(Err(Error::Unauthorized)); - } - validity.check(waiter)?; - Self::poll_handoff(handoff, waiter).map(Ok) - }) - .await; - match adopted { - Ok(adopted) => adopted, - Err(err) => { - self.unwant(key, handoff); - target.fail(err); - return None; + // The handoff waits for the held stream to flush, which a peer withholding + // credit can stall, so nobody wanting the track any more ends the wait too. + enum Abandon { + Failed(Error), + Unused, + } + loop { + let adopt = kio::wait(|waiter| { + if gate.poll_denied(waiter).is_ready() { + return Poll::Ready(Err(Abandon::Failed(Error::Unauthorized))); + } + if let Err(err) = validity.check(waiter) { + return Poll::Ready(Err(Abandon::Failed(err))); + } + if let Poll::Ready(adopted) = Self::poll_handoff(handoff, waiter) { + return Poll::Ready(Ok(adopted)); + } + let unwanted = match &mut target { + Target::Request(request) => crate::util::poll_unwanted(&mut request.demand(), waiter), + Target::Resume(idle) => crate::util::poll_unwanted(&mut idle.demand, waiter), + }; + unwanted.map(|_| Err(Abandon::Unused)) + }) + .await; + match adopt { + Ok(adopted) => break adopted, + Err(Abandon::Failed(err)) => { + self.unwant(key, handoff); + target.fail(err); + return None; + } + // Nobody wants the track: a resumed copy lingers again, a request is refused. + Err(Abandon::Unused) => match target { + Target::Resume(idle) => { + self.unwant(key, handoff); + return Some(idle); + } + Target::Request(ref request) if request.reject_unused(Error::Cancel) => { + self.unwant(key, handoff); + return None; + } + Target::Request(_) => continue, + }, } } } @@ -3069,6 +3109,8 @@ impl Subscriber { let order = aggregate.map(|s| s.order).unwrap_or_default(); // Without an Established subscription, a relay includes the request it holds. let new_group = target.new_group(); + // Buffered only: the wait for the answer flushes it, so the local terminals still end a + // request the peer's send window holds back. let written = match &held { // Resume the held subscription: its group order is the one it was made with, // since no update can change it. Its groups wait on the answer from here. @@ -3083,37 +3125,32 @@ impl Subscriber { // Only a track declaring DYNAMIC_GROUPS takes one on an update. new_group: new_group.filter(|_| held.properties.dynamic_groups), }; - self.write_forward(&mut stream.writer, &forward).await - } - None => { - async { - stream - .writer - .encode_message(&ietf::Subscribe { - max_age, - new_group, - request_id, - track_namespace: broadcast_path.clone(), - track_name: track_name.into(), - subscriber_priority: super::priority::to_wire(priority), - group_order: order.into(), - filter: join.filter, - fill: join.fill, - // A resumed copy already knows the track, from the answer that accepted it. - properties_wanted: matches!(target, Target::Request(_)), - forward: true, - rendezvous: false, - range_filters: false, - authorization_token: None, - epoch: self.epoch.clone(), - }) - .await - } - .await + self.buffer_forward(&mut stream.writer, &forward) } + None => stream.writer.buffer_message( + ietf::Subscribe::ID, + &ietf::Subscribe { + max_age, + new_group, + request_id, + track_namespace: broadcast_path.clone(), + track_name: track_name.into(), + subscriber_priority: super::priority::to_wire(priority), + group_order: order.into(), + filter: join.filter, + fill: join.fill, + // A resumed copy already knows the track, from the answer that accepted it. + properties_wanted: matches!(target, Target::Request(_)), + forward: true, + rendezvous: false, + range_filters: false, + authorization_token: None, + epoch: self.epoch.clone(), + }, + ), }; if let Err(err) = written { - tracing::debug!(%err, "failed to write subscribe"); + tracing::debug!(%err, "failed to encode subscribe"); self.remove_subscribe(request_id); self.entry_ended(&key, request_id, Err(err.clone())); target.fail(err); @@ -3154,7 +3191,10 @@ impl Subscriber { // A local terminal ended the wait, so the peer still holds the request to cancel. let mut unanswered = false; let setup = { - let mut response = std::pin::pin!(self.read_answer(&mut stream, &mut messages, held.as_mut())); + let mut response = std::pin::pin!(async { + kio::wait(|waiter| stream.writer.poll_flush(waiter)).await?; + self.read_answer(&mut stream, &mut messages, held.as_mut()).await + }); loop { let setup = kio::wait(|waiter| { let response = waiter.poll_future(response.as_mut()); @@ -3246,7 +3286,7 @@ impl Subscriber { owed: held.owed + 1, copy: resumed.as_ref().map(|idle| idle.track.clone()), }; - self.pause(key, upstream).await; + self.pause(key, upstream); return resumed; } // The publisher may already be serving before it answers. A session abort @@ -3437,6 +3477,8 @@ impl Subscriber { Fill(track::Fill), Update(Update), Invalid(Error), + /// An update failed to go out. + Failed(Error), } // The NEW_GROUP_REQUEST last sent, the SUBSCRIBE's included. It stays outstanding @@ -3489,6 +3531,11 @@ impl Subscriber { if gate.poll_denied(waiter).is_ready() { return Poll::Ready(End::Revoked); } + // An update the peer's send window holds back goes out from here, so it never + // holds back the ends below. + if let Poll::Ready(Err(err)) = stream.writer.poll_flush(waiter) { + return Poll::Ready(End::Failed(err)); + } if !fetch_done && let Some(fut) = fetching.as_mut() && let Poll::Ready(res) = fut.poll(waiter) @@ -3652,7 +3699,7 @@ impl Subscriber { sent_bounds = wanted_bounds.clone().expect("validated update bounds"); } let (asked, new_group, asked_age) = (update.priority, update.new_group, update.max_age); - if let Err(err) = self.write_update(&mut stream.writer, request_id, update).await { + if let Err(err) = self.buffer_update(&mut stream.writer, request_id, update) { tracing::debug!(%err, "failed to update the subscription"); ended = Some(Err(err.clone())); let _ = track.clone().abort(err); @@ -3672,6 +3719,12 @@ impl Subscriber { } unanswered.fetch_add(1, Ordering::Relaxed); } + End::Failed(err) => { + tracing::debug!(%err, "failed to update the subscription"); + ended = Some(Err(err.clone())); + let _ = track.clone().abort(err); + break false; + } End::Revoked => { tracing::info!(broadcast = %crate::path::Rooted(self.origin.root(), broadcast_path), track = %track_name, "subscription no longer authorized"); let _ = track.clone().abort(Error::Unauthorized); @@ -3764,7 +3817,7 @@ impl Subscriber { } }; - // Release an unallocated join now, or let the session finish its committed header. + // Release an unallocated join now, or let the session cancel its committed request. drop(fetching); if let Some(end) = ended { @@ -3781,7 +3834,7 @@ impl Subscriber { owed: unanswered.load(Ordering::Relaxed), copy: Some(track.clone()), }; - self.pause(key, upstream).await; + self.pause(key, upstream); return Some(Idle { demand: track.demand(), track, @@ -3795,8 +3848,14 @@ impl Subscriber { let aborted = self.remove_subscribe(request_id).is_none(); if !idle { - // The publisher already ended the request, so a FIN is all we owe it. - stream.writer.finish().ok(); + // The publisher already ended the request, so a FIN is all we owe it, unless an + // update its send window held back would go out truncated behind it. + match stream.writer.is_flushed() { + true => { + stream.writer.finish().ok(); + } + false => stream.writer.abort(&Error::Cancel), + } // A reader may still miss history the publisher holds, such as a group whose // fetch was in flight as the track ended, so a finished copy keeps serving fills. return (finished && !aborted).then(|| Idle { @@ -3983,9 +4042,9 @@ impl Subscriber { } } - /// Update an Established subscription: REQUEST_UPDATE carrying only what changed, under a - /// Request ID of its own. - async fn write_update( + /// Buffer the update to an Established subscription, for its wait to flush: + /// REQUEST_UPDATE carrying only what changed, under a Request ID of its own. + fn buffer_update( &self, writer: &mut Writer, subscription: RequestId, @@ -4006,8 +4065,9 @@ impl Subscriber { } state.fetches.insert(request_id, FillRequest::Update(subscription)); } - writer - .encode_message(&ietf::SubscribeUpdate { + writer.buffer_message( + ietf::SubscribeUpdate::ID, + &ietf::SubscribeUpdate { max_age: update.max_age, request_id, subscriber_priority: update.priority.map(super::priority::to_wire), @@ -4016,8 +4076,8 @@ impl Subscriber { authorization_token: None, new_group: update.new_group, fill: update.fill, - }) - .await + }, + ) } /// Tell the publisher to stop serving a subscription we are walking away from. @@ -4040,7 +4100,7 @@ impl Subscriber { // Finishing alone would leave the writer's Drop free to RESET_STREAM, and a stream // that has sent its FIN is still retransmitting: the reset would discard what we // already wrote. Closing consumes the writer, removing that fallback, and waits for - // the acknowledgement. + // the acknowledgement. A request the peer's send window held back resets instead. if let Err(err) = writer.close().await { tracing::debug!(%err, "failed to close the subscribe stream"); } @@ -4087,33 +4147,27 @@ impl Subscriber { state.fetches.insert(fetch_id, FillRequest::Joining(subscribe_id)); } } - if let Err(err) = async { - stream - .writer - .encode_message(&ietf::Fetch { - request_id: fetch_id, - subscriber_priority: super::priority::to_wire( - track.subscription().map(|s| s.priority).unwrap_or(0), - ), - group_order: GroupOrder::Ascending, - fetch_type: FetchType::RelativeJoining { - subscriber_request_id: subscribe_id, - group_offset: 0, - }, - range_filters: false, - fill_timeout: false, - properties_wanted: true, - // Served by the subscription it joins, which named the instance. - epoch: None, - }) - .await?; - Ok::<(), Error>(()) - } - .await - { - tracing::debug!(%err, "failed to write joining FETCH"); - return Err(err); - } + // Flushed while waiting for the answer, so a cancellation still ends a request the + // peer's send window holds back. + stream.writer.buffer_message( + ietf::Fetch::ID, + &ietf::Fetch { + request_id: fetch_id, + subscriber_priority: super::priority::to_wire( + track.subscription().map(|s| s.priority).unwrap_or(0), + ), + group_order: GroupOrder::Ascending, + fetch_type: FetchType::RelativeJoining { + subscriber_request_id: subscribe_id, + group_offset: 0, + }, + range_filters: false, + fill_timeout: false, + properties_wanted: true, + // Served by the subscription it joins, which named the instance. + epoch: None, + }, + )?; this.finish_joining_fetch(stream, progress).await } .await; @@ -4160,7 +4214,10 @@ impl Subscriber { let mut messages = super::request_stream::Messages::new(self.session.clone(), self.control.client()); let res = async { let answer = { - let mut response = std::pin::pin!(self.read_fetch_response(&mut stream, &mut messages)); + let mut response = std::pin::pin!(async { + kio::wait(|waiter| stream.writer.poll_flush(waiter)).await?; + self.read_fetch_response(&mut stream, &mut messages).await + }); kio::wait(|waiter| { if cancelled(waiter).is_ready() { return Poll::Ready(None); @@ -4256,19 +4313,18 @@ impl Subscriber { stream = stream.with_stats(self.origin.stats()); let request_id = self.control.next_request_id(); - let written = async { - stream - .writer - .encode_message(&ietf::TrackStatus { - request_id, - track_namespace: broadcast_path.clone(), - track_name: track_name.into(), - properties_wanted: true, - epoch: self.epoch.clone(), - }) - .await - } - .await; + // Flushed while waiting for the answer, so the terminals below still end a request + // the peer's send window holds back. + let written = stream.writer.buffer_message( + ietf::TrackStatus::ID, + &ietf::TrackStatus { + request_id, + track_namespace: broadcast_path.clone(), + track_name: track_name.into(), + properties_wanted: true, + epoch: self.epoch.clone(), + }, + ); if let Err(err) = written { request.reject(err); return None; @@ -4279,7 +4335,10 @@ impl Subscriber { let mut demand = request.demand(); let closed = self.session.clone(); let response = { - let mut response = std::pin::pin!(self.read_track_status_response(&mut stream)); + let mut response = std::pin::pin!(async { + kio::wait(|waiter| stream.writer.poll_flush(waiter)).await?; + self.read_track_status_response(&mut stream).await + }); loop { let res = kio::wait(|waiter| { let response = waiter.poll_future(response.as_mut()); @@ -5381,30 +5440,31 @@ impl Subscriber { end: ietf::Location { group: last, object: 0 }, }, }; - let res = async { - stream - .writer - .encode_message(&ietf::Fetch { - request_id: fetch_id, - subscriber_priority: super::priority::to_wire(fill.priority()), - group_order: GroupOrder::Ascending, - fetch_type, - range_filters: false, - fill_timeout: false, - // The copy learned the track before any fetch, from SUBSCRIBE_OK or - // TRACK_STATUS_OK, so a FETCH_OK repeating it is waste. - properties_wanted: false, - epoch: self.epoch.clone(), - }) - .await?; - Ok::<_, Error>(()) - } - .await; + // Flushed while waiting for the answer, so the terminals below still end a request the + // peer's send window holds back. + let res = stream.writer.buffer_message( + ietf::Fetch::ID, + &ietf::Fetch { + request_id: fetch_id, + subscriber_priority: super::priority::to_wire(fill.priority()), + group_order: GroupOrder::Ascending, + fetch_type, + range_filters: false, + fill_timeout: false, + // The copy learned the track before any fetch, from SUBSCRIBE_OK or + // TRACK_STATUS_OK, so a FETCH_OK repeating it is waste. + properties_wanted: false, + epoch: self.epoch.clone(), + }, + ); let mut messages = super::request_stream::Messages::new(self.session.clone(), self.control.client()); let res = match res { Err(err) => SetupResponse::Response(Err(err)), Ok(()) => { - let mut response = std::pin::pin!(self.read_fetch_response(&mut stream, &mut messages)); + let mut response = std::pin::pin!(async { + kio::wait(|waiter| stream.writer.poll_flush(waiter)).await?; + self.read_fetch_response(&mut stream, &mut messages).await + }); kio::wait(|waiter| { let response = waiter.poll_future(response.as_mut()); if let Poll::Ready(Err(err)) = response { @@ -6311,6 +6371,58 @@ mod tests { assert_eq!(occurrences(&log, b"rootns"), 0, "asked the peer for our local root"); } + /// A SUBSCRIBE_NAMESPACE the peer's send window holds back part-way still ends as soon as + /// `live` closes, resetting the stream: a FIN would hand the peer a truncated request, + /// and there is no answer to wait for. + #[moq_clock::test] + async fn a_namespace_request_stalled_on_credit_ends_when_cancelled() { + for version in [Version::Draft18, Version::Draft22] { + let session = crate::lite::test_transport::ScriptedSession::new(Vec::new()).with_send_window(1); + let log = session.log.clone(); + let (tasks, _task_set) = crate::util::TaskSet::new(); + let mut subscriber = Subscriber::new( + crate::time::sim(), + crate::driver::Transport::new(session.clone()), + crate::origin::Config::new(crate::Hop::new(1).unwrap()).produce(), + Control::new(false), + None, + peer::PeerSetup::declared(peer::Peer::default()), + crate::Hop::new(1).unwrap(), + None, + version, + tasks, + Default::default(), + ); + + let (cancel, live) = live(); + let stream = Stream::open(&mut crate::driver::Transport::new(session.clone()), version) + .await + .unwrap(); + let mut run = std::pin::pin!(subscriber.run_subscribe_namespace( + stream, + Path::from_static("room"), + crate::model::Snapshot::landed(), + live + )); + assert!(futures::poll!(run.as_mut()).is_pending()); + assert_eq!(log.writes.lock().unwrap().len(), 1, "{version}: stalled mid-message"); + + drop(cancel); + let Poll::Ready(res) = futures::poll!(run.as_mut()) else { + panic!("{version}: the cancelled request stayed alive"); + }; + assert!(matches!(res, Ok(namespaces::Ended::Local)), "{version}: {res:?}"); + assert_eq!(log.resets(), vec![crate::ietf::error::CANCELLED], "{version}"); + assert_eq!(log.stops(), vec![crate::ietf::error::CANCELLED], "{version}"); + assert!( + !log.trail() + .iter() + .any(|(_, sent)| matches!(sent, crate::lite::test_transport::Sent::Finish)), + "{version}: finished a truncated request" + ); + } + } + /// The peer's REQUEST_OK followed by one NAMESPACE, framed exactly as /// `run_subscribe_namespace` reads it -- built with the crate's own writer so the /// framing can't drift from the encoder under test. @@ -7490,9 +7602,13 @@ mod tests { #[derive(Clone, Copy, Debug)] enum SetupWait { SubscribeOpen, + /// The request's first message, stalled part-way on send credit. + SubscribeWrite, StatusOpen, + StatusWrite, StatusResponse, FetchOpen, + FetchWrite, FetchResponse, } @@ -7511,9 +7627,15 @@ mod tests { stage, SetupWait::SubscribeOpen | SetupWait::StatusOpen | SetupWait::FetchOpen ); + let writing = matches!( + stage, + SetupWait::SubscribeWrite | SetupWait::StatusWrite | SetupWait::FetchWrite + ); let open = kio::Producer::new(false); let session = if opening { crate::lite::test_transport::ScriptedSession::gated_open(Vec::new(), open.consume()) + } else if writing { + crate::lite::test_transport::ScriptedSession::new(Vec::new()).with_send_window(1) } else { crate::lite::test_transport::ScriptedSession::new(Vec::new()).with_unacked_fin() }; @@ -7555,7 +7677,7 @@ mod tests { let mut group = None; let mut accepted = None; let work: std::pin::Pin>> = match stage { - SetupWait::SubscribeOpen => Box::pin(async { + SetupWait::SubscribeOpen | SetupWait::SubscribeWrite => Box::pin(async { assert!( subscriber .subscribe_once( @@ -7569,7 +7691,7 @@ mod tests { .is_none() ); }), - SetupWait::StatusOpen | SetupWait::StatusResponse => Box::pin(async { + SetupWait::StatusOpen | SetupWait::StatusWrite | SetupWait::StatusResponse => Box::pin(async { assert!( subscriber .track_status(&Path::from_static("broadcast"), "video", request, &mut gate) @@ -7598,6 +7720,13 @@ mod tests { if opening { assert!(log.writes.lock().unwrap().is_empty()); } + if writing { + assert_eq!( + log.writes.lock().unwrap().len(), + 1, + "{stage:?}/{version}: stalled mid-message" + ); + } let expected = match cause { Invalidated::Retired => { drop(route); @@ -7640,6 +7769,25 @@ mod tests { if opening { assert!(done, "{stage:?}/{version}/{cause:?}: cancelled wait stayed alive"); assert!(log.writes.lock().unwrap().is_empty()); + } else if writing { + assert!(done, "{stage:?}/{version}/{cause:?}: cancelled write stayed alive"); + // A FIN would hand the peer a truncated message. + assert_eq!( + log.resets(), + vec![crate::ietf::error::CANCELLED], + "{stage:?}/{version}/{cause:?}" + ); + assert!( + !log.trail() + .iter() + .any(|(_, sent)| matches!(sent, crate::lite::test_transport::Sent::Finish)), + "{stage:?}/{version}/{cause:?}: finished a truncated request" + ); + assert_eq!( + log.stops(), + vec![crate::ietf::error::CANCELLED], + "{stage:?}/{version}/{cause:?}" + ); } else { assert_eq!( log.stops(), @@ -7668,9 +7816,12 @@ mod tests { } setup_wait_test!(invalid_subscribe_open_is_refused, SubscribeOpen, [Draft22]); + setup_wait_test!(invalid_subscribe_write_is_refused, SubscribeWrite, [Draft18, Draft22]); setup_wait_test!(invalid_status_open_is_refused, StatusOpen, [Draft22]); + setup_wait_test!(invalid_status_write_is_refused, StatusWrite, [Draft18, Draft22]); setup_wait_test!(invalid_status_response_is_refused, StatusResponse, [Draft22]); setup_wait_test!(invalid_fetch_open_is_refused, FetchOpen, [Draft22]); + setup_wait_test!(invalid_fetch_write_is_refused, FetchWrite, [Draft18, Draft22]); setup_wait_test!(invalid_fetch_response_is_refused, FetchResponse, [Draft22]); /// The control messages that actually reached the wire, by type id. @@ -7719,6 +7870,118 @@ mod tests { } } + /// Run a draft-22 subscription to Established over a peer whose request stream takes only + /// `window` bytes. Returns the session, the running task, the track, and its first reader. + async fn update_under_window( + window: Option, + ) -> ( + crate::lite::test_transport::ScriptedSession, + moq_clock::sim::JoinHandle<()>, + crate::track::Consumer, + kio::Pending, + ) { + let version = Version::Draft22; + let subscribe_ok = message( + ietf::SubscribeOk::ID, + &ietf::SubscribeOk { + track_alias: 7, + largest: None, + properties: Default::default(), + }, + version, + ); + let mut session = crate::lite::test_transport::ScriptedSession::new(subscribe_ok); + if let Some(window) = window { + session = session.with_send_window(window); + } + let (tasks, task_set) = crate::util::TaskSet::new(); + std::mem::forget(task_set); + let mut subscriber = Subscriber::new( + crate::time::sim(), + crate::driver::Transport::new(session.clone()), + crate::origin::Config::new(crate::Hop::new(1).unwrap()).produce(), + Control::new(false), + None, + peer::PeerSetup::declared(peer::Peer::default()), + crate::Hop::new(1).unwrap(), + None, + version, + tasks, + Default::default(), + ); + let producer = crate::broadcast::Info::default().produce(); + let mut dynamic = producer.requests(); + let track = producer.consume().track("video").unwrap(); + let first = track.subscribe(None); + let request = dynamic.requested_track().await.expect("no track requested"); + let serving = moq_clock::sim::spawn(async move { + let _producer = producer; + subscriber + .run_subscribe(Path::from_static("broadcast"), dynamic, None, request) + .await; + }); + settle().await; + (session, serving, track, first) + } + + /// A REQUEST_UPDATE the peer's send window holds back part-way is still buffered when + /// PUBLISH_DONE ends the subscription, so the stream resets rather than FIN a truncated + /// update. + #[moq_clock::test] + async fn a_stalled_update_resets_when_the_publisher_ends() { + // The SUBSCRIBE alone, measured over an open window. + let subscribe = { + let (session, serving, _track, _first) = update_under_window(None).await; + serving.abort(); + session.log.writes.lock().unwrap().len() + }; + + let (session, serving, track, _first) = update_under_window(Some(subscribe + 1)).await; + let _raised = track + .subscribe(track::Subscription { + priority: 5, + ..Default::default() + }) + .await + .unwrap(); + settle().await; + assert_eq!( + session.log.writes.lock().unwrap().len(), + subscribe + 1, + "the update stalled mid-message" + ); + + session.push(&message( + ietf::PublishDone::ID, + &ietf::PublishDone { + status_code: 0x2, + stream_count: 0, + reason_phrase: "done".into(), + }, + Version::Draft22, + )); + // The script push wakes nothing, so a subscription change re-polls the loop. + let _again = track + .subscribe(track::Subscription { + priority: 7, + ..Default::default() + }) + .await + .unwrap(); + // Past the grace PUBLISH_DONE waits out for its data streams; the copy then lingers. + moq_clock::sleep(crate::ledger::GRACE * 2).await; + assert_eq!(session.log.resets(), vec![crate::ietf::error::CANCELLED]); + assert!( + !session + .log + .trail() + .iter() + .any(|(_, sent)| matches!(sent, crate::lite::test_transport::Sent::Finish)), + "finished a truncated update" + ); + serving.abort(); + } + /// When `conflict` is set, an alias-7 binding for the same full track name is seeded /// first, so the subscription under test loses the race to bind it. async fn cancel_a_subscription_inner(version: Version, conflict: bool) -> crate::lite::test_transport::Log { @@ -15055,6 +15318,31 @@ mod joining_fetch_tests { } } + /// A committed joining FETCH the peer's send window holds back part-way is reset once + /// its subscription lets go, rather than waiting for credit to finish the header. + #[moq_clock::test] + async fn a_joining_fetch_stalled_on_credit_resets_when_cancelled() { + for version in JOINING_DRAFTS { + let session = ScriptedSession::new(Vec::new()).with_send_window(1); + let log = session.log.clone(); + let (subscriber, track, mut tasks) = pending_join(crate::driver::Transport::new(session.clone()), version); + let mut joining = subscriber.start_joining_fetch(RequestId(99), &track).unwrap(); + assert!(joining.poll(&kio::Waiter::noop()).is_pending()); + assert_eq!(log.writes.lock().unwrap().len(), 1, "{version}: stalled mid-message"); + + // The session's tasks now own the request. + drop(joining); + let _ = tasks.poll(&kio::Waiter::noop()); + assert_eq!(log.resets(), vec![crate::ietf::error::CANCELLED], "{version}"); + assert!( + !log.trail() + .iter() + .any(|(_, sent)| matches!(sent, crate::lite::test_transport::Sent::Finish)), + "{version}: finished a truncated request" + ); + } + } + /// Abandoning a blocked open leaves the next sequential ID available to unrelated work. #[moq_clock::test] async fn cancelling_a_joining_fetch_open_spends_no_id() { diff --git a/rs/moq-net/src/ietf/subscriber/tracks.rs b/rs/moq-net/src/ietf/subscriber/tracks.rs index d27a291ba33..4759412e29f 100644 --- a/rs/moq-net/src/ietf/subscriber/tracks.rs +++ b/rs/moq-net/src/ietf/subscriber/tracks.rs @@ -509,6 +509,12 @@ impl Subscriber { // made stay ours. tracing::debug!(prefix = %crate::path::Rooted(self.origin.root(), prefix), "subscribe_tracks cancelled"); let Stream { mut writer, mut reader } = stream; + // The request never reached the peer whole, so it holds nothing to wait on. + if !writer.is_flushed() { + reader.abort(&Error::Cancel); + writer.abort(&Error::Cancel); + return Ok(()); + } match request_stream::fin_cancels(self.version) { true => { let _ = writer.finish(); @@ -630,9 +636,11 @@ impl Subscriber { ); let mut messages = Messages::new(self.session.clone(), self.control.client()); let accepted = async { - stream - .writer - .encode_message(&ietf::Subscribe { + // Flushed while waiting for the answer, so the entry going still ends a request + // the peer's send window holds back. + stream.writer.buffer_message( + ietf::Subscribe::ID, + &ietf::Subscribe { max_age: None, new_group: None, request_id, @@ -649,9 +657,12 @@ impl Subscriber { range_filters: false, authorization_token: None, epoch: self.epoch.clone(), - }) - .await?; - let mut response = std::pin::pin!(self.read_subscribe_response(&mut stream, &mut messages)); + }, + )?; + let mut response = std::pin::pin!(async { + kio::wait(|waiter| stream.writer.poll_flush(waiter)).await?; + self.read_subscribe_response(&mut stream, &mut messages).await + }); let answered = kio::wait(|waiter| { if gone(waiter) { return Poll::Ready(None); @@ -902,10 +913,18 @@ impl Subscriber { if session.poll_closed(waiter).is_ready() { return Poll::Ready(Step::Closed); } - if let Poll::Ready(handoff) = handoff.poll(waiter, |handoff| match handoff.gone || handoff.wanted { - true => Poll::Ready(()), - false => Poll::Pending, - }) { + // A held subscription is handed over only once flushed, so resumes and pauses + // can't pile up behind a peer withholding credit. + let flushed = match upstream.stream.writer.poll_flush(waiter) { + Poll::Ready(Err(err)) => return Poll::Ready(Step::Message(Err(err))), + Poll::Ready(Ok(())) => true, + Poll::Pending => false, + }; + if let Poll::Ready(handoff) = + handoff.poll(waiter, |handoff| match handoff.gone || (handoff.wanted && flushed) { + true => Poll::Ready(()), + false => Poll::Pending, + }) { return Poll::Ready(match handoff.gone { true => Step::Gone, false => Step::Wanted, @@ -964,7 +983,13 @@ impl Subscriber { }; tracing::debug!(broadcast = %key.0, track = %key.1, ?end, "held subscription ended"); self.remove_subscribe(upstream.request_id); - let _ = upstream.stream.writer.finish(); + // A pause still held back would go out truncated behind a FIN. + match upstream.stream.writer.is_flushed() { + true => { + let _ = upstream.stream.writer.finish(); + } + false => upstream.stream.writer.abort(&Error::Cancel), + } // A failed upstream fails the copy, as it would while forwarding. TRACK_ENDED leaves it // idle: the end is past groups it never saw, so a returning reader's SUBSCRIBE learns // where, as for a copy whose subscription was cancelled. @@ -1074,14 +1099,15 @@ impl Subscriber { }); } - /// Pause a subscription nobody reads that is its track's entry, and hold it again. - pub(super) async fn pause(&self, key: TrackKey, mut upstream: Upstream) { + /// Pause a subscription nobody reads that is its track's entry, and hold it again. The + /// held watch flushes the pause, so the entry going still ends it under send pressure. + pub(super) fn pause(&self, key: TrackKey, mut upstream: Upstream) { self.park_subscribe(upstream.request_id); let pause = Forward { forward: false, ..Default::default() }; - match self.write_forward(&mut upstream.stream.writer, &pause).await { + match self.buffer_forward(&mut upstream.stream.writer, &pause) { Ok(()) => { upstream.owed += 1; tracing::debug!(broadcast = %key.0, track = %key.1, "subscription paused"); @@ -1124,15 +1150,17 @@ impl Subscriber { } } - /// Set a held subscription's Forward State with REQUEST_UPDATE. - pub(super) async fn write_forward( + /// Buffer the REQUEST_UPDATE setting a held subscription's Forward State, for the + /// caller to flush. + pub(super) fn buffer_forward( &self, writer: &mut Writer, forward: &Forward, ) -> Result<(), Error> { let request_id = self.control.next_request_id(); - writer - .encode_message(&ietf::SubscribeUpdate { + writer.buffer_message( + ietf::SubscribeUpdate::ID, + &ietf::SubscribeUpdate { max_age: forward.max_age, request_id, subscriber_priority: forward.priority.map(super::super::priority::to_wire), @@ -1141,8 +1169,8 @@ impl Subscriber { authorization_token: None, new_group: forward.new_group, fill: forward.fill, - }) - .await + }, + ) } /// Check the answer to a REQUEST_UPDATE a held subscription still owes. It carries @@ -1253,7 +1281,11 @@ mod tests { } fn harness(version: Version) -> Harness { - let (session, peer) = memory::pair(memory::Config::default()); + harness_with(version, memory::Config::default()) + } + + fn harness_with(version: Version, config: memory::Config) -> Harness { + let (session, peer) = memory::pair(config); let (tasks, mut task_set) = TaskSet::new(); let driver = moq_clock::sim::spawn(async move { kio::wait(|waiter| task_set.poll(waiter)).await }); let subscriber = Subscriber::new( @@ -1294,9 +1326,10 @@ mod tests { .expect("the subscriber sent nothing") } + /// One message in one write, so a peer holding one message per stream still delivers it. async fn write<'m, M: ietf::Message<'m> + std::fmt::Debug>(stream: &mut PeerStream, id: u64, msg: &M) { - stream.writer.varint(id).await.unwrap(); - stream.writer.encode(msg).await.unwrap(); + stream.writer.buffer_message(id, msg).unwrap(); + kio::wait(|waiter| stream.writer.poll_flush(waiter)).await.unwrap(); settle().await; } @@ -1414,8 +1447,7 @@ mod tests { let mut stream = Stream::open(&mut crate::driver::Transport::new(self.peer.clone()), self.version) .await .unwrap(); - stream.writer.varint(ietf::Publish::ID).await.unwrap(); - stream.writer.encode(&publish).await.unwrap(); + stream.writer.encode_message(&publish).await.unwrap(); let mut ours = Stream::accept(&mut self.subscriber.session.clone(), self.version) .await @@ -1434,6 +1466,42 @@ mod tests { } } + /// Readers coming and going while the peer withholds credit on a held track never pile + /// updates up behind it: the held subscription is handed over only once its pause is out, + /// and a reader leaving meanwhile stops waiting for it. + #[moq_clock::test] + async fn churn_under_withheld_credit_buffers_one_resume_and_pause() { + for version in [Version::Draft18, Version::Draft22] { + // One message in flight per stream, so a peer that stops reading withholds credit. + let config = memory::Config { + stream_buffer: 0, + ..Default::default() + }; + let mut h = harness_with(version, config); + let _listing = h.establish().await; + let mut video = h.publish("video", 1).await; + + for _ in 0..3 { + let (track, reader, serving) = h.read_track("video").await; + settle().await; + drop(reader); + drop(track); + moq_clock::timeout(TIMEOUT, serving) + .await + .unwrap_or_else(|_| panic!("{version}: the abandoned resume stayed alive")) + .unwrap(); + } + + let mut updates = Vec::new(); + while let Ok(Ok(id)) = moq_clock::timeout(TIMEOUT, video.reader.varint()).await { + let body: ietf::Body = video.reader.decode().await.unwrap(); + assert_eq!(id, ietf::SubscribeUpdate::ID, "{version}"); + updates.push(body.message::(version).unwrap().forward); + } + assert_eq!(updates, [Some(true), Some(false)], "{version}"); + } + } + /// A paused PUBLISH of `name` in `namespace`, aliased by its request ID. fn offered<'a>(namespace: &'a str, name: &'a str, request_id: u64) -> ietf::Publish<'a> { ietf::Publish { diff --git a/rs/moq-net/src/lite/test_transport.rs b/rs/moq-net/src/lite/test_transport.rs index ea1b0cbdaf0..a17bf539c68 100644 --- a/rs/moq-net/src/lite/test_transport.rs +++ b/rs/moq-net/src/lite/test_transport.rs @@ -948,6 +948,8 @@ pub struct ScriptedSession { datagram_size: usize, /// Set by [`Self::with_groups`]. groups: bool, + /// Set by [`Self::with_send_window`]. + send_window: Option, } impl ScriptedSession { @@ -985,9 +987,17 @@ impl ScriptedSession { unacked_unis: Default::default(), datagram_size: 0, groups: false, + send_window: None, } } + /// Let each opened bidi stream send only its first `bytes`, then park its writes for good, + /// like a peer that never grants more send credit. + pub fn with_send_window(mut self, bytes: usize) -> Self { + self.send_window = Some(bytes); + self + } + /// Open send groups, as native QUIC does, recording them in the log. pub fn with_groups(mut self) -> Self { self.groups = true; @@ -1141,7 +1151,15 @@ impl poll::Session for ScriptedSession { Some(queue) => Arc::new(Mutex::new(queue.lock().unwrap().pop_front().unwrap_or_default())), None => self.script.clone(), }; - let mut send = SinkSend::new(self.log.clone()).opened(options); + let mut send = match self.send_window { + // The gate's producer is gone, so the writes past the prefix park forever. + Some(window) => SinkSend { + prefix: window, + ..SinkSend::gated(self.log.clone(), kio::Producer::new(false).consume()) + }, + None => SinkSend::new(self.log.clone()), + } + .opened(options); send.unacked_fin = self.unacked_fin; Poll::Ready(Ok(( send,