From 28168984d325a039b48bd3bb6a37c45263b19d8b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 11:00:03 -0700 Subject: [PATCH 1/8] quest: claim track-tail-hardening Co-Authored-By: Claude Opus 5.5 From d498efbd5949f542d2e189f2282ad137a0bb4feb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 11:46:22 -0700 Subject: [PATCH 2/8] fix(net): harden the Rust track tail Cancel an IETF END_OF_TRACK stuck on stream credit with the request, count it once opened, abort a group whose first object is cut short, and hold the tail open while any stream is still being read. A group at or past the declared end aborts the track. The lite owed floor follows the demand, and tail gaps older than the grace fold away. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/ietf/publisher.rs | 34 +++- rs/moq-net/src/ietf/subscriber.rs | 110 ++++++++---- rs/moq-net/src/lite/subscriber.rs | 166 +++++++++++++++-- rs/moq-net/src/tail.rs | 288 +++++++++++++++++++++++++----- rs/moq-net/tests/support/mock.rs | 34 +++- rs/moq-net/tests/track_tail.rs | 163 ++++++++++++++++- 6 files changed, 685 insertions(+), 110 deletions(-) diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 2243679f4ef..48de95a3a9d 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -620,26 +620,40 @@ where .await }; - // Every data stream this subscription opened is closed by now, which PUBLISH_DONE - // requires, so the count it reports is final. let completed = served.is_some(); let (res, filled) = served.unwrap_or((Ok(()), false)); - let mut streams = track_serve.opened() + u64::from(filled); // Draft-14 on carries no end location in PUBLISH_DONE: an END_OF_TRACK object is // what tells the subscriber where the track ended. A cancelled subscription is - // owed nothing more. + // owed nothing more, and one cancelled while the marker waits for stream credit + // abandons it. if completed && res.is_ok() && let Some(end) = track_serve.end() { - match track_serve.write_end_of_track(end, priority).await { - Ok(()) => streams += 1, - // A failure only costs the subscriber the early boundary. - Err(err) => tracing::debug!(%err, id = %request_id, "end of track failed"), + let mut marker = std::pin::pin!(track_serve.write_end_of_track(end, priority)); + let mut closed_session = self.session.clone(); + let written = kio::wait(|waiter| { + if let Poll::Ready(res) = waiter.poll_future(marker.as_mut()) { + return Poll::Ready(res); + } + let mut cx = std::task::Context::from_waker(waiter.waker()); + if stream.reader.poll_closed(&mut cx).is_ready() || closed_session.poll_closed(&mut cx).is_ready() { + return Poll::Ready(Err(Error::Cancel)); + } + Poll::Pending + }) + .await; + // A failure only costs the subscriber the early boundary. + if let Err(err) = written { + tracing::debug!(%err, id = %request_id, "end of track failed"); } } + // Every data stream this subscription opened is closed by now, which PUBLISH_DONE + // requires, so the count it reports is final. + let streams = track_serve.opened() + u64::from(filled); + // Send PublishDone let (status, reason) = match &res { Ok(()) => (ietf::PublishDoneStatus::TrackEnded, "track ended"), @@ -2015,11 +2029,13 @@ impl TrackServe { /// the group that will never exist. /// /// The last group's stream has usually finished before the track ends, so the marker - /// cannot ride on it. The stream counts toward PUBLISH_DONE once it is open. + /// cannot ride on it. Like a group stream, it counts toward PUBLISH_DONE once open, + /// since a reset can still deliver its header. async fn write_end_of_track(&mut self, end: u64, priority: u8) -> Result<(), Error> { let mut stream = std::future::poll_fn(|cx| self.session.poll_open_uni(cx)) .await .map_err(Error::from_transport)?; + self.opened.fetch_add(1, Ordering::Relaxed); stream.set_priority(priority); let mut writer = Writer::new(stream, self.version); diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 000610327d2..a4c8e965a39 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -15,7 +15,7 @@ use crate::{ }; use super::{Message, Version, cluster, error::request, peer}; -use crate::tail::{self, Settle, Tail}; +use crate::tail::{Reading, Settle, Tail}; use kio::Lock; @@ -374,20 +374,6 @@ impl TrackState { } } -/// Counts a data stream toward its subscription's [`Tail`] once its handler is done with it. -/// -/// Counting at the end rather than at the header means a Stream Count that is reached never -/// races the stream's own work, such as the END_OF_TRACK that declares where the track ends. -struct Counted(kio::Producer); - -impl Drop for Counted { - fn drop(&mut self) { - if let Ok(mut tail) = self.0.write() { - tail.stream(); - } - } -} - struct BroadcastState { // The route announced into our origin for this namespace, post-charge. route: crate::origin::Route, @@ -1840,10 +1826,10 @@ where tracing::info!(broadcast = %self.origin.absolute(&broadcast_path), track = %track_name, "subscribe complete"); // The publisher sends PUBLISH_DONE once every data stream it opened // is closed, but QUIC does not order them, so some can still be on - // their way. Wait until Stream Count of them are read, or a bounded - // grace for any reset before its header (the draft says to use a - // timeout). The count is a hint: a published peer sends 0, which - // waits out the grace. + // their way. Wait until Stream Count of their headers arrived and + // each is read to its end, or a bounded grace for any reset before + // its header (the draft says to use a timeout). The count is a hint: + // a published peer sends 0, which waits out the grace. let tail = self .state .lock() @@ -1851,7 +1837,7 @@ where .get(&request_id) .map(|held| held.tail.consume()); if let Some(tail) = tail { - let mut settle = Settle::new(&self.runtime, tail, tail::GRACE); + let mut settle = Settle::new(&self.runtime, tail); kio::wait(|waiter| { if !fetch_done && let Some(fut) = fetching.as_mut() @@ -2179,7 +2165,7 @@ where } }; - let (mut track, timescale, fill, _counted) = { + let (mut track, timescale, fill, mut reading) = { let state = self.state.lock(); let track = state.subscribes.get(&request_id).ok_or(Error::NotFound)?; ( @@ -2187,8 +2173,8 @@ where track.timescale, track.fill.clone(), // Every data stream counts toward PUBLISH_DONE's Stream Count, even one - // dropped below. - Counted(track.tail.clone()), + // dropped below, and the subscription's end waits until it is read. + Reading::open(&track.tail, Some(group.group_id), self.runtime.now()), ) }; @@ -2225,14 +2211,25 @@ where // Otherwise dropping the local subscriber cannot end this handler. let opened = { let mut opening = track.clone(); - let mut open = std::pin::pin!(self.open_group(stream, &mut opening, &fill, &group)); + let mut open = std::pin::pin!(self.open_group(stream, &mut opening, &fill, &group, &mut reading)); kio::wait(|waiter| { if let Poll::Ready(err) = track.poll_closed(waiter) { return Poll::Ready(Err(err)); } waiter.poll_future(open.as_mut()) }) - .await? + .await + }; + let opened = match opened { + // The group is at or past the end the publisher declared, which no later stream + // can repair. + Err(Error::Closed) => { + tracing::warn!(group = group.group_id, "group past the declared end of track"); + let _ = track.abort(Error::ProtocolViolation); + return Err(Error::ProtocolViolation); + } + Err(err) => return Err(err), + Ok(opened) => opened, }; let (producer, start) = match opened { Opened::Group(producer, start) => (producer, start), @@ -2388,15 +2385,34 @@ where track: &mut track::Producer, fill: &kio::Producer, header: &ietf::GroupHeader, + reading: &mut Reading, ) -> Result { let sequence = header.group_id; // Stats (groups/frames/bytes) are counted in the model as the group is written, // through the tagged `track::Producer`. let create = |track: &mut track::Producer| track.create_group(group::Info { sequence }); - let first = match header.flags.has_extensions { - true => stream.decode_peek_maybe::>().await?.map(|peek| peek.0), - false => stream.decode_peek_maybe::>().await?.map(|peek| peek.0), + let peeked = match header.flags.has_extensions { + true => stream + .decode_peek_maybe::>() + .await + .map(|peek| peek.map(|peek| peek.0)), + false => stream + .decode_peek_maybe::>() + .await + .map(|peek| peek.map(|peek| peek.0)), + }; + let first = match peeked { + Ok(first) => first, + // The header arrived, so the stream counts toward the track's end. Abort the + // group it named rather than let the track end clean without it. A fill that + // already holds the group reports it through its own producer. + Err(err) => { + if let Ok(group) = create(track) { + let _ = group.abort(err.clone()); + } + return Err(err); + } }; if first.is_some_and(|first| first.id == 0 && first.end_of_track) { return Ok(Opened::EndOfTrack); @@ -2427,7 +2443,7 @@ where // A group starting partway through is the tail of one the fill began, and // without that head it has a hole at the front. - Some(start) => match self.claim_fill(fill, track, sequence, Some(start)).await? { + Some(start) => match self.claim_fill(fill, track, sequence, Some(start), reading).await? { Some(producer) => Ok(Opened::Group(producer, start)), None => { tracing::warn!(sequence, start, "no fill to stitch a mid-group stream onto"); @@ -2437,7 +2453,7 @@ where // A stream that ends without an object: the group is over and had nothing // outside the fill's range, so the head it delivered is the whole group. - None => match self.claim_fill(fill, track, sequence, None).await? { + None => match self.claim_fill(fill, track, sequence, None, reading).await? { Some(producer) => Ok(Opened::Group(producer, 0)), None => Ok(Opened::Group(create(track)?, 0)), }, @@ -2454,15 +2470,19 @@ where /// /// Waiting is what keeps the two streams from interleaving into one producer. It ends /// with the subscription, so a publisher that promises a fill and never delivers one - /// costs this stream and nothing else. + /// costs this stream and nothing else. Meanwhile this stream stops holding the + /// subscription's end open: the fill's own stream holds it if its header arrived, and + /// the grace gives up on it if not. async fn claim_fill( &self, fill: &kio::Producer, track: &track::Producer, sequence: u64, start: Option, + reading: &mut Reading, ) -> Result, Error> { - kio::wait(|waiter| { + reading.park(); + let claimed = kio::wait(|waiter| { if let Poll::Ready(err) = track.poll_closed(waiter) { return Poll::Ready(Err(err)); } @@ -2479,7 +2499,9 @@ where Poll::Pending => Poll::Pending, } }) - .await + .await; + reading.resume(); + claimed } } @@ -2561,7 +2583,9 @@ where let track = state.subscribes.get(&subscribe_id).ok_or(Error::NotFound)?; // A fill is one of the subscription's own data streams, so PUBLISH_DONE counts // it. A joining FETCH is a request of its own. - let counted = joined.is_none().then(|| Counted(track.tail.clone())); + let counted = joined + .is_none() + .then(|| Reading::open(&track.tail, None, self.runtime.now())); (subscribe_id, track.fill.clone(), track.joining, track.largest, counted) }; @@ -6102,6 +6126,24 @@ mod stitch_tests { assert!(consumer.recv_group().await.unwrap().is_none(), "then the track ends"); } + /// A group at or past the end an END_OF_TRACK declared contradicts that end, which no + /// later stream can repair, so the whole track fails rather than ending clean without it. + #[tokio::test] + async fn a_group_past_the_declared_end_aborts_the_track() { + use futures::FutureExt; + + let mut h = Harness::new(Fill::Done, vec![tail_stream(SEQUENCE, 0, &[b"late"])]); + h.track.finish_at(SEQUENCE).unwrap(); + let mut stream = h.stream().await; + + let res = h.subscriber.clone().recv_group(&mut stream).await; + assert!(matches!(res, Err(Error::ProtocolViolation)), "{res:?}"); + assert!(matches!( + h.track.closed().now_or_never(), + Some(Error::ProtocolViolation) + )); + } + /// END_OF_TRACK at object 0 says the group does not exist, so the track ends before it /// and no group is created for it. #[tokio::test] diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index a648b2f2c84..2d5081515ff 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -15,7 +15,7 @@ use crate::{ }; use super::Version; -use crate::tail::{self, Settle, Tail}; +use crate::tail::{self, Reading, Settle, Tail}; use kio::Lock; @@ -456,7 +456,7 @@ impl Subscriber { // A datagram is never owed a stream, so it never holds the subscription's end open. if let Ok(mut tail) = entry.tail.write() { - tail.account(dg.sequence..dg.sequence.saturating_add(1)); + tail.account(dg.sequence..dg.sequence.saturating_add(1), self.runtime.now()); } entry.producer.insert_datagram(dg.sequence, timestamp, dg.payload)?; Ok(()) @@ -740,6 +740,7 @@ enum GroupRecvState { group: crate::recv::Group, track: track::Producer, ingest: FrameIngest, + _reading: Reading, }, Done, } @@ -760,22 +761,31 @@ impl GroupRecv { let mut cx = std::task::Context::from_waker(waiter.waker()); let hdr = ready!(self.reader.poll_decode::(&mut cx))?; - let (group, track, timescale) = { + let (group, track, timescale, reading) = { let mut subs = self.subscriber.subscribes.lock(); let entry = subs.get_mut(&hdr.subscribe).ok_or(Error::Cancel)?; - if let Ok(mut tail) = entry.tail.write() { - tail.open(hdr.sequence); - } + // The subscription's end waits until this stream is read. + let reading = Reading::open(&entry.tail, Some(hdr.sequence), self.subscriber.runtime.now()); let group_info = group::Info { sequence: hdr.sequence }; // Stats (groups/frames/bytes) are counted in the model as the group // is written, through the tagged `track::Producer`. - let mut group = entry.producer.create_group(group_info)?; + let mut group = match entry.producer.create_group(group_info) { + Ok(group) => group, + // The group is at or past the end the publisher declared, which no + // later stream can repair. + Err(Error::Closed) => { + tracing::warn!(group = hdr.sequence, "group past the declared end of track"); + let _ = entry.producer.clone().abort(Error::ProtocolViolation); + return Poll::Ready(Err(Error::ProtocolViolation)); + } + Err(err) => return Poll::Ready(Err(err)), + }; // The stream may carry only the tail of the group; number the frames // from where the publisher said they start so a reader splicing // across routes lines them up. group.start_at(hdr.frame_start)?; - (group, entry.producer.clone(), entry.timescale) + (group, entry.producer.clone(), entry.timescale, reading) }; // The timescale came from TRACK_INFO (read before this subscription was @@ -785,9 +795,12 @@ impl GroupRecv { group: crate::recv::Group::new(group), track, ingest: FrameIngest::new(self.subscriber.runtime.clone(), timescale), + _reading: reading, }; } - GroupRecvState::Serve { group, track, ingest } => { + GroupRecvState::Serve { + group, track, ingest, .. + } => { // The track or group dying cancels the stream; the peer's own close // arrives through the ingest's reads. let res = 'serve: { @@ -806,7 +819,10 @@ impl GroupRecv { // Held until the group settles below, so a frame the track or group cut // short drops into an already-aborted group instead of reporting a loss. let GroupRecvState::Serve { - group, ingest: _ingest, .. + group, + ingest: _ingest, + _reading, + .. } = std::mem::replace(&mut self.state, GroupRecvState::Done) else { unreachable!() @@ -1379,6 +1395,96 @@ mod tests { const VERSION: Version = Version::Lite05; + /// The owed groups start at the floor the demand last asked for, which an update can move + /// either way after SUBSCRIBE_START answered the SUBSCRIBE. + #[tokio::test] + async fn the_owed_floor_follows_the_demand() { + let mut session = crate::lite::test_transport::ScriptedSession::new(Vec::new()); + let stream = Stream::open(&mut session, VERSION).await.unwrap(); + let mut sub = SubStream { + stream, + id: 0, + max_age: Duration::ZERO, + start: Some(Position::group(2)), + priority: 0, + requested: Some(Position::group(2)), + tail: Default::default(), + served: None, + end: Some(lite::SubscribeEnd { group: 10, streams: 0 }), + }; + assert_eq!( + sub.owed(None), + Some(10..10), + "without SUBSCRIBE_START nothing was served" + ); + + sub.served = Some(4); + assert_eq!( + sub.owed(None), + Some(2..10), + "the groups below START are accounted for on arrival" + ); + sub.start = Some(Position::group(6)); + assert_eq!(sub.owed(None), Some(6..10), "a raised floor owes nothing below it"); + sub.start = Some(Position::group(1)); + assert_eq!( + sub.owed(None), + Some(1..10), + "a lowered floor owes what it newly asked for" + ); + sub.start = None; + assert_eq!( + sub.owed(Some(8)), + Some(4..8), + "a live-edge floor starts where START resolved it" + ); + } + + /// A group at or past the end the publisher declared contradicts that end, which no later + /// stream can repair, so the whole track fails rather than ending clean without it. + #[tokio::test] + async fn a_group_past_the_declared_end_aborts_the_track() { + use crate::transport::poll::Session as _; + + let mut script = Vec::new(); + lite::Group { + subscribe: 7, + sequence: 3, + frame_start: 0, + } + .encode(&mut script, VERSION) + .unwrap(); + let mut session = crate::lite::test_transport::ScriptedSession::eof(script); + let subscriber = Subscriber::new(SubscriberConfig { + runtime: crate::time::Clock::tokio(), + session: session.clone(), + origin: origin::Config::new(crate::Hop::new(1).unwrap()).produce(), + recv_bandwidth: None, + version: VERSION, + peer_setup: Default::default(), + peer_hop: None, + cost: None, + going_away: Default::default(), + }); + + let mut track = track::Producer::new(std::sync::Arc::new(crate::broadcast::Info::default()), "video", None); + track.finish_at(3).unwrap(); + subscriber.subscribes.lock().insert( + 7, + TrackEntry { + producer: track.clone(), + timescale: Some(Timescale::default()), + tail: Default::default(), + }, + ); + + let (_, recv) = session.open_bi().await.unwrap(); + let mut group = GroupRecv::new(subscriber, Reader::new(recv, VERSION)); + let res = kio::wait(|waiter| group.poll_serve(waiter)).await; + assert!(matches!(res, Err(Error::ProtocolViolation)), "{res:?}"); + assert!(matches!(track.closed().now_or_never(), Some(Error::ProtocolViolation))); + } + /// Drive the subscriber with a peer's response bytes followed by FIN. async fn check_subscription_fin(version: Version, responses: Vec, clean: bool) { let session = crate::lite::test_transport::ScriptedSession::eof(responses); @@ -2778,11 +2884,18 @@ impl SubStream { /// The groups the publisher still owes once it has ended the subscription. /// /// `None` when nothing says which: drafts before SUBSCRIBE_END only have the FIN. - /// Without a SUBSCRIBE_START the publisher served no group at all. + /// Without a SUBSCRIBE_START the publisher served no group at all. Otherwise they start + /// at the floor the demand last asked for, which an update can move either way, or + /// where SUBSCRIBE_START resolved a live-edge one. The groups the SUBSCRIBE asked for + /// below its SUBSCRIBE_START are accounted for as unavailable when it arrives. fn owed(&self, requested_end: Option) -> Option> { let end = self.end.as_ref()?.group; let end = requested_end.map_or(end, |requested| requested.min(end)); - Some(self.served.unwrap_or(end)..end) + let start = match self.served { + Some(served) => self.start.map_or(served, |start| start.group), + None => end, + }; + Some(start..end) } } @@ -3055,6 +3168,9 @@ impl TrackServe { let start_moved = active.start != subscription.start; active.priority = subscription.priority; active.max_age = subscription.max_age; + if let Ok(mut tail) = active.tail.write() { + tail.set_grace(tail::grace(subscription.max_age)); + } active.start = subscription.start; if supports_update { // The floor follows the requested start, in both directions: @@ -3127,7 +3243,7 @@ impl TrackServe { tracing::info!(id, broadcast = %self.subscriber.log_path(&self.path), track = %self.name, "subscribe started"); - let tail = kio::Producer::new(Tail::default()); + let tail = kio::Producer::new(Tail::new(tail::grace(subscription.max_age))); self.subscriber.subscribes.lock().insert( id, TrackEntry { @@ -3763,12 +3879,22 @@ impl ServeLoop { let _ = self.serving.start_at(start.group); } active.served = Some(start.group); + // The groups the SUBSCRIBE asked for below it are + // unavailable, whatever the demand asks later. + if let Some(requested) = active.requested + && let Ok(mut tail) = active.tail.write() + { + tail.account(requested.group..start.group, serve.subscriber.runtime.now()); + } } // The publisher will never send these groups, so they // are accounted for without a stream. lite::SubscribeResponse::Drop(dropped) => { if let Ok(mut tail) = active.tail.write() { - tail.account(dropped.start..dropped.end.saturating_add(1)); + tail.account( + dropped.start..dropped.end.saturating_add(1), + serve.subscriber.runtime.now(), + ); } } // OK just resolves the range (the producer already orders @@ -3800,12 +3926,14 @@ impl ServeLoop { // The effective max age is the stopgap grace: the wrong clock // (it bounds presentation-time drift), but it is how long the // subscriber was willing to wait for a late group anyway. - let grace = subscription - .map(|sub| sub.max_age) - .filter(|max_age| !max_age.is_zero()) - .unwrap_or(tail::GRACE); + if let Ok(mut tail) = active.tail.write() { + tail.set_grace(tail::grace( + subscription.map(|sub| sub.max_age).unwrap_or_default(), + )); + tail.expire(serve.subscriber.runtime.now()); + } self.mode = ServeMode::Tail { - settle: Settle::new(&serve.subscriber.runtime, active.tail.consume(), grace), + settle: Settle::new(&serve.subscriber.runtime, active.tail.consume()), owed: active.owed(requested_end), streams: active .end diff --git a/rs/moq-net/src/tail.rs b/rs/moq-net/src/tail.rs index 34e2ff6a292..67c1361d81a 100644 --- a/rs/moq-net/src/tail.rs +++ b/rs/moq-net/src/tail.rs @@ -5,10 +5,14 @@ //! it. The subscriber keeps the subscription routable until each owed group is accounted //! for, or a grace expires for a group whose stream was reset before its header arrived. //! A reset that keeps the header (reliable reset) would make the grace unnecessary. +//! +//! A stream whose header arrived is read to its end whatever the grace says: a group ends +//! on its own stream's FIN or reset, never because its track ended. A lost datagram is not +//! owed: it leaves a hole that waits out the grace like a stream reset before its header. use std::{ops::Range, task::Poll, time::Duration}; -use crate::runtime::Deadline; +use crate::runtime::{Deadline, Instant}; /// How long a subscriber waits for a group stream it cannot account for once the /// publisher has ended the subscription. @@ -17,52 +21,110 @@ use crate::runtime::Deadline; /// it with. Matches `@moq/net`, so a reader cannot tell which side it is talking to. pub(crate) const GRACE: Duration = Duration::from_secs(1); +/// The grace for a subscription with `max_age`: how long the subscriber was willing to +/// wait for a late group anyway, or [`GRACE`] without one. +pub(crate) fn grace(max_age: Duration) -> Duration { + match max_age.is_zero() { + true => GRACE, + false => max_age, + } +} + +/// A run of accounted sequences. +#[derive(Debug, Clone, PartialEq, Eq)] +struct Run { + groups: Range, + /// When the gap below this run opened, which is when its missing groups became late. + since: Instant, +} + /// The data streams a subscription has received, and the groups they account for. -#[derive(Default, Debug)] +#[derive(Debug)] pub(crate) struct Tail { - // Disjoint, sorted, non-adjacent ranges of accounted sequences, so this grows with the - // number of gaps rather than the number of groups. - accounted: Vec>, + // Disjoint, sorted, non-adjacent runs of accounted sequences. A gap older than the grace + // can no longer be waited for, so it folds into the runs around it, which bounds this by + // the gaps opened within the grace rather than every gap in the subscription. + runs: Vec, + grace: Duration, streams: u64, + // Streams whose header arrived and that are still being read. + active: u64, +} + +impl Default for Tail { + fn default() -> Self { + Self { + runs: Vec::new(), + grace: GRACE, + streams: 0, + active: 0, + } + } } impl Tail { - /// Record a group stream whose header arrived. - pub fn open(&mut self, sequence: u64) { - self.stream(); - self.account(sequence..sequence.saturating_add(1)); + /// A tail that gives up on a missing group after `grace`. + pub fn new(grace: Duration) -> Self { + Self { + grace, + ..Self::default() + } } - /// Record a data stream that names no single group, such as a fill. - pub fn stream(&mut self) { - self.streams += 1; + /// Change the grace, for a subscription whose max age changed. + pub fn set_grace(&mut self, grace: Duration) { + self.grace = grace; } /// Record groups as accounted for without a stream: dropped, or sent as a datagram. - pub fn account(&mut self, groups: Range) { + pub fn account(&mut self, groups: Range, now: Instant) { if groups.is_empty() { return; } - // Every range the insert overlaps or touches merges with it into one. - let first = self.accounted.partition_point(|range| range.end < groups.start); - let last = self.accounted.partition_point(|range| range.start <= groups.end); - let merged = match &self.accounted[first..last] { - [] => groups, - [head, .., tail] => head.start.min(groups.start)..tail.end.max(groups.end), - [only] => only.start.min(groups.start)..only.end.max(groups.end), + // Every run the insert overlaps or touches merges with it into one. + let first = self.runs.partition_point(|run| run.groups.end < groups.start); + let last = self.runs.partition_point(|run| run.groups.start <= groups.end); + let merged = match &self.runs[first..last] { + [] => Run { + groups, + // Splitting a gap leaves both halves as late as it was. A run past every + // other one opens a new gap. + since: self.runs.get(first).map_or(now, |above| above.since), + }, + [head, .., tail] => Run { + groups: head.groups.start.min(groups.start)..tail.groups.end.max(groups.end), + since: head.since, + }, + [only] => Run { + groups: only.groups.start.min(groups.start)..only.groups.end.max(groups.end), + since: only.since, + }, }; - self.accounted.splice(first..last, [merged]); + self.runs.splice(first..last, [merged]); + self.expire(now); + } + + /// Fold every gap older than the grace into the runs around it. + pub fn expire(&mut self, now: Instant) { + let grace = self.grace; + self.runs.dedup_by(|run, below| { + let expired = now.duration_since(run.since) >= grace; + if expired { + below.groups.end = run.groups.end; + } + expired + }); } /// Whether every group in `groups` is accounted for. pub fn covers(&self, groups: Range) -> bool { - // Ranges merge on insert, so one range covers the span or none does. + // Runs merge on insert, so one run covers the span or none does. groups.is_empty() || self - .accounted + .runs .iter() - .any(|range| range.start <= groups.start && groups.end <= range.end) + .any(|run| run.groups.start <= groups.start && groups.end <= run.groups.end) } /// Data streams received, whether they finished or were reset. @@ -71,6 +133,65 @@ impl Tail { } } +/// A data stream whose header arrived, holding its subscription's end open until dropped. +pub(crate) struct Reading { + tail: Option>, + active: bool, +} + +impl Reading { + /// Count a data stream, accounting for `group` when it names one (a fill does not). + pub fn open(tail: &kio::Producer, group: Option, now: Instant) -> Self { + let Ok(mut state) = tail.write() else { + return Self { + tail: None, + active: false, + }; + }; + state.streams += 1; + state.active += 1; + if let Some(group) = group { + state.account(group..group.saturating_add(1), now); + } + Self { + tail: Some(tail.clone()), + active: true, + } + } + + /// Stop holding the end open while the stream waits on another one, which holds it + /// open itself if its header arrived. + pub fn park(&mut self) { + self.set_active(false); + } + + /// Hold the end open again once the stream resumes reading. + pub fn resume(&mut self) { + self.set_active(true); + } + + fn set_active(&mut self, active: bool) { + if self.active == active { + return; + } + self.active = active; + if let Some(tail) = &self.tail + && let Ok(mut state) = tail.write() + { + match active { + true => state.active += 1, + false => state.active -= 1, + } + } + } +} + +impl Drop for Reading { + fn drop(&mut self) { + self.set_active(false); + } +} + /// Waits out a subscription's tail: until the owed streams are accounted for, or the grace. pub(crate) struct Settle { tail: kio::Consumer, @@ -78,21 +199,21 @@ pub(crate) struct Settle { } impl Settle { - /// Start waiting on `tail`, giving up on missing streams after `grace`. - pub fn new(runtime: &crate::time::Clock, tail: kio::Consumer, grace: Duration) -> Self { + /// Start waiting on `tail`, giving up on missing streams after its grace. + pub fn new(runtime: &crate::time::Clock, tail: kio::Consumer) -> Self { + let grace = tail.read().grace; Self { tail, grace: Deadline::after(runtime, grace), } } - /// Ready once `complete` holds, the grace expires, or the subscription is gone. + /// Ready once no stream is being read and either `complete` holds or the grace + /// expired, or once the subscription is gone. pub fn poll(&mut self, waiter: &kio::Waiter, mut complete: impl FnMut(&Tail) -> bool) -> Poll<()> { - if self.grace.poll(waiter).is_ready() { - return Poll::Ready(()); - } + let expired = self.grace.poll(waiter).is_ready(); self.tail - .poll(waiter, |tail| match complete(tail) { + .poll(waiter, |tail| match tail.active == 0 && (expired || complete(tail)) { true => Poll::Ready(()), false => Poll::Pending, }) @@ -104,24 +225,107 @@ impl Settle { mod tests { use super::*; + fn groups(tail: &Tail) -> Vec> { + tail.runs.iter().map(|run| run.groups.clone()).collect() + } + #[test] - fn ranges_merge_across_gaps() { + fn runs_merge_across_gaps() { + let now = Instant::now(); let mut tail = Tail::default(); - tail.open(0); - tail.open(2); + tail.account(0..1, now); + tail.account(2..3, now); assert!(!tail.covers(0..3), "group 1 is missing"); - assert_eq!(tail.accounted, vec![0..1, 2..3]); + assert_eq!(groups(&tail), vec![0..1, 2..3]); - tail.account(1..2); - assert!(tail.covers(0..3) && tail.accounted.len() == 1, "adjacent ranges merge"); + tail.account(1..2, now); + assert!(tail.covers(0..3) && tail.runs.len() == 1, "adjacent runs merge"); - tail.account(5..7); - tail.account(9..10); - tail.account(4..9); - assert_eq!(tail.accounted, vec![0..3, 4..10], "one insert swallows several ranges"); + tail.account(5..7, now); + tail.account(9..10, now); + tail.account(4..9, now); + assert_eq!(groups(&tail), vec![0..3, 4..10], "one insert swallows several runs"); assert!(tail.covers(4..10)); assert!(!tail.covers(2..5)); assert!(tail.covers(7..7), "an empty range is always covered"); - assert_eq!(tail.streams(), 2, "only streams count, not drops"); + } + + /// A gap older than the grace folds away, so a lossy track keeps only the gaps it can + /// still wait for. + #[test] + fn a_gap_past_the_grace_folds_away() { + let start = Instant::now(); + let mut tail = Tail::new(GRACE); + tail.account(0..1, start); + tail.account(2..3, start); + tail.account(4..5, start + GRACE / 2); + assert_eq!(groups(&tail), vec![0..1, 2..3, 4..5]); + + // Splitting the younger gap keeps it as late as it was. + tail.account(6..7, start + GRACE / 2); + tail.account(8..9, start + GRACE); + assert_eq!( + groups(&tail), + vec![0..3, 4..5, 6..7, 8..9], + "only the oldest gap expired" + ); + + tail.expire(start + GRACE * 2); + assert_eq!(groups(&tail), vec![0..9]); + assert!(tail.covers(0..9)); + } + + /// Splitting a gap leaves both halves with its age. + #[test] + fn a_split_gap_keeps_its_age() { + let start = Instant::now(); + let mut tail = Tail::new(GRACE); + tail.account(0..1, start); + tail.account(9..10, start); + tail.account(5..6, start + GRACE / 2); + assert_eq!(groups(&tail), vec![0..1, 5..6, 9..10]); + + tail.expire(start + GRACE); + assert_eq!(groups(&tail), vec![0..10], "both halves opened at the start"); + } + + /// The grace gives up on streams that never arrived, never on one still being read: + /// its group, or the END_OF_TRACK it carries, still belongs to the track. + #[tokio::test(start_paused = true)] + async fn the_grace_waits_for_a_stream_being_read() { + use crate::runtime::Timers as _; + + let runtime = crate::time::Clock::tokio(); + let tail = kio::Producer::new(Tail::default()); + let reading = Reading::open(&tail, Some(0), runtime.now()); + let mut settle = Settle::new(&runtime, tail.consume()); + let mut settled = std::pin::pin!(kio::wait(|waiter| settle.poll(waiter, |_| false))); + + tokio::time::sleep(GRACE * 2).await; + assert!( + futures::poll!(settled.as_mut()).is_pending(), + "a stream is still being read" + ); + + drop(reading); + settled.await; + } + + /// A stream parked on another one does not hold the end open. + #[tokio::test(start_paused = true)] + async fn a_parked_stream_does_not_hold_the_end() { + use crate::runtime::Timers as _; + + let runtime = crate::time::Clock::tokio(); + let tail = kio::Producer::new(Tail::default()); + let mut reading = Reading::open(&tail, None, runtime.now()); + reading.park(); + let mut settle = Settle::new(&runtime, tail.consume()); + kio::wait(|waiter| settle.poll(waiter, |tail| tail.streams() == 1)).await; + + reading.resume(); + assert_eq!(tail.read().active, 1); + drop(reading); + assert_eq!(tail.read().active, 0); } } diff --git a/rs/moq-net/tests/support/mock.rs b/rs/moq-net/tests/support/mock.rs index c23cfbb5bff..36e5f74ea05 100644 --- a/rs/moq-net/tests/support/mock.rs +++ b/rs/moq-net/tests/support/mock.rs @@ -366,6 +366,10 @@ struct SessionSide { conn: Arc, /// Uni streams this side opened that the peer has not accepted yet, while held. held: Mutex>>, + /// Whether the peer has withheld uni stream credit, parking every open. + withheld: Mutex, + /// Whether the datagrams this side sends are lost. + lossy: Mutex, } /// An in-memory mock WebTransport session. @@ -382,6 +386,7 @@ pub struct MockSession { accept_bi: kio::Park, datagram: kio::Park, closed: kio::Park, + open_uni: kio::Park, } impl poll::Session for MockSession { @@ -439,7 +444,15 @@ impl poll::Session for MockSession { } } - fn poll_open_uni(&mut self, _cx: &mut Context<'_>) -> Poll> { + fn poll_open_uni(&mut self, cx: &mut Context<'_>) -> Poll> { + if *self.side.withheld.lock().unwrap() { + self.side.conn.waiters.register(self.open_uni.hold(cx)); + return match self.side.conn.error() { + Some(err) => Poll::Ready(Err(err)), + None => Poll::Pending, + }; + } + let (mut our_send, peer_recv) = new_stream_pair(&self.side.conn); if let Some(held) = self.side.held.lock().unwrap().as_mut() { @@ -456,6 +469,9 @@ impl poll::Session for MockSession { } fn poll_send_datagram(&mut self, _cx: &mut Context<'_>, payload: &[u8]) -> Poll> { + if *self.side.lossy.lock().unwrap() { + return Poll::Ready(Ok(())); + } match self.side.peer_datagrams.try_push(Bytes::copy_from_slice(payload)) { Ok(()) => Poll::Ready(Ok(())), Err(_) => Poll::Ready(Err(self.close_error())), @@ -553,6 +569,17 @@ impl MockSession { pub fn drop_unis(&self) { self.side.held.lock().unwrap().take(); } + + /// Park every uni stream this side opens from now on, like a peer that has granted + /// no more stream credit. + pub fn withhold_unis(&self) { + *self.side.withheld.lock().unwrap() = true; + } + + /// Lose every datagram this side sends from now on. + pub fn lose_datagrams(&self) { + *self.side.lossy.lock().unwrap() = true; + } } impl MockSession { @@ -590,6 +617,8 @@ pub fn create_mock_session_pair(protocol: Option<&'static str>) -> (MockSession, protocol, conn: conn.clone(), held: Mutex::default(), + withheld: Mutex::default(), + lossy: Mutex::default(), }); let server_side = Arc::new(SessionSide { @@ -602,6 +631,8 @@ pub fn create_mock_session_pair(protocol: Option<&'static str>) -> (MockSession, protocol, conn, held: Mutex::default(), + withheld: Mutex::default(), + lossy: Mutex::default(), }); let new = |side| MockSession { @@ -610,6 +641,7 @@ pub fn create_mock_session_pair(protocol: Option<&'static str>) -> (MockSession, accept_bi: kio::Park::default(), datagram: kio::Park::default(), closed: kio::Park::default(), + open_uni: kio::Park::default(), }; (new(client_side), new(server_side)) diff --git a/rs/moq-net/tests/track_tail.rs b/rs/moq-net/tests/track_tail.rs index a2e21403bf9..71857fc61b7 100644 --- a/rs/moq-net/tests/track_tail.rs +++ b/rs/moq-net/tests/track_tail.rs @@ -56,12 +56,22 @@ struct Outcome { elapsed: Duration, } -/// Publish a group below the declared end (or none for an empty track), hold its stream -/// back until the subscription ends, then deliver or lose it. -async fn round(version: &str, late: Late, final_sequence: u64) -> Outcome { +/// A publisher and subscriber over the mock, with the subscriber holding `bcast`. +struct Pair { + pair: support::harness::MockPair, + track: moq_net::track::Producer, + remote: moq_net::broadcast::Consumer, + _keep: ( + moq_net::broadcast::Producer, + moq_net::origin::Producer, + moq_net::origin::Producer, + ), +} + +async fn connect(version: &str) -> Pair { let publisher = produce_origin(1); let broadcast = publisher.create_broadcast("bcast").unwrap(); - let mut track = broadcast.create_track("video", None).unwrap(); + let track = broadcast.create_track("video", None).unwrap(); broadcast.announce(Default::default()).unwrap(); let subscriber = produce_origin(2); @@ -80,6 +90,24 @@ async fn round(version: &str, late: Late, final_sequence: u64) -> Outcome { .expect("resolve timeout") .expect("broadcast resolves"); + Pair { + pair, + track, + remote, + _keep: (broadcast, publisher, subscriber), + } +} + +/// Publish a group below the declared end (or none for an empty track), hold its stream +/// back until the subscription ends, then deliver or lose it. +async fn round(version: &str, late: Late, final_sequence: u64) -> Outcome { + let Pair { + pair, + mut track, + remote, + _keep, + } = connect(version).await; + let reader = tokio::spawn(async move { let subscription = moq_net::track::Subscription::default().with_start(moq_net::track::Position::group(0)); let mut sub = remote @@ -140,7 +168,7 @@ async fn round(version: &str, late: Late, final_sequence: u64) -> Outcome { .await .expect("the subscription never ended") .expect("reader panicked"); - drop((pair, broadcast, publisher, subscriber)); + drop((pair, _keep)); Outcome { frames, err, @@ -214,3 +242,128 @@ async fn lite07_zero_streams_end_without_the_grace() { assert!(outcome.frames.is_empty()); assert!(outcome.elapsed < GRACE / 10, "ended after {:?}", outcome.elapsed); } + +/// IETF drafts over the control stream adapter (14), on their own streams (17), and with +/// subscription fills (20+). +const IETF: &[&str] = &[ + "moq-transport-14", + "moq-transport-17", + "moq-transport-20", + "moq-transport-22", +]; + +/// A subscriber that leaves while END_OF_TRACK waits for stream credit ends the publisher's +/// request, instead of parking it until credit that may never come. +#[tokio::test] +async fn ietf_leaving_cancels_a_blocked_end_of_track() { + tokio::time::pause(); + for version in IETF { + let Pair { + pair, + track, + remote, + _keep, + } = connect(version).await; + + let subscription = moq_net::track::Subscription::default().with_start(moq_net::track::Position::group(0)); + let mut sub = remote + .track("video") + .unwrap() + .subscribe(subscription) + .await + .expect("subscribe"); + tokio::time::timeout(TIMEOUT, track.used()) + .await + .expect("no subscriber appeared") + .unwrap(); + + let mut group = track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, PAYLOAD).unwrap(); + group.finish().unwrap(); + let mut group = tokio::time::timeout(TIMEOUT, sub.recv_group()) + .await + .expect("group timeout") + .unwrap() + .expect("a group"); + while group.read_frame().await.unwrap().is_some() {} + + // Out of stream credit, so the marker cannot open. + pair.server_transport.withhold_unis(); + track.finish().unwrap(); + tokio::time::sleep(GRACE / 10).await; + assert!( + track.subscription().is_some(), + "{version}: the publisher is still ending the request" + ); + + // The request task holds the publisher's subscription until it ends. + drop((group, sub)); + tokio::time::timeout(TIMEOUT, async { + while track.subscription().is_some() { + tokio::time::sleep(GRACE / 100).await; + } + }) + .await + .unwrap_or_else(|_| panic!("{version}: the publisher's request never ended")); + drop((pair, _keep)); + } +} + +/// A lost datagram is not owed. Nothing tells the subscriber its sequence was a datagram, so +/// on the drafts that account for the owed range its hole keeps the subscription routable for +/// the grace, like a stream reset before its header. A reader never waits on it: the track +/// ends for readers once the live edge reaches the declared end. +#[tokio::test] +async fn a_lost_datagram_never_delays_the_end() { + tokio::time::pause(); + for version in ["moq-lite-05", "moq-lite-07-wip"] { + let Pair { + pair, + mut track, + remote, + _keep, + } = connect(version).await; + + let reader = tokio::spawn(async move { + let subscription = moq_net::track::Subscription::default().with_start(moq_net::track::Position::group(0)); + let mut sub = remote + .track("video") + .unwrap() + .subscribe(subscription) + .await + .expect("subscribe"); + let mut groups = Vec::new(); + while let Some(group) = sub.recv_group().await.expect("track aborted") { + groups.push(group.sequence); + } + (groups, tokio::time::Instant::now()) + }); + tokio::time::timeout(TIMEOUT, track.used()) + .await + .expect("no subscriber appeared") + .unwrap(); + + for datagram in [false, true, false] { + if datagram { + pair.server_transport.lose_datagrams(); + track.append_datagram(Timestamp::ZERO, PAYLOAD).unwrap(); + } else { + let mut group = track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, PAYLOAD).unwrap(); + group.finish().unwrap(); + } + tokio::time::sleep(GRACE / 100).await; + } + let finished = tokio::time::Instant::now(); + track.finish().unwrap(); + + let (groups, ended) = tokio::time::timeout(TIMEOUT, reader) + .await + .expect("the subscription never ended") + .expect("reader panicked"); + assert_eq!(groups, [0, 2], "{version}"); + let elapsed = ended - finished; + assert!(elapsed < GRACE / 10, "{version}: ended after {elapsed:?}"); + drop((pair, _keep)); + } +} From 75df2bf75fc5c6467859c94b1cb779a363f3b5e7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 12:07:27 -0700 Subject: [PATCH 3/8] fix(net): harden the JS track tail to match Rust A group at or past the declared end aborts the track, the lite owed floor follows the demand with the groups below SUBSCRIBE_START accounted, and tail gaps older than the grace fold away. Finishes the track tail hardening quest. Co-Authored-By: Claude Opus 5.5 --- doc/concept/moq-lite.md | 10 ++-- js/net/src/ietf/subscriber.ts | 13 ++++- js/net/src/ietf/tail.test.ts | 14 +++++ js/net/src/lite/subscriber.ts | 41 +++++++++++--- js/net/src/lite/tail.test.ts | 69 +++++++++++++++++++++++- js/net/src/tail.test.ts | 59 ++++++++++++++++++++- js/net/src/tail.ts | 91 ++++++++++++++++++++++++-------- quest/m1/README.md | 1 - quest/m1/session-death.md | 1 - quest/m1/subscribe-drop.md | 1 - quest/m1/track-tail-hardening.md | 79 --------------------------- 11 files changed, 258 insertions(+), 121 deletions(-) delete mode 100644 quest/m1/track-tail-hardening.md diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index c2947245ed4..5f47e803dc4 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -38,14 +38,16 @@ sides explicitly enable it. On moq-lite 07, `SUBSCRIBE_END` counts the group streams opened for the subscription. Rust and TypeScript stop waiting for missing streams once that -many headers have arrived; skipped group sequences add no wait. Groups already -being received continue until their own stream ends or resets. +many headers have arrived; skipped group sequences add no wait. A stream whose +header arrived is always read to its end before the subscription completes. A stream reset before its header arrived cannot be counted, so the subscriber still allows a grace period for late streams. The grace uses the subscription's nonzero effective maximum age, or one second when no maximum age is set. -moq-lite 05 and 06 instead account for group sequences using received headers -and `SUBSCRIBE_DROP`. +moq-lite 05 and 06 instead account for group sequences using received headers, +datagrams, and `SUBSCRIBE_DROP`, from the start group the subscription last +asked for. A lost datagram is not owed, but its hole waits out the grace like a +lost stream. A group at or past the declared end aborts the track. ## Discovery diff --git a/js/net/src/ietf/subscriber.ts b/js/net/src/ietf/subscriber.ts index 9b9833d15ee..69dcdb0dce3 100644 --- a/js/net/src/ietf/subscriber.ts +++ b/js/net/src/ietf/subscriber.ts @@ -8,7 +8,7 @@ import { Cost, type Route, routesEqual, UNKNOWN_HOP } from "../hop.ts"; import { hiddenBelow, hooks, scopeCaptures, scopeHead, scopeOverlaps } from "../internal.ts"; import * as Path from "../path.ts"; import type { Cursor, Reader, Stream } from "../stream.ts"; -import { TAIL_GRACE_MS, Tail } from "../tail.ts"; +import { Tail } from "../tail.ts"; import { type Timescale, Timestamp } from "../time.ts"; import type * as track from "../track.ts"; import { TimeoutError, withTimeout } from "../util/timeout.ts"; @@ -52,6 +52,8 @@ type Subscription = { // The group streams received, so the subscription can wait for the ones PUBLISH_DONE // says are still owed. tail: Tail; + // The track's exclusive end, once an END_OF_TRACK declares it. + end?: number; }; // Out-parameter for #openSubscribe: lets the caller observe partial progress @@ -674,7 +676,7 @@ export class Subscriber { const { tail, track } = subscription; const complete = () => count > 0n && BigInt(tail.streams) >= count; - await tail.settle(complete, TAIL_GRACE_MS, track.closed); + await tail.settle(complete, track.closed); } /** @@ -1018,6 +1020,12 @@ export class Subscriber { let producer: netGroup.Producer | undefined; const open = () => { if (!producer) { + // The publisher contradicted its own end, which no later group can repair. + if (subscription.end !== undefined && group.groupId >= subscription.end) { + throw new ProtocolViolation( + `group ${group.groupId} is at or past the declared end ${subscription.end}`, + ); + } producer = new netGroup.Producer(group.groupId); track.writeGroup(producer); } @@ -1068,6 +1076,7 @@ export class Subscriber { } catch (err: unknown) { throw new ProtocolViolation(`invalid END_OF_TRACK: ${reason(error(err))}`); } + subscription.end ??= end; return; } if (frame.payload === undefined) break; diff --git a/js/net/src/ietf/tail.test.ts b/js/net/src/ietf/tail.test.ts index da233686e77..3f43b53dc92 100644 --- a/js/net/src/ietf/tail.test.ts +++ b/js/net/src/ietf/tail.test.ts @@ -178,6 +178,20 @@ test("END_OF_TRACK at object 0 ends the track before its group, which never exis expect(reader.final()).toBe(2); }); +test("a group at or past the END_OF_TRACK aborts the track", async () => { + const { subscriber, reader } = await subscribed(); + const end = groupStream(subscriber, 2); + end.write(END_OF_TRACK); + end.finish(); + expect(await reader.finished()).toBe(2); + + const late = groupStream(subscriber, 2); + late.write(object("2.0")); + late.finish(); + await late.handled; + expect(await reader.closed).toBeInstanceOf(ProtocolViolation); +}); + test("a bare FIN without PUBLISH_DONE aborts the track", async () => { const { reader, fin } = await subscribed(); await fin(); diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 61c3e7aaf25..fe3d85bafe3 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -82,6 +82,8 @@ interface SubscribeEntry { // The group streams received, so the subscription can wait for the ones still owed // after the publisher ends it. tail: Tail; + // The first group the SUBSCRIBE asked for, if any. + requested?: number; // The first group the publisher serves (SUBSCRIBE_START) and the track's exclusive end // (SUBSCRIBE_END), once it declares them. start?: number; @@ -652,7 +654,20 @@ export class Subscriber { } // Register before opening SUBSCRIBE so a racing GROUP stream finds the entry. - const entry: SubscribeEntry = { track: producer, timescale, tail: new Tail() }; + const entry: SubscribeEntry = { + track: producer, + timescale, + // The effective max age is the stopgap grace: the wrong clock (it bounds + // presentation-time drift), but it is how long the subscriber was willing to wait + // for a late group anyway. Already the smaller of the subscriber's and the track's. + tail: new Tail({ + grace: () => { + const maxAge = producer.subscription.peek()?.maxAge ?? Time.Milli.zero; + return maxAge > 0 ? maxAge : TAIL_GRACE_MS; + }, + }), + requested: msg.startGroup, + }; this.#subscribes.set(id, entry); state.stream = await Stream.open(this.#quic); @@ -872,6 +887,9 @@ export class Subscriber { if ("start" in resp) { entry.start = resp.start.group; + // The groups the SUBSCRIBE asked for below it are unavailable, whatever the + // demand asks later. + if (entry.requested !== undefined) entry.tail.account(entry.requested, entry.start); } else if ("end" in resp) { if (entry.end !== undefined) throw new ProtocolViolation("duplicate SUBSCRIBE_END"); entry.end = resp.end.group; @@ -897,9 +915,6 @@ export class Subscriber { // headers arrived keep reading until their own FIN or reset. #settleTail(entry: SubscribeEntry): Promise { const { tail, track } = entry; - // Already the smaller of the subscriber's and the track's max age. - const maxAge = track.subscription.peek()?.maxAge ?? Time.Milli.zero; - const grace = maxAge > 0 ? maxAge : TAIL_GRACE_MS; const complete = () => { if (entry.streams !== undefined) return tail.streams >= entry.streams; @@ -907,13 +922,17 @@ export class Subscriber { if (entry.end === undefined) return false; // Without SUBSCRIBE_START the publisher served no group at all. if (entry.start === undefined) return true; - const bounds = groupBounds(track.subscription.peek()?.groups ?? {}); - const start = Math.max(entry.start, bounds.start); + // Owed from the floor the demand last asked for, which an update can move either + // way, or where SUBSCRIBE_START resolved a live-edge one. The groups the SUBSCRIBE + // asked for below its SUBSCRIBE_START were accounted for when it arrived. + const groups = track.subscription.peek()?.groups; + const bounds = groupBounds(groups ?? {}); + const start = groups?.start === undefined ? entry.start : bounds.start; const end = bounds.end === undefined ? entry.end : Math.min(entry.end, bounds.end); return tail.covers(start, end); }; - return tail.settle(complete, grace, track.closed); + return tail.settle(complete, track.closed); } /** @@ -1005,6 +1024,14 @@ export class Subscriber { const read = tail.open(group.sequence); try { + // The publisher contradicted its own end, which no later group can repair. + if (entry.end !== undefined && group.sequence >= entry.end) { + const violation = new ProtocolViolation( + `group ${group.sequence} is at or past the declared end ${entry.end}`, + ); + track.close(violation); + throw violation; + } track.writeGroup(producer); // Block until the timescale is known; the group's stream can arrive before diff --git a/js/net/src/lite/tail.test.ts b/js/net/src/lite/tail.test.ts index 245791287da..852d52ee1d5 100644 --- a/js/net/src/lite/tail.test.ts +++ b/js/net/src/lite/tail.test.ts @@ -6,6 +6,7 @@ import { createMockTransportPair } from "../mock.ts"; import * as Path from "../path.ts"; import { Reader, Stream } from "../stream.ts"; import { Milli } from "../time.ts"; +import type { Groups } from "../track.ts"; import { Group as GroupMessage } from "./group.ts"; import { StreamId } from "./stream.ts"; import { @@ -51,10 +52,10 @@ function groupStream(subscriber: Subscriber, sequence: number) { * it answers TRACK_INFO, then writes whatever responses the test asks for on the subscribe * stream and FINs it when told. */ -async function subscribed(version: Version, maxAge = GRACE) { +async function subscribed(version: Version, maxAge = GRACE, groups?: Groups) { const pair = createMockTransportPair(ALPN_05); const subscriber = new Subscriber(pair.client, version, randomHop()); - const reader = subscriber.consume(Path.from("room")).track("video").subscribe({ maxAge }); + const reader = subscriber.consume(Path.from("room")).track("video").subscribe({ maxAge, groups }); const info = await Stream.accept(pair.server); if (!info) throw new Error("the subscriber never asked for TRACK_INFO"); @@ -210,6 +211,70 @@ describe.each([Version.DRAFT_05, Version.DRAFT_06, Version.DRAFT_07])("%s", (ver expect(reader.final()).toBe(0); }); + test("a group at or past SUBSCRIBE_END aborts the track", async () => { + const { subscriber, reader, respond } = await subscribed(version); + await respond({ start: new SubscribeStart(0) }); + await respond({ end: new SubscribeEnd(2, 2) }); + expect(await reader.finished()).toBe(2); + + await groupStream(subscriber, 2).handled; + expect(await reader.closed).toBeInstanceOf(ProtocolViolation); + }); + + test("a floor below SUBSCRIBE_START owes nothing below it", async () => { + const { subscriber, reader, respond, fin } = await subscribed(version, Milli(60_000), { + start: { included: 1 }, + }); + await respond({ start: new SubscribeStart(3) }); + const group = groupStream(subscriber, 3); + group.finish(); + await group.handled; + await respond({ end: new SubscribeEnd(4, 1) }); + await fin(); + + expect((await reader.recvGroup())?.sequence).toBe(3); + expect(await settlesWithin(reader.recvGroup(), 1000)).toBe(true); + expect(await reader.closed).toBeNull(); + }); + + test.skipIf(version === Version.DRAFT_07)("a lowered floor owes the groups it newly asked for", async () => { + const maxAge = Milli(60_000); + const { subscriber, respond, fin } = await subscribed(version, maxAge, { start: { included: 3 } }); + await respond({ start: new SubscribeStart(3) }); + const group = groupStream(subscriber, 3); + group.write("3.0"); + group.finish(); + await group.handled; + + // A second reader lowers the floor, which the subscription forwards as an update. + const lower = subscriber + .consume(Path.from("room")) + .track("video") + .subscribe({ maxAge, groups: { start: { included: 1 } } }); + await respond({ end: new SubscribeEnd(4, 1) }); + await fin(); + // Long enough for a subscription that owed nothing more to have ended. + await new Promise((resolve) => setTimeout(resolve, 50)); + + // QUIC does not order streams, so the lowered groups land after the FIN. + for (const sequence of [1, 2]) { + const late = groupStream(subscriber, sequence); + late.write(`${sequence}.0`); + late.finish(); + await late.handled; + } + + const received: number[] = []; + for (;;) { + const next = await lower.recvGroup(); + if (!next) break; + received.push(next.sequence); + } + expect(received).toContain(1); + expect(received).toContain(2); + expect(await lower.closed).toBeNull(); + }); + test("a SUBSCRIBE_END below a group already received aborts the track", async () => { const { subscriber, reader, respond } = await subscribed(version); await respond({ start: new SubscribeStart(0) }); diff --git a/js/net/src/tail.test.ts b/js/net/src/tail.test.ts index 9daa4eaa56e..e40fefb8ab2 100644 --- a/js/net/src/tail.test.ts +++ b/js/net/src/tail.test.ts @@ -1,11 +1,12 @@ -import { expect, test } from "bun:test"; +import { describe, expect, test } from "bun:test"; +import { Once } from "@moq/signals"; import { accept, connect } from "./connection/index.ts"; import * as Ietf from "./ietf/index.ts"; import * as Lite from "./lite/index.ts"; import { createMockTransportPair } from "./mock.ts"; import { Producer as OriginProducer } from "./origin.ts"; import * as Path from "./path.ts"; -import { TAIL_GRACE_MS } from "./tail.ts"; +import { TAIL_GRACE_MS, Tail } from "./tail.ts"; import { Milli } from "./time.ts"; import type { Ordered } from "./track.ts"; import { wireOf } from "./wire.ts"; @@ -103,3 +104,57 @@ test.each([Ietf.ALPN.DRAFT_16, Ietf.ALPN.DRAFT_17, Ietf.ALPN.DRAFT_20])( } }, ); + +describe("Tail", () => { + const GRACE = Milli(1000); + + function tail() { + const clock = { now: 0 }; + return { clock, tail: new Tail({ grace: () => GRACE, now: () => clock.now }) }; + } + + test("a gap older than the grace folds away", () => { + const { clock, tail: t } = tail(); + t.account(0, 1); + t.account(2, 3); + clock.now = GRACE / 2; + t.account(4, 5); + expect(t.covers(0, 3)).toBe(false); + + // Splitting the younger gap keeps it as late as it was. + t.account(6, 7); + clock.now = GRACE; + t.account(8, 9); + expect(t.covers(0, 3)).toBe(true); + expect(t.covers(0, 5)).toBe(false); + }); + + test("splitting a gap leaves both halves with its age", () => { + const { clock, tail: t } = tail(); + t.account(0, 1); + t.account(9, 10); + clock.now = GRACE / 2; + t.account(5, 6); + expect(t.covers(0, 10)).toBe(false); + + clock.now = GRACE; + t.account(9, 10); + expect(t.covers(0, 10)).toBe(true); + }); + + test("the grace waits for a stream being read", async () => { + const t = new Tail({ grace: () => Milli(10) }); + const read = t.open(0); + let settled = false; + const settling = t + .settle(() => false, new Once()) + .then(() => { + settled = true; + }); + + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(settled).toBe(false); + read(); + await settling; + }); +}); diff --git a/js/net/src/tail.ts b/js/net/src/tail.ts index e516f1ea089..178c50d002f 100644 --- a/js/net/src/tail.ts +++ b/js/net/src/tail.ts @@ -14,6 +14,18 @@ export const TAIL_GRACE_MS = Milli(1000); // setTimeout truncates a longer delay to a signed 32-bit int and fires at once. const MAX_TIMEOUT_MS = 2 ** 31 - 1; +// A run of accounted sequences `[start, end)`, and when the gap below it opened (in +// `performance.now()` milliseconds), which is when its missing groups became late. +type Run = { start: number; end: number; since: number }; + +/** How a {@link Tail} measures its grace. */ +export interface TailOptions { + /** How long to wait for a group that cannot be accounted for. Read each time it is needed. */ + grace?: () => Milli; + /** The clock, in `performance.now()` milliseconds. */ + now?: () => number; +} + /** * The group streams a subscription has received, so its end can wait for the ones still owed. * @@ -22,16 +34,27 @@ const MAX_TIMEOUT_MS = 2 ** 31 - 1; * the publisher dropped them) and how many streams are still being read, and {@link settle} * waits on them. * + * A lost datagram is not owed: it leaves a hole that waits out the grace like a stream reset + * before its header. + * * @internal */ export class Tail { - // Disjoint, sorted, exclusive-end ranges of accounted sequences. A gap splits a range, so - // this stays as small as the number of gaps rather than the number of groups. - #accounted: [number, number][] = []; + // Disjoint, sorted, non-adjacent runs of accounted sequences. A gap older than the grace + // can no longer be waited for, so it folds into the runs around it, which bounds this by + // the gaps opened within the grace rather than every gap in the subscription. + #runs: Run[] = []; // Group streams whose header arrived, and those still being read. #streams = 0; #active = 0; #changed = new Signal(0); + #grace: () => Milli; + #now: () => number; + + constructor(options: TailOptions = {}) { + this.#grace = options.grace ?? (() => TAIL_GRACE_MS); + this.#now = options.now ?? (() => performance.now()); + } /** Group streams whose header arrived, whether they finished or were reset. */ get streams(): number { @@ -64,26 +87,29 @@ export class Tail { /** Whether every sequence in `[start, end)` is accounted for. */ covers(start: number, end: number): boolean { if (start >= end) return true; - // Ranges are merged on insert, so one range covers the span or none does. - return this.#accounted.some(([lo, hi]) => lo <= start && end <= hi); + // Runs are merged on insert, so one run covers the span or none does. + return this.#runs.some((run) => run.start <= start && end <= run.end); } /** - * Wait until every stream is read to its end and `complete()` holds, or `grace` - * milliseconds pass with nothing left being read, or `closed` settles. + * Wait until every stream is read to its end and `complete()` holds, or the grace passes + * with nothing left being read, or `closed` settles. * * A stream still being read is always waited for: a group ends on its own stream's FIN * or reset, never because its track ended. The grace only gives up on streams that never * arrived. */ - async settle(complete: () => boolean, grace: Milli, closed: GetPromise): Promise { + async settle(complete: () => boolean, closed: GetPromise): Promise { + // A gap that aged past the grace since the last insert is no longer waited for either. + this.#expire(this.#now()); + let expired = false; const timer = setTimeout( () => { expired = true; this.#bump(); }, - Math.min(grace, MAX_TIMEOUT_MS), + Math.min(this.#grace(), MAX_TIMEOUT_MS), ); try { while (closed.peek() === undefined) { @@ -98,27 +124,48 @@ export class Tail { #account(start: number, end: number): void { if (start >= end) return; - const merged: [number, number][] = []; - let lo = start; - let hi = end; + const now = this.#now(); + const runs: Run[] = []; + let merged: Run | undefined; let placed = false; - for (const range of this.#accounted) { - if (range[1] < lo) { - merged.push(range); - } else if (hi < range[0]) { - if (!placed) merged.push([lo, hi]); + for (const run of this.#runs) { + if (run.end < start) { + runs.push(run); + } else if (end < run.start) { + // Splitting a gap leaves both halves as late as it was. + if (!placed) runs.push(merged ?? { start, end, since: run.since }); placed = true; - merged.push(range); + runs.push(run); } else { - lo = Math.min(lo, range[0]); - hi = Math.max(hi, range[1]); + merged = { + start: Math.min(merged?.start ?? start, run.start), + end: Math.max(merged?.end ?? end, run.end), + since: merged?.since ?? run.since, + }; } } - if (!placed) merged.push([lo, hi]); - this.#accounted = merged; + // A run past every other one opens a new gap. + if (!placed) runs.push(merged ?? { start, end, since: now }); + this.#runs = runs; + this.#expire(now); this.#bump(); } + // Fold every gap older than the grace into the runs around it. + #expire(now: number): void { + const grace = this.#grace(); + const runs: Run[] = []; + for (const run of this.#runs) { + const below = runs.at(-1); + if (below && now - run.since >= grace) { + runs[runs.length - 1] = { ...below, end: run.end }; + } else { + runs.push(run); + } + } + this.#runs = runs; + } + #bump(): void { this.#changed.update((revision) => revision + 1); } diff --git a/quest/m1/README.md b/quest/m1/README.md index 55b4f0a17f6..2ae909d46a0 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -33,7 +33,6 @@ transport, benchmark tooling); worktrees isolate commits, not semantics. - [Watch refusal](/quest/m1/watch-refusal.md) - `` shows an origin refusal as an error instead of sitting offline - [Splice edge cases](/quest/m1/splice-edges.md) - an unstamped successor, a pruned segment's boundary group, and a warm head during a takeover are each handled correctly - [Resumed groups](/quest/m1/resume-latest.md) - a half-delivered group ends once the new copy is past it, so a group-only reader never parks after a mid-group failover -- [Track tail hardening](/quest/m1/track-tail-hardening.md) - Rust and JS wait out a track's tail by the same rules, with the known hang, count, truncation, grace, and memory holes closed - [JS close end](/quest/m1/js-track-close-end.md) - a JS track's clean close ends at its own last group, not a sibling producer's - [SUBSCRIBE_DROP](/quest/m1/subscribe-drop.md) - every stream group in a lite subscription arrives or is dropped by name, and lite-07 drops its stream count for it - [FIN wait expiry](/quest/m1/fin-wait-expiry.md) - a group awaiting its FIN ack still expires and follows priority updates on lite and IETF diff --git a/quest/m1/session-death.md b/quest/m1/session-death.md index 0eec5aa84ae..af67ddd7151 100644 --- a/quest/m1/session-death.md +++ b/quest/m1/session-death.md @@ -43,5 +43,4 @@ clean end needs a public method. ## Related -- [Track tail hardening](/quest/m1/track-tail-hardening.md) - the tail rules this must not mask - [Graceful session close](/quest/m1/session-close.md) - what a local close sends the peer diff --git a/quest/m1/subscribe-drop.md b/quest/m1/subscribe-drop.md index 3f93210d63d..a12a865a6a1 100644 --- a/quest/m1/subscribe-drop.md +++ b/quest/m1/subscribe-drop.md @@ -46,5 +46,4 @@ without waiting out the grace. ## Related -- [Track tail hardening](/quest/m1/track-tail-hardening.md) - the same tail accounting, in both languages - [Track tail interop](/quest/m1/track-tail-interop.md) - the Rust-JS proof of the lite-07 drop case diff --git a/quest/m1/track-tail-hardening.md b/quest/m1/track-tail-hardening.md deleted file mode 100644 index 74d412bd042..00000000000 --- a/quest/m1/track-tail-hardening.md +++ /dev/null @@ -1,79 +0,0 @@ -# [M] Track tail hardening - -## Goal - -moq-net and `@moq/net` wait out a subscription's tail by the same rules, and -the known holes in those rules are closed. A finished IETF subscription never -hangs its request task or under-reports its stream count, a group whose -header arrived is never silently dropped from a clean end, the grace never -cuts off a stream still being read, and tail bookkeeping stays bounded on a -lossy track. - -## Plan - -Track tail landed in https://github.com/moq-dev/moq/pull/4086 (JS) and -https://github.com/moq-dev/moq/pull/4116 (Rust), and Codex's findings on both -were left for this pass. Fix each in the language named and check whether -the other has the same hole. - -Bugs: - -- **Rust IETF publisher, END_OF_TRACK not cancellable.** After the group - streams drain, `write_end_of_track` in `rs/moq-net/src/ietf/publisher.rs` - runs outside the race against `stream.reader.poll_closed` and the session, - so with uni-stream credit exhausted an unsubscribe leaves the request task - parked forever, never sending PUBLISH_DONE. -- **Rust IETF publisher, stream count.** END_OF_TRACK is counted only if the - whole write succeeds, while every other stream counts once opened. A reset - END_OF_TRACK whose header still arrives can then land after the subscriber - met the count and retired the alias. -- **Rust IETF subscriber, truncated first object.** `open_group` peeks the - first object with `?` before creating the group, so a stream reset or - truncated after its header drops that group, and since the stream still - counts, the track can end clean without it. Keep the END_OF_TRACK peek, but - create and abort the named group on any other peek failure. -- **Rust grace cuts off an arrived stream.** `Settle::poll` in - `rs/moq-net/src/tail.rs` returns when the grace fires even if an - END_OF_TRACK stream is mid-read, so the track finishes at the live edge and - the later marker is ignored. JS's `Tail.settle` already waits for active - streams; do the same. -- **JS lite floor.** `Math.max(entry.start, bounds.start)` in - `js/net/src/lite/subscriber.ts` ignores a SUBSCRIBE_UPDATE that lowered the - floor after SUBSCRIBE_START, so a newly requested lower group reordered - behind the FIN is dropped. `bounds.start` alone is wrong too: a subscriber - that asked below the announced start would wait for groups never promised. - The owed start has to remember the floor each request was answered at. - Check whether Rust's `SubStream::owed` has the same shape. - -Open questions. The maintainer chose to plan these without settling them; -decide in the PR, applying the choice to both languages: - -- **Are lost datagrams owed before a track ends?** Both `Tail`s account a - datagram only when it arrives, so a lost one leaves a hole in the owed span - and the end waits out the whole grace (JS was reported to wait and Rust - not, but Rust's `covers(owed)` reads the same way despite the comment in - `route_datagram`; confirm with a test first). Recommendation: no, datagrams - are best effort. [SUBSCRIBE_DROP](/quest/m1/subscribe-drop.md) decided - the same: datagram groups are not owed, so an uncovered hole waits out the - grace on every version. -- **Does a group at or past the declared end abort the track, or only that - group?** Rust aborts the track with `ProtocolViolation`; JS aborts the group - and ends clean. Recommendation: abort the track in both. The peer - contradicted its own end, and the repo fails loud on malformed input. -- **How is tail memory bounded?** Rust `Tail.accounted` gains a range per - permanent gap for the life of the subscription, and so does JS's. A gap - older than the grace can no longer be waited for, so folding it in as - accounted loses nothing. Recommendation: that, which bounds the ranges by - the gaps inside the grace window. Once publishers send SUBSCRIBE_DROP for - every gap, only datagram holes remain. - -Regression tests go in `rs/moq-net/tests/track_tail.rs` (its `hold_unis` -mock makes the reorder deterministic) and the JS tail tests. Wire output only -changes if the stream-count fix does, and that is a correction to what the -drafts already require. - -## Related - -- [Track tail interop](/quest/m1/track-tail-interop.md) - the Rust-JS check that both sides now agree -- [SUBSCRIBE_DROP](/quest/m1/subscribe-drop.md) - every group is a stream or a drop, replacing lite-07's stream count -- [Session death](/quest/m1/session-death.md) - how a tail ends when the session dies under it From eba9e2b9ad4b6fd73a0e34b1c633f1dbc4be4b99 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 13:14:01 -0700 Subject: [PATCH 4/8] fix(net): abort on a contradicted lite end from lite-06 A SUBSCRIBE_END below a received group, or a group at or past the declared end, aborts the track on lite-06 and later in both languages. lite-05 specified an inclusive end, and @moq/net 0.1.3 to 0.1.9 sent one, so there it only warns or drops that group. Co-Authored-By: Claude Opus 5.5 --- doc/concept/moq-lite.md | 7 +- js/net/src/lite/subscriber.ts | 14 ++- js/net/src/lite/tail.test.ts | 23 +++- rs/moq-net/src/lite/subscriber.rs | 182 ++++++++++++++++++++++-------- 4 files changed, 171 insertions(+), 55 deletions(-) diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index 5f47e803dc4..e5b133ca693 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -47,7 +47,12 @@ nonzero effective maximum age, or one second when no maximum age is set. moq-lite 05 and 06 instead account for group sequences using received headers, datagrams, and `SUBSCRIBE_DROP`, from the start group the subscription last asked for. A lost datagram is not owed, but its hole waits out the grace like a -lost stream. A group at or past the declared end aborts the track. +lost stream. + +From moq-lite 06, an end that contradicts the groups received aborts the track: +a group at or past `SUBSCRIBE_END`, or a `SUBSCRIBE_END` below a group already +received. moq-lite 05 specified an inclusive end, so there it only drops that +group or the early boundary. ## Discovery diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index fe3d85bafe3..cc07c24f17b 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -899,7 +899,13 @@ export class Subscriber { try { entry.track.finishAt(entry.end); } catch (err) { - throw new ProtocolViolation(`invalid SUBSCRIBE_END: ${reason(error(err))}`); + // lite-05 specified an inclusive end, and @moq/net 0.1.3 to 0.1.9 sent one, so + // there an end below a received group only costs the early boundary: the FIN + // still finishes the track. Later drafts made it exclusive. + if (this.version !== Version.DRAFT_05) { + throw new ProtocolViolation(`invalid SUBSCRIBE_END: ${reason(error(err))}`); + } + console.warn(`invalid SUBSCRIBE_END: ${reason(error(err))}`); } } else if ("drop" in resp) { entry.tail.account(resp.drop.start, resp.drop.end + 1); @@ -1024,8 +1030,10 @@ export class Subscriber { const read = tail.open(group.sequence); try { - // The publisher contradicted its own end, which no later group can repair. - if (entry.end !== undefined && group.sequence >= entry.end) { + // The publisher contradicted its own end, which no later group can repair. lite-05 + // specified an inclusive end, so its last group lands on it: the write below drops + // only that group there. + if (entry.end !== undefined && group.sequence >= entry.end && this.version !== Version.DRAFT_05) { const violation = new ProtocolViolation( `group ${group.sequence} is at or past the declared end ${entry.end}`, ); diff --git a/js/net/src/lite/tail.test.ts b/js/net/src/lite/tail.test.ts index 852d52ee1d5..87e0a9a474a 100644 --- a/js/net/src/lite/tail.test.ts +++ b/js/net/src/lite/tail.test.ts @@ -211,14 +211,19 @@ describe.each([Version.DRAFT_05, Version.DRAFT_06, Version.DRAFT_07])("%s", (ver expect(reader.final()).toBe(0); }); - test("a group at or past SUBSCRIBE_END aborts the track", async () => { + // lite-05 specified an inclusive end, so there the group costs only its own stream. + test(`a group at or past SUBSCRIBE_END ${version === Version.DRAFT_05 ? "is dropped" : "aborts the track"}`, async () => { const { subscriber, reader, respond } = await subscribed(version); await respond({ start: new SubscribeStart(0) }); await respond({ end: new SubscribeEnd(2, 2) }); expect(await reader.finished()).toBe(2); await groupStream(subscriber, 2).handled; - expect(await reader.closed).toBeInstanceOf(ProtocolViolation); + if (version === Version.DRAFT_05) { + expect(await settlesWithin(Promise.resolve(reader.closed), 50)).toBe(false); + } else { + expect(await reader.closed).toBeInstanceOf(ProtocolViolation); + } }); test("a floor below SUBSCRIBE_START owes nothing below it", async () => { @@ -275,16 +280,22 @@ describe.each([Version.DRAFT_05, Version.DRAFT_06, Version.DRAFT_07])("%s", (ver expect(await lower.closed).toBeNull(); }); - test("a SUBSCRIBE_END below a group already received aborts the track", async () => { - const { subscriber, reader, respond } = await subscribed(version); + // lite-05 specified an inclusive end, and @moq/net 0.1.3 to 0.1.9 sent one, so there an end + // below a received group only costs the early boundary. + test(`a SUBSCRIBE_END below a group already received ${version === Version.DRAFT_05 ? "finishes clean" : "aborts the track"}`, async () => { + const { subscriber, reader, respond, fin } = await subscribed(version); await respond({ start: new SubscribeStart(0) }); const group = groupStream(subscriber, 3); group.finish(); await group.handled; await respond({ end: new SubscribeEnd(2, 2) }); - const closed = await reader.closed; - expect(closed).toBeInstanceOf(Error); + if (version === Version.DRAFT_05) { + await fin(); + expect(await reader.closed).toBeNull(); + } else { + expect(await reader.closed).toBeInstanceOf(ProtocolViolation); + } }); }); diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 2d5081515ff..b393f33ac22 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -773,8 +773,17 @@ impl GroupRecv { let mut group = match entry.producer.create_group(group_info) { Ok(group) => group, // The group is at or past the end the publisher declared, which no - // later stream can repair. - Err(Error::Closed) => { + // later stream can repair. Before lite-05 only a local finish sets + // an end, and lite-05 specified an inclusive one, so its last + // group lands on it: drop only that stream there. + Err(Error::Closed) + if !matches!( + self.subscriber.version, + Version::Lite01 + | Version::Lite02 | Version::Lite03 + | Version::Lite04 | Version::Lite05 + ) => + { tracing::warn!(group = hdr.sequence, "group past the declared end of track"); let _ = entry.producer.clone().abort(Error::ProtocolViolation); return Poll::Ready(Err(Error::ProtocolViolation)); @@ -1441,48 +1450,124 @@ mod tests { } /// A group at or past the end the publisher declared contradicts that end, which no later - /// stream can repair, so the whole track fails rather than ending clean without it. + /// stream can repair, so the whole track fails rather than ending clean without it. lite-05 + /// specified an inclusive end, so there it costs only that group's stream. #[tokio::test] async fn a_group_past_the_declared_end_aborts_the_track() { use crate::transport::poll::Session as _; - let mut script = Vec::new(); - lite::Group { - subscribe: 7, - sequence: 3, - frame_start: 0, + for version in [Version::Lite05, Version::Lite06, Version::Lite07] { + let mut script = Vec::new(); + lite::Group { + subscribe: 7, + sequence: 3, + frame_start: 0, + } + .encode(&mut script, version) + .unwrap(); + let mut session = crate::lite::test_transport::ScriptedSession::eof(script); + let subscriber = Subscriber::new(SubscriberConfig { + runtime: crate::time::Clock::tokio(), + session: session.clone(), + origin: origin::Config::new(crate::Hop::new(1).unwrap()).produce(), + recv_bandwidth: None, + version, + peer_setup: Default::default(), + peer_hop: None, + cost: None, + going_away: Default::default(), + }); + + let mut track = track::Producer::new(std::sync::Arc::new(crate::broadcast::Info::default()), "video", None); + track.finish_at(3).unwrap(); + subscriber.subscribes.lock().insert( + 7, + TrackEntry { + producer: track.clone(), + timescale: Some(Timescale::default()), + tail: Default::default(), + }, + ); + + let (_, recv) = session.open_bi().await.unwrap(); + let mut group = GroupRecv::new(subscriber, Reader::new(recv, version)); + let res = kio::wait(|waiter| group.poll_serve(waiter)).await; + match version { + Version::Lite05 => { + assert!(matches!(res, Err(Error::Closed)), "{version:?}: {res:?}"); + assert!( + track.closed().now_or_never().is_none(), + "{version:?}: the track lives on" + ); + } + _ => { + assert!(matches!(res, Err(Error::ProtocolViolation)), "{version:?}: {res:?}"); + assert!(matches!(track.closed().now_or_never(), Some(Error::ProtocolViolation))); + } + } } - .encode(&mut script, VERSION) - .unwrap(); - let mut session = crate::lite::test_transport::ScriptedSession::eof(script); - let subscriber = Subscriber::new(SubscriberConfig { - runtime: crate::time::Clock::tokio(), - session: session.clone(), - origin: origin::Config::new(crate::Hop::new(1).unwrap()).produce(), - recv_bandwidth: None, - version: VERSION, - peer_setup: Default::default(), - peer_hop: None, - cost: None, - going_away: Default::default(), - }); + } - let mut track = track::Producer::new(std::sync::Arc::new(crate::broadcast::Info::default()), "video", None); - track.finish_at(3).unwrap(); - subscriber.subscribes.lock().insert( - 7, - TrackEntry { - producer: track.clone(), - timescale: Some(Timescale::default()), - tail: Default::default(), - }, - ); + /// A SUBSCRIBE_END below a group already received contradicts that group. lite-05 + /// specified an inclusive end, and `@moq/net` 0.1.3 to 0.1.9 sent one, so there it only + /// costs the early boundary; later drafts abort the track. + #[tokio::test(start_paused = true)] + async fn a_subscribe_end_below_a_received_group() { + for version in [Version::Lite05, Version::Lite06, Version::Lite07] { + let mut responses = Vec::new(); + lite::SubscribeResponse::Start(lite::SubscribeStart { group: 0 }) + .encode(&mut responses, version) + .unwrap(); + lite::SubscribeResponse::End(lite::SubscribeEnd { group: 2, streams: 1 }) + .encode(&mut responses, version) + .unwrap(); + let session = crate::lite::test_transport::ScriptedSession::eof(responses); + let subscriber = Subscriber::new(SubscriberConfig { + runtime: crate::time::Clock::tokio(), + session, + origin: origin::Config::new(crate::Hop::new(1).unwrap()).produce(), + recv_bandwidth: None, + version, + peer_setup: Default::default(), + peer_hop: None, + cost: None, + going_away: Default::default(), + }); + let serve = TrackServe { + subscriber, + path: Path::new("room").to_owned(), + name: "video".to_string(), + }; + let broadcast = crate::broadcast::Info::new().produce(); + let request = broadcast.reserve_track("video").unwrap(); + let serving = ServeLoop::new(&serve, request, Default::default(), Some(Timescale::default())); + let mut group = serving.serving.create_group(group::Info { sequence: 2 }).unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"2".as_slice()).unwrap(); + group.finish().unwrap(); + let mut reader = broadcast + .consume() + .track("video") + .unwrap() + .subscribe(None) + .await + .unwrap(); + let mut running = TrackServeRun { + serve, + state: TrackRunState::Serve(serving), + }; + kio::wait(|waiter| kio::Task::poll(&mut running, waiter)).await; - let (_, recv) = session.open_bi().await.unwrap(); - let mut group = GroupRecv::new(subscriber, Reader::new(recv, VERSION)); - let res = kio::wait(|waiter| group.poll_serve(waiter)).await; - assert!(matches!(res, Err(Error::ProtocolViolation)), "{res:?}"); - assert!(matches!(track.closed().now_or_never(), Some(Error::ProtocolViolation))); + let end = loop { + match reader.recv_group().await { + Ok(Some(_)) => continue, + end => break end.map(|group| group.map(|group| group.sequence)), + } + }; + match version { + Version::Lite05 => assert!(matches!(end, Ok(None)), "{version:?}: {end:?}"), + _ => assert!(matches!(end, Err(Error::ProtocolViolation)), "{version:?}: {end:?}"), + } + } } /// Drive the subscriber with a peer's response bytes followed by FIN. @@ -3851,15 +3936,22 @@ impl ServeLoop { // consumers learn the boundary early; the later stream FIN // then finds the track already finished. lite::SubscribeResponse::End(end) => { - // finish_at rejects a boundary at or below the live - // edge, which is what a peer sending an inclusive bound - // looks like once the final group has already arrived. - // Don't abort: the stream FIN still finishes the track, - // so this only costs the early boundary. Warn anyway, - // since it's our only signal that a peer disagrees - // about the encoding. + // finish_at rejects a boundary at or below a group + // already received. lite-05 specified an inclusive end, + // and `@moq/net` 0.1.3 to 0.1.9 sent one, so there it + // only costs the early boundary: warn, and let the FIN + // finish the track. Later drafts made it exclusive, so + // the publisher contradicted its own end. if let Err(err) = self.serving.finish_at(end.group) { - tracing::warn!(track = %serve.name, group = end.group, %err, "invalid subscribe end"); + match serve.subscriber.version { + Version::Lite05 => { + tracing::warn!(track = %serve.name, group = end.group, %err, "invalid subscribe end") + } + _ => { + tracing::warn!(track = %serve.name, group = end.group, %err, "subscribe end below a received group"); + return Poll::Ready(ServeEnd::GiveBack(Error::ProtocolViolation)); + } + } } active.end = Some(end.clone()); } From 17df80858457846f1176841a8c36c68809fae365 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 13:15:00 -0700 Subject: [PATCH 5/8] quest(m1): plan reader end parity and visible dropped groups Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 1 + quest/m1/reader-end-parity.md | 32 ++++++++++++++++++++++++++++++++ quest/m1/subscribe-drop.md | 3 +++ 3 files changed, 36 insertions(+) create mode 100644 quest/m1/reader-end-parity.md diff --git a/quest/m1/README.md b/quest/m1/README.md index 2ae909d46a0..7443d289d1e 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -35,6 +35,7 @@ transport, benchmark tooling); worktrees isolate commits, not semantics. - [Resumed groups](/quest/m1/resume-latest.md) - a half-delivered group ends once the new copy is past it, so a group-only reader never parks after a mid-group failover - [JS close end](/quest/m1/js-track-close-end.md) - a JS track's clean close ends at its own last group, not a sibling producer's - [SUBSCRIBE_DROP](/quest/m1/subscribe-drop.md) - every stream group in a lite subscription arrives or is dropped by name, and lite-07 drops its stream count for it +- [Reader end parity](/quest/m1/reader-end-parity.md) - JS readers see a track's end once the newest group reaches the declared end, as Rust readers do - [FIN wait expiry](/quest/m1/fin-wait-expiry.md) - a group awaiting its FIN ack still expires and follows priority updates on lite and IETF - [Cross-relay bursts](/quest/m1/cross-relay-bursts.md) - bursty small-group tracks cross two relays without lost groups, unanswered FETCHes, or stalls - [Request stream cancel](/quest/m1/request-stream-serve.md) - a lite publisher stops resolving a SUBSCRIBE or FETCH once the requester FINs or resets, through one wrapper that owns every request stream's reader diff --git a/quest/m1/reader-end-parity.md b/quest/m1/reader-end-parity.md new file mode 100644 index 00000000000..dbceda77167 --- /dev/null +++ b/quest/m1/reader-end-parity.md @@ -0,0 +1,32 @@ +# [S] Readers see a track's end at the same moment in Rust and JS + +## Goal + +A track reader in `@moq/net` sees the end as soon as the newest group reaches +the declared end, as Rust readers already do, instead of waiting for the track +to close after the tail settles. The same subscription ends for a reader at +the same point in both languages. + +## Plan + +Found in [#4533](https://github.com/moq-dev/moq/pull/4533): the tails agree +on when a subscription settles, but the reader-facing end differs. The Rust +track model ends for readers once the newest group reaches the declared end, +so a hole waiting out the grace never delays a reader. JS readers wait for the +track to close, grace included. + +Decided with the maintainer: adopt Rust's behavior in JS. A reader isn't held +for the grace over groups it would skip anyway; the tail still settles +separately for accounting. Keep a clean end that has not settled distinct from +an abort, as #4120 and #4116 require. + +Tests: the same scenario in both languages (a declared end with a missing +group in the grace window) reports the end to readers without waiting out the +grace. Consider asserting it across languages in [Track tail +interop](/quest/m1/track-tail-interop.md). + +Public API: none expected; a JS behavior change. Wire: none. + +## Related + +- [SUBSCRIBE_DROP](/quest/m1/subscribe-drop.md) - names every missing group, so fewer holes reach the grace at all diff --git a/quest/m1/subscribe-drop.md b/quest/m1/subscribe-drop.md index a12a865a6a1..ecd671811c5 100644 --- a/quest/m1/subscribe-drop.md +++ b/quest/m1/subscribe-drop.md @@ -34,6 +34,9 @@ Decided: grace, as today. - A resumed group ([Resumed groups](/quest/m1/resume-latest.md)) that is the new copy's latest ends with the DROP's error when the copy drops it. +- A dropped or aborted group is visible to readers, not silently skipped. + #4533 found the Rust model releases an aborted group's sequence and skips it, + so a truncated first object is indistinguishable from a group never sent. Update `drafts/draft-lcurley-moq-lite.md` (SUBSCRIBE_DROP, SUBSCRIBE_END, the lite-07 changelog), `doc/concept/moq-lite.md`, and the Rust and JS lite From 03bd421726363ab89ef76446581652aa7ef51fe0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 14:09:46 -0700 Subject: [PATCH 6/8] fix(net): restart a gap's age when a lowered floor asks for it A gap the demand newly asks for was aging from the run above it, so a floor lowered on a long-lived subscription could fold the new gap away at once and end the subscription before its groups arrived. Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/subscriber.ts | 11 ++++++++-- js/net/src/tail.test.ts | 17 +++++++++++++++ js/net/src/tail.ts | 12 +++++++++++ rs/moq-net/src/lite/subscriber.rs | 5 +++++ rs/moq-net/src/tail.rs | 36 +++++++++++++++++++++++++++++++ 5 files changed, 79 insertions(+), 2 deletions(-) diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index cc07c24f17b..ee919a230dc 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -589,7 +589,7 @@ export class Subscriber { const subscriptionUpdates = this.version === Version.DRAFT_01 || this.version === Version.DRAFT_02 ? undefined - : this.#runSubscriptionUpdates(id, broadcast, producer, msg, stream); + : this.#runSubscriptionUpdates(id, broadcast, entry, msg, stream); // Terminal conditions (stream end, track close, a failed subscription update) settle at most // once; race them into one stable promise so the demand loop doesn't re-subscribe each pass. @@ -955,10 +955,11 @@ export class Subscriber { async #runSubscriptionUpdates( id: bigint, broadcast: Path.Valid, - track: track.Producer, + entry: SubscribeEntry, msg: Subscribe, stream: Stream, ): Promise { + const track = entry.track; const stopped: Promise = race([track.closed, stream.reader.closed]).then(() => null); let lastSent: track.Subscription = { priority: msg.priority, @@ -983,6 +984,12 @@ export class Subscriber { const bounds = groupBounds(current.groups); if (emptyRange({ startGroup: bounds.start, endGroup: bounds.end })) throw new Error(EMPTY_RANGE); + // A lowered floor owes groups nobody asked for until now. + if (current.groups?.start !== undefined) { + const floor = lastSent.groups?.start === undefined ? entry.start : groupBounds(lastSent.groups).start; + entry.tail.demand(bounds.start, floor ?? Number.POSITIVE_INFINITY); + } + // Round-trip the other Subscribe parameters so the publisher doesn't // interpret SUBSCRIBE_UPDATE as a reset of ordered/maxAge/etc. const update = new SubscribeUpdate({ diff --git a/js/net/src/tail.test.ts b/js/net/src/tail.test.ts index e40fefb8ab2..62c4ebfe941 100644 --- a/js/net/src/tail.test.ts +++ b/js/net/src/tail.test.ts @@ -142,6 +142,23 @@ describe("Tail", () => { expect(t.covers(0, 10)).toBe(true); }); + test("a lowered floor restarts the gap age", () => { + const { clock, tail: t } = tail(); + t.account(3, 4); + t.account(9, 10); + + clock.now = GRACE * 2; + t.demand(1, 3); + t.account(1, 2); + expect(t.covers(1, 4)).toBe(false); + // Only the gap the floor reached restarts. + expect(t.covers(3, 10)).toBe(true); + + clock.now = GRACE * 3; + t.account(9, 10); + expect(t.covers(1, 10)).toBe(true); + }); + test("the grace waits for a stream being read", async () => { const t = new Tail({ grace: () => Milli(10) }); const read = t.open(0); diff --git a/js/net/src/tail.ts b/js/net/src/tail.ts index 178c50d002f..51e5f20bb99 100644 --- a/js/net/src/tail.ts +++ b/js/net/src/tail.ts @@ -84,6 +84,18 @@ export class Tail { this.#account(start, end); } + /** Restart the age of every gap reaching into `[start, end)`, which the demand newly asks for. */ + demand(start: number, end: number): void { + if (start >= end) return; + const now = this.#now(); + let below = 0; + this.#runs = this.#runs.map((run) => { + const reached = below < end && start < run.start; + below = run.end; + return reached ? { ...run, since: now } : run; + }); + } + /** Whether every sequence in `[start, end)` is accounted for. */ covers(start: number, end: number): boolean { if (start >= end) return true; diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index b393f33ac22..0fbc2097fc8 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -3255,6 +3255,11 @@ impl TrackServe { active.max_age = subscription.max_age; if let Ok(mut tail) = active.tail.write() { tail.set_grace(tail::grace(subscription.max_age)); + // A lowered floor owes groups nobody asked for until now. + if let Some(start) = subscription.start { + let floor = active.start.map(|start| start.group).or(active.served); + tail.demand(start.group..floor.unwrap_or(u64::MAX), self.subscriber.runtime.now()); + } } active.start = subscription.start; if supports_update { diff --git a/rs/moq-net/src/tail.rs b/rs/moq-net/src/tail.rs index 67c1361d81a..65e0a721eae 100644 --- a/rs/moq-net/src/tail.rs +++ b/rs/moq-net/src/tail.rs @@ -105,6 +105,20 @@ impl Tail { self.expire(now); } + /// Restart the age of every gap reaching into `groups`, which the demand newly asks for. + pub fn demand(&mut self, groups: Range, now: Instant) { + if groups.is_empty() { + return; + } + let mut below = 0; + for run in &mut self.runs { + if below < groups.end && groups.start < run.groups.start { + run.since = now; + } + below = run.groups.end; + } + } + /// Fold every gap older than the grace into the runs around it. pub fn expire(&mut self, now: Instant) { let grace = self.grace; @@ -289,6 +303,28 @@ mod tests { assert_eq!(groups(&tail), vec![0..10], "both halves opened at the start"); } + /// A gap the demand newly asks for is late only from then, however old the run above it. + #[test] + fn a_lowered_floor_restarts_the_gap_age() { + let start = Instant::now(); + let mut tail = Tail::new(GRACE); + tail.account(3..4, start); + tail.account(9..10, start); + + let lowered = start + GRACE * 2; + tail.demand(1..3, lowered); + tail.account(1..2, lowered); + assert_eq!( + groups(&tail), + vec![1..2, 3..10], + "only the gap the floor reached restarts" + ); + assert!(!tail.covers(1..4)); + + tail.expire(lowered + GRACE); + assert!(tail.covers(1..10)); + } + /// The grace gives up on streams that never arrived, never on one still being read: /// its group, or the END_OF_TRACK it carries, still belongs to the track. #[tokio::test(start_paused = true)] From 3c144e68c83681bc0f5a986071066b03bba46589 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 15:14:42 -0700 Subject: [PATCH 7/8] fix(net): hold an IETF subscription's end open across the fill handoff A tail parked on its fill stops holding the end open, and the fill's own stream let go once it installed the head, so the subscription could settle and end before the tail woke to claim it. An outstanding fill is now owed, and the tail resumes before it takes the head. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/ietf/subscriber.rs | 62 +++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index a4c8e965a39..5871f5bec56 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -1830,13 +1830,13 @@ where // each is read to its end, or a bounded grace for any reset before // its header (the draft says to use a timeout). The count is a hint: // a published peer sends 0, which waits out the grace. - let tail = self + let held = self .state .lock() .subscribes .get(&request_id) - .map(|held| held.tail.consume()); - if let Some(tail) = tail { + .map(|held| (held.tail.consume(), held.fill.clone())); + if let Some((tail, fill)) = held { let mut settle = Settle::new(&self.runtime, tail); kio::wait(|waiter| { if !fetch_done @@ -1845,7 +1845,7 @@ where { fetch_done = true; } - settle.poll(waiter, |tail| count > 0 && tail.streams() >= count) + poll_settled(&mut settle, waiter, &fill, count) }) .await; } @@ -2482,7 +2482,7 @@ where reading: &mut Reading, ) -> Result, Error> { reading.park(); - let claimed = kio::wait(|waiter| { + let settled = kio::wait(|waiter| { if let Poll::Ready(err) = track.poll_closed(waiter) { return Poll::Ready(Err(err)); } @@ -2493,15 +2493,18 @@ where }); match settled { - Poll::Ready(Ok(mut fill)) => Poll::Ready(fill.claim(sequence, start)), + Poll::Ready(Ok(_)) => Poll::Ready(Ok(())), // The subscription went away underneath us. Poll::Ready(Err(_)) => Poll::Ready(Err(Error::Dropped)), Poll::Pending => Poll::Pending, } }) .await; + // Hold the end open again before taking the head: once the fill is claimed, this + // stream is the only thing saying the group is still being read. reading.resume(); - claimed + settled?; + fill.write().map_err(|_| Error::Dropped)?.claim(sequence, start) } } @@ -2943,6 +2946,15 @@ fn advance_fill_group( open_fill_group(track, head, sequence) } +/// Ready once a finished subscription's data streams are accounted for: Stream Count of +/// their headers, and no fill outstanding. A tail parked on its head holds nothing open +/// itself, so the fill does until the head is claimed. +fn poll_settled(settle: &mut Settle, waiter: &kio::Waiter, fill: &kio::Producer, count: u64) -> Poll<()> { + // Read before the tail: Done is terminal, so the answer cannot go stale. + let filled = !fill.read().outstanding(); + settle.poll(waiter, |tail| filled && count > 0 && tail.streams() >= count) +} + /// A refused or missing joining FETCH continues the subscription live: drop the outstanding /// fill so a mid-group tail is not left waiting on a head that is never coming. fn settle_join_live(fill: &kio::Producer) { @@ -6046,6 +6058,42 @@ mod stitch_tests { assert!(matches!(*h.fill.read(), Fill::Done), "the head was claimed"); } + /// A tail parked on its head stops holding the end open, so the finished fill does until + /// the head is claimed: otherwise the subscription looks settled between the fill + /// finishing and the tail waking, and ends before the tail is read. + #[tokio::test] + async fn an_unclaimed_fill_holds_the_end_open() { + let h = Harness::new( + Fill::Serving(Some(Timescale::MICRO)), + vec![ + fill_stream(SEQUENCE, &[b"head-0", b"head-1"]), + tail_stream(SEQUENCE, 2, &[b"tail-2"]), + ], + ); + let mut fill = h.stream().await; + let mut tail = h.stream().await; + + let mut serve_tail = h.subscriber.clone(); + let mut tailing = std::pin::pin!(serve_tail.recv_group(&mut tail)); + assert!( + futures::poll!(tailing.as_mut()).is_pending(), + "the tail waits for its head" + ); + h.subscriber.clone().recv_fill(&mut fill).await.expect("fill"); + + let state = h.subscriber.state.lock().subscribes[&REQUEST].tail.consume(); + let count = state.read().streams(); + let mut settle = Settle::new(&crate::time::Clock::tokio(), state); + let mut settled = std::pin::pin!(kio::wait(|waiter| poll_settled(&mut settle, waiter, &h.fill, count))); + assert!( + futures::poll!(settled.as_mut()).is_pending(), + "the head is still owed to the tail" + ); + + tailing.await.expect("tail"); + settled.await; + } + #[tokio::test] async fn object_extension_limit() { for size in [65536usize, 65537] { From 9d2f0a7451904f0224692da271801123df30e175 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 15:25:14 -0700 Subject: [PATCH 8/8] docs(net): note that a folded tail gap stays folded Co-Authored-By: Claude Opus 5.5 --- js/net/src/tail.ts | 3 ++- rs/moq-net/src/tail.rs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/js/net/src/tail.ts b/js/net/src/tail.ts index 51e5f20bb99..b732ae11338 100644 --- a/js/net/src/tail.ts +++ b/js/net/src/tail.ts @@ -42,7 +42,8 @@ export interface TailOptions { export class Tail { // Disjoint, sorted, non-adjacent runs of accounted sequences. A gap older than the grace // can no longer be waited for, so it folds into the runs around it, which bounds this by - // the gaps opened within the grace rather than every gap in the subscription. + // the gaps opened within the grace rather than every gap in the subscription. A fold is + // final: a later, longer grace or a lowered floor cannot reopen it. #runs: Run[] = []; // Group streams whose header arrived, and those still being read. #streams = 0; diff --git a/rs/moq-net/src/tail.rs b/rs/moq-net/src/tail.rs index 65e0a721eae..b32a2f66487 100644 --- a/rs/moq-net/src/tail.rs +++ b/rs/moq-net/src/tail.rs @@ -43,7 +43,8 @@ struct Run { pub(crate) struct Tail { // Disjoint, sorted, non-adjacent runs of accounted sequences. A gap older than the grace // can no longer be waited for, so it folds into the runs around it, which bounds this by - // the gaps opened within the grace rather than every gap in the subscription. + // the gaps opened within the grace rather than every gap in the subscription. A fold is + // final: a later, longer grace or a lowered floor cannot reopen it. runs: Vec, grace: Duration, streams: u64,