From 5d0113280d8bc7f49bebc2f64bffef19d5252414 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:00:19 -0800 Subject: [PATCH 01/18] chore(quest): claim quest/m1/stream-credit/rank-opening.md Quest-Claim: 08cda3959a1c52bb From 2b6711ca286392868b0c3af6e7c818b7ea9a8fec Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:11:01 -0800 Subject: [PATCH 02/18] fix(moq-net): a group waiting for stream credit gives way to one that outranks it Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-subscribe/SKILL.md | 1 + rs/kio/src/task.rs | 5 + rs/moq-net/src/lite/publisher.rs | 129 +++++++++-------- rs/moq-net/src/model/track.rs | 35 ++++- rs/moq-net/tests/integration/stream_credit.rs | 136 ++++++++++++++++++ 5 files changed, 242 insertions(+), 64 deletions(-) diff --git a/.claude/skills/verify-subscribe/SKILL.md b/.claude/skills/verify-subscribe/SKILL.md index 01c5a06e290..e50e4e4b8ce 100644 --- a/.claude/skills/verify-subscribe/SKILL.md +++ b/.claude/skills/verify-subscribe/SKILL.md @@ -46,6 +46,7 @@ Sync commit: `81a4d971fc766431ef49361c640a36d69b7f3ac2`. Run `git log 81a4d971fc | Code | Model | | --- | --- | | `rs/moq-net/src/lite/publisher.rs` `TrackRun::poll_track` (`Recv::Group`, the re-offer pop), `serve`, `mark_served` | `CursorHand`, `PubReoffer`, `Serve` | +| `rs/moq-net/src/lite/publisher.rs` `TrackRun::give_way` | a new group as `Serve`, then `StreamFail` with `old` whose header never arrives; a continuation as `StreamYield` of a stream whose header never arrives | | `rs/moq-net/src/lite/publisher.rs` `poll_forward_drop`, `forward_range`, `drop_range`, `replace_stale`, `sent_stale`, `hole` | `CursorGiveUp`, `ReplaceStale` | | `rs/moq-net/src/lite/publisher.rs` `TrackRun::poll_step` (outcomes, `redrop`) | `PubOutcomes`, `PubRedrop` | | `rs/moq-net/src/lite/publisher.rs` `TrackRun::update`, `owe`; `rs/moq-net/src/model/track.rs` `reoffer_stale`, `reoffer_groups` | `PubUpdate` | diff --git a/rs/kio/src/task.rs b/rs/kio/src/task.rs index 838da944839..2d02270ac2c 100644 --- a/rs/kio/src/task.rs +++ b/rs/kio/src/task.rs @@ -330,6 +330,11 @@ impl Tasks { self.len == 0 } + /// The tasks still running, in no particular order. + pub fn iter(&self) -> impl Iterator { + self.children.iter().flatten().map(|occupant| &occupant.task) + } + /// Add a task, queued for its first poll. /// /// Wakes the owner, so a task pushed between polls (from another arm of the diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 5887a4ffeca..94ae01770e7 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -3557,7 +3557,7 @@ mod test { .finish_at(track::Position::group(7), Some(Error::App(7))) .unwrap(); let next = |subscriber: &mut track::Subscriber| { - kio::wait(|waiter| poll_recv_next(subscriber, true, true, end_error, waiter)).now_or_never() + kio::wait(|waiter| poll_recv_next(subscriber, Some(ALL), true, end_error, waiter)).now_or_never() }; match (end_error || capped, next(&mut subscriber)) { (true, Some(Ok(Recv::Boundary(end)))) => { @@ -4687,6 +4687,9 @@ fn final_sequence(end: track::Position) -> u64 { end.final_sequence().unwrap_or(u64::MAX) } +/// Every group sequence, for [`poll_recv_next`] to hand out any. +const ALL: (Bound, Bound) = (Bound::Unbounded, Bound::Unbounded); + /// Poll a single [`track::Subscriber`] for the next group (cap-aware, in arrival order). /// /// Groups are served in arrival order (`poll_recv_group`), not sequence order: on a relay, a @@ -4694,8 +4697,9 @@ fn final_sequence(end: track::Position) -> u64 { /// permanently skip the older group even though it is cached and in demand. Staleness is /// governed by max age and the subscriber's max delay, not arrival raciness. /// -/// Without `groups` the cursor hands out none, leaving them in the cache, so it never reports -/// [`Recv::Finished`] either. +/// The cursor hands out the next group only if its sequence lies in `groups`, leaving it in +/// the cache otherwise, and without `groups` none at all. Either way it then never reports +/// [`Recv::Finished`]. /// /// When `emit_boundary` is set, a declared-but-not-yet-reached final sequence surfaces as /// [`Recv::Boundary`] in an idle moment (after groups), so the caller can send @@ -4706,7 +4710,7 @@ fn final_sequence(end: track::Position) -> u64 { /// that end finishes like a clean one. Otherwise reaching it is the error, which resets. fn poll_recv_next( track: &mut track::Subscriber, - groups: bool, + groups: Option<(Bound, Bound)>, emit_boundary: bool, end_error: bool, waiter: &kio::Waiter, @@ -4714,8 +4718,8 @@ fn poll_recv_next( { let mut groups_finished = false; let next = match groups { - true => track.poll_recv_group(waiter), - false => Poll::Pending, + Some(admit) => track.poll_recv_group_in(admit, waiter), + None => Poll::Pending, }; match next { Poll::Ready(Ok(Some(group))) => return Poll::Ready(Ok(Recv::Group(group))), @@ -4755,7 +4759,7 @@ fn poll_recv_next( /// The async form of [`poll_recv_next`], for callers with nothing else to poll. #[cfg(test)] async fn recv_next(track: &mut track::Subscriber, emit_boundary: bool) -> Result { - kio::wait(|waiter| poll_recv_next(track, true, emit_boundary, true, waiter)).await + kio::wait(|waiter| poll_recv_next(track, Some(ALL), emit_boundary, true, waiter)).await } /// Bound `group` to what this subscription asked for, or `None` when it cannot be served @@ -5033,11 +5037,9 @@ struct Paused { } impl Paused { - /// Whether `a` ranks above `b` for a subscription's stream credit, by sequence in its - /// `order`, as [`crate::priority::in_group`] ranks them. - fn outranks(order: crate::track::Order, a: &Self, b: &Self) -> bool { - crate::priority::in_group(a.group.sequence, a.frame_start, order) - > crate::priority::in_group(b.group.sequence, b.frame_start, order) + /// Its rank for the subscription's stream credit in `order`, higher first. + fn rank(&self, order: crate::track::Order) -> i64 { + crate::priority::in_group(self.group.sequence, self.frame_start, order) } } @@ -5133,9 +5135,6 @@ struct TrackRun { paused: std::collections::VecDeque, // Lite-07: the group `opening` holds continues a paused one. continuing: bool, - // Lite-07: a group the cursor handed out while a continuation waited for credit, ranked - // below it. One at most: the cursor hands out no other until it opens. - stashed: Option, end_sent: bool, // Lite-07 with every range bounded: once every group in its ranges is given up, it is // answered without a group to resolve its start, rather than left unanswered. @@ -5143,7 +5142,8 @@ struct TrackRun { /// The one group waiting for stream credit. The rest of the backlog waits in the cache, /// which bills, ages, and evicts it, and the cursor's budget skips whatever went stale /// meanwhile: a machine per parked group would hold state for as long as the peer - /// withholds credit. + /// withholds credit. One that cannot make room gives way to a group that outranks it + /// (see [`Self::stuck`]). opening: Option, /// The groups with a stream, serving or awaiting the acknowledgement of their FIN. children: kio::Tasks, @@ -5202,7 +5202,6 @@ impl TrackRun { reoffered: Default::default(), paused: Default::default(), continuing: false, - stashed: None, end_sent: false, opening: None, children: kio::Tasks::new(), @@ -5605,27 +5604,16 @@ impl TrackRun { // A paused group takes the next stream credit, ahead of re-offers and new groups, the // highest-ranked first. It is owed already, and the subscriber holds its group open until - // it continues. A group stashed behind a continuation goes first if it outranks them all. - let best = match self.opening.is_none() { - true => self.best_paused(), - false => None, - }; - if self.opening.is_none() - && let Some(stashed) = self.stashed.take() - { - let order = self.ctx.steer_current().order; - match best.map(|best| &self.paused[best]) { - Some(paused) if !Paused::outranks(order, &stashed, paused) => self.stashed = Some(stashed), - _ => { - self.open(stashed.group, stashed.frame_start); - return Poll::Ready(Ok(ControlFlow::Continue(()))); - } - } - } - if self.opening.is_none() - && let Some(Paused { group, frame_start }) = best.and_then(|best| self.paused.remove(best)) + // it continues. A group waiting for credit it cannot make room for gives way to one that + // outranks it. + let order = self.ctx.steer_current().order; + let stuck = self.stuck(); + if let Some(best) = self.best_paused() + && (self.opening.is_none() || stuck.is_some_and(|waiting| self.paused[best].rank(order) > waiting)) { + let Paused { group, frame_start } = self.paused.remove(best).expect("best is in range"); tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, sequence = group.sequence, frame_start, "continuing paused group"); + self.give_way(); self.open(group, frame_start); self.continuing = true; return Poll::Ready(Ok(ControlFlow::Continue(()))); @@ -5657,11 +5645,18 @@ impl TrackRun { // One cursor drives the whole subscription: poll the cap-aware arrival-order group. let emit_boundary = self.emit_range && !self.end_sent; - // A continuation waiting for credit ranks below every stream it could take, so it must - // not hold back a newer group that outranks one of them: the cursor hands one more out. - let groups = self.opening.is_none() || (self.continuing && self.stashed.is_none()); + // A group waiting for credit it cannot make room for must not hold back a newer one + // that could: the cursor hands out a group that outranks it, which takes its place. + // Behind one that can, the backlog waits in the cache. + let admit = match &self.opening { + None => Some(ALL), + Some(waiting) => stuck.map(|_| match order { + Order::Descending => (Bound::Excluded(waiting.sequence), Bound::Unbounded), + Order::Ascending => (Bound::Unbounded, Bound::Excluded(waiting.sequence)), + }), + }; let end_error = self.ctx.version.has_end_error(); - if let Poll::Ready(res) = poll_recv_next(&mut self.track, groups, emit_boundary, end_error, waiter) { + if let Poll::Ready(res) = poll_recv_next(&mut self.track, admit, emit_boundary, end_error, waiter) { match res? { Recv::Group(group) => { // Never a second stream for a sequence this subscription already accounted @@ -6053,32 +6048,44 @@ impl TrackRun { Ok(()) } - /// Open a group machine for `group`, which waits for stream credit before the cursor - /// hands out another. + /// Open a group machine for `group`, which waits for stream credit. The cursor hands it + /// out past a group already waiting only if it outranks that one, which gives way. fn serve(&mut self, group: group::Consumer) { let sequence = group.sequence; let frame_start = group.index(); self.mark_served(sequence); tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, sequence, "serving group"); - let Some(waiting) = self.opening.take() else { - return self.open(group, frame_start); - }; - // Lite-07: handed out while a continuation waits for credit. The higher-ranked of the two - // waits for it, and the other right behind; a continuation goes back among the paused. - debug_assert!(self.continuing, "only a continuation shares the wait"); - let serving = Paused { group, frame_start }; - let continuation = Paused { - group: waiting.group.clone(), - frame_start: waiting.frame_start, + self.give_way(); + self.open(group, frame_start); + } + + /// The rank of the group waiting for stream credit, if it cannot make room: it ranks below + /// every stream this subscription holds, so it waits for credit to return, while a group + /// that outranks one of those streams would take it over now. + fn stuck(&mut self) -> Option { + let waiting = self.opening.as_ref()?; + let order = self.ctx.steer_current().order; + let rank = |serve: &GroupServe| crate::priority::in_group(serve.sequence, serve.frame_start, order); + let waiting = rank(waiting); + let mut held = self.children.iter().map(rank).peekable(); + (held.peek().is_some() && held.all(|held| held > waiting)).then_some(waiting) + } + + /// Take the group waiting for stream credit out of line, for one that outranks it: a + /// continuation goes back among the paused, and a new group is given up as one expired + /// under the subscription's budget is, which a lite-07 update may re-offer. Either way at + /// most one group per subscription waits for credit. + fn give_way(&mut self) { + let Some(mut waiting) = self.opening.take() else { + return; }; - if Paused::outranks(self.ctx.steer_current().order, &serving, &continuation) { - drop(waiting); - self.continuing = false; - self.paused.push_back(continuation); - self.open(serving.group, serving.frame_start); - } else { - self.opening = Some(waiting); - self.stashed = Some(serving); + tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, sequence = waiting.sequence, "giving way to a higher-ranked group"); + match std::mem::take(&mut self.continuing) { + true => self.paused.push_back(Paused { + group: waiting.group.clone(), + frame_start: waiting.frame_start, + }), + false => waiting.settle_failed(&Error::Old), } } @@ -6087,7 +6094,7 @@ impl TrackRun { fn best_paused(&mut self) -> Option { let order = self.ctx.steer_current().order; (0..self.paused.len()).reduce(|best, next| { - match Paused::outranks(order, &self.paused[next], &self.paused[best]) { + match self.paused[next].rank(order) > self.paused[best].rank(order) { true => next, false => best, } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index a93dce1acb1..6dbaa44dc65 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -9610,10 +9610,30 @@ impl Subscriber { self.poll_group(false, waiter) } + /// [`Self::poll_recv_group`], handing out the next group only if its sequence lies in + /// `admit`: one outside stays next, and the read is `Pending` without parking on it. + pub(crate) fn poll_recv_group_in( + &mut self, + admit: (ops::Bound, ops::Bound), + waiter: &kio::Waiter, + ) -> Poll>> { + self.poll_group_in(false, admit, waiter) + } + /// The next group in arrival order, or in sequence order for `ordered`, skipping any /// already handed out. fn poll_group(&mut self, ordered: bool, waiter: &kio::Waiter) -> Poll>> { - let res = self.poll_group_unsettled(ordered, waiter); + self.poll_group_in(ordered, (ops::Bound::Unbounded, ops::Bound::Unbounded), waiter) + } + + /// [`Self::poll_group`], refusing a next group whose sequence lies outside `admit`. + fn poll_group_in( + &mut self, + ordered: bool, + admit: (ops::Bound, ops::Bound), + waiter: &kio::Waiter, + ) -> Poll>> { + let res = self.poll_group_unsettled(ordered, admit, waiter); // What the read settled is billed now, and pays what it can of its readers' debt. The // group it hands out stays above the fold until its reader accounts for it. let handed = match &res { @@ -9626,7 +9646,12 @@ impl Subscriber { } } - fn poll_group_unsettled(&mut self, ordered: bool, waiter: &kio::Waiter) -> Poll>> { + fn poll_group_unsettled( + &mut self, + ordered: bool, + admit: (ops::Bound, ops::Bound), + waiter: &kio::Waiter, + ) -> Poll>> { if self.record().failed() { return Poll::Ready(Err(Error::Lagged)); } @@ -9644,12 +9669,16 @@ impl Subscriber { _ => {} } match self.poll_peek_group(ordered, waiter) { - Poll::Ready(Ok(Some(reserved))) => Poll::Ready(self.take(reserved).map(Some)), + Poll::Ready(Ok(Some(reserved))) if admit.contains(&reserved.sequence()) => { + Poll::Ready(self.take(reserved).map(Some)) + } res => { // What the peek skipped for age counts even when nothing is handed out. self.count_stale(&self.stats.meter()); match res { Poll::Ready(Ok(None)) => Poll::Ready(self.ended()), + // Refused: it stays next, so nothing behind it is either. + Poll::Ready(Ok(Some(_))) => Poll::Pending, res => res.map(|res| res.map(|_| None)), } } diff --git a/rs/moq-net/tests/integration/stream_credit.rs b/rs/moq-net/tests/integration/stream_credit.rs index d261c4af0c1..938d941f741 100644 --- a/rs/moq-net/tests/integration/stream_credit.rs +++ b/rs/moq-net/tests/integration/stream_credit.rs @@ -505,3 +505,139 @@ async fn an_abandoned_setup_returns_credit_with_the_publisher_still_live() { drop(first); } } + +/// Out of credit, a late group that ranks below the stream holding it waits for credit +/// without holding back a newer group that outranks that stream: the newer group still +/// resets it, on each protocol. +#[kio::test] +async fn a_waiting_late_group_lets_a_newer_group_through() { + for version in ["moq-lite-06", "moq-lite-07-wip"] { + let version: Version = version.parse().unwrap(); + let publisher = origin(1); + let viewer = origin(2); + let broadcast = publisher + .route("live", moq_net::origin::RouteOptions::hidden()) + .unwrap(); + let info = track::Info::default() + .with_timescale(moq_net::Timescale::MILLI) + .with_max_age(None); + let track = broadcast.broadcast().create_track("0", info).unwrap(); + broadcast.announce().unwrap(); + + let mut options = MockConnectOptions::new(version); + options.transport.max_concurrent_uni_streams = Some(1); + options.server_publish = Some(publisher.consume()); + options.client_subscribe = Some(viewer.clone()); + let _pair = connect_mock(options).await; + + let remote = viewer + .consume() + .follow("live", None) + .unwrap() + .available() + .await + .unwrap() + .unwrap(); + let prefs = track::Subscription::default() + .with_order(track::Order::Descending) + .with_delay(track::Delay::default().with_max(Duration::from_secs(30))); + let mut subscriber = remote.track("0").unwrap().subscribe(prefs).await.unwrap(); + macro_rules! next { + ($what:literal) => { + kio::clock::timeout(Duration::from_secs(5), subscriber.recv_group()) + .await + .unwrap_or_else(|_| panic!("{version}: {}", $what)) + .unwrap() + .unwrap() + }; + } + + // Unfinished, so its stream holds the only credit. + let mut held = track.create_group(moq_net::group::Info { sequence: 2 }).unwrap(); + held.write_frame(Timestamp::ZERO, b"held".as_slice()).unwrap(); + assert_eq!(next!("the held group arrives").sequence, 2, "{version}"); + + // Late and below the held stream: it waits for credit it cannot make room for. + let mut late = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + late.write_frame(Timestamp::ZERO, b"late".as_slice()).unwrap(); + late.finish().unwrap(); + + let mut newer = track.create_group(moq_net::group::Info { sequence: 3 }).unwrap(); + newer.write_frame(Timestamp::ZERO, b"new".as_slice()).unwrap(); + newer.finish().unwrap(); + assert_eq!( + next!("the newer group waits behind the late one").sequence, + 3, + "{version}" + ); + drop(held); + } +} + +/// Oldest first, a backlog behind a group waiting for credit stays in the cache rather than +/// giving it up: a newer group ranks below the waiting one, so every group still arrives once +/// the stream holding the credit finishes, on each protocol. +#[kio::test] +async fn an_ascending_backlog_waits_behind_a_waiting_group() { + for version in ["moq-lite-06", "moq-lite-07-wip"] { + let version: Version = version.parse().unwrap(); + let publisher = origin(1); + let viewer = origin(2); + let broadcast = publisher + .route("live", moq_net::origin::RouteOptions::hidden()) + .unwrap(); + let info = track::Info::default() + .with_timescale(moq_net::Timescale::MILLI) + .with_max_age(None); + let track = broadcast.broadcast().create_track("0", info).unwrap(); + broadcast.announce().unwrap(); + + let mut options = MockConnectOptions::new(version); + options.transport.max_concurrent_uni_streams = Some(1); + options.server_publish = Some(publisher.consume()); + options.client_subscribe = Some(viewer.clone()); + let _pair = connect_mock(options).await; + + let remote = viewer + .consume() + .follow("live", None) + .unwrap() + .available() + .await + .unwrap() + .unwrap(); + let prefs = track::Subscription::default() + .with_order(track::Order::Ascending) + .with_delay(track::Delay::default().with_max(Duration::from_secs(30))); + let mut subscriber = remote.track("0").unwrap().subscribe(prefs).await.unwrap(); + macro_rules! next { + ($what:literal) => { + kio::clock::timeout(Duration::from_secs(5), subscriber.recv_group()) + .await + .unwrap_or_else(|_| panic!("{version}: {}", $what)) + .unwrap() + .unwrap() + }; + } + + // Unfinished, so its stream holds the only credit, and outranks every newer group. + let mut held = track.append_group().unwrap(); + held.write_frame(Timestamp::ZERO, b"held".as_slice()).unwrap(); + assert_eq!(next!("the held group arrives").sequence, 0, "{version}"); + + for _ in 0..3 { + let mut group = track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, b"x".as_slice()).unwrap(); + group.finish().unwrap(); + } + // Let the publisher take what it would before credit returns. + kio::clock::sleep(Duration::from_millis(100)).await; + held.finish().unwrap(); + + let mut received = Vec::new(); + while received.len() < 3 { + received.push(next!("the backlog arrives").sequence); + } + assert_eq!(received, [1, 2, 3], "{version}"); + } +} From 50eab3f85301067f55a1b0c593c67144d46ede0a Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:47:11 -0800 Subject: [PATCH 03/18] test(js/net): a newer group passes a late one waiting for credit; finish the quest Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-subscribe/SKILL.md | 2 +- js/net/src/integration.test.ts | 56 ++++++++++++++++++++++++ quest/m1/stream-credit/rank-opening.md | 43 ------------------ rs/moq-net/src/model/track.rs | 1 + 4 files changed, 58 insertions(+), 44 deletions(-) delete mode 100644 quest/m1/stream-credit/rank-opening.md diff --git a/.claude/skills/verify-subscribe/SKILL.md b/.claude/skills/verify-subscribe/SKILL.md index e50e4e4b8ce..e26c7ff33f3 100644 --- a/.claude/skills/verify-subscribe/SKILL.md +++ b/.claude/skills/verify-subscribe/SKILL.md @@ -41,7 +41,7 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `81a4d971fc766431ef49361c640a36d69b7f3ac2`. Run `git log 81a4d971fc.. -- rs/moq-net/src/lite/publisher.rs rs/moq-net/src/lite/subscriber.rs rs/moq-net/src/tail.rs rs/moq-net/src/model/track.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. +Sync commit: `2b6711ca286392868b0c3af6e7c818b7ea9a8fec`. Run `git log 2b6711ca28.. -- rs/moq-net/src/lite/publisher.rs rs/moq-net/src/lite/subscriber.rs rs/moq-net/src/tail.rs rs/moq-net/src/model/track.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. | Code | Model | | --- | --- | diff --git a/js/net/src/integration.test.ts b/js/net/src/integration.test.ts index 2e7016a4104..7e48c1c73f3 100644 --- a/js/net/src/integration.test.ts +++ b/js/net/src/integration.test.ts @@ -3010,3 +3010,59 @@ test("integration: lite-07 resumes a group reset to make room without loss", asy server.abort(); } }); + +// Out of credit, a late group that ranks below the stream holding it waits for credit without +// holding back a newer group that outranks that stream. Mirrors the Rust +// `a_waiting_late_group_lets_a_newer_group_through`. +for (const alpn of [Lite.ALPN_06, Lite.ALPN_07_WIP]) { + test(`integration: ${alpn} lets a newer group past a late one waiting for credit`, async () => { + using clock = manual(); + const settled = async (work: Promise, what: string): Promise => { + for (let i = 0; i < 100; i++) if (await settlesWithin(work, clock, Duration.millis(10))) return work; + throw new Error(`${what} never settled`); + }; + const pair = qmuxPair(alpn, 1n); + await Promise.all([pair.client.ready, pair.server.ready]); + const origin = new OriginProducer(); + const [client, server] = await Promise.all([ + connect(url, { transport: pair.client }), + accept(pair.server, url, { publish: origin.consume() }), + ]); + const broadcast = publish(origin, Path.from("live")); + const producer = broadcast.createTrack("0", { timescale: Timescale.MILLI, maxAge: null }); + const remote = wireOf(client).consume(Path.from("live")); + const subscriber = remote + .track("0") + .subscribe({ order: "descending", delay: { max: Duration.millis(30_000) } }); + try { + // Unfinished, so its stream holds the only credit. + // Each group stamped in sequence order, since the track checks a late group's start. + const at = (sequence: bigint) => Timestamp.fromMillis(1_000 + Number(sequence)); + const held = producer.createGroup(2n); + held.writeFrame(textFrame("held", at(2n))); + expect((await settled(subscriber.recvGroup(), "the held group"))?.sequence).toBe(2n); + + // Late and below the held stream: it waits for credit it cannot make room for. + const late = producer.createGroup(0n); + late.writeFrame(textFrame("late", at(0n))); + late.finish(); + + const newer = producer.createGroup(3n); + newer.writeFrame(textFrame("new", at(3n))); + newer.finish(); + // lite-07 may fill the late group by a fetch first; only group 3 must not wait behind it. + for (;;) { + const group = await settled(subscriber.recvGroup(), "the newer group past the late one"); + if (!group) throw new Error("the subscription ended"); + if (group.sequence === 3n) break; + expect(group.sequence).toBe(0n); + } + } finally { + subscriber.close(); + remote.close(); + broadcast.close(); + client.abort(); + server.abort(); + } + }); +} diff --git a/quest/m1/stream-credit/rank-opening.md b/quest/m1/stream-credit/rank-opening.md deleted file mode 100644 index 32ac4beb2e9..00000000000 --- a/quest/m1/stream-credit/rank-opening.md +++ /dev/null @@ -1,43 +0,0 @@ -# [M] A lower-ranked group waiting for stream credit holds back higher-ranked ones - -## Goal - -Out of stream credit, a Rust publisher's subscription never holds a -higher-ranked group back behind a lower-ranked one waiting for credit: a -newer group that would reset a lower stream reaches the open line even while -another group of the subscription waits there. - -Today `TrackRun` (`rs/moq-net/src/lite/publisher.rs`) keeps one group waiting -for credit in `opening`, and its cursor hands out nothing while one waits. -A group that ranks below every stream the subscription holds can never make -room, so it blocks the cursor until credit returns or a stream finishes. With -one credit and descending order, unfinished group 2 holds the stream, late -group 0 waits in `opening`, and newer group 3, which would reset group 2, -never reaches the line. This reproduces on lite-06 at `4265dc5b57`, before -any lite-07 YIELDED: a test that publishes groups 2 (unfinished), 0, then 3 -times out waiting for group 3. -[moq-dev/moq#8545](https://github.com/moq-dev/moq/pull/8545) handles one case, -a lite-07 continuation waiting in `opening`, by letting the cursor hand out -one more group (`TrackRun::stashed`); a late lower-ranked group stashed there -blocks the cursor again. - -## Plan - -Recommended: while a group waits in `opening`, keep polling the cursor, and -keep the higher-ranked of the waiting group and the one it hands out -(`Paused::outranks`). The other leaves the line: a lite-07 continuation goes -back to `paused`, and a new group is given up as a group expired under the -subscription's budget is (a lite-07 stale drop, which the next -SUBSCRIBE_UPDATE may re-offer). This keeps one waiting group per -subscription plus the paused continuations stream credit bounds, and folds -in #8545's `stashed`. Check whether `SubscribeAccounting.tla`'s accounting -rules still hold, since a handed-out group may now end in a stale drop -without a stream. - -JS (`js/net/src/lite/publisher.ts`) opens every group concurrently, ranked by -send order, so it has no counterpart; confirm with the same test there. - -Tests: the repro above, on lite-06 and lite-07, in -`rs/moq-net/tests/integration/stream_credit.rs`. - -Public API: none. Wire: none. diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 6dbaa44dc65..8f8645fa559 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -9612,6 +9612,7 @@ impl Subscriber { /// [`Self::poll_recv_group`], handing out the next group only if its sequence lies in /// `admit`: one outside stays next, and the read is `Pending` without parking on it. + #[cfg(feature = "async")] pub(crate) fn poll_recv_group_in( &mut self, admit: (ops::Bound, ops::Bound), From d4c806ddc6df6743cd54750c6cfaee4f30d6efb7 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:12:13 -0800 Subject: [PATCH 04/18] fix(moq-net): newest first sheds a group ranked below one waiting for credit Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-subscribe/SKILL.md | 4 +- rs/moq-net/src/lite/publisher.rs | 33 +++++++-- rs/moq-net/tests/integration/stream_credit.rs | 70 +++++++++++++++++++ 3 files changed, 101 insertions(+), 6 deletions(-) diff --git a/.claude/skills/verify-subscribe/SKILL.md b/.claude/skills/verify-subscribe/SKILL.md index e26c7ff33f3..687ac14ced0 100644 --- a/.claude/skills/verify-subscribe/SKILL.md +++ b/.claude/skills/verify-subscribe/SKILL.md @@ -41,12 +41,12 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `2b6711ca286392868b0c3af6e7c818b7ea9a8fec`. Run `git log 2b6711ca28.. -- rs/moq-net/src/lite/publisher.rs rs/moq-net/src/lite/subscriber.rs rs/moq-net/src/tail.rs rs/moq-net/src/model/track.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. +Sync commit: `1be43c735b4ddaf5811433d82390e377cd1fe15a`. Run `git log 1be43c735b.. -- rs/moq-net/src/lite/publisher.rs rs/moq-net/src/lite/subscriber.rs rs/moq-net/src/tail.rs rs/moq-net/src/model/track.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. | Code | Model | | --- | --- | | `rs/moq-net/src/lite/publisher.rs` `TrackRun::poll_track` (`Recv::Group`, the re-offer pop), `serve`, `mark_served` | `CursorHand`, `PubReoffer`, `Serve` | -| `rs/moq-net/src/lite/publisher.rs` `TrackRun::give_way` | a new group as `Serve`, then `StreamFail` with `old` whose header never arrives; a continuation as `StreamYield` of a stream whose header never arrives | +| `rs/moq-net/src/lite/publisher.rs` `TrackRun::give_way`, `shed` | a new group, or one shed, as `Serve`, then `StreamFail` with `old` whose header never arrives; a continuation as `StreamYield` of a stream whose header never arrives | | `rs/moq-net/src/lite/publisher.rs` `poll_forward_drop`, `forward_range`, `drop_range`, `replace_stale`, `sent_stale`, `hole` | `CursorGiveUp`, `ReplaceStale` | | `rs/moq-net/src/lite/publisher.rs` `TrackRun::poll_step` (outcomes, `redrop`) | `PubOutcomes`, `PubRedrop` | | `rs/moq-net/src/lite/publisher.rs` `TrackRun::update`, `owe`; `rs/moq-net/src/model/track.rs` `reoffer_stale`, `reoffer_groups` | `PubUpdate` | diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 94ae01770e7..2ac9077b5d4 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -5645,13 +5645,16 @@ impl TrackRun { // One cursor drives the whole subscription: poll the cap-aware arrival-order group. let emit_boundary = self.emit_range && !self.end_sent; - // A group waiting for credit it cannot make room for must not hold back a newer one - // that could: the cursor hands out a group that outranks it, which takes its place. - // Behind one that can, the backlog waits in the cache. + // A group waiting for credit it cannot make room for must not hold back one that could: + // a group that outranks it takes its place. Newest first sheds the backlog, so the cursor + // hands out every group and one ranked below the waiting one is given up. Oldest first + // drains it in order, so the cursor hands out only a group that outranks the waiting + // one, and the rest waits in the cache. Behind a group that can make room, the backlog + // waits in the cache either way. let admit = match &self.opening { None => Some(ALL), Some(waiting) => stuck.map(|_| match order { - Order::Descending => (Bound::Excluded(waiting.sequence), Bound::Unbounded), + Order::Descending => ALL, Order::Ascending => (Bound::Unbounded, Bound::Excluded(waiting.sequence)), }), }; @@ -5687,6 +5690,15 @@ impl TrackRun { } return Poll::Ready(Ok(ControlFlow::Continue(()))); }; + if let Some(waiting) = &self.opening + && crate::priority::in_group(sequence, 0, order) + < crate::priority::in_group(waiting.sequence, 0, order) + { + tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, sequence, "shedding a group ranked below one waiting for credit"); + self.mark_served(sequence); + self.shed(sequence); + return Poll::Ready(Ok(ControlFlow::Continue(()))); + } match self.emit_range && self.start.is_none() { true => self.first = Some(group), false => self.serve(group), @@ -6089,6 +6101,19 @@ impl TrackRun { } } + /// Give up a group the cursor handed out without a stream, as one expired under the + /// subscription's budget is. + fn shed(&self, sequence: u64) { + if self.ctx.version.has_track_stream() { + self.ctx + .outcomes + .dropped + .lock() + .unwrap() + .push((sequence, StreamError::Old)); + } + } + /// The position of the highest-ranked paused group in this subscription's current order, /// the next to continue. fn best_paused(&mut self) -> Option { diff --git a/rs/moq-net/tests/integration/stream_credit.rs b/rs/moq-net/tests/integration/stream_credit.rs index 938d941f741..4cc282622ac 100644 --- a/rs/moq-net/tests/integration/stream_credit.rs +++ b/rs/moq-net/tests/integration/stream_credit.rs @@ -574,6 +574,76 @@ async fn a_waiting_late_group_lets_a_newer_group_through() { } } +/// Newest first sheds the backlog: under a late group waiting for credit, a later late group +/// ranked below it is given up rather than holding back a newer group behind it, on each +/// protocol. +#[kio::test] +async fn a_shed_late_group_lets_a_newer_group_through() { + for version in ["moq-lite-06", "moq-lite-07-wip"] { + let version: Version = version.parse().unwrap(); + let publisher = origin(1); + let viewer = origin(2); + let broadcast = publisher + .route("live", moq_net::origin::RouteOptions::hidden()) + .unwrap(); + let info = track::Info::default() + .with_timescale(moq_net::Timescale::MILLI) + .with_max_age(None); + let track = broadcast.broadcast().create_track("0", info).unwrap(); + broadcast.announce().unwrap(); + + let mut options = MockConnectOptions::new(version); + options.transport.max_concurrent_uni_streams = Some(1); + options.server_publish = Some(publisher.consume()); + options.client_subscribe = Some(viewer.clone()); + let _pair = connect_mock(options).await; + + let remote = viewer + .consume() + .follow("live", None) + .unwrap() + .available() + .await + .unwrap() + .unwrap(); + let prefs = track::Subscription::default() + .with_order(track::Order::Descending) + .with_delay(track::Delay::default().with_max(Duration::from_secs(30))); + let mut subscriber = remote.track("0").unwrap().subscribe(prefs).await.unwrap(); + macro_rules! next { + ($what:literal) => { + kio::clock::timeout(Duration::from_secs(5), subscriber.recv_group()) + .await + .unwrap_or_else(|_| panic!("{version}: {}", $what)) + .unwrap() + .unwrap() + }; + } + + // Unfinished, so its stream holds the only credit. + let mut held = track.create_group(moq_net::group::Info { sequence: 3 }).unwrap(); + held.write_frame(Timestamp::ZERO, b"held".as_slice()).unwrap(); + assert_eq!(next!("the held group arrives").sequence, 3, "{version}"); + + // Group 1 waits for credit it cannot make room for, and group 0 ranks below it. + for sequence in [1, 0] { + let mut late = track.create_group(moq_net::group::Info { sequence }).unwrap(); + late.write_frame(Timestamp::ZERO, b"late".as_slice()).unwrap(); + late.finish().unwrap(); + } + + let mut newer = track.create_group(moq_net::group::Info { sequence: 4 }).unwrap(); + newer.write_frame(Timestamp::ZERO, b"new".as_slice()).unwrap(); + newer.finish().unwrap(); + assert_eq!( + next!("the newer group waits behind the late ones").sequence, + 4, + "{version}" + ); + drop(held); + } +} + /// Oldest first, a backlog behind a group waiting for credit stays in the cache rather than /// giving it up: a newer group ranks below the waiting one, so every group still arrives once /// the stream holding the credit finishes, on each protocol. From cd880004a0c281227f8c51d30bb5f3d21be755b9 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:21:27 -0800 Subject: [PATCH 05/18] chore(quest): claim quest/m1/stream-credit/yield-findings.md Quest-Claim: 451926cae618e4ec From 7b60fe9f7c86ef9f7631cd500ec320475284099f Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:43:54 -0800 Subject: [PATCH 06/18] fix(moq-net): bound paused groups and fix the yield and resume review findings - TrackRun drops a paused group the budget or the cache gave up while it waited, keyed by sequence, so sustained congestion no longer grows `paused` without bound. - A stale drop superseded by a later SUBSCRIBE_UPDATE no longer ends a yielded group; it pauses (Rust and JS). - JS: a continuation whose group failed drops with that error, not a stale one. - Rust subscriber: a Frame Start near u64::MAX is a protocol violation, not a panic. - copy_fill.test.ts mocks time. Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/copy_fill.test.ts | 18 +- js/net/src/lite/group.test.ts | 27 ++- js/net/src/lite/publisher.test.ts | 24 +++ js/net/src/lite/publisher.ts | 13 +- js/net/src/lite/subscriber.test.ts | 48 +++++ js/net/src/lite/subscriber.ts | 24 ++- quest/m1/stream-credit/yield-findings.md | 35 ---- rs/moq-net/src/lite/publisher.rs | 63 ++++-- rs/moq-net/src/lite/subscriber.rs | 187 ++++++++++++++++-- rs/moq-net/src/tail.rs | 9 + rs/moq-net/tests/integration/stream_credit.rs | 65 ++++++ 11 files changed, 431 insertions(+), 82 deletions(-) delete mode 100644 quest/m1/stream-credit/yield-findings.md diff --git a/js/net/src/lite/copy_fill.test.ts b/js/net/src/lite/copy_fill.test.ts index 556534937d3..a4b5a0c0894 100644 --- a/js/net/src/lite/copy_fill.test.ts +++ b/js/net/src/lite/copy_fill.test.ts @@ -371,9 +371,11 @@ test("copy fill: a filled group nobody holds any more stops downloading", async }); // Lite-07: a filled group whose stream the publisher yielded, then gave up, ends with the fill's -// drop naming it, as one on a subscription does, rather than waiting paused on a continuation that -// never comes. +// drop naming it with the group's own error, as one on a subscription does, rather than waiting +// paused on a continuation that never comes. test("copy fill: a yielded group the fill's drop names ends", async () => { + const clock = new Manual(); + const installed = clock.install(); const pair = createMockTransportPair(Lite.ALPN_07_WIP); const origin = new OriginProducer(); const [client, server] = await Promise.all([ @@ -411,12 +413,15 @@ test("copy fill: a yielded group the fill's drop names ends", async () => { const open = lines(pair.server).streams.list(); expect(open.length).toBe(1); open[0]?.preempt(); - source.abort(new Error("gone")); + source.abort(new StreamError(StreamCode(70), { message: "gone" })); - // Still paused without it, which the deadline reports as elapsed. - const ended = await timeout(Duration.millis(1000), Promise.resolve(group?.closed)).catch((err: unknown) => err); + // Still paused without it, which the deadline, fired once every pending task settled, + // reports as elapsed. + const ended = await clock + .drive(timeout(Duration.millis(1000), Promise.resolve(group?.closed))) + .catch((err: unknown) => err); expect(ended).toBeInstanceOf(StreamError); - expect((ended as StreamError).code).toBe(StreamCode.Old); + expect((ended as StreamError).code).toBe(StreamCode(70)); history.close(); reader.close(); } finally { @@ -427,5 +432,6 @@ test("copy fill: a yielded group the fill's drop names ends", async () => { client.abort(); server.abort(); origin.close(); + installed.close(); } }); diff --git a/js/net/src/lite/group.test.ts b/js/net/src/lite/group.test.ts index dc0cf436de0..7c7d2009ee9 100644 --- a/js/net/src/lite/group.test.ts +++ b/js/net/src/lite/group.test.ts @@ -1,6 +1,7 @@ import { expect, test } from "bun:test"; -import { StreamCode, StreamError } from "../error.ts"; +import { ProtocolViolation, StreamCode, StreamError } from "../error.ts"; import { Producer } from "../group.ts"; +import { hooks } from "../internal.ts"; import { concat, encodeVarint, Reader } from "../stream.ts"; import { SEGMENT_BYTES } from "../util/limits.ts"; import * as Varint from "../varint.ts"; @@ -230,3 +231,27 @@ test("a lite-07 stream ending at the Payload Length ends the group", async () => readFrames(streamOf([cut], version), new Producer(0n), { scale: SCALE, lengthFirst: true }), ).rejects.toThrow("unexpected end of stream"); }); + +// A peer's huge Frame Start numbers a frame past what a number counts exactly, which is refused +// as a protocol violation rather than counted wrong. Mirrors the Rust +// `group_frame_start_past_any_count_is_refused`. +test("a lite-07 frame numbered past the safe integer range is refused", async () => { + const version = Version.DRAFT_07; + const frame = concat([frameHeader(version, 0n, undefined, 1, true), new Uint8Array([7])]); + const start = Number.MAX_SAFE_INTEGER - 1; + const producer = new Producer(0n); + hooks.startGroup(producer, start); + const err = await readFrames( + streamOf([frame, frame], version), + producer, + { scale: SCALE, lengthFirst: true }, + { + index: start, + }, + ).then( + () => undefined, + (err: unknown) => err, + ); + expect(err).toBeInstanceOf(ProtocolViolation); + expect(producer.consume().frameCount).toBe(Number.MAX_SAFE_INTEGER); +}); diff --git a/js/net/src/lite/publisher.test.ts b/js/net/src/lite/publisher.test.ts index ea17ef20996..3dcf268056c 100644 --- a/js/net/src/lite/publisher.test.ts +++ b/js/net/src/lite/publisher.test.ts @@ -967,6 +967,30 @@ test("lite-07 continues a yielded cached tail from its Frame Start", async () => } }); +// Lite-07: a yielded group the publisher aborts before its continuation is dropped with the +// publisher's error, not as stale. Mirrors the Rust `lite07_yielded_group_aborted_drops_with_its_error`. +test("lite-07 drops a yielded group the publisher aborted with its error", async () => { + const version = Version.DRAFT_07; + const sub = await servedSubscription({ version, startGroup: 0n }); + try { + const group = sub.track.createGroup(0n); + group.writeFrame(textFrame("a", Timestamp.fromMillis(0))); + expect(await sub.servedHeader()).toBe(0n); + expect("start" in (await decodeSubscribeResponse(sub.client.reader, version))).toBe(true); + + const [open] = lines(sub.quic).streams.list(); + if (!open) throw new Error("the group stream is not open"); + open.preempt(); + group.abort(new StreamError(StreamCode(70), { message: "cut" })); + + const resp = await decodeSubscribeResponse(sub.client.reader, version); + if (!("drop" in resp)) throw new Error("expected SUBSCRIBE_DROP"); + expect([resp.drop.start, resp.drop.end, resp.drop.error]).toEqual([0n, 1n, 70]); + } finally { + await sub.close(); + } +}); + // A range an update adds learns where it starts from SUBSCRIBE_STARTED, and keeps that start however // many later updates leave it live, rather than counting back from a newer latest group. test("lite-07 SUBSCRIBE_STARTED reports an added range's start, which later updates keep", async () => { diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index 5abd17a83f3..a96ffabfa39 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -2062,7 +2062,12 @@ export class Publisher { // The subscriber keeps what arrived, and a new stream continues the group from // the reset one's Frame Start, owing no drop. Mirrors the Rust `GroupServe::settle_failed`. if (!(writer instanceof Error) && writer.preempted && hasYield(this.version)) { - const resumed = continuation(serving); + // The group's own failure, such as the publisher's abort or an eviction, ends + // it with that error, as the Rust continuation reads it off the group. Read + // before the release replaces it with the reset's. + const fate = serving.group.closed.peek(); + hooks.releaseGroupReader(serving.group, failed); + const resumed = fate instanceof Error ? undefined : continuation(serving); if (resumed) { console.debug( `continuing paused group: track=${track.name} sequence=${group.sequence} frame_start=${serving.start}`, @@ -2071,7 +2076,7 @@ export class Publisher { continue; } // Given up meanwhile, such as by the subscription's budget: the usual stale drop. - failed = new StreamError(StreamCode.Old); + failed = fate instanceof Error ? fate : new StreamError(StreamCode.Old); } if (emitRange) { pendingDrops.push({ start: group.sequence, end: group.sequence + 1n, error: failed }); @@ -2645,7 +2650,9 @@ export class Publisher { } catch (err: unknown) { const e = error(err); stream.reset(e); - hooks.releaseGroupReader(group, e); + // A reset that pauses the group leaves the reader to the caller, which reads its fate + // first. A reset stream is never preempted after, so the caller sees the same flag. + if (!(stream.preempted && hasYield(this.version))) hooks.releaseGroupReader(group, e); return e; } finally { priority.remove(stream); diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index 8e3c92a4f72..26386c48b4e 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -1556,6 +1556,54 @@ test("a yielded group a drop named ends", async () => { } }); +// Lite-07: a stale drop that overtook the YIELDED reset of the stream it names holds only under the +// update it names: once the subscriber sends another, the reset pauses the group, which the +// publisher then continues or drops again. Mirrors the Rust +// `a_yielded_group_pauses_once_an_update_supersedes_its_stale_drop`. +test("a yielded group pauses once an update supersedes its stale drop", async () => { + const version = Version.DRAFT_07; + const { quic, streams, opened } = fakeSession(); + const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); + const track = subscriber + .consume(Path.from("room")) + .track("video") + .subscribe({ delay: { max: Duration.millis(60_000) } }); + try { + await opened(0); + await answerTrackInfo(streams[0], version, 1000); + const request = await opened(1); + await request.reading; + await respond(request, version, [{ start: new SubscribeStart([0n]) }]); + + const first = handStream(version); + first.write(await liteFrame(300, new Uint8Array([1]))); + const yielded = subscriber.runGroup(new Group({ subscribe: 0n, sequence: 3n }), first.reader); + const group = await track.recvGroup(); + expect(group?.sequence).toBe(3n); + expect((await group?.readFrame())?.payload).toEqual(new Uint8Array([1])); + + await respond(request, version, [{ drop: new SubscribeDrop({ start: 3n, end: 4n, error: StreamCode.Old }) }]); + await drainReactions(); + track.update({ priority: 1, delay: { max: Duration.millis(60_000) } }); + await drainReactions(); + first.reset(StreamCode.Yielded); + await yielded; + await drainReactions(); + + expect(group?.closed.peek()).toBeUndefined(); + // The continuation picks the group up where the reset left it. + const second = handStream(version); + second.write(await liteFrame(10, new Uint8Array([2]))); + second.finish(); + await subscriber.runGroup(new Group({ subscribe: 0n, sequence: 3n, frameStart: 1 }), second.reader); + expect((await group?.readFrame())?.payload).toEqual(new Uint8Array([2])); + expect(await group?.readFrame()).toBeUndefined(); + } finally { + track.close(); + subscriber.close(); + } +}); + // Lite-07: a malformed FRAME on a Group Stream ends only that stream and its group, never the // session: a Payload Length of 0, or a continuation that resends the cut frame at another size, ends // before completing it, or starts past the frames held. Mirrors the Rust `UniState::Group`. diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 13afdd14454..e73370671ae 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -272,6 +272,8 @@ interface Yielding { owed: boolean; // The error a drop named the group with while the stream was read. dropped?: Error; + // For a stale drop, the update it named: the next update supersedes it. + staleUnder?: bigint; } // The paused groups of `entry`, created on first use. @@ -2936,7 +2938,7 @@ export class Subscriber { const overtaken = stale ? entry.tail.overtaken(start, end) : []; entry.tail.account(start, end); // A paused group the drop names ends there: no continuation follows. - this.#dropPaused(entry, start, end, error); + this.#dropPaused(entry, start, end, error, stale ? update : undefined); // Judged under the budget this subscriber sent with that update. const hole = stale ? staleHole(entry.maxDelay ?? 0) : { error }; // A group whose stream finished under this update was served over a stale drop, so it @@ -3325,7 +3327,10 @@ export class Subscriber { this.#endYielding(entry, group.sequence, yielding); // A drop named the group first: the publisher gave it up, so nothing continues it. One // that overtook the stream's header carried a stale verdict. - const given = yielding.dropped ?? (yielding.owed ? undefined : new StreamError(StreamCode.Old)); + // A stale drop an update sent since superseded no longer ends the group. + const superseded = yielding.staleUnder !== undefined && yielding.staleUnder < entry.update; + const dropped = superseded ? undefined : yielding.dropped; + const given = dropped ?? (yielding.owed ? undefined : new StreamError(StreamCode.Old)); if (given) { producer.abort(given); return; @@ -3390,12 +3395,17 @@ export class Subscriber { if (entry.yielding?.get(sequence) === yielding) entry.yielding.delete(sequence); } - // A SUBSCRIBE_DROP named `[start, end)` with `error`: a paused group ends there, and one a stream - // is reading ends with it if a YIELDED reset would otherwise pause it, since no continuation - // follows. Mirrors the Rust `TrackEntry::dropped`. - #dropPaused(entry: SubscribeEntry, start: bigint, end: bigint, error: Error): void { + // A SUBSCRIBE_DROP named `[start, end)` with `error`, under update `staleUnder` when it is a + // lite-07 stale verdict: a paused group ends there, and one a stream is reading ends with it if + // a YIELDED reset would otherwise pause it, since no continuation follows, unless an update + // supersedes a stale drop first. Mirrors the Rust `TrackEntry::dropped`. + #dropPaused(entry: SubscribeEntry, start: bigint, end: bigint, error: Error, staleUnder?: bigint): void { for (const [sequence, yielding] of entry.yielding ?? []) { - if (sequence >= start && sequence < end) yielding.dropped = error; + if (sequence < start || sequence >= end) continue; + // A final verdict stands, whatever stale one follows. + if (staleUnder !== undefined && yielding.dropped && yielding.staleUnder === undefined) continue; + yielding.dropped = error; + yielding.staleUnder = staleUnder; } const paused = entry.paused; if (!paused) return; diff --git a/quest/m1/stream-credit/yield-findings.md b/quest/m1/stream-credit/yield-findings.md deleted file mode 100644 index 5fcb5eed0fb..00000000000 --- a/quest/m1/stream-credit/yield-findings.md +++ /dev/null @@ -1,35 +0,0 @@ -# [M] Fix review findings on group yield and resume - -## Goal - -The five Codex findings on [moq-dev/moq#8545](https://github.com/moq-dev/moq/pull/8545)'s -merged code are each reproduced, and every real one is fixed at its root -cause in Rust and JS alike; a finding that does not reproduce is recorded -here with why. - -## Plan - -Decided 2026-11-13 (maintainer): ✅ one follow-up quest (rejected: fix now -in a new PR without a quest; ignore). Reproduce each finding with a failing -test before fixing it; fix 1 first. - -1. (high) Rust `TrackRun::paused` in `rs/moq-net/src/lite/publisher.rs` has - no bound under sustained congestion. Its comment says stream credit - bounds it, but a continued group can be paused again, so the real bound - may be the cached groups. Bound it by design (by group identity, so a - group re-paused replaces its entry) and fix the comment; mirror in - `js/net/src/lite/publisher.ts`. -2. A stale drop arriving before the YIELDED reset survives a later - SUBSCRIBE_UPDATE that supersedes it, so the group ends instead of - pausing (Rust and JS). Check against the `/verify-subscribe` model. -3. JS `js/net/src/lite/publisher.ts`: a continuation missing from the cache - reports stale instead of the publisher's real error. -4. Rust `rs/moq-net/src/lite/subscriber.rs`: a huge Frame Start can overflow - the frame counter and panic. Peer input must never panic: reject it as a - protocol error. Check the JS subscriber for the same. -5. `js/net/src/lite/copy_fill.test.ts` waits on a real 1-second clock; - mock time instead. - -## Related - -- [Takeover](/quest/m1/stream-credit/takeover.md) - the yield and resume work these findings review diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 2ac9077b5d4..74163cb4156 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -5129,10 +5129,12 @@ struct TrackRun { // The expired groups an update re-offers, still cached and in range, each served again // on its own stream as stream credit allows. reoffered: std::collections::VecDeque, - // Lite-07: the paused groups, each continued on its own stream ahead of re-offers and new - // groups, the highest-ranked first. Each paused while holding a stream, so stream credit - // bounds them. - paused: std::collections::VecDeque, + // Lite-07: the paused groups by sequence, each continued on its own stream ahead of + // re-offers and new groups, the highest-ranked first. Credit returns with each pause, so + // sustained congestion can pause every group in turn: one the budget or the cache gives + // up while it waits is dropped then, which bounds them by what the cache holds, and a + // group paused again replaces its entry. + paused: std::collections::BTreeMap, // Lite-07: the group `opening` holds continues a paused one. continuing: bool, end_sent: bool, @@ -5508,8 +5510,10 @@ impl TrackRun { if !paused.is_empty() { // The track side continues them. Idle::disarm(&mut self.idle); - self.paused.extend(paused); + self.paused + .extend(paused.into_iter().map(|paused| (paused.group.sequence, paused))); } + self.drop_given_up_paused(waiter); let dropped = std::mem::take(&mut *self.ctx.outcomes.dropped.lock().unwrap()); if !dropped.is_empty() { for (sequence, err) in dropped { @@ -5609,9 +5613,9 @@ impl TrackRun { let order = self.ctx.steer_current().order; let stuck = self.stuck(); if let Some(best) = self.best_paused() - && (self.opening.is_none() || stuck.is_some_and(|waiting| self.paused[best].rank(order) > waiting)) + && (self.opening.is_none() || stuck.is_some_and(|waiting| self.paused[&best].rank(order) > waiting)) { - let Paused { group, frame_start } = self.paused.remove(best).expect("best is in range"); + let Paused { group, frame_start } = self.paused.remove(&best).expect("best is paused"); tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, sequence = group.sequence, frame_start, "continuing paused group"); self.give_way(); self.open(group, frame_start); @@ -6093,10 +6097,13 @@ impl TrackRun { }; tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, sequence = waiting.sequence, "giving way to a higher-ranked group"); match std::mem::take(&mut self.continuing) { - true => self.paused.push_back(Paused { - group: waiting.group.clone(), - frame_start: waiting.frame_start, - }), + true => { + let paused = Paused { + group: waiting.group.clone(), + frame_start: waiting.frame_start, + }; + self.paused.insert(waiting.sequence, paused); + } false => waiting.settle_failed(&Error::Old), } } @@ -6114,16 +6121,34 @@ impl TrackRun { } } - /// The position of the highest-ranked paused group in this subscription's current order, + /// The sequence of the highest-ranked paused group in this subscription's current order, /// the next to continue. - fn best_paused(&mut self) -> Option { + fn best_paused(&mut self) -> Option { let order = self.ctx.steer_current().order; - (0..self.paused.len()).reduce(|best, next| { - match self.paused[next].rank(order) > self.paused[best].rank(order) { - true => next, - false => best, - } - }) + self.paused + .iter() + .max_by_key(|(_, paused)| paused.rank(order)) + .map(|(&sequence, _)| sequence) + } + + /// Drop each paused group the subscription's budget or the cache gave up while it waited + /// to continue, as a group waiting for credit would be. + fn drop_given_up_paused(&mut self, waiter: &kio::Waiter) { + if self.paused.is_empty() { + return; + } + let mut dropped = Vec::new(); + self.paused + .retain(|&sequence, paused| match poll_given_up(&mut paused.group, waiter) { + Some(err) => { + dropped.push((sequence, StreamError::from(&err))); + false + } + None => true, + }); + if !dropped.is_empty() && self.ctx.version.has_track_stream() { + self.ctx.outcomes.dropped.lock().unwrap().extend(dropped); + } } /// Open a group machine streaming `group` from `frame_start`, which waits for stream diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 48f72cd6c1e..4a49bf25001 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -268,6 +268,8 @@ struct Yielding { owed: bool, /// The error a drop named the group with while the stream was read. dropped: Option, + /// For a stale drop, the updates sent when it arrived: the next update supersedes it. + stale_under: Option, } /// What a group stream's header opened in its subscription's copy. @@ -577,6 +579,7 @@ impl TrackEntry { stream, owed, dropped: None, + stale_under: None, }, ); stream @@ -584,20 +587,36 @@ impl TrackEntry { /// Stop tracking the stream reading `sequence`, returning whether a YIELDED reset pauses it. /// A stream that a later one for the same group replaced finds nothing, leaving the - /// later stream's record in place. + /// later stream's record in place. A stale drop an update sent since superseded no + /// longer ends the group. fn end_yielding(&mut self, sequence: u64, stream: u64) -> Option { - match self.yielding.get(&sequence) { - Some(yielding) if yielding.stream == stream => self.yielding.remove(&sequence), - _ => None, + let mut yielding = match self.yielding.get(&sequence) { + Some(yielding) if yielding.stream == stream => self.yielding.remove(&sequence)?, + _ => return None, + }; + if yielding + .stale_under + .is_some_and(|updates| updates < self.tail.read().updates()) + { + yielding.dropped = None; + yielding.stale_under = None; } + Some(yielding) } - /// A SUBSCRIBE_DROP named `groups` with `err`: a paused group ends there, and one a stream - /// is reading ends with it if a YIELDED reset would otherwise pause it, since no - /// continuation follows. - fn dropped(&mut self, groups: std::ops::Range, err: &Error) { + /// A SUBSCRIBE_DROP named `groups` with `err`, `stale` when it is a lite-07 stale verdict: a + /// paused group ends there, and one a stream is reading ends with it if a YIELDED reset + /// would otherwise pause it, since no continuation follows, unless an update supersedes a + /// stale drop first. + fn dropped(&mut self, groups: std::ops::Range, err: &Error, stale: bool) { + let updates = stale.then(|| self.tail.read().updates()); for (_, yielding) in self.yielding.range_mut(groups.clone()) { + // A final verdict stands, whatever stale one follows. + if stale && yielding.dropped.is_some() && yielding.stale_under.is_none() { + continue; + } yielding.dropped = Some(err.clone()); + yielding.stale_under = updates; } let Some(registry) = &self.paused else { return; @@ -1982,7 +2001,8 @@ impl FrameIngest { /// one the group holds already, skipped, or the one a YIELDED reset cut short, completed. fn dispatch(&mut self, group: &crate::recv::Group, header: frame::Header) -> Result { let index = self.index; - self.index += 1; + // The peer's Frame Start can number a frame past any count. + self.index = index.checked_add(1).ok_or(Error::ProtocolViolation)?; let held = group.frame_count() as u64; if index >= held { return Ok(IngestPhase::Admit { header }); @@ -5857,7 +5877,7 @@ mod tests { .lock() .get_mut(&7) .unwrap() - .dropped(3..4, &Error::Stream(crate::StreamError::Old)); + .dropped(3..4, &Error::Stream(crate::StreamError::Old), true); yielded.close(Close::ResetStream(crate::StreamError::Yielded.to_code())); let res = kio::wait(|waiter| first.poll_serve(waiter)).await; assert!(res.is_ok(), "{res:?}"); @@ -5870,6 +5890,89 @@ mod tests { ); } + /// Lite-07: a stale drop that overtook the YIELDED reset of the stream it names holds only + /// under the update it names: once the subscriber sends another, the reset pauses the + /// group, which the publisher then continues or drops again. + #[kio::test] + async fn a_yielded_group_pauses_once_an_update_supersedes_its_stale_drop() { + use crate::lite::test_transport::{Close, ScriptedSession}; + use crate::transport::poll::Session as _; + + let version = Version::Lite07; + let mut script = Vec::new(); + let mut encoder = crate::coding::Encoder::new(&mut script, version.into()); + lite::Group { + subscribe: 7, + sequence: 3, + frame_start: 0, + } + .encode(&mut encoder, version) + .unwrap(); + encoder.varint(2).unwrap(); + encoder.varint(crate::coding::varint::zigzag(300)).unwrap(); + script.push(b'x'); + let yielded = ScriptedSession::new(script); + let clock = crate::time::Deadlines::new(crate::time::Instant::now()); + let subscriber = Subscriber::new(SubscriberConfig { + runtime: clock.clone(), + session: crate::driver::Transport::new(yielded.clone()), + origin: origin::Config::new(std::num::NonZeroU64::new(1).unwrap()).produce(), + recv_bandwidth: None, + version, + peer_setup: crate::lite::PeerSetup::new(false, crate::transport::Address::Setup), + peer_hop: None, + cost: None, + going_away: Default::default(), + auth: crate::auth::Link::new(false), + }); + let track = track::Producer::new( + std::sync::Arc::new(crate::broadcast::Info::default()), + "video", + crate::track::Info::timed(), + ); + let subscription = crate::track::Subscription::default() + .with_delay(crate::track::Delay::default().with_max(std::time::Duration::from_secs(10))) + .with_range(0..); + let mut reader = track.subscribe(subscription); + let framing = lite::Framing { + timescale: Some(Timescale::MILLI), + ietf: None, + compressed: false, + length_first: true, + }; + let tail = kio::Producer::new(Tail::default()); + subscriber + .subscribes + .lock() + .insert(7, TrackEntry::accepted(track.clone(), framing, tail.clone())); + + let (_, recv) = yielded.open_bi(Default::default()).await.unwrap(); + let mut first = GroupRecv::new( + subscriber.clone(), + Reader::new(crate::transport::boxed::RecvStream::new(recv), version), + ); + assert!(first.poll_serve(&kio::Waiter::noop()).is_pending()); + let mut group = reader.assert_group(); + // The publisher gave the group up after pausing it, and its drop lands first. + subscriber + .subscribes + .lock() + .get_mut(&7) + .unwrap() + .dropped(3..4, &Error::Stream(crate::StreamError::Old), true); + // An update supersedes it, as `buffer_update` does. + tail.write().ok().expect("the tail is open").updated(); + yielded.close(Close::ResetStream(crate::StreamError::Yielded.to_code())); + let res = kio::wait(|waiter| first.poll_serve(waiter)).await; + assert!(res.is_ok(), "{res:?}"); + + let read = group.read_frame().await; + assert!(matches!(read, Ok(Some(_))), "the frame that arrived stays: {read:?}"); + let subs = subscriber.subscribes.lock(); + let paused = subs.get(&7).unwrap().paused.as_ref().unwrap().read(); + assert!(paused.groups.contains_key(&3), "the group pauses"); + } + /// A SUBSCRIBE_FINISH below a group already received contradicts that group. lite-05 /// specified an inclusive end, and `@moq/net` 0.1.3 to 0.1.9 sent one, so there it only /// costs the early boundary; later drafts abort the track. @@ -10100,6 +10203,68 @@ mod tests { let _a = slots.acquire().expect("the failed stream kept a slot"); let _b = slots.acquire().expect("the failed stream kept a slot"); } + /// A frame numbered past `u64::MAX` by a huge Frame Start is a protocol violation, not + /// a counter overflow. + #[kio::test] + async fn group_frame_start_past_any_count_is_refused() { + use crate::transport::poll::Session as _; + let version = Version::Lite07; + let session = crate::lite::test_transport::ScriptedSession::eof(Vec::new()); + let subscriber = Subscriber::new(SubscriberConfig { + runtime: crate::time::sim(), + session: crate::driver::Transport::new(session), + origin: origin::Config::new(std::num::NonZeroU64::new(1).unwrap()).produce(), + recv_bandwidth: None, + version, + peer_setup: crate::lite::PeerSetup::new(false, crate::transport::Address::Setup), + peer_hop: None, + cost: None, + going_away: Default::default(), + auth: crate::auth::Link::new(false), + }); + let track = track::Producer::new( + std::sync::Arc::new(crate::broadcast::Info::default()), + "video", + crate::track::Info::timed(), + ); + subscriber.subscribes.lock().insert( + 7, + TrackEntry::accepted( + track.clone(), + lite::Framing { + timescale: Some(Timescale::MILLI), + ietf: None, + compressed: false, + length_first: false, + }, + Default::default(), + ), + ); + // The model takes the first frame at the last countable index, so the second frame's + // index is the one that cannot be counted. + let mut script = Vec::new(); + let mut encoder = crate::coding::Encoder::new(&mut script, version.into()); + lite::Group { + subscribe: 7, + sequence: 0, + frame_start: u64::MAX - 1, + } + .encode(&mut encoder, version) + .unwrap(); + for _ in 0..2 { + encoder.varint(crate::coding::varint::zigzag(0)).unwrap(); + encoder.bytes(b"x").unwrap(); + } + let incoming = crate::lite::test_transport::ScriptedSession::eof(script); + let (_, recv) = incoming.open_bi(Default::default()).await.unwrap(); + let mut group = GroupRecv::new( + subscriber.clone(), + Reader::new(crate::transport::boxed::RecvStream::new(recv), version), + ); + let err = kio::wait(|waiter| group.poll_serve(waiter)).await.unwrap_err(); + assert!(matches!(err, Error::ProtocolViolation), "{err:?}"); + } + /// Distinct live GROUP starts are copies of the same group, including while /// readers retain every earlier tail. Exercise the real header/ingest path. #[kio::test] @@ -12694,7 +12859,7 @@ impl ServeLoop { } // A paused group the drop names ends there: no continuation follows. if let Some(entry) = serve.subscriber.subscribes.lock().get_mut(&active.id) { - entry.dropped(groups.clone(), &Error::Stream(error.clone())); + entry.dropped(groups.clone(), &Error::Stream(error.clone()), stale); } match stale { // Judged under the budget this subscriber sent with that update. A diff --git a/rs/moq-net/src/tail.rs b/rs/moq-net/src/tail.rs index a379276dd75..66ee5f1058a 100644 --- a/rs/moq-net/src/tail.rs +++ b/rs/moq-net/src/tail.rs @@ -38,6 +38,8 @@ pub(crate) struct Tail { // A stale drop there for a group the ledger already accounted may be one its stream // overtook, so the next update owes none of it again. folded: Option>, + // How many updates were sent, so a stale drop can tell whether a later one superseded it. + updates: u64, // What `finished` bills to the track's account, [`FINISHED_OVERHEAD`] per run, once the // subscription has a copy to bill. bill: Option, @@ -64,6 +66,7 @@ impl Tail { ended: false, finished: RangeSet::new(), folded: None, + updates: 0, bill: None, } } @@ -187,11 +190,17 @@ impl Tail { /// Forget which streams finished, for an update being sent: a stale drop under an earlier /// one is no verdict, so none needs them. pub fn updated(&mut self) { + self.updates += 1; self.finished = RangeSet::new(); self.folded = None; self.rebill(); } + /// How many updates were sent, each superseding the stale drops before it. + pub fn updates(&self) -> u64 { + self.updates + } + /// Whether a data stream whose header arrived is still being read. pub fn is_reading(&self) -> bool { self.active > 0 diff --git a/rs/moq-net/tests/integration/stream_credit.rs b/rs/moq-net/tests/integration/stream_credit.rs index 4cc282622ac..d876027deb5 100644 --- a/rs/moq-net/tests/integration/stream_credit.rs +++ b/rs/moq-net/tests/integration/stream_credit.rs @@ -711,3 +711,68 @@ async fn an_ascending_backlog_waits_behind_a_waiting_group() { assert_eq!(received, [1, 2, 3], "{version}"); } } + +/// On lite-07 a paused group waiting behind another paused group's continuation is still +/// given up once its publisher aborts it, rather than held until credit returns: the paused +/// groups are bounded by what the cache still holds, however long congestion lasts. +#[kio::test] +async fn a_paused_group_given_up_while_it_waits_is_dropped() { + let version: Version = "moq-lite-07-wip".parse().unwrap(); + let publisher = origin(1); + let viewer = origin(2); + let broadcast = publisher + .route("live", moq_net::origin::RouteOptions::hidden()) + .unwrap(); + let info = track::Info::default() + .with_timescale(moq_net::Timescale::MILLI) + .with_max_age(None); + let track = broadcast.broadcast().create_track("0", info).unwrap(); + broadcast.announce().unwrap(); + + let mut options = MockConnectOptions::new(version); + options.transport.max_concurrent_uni_streams = Some(1); + options.server_publish = Some(publisher.consume()); + options.client_subscribe = Some(viewer.clone()); + let _pair = connect_mock(options).await; + + let remote = viewer + .consume() + .follow("live", None) + .unwrap() + .available() + .await + .unwrap() + .unwrap(); + let prefs = track::Subscription::default() + .with_order(track::Order::Descending) + .with_delay(track::Delay::default().with_max(Duration::from_secs(30))); + let mut subscriber = remote.track("0").unwrap().subscribe(prefs).await.unwrap(); + + // Each unfinished group resets the one before: the newest holds the only credit, group + // 1's continuation waits for it, and group 0 waits among the paused behind that. + let mut groups = Vec::new(); + let mut readers = Vec::new(); + for sequence in 0..3 { + let mut group = track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, b"a".as_slice()).unwrap(); + let mut reader = kio::clock::timeout(Duration::from_secs(5), subscriber.recv_group()) + .await + .expect("a group arrives") + .unwrap() + .unwrap(); + assert_eq!(reader.sequence, sequence); + let first = kio::clock::timeout(Duration::from_secs(5), reader.read_frame()) + .await + .expect("its first frame arrives") + .unwrap(); + assert_eq!(first.map(|frame| frame.payload), Some(bytes::Bytes::from_static(b"a"))); + groups.push(group); + readers.push(reader); + } + + groups.remove(0).abort(moq_net::Error::Cancel).unwrap(); + let end = kio::clock::timeout(Duration::from_secs(5), readers[0].read_frame()) + .await + .expect("the paused group is dropped while the others hold the credit"); + assert!(end.is_err(), "the aborted paused group fails: {end:?}"); +} From af7e4b430ba28f44fc0a15e33220ad0412d8eb88 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:41:18 -0800 Subject: [PATCH 07/18] fix(moq-net): supersede a pre-header stale drop on update, and rank only resettable streams as held Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/subscriber.test.ts | 47 +++++++ js/net/src/lite/subscriber.ts | 15 ++- rs/moq-net/src/lite/publisher.rs | 12 +- rs/moq-net/src/lite/subscriber.rs | 118 ++++++++++++++++-- rs/moq-net/src/model/track.rs | 11 ++ rs/moq-net/tests/integration/stream_credit.rs | 73 +++++++++++ 6 files changed, 260 insertions(+), 16 deletions(-) diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index 26386c48b4e..80e70a63702 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -1604,6 +1604,53 @@ test("a yielded group pauses once an update supersedes its stale drop", async () } }); +// Lite-07: a stale drop that overtook the header of the stream it names holds only under the update +// it names: once the subscriber sends another, a YIELDED reset of that stream pauses the group. +// Mirrors the Rust `a_yielded_group_pauses_once_an_update_supersedes_a_stale_drop_before_its_header`. +test("a yielded group pauses once an update supersedes a stale drop before its header", async () => { + const version = Version.DRAFT_07; + const { quic, streams, opened } = fakeSession(); + const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); + const track = subscriber + .consume(Path.from("room")) + .track("video") + .subscribe({ delay: { max: Duration.millis(60_000) } }); + try { + await opened(0); + await answerTrackInfo(streams[0], version, 1000); + const request = await opened(1); + await request.reading; + await respond(request, version, [{ start: new SubscribeStart([0n]) }]); + + await respond(request, version, [{ drop: new SubscribeDrop({ start: 3n, end: 4n, error: StreamCode.Old }) }]); + await drainReactions(); + const first = handStream(version); + first.write(await liteFrame(300, new Uint8Array([1]))); + const yielded = subscriber.runGroup(new Group({ subscribe: 0n, sequence: 3n }), first.reader); + const group = await track.recvGroup(); + expect(group?.sequence).toBe(3n); + expect((await group?.readFrame())?.payload).toEqual(new Uint8Array([1])); + + track.update({ priority: 1, delay: { max: Duration.millis(60_000) } }); + await drainReactions(); + first.reset(StreamCode.Yielded); + await yielded; + await drainReactions(); + + expect(group?.closed.peek()).toBeUndefined(); + // The continuation picks the group up where the reset left it. + const second = handStream(version); + second.write(await liteFrame(10, new Uint8Array([2]))); + second.finish(); + await subscriber.runGroup(new Group({ subscribe: 0n, sequence: 3n, frameStart: 1 }), second.reader); + expect((await group?.readFrame())?.payload).toEqual(new Uint8Array([2])); + expect(await group?.readFrame()).toBeUndefined(); + } finally { + track.close(); + subscriber.close(); + } +}); + // Lite-07: a malformed FRAME on a Group Stream ends only that stream and its group, never the // session: a Payload Length of 0, or a continuation that resends the cut frame at another size, ends // before completing it, or starts past the frames held. Mirrors the Rust `UniState::Group`. diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index e73370671ae..594edff9910 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -274,6 +274,9 @@ interface Yielding { dropped?: Error; // For a stale drop, the update it named: the next update supersedes it. staleUnder?: bigint; + // When a stale drop that overtook the stream is all that left the group unowed, the update it + // named: once another is sent, the group is owed again. Mirrors the Rust `Yielding::unowed_under`. + unowedUnder?: bigint; } // The paused groups of `entry`, created on first use. @@ -3251,6 +3254,12 @@ export class Subscriber { const yields = hasYield(this.version); // Asked before this stream accounts for the group: a drop that overtook it named it. let owed = !tail.covers(group.sequence, group.sequence + 1n); + // A stale drop that overtook this stream: the next update supersedes that verdict, so a + // YIELDED reset after it pauses the group. + const unowedUnder = + !owed && entry.stale?.some((run) => run.start <= group.sequence && group.sequence < run.end) + ? entry.update + : undefined; let producer: netGroup.Producer | undefined; let ingest: Ingest | undefined; let yielding: Yielding | undefined; @@ -3298,7 +3307,7 @@ export class Subscriber { }); if (yields) { ingest ??= { index: group.frameStart }; - yielding = { owed }; + yielding = { owed, unowedUnder: owed ? undefined : unowedUnder }; entry.yielding ??= new Map(); entry.yielding.set(group.sequence, yielding); } @@ -3330,7 +3339,9 @@ export class Subscriber { // A stale drop an update sent since superseded no longer ends the group. const superseded = yielding.staleUnder !== undefined && yielding.staleUnder < entry.update; const dropped = superseded ? undefined : yielding.dropped; - const given = dropped ?? (yielding.owed ? undefined : new StreamError(StreamCode.Old)); + const owedNow = + yielding.owed || (yielding.unowedUnder !== undefined && yielding.unowedUnder < entry.update); + const given = dropped ?? (owedNow ? undefined : new StreamError(StreamCode.Old)); if (given) { producer.abort(given); return; diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 74163cb4156..7caabc54e6e 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -6076,14 +6076,20 @@ impl TrackRun { } /// The rank of the group waiting for stream credit, if it cannot make room: it ranks below - /// every stream this subscription holds, so it waits for credit to return, while a group - /// that outranks one of those streams would take it over now. + /// every stream this subscription holds that a reset can still take, so it waits for credit + /// to return, while a group that outranks one of those streams would take it over now. A + /// stream past its FIN holds credit until acknowledged, but nothing can reset it. fn stuck(&mut self) -> Option { let waiting = self.opening.as_ref()?; let order = self.ctx.steer_current().order; let rank = |serve: &GroupServe| crate::priority::in_group(serve.sequence, serve.frame_start, order); let waiting = rank(waiting); - let mut held = self.children.iter().map(rank).peekable(); + let mut held = self + .children + .iter() + .filter(|serve| matches!(serve.state, GroupState::Serve { .. })) + .map(rank) + .peekable(); (held.peek().is_some() && held.all(|held| held > waiting)).then_some(waiting) } diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 4a49bf25001..a8129dbd88a 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -270,6 +270,9 @@ struct Yielding { dropped: Option, /// For a stale drop, the updates sent when it arrived: the next update supersedes it. stale_under: Option, + /// When a stale drop that overtook the stream is all that left the group unowed, the + /// updates sent when the stream arrived: once another is sent, the group is owed again. + unowed_under: Option, } /// What a group stream's header opened in its subscription's copy. @@ -286,6 +289,8 @@ struct Opened { partial: Option>, /// The publisher owed the group when the stream arrived; see [`Yielding::owed`]. owed: bool, + /// See [`Yielding::unowed_under`]. + unowed_under: Option, track: track::Producer, framing: lite::Framing, reading: Reading, @@ -462,6 +467,9 @@ impl TrackEntry { return Err(Error::Cancel); }; let group_info = group::Info { sequence: hdr.sequence }; + // A stale drop that overtook this stream, which the group replaces: the next update + // supersedes that verdict, so a YIELDED reset after it pauses the group. + let stale = producer.holds_stale(hdr.sequence); // Stats (groups/frames/bytes) are counted in the model as the group // is written, through the tagged `track::Producer`. The track counts // it as content once its first frame lands. @@ -493,7 +501,8 @@ impl TrackEntry { } } } - self.place(hdr, version, received, reading, owed) + let unowed_under = (!owed && stale).then(|| self.tail.read().updates()); + self.place(hdr, version, received, reading, owed, unowed_under) } /// Whether a YIELDED reset paused the group `sequence`, which the cache still holds. One the @@ -546,6 +555,7 @@ impl TrackEntry { group, partial: partial.map(Box::new), owed: true, + unowed_under: None, track: producer.clone(), framing: *framing, reading, @@ -567,10 +577,10 @@ impl TrackEntry { } /// Track a stream reading `sequence` that a YIELDED reset would pause, which `owed` says - /// the publisher owed when it arrived. + /// the publisher owed when it arrived; see [`Yielding::unowed_under`]. /// /// Returns the stream's token for [`Self::end_yielding`]. - fn read_yielding(&mut self, sequence: u64, owed: bool) -> u64 { + fn read_yielding(&mut self, sequence: u64, owed: bool, unowed_under: Option) -> u64 { self.yield_streams += 1; let stream = self.yield_streams; self.yielding.insert( @@ -580,6 +590,7 @@ impl TrackEntry { owed, dropped: None, stale_under: None, + unowed_under, }, ); stream @@ -594,13 +605,15 @@ impl TrackEntry { Some(yielding) if yielding.stream == stream => self.yielding.remove(&sequence)?, _ => return None, }; - if yielding - .stale_under - .is_some_and(|updates| updates < self.tail.read().updates()) - { + let updates = self.tail.read().updates(); + if yielding.stale_under.is_some_and(|under| under < updates) { yielding.dropped = None; yielding.stale_under = None; } + if yielding.unowed_under.is_some_and(|under| under < updates) { + yielding.owed = true; + yielding.unowed_under = None; + } Some(yielding) } @@ -645,7 +658,7 @@ impl TrackEntry { }; let received = producer.create_group_at(group::Info { sequence: hdr.sequence }, hdr.frame_start); // Served again, so owed again. - self.place(hdr, version, received, reading, true) + self.place(hdr, version, received, reading, true, None) } /// Give a stream what the cache answered for its header. @@ -656,6 +669,7 @@ impl TrackEntry { received: Result, reading: Reading, owed: bool, + unowed_under: Option, ) -> Result { let Some(Accepted { producer, framing }) = &self.copy else { return Err(Error::Cancel); @@ -690,6 +704,7 @@ impl TrackEntry { group: crate::recv::Group::new(group), partial: None, owed, + unowed_under, track: producer.clone(), framing: *framing, reading, @@ -1772,6 +1787,7 @@ impl GroupRecv { group, partial: cut, owed: arrived_owed, + unowed_under, track, framing, reading, @@ -1782,7 +1798,7 @@ impl GroupRecv { .subscribes .lock() .get_mut(&hdr.subscribe) - .map(|entry| entry.read_yielding(hdr.sequence, arrived_owed)), + .map(|entry| entry.read_yielding(hdr.sequence, arrived_owed, unowed_under)), false => None, }; let (priority, order) = track @@ -5709,8 +5725,8 @@ mod tests { length_first: true, }; let mut entry = TrackEntry::accepted(track, framing, kio::Producer::new(Tail::default())); - let replaced = entry.read_yielding(3, true); - let replacement = entry.read_yielding(3, true); + let replaced = entry.read_yielding(3, true, None); + let replacement = entry.read_yielding(3, true, None); assert!( entry.end_yielding(3, replaced).is_none(), "the replaced stream's record is gone" @@ -5973,6 +5989,86 @@ mod tests { assert!(paused.groups.contains_key(&3), "the group pauses"); } + /// Lite-07: a stale drop that overtook the header of the stream it names holds only under + /// the update it names: once the subscriber sends another, a YIELDED reset of that stream + /// pauses the group rather than ending it as stale. + #[kio::test] + async fn a_yielded_group_pauses_once_an_update_supersedes_a_stale_drop_before_its_header() { + use crate::lite::test_transport::{Close, ScriptedSession}; + use crate::transport::poll::Session as _; + + let version = Version::Lite07; + let mut script = Vec::new(); + let mut encoder = crate::coding::Encoder::new(&mut script, version.into()); + lite::Group { + subscribe: 7, + sequence: 3, + frame_start: 0, + } + .encode(&mut encoder, version) + .unwrap(); + encoder.varint(2).unwrap(); + encoder.varint(crate::coding::varint::zigzag(300)).unwrap(); + script.push(b'x'); + let yielded = ScriptedSession::new(script); + let clock = crate::time::Deadlines::new(crate::time::Instant::now()); + let subscriber = Subscriber::new(SubscriberConfig { + runtime: clock.clone(), + session: crate::driver::Transport::new(yielded.clone()), + origin: origin::Config::new(std::num::NonZeroU64::new(1).unwrap()).produce(), + recv_bandwidth: None, + version, + peer_setup: crate::lite::PeerSetup::new(false, crate::transport::Address::Setup), + peer_hop: None, + cost: None, + going_away: Default::default(), + auth: crate::auth::Link::new(false), + }); + let track = track::Producer::new( + std::sync::Arc::new(crate::broadcast::Info::default()), + "video", + crate::track::Info::timed(), + ); + let subscription = crate::track::Subscription::default() + .with_delay(crate::track::Delay::default().with_max(std::time::Duration::from_secs(10))) + .with_range(0..); + let mut reader = track.subscribe(subscription); + let framing = lite::Framing { + timescale: Some(Timescale::MILLI), + ietf: None, + compressed: false, + length_first: true, + }; + let tail = kio::Producer::new(Tail::default()); + subscriber + .subscribes + .lock() + .insert(7, TrackEntry::accepted(track.clone(), framing, tail.clone())); + + // The drop lands before the stream's header, as `apply_response` records it. + track.give_up(3..4, track::Hole::stale(std::time::Duration::from_secs(10))); + tail.write().ok().expect("the tail is open").account(3..4, clock.now()); + + let (_, recv) = yielded.open_bi(Default::default()).await.unwrap(); + let mut first = GroupRecv::new( + subscriber.clone(), + Reader::new(crate::transport::boxed::RecvStream::new(recv), version), + ); + assert!(first.poll_serve(&kio::Waiter::noop()).is_pending()); + let mut group = reader.assert_group(); + // An update supersedes it, as `buffer_update` does. + tail.write().ok().expect("the tail is open").updated(); + yielded.close(Close::ResetStream(crate::StreamError::Yielded.to_code())); + let res = kio::wait(|waiter| first.poll_serve(waiter)).await; + assert!(res.is_ok(), "{res:?}"); + + let read = group.read_frame().await; + assert!(matches!(read, Ok(Some(_))), "the frame that arrived stays: {read:?}"); + let subs = subscriber.subscribes.lock(); + let paused = subs.get(&7).unwrap().paused.as_ref().unwrap().read(); + assert!(paused.groups.contains_key(&3), "the group pauses"); + } + /// A SUBSCRIBE_FINISH below a group already received contradicts that group. lite-05 /// specified an inclusive end, and `@moq/net` 0.1.3 to 0.1.9 sent one, so there it only /// costs the early boundary; later drafts abort the track. diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 8f8645fa559..d1d037ce5a0 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -4469,6 +4469,17 @@ impl Producer { state.supersede_stale() } + /// Whether a stale drop under the current terms gave up `sequence`, which a group arriving + /// for it replaces and an update supersedes. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn holds_stale(&self, sequence: u64) -> bool { + self.state + .read() + .drops + .get(&sequence) + .is_some_and(|hole| hole.stale_budget().is_some()) + } + /// Record `groups` given up as `hole` describes, returning the runs recorded: a drop /// leaves a skipped sequence skipped, and a group a copy still serves whole, since a group /// stream wins over its drop: a stale drop leaves it still arriving or not, and a final one diff --git a/rs/moq-net/tests/integration/stream_credit.rs b/rs/moq-net/tests/integration/stream_credit.rs index d876027deb5..5dbe2ce7490 100644 --- a/rs/moq-net/tests/integration/stream_credit.rs +++ b/rs/moq-net/tests/integration/stream_credit.rs @@ -776,3 +776,76 @@ async fn a_paused_group_given_up_while_it_waits_is_dropped() { .expect("the paused group is dropped while the others hold the credit"); assert!(end.is_err(), "the aborted paused group fails: {end:?}"); } + +/// On lite-07 a finished group whose FIN awaits its acknowledgement still holds stream credit +/// but can no longer be reset, so a continuation waiting for credit is weighed only against +/// the streams that can: a newer group still resets the one that outranks the continuation, +/// without waiting a round trip for the acknowledgement. +#[kio::test] +async fn a_closing_stream_does_not_hold_back_a_newer_group() { + let version: Version = "moq-lite-07-wip".parse().unwrap(); + let publisher = origin(1); + let viewer = origin(2); + let broadcast = publisher + .route("live", moq_net::origin::RouteOptions::hidden()) + .unwrap(); + let info = track::Info::default() + .with_timescale(moq_net::Timescale::MILLI) + .with_max_age(None); + let track = broadcast.broadcast().create_track("0", info).unwrap(); + broadcast.announce().unwrap(); + + let latency = Duration::from_secs(1); + let mut options = MockConnectOptions::new(version); + options.transport.max_concurrent_uni_streams = Some(2); + options.latency = latency; + options.server_publish = Some(publisher.consume()); + options.client_subscribe = Some(viewer.clone()); + let _pair = connect_mock(options).await; + + let remote = viewer + .consume() + .follow("live", None) + .unwrap() + .available() + .await + .unwrap() + .unwrap(); + let prefs = track::Subscription::default() + .with_order(track::Order::Descending) + .with_delay(track::Delay::default().with_max(Duration::from_secs(300))); + let mut subscriber = remote.track("0").unwrap().subscribe(prefs).await.unwrap(); + + // Group 0 finishes, so its stream only waits for the acknowledgement. Group 1 stays + // unfinished, then group 2 resets it, and group 1's continuation waits for credit. + let mut zero = track.append_group().unwrap(); + zero.write_frame(Timestamp::ZERO, b"a".as_slice()).unwrap(); + zero.finish().unwrap(); + let mut open = Vec::new(); + for _ in 1..3 { + let mut group = track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, b"a".as_slice()).unwrap(); + open.push(group); + } + // Group 3 outranks group 2's stream, the only one that can still be reset. + let mut three = track.append_group().unwrap(); + three.write_frame(Timestamp::ZERO, b"new".as_slice()).unwrap(); + three.finish().unwrap(); + + let start = kio::clock::now(); + loop { + let group = kio::clock::timeout(4 * latency, subscriber.recv_group()) + .await + .expect("a group arrives") + .unwrap() + .unwrap(); + if group.sequence == 3 { + break; + } + } + let waited = kio::clock::now() - start; + assert!( + waited < 2 * latency, + "group 3 waited for an acknowledgement: {waited:?}" + ); +} From caad387c1ebfa8c234c9de867c8450f85ac91869 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 12 Nov 2026 20:06:36 -0800 Subject: [PATCH 08/18] quest: size stream-credit's remaining work now that its last child is done Co-Authored-By: Claude Opus 5.5 --- quest/m1/stream-credit/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quest/m1/stream-credit/README.md b/quest/m1/stream-credit/README.md index bc4388b248b..aa945f1cbeb 100644 --- a/quest/m1/stream-credit/README.md +++ b/quest/m1/stream-credit/README.md @@ -1,4 +1,4 @@ -# Stream credit by priority +# [L] Stream credit by priority ## Goal From a7e01234472ce0fea88a7121ae02687ac89d4f4d Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:17:42 -0800 Subject: [PATCH 09/18] perf(moq-net): wake only the paused group a frame lands on Each paused group waits in its own kio::Tasks slot, so a frame written to one re-polls just that group instead of every paused group, and the next group to continue is an end of the sequence map. The publisher bench sweeps paused count against frames per wake. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/benches/publisher.rs | 54 +++++++++++- rs/moq-net/src/fuzz.rs | 4 +- rs/moq-net/src/lite/mod.rs | 2 +- rs/moq-net/src/lite/publisher.rs | 141 ++++++++++++++++++++++++------- 4 files changed, 164 insertions(+), 37 deletions(-) diff --git a/rs/moq-net/benches/publisher.rs b/rs/moq-net/benches/publisher.rs index 920486d547d..d62cd59b075 100644 --- a/rs/moq-net/benches/publisher.rs +++ b/rs/moq-net/benches/publisher.rs @@ -7,10 +7,23 @@ //! up as a slope over `skipped`. `adjacent` gives up the groups of one range instead, each //! with a different error from its neighbor, so the record holds one entry per group. //! +//! `publisher_paused_frames` holds `paused` unfinished paused groups and writes `frames` +//! frames to them per wake, one group each in turn, before the run checks its paused groups +//! for one the budget or the cache gave up. A wake re-polls only the groups written to, so +//! the cost per frame stays flat over `paused`. +//! //! Run with `cargo bench -p moq-net --features fuzz --bench publisher`. +#![expect( + clippy::disallowed_methods, + clippy::disallowed_types, + reason = "custom timings measure the real time the work took, not the ambient clock" +)] + use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; -use moq_net::fuzz::CoverageBench; +use std::time::{Duration, Instant}; + +use moq_net::fuzz::{CoverageBench, PausedBench}; /// Ranges the source gave up before the wake. const SKIPPED: [u64; 4] = [1, 32, 1024, 8192]; @@ -38,5 +51,42 @@ fn bench_covered_before_start(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench_covered_before_start); +/// Paused groups the run holds. +const PAUSED: [u64; 4] = [1, 32, 1024, 8192]; + +/// Frames written to paused groups per wake. +const FRAMES: [usize; 3] = [1, 16, 256]; + +/// Wakes per fixture: each frame stays in its group, so the fixture is rebuilt, untimed, +/// before it holds too many. +const WAKES: u64 = 1024; + +fn bench_paused_frames(c: &mut Criterion) { + let mut group = c.benchmark_group("publisher_paused_frames"); + for paused in PAUSED { + for frames in FRAMES { + group.throughput(Throughput::Elements(frames as u64)); + group.bench_function(BenchmarkId::new(format!("paused={paused}"), frames), |b| { + b.iter_custom(|iters| { + let mut total = Duration::ZERO; + let mut left = iters; + while left > 0 { + let mut run = PausedBench::new(paused); + let wakes = left.min(WAKES); + let start = Instant::now(); + for _ in 0..wakes { + run.wake(frames); + } + total += start.elapsed(); + left -= wakes; + } + total + }); + }); + } + } + group.finish(); +} + +criterion_group!(benches, bench_covered_before_start, bench_paused_frames); criterion_main!(benches); diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index ba7746bea8e..c4c58e9722a 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -19,8 +19,8 @@ pub use ingest::ingest; pub use request::request; pub use session::{session_ietf, session_lite}; -/// A lite-07 publisher run loop waiting on SUBSCRIBE_START, for the publisher bench. -pub use crate::lite::CoverageBench; +/// Lite-07 publisher run loops for the publisher bench. +pub use crate::lite::{CoverageBench, PausedBench}; use bytes::Bytes; diff --git a/rs/moq-net/src/lite/mod.rs b/rs/moq-net/src/lite/mod.rs index bf34b320e9f..fec8f1f696e 100644 --- a/rs/moq-net/src/lite/mod.rs +++ b/rs/moq-net/src/lite/mod.rs @@ -33,7 +33,7 @@ mod probe; #[cfg(feature = "async")] mod publisher; #[cfg(all(feature = "async", any(test, feature = "fuzz")))] -pub use publisher::CoverageBench; +pub use publisher::{CoverageBench, PausedBench}; #[cfg(feature = "async")] mod replay; #[cfg(feature = "async")] diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index a2ac7f657fb..4ac6136062b 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -5036,13 +5036,26 @@ struct Paused { frame_start: u64, } -impl Paused { - /// Its rank for the subscription's stream credit in `order`, higher first. - fn rank(&self, order: crate::track::Order) -> i64 { - crate::priority::in_group(self.group.sequence, self.frame_start, order) +/// Waits for the subscription's budget or the cache to give the paused group up. Each +/// paused group wakes only its own task, so a frame written to one re-polls just that one. +impl kio::Task for Paused { + type Output = Error; + + fn poll(&mut self, waiter: &kio::Waiter) -> Poll { + match poll_given_up(&mut self.group, waiter) { + Some(err) => Poll::Ready(err), + None => Poll::Pending, + } } } +/// Where a paused group waits in [`TrackRun::paused_tasks`], and where its continuation starts. +#[derive(Clone, Copy)] +struct PausedKey { + key: kio::TaskKey, + frame_start: u64, +} + impl Subscription { fn group_context(&self) -> GroupContext { GroupContext { @@ -5134,7 +5147,9 @@ struct TrackRun { // sustained congestion can pause every group in turn: one the budget or the cache gives // up while it waits is dropped then, which bounds them by what the cache holds, and a // group paused again replaces its entry. - paused: std::collections::BTreeMap, + paused: std::collections::BTreeMap, + // Lite-07: the paused groups themselves, each waiting to be given up. + paused_tasks: kio::Tasks, // Lite-07: the group `opening` holds continues a paused one. continuing: bool, end_sent: bool, @@ -5203,6 +5218,7 @@ impl TrackRun { redrop: RangeSet::new(), reoffered: Default::default(), paused: Default::default(), + paused_tasks: kio::Tasks::new(), continuing: false, end_sent: false, opening: None, @@ -5510,8 +5526,9 @@ impl TrackRun { if !paused.is_empty() { // The track side continues them. Idle::disarm(&mut self.idle); - self.paused - .extend(paused.into_iter().map(|paused| (paused.group.sequence, paused))); + for paused in paused { + self.pause(paused); + } } self.drop_given_up_paused(waiter); let dropped = std::mem::take(&mut *self.ctx.outcomes.dropped.lock().unwrap()); @@ -5612,10 +5629,11 @@ impl TrackRun { // outranks it. let order = self.ctx.steer_current().order; let stuck = self.stuck(); - if let Some(best) = self.best_paused() - && (self.opening.is_none() || stuck.is_some_and(|waiting| self.paused[&best].rank(order) > waiting)) + if let Some((best, rank)) = self.best_paused(order) + && (self.opening.is_none() || stuck.is_some_and(|waiting| rank > waiting)) { - let Paused { group, frame_start } = self.paused.remove(&best).expect("best is paused"); + let PausedKey { key, .. } = self.paused.remove(&best).expect("best is paused"); + let Paused { group, frame_start } = self.paused_tasks.remove(key).expect("a paused group waits"); tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, sequence = group.sequence, frame_start, "continuing paused group"); self.give_way(); self.open(group, frame_start); @@ -6104,11 +6122,10 @@ impl TrackRun { tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, sequence = waiting.sequence, "giving way to a higher-ranked group"); match std::mem::take(&mut self.continuing) { true => { - let paused = Paused { + self.pause(Paused { group: waiting.group.clone(), frame_start: waiting.frame_start, - }; - self.paused.insert(waiting.sequence, paused); + }); } false => waiting.settle_failed(&Error::Old), } @@ -6127,32 +6144,43 @@ impl TrackRun { } } - /// The sequence of the highest-ranked paused group in this subscription's current order, - /// the next to continue. - fn best_paused(&mut self) -> Option { - let order = self.ctx.steer_current().order; - self.paused - .iter() - .max_by_key(|(_, paused)| paused.rank(order)) - .map(|(&sequence, _)| sequence) + /// Hold a paused group until its continuation, replacing the entry of an earlier pause. + fn pause(&mut self, paused: Paused) { + let sequence = paused.group.sequence; + let frame_start = paused.frame_start; + let key = self.paused_tasks.insert(paused); + if let Some(old) = self.paused.insert(sequence, PausedKey { key, frame_start }) { + self.paused_tasks.remove(old.key); + } + } + + /// The sequence and rank of the highest-ranked paused group in `order`, the next to + /// continue. The rank orders groups by sequence first, so it is an end of the map. + fn best_paused(&self, order: crate::track::Order) -> Option<(u64, i64)> { + let (&sequence, paused) = match order { + crate::track::Order::Descending => self.paused.last_key_value(), + crate::track::Order::Ascending => self.paused.first_key_value(), + }?; + Some((sequence, crate::priority::in_group(sequence, paused.frame_start, order))) } /// Drop each paused group the subscription's budget or the cache gave up while it waited - /// to continue, as a group waiting for credit would be. + /// to continue, as a group waiting for credit would be. Only the woken groups are polled. fn drop_given_up_paused(&mut self, waiter: &kio::Waiter) { - if self.paused.is_empty() { + if self.paused_tasks.is_empty() { return; } let mut dropped = Vec::new(); - self.paused - .retain(|&sequence, paused| match poll_given_up(&mut paused.group, waiter) { - Some(err) => { - dropped.push((sequence, StreamError::from(&err))); - false - } - None => true, - }); - if !dropped.is_empty() && self.ctx.version.has_track_stream() { + let _ = self.paused_tasks.poll_each(waiter, |paused, err| { + dropped.push((paused.group.sequence, StreamError::from(&err))); + }); + if dropped.is_empty() { + return; + } + for (sequence, _) in &dropped { + self.paused.remove(sequence); + } + if self.ctx.version.has_track_stream() { self.ctx.outcomes.dropped.lock().unwrap().extend(dropped); } } @@ -6299,6 +6327,55 @@ impl CoverageBench { } } +/// A lite-07 subscription's run loop holding paused groups that are still being written, for +/// the `publisher` bench: each wake writes frames to some of them, then checks the paused +/// groups for one given up and picks the next to continue. +#[cfg(any(test, feature = "fuzz"))] +pub struct PausedBench { + coverage: CoverageBench, + groups: Vec, + next: usize, +} + +#[cfg(any(test, feature = "fuzz"))] +impl PausedBench { + /// Pause `paused` unfinished groups, each holding one frame. + pub fn new(paused: u64) -> Self { + let mut coverage = CoverageBench::new(2, 0); + let mut groups = Vec::new(); + // Past the subscribed ranges, so the run itself never serves them. + for sequence in 100..100 + paused { + let mut group = coverage.track.create_group(group::Info { sequence }).unwrap(); + group.write_frame(None, b"x".to_vec()).unwrap(); + coverage.run.pause(Paused { + group: group.consume(), + frame_start: 0, + }); + groups.push(group); + } + let mut this = Self { + coverage, + groups, + next: 0, + }; + this.wake(0); + this + } + + /// Write one frame to each of the next `frames` paused groups in turn, then wake once. + pub fn wake(&mut self, frames: usize) { + for _ in 0..frames { + let group = &mut self.groups[self.next]; + group.write_frame(None, b"x".to_vec()).unwrap(); + self.next = (self.next + 1) % self.groups.len(); + } + let run = &mut self.coverage.run; + run.drop_given_up_paused(&kio::Waiter::noop()); + let order = run.ctx.steer_current().order; + assert!(run.best_paused(order).is_some(), "no paused group is given up"); + } +} + /// Where a group stream is ranked. enum Rank { /// In the subscription's send group, by its sequence in the subscription's order. From 2e122cf122f309b65f8009edad83fae0ab8c5e20 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:34:08 -0800 Subject: [PATCH 10/18] test(moq-net): a re-paused group waits once, and the best paused group follows the order Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/publisher.rs | 36 ++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 4ac6136062b..eae30b3981e 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -12415,4 +12415,40 @@ mod tests { assert!(groups[1].1 > groups[0].1, "the publisher's priority breaks the tie"); assert!(groups[2].1 > groups[1].1, "the subscriber's priority dominates"); } + + /// A group paused again waits once, at its new start, and is dropped once when given up. + #[test] + fn a_group_paused_again_replaces_its_wait() { + let mut bench = PausedBench::new(1); + let run = &mut bench.coverage.run; + run.pause(Paused { + group: bench.groups[0].consume(), + frame_start: 1, + }); + assert_eq!(run.paused_tasks.len(), 1, "the earlier wait is cancelled"); + assert_eq!(run.paused[&100].frame_start, 1); + + bench.groups.remove(0).abort(Error::Cancel).unwrap(); + let run = &mut bench.coverage.run; + run.drop_given_up_paused(&kio::Waiter::noop()); + assert!(run.paused.is_empty() && run.paused_tasks.is_empty()); + let dropped = run.ctx.outcomes.dropped.lock().unwrap(); + assert_eq!(dropped.iter().map(|(sequence, _)| *sequence).collect::>(), [100]); + } + + /// The next paused group to continue is the newest when descending, the oldest when + /// ascending, and only the woken group leaves when it is given up. + #[test] + fn the_best_paused_group_follows_the_order() { + let mut bench = PausedBench::new(3); + let best = |run: &TrackRun, order| run.best_paused(order).map(|(sequence, _)| sequence); + assert_eq!(best(&bench.coverage.run, track::Order::Descending), Some(102)); + assert_eq!(best(&bench.coverage.run, track::Order::Ascending), Some(100)); + + bench.groups.remove(2).abort(Error::Cancel).unwrap(); + bench.wake(0); + let run = &bench.coverage.run; + assert_eq!(best(run, track::Order::Descending), Some(101)); + assert_eq!(run.paused_tasks.len(), 2); + } } From 2d1adc99355c96c051ba5ab5516baa53b777b456 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:02:51 -0800 Subject: [PATCH 11/18] fix(moq-net): drop a paused group the cache released with nothing unread A finished group whose cursor has nothing left reads as a clean end even after the cache releases it, so it never counted as given up and could outlive the cache's bound. The paused wait also ends on the group's abort. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/publisher.rs | 30 +++++++++++++++++++++++++++--- rs/moq-net/src/model/group.rs | 12 ++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index eae30b3981e..63c36f65192 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -5042,10 +5042,12 @@ impl kio::Task for Paused { type Output = Error; fn poll(&mut self, waiter: &kio::Waiter) -> Poll { - match poll_given_up(&mut self.group, waiter) { - Some(err) => Poll::Ready(err), - None => Poll::Pending, + if let Some(err) = poll_given_up(&mut self.group, waiter) { + return Poll::Ready(err); } + // A finished group this cursor has nothing left of is no loss once the cache releases + // it, but holding it past that would outlive the cache's bound on the paused groups. + self.group.poll_aborted(waiter) } } @@ -12451,4 +12453,26 @@ mod tests { assert_eq!(best(run, track::Order::Descending), Some(101)); assert_eq!(run.paused_tasks.len(), 2); } + + /// A paused group with nothing left to send is dropped once the cache releases it. + #[test] + fn a_released_paused_group_with_nothing_unread_is_dropped() { + let mut bench = PausedBench::new(1); + let run = &mut bench.coverage.run; + let empty = bench.coverage.track.create_group(group::Info { sequence: 200 }).unwrap(); + let empty = empty.finish().unwrap(); + run.pause(Paused { + group: empty.consume(), + frame_start: 0, + }); + run.drop_given_up_paused(&kio::Waiter::noop()); + assert!(run.paused.contains_key(&200), "a finished group waits to continue"); + + empty.abort(Error::Evicted).unwrap(); + run.drop_given_up_paused(&kio::Waiter::noop()); + assert!(!run.paused.contains_key(&200)); + assert_eq!(run.paused_tasks.len(), 1, "only the released group leaves"); + let dropped = run.ctx.outcomes.dropped.lock().unwrap(); + assert_eq!(dropped.iter().map(|(sequence, _)| *sequence).collect::>(), [200]); + } } diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index e28ed1925f7..ce90944874c 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -3802,6 +3802,18 @@ impl Consumer { self.cursor.poll(waiter, |state| state.poll_outcome()) } + /// Resolves once the group is aborted, including when the cache releases it after it + /// finished, with the cause. Finishing alone does not end it. + #[cfg(feature = "async")] + pub(crate) fn poll_aborted(&self, waiter: &kio::Waiter) -> Poll { + self.cursor + .poll(waiter, |state| match state.outcome.error() { + Some(err) => Poll::Ready(Ok(err.clone())), + None => Poll::Pending, + }) + .map(|res| res.unwrap_or_else(|err| err)) + } + /// The index of the first frame the group holds (see [`Producer::start_at`]). pub(crate) fn first_frame(&self) -> u64 { self.cursor.state.read().offset as u64 From 105790b2b99c4fdd75dbfadbe5fc32f32437d777 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:06:07 -0800 Subject: [PATCH 12/18] style: cargo fmt Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/publisher.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 63c36f65192..844c7174123 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -12459,7 +12459,11 @@ mod tests { fn a_released_paused_group_with_nothing_unread_is_dropped() { let mut bench = PausedBench::new(1); let run = &mut bench.coverage.run; - let empty = bench.coverage.track.create_group(group::Info { sequence: 200 }).unwrap(); + let empty = bench + .coverage + .track + .create_group(group::Info { sequence: 200 }) + .unwrap(); let empty = empty.finish().unwrap(); run.pause(Paused { group: empty.consume(), From 30f04478b64ecf2c14896158c658001e3c98dffb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 02:19:21 -0800 Subject: [PATCH 13/18] fix(net): preserve finished groups after their writers drop --- rs/moq-net/src/model/group.rs | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index ce90944874c..e0e7611a843 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -3806,12 +3806,13 @@ impl Consumer { /// finished, with the cause. Finishing alone does not end it. #[cfg(feature = "async")] pub(crate) fn poll_aborted(&self, waiter: &kio::Waiter) -> Poll { - self.cursor - .poll(waiter, |state| match state.outcome.error() { - Some(err) => Poll::Ready(Ok(err.clone())), - None => Poll::Pending, - }) - .map(|res| res.unwrap_or_else(|err| err)) + let abort = |state: &GroupState| state.outcome.error().cloned().map_or(Poll::Pending, Poll::Ready); + // Not `Cursor::poll`: it reads a closed channel as `Dropped`, but every writer dropping + // after a finish aborts nothing, and with no writer left nothing ever will. + match ready!(self.cursor.state.poll(waiter, |state| abort(state))) { + Ok(err) => Poll::Ready(err), + Err(state) => abort(&state), + } } /// The index of the first frame the group holds (see [`Producer::start_at`]). @@ -6516,6 +6517,20 @@ mod test { assert!(matches!(consumer.poll_read_frame(&waiter), Poll::Ready(Ok(None)))); } + /// Every writer dropping after a finish aborts nothing, so a reader keeps its frames. + #[test] + fn writers_dropped_after_a_finish_abort_nothing() { + let mut producer = Info { sequence: 0 }.produce(); + producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"a")).unwrap(); + let mut consumer = producer.consume(); + drop(producer.finish().unwrap()); + let waiter = kio::Waiter::noop(); + + assert!(consumer.poll_aborted(&waiter).is_pending()); + assert!(matches!(consumer.poll_read_frame(&waiter), Poll::Ready(Ok(Some(_))))); + assert!(matches!(consumer.poll_read_frame(&waiter), Poll::Ready(Ok(None)))); + } + /// An abort before the finish is the outcome, on every observer. #[test] fn abort_before_finish_is_the_outcome() { From 5fb55fb0da07ae601d92f22d7abf7f7ded789f79 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 02:32:26 -0800 Subject: [PATCH 14/18] refactor(net): keep the yielding verdict in one state --- js/net/src/lite/subscriber.ts | 62 +++++++++----- rs/moq-net/src/lite/subscriber.rs | 132 +++++++++++++++--------------- 2 files changed, 110 insertions(+), 84 deletions(-) diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 594edff9910..6edcfde447b 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -268,15 +268,30 @@ interface PausedGroup { // Whether a YIELDED reset of a stream being read pauses its group. interface Yielding { - // The publisher still owed the group when the stream arrived: no drop named it first. - owed: boolean; - // The error a drop named the group with while the stream was read. - dropped?: Error; - // For a stale drop, the update it named: the next update supersedes it. - staleUnder?: bigint; - // When a stale drop that overtook the stream is all that left the group unowed, the update it - // named: once another is sent, the group is owed again. Mirrors the Rust `Yielding::unowed_under`. - unowedUnder?: bigint; + verdict: YieldVerdict; +} + +// Stale drops expire on the next update. An already accounted header stays accounted. +type YieldVerdict = + | { kind: "owed" } + | { kind: "accounted" } + | { kind: "stale"; under: bigint; error: Error } + | { kind: "accounted-stale"; under: bigint; error: Error } + | { kind: "dropped"; error: Error }; + +function yieldError(verdict: YieldVerdict, update: bigint): Error | undefined { + switch (verdict.kind) { + case "owed": + return undefined; + case "accounted": + return new StreamError(StreamCode.Old); + case "stale": + return verdict.under < update ? undefined : verdict.error; + case "accounted-stale": + return verdict.under < update ? new StreamError(StreamCode.Old) : verdict.error; + case "dropped": + return verdict.error; + } } // The paused groups of `entry`, created on first use. @@ -3307,7 +3322,13 @@ export class Subscriber { }); if (yields) { ingest ??= { index: group.frameStart }; - yielding = { owed, unowedUnder: owed ? undefined : unowedUnder }; + yielding = { + verdict: owed + ? { kind: "owed" } + : unowedUnder !== undefined + ? { kind: "stale", under: unowedUnder, error: new StreamError(StreamCode.Old) } + : { kind: "accounted" }, + }; entry.yielding ??= new Map(); entry.yielding.set(group.sequence, yielding); } @@ -3337,11 +3358,7 @@ export class Subscriber { // A drop named the group first: the publisher gave it up, so nothing continues it. One // that overtook the stream's header carried a stale verdict. // A stale drop an update sent since superseded no longer ends the group. - const superseded = yielding.staleUnder !== undefined && yielding.staleUnder < entry.update; - const dropped = superseded ? undefined : yielding.dropped; - const owedNow = - yielding.owed || (yielding.unowedUnder !== undefined && yielding.unowedUnder < entry.update); - const given = dropped ?? (owedNow ? undefined : new StreamError(StreamCode.Old)); + const given = yieldError(yielding.verdict, entry.update); if (given) { producer.abort(given); return; @@ -3413,10 +3430,17 @@ export class Subscriber { #dropPaused(entry: SubscribeEntry, start: bigint, end: bigint, error: Error, staleUnder?: bigint): void { for (const [sequence, yielding] of entry.yielding ?? []) { if (sequence < start || sequence >= end) continue; - // A final verdict stands, whatever stale one follows. - if (staleUnder !== undefined && yielding.dropped && yielding.staleUnder === undefined) continue; - yielding.dropped = error; - yielding.staleUnder = staleUnder; + const verdict = yielding.verdict; + if (staleUnder === undefined) yielding.verdict = { kind: "dropped", error }; + else if (verdict.kind !== "dropped") + yielding.verdict = { + kind: + verdict.kind === "accounted" || verdict.kind === "accounted-stale" + ? "accounted-stale" + : "stale", + under: staleUnder, + error, + }; } const paused = entry.paused; if (!paused) return; diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index a8129dbd88a..0be213aa025 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -264,15 +264,46 @@ impl PausedGroup { struct Yielding { /// The stream reading the group, as [`TrackEntry::read_yielding`] numbered it. stream: u64, - /// The publisher still owed the group when the stream arrived: no drop named it first. - owed: bool, - /// The error a drop named the group with while the stream was read. - dropped: Option, - /// For a stale drop, the updates sent when it arrived: the next update supersedes it. - stale_under: Option, - /// When a stale drop that overtook the stream is all that left the group unowed, the - /// updates sent when the stream arrived: once another is sent, the group is owed again. - unowed_under: Option, + verdict: YieldVerdict, +} + +/// A stale verdict expires on the next update; a final drop never does. A group already +/// accounted before its header stays accounted even if a later stale drop expires. +enum YieldVerdict { + Owed, + Accounted, + Stale { under: u64, error: Error }, + AccountedStale { under: u64, error: Error }, + Dropped(Error), +} + +impl YieldVerdict { + fn updated(self, updates: u64) -> Self { + match self { + Self::Stale { under, .. } if under < updates => Self::Owed, + Self::AccountedStale { under, .. } if under < updates => Self::Accounted, + verdict => verdict, + } + } + + fn drop_with(&mut self, error: Error, under: Option) { + *self = match under { + Some(_) if matches!(self, Self::Dropped(_)) => return, + Some(under) if matches!(self, Self::Accounted | Self::AccountedStale { .. }) => { + Self::AccountedStale { under, error } + } + Some(under) => Self::Stale { under, error }, + None => Self::Dropped(error), + }; + } + + fn error(self) -> Option { + match self { + Self::Owed => None, + Self::Accounted => Some(Error::Stream(crate::StreamError::Old)), + Self::Stale { error, .. } | Self::AccountedStale { error, .. } | Self::Dropped(error) => Some(error), + } + } } /// What a group stream's header opened in its subscription's copy. @@ -287,10 +318,7 @@ struct Opened { /// A continuation's frame its YIELDED reset cut short, which it completes. Boxed, as /// most groups have none. partial: Option>, - /// The publisher owed the group when the stream arrived; see [`Yielding::owed`]. - owed: bool, - /// See [`Yielding::unowed_under`]. - unowed_under: Option, + verdict: YieldVerdict, track: track::Producer, framing: lite::Framing, reading: Reading, @@ -501,8 +529,17 @@ impl TrackEntry { } } } - let unowed_under = (!owed && stale).then(|| self.tail.read().updates()); - self.place(hdr, version, received, reading, owed, unowed_under) + let verdict = if owed { + YieldVerdict::Owed + } else if stale { + YieldVerdict::Stale { + under: self.tail.read().updates(), + error: Error::Stream(crate::StreamError::Old), + } + } else { + YieldVerdict::Accounted + }; + self.place(hdr, version, received, reading, verdict) } /// Whether a YIELDED reset paused the group `sequence`, which the cache still holds. One the @@ -554,8 +591,7 @@ impl TrackEntry { Some(Ok(Open::Group(Opened { group, partial: partial.map(Box::new), - owed: true, - unowed_under: None, + verdict: YieldVerdict::Owed, track: producer.clone(), framing: *framing, reading, @@ -576,23 +612,11 @@ impl TrackEntry { state.groups.insert(sequence, paused); } - /// Track a stream reading `sequence` that a YIELDED reset would pause, which `owed` says - /// the publisher owed when it arrived; see [`Yielding::unowed_under`]. - /// - /// Returns the stream's token for [`Self::end_yielding`]. - fn read_yielding(&mut self, sequence: u64, owed: bool, unowed_under: Option) -> u64 { + /// Track a stream reading `sequence`, returning its token for [`Self::end_yielding`]. + fn read_yielding(&mut self, sequence: u64, verdict: YieldVerdict) -> u64 { self.yield_streams += 1; let stream = self.yield_streams; - self.yielding.insert( - sequence, - Yielding { - stream, - owed, - dropped: None, - stale_under: None, - unowed_under, - }, - ); + self.yielding.insert(sequence, Yielding { stream, verdict }); stream } @@ -605,15 +629,7 @@ impl TrackEntry { Some(yielding) if yielding.stream == stream => self.yielding.remove(&sequence)?, _ => return None, }; - let updates = self.tail.read().updates(); - if yielding.stale_under.is_some_and(|under| under < updates) { - yielding.dropped = None; - yielding.stale_under = None; - } - if yielding.unowed_under.is_some_and(|under| under < updates) { - yielding.owed = true; - yielding.unowed_under = None; - } + yielding.verdict = yielding.verdict.updated(self.tail.read().updates()); Some(yielding) } @@ -624,12 +640,7 @@ impl TrackEntry { fn dropped(&mut self, groups: std::ops::Range, err: &Error, stale: bool) { let updates = stale.then(|| self.tail.read().updates()); for (_, yielding) in self.yielding.range_mut(groups.clone()) { - // A final verdict stands, whatever stale one follows. - if stale && yielding.dropped.is_some() && yielding.stale_under.is_none() { - continue; - } - yielding.dropped = Some(err.clone()); - yielding.stale_under = updates; + yielding.verdict.drop_with(err.clone(), updates); } let Some(registry) = &self.paused else { return; @@ -658,7 +669,7 @@ impl TrackEntry { }; let received = producer.create_group_at(group::Info { sequence: hdr.sequence }, hdr.frame_start); // Served again, so owed again. - self.place(hdr, version, received, reading, true, None) + self.place(hdr, version, received, reading, YieldVerdict::Owed) } /// Give a stream what the cache answered for its header. @@ -668,8 +679,7 @@ impl TrackEntry { version: Version, received: Result, reading: Reading, - owed: bool, - unowed_under: Option, + verdict: YieldVerdict, ) -> Result { let Some(Accepted { producer, framing }) = &self.copy else { return Err(Error::Cancel); @@ -703,8 +713,7 @@ impl TrackEntry { Ok(Open::Group(Opened { group: crate::recv::Group::new(group), partial: None, - owed, - unowed_under, + verdict, track: producer.clone(), framing: *framing, reading, @@ -1786,8 +1795,7 @@ impl GroupRecv { let Opened { group, partial: cut, - owed: arrived_owed, - unowed_under, + verdict, track, framing, reading, @@ -1798,7 +1806,7 @@ impl GroupRecv { .subscribes .lock() .get_mut(&hdr.subscribe) - .map(|entry| entry.read_yielding(hdr.sequence, arrived_owed, unowed_under)), + .map(|entry| entry.read_yielding(hdr.sequence, verdict)), false => None, }; let (priority, order) = track @@ -1854,13 +1862,7 @@ impl GroupRecv { // A drop named the group first: the publisher gave it up, so nothing continues it. // One that overtook the stream's header carried a stale verdict. let err = match yielding { - Some(Yielding { - owed: true, - dropped: None, - .. - }) => None, - Some(Yielding { dropped: Some(err), .. }) => Some(err), - Some(Yielding { owed: false, .. }) => Some(Error::Stream(crate::StreamError::Old)), + Some(yielding) => yielding.verdict.error(), None => Some(Error::Cancel), }; if let Some(err) = err { @@ -5725,8 +5727,8 @@ mod tests { length_first: true, }; let mut entry = TrackEntry::accepted(track, framing, kio::Producer::new(Tail::default())); - let replaced = entry.read_yielding(3, true, None); - let replacement = entry.read_yielding(3, true, None); + let replaced = entry.read_yielding(3, YieldVerdict::Owed); + let replacement = entry.read_yielding(3, YieldVerdict::Owed); assert!( entry.end_yielding(3, replaced).is_none(), "the replaced stream's record is gone" @@ -5734,7 +5736,7 @@ mod tests { let yielding = entry .end_yielding(3, replacement) .expect("the replacement's record stays"); - assert!(yielding.owed && yielding.dropped.is_none()); + assert!(matches!(yielding.verdict, YieldVerdict::Owed)); } /// Lite-07: a paused group the cache gave up on lets go of the subscription's end once a From bdee14609034a0e1423984af3edce45c0d284fcf Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 02:38:10 -0800 Subject: [PATCH 15/18] test(net): gate the abort poll regression on async --- rs/moq-net/src/model/group.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index e0e7611a843..e16b987db1d 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -6518,6 +6518,7 @@ mod test { } /// Every writer dropping after a finish aborts nothing, so a reader keeps its frames. + #[cfg(feature = "async")] #[test] fn writers_dropped_after_a_finish_abort_nothing() { let mut producer = Info { sequence: 0 }.produce(); From 5e4ba85a7e7890ce3737fce1ffef14a546ac237b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 02:41:35 -0800 Subject: [PATCH 16/18] docs(net): resynchronize the subscription model coverage --- .claude/skills/verify-subscribe/SKILL.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.claude/skills/verify-subscribe/SKILL.md b/.claude/skills/verify-subscribe/SKILL.md index 687ac14ced0..38d07c9f58d 100644 --- a/.claude/skills/verify-subscribe/SKILL.md +++ b/.claude/skills/verify-subscribe/SKILL.md @@ -41,7 +41,7 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `1be43c735b4ddaf5811433d82390e377cd1fe15a`. Run `git log 1be43c735b.. -- rs/moq-net/src/lite/publisher.rs rs/moq-net/src/lite/subscriber.rs rs/moq-net/src/tail.rs rs/moq-net/src/model/track.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. +Sync commit: `6aa4f9c1cad482aa86adcf3cf26bd7779697a01a`. Run `git log 6aa4f9c1ca.. -- rs/moq-net/src/lite/publisher.rs rs/moq-net/src/lite/subscriber.rs rs/moq-net/src/tail.rs rs/moq-net/src/model/track.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. | Code | Model | | --- | --- | @@ -58,7 +58,7 @@ Sync commit: `1be43c735b4ddaf5811433d82390e377cd1fe15a`. Run `git log 1be43c735b | `rs/moq-net/src/tail.rs` `Tail::rebill`, `overtaken`, `updated`; `rs/moq-net/src/lite/subscriber.rs` `ServeLoop::apply_response`; `rs/moq-net/src/model/record.rs` `Record::folded` | `TailFold`, `forgot`, `PubFold`, `folded` | | `rs/moq-net/src/model/track.rs` `Producer::give_up`, `skip_received` | `rec`, `cached` | | `rs/moq-net/src/lite/publisher.rs` `GroupServe::settle_failed`, `TrackRun::open`; `rs/moq-net/src/transport/boxed.rs` `SendStream::preempted` | `StreamYield`, `PubContinue`, `PausedGiveUp` | -| `rs/moq-net/src/lite/subscriber.rs` `TrackEntry::pause`, `resume`, `dropped`, `read_yielding`, `end_yielding`, `GroupRecv::poll_serve`, `FrameIngest::dispatch` | the YIELDED branch of `SubReset`, `SubFrame`, `reading`, `sdrop`, `paused` | +| `rs/moq-net/src/lite/subscriber.rs` `TrackEntry::pause`, `resume`, `dropped`, `read_yielding`, `end_yielding`, `YieldVerdict`, `GroupRecv::poll_serve`, `FrameIngest::dispatch` | the YIELDED branch of `SubReset`, `SubFrame`, `reading`, `sdrop`, `paused` | What the model asks of that code, each found by TLC as a broken rule without it: From caec521f3b85f9fa1e22dfe9237d80f8bba014c3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 02:48:22 -0800 Subject: [PATCH 17/18] test(net): settle the live stream before preempting the fill --- js/net/src/lite/copy_fill.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/js/net/src/lite/copy_fill.test.ts b/js/net/src/lite/copy_fill.test.ts index a4b5a0c0894..c0e626a27f5 100644 --- a/js/net/src/lite/copy_fill.test.ts +++ b/js/net/src/lite/copy_fill.test.ts @@ -410,6 +410,8 @@ test("copy fill: a yielded group the fill's drop names ends", async () => { // The session resets the group's stream to make room, and the group is gone before a new // stream could continue it, so the publisher drops it on the fill's subscription. + // Receiving the live group's header did not wait for its stream's FIN to retire. + await clock.advance(Duration.ZERO); const open = lines(pair.server).streams.list(); expect(open.length).toBe(1); open[0]?.preempt(); From 47c5e2c0bff5a269337fb3c75c1ae7d103e594ca Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 02:54:28 -0800 Subject: [PATCH 18/18] test(net): preempt every stream serving the archived group --- js/net/src/lite/copy_fill.test.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/js/net/src/lite/copy_fill.test.ts b/js/net/src/lite/copy_fill.test.ts index c0e626a27f5..d6a8ce7a9d2 100644 --- a/js/net/src/lite/copy_fill.test.ts +++ b/js/net/src/lite/copy_fill.test.ts @@ -399,7 +399,11 @@ test("copy fill: a yielded group the fill's drop names ends", async () => { const remote = wireOf(client).consume(Path.from("room")); try { const reader = remote.track("video").subscribe({ ranges: [{ start: { relative: 0n } }] }); - expect((await reader.ordered().nextGroup())?.sequence).toBe(1n); + const live = await reader.ordered().nextGroup(); + expect(live?.sequence).toBe(1n); + expect(dec.decode((await live?.readFrame())?.payload)).toBe("live 1"); + expect(await live?.readFrame()).toBeUndefined(); + live?.close(); const history = remote .track("video") .subscribe({ ...WIDE, ranges: [{ start: { absolute: 0n }, end: { absolute: 1n } }] }); @@ -410,11 +414,12 @@ test("copy fill: a yielded group the fill's drop names ends", async () => { // The session resets the group's stream to make room, and the group is gone before a new // stream could continue it, so the publisher drops it on the fill's subscription. - // Receiving the live group's header did not wait for its stream's FIN to retire. + // The live stream has finished. The archive may be served on both the fill and + // the copy's expanded feed, so reset every stream still serving it. await clock.advance(Duration.ZERO); const open = lines(pair.server).streams.list(); - expect(open.length).toBe(1); - open[0]?.preempt(); + expect(open.length).toBeGreaterThan(0); + for (const stream of open) stream.preempt(); source.abort(new StreamError(StreamCode(70), { message: "gone" })); // Still paused without it, which the deadline, fired once every pending task settled,