From 5c24f17018fa521e5f39b8077c123a2cc7401248 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 00:56:52 -0800 Subject: [PATCH 1/4] chore(quest): claim quest/m1/export-evicts-every-group.md Quest-Claim: 7d12b7c80140adb2 From d87bf2622d6c48eab2b09db587d770f83dfdf874 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 02:09:36 -0800 Subject: [PATCH 2/4] fix(net): a pre-lite-07 group stream whose FIN went out is never reset for going stale Before lite-07 the publisher reset a group stream with OLD when its group went stale while the FIN awaited acknowledgement. Under a zero max-delay budget, the default, every live group's successor lands before that acknowledgement, so every group was reset after it was sent whole. quinn clears a reset stream's receive buffer, so the subscriber lost each group's unread tail, logged an eviction, and counted a loss per group. Lite-07 already leaves such a stream alone; lite-06 and older now do the same. Co-Authored-By: Claude Opus 5.5 --- quest/m1/export-evicts-every-group.md | 30 -------------- rs/moq-net/src/lite/publisher.rs | 56 +++++++++++---------------- 2 files changed, 22 insertions(+), 64 deletions(-) delete mode 100644 quest/m1/export-evicts-every-group.md diff --git a/quest/m1/export-evicts-every-group.md b/quest/m1/export-evicts-every-group.md deleted file mode 100644 index 7415614c30..0000000000 --- a/quest/m1/export-evicts-every-group.md +++ /dev/null @@ -1,30 +0,0 @@ -# [M] A live export reads each group to its end instead of evicting it - -## Goal - -A container export following a steady live broadcast reads every group to its -clean end: it logs no `current group evicted` and counts no loss while nothing -is lost. Each group's tail frames reach the export. - -## Plan - -Seen 2026-11-17 while validating -[moq-dev/moq#9049](https://github.com/moq-dev/moq/pull/9049) on `main`: a -local `moq-relay`, ffmpeg pushing 30 fps H.264 with a 1 s GOP over RTMP into -`moq import rtmp`, and `moq export webrtc --listen` from the relay (both on -moq-lite-06, WebTransport). The export logged `current group evicted; skipping -to next buffered group ... error=Moq(Stream(Old))` from -`rs/moq-mux/src/container/consumer.rs` once per group (33 in 33 s), while -Chrome still decoded about 30 fps. Every evicted group calls `lose()`, so the -export's loss accounting reports a loss per group. - -Reproduce first, then find which side ends a finished group as `Old`: the -relay hop, `moq-rtmp`'s group writes, or the consumer reading a group whose -successor started. Recent candidates touching this path: -[#8910](https://github.com/moq-dev/moq/pull/8910) (an ordered track reports -the groups it gives up on) and -[#8961](https://github.com/moq-dev/moq/pull/8961) (a group's end in one word). - -## Related - -- [Export order](/quest/m3/export-order.md) - the same warning after an outage, where the eviction is real diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 619aa41303..974ebae034 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -6048,7 +6048,7 @@ enum GroupState { ending: Ending, }, /// Every frame is written and the FIN sent: wait for the acknowledgement so a late - /// cancel can still reset the stream, and before lite-07 a late expiry too. + /// cancel can still reset the stream. Closed { writer: Writer, }, @@ -6476,22 +6476,13 @@ impl GroupServe { // poll_close releases the stream on completion: the peer acknowledged // everything, so the Drop fallback must not reset the stream and // discard bytes still retransmitting. - let res = match writer.poll_close(waiter) { - Poll::Ready(res) => res, - // Those bytes still hold the connection until acknowledged, so before - // lite-07 a group gone stale meanwhile releases them like one still - // serving. Lite-07 already said how the group ended, which a reset - // would take back. - Poll::Pending if !self.ctx.version.has_group_end() && self.group.poll_expired(waiter) => { - let GroupState::Closed { writer } = std::mem::replace(&mut self.state, GroupState::Done) - else { - unreachable!() - }; - writer.abort(&Error::Old); - return Poll::Ready(Err(Error::Old)); - } - Poll::Pending => return Poll::Pending, - }; + // + // Going stale meanwhile resets nothing: the FIN said the group ended whole, + // and a reset takes that back, discarding the tail the peer has yet to read + // (quinn clears a reset stream's buffer). A live group's successor usually + // lands before its FIN is acknowledged, so a zero budget would cut nearly + // every group. + let res = ready!(writer.poll_close(waiter)); self.state = GroupState::Done; return Poll::Ready(res.map(|()| { tracing::debug!(key = self.key, "finished group"); @@ -7491,10 +7482,10 @@ mod serve_group_test { ); } - /// A FIN holds the group's bytes until the peer acknowledges it, so a group that - /// goes stale while waiting still expires instead of pinning them. + /// A group whose FIN went out is whole on the wire, so going stale before the + /// acknowledgement resets nothing: a reset would discard the tail the peer has yet to read. #[kio::test] - async fn unacknowledged_fin_expires_with_the_group() { + async fn an_unacknowledged_fin_outlives_its_group_going_stale() { let session = SinkSession::new(Log::default()).with_unacked_fin(); let log = session.log.clone(); let steer = kio::Producer::new(Steer::default()); @@ -7535,8 +7526,11 @@ mod serve_group_test { .unwrap(); edge.finish().unwrap(); - assert!(matches!(futures::poll!(serve.as_mut()), Poll::Ready(Err(Error::Old)))); - assert_eq!(log.resets(), vec![crate::StreamError::Old.to_code()]); + assert!( + futures::poll!(serve.as_mut()).is_pending(), + "still awaiting the acknowledgement" + ); + assert!(log.resets().is_empty(), "the stale group's stream is not reset"); } /// The transport keeps scheduling a finished stream's retransmissions until the @@ -8651,11 +8645,9 @@ mod serve_group_test { } } - /// A group whose stream wrote its FIN but goes stale before the acknowledgement is never - /// served again, though the subscriber may have read the first stream whole. Before lite-07 - /// the stream resets with OLD and a stale drop names it, which is final even when a later - /// update widens the budget; lite-07 leaves a stream whose end it wrote alone, and needs no - /// drop. + /// A group whose stream wrote its FIN but goes stale before the acknowledgement is left + /// alone, with no reset and no drop, and is never served again when a later update widens + /// the budget. #[kio::test] async fn a_stale_unacknowledged_fin_is_never_served_again() { for version in [Version::Lite06, Version::Lite07] { @@ -8693,12 +8685,8 @@ mod serve_group_test { write_group(&mut track, 1, 1000); write_group(&mut track, 2, 6000); while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} - let (dropped, resets): (&[(u64, u64)], &[u32]) = match version.has_group_end() { - true => (&[], &[]), - false => (&[(0, 0)], &[StreamError::Old.to_code()]), - }; - assert_eq!(drops(&mut writer, &log, version).await, dropped, "{version}"); - assert_eq!(session.resets(), resets, "{version}"); + assert!(drops(&mut writer, &log, version).await.is_empty(), "{version}"); + assert!(session.resets().is_empty(), "{version}"); let opened = run.ctx.outcomes.opened.load(Ordering::Relaxed); run.update(update(Duration::from_secs(10))); @@ -8708,7 +8696,7 @@ mod serve_group_test { opened, "{version}: group 0 is not served again" ); - assert_eq!(drops(&mut writer, &log, version).await, dropped, "{version}"); + assert!(drops(&mut writer, &log, version).await.is_empty(), "{version}"); } } From 08ea7be64711b343ad5db6301dc435f0dc47521b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 03:24:44 -0800 Subject: [PATCH 3/4] fix(net): the JS lite publisher never resets a group stream whose FIN went out Mirrors the Rust GroupServe: before lite-07 the JS publisher still reset a FIN'd group stream with OLD once its group went stale before the acknowledgement, discarding the tail the subscriber had yet to read. Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/publisher.test.ts | 25 +++++++++---------------- js/net/src/lite/publisher.ts | 7 +++---- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/js/net/src/lite/publisher.test.ts b/js/net/src/lite/publisher.test.ts index 3ff309467b..ee505613d3 100644 --- a/js/net/src/lite/publisher.test.ts +++ b/js/net/src/lite/publisher.test.ts @@ -4,7 +4,7 @@ import { Manual } from "@moq/clock/test"; import { Signal } from "@moq/signals"; import type { Grant } from "../auth.ts"; import { Producer as BroadcastProducer } from "../broadcast.ts"; -import { fromTransport, ProtocolViolation, SessionCode, StreamCode, StreamError, toStreamCode } from "../error.ts"; +import { ProtocolViolation, SessionCode, StreamCode, StreamError, toStreamCode } from "../error.ts"; import { Consumer as GroupConsumer, type Producer as GroupProducer } from "../group.ts"; import { randomHop } from "../hop.ts"; import { hooks, NO_MAX_DELAY, staleHole } from "../internal.ts"; @@ -1404,11 +1404,10 @@ test("lite draft-06: a reported cut stays dropped when re-created", async () => } }); -// A group stream that never completes after its FIN still holds the connection. Like the -// Rust `GroupServe`, the publisher waits on it only while the age budget keeps the group: -// the newest group is never stale, and once its successor arrived a full budget ago the -// stream is reset with `Old` and the group dropped. -test("lite draft-06: a group stream that never completes is reset once the group goes stale", async () => { +// A group stream whose FIN went out is whole on the wire, so, like the Rust `GroupServe`, the +// publisher never resets it as its group goes stale while the acknowledgement is pending: a reset +// would discard the tail the subscriber has yet to read. No drop names it either. +test("lite draft-06: a group stream awaiting its FIN acknowledgement is not reset as the group goes stale", async () => { const clock = new Manual(); const installed = clock.install(); const budget = 1_000; @@ -1424,25 +1423,19 @@ test("lite draft-06: a group stream that never completes is reset once the group expect("start" in (await decodeSubscribeResponse(sub.client.reader, Version.DRAFT_06))).toBe(true); let reason: unknown; - const reset = sub.stall().then((value) => { + void sub.stall().then((value) => { reason = value; }); sub.serve(1n); expect(await sub.servedHeader()).toBe(1n); await flush(); - await clock.advance(Duration.millis(budget * 10)); - expect(reason).toBeUndefined(); sub.serve(2n); expect(await sub.servedSequence()).toBe(2n); - await clock.advance(Duration.millis(budget - 1)); + const response = decodeSubscribeResponseMaybe(sub.client.reader, Version.DRAFT_06); + expect(await settlesWithin(response, clock, Duration.millis(budget * 10))).toBe(false); + await flush(); expect(reason).toBeUndefined(); - await clock.advance(Duration.millis(1)); - await reset; - expect(toStreamCode(fromTransport(reason))).toBe(StreamCode.Old); - const resp = await decodeSubscribeResponse(sub.client.reader, Version.DRAFT_06); - if (!("drop" in resp)) throw new Error("expected SUBSCRIBE_DROP"); - expect([resp.drop.start, resp.drop.end]).toEqual([1n, 2n]); } finally { await sub.close(); installed.close(); diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index 28c4e2414a..960135750e 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -2475,10 +2475,9 @@ export class Publisher { ended = true; group.close(); // A stream that fails after its FIN may have lost its header, so report it for a DROP. - // Its bytes still hold the connection until acknowledged, so before lite-07 a group gone - // stale meanwhile is reset like one still serving, as the Rust `GroupServe` does. - if (closes) await stream.closed; - else await write(() => stream.closed); + // Going stale meanwhile resets nothing, as in the Rust `GroupServe`: the FIN said the + // group ended whole, and a reset would discard the tail the peer has yet to read. + await stream.closed; return undefined; } catch (err: unknown) { let e = error(err); From b951069aba3eecbd0f1fc130e6c82d45b2bc5fd2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 11:44:43 -0800 Subject: [PATCH 4/4] fix(moq-webrtc): the egress subscribes with a 100 ms max delay A live group finishes the instant its successor starts, so under the default budget of 0 the publisher reset each one with OLD right after its FIN, and the transport dropped its unread tail: one eviction per group on every lite version. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/source.rs | 2 +- rs/moq-webrtc/src/egress.rs | 27 +++++++++++++++++++++------ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/rs/moq-mux/src/source.rs b/rs/moq-mux/src/source.rs index 8d79e9d6a7..4d359ee196 100644 --- a/rs/moq-mux/src/source.rs +++ b/rs/moq-mux/src/source.rs @@ -372,7 +372,7 @@ impl Source { /// [`Error::EscapingBroadcast`](crate::Error::EscapingBroadcast). /// /// This is the async counterpart to the poll-driven container exporters: consumers - /// that wrap a raw [`moq_net::track::Subscriber`] themselves (e.g. the WebRTC egress) + /// that wrap a raw [`moq_net::track::Subscriber`] themselves /// use it to honor cross-broadcast renditions without reimplementing the path math. #[cfg(feature = "async")] pub async fn subscribe_track( diff --git a/rs/moq-webrtc/src/egress.rs b/rs/moq-webrtc/src/egress.rs index cc5de8c3b3..93bd50ea5d 100644 --- a/rs/moq-webrtc/src/egress.rs +++ b/rs/moq-webrtc/src/egress.rs @@ -324,21 +324,36 @@ impl Rendition { match &self { Self::Audio(name, config) => { - let track = source - .subscribe_track(config.broadcast(), config.instance(), config.track(name)) - .await?; + let track = subscribe_live(&source, config.broadcast(), config.instance(), config.track(name)).await?; codec::Track::opus(track, config) } Self::Video(name, config) => { - let track = source - .subscribe_track(config.broadcast(), config.instance(), config.track(name)) - .await?; + let track = subscribe_live(&source, config.broadcast(), config.instance(), config.track(name)).await?; codec::Track::video(track, config) } } } } +/// How long a group may trail its successor before the publisher gives up on it. A live group +/// finishes the instant the next starts, so a zero budget would reset each one just after its +/// last frame went out and the transport would drop that unread tail. +const MAX_DELAY: Duration = Duration::from_millis(100); + +/// Subscribe to a rendition's track with the [`MAX_DELAY`] budget, resolving its +/// cross-broadcast reference (the catalog `broadcast` field) and `instance`. +async fn subscribe_live( + source: &moq_mux::Source, + rel: Option<&moq_net::path::Relative>, + instance: Option<&moq_net::Instance>, + name: &str, +) -> Result { + let broadcast = source.resolve(rel, instance).await?; + let delay = moq_net::track::Delay::default().with_min(None).with_max(MAX_DELAY); + let subscription = moq_net::track::Subscription::default().with_delay(delay); + Ok(broadcast.track(name)?.subscribe(subscription).await?) +} + /// The best [enabled](hang::catalog::Audio::enabled) Opus rendition that `source` can reach. fn pick_audio<'a>( source: &moq_mux::Source,