diff --git a/js/hang/src/container/consumer.holes.test.ts b/js/hang/src/container/consumer.holes.test.ts new file mode 100644 index 00000000000..5f17293e93d --- /dev/null +++ b/js/hang/src/container/consumer.holes.test.ts @@ -0,0 +1,104 @@ +import { expect, test } from "bun:test"; +import { Manual, Micro, Milli, Nano } from "@moq/clock"; +import { Group, Time, Track } from "@moq/net"; +import { Consumer, type Read } from "./consumer.ts"; +import { encodeFrame, Format } from "./legacy.ts"; + +const flush = () => new Promise((resolve) => setImmediate(resolve)); + +test.each([0, 100])("a sequence hole before a paused data marker expires at %sms", async (budget) => { + const clock = new Manual(); + const restore = clock.install(); + const track = new Track.Producer("data"); + const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { + format: new Format("data"), + maxDelay: Milli(budget), + }); + const results: Read[] = []; + const reading = (async () => { + for (;;) { + const read = await consumer.next(); + if (!read) return; + results.push(read); + } + })(); + try { + const first = new Group.Producer(0); + track.writeGroup(first); + first.writeFrame({ payload: encodeFrame(new Uint8Array([1]), Micro(10_000)), timestamp: Time.Timestamp.now() }); + first.close(); + await flush(); + expect(results).toHaveLength(1); + const marker = new Group.Producer(2); + track.writeGroup(marker); + marker.writeFrame({ payload: new Uint8Array(), timestamp: Time.Timestamp.now() }); + marker.close(); + await flush(); + if (budget > 0) { + expect(results).toHaveLength(1); + await clock.advanceAsync(Nano.fromMilli(Milli(budget - 1))); + expect(results).toHaveLength(1); + await clock.advanceAsync(Nano.fromMilli(Milli(1))); + } + expect(results.slice(1)).toEqual([ + { kind: "gap", timestamp: Micro(10_000), duration: Micro.zero }, + { kind: "end", group: 2, end: Micro(10_000) }, + ]); + } finally { + consumer.close(); + track.close(); + await reading; + restore(); + } +}); + +test("a filled hole cancels its timer and the next hole gets a fresh budget", async () => { + const clock = new Manual(); + const restore = clock.install(); + const track = new Track.Producer("data"); + const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { + format: new Format("data"), + maxDelay: Milli(100), + }); + const results: Read[] = []; + const reading = (async () => { + for (;;) { + const read = await consumer.next(); + if (!read) return; + results.push(read); + } + })(); + const write = (sequence: number, timestamp?: number) => { + const group = new Group.Producer(sequence); + track.writeGroup(group); + group.writeFrame({ + payload: timestamp === undefined ? new Uint8Array() : encodeFrame(new Uint8Array([1]), Micro(timestamp)), + timestamp: Time.Timestamp.fromMicros(timestamp ?? 20_000), + }); + group.close(); + }; + try { + write(0, 0); + await flush(); + write(2, 20_000); + await flush(); + await clock.advanceAsync(Nano.fromMilli(Milli(50))); + write(1, 10_000); + await flush(); + expect(results.map((read) => read.kind)).toEqual(["frame", "frame", "frame"]); + write(4); + await flush(); + await clock.advanceAsync(Nano.fromMilli(Milli(99))); + expect(results).toHaveLength(3); + await clock.advanceAsync(Nano.fromMilli(Milli(1))); + expect(results.slice(3)).toEqual([ + { kind: "gap", timestamp: Micro(20_000), duration: Micro.zero }, + { kind: "end", group: 4, end: Micro(20_000) }, + ]); + } finally { + consumer.close(); + track.close(); + await reading; + restore(); + } +}); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 1d02709aebf..51770c432a4 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -1,4 +1,4 @@ -import { Micro, Milli, Nano, now } from "@moq/clock"; +import { Instant, Micro, Milli, Nano, now } from "@moq/clock"; import * as Moq from "@moq/net"; import { Effect, type Getter, type GetterInit, getter, Once, Signal } from "@moq/signals"; import type { Format } from "./format"; @@ -13,7 +13,9 @@ export interface ConsumerProps { * How stale a group may get before it is skipped, in milliseconds (default: 0). * * Measured as the span from the oldest buffered frame to the newest, so it bounds how long a - * late or missing group is waited for. The local half of the subscription's + * late group is waited for. A missing sequence also expires after this much monotonic + * time blocked on it, even when the next group is a marker on a paused track. + * The local half of the subscription's * `maxDelay`; both measure the same budget, one on the wire and one as frames are read. */ // Read-only: a Getter (e.g. another component's output) is accepted directly. @@ -103,6 +105,7 @@ export class Consumer { #maxDelay: Getter; #groups: Group[] = []; #active?: number; // the active group sequence number + #hole?: { sequence: number; since: Instant }; // Presentation end (max PTS + duration) of the group we most recently advanced past, so next()'s // promotion guard can tell a timeline-continuous next group from one sitting after a gap. // Maintained only via #recordPresented; see its comment for the invariant. @@ -501,6 +504,9 @@ export class Consumer { if (this.#error) throw this.#error; const ended = this.#closed.peek(); + if (this.#groups[0]?.consumer.sequence === this.#active || this.#groups.length === 0) { + this.#hole = undefined; + } if (this.#groups.length === 0) { if (ended !== undefined) { if (ended instanceof Error) throw ended; @@ -520,11 +526,14 @@ export class Consumer { // max delay, so anything still missing in between would arrive too old to play. Proving it // here matters: #checkMaxDelay would instead drop the head, the very group to play next. // After track termination no missing group can arrive, so drain across any remaining gap. - // Otherwise wait: #checkMaxDelay skips once the budget is spent, and #tryDurationSkip once - // the duration covers it. + // A missing sequence also expires on elapsed monotonic time, so timestamp-free markers + // behind a hole can end a paused track without more arrivals. if (this.#active !== undefined && this.#groups.length > 0) { const head = this.#groups[0]; if (head.consumer.sequence > this.#active) { + if (this.#hole?.sequence !== this.#active) { + this.#hole = { sequence: this.#active, since: now() }; + } const startup = this.#start === undefined; const contiguous = ptsContiguous(this.#presentedEnd, head.frames.at(0)?.timestamp); const empty = head.empty && head.consumer.done; @@ -535,9 +544,18 @@ export class Consumer { (this.#presentedEnd !== undefined && head.latest !== undefined && head.latest - this.#presentedEnd > maxDelay)); - if (startup || empty || contiguous || skipHole || ended !== undefined) { - if (!contiguous && !empty) this.#lost = true; + const elapsed = Instant.durationSince(now(), this.#hole.since) >= Nano.fromMicro(maxDelay); + if (startup || empty || contiguous || skipHole || elapsed || ended !== undefined) { + if (!contiguous && !empty) { + this.#lost = true; + if (!startup) { + console.warn( + `skipping missing groups: track=${this.#track.name} ${this.#active} -> ${head.consumer.sequence}`, + ); + } + } this.#active = head.consumer.sequence; + this.#hole = undefined; } } } @@ -621,6 +639,16 @@ export class Consumer { const abort = this.#signals.abort; if (abort.aborted) return undefined; + let timer: Effect | undefined; + if (this.#hole) { + timer = new Effect(); + const since = this.#hole.since; + timer.run((effect) => { + const budget = Nano.fromMilli(effect.get(this.#maxDelay)); + const elapsed = Instant.durationSince(now(), since); + effect.timer(() => this.#notify?.(), Nano(Math.max(0, budget - elapsed))); + }); + } const aborted = await new Promise((resolve) => { const onAbort = () => resolve(true); @@ -630,6 +658,7 @@ export class Consumer { resolve(false); }; }); + timer?.close(); this.#notify = undefined; if (aborted) return undefined; diff --git a/quest/m1/ts-reliability/README.md b/quest/m1/ts-reliability/README.md index a01e14cd507..f543017f979 100644 --- a/quest/m1/ts-reliability/README.md +++ b/quest/m1/ts-reliability/README.md @@ -100,7 +100,6 @@ remain with their current quests. ## Required - [Plan: semantic fidelity](/quest/m1/ts-reliability/fidelity.md) - explicit profiles and PMT lifecycle contracts produce implementation quests -- [Sequence holes get a time budget](/quest/m1/ts-reliability/hole-budget.md) - a hole gives up on max(wall, pts), so a break marker behind it on a paused track is delivered - [Loss and recovery tests](/quest/m1/ts-reliability/recovery.md) - actual MoQ omissions exercise reacquisition and unaffected tracks ## Related diff --git a/quest/m1/ts-reliability/hole-budget.md b/quest/m1/ts-reliability/hole-budget.md deleted file mode 100644 index dcd31839ea4..00000000000 --- a/quest/m1/ts-reliability/hole-budget.md +++ /dev/null @@ -1,44 +0,0 @@ -# [S] Sequence holes get a time budget - -## Goal - -A container consumer waiting on a missing group sequence gives up on it once -a budget expires on max(wall clock, pts), then reports the gap and what -follows. A break marker behind a hole on a paused track is delivered without -waiting for media to resume or the track to finish. Rust -(`rs/moq-mux/src/container/consumer.rs`) and JS -(`js/hang/src/container/consumer.ts`) agree. - -## Plan - -Decided with the maintainer on 2026-10-17: the goal as drafted, from the -review of [#5815](https://github.com/moq-dev/moq/pull/5815). The budget covers -every kind of hole, not only one before a data marker, and runs on -max(wall, pts) so it fires on a paused track. - -Today a marker carries no timestamp, so a hole before it on an open, paused -track waits indefinitely. A buffered higher sequence alone proves nothing, -since a late group is not a lost one; the budget is what decides. - -The two anchors, matching the timers the TS line already uses: - -- PTS: the existing age measure, how far the newest content - (`max_timestamp`) runs past where the next stamped group begins - (`next_start`), in `rs/moq-mux/src/container/consumer.rs`, and its - counterpart in `js/hang/src/container/consumer.ts`. A bare marker has no - timestamp, so this side never fires behind one. -- Wall: the `moq_clock` instant the reader first found itself blocked on - the hole, the way the TS export anchors a frame's arrival at the instant - its source was last found empty (`Track::empty` in - `rs/moq-mux/src/container/ts/export.rs`, judged by `jitter::Buffer` in - `rs/moq-mux/src/jitter.rs`). A timer wakes the reader at the anchor plus - the budget, with no new group needed. - -The hole is lost once either side reaches the budget. Decided with the -maintainer on 2026-10-17: the budget is the subscription's max delay, which -already drives latency skips, so both sides share one number. With the -default of zero, a hole counts as lost as soon as the reader blocks on it. - -Test with mocked time in both languages: read group N, lose N+1, receive a -closed marker as N+2 on a paused track; after the budget the reader reports a -gap, then the end. It fails without the fix. diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 19bd5ccb248..6683bbfad81 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -43,6 +43,8 @@ pub(crate) enum Item { /// With a non-zero max delay, slow groups are tolerated up to that budget before being /// skipped. A missing sequence gets the same tolerance: there is no way to tell a stream /// that lost the delivery race from one the cache evicted. +/// Its budget also expires after that much monotonic time blocked on it, so a marker +/// behind a hole is delivered even when the track is paused. /// /// Delivery starts at the lowest absolute start among the /// [`Subscription::ranges`](moq_net::track::Subscription::ranges) when every range names one, waiting for that group under the budget; without one it @@ -98,6 +100,10 @@ pub struct Consumer { // How far we may drift from the live edge before skipping a group. max_delay: std::time::Duration, + // Missing sequence and when it first blocked delivery; cleared when the cursor moves. + hole: Option<(u64, moq_clock::Instant)>, + hole_timer: Option>>, + // The furthest presentation point delivered so far: max(timestamp + duration), endpoints // included. `None` until the first item, so a loss before it is no gap. edge: Option, @@ -192,6 +198,8 @@ impl Consumer { startup: start.is_none(), newest: false, max_delay, + hole: None, + hole_timer: None, edge: None, start: None, floor: None, @@ -363,6 +371,11 @@ impl Consumer { })?; 'read: loop { + if self.pending.front().is_none_or(|group| group.sequence <= self.current) { + self.hole = None; + self.hole_timer = None; + } + // Return the next frame from the current group if possible. // If the current group is finished or errored, advance to the next group. if let Some(group) = self.pending.front_mut() @@ -498,19 +511,37 @@ impl Consumer { // sequences that never arrived: finishing the track ends new groups, not // the frames still flowing on ones already open, so arrived groups are // settled below by their own FIN, abort, or the budget. - if let Some(front_sequence) = self.pending.front().map(|g| g.sequence) - && front_sequence > self.current - && let Some((_, next_start)) = next_group - && (finished || max_timestamp.saturating_sub(next_start) >= self.max_delay) + if let Some(front) = self.pending.front() + && front.sequence > self.current { - // A data marker has no timestamp to show the missing sequences held nothing. - let bare = self.pending.front().is_some_and(|group| group.bare); - if bare || !pts_contiguous(current_end.or(self.presented_end), next_start) { - self.lose(); + let sequence = front.sequence; + if self.hole.is_none_or(|(sequence, _)| sequence != self.current) { + self.hole = Some((self.current, moq_clock::now())); + } + let since = self.hole.unwrap().1; + let elapsed = since.elapsed() >= self.max_delay; + let aged = next_group.is_some_and(|(_, start)| max_timestamp.saturating_sub(start) >= self.max_delay); + if finished || elapsed || aged { + let _ = self.pending.front_mut().unwrap().look_all(waiter, &self.format); + let front = self.pending.front_mut().unwrap(); + let empty = front.empty && matches!(front.group.poll_finished(waiter), Poll::Ready(Ok(_))); + let front = self.pending.front().unwrap(); + let next_start = front.min_timestamp.map(std::time::Duration::from).or(front.max_end); + // A data marker has no timestamp to prove continuity across the hole. + if !empty && next_start.is_none_or(|start| !pts_contiguous(self.presented_end, start)) { + self.lose(); + } + self.current = sequence; + self.note_group_edge(); + continue; + } + if let Some(wake) = since.checked_add(self.max_delay) { + let timer = self + .hole_timer + .get_or_insert_with(|| Box::pin(moq_clock::sleep_until(wake))); + timer.as_mut().reset(wake); + let _ = waiter.poll_future(timer.as_mut()); } - self.current = front_sequence; - self.note_group_edge(); - continue; } // The current group is blocking. Everything it could still present ends @@ -551,26 +582,6 @@ impl Consumer { continue; } - if finished - && let Some(front_sequence) = self.pending.front().map(|g| g.sequence) - && front_sequence > self.current - { - let _ = self.pending.front_mut().unwrap().look_all(waiter, &self.format); - let front = self.pending.front().unwrap(); - let next_start = front.min_timestamp.map(std::time::Duration::from).or(front.max_end); - // A data marker has no timestamp to show the missing sequences held nothing. - let lost = match next_start { - Some(start) => !pts_contiguous(current_end.or(self.presented_end), start), - None => front.bare, - }; - if lost { - self.lose(); - } - self.current = front_sequence; - self.note_group_edge(); - continue; - } - if finished && self.pending.is_empty() { if let Some(err) = self.track_error.take() { // The caller only gets the bare error, with nothing to say which track died. @@ -601,6 +612,8 @@ impl Consumer { } fn note_group_edge(&mut self) { + self.hole = None; + self.hole_timer = None; if let Some((_, ts)) = self.start { self.floor = Some(ts); } @@ -1214,6 +1227,84 @@ mod tests { assert_eq!(items(&mut reader).await, ["0", "10000", "gap 10000+0", "end 10000"]); } + #[tokio::test(start_paused = true)] + async fn a_sequence_hole_before_a_paused_data_marker_expires() { + for budget in [Duration::ZERO, Duration::from_millis(100)] { + let mut track = track_producer("data", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let subscriber = track.subscribe(moq_net::track::Subscription::default().with_max_delay(budget)); + let mut reader = Consumer::new(subscriber, Container::Legacy(crate::container::Kind::Data)); + write_group(&mut track, 0, &[ts(10_000)]); + assert!(matches!(reader.read().await.unwrap(), Some(Read::Frame { .. }))); + let mut marker = track.create_group(moq_net::group::Info { sequence: 2 }).unwrap(); + marker.write_frame(ts(10_000), Bytes::new()).unwrap(); + marker.finish().unwrap(); + + if !budget.is_zero() { + assert!(reader.poll_read(&kio::Waiter::noop()).is_pending()); + tokio::time::advance(budget - Duration::from_millis(1)).await; + assert!(reader.poll_read(&kio::Waiter::noop()).is_pending()); + let before = moq_clock::now(); + let read = moq_clock::timeout(Duration::from_millis(2), reader.read()) + .await + .unwrap() + .unwrap(); + assert!(matches!(read, Some(Read::Gap { .. }))); + assert_eq!( + before.elapsed(), + Duration::from_millis(1), + "the hole timer wakes the reader" + ); + } else { + assert!(matches!( + reader.poll_read(&kio::Waiter::noop()), + Poll::Ready(Ok(Some(Read::Gap { .. }))) + )); + } + assert!( + matches!(reader.poll_read(&kio::Waiter::noop()), Poll::Ready(Ok(Some(Read::End { group: 2, end }))) if end == ts(10_000)) + ); + track.finish().unwrap(); + } + } + + #[tokio::test(start_paused = true)] + async fn a_filled_sequence_hole_gives_the_next_hole_a_fresh_budget() { + let mut track = track_producer("data", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let subscriber = + track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_millis(100))); + let mut reader = Consumer::new(subscriber, Container::Legacy(crate::container::Kind::Data)); + write_group(&mut track, 0, &[ts(0)]); + assert!(matches!( + reader.read().await.unwrap(), + Some(Read::Frame { group: 0, .. }) + )); + write_group(&mut track, 2, &[ts(20_000)]); + assert!(reader.poll_read(&kio::Waiter::noop()).is_pending()); + tokio::time::advance(Duration::from_millis(50)).await; + write_group(&mut track, 1, &[ts(10_000)]); + assert!(matches!( + reader.read().await.unwrap(), + Some(Read::Frame { group: 1, .. }) + )); + assert!(matches!( + reader.read().await.unwrap(), + Some(Read::Frame { group: 2, .. }) + )); + let mut marker = track.create_group(moq_net::group::Info { sequence: 4 }).unwrap(); + marker.write_frame(ts(20_000), Bytes::new()).unwrap(); + marker.finish().unwrap(); + assert!(reader.poll_read(&kio::Waiter::noop()).is_pending()); + tokio::time::advance(Duration::from_millis(99)).await; + assert!(reader.poll_read(&kio::Waiter::noop()).is_pending()); + tokio::time::advance(Duration::from_millis(1)).await; + assert!(matches!( + reader.poll_read(&kio::Waiter::noop()), + Poll::Ready(Ok(Some(Read::Gap { .. }))) + )); + assert!(matches!(reader.read().await.unwrap(), Some(Read::End { group: 4, .. }))); + track.finish().unwrap(); + } + /// A malformed data marker group found while looking ahead fails the reader, rather than /// reading as a gap when a latency skip passes over it. #[tokio::test(start_paused = true)]