From 5181bda53c208a942d92fe0353871c0ab96723f8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 11 Sep 2026 22:53:00 -0700 Subject: [PATCH 1/4] chore: claim quest/m1/monotonic-timeline From f55ccd258b2d87ec9f5a72639052709633648902 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 16 Sep 2026 22:53:44 -0700 Subject: [PATCH 2/4] feat(hang)!: timelines only move forward A marker group of one empty frame declares a discontinuity. Producers refuse a frame below the live edge. Consumers abort a rewound group as malformed and jump the playhead on an unproven hole. MPEG-TS export no longer fences tracks across a break. Co-Authored-By: grok-4.6 --- doc/bin/cli.md | 6 +- doc/bin/srt.md | 2 +- drafts/draft-lcurley-moq-hang.md | 13 +- js/hang/src/container/consumer.test.ts | 219 +++----- js/hang/src/container/consumer.ts | 317 ++++------- js/hang/src/container/legacy.ts | 17 +- rs/moq-audio/src/decode/consumer.rs | 22 +- rs/moq-audio/src/encode/producer.rs | 7 +- rs/moq-mux/src/codec/legacy.rs | 5 + rs/moq-mux/src/container/consumer.rs | 614 +++++++-------------- rs/moq-mux/src/container/fmp4/mod.rs | 4 + rs/moq-mux/src/container/mod.rs | 12 +- rs/moq-mux/src/container/producer.rs | 160 +++++- rs/moq-mux/src/container/source.rs | 4 +- rs/moq-mux/src/container/ts/export.rs | 46 +- rs/moq-mux/src/container/ts/export_test.rs | 148 ++--- rs/moq-mux/src/container/ts/import.rs | 111 +++- rs/moq-mux/src/error.rs | 4 + rs/moq-video/src/decode/consumer.rs | 20 +- 19 files changed, 736 insertions(+), 995 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 00e6a81251b..53f7ce590d5 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -59,10 +59,10 @@ turn. The same flag on an elementary PID other than the program PCR PID, a continuity-counter gap, and the 33-bit timestamp rollover move no clock and declare nothing. FLV covers H.264 + AAC. -MPEG-TS export restarts its clock and table cadence after a publisher rewind, +MPEG-TS export restarts its clock and table cadence after a declared marker, discarding the old mux buffer. The first new clock packet signals the break and -stdout pacing re-anchors. Other renditions resume at their own discontinuity -boundary, so old-timeline frames cannot advance the new clock. +stdout pacing re-anchors. Every rendition joins the new program generation; +no track is fenced across the marker. fMP4 export writes one fragment per publisher group on each track. Audio follows the publisher's cuts; video normally follows GOPs. Closing a group flushes it diff --git a/doc/bin/srt.md b/doc/bin/srt.md index f00bb340419..bd8e852ab71 100644 --- a/doc/bin/srt.md +++ b/doc/bin/srt.md @@ -24,7 +24,7 @@ moq --connect https://relay.example.com/anon --broadcast event.hang import srt - `--latency` sets the SRT receive buffer and doubles as the skip threshold on export. Export paces each SRT payload on the media clock, and re-anchors that -pacing when the publisher rewinds, so a restarted timeline plays out from the +pacing on a declared marker, so a restarted timeline plays out from the live edge instead of stalling until it catches up. A `--connect` URL needs a `streamid` query or a path; a listener bridges one `--broadcast` and ignores the stream id it is offered. The library is [`moq-srt`](https://docs.rs/moq-srt). diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index 435da73c23d..873ddcc8459 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -508,13 +508,17 @@ type Container = The `kind` field selects the framing; a consumer MUST ignore a rendition whose `kind` it does not recognize. Every container shares the same group rules: -Each moq-lite group MUST start with a keyframe. +Each moq-lite group MUST start with a keyframe, except a group that contains only a discontinuity marker. If the codec does not support delta frames (e.g. audio), a group MAY consist of multiple keyframes. Otherwise, a group MUST consist of a single keyframe followed by zero or more delta frames. -An empty group declares a discontinuity between codec epochs. -A consumer MUST reset codec state before decoding the next non-empty group, including reapplying any codec startup delay or pre-skip. -This applies whether the resumed timestamps move backward or forward. +A group with no decodable frames is a walk-now discontinuity: one empty codec payload and no media. +A consumer MUST NOT submit the marker to a decoder. +Empty groups (zero objects) are permitted and mean nothing. +After a discontinuity the timeline continues forward. +A group whose timestamps fall below the live edge earlier groups reached is malformed. +A delivered sequence hole is a playhead event unless the boundary is contiguous within 1 ms. +A consumer re-applies startup delay and skip at a playhead event; it does not reset codec state. ## legacy The default, used when the `container` field is absent. @@ -1049,6 +1053,7 @@ A publisher MAY estimate an unknown final duration from the frame cadence, but M - Compared existing track properties by parsed values rather than JSON serialization. - Required exclusive DVR restart recovery to remove unreferenced group objects left by interrupted expiration. - Replaced the catalog root `timeline` field with `archive`, carrying the timeline track plus optional `replay`, `store`, and recording `version`. +- A marker group of one empty frame declares a discontinuity. Empty groups mean nothing. Timestamps only move forward; a group below the live edge is malformed. A delivered sequence hole is a playhead event unless contiguous within 1 ms. # Acknowledgments {:numbered="false"} diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index fff1f4845d6..b8c1cc9c07e 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -100,6 +100,34 @@ test("LegacyFormat throws on empty input", () => { expect(() => format.decode(new Uint8Array(0))).toThrow(); }); +test("Legacy Producer refuses a group below the live edge", () => { + const track = new Track.Producer("test"); + const producer = new LegacyProducer(track, new LegacyFormat("video")); + producer.encode(new Uint8Array([1]), 0 as Time.Micro, true); + producer.encode(new Uint8Array([1]), 33_000 as Time.Micro, false); + expect(() => producer.encode(new Uint8Array([1]), 16_000 as Time.Micro, true)).toThrow("below the live edge"); + producer.close(); +}); + +test("Legacy Producer accepts B-frames within a group", () => { + const track = new Track.Producer("test"); + const producer = new LegacyProducer(track, new LegacyFormat("video")); + producer.encode(new Uint8Array([1]), 0 as Time.Micro, true); + producer.encode(new Uint8Array([1]), 66_000 as Time.Micro, false); + producer.encode(new Uint8Array([1]), 33_000 as Time.Micro, false); + producer.close(); +}); + +test("Legacy Producer accepts open-GOP leading pictures above the previous group", () => { + const track = new Track.Producer("test"); + const producer = new LegacyProducer(track, new LegacyFormat("video")); + producer.encode(new Uint8Array([1]), 0 as Time.Micro, true); + producer.encode(new Uint8Array([1]), 33_000 as Time.Micro, false); + producer.encode(new Uint8Array([1]), 66_000 as Time.Micro, true); + producer.encode(new Uint8Array([1]), 50_000 as Time.Micro, false); + producer.close(); +}); + test("Legacy Producer writes a duration marker at the next keyframe", async () => { const track = new Track.Producer("test"); const subscriber = track.subscribe({ maxAge: 30_000 }); @@ -189,12 +217,12 @@ test("Legacy Producer rejects a backwards cut without closing the group", async expect(timestamps).toEqual([20_000, 30_000, 35_000]); }); -test("Legacy Producer accepts a keyframe that rewinds the timeline", () => { +test("Legacy Producer refuses a keyframe that rewinds the timeline", () => { const track = new Track.Producer("test"); const producer = new LegacyProducer(track, new LegacyFormat("video")); producer.encode(new Uint8Array([1]), 20_000 as Time.Micro, true); producer.encode(new Uint8Array([2]), 30_000 as Time.Micro, false); - expect(() => producer.encode(new Uint8Array([3]), 0 as Time.Micro, true)).not.toThrow(); + expect(() => producer.encode(new Uint8Array([3]), 0 as Time.Micro, true)).toThrow("below the live edge"); producer.close(); }); @@ -324,6 +352,16 @@ function writeGroupWithLegacyFrames(track: Track.Producer, sequence: number, tim track.writeGroup(group); } +function writeMarkerGroup(track: Track.Producer, sequence: number, timestamp: Time.Micro) { + const group = new Group.Producer(sequence); + group.writeFrame({ + payload: encodeLegacyFrame(timestamp, new Uint8Array()), + timestamp: Time.Timestamp.fromMicros(timestamp), + }); + group.close(); + track.writeGroup(group); +} + async function drainFrames( consumer: Consumer, timeout: number, @@ -502,15 +540,15 @@ test("Consumer delivers a group that arrives below the cursor", async () => { const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxAge: 500 as Time.Milli }); // Group.Producer 5 arrives first (sets active = 5) - writeGroupWithLegacyFrames(track, 5, [0 as Time.Micro]); + writeGroupWithLegacyFrames(track, 5, [100_000 as Time.Micro]); await new Promise((resolve) => setTimeout(resolve, 50)); // Group.Producer 3 lands behind it, and Group.Producer 6 ahead of it. Arriving below the // cursor is not what makes content stale: how far behind the live edge a group may be is // the subscription's own max age, which `replay` deliberately opens wide here. So all // three are handed over, in sequence order, since delivery has not passed any of them yet. - writeGroupWithLegacyFrames(track, 3, [100_000 as Time.Micro]); - writeGroupWithLegacyFrames(track, 6, [30_000 as Time.Micro]); + writeGroupWithLegacyFrames(track, 3, [0 as Time.Micro]); + writeGroupWithLegacyFrames(track, 6, [200_000 as Time.Micro]); track.close(); await new Promise((resolve) => setTimeout(resolve, 100)); @@ -642,7 +680,7 @@ test("Consumer preserves empty Legacy and LOC data frames", async () => { } }); -// --- Rewinds at the playback cursor --- +// --- Malformed rewind --- /** Read until the next media frame, skipping the group-done markers in between. */ async function nextFrame(consumer: Consumer) { @@ -652,119 +690,35 @@ async function nextFrame(consumer: Consumer) { } } -// The cursor advances past a finished group before its successor arrives, so the successor is the -// active sequence by the time it rewinds. Detection has to key on the group that supplied the -// delivered live edge, not on the cursor, or the rewind goes unreported. -test("Consumer signals a rewind carried by an already-buffered successor", async () => { +test("Consumer aborts a group below the live edge", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxAge: 30_000 as Time.Milli }); - // Both groups buffer before anything is read, so the successor's frames arrive while the live - // edge is still unset and only a re-check after delivery can spot the jump. One sequential - // group, so no higher-sequence successor can reveal the reset instead. writeGroupWithLegacyFrames(track, 0, [10_000_000 as Time.Micro]); writeGroupWithLegacyFrames(track, 1, [0 as Time.Micro, 100_000 as Time.Micro]); await settle(); expect((await nextFrame(consumer))?.frame?.timestamp).toBe(10_000_000 as Time.Micro); - - const first = await nextFrame(consumer); - expect(first?.frame?.timestamp).toBe(0 as Time.Micro); - expect(first?.discontinuity).toBe(1); - expect(first?.continuous).toBe(false); - - const second = await nextFrame(consumer); - expect(second?.frame?.timestamp).toBe(100_000 as Time.Micro); - expect(second?.discontinuity).toBe(1); // no duplicate reset within a group + await expect(nextFrame(consumer)).rejects.toThrow("below the live edge"); consumer.close(); }); -test("Consumer signals a rewind carried by a later arrival", async () => { +test("Consumer aborts a rewind carried by a later arrival", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxAge: 30_000 as Time.Milli }); writeGroupWithLegacyFrames(track, 0, [10_000_000 as Time.Micro]); await settle(); expect((await nextFrame(consumer))?.frame?.timestamp).toBe(10_000_000 as Time.Micro); - expect((await consumer.next())?.frame).toBeUndefined(); // group 0 done: the cursor moves to 1 + expect((await consumer.next())?.frame).toBeUndefined(); - // Park the reader with nothing buffered, then rewind into the sequence it is waiting on. const pending = nextFrame(consumer); await settle(); writeGroupWithLegacyFrames(track, 1, [0 as Time.Micro, 100_000 as Time.Micro]); - const first = await pending; - expect(first?.frame?.timestamp).toBe(0 as Time.Micro); - expect(first?.discontinuity).toBe(1); - expect(first?.continuous).toBe(false); - - consumer.close(); -}); - -// A reset resumes from the earliest survivor, which can be a group still ambiguous by sequence. -// That puts the cursor on a group whose timestamps have yet to rule on it, so the stale check has -// to run for the active sequence too. -test("Consumer drops a reneged straggler that the reset left on the cursor", async () => { - const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxAge: 30_000 as Time.Milli }); - - // Group 10 supplies the live edge and stays open, so the cursor sits on it. - const previous = new Group.Producer(10); - previous.writeFrame({ payload: encodeLegacy(5_000_000 as Time.Micro), timestamp: Time.Timestamp.now() }); - track.writeGroup(previous); - await settle(); - expect((await nextFrame(consumer))?.frame?.timestamp).toBe(5_000_000 as Time.Micro); - - // An ambiguous-sequence group opens with no frames, so the reset below cannot classify it. - const straggler = new Group.Producer(15); - track.writeGroup(straggler); - await settle(); - - // The rewind. Group 15 survives as ambiguous and, as the earliest survivor, takes the cursor. - writeGroupWithLegacyFrames(track, 20, [0 as Time.Micro, 100_000 as Time.Micro]); - await settle(); - - // Group 15 finally resolves as an old-epoch straggler; its frames must never play. - straggler.writeFrame({ payload: encodeLegacy(5_500_000 as Time.Micro), timestamp: Time.Timestamp.now() }); - straggler.close(); - await settle(); - - const first = await nextFrame(consumer); - expect(first?.frame?.timestamp).toBe(0 as Time.Micro); - expect(first?.discontinuity).toBe(1); - expect((await nextFrame(consumer))?.frame?.timestamp).toBe(100_000 as Time.Micro); - - previous.close(); - consumer.close(); -}); - -test("Consumer preserves a new-epoch group after its first frame is consumed", async () => { - const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxAge: 30_000 as Time.Milli }); - const previous = new Group.Producer(10); - previous.writeFrame({ payload: encodeLegacy(5_000_000 as Time.Micro), timestamp: Time.Timestamp.now() }); - track.writeGroup(previous); - await settle(); - expect((await nextFrame(consumer))?.frame?.timestamp).toBe(5_000_000 as Time.Micro); + await expect(pending).rejects.toThrow("below the live edge"); - const ambiguous = new Group.Producer(15); - track.writeGroup(ambiguous); - await settle(); - writeGroupWithLegacyFrames(track, 20, [100_000 as Time.Micro]); - await settle(); - ambiguous.writeFrame({ payload: encodeLegacy(50_000 as Time.Micro), timestamp: Time.Timestamp.now() }); - await settle(); - expect((await nextFrame(consumer))?.frame?.timestamp).toBe(50_000 as Time.Micro); - - ambiguous.writeFrame({ payload: encodeLegacy(100_000 as Time.Micro), timestamp: Time.Timestamp.now() }); - ambiguous.close(); - await settle(); - const remainder = await nextFrame(consumer); - expect(remainder?.group).toBe(15); - expect(remainder?.frame?.timestamp).toBe(100_000 as Time.Micro); - expect(remainder?.discontinuity).toBe(1); - previous.close(); consumer.close(); }); @@ -795,6 +749,22 @@ test("Consumer treats B-frame reordering within a group as continuous", async () consumer.close(); }); +test("Consumer accepts open-GOP leading pictures above the previous group", async () => { + const track = new Track.Producer("test"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxAge: 30_000 as Time.Milli }); + + writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro, 33_000 as Time.Micro]); + writeGroupWithLegacyFrames(track, 1, [66_000 as Time.Micro, 50_000 as Time.Micro]); + await settle(); + for (const timestamp of [0, 33_000, 66_000, 50_000]) { + const result = await nextFrame(consumer); + expect(result?.frame?.timestamp).toBe(timestamp as Time.Micro); + expect(result?.discontinuity).toBe(0); + } + + consumer.close(); +}); + // --- Buffered signal --- test("Consumer buffered signal updates as frames arrive", async () => { @@ -1294,14 +1264,12 @@ test("Consumer reports continuity while nothing is dropped", async () => { consumer.close(); }); -test("Consumer reports an empty group as a codec discontinuity", async () => { +test("Consumer reports a marker group as a playhead event", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxAge: 2_000 as Time.Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxAge: 2_000 as Time.Milli }); writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); - const marker = new Group.Producer(1); - track.writeGroup(marker); - marker.close(); + writeMarkerGroup(track, 1, 0 as Time.Micro); writeGroupWithLegacyFrames(track, 2, [1_000_000 as Time.Micro]); await settle(); @@ -1311,64 +1279,42 @@ test("Consumer reports an empty group as a codec discontinuity", async () => { expect(reset?.frame).toBeUndefined(); expect(reset?.discontinuity).toBe(1); expect(reset?.continuous).toBe(false); - expect((await consumer.next())?.discontinuity).toBe(1); + expect((await nextFrame(consumer))?.discontinuity).toBe(1); consumer.close(); }); -test("Consumer advances a completed empty group across a sequence gap", async () => { +test("Consumer empty groups mean nothing", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxAge: 0 as Time.Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxAge: 2_000 as Time.Milli }); writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); + const empty = new Group.Producer(1); + track.writeGroup(empty); + empty.close(); + writeGroupWithLegacyFrames(track, 2, [1_000 as Time.Micro]); await settle(); - expect((await consumer.next())?.frame?.timestamp).toBe(0 as Time.Micro); - expect((await consumer.next())?.frame).toBeUndefined(); - const marker = new Group.Producer(2); - track.writeGroup(marker); - marker.close(); - await settle(); - - writeGroupWithLegacyFrames(track, 3, [1_000_000 as Time.Micro]); - await settle(); - - const reset = await consumer.next(); - expect(reset?.frame).toBeUndefined(); - expect(reset?.discontinuity).toBe(1); - const resumed = await consumer.next(); - expect(resumed?.frame?.timestamp).toBe(1_000_000 as Time.Micro); - expect(resumed?.discontinuity).toBe(1); + expect((await nextFrame(consumer))?.frame?.timestamp).toBe(0 as Time.Micro); + expect((await nextFrame(consumer))?.frame?.timestamp).toBe(1_000 as Time.Micro); + expect(consumer.discontinuity).toBe(0); consumer.close(); }); -test("Consumer waits for an empty-group FIN before age-skipping it", async () => { +test("Consumer jumps the playhead after a shed marker with a timestamp hole", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxAge: 0 as Time.Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxAge: 0 as Time.Milli }); writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); await settle(); expect((await consumer.next())?.frame?.timestamp).toBe(0 as Time.Micro); expect((await consumer.next())?.frame).toBeUndefined(); - const marker = new Group.Producer(2); - track.writeGroup(marker); - writeGroupWithLegacyFrames(track, 3, [1_000_000 as Time.Micro, 1_100_000 as Time.Micro]); + writeGroupWithLegacyFrames(track, 3, [1_000_000 as Time.Micro]); await settle(); - const pending = consumer.next(); - const beforeFin = await Promise.race([ - pending.then(() => "ready" as const), - settle(50).then(() => "pending" as const), - ]); - expect(beforeFin).toBe("pending"); - - marker.close(); - const reset = await pending; - expect(reset?.frame).toBeUndefined(); - expect(reset?.discontinuity).toBe(1); - const resumed = await consumer.next(); + const resumed = await nextFrame(consumer); expect(resumed?.frame?.timestamp).toBe(1_000_000 as Time.Micro); expect(resumed?.discontinuity).toBe(1); @@ -1399,6 +1345,7 @@ test("Consumer reports continuity across a PTS-contiguous group id jump (CMAF)", const result = await consumer.next(); expect(result?.frame?.payload).toEqual(new Uint8Array([0x02])); expect(result?.continuous).toBe(true); + expect(result?.discontinuity).toBe(0); consumer.close(); }); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 7d5ba4d114e..0fd36e7142b 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -23,66 +23,16 @@ export interface ConsumerProps { interface Group { consumer: Moq.Group.Consumer; frames: Frame[]; // decode order - empty: boolean; // no wire frame was published, which declares a discontinuity - start?: Time.Micro; // First decoded timestamp, retained after delivery for epoch classification + empty: boolean; // no wire frame was published; empty groups mean nothing + media: boolean; // a decodable (non-marker) frame was buffered + start?: Time.Micro; // First decoded timestamp + minMedia?: Time.Micro; // Lowest decodable timestamp latest?: Time.Micro; // The timestamp of the latest known frame end?: Time.Micro; // The furthest presentation point so far, i.e. max(timestamp + duration) done?: boolean; // Set when #runGroup finishes reading all frames truncated?: boolean; // The missing tail becomes a gap after the buffered frames are delivered. } -/** - * A recorded rewind boundary. - * - * After a backwards timestamp jump, groups can still arrive out of order, so a single - * sequence floor is not enough: a late new-epoch group can have a lower sequence than the - * group that triggered detection. This keeps just enough state to classify any group by - * its (sequence, timestamp). - */ -class Reset { - /** Highest-sequence old-epoch group seen at detection. At or below this is old: drop. */ - readonly prevMax: number; - /** The group whose backwards timestamp triggered detection. At or above this is new: keep. */ - readonly group: number; - /** That group's timestamp; in the ambiguous span, old stragglers sit at or above it. */ - readonly timestamp: Time.Micro; - - constructor(prevMax: number, group: number, timestamp: Time.Micro) { - this.prevMax = prevMax; - this.group = group; - this.timestamp = timestamp; - } - - /** Classify by sequence alone: true=old, false=new, undefined=ambiguous (resolve by timestamp). */ - bySequence(sequence: number): boolean | undefined { - if (sequence <= this.prevMax) return true; - if (sequence >= this.group) return false; - return undefined; - } - - /** Whether a group belongs to the reneged old epoch and should be dropped. */ - isStale(sequence: number, timestamp: Time.Micro): boolean { - return this.bySequence(sequence) ?? timestamp >= this.timestamp; - } -} - -/** - * Live state for declared discontinuities and detected timeline rewinds. - * - * A publisher reneges its buffered tail by rewinding timestamps while group sequence keeps - * climbing (e.g. a voice agent interrupted mid-utterance). We track the live edge to spot the - * jump and a {@link Reset} boundary to classify out-of-order groups across it. The counter also - * advances when delivery reaches an explicit empty-group discontinuity. - */ -class Rewind { - /** The live edge of playback: max delivered timestamp and the group that carried it. */ - liveEdge?: { group: number; timestamp: Time.Micro }; - /** The active rewind boundary, if any. */ - boundary?: Reset; - /** Increments on every declared discontinuity or rewind. */ - discontinuity = 0; -} - // Two adjacent groups are treated as timeline-contiguous when the next group's first PTS is within // this slack of the current group's end. Per-sample durations and base-decode-times are each rounded // to microseconds independently, so a genuinely contiguous boundary can be off by ~1µs (seen on 48kHz @@ -96,8 +46,8 @@ const CONTIGUITY_TOLERANCE = Moq.Time.Micro.fromMilli(1 as Time.Milli); * durations and base-decode-times. Undefined on either side means continuity can't be proven. * * The bound is one-sided (upper only) by design: a next start at or before `end` continues the - * timeline, a start past `end` beyond the tolerance is a gap. A start well before `end` (a backward - * jump) is a rewind, handled by Rewind / #checkReset, not here. + * timeline, a start past `end` beyond the tolerance is a gap. A start well before `end` is + * malformed and aborts the track. */ function ptsContiguous(end: Time.Micro | undefined, nextStart: Time.Micro | undefined): boolean { return end !== undefined && nextStart !== undefined && nextStart <= Moq.Time.Micro.add(end, CONTIGUITY_TOLERANCE); @@ -128,15 +78,20 @@ export class Consumer { // Maintained only via #recordPresented; see its comment for the invariant. #presentedEnd?: Time.Micro; // Group of the last frame next() returned, so it can report whether the following result - // continues that frame's timeline. Undefined until the first delivery and after a rewind. + // continues that frame's timeline. Undefined until the first delivery and after a playhead event. #deliveredGroup?: number; // Set whenever the consumer throws content away: a group that aged past `maxAge`, - // a group truncated by a decode error, a reneged straggler, a rewind. Reported (and + // a group truncated by a decode error. Reported (and // cleared) on the first frame delivered from the next group, which is where the missing span // sits. Only the consumer can know this, which is why next() reports it instead of leaving // callers to guess from group numbers. #gap = false; - #rewind = new Rewind(); // live edge + active boundary + discontinuity count + // The live edge of playback: max delivered timestamp and the group that carried it. + #liveEdge?: { group: number; timestamp: Time.Micro }; + // Increments on a declared marker, an unproven delivered hole, and a latency skip. + #discontinuity = 0; + // A group below the live edge aborts the track. + #error?: Error; // Wake up the consumer when a new frame is available. #notify?: () => void; @@ -188,21 +143,11 @@ export class Consumer { // was served alongside, and both consumers can still place one: audio writes into // a timestamp-indexed ring, video drops a late frame at render. How far back one // may be is the subscription's own max age, applied before it ever reaches here. - // - // A group the reset boundary proves reneged is different, and still dropped: it - // belongs to a timeline the publisher withdrew rather than one that arrived late. - // An ambiguous one is admitted so #runGroup can rule on it once its timestamps - // arrive. - if (this.#rewind.boundary?.bySequence(consumer.sequence) === true) { - console.warn(`skipping reneged group: track=${this.#track.name} ${consumer.sequence}`); - consumer.close(); - continue; - } - const group: Group = { consumer, frames: [], empty: true, + media: false, }; // Insert into #groups based on the group sequence number (ascending). @@ -246,7 +191,13 @@ export class Consumer { duration: sample.duration, }; - if (!marker) index++; + if (!marker) { + index++; + group.media = true; + if (group.minMedia === undefined || frame.timestamp < group.minMedia) { + group.minMedia = frame.timestamp; + } + } group.start ??= frame.timestamp; group.frames.push(frame); @@ -262,12 +213,7 @@ export class Consumer { this.#updateBuffered(); - // Resolve the group against an active reset (dropping a reneged straggler), - // else detect a new rewind. Both run for the active sequence too: the cursor - // advances past a finished group before its successor arrives, so the group - // that rewinds is routinely the active one. - if (this.#classifyStale(group)) return; - this.#checkReset(group); + if (!marker && this.#abortIfRewound(group, frame.timestamp)) return; let skipped = false; if (group.consumer.sequence !== this.#active) { @@ -297,6 +243,7 @@ export class Consumer { } } } catch (err) { + if (this.#error) return; // Stop reading the group but keep already-decoded frames. // A decode error or stream RESET truncates the tail of the GoP; // frames decoded before the error are still valid and playable. @@ -358,6 +305,7 @@ export class Consumer { if (this.#active === undefined) return; let skipped = false; + let hole = false; // Keep skipping the oldest group while the buffered span exceeds the max age. // This also handles gaps in group sequence numbers: if #active points to a missing @@ -365,7 +313,6 @@ export class Consumer { while (this.#groups.length >= 2) { const threshold = Moq.Time.Micro.fromMilli(this.#maxAge.peek()); const first = this.#groups[0]; - if (first.empty && !first.consumer.done) break; // Check the difference between the earliest and latest known frames. let min: number | undefined; @@ -390,13 +337,18 @@ export class Consumer { `skipping slow group: track=${this.#track.name} ${first.consumer.sequence} -> ${this.#active}`, ); - if (first.empty) this.#markDiscontinuity(); + const nextStart = this.#groups[0]?.frames.at(0)?.timestamp ?? this.#groups[0]?.end; + if (!ptsContiguous(first.end ?? this.#presentedEnd, nextStart)) { + hole = true; + } first.consumer.close(); first.frames.length = 0; skipped = true; this.#gap = true; } + if (hole) this.#markPlayhead(); + if (skipped) { this.#updateBuffered(); @@ -432,115 +384,36 @@ export class Consumer { return true; } - // Detect a publisher "rewind" and record the reneged boundary. A newer group (sequence - // climbs) whose earliest frame lands before the live edge (timestamp goes backwards) can - // only be an explicit reneg of the buffered tail; record the boundary, bump the - // discontinuity counter, drop the groups it proves stale, and resume from the earliest - // survivor. Groups still ambiguous (a late new-epoch group vs. an old straggler) are kept - // and resolved by #classifyStale once their timestamps arrive. - #checkReset(group: Group) { - if (this.#active === undefined) return; - - const live = this.#rewind.liveEdge; + // A group whose media timestamps sit below the live edge earlier groups reached is + // malformed. Returns true if the track was aborted. + #checkMalformed(): void { + const live = this.#liveEdge; if (live === undefined) return; - - // Only a group newer than the one that supplied the live edge can rewind the timeline. - // The cursor is not the bound: it advances past a finished group before the successor - // arrives, so a rewind at the cursor is the common case rather than an impossible one. - // Ruling out the live edge's own group is what keeps B-frame reordering inside it - // continuous, and it matches the prevMax the Reset below is built from. - if (group.consumer.sequence <= live.group) return; - - const start = group.start; - if (start === undefined) return; - - // A rewind is the timestamp going strictly backwards past the live edge. Anything at - // or ahead of it is normal forward motion. - if (start >= live.timestamp) return; - - const reset = new Reset(live.group, group.consumer.sequence, start); - this.#rewind.boundary = reset; - this.#rewind.discontinuity++; - this.#gap = true; - - // Drop buffered groups the boundary can already prove stale; keep ambiguous ones. - this.#groups = this.#groups.filter((g) => { - const verdict = reset.bySequence(g.consumer.sequence); - const start = g.start; - const stale = verdict ?? (start !== undefined && reset.isStale(g.consumer.sequence, start)); - if (stale) { - g.consumer.close(); - g.frames.length = 0; + for (const group of this.#groups) { + if (group.consumer.sequence <= live.group) continue; + if (group.minMedia !== undefined && group.minMedia < live.timestamp) { + this.#abort(new Error("group timestamp is below the live edge")); + return; } - return !stale; - }); - - console.warn( - `buffer reset: track=${this.#track.name} group timestamps rewound (prevMax ${reset.prevMax}, group ${reset.group})`, - ); - - // Resume from the earliest survivor; if none, from the rewound group. Anchor the live - // edge at the rewind point (delivered), not group.latest (buffered ahead of playback). - this.#active = this.#groups[0]?.consumer.sequence ?? reset.group; - // Clear the presented-timeline end. After a rewind the old value is a stale, higher timeline - // end; since the promotion guard's contiguity check is upper-bound only (see ptsContiguous), a - // new higher-sequence group with a low post-rewind PTS would otherwise look contiguous and be - // promoted across the discontinuity. Undefined means "unknown", so the guard waits until the - // resumed group's end is recomputed as it delivers. - this.#presentedEnd = undefined; - // Nothing on the new timeline has been delivered yet, so the next frame continues nothing. - this.#deliveredGroup = undefined; - this.#rewind.liveEdge = { group: reset.group, timestamp: start }; - this.#updateBuffered(); - - // Wake up any consumer waiting for a new frame. - this.#notify?.(); - this.#notify = undefined; + } } - // Drop a group that an active reset resolves as a reneged old straggler (its timestamp - // landed at or above the reset point). Returns true if the group was dropped. - #classifyStale(group: Group): boolean { - const reset = this.#rewind.boundary; - if (!reset) return false; - - const start = group.start; - if (start === undefined) return false; - if (!reset.isStale(group.consumer.sequence, start)) return false; - - this.#groups = this.#groups.filter((g) => g !== group); - group.consumer.close(); - group.frames.length = 0; - this.#gap = true; - - // A reset resumes from the earliest survivor, which may still be ambiguous, so the cursor - // can be sitting on the group just dropped. Move it here rather than leaving #runGroup's - // finally block to do it: that path would call #recordPresented and adopt this group's - // old-epoch end as the presented timeline, which is exactly the stale value the reset - // cleared. Survivors below still deliver first, since delivery only needs the head at or - // below the cursor. - if (group.consumer.sequence === this.#active) { - this.#active = - this.#groups.find((g) => g.consumer.sequence > group.consumer.sequence)?.consumer.sequence ?? - group.consumer.sequence + 1; - } + #abortIfRewound(group: Group, timestamp: Time.Micro): boolean { + const live = this.#liveEdge; + if (live === undefined) return false; + if (group.consumer.sequence <= live.group) return false; + if (timestamp >= live.timestamp) return false; - this.#updateBuffered(); + this.#abort(new Error("group timestamp is below the live edge")); return true; } - // Re-check buffered newer groups against the current live edge. #checkReset otherwise only - // runs when a group receives a frame, so a group that buffered while the live edge was lower - // (or undefined) is never reconsidered once delivery advances the edge past it, and the - // rewind would be missed. Highest sequence first, mirroring the Rust scan: the first rewound - // group becomes the boundary, and #checkReset's own guards make the rest no-ops. - #checkBufferedReset(): void { - const live = this.#rewind.liveEdge; - if (this.#active === undefined || live === undefined) return; - for (const group of [...this.#groups].reverse()) { - if (group.consumer.sequence <= live.group) break; - this.#checkReset(group); - } + #abort(error: Error): void { + this.#error = error; + this.#finish(error); + this.#track.close(error); + this.#notify?.(); + this.#notify = undefined; } /** @@ -548,17 +421,17 @@ export class Consumer { * {@link discontinuity} count, awaiting one if needed. A `frame` of undefined signals either * the end of that group or, when `end` is present, an exclusive media endpoint carried by a * legacy marker. The overall result is undefined once closed. When `discontinuity` - * jumps relative to the previous call, the publisher declared a break or rewound the - * timeline: reset codec state and flush downstream render buffers before playing this frame. + * jumps relative to the previous call, re-apply startup delay and skip: it is a playhead + * event, not a decoder flush. * * `continuous` is true when this result picks up exactly where the previous frame left off, so * the span between them can be treated as delivered. It is false on the first frame, after a - * rewind, and whenever the consumer threw content away to keep up: a slow group skipped for the - * max age, a group truncated by a decode error, a reneged straggler. Use it rather than - * comparing group numbers, which are not required to be sequential: adjacency neither proves - * the timeline is unbroken nor catches a group dropped on the way past. + * playhead event, and whenever the consumer threw content away to keep up: a slow group skipped + * for the max age, a group truncated by a decode error. Use it rather than comparing group + * numbers, which are not required to be sequential: adjacency neither proves the timeline is + * unbroken nor catches a group dropped on the way past. * - * It reports what this consumer dropped plus empty-group discontinuities the publisher declared. + * It reports what this consumer dropped plus marker groups the publisher declared. * An unmarked forward timestamp jump still reads as continuous because nothing on the wire says * the missing span will never arrive. * After buffered groups drain, a finished track returns undefined and an aborted track throws. @@ -574,6 +447,10 @@ export class Consumer { | undefined > { for (;;) { + if (this.#error) throw this.#error; + this.#checkMalformed(); + if (this.#error) throw this.#error; + const ended = this.#closed.peek(); if (this.#groups.length === 0) { if (ended !== undefined) { @@ -582,29 +459,26 @@ export class Consumer { } } - // A group may have buffered a rewind while the live edge was still behind it; catch it - // now that delivery has advanced the edge. - this.#checkBufferedReset(); - // If #active points below all buffered groups -- e.g. the finally block's `+ 1` // fallback fired because no later group was buffered yet, and the real (large-gap, // non-sequential) next group has since arrived -- promote #active to the first real // group so delivery resumes instead of stalling on a nonexistent sequence. // Promote #active to the first buffered group when it continues the timeline we left off at, - // or when a completed empty group declares a boundary that makes the earlier gap irrelevant. - // Otherwise a real gap sits before it and an in-transit group may still arrive, so wait. - // #checkMaxAge skips the gap once the buffered span exceeds the budget, and - // #tryDurationSkip once the duration covers it. - // After track termination no missing group can arrive, so drain across any remaining gap. + // when a completed empty group can be walked (empty groups mean nothing), or when a + // zero-budget hole is already proven. After track termination no missing group can arrive, + // so drain across any remaining gap. Otherwise wait: #checkMaxAge skips once the budget + // is spent, and #tryDurationSkip once the duration covers it. if (this.#active !== undefined && this.#groups.length > 0) { const head = this.#groups[0]; - const contiguous = ptsContiguous(this.#presentedEnd, head.frames.at(0)?.timestamp); - if ( - head.consumer.sequence > this.#active && - ((head.empty && head.consumer.done) || contiguous || ended !== undefined) - ) { - if (!contiguous) this.#gap = true; - this.#active = head.consumer.sequence; + if (head.consumer.sequence > this.#active) { + const contiguous = ptsContiguous(this.#presentedEnd, head.frames.at(0)?.timestamp); + const empty = head.empty && head.consumer.done; + const skipHole = this.#maxAge.peek() === 0 && head.frames.length > 0; + if (empty || contiguous || skipHole || ended !== undefined) { + if ((skipHole || ended !== undefined) && !contiguous && !empty) this.#markPlayhead(); + if (!contiguous) this.#gap = true; + this.#active = head.consumer.sequence; + } } } @@ -616,29 +490,31 @@ export class Consumer { const frame = this.#groups[0].frames.shift(); if (frame) { const seq = this.#groups[0].consumer.sequence; - const continuous = this.#continuesDelivery(seq); const end = this.#format.end?.(frame); if (end !== undefined) { + if (!this.#groups[0].media) this.#markPlayhead(); + if (this.#liveEdge === undefined || end > this.#liveEdge.timestamp) { + this.#liveEdge = { group: seq, timestamp: end }; + } this.#updateBuffered(); return { frame: undefined, group: seq, - discontinuity: this.#rewind.discontinuity, - continuous, + discontinuity: this.#discontinuity, + continuous: this.#continuesDelivery(seq), end, }; } + const continuous = this.#continuesDelivery(seq); if (seq !== this.#deliveredGroup) this.#gap = false; this.#deliveredGroup = seq; - // Track the live edge (max timestamp + its group) so a later backwards jump - // is detectable and the old epoch's tail is anchored. - const live = this.#rewind.liveEdge; + const live = this.#liveEdge; if (live === undefined || frame.timestamp > live.timestamp) { - this.#rewind.liveEdge = { group: seq, timestamp: frame.timestamp }; + this.#liveEdge = { group: seq, timestamp: frame.timestamp }; } this.#updateBuffered(); - return { frame, group: seq, discontinuity: this.#rewind.discontinuity, continuous }; + return { frame, group: seq, discontinuity: this.#discontinuity, continuous }; } // Check if the group is done and then remove it. A group is removable only @@ -662,12 +538,11 @@ export class Consumer { if (group) { const seq = group.consumer.sequence; if (group.truncated) this.#gap = true; - if (group.empty) this.#markDiscontinuity(); this.#updateBuffered(); return { frame: undefined, group: seq, - discontinuity: this.#rewind.discontinuity, + discontinuity: this.#discontinuity, // A marker carries no content of its own, so this just reports whether // the group it closes was itself reached without a gap. continuous: this.#continuesDelivery(seq), @@ -702,13 +577,8 @@ export class Consumer { } } - // An empty group is an ordered codec boundary. Unlike a detected rewind it needs no - // stale-group classification, because delivery has already reached the marker in sequence. - #markDiscontinuity(): void { - this.#rewind.discontinuity++; - this.#rewind.liveEdge = undefined; - this.#rewind.boundary = undefined; - this.#presentedEnd = undefined; + #markPlayhead(): void { + this.#discontinuity++; this.#deliveredGroup = undefined; this.#gap = true; } @@ -740,12 +610,11 @@ export class Consumer { } /** - * A counter that increments at each declared discontinuity or detected timeline rewind. - * Also surfaced per-read via {@link next}; downstream consumers reset codec state and flush - * render buffers when it changes. + * A counter that increments at each playhead event: a declared marker group, an unproven + * delivered hole, or a latency skip. Also surfaced per-read via {@link next}. */ get discontinuity(): number { - return this.#rewind.discontinuity; + return this.#discontinuity; } /** Stop consuming and release the track and all buffered groups. */ diff --git a/js/hang/src/container/legacy.ts b/js/hang/src/container/legacy.ts index fc798816cd0..b81de8b4b99 100644 --- a/js/hang/src/container/legacy.ts +++ b/js/hang/src/container/legacy.ts @@ -80,6 +80,8 @@ export class Producer { // The newest timestamp written, reported to the timeline when the track closes: the last // group has no successor to bound it, so its segment would be published a group short. #end?: Time.Micro; + // Exclusive presentation end of finished groups. A frame below this is refused. + #liveEdge?: Time.Micro; // Gap between consecutive timestamps, used to close the last group when no successor exists. #interval?: Time.Micro; @@ -90,17 +92,20 @@ export class Producer { this.#timeline = props.timeline; } - /** Encode and append a frame; a keyframe starts a new group. Throws if the first frame is not a keyframe. */ + /** Encode and append a frame; a keyframe starts a new group. Throws if the first frame is not a keyframe, or if the timestamp sits below the live edge earlier groups reached. */ encode(data: Uint8Array | Source, timestamp: Time.Micro, keyframe: boolean) { if (keyframe) { const rewound = this.#previous !== undefined && timestamp < this.#previous; this.cut(rewound ? undefined : timestamp); if (rewound) this.#interval = undefined; + this.#refuse(timestamp); this.#group = this.#track.appendGroup(); // Report the group the moment it opens: its start is this keyframe's timestamp. this.#timeline?.record(this.#group.sequence, timestamp, true); } else if (!this.#group) { throw new Error("must start with a keyframe"); + } else { + this.#refuse(timestamp); } this.#group?.writeFrame({ @@ -139,11 +144,21 @@ export class Producer { if (bound !== undefined) this.#timeline?.end(bound); this.#group.close(); this.#group = undefined; + if (this.#end !== undefined) { + this.#liveEdge = + this.#liveEdge === undefined ? this.#end : (Math.max(this.#liveEdge, this.#end) as Time.Micro); + } this.#end = undefined; this.#previous = undefined; this.#reordered = false; } + #refuse(timestamp: Time.Micro) { + if (this.#liveEdge !== undefined && timestamp < this.#liveEdge) { + throw new Error("frame timestamp is below the live edge"); + } + } + /** Close the track and current group, optionally with an error. */ close(err?: Error) { if (!err) this.cut(); diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index f49ad2acc8b..1833644b774 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -46,7 +46,7 @@ pub struct Consumer { end: Option, /// Presentation time of the first decoded terminal frame. terminal_start: Option, - /// Last container discontinuity applied to codec and resampler state. + /// Last container playhead generation applied to timeline state. discontinuity: u64, } @@ -299,7 +299,8 @@ impl Consumer { } } - /// Reset every stateful decode stage before the first packet of a new epoch. + /// A playhead event re-applies startup delay and skip. The decoder is not reset: + /// the next group already starts on a keyframe, and pre-skip is a play-path concern. fn apply_discontinuity(&mut self) -> Result<(), Error> { let discontinuity = self.track.discontinuity(); if discontinuity == self.discontinuity { @@ -307,15 +308,9 @@ impl Consumer { } self.discontinuity = discontinuity; - self.decoder.reset()?; - if let Some(resampler) = self.resampler.as_mut() { - resampler.reset(); - } self.next_start = None; self.spans.clear(); self.trailing = Activity::Active; - self.epoch = None; - self.delay_trimmed = 0; self.frames_decoded = 0; self.end = None; self.terminal_start = None; @@ -914,20 +909,21 @@ mod tests { // A 20 ms Opus packet decodes 960 frames, less the pre-skip on the first one, // so it doesn't fill the 960-frame chunk and is held whole. - let write = |producer: &mut moq_mux::container::Producer<_>, frames: u64, payload: Bytes| { + let write = |producer: &mut moq_mux::container::Producer<_>, frames: u64, payload: Bytes, keyframe: bool| { producer .write(moq_mux::container::Frame { timestamp: Timestamp::from_scale(frames, 48_000).unwrap(), duration: None, payload, - keyframe: true, + keyframe, }) .unwrap(); }; - write(&mut producer, 0, active.payload); + write(&mut producer, 0, active.payload, true); // The end marker, then the terminal packet a second past where it belongs. - write(&mut producer, 3 * 48_000, Bytes::new()); - write(&mut producer, 48_000, dtx.payload); + // Same group: a new group at 1s would sit below the marker's live edge. + write(&mut producer, 3 * 48_000, Bytes::new(), false); + write(&mut producer, 48_000, dtx.payload, false); producer.finish().unwrap(); let frame = consumer.read().await.unwrap().expect("decoded frame"); diff --git a/rs/moq-audio/src/encode/producer.rs b/rs/moq-audio/src/encode/producer.rs index 644ab3de26a..6c9a313c4bf 100644 --- a/rs/moq-audio/src/encode/producer.rs +++ b/rs/moq-audio/src/encode/producer.rs @@ -729,10 +729,11 @@ mod tests { let mut resumed_frames = 0; while let Some(frame) = audio.read().await.unwrap() { - assert!(frame.timestamp.as_micros() >= 1_000_000); - resumed_frames += frame.data.len() / size_of::(); + if frame.timestamp.as_micros() >= 1_000_000 { + resumed_frames += frame.data.len() / size_of::(); + } } - assert_eq!(resumed_frames, 960, "the resumed epoch must trim its own pre-skip once"); + assert!(resumed_frames > 0, "the resumed epoch still decodes"); } #[tokio::test] diff --git a/rs/moq-mux/src/codec/legacy.rs b/rs/moq-mux/src/codec/legacy.rs index c09bcb723dc..e64db958291 100644 --- a/rs/moq-mux/src/codec/legacy.rs +++ b/rs/moq-mux/src/codec/legacy.rs @@ -164,6 +164,11 @@ impl Import { self.track.name() } + /// The exclusive presentation end earlier groups have reached, if any. + pub fn live_edge(&self) -> Option { + self.track.live_edge() + } + /// Finish the track, flushing the current group. pub fn finish(&mut self) -> crate::Result<()> { self.track.finish()?; diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 36487686708..73a6a020c68 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -3,7 +3,7 @@ use std::task::{Poll, ready}; use moq_net::Timestamp; -use super::{Container, Frame}; +use super::{Container, Frame, TimestampRewind}; /// Media and clean group boundaries in delivery order. pub(crate) enum Event { @@ -52,11 +52,12 @@ pub(crate) enum Event { /// /// ## Timeline discontinuities /// -/// An empty group declares a codec boundary. A newer group whose timestamps jump backwards -/// past the live edge also reneges the buffered tail (e.g. a voice agent interrupted -/// mid-utterance). The consumer bumps [`discontinuity`](Self::discontinuity) for either, -/// dropping reneged groups when needed so downstream consumers can reset codec and render -/// buffers. This is always on. +/// A marker group (no decodable frames, one empty payload) is a walk-now discontinuity. +/// A delivered sequence hole is a playhead event unless the boundary is contiguous within +/// 1 ms, and a latency skip is the same event. [`discontinuity`](Self::discontinuity) is +/// that playhead generation: re-apply startup delay and skip, not a decoder flush. +/// Empty groups (zero objects) mean nothing. A group whose timestamps fall below the live +/// edge earlier groups reached is malformed and aborts the track. pub struct Consumer { track: moq_net::track::Subscriber, @@ -75,73 +76,30 @@ pub struct Consumer { // How far we may drift from the live edge before skipping a group. max_age: std::time::Duration, - // Timeline-discontinuity tracking: the live edge, rewind boundary, and event count. - rewind: Rewind, - - // Exclusive audio endpoint delivered before terminal codec packets. - end: Option, -} - -/// Live state for detecting timeline rewinds and classifying out-of-order groups. -/// -/// Tracks the live edge and [`Reset`] boundary used to classify a timestamp rewind, plus the -/// counter shared by rewinds and explicit empty-group discontinuities. -#[derive(Default)] -struct Rewind { // The live edge of playback: the largest timestamp delivered so far and the group that - // carried it. `None` until the first frame is delivered. + // carried it. `None` until the first frame is delivered. A later group below this is + // malformed. live_edge: Option<(u64, Timestamp)>, - // The active rewind boundary, if any. Out-of-order groups are classified against it so a - // late new-epoch group is kept while a reneged old-epoch straggler is dropped. - boundary: Option, + // Presentation end of the group we most recently advanced past, for the 1 ms + // contiguity check on a delivered hole. + presented_end: Option, - // Increments on every declared discontinuity or rewind. Downstream consumers compare it - // across reads and reset codec and render buffers when it changes. + // Increments on a declared marker, an unproven delivered hole, and a latency skip. discontinuity: u64, -} -/// A recorded rewind boundary. -/// -/// After a backwards timestamp jump, groups can still arrive out of order, so a single -/// sequence floor is not enough: a late new-epoch group can have a *lower* sequence than -/// the group that triggered detection. We keep just enough state to classify any group by -/// `(sequence, timestamp)`. -#[derive(Clone, Copy)] -struct Reset { - // Highest-sequence old-epoch group seen at detection (it held the old live edge). - // Sequences at or below this are old: drop. - prev_max: u64, - - // The group whose backwards timestamp triggered detection. Sequences at or above this - // are new: keep. - group: u64, - - // That group's timestamp. Within the ambiguous span `(prev_max, group)` a group is a - // new-epoch gap-filler if its timestamp is below this, else an old straggler whose - // higher timestamp simply hadn't arrived yet. - timestamp: Timestamp, + // Exclusive audio endpoint delivered before terminal codec packets. + end: Option, } -impl Reset { - // Classify by sequence alone. `Some(true)` = old/drop, `Some(false)` = new/keep, - // `None` = ambiguous (the caller must resolve it with the group's timestamp). - fn by_sequence(&self, sequence: u64) -> Option { - if sequence <= self.prev_max { - Some(true) - } else if sequence >= self.group { - Some(false) - } else { - None - } - } +/// Two adjacent groups are timeline-contiguous when the next start is within this slack +/// of the current end. Per-sample durations and base-decode-times round to microseconds +/// independently, so a genuine boundary can be off by ~1 µs; a real missing group spans +/// about one group duration. +const CONTIGUITY_TOLERANCE: std::time::Duration = std::time::Duration::from_millis(1); - // Whether a group belongs to the reneged old epoch and should be dropped. In the - // ambiguous span, old stragglers sit at or above the reset timestamp; new gap-fillers - // fall below it. - fn is_stale(&self, sequence: u64, timestamp: Timestamp) -> bool { - self.by_sequence(sequence).unwrap_or(timestamp >= self.timestamp) - } +fn pts_contiguous(end: Option, next_start: std::time::Duration) -> bool { + end.is_some_and(|end| next_start <= end.saturating_add(CONTIGUITY_TOLERANCE)) } impl Consumer { @@ -168,21 +126,20 @@ impl Consumer { pending: VecDeque::new(), startup: start.is_none(), max_age, - rewind: Rewind::default(), + live_edge: None, + presented_end: None, + discontinuity: 0, end: None, } } - /// A counter that increments each time the consumer reaches a declared discontinuity or - /// detects a timeline rewind and drops the reneged buffer. + /// A counter that increments at each playhead event: a declared marker group, an + /// unproven delivered hole, or a latency skip. /// - /// A publisher declares a discontinuity with an empty group. A newer group whose timestamps - /// jump backwards past the live edge also reneges everything buffered after that point. - /// Downstream consumers should compare this across reads and, when it changes, reset codec - /// state and flush media still queued in decoder or render buffers. The frame returned by - /// the read that bumps it is the first of the new timeline. + /// Downstream consumers re-apply startup delay and skip when it changes. It is not a + /// decoder flush: the next group already starts on a keyframe with parameter sets. pub fn discontinuity(&self) -> u64 { - self.rewind.discontinuity + self.discontinuity } /// The exclusive audio endpoint delivered before terminal codec packets. @@ -255,14 +212,7 @@ impl Consumer { .retain_mut(|group| group.sequence <= current || !group.buffered.is_empty() || !group.poll_aborted(waiter)); 'read: loop { - // A newer group whose timestamps jumped backwards means the publisher reneged - // the buffered tail. Record the boundary and resume from the new epoch, then restart. - if self.poll_reset(waiter)? { - continue; - } - - // Drop any reneged stragglers whose timestamps have since resolved them as old. - self.poll_classify(waiter)?; + self.poll_malformed(waiter)?; // Return the next frame from the current group if possible. // If the current group is finished or errored, advance to the next group. @@ -271,15 +221,20 @@ impl Consumer { { match group.poll_read(waiter, &self.format) { Poll::Ready(Ok(Some(Event::Frame(frame)))) => { - // Track the live edge (the max timestamp and the group that carries it) so a - // later backwards jump is detectable and the old epoch's tail is anchored. let seq = group.group.sequence; let ts = frame.timestamp; - if self.rewind.live_edge.is_none_or(|(_, high)| ts > high) { - self.rewind.live_edge = Some((seq, ts)); + if self.live_edge.is_none_or(|(_, high)| ts.as_micros() > high.as_micros()) { + self.live_edge = Some((seq, ts)); } return Poll::Ready(Ok(Some(Event::Frame(frame)))); } + Poll::Ready(Ok(Some(Event::FrameEnd(end)))) => { + let seq = group.group.sequence; + if self.live_edge.is_none_or(|(_, high)| end.as_micros() > high.as_micros()) { + self.live_edge = Some((seq, end)); + } + return Poll::Ready(Ok(Some(Event::FrameEnd(end)))); + } Poll::Ready(Ok(Some(event))) => return Poll::Ready(Ok(Some(event))), // Still blocked on this group, don't skip it yet. Poll::Pending => {} @@ -305,11 +260,14 @@ impl Consumer { } // Cleanly finished group: advance to the next sequence. Poll::Ready(Ok(None)) => { - let empty = group.empty; + let marker = group.marker(); + if let Some(end) = group.max_end { + self.presented_end = Some(end); + } self.pending.pop_front(); self.current += 1; - if empty { - self.mark_discontinuities(1); + if marker { + self.bump_playhead(); } return Poll::Ready(Ok(Some(Event::GroupEnd))); } @@ -370,6 +328,9 @@ impl Consumer { && let Some((_, next_start)) = next_group && (finished || max_timestamp.saturating_sub(next_start) >= self.max_age) { + if !pts_contiguous(current_end.or(self.presented_end), next_start) { + self.bump_playhead(); + } self.current = front_sequence; continue; } @@ -393,25 +354,14 @@ impl Consumer { false }; - if let Some((new_idx, _)) = next_group + if let Some((new_idx, next_start)) = next_group && should_skip { - // A zero-frame group is ambiguous until its FIN arrives. Keep it in order so a - // delayed empty-group boundary cannot be discarded before it is recognized. - // With nothing delivered since the last boundary there is no codec state to - // reset, so a would-be discontinuity is redundant and the wait is skipped. - let mut discontinuities = 0; - if self.rewind.live_edge.is_some() { - for group in self.pending.iter_mut().take(new_idx) { - match group.poll_empty(waiter) { - Poll::Ready(true) => discontinuities += 1, - Poll::Ready(false) => {} - Poll::Pending => return Poll::Pending, - } - } - } + let hole = !pts_contiguous(current_end.or(self.presented_end), next_start); self.pending.drain(0..new_idx); - self.mark_discontinuities(discontinuities); + if hole { + self.bump_playhead(); + } let new_current = self.pending.front().map(|g| g.sequence).unwrap(); tracing::debug!(old = self.current, new = new_current, "skipping slow groups"); @@ -421,11 +371,22 @@ impl Consumer { } if finished - && let Some(group) = self.pending.front_mut() - && group.sequence > self.current - && matches!(group.poll_empty(waiter), Poll::Ready(true)) + && let Some(front_sequence) = self.pending.front().map(|g| g.sequence) + && front_sequence > self.current { - self.current = group.sequence; + let _ = self.pending.front_mut().unwrap().buffer_all(waiter, &self.format); + let next_start = self.pending.front().and_then(|group| { + group + .min_timestamp + .map(std::time::Duration::from) + .or(group.max_end) + }); + if let Some(start) = next_start + && !pts_contiguous(current_end.or(self.presented_end), start) + { + self.bump_playhead(); + } + self.current = front_sequence; continue; } @@ -437,15 +398,9 @@ impl Consumer { } } - fn mark_discontinuities(&mut self, count: u64) { - if count == 0 { - return; - } - - self.rewind.discontinuity += count; + fn bump_playhead(&mut self) { + self.discontinuity += 1; self.end = None; - self.rewind.live_edge = None; - self.rewind.boundary = None; } // Reads any new groups from the track until we're completely finished. @@ -461,19 +416,7 @@ impl Consumer { let reader = GroupBuffer::new(group); let sequence = reader.group.sequence; - // Normally we drop anything behind the playback cursor. With an active reset the - // cursor isn't a valid floor: a late new-epoch group can sit below it. Defer to - // the boundary, admitting ambiguous groups so poll_classify can rule on them once - // their timestamps arrive. - let drop = match &self.rewind.boundary { - Some(reset) => match reset.by_sequence(sequence) { - Some(true) => true, // old epoch: reneged - Some(false) => sequence < self.current, // new epoch, but already played past - None => false, // ambiguous: admit, classify later - }, - None => sequence < self.current, - }; - if drop { + if sequence < self.current { tracing::debug!(old = ?sequence, current = ?self.current, "skipping old group"); continue; } @@ -485,105 +428,21 @@ impl Consumer { } } - // Detect a publisher "rewind" and record the reneged boundary. - // - // A newer group (sequence climbs) whose first frame lands before the live edge (timestamp - // goes backwards) can only be an explicit reneg of the buffered tail. We record a [`Reset`] - // from `(live-edge group, rewound group, rewound timestamp)`, drop the buffered groups it - // can already prove stale, bump the discontinuity counter, and resume playback from the - // earliest survivor. Groups still ambiguous (a late new-epoch group vs. an old straggler) - // are kept and resolved by [`poll_classify`](Self::poll_classify) once their timestamps - // arrive. - // - // Returns true if a reset happened, signalling the caller to restart the read loop. - fn poll_reset(&mut self, waiter: &kio::Waiter) -> Result { - let Some((prev_max, live_edge)) = self.rewind.live_edge else { - return Ok(false); - }; - - // Scan newer groups from the back (highest sequence first) for a rewind: a group whose - // timestamp went strictly backwards past the live edge. Checking only `back()` would - // miss a rewind that a higher-sequence group masks by having already caught back up - // (timestamp >= live edge) or by having no frame yet. Pending is sequence-sorted, so we - // take the highest-sequence group that actually rewound. - let reset = { - let mut found = None; - for group in self.pending.iter_mut().rev() { - // The cursor may already point at an unread group. Only groups that - // supplied the old live edge (or preceded it) are ruled out. - if group.group.sequence <= prev_max { - break; - } - - // Skip groups with no frame yet; a lower-sequence one may still have rewound. - let Poll::Ready(Ok(min)) = group.poll_min_timestamp(waiter, &self.format) else { - continue; - }; - - if min < live_edge { - found = Some(Reset { - prev_max, - group: group.group.sequence, - timestamp: min, - }); - break; - } - } - - let Some(reset) = found else { - return Ok(false); - }; - reset - }; - - // Drop buffered groups the boundary can already prove are old-epoch. Ambiguous ones - // (no verdict by sequence, or timestamp not read yet) are kept for poll_classify. - self.pending.retain(|g| match reset.by_sequence(g.group.sequence) { - Some(stale) => !stale, - None => g.min_timestamp.is_none_or(|ts| !reset.is_stale(g.group.sequence, ts)), - }); - - self.rewind.discontinuity += 1; - self.end = None; - tracing::debug!( - prev_max = reset.prev_max, - group = reset.group, - discontinuity = self.rewind.discontinuity, - "buffer reset: group timestamps rewound" - ); - self.rewind.boundary = Some(reset); - // Resume from the earliest survivor; if none buffered yet, from the rewound group. - self.current = self.pending.front().map_or(reset.group, |g| g.group.sequence); - self.rewind.live_edge = Some((reset.group, reset.timestamp)); - - Ok(true) - } - - // Resolve groups left ambiguous by a reset once their timestamps arrive. - // - // A group whose sequence falls in the reset's ambiguous span could be a late new-epoch - // group (keep) or an old straggler whose higher timestamp simply hadn't been seen at - // detection time (drop). We can only tell once it has a frame, so we re-check each loop - // iteration and drop the ones that resolve to stale. - fn poll_classify(&mut self, waiter: &kio::Waiter) -> Result<(), F::Error> { - let Some(reset) = self.rewind.boundary else { + // A group whose media timestamps sit below the live edge earlier groups reached is + // malformed. Markers have no media timestamp, so they are not this check. + fn poll_malformed(&mut self, waiter: &kio::Waiter) -> Result<(), F::Error> { + let Some((prev_group, edge)) = self.live_edge else { return Ok(()); }; - let mut i = 0; - while i < self.pending.len() { - let group = &mut self.pending[i]; - // Only ambiguous-by-sequence groups need a timestamp verdict. - if reset.by_sequence(group.group.sequence).is_some() { - i += 1; + for group in self.pending.iter_mut() { + if group.group.sequence <= prev_group { continue; } - - match group.poll_min_timestamp(waiter, &self.format) { - Poll::Ready(Ok(min)) if reset.is_stale(group.group.sequence, min) => { - self.pending.remove(i); - } - _ => i += 1, + if let Poll::Ready(Ok(min)) = group.poll_min_timestamp(waiter, &self.format) + && min.as_micros() < edge.as_micros() + { + return Err(TimestampRewind.into()); } } @@ -613,10 +472,13 @@ struct GroupBuffer { // The current frame index within the group. index: usize, - // Whether the group has carried any wire frame. A cleanly finished group with - // none is an explicit discontinuity marker. + // Whether the group has carried any wire frame. Empty groups (zero objects) mean + // nothing; a finished group with wire frames and no media is a marker. empty: bool, + // Whether a decodable (non-marker) frame was buffered. + media: bool, + // Read frames that haven't been consumed yet. buffered: VecDeque, markers: VecDeque<(usize, Timestamp)>, @@ -640,6 +502,7 @@ impl GroupBuffer { group, index: 0, empty: true, + media: false, buffered: VecDeque::new(), markers: VecDeque::new(), delivered: 0, @@ -705,6 +568,7 @@ impl GroupBuffer { // the container's flag otherwise so CMAF mid-group keyframes survive. frame.keyframe = frame.keyframe || self.index == 0; self.index += 1; + self.media = true; self.buffered.push_back(frame); } @@ -782,16 +646,8 @@ impl GroupBuffer { self.max_end = Some(self.max_end.map_or(end, |existing| existing.max(end))); } - fn poll_empty(&mut self, waiter: &kio::Waiter) -> Poll { - if !self.empty { - return Poll::Ready(false); - } - - match self.group.poll_finished(waiter) { - Poll::Ready(Ok(_)) => Poll::Ready(true), - Poll::Ready(Err(_)) => Poll::Ready(false), - Poll::Pending => Poll::Pending, - } + fn marker(&self) -> bool { + !self.empty && !self.media } } @@ -969,12 +825,46 @@ mod tests { assert!(consumer.read().await.unwrap().is_none()); } + fn write_marker_group(track: &mut moq_net::track::Producer, sequence: u64, timestamp: Timestamp) { + let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + Container::Legacy(crate::container::Kind::Audio) + .write( + &mut group, + &[Frame { + timestamp, + payload: Bytes::new(), + keyframe: false, + duration: None, + }], + ) + .unwrap(); + group.finish().unwrap(); + } + #[tokio::test] async fn empty_group_declares_a_discontinuity() { - let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); let consumer_track = track.subscribe(moq_net::track::Subscription::default().with_max_age(Duration::from_secs(2))); - let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); + let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Audio)); + + write_group(&mut track, 0, &[ts(0)]); + write_marker_group(&mut track, 1, ts(0)); + write_group(&mut track, 2, &[ts(1_000_000)]); + track.finish().unwrap(); + + assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.discontinuity(), 0); + assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000_000)); + assert_eq!(consumer.discontinuity(), 1); + } + + #[tokio::test] + async fn empty_groups_mean_nothing() { + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let consumer_track = + track.subscribe(moq_net::track::Subscription::default().with_max_age(Duration::from_secs(2))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Audio)); write_group(&mut track, 0, &[ts(0)]); track @@ -982,40 +872,35 @@ mod tests { .unwrap() .finish() .unwrap(); - write_group(&mut track, 2, &[ts(1_000_000)]); + write_group(&mut track, 2, &[ts(1_000)]); track.finish().unwrap(); assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); - assert_eq!(consumer.discontinuity(), 0); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000_000)); - assert_eq!(consumer.discontinuity(), 1); + assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000)); + assert_eq!(consumer.discontinuity(), 0, "an empty group is not a marker"); } #[tokio::test] async fn latency_skip_preserves_empty_group_discontinuity() { - let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); let consumer_track = track.subscribe(moq_net::track::Subscription::default().with_max_age(Duration::from_secs(2))); - let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); + let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Audio)); // Keep transport filtering out of this test so it isolates the mux skip logic. consumer.max_age = Duration::ZERO; write_group(&mut track, 0, &[ts(0)]); - let mut marker = track.create_group(moq_net::group::Info { sequence: 2 }).unwrap(); write_group(&mut track, 3, &[ts(1_000_000)]); + track.finish().unwrap(); assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); assert_eq!(consumer.discontinuity(), 0); - assert!( - tokio::time::timeout(Duration::from_millis(20), consumer.read()) - .await - .is_err() - ); - - marker.finish().unwrap(); - track.finish().unwrap(); assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000_000)); - assert_eq!(consumer.discontinuity(), 1); + assert_eq!( + consumer.discontinuity(), + 1, + "a shed marker is a timestamp hole, so the playhead jumps" + ); } #[tokio::test] @@ -1179,182 +1064,82 @@ mod tests { finisher.await.expect("finisher task panicked"); } - // ---- Rewind / reneg ---- - - /// The reset boundary classifies out-of-order groups by `(sequence, timestamp)`. - /// Old epoch peaked at group 55 (ts 100); group 58 rewound to ts 90. - #[test] - fn reset_classifies_out_of_order_groups() { - let reset = Reset { - prev_max: 55, - group: 58, - timestamp: ts(90), - }; - - // Late new-epoch gap-filler: sequence in (55, 58), ts below the rewind. Keep. - assert!(!reset.is_stale(57, ts(88))); - // Old straggler from before the peak (low sequence). Drop, even though its ts (86) - // is below the rewind — sequence is what separates it from group 57. - assert!(reset.is_stale(52, ts(86))); - // Old straggler in the gap whose higher ts hadn't arrived at detection. Drop. - assert!(reset.is_stale(56, ts(105))); - // At or after the rewound group: new epoch. Keep. - assert!(!reset.is_stale(58, ts(90))); - assert!(!reset.is_stale(59, ts(92))); - // At or before the old peak: old epoch. Drop. - assert!(reset.is_stale(55, ts(100))); - } + // ---- Malformed rewind ---- #[tokio::test] - async fn rewind_at_the_cursor_signals_the_first_frame() { - tokio::time::pause(); - for drained in [false, true] { - let mut track = track_producer( - "cursor-rewind", - hang::container::track_info(hang::catalog::PRIORITY.video), - ); - let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Data)); - write_group(&mut track, 0, &[ts(600_000_000)]); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(600_000_000)); - if drained { - assert!( - tokio::time::timeout(Duration::from_millis(1), consumer.read()) - .await - .is_err() - ); - } - // One new group, so no higher-sequence successor can reveal the reset. - write_group(&mut track, 1, &[ts(0), ts(100_000)]); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); - assert_eq!(consumer.discontinuity(), 1, "first rewound frame, drained={drained}"); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(100_000)); - assert_eq!(consumer.discontinuity(), 1, "no duplicate reset within a group"); - } + async fn a_group_below_the_live_edge_aborts() { + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Data)); + write_group(&mut track, 0, &[ts(100_000)]); + assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(100_000)); + write_group(&mut track, 1, &[ts(0)]); + track.finish().unwrap(); + let err = consumer.read().await.unwrap_err(); + assert!(matches!(err, crate::Error::TimestampRewind(_))); } - /// A new-epoch group that arrives out of order *below* the resume point is kept and - /// played, not dropped — the bug a plain "floor = detection group" would have. #[tokio::test] - async fn reset_keeps_out_of_order_new_group() { - tokio::time::pause(); + async fn b_frames_within_a_group_are_accepted() { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); - let consumer_track = - track.subscribe(moq_net::track::Subscription::default().with_max_age(Duration::from_secs(10))); - let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); - - // Old epoch, played forward until the live edge passes the rewind point. - write_group(&mut track, 0, &[ts(0)]); - write_group(&mut track, 1, &[ts(100_000)]); - write_group(&mut track, 2, &[ts(200_000)]); - // New epoch's later group (seq 5, ts 3 ms) arrives first and triggers the reset. - write_group(&mut track, 5, &[ts(3_000)]); - - // Its earlier gap-fillers (seq 3, 4) land after the reset, below the resume point. - let finisher = tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(50)).await; - write_group(&mut track, 3, &[ts(1_000)]); - write_group(&mut track, 4, &[ts(2_000)]); - track.finish().unwrap(); - }); - + let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Data)); + write_group(&mut track, 0, &[ts(0), ts(66_000), ts(33_000)]); + track.finish().unwrap(); let frames = read_all(&mut consumer).await.unwrap(); - let micros: Vec = frames.iter().map(|f| f.timestamp.as_micros()).collect(); - - // Old epoch played before the reset, and all three new-epoch groups survived — - // including the two out-of-order gap-fillers that arrived below the resume point. - assert!(micros.contains(&100_000), "old epoch played before the reset"); - assert!( - micros.contains(&1_000) && micros.contains(&2_000) && micros.contains(&3_000), - "out-of-order new-epoch groups kept, got {micros:?}" + assert_eq!( + frames.iter().map(|f| f.timestamp).collect::>(), + vec![ts(0), ts(66_000), ts(33_000)] ); - assert_eq!(consumer.discontinuity(), 1, "one rewind detected"); - finisher.await.expect("finisher task panicked"); + assert_eq!(consumer.discontinuity(), 0); } - /// A rewind is detected even when a higher-sequence group has already caught back up past - /// the live edge (so the newest pending group looks forward). Scanning only `back()` would - /// miss the lower-sequence rewound group and play the reneged tail without a discontinuity. #[tokio::test] - async fn reset_detected_behind_forward_newest_group() { - tokio::time::pause(); + async fn open_gop_leading_pictures_above_the_previous_group_are_accepted() { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); - let consumer_track = - track.subscribe(moq_net::track::Subscription::default().with_max_age(Duration::from_secs(10))); - let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); - - // Old timeline, played to a live edge of 200 ms. - write_group(&mut track, 0, &[ts(0)]); - write_group(&mut track, 1, &[ts(100_000)]); - write_group(&mut track, 2, &[ts(200_000)]); - // Group 6 (highest sequence) is forward of the live edge, masking... - write_group(&mut track, 6, &[ts(250_000)]); - // ...group 5, a lower-sequence group that rewound below it. - write_group(&mut track, 5, &[ts(50_000)]); + let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Data)); + write_group(&mut track, 0, &[ts(0), ts(33_000)]); + write_group(&mut track, 1, &[ts(66_000), ts(50_000)]); track.finish().unwrap(); - let frames = read_all(&mut consumer).await.unwrap(); - let micros: Vec = frames.iter().map(|f| f.timestamp.as_micros()).collect(); - assert_eq!( - consumer.discontinuity(), - 1, - "rewind detected behind a forward newest group" - ); - assert!(micros.contains(&50_000), "resumed at the rewound group, got {micros:?}"); - assert!( - !micros.contains(&200_000), - "the reneged tail was dropped, got {micros:?}" + frames.iter().map(|f| f.timestamp.as_micros()).collect::>(), + vec![0, 33_000, 66_000, 50_000] ); + assert_eq!(consumer.discontinuity(), 0); } - /// A newer group whose timestamps jump backwards past the buffered tail drops the - /// reneged groups and resumes from the rewound group. Models a voice agent that - /// runs ahead of playback and then interrupts to start a new utterance. #[tokio::test] - async fn backwards_timestamp_resets_buffer() { - tokio::time::pause(); - let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); - let consumer_track = - track.subscribe(moq_net::track::Subscription::default().with_max_age(Duration::from_secs(10))); - // Large max age so the slow-group skip never fires; isolate the rewind path. - let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); - - // Publisher runs ahead: groups 0-4 at 0, 100, 200, 300, 400 ms. - for i in 0..5u64 { - write_group(&mut track, i, &[ts(i * 100_000)]); - } - // Then it reneges and rewinds: group 5 restarts the timeline at 0 ms. - write_group(&mut track, 5, &[ts(0), ts(20_000)]); + async fn a_marker_group_and_forward_jump_bumps_playhead_once() { + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let mut consumer = Consumer::new( + track.subscribe(moq_net::track::Subscription::default().with_max_age(Duration::from_secs(2))), + Container::Legacy(crate::container::Kind::Audio), + ); + write_group(&mut track, 0, &[ts(0)]); + write_marker_group(&mut track, 1, ts(0)); + write_group(&mut track, 2, &[ts(1_000_000)]); track.finish().unwrap(); - - let frames = read_all(&mut consumer).await.unwrap(); - let timestamps: Vec<_> = frames.iter().map(|f| f.timestamp).collect(); - - // We play forward until the live edge passes the rewind point (through 100 ms), then - // the rewind drops the buffered-ahead groups (200/300/400 ms) and resumes at group 5. - assert_eq!(timestamps, vec![ts(0), ts(100_000), ts(0), ts(20_000)]); + assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.discontinuity(), 0); + assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000_000)); assert_eq!(consumer.discontinuity(), 1); } - /// Rewind detection is always on: a backwards group timestamp resets the buffer with no - /// configuration. Here group 2 rewinds the timeline and bumps the discontinuity counter. #[tokio::test] - async fn backwards_timestamp_always_resets() { - let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); - let consumer_track = - track.subscribe(moq_net::track::Subscription::default().with_max_age(Duration::from_secs(10))); - let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); - + async fn a_zero_budget_idle_resume_jumps_the_playhead_after_a_shed_marker() { + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Audio)); + consumer.max_age = Duration::ZERO; write_group(&mut track, 0, &[ts(0)]); - write_group(&mut track, 1, &[ts(500_000)]); - write_group(&mut track, 2, &[ts(0)]); // rewind + assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); + write_marker_group(&mut track, 1, ts(0)); + write_group(&mut track, 2, &[ts(1_000_000)]); track.finish().unwrap(); - - let frames = read_all(&mut consumer).await.unwrap(); - let timestamps: Vec<_> = frames.iter().map(|f| f.timestamp).collect(); - - assert_eq!(timestamps, vec![ts(0), ts(500_000), ts(0)]); - assert_eq!(consumer.discontinuity(), 1, "the backwards group triggered a reset"); + assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000_000)); + assert_eq!( + consumer.discontinuity(), + 1, + "a shed marker with a timestamp jump is a playhead event" + ); } // ---- Empty payloads ---- @@ -2682,11 +2467,34 @@ mod tests { assert_eq!(frames.len(), 2); assert_eq!(frames[0].timestamp, ts(0)); assert_eq!(frames[1].timestamp, ts(33_000)); + assert_eq!(consumer.discontinuity(), 0, "a contiguous duration skip is not a hole"); // group0 is intentionally never finished. drop(group0); } + #[tokio::test] + async fn a_nonsequential_contiguous_jump_does_not_bump_playhead() { + let mut track = track_producer("test", None); + let mut consumer = Consumer::new( + track.subscribe(moq_net::track::Subscription::default().with_max_age(Duration::from_secs(10))), + DurationWire, + ); + let mut group0 = track.create_group(moq_net::group::Info { sequence: 1_000_000 }).unwrap(); + write_duration_frame(&mut group0, ts(0), ts(33_000)); + group0.finish().unwrap(); + let mut group1 = track.create_group(moq_net::group::Info { sequence: 1_090_000 }).unwrap(); + write_duration_frame(&mut group1, ts(33_000), ts(33_000)); + group1.finish().unwrap(); + track.finish().unwrap(); + let mut frames = Vec::new(); + while let Some(frame) = consumer.read().await.unwrap() { + frames.push(frame); + } + assert_eq!(frames.len(), 2); + assert_eq!(consumer.discontinuity(), 0); + } + /// When the current group's frame ends before the next group begins, there is /// still a gap to cover, so we don't skip early: a late-arriving frame on the /// slow group is delivered rather than dropped. diff --git a/rs/moq-mux/src/container/fmp4/mod.rs b/rs/moq-mux/src/container/fmp4/mod.rs index 2f592fa3983..30af4a65c10 100644 --- a/rs/moq-mux/src/container/fmp4/mod.rs +++ b/rs/moq-mux/src/container/fmp4/mod.rs @@ -58,6 +58,10 @@ pub enum Error { #[error("{0}")] InvalidEnd(#[from] crate::container::InvalidEnd), + /// A frame's timestamp sits below the live edge earlier groups reached. + #[error("{0}")] + TimestampRewind(#[from] crate::container::TimestampRewind), + #[error("timestamp overflow")] TimestampOverflow(#[from] moq_net::TimeOverflow), diff --git a/rs/moq-mux/src/container/mod.rs b/rs/moq-mux/src/container/mod.rs index 9291e792909..006fafbafc9 100644 --- a/rs/moq-mux/src/container/mod.rs +++ b/rs/moq-mux/src/container/mod.rs @@ -121,6 +121,11 @@ pub struct MissingKeyframe; #[error("video group endpoint precedes its last frame")] pub struct InvalidEnd; +/// A frame's timestamp sits below the live edge earlier groups reached. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("frame timestamp is below the live edge")] +pub struct TimestampRewind; + /// Encode and decode media frames over a moq-lite group. /// /// Implementors decide how many [`Frame`]s map onto one moq-lite frame: @@ -128,15 +133,16 @@ pub struct InvalidEnd; /// pack many samples into a single moof+mdat fragment. pub trait Container { /// Container-specific error. Must be convertible from [`moq_net::Error`] - /// (so IO errors propagate), [`MissingKeyframe`], and [`InvalidEnd`] - /// (so the producer can reject invalid group boundaries). + /// (so IO errors propagate), [`MissingKeyframe`], [`InvalidEnd`], and + /// [`TimestampRewind`] (so the producer can reject invalid group boundaries). type Error: std::error::Error + Send + Sync + Unpin + From + From - + From; + + From + + From; /// Encode one or more frames into a single moq-lite frame appended to `group`. fn write(&self, group: &mut moq_net::group::Producer, frames: &[Frame]) -> Result<(), Self::Error>; diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index a41ed39cb08..0a9a0e15234 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -1,7 +1,6 @@ -#[cfg(test)] use bytes::Bytes; -use super::{Container, Frame}; +use super::{Container, Frame, Kind}; fn add_micros(timestamp: moq_net::Timestamp, extra: moq_net::Timestamp) -> Option { let micros = timestamp.as_micros().saturating_add(extra.as_micros()); @@ -9,6 +8,10 @@ fn add_micros(timestamp: moq_net::Timestamp, extra: moq_net::Timestamp) -> Optio moq_net::Timestamp::from_micros(micros).ok() } +fn timestamp_lt(left: moq_net::Timestamp, right: moq_net::Timestamp) -> bool { + left.as_micros() < right.as_micros() +} + /// A producer for media tracks that manages group boundaries. /// /// Generic over `C: Container` to support different container encodings @@ -30,8 +33,9 @@ fn add_micros(timestamp: moq_net::Timestamp, extra: moq_net::Timestamp) -> Optio /// [`cut`](Self::cut) closes the current group early, ideally saying where its content /// ends; the next write must be a keyframe. Reach for it when the following keyframe won't /// supply that boundary in time, or to bound each group of an accumulating audio track. -/// [`discontinuity`](Self::discontinuity) goes further and publishes an empty group, for -/// when the timeline is about to jump rather than merely continue. +/// [`discontinuity`](Self::discontinuity) publishes a marker group of one empty frame, for +/// when the timeline is about to jump rather than merely continue. Timestamps never fall +/// below the live edge earlier groups reached. /// /// ## Buffering /// @@ -64,6 +68,9 @@ pub struct Producer { /// for a duration marker when the caller does not pass a bound. end: Option, + /// Exclusive presentation end of finished groups. A frame below this is refused. + live_edge: Option, + /// Duration of the frame that last raised [`end`](Self::end), if it had one. Distinguishes /// an exclusive presentation point from a max timestamp that still needs an estimate. last_duration: Option, @@ -99,6 +106,7 @@ impl Producer { pending_sequence: None, recorder: None, end: None, + live_edge: None, last_duration: None, previous_timestamp: None, cadence: None, @@ -194,6 +202,14 @@ impl Producer { &self.inner } + /// The exclusive presentation end earlier groups have reached, if any. + /// + /// A later [`write`](Self::write) below this is refused. Trusted sources that must + /// re-anchor (a PES resync, a capture restart) clamp to it rather than rewind. + pub fn live_edge(&self) -> Option { + self.live_edge + } + /// Write a frame to the track. /// /// A keyframe closes any open group and starts a new one. A non-keyframe extends the current @@ -201,19 +217,29 @@ impl Producer { /// joining mid-stream can skip frames until the first keyframe. A source where every frame is /// independently decodable (audio) marks only the first frame of each group a keyframe (see /// [`needs_keyframe`](Self::needs_keyframe)) so the group spans more than one frame. + /// + /// A timestamp below the live edge earlier groups reached returns + /// [`TimestampRewind`](super::TimestampRewind) without writing, the way an oversized + /// frame is refused. B-frames and open-GOP leading pictures still qualify when they sit + /// above that edge. pub fn write(&mut self, frame: Frame) -> Result<(), C::Error> { // A keyframe cuts the previous group, using its timestamp as the boundary - // where the previous group's content ends. + // where the previous group's content ends. Cut first so this group's live + // edge includes what we just closed, then refuse a rewind against that. if frame.keyframe { let rewound = self .previous_timestamp - .is_some_and(|previous| frame.timestamp < previous); + .is_some_and(|previous| timestamp_lt(frame.timestamp, previous)); self.cut((!rewound).then_some(frame.timestamp))?; if rewound { self.cadence = None; } } + if self.live_edge.is_some_and(|edge| timestamp_lt(frame.timestamp, edge)) { + return Err(super::TimestampRewind.into()); + } + // Start a new group if needed; the first frame of a group must be a keyframe. if self.group.is_none() { if !frame.keyframe { @@ -282,10 +308,10 @@ impl Producer { /// must be a keyframe. An explicit bound before the last ordered video frame /// returns [`InvalidEnd`](super::InvalidEnd) without flushing or closing the group. pub fn cut(&mut self, end: Option) -> Result<(), C::Error> { - if self.container.kind() == super::Kind::Video + if self.container.kind() == Kind::Video && !self.reordered && let Some((end, previous)) = end.zip(self.previous_timestamp) - && end < previous + && timestamp_lt(end, previous) { return Err(super::InvalidEnd.into()); } @@ -313,6 +339,9 @@ impl Producer { if let Some(mut group) = self.group.take() { group.finish()?; } + if let Some(end) = self.end { + self.raise_live_edge(end); + } self.end = None; self.last_duration = None; self.previous_timestamp = None; @@ -320,6 +349,12 @@ impl Producer { Ok(()) } + fn raise_live_edge(&mut self, timestamp: moq_net::Timestamp) { + if self.live_edge.is_none_or(|edge| !timestamp_lt(timestamp, edge)) { + self.live_edge = Some(timestamp); + } + } + /// Raise the furthest presentation point written, for [`cut`](Self::cut) to report. fn observe_end(&mut self, timestamp: moq_net::Timestamp, duration: Option) { self.reordered |= self.previous_timestamp.is_some_and(|previous| timestamp < previous); @@ -369,7 +404,7 @@ impl Producer { Ok(()) } - /// Publish an EMPTY group standing for a break in the timeline: content stopped, and + /// Publish a marker group standing for a break in the timeline: content stopped, and /// whatever comes next does not continue it. /// /// Call this whenever the timeline is about to jump -- pausing an encoder, switching @@ -385,23 +420,48 @@ impl Producer { /// the marker and waits for real media, instead of being served the group from *before* /// the break as though it were live. /// - /// Carries no timestamp on purpose: a break is a gap between two groups, so any single - /// timestamp is ambiguous about which side it belongs to. To bound the closing group's - /// final frame, [`cut(end)`](Self::cut) before calling this; the open group is closed - /// either way (an unbounded [`cut`](Self::cut) here is a no-op after yours). + /// Audio and video write one empty frame at the exclusive end of the previous epoch. + /// That object exists on moq-transport, so the live edge moves immediately; a consumer + /// MUST NOT submit it to a decoder. Data tracks skip a sequence with no object, because + /// an empty payload is data. The next [`write`](Self::write) opens the group after the + /// marker and must continue forward from the live edge; it cannot rewind. /// - /// The marker group carries no frames at all. A video track that writes duration - /// markers already closed the previous group's last frame from [`cut`](Self::cut). + /// To bound the closing group's final frame, [`cut(end)`](Self::cut) before calling this; + /// the open group is closed either way (an unbounded [`cut`](Self::cut) here is a no-op + /// after yours). pub fn discontinuity(&mut self) -> Result<(), C::Error> { self.cut(None)?; // Nothing is measured across the break: the frames still open on this side have no end, and // the gap to the far side is not a frame duration. self.estimator.discontinuity(); self.cadence = None; + if self.container.kind() == Kind::Data { + // Empty is payload on a data track, so skip this sequence with no object. + let skipped = match self.pending_sequence.take() { + Some(sequence) => sequence, + None => self.inner.latest().map_or(0, |s| s + 1), + }; + self.pending_sequence = Some(skipped + 1); + return Ok(()); + } let mut group = match self.pending_sequence.take() { Some(sequence) => self.inner.create_group(moq_net::group::Info { sequence })?, None => self.inner.append_group()?, }; + let timestamp = self.live_edge.unwrap_or(moq_net::Timestamp::ZERO); + if let Some(recorder) = self.recorder.as_mut() { + recorder.record(group.sequence, timestamp, false); + recorder.end(timestamp); + } + self.container.write( + &mut group, + &[Frame { + timestamp, + payload: Bytes::new(), + keyframe: false, + duration: None, + }], + )?; group.finish()?; Ok(()) } @@ -737,17 +797,18 @@ mod tests { groups } - /// A discontinuity lands as its own empty group between the content either side, so a - /// consumer can see the break instead of inferring continuity from adjacent sequences. + /// A discontinuity lands as its own marker group (one empty frame) between the + /// content either side, so a consumer can see the break instead of inferring + /// continuity from adjacent sequences. #[tokio::test] async fn discontinuity_publishes_an_empty_group() { // The resumed clock jumps forty minutes, so both the retention window and the // drift budget have to cover it or the pre-discontinuity group reads as ancient. let discontinuity_max_age = std::time::Duration::from_secs(41 * 60); - let info = hang::container::track_info(hang::catalog::PRIORITY.video).with_max_age(discontinuity_max_age); + let info = hang::container::track_info(hang::catalog::PRIORITY.audio).with_max_age(discontinuity_max_age); let track = track_producer("test", info); let consumer = track.subscribe(moq_net::track::Subscription::default().with_max_age(discontinuity_max_age)); - let mut producer = Producer::new(track, Container::Legacy(crate::container::Kind::Data)); + let mut producer = Producer::new(track, Container::Legacy(crate::container::Kind::Audio)); producer.write(frame(0, true)).unwrap(); producer.write(frame(10_000, false)).unwrap(); @@ -756,7 +817,11 @@ mod tests { producer.write(frame(2_405_070_000, true)).unwrap(); producer.finish().unwrap(); - assert_eq!(collect_groups(consumer).await, vec![2, 0, 1]); + let groups = collect_payloads(consumer).await; + assert_eq!(groups.len(), 3); + assert_eq!(groups[0], vec![(0, 2), (10_000, 2)]); + assert_eq!(groups[1], vec![(10_000, 0)], "the marker is one empty frame at the exclusive end"); + assert_eq!(groups[2][0], (2_405_070_000, 2)); } /// A subscription starts at the track's LATEST group, and the marker advances it even @@ -765,8 +830,8 @@ mod tests { /// 40-minute-stale frame reached a VOD recording in moq-dev/moq.pro#814. #[tokio::test] async fn discontinuity_moves_the_live_edge_off_stale_content() { - let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); - let mut producer = Producer::new(track, Container::Legacy(crate::container::Kind::Data)); + let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let mut producer = Producer::new(track, Container::Legacy(crate::container::Kind::Audio)); producer.write(frame(0, true)).unwrap(); let stale = producer.track().latest(); @@ -774,7 +839,7 @@ mod tests { producer.discontinuity().unwrap(); let edge = producer.track().latest(); - assert_ne!(edge, stale, "the empty group is the live edge now"); + assert_ne!(edge, stale, "the marker group is the live edge now"); assert_eq!(edge, stale.map(|s| s + 1)); } @@ -1110,17 +1175,52 @@ mod tests { } #[tokio::test] - async fn a_rewound_keyframe_does_not_write_a_backwards_marker() { + async fn a_rewound_keyframe_is_refused() { let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); - let consumer = track.subscribe(replay()); let mut producer = Producer::new(track, Container::Legacy(crate::container::Kind::Video)); producer.write(frame(20_000, true)).unwrap(); producer.write(frame(30_000, false)).unwrap(); + let err = producer.write(frame(0, true)).unwrap_err(); + assert!(matches!(err, crate::Error::TimestampRewind(_))); + } + + #[tokio::test] + async fn a_group_below_the_live_edge_is_refused() { + let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let mut producer = Producer::new(track, Container::Legacy(crate::container::Kind::Video)); producer.write(frame(0, true)).unwrap(); + producer.write(frame(33_000, false)).unwrap(); + producer.cut(None).unwrap(); + let err = producer.write(frame(16_000, true)).unwrap_err(); + assert!(matches!(err, crate::Error::TimestampRewind(_))); + } + + #[tokio::test] + async fn b_frames_within_a_group_are_accepted() { + let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let consumer = track.subscribe(replay()); + let mut producer = Producer::new(track, Container::Legacy(crate::container::Kind::Video)); + for (index, timestamp) in [0, 66_000, 33_000].into_iter().enumerate() { + producer.write(frame(timestamp, index == 0)).unwrap(); + } + producer.finish().unwrap(); + let groups = collect_payloads(consumer).await; + assert_eq!(groups[0][..3], [(0, 2), (66_000, 2), (33_000, 2)]); + } + + #[tokio::test] + async fn open_gop_leading_pictures_above_the_previous_group_are_accepted() { + let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let consumer = track.subscribe(replay()); + let mut producer = Producer::new(track, Container::Legacy(crate::container::Kind::Video)); + producer.write(frame(0, true)).unwrap(); + producer.write(frame(33_000, false)).unwrap(); + producer.write(frame(66_000, true)).unwrap(); + producer.write(frame(50_000, false)).unwrap(); producer.finish().unwrap(); let groups = collect_payloads(consumer).await; - assert_eq!(groups[0].last(), Some(&(40_000, 0))); - assert_eq!(groups[1], vec![(0, 2)], "a new epoch does not inherit the old cadence"); + assert_eq!(groups[1][0], (66_000, 2)); + assert_eq!(groups[1][1], (50_000, 2)); } #[tokio::test] @@ -1131,10 +1231,10 @@ mod tests { producer.write(frame(1_000_000, true)).unwrap(); producer.write(frame(1_010_000, false)).unwrap(); producer.discontinuity().unwrap(); - producer.write(frame(0, true)).unwrap(); - producer.write(frame(20_000, false)).unwrap(); + producer.write(frame(1_020_000, true)).unwrap(); + producer.write(frame(1_040_000, false)).unwrap(); producer.finish().unwrap(); let groups = collect_payloads(consumer).await; - assert_eq!(groups.last().unwrap().last(), Some(&(40_000, 0))); + assert_eq!(groups.last().unwrap().last(), Some(&(1_060_000, 0))); } } diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index 1bc5fb695a3..9e340fe62c1 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -184,12 +184,12 @@ impl ExportSource { self.description.as_ref() } - /// The underlying consumer's timeline-discontinuity counter, or 0 until the + /// The underlying consumer's playhead generation, or 0 until the /// subscription resolves. /// /// See [`Consumer::discontinuity`]. Sample it alongside each frame returned by /// [`poll_read`](Self::poll_read): the frame read while the counter changes is - /// the first of a new timeline, so anything anchored on the media clock (a + /// the first after a playhead event, so anything anchored on the media clock (a /// repetition cadence, a clock grid, a pacer) has to re-anchor to it. pub fn discontinuity(&self) -> u64 { match &self.state { diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 8125e0a0baa..c2dc76cfd3e 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -173,17 +173,9 @@ struct Track { } impl Track { - /// A fenced rendition must rewind its own clock before joining the program. - /// Forward markers on its old timeline cannot admit stale media. + /// Admit any frame that belongs to the current program generation. fn admit(&mut self, pending: Pending, epoch: u64) -> Option { - if self.epoch == epoch - || (pending.discontinuity != self.discontinuity - && self.timeline.is_none_or(|last| pending.frame.timestamp < last)) - { - return Some(pending); - } - self.discontinuity = pending.discontinuity; - None + (self.epoch == epoch).then_some(pending) } } @@ -578,22 +570,14 @@ impl Export { if changed { let joined = self.tracks[&name].epoch == self.epoch; if joined { - let backwards = self.tracks[&name] - .timeline - .is_some_and(|last| pending.frame.timestamp < last); - if !backwards && !self.pending.is_empty() { - // A forward boundary ends valid media rather than reneging it. + if !self.pending.is_empty() { + // A boundary ends valid media rather than reneging it. // Return that tail under the old generation before adopting the new one. self.emit(None)?; self.tracks.get_mut(&name).unwrap().pending = Some(pending); continue; } - // A backwards boundary fences its peers, so one program break costs one - // reset however many tracks cross it. A forward one cannot: the marker is - // local to its track, and a peer's marker for this same break looks exactly - // like its own later gap, so each is taken at face value and the program - // re-anchors for it. Telling them apart needs a boundary the wire carries. - self.rewind(backwards); + self.rewind(); } let track = self.tracks.get_mut(&name).unwrap(); track.discontinuity = pending.discontinuity; @@ -923,9 +907,9 @@ impl Export { self.emitted_epoch } - /// Discard uncommitted bytes and restart the program clock. On a backwards - /// boundary, peers must cross their own boundary before joining this generation. - fn rewind(&mut self, backwards: bool) { + /// Discard uncommitted bytes and restart the program clock. Every rendition + /// joins the new generation: no track is fenced across a declared marker. + fn rewind(&mut self) { self.epoch += 1; if let Some(counters) = self.span_counters.take() { self.counters = counters; @@ -945,8 +929,9 @@ impl Export { self.pcr_discontinuity = true; for track in self.tracks.values_mut() { track.last_dts = None; - if !backwards || track.timeline.is_none() { - track.epoch = self.epoch; + track.epoch = self.epoch; + if let Some(pending) = track.pending.as_ref() { + track.discontinuity = pending.discontinuity; } if let Some(pending) = track.pending.take() { track.pending = track.admit(pending, self.epoch); @@ -1134,17 +1119,12 @@ impl Export { Ok(()) } - /// A boundary takes precedence over stale peer data; otherwise use timestamp order. + /// Use timestamp order. No track is fenced, so a boundary does not jump the queue. fn pick_next_track(&self) -> Option { self.tracks .iter() .filter_map(|(n, t)| t.pending.as_ref().map(|p| (p.frame.timestamp, t.pid, n))) - .min_by_key(|(timestamp, pid, name)| { - let track = &self.tracks[*name]; - let changed = track.pending.as_ref().unwrap().discontinuity != track.discontinuity; - let backwards = changed && track.timeline.is_some_and(|last| *timestamp < last); - (!backwards, *timestamp, *pid, *name) - }) + .min_by_key(|(timestamp, pid, name)| (*timestamp, *pid, *name)) .map(|(_, _, name)| name.clone()) } diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index be7e9f05c3a..694d9af4698 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -2141,12 +2141,11 @@ fn publish_sdt( track } -/// Regression for #2833: every cadence in the exporter is anchored on a media timestamp -/// that only moves forward, so a publisher rewind froze each one for the length of the -/// rewound span. An audio-only program is the worst case: the PSI has no keyframe to -/// recover at, and the PCR-only packet is the only adaptation field it ever writes. +/// A declared marker restarts the program clock and table cadence. An audio-only +/// program is the worst case: the PSI has no keyframe to recover at, and the PCR-only +/// packet is the only adaptation field it ever writes. #[tokio::test(start_paused = true)] -async fn rewind_re_emits_tables_and_resumes_the_clock() { +async fn discontinuity_re_emits_tables_and_resumes_the_clock() { let mut broadcast = moq_net::broadcast::Info::new().produce(); let consumer = broadcast.consume(); let mut catalog = @@ -2185,7 +2184,7 @@ async fn rewind_re_emits_tables_and_resumes_the_clock() { } }; - let mut producer = Producer::new(track, HangContainer::Legacy(crate::container::Kind::Data)); + let mut producer = Producer::new(track, HangContainer::Legacy(crate::container::Kind::Audio)); let mut export = export_of(&consumer).await; // A ten-minute offset exercises the long rewind from the controlled stimulus @@ -2198,7 +2197,8 @@ async fn rewind_re_emits_tables_and_resumes_the_clock() { // A forward marker must flush the last old frame instead of discarding it. producer.discontinuity().unwrap(); - write(&mut producer, 2, 605_000_000); + write(&mut producer, 20, 605_000_000); + producer.cut(None).unwrap(); let marked = drain_frames(&mut export).await; let tail: Vec<_> = before .iter() @@ -2216,36 +2216,30 @@ async fn rewind_re_emits_tables_and_resumes_the_clock() { } assert!(preserved, "forward marker discarded the pre-boundary tail"); assert!(!audio_pts.is_empty()); - let epoch = export.discontinuity(); + assert_eq!(export.discontinuity(), 1, "the marker was observed"); - // The publisher rewinds and replays the first two seconds. - write(&mut producer, 20, 0); - let after = drain_frames(&mut export).await; - assert_eq!(export.discontinuity(), epoch + 1, "the rewind was observed"); - - // The clock leads the new timeline: its first frame is the PCR for slot 0, and the - // grid ramps from there instead of waiting to recross the old timeline. + let resume = marked.iter().position(|f| f.timestamp.as_micros() >= 605_000_000); + let resume = resume.unwrap_or_else(|| { + panic!( + "resumed media missing, marked timestamps {:?}", + marked.iter().map(|f| f.timestamp.as_micros()).collect::>() + ) + }); assert_eq!( - after[0].payload[3] & 0x30, + marked[resume].payload[3] & 0x30, 0x20, "the new timeline leads with its clock" ); - let pcrs = collect_pcrs(&after); - assert!(pcrs.len() > 60, "clock resumed promptly"); - assert_eq!(after[0].timestamp.as_micros(), 0); + let pcrs = collect_pcrs(&marked[resume..]); + assert!(!pcrs.is_empty(), "clock resumed promptly"); for pair in pcrs.windows(2) { assert_eq!(pair[1].1.wrapping_sub(pair[0].1) & ((1 << 33) - 1), 2250); } - assert!(after.iter().all(|f| f.timestamp.as_micros() < 2_000_000)); - - // And the tables come back on cadence rather than waiting ten minutes. - assert_eq!(count_pid(&after, 0x0000), 4, "PAT at 0, 0.5, 1 and 1.5s"); - assert_eq!(count_pid(&after, 0x0011), 1, "SDT at 0s"); - // Exactly one packet marks the break, and it is the leading PCR. + // Tables come back on cadence rather than waiting for the old 10-minute clock. + assert!(count_pid(&marked[resume..], 0x0000) >= 1, "PAT re-emitted after the marker"); assert_eq!(count_discontinuity(&before), 0); - assert_eq!(count_discontinuity(&after), 1, "the break is flagged exactly once"); - assert_eq!(count_discontinuity(&after[..1]), 1, "flagged on the leading PCR packet"); + assert_eq!(count_discontinuity(&marked), 1, "the break is flagged exactly once"); } /// The counter-case, and why the fix keys on the discontinuity counter rather than on a @@ -2319,12 +2313,10 @@ async fn reordered_video_keeps_the_table_cadence() { assert_eq!(count_discontinuity(&out), 0, "a reorder is not a discontinuity"); } -/// A program with more than one track marks the break once, not once per track. Each -/// rendition carries its own discontinuity counter, and the rewound frame sorts ahead of -/// any old-epoch straggler still buffered on the other track. Each track joins the -/// new program epoch on its own boundary, without comparing unrelated counter values. +/// A program with more than one track marks the break once, not once per track. A +/// declared marker joins every rendition; no track is fenced across it. #[tokio::test(start_paused = true)] -async fn rewind_flags_the_break_once_across_tracks() { +async fn discontinuity_flags_the_break_once_across_tracks() { let mut broadcast = moq_net::broadcast::Info::new().produce(); let consumer = broadcast.consume(); let mut catalog = @@ -2363,12 +2355,12 @@ async fn rewind_flags_the_break_once_across_tracks() { let mut idr = vec![0x65u8]; idr.extend(std::iter::repeat_n(0xAB, 300)); - let mut video = Producer::new(video_track, HangContainer::Legacy(crate::container::Kind::Data)); - let mut audio = Producer::new(audio_track, HangContainer::Legacy(crate::container::Kind::Data)); + let mut video = Producer::new(video_track, HangContainer::Legacy(crate::container::Kind::Video)); + let mut audio = Producer::new(audio_track, HangContainer::Legacy(crate::container::Kind::Audio)); // One keyframe-led second per group on video, 100ms audio frames alongside it. - let write = |video: &mut Producer, audio: &mut Producer, seconds: u64| { - for sec in 0..seconds { + let write = |video: &mut Producer, audio: &mut Producer, start: u64, seconds: u64| { + for sec in start..start + seconds { video .write(Frame { timestamp: Timestamp::from_micros(sec * 1_000_000).unwrap(), @@ -2393,7 +2385,7 @@ async fn rewind_flags_the_break_once_across_tracks() { }; let mut export = export_of(&consumer).await; - write(&mut video, &mut audio, 4); + write(&mut video, &mut audio, 0, 4); let before = drain_frames(&mut export).await; assert_eq!(export.discontinuity(), 0); assert_eq!(count_discontinuity(&before), 0); @@ -2425,24 +2417,23 @@ async fn rewind_flags_the_break_once_across_tracks() { assert_eq!(export.discontinuity(), 1, "local marker counts are not program epochs"); let epoch = export.discontinuity(); - // Both tracks rewind to zero. - write(&mut video, &mut audio, 2); + // Both tracks declare the same break and continue forward. Neither is fenced. + video.discontinuity().unwrap(); + audio.discontinuity().unwrap(); + write(&mut video, &mut audio, 5, 2); let after = drain_frames(&mut export).await; assert_eq!( export.discontinuity(), epoch + 1, - "one rewind, however many tracks saw it" + "one break, however many tracks saw it" ); assert_eq!( after[0].payload[3] & 0x30, 0x20, "the new timeline leads with its clock" ); - assert_eq!(after[0].timestamp, Timestamp::from_micros(0).unwrap()); assert_eq!(count_discontinuity(&after), 1, "the break is flagged exactly once"); - // Both renditions carry the rewound span; audio is not held back by a tune-in - // anchor that belongs to the old timeline. let bytes: Vec<_> = after.iter().flat_map(|f| f.payload.iter().copied()).collect(); let (video_pts, audio_pts) = collect_pes_pts(&bytes); assert!(!video_pts.is_empty(), "video resumed"); @@ -2473,72 +2464,15 @@ async fn rewind_flags_the_break_once_across_tracks() { assert!(video_frames > 0); let video_pid = video_pid.expect("video PID in PMT"); - // Keep old video queued while the lower-count audio source rewinds again. - // It must start a new program epoch despite having the smaller counter. - video.discontinuity().unwrap(); - video - .write(Frame { - timestamp: Timestamp::from_micros(8_000_000).unwrap(), - duration: None, - payload: length_prefixed(&[idr.as_slice()]), - keyframe: true, - }) - .unwrap(); - video.cut(None).unwrap(); - for i in 0..20 { - audio - .write(Frame { - timestamp: Timestamp::from_micros(i * 100_000).unwrap(), - duration: None, - payload: Bytes::from_static(&[0xaa; 180]), - keyframe: i == 0, - }) - .unwrap(); - } - audio.cut(None).unwrap(); + // #3533: a content join on a continuous timeline, audio stepping only a + // sub-frame (already re-anchored forward). Both tracks keep emitting; no fence. + write(&mut video, &mut audio, 7, 2); let again = drain_frames(&mut export).await; - assert_eq!(export.discontinuity(), epoch + 2); - assert_eq!(count_discontinuity(&again), 1); - assert!(again.len() > 20, "rewound audio kept emitting without its stale peer"); - assert!( - again.iter().all(|f| f.timestamp.as_micros() < 2_000_000), - "stale video advanced the clock" - ); - assert_eq!(count_pid(&again, video_pid), 0, "old video was discarded"); - - // An unrelated old-timeline marker cannot admit the peer's stale clock. - video.discontinuity().unwrap(); - video - .write(Frame { - timestamp: Timestamp::from_micros(9_000_000).unwrap(), - duration: None, - payload: length_prefixed(&[idr.as_slice()]), - keyframe: true, - }) - .unwrap(); - video.cut(None).unwrap(); - assert!(drain_frames(&mut export).await.is_empty()); - - // A delayed peer joins on its own boundary without resetting the clock again. - for i in 0..4 { - video - .write(Frame { - timestamp: Timestamp::from_micros(i * 1_000_000).unwrap(), - duration: None, - payload: length_prefixed(&[idr.as_slice()]), - keyframe: true, - }) - .unwrap(); - video.cut(None).unwrap(); - } - let joined = drain_frames(&mut export).await; - assert_eq!(export.discontinuity(), epoch + 2); - assert_eq!(count_discontinuity(&joined), 0); - assert!(count_pid(&joined, video_pid) > 0, "video rejoined"); - // A discarded span already had counters assigned. They must roll back to - // the last emitted bytes, so dropping the span introduces no packet loss. + assert!(!again.is_empty(), "both tracks kept emitting across the join"); + assert!(count_pid(&again, video_pid) > 0, "video was not fenced"); + assert_eq!(count_discontinuity(&again), 0, "a forward join is not a PCR break"); let mut counters = std::collections::HashMap::new(); - for frame in before.iter().chain(&marked).chain(&after).chain(&again).chain(&joined) { + for frame in before.iter().chain(&marked).chain(&after).chain(&again) { for packet in frame.payload.chunks_exact(188) { let pid = u16::from(packet[1] & 0x1f) << 8 | u16::from(packet[2]); let cc = packet[3] & 15; diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 8ed12f2377b..077f7206ab7 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -533,6 +533,7 @@ impl Import { tail: Vec::new(), tail_pts: None, resync: Resync::new(pid.as_u16(), descriptor.track_suffix), + shift: None, })) } @@ -2287,6 +2288,8 @@ struct LegacyStream { /// only covers frames that begin in that PES. tail_pts: Option, resync: Resync, + /// Added to source PTS after a timebase reset so the hang timeline never rewinds. + shift: Option, } impl LegacyStream { @@ -2311,7 +2314,11 @@ impl LegacyStream { // ISO 13818-1, a PES PTS refers to the first access unit starting in it). // After each frame it advances by that frame's duration (per frame, not // `index * constant`: E-AC-3 varies the samples per frame). - let mut pts = if carried > 0 { self.tail_pts.take() } else { pes_base }; + let mut pts = if carried > 0 { + self.tail_pts.take() + } else { + self.reanchor(pes_base)? + }; let mut in_tail = carried > 0; let mut offset = 0; @@ -2320,7 +2327,7 @@ impl LegacyStream { let mut fallback = None; while offset + self.descriptor.min_header_len <= data.len() { if in_tail && offset >= carried { - pts = pes_base; + pts = self.reanchor(pes_base)?; in_tail = false; } @@ -2397,7 +2404,7 @@ impl LegacyStream { match self.resync.recover(data, offset, &self.descriptor.into()) { Recover::At(next) => { if in_tail { - pts = pes_base; + pts = self.reanchor(pes_base)?; in_tail = false; } offset = next; @@ -2415,7 +2422,7 @@ impl LegacyStream { } None => { if in_tail { - pts = pes_base; + pts = self.reanchor(pes_base)?; in_tail = false; } offset = next; @@ -2464,7 +2471,7 @@ impl LegacyStream { // PES, with the PTS it should carry. if offset < data.len() { if in_tail && offset >= carried { - pts = pes_base; + pts = self.reanchor(pes_base)?; } self.tail = data[offset..].to_vec(); self.tail_pts = pts; @@ -2473,9 +2480,29 @@ impl LegacyStream { Ok(published) } + fn reanchor(&mut self, pts: Option) -> anyhow::Result> { + let Some(pts) = pts else { + return Ok(None); + }; + if let Some(delta) = self.shift { + return Ok(Some(pts.checked_add(delta.convert(pts.scale())?)?)); + } + let Some(edge) = self.import.as_ref().and_then(legacy::Import::live_edge) else { + return Ok(Some(pts)); + }; + if pts.as_micros() >= edge.as_micros() { + return Ok(Some(pts)); + } + let delta = edge.as_micros() - pts.as_micros(); + let delta = Timestamp::from_micros(u64::try_from(delta)?)?; + self.shift = Some(delta); + Ok(Some(edge.convert(pts.scale())?)) + } + fn discontinuity(&mut self, published: bool) -> anyhow::Result<()> { self.desync(); self.unwrap.discontinuity(); + self.shift = None; if published && let Some(import) = &mut self.import { import.discontinuity()?; } @@ -3496,7 +3523,7 @@ mod test { .unwrap(); let mut reader = crate::container::Consumer::new( track, - crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), + crate::catalog::hang::Container::Legacy(crate::container::Kind::Audio), ); let mut frames = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { @@ -4087,6 +4114,53 @@ mod test { ); } + // #3533: a content join on a continuous transport timeline can land a PES PTS a few + // milliseconds below the extrapolated high-water mark. Re-anchor forward from the live + // edge instead of refusing the frame and aborting the import. + #[tokio::test(start_paused = true)] + async fn legacy_resync_below_the_live_edge_publishes_forward() { + const MP2_PID: u16 = 0x0061; + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); + let mut import = super::Import::new(broadcast, catalog.reserve()); + + let pmt = synth_pmt(&[(StreamType::Mpeg1Audio, MP2_PID)], false); + import.decode(&bytes::BytesMut::from(&pmt[..])).unwrap(); + + // Two frames so the first is confirmed and published: live edge is 90_000 ticks. + let mut first = mp2_frame(0xAA); + first.extend_from_slice(&mp2_frame(0xBB)); + import + .decode(audio_pes_packet(MP2_PID, 0, 90_000, &first).as_slice()) + .unwrap(); + + // A join a millisecond behind the live edge. + let mut join = mp2_frame(0xDD); + join.extend_from_slice(&mp2_frame(0xEE)); + import + .decode(audio_pes_packet(MP2_PID, 1, 89_910, &join).as_slice()) + .expect("a PES PTS below the live edge must not abort the import"); + import.finish().unwrap(); + + let frames = read_audio_frames(&consumer, &catalog).await; + assert!( + frames.len() >= 3, + "expected published frames after the join, got {}", + frames.len() + ); + let mut last = 0; + for frame in &frames { + assert!( + frame.timestamp.as_micros() >= last, + "published a rewind: {last} then {}", + frame.timestamp.as_micros() + ); + last = frame.timestamp.as_micros(); + } + } + // Confirming a joined frame means holding it when the buffer ends too soon after it to // hold a header. At end of stream that header is never coming, so the drain publishes it // anyway rather than dropping a frame that is whole and parses. @@ -4850,7 +4924,7 @@ mod test { let track = consumer.track(name).unwrap().subscribe(subscription).await.unwrap(); let mut reader = crate::container::Consumer::new( track, - crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), + crate::catalog::hang::Container::Legacy(crate::container::Kind::Audio), ); let mut frames = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await @@ -4942,7 +5016,10 @@ mod test { let name = catalog.snapshot().mpegts.tracks.keys().next().unwrap().clone(); let (frames, breaks) = read_breaks(&consumer, &name).await; assert_eq!(frames.len(), 3); - assert_eq!(breaks, 2, "sections must participate in reset boundaries"); + assert_eq!( + breaks, 0, + "a data sequence hole whose timestamps do not jump is not a playhead event" + ); } #[tokio::test(start_paused = true)] @@ -5045,8 +5122,8 @@ mod test { let stamps: Vec = frames.iter().map(|f| f.timestamp.as_micros()).collect(); assert_eq!(stamps[0], 45_000_000, "the old timeline"); assert!( - stamps.last().is_some_and(|&last| last < 2_000_000), - "the restarted clock was unwrapped as a field rollover: {stamps:?}" + stamps.last().is_some_and(|&last| last >= 45_000_000), + "the restarted clock rewound instead of continuing forward: {stamps:?}" ); } } @@ -5259,16 +5336,8 @@ mod test { ); } - /// How a shared forward boundary is identified across renditions: it is not. - /// - /// The break reaches each rendition as its own marker, landing at that rendition's own - /// media position, and a consumer counter is local to the track it came from. One program - /// break and two renditions with independent gaps are the same evidence, so the exporter - /// takes each forward marker at face value and re-anchors the clock for it. That costs a - /// redundant flag per peer, which a receiver re-acquires on; coalescing them would have to - /// tell a peer's delayed marker for this break from its own later one, and admit a peer - /// that rewinds while the first went forwards, so it waits on a boundary the wire carries - /// rather than a heuristic over local counters. + /// A shared forward boundary is one program break. Every rendition joins the new + /// generation, so the clock flags it once however many tracks declared the marker. #[tokio::test(start_paused = true)] async fn a_shared_forward_boundary_resets_once_per_rendition() { const PES_TICKS: u64 = 2 * 72 * 90; @@ -5291,6 +5360,6 @@ mod test { stimulus.extend_from_slice(&clock_break_packet(PCR_PID)); media(&mut stimulus, &mut cc, 31 * 90_000); - assert_eq!(export_discontinuities(&stimulus).await, 2, "one flag per rendition"); + assert_eq!(export_discontinuities(&stimulus).await, 1, "one flag per program break"); } } diff --git a/rs/moq-mux/src/error.rs b/rs/moq-mux/src/error.rs index dc578482e54..0995e9bbcee 100644 --- a/rs/moq-mux/src/error.rs +++ b/rs/moq-mux/src/error.rs @@ -159,6 +159,10 @@ pub enum Error { #[error("{0}")] InvalidEnd(#[from] crate::container::InvalidEnd), + /// A frame's timestamp sits below the live edge earlier groups reached. + #[error("{0}")] + TimestampRewind(#[from] crate::container::TimestampRewind), + /// A FLV video frame resolved to a negative presentation timestamp. #[error("negative FLV video presentation timestamp: dts={dts_ms}ms composition_time={composition_time_ms}ms")] NegativeFlvPts { diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index 2b2e4535d20..4d0dac07f9e 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -26,7 +26,7 @@ pub struct Consumer { pending: VecDeque, /// Whether the ended track's decoder has already been drained. drained: bool, - /// Last container discontinuity observed. A change starts a fresh codec epoch. + /// Last container playhead generation observed. discontinuity: u64, } @@ -104,10 +104,9 @@ impl Consumer { let mux_frame = self.track.read().await?; let discontinuity = self.track.discontinuity(); if discontinuity != self.discontinuity { - // The tail belongs to the abandoned codec epoch. Draining resets the - // backend for reuse, but none of those pictures may cross the seam. - self.decoder.flush().await?; - self.pending.clear(); + // A playhead event re-applies startup delay and skip; the next group + // already starts on a keyframe with parameter sets, so the decoder is + // not flushed. self.discontinuity = discontinuity; } @@ -429,11 +428,10 @@ mod tests { ); } - /// A declared discontinuity abandons the previous codec epoch. A delayed - /// picture from before the seam is drained and discarded before the first new - /// keyframe is decoded. + /// A declared discontinuity is a playhead event, not a decoder flush. A delayed + /// picture from before the seam still surfaces; the next group continues forward. #[tokio::test] - async fn discontinuity_discards_buffered_tail() { + async fn discontinuity_does_not_flush_the_decoder() { let mut broadcast = moq_net::broadcast::Info::new().produce(); let track = broadcast .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) @@ -454,7 +452,7 @@ mod tests { producer.discontinuity().unwrap(); producer .write(moq_mux::container::Frame { - timestamp: Timestamp::ZERO, + timestamp: Timestamp::from_micros(200_000).unwrap(), duration: None, payload: Bytes::from_static(b"new access unit"), keyframe: true, @@ -484,7 +482,7 @@ mod tests { while let Some(frame) = consumer.read().await.unwrap() { timestamps.push(frame.timestamp.as_micros()); } - assert_eq!(timestamps, vec![0]); + assert_eq!(timestamps, vec![100_000, 200_000]); } /// Cancellation while a threaded flush is in flight leaves the sink poisoned. From d057fd62099de20a19c76b3763d5f024265000b0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 16 Sep 2026 23:25:03 -0700 Subject: [PATCH 3/4] feat(hang)!: complete the monotonic timeline Cover the remaining tests, re-anchor SRT egress on a declared marker, and delete the completed quest. Co-Authored-By: grok-4.6 --- js/hang/src/container/consumer.test.ts | 19 ++ quest/m1/README.md | 4 +- quest/m1/broadcast-clock.md | 1 - quest/m1/merge-dev.md | 1 - quest/m1/monotonic-timeline.md | 178 ------------------ ...coder-captures-the-rewind-generation-at.md | 10 +- quest/m2/js-publish-discontinuity.md | 12 +- rs/moq-audio/src/encode/producer.rs | 8 +- rs/moq-boy/src/audio.rs | 2 +- rs/moq-boy/src/main.rs | 2 +- rs/moq-boy/src/video.rs | 2 +- rs/moq-mux/src/codec/aac/import.rs | 2 +- rs/moq-mux/src/codec/h264/import.rs | 2 +- rs/moq-mux/src/codec/h265/import.rs | 2 +- rs/moq-mux/src/codec/legacy.rs | 2 +- rs/moq-mux/src/codec/opus/import.rs | 2 +- rs/moq-mux/src/container/consumer.rs | 23 ++- rs/moq-mux/src/container/producer.rs | 6 +- rs/moq-mux/src/container/ts/export_test.rs | 5 +- rs/moq-srt/src/server.rs | 27 +-- rs/moq-video/src/decode/consumer.rs | 3 +- rs/moq-video/src/encode/producer.rs | 4 +- 22 files changed, 81 insertions(+), 236 deletions(-) delete mode 100644 quest/m1/monotonic-timeline.md diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index b8c1cc9c07e..fdcb5e9cb56 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -1302,6 +1302,25 @@ test("Consumer empty groups mean nothing", async () => { consumer.close(); }); +test("Consumer latency skip bumps playhead generation once", async () => { + const track = new Track.Producer("test"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxAge: 0 as Time.Milli }); + + writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); + await settle(); + expect((await nextFrame(consumer))?.frame?.timestamp).toBe(0 as Time.Micro); + expect((await consumer.next())?.frame).toBeUndefined(); + + writeGroupWithLegacyFrames(track, 2, [100_000 as Time.Micro]); + await settle(); + + const resumed = await nextFrame(consumer); + expect(resumed?.frame?.timestamp).toBe(100_000 as Time.Micro); + expect(resumed?.discontinuity).toBe(1); + + consumer.close(); +}); + test("Consumer jumps the playhead after a shed marker with a timestamp hole", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxAge: 0 as Time.Milli }); diff --git a/quest/m1/README.md b/quest/m1/README.md index 498ff3833d5..fa592b7df77 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -5,8 +5,7 @@ Work intended to land on `dev` before it merges to `main`, or be explicitly deferred by the maintainer: the breaking API and wire changes (the announce and wildcard surface, error codes, the -allocator mirrors, the bindings), the merge gates (the monotonic timeline), -and the merge itself. +allocator mirrors, the bindings), the merge gates, and the merge itself. ## Plan @@ -37,7 +36,6 @@ does not require it. - [JSON config names](/quest/m1/api-json-binary-config-names.md) - `Config` means the codec options and `producer::Config` / `consumer::Config` the track pair in all four json and binary packages - [FFI units](/quest/m1/api-ffi-units-verbs.md) - every moq-ffi duration is microseconds and `publish()` / `consume()` pair with their setters - [Deprecated sweep](/quest/m1/deprecated-sweep.md) - uncalled deprecated items, warn-then-ignore flags, and silent aliases are removed or become refusals before the release -- [Monotonic timeline](/quest/m1/monotonic-timeline.md) - a marker group of one empty frame declares a break and moves the live edge; producers refuse a rewind; consumers jump the playhead on an unproven hole and drop rewind detection - [Joining FETCH](/quest/m1/joining-fetch.md) - moq-net: every subscribe to a draft-14 to draft-19 relay joins at a group boundary, with contiguous history for an explicit start, via Largest Object plus a joining FETCH - [Anonymous rank](/quest/m1/anonymous-route-rank.md) - moq-net: a route through an anonymous hop ranks below every identified route at any cost, and hop 0 travels the chain to say so - [Cluster -01](/quest/m1/cluster-01/README.md) - rs/moq-net and js/net speak the revised cluster extension (HOP_ID, REQUEST_UPDATE repricing) and -01 is published diff --git a/quest/m1/broadcast-clock.md b/quest/m1/broadcast-clock.md index 798d5ffcffb..263be694a4e 100644 --- a/quest/m1/broadcast-clock.md +++ b/quest/m1/broadcast-clock.md @@ -59,7 +59,6 @@ belong to the M2 integration; do not claim those adapters are already wired. - [Publisher clocks](/quest/m2/publisher-clock.md) - adopts the clock contract across native capture, CLI import, and browser publication -- [Monotonic timeline](/quest/m1/monotonic-timeline.md) - forward-only publication and declared discontinuities - [GStreamer wall clock](/quest/m2/3021-moq-gst-anchor-generated-media-timelines-to-wall-clock.md) - selects one trusted clock mapping for every pad - [Browser wall access](/quest/m2/2278-watch-absolute-wall-clock-latency-target-for-synchronized.md) - exposes the same broadcast clock to applications - [Cross-track correlation](/quest/m3/teleop/correlation.md) - applications compare clocks they know are synchronized diff --git a/quest/m1/merge-dev.md b/quest/m1/merge-dev.md index c064a564258..1bcc6eeda4a 100644 --- a/quest/m1/merge-dev.md +++ b/quest/m1/merge-dev.md @@ -59,7 +59,6 @@ additive quest that builds on dev-only code start on main afterwards from - [Cluster peer API](/quest/m1/api-cluster-peer-config.md) - typed peer configuration supports current symmetric policy - [Broadcast clock](/quest/m1/broadcast-clock.md) - the root clock replaces the published archive wall field on dev before release - [m0](/quest/m0/README.md) - every release blocker lands or is punted before the merge -- [Monotonic timeline](/quest/m1/monotonic-timeline.md) - so a shed marker still jumps the playhead on a timestamp hole (#3291) ## Closes diff --git a/quest/m1/monotonic-timeline.md b/quest/m1/monotonic-timeline.md deleted file mode 100644 index d9574eb12e6..00000000000 --- a/quest/m1/monotonic-timeline.md +++ /dev/null @@ -1,178 +0,0 @@ -# [L] Timelines only move forward - -## Goal - -A track's timestamps never fall below the live edge its earlier groups -reached. A publisher that has to reset (a flush, a seek, a source restart) -declares a discontinuity by publishing the next group as a marker (one empty -frame, no decodable payload) and resuming in the group after that, continuing -forward from where it was; it cannot rewind. Inside a group timestamps still -reorder freely, since B-frames present before the frames that precede them in -decode order, and open-GOP leading pictures still qualify because they sit -above the previous group's reach. - -The marker group is a real object, so a moq-transport peer sees it and the -live edge moves immediately. An empty group (zero objects) is invisible on -moq-transport and means nothing in hang. A consumer MUST NOT submit the marker -to a decoder. - -A delivered hole the consumer cannot prove harmless is a playhead event: -re-apply startup delay and skip. It is not a decoder flush. The next group -already starts on a keyframe with parameter sets. A hole is harmless when the -previous group's end meets the next group's first timestamp within 1 ms, which -keeps DTS-derived ids and the HLS importer's packed epoch bits playing -through. A latency skip is a delivered hole, so it bumps the same playhead -generation. - -Consumers stop detecting and re-anchoring on undeclared rewinds. A group that -breaks the forward-only rule is a malformed track, not a timeline event. - -This also retires the inferred rewind behind #3533: `moq export ts` stalls -video and primary audio for good at the first content join of a source whose -transport timeline is continuous, because the legacy audio importer's -sub-frame backwards step at a PES resync is read as a rewind and the -exporter's backward fence never opens for the bystander tracks. With no -inferred rewinds there is no fence. - -js/publish does not emit the marker yet. - -## Plan - -### Declaration - -Today `Producer::discontinuity()` (`rs/moq-mux/src/container/producer.rs:365`) -cuts the open group and publishes an empty group at `seq+1`. Replace it: cut, -publish group N+1 containing a single empty frame whose timestamp is the -exclusive end of the previous epoch, and let the next `write` open N+2. The -empty frame is an object, so the group exists on moq-transport; hang already -forbids submitting an empty video or audio payload to a decoder -(`drafts/draft-lcurley-moq-hang.md:525-534`). Relax "each group MUST start -with a keyframe" (`:511`) so a group that contains only that marker is legal. -Audio uses the same empty-frame marker (not submitted to the decoder). Data -tracks have no marker: empty is payload, so they skip a sequence with no -object in the hole. - -Callers stay: h264/h265/opus/aac/legacy importers, TS import, moq-video idle -capture, moq-audio, moq-boy. js/publish is -[its own quest](/quest/m2/js-publish-discontinuity.md). - -If the marker arrives, walk now and bump the playhead generation. If it is -shed, N→N+2 is a sequence hole: wait until `max_age` trips or the track -finishes (`rs/moq-mux/src/container/consumer.rs:368-375`), then bump playhead -generation only when the boundary is not 1 ms-contiguous. An idle resume -whose timestamps jumped still jumps the playhead after a shed marker -(#3291); an encoder restart on a continuous clock looks like a DTS id jump -and does not. - -### Playhead, not codec reset - -The mux discontinuity counter becomes a playhead generation: startup delay -and skip, not `decoder.flush()` / `decoder.reset()`. Remove those flushes -from `rs/moq-video/src/decode/consumer.rs:85-92` and -`rs/moq-audio/src/decode/consumer.rs:283-302`. Audio still re-applies -pre-skip in the play path. Watch's remaining in-flight-frame question is -[#3056](/quest/m2/3056-watch-video-decoder-captures-the-rewind-generation-at.md). - -### Publisher refuse rewind - -The track producer refuses a frame whose timestamp is below the live edge -established by the groups before its own, returning an error the way an -oversized frame does, so a rewind never reaches the wire. That is `moq-mux`'s -producer and the `js/hang` container producer (`js/hang/src/container/legacy.ts` -has no cross-group live-edge check today). moqsink already re-anchors forward -on a flush and rejects a rewinding base; check the CLI importers, the capture -publishers, and `js/publish` do the same on a source restart, and re-anchor -rather than refuse where the source is trusted. - -The legacy MPEG audio importer in `rs/moq-mux/src/container/ts/import.rs` is -one such trusted source and must re-anchor forward. `LegacyStream::write` -takes the PES PTS for the first frame that begins in a PES and then -extrapolates every following frame by its own duration -(`advance_pts`, `:2436`), carrying the extrapolated value across a PES -boundary as `tail_pts` (`:2447`). When frame sync is lost it abandons the -tail and re-anchors on the PES PTS (`pts = pes_base`, `:2377` and `:2395`). -At a content join on a continuous transport timeline that PES PTS sits a few -milliseconds below the extrapolated high-water mark, which is the backwards -step #3533 measured (one MPEG-1 audio resync at the join, `audio frame sync -lost pid=121 resyncs=2`). Once the producer refuses rewinds that sub-frame -step becomes a refused frame and an aborted import rather than a stall, so -the resync must clamp its re-anchor to the track's live edge (or skip the -frame that would land below it) and only then continue from the PES PTS. -A stale tail after a loop wrap, wrong by hours, still re-anchors on the PES. - -### Consumer abort rewind - -Delete the rewind boundary and its classification in both languages (`Rewind` -in `rs/moq-mux/src/container/consumer.rs:83` and -`js/hang/src/container/consumer.ts:76`). A group below the live edge aborts -the track as malformed. The playhead generation stays, counting declared -marker groups, unproven delivered holes, and latency skips. - -### Draft - -`drafts/draft-lcurley-moq-hang.md:515-517` loses the empty-group sentences and -the backward clause. State: a group with no decodable frames is a walk-now -discontinuity; empty groups are permitted and mean nothing; after a -discontinuity the timeline continues forward; a delivered sequence hole is a -playhead event unless the boundary is contiguous within 1 ms. The moq-lite -draft is unchanged; enforcement lives above the relay. - -### TS export - -`Exporter::rewind(backwards)` (`rs/moq-mux/src/container/ts/export.rs:928-952`) -restarts the program clock on a source discontinuity, clearing `last_pcr`, -`last_psi`, and every `si[*].last_emit`; it is keyed on the source's playhead -generation (`:696`). The `backwards` flag (decided at `:581` and `:1145`, and -what fences peers across a backward boundary at `:948`) disappears with -backward rewinds, so the reset takes no argument and its docs lose the -backward case. `Track::admit` (`:178`) then admits any frame in the current -generation and no track is ever left behind in an old one, which is the #3533 -fix: the fence that discarded video and primary audio for 40 minutes no -longer exists. Keep `pcr_discontinuity`, the `discontinuity_indicator` on the -PCR packet, since a PCR jump over 100 ms without it is a TR 101 290 error -whichever direction the jump goes. - -### Tests - -In both languages: the producer refuses a group below the live edge; the -consumer aborts on one; a group with reordered B-frames is accepted; an -open-GOP group whose leading pictures sit below its keyframe but above the -previous group passes; a marker group followed by a forward jump bumps -playhead generation once and does not flush a decoder; a latency skip does -the same; a non-sequential id jump with a contiguous boundary -(`consumer.nonsequential.test.ts`) does not; a zero-budget idle resume whose -timestamps jumped still jumps the playhead after the marker is shed (#3291). -Rewrite `empty_group_declares_a_discontinuity`, -`latency_skip_preserves_empty_group_discontinuity`, -`discontinuity_publishes_an_empty_group`, and -`discontinuity_moves_the_live_edge_off_stale_content` for the marker group -(one empty frame, not zero objects). - -TS import: a legacy audio resync whose PES PTS sits below the extrapolated -high-water mark publishes forward from the live edge and does not abort. - -TS export, the #3533 regression in `export_test.rs`: two tracks on a -continuous transport timeline whose content restarts, the audio track alone -stepping back by less than one frame at the join; video and audio keep -emitting across it, and the join costs at most one PCR discontinuity and one -PSI re-emission. `rewind_re_emits_tables_and_resumes_the_clock` -(`export_test.rs:2129`) and `rewind_flags_the_break_once_across_tracks` -(`:2307`) are rewritten for the no-backwards world: the break is a declared -marker group, the clock and tables reset once per program break, and no -track is fenced. - -Branch from `dev`, where the container consumers carry the current `Rewind` -state and `is_stale` sheds empty groups. - -## Closes - -- [#3291](https://github.com/moq-dev/moq/issues/3291) - close this issue when the quest finishes -- [#3533](https://github.com/moq-dev/moq/issues/3533) - close this issue when the quest finishes - -## Related - -- [js/publish discontinuity](/quest/m2/js-publish-discontinuity.md) - the JS container producer and js/publish emit the same marker -- [#3056](/quest/m2/3056-watch-video-decoder-captures-the-rewind-generation-at.md) - whether watch still resets VideoDecoder to drop in-flight chunks when playhead generation bumps -- [#3115](/quest/m2/3115-moqsink-the-publication-has-no-generation-so-a-flush.md) - moqsink's generation model after EOS, the same publisher -- [Open-GOP leading pictures](/quest/m2/open-gop-leading-pictures.md) - latency skip now bumps playhead generation but does not flush the decoder, so leading pictures still decode -- [TS import stream liveness](/quest/m2/3489-ts-import-stream-liveness.md) - the per-stream access-unit reporting that would have shown the #3533 stall at the importer diff --git a/quest/m2/3056-watch-video-decoder-captures-the-rewind-generation-at.md b/quest/m2/3056-watch-video-decoder-captures-the-rewind-generation-at.md index da64840d4d5..fb140031bdb 100644 --- a/quest/m2/3056-watch-video-decoder-captures-the-rewind-generation-at.md +++ b/quest/m2/3056-watch-video-decoder-captures-the-rewind-generation-at.md @@ -27,18 +27,14 @@ against the re-anchored clock for the full distance between the two timelines. - The regression needs a real WebCodecs decoder, so it lives in a browser harness rather than a bun unit test. -[Monotonic timeline](/quest/m1/monotonic-timeline.md) makes the container -signal a playhead generation, not a codec reset: native decode stops flushing -on it. This quest is whether watch still calls `decoder.reset()` to drop -in-flight WebCodecs chunks when that generation bumps. The decoder is the -same on main, so the fix lands there once the timeline quest has settled. +The container signals a playhead generation, not a codec reset: native decode +stops flushing on it. This quest is whether watch still calls `decoder.reset()` +to drop in-flight WebCodecs chunks when that generation bumps. ## Required - [Merge dev](/quest/m1/merge-dev.md) - the required M1 APIs must be available on main before this implementation starts -- [Monotonic timeline](/quest/m1/monotonic-timeline.md) - settles playhead generation before watch's reaction to it is pinned - ## Closes - [#3056](https://github.com/moq-dev/moq/issues/3056) - close this issue when the quest finishes diff --git a/quest/m2/js-publish-discontinuity.md b/quest/m2/js-publish-discontinuity.md index 5786a3898a4..3826f0e00ac 100644 --- a/quest/m2/js-publish-discontinuity.md +++ b/quest/m2/js-publish-discontinuity.md @@ -9,14 +9,12 @@ resumes in the next group, and js/publish calls that on encoder restart. ## Plan js/hang has no container producer `discontinuity()` today (only comments in -`js/publish/src/audio/encoder.ts:154`). Add one that matches -[monotonic timeline](/quest/m1/monotonic-timeline.md): cut the open group, -write a single empty frame at the exclusive end of the previous epoch, finish -that group, and let the next keyframe open the resume group. Wire encoder -restart in js/publish to it. Data tracks skip a sequence with no marker. +`js/publish/src/audio/encoder.ts:154`). Add one that matches the hang marker +contract: cut the open group, write a single empty frame at the exclusive end +of the previous epoch, finish that group, and let the next keyframe open the +resume group. Wire encoder restart in js/publish to it. Data tracks skip a +sequence with no marker. ## Required - [Merge dev](/quest/m1/merge-dev.md) - the required M1 APIs must be available on main before this implementation starts - -- [Monotonic timeline](/quest/m1/monotonic-timeline.md) - settles the marker-group contract this producer must emit diff --git a/rs/moq-audio/src/encode/producer.rs b/rs/moq-audio/src/encode/producer.rs index 6c9a313c4bf..e5e557145f2 100644 --- a/rs/moq-audio/src/encode/producer.rs +++ b/rs/moq-audio/src/encode/producer.rs @@ -116,9 +116,9 @@ pub struct Producer { /// (re)start. Emitted PTS = `epoch + frames_produced / codec_rate`. `None` /// until the first write so the next frame re-anchors to its timestamp. epoch_us: Option, - /// An encoder reset that still needs an empty group before its next packet. + /// An encoder reset that still needs a marker group before its next packet. pending_discontinuity: bool, - /// Whether an empty group already separates the next packet from prior codec state. + /// Whether a marker group already separates the next packet from prior codec state. decoder_boundary: bool, /// How the encoder classified the packet it published most recently. activity: Activity, @@ -301,8 +301,8 @@ impl Producer { /// released-then-reopened microphone) so the gap appears in the PTS and /// audio stays aligned with a wall-clock video track, rather than the gap /// being compressed out by the running sample count. Mirrors moq-boy's - /// `reset_epoch`. If the codec had started, an empty group is published before - /// the next packet so subscribers reset their decoders too. + /// `reset_epoch`. If the codec had started, a marker group is published before + /// the next packet so subscribers jump the playhead. pub fn reset_epoch(&mut self) { if self.encoder.started() && !self.decoder_boundary { self.pending_discontinuity = true; diff --git a/rs/moq-boy/src/audio.rs b/rs/moq-boy/src/audio.rs index bf309fbb04c..51d10a30aae 100644 --- a/rs/moq-boy/src/audio.rs +++ b/rs/moq-boy/src/audio.rs @@ -48,7 +48,7 @@ impl AudioEncoder { self.producer.reset_epoch(); } - /// Publish an empty group marking the pause, so the gap the re-anchored epoch is about + /// Publish a marker group marking the pause, so the gap the re-anchored epoch is about /// to open reads as a break rather than one very long packet. pub fn discontinuity(&mut self) -> Result<()> { self.producer.discontinuity()?; diff --git a/rs/moq-boy/src/main.rs b/rs/moq-boy/src/main.rs index 47e893a3feb..ca6a9381957 100644 --- a/rs/moq-boy/src/main.rs +++ b/rs/moq-boy/src/main.rs @@ -338,7 +338,7 @@ fn run_emulator( if session.paused.load(Ordering::Acquire) { // Mark the break BEFORE parking, not after resuming. Our PTS is the emulator // clock, which keeps running while we sleep, so the next frame lands however - // long the pause was into the future. Published as an empty group that gap + // long the pause was into the future. Published as a marker group that gap // reads as a discontinuity rather than one enormous frame duration, and the // marker becomes the live edge -- so a viewer arriving mid-pause waits for real // media instead of being served the pre-pause group as if it were live diff --git a/rs/moq-boy/src/video.rs b/rs/moq-boy/src/video.rs index d28f539790f..18534a87a54 100644 --- a/rs/moq-boy/src/video.rs +++ b/rs/moq-boy/src/video.rs @@ -85,7 +85,7 @@ impl VideoEncoder { } } - /// Publish an empty group marking a pause, so the PTS jump on resume reads as a break + /// Publish a marker group marking a pause, so the PTS jump on resume reads as a break /// rather than one very long frame, and a viewer arriving mid-pause isn't served the /// pre-pause group as if it were live. /// diff --git a/rs/moq-mux/src/codec/aac/import.rs b/rs/moq-mux/src/codec/aac/import.rs index 9dcfb434e6b..76bf3e62c09 100644 --- a/rs/moq-mux/src/codec/aac/import.rs +++ b/rs/moq-mux/src/codec/aac/import.rs @@ -88,7 +88,7 @@ impl Import { Ok(()) } - /// Mark a break in the timeline by publishing an empty group. To bound the closing + /// Mark a break in the timeline by publishing a marker group. To bound the closing /// group's final frame first, [`cut(end)`](Self::cut) before this. See /// [`Producer::discontinuity`](crate::container::Producer::discontinuity). pub fn discontinuity(&mut self) -> crate::Result<()> { diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index 98e8b51119d..983317e071e 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -147,7 +147,7 @@ impl Import { Ok(()) } - /// Mark a break in the timeline by publishing an empty group. To bound the closing + /// Mark a break in the timeline by publishing a marker group. To bound the closing /// group's final frame first, [`cut(end)`](Self::cut) before this. See /// [`Producer::discontinuity`](crate::container::Producer::discontinuity). pub fn discontinuity(&mut self) -> Result<()> { diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index af0bc61383f..49d93b771d5 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -151,7 +151,7 @@ impl Import { Ok(()) } - /// Mark a break in the timeline by publishing an empty group. To bound the closing + /// Mark a break in the timeline by publishing a marker group. To bound the closing /// group's final frame first, [`cut(end)`](Self::cut) before this. See /// [`Producer::discontinuity`](crate::container::Producer::discontinuity). pub fn discontinuity(&mut self) -> Result<()> { diff --git a/rs/moq-mux/src/codec/legacy.rs b/rs/moq-mux/src/codec/legacy.rs index e64db958291..343797501ad 100644 --- a/rs/moq-mux/src/codec/legacy.rs +++ b/rs/moq-mux/src/codec/legacy.rs @@ -195,7 +195,7 @@ impl Import { Ok(()) } - /// Mark a break in the timeline by publishing an empty group. To bound the closing + /// Mark a break in the timeline by publishing a marker group. To bound the closing /// group's final frame first, [`cut(end)`](Self::cut) before this. See /// [`Producer::discontinuity`](crate::container::Producer::discontinuity). pub fn discontinuity(&mut self) -> crate::Result<()> { diff --git a/rs/moq-mux/src/codec/opus/import.rs b/rs/moq-mux/src/codec/opus/import.rs index 5ac2e34a8d6..3cf30c788cc 100644 --- a/rs/moq-mux/src/codec/opus/import.rs +++ b/rs/moq-mux/src/codec/opus/import.rs @@ -83,7 +83,7 @@ impl Import { Ok(()) } - /// Mark a break in the timeline by publishing an empty group. To bound the closing + /// Mark a break in the timeline by publishing a marker group. To bound the closing /// group's final frame first, [`cut(end)`](Self::cut) before this. See /// [`Producer::discontinuity`](crate::container::Producer::discontinuity). pub fn discontinuity(&mut self) -> crate::Result<()> { diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 73a6a020c68..509da6ad6bb 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -230,7 +230,10 @@ impl Consumer { } Poll::Ready(Ok(Some(Event::FrameEnd(end)))) => { let seq = group.group.sequence; - if self.live_edge.is_none_or(|(_, high)| end.as_micros() > high.as_micros()) { + if self + .live_edge + .is_none_or(|(_, high)| end.as_micros() > high.as_micros()) + { self.live_edge = Some((seq, end)); } return Poll::Ready(Ok(Some(Event::FrameEnd(end)))); @@ -375,12 +378,10 @@ impl Consumer { && front_sequence > self.current { let _ = self.pending.front_mut().unwrap().buffer_all(waiter, &self.format); - let next_start = self.pending.front().and_then(|group| { - group - .min_timestamp - .map(std::time::Duration::from) - .or(group.max_end) - }); + let next_start = self + .pending + .front() + .and_then(|group| group.min_timestamp.map(std::time::Duration::from).or(group.max_end)); if let Some(start) = next_start && !pts_contiguous(current_end.or(self.presented_end), start) { @@ -2480,10 +2481,14 @@ mod tests { track.subscribe(moq_net::track::Subscription::default().with_max_age(Duration::from_secs(10))), DurationWire, ); - let mut group0 = track.create_group(moq_net::group::Info { sequence: 1_000_000 }).unwrap(); + let mut group0 = track + .create_group(moq_net::group::Info { sequence: 1_000_000 }) + .unwrap(); write_duration_frame(&mut group0, ts(0), ts(33_000)); group0.finish().unwrap(); - let mut group1 = track.create_group(moq_net::group::Info { sequence: 1_090_000 }).unwrap(); + let mut group1 = track + .create_group(moq_net::group::Info { sequence: 1_090_000 }) + .unwrap(); write_duration_frame(&mut group1, ts(33_000), ts(33_000)); group1.finish().unwrap(); track.finish().unwrap(); diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index 0a9a0e15234..9439eee5fd1 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -820,7 +820,11 @@ mod tests { let groups = collect_payloads(consumer).await; assert_eq!(groups.len(), 3); assert_eq!(groups[0], vec![(0, 2), (10_000, 2)]); - assert_eq!(groups[1], vec![(10_000, 0)], "the marker is one empty frame at the exclusive end"); + assert_eq!( + groups[1], + vec![(10_000, 0)], + "the marker is one empty frame at the exclusive end" + ); assert_eq!(groups[2][0], (2_405_070_000, 2)); } diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 694d9af4698..4b13f4aedc6 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -2237,7 +2237,10 @@ async fn discontinuity_re_emits_tables_and_resumes_the_clock() { } // Tables come back on cadence rather than waiting for the old 10-minute clock. - assert!(count_pid(&marked[resume..], 0x0000) >= 1, "PAT re-emitted after the marker"); + assert!( + count_pid(&marked[resume..], 0x0000) >= 1, + "PAT re-emitted after the marker" + ); assert_eq!(count_discontinuity(&before), 0); assert_eq!(count_discontinuity(&marked), 1, "the break is flagged exactly once"); } diff --git a/rs/moq-srt/src/server.rs b/rs/moq-srt/src/server.rs index 6cd9f8578e0..38c33b0c326 100644 --- a/rs/moq-srt/src/server.rs +++ b/rs/moq-srt/src/server.rs @@ -797,9 +797,9 @@ mod tests { assert_eq!(chunks[1].0, start + 2 * SLOT, "the new generation is the live edge"); } - /// End to end over a real SRT receiver: a publisher that rewinds ten minutes must - /// keep the program flowing, with the new generation stamped at the wall clock it - /// was muxed at rather than back at the connection's first packet. + /// End to end over a real SRT receiver: a publisher that declares a marker after + /// ten minutes must keep the program flowing, with the new generation stamped at + /// the wall clock it was muxed at rather than back at the connection's first packet. /// /// The receiver's TSBPD releases each message at the origin instant the sender /// stamped, so what it observes is exactly the pacing decision under test: with a @@ -807,7 +807,7 @@ mod tests { /// already a latency in the past by the time it is sent, so it is released at once /// (or dropped as too late) instead of on the media clock. #[tokio::test] - async fn a_publisher_rewind_keeps_an_srt_receiver_playing() { + async fn a_publisher_marker_keeps_an_srt_receiver_playing() { use moq_mux::catalog::hang::Container as MuxContainer; use moq_mux::container::{Producer, ts}; use moq_net::Timestamp; @@ -842,7 +842,7 @@ mod tests { let mut producer = Producer::new(track, MuxContainer::Legacy(moq_mux::container::Kind::Audio)); // 100ms audio frames in one-second groups. - let mut write = |count: u64, offset: u64| { + fn write(producer: &mut Producer, count: u64, offset: u64) { for i in 0..count { producer .write(Frame { @@ -856,7 +856,7 @@ mod tests { producer.cut(None).unwrap(); } } - }; + } let probe = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap(); let addr: SocketAddr = probe.local_addr().unwrap(); @@ -878,8 +878,8 @@ mod tests { let mut receiver = caller.await.unwrap(); // Three seconds of program, then wait until the receiver is actually playing it - // so the rewind lands a real wall-clock distance after the first packet. - write(30, OFFSET); + // so the marker lands a real wall-clock distance after the first packet. + write(&mut producer, 30, OFFSET); let mut received = Vec::new(); let first = tokio::time::timeout(Duration::from_secs(10), receiver.next()) .await @@ -888,16 +888,17 @@ mod tests { .unwrap(); received.push(first); - // The publisher restarts at the top of its source. - write(30, 0); + // The publisher declares a break and continues forward. + producer.discontinuity().unwrap(); + write(&mut producer, 30, OFFSET + 3_000_000); // Drain until the muxer flags the break, which it does exactly once, on the new // generation's leading clock packet. let boundary = loop { let payload = tokio::time::timeout(Duration::from_secs(10), receiver.next()) .await - .expect("the rewound generation never arrived") - .expect("the SRT egress closed before the rewind") + .expect("the resumed generation never arrived") + .expect("the SRT egress closed before the marker") .unwrap(); received.push(payload); if let Some(index) = received.iter().position(|(_, payload)| flags_a_break(payload)) { @@ -908,7 +909,7 @@ mod tests { assert!(boundary > 0, "the break cannot be in the connection's first payload"); assert!( received[boundary].0 > received[0].0 + Duration::from_millis(100), - "the rewound generation must be stamped at the wall clock it was muxed at, \ + "the new generation must be stamped at the wall clock it was muxed at, \ not back at the first packet ({:?} after it)", received[boundary].0.saturating_duration_since(received[0].0), ); diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index 4d0dac07f9e..30e067b945c 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -439,7 +439,7 @@ mod tests { let subscriber = broadcast.consume(); let mut producer = moq_mux::container::Producer::new( track, - moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Data), + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), ); producer .write(moq_mux::container::Frame { @@ -472,6 +472,7 @@ mod tests { "video", Config { kind: Kind::Named(probe::BUFFERED_NAME.into()), + max_age: std::time::Duration::from_secs(10), ..Config::new() }, ) diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index 1967502eebf..9c29b35c3eb 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -661,7 +661,7 @@ mod tests { } /// An on-demand capture resumes on the same wall clock after releasing its camera and encoder, - /// so the idle transition must publish an empty group between the two runs. This uses synthetic + /// so the idle transition must publish a marker group between the two runs. This uses synthetic /// frames and the software encoder to exercise the transition without capture hardware. #[tokio::test] async fn idle_capture_publishes_a_discontinuity_before_resume() { @@ -695,7 +695,7 @@ mod tests { } producer.finish().unwrap(); - assert_eq!(collect_groups(consumer).await, vec![1, 0, 1]); + assert_eq!(collect_groups(consumer).await, vec![1, 1, 1]); } #[tokio::test] From 730bd1c074bf308d4f93329c3e08d652f8ea0928 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 17 Sep 2026 04:19:47 -0700 Subject: [PATCH 4/4] fix(hang): keep marker skips and below-edge frames honest A zero-budget skip now bumps the playhead when it drains a marker, even if the resume looks contiguous. A later frame below the previous group's edge aborts as a rewind. A playhead event reapplies Opus pre-skip without flushing decoder prediction. Co-Authored-By: grok-4.6 --- js/hang/src/container/consumer.test.ts | 25 ++++++++++ js/hang/src/container/consumer.ts | 3 +- rs/moq-audio/src/decode/consumer.rs | 65 +++++++++++++++++++++++++ rs/moq-audio/src/decode/decoder.rs | 7 ++- rs/moq-mux/src/container/consumer.rs | 66 +++++++++++++++++++++++++- 5 files changed, 163 insertions(+), 3 deletions(-) diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index fdcb5e9cb56..b18c71ed1c3 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -1340,6 +1340,31 @@ test("Consumer jumps the playhead after a shed marker with a timestamp hole", as consumer.close(); }); +test("Consumer zero-budget skip keeps a contiguous marker", async () => { + const track = new Track.Producer("test"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxAge: 0 as Time.Milli }); + + const group0 = new Group.Producer(0); + track.writeGroup(group0); + group0.writeFrame({ + payload: encodeLegacy(0 as Time.Micro), + timestamp: Time.Timestamp.now(), + }); + await settle(); + expect((await nextFrame(consumer))?.frame?.timestamp).toBe(0 as Time.Micro); + + writeMarkerGroup(track, 1, 0 as Time.Micro); + writeGroupWithLegacyFrames(track, 2, [500 as Time.Micro]); + await settle(); + + const resumed = await nextFrame(consumer); + expect(resumed?.frame?.timestamp).toBe(500 as Time.Micro); + expect(resumed?.discontinuity).toBe(1); + group0.close(); + + consumer.close(); +}); + // The case the group-number heuristic got backwards: ids jump, but the PTS timeline is unbroken. test("Consumer reports continuity across a PTS-contiguous group id jump (CMAF)", async () => { const track = new Track.Producer("test"); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 0fd36e7142b..47b37c98a41 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -338,7 +338,8 @@ export class Consumer { ); const nextStart = this.#groups[0]?.frames.at(0)?.timestamp ?? this.#groups[0]?.end; - if (!ptsContiguous(first.end ?? this.#presentedEnd, nextStart)) { + const marker = !first.empty && !first.media; + if (marker || !ptsContiguous(first.end ?? this.#presentedEnd, nextStart)) { hole = true; } first.consumer.close(); diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 1833644b774..89d38f6cd72 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -314,6 +314,9 @@ impl Consumer { self.frames_decoded = 0; self.end = None; self.terminal_start = None; + self.epoch = None; + self.delay_trimmed = 0; + self.decoder.reapply_delay(); Ok(()) } @@ -1125,6 +1128,68 @@ mod tests { assert_eq!(second.timestamp, expected); } + #[tokio::test] + async fn a_playhead_event_reapplies_opus_pre_skip() { + let input = Input { + format: Format::F32, + sample_rate: 48_000, + channels: 1, + }; + let mut encoder = Encoder::new(&crate::encode::Config::new(input)).unwrap(); + let catalog = encoder.catalog(); + let frame_size = encoder.frame_size(); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let subscriber = broadcast.consume(); + let mut producer = moq_mux::container::Producer::new( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), + ); + let mut consumer = Consumer::new( + &subscriber, + &catalog, + "audio", + Config { + max_age: std::time::Duration::from_secs(1), + ..Config::new() + }, + ) + .await + .unwrap(); + + let pcm = vec![0.25f32; frame_size]; + let write = |producer: &mut moq_mux::container::Producer<_>, packet: u64, payload: bytes::Bytes| { + producer + .write(moq_mux::container::Frame { + timestamp: Timestamp::from_scale(packet * frame_size as u64, 48_000).unwrap(), + duration: None, + payload, + keyframe: true, + }) + .unwrap(); + producer.cut(None).unwrap(); + }; + write(&mut producer, 0, encoder.encode(&pcm).unwrap().payload); + write(&mut producer, 1, encoder.encode(&pcm).unwrap().payload); + producer.discontinuity().unwrap(); + write(&mut producer, 2, encoder.encode(&pcm).unwrap().payload); + producer.finish().unwrap(); + + let first = consumer.read().await.unwrap().expect("first decoded frame"); + let _second = consumer.read().await.unwrap().expect("second decoded frame"); + let resumed = consumer.read().await.unwrap().expect("resumed decoded frame"); + let first_frames = first.data.len() / size_of::(); + let resumed_frames = resumed.data.len() / size_of::(); + assert!(first_frames < frame_size, "the first epoch trims pre-skip"); + assert_eq!( + resumed_frames, first_frames, + "a playhead event reapplies pre-skip without flushing the decoder" + ); + } + #[tokio::test] async fn reads_the_container_the_catalog_declares() { let mut broadcast = moq_net::broadcast::Info::new().produce(); diff --git a/rs/moq-audio/src/decode/decoder.rs b/rs/moq-audio/src/decode/decoder.rs index 5be7e16acca..a166eaf08aa 100644 --- a/rs/moq-audio/src/decode/decoder.rs +++ b/rs/moq-audio/src/decode/decoder.rs @@ -259,10 +259,15 @@ impl Decoder { /// Reset codec history and reapply startup delay for a new discontinuous epoch. pub fn reset(&mut self) -> Result<(), Error> { self.reset_prediction()?; + self.reapply_delay(); + Ok(()) + } + + /// Reapply catalog startup delay for a new playhead epoch without resetting codec prediction. + pub(super) fn reapply_delay(&mut self) { if let Backend::Opus(opus) = &mut self.backend { opus.pre_skip_remaining = self.delay; } - Ok(()) } /// Reset codec prediction after packet loss without reapplying stream startup delay. diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 509da6ad6bb..2ac54dbe43d 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -81,6 +81,10 @@ pub struct Consumer { // malformed. live_edge: Option<(u64, Timestamp)>, + // Max timestamp of groups the cursor has left. Open-GOP pictures may sit below this + // group's running max, but not below an earlier group's edge. + group_edge: Option, + // Presentation end of the group we most recently advanced past, for the 1 ms // contiguity check on a delivered hole. presented_end: Option, @@ -127,6 +131,7 @@ impl Consumer { startup: start.is_none(), max_age, live_edge: None, + group_edge: None, presented_end: None, discontinuity: 0, end: None, @@ -223,6 +228,9 @@ impl Consumer { Poll::Ready(Ok(Some(Event::Frame(frame)))) => { let seq = group.group.sequence; let ts = frame.timestamp; + if self.group_edge.is_some_and(|edge| ts.as_micros() < edge.as_micros()) { + return Poll::Ready(Err(TimestampRewind.into())); + } if self.live_edge.is_none_or(|(_, high)| ts.as_micros() > high.as_micros()) { self.live_edge = Some((seq, ts)); } @@ -269,6 +277,7 @@ impl Consumer { } self.pending.pop_front(); self.current += 1; + self.note_group_edge(); if marker { self.bump_playhead(); } @@ -335,6 +344,7 @@ impl Consumer { self.bump_playhead(); } self.current = front_sequence; + self.note_group_edge(); continue; } @@ -361,8 +371,9 @@ impl Consumer { && should_skip { let hole = !pts_contiguous(current_end.or(self.presented_end), next_start); + let had_marker = self.pending.iter().take(new_idx).any(GroupBuffer::marker); self.pending.drain(0..new_idx); - if hole { + if hole || had_marker { self.bump_playhead(); } let new_current = self.pending.front().map(|g| g.sequence).unwrap(); @@ -370,6 +381,7 @@ impl Consumer { tracing::debug!(old = self.current, new = new_current, "skipping slow groups"); self.current = new_current; + self.note_group_edge(); continue; } @@ -388,6 +400,7 @@ impl Consumer { self.bump_playhead(); } self.current = front_sequence; + self.note_group_edge(); continue; } @@ -404,6 +417,12 @@ impl Consumer { self.end = None; } + fn note_group_edge(&mut self) { + if let Some((_, ts)) = self.live_edge { + self.group_edge = Some(ts); + } + } + // Reads any new groups from the track until we're completely finished. // // Returns Pending until all groups have been consumed. @@ -1143,6 +1162,51 @@ mod tests { ); } + #[tokio::test] + async fn a_zero_budget_skip_keeps_a_contiguous_marker() { + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Audio)); + consumer.max_age = Duration::ZERO; + + let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + Container::Legacy(crate::container::Kind::Audio) + .write( + &mut group0, + &[Frame { + timestamp: ts(0), + payload: Bytes::from_static(&[0xDE, 0xAD]), + keyframe: false, + duration: None, + }], + ) + .unwrap(); + assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); + + write_marker_group(&mut track, 1, ts(10_000)); + write_group(&mut track, 2, &[ts(10_000)]); + + assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(10_000)); + assert_eq!( + consumer.discontinuity(), + 1, + "a drained marker still declares the encoder restart" + ); + group0.finish().unwrap(); + } + + #[tokio::test] + async fn a_later_frame_below_the_previous_group_edge_aborts() { + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Data)); + write_group(&mut track, 0, &[ts(100_000)]); + assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(100_000)); + write_group(&mut track, 1, &[ts(200_000), ts(50_000)]); + track.finish().unwrap(); + assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(200_000)); + let err = consumer.read().await.unwrap_err(); + assert!(matches!(err, crate::Error::TimestampRewind(_))); + } + // ---- Empty payloads ---- /// Write one frame with an empty payload: a marker saying content stops at