diff --git a/doc/lib/rs/moq-mux.md b/doc/lib/rs/moq-mux.md index 9388626c0b1..82255ac22ce 100644 --- a/doc/lib/rs/moq-mux.md +++ b/doc/lib/rs/moq-mux.md @@ -30,6 +30,10 @@ set at the init segment, so a new rendition or a changed configuration ends the export. The [CLI page](/bin/cli#import) covers these and the MPEG-TS specifics: one program per broadcast, damaged packets, and feed checks. +A container read yields frames plus two markers in delivery order: a gap where +media will never arrive (the join, a skip, a hole) and an end where the +publisher declared its media stops (a pause, a source switch, the finish). + Data tracks are catalog entries too. The [hang page](/concept/hang#data-tracks) describes the modes. A capture time converts onto the broadcast clock; a timestamp taken from the source is published as given and lines up with media diff --git a/js/hang/src/container/consumer.nonsequential.test.ts b/js/hang/src/container/consumer.nonsequential.test.ts index fca4c3f4a76..61a62ea97a9 100644 --- a/js/hang/src/container/consumer.nonsequential.test.ts +++ b/js/hang/src/container/consumer.nonsequential.test.ts @@ -5,6 +5,14 @@ import { Format as LegacyFormat } from "./legacy.ts"; // Standalone from consumer.test.ts so it only pulls the Legacy format, not the CMAF decoder. +/** The next frame, passing over gaps and ends. */ +async function nextFrame(consumer: Consumer) { + for (;;) { + const next = await consumer.next(); + if (!next || next.kind === "frame") return next; + } +} + function encodeLegacyFrame(timestamp: Time.Micro, payload: Uint8Array): Uint8Array { const tsBytes = Varint.encode(timestamp); const data = new Uint8Array(tsBytes.byteLength + payload.byteLength); @@ -40,10 +48,9 @@ test("Consumer waits on a non-sequential hang group because it has no per-frame a.close(); track.writeGroup(a); - // Drain A: its frame, then its group-done marker. - const firstFrame = await consumer.next(); - expect(firstFrame?.frame?.payload).toEqual(new Uint8Array([0x01])); - await consumer.next(); + // Drain A's frame. + const firstFrame = await nextFrame(consumer); + expect(firstFrame?.frame.payload).toEqual(new Uint8Array([0x01])); // Park next() before the next group's frames arrive (the live streaming case). const pending = consumer.next(); diff --git a/js/hang/src/container/consumer.outoforder.test.ts b/js/hang/src/container/consumer.outoforder.test.ts index 89453f3d1f6..2bccec79ca7 100644 --- a/js/hang/src/container/consumer.outoforder.test.ts +++ b/js/hang/src/container/consumer.outoforder.test.ts @@ -5,6 +5,14 @@ import { Format as LegacyFormat } from "./legacy.ts"; // Standalone from consumer.test.ts so it only pulls the Legacy format, not the CMAF decoder. +/** The next frame, passing over gaps and ends. */ +async function nextFrame(consumer: Consumer) { + for (;;) { + const next = await consumer.next(); + if (!next || next.kind === "frame") return next; + } +} + function encodeLegacyFrame(timestamp: Time.Micro, payload: Uint8Array): Uint8Array { const tsBytes = Varint.encode(timestamp); const data = new Uint8Array(tsBytes.byteLength + payload.byteLength); @@ -29,14 +37,16 @@ function publish(track: Track.Producer, sequence: number, millis: number) { group.close(); } -/** Every group and frame the consumer will hand over right now, as [group, payload] pairs. */ +/** Every frame the consumer will hand over right now, as [group, payload] pairs. */ async function drain(consumer: Consumer): Promise<[number, number | undefined][]> { const seen: [number, number | undefined][] = []; for (;;) { - const next = await Promise.race([consumer.next(), new Promise((resolve) => setTimeout(resolve, 50))]); + const next = await Promise.race([ + consumer.next(), + new Promise((resolve) => setTimeout(resolve, 50)), + ]); if (!next) break; - const { group, frame } = next as { group: number; frame?: { payload: Uint8Array } }; - seen.push([group, frame?.payload[0]]); + if (next.kind === "frame") seen.push([next.group, next.frame.payload[0]]); } return seen; } @@ -55,7 +65,7 @@ test("out-of-order groups are delivered rather than dropped", async () => { // The live edge lands first, and delivery starts there rather than waiting. publish(track, 3, 30); - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([3])); + expect((await nextFrame(consumer))?.frame.payload).toEqual(new Uint8Array([3])); // The window's head follows it. Every group is still within both budgets, so all three // are handed over even though each sits below the group already delivered. @@ -63,8 +73,7 @@ test("out-of-order groups are delivered rather than dropped", async () => { publish(track, 1, 10); publish(track, 2, 20); - const delivered = (await drain(consumer)).filter(([, payload]) => payload !== undefined); - expect(delivered).toEqual([ + expect(await drain(consumer)).toEqual([ [0, 0], [1, 1], [2, 2], @@ -87,25 +96,25 @@ test("a below-cursor group still downloading is not truncated", async () => { const live = new Group.Producer(3); track.writeGroup(live); publishFrame(live, 3000, 30); - expect((await consumer.next())?.frame?.payload[0]).toBe(30); + expect((await nextFrame(consumer))?.frame.payload[0]).toBe(30); // A backlog group lands below the cursor, one frame at a time (mid-download). const backlog = new Group.Producer(1); track.writeGroup(backlog); publishFrame(backlog, 1000, 10); - const first = await consumer.next(); + const first = await nextFrame(consumer); expect(first?.group).toBe(1); - expect(first?.frame?.payload[0]).toBe(10); + expect(first?.frame.payload[0]).toBe(10); // The consumer finds the group's buffer empty while it is still open. It must wait // rather than shift the group out, so the second frame still arrives. - const pending = consumer.next(); + const pending = nextFrame(consumer); publishFrame(backlog, 1500, 11); backlog.close(); - const second = (await pending) as { group: number; frame?: { payload: Uint8Array } }; - expect(second.group).toBe(1); - expect(second.frame?.payload[0]).toBe(11); + const second = await pending; + expect(second?.group).toBe(1); + expect(second?.frame.payload[0]).toBe(11); live.close(); track.close(); @@ -120,30 +129,24 @@ test("delayed older groups survive a floor established by two newer groups", asy format: new LegacyFormat("data"), maxDelay: Time.Milli(5000), }); - async function nextFrame() { - for (;;) { - const next = await consumer.next(); - if (!next || next.frame) return next; - } - } try { publish(track, 3, 30); - expect((await nextFrame())?.group).toBe(3); + expect((await nextFrame(consumer))?.group).toBe(3); const latest = new Group.Producer(4); track.writeGroup(latest); publishFrame(latest, 40, 4); - expect((await nextFrame())?.group).toBe(4); + expect((await nextFrame(consumer))?.group).toBe(4); // Group 1's 10ms frame is valid history, below group 4's 30ms floor. publish(track, 1, 10); - const backlog = await nextFrame(); + const backlog = await nextFrame(consumer); expect(backlog?.group).toBe(1); - expect(backlog?.frame?.timestamp).toBe(Time.Micro(10_000)); + expect(backlog?.frame.timestamp).toBe(Time.Micro(10_000)); // Delivering the backlog must not erase the floor for group 4 itself. publishFrame(latest, 29, 5); - await expect(nextFrame()).rejects.toThrow("below the previous group start"); + await expect(nextFrame(consumer)).rejects.toThrow("below the previous group start"); } finally { consumer.close(); track.close(); diff --git a/js/hang/src/container/consumer.rewind.test.ts b/js/hang/src/container/consumer.rewind.test.ts index c826fbe1b58..34adadab02e 100644 --- a/js/hang/src/container/consumer.rewind.test.ts +++ b/js/hang/src/container/consumer.rewind.test.ts @@ -38,7 +38,7 @@ function group(track: Track.Producer, timestamps: number[], end?: number) { async function frame(consumer: Consumer) { for (;;) { const result = await consumer.next(); - if (!result || result.frame) return result; + if (!result || result.kind === "frame") return result; } } @@ -50,7 +50,6 @@ test("Consumer accepts a resuming keyframe before the estimated group end", asyn group(track, [0, 33_000], 66_000); expect((await frame(consumer))?.frame?.timestamp).toBe(Time.Micro(0)); expect((await frame(consumer))?.frame?.timestamp).toBe(Time.Micro(33_000)); - expect((await consumer.next())?.end).toBe(Time.Micro(66_000)); group(track, [50_000]); expect((await frame(consumer))?.frame?.timestamp).toBe(Time.Micro(50_000)); } finally { diff --git a/js/hang/src/container/consumer.skips.test.ts b/js/hang/src/container/consumer.skips.test.ts index 591ab00aa1b..17447495cf3 100644 --- a/js/hang/src/container/consumer.skips.test.ts +++ b/js/hang/src/container/consumer.skips.test.ts @@ -34,7 +34,8 @@ for (const count of [1, 16, 256]) { expect(warn).toHaveBeenLastCalledWith(`skipping slow groups: track=tone 0 -> ${count * 2} count=${count}`); expect(consumer.buffered.peek()[0]?.start).toBe(Time.Milli(count)); // The retained group is still delivered, and another catch-up reports separately. - expect((await consumer.next())?.group).toBe(count * 2); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(await consumer.next()).toMatchObject({ kind: "frame", group: count * 2 }); newest.writeFrame({ payload: Varint.encode(count * 1000), timestamp: Time.Timestamp.now() }); write(count * 2 + 2, (count + 1) * 1000); await buffered(count + 1); diff --git a/js/hang/src/container/consumer.startup.test.ts b/js/hang/src/container/consumer.startup.test.ts index 008116a911f..5d8ecfacd8f 100644 --- a/js/hang/src/container/consumer.startup.test.ts +++ b/js/hang/src/container/consumer.startup.test.ts @@ -1,6 +1,6 @@ import { expect, jest, onTestFinished, spyOn, test } from "bun:test"; import { Group, Error as NetError, StreamCode, Time, Track } from "@moq/net"; -import { Consumer } from "./consumer.ts"; +import { Consumer, type Read } from "./consumer.ts"; import { encodeFrame, Format } from "./legacy.ts"; // Drain the in-memory readers and their continuations without advancing mocked time. @@ -13,7 +13,7 @@ function setup(maxDelay = 100) { format: new Format("video"), maxDelay: Time.Milli(maxDelay), }); - const results: NonNullable>>[] = []; + const results: Read[] = []; const warn = spyOn(console, "warn").mockImplementation(() => {}); const reading = (async () => { for (;;) { @@ -35,6 +35,19 @@ function setup(maxDelay = 100) { return { track, consumer, results, warn }; } +// Each item as "gap", "@<µs>" for a frame, or "end @<µs>". +function items(results: Read[]): string[] { + return results.map((read) => { + if (read.kind === "gap") return "gap"; + if (read.kind === "end") return `end ${read.group}@${read.end}`; + return `${read.group}@${read.frame.timestamp}`; + }); +} + +function frames(results: Read[]) { + return results.flatMap((read) => (read.kind === "frame" ? [read] : [])); +} + function write(group: Group.Producer, timestamp: number, marker = false) { group.writeFrame({ payload: encodeFrame(marker ? new Uint8Array() : new Uint8Array([1]), Time.Micro(timestamp)), @@ -46,7 +59,7 @@ for (const emptySuccessor of [false, true]) { test.each([0, 100])( `Consumer starts across a hole after an empty reset (empty successor: ${emptySuccessor}, maxDelay: %s)`, async (maxDelay) => { - const { track, consumer, results, warn } = setup(maxDelay); + const { track, results, warn } = setup(maxDelay); const stale = new Group.Producer(16); track.writeGroup(stale); await flush(); @@ -58,8 +71,7 @@ for (const emptySuccessor of [false, true]) { } stale.close(new NetError.Stream(StreamCode.Old)); await flush(); - expect(results.some((result) => result.group === 16 && !result.frame)).toBe(true); - if (emptySuccessor) expect(results.some((result) => result.group === 17 && !result.frame)).toBe(true); + expect(results).toEqual([]); const sequence = emptySuccessor ? 19 : 18; const live = new Group.Producer(sequence); @@ -67,36 +79,34 @@ for (const emptySuccessor of [false, true]) { for (let i = 0; i < 5; i++) write(live, 1_000_000 + i * 33_333); await flush(); - const frames = results.filter((result) => result.frame); - expect(frames.map((result) => result.group)).toEqual(Array(5).fill(sequence)); - expect(frames.map((result) => result.frame?.timestamp)).toEqual( - [1_000_000, 1_033_333, 1_066_666, 1_099_999, 1_133_332].map((timestamp) => Time.Micro(timestamp)), - ); - expect(frames.map((result) => result.frame?.keyframe)).toEqual([true, false, false, false, false]); - expect(frames.map((result) => result.continuous)).toEqual([false, true, true, true, true]); - expect(consumer.discontinuity).toBe(0); + expect(items(results)).toEqual([ + "gap", + ...[1_000_000, 1_033_333, 1_066_666, 1_099_999, 1_133_332].map( + (timestamp) => `${sequence}@${timestamp}`, + ), + ]); + expect(frames(results).map((read) => read.frame.keyframe)).toEqual([true, false, false, false, false]); expect(warn).not.toHaveBeenCalled(); }, ); } test("Consumer preserves a declared marker before its first media across a hole", async () => { - const { track, consumer, results, warn } = setup(); + const { track, results, warn } = setup(); const marker = new Group.Producer(16); track.writeGroup(marker); write(marker, 0, true); marker.close(); await flush(); - expect(consumer.discontinuity).toBe(1); + expect(items(results)).toEqual(["gap", "end 16@0"]); const live = new Group.Producer(18); track.writeGroup(live); write(live, 50_000); await flush(); - const frames = results.filter((result) => result.frame); - expect(frames).toHaveLength(1); - expect(frames[0]).toMatchObject({ group: 18, discontinuity: 1, continuous: false }); - expect(frames[0].frame?.keyframe).toBe(true); + // Group 17 never arrived, so the media after the marker is past a gap. + expect(items(results)).toEqual(["gap", "end 16@0", "gap", "18@50000"]); + expect(frames(results)[0].frame.keyframe).toBe(true); expect(warn).not.toHaveBeenCalled(); }); @@ -107,7 +117,7 @@ test.each([false, true])("Consumer still waits for a gap after media (playhead r write(first, 0); first.close(); await flush(); - expect(results.filter((result) => result.frame)).toHaveLength(1); + expect(items(results)).toEqual(["gap", "16@0"]); if (reset) { const marker = new Group.Producer(17); @@ -115,7 +125,7 @@ test.each([false, true])("Consumer still waits for a gap after media (playhead r write(marker, 0, true); marker.close(); await flush(); - expect(consumer.discontinuity).toBe(1); + expect(items(results)).toEqual(["gap", "16@0", "end 17@0"]); } const sequence = reset ? 20 : 19; const later = new Group.Producer(sequence); @@ -123,7 +133,7 @@ test.each([false, true])("Consumer still waits for a gap after media (playhead r write(later, 50_000); await flush(); expect(consumer.buffered.peek().length).toBeGreaterThan(0); - expect(results.filter((result) => result.frame)).toHaveLength(1); + expect(frames(results)).toHaveLength(1); // Supply the missing, timestamp-contiguous bridge and prove the queued frame survives. const bridge = new Group.Producer(sequence - 1); @@ -131,11 +141,13 @@ test.each([false, true])("Consumer still waits for a gap after media (playhead r write(bridge, 0); bridge.close(); await flush(); - expect(results.filter((result) => result.frame).map((result) => result.group)).toEqual([ - 16, - sequence - 1, - sequence, + // The bridge closes the hole, so nothing reads as a gap past the first. + expect(items(results)).toEqual([ + "gap", + "16@0", + ...(reset ? ["end 17@0"] : []), + `${sequence - 1}@0`, + `${sequence}@50000`, ]); - expect(consumer.discontinuity).toBe(reset ? 1 : 0); expect(warn).not.toHaveBeenCalled(); }); diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index 0443a095d38..8d9a9f8f4c1 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -5,7 +5,7 @@ import { AudioConfigSchema } from "../catalog/audio.ts"; import { decodeInitSegment, type InitSegment } from "./cmaf/decode.ts"; import { createAudioInitSegment, encodeDataSegment } from "./cmaf/encode.ts"; import { Format as CmafFormat } from "./cmaf/format.ts"; -import { Consumer } from "./consumer.ts"; +import { Consumer, type Read } from "./consumer.ts"; import type { Format as ContainerFormat } from "./format.ts"; import { Format as LegacyFormat, Producer as LegacyProducer } from "./legacy.ts"; import type { Frame } from "./types.ts"; @@ -522,13 +522,27 @@ async function drainFrames( new Promise((resolve) => setTimeout(() => resolve(null), timeout)), ]); if (result === null || result === undefined) break; - if (result.frame) { + if (result.kind === "frame") { frames.push({ timestamp: result.frame.timestamp, group: result.group, keyframe: result.frame.keyframe }); } } return frames; } +/** An item as "gap", a frame's µs, or "end <µs>". */ +function describe(read: Read): string { + if (read.kind === "gap") return "gap"; + if (read.kind === "end") return `end ${read.end}`; + return `${read.frame.timestamp}`; +} + +/** The frame a read carries, failing on anything else. */ +function frameOf(read: Read | string | undefined): Frame { + if (typeof read !== "object" || read.kind !== "frame") + throw new Error(`expected a frame, got ${JSON.stringify(read)}`); + return read.frame; +} + test("Consumer delivers frames from a single group", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 500 as Time.Milli }); @@ -731,7 +745,7 @@ test("Consumer delivers a group that arrives below the cursor", async () => { // --- Group.Producer boundary signals --- -test("Consumer next() returns group-done signals", async () => { +test("Consumer next() yields a gap before the first frame only", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 500 as Time.Milli }); @@ -741,26 +755,21 @@ test("Consumer next() returns group-done signals", async () => { await new Promise((resolve) => setTimeout(resolve, 50)); - const allResults: { frame: boolean; group: number }[] = []; + const seen: string[] = []; for (;;) { const result = await Promise.race([ consumer.next(), new Promise((resolve) => setTimeout(() => resolve(null), 500)), ]); if (result === null || result === undefined) break; - allResults.push({ frame: result.frame !== undefined, group: result.group }); + seen.push(describe(result)); } - const frameResults = allResults.filter((r) => r.frame); - const boundaries = allResults.filter((r) => !r.frame); - expect(frameResults).toHaveLength(3); - expect(boundaries).toHaveLength(2); - expect(boundaries[0].group).toBe(0); - expect(boundaries[1].group).toBe(1); + expect(seen).toEqual(["gap", "0", "33000", "66000"]); consumer.close(); }); -test("Consumer reports a duration marker as metadata", async () => { +test("Consumer keeps a trailing duration marker internal", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe(), { format: new LegacyFormat("video"), @@ -781,18 +790,17 @@ test("Consumer reports a duration marker as metadata", async () => { track.close(); await settle(); - const media = await consumer.next(); - expect(media?.frame?.payload).toEqual(new Uint8Array([0xde, 0xad])); - expect(media?.frame?.keyframe).toBe(true); - expect(media?.frame?.duration).toBeUndefined(); - - const done = await consumer.next(); - expect(done?.frame).toBeUndefined(); - expect(done?.end).toBe(33_000 as Time.Micro); + expect(await consumer.next()).toEqual({ kind: "gap" }); + const media = frameOf(await consumer.next()); + expect(media.payload).toEqual(new Uint8Array([0xde, 0xad])); + expect(media.keyframe).toBe(true); + expect(media.duration).toBeUndefined(); + // The marker is the frame's duration, not an item. + expect(await consumer.next()).toBeUndefined(); consumer.close(); }); -test("Consumer skips a leading marker and keeps the first media keyframe", async () => { +test("Consumer reads a leading marker as an end and keeps the first media keyframe", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe(), { format: new LegacyFormat("video"), @@ -812,17 +820,16 @@ test("Consumer skips a leading marker and keeps the first media keyframe", async track.writeGroup(group); track.close(); - const marker = await consumer.next(); - expect(marker?.end).toBe(20_000 as Time.Micro); - expect(marker?.frame).toBeUndefined(); - const media = await consumer.next(); - expect(media?.frame?.payload).toEqual(new Uint8Array([0xde, 0xad])); - expect(media?.frame?.keyframe).toBe(true); - expect(media?.frame?.duration).toBeUndefined(); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(await consumer.next()).toEqual({ kind: "end", group: 0, end: 20_000 as Time.Micro }); + const media = frameOf(await consumer.next()); + expect(media.payload).toEqual(new Uint8Array([0xde, 0xad])); + expect(media.keyframe).toBe(true); + expect(media.duration).toBeUndefined(); consumer.close(); }); -test("Consumer skips an empty LOC payload", async () => { +test("Consumer reads an empty LOC payload as the frame's duration", async () => { const track = new Track.Producer("test"); const producer = new LocProducer(track); producer.encode(new Uint8Array([0xde, 0xad]), 0 as Time.Micro, true); @@ -831,16 +838,15 @@ test("Consumer skips an empty LOC payload", async () => { const consumer = new Consumer(replay(track), { format: new LocFormat("video"), maxDelay: 500 as Time.Milli }); await settle(); - const media = await consumer.next(); - expect(media?.frame?.payload).toEqual(new Uint8Array([0xde, 0xad])); - expect(media?.frame?.duration).toBeUndefined(); - const marker = await consumer.next(); - expect(marker?.end).toBe(33_000 as Time.Micro); - expect(marker?.frame).toBeUndefined(); + expect(await consumer.next()).toEqual({ kind: "gap" }); + const media = frameOf(await consumer.next()); + expect(media.payload).toEqual(new Uint8Array([0xde, 0xad])); + expect(media.duration).toBeUndefined(); + expect(await consumer.next()).toBeUndefined(); consumer.close(); }); -test("Consumer skips the duration marker the LOC producer writes", async () => { +test("Consumer passes over the duration markers the LOC producer writes", async () => { const track = new Track.Producer("test"); const producer = new LocProducer(track); producer.encode(new Uint8Array([0xde, 0xad]), 0 as Time.Micro, true); @@ -850,23 +856,15 @@ test("Consumer skips the duration marker the LOC producer writes", async () => { const consumer = new Consumer(replay(track), { format: new LocFormat("video"), maxDelay: 500 as Time.Milli }); await settle(); - const first = await consumer.next(); - expect(first?.frame?.payload).toEqual(new Uint8Array([0xde, 0xad])); - expect(first?.frame?.keyframe).toBe(true); - const second = await consumer.next(); - expect(second?.frame?.payload).toEqual(new Uint8Array([0xbe, 0xef])); - const boundary = await consumer.next(); - expect(boundary?.frame).toBeUndefined(); - expect(boundary?.end).toBe(20_000 as Time.Micro); - const keyframe = await consumer.next(); - expect(keyframe?.frame).toBeUndefined(); - expect(keyframe?.end).toBeUndefined(); - const next = await consumer.next(); - expect(next?.frame?.payload).toEqual(new Uint8Array([0xca, 0xfe])); - expect(next?.frame?.keyframe).toBe(true); - const tail = await consumer.next(); - expect(tail?.frame).toBeUndefined(); - expect(tail?.end).toBe(30_000 as Time.Micro); + expect(await consumer.next()).toEqual({ kind: "gap" }); + const first = frameOf(await consumer.next()); + expect(first.payload).toEqual(new Uint8Array([0xde, 0xad])); + expect(first.keyframe).toBe(true); + expect(frameOf(await consumer.next()).payload).toEqual(new Uint8Array([0xbe, 0xef])); + const next = frameOf(await consumer.next()); + expect(next.payload).toEqual(new Uint8Array([0xca, 0xfe])); + expect(next.keyframe).toBe(true); + expect(await consumer.next()).toBeUndefined(); consumer.close(); }); @@ -879,21 +877,21 @@ test("Consumer preserves empty Legacy and LOC data frames", async () => { producer.close(); const format = kind === "legacy" ? new LegacyFormat("data") : new LocFormat(); const consumer = new Consumer(replay(track), { format, maxDelay: 500 as Time.Milli }); - const result = await consumer.next(); - expect(result?.frame?.payload).toEqual(new Uint8Array()); - expect(result?.frame?.keyframe).toBe(true); - expect(result?.end).toBeUndefined(); + expect(await consumer.next()).toEqual({ kind: "gap" }); + const frame = frameOf(await consumer.next()); + expect(frame.payload).toEqual(new Uint8Array()); + expect(frame.keyframe).toBe(true); consumer.close(); } }); // --- Malformed rewind --- -/** Read until the next media frame, skipping the group-done markers in between. */ +/** Read until the next media frame, passing over gaps and ends. */ async function nextFrame(consumer: Consumer) { for (;;) { const result = await consumer.next(); - if (!result || result.frame) return result; + if (!result || result.kind === "frame") return result; } } @@ -918,7 +916,6 @@ test("Consumer aborts a rewind carried by a later arrival", async () => { writeGroupWithLegacyFrames(track, 0, [10_000_000 as Time.Micro]); await settle(); expect((await nextFrame(consumer))?.frame?.timestamp).toBe(10_000_000 as Time.Micro); - expect((await consumer.next())?.frame).toBeUndefined(); const pending = nextFrame(consumer); await settle(); @@ -937,20 +934,17 @@ test("Consumer treats B-frame reordering within a group as continuous", async () writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro, 66_000 as Time.Micro, 33_000 as Time.Micro]); await settle(); + expect(await consumer.next()).toEqual({ kind: "gap" }); for (const timestamp of [0, 66_000, 33_000]) { - const result = await nextFrame(consumer); - expect(result?.frame?.timestamp).toBe(timestamp as Time.Micro); - expect(result?.discontinuity).toBe(0); + expect(frameOf(await consumer.next()).timestamp).toBe(timestamp as Time.Micro); } // The cursor is now on group 1, whose own frames reorder the same way. writeGroupWithLegacyFrames(track, 1, [100_000 as Time.Micro, 166_000 as Time.Micro, 133_000 as Time.Micro]); await settle(); for (const timestamp of [100_000, 166_000, 133_000]) { - const result = await nextFrame(consumer); - expect(result?.frame?.timestamp).toBe(timestamp as Time.Micro); - expect(result?.discontinuity).toBe(0); - expect(result?.continuous).toBe(true); + // Read directly: a gap here would fail the frame check. + expect(frameOf(await consumer.next()).timestamp).toBe(timestamp as Time.Micro); } consumer.close(); @@ -963,15 +957,108 @@ test("Consumer accepts open-GOP leading pictures above the previous group", asyn writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro, 33_000 as Time.Micro]); writeGroupWithLegacyFrames(track, 1, [66_000 as Time.Micro, 50_000 as Time.Micro]); await settle(); + expect(await consumer.next()).toEqual({ kind: "gap" }); for (const timestamp of [0, 33_000, 66_000, 50_000]) { - const result = await nextFrame(consumer); - expect(result?.frame?.timestamp).toBe(timestamp as Time.Micro); - expect(result?.discontinuity).toBe(0); + expect(frameOf(await consumer.next()).timestamp).toBe(timestamp as Time.Micro); } consumer.close(); }); +// A delivered group still awaiting its done result holds no unread media, so retiring it for +// the max delay loses nothing. +test("Consumer reads no gap when retiring a fully delivered group", async () => { + const track = new Track.Producer("test"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 150 as Time.Milli }); + + writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); + await settle(); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + + writeGroupWithLegacyFrames(track, 1, [100_000 as Time.Micro]); + writeGroupWithLegacyFrames(track, 2, [200_000 as Time.Micro]); + await settle(); + + // Read directly: a gap here would fail the frame check. + expect(frameOf(await consumer.next()).timestamp).toBe(100_000 as Time.Micro); + + consumer.close(); +}); + +// An unread trailing endpoint carries no media, so retiring its group loses nothing. +test("Consumer reads no gap when retiring a group holding only its endpoint", async () => { + const track = new Track.Producer("test"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("video"), maxDelay: 150 as Time.Milli }); + + const group = new Group.Producer(0); + group.writeFrame({ payload: encodeLegacy(0 as Time.Micro), timestamp: Time.Timestamp.now() }); + group.writeFrame({ + payload: encodeLegacyFrame(33_000 as Time.Micro, new Uint8Array()), + timestamp: Time.Timestamp.now(), + }); + group.close(); + track.writeGroup(group); + await settle(); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + + writeGroupWithLegacyFrames(track, 1, [100_000 as Time.Micro]); + writeGroupWithLegacyFrames(track, 2, [200_000 as Time.Micro]); + await settle(); + + expect(frameOf(await consumer.next()).timestamp).toBe(100_000 as Time.Micro); + + consumer.close(); +}); + +// A truncated group lost its tail, so retiring it for the max delay is a gap even when the +// next group is adjacent. +test("Consumer reads a gap when retiring a truncated group", async () => { + const track = new Track.Producer("test"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 150 as Time.Milli }); + + const group = new Group.Producer(0); + group.writeFrame({ payload: encodeLegacy(0 as Time.Micro), timestamp: Time.Timestamp.now() }); + track.writeGroup(group); + await settle(); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + + group.close(new NetError.Stream(StreamCode.Cancel)); + writeGroupWithLegacyFrames(track, 1, [100_000 as Time.Micro]); + writeGroupWithLegacyFrames(track, 2, [200_000 as Time.Micro]); + await settle(); + + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(100_000 as Time.Micro); + + consumer.close(); +}); + +// An open group may still have a tail in flight, so adjacency proves nothing when it is retired. +test("Consumer reads a gap when retiring an open group", async () => { + const track = new Track.Producer("test"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 150 as Time.Milli }); + + const group = new Group.Producer(0); + group.writeFrame({ payload: encodeLegacy(0 as Time.Micro), timestamp: Time.Timestamp.now() }); + track.writeGroup(group); + await settle(); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + + writeGroupWithLegacyFrames(track, 1, [100_000 as Time.Micro]); + writeGroupWithLegacyFrames(track, 2, [200_000 as Time.Micro]); + await settle(); + + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(100_000 as Time.Micro); + + group.close(); + consumer.close(); +}); + // --- Buffered signal --- test("Consumer buffered signal updates as frames arrive", async () => { @@ -1061,8 +1148,7 @@ test("Consumer preserves empty media from formats without endpoint markers", asy track.writeGroup(group); track.close(); - const result = await consumer.next(); - expect(result?.frame?.payload).toHaveLength(0); + expect(frameOf(await nextFrame(consumer)).payload).toHaveLength(0); consumer.close(); }); @@ -1256,9 +1342,7 @@ test("Consumer delivers a PTS-contiguous next group whose sequence jumped (CMAF) a.close(); track.writeGroup(a); - const firstFrame = await consumer.next(); - expect(firstFrame?.frame?.payload).toEqual(new Uint8Array([0x01])); - await consumer.next(); // group-done marker + expect(frameOf(await nextFrame(consumer)).payload).toEqual(new Uint8Array([0x01])); const pending = consumer.next(); @@ -1285,7 +1369,7 @@ test("Consumer delivers a PTS-contiguous next group whose sequence jumped (CMAF) new Promise<"timeout">((resolve) => setTimeout(() => resolve("timeout"), 500)), ]); expect(result).not.toBe("timeout"); - expect((result as { frame?: Frame } | undefined)?.frame?.payload).toEqual(new Uint8Array([0x02])); + expect(frameOf(result).payload).toEqual(new Uint8Array([0x02])); consumer.close(); }); @@ -1317,9 +1401,7 @@ test("Consumer waits on a PTS gap instead of skipping to a later buffered group a.close(); track.writeGroup(a); - const firstFrame = await consumer.next(); - expect(firstFrame?.frame?.payload).toEqual(new Uint8Array([0x01])); - await consumer.next(); // group-done marker + expect(frameOf(await nextFrame(consumer)).payload).toEqual(new Uint8Array([0x01])); const pending = consumer.next(); @@ -1362,7 +1444,7 @@ test("Consumer waits on a PTS gap instead of skipping to a later buffered group }); const result = await pending; - expect((result as { frame?: Frame } | undefined)?.frame?.payload).toEqual(new Uint8Array([0x02])); + expect(frameOf(result).payload).toEqual(new Uint8Array([0x02])); consumer.close(); }); @@ -1383,7 +1465,7 @@ test("Consumer delivers a contiguous group after one that completed out of order track.writeGroup(a); a.writeFrame(cmafFrame(0x01, 0, 0)); - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([0x01])); + expect(frameOf(await nextFrame(consumer)).payload).toEqual(new Uint8Array([0x01])); // A's second frame stays buffered so A can't be duration-skipped. A ends at 6000 ticks (66_666µs). a.writeFrame(cmafFrame(0x02, 3000, 1)); @@ -1398,10 +1480,8 @@ test("Consumer delivers a contiguous group after one that completed out of order await settle(); a.close(); - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([0x02])); - expect((await consumer.next())?.frame).toBeUndefined(); // A group-done - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([0x03])); - expect((await consumer.next())?.frame).toBeUndefined(); // B group-done + expect(frameOf(await consumer.next()).payload).toEqual(new Uint8Array([0x02])); + expect(frameOf(await consumer.next()).payload).toEqual(new Uint8Array([0x03])); // C (seq 1_090_000) starts at B's end, so it continues the timeline and must deliver. const pending = consumer.next(); @@ -1411,7 +1491,7 @@ test("Consumer delivers a contiguous group after one that completed out of order const result = await Promise.race([pending, settle(300).then(() => "timeout" as const)]); expect(result).not.toBe("timeout"); - expect((result as { frame?: Frame } | undefined)?.frame?.payload).toEqual(new Uint8Array([0x04])); + expect(frameOf(result).payload).toEqual(new Uint8Array([0x04])); consumer.close(); }); @@ -1430,9 +1510,8 @@ test("Consumer plays the head once a waited-out gap exceeds the budget (CMAF)", const a = new Group.Producer(1000); track.writeGroup(a); a.writeFrame(cmafFrame(0x01, 0, 0)); - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([0x01])); + expect(frameOf(await nextFrame(consumer)).payload).toEqual(new Uint8Array([0x01])); a.close(); - expect((await consumer.next())?.frame).toBeUndefined(); // #active falls back to the 1001 phantom // The reader is parked on the missing 1001, as a decoder is while it waits for the next group. const pending = consumer.next(); @@ -1443,12 +1522,10 @@ test("Consumer plays the head once a waited-out gap exceeds the budget (CMAF)", track.writeGroup(b); b.writeFrame(cmafFrame(0x02, 90_000, 1)); + // The missing group's span was never delivered, so a gap comes first. const result = await Promise.race([pending, settle(300).then(() => "timeout" as const)]); - expect(result).not.toBe("timeout"); - const delivered = result as { frame?: Frame; continuous?: boolean } | undefined; - expect(delivered?.frame?.payload).toEqual(new Uint8Array([0x02])); - // The missing group's span was never delivered, so downstream must not bridge it. - expect(delivered?.continuous).toBe(false); + expect(result).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).payload).toEqual(new Uint8Array([0x02])); consumer.close(); }); @@ -1465,9 +1542,8 @@ test("Consumer waits on a gap within the budget, then plays the head once it exc const a = new Group.Producer(1000); track.writeGroup(a); a.writeFrame(cmafFrame(0x01, 0, 0)); - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([0x01])); + expect(frameOf(await nextFrame(consumer)).payload).toEqual(new Uint8Array([0x01])); a.close(); - expect((await consumer.next())?.frame).toBeUndefined(); // B (seq 2000) starts at 6000 ticks (66_667µs): a gap, but only 33ms past A's end. const b = new Group.Producer(2000); @@ -1485,36 +1561,36 @@ test("Consumer waits on a gap within the budget, then plays the head once it exc } const result = await Promise.race([pending, settle(300).then(() => "timeout" as const)]); - expect(result).not.toBe("timeout"); - expect((result as { frame?: Frame } | undefined)?.frame?.payload).toEqual(new Uint8Array([0x02])); + expect(result).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).payload).toEqual(new Uint8Array([0x02])); consumer.close(); }); -// --- Continuity reporting --- +// --- Gaps and ends --- -// `continuous` is what downstream buffer accounting keys on, so it must be false exactly when the -// consumer dropped something. Group numbers can't answer that: they aren't required to be -// sequential, and adjacency doesn't rule out a group the max delay check dropped on the way past. -test("Consumer reports continuity while nothing is dropped", async () => { +// A gap is what downstream buffer accounting keys on, so it must appear exactly when the consumer +// dropped something. Group numbers can't answer that: they aren't required to be sequential, and +// adjacency doesn't rule out a group the max delay check dropped on the way past. +test("Consumer yields no gap while nothing is dropped", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 100 as Time.Milli }); writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro, 33_000 as Time.Micro]); await settle(); - expect((await consumer.next())?.continuous).toBe(false); // nothing precedes the first frame - expect((await consumer.next())?.continuous).toBe(true); // same group: consecutive by protocol - expect((await consumer.next())?.frame).toBeUndefined(); // group 0 done + expect(await consumer.next()).toEqual({ kind: "gap" }); // nothing precedes the first frame + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + expect(frameOf(await consumer.next()).timestamp).toBe(33_000 as Time.Micro); writeGroupWithLegacyFrames(track, 1, [66_000 as Time.Micro]); await settle(); - expect((await consumer.next())?.continuous).toBe(true); // group 1 continues group 0 + expect(frameOf(await consumer.next()).timestamp).toBe(66_000 as Time.Micro); // group 1 continues group 0 consumer.close(); }); -test("Consumer reports a marker group as a playhead event", async () => { +test("Consumer reads a marker group as an end, not a gap", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 2_000 as Time.Milli }); @@ -1523,13 +1599,11 @@ test("Consumer reports a marker group as a playhead event", async () => { writeGroupWithLegacyFrames(track, 2, [1_000_000 as Time.Micro]); await settle(); - expect((await consumer.next())?.discontinuity).toBe(0); - expect((await consumer.next())?.discontinuity).toBe(0); // group 0 done - const reset = await consumer.next(); - expect(reset?.frame).toBeUndefined(); - expect(reset?.discontinuity).toBe(1); - expect(reset?.continuous).toBe(false); - expect((await nextFrame(consumer))?.discontinuity).toBe(1); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + expect(await consumer.next()).toEqual({ kind: "end", group: 1, end: 0 as Time.Micro }); + // A declared stop loses nothing, so no gap follows it. + expect(frameOf(await consumer.next()).timestamp).toBe(1_000_000 as Time.Micro); consumer.close(); }); @@ -1545,52 +1619,48 @@ test("Consumer empty groups mean nothing", async () => { writeGroupWithLegacyFrames(track, 2, [1_000 as Time.Micro]); await settle(); - expect((await nextFrame(consumer))?.frame?.timestamp).toBe(0 as Time.Micro); - expect((await nextFrame(consumer))?.frame?.timestamp).toBe(1_000 as Time.Micro); - expect(consumer.discontinuity).toBe(0); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + expect(frameOf(await consumer.next()).timestamp).toBe(1_000 as Time.Micro); consumer.close(); }); -test("Consumer latency skip bumps playhead generation once", async () => { +test("Consumer reads a latency skip as one gap", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 0 as Time.Milli }); writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); await settle(); expect((await nextFrame(consumer))?.frame?.timestamp).toBe(0 as Time.Micro); - expect((await consumer.next())?.frame).toBeUndefined(); writeGroupWithLegacyFrames(track, 2, [100_000 as Time.Micro]); await settle(); - const resumed = await nextFrame(consumer); - expect(resumed?.frame?.timestamp).toBe(100_000 as Time.Micro); - expect(resumed?.discontinuity).toBe(1); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(100_000 as Time.Micro); consumer.close(); }); -test("Consumer jumps the playhead after a shed marker with a timestamp hole", async () => { +test("Consumer reads a timestamp hole as a gap", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 0 as Time.Milli }); writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); await settle(); - expect((await consumer.next())?.frame?.timestamp).toBe(0 as Time.Micro); - expect((await consumer.next())?.frame).toBeUndefined(); + expect((await nextFrame(consumer))?.frame?.timestamp).toBe(0 as Time.Micro); writeGroupWithLegacyFrames(track, 3, [1_000_000 as Time.Micro]); await settle(); - const resumed = await nextFrame(consumer); - expect(resumed?.frame?.timestamp).toBe(1_000_000 as Time.Micro); - expect(resumed?.discontinuity).toBe(1); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(1_000_000 as Time.Micro); consumer.close(); }); -test("Consumer zero-budget skip keeps a contiguous marker", async () => { +test("Consumer reads a shed marker group as a gap", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 0 as Time.Milli }); @@ -1607,16 +1677,16 @@ test("Consumer zero-budget skip keeps a contiguous marker", async () => { writeGroupWithLegacyFrames(track, 2, [500 as Time.Micro]); await settle(); - const resumed = await nextFrame(consumer); - expect(resumed?.frame?.timestamp).toBe(500 as Time.Micro); - expect(resumed?.discontinuity).toBe(1); + // The zero budget sheds the marker unread, and what it declared is lost with it. + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(500 as Time.Micro); group0.close(); consumer.close(); }); // The case the group-number heuristic got backwards: ids jump, but the PTS timeline is unbroken. -test("Consumer reports continuity across a PTS-contiguous group id jump (CMAF)", async () => { +test("Consumer yields no gap across a PTS-contiguous group id jump (CMAF)", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new CmafFormat(TEST_INIT), @@ -1628,23 +1698,19 @@ test("Consumer reports continuity across a PTS-contiguous group id jump (CMAF)", a.writeFrame(cmafFrame(0x01, 0, 0)); a.close(); - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([0x01])); - expect((await consumer.next())?.frame).toBeUndefined(); // group-done + expect(frameOf(await nextFrame(consumer)).payload).toEqual(new Uint8Array([0x01])); // Sequence jumps +90_000 but the first PTS meets A's end, so nothing is missing. const b = new Group.Producer(1_090_000); track.writeGroup(b); b.writeFrame(cmafFrame(0x02, 3000, 1)); - const result = await consumer.next(); - expect(result?.frame?.payload).toEqual(new Uint8Array([0x02])); - expect(result?.continuous).toBe(true); - expect(result?.discontinuity).toBe(0); + expect(frameOf(await consumer.next()).payload).toEqual(new Uint8Array([0x02])); consumer.close(); }); -test("live duration marker follows an immediately delivered video frame", async () => { +test("a live video frame is delivered before its duration marker arrives", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe(), { format: new LegacyFormat("video"), @@ -1662,14 +1728,16 @@ test("live duration marker follows an immediately delivered video frame", async }); await settle(); expect(delivered).toBe(true); - expect((await read)?.frame?.timestamp).toBe(0 as Time.Micro); + expect(await read).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); group.writeFrame({ payload: encodeLegacyFrame(15_000 as Time.Micro, new Uint8Array()), timestamp: Time.Timestamp.now(), }); - const result = await consumer.next(); - expect(result?.frame).toBeUndefined(); - expect(result?.end).toBe(15_000 as Time.Micro); + group.close(); + track.close(); + // The marker is the frame's duration, not an item. + expect(await consumer.next()).toBeUndefined(); consumer.close(); track.close(); }); @@ -1700,18 +1768,18 @@ for (const code of [StreamCode.Cancel, StreamCode.Internal, StreamCode.Old, Stre await settle(); group.close(new NetError.Stream(code)); await settle(); - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([1])); - expect((await consumer.next())?.frame).toBeUndefined(); + expect(frameOf(await nextFrame(consumer)).payload).toEqual(new Uint8Array([1])); const next = track.appendGroup(); next.writeFrame({ payload: encodeLegacyFrame(1 as Time.Micro, new Uint8Array([2])), timestamp: Time.Timestamp.now(), }); next.close(); - const resumed = await consumer.next(); - expect(resumed?.frame?.payload).toEqual(new Uint8Array([2])); - expect(resumed?.continuous).toBe(false); - expect(resumed?.frame?.keyframe).toBe(true); + // The reset cut the group's tail. + expect(await consumer.next()).toEqual({ kind: "gap" }); + const resumed = frameOf(await consumer.next()); + expect(resumed.payload).toEqual(new Uint8Array([2])); + expect(resumed.keyframe).toBe(true); expect(error).not.toHaveBeenCalled(); } finally { consumer.close(); @@ -1775,8 +1843,7 @@ for (const end of [ }); try { writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); - expect((await consumer.next())?.group).toBe(0); - expect((await consumer.next())?.frame).toBeUndefined(); + expect(await nextFrame(consumer)).toMatchObject({ group: 0 }); writeGroupWithLegacyFrames(track, 2, [1_000_000 as Time.Micro]); await settle(); const pending = consumer.next(); @@ -1784,10 +1851,8 @@ for (const end of [ const resumed = await Promise.race([pending, settle(100).then(() => "stalled" as const)]); expect(resumed).not.toBe("stalled"); if (resumed === "stalled") return; - expect(resumed?.group).toBe(2); - expect(resumed?.frame?.timestamp).toBe(1_000_000 as Time.Micro); - expect(resumed?.continuous).toBe(false); - expect((await consumer.next())?.frame).toBeUndefined(); + expect(resumed).toEqual({ kind: "gap" }); + expect(await consumer.next()).toMatchObject({ kind: "frame", group: 2, frame: { timestamp: 1_000_000 } }); if (end) await expect(consumer.next()).rejects.toBe(end); else expect(await consumer.next()).toBeUndefined(); } finally { diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index a7e563d6fd2..a4bf7c3c188 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -21,6 +21,23 @@ export interface ConsumerProps { maxDelay?: GetterInit; } +/** One item of a {@link Consumer.next} read, in delivery order. Mirrors Rust's `container::Read`. */ +export type Read = + /** A media frame from `group`. Only group equality means anything: numbers may skip. */ + | { kind: "frame"; group: number; frame: Frame } + /** + * Media before the next item will never reach this reader: it joined mid-track, or a group was + * skipped, evicted, truncated, or never arrived. A decoder resumes without the references or + * state that media built. + */ + | { kind: "gap" } + /** + * The publisher declared that its media stops at `end`: it paused, switched source, or finished. + * Frames that follow from the same `group` are the stopped encoder's final packets (an audio + * codec's drain), whose output at or past `end` is padding. The next group starts fresh. + */ + | { kind: "end"; group: number; end: Time.Micro }; + interface Group { consumer: Moq.Group.Consumer; frames: Frame[]; // decode order @@ -78,21 +95,18 @@ export class Consumer { // promotion guard can tell a timeline-continuous next group from one sitting after a gap. // Maintained only via #recordPresented; see its comment for the invariant. #presentedEnd?: Time.Micro; - // Group of the last frame next() returned, so it can report whether the following result - // continues that frame's timeline. Undefined until the first delivery and after a playhead event. - #deliveredGroup?: number; - // Set whenever the consumer throws content away: a group that fell `maxDelay` behind, - // a group truncated by a decode error. Reported (and - // cleared) on the first frame delivered from the next group, which is where the missing span - // sits. Only the consumer can know this, which is why next() reports it instead of leaving - // callers to guess from group numbers. - #gap = false; + // Group of the last media frame next() returned, so an endpoint after it reads as that frame's + // duration rather than a declared stop. + #mediaGroup?: number; + // Media went missing since the last item read, so the next one is preceded by a gap. Starts set: + // nothing before the first frame was read. + #lost = true; + // The item a gap was returned ahead of. + #next?: Read; // The first media timestamp of the latest delivered group, and its predecessor's start. // Endpoints and later frames can overlap the next group without moving this floor. #start?: { group: number; timestamp: Time.Micro }; #floor?: Time.Micro; - // Increments on a declared marker, an unproven delivered hole, and a latency skip. - #discontinuity = 0; // A group below the previous group start aborts the track. #error?: Error; @@ -313,22 +327,11 @@ export class Consumer { if (group.end !== undefined) this.#presentedEnd = group.end; } - // Whether delivering from group `sequence` continues the timeline of the last frame returned. - // Frames within a group are consecutive by protocol, so only a group boundary can break it, and - // there it comes down to whether anything was dropped in between. Deliberately not derived from - // group numbers: they need not be sequential, so adjacency neither proves continuity nor catches - // a group the max delay check truncated on the way past. - #continuesDelivery(sequence: number): boolean { - if (this.#deliveredGroup === undefined) return false; - return sequence === this.#deliveredGroup || !this.#gap; - } - #checkMaxDelay() { if (this.#active === undefined) return; let skipped = 0; const start = this.#groups[0]?.consumer.sequence; - let hole = false; // Keep skipping the oldest group while the buffered span exceeds the max delay. // This also handles gaps in group sequence numbers: if #active points to a missing @@ -357,19 +360,23 @@ export class Consumer { this.#groups.shift(); this.#active = this.#groups[0]?.consumer.sequence; - const nextStart = this.#groups[0]?.frames.at(0)?.timestamp ?? this.#groups[0]?.end; + const next = this.#groups[0]; + const nextStart = next?.frames.at(0)?.timestamp ?? next?.end; const marker = !first.empty && !first.media; - if (marker || !ptsContiguous(first.end ?? this.#presentedEnd, nextStart)) { - hole = true; - } + // Unread media is thrown away, and `first.end` counts it, so contiguity alone would hide it. + // A trailing endpoint carries no media, so leaving it unread discards nothing. + const discarded = first.frames.some((frame) => this.#format.end?.(frame) === undefined); + // Adjacency proves nothing is missing only once the group finished cleanly: a truncated + // or still-open group may have lost its tail. + const finished = first.done && !first.truncated; + const adjacent = next?.consumer.sequence === first.consumer.sequence + 1; + const continued = (finished && adjacent) || ptsContiguous(first.end ?? this.#presentedEnd, nextStart); + if (marker || discarded || first.truncated || !continued) this.#lost = true; first.consumer.close(); first.frames.length = 0; skipped++; - this.#gap = true; } - if (hole) this.#markPlayhead(); - if (skipped) { console.warn( `skipping slow groups: track=${this.#track.name} ${start} -> ${this.#active} count=${skipped}`, @@ -441,35 +448,17 @@ export class Consumer { } /** - * Returns the next frame in order along with its group number and the current - * {@link discontinuity} count, awaiting one if needed. A `frame` of undefined signals either - * the end of that group or, when `end` is present, an exclusive media endpoint carried by a - * legacy marker. The overall result is undefined once closed. When `discontinuity` - * jumps relative to the previous call, re-apply startup delay and skip: it is a playhead - * event, and the next frame may be a delta that continues the interrupted group. - * - * `continuous` is true when this result picks up exactly where the previous frame left off, so - * the span between them can be treated as delivered. It is false on the first frame, after a - * playhead event, and whenever the consumer threw content away to keep up: a slow group skipped - * for the max delay, a group truncated by a decode error. Use it rather than comparing group - * numbers, which are not required to be sequential: adjacency neither proves the timeline is - * unbroken nor catches a group dropped on the way past. - * - * It reports what this consumer dropped plus marker groups the publisher declared. - * An unmarked forward timestamp jump still reads as continuous because nothing on the wire says - * the missing span will never arrive. + * Returns the next {@link Read} in delivery order, awaiting one if needed: a frame, a gap where + * media will never arrive (always one before the first frame), or an end the publisher declared. + * An unmarked forward timestamp jump is not a gap, since nothing on the wire says media is missing. * After buffered groups drain, a finished track returns undefined and an aborted track throws. */ - async next(): Promise< - | { - frame: Frame | undefined; - group: number; - discontinuity: number; - continuous: boolean; - end?: Time.Micro; - } - | undefined - > { + async next(): Promise { + if (this.#next) { + const next = this.#next; + this.#next = undefined; + return next; + } for (;;) { if (this.#error) throw this.#error; this.#checkMalformed(); @@ -487,8 +476,8 @@ export class Consumer { // fallback fired because no later group was buffered yet, and the real (large-gap, // non-sequential) next group has since arrived -- promote #active to the first real // group so delivery resumes instead of stalling on a nonexistent sequence. - // Before any media is delivered there is no timeline hole to wait for. #start survives - // playhead events, unlike #deliveredGroup, so this exception only applies at startup. + // Before any media is delivered there is no timeline hole to wait for, so this exception + // only applies at startup. // Otherwise promote #active when the first buffered group continues the timeline we left off at, // when a completed empty group can be walked (empty groups mean nothing), or when the hole // is proven: the head already reaches past where presentation left off by more than the @@ -511,9 +500,7 @@ export class Consumer { head.latest !== undefined && head.latest - this.#presentedEnd > maxDelay)); if (startup || empty || contiguous || skipHole || ended !== undefined) { - if (!startup && (skipHole || ended !== undefined) && !contiguous && !empty) - this.#markPlayhead(); - if (!contiguous) this.#gap = true; + if (!contiguous && !empty) this.#lost = true; this.#active = head.consumer.sequence; } } @@ -529,19 +516,12 @@ export class Consumer { const seq = this.#groups[0].consumer.sequence; const end = this.#format.end?.(frame); if (end !== undefined) { - if (!this.#groups[0].media) this.#markPlayhead(); this.#updateBuffered(); - return { - frame: undefined, - group: seq, - discontinuity: this.#discontinuity, - continuous: this.#continuesDelivery(seq), - end, - }; + // A trailing endpoint is the last frame's duration, which only the skip logic needs. + if (seq === this.#mediaGroup) continue; + return this.#emit({ kind: "end", group: seq, end }); } - const continuous = this.#continuesDelivery(seq); - if (seq !== this.#deliveredGroup) this.#gap = false; - this.#deliveredGroup = seq; + this.#mediaGroup = seq; if (this.#start === undefined || seq > this.#start.group) { this.#floor = this.#start?.timestamp; @@ -553,7 +533,7 @@ export class Consumer { throw this.#error; } this.#updateBuffered(); - return { frame, group: seq, discontinuity: this.#discontinuity, continuous }; + return this.#emit({ kind: "frame", group: seq, frame }); } // Check if the group is done and then remove it. A group is removable only @@ -575,17 +555,9 @@ export class Consumer { const group = this.#groups.shift(); if (group) { - const seq = group.consumer.sequence; - if (group.truncated) this.#gap = true; + if (group.truncated) this.#lost = true; this.#updateBuffered(); - return { - frame: undefined, - group: seq, - discontinuity: this.#discontinuity, - // A marker carries no content of its own, so this just reports whether - // the group it closes was itself reached without a gap. - continuous: this.#continuesDelivery(seq), - }; + continue; } } @@ -616,10 +588,12 @@ export class Consumer { } } - #markPlayhead(): void { - this.#discontinuity++; - this.#deliveredGroup = undefined; - this.#gap = true; + // Return `read`, or a gap ahead of it when media went missing first. + #emit(read: Read): Read { + if (!this.#lost) return read; + this.#lost = false; + this.#next = read; + return { kind: "gap" }; } #updateBuffered(): void { @@ -648,14 +622,6 @@ export class Consumer { this.#buffered.set(ranges); } - /** - * A counter that increments at each playhead event: a declared marker group, an unproven - * delivered hole, or a latency skip. Also surfaced per-read via {@link next}. - */ - get discontinuity(): number { - return this.#discontinuity; - } - /** Stop consuming and release the track and all buffered groups. */ close(): void { this.#finish(null); diff --git a/js/hang/src/container/index.ts b/js/hang/src/container/index.ts index 5ef19f7f7a8..135c5600d8f 100644 --- a/js/hang/src/container/index.ts +++ b/js/hang/src/container/index.ts @@ -7,7 +7,7 @@ export * as Loc from "@moq/loc"; export * as Cmaf from "./cmaf"; -export { Consumer, type ConsumerProps } from "./consumer"; +export { Consumer, type ConsumerProps, type Read } from "./consumer"; export type { Format } from "./format"; export * as Legacy from "./legacy"; export { type TrackInfoOptions, trackInfo } from "./track"; diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 77da93f5c88..15c8b468035 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -67,15 +67,14 @@ class FakeData { close() {} } -type Read = NonNullable>>; +type Read = Container.Read; let frameTimestamp = 0; -function frame(): Read { +function frame(): Extract { frameTimestamp += 20_000; return { + kind: "frame", group: 0, - discontinuity: 0, - continuous: true, frame: { timestamp: Time.Micro(frameTimestamp), payload: new Uint8Array([1]), keyframe: true }, }; } @@ -234,7 +233,7 @@ async function play(initial: Delay, buffered = false, cmaf = false) { const timestamps: number[] = []; for (let i = 0; i < count; i++) { const read = frame(); - timestamps.push(read.frame?.timestamp ?? 0); + timestamps.push(read.frame.timestamp); push(read); await microtasks(); } @@ -291,13 +290,12 @@ async function play(initial: Delay, buffered = false, cmaf = false) { }; } -it("decodes an audio packet inside a group after a playhead discontinuity", async () => { +it("decodes an audio packet inside a group after a gap", async () => { const playback = await play(Time.Milli(100)); try { await playback.play(); + playback.push({ kind: "gap" }); const next = frame(); - next.discontinuity = 1; - if (!next.frame) throw new Error("missing test frame"); next.frame.keyframe = false; playback.push(next); await microtasks(); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 348f378ff96..d618f87db53 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -119,7 +119,7 @@ export class Decoder { // The rendition is absent and the ring has drained its tail, so the graph has nothing to play. readonly #idle: Computed; - // Ordered discontinuity and endpoint state from the container consumer. + // Ordered epoch and endpoint state from the container consumer. #terminal = new Terminal(); // The container consumer's arrival estimate, unset while nothing is subscribed. @@ -471,16 +471,12 @@ export class Decoder { for (;;) { const next = await nextMedia(consumer); if (!next) break; - if (this.#onNext(next)) { + if (this.#onRead(next)) { decoder.reset(); decoder.configure(decoderConfig); } - if (next.end !== undefined) { - continue; - } - + if (next.kind !== "frame") continue; const { frame } = next; - if (!frame) continue; // Mark that we received this frame right now. const timestamp = Time.Milli.fromMicro(frame.timestamp as Time.Micro); @@ -581,13 +577,12 @@ export class Decoder { if (!next) break; // Reset and re-anchor before decoding the first frame of a new codec epoch. - if (this.#onNext(next)) { + if (this.#onRead(next)) { decoder.reset(); decoder.configure(decoderConfig); } - + if (next.kind !== "frame") continue; const { frame } = next; - if (!frame) continue; const timestamp = Time.Milli.fromMicro(frame.timestamp); this.sync.received(timestamp, "audio"); @@ -732,15 +727,10 @@ export class Decoder { this.#ring.peek()?.reset(); } - // Apply ordered container metadata before handling the result. An endpoint that also - // starts a new epoch must survive the reset so its following drain is trimmed. - #onNext(next: { - discontinuity: number; - group: number; - end?: Time.Micro; - frame?: { timestamp: Time.Micro }; - }): boolean { - if (!this.#terminal.update(next)) return false; + // Apply an ordered container read before handling it, reporting whether a codec epoch ended: + // a gap, or a later group after a declared end. + #onRead(read: Container.Read): boolean { + if (!this.#terminal.update(read)) return false; this.#ring.peek()?.reset(); this.sync.reset(); return true; diff --git a/js/watch/src/audio/playout.test.ts b/js/watch/src/audio/playout.test.ts index 362681f2186..e2e05d9e944 100644 --- a/js/watch/src/audio/playout.test.ts +++ b/js/watch/src/audio/playout.test.ts @@ -89,7 +89,7 @@ async function measure(t: Arrival[], budget: (target: Time.Milli) => Time.Milli) for (;;) { const next = await consumer.next(); if (!next) return; - if (next.frame) result.delivered++; + if (next.kind === "frame") result.delivered++; } })(); diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index 3c18faf97ba..e7ae9424f08 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -3,7 +3,7 @@ * * Every frame is written into its group at the instant the trace says it arrived, and the player's * real `Container.Consumer` decides what to deliver, wait for, and skip, at the max delay the delay - * sets. Delivered frames go into the real ring, and a discontinuity resets it, the way the decoder + * sets. Delivered frames go into the real ring, and a new codec epoch resets it, the way the decoder * does. One render quantum is read every quantum's worth of that same clock, which is what the * AudioWorklet does. A real {@link Sync} resolves the delay from the playout target the decoder * registers, measured by that same consumer from the recorded catalog config, and the ring follows @@ -221,7 +221,7 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato const next = await consumer.next(); if (!next) return; if (terminal.update(next)) ring.reset(); - if (next.end !== undefined || !next.frame) continue; + if (next.kind !== "frame") continue; const { timestamp } = next.frame; const length = samples.get(timestamp); diff --git a/js/watch/src/audio/terminal.test.ts b/js/watch/src/audio/terminal.test.ts index 6c6d9607cad..44eb0d06c33 100644 --- a/js/watch/src/audio/terminal.test.ts +++ b/js/watch/src/audio/terminal.test.ts @@ -2,21 +2,31 @@ import { expect, test } from "bun:test"; import type { Time } from "@moq/net"; import { Terminal } from "./terminal"; +const frame = (group: number, timestamp: number) => ({ + kind: "frame" as const, + group, + frame: { timestamp: timestamp as Time.Micro }, +}); +const end = (group: number, timestamp: number) => ({ kind: "end" as const, group, end: timestamp as Time.Micro }); +const gap = { kind: "gap" as const }; + test("Terminal trims a partial packet to the source endpoint", () => { const terminal = new Terminal(); - terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 0 as Time.Micro }, end: 17_916 as Time.Micro }); + terminal.update(end(0, 17_916)); + terminal.update(frame(0, 0)); expect(terminal.span({ timestamp: 0, sampleRate: 48_000, numberOfFrames: 960 }).frames).toBe(860); }); test("Terminal maps Opus delay back before trimming terminal output", () => { const terminal = new Terminal(); terminal.clear(312); - terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 0 as Time.Micro } }); + terminal.update(frame(0, 0)); const body = terminal.span({ timestamp: 0, sampleRate: 48_000, numberOfFrames: 960 }); expect(body).toEqual({ timestamp: 0 as Time.Micro, frameOffset: 312, frames: 648 }); - terminal.update({ discontinuity: 0, group: 0, end: 20_000 as Time.Micro }); - terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 20_000 as Time.Micro } }); + expect(terminal.update(end(1, 20_000))).toBe(false); + // The drain shares the end's group, so it continues the epoch. + expect(terminal.update(frame(1, 20_000))).toBe(false); const drain = terminal.span({ timestamp: 20_000, sampleRate: 48_000, numberOfFrames: 960 }); expect(drain).toEqual({ timestamp: 13_500 as Time.Micro, frameOffset: 0, frames: 312 }); expect(body.frames + drain.frames).toBe(960); @@ -24,41 +34,48 @@ test("Terminal maps Opus delay back before trimming terminal output", () => { test("Terminal preserves samples before the source endpoint", () => { const terminal = new Terminal(); - terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 0 as Time.Micro }, end: 20_000 as Time.Micro }); + terminal.update(end(0, 20_000)); + terminal.update(frame(0, 0)); expect(terminal.span({ timestamp: 0, sampleRate: 48_000, numberOfFrames: 960 }).frames).toBe(960); }); -test("a rewound endpoint survives the discontinuity reset and trims its terminal packet", () => { +test("the gap before the first frame ends no epoch", () => { + const terminal = new Terminal(); + expect(terminal.update(gap)).toBe(false); +}); + +test("an endpoint after a gap trims its own group", () => { const terminal = new Terminal(); terminal.clear(312); - terminal.update({ discontinuity: 0, group: 0, end: 40_000 as Time.Micro }); + terminal.update(frame(0, 0)); + expect(terminal.update(gap)).toBe(true); - const reset = terminal.update({ discontinuity: 1, group: 1, end: 20_000 as Time.Micro }); - expect(reset).toBe(true); + terminal.update(end(1, 20_000)); expect(terminal.end).toBe(20_000 as Time.Micro); - - terminal.update({ discontinuity: 1, group: 1, frame: { timestamp: 20_000 as Time.Micro } }); + terminal.update(frame(1, 20_000)); const span = terminal.span({ timestamp: 20_000, sampleRate: 48_000, numberOfFrames: 960 }); expect(span).toEqual({ timestamp: 20_000 as Time.Micro, frameOffset: 312, frames: 0 }); }); -test("a discontinuity reapplies Opus pre-skip in the resumed epoch", () => { +test("a gap reapplies Opus pre-skip in the resumed epoch", () => { const terminal = new Terminal(); terminal.clear(312); - terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 0 as Time.Micro } }); + terminal.update(frame(0, 0)); expect(terminal.span({ timestamp: 0, sampleRate: 48_000, numberOfFrames: 960 }).frameOffset).toBe(312); - terminal.update({ discontinuity: 1, group: 1, frame: { timestamp: 1_000_000 as Time.Micro } }); + expect(terminal.update(gap)).toBe(true); + terminal.update(frame(1, 1_000_000)); const resumed = terminal.span({ timestamp: 1_000_000, sampleRate: 48_000, numberOfFrames: 960 }); expect(resumed).toEqual({ timestamp: 1_000_000 as Time.Micro, frameOffset: 312, frames: 648 }); }); test("an endpoint does not trim media resumed in a later group", () => { const terminal = new Terminal(); - terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 0 as Time.Micro } }); + terminal.update(frame(0, 0)); // A discontinuity marker: its own group, ending the epoch at the live edge. - terminal.update({ discontinuity: 1, group: 1, end: 20_000 as Time.Micro }); - terminal.update({ discontinuity: 1, group: 2, frame: { timestamp: 5_000_000 as Time.Micro } }); + expect(terminal.update(end(1, 20_000))).toBe(false); + // The next group comes from a fresh encoder. + expect(terminal.update(frame(2, 5_000_000))).toBe(true); expect(terminal.end).toBeUndefined(); expect(terminal.span({ timestamp: 5_000_000, sampleRate: 48_000, numberOfFrames: 960 }).frames).toBe(960); }); diff --git a/js/watch/src/audio/terminal.ts b/js/watch/src/audio/terminal.ts index f89a8b15e74..59e501ae176 100644 --- a/js/watch/src/audio/terminal.ts +++ b/js/watch/src/audio/terminal.ts @@ -6,12 +6,11 @@ interface SampleSpan { readonly numberOfFrames: number; } -interface Update { - readonly discontinuity: number; - readonly group: number; - readonly end?: Time.Micro; - readonly frame?: { readonly timestamp: Time.Micro }; -} +/** The parts of a container read the terminal tracks. */ +type Update = + | { readonly kind: "frame"; readonly group: number; readonly frame: { readonly timestamp: Time.Micro } } + | { readonly kind: "gap" } + | { readonly kind: "end"; readonly group: number; readonly end: Time.Micro }; /** The portion of decoded audio that belongs to the source timeline. */ export interface DecodedSpan { @@ -25,7 +24,8 @@ export interface DecodedSpan { /** Track codec epochs and map decoded output onto the source timeline. */ export class Terminal { - #discontinuity = 0; + // Whether a frame arrived since the last reset, so there is an epoch to end. + #started = false; #end?: Time.Micro; // The group that carried `#end`. It bounds only the terminal packets that follow it there. #endGroup?: number; @@ -40,30 +40,39 @@ export class Terminal { /** Reset state for a new container consumer with Opus pre-skip in 48 kHz frames. */ clear(preSkip = 0): void { - this.#discontinuity = 0; + this.#started = false; this.#end = undefined; + this.#endGroup = undefined; this.#preSkip = preSkip; this.#resetEpoch(); } - /** Apply one ordered consumer result and report whether its codec epoch changed. */ - update(next: Update): boolean { - const reset = next.discontinuity !== this.#discontinuity; - if (reset) { - this.#discontinuity = next.discontinuity; - this.#end = undefined; - this.#resetEpoch(); - } - // A frame from a later group is new media, e.g. resuming after a discontinuity marker. - if (next.frame && next.group !== this.#endGroup) this.#end = undefined; - if (next.frame && this.#epoch === undefined) this.#epoch = next.frame.timestamp; - if (next.end !== undefined) { - this.#end = next.end; - this.#endGroup = next.group; + /** Apply one ordered consumer read and report whether a codec epoch ended before it. */ + update(read: Update): boolean { + if (read.kind === "gap") return this.#restart(); + if (read.kind === "end") { + this.#end = read.end; + this.#endGroup = read.group; + return false; } + // A later group after a declared end comes from a fresh encoder; the end's own group carries + // the stopped encoder's final packets. + const reset = this.#end !== undefined && read.group !== this.#endGroup && this.#restart(); + this.#started = true; + this.#epoch ??= read.frame.timestamp; return reset; } + // Start a new epoch, reporting whether one was running. + #restart(): boolean { + const started = this.#started; + this.#started = false; + this.#end = undefined; + this.#endGroup = undefined; + this.#resetEpoch(); + return started; + } + /** Remove codec pre-skip and terminal padding from one decoded sample. */ span(sample: SampleSpan): DecodedSpan { const epoch = this.#epoch ?? (sample.timestamp as Time.Micro); diff --git a/js/watch/src/retention.test.ts b/js/watch/src/retention.test.ts index 453c545f137..457b2ec975d 100644 --- a/js/watch/src/retention.test.ts +++ b/js/watch/src/retention.test.ts @@ -100,10 +100,10 @@ test("a long subscription through the player path leaves nothing behind", async const play = async (count: number) => { for (let i = 0; i < count; i++) { producer.encode(new Uint8Array([i & 0xff]), Math.round(Time.Micro.now()) as Time.Micro, true); - // A new group closes the previous one with a duration marker, which reads as no frame. + // Pass over the gap before the first frame. let next = await nextMedia(consumer); - while (next && !next.frame) next = await nextMedia(consumer); - if (!next?.frame) throw new Error("the track ended"); + while (next && next.kind !== "frame") next = await nextMedia(consumer); + if (!next) throw new Error("the track ended"); const timestamp = Time.Milli.fromMicro(next.frame.timestamp); sync.received(timestamp); diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index d18bc4775be..e2acb831415 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -173,15 +173,15 @@ class Picture { } } -type Read = NonNullable>>; -function sample(group: number, timestamp: number, discontinuity = 0, keyframe = true): Read { +type Read = Container.Read; +function sample(group: number, timestamp: number, keyframe = true): Read { return { + kind: "frame", group, - discontinuity, - continuous: true, frame: { timestamp: Time.Micro(timestamp), payload: new Uint8Array([1]), keyframe }, }; } +const gap: Read = { kind: "gap" }; async function guardedPlayback(kind: "legacy" | "cmaf", reads: Read[], hold: readonly number[] = []) { const originalDecoder = Object.getOwnPropertyDescriptor(globalThis, "VideoDecoder"); @@ -420,20 +420,20 @@ it("clears the picture once no rendition is selectable", async () => { } }); -// A late marker from older history raises a discontinuity without making older groups newer. -function marker(group: number, end: number, discontinuity: number): Read { - return { group, discontinuity, continuous: false, frame: undefined, end: Time.Micro(end) }; +// A late marker from older history breaks the timeline without making older groups newer. +function marker(group: number, end: number): Read { + return { kind: "end", group, end: Time.Micro(end) }; } for (const kind of ["legacy", "cmaf"] as const) { - it(`${kind} skips older groups before decode, even across a stale marker's discontinuity`, async () => { + it(`${kind} skips older groups before decode, even across a stale marker`, async () => { const playback = await guardedPlayback(kind, [ sample(10, 1000), sample(9, 900), sample(10, 1100), - marker(8, 850, 1), - sample(7, 700, 1), - sample(11, 1200, 1), + marker(8, 850), + sample(7, 700), + sample(11, 1200), ]); try { expect(playback.submitted).toEqual([1000, 1100, 1200]); @@ -446,9 +446,9 @@ for (const kind of ["legacy", "cmaf"] as const) { // A group may restart at its own start, so a picture already queued from later in that // group is above the new keyframe. It must not come out afterwards and late-reject it. - it(`${kind} drops a picture queued before a discontinuity`, async () => { + it(`${kind} drops a picture queued before a gap`, async () => { const queued = 60_000_000; - const playback = await guardedPlayback(kind, [sample(1, 0), sample(1, queued), sample(2, 0, 1)], [queued]); + const playback = await guardedPlayback(kind, [sample(1, 0), sample(1, queued), gap, sample(2, 0)], [queued]); try { expect(playback.submitted).toEqual([0, queued, 0]); expect(playback.outputs).toEqual([0, 0]); @@ -459,15 +459,15 @@ for (const kind of ["legacy", "cmaf"] as const) { } }); - // A stale marker (or a max-delay hole) can bump the playhead while the live group is only - // partly delivered, so the next frame is a delta that still needs the decoder's references. - it(`${kind} keeps decoding a delta that follows a discontinuity`, async () => { + // A stale marker can break the timeline while the live group is only partly delivered, so the + // next frame is a delta that still needs the decoder's references. + it(`${kind} keeps decoding a delta that follows a stale marker`, async () => { const playback = await guardedPlayback(kind, [ sample(10, 1000), - sample(10, 1100, 0, false), - marker(8, 850, 1), - sample(10, 1200, 1, false), - sample(11, 1300, 1), + sample(10, 1100, false), + marker(8, 850), + sample(10, 1200, false), + sample(11, 1300), ]); try { expect(playback.submitted).toEqual([1000, 1100, 1200, 1300]); @@ -603,7 +603,7 @@ it("a shared audio clock reset cannot present a parked video frame", async () => playback.enabled.set(false); playback.delay.set(Time.Milli(1000)); await microtasks(); - reads.push(sample(0, 0), sample(0, 100_000, 0, false)); + reads.push(sample(0, 0), sample(0, 100_000, false)); playback.enabled.set(true); await microtasks(); expect(playback.submitted).toEqual([0, 100_000]); @@ -645,7 +645,7 @@ it("closing the track releases a video wait parked across a reset", async () => playback.enabled.set(false); playback.delay.set(Time.Milli(1000)); await microtasks(); - reads.push(sample(0, 0), sample(0, 100_000, 0, false)); + reads.push(sample(0, 0), sample(0, 100_000, false)); playback.enabled.set(true); await microtasks(); expect(playback.submitted).toEqual([0, 100_000]); @@ -680,7 +680,7 @@ it("a reset drops a parked video frame even when the next anchor restores the re playback.enabled.set(false); playback.delay.set(Time.Milli(1000)); await microtasks(); - reads.push(sample(0, 0), sample(0, 100_000, 0, false)); + reads.push(sample(0, 0), sample(0, 100_000, false)); playback.enabled.set(true); await microtasks(); expect(playback.submitted).toEqual([0, 100_000]); diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index a1f915fee1f..173047d44f0 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -410,9 +410,6 @@ class DecoderTrack { // Decoded frames waiting to be rendered. #buffered = new Signal([]); - // The last discontinuity count seen from the container consumer. - #discontinuity = 0; - // Codec description from the last configure, reused after a playhead reset. #description?: Uint8Array; @@ -543,15 +540,18 @@ class DecoderTrack { const next = await nextMedia(consumer); if (!next) break; - // Playhead moved: re-anchor before decoding further. - if (this.#onDiscontinuity(next.discontinuity)) previous = undefined; - + // A gap or a declared end breaks the timeline: re-anchor before decoding further, and + // never bridge the buffer across it. + if (next.kind !== "frame") { + if (latest !== undefined) this.#onBreak(); + previous = undefined; + continue; + } const { frame } = next; - if (!frame) continue; // The group is done // An older keyframe would replace the codec references needed by live deltas. Group - // sequences only grow within a subscription, so a discontinuity (which a stale marker - // can raise) never resets this guard. Rejected payloads still count as received bytes. + // sequences only grow within a subscription, so a break (which a stale marker can + // raise) never resets this guard. Rejected payloads still count as received bytes. const stale = latest !== undefined && next.group < latest; this.stats.update((current) => ({ frameCount: (current?.frameCount ?? 0) + (stale ? 0 : 1), @@ -570,11 +570,11 @@ class DecoderTrack { }); // Track decode buffer: frames sent to decoder but not yet rendered. Only bridge from - // the previous frame when the consumer says nothing is missing in between. Group ids - // can't answer that: they aren't required to be sequential (some encoders derive them - // from DTS), so adjacency neither proves continuity nor rules out a gap the consumer - // skipped, and reporting a skipped span as decoded overstates the buffer. - if (previous !== undefined && next.continuous) { + // the previous frame when no gap or end came between them. Group ids can't answer that: + // they aren't required to be sequential (some encoders derive them from DTS), so + // adjacency neither proves continuity nor rules out a gap the consumer skipped, and + // reporting a skipped span as decoded overstates the buffer. + if (previous !== undefined) { this.#addBuffered(Time.Milli.fromMicro(previous), Time.Milli.fromMicro(frame.timestamp)); } @@ -623,15 +623,17 @@ class DecoderTrack { const next = await nextMedia(consumer); if (!next) break; - // Playhead moved: re-anchor before decoding further. - if (this.#onDiscontinuity(next.discontinuity)) previous = undefined; - + // A gap or a declared end breaks the timeline (see #runLegacy). + if (next.kind !== "frame") { + if (latest !== undefined) this.#onBreak(); + previous = undefined; + continue; + } const { frame } = next; - if (!frame) continue; // An older keyframe would replace the codec references needed by live deltas. Group - // sequences only grow within a subscription, so a discontinuity (which a stale marker - // can raise) never resets this guard. Rejected payloads still count as received bytes. + // sequences only grow within a subscription, so a break (which a stale marker can + // raise) never resets this guard. Rejected payloads still count as received bytes. const stale = latest !== undefined && next.group < latest; this.stats.update((current) => ({ frameCount: (current?.frameCount ?? 0) + (stale ? 0 : 1), @@ -643,9 +645,9 @@ class DecoderTrack { const timestamp = Time.Milli.fromMicro(frame.timestamp); this.sync.received(timestamp, "video"); - // Track decode buffer (see #runLegacy: bridge on the consumer's continuity signal, + // Track decode buffer (see #runLegacy: bridge only when no gap or end came between, // never on group adjacency, which proves nothing about the timeline). - if (previous !== undefined && next.continuous) { + if (previous !== undefined) { this.#addBuffered(Time.Milli.fromMicro(previous), Time.Milli.fromMicro(frame.timestamp)); } @@ -665,25 +667,21 @@ class DecoderTrack { }); } - // The container bumped its playhead (a marker, a hole, or a latency skip). Re-anchor and - // let the next decoded frame decide the decoder reset (see #resetAtKeyframe). Clearing - // `timestamp` keeps a stale high value from late-rejecting the following frames, which may - // overlap the previous group's tail. The held picture stays until the new keyframe renders. - // Returns true when a new generation was handled. - #onDiscontinuity(count: number): boolean { - if (count === this.#discontinuity) return false; - this.#discontinuity = count; + // The container read a gap or a declared end after media. Re-anchor and let the next decoded + // frame decide the decoder reset (see #resetAtKeyframe). Clearing `timestamp` keeps a stale + // high value from late-rejecting the following frames, which may overlap the previous group's + // tail. The held picture stays until the new keyframe renders. + #onBreak(): void { this.timestamp.set(undefined); this.#buffered.set([]); this.sync.reset(); this.#resetPending = true; - return true; } - // Call before each decode. After a playhead bump, a keyframe starts a new timeline, so drop - // the pictures still queued in the decoder: one from later in a group that restarts would + // Call before each decode. After a break, a keyframe starts a new timeline, so drop the + // pictures still queued in the decoder: one from later in a group that restarts would // otherwise come out above the new keyframe and late-reject it. A delta continues a group the - // bump interrupted (a stale marker or a max-delay hole), so it keeps the decoder's references; + // break interrupted (a stale marker or a max-delay hole), so it keeps the decoder's references; // a reset would make WebCodecs require a key chunk and throw. `reset()` cannot cancel an // output callback that already holds a frame; the clock epoch it captured rejects that one. #resetAtKeyframe(decoder: VideoDecoder, keyframe: boolean): void { diff --git a/quest/m1/README.md b/quest/m1/README.md index 38a87c57bec..f7cf24fc148 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -189,7 +189,6 @@ disabled rung strands WHEP, HLS, and moq-mux export viewers. - [TS fidelity and recovery](/quest/m1/ts-reliability/README.md) - both modes have explicit profiles, automated validation, loss tests, and a dedicated guide - [T-STD TS export](/quest/m1/tstd/README.md) - `moq export ts` is a proper remux that passes the supported T-STD buffer models, starting with a fixed `--delay` - [ATSC AC-3](/quest/m1/ts-atsc-ac3.md) - `export ts` carries 48 kHz ATSC AC-3 at every A/52 rate up to 640 kb/s instead of aborting from 384 kb/s up -- [Rust non-continuous signal](/quest/m1/rust-continuous.md) - the Rust container consumer reports a frame after a subscribe or discontinuity as non-continuous, like JS, for the tune-in and warmup trims - [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - a viewer joining at a recovery point drops the leading pictures it cannot decode; continuous viewers keep them - [Audio warmup](/quest/m1/audio-warmup.md) - a viewer joining an Opus rendition mid-stream never hears the unconverged first 80 ms - [#3021](/quest/m1/3021-moq-gst-anchor-generated-media-timelines-to-wall-clock.md) - moq-gst picks the broadcast wall epoch; a restarted source is a new epoch, not a forward re-anchor diff --git a/quest/m1/audio-warmup.md b/quest/m1/audio-warmup.md index 9160f0e82dd..1f8888b360a 100644 --- a/quest/m1/audio-warmup.md +++ b/quest/m1/audio-warmup.md @@ -14,9 +14,9 @@ frames decode independently and set nothing; HE-AAC is out of scope. - `rs/moq-audio/src/encode/producer.rs` and `rs/moq-mux/src/codec/opus` publish `warmup` for Opus; `js/publish` does the same for its Opus track. - `rs/moq-audio/src/decode` already trims Opus `pre_skip` at stream start; - the warmup trim is the same mechanism keyed on the container consumer's - non-continuous signal (added in Rust by - [Rust non-continuous signal](/quest/m1/rust-continuous.md)), and the + the warmup trim is the same mechanism keyed on the container read's gap + item (`container::Read::Gap` in Rust, `{ kind: "gap" }` in JS), never on a + declared end, which would cut 80 ms at every pause, and the subscription's maximum age grows by `warmup` as the video consumer quest does. `js/watch` audio mirrors it. - The trim replaces `LEGACY_WARMUP_CALLBACKS` in `js/watch/src/audio/decoder.ts`, @@ -26,10 +26,6 @@ frames decode independently and set nothing; HE-AAC is out of scope. - Tests in both languages: a mid-stream join discards exactly the warmup span and a continuous listener loses nothing. -## Required - -- [Rust non-continuous signal](/quest/m1/rust-continuous.md) - the signal the trim keys on - ## Related - [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - trims video at tune-in from the same signal diff --git a/quest/m1/open-gop-leading-pictures.md b/quest/m1/open-gop-leading-pictures.md index be7c8d0d01d..fa317d9f3c6 100644 --- a/quest/m1/open-gop-leading-pictures.md +++ b/quest/m1/open-gop-leading-pictures.md @@ -2,8 +2,7 @@ ## Goal -A viewer joining or resuming an open-GOP broadcast after a non-continuous -transition never hands the decoder a leading picture whose references it +A viewer joining or resuming an open-GOP broadcast after a gap never hands the decoder a leading picture whose references it does not have. Continuous groups keep every frame. The rule is the same for H.264 recovery-point keyframes with `recovery_frame_cnt = 0` (the broadcast contribution case, and what the reference capture carries) and H.265 CRA @@ -39,24 +38,21 @@ decode, and the JS test should drive the software decoder. - In the JS consumer (`js/hang/src/container/consumer.ts` forces the first sample of a group to `keyframe`, and `js/watch/src/video/decoder.ts` submits - it as `"key"`): for the first group after any non-continuous transition, - skip delta frames stamped before that group's keyframe. That covers a - subscribe, a declared discontinuity, and a latency skip: `#checkMaxDelay` - records the skip through `#gap` and `next()` reports the next frame with - `continuous: false`. Latency skip also bumps playhead generation (startup - delay) but does not flush the decoder. Leading pictures after that - non-continuous transition are still skipped, as above; a viewer that skipped - into a later GOP lacks its references just like a cold join. - Every continuous group is passed through untouched. + it as `"key"`): for the first group after a gap item, skip delta frames + stamped before that group's keyframe. `next()` yields `{ kind: "gap" }` + before the first frame of a subscribe and after a latency skip, eviction, or + hole; a declared end is not a gap, since a fresh encoder has no earlier GOP + to reference. A latency skip does not flush the decoder, and a viewer that + skipped into a later GOP lacks its references just like a cold join. + Every group without a gap before it is passed through untouched. - The same rule in the Rust decode path (`moq-video` decode consumers), so native playback and the transcoder tune in the same way, keyed on the - non-continuous signal from [Rust non-continuous - signal](/quest/m1/rust-continuous.md). + container read's `Read::Gap` item. - The same rule in `moq export ts`. Decided (2026-10-01): the fixed-delay export still sends a join's orphaned leading pictures, so 3 of 500 frames on the open-GOP fixture decode after they present (`dts-before-pts`). Trim them at tune-in from the same signal, on the - first group after each non-continuous transition, and make + first group after each gap, and make `just test ts --open-gop` pass under `--strict`. Decided in the 2026-10-05 audit: the grader changes with it. `test/ts/open-gop.py`'s decode-order check (a contiguous run of source @@ -74,12 +70,8 @@ decode, and the JS test should drive the software decoder. group's leading pictures trimmed too, so an implementation that only trims the initial group fails. Both cases in both languages. -Decided 2026-10-08: the Rust non-continuous signal is its own quest, so -audio warmup and consumer warmup don't wait on this one. - -## Required - -- [Rust non-continuous signal](/quest/m1/rust-continuous.md) - the Rust decode path keys the trim on it +Decided 2026-10-12: both container reads yield the gap item, so nothing +here waits on another quest. ## Related diff --git a/quest/m1/rust-continuous.md b/quest/m1/rust-continuous.md deleted file mode 100644 index a6032b7b82d..00000000000 --- a/quest/m1/rust-continuous.md +++ /dev/null @@ -1,33 +0,0 @@ -# [S] Rust non-continuous signal - -## Goal - -`moq_mux::container::Consumer` tells its reader whether a frame picks up -exactly where the previous one left off, like `continuous` in -`js/hang/src/container/consumer.ts`: false on the first frame after the -subscribe and after every discontinuity, true otherwise. -`rs/moq-video/src/decode` and `rs/moq-audio/src/decode` pass it through, so -the open-GOP trim, audio warmup, and consumer warmup key on one signal -instead of each adding its own. - -## Plan - -Split out of [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) -in the 2026-10-08 audit, so audio and consumer warmup need not wait on the -open-GOP trim. - -Today `poll_read` returns a bare frame, and `discontinuity()` is a counter -bumped on a declared marker group, an unproven delivered hole, or a latency -skip, but not on the subscribe itself. Report the flag with each frame and -let the counter go if nothing else needs it. Mirror the JS name. - -Tests: the first frame after a subscribe, a marker group, a delivered hole, -and a latency skip each read as non-continuous, and every other frame as -continuous. - -Public API: breaking in moq-mux (the consumer's read result changes). Wire: -none. - -## Related - -- [Consumer warmup](/quest/m2/intra-refresh/consumer-warmup.md) - withholds frames after the same signal diff --git a/quest/m1/ts-reliability/recovery.md b/quest/m1/ts-reliability/recovery.md index 48014c19425..cfc4db438ad 100644 --- a/quest/m1/ts-reliability/recovery.md +++ b/quest/m1/ts-reliability/recovery.md @@ -57,6 +57,5 @@ and measured release timing as separate checks. ## Related -- [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - trims after non-continuous transitions +- [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - trims after a gap - [Gradual recovery](/quest/m2/intra-refresh/README.md) - recovery counts and per-group warmup -- [Rust continuity](/quest/m1/rust-continuous.md) - consumers observe delivery breaks diff --git a/quest/m2/intra-refresh/consumer-warmup.md b/quest/m2/intra-refresh/consumer-warmup.md index b860afe142b..0ad9d0e0ffa 100644 --- a/quest/m2/intra-refresh/consumer-warmup.md +++ b/quest/m2/intra-refresh/consumer-warmup.md @@ -19,21 +19,18 @@ This absorbs the old gradual-recovery quest, which planned to carry `recovery_frame_cnt` on the wire and count `frame_num`. The catalog duration replaces both: the rule is timestamp arithmetic on the group start. -- Withhold rule, keyed on the same non-continuous signal in both consumers: - `js/hang/src/container/consumer.ts` `next()` reports `continuous: false` - after a subscribe, a declared discontinuity, or any skip (`#gap`). The Rust - `moq_mux::container::Consumer` gains the equivalent in - [Rust non-continuous signal](/quest/m1/rust-continuous.md) - (today `poll_read` returns a bare frame and only the `discontinuity()` - counter moves); this quest reuses it, and - `rs/moq-video/src/decode/consumer.rs` propagates it. For the first - group after that signal, every frame is decoded (the decoder needs them to +- Withhold rule, keyed on the gap item both container reads yield before the + first frame and after any skip: `{ kind: "gap" }` from + `js/hang/src/container/consumer.ts` `next()` and `Read::Gap` from the Rust + `moq_mux::container::Consumer`, which `rs/moq-video/src/decode/consumer.rs` + reads directly. A declared end is not a gap and keeps everything. For the + first group after a gap, every frame is decoded (the decoder needs them to build reference state) and frames stamped below `group.start + warmup` are not presented; frames stamped at or above that boundary are, so the recovery picture itself is never withheld. `js/watch/src/video/decoder.ts` and `rs/moq-video/src/decode` are where presentation happens; the transcoder gets the skip for free because it consumes the same decode path. Keep the - last painted frame on screen, as `#onDiscontinuity` already does. + last painted frame on screen, as `#onBreak` already does. - IDR exception: before withholding, check the first slice NAL type of the group's first frame (H.264 type 5; H.265 `IDR_W_RADL`, `IDR_N_LP`, the BLA types). Rust has the NAL walkers in `rs/moq-mux/src/codec/{h264,h265}`; @@ -61,7 +58,6 @@ replaces both: the rule is timestamp arithmetic on the group start. ## Required - [Plan: recovery boundaries](/quest/m2/intra-refresh/plan-boundary.md) - consumers need a safe boundary before presenting the affected pictures -- [Rust non-continuous signal](/quest/m1/rust-continuous.md) - the Rust signal this keys on ## Related diff --git a/rs/hang/examples/subscribe.rs b/rs/hang/examples/subscribe.rs index b929f4b8c7e..f8fdb09e2f7 100644 --- a/rs/hang/examples/subscribe.rs +++ b/rs/hang/examples/subscribe.rs @@ -115,13 +115,17 @@ async fn play(consumer: moq_net::origin::Consumer, announce: moq_net::announce:: moq_mux::container::Consumer::new(track_consumer, moq_mux::catalog::hang::Container::try_from(config)?); // Read frames in latency-bounded presentation order. - while let Some(frame) = ordered.read().await? { - tracing::info!( - timestamp = ?frame.timestamp, - keyframe = frame.keyframe, - bytes = frame.payload.len(), - "received frame" - ); + while let Some(read) = ordered.read().await? { + match read { + moq_mux::container::Read::Frame { frame, .. } => tracing::info!( + timestamp = ?frame.timestamp, + keyframe = frame.keyframe, + bytes = frame.payload.len(), + "received frame" + ), + moq_mux::container::Read::Gap => tracing::info!("media skipped"), + moq_mux::container::Read::End { end, .. } => tracing::info!(?end, "media stopped"), + } } Ok(()) diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 9b9cd6c49dc..80e7585b265 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -1,6 +1,8 @@ //! Subscribe to an encoded audio track and emit raw PCM. use std::collections::VecDeque; + +use moq_mux::container::Read; use std::time::{Duration, Instant}; use bytes::Bytes; @@ -99,10 +101,10 @@ pub struct Consumer { frames_decoded: usize, /// Logical endpoint carried by an empty legacy frame before terminal packets. end: Option, + /// The group `end` came from: it bounds only the terminal packets that follow it there. + end_group: u64, /// Presentation time of the first decoded terminal frame. terminal_start: Option, - /// Last container playhead generation applied to timeline state. - discontinuity: u64, /// The playout delay, fixed or estimated from arrivals. target: Target, /// Where arrival times are measured from. @@ -206,8 +208,8 @@ impl Consumer { delay_trimmed: 0, frames_decoded: 0, end: None, + end_group: 0, terminal_start: None, - discontinuity: 0, origin: now(), }) } @@ -267,10 +269,27 @@ impl Consumer { return Ok(Some(frame)); } - let mux_frame = self.track.read().await?; - self.apply_discontinuity()?; - let Some(mux_frame) = mux_frame else { - return self.flush(); + let mux_frame = match self.track.read().await? { + Some(Read::Frame { group, frame }) => { + // A later group after a declared end comes from a fresh encoder; the end's + // own group carries the stopped encoder's final packets, trimmed below. + if self.end.is_some() && group != self.end_group { + self.restart()?; + } + frame + } + Some(Read::Gap) => { + self.restart()?; + continue; + } + Some(Read::End { group, end }) => { + self.end = Some(end); + self.end_group = group; + self.frames_decoded = 0; + self.terminal_start = None; + continue; + } + None => return self.flush(), }; // Every packet the container hands over is an arrival, including one the @@ -280,14 +299,6 @@ impl Consumer { let media = timestamp.value() as f64 * 1000.0 / timestamp.scale().as_u64() as f64; self.target.observe(arrival, media); - if let Some(end) = self.track.end() - && self.end != Some(end) - { - self.end = Some(end); - self.frames_decoded = 0; - self.terminal_start = None; - } - // Undeclared holes are routine: a skipped stalled group, a packet the // decoder refused, an ingest that resynced. Drop every stage's state at // the edge, before the packet after it goes anywhere near the decoder. @@ -401,15 +412,15 @@ impl Consumer { } } - /// A playhead event re-applies startup delay and skip. The decoder is not reset: + /// Start a new timeline after a gap or past a declared end. The decoder is not reset: /// the next group already starts on a keyframe, and pre-skip is a play-path concern. - fn apply_discontinuity(&mut self) -> Result<(), Error> { - let discontinuity = self.track.discontinuity(); - if discontinuity == self.discontinuity { - return Ok(()); + /// + /// Samples the resampler still holds arrived before the break, so they are drained + /// as their own frame first rather than filtered together with what follows it. + fn restart(&mut self) -> Result<(), Error> { + if let Some(frame) = self.drain()? { + self.ready.push_back(frame); } - - self.discontinuity = discontinuity; self.next_start = None; self.spans.clear(); self.trailing = Activity::Active; @@ -433,13 +444,7 @@ impl Consumer { fn gap(&mut self) -> Result, Error> { self.decoder.reset_prediction()?; - let mut frame = None; - if let Some(resampler) = self.resampler.as_mut() { - let held = resampler.held_at(); - let skipped = resampler.skipped(); - let pcm = resampler.drain()?; - frame = self.tail(pcm, held, skipped)?; - } + let frame = self.drain()?; self.next_start = None; self.spans.clear(); @@ -449,6 +454,17 @@ impl Consumer { Ok(frame) } + /// Drain what the resampler holds while keeping it for what follows. + fn drain(&mut self) -> Result, Error> { + let Some(resampler) = self.resampler.as_mut() else { + return Ok(None); + }; + let held = resampler.held_at(); + let skipped = resampler.skipped(); + let pcm = resampler.drain()?; + self.tail(pcm, held, skipped) + } + /// The tail the resampler is still holding when the track ends, once. /// /// Without it the last partial chunk is dropped, which is up to a chunk of @@ -595,6 +611,61 @@ mod tests { use crate::encode::{Encoder, Input, Options as EncodeOptions, Producer, Settings}; use crate::{Format, Layout}; + /// What the resampler holds at a marker belongs before it: draining it after the next + /// packet would splice it onto the resumed audio and stamp that late. + #[tokio::test(start_paused = true)] + async fn resampled_tail_stays_before_a_marker() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let subscriber = broadcast.consume(); + let catalog = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Pcm, 44_100, 1); + let mut producer = moq_mux::container::Producer::new( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), + ); + let mut consumer = Consumer::new( + &subscriber, + &catalog, + "audio", + Options { + max_delay: Duration::from_secs(10), + output: Output { + sample_rate: Some(48_000), + ..Output::default() + }, + ..Options::new() + }, + ) + .await + .unwrap(); + for index in 0..2 { + if index != 0 { + producer.discontinuity().unwrap(); + } + producer + .write(moq_mux::container::Frame { + timestamp: Timestamp::from_micros(index * 1_000_000).unwrap(), + duration: None, + payload: vec![0u8; 1024 * size_of::()].into(), + keyframe: true, + }) + .unwrap(); + } + producer.finish().unwrap(); + let mut stamps = Vec::new(); + while let Some(frame) = consumer.read().await.unwrap() { + stamps.push(frame.timestamp.as_micros()); + } + assert_eq!(stamps.len(), 4, "each span's body and tail: {stamps:?}"); + assert!( + stamps[1] < 1_000_000, + "the old tail stays before the marker: {stamps:?}" + ); + assert_eq!(stamps[2], 1_000_000, "the resumed span starts on its source timestamp"); + } + #[tokio::test] async fn remixes_mono_stream_to_stereo_output() { let mut broadcast = moq_net::broadcast::Info::new().produce(); diff --git a/rs/moq-audio/src/encode/capture.rs b/rs/moq-audio/src/encode/capture.rs index 8fe2be135fc..eb4f92b9ea5 100644 --- a/rs/moq-audio/src/encode/capture.rs +++ b/rs/moq-audio/src/encode/capture.rs @@ -2193,7 +2193,10 @@ mod tests { async fn read_new(&self, track: &mut Track, seen: &[u64]) -> u64 { loop { - let packet = track.read().await.unwrap().expect("a published packet"); + let Some(moq_mux::container::Read::Frame { frame: packet, .. }) = track.read().await.unwrap() + else { + continue; + }; let timestamp = u64::try_from(packet.timestamp.as_micros()).unwrap(); if !seen.contains(×tamp) { return timestamp; diff --git a/rs/moq-audio/src/encode/producer.rs b/rs/moq-audio/src/encode/producer.rs index 85c479b0545..bc68c6a7034 100644 --- a/rs/moq-audio/src/encode/producer.rs +++ b/rs/moq-audio/src/encode/producer.rs @@ -565,6 +565,18 @@ mod tests { use crate::decode::{Consumer as AudioConsumer, Options as DecodeOptions}; use crate::{Activity, Format, Layout}; + /// The next published packet, passing over gaps and ends. + async fn packet(reader: &mut moq_mux::container::Consumer) -> Option + where + F::Error: std::fmt::Debug, + { + loop { + if let moq_mux::container::Read::Frame { frame, .. } = reader.read().await.unwrap()? { + return Some(frame); + } + } + } + #[tokio::test] async fn demand_follows_subscribers_and_closes_with_the_producer() { let mut broadcast = moq_net::broadcast::Info::new().produce(); @@ -699,7 +711,7 @@ mod tests { producer.finish().unwrap(); let mut packets = 0; - while track.read().await.unwrap().is_some() { + while packet(&mut track).await.is_some() { packets += 1; } assert_eq!(packets, 11); @@ -782,8 +794,8 @@ mod tests { producer.reanchor(); producer.finish().unwrap(); - assert!(track.read().await.unwrap().is_some()); - assert!(track.read().await.unwrap().is_none()); + assert!(packet(&mut track).await.is_some()); + assert!(packet(&mut track).await.is_none()); } /// A codec reset starts a new pre-skip interval at the receiver too. @@ -928,7 +940,7 @@ mod tests { producer.reanchor(); } producer.write(frame).unwrap(); - let read = reader.read().await.unwrap().expect("a packet per full frame"); + let read = packet(&mut reader).await.expect("a packet per full frame"); pts.push(read.timestamp.as_micros()); } pts @@ -1118,7 +1130,7 @@ mod tests { producer.write(&pcm_frame(&[0.1; 4 * 1024], anchor_us)).unwrap(); let mut pts = Vec::new(); for _ in 0..4 { - pts.push(reader.read().await.unwrap().expect("a packet").timestamp.as_micros()); + pts.push(packet(&mut reader).await.expect("a packet").timestamp.as_micros()); } pts } @@ -1167,7 +1179,7 @@ mod tests { // A second of audio, so the filter's delay is nowhere near the whole write. producer.write(&pcm_frame(&vec![0.1; 44_100], 1_000_000)).unwrap(); - let first = reader.read().await.unwrap().expect("a packet"); + let first = packet(&mut reader).await.expect("a packet"); assert_eq!(first.timestamp.as_micros(), 1_000_000); } @@ -1219,7 +1231,7 @@ mod tests { producer.write(&full_frame(0)).unwrap(); producer.finish().unwrap(); - assert!(track.read().await.unwrap().is_some()); + assert!(packet(&mut track).await.is_some()); producer.abort(moq_net::Error::Cancel); } diff --git a/rs/moq-c/src/consume.rs b/rs/moq-c/src/consume.rs index 3bc2dff3eae..8c0efdfb479 100644 --- a/rs/moq-c/src/consume.rs +++ b/rs/moq-c/src/consume.rs @@ -539,8 +539,10 @@ impl Consume { let frame = tokio::select! { biased; _ = &mut close => return Ok(()), - frame = track.read() => match frame? { - Some(frame) => frame, + read = track.read() => match read? { + Some(moq_mux::container::Read::Frame { frame, .. }) => frame, + // The C ABI carries frames only. + Some(moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. }) => continue, None => return Ok(()), }, }; diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index e80f837b795..35a0cc608e5 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -4,7 +4,7 @@ use std::collections::{BTreeSet, VecDeque}; use std::sync::{Arc, Mutex}; use std::time::Duration; -use moq_mux::container::Frame; +use moq_mux::container::{Frame, Read}; use tokio::time::Instant; use super::buffer::Buffer; @@ -73,30 +73,46 @@ impl Video { pub async fn run(self, mut track: Track, mut decoder: impl Decoder) -> anyhow::Result<()> { let buffer = Mutex::new(Buffer::default()); let receive = async { - let mut discontinuity = track.discontinuity(); + // The group of a declared end whose restart waits for a later group, so the + // media buffered before the end still plays out. + let mut end = None; loop { - let frame = track.read().await?; + let read = track.read().await?; let mut buffer = buffer.lock().unwrap(); let before = buffer.generation; - if track.discontinuity() != discontinuity { - discontinuity = track.discontinuity(); - buffer.clear(); - self.presentation.lock().unwrap().video_restarted(); - } - let ended = frame.is_none(); - if let Some(frame) = frame { - if self - .presentation - .lock() - .unwrap() - .video(frame.timestamp, Instant::now().into_std()) - { - self.output.send(Event::Wake); + let ended = match read { + Some(Read::Frame { group, frame }) => { + if end.take_if(|end| *end != group).is_some() { + buffer.clear(); + self.presentation.lock().unwrap().video_restarted(); + } + if self + .presentation + .lock() + .unwrap() + .video(frame.timestamp, Instant::now().into_std()) + { + self.output.send(Event::Wake); + } + buffer.push(frame, self.max_delay); + false } - buffer.push(frame, self.max_delay); - } else { - buffer.ended = true; - } + // Applied in delivery order, so the restart lands before the frame after it. + Some(Read::Gap) => { + end = None; + buffer.clear(); + self.presentation.lock().unwrap().video_restarted(); + false + } + Some(Read::End { group, .. }) => { + end = Some(group); + false + } + None => { + buffer.ended = true; + true + } + }; if buffer.generation != before { self.frames.lock().unwrap().clear(); self.output.send(Event::Wake); @@ -482,6 +498,48 @@ mod tests { assert_eq!(media.output.present(&media).unwrap().as_millis(), 500); } + #[tokio::test] + async fn a_declared_end_plays_out_the_pictures_before_it() { + tokio::time::pause(); + let delay = Duration::from_secs(2); + let media = media(delay); + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let subscriber = track + .consume() + .subscribe(moq_net::track::Subscription::default().with_max_delay(delay)) + .await + .unwrap(); + let mut producer = moq_mux::container::Producer::new( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), + ); + let track = moq_mux::container::Consumer::new( + subscriber, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), + ); + for ms in [0, 33, 66] { + producer.write(frame(ms, ms == 0)).unwrap(); + } + // The publisher pauses and sends nothing more for now. + producer.discontinuity().unwrap(); + // A codec that holds nothing back, so every picture is owed before the track ends. + let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(0))); + let mut shown = Vec::new(); + for _ in 0..2_200 { + tokio::task::yield_now().await; + if let Some(timestamp) = media.output.present(&media) { + shown.push(timestamp.as_millis()); + } + tokio::time::advance(Duration::from_millis(1)).await; + } + assert_eq!(shown, [0, 33, 66], "the end discarded the pictures before it"); + producer.finish().unwrap(); + task.await.unwrap().unwrap(); + } + /// Play one burst of 33 ms pictures, given by slot in decode order, through a /// codec holding `depth` pictures, and check the window shows every one on /// time. diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index aef9f9c72de..ec1c356720b 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -1173,12 +1173,7 @@ pub(crate) mod tests { let (name, config) = catalog.video.renditions.iter().next().expect("a video rendition"); let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); let container = moq_mux::catalog::hang::Container::try_from(config).unwrap(); - let first = Consumer::new(track, container) - .read() - .await - .unwrap() - .expect("a video frame") - .timestamp; + let first = next_frame(&mut Consumer::new(track, container)).await.timestamp; assert!( first.as_micros() < Duration::from_secs(5).as_micros(), "the feed's own PTS: {first:?}" @@ -1454,11 +1449,22 @@ pub(crate) mod tests { async fn read_frame(consumer: &moq_net::broadcast::Consumer, name: &str) -> Vec { let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); let mut reader = Consumer::new(track, Container::Legacy(moq_mux::container::Kind::Data)); - let frame = tokio::time::timeout(Duration::from_secs(1), reader.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), next_frame(&mut reader)) .await - .expect("verbatim read timed out") - .unwrap() - .expect("a published verbatim frame"); + .expect("verbatim read timed out"); frame.payload.to_vec() } + + /// The next frame, passing over gaps and ends. + async fn next_frame(reader: &mut Consumer) -> moq_mux::container::Frame + where + C::Error: std::fmt::Debug, + { + loop { + let read = reader.read().await.unwrap().expect("a published frame"); + if let moq_mux::container::Read::Frame { frame, .. } = read { + return frame; + } + } + } } diff --git a/rs/moq-ffi/src/media.rs b/rs/moq-ffi/src/media.rs index db5e40ae0db..753745fc152 100644 --- a/rs/moq-ffi/src/media.rs +++ b/rs/moq-ffi/src/media.rs @@ -1101,7 +1101,14 @@ struct Media { impl Media { async fn next(&mut self) -> Result, MoqError> { - self.inner.read().await?.map(media_frame).transpose() + loop { + match self.inner.read().await? { + Some(moq_mux::container::Read::Frame { frame, .. }) => return media_frame(frame).map(Some), + // The bindings carry frames only. + Some(moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. }) => continue, + None => return Ok(None), + } + } } } diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 98372792217..9b9bb29ca6d 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -1094,7 +1094,8 @@ impl Pump { // This rendition is being torn down (its run ended, or a catalog update replaced it). _ = cancel.changed() => return None, frame = track.read() => match frame { - Ok(Some(frame)) => { + Ok(Some(moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. })) => {} + Ok(Some(moq_mux::container::Read::Frame { frame, .. })) => { // PTS restarts at zero with every pump, so the segment places that zero at the // current running time, or a synced sink would drop a restarted run as late. let segment = reference_ts.is_none().then(|| { diff --git a/rs/moq-mux/src/codec/opus/import.rs b/rs/moq-mux/src/codec/opus/import.rs index 3ee70a185a7..1abbbed143e 100644 --- a/rs/moq-mux/src/codec/opus/import.rs +++ b/rs/moq-mux/src/codec/opus/import.rs @@ -173,7 +173,7 @@ mod tests { subscriber, crate::catalog::hang::Container::Loc(crate::container::Kind::Data), ); - let frame = tokio::time::timeout(Duration::from_secs(1), media.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), media.read_frame()) .await .unwrap() .unwrap() diff --git a/rs/moq-mux/src/codec/vp8/import.rs b/rs/moq-mux/src/codec/vp8/import.rs index dde79d1f314..0d6321b68a4 100644 --- a/rs/moq-mux/src/codec/vp8/import.rs +++ b/rs/moq-mux/src/codec/vp8/import.rs @@ -206,7 +206,7 @@ mod tests { subscriber, crate::catalog::hang::Container::Loc(crate::container::Kind::Data), ); - let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read()) + let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read_frame()) .await .unwrap() .unwrap() diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 1ea8a3f1940..a83066c24af 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -3,12 +3,17 @@ use std::task::{Poll, ready}; use moq_net::Timestamp; -use super::{Container, Frame, TimestampRewind}; +use super::{Container, Frame, Read, TimestampRewind}; /// Media and clean group boundaries in delivery order. pub(crate) enum Event { Frame(Frame), - FrameEnd(Timestamp), + /// An endpoint marker. `opens` when no media precedes it in its group: a declared stop + /// (a marker group or an audio terminal) rather than the previous frame's duration. + FrameEnd { + end: Timestamp, + opens: bool, + }, GroupEnd, } @@ -49,12 +54,12 @@ pub(crate) enum Event { /// subscribing. [`new`](Self::new) inherits that budget, and /// [`set_max_delay`](Self::set_max_delay) changes it mid-stream. /// -/// ## Timeline discontinuities +/// ## Gaps and ends /// -/// A marker group (no decodable frames, one empty payload) is a walk-now discontinuity. -/// A delivered sequence hole is a playhead event unless the boundary is contiguous within -/// 1 ms, and a latency skip is the same event. [`discontinuity`](Self::discontinuity) is -/// that playhead generation: re-apply startup delay and skip, not a decoder flush. +/// [`read`](Self::read) yields [`Read::Gap`] before the first frame and wherever media +/// will never arrive: a latency skip, an evicted current group, or a sequence hole whose +/// boundary is not contiguous within 1 ms. A marker group (no decodable frames, one empty +/// payload) is the publisher declaring a stop, read as [`Read::End`]; nothing is missing. /// Empty groups (zero objects) mean nothing. A group whose timestamps fall below the live /// edge earlier groups reached is malformed and aborts the track. pub struct Consumer { @@ -98,8 +103,15 @@ pub struct Consumer { // Increments on a declared marker only: the publisher paused or broke its timeline forward. markers: u64, - // Exclusive audio endpoint delivered before terminal codec packets. - end: Option, + // Media went missing since the last item read, so the next one is preceded by a gap. + // Starts set: nothing before the first frame was read. + gap: bool, + + // The item a gap was returned ahead of. + next: Option, + + // The group the last event came from. + group: u64, } /// Two adjacent groups are timeline-contiguous when the next start is within this slack @@ -144,16 +156,15 @@ impl Consumer { presented_end: None, discontinuity: 0, markers: 0, - end: None, + gap: true, + next: None, + group: 0, } } /// A counter that increments at each playhead event: a declared marker group, an - /// unproven delivered hole, or a latency skip. - /// - /// Downstream consumers re-apply startup delay and skip when it changes. It is not a - /// decoder flush: the next group already starts on a keyframe with parameter sets. - pub fn discontinuity(&self) -> u64 { + /// unproven delivered hole, a latency skip, or an evicted current group. + pub(crate) fn discontinuity(&self) -> u64 { self.discontinuity } @@ -165,38 +176,54 @@ impl Consumer { self.markers } - /// The exclusive audio endpoint delivered before terminal codec packets. - pub fn end(&self) -> Option { - self.end - } - /// Read the next frame from the track. /// /// This method handles timestamp decoding, group ordering, and age management /// automatically. It will skip groups that are too far behind to maintain the /// configured max delay. /// - /// Returns `None` when the track has ended. - pub async fn read(&mut self) -> Result, F::Error> { + /// Returns `None` when the track has ended. See [`Read`] for the items between frames; + /// an undeclared forward timestamp jump is not a gap, since nothing says media is missing. + pub async fn read(&mut self) -> Result, F::Error> { kio::wait(|waiter| self.poll_read(waiter)).await } + /// The next frame, passing over gaps and ends. + #[cfg(test)] + pub(crate) async fn read_frame(&mut self) -> Result, F::Error> { + loop { + match self.read().await? { + Some(Read::Frame { frame, .. }) => return Ok(Some(frame)), + Some(Read::Gap | Read::End { .. }) => continue, + None => return Ok(None), + } + } + } + /// Poll-based implementation of the read loop. /// /// Uses a single waiter that gets registered on all relevant kio channels, /// avoiding the need for `tokio::select!` or `FuturesUnordered`. - pub fn poll_read(&mut self, waiter: &kio::Waiter) -> Poll, F::Error>> { + pub fn poll_read(&mut self, waiter: &kio::Waiter) -> Poll, F::Error>> { + if let Some(read) = self.next.take() { + return Poll::Ready(Ok(Some(read))); + } loop { - match ready!(self.poll_event(waiter))? { - Some(Event::Frame(frame)) => return Poll::Ready(Ok(Some(frame))), - Some(Event::FrameEnd(end)) => { - if self.format.kind() == super::Kind::Audio { - self.end = Some(end); - } - } - Some(Event::GroupEnd) => continue, + let read = match ready!(self.poll_event(waiter))? { + Some(Event::Frame(frame)) => Read::Frame { + group: self.group, + frame, + }, + Some(Event::FrameEnd { end, opens: true }) => Read::End { group: self.group, end }, + // A trailing endpoint is the last frame's duration, which only the skip logic needs. + Some(Event::FrameEnd { opens: false, .. } | Event::GroupEnd) => continue, None => return Poll::Ready(Ok(None)), + }; + if std::mem::take(&mut self.gap) { + self.next = Some(read); + return Poll::Ready(Ok(Some(Read::Gap))); } + return Poll::Ready(Ok(Some(read))); } } @@ -212,11 +239,12 @@ impl Consumer { // first arrivals instead of assuming group 0. Skipping stays the budget's job. if self.startup { // NOTE: poll_min_timestamp buffers at least one frame per group and - // registers the waiter on the ones still empty. - let any_frame = self - .pending - .iter_mut() - .any(|group| matches!(group.poll_min_timestamp(waiter, &self.format), Poll::Ready(Ok(_)))); + // registers the waiter on the ones still empty. Any wire frame counts, so a + // paused track whose only group is a marker still reads its end. + let any_frame = self.pending.iter_mut().any(|group| { + let _ = group.poll_min_timestamp(waiter, &self.format); + !group.empty + }); if any_frame { self.current = self.pending.front().expect("a group has a frame").sequence; self.startup = false; @@ -257,9 +285,10 @@ impl Consumer { if self.live_edge.is_none_or(|(_, high)| ts.as_micros() > high.as_micros()) { self.live_edge = Some((seq, ts)); } + self.group = seq; return Poll::Ready(Ok(Some(Event::Frame(frame)))); } - Poll::Ready(Ok(Some(Event::FrameEnd(end)))) => { + Poll::Ready(Ok(Some(Event::FrameEnd { end, opens }))) => { let seq = group.group.sequence; if self .live_edge @@ -267,7 +296,8 @@ impl Consumer { { self.live_edge = Some((seq, end)); } - return Poll::Ready(Ok(Some(Event::FrameEnd(end)))); + self.group = seq; + return Poll::Ready(Ok(Some(Event::FrameEnd { end, opens }))); } Poll::Ready(Ok(Some(event))) => return Poll::Ready(Ok(Some(event))), // Still blocked on this group, don't skip it yet. @@ -299,6 +329,8 @@ impl Consumer { error = ?e, "current group evicted; skipping to next buffered group" ); + // Whatever the group still held is lost, so this is a skip like any other. + self.lose(); self.pending.pop_front(); self.current = self.pending.front().map_or(self.current + 1, |g| g.sequence); continue 'read; @@ -375,7 +407,7 @@ impl Consumer { && (finished || max_timestamp.saturating_sub(next_start) >= self.max_delay) { if !pts_contiguous(current_end.or(self.presented_end), next_start) { - self.bump_playhead(); + self.lose(); } self.current = front_sequence; self.note_group_edge(); @@ -400,9 +432,15 @@ impl Consumer { { let hole = !pts_contiguous(current_end.or(self.presented_end), next_start); let had_marker = self.pending.iter().take(new_idx).any(GroupBuffer::marker); + // Buffered frames are thrown away, and `current_end` counts them, so contiguity alone would hide them. + let discarded = self + .pending + .iter() + .take(new_idx) + .any(|group| !group.buffered.is_empty()); self.pending.drain(0..new_idx); - if hole || had_marker { - self.bump_playhead(); + if hole || had_marker || discarded { + self.lose(); } self.markers += u64::from(had_marker); let new_current = self.pending.front().map(|g| g.sequence).unwrap(); @@ -426,7 +464,7 @@ impl Consumer { if let Some(start) = next_start && !pts_contiguous(current_end.or(self.presented_end), start) { - self.bump_playhead(); + self.lose(); } self.current = front_sequence; self.note_group_edge(); @@ -443,7 +481,12 @@ impl Consumer { fn bump_playhead(&mut self) { self.discontinuity += 1; - self.end = None; + } + + // Media this reader will never deliver: the next item is preceded by a gap. + fn lose(&mut self) { + self.bump_playhead(); + self.gap = true; } fn note_group_edge(&mut self) { @@ -575,8 +618,8 @@ impl GroupBuffer { fn poll_read(&mut self, waiter: &kio::Waiter, format: &F) -> Poll, F::Error>> { loop { if self.markers.front().is_some_and(|(index, _)| *index <= self.delivered) { - let (_, end) = self.markers.pop_front().unwrap(); - return Poll::Ready(Ok(Some(Event::FrameEnd(end)))); + let (index, end) = self.markers.pop_front().unwrap(); + return Poll::Ready(Ok(Some(Event::FrameEnd { end, opens: index == 0 }))); } if let Some(frame) = self.buffered.pop_front() { self.delivered += 1; @@ -826,7 +869,7 @@ mod tests { async fn read_all(consumer: &mut Consumer) -> Result, crate::Error> { let mut frames = Vec::new(); loop { - match tokio::time::timeout(Duration::from_millis(200), consumer.read()).await { + match tokio::time::timeout(Duration::from_millis(200), consumer.read_frame()).await { Ok(Ok(Some(frame))) => frames.push(frame), Ok(Ok(None)) => break, Ok(Err(e)) => return Err(e), @@ -908,6 +951,135 @@ mod tests { group.finish().unwrap(); } + /// Every item until the track ends: a frame as its timestamp in µs, a gap, or an end. + async fn items(consumer: &mut Consumer) -> Vec { + let mut items = Vec::new(); + while let Some(read) = consumer.read().await.unwrap() { + items.push(match read { + Read::Frame { frame, .. } => frame.timestamp.as_micros().to_string(), + Read::Gap => "gap".to_string(), + Read::End { end, .. } => format!("end {}", end.as_micros()), + }); + } + items + } + + /// Joining a paused track, whose only served group is a marker, reads the end without + /// waiting for media to resume. + #[tokio::test(start_paused = true)] + async fn a_join_on_a_marker_group_reads_its_end() { + let mut track = track_producer("gaps", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let subscription = + track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))); + let mut reader = Consumer::new(subscription, Container::Legacy(crate::container::Kind::Audio)); + write_marker_group(&mut track, 16, ts(1_000_000)); + + let mut read = async || { + tokio::time::timeout(Duration::from_secs(1), reader.read()) + .await + .expect("the end is read while the track stays open") + .unwrap() + }; + assert!(matches!(read().await, Some(Read::Gap))); + assert!(matches!(read().await, Some(Read::End { group: 16, end }) if end == ts(1_000_000))); + } + + #[tokio::test] + async fn gaps_mark_the_join_and_holes_while_a_marker_reads_as_an_end() { + let mut track = track_producer("gaps", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let subscription = + track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))); + let mut reader = Consumer::new(subscription, Container::Legacy(crate::container::Kind::Audio)); + write_group(&mut track, 0, &[ts(0), ts(10_000)]); + write_group(&mut track, 2, &[ts(1_000_000), ts(1_010_000)]); + write_marker_group(&mut track, 3, ts(1_020_000)); + write_group(&mut track, 4, &[ts(2_000_000), ts(2_010_000)]); + track.finish().unwrap(); + assert_eq!( + items(&mut reader).await, + [ + "gap", + "0", + "10000", + "gap", + "1000000", + "1010000", + "end 1020000", + "2000000", + "2010000" + ] + ); + } + + #[tokio::test] + async fn a_latency_skip_is_a_gap() { + let mut track = track_producer("gaps", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let subscription = + track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))); + let mut reader = Consumer::new(subscription, Container::Legacy(crate::container::Kind::Audio)); + reader.max_delay = Duration::ZERO; + write_group(&mut track, 0, &[ts(0)]); + write_group(&mut track, 3, &[ts(1_000_000)]); + track.finish().unwrap(); + assert_eq!(items(&mut reader).await, ["gap", "0", "gap", "1000000"]); + } + + /// A video group's trailing duration marker is that frame's end, not a declared stop. + #[tokio::test] + async fn a_trailing_duration_marker_is_not_an_end() { + let track = track_producer("gaps", hang::container::track_info(hang::catalog::PRIORITY.video)); + let subscription = + track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))); + let mut reader = Consumer::new(subscription, Container::Legacy(crate::container::Kind::Video)); + let wire = Container::Legacy(crate::container::Kind::Video); + for (sequence, start) in [(0, 0), (1, 33_000)] { + let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let frame = |timestamp: u64, payload| Frame { + timestamp: ts(timestamp), + payload, + keyframe: timestamp == start, + duration: None, + }; + wire.write( + &mut group, + &[ + frame(start, Bytes::from_static(&[1])), + frame(start + 33_000, Bytes::new()), + ], + ) + .unwrap(); + group.finish().unwrap(); + } + track.finish().unwrap(); + assert_eq!(items(&mut reader).await, ["gap", "0", "33000"]); + } + + /// An audio terminal opens its group with the endpoint, then the codec's final packets. + #[tokio::test] + async fn an_audio_terminal_reads_its_end_before_the_final_packets() { + let mut track = track_producer("gaps", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let subscription = + track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))); + let mut reader = Consumer::new(subscription, Container::Legacy(crate::container::Kind::Audio)); + write_group(&mut track, 0, &[ts(0), ts(20_000)]); + let mut group = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let frame = |timestamp: u64, payload| Frame { + timestamp: ts(timestamp), + payload, + keyframe: false, + duration: None, + }; + Container::Legacy(crate::container::Kind::Audio) + .write( + &mut group, + &[frame(50_000, Bytes::new()), frame(40_000, Bytes::from_static(&[1]))], + ) + .unwrap(); + group.finish().unwrap(); + track.finish().unwrap(); + assert_eq!(items(&mut reader).await, ["gap", "0", "20000", "end 50000", "40000"]); + } + #[tokio::test] async fn empty_group_declares_a_discontinuity() { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); @@ -920,9 +1092,9 @@ mod tests { write_group(&mut track, 2, &[ts(1_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); assert_eq!(consumer.discontinuity(), 0); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(1_000_000)); assert_eq!(consumer.discontinuity(), 1); } @@ -942,11 +1114,11 @@ mod tests { write_group(&mut track, 4, &[ts(2_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(1_000_000)); assert_eq!(consumer.discontinuity(), 1, "the hole moves the playhead"); assert_eq!(consumer.markers(), 0, "but the timeline carried on"); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(2_000_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(2_000_000)); assert_eq!(consumer.discontinuity(), 2); assert_eq!(consumer.markers(), 1, "the marker declares a break"); } @@ -967,8 +1139,8 @@ mod tests { write_group(&mut track, 2, &[ts(1_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(1_000)); assert_eq!(consumer.discontinuity(), 0, "an empty group is not a marker"); } @@ -985,9 +1157,9 @@ mod tests { write_group(&mut track, 3, &[ts(1_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); assert_eq!(consumer.discontinuity(), 0); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(1_000_000)); assert_eq!( consumer.discontinuity(), 1, @@ -1176,13 +1348,13 @@ mod tests { .write(group, &[frame]) .unwrap(); }; - let read = |consumer: &mut Consumer| match consumer.poll_read(&waiter) { - Poll::Ready(Ok(Some(frame))) => Some(frame.timestamp), - Poll::Pending => None, - other => panic!( - "unexpected read: {:?}", - other.map(|r| r.map(|f| f.map(|f| f.timestamp))) - ), + let read = |consumer: &mut Consumer| loop { + match consumer.poll_read(&waiter) { + Poll::Ready(Ok(Some(Read::Frame { frame, .. }))) => break Some(frame.timestamp), + Poll::Ready(Ok(Some(Read::Gap))) => continue, + Poll::Pending => break None, + other => panic!("unexpected read: {other:?}"), + } }; // A 2.5 s group, half arrived, and the next one already 0.5 s in. @@ -1219,7 +1391,7 @@ mod tests { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Data)); write_group(&mut track, 0, &[ts(100_000)]); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(100_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(100_000)); write_group(&mut track, 1, &[ts(0)]); track.finish().unwrap(); let err = consumer.read().await.unwrap_err(); @@ -1253,10 +1425,10 @@ mod tests { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Data)); write_group(&mut track, 0, &[ts(100_000)]); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(100_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(100_000)); write_group(&mut track, 1, &[ts(100_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(100_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(100_000)); } /// A keyframe one frame below the previous group's last frame, but above its start, is an @@ -1269,7 +1441,7 @@ mod tests { // Read the first group before the second arrives: a blocked group that the next one // already covers is skipped for latency, which is not what this checks. for expected in [0, 33_000, 66_000] { - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(expected)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(expected)); } write_group(&mut track, 1, &[ts(50_000), ts(83_000)]); track.finish().unwrap(); @@ -1306,9 +1478,9 @@ mod tests { write_marker_group(&mut track, 1, ts(0)); write_group(&mut track, 2, &[ts(1_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); assert_eq!(consumer.discontinuity(), 0); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(1_000_000)); assert_eq!(consumer.discontinuity(), 1); } @@ -1318,11 +1490,11 @@ mod tests { let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Audio)); consumer.max_delay = Duration::ZERO; write_group(&mut track, 0, &[ts(0)]); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); write_marker_group(&mut track, 1, ts(0)); write_group(&mut track, 2, &[ts(1_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(1_000_000)); assert_eq!( consumer.discontinuity(), 1, @@ -1348,12 +1520,12 @@ mod tests { }], ) .unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); write_marker_group(&mut track, 1, ts(10_000)); write_group(&mut track, 2, &[ts(10_000)]); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(10_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(10_000)); assert_eq!( consumer.discontinuity(), 1, @@ -1367,10 +1539,10 @@ mod tests { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Data)); write_group(&mut track, 0, &[ts(100_000)]); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(100_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(100_000)); write_group(&mut track, 1, &[ts(200_000), ts(50_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(200_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(200_000)); let err = consumer.read().await.unwrap_err(); assert!(matches!( err, @@ -1456,7 +1628,7 @@ mod tests { group.finish().unwrap(); track.finish().unwrap(); - let frame = consumer.read().await.unwrap().unwrap(); + let frame = consumer.read_frame().await.unwrap().unwrap(); assert!(frame.keyframe, "the marker does not consume the first-media slot"); assert!(frame.duration.is_none(), "a leading marker has no previous frame"); } @@ -1754,7 +1926,7 @@ mod tests { route.requested_broadcast().await.unwrap().accept(&source); track.demand().used().await.unwrap(); if sequence == 1 { - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); } for sequence in 1..=sequence { write_group(&mut track, sequence, &[ts(sequence * 1_000_000)]); @@ -1782,7 +1954,7 @@ mod tests { matches!(consumer.poll_event(&waiter), Poll::Ready(Ok(Some(Event::GroupEnd)))), "the complete successor still emits its clean boundary" ); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(2_000_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(2_000_000)); } // ---- Eviction recovery (pause/resume) ---- @@ -1812,7 +1984,7 @@ mod tests { }], ) .unwrap(); - let first = consumer.read().await.unwrap().unwrap(); + let first = consumer.read_frame().await.unwrap().unwrap(); assert_eq!(first.timestamp, ts(0)); // A live group arrives far ahead -- sequences 1..4 never come (evicted). The @@ -1829,6 +2001,8 @@ mod tests { .expect("consumer hung on an evicted group / gap") .unwrap() .unwrap(); + assert!(matches!(next, Read::Gap), "the evicted media is a gap"); + let next = consumer.read_frame().await.unwrap().unwrap(); assert_eq!(next.timestamp, ts(150_000), "skipped the evicted gap to the live group"); } @@ -1855,7 +2029,7 @@ mod tests { // Reading must reach group 2 across the gap instead of waiting forever for 1. let reached = tokio::time::timeout(Duration::from_secs(1), async { loop { - let frame = consumer.read().await.unwrap().unwrap(); + let frame = consumer.read_frame().await.unwrap().unwrap(); if frame.timestamp == ts(200_000) { return; } @@ -1881,7 +2055,7 @@ mod tests { write_group(&mut track, 0, &[ts(0)]); write_group(&mut track, 2, &[ts(200_000)]); - let first = consumer.read().await.unwrap().expect("group 0 frame"); + let first = consumer.read_frame().await.unwrap().expect("group 0 frame"); assert_eq!(first.timestamp, ts(0)); // Group 0 is done; group 1's stream is still racing. Within the budget the @@ -2036,7 +2210,7 @@ mod tests { track.finish().unwrap(); // The first frame decodes; the malformed second frame must surface as an error. - let first = consumer.read().await; + let first = consumer.read_frame().await; assert!(matches!(first, Ok(Some(_))), "first frame should decode, got {first:?}"); let second = tokio::time::timeout(Duration::from_millis(200), consumer.read()) @@ -2143,7 +2317,7 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { + while let Some(frame) = consumer.read_frame().await.unwrap() { frames.push(frame); } frames @@ -2184,7 +2358,7 @@ mod tests { write_group(&mut track, 0, &[ts(0)]); track.finish().unwrap(); - let frame = consumer.read().await.unwrap().unwrap(); + let frame = consumer.read_frame().await.unwrap().unwrap(); assert_eq!(frame.timestamp, ts(0)); consumer.set_max_delay(Duration::from_millis(100)); @@ -2227,7 +2401,7 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { + while let Some(frame) = consumer.read_frame().await.unwrap() { frames.push(frame); } frames @@ -2288,7 +2462,7 @@ mod tests { let _frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { + while let Some(frame) = consumer.read_frame().await.unwrap() { frames.push(frame); } frames @@ -2434,7 +2608,7 @@ mod tests { // group is served, and nothing else arrives. write_group(&mut track, 100, &[ts(3_000_000)]); - let frame = tokio::time::timeout(Duration::from_millis(200), consumer.read()) + let frame = tokio::time::timeout(Duration::from_millis(200), consumer.read_frame()) .await .expect("an unfloored join must not wait out a gap below the served start") .unwrap() @@ -2618,7 +2792,7 @@ mod tests { track.finish().unwrap(); // Take the first frame only, leaving the second unread. - let first = consumer.read().await.unwrap().unwrap(); + let first = consumer.read_frame().await.unwrap().unwrap(); assert_eq!(first.timestamp, ts(0)); // Expiry aborts the finished group while the reader is still inside it. @@ -2693,7 +2867,7 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { + while let Some(frame) = consumer.read_frame().await.unwrap() { frames.push(frame); } frames @@ -2750,7 +2924,7 @@ mod tests { track.finish().unwrap(); let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { + while let Some(frame) = consumer.read_frame().await.unwrap() { frames.push(frame); } @@ -2792,7 +2966,7 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { + while let Some(frame) = consumer.read_frame().await.unwrap() { frames.push(frame); } frames @@ -2828,7 +3002,7 @@ mod tests { group1.finish().unwrap(); track.finish().unwrap(); let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { + while let Some(frame) = consumer.read_frame().await.unwrap() { frames.push(frame); } assert_eq!(frames.len(), 2); @@ -2867,7 +3041,7 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { + while let Some(frame) = consumer.read_frame().await.unwrap() { frames.push(frame); } frames @@ -2898,7 +3072,7 @@ mod tests { }], ) .unwrap(); - let frame = tokio::time::timeout(Duration::from_secs(1), consumer.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), consumer.read_frame()) .await .unwrap() .unwrap() @@ -2906,6 +3080,6 @@ mod tests { assert_eq!(frame.timestamp, ts(0)); write_marker(&mut group, ts(15_000)); let event = kio::wait(|waiter| consumer.poll_event(waiter)).await.unwrap(); - assert!(matches!(event, Some(Event::FrameEnd(end)) if end == ts(15_000))); + assert!(matches!(event, Some(Event::FrameEnd { end, opens: false }) if end == ts(15_000))); } } diff --git a/rs/moq-mux/src/container/flv/import_test.rs b/rs/moq-mux/src/container/flv/import_test.rs index 96fa2c287f2..3f0b3b29349 100644 --- a/rs/moq-mux/src/container/flv/import_test.rs +++ b/rs/moq-mux/src/container/flv/import_test.rs @@ -181,7 +181,7 @@ async fn import_emits_frames() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = decoder.read().await.unwrap().expect("a video frame"); + let frame = decoder.read_frame().await.unwrap().expect("a video frame"); assert!(frame.keyframe); // The payload is the length-prefixed NALU, carried through verbatim. assert_eq!(frame.payload.as_ref(), &[0, 0, 0, 5, 0x65, 0x88, 0x84, 0x21, 0x00]); @@ -207,7 +207,7 @@ async fn public_container_preserves_loc_for_flv() { track, crate::catalog::hang::Container::Loc(crate::container::Kind::Data), ); - let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read()) + let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read_frame()) .await .unwrap() .unwrap() @@ -504,7 +504,7 @@ async fn import_enhanced_av1() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = decoder.read().await.unwrap().expect("an AV1 frame"); + let frame = decoder.read_frame().await.unwrap().expect("an AV1 frame"); assert!(frame.keyframe); assert_eq!(frame.payload.as_ref(), payload); } @@ -638,7 +638,7 @@ async fn import_reports_negative_pts_and_can_resume() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = decoder.read().await.unwrap().expect("the good frame"); + let frame = decoder.read_frame().await.unwrap().expect("the good frame"); assert_eq!(frame.timestamp.as_millis(), 10); } @@ -678,7 +678,7 @@ async fn import_enhanced_hvc1_applies_composition_time() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = decoder.read().await.unwrap().expect("a video frame"); + let frame = decoder.read_frame().await.unwrap().expect("a video frame"); assert_eq!(frame.timestamp.as_millis(), 17); } diff --git a/rs/moq-mux/src/container/fmp4/export.rs b/rs/moq-mux/src/container/fmp4/export.rs index 81094b85080..686107e362c 100644 --- a/rs/moq-mux/src/container/fmp4/export.rs +++ b/rs/moq-mux/src/container/fmp4/export.rs @@ -563,7 +563,7 @@ impl Export { track.pending = Some(frame); break; } - Poll::Ready(Some(Event::FrameEnd(end))) => { + Poll::Ready(Some(Event::FrameEnd { end, .. })) => { if track.is_video && let Some(last) = track.buffer.last_mut() && last.duration.is_none() diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index e1a6b0b13f7..ea2b2f7c503 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -1815,7 +1815,7 @@ async fn webvtt_samples_become_timed_text_with_source_metadata() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = reader.read().await.unwrap().unwrap(); + let frame = reader.read_frame().await.unwrap().unwrap(); assert_eq!(frame.timestamp.as_micros(), 250_000); assert_eq!( &frame.payload[..], @@ -1885,7 +1885,7 @@ async fn webvtt_kind_and_lone_cr_terminators() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = reader.read().await.unwrap().unwrap(); + let frame = reader.read_frame().await.unwrap().unwrap(); assert_eq!( &frame.payload[..], b"WEBVTT\nKind: captions\n\n00:00:00.250 --> 00:00:01.500\nHello\nworld\n\n" @@ -1996,6 +1996,6 @@ async fn webvtt_sub_millisecond_cue_stays_positive() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = reader.read().await.unwrap().unwrap(); + let frame = reader.read_frame().await.unwrap().unwrap(); assert_eq!(&frame.payload[..], b"WEBVTT\n\n00:00:00.250 --> 00:00:00.251\nhi\n\n"); } diff --git a/rs/moq-mux/src/container/group.rs b/rs/moq-mux/src/container/group.rs index db0d677981e..31fcd7d84e4 100644 --- a/rs/moq-mux/src/container/group.rs +++ b/rs/moq-mux/src/container/group.rs @@ -144,11 +144,11 @@ mod tests { subscription, Hang::new(&config, crate::container::Kind::Data).unwrap(), ); - let first = live.read().await.unwrap().unwrap(); + let first = live.read_frame().await.unwrap().unwrap(); assert!(first.payload.is_empty()); assert!(first.keyframe); - assert_eq!(live.read().await.unwrap().unwrap().payload, b"data".as_slice()); - assert!(live.read().await.unwrap().is_none()); + assert_eq!(live.read_frame().await.unwrap().unwrap().payload, b"data".as_slice()); + assert!(live.read_frame().await.unwrap().is_none()); let group = consumer.track("data").unwrap().fetch_group(0, None).await.unwrap(); let mut fetched = GroupConsumer::new(group, Hang::new(&config, crate::container::Kind::Data).unwrap()); assert!(fetched.read().await.unwrap().unwrap().payload.is_empty()); diff --git a/rs/moq-mux/src/container/mkv/import_test.rs b/rs/moq-mux/src/container/mkv/import_test.rs index f931a97ff02..9a1e8f080bb 100644 --- a/rs/moq-mux/src/container/mkv/import_test.rs +++ b/rs/moq-mux/src/container/mkv/import_test.rs @@ -263,7 +263,7 @@ async fn public_container_preserves_loc_for_mkv() { track, crate::catalog::hang::Container::Loc(crate::container::Kind::Data), ); - let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read()) + let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read_frame()) .await .unwrap() .unwrap() @@ -683,7 +683,7 @@ async fn subtitle_languages_and_empty_tracks_are_published() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = reader.read().await.unwrap().unwrap(); + let frame = reader.read_frame().await.unwrap().unwrap(); assert_eq!( &frame.payload[..], b"WEBVTT\n\n00:00:00.250 --> 00:00:01.500\nHello\n\n" @@ -730,7 +730,7 @@ async fn subtitle_duration_survives_as_a_timed_cue() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = reader.read().await.unwrap().unwrap(); + let frame = reader.read_frame().await.unwrap().unwrap(); assert_eq!(frame.timestamp.as_micros(), 250_000); assert_eq!( &frame.payload[..], @@ -786,7 +786,7 @@ async fn webvtt_preserves_cue_metadata_and_relative_timestamps() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = reader.read().await.unwrap().unwrap(); + let frame = reader.read_frame().await.unwrap().unwrap(); assert_eq!(&frame.payload[..], b"WEBVTT\n\nSTYLE\n::cue { color: lime; }\n\nNOTE source\n\ncue-1\n00:00:00.250 --> 00:00:01.500 align:start\nHello <00:00:00.750>world\n\n"); } @@ -904,11 +904,11 @@ async fn laced_opus_frames() { let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::try_from(config).unwrap()); for (index, packet) in packets.iter().enumerate() { - let frame = reader.read().await.unwrap().expect("one frame per packet"); + let frame = reader.read_frame().await.unwrap().expect("one frame per packet"); assert_eq!(frame.payload.as_ref(), *packet, "{lacing:?}, grouped={grouped}"); assert_eq!(frame.timestamp.as_micros(), 107_000 + index as u128 * 20_000); } - assert!(reader.read().await.unwrap().is_none()); + assert!(reader.read_frame().await.unwrap().is_none()); } } } diff --git a/rs/moq-mux/src/container/mod.rs b/rs/moq-mux/src/container/mod.rs index b15f5d8839f..26602015cdc 100644 --- a/rs/moq-mux/src/container/mod.rs +++ b/rs/moq-mux/src/container/mod.rs @@ -46,6 +46,30 @@ pub enum Kind { Data, } +/// One item of a [`Consumer`] read, in delivery order. +#[derive(Clone, Debug)] +pub enum Read { + /// A media frame from `group`. + Frame { + /// The group's sequence number. Only equality means anything: numbers may skip. + group: u64, + frame: Frame, + }, + /// Media before the next item will never reach this reader: it joined mid-track, or a + /// group was skipped, evicted, or never arrived. A decoder resumes from here without + /// the references or state that media built. + Gap, + /// The publisher declared that its media stops at `end`: it paused, switched source, or + /// finished. Frames that follow from the same `group` are the stopped encoder's final + /// packets (an audio codec's drain), whose output at or past `end` is padding. The next + /// group starts fresh. + End { + /// The group's sequence number. Only equality means anything: numbers may skip. + group: u64, + end: moq_net::Timestamp, + }, +} + /// A decoded media frame: timestamp, payload bytes, keyframe flag. /// /// `payload` is the raw codec bitstream that gets handed to the decoder. diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index 6958adf6cf8..f2afb8b9991 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -1717,7 +1717,7 @@ mod tests { let mut consumer = crate::container::Consumer::new(subscriber, Container::Legacy(crate::container::Kind::Video)); let mut timestamps = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { + while let Some(frame) = consumer.read_frame().await.unwrap() { timestamps.push(frame.timestamp.as_micros()); } assert_eq!(timestamps, [1_000_000, 1_040_000, 1_050_000]); diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index 9c25239af38..2fcc76aad27 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -315,7 +315,7 @@ impl ExportSource { loop { match ready!(self.poll_event(waiter))? { Some(Event::Frame(frame)) => return Poll::Ready(Ok(Some(frame))), - Some(Event::GroupEnd | Event::FrameEnd(_)) => continue, + Some(Event::GroupEnd | Event::FrameEnd { .. }) => continue, None => return Poll::Ready(Ok(None)), } } diff --git a/rs/moq-mux/src/container/test_util.rs b/rs/moq-mux/src/container/test_util.rs index 5b4829cada1..9c65a3995eb 100644 --- a/rs/moq-mux/src/container/test_util.rs +++ b/rs/moq-mux/src/container/test_util.rs @@ -152,7 +152,7 @@ pub(crate) async fn published( let track = consumer.track(&name).unwrap().subscribe(replay).await.unwrap(); let mut reader = crate::container::Consumer::new(track, container); let mut timestamps = Vec::new(); - while let Some(frame) = tokio::time::timeout(std::time::Duration::from_secs(5), reader.read()) + while let Some(frame) = tokio::time::timeout(std::time::Duration::from_secs(5), reader.read_frame()) .await .expect("the importer finished its tracks") .unwrap() diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 214d4b870c1..20deb83a167 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -1431,7 +1431,7 @@ async fn export_scte35_roundtrip() { .unwrap(); let mut scte_reader = crate::container::Consumer::new(track, HangContainer::Legacy(crate::container::Kind::Data)); let frame = scte_reader - .read() + .read_frame() .await .unwrap() .expect("no SCTE-35 frame after round-trip"); @@ -1550,7 +1550,7 @@ async fn export_pes_verbatim_roundtrip() { .unwrap(); let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy(crate::container::Kind::Data)); let frame = reader - .read() + .read_frame() .await .unwrap() .expect("no verbatim PES frame after round-trip"); @@ -1628,7 +1628,7 @@ async fn read_frames(consumer: &moq_net::broadcast::Consumer, name: &str, kind: .unwrap(); let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy(kind)); let mut frames = Vec::new(); - while let Ok(res) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { + while let Ok(res) = tokio::time::timeout(Duration::from_millis(50), reader.read_frame()).await { let Some(frame) = res.unwrap() else { break }; frames.push(frame.payload.to_vec()); } @@ -2209,7 +2209,7 @@ async fn read_cues(consumer: &moq_net::broadcast::Consumer, name: &str) -> Vec<( .unwrap(); let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy(crate::container::Kind::Data)); let mut cues = Vec::new(); - while let Ok(res) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { + while let Ok(res) = tokio::time::timeout(Duration::from_millis(50), reader.read_frame()).await { let Some(frame) = res.unwrap() else { break }; cues.push((frame.payload.to_vec(), frame.timestamp)); } diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 4edb6a118c9..36a4cd7fc5f 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -3885,7 +3885,7 @@ pub(super) mod test { .await .unwrap(); let mut reader = Consumer::new(track, Container::Legacy(crate::container::Kind::Data)); - let frame = tokio::time::timeout(Duration::from_secs(1), reader.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), reader.read_frame()) .await .expect("cue read timed out") .unwrap() @@ -4554,7 +4554,7 @@ pub(super) mod test { crate::catalog::hang::Container::Legacy(crate::container::Kind::Audio), ); let mut frames = Vec::new(); - while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { + while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read_frame()).await { frames.push(frame); } frames @@ -4608,7 +4608,7 @@ pub(super) mod test { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let mut frames = Vec::new(); - while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { + while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read_frame()).await { frames.push(frame); } frames @@ -5432,7 +5432,7 @@ pub(super) mod test { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let published = tokio::time::timeout(Duration::from_millis(50), reader.read()).await; + let published = tokio::time::timeout(Duration::from_millis(50), reader.read_frame()).await; assert!( matches!(published, Ok(Ok(Some(_)))), "a PID that never published held the catalog shut: {published:?}" @@ -5822,7 +5822,7 @@ pub(super) mod test { .await .unwrap(); let mut reader = Consumer::new(track, Container::Legacy(crate::container::Kind::Data)); - let frame = tokio::time::timeout(Duration::from_secs(1), reader.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), reader.read_frame()) .await .expect("cue read timed out") .unwrap() @@ -6008,7 +6008,7 @@ pub(super) mod test { .await .unwrap(); let mut reader = Consumer::new(track, Container::Legacy(crate::container::Kind::Data)); - let frame = tokio::time::timeout(Duration::from_secs(1), reader.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), reader.read_frame()) .await .expect("verbatim read timed out") .unwrap() @@ -6099,7 +6099,8 @@ pub(super) mod test { crate::catalog::hang::Container::Legacy(crate::container::Kind::Audio), ); let mut frames = Vec::new(); - while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await + while let Ok(Ok(Some(frame))) = + tokio::time::timeout(std::time::Duration::from_millis(50), reader.read_frame()).await { frames.push(frame); } @@ -6707,7 +6708,7 @@ pub(super) mod test { let track = consumer.track(name).unwrap().subscribe(subscription).await.unwrap(); let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy(kind)); let mut frames = Vec::new(); - while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { + while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read_frame()).await { if !frame.payload.is_empty() { frames.push(frame); } diff --git a/rs/moq-mux/src/container/ts/import_test.rs b/rs/moq-mux/src/container/ts/import_test.rs index 03a6217b0a1..5a31923c29b 100644 --- a/rs/moq-mux/src/container/ts/import_test.rs +++ b/rs/moq-mux/src/container/ts/import_test.rs @@ -96,7 +96,7 @@ async fn public_container_preserves_loc_for_ts() { track, crate::catalog::hang::Container::Loc(crate::container::Kind::Data), ); - let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read()) + let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read_frame()) .await .unwrap() .unwrap() @@ -262,7 +262,7 @@ async fn import_opus_frames() { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let mut frames = Vec::new(); - while let Ok(res) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { + while let Ok(res) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read_frame()).await { let Some(frame) = res.unwrap() else { break }; frames.push(frame.payload); } @@ -541,7 +541,9 @@ async fn survives_midstream_join() { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let mut frames = Vec::new(); - while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { + while let Ok(Ok(Some(frame))) = + tokio::time::timeout(std::time::Duration::from_millis(50), reader.read_frame()).await + { frames.push(frame); } assert_eq!(frames.len(), 1, "expected only the post-join IDR, got {}", frames.len()); @@ -595,7 +597,9 @@ async fn kyrion_dirtystart_extracts_real_cues() { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let mut cues = Vec::new(); - while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { + while let Ok(Ok(Some(frame))) = + tokio::time::timeout(std::time::Duration::from_millis(50), reader.read_frame()).await + { cues.push((frame.payload.to_vec(), frame.timestamp)); } assert_eq!(cues.len(), 6, "expected the six real splice_inserts"); diff --git a/rs/moq-mux/src/container/ts/programs.rs b/rs/moq-mux/src/container/ts/programs.rs index 49ab1ffda56..5324bd19e8f 100644 --- a/rs/moq-mux/src/container/ts/programs.rs +++ b/rs/moq-mux/src/container/ts/programs.rs @@ -376,7 +376,11 @@ mod test { let (name, config) = renditions[0]; let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); let container = Container::try_from(config).unwrap(); - let frame = Consumer::new(track, container).read().await.unwrap().expect("a frame"); + let frame = Consumer::new(track, container) + .read_frame() + .await + .unwrap() + .expect("a frame"); assert!(fills.contains(&frame.payload[4]), "{path} carries only its own program"); let skew = Duration::from_secs(2); let wall = catalog diff --git a/rs/moq-mux/src/import/track.rs b/rs/moq-mux/src/import/track.rs index 9aca70feae6..58582eb5059 100644 --- a/rs/moq-mux/src/import/track.rs +++ b/rs/moq-mux/src/import/track.rs @@ -1050,7 +1050,7 @@ mod tests { .decode(&payload, Some(Timestamp::from_micros(1_000).unwrap())) .unwrap(); - let frame = tokio::time::timeout(Duration::from_secs(1), media.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), media.read_frame()) .await .unwrap() .unwrap() @@ -1378,7 +1378,7 @@ mod tests { .decode(&au, Some(Timestamp::from_micros(1_000).unwrap())) .unwrap(); - let frame = tokio::time::timeout(Duration::from_secs(1), media.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), media.read_frame()) .await .unwrap() .unwrap() diff --git a/rs/moq-rtc/src/codec/mod.rs b/rs/moq-rtc/src/codec/mod.rs index f83ca447faf..20d24b7a712 100644 --- a/rs/moq-rtc/src/codec/mod.rs +++ b/rs/moq-rtc/src/codec/mod.rs @@ -230,8 +230,11 @@ impl Track { /// Pull the next RTP-ready frame. Returns `None` when the track ends. pub async fn next(&mut self) -> Result> { loop { - let Some(frame) = self.consumer.read().await? else { - return Ok(None); + let frame = match self.consumer.read().await? { + Some(moq_mux::container::Read::Frame { frame, .. }) => frame, + // RTP sequence numbers and timestamps already show the hole to a receiver. + Some(moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. }) => continue, + None => return Ok(None), }; let payload = match &self.convert { TrackConvert::Passthrough => frame.payload, diff --git a/rs/moq-srt/src/ts.rs b/rs/moq-srt/src/ts.rs index 8a2d36cb41e..2766366623c 100644 --- a/rs/moq-srt/src/ts.rs +++ b/rs/moq-srt/src/ts.rs @@ -647,10 +647,9 @@ mod tests { let track = broadcast.track(name.as_str()).unwrap().subscribe(None).await.unwrap(); let mut reader = moq_mux::container::Consumer::new(track, Container::Legacy(moq_mux::container::Kind::Data)); - let cue = timeout(Duration::from_secs(5), reader.read()) + let cue = timeout(Duration::from_secs(5), next_frame(&mut reader)) .await .expect("cue read timed out") - .unwrap() .expect("a published cue section"); let expected_cue = cue.payload; assert_eq!(expected_cue[0], 0xFC, "a verbatim splice_info_section (table_id 0xFC)"); @@ -696,10 +695,9 @@ mod tests { .expect("SRT egress preserves the SCTE-35 track"); let track = roundtrip_consumer.track(name).unwrap().subscribe(None).await.unwrap(); let mut reader = moq_mux::container::Consumer::new(track, Container::Legacy(moq_mux::container::Kind::Data)); - let cue = timeout(Duration::from_secs(5), reader.read()) + let cue = timeout(Duration::from_secs(5), next_frame(&mut reader)) .await .expect("round-trip cue read timed out") - .unwrap() .expect("SRT egress preserves a cue section"); assert_eq!(cue.payload[0], 0xFC, "the round-trip cue is a splice_info_section"); assert_eq!( @@ -707,4 +705,14 @@ mod tests { "SRT egress preserves the complete cue section" ); } + + /// The next frame, passing over gaps and ends. + async fn next_frame(reader: &mut moq_mux::container::Consumer) -> Option { + loop { + match reader.read().await.unwrap()? { + moq_mux::container::Read::Frame { frame, .. } => return Some(frame), + moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. } => continue, + } + } + } } diff --git a/rs/moq-video/src/capture/wgc.rs b/rs/moq-video/src/capture/wgc.rs index ce0c7ede8c9..f2d44221d95 100644 --- a/rs/moq-video/src/capture/wgc.rs +++ b/rs/moq-video/src/capture/wgc.rs @@ -327,7 +327,11 @@ mod tests { let (surface, timestamp) = delivery.next(now).unwrap().unwrap(); let frame = crate::Frame::new(crate::Surface::I420(surface.clone()), timestamp); producer.publish(&encoder.encode(&frame).unwrap()).unwrap(); - let received = subscriber.read().await.unwrap().unwrap(); + let received = loop { + if let moq_mux::container::Read::Frame { frame, .. } = subscriber.read().await.unwrap().unwrap() { + break frame; + } + }; assert_eq!(received.timestamp.as_micros(), u128::from(step) * 20_000); } } diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index 4add91aca90..327e3988017 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -3,6 +3,7 @@ use std::collections::VecDeque; use hang::catalog::VideoConfig; +use moq_mux::container::Read; use super::decoder::Config; use super::sink::Sink; @@ -85,8 +86,6 @@ pub struct Consumer { pending: VecDeque, /// Whether the ended track's decoder has already been drained. drained: bool, - /// Last container playhead generation observed. - discontinuity: u64, } impl Consumer { @@ -140,7 +139,6 @@ impl Consumer { track, pending: VecDeque::new(), drained: false, - discontinuity: 0, }) } @@ -166,14 +164,13 @@ impl Consumer { return Ok(None); } - let mux_frame = self.track.read().await?; - let discontinuity = self.track.discontinuity(); - if discontinuity != self.discontinuity { - // A playhead event re-applies startup delay and skip; the next group - // already starts on a keyframe with parameter sets, so the decoder is - // not flushed. - self.discontinuity = discontinuity; - } + let mux_frame = match self.track.read().await? { + Some(Read::Frame { frame, .. }) => Some(frame), + // The next group starts on a keyframe with parameter sets, so neither a + // gap nor a declared end flushes the decoder. + Some(Read::Gap | Read::End { .. }) => continue, + None => None, + }; let Some(mux_frame) = mux_frame else { // The flag goes up only once the tail is in hand, so a read diff --git a/rs/moq-video/src/decode/decoder.rs b/rs/moq-video/src/decode/decoder.rs index 9fa8ba8ec32..5ce8d34910e 100644 --- a/rs/moq-video/src/decode/decoder.rs +++ b/rs/moq-video/src/decode/decoder.rs @@ -598,7 +598,12 @@ mod tests { .unwrap(); let mut track = moq_mux::container::Consumer::new(track, moq_mux::catalog::hang::Container::try_from(config).unwrap()); - let sample = track.read().await.unwrap().expect("an imported sample"); + let sample = loop { + let read = track.read().await.unwrap().expect("an imported sample"); + if let moq_mux::container::Read::Frame { frame, .. } = read { + break frame; + } + }; assert!(sample.keyframe); let decoder = diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index 44be26c103a..3241d3c096a 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -1208,8 +1208,12 @@ mod tests { } async fn read(track: &mut moq_mux::container::Consumer) -> u64 { - let frame = track.read().await.unwrap().expect("a published frame"); - u64::try_from(frame.timestamp.as_micros()).unwrap() + loop { + let read = track.read().await.unwrap().expect("a published frame"); + if let moq_mux::container::Read::Frame { frame, .. } = read { + return u64::try_from(frame.timestamp.as_micros()).unwrap(); + } + } } /// Read the next frame not already in `seen`: a resubscription replays retained groups first. diff --git a/test/interop/clients/js-native/subscribe.ts b/test/interop/clients/js-native/subscribe.ts index 4a11e8f3677..b7b3b17f982 100644 --- a/test/interop/clients/js-native/subscribe.ts +++ b/test/interop/clients/js-native/subscribe.ts @@ -129,7 +129,7 @@ async function run(): Promise { for (;;) { const next = await consumer.next(); if (!next) break; - const bytes = next.frame?.payload.byteLength ?? 0; + const bytes = next.kind === "frame" ? next.frame.payload.byteLength : 0; if (bytes > 0) { // The harness judges success by this marker, not the exit code: the // @moq/web-transport NAPI addon can segfault during the runtime's exit