diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index c629b3c0f7e..1bd9e2641bf 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -133,7 +133,7 @@ export class Backpressure { this.#release(); } - /** Note that the decoder closed, dropping every frame still decoding. */ + /** Note that the decoder closed or reset, dropping every frame still decoding. */ dropped(): void { this.#decoding = []; this.#release(); @@ -344,7 +344,7 @@ export class AudioBuffer { this.#backpressure.decoded(end, landed); } - /** Note that the decoder closed, dropping every frame still decoding. */ + /** Note that the decoder closed or reset, dropping every frame still decoding. */ dropped(): void { this.#backpressure.dropped(); } diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 49f72ccd810..a755c447131 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -154,12 +154,18 @@ let codecs = 0; let codecErrors: AudioDecoderInit["error"][] = []; // Thrown by the next `decode`, as WebCodecs does synchronously for a chunk it refuses. let decodeThrows: Error | undefined; +let deferAudioOutput = false; +let pendingAudio: (() => void)[] = []; +let audioInputs: number[] = []; beforeEach(() => { frameTimestamp = 0; codecs = 0; codecErrors = []; decodeThrows = undefined; + deferAudioOutput = false; + pendingAudio = []; + audioInputs = []; contexts = []; worklets = []; moduleLoaded = Promise.resolve(); @@ -179,12 +185,16 @@ beforeEach(() => { } reset() { this.needsKey = true; + pendingAudio = []; } decode(chunk: { timestamp: number; type: EncodedAudioChunkType }) { if (decodeThrows) throw decodeThrows; if (this.needsKey && chunk.type !== "key") throw new DOMException("key chunk required", "DataError"); this.needsKey = false; - this.init.output(new FakeData(chunk.timestamp) as unknown as AudioData); + audioInputs.push(chunk.timestamp); + const emit = () => this.init.output(new FakeData(chunk.timestamp) as unknown as AudioData); + if (deferAudioOutput) pendingAudio.push(emit); + else emit(); } close() { this.state = "closed"; @@ -380,6 +390,40 @@ async function play(initial: Delay, buffered = false, cmaf = false) { }; } +for (const cmaf of [false, true]) { + it(`a ${cmaf ? "CMAF" : "legacy"} gap releases accounting for codec outputs it discarded`, async () => { + const clock = new Manual(); + const installed = clock.install(); + deferAudioOutput = true; + const playback = await play(Milli(100), true, cmaf); + try { + for (let n = 0; n < 6; n++) { + playback.push({ ...frame(), group: n === 0 ? 0 : 1 }); + await microtasks(); + } + expect(audioInputs).toHaveLength(6); + expect(pendingAudio).toHaveLength(6); + expect(playback.inserted()).toHaveLength(0); + playback.push({ kind: "gap", timestamp: Micro(140_000), duration: Micro(20_000) }); + await microtasks(); + expect(pendingAudio).toHaveLength(0); + deferAudioOutput = false; + frameTimestamp = 160_000; + const resumed = frame(); + playback.push({ ...resumed, group: 2 }); + await microtasks(); + expect(audioInputs).toHaveLength(7); + expect(playback.inserted()).toContain(resumed.frame.timestamp); + await playback.play(5); + expect(playback.sync.playing.peek()).toBe(true); + } finally { + playback.close(); + await microtasks(); + installed.close(); + } + }); +} + it("decodes an audio packet inside a group after a gap", async () => { const playback = await play(Milli(100)); try { @@ -683,20 +727,42 @@ describe("Decoder across a rendition's absence", () => { } }); - it("keeps the graph when only the container changes", async () => { - const playback = await play(Milli(100)); - try { - await playback.play(); - playback.restore({ container: { kind: "loc" } }); - await microtasks(); - await playback.play(); - - expect(playback.subscriptions()).toBe(2); - expect(contexts).toHaveLength(1); - } finally { - playback.close(); - } - }); + for (const [name, patch] of Object.entries({ + container: { container: { kind: "loc" } }, + codec: { codec: "mp4a.40.2" }, + })) { + it(`keeps compatible PCM and Sync when the ${name} changes on the same instance`, async () => { + const clock = new Manual(); + const installed = clock.install(); + const playback = await play(Milli(100), true); + const reset = spyOn(playback.sync, "reset"); + try { + await playback.play(7); + const ring = playback.rings().at(-1); + if (!ring) throw new Error("missing audio ring"); + const held = ring.length; + const reference = playback.sync.out.reference.peek(); + expect(held).toBeGreaterThan(0); + playback.restore(patch); + await microtasks(); + expect(playback.subscriptions()).toBe(2); + expect(playback.codecs()).toBe(2); + expect(contexts).toHaveLength(1); + expect(playback.rings()).toEqual([ring]); + expect(ring.length).toBe(held); + expect(playback.resets()).toBe(0); + expect(reset).not.toHaveBeenCalled(); + expect(playback.sync.out.reference.peek()).toBe(reference); + expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(960); + await playback.play(); + expect(reset).not.toHaveBeenCalled(); + } finally { + playback.close(); + await microtasks(); + installed.close(); + } + }); + } it("rebuilds the graph on a new channel count", async () => { const playback = await play(Milli(100)); @@ -930,6 +996,14 @@ async function livePlayback(burst: number) { return { sync, error: decoder.out.error, + arrivals: () => decoder.arrivals(), + // Lose one whole group without changing the following media timestamps or arrival cadence. + async hole() { + if (held.length) throw new Error("a hole needs an empty delivery burst"); + sequence++; + frame++; + await clock.advance(FRAME); + }, counts, ring, get burst() { @@ -993,6 +1067,36 @@ async function livePlayback(burst: number) { }; } +it("a missing live group re-primes without clearing decoded PCM or resetting Sync", async () => { + const playback = await livePlayback(1); + const reset = spyOn(playback.sync, "reset"); + const prime = spyOn(AudioDecoder.prototype, "reset"); + const arrivals = playback.arrivals(); + const received: { kind: string }[] = []; + const reading = (async () => { + for await (const arrival of arrivals) received.push(arrival); + })(); + try { + await playback.run(1000); + const ring = playback.ring(); + expect(ring?.stalled).toBe(false); + const buffered = ring?.length; + await playback.hole(); + expect(ring?.length).toBe(buffered); + await playback.run(2000); + expect(received.some((arrival) => arrival.kind === "gap")).toBe(true); + expect(prime).toHaveBeenCalledTimes(1); + expect(reset).not.toHaveBeenCalled(); + expect(playback.ring()).toBe(ring); + expect(ring?.stalled).toBe(false); + expect(playback.error.peek()).toBeUndefined(); + } finally { + arrivals.close(); + await reading; + await playback.close(); + } +}); + describe("Decoder across a rename", () => { // A codec change renames the audio track, and the replacement's consumer reads the same path. // Clearing the estimate at teardown and restarting it from the cold-start prior dropped the @@ -1179,30 +1283,47 @@ it("waits for video support before allowing audio-only startup", async () => { } }); -it("does not count flushed audio toward a new startup floor after a gap", async () => { - const playback = await play(Milli(100), true); - try { - await playback.play(7); - expect(playback.sync.playing.peek()).toBe(true); - const openingRing = playback.rings().at(-1); - if (!openingRing) throw new Error("missing audio ring"); - openingRing.read([new Float32Array(960), new Float32Array(960)]); - await microtasks(); - playback.push({ kind: "gap", timestamp: Micro(160_000), duration: Micro(840_000) }); - await microtasks(); - frameTimestamp = 1_000_000; - await playback.play(1); - expect(playback.sync.playing.peek()).toBe(false); - const ring = playback.rings().at(-1); - if (!ring) throw new Error("missing audio ring"); - expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(0); - await playback.play(5); - expect(playback.sync.playing.peek()).toBe(true); - } finally { - playback.close(); - await microtasks(); - } -}); +for (const cmaf of [false, true]) { + it(`a ${cmaf ? "CMAF" : "legacy"} hole re-primes the codec while keeping PCM and the clock`, async () => { + const clock = new Manual(); + const installed = clock.install(); + const playback = await play(Milli(100), true, cmaf); + const reset = spyOn(playback.sync, "reset"); + const prime = spyOn(AudioDecoder.prototype, "reset"); + try { + await playback.play(7); + expect(playback.sync.playing.peek()).toBe(true); + const ring = playback.rings().at(-1); + if (!ring) throw new Error("missing audio ring"); + ring.read([new Float32Array(960), new Float32Array(960)]); + await microtasks(); + const held = ring.length; + const reference = playback.sync.out.reference.peek(); + const resets = playback.resets(); + const parked = playback.sync.wait(Milli(1000)); + expect(held).toBeGreaterThan(0); + playback.push({ kind: "gap", timestamp: Micro(160_000), duration: Micro(20_000) }); + await microtasks(); + expect(prime).toHaveBeenCalledTimes(1); + expect(reset).not.toHaveBeenCalled(); + expect(playback.resets()).toBe(resets); + expect(ring.length).toBe(held); + expect(playback.sync.out.reference.peek()).toBe(reference); + expect(playback.sync.playing.peek()).toBe(true); + expect(Bun.peek.status(parked)).toBe("pending"); + frameTimestamp = 160_000; + await playback.play(1); + expect(playback.sync.playing.peek()).toBe(true); + expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(960); + await clock.advance(Nano.fromMilli(Milli(1200))); + expect(await parked).toBe(true); + } finally { + playback.close(); + await microtasks(); + installed.close(); + } + }); +} it.each([100, 1000])( "closing during shared startup at %dms releases the audio graph and backpressured reader", diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 1a6c8792cee..b1ccf7149dd 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -512,9 +512,7 @@ export class Decoder { if (!(await play(order.flush()))) return; if (!next) break; if (this.#onRead(next)) { - decoder.reset(); - decoder.configure(decoderConfig); - order.reset(); + this.#reprime(decoder, decoderConfig, order); } if (next.kind === "gap") this.#arrivals.write(gap(next)); continue; @@ -601,9 +599,7 @@ export class Decoder { if (!(await play(order.flush()))) return; if (!next) break; if (this.#onRead(next)) { - decoder.reset(); - decoder.configure(decoderConfig); - order.reset(); + this.#reprime(decoder, decoderConfig, order); } if (next.kind === "gap") this.#arrivals.write(gap(next)); continue; @@ -639,9 +635,7 @@ export class Decoder { ): Promise { for (const read of reads) { if (this.#onRead(read)) { - decoder.reset(); - decoder.configure(config); - order.reset(); + this.#reprime(decoder, config, order); } if (!order.admit(read)) continue; if (!(await this.#decode(effect, decoder, read.frame))) return false; @@ -649,6 +643,14 @@ export class Decoder { return true; } + // A reset discards queued codec output, so release its accounting without clearing retained PCM. + #reprime(decoder: AudioDecoder, config: AudioDecoderConfig, order: Order): void { + decoder.reset(); + this.#ring.peek()?.dropped(); + decoder.configure(config); + order.reset(); + } + // Decode one frame once backpressure admits it, reporting whether to keep going. async #decode(effect: Effect, decoder: AudioDecoder, frame: Container.ReceivedFrame): Promise { const ring = await this.#ready(effect); @@ -891,8 +893,11 @@ export class Decoder { // a gap, or a later group after a declared end. #onRead(read: Container.Read): boolean { if (!this.#terminal.update(read)) return false; - this.sync.reset(); - this.reset(); + // A hole restarts codec prediction on the same timeline, keeping decoded PCM and parked video. + if (read.kind === "frame") { + this.sync.reset(); + this.reset(); + } return true; } diff --git a/js/watch/src/audio/terminal.test.ts b/js/watch/src/audio/terminal.test.ts index fab05c830cc..e858994beb6 100644 --- a/js/watch/src/audio/terminal.test.ts +++ b/js/watch/src/audio/terminal.test.ts @@ -100,3 +100,15 @@ test("an endpoint does not trim media resumed in a later group", () => { expect(terminal.end).toBeUndefined(); expect(terminal.span({ timestamp: 5_000_000, sampleRate: 48_000, numberOfFrames: 960 }).frames).toBe(960); }); + +test("a hole preserves a declared endpoint until a later group starts a new timeline", () => { + const terminal = new Terminal(); + terminal.update(frame(0, 0)); + terminal.update(end(1, 20_000)); + expect(terminal.update(gap)).toBe(true); + expect(terminal.end).toBe(20_000 as Micro); + expect(terminal.update(frame(1, 10_000))).toBe(false); + expect(terminal.span({ timestamp: 10_000, sampleRate: 48_000, numberOfFrames: 960 }).frames).toBe(480); + expect(terminal.update(frame(2, 1_000_000))).toBe(true); + expect(terminal.end).toBeUndefined(); +}); diff --git a/js/watch/src/audio/terminal.ts b/js/watch/src/audio/terminal.ts index 39aaf26ec9c..aade858314d 100644 --- a/js/watch/src/audio/terminal.ts +++ b/js/watch/src/audio/terminal.ts @@ -24,7 +24,7 @@ export interface DecodedSpan { /** Track codec epochs and map decoded output onto the source timeline. */ export class Terminal { - // Whether a frame arrived since the last reset, so there is an epoch to end. + // Whether media has started on this timeline. #started = false; #end?: Micro; // The group that carried `#end`. It bounds only the terminal packets that follow it there. @@ -57,7 +57,9 @@ export class Terminal { update(read: Update): boolean { if (read.kind === "gap") { this.#prime = true; - return this.#restart(); + this.#before = undefined; + this.#resetEpoch(); + return this.#started; } if (read.kind === "end") { this.#end = read.end; @@ -66,7 +68,8 @@ export class Terminal { } // A later group after a declared end comes from a fresh encoder; the end's own group carries // the stopped encoder's final packets. - const reset = this.#end !== undefined && read.group !== this.#endGroup && this.#restart(); + const reset = this.#endGroup !== undefined && read.group > this.#endGroup; + if (reset) this.#restart(); this.#started = true; this.#epoch ??= read.frame.timestamp; if (this.#prime) { @@ -76,15 +79,13 @@ export class Terminal { return reset; } - // Start a new epoch, reporting whether one was running. - #restart(): boolean { - const started = this.#started; + // Start a fresh timeline after its declared end. + #restart(): void { this.#started = false; this.#end = undefined; this.#endGroup = undefined; this.#before = undefined; this.#resetEpoch(); - return started; } /** Remove codec pre-skip and terminal padding from one decoded sample. */ diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index ed93899e50e..a984114ebe6 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -497,7 +497,7 @@ it("clears the picture once no rendition is selectable", async () => { } }); -// A late marker from older history breaks the timeline without making older groups newer. +// A marker from older history cannot make older groups newer. function marker(group: number, end: number): Read { return { kind: "end", group, end: Micro(end) }; } @@ -584,11 +584,15 @@ for (const kind of ["legacy", "cmaf"] as const) { } }); - // A group may restart at its own start, so a picture already queued from later in that - // group is above the new keyframe. It must not come out afterwards and late-reject it. - it(`${kind} drops a picture queued before a gap`, async () => { + // A declared end permits the next group to start a fresh timeline. Pictures still inside + // the codec must not come out above that new keyframe and late-reject it. + it.each([false, true])(`${kind} drops codec-queued pictures at a declared restart (hole=%s)`, async (hole) => { const queued = 60_000_000; - const playback = await guardedPlayback(kind, [sample(1, 0), sample(1, queued), gap, sample(2, 0)], [queued]); + const playback = await guardedPlayback( + kind, + [sample(1, 0), sample(1, queued), marker(1, queued), ...(hole ? [gap] : []), sample(2, 0)], + [queued], + ); try { expect(playback.submitted).toEqual([0, queued, 0]); expect(playback.outputs).toEqual([0, 0]); @@ -599,8 +603,36 @@ for (const kind of ["legacy", "cmaf"] as const) { } }); - // A stale marker can break the timeline while the live group is only partly delivered, so the - // next frame is a delta that still needs the decoder's references. + for (const boundary of ["hole", "end"] as const) { + it(`${kind} keeps parked pictures and the anchor across a ${boundary}`, async () => { + const clock = manualAt(1000); + const reads: Read[] = []; + const playback = await guardedPlayback(kind, reads); + const reset = spyOn(playback.sync, "reset"); + try { + playback.enabled.set(false); + playback.delay.set(Milli(1000)); + await microtasks(); + reads.push(sample(0, 0), sample(0, 100_000, false), boundary === "hole" ? gap : marker(0, 200_000)); + if (boundary === "hole") reads.push(sample(2, 200_000)); + playback.enabled.set(true); + await microtasks(); + expect(reset).not.toHaveBeenCalled(); + expect(playback.pictures.find((picture) => picture.timestamp === 100_000)?.closed).toBe(false); + expect(playback.resets).toBe(boundary === "hole" ? 1 : 0); + await clock.advance(1200); + expect(playback.decoder.out.timestamp.peek()).toBe(Milli(boundary === "hole" ? 200 : 100)); + } finally { + reset.mockRestore(); + playback.close(); + await microtasks(); + clock.installed.close(); + } + }); + } + + // A stale marker can arrive while the live group is only partly delivered. Its next delta + // still needs the decoder's references. it(`${kind} keeps decoding a delta that follows a stale marker`, async () => { const playback = await guardedPlayback(kind, [ sample(10, 1000), @@ -1037,6 +1069,74 @@ it("buffered video holds its poster until its own span reaches the floor", async } }); +for (const kind of ["legacy", "cmaf"] as const) { + it(`${kind} bounds startup across disconnected ranges, then starts on the retained timeline`, async () => { + const clock = manualAt(0); + const reads: Read[] = []; + const playback = await guardedPlayback(kind, reads, [], false, 1000); + const audio = new Signal<{ start: Milli; end: Milli } | null>(null); + const remove = playback.sync.register(new Signal(Milli(100)), audio); + const reset = spyOn(playback.sync, "reset"); + try { + playback.enabled.set(false); + await microtasks(); + for (let group = 0; group < 10; group++) { + if (group) reads.push(gap); + reads.push(sample(group, group * 100_000), sample(group, group * 100_000 + 40_000, false)); + } + playback.enabled.set(true); + await microtasks(); + await clock.advance(0); + // Three received 40ms spans cover the 100ms floor; their holes count for nothing. + expect(playback.submitted).toEqual([0, 40_000, 100_000, 140_000, 200_000, 240_000]); + expect(playback.sync.playing.peek()).toBe(false); + expect(reset).not.toHaveBeenCalled(); + audio.set({ start: Milli(0), end: Milli(100) }); + await microtasks(); + await clock.advance(0); + expect(playback.sync.playing.peek()).toBe(true); + expect(playback.submitted).toHaveLength(20); + await clock.advance(1000); + expect(playback.decoder.out.timestamp.peek()).toBe(Milli(940)); + expect(reset).not.toHaveBeenCalled(); + } finally { + remove(); + reset.mockRestore(); + playback.close(); + await microtasks(); + clock.installed.close(); + } + }); +} + +// Single-picture groups split by holes never build a received span, so coverage alone never stops +// startup decoding. Parked pictures must stay within the max delay of the opening one instead. +for (const kind of ["legacy", "cmaf"] as const) { + it(`${kind} bounds startup pictures by the max delay when holes keep coverage empty`, async () => { + const clock = manualAt(0); + const reads: Read[] = []; + for (let group = 0; group < 64; group++) { + if (group) reads.push(gap); + reads.push(sample(group * 2, group * 40_000)); + } + const playback = await guardedPlayback(kind, reads, [], false, 1000); + const remove = playback.sync.register(new Signal(Milli(100)), new Signal(null)); + try { + await microtasks(); + await clock.advance(0); + // The 1100ms max delay holds the opening picture through the first one at or past it. + expect(playback.submitted).toHaveLength(29); + expect(playback.submitted.at(-1)).toBe(1_120_000); + expect(playback.sync.playing.peek()).toBe(false); + } finally { + remove(); + playback.close(); + await microtasks(); + clock.installed.close(); + } + }); +} + it("bounds startup decoding while another track is missing and releases it on close", async () => { const clock = manualAt(0); const playback = await guardedPlayback("legacy", [], [], true, 1000); diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 943cc72f494..4d0454eea9f 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -32,6 +32,13 @@ import type { Source } from "./source"; // The amount of time to wait before considering the video to be buffering. const BUFFERING = Milli(500); +// The startup floor counts received media across holes, without counting the missing spans. +function startupSpan(ranges: Container.BufferedRanges, first: Milli): { start: Milli; end: Milli } { + let duration = 0; + for (const range of ranges) duration += Math.max(0, range.end - Math.max(range.start, first)); + return { start: first, end: Milli(first + duration) }; +} + /** Signals the decoder reads. */ export type DecoderInput = { /** Whether to download the video track. Defaults to true; the parent may wire it from the renderer's output. */ @@ -157,8 +164,7 @@ export class Decoder { const timestamp = effect.get(active.first); if (timestamp === undefined) return null; const ranges = effect.get(active.buffered); - const span = ranges.find((range) => range.start <= timestamp && range.end >= timestamp); - return span ? { ...span } : { start: timestamp, end: timestamp }; + return startupSpan(ranges, timestamp); }); this.#signals.cleanup(this.sync.register(target, startup)); this.#identity = this.#signals.computed((effect) => { @@ -452,8 +458,10 @@ class DecoderTrack { // Codec description from the last configure, reused after a playhead reset. #description?: Uint8Array; - // The playhead moved and the next decoded frame decides whether the decoder resets. + // A hole or timeline restart makes the next decoded frame decide whether the codec resets. #resetPending = false; + // A declared endpoint stays in force through holes until a later group starts a new timeline. + #endGroup?: number; // The newest consumer reading this track, whose estimate a replacement continues. The predecessor // is only held until the first one opens. @@ -599,20 +607,19 @@ class DecoderTrack { for (;;) { const next = await nextMedia(consumer); if (!next) break; + this.#onRead(next, latest); - // A gap or a declared end breaks the timeline: re-anchor before decoding further, and - // never bridge the buffer across it. + // A gap or declared end cuts the decoded span; neither bridges buffered ranges. if (next.kind !== "frame") { if (next.kind === "gap") recovery.gap(); - if (latest !== undefined) this.#onBreak(); previous = undefined; continue; } const { frame } = next; // An older keyframe would replace the codec references needed by live deltas. Group - // sequences only grow within a subscription, so a break (which a stale marker can - // raise) never resets this guard. Rejected payloads still count as received bytes. + // sequences only grow within a subscription, so neither a hole nor an endpoint + // resets this guard. Rejected payloads still count as received bytes. const stale = latest !== undefined && next.group < latest; const accepted = !stale && recovery.accept(frame); this.stats.update((current) => ({ @@ -644,7 +651,8 @@ class DecoderTrack { latest = next.group; this.#resetAtKeyframe(decoder, frame.keyframe); decoder.decode(chunk); - if (!this.sync.playing.peek()) await this.#waitStartup(effect); + if (!this.sync.playing.peek()) + await this.#waitStartup(effect, Milli.fromMicro(frame.timestamp as Micro)); } }); } @@ -685,19 +693,19 @@ class DecoderTrack { for (;;) { const next = await nextMedia(consumer); if (!next) break; + this.#onRead(next, latest); - // A gap or a declared end breaks the timeline (see #runLegacy). + // Do not bridge buffered ranges across a gap or end (see #runLegacy). if (next.kind !== "frame") { if (next.kind === "gap") recovery.gap(); - if (latest !== undefined) this.#onBreak(); previous = undefined; continue; } const { frame } = next; // An older keyframe would replace the codec references needed by live deltas. Group - // sequences only grow within a subscription, so a break (which a stale marker can - // raise) never resets this guard. Rejected payloads still count as received bytes. + // sequences only grow within a subscription, so neither a hole nor an endpoint + // resets this guard. Rejected payloads still count as received bytes. const stale = latest !== undefined && next.group < latest; const accepted = !stale && recovery.accept(frame); this.stats.update((current) => ({ @@ -727,39 +735,58 @@ class DecoderTrack { timestamp: frame.timestamp, }), ); - if (!this.sync.playing.peek()) await this.#waitStartup(effect); + if (!this.sync.playing.peek()) await this.#waitStartup(effect, Milli.fromMicro(frame.timestamp)); } }); } - // Once this track has filled its floor, leave further media encoded upstream while the other - // track fills. Otherwise a missing audio track could retain unbounded decoded GPU frames. - async #waitStartup(effect: Effect): Promise { + // Decoded pictures park until startup releases, so leave further media encoded upstream once this + // track covers its floor, or once `timestamp` sits the max delay past the opening picture: the + // most any held frame may lead the playhead, which startup anchors there. Holes keep coverage + // short without bound, so only the second limit bounds the parked pictures. + async #waitStartup(effect: Effect, timestamp: Milli): Promise { // Some codecs need more input before their first output. Do not park that input // behind a gate that needs the very picture the decoder has not produced yet. - if (this.first.peek() === undefined) return; - const span = this.#buffered.peek()[0]; - if (!span || span.end - span.start < this.sync.out.delay.peek()) return; + const first = this.first.peek(); + if (first === undefined) return; + const filled = (get: (signal: Getter) => T) => { + const span = startupSpan(get(this.#buffered), first); + return ( + span.end - span.start >= get(this.sync.out.delay) || timestamp - first >= get(this.sync.out.maxDelay) + ); + }; + if (!filled((signal) => signal.peek())) return; const waiting = new Effect(); const refill = new Once(); waiting.run((inner) => { // An adaptive floor can rise while another track fills. Resume decoding to reach it, // rather than waiting on a clock whose startup now needs more video. - if (span.end - span.start < inner.get(this.sync.out.delay)) refill.set(true); + if (!filled((signal) => inner.get(signal))) refill.set(true); }); try { - // Wait on the span's first picture, which is due the moment startup releases, so decoding + // Wait on the first picture, which is due the moment startup releases, so decoding // resumes with the whole floor still ahead instead of once it has played out. - await effect.race(refill, this.sync.wait(span.start, waiting.abort)); + await effect.race(refill, this.sync.wait(first, waiting.abort)); } finally { waiting.close(); } } - // The container read a gap or a declared end after media. Re-anchor and let the next decoded - // frame decide the decoder reset (see #resetAtKeyframe). Clearing `timestamp` keeps a stale - // high value from late-rejecting the following frames, which may overlap the previous group's - // tail. The held picture stays until the new keyframe renders. + // Container gaps restart prediction on the existing clock. Only a declared end followed by + // a later group establishes another timeline; a marker from older history cannot end live media. + #onRead(read: Container.Read, latest: number | undefined): void { + if (read.kind === "gap") { + if (latest !== undefined) this.#resetPending = true; + } else if (read.kind === "end") { + if (latest === undefined || read.group >= latest) this.#endGroup = read.group; + } else if (this.#endGroup !== undefined && read.group > this.#endGroup) { + this.#endGroup = undefined; + this.#onBreak(); + } + } + + // A fresh timeline invalidates parked pictures and its previous decoded span. The held picture + // stays until the new keyframe renders; the codec resets before that keyframe is decoded. #onBreak(): void { this.first.set(undefined); this.timestamp.set(undefined); @@ -768,12 +795,10 @@ class DecoderTrack { this.#resetPending = true; } - // Call before each decode. After a break, a keyframe starts a new timeline, so drop the - // pictures still queued in the decoder: one from later in a group that restarts would - // otherwise come out above the new keyframe and late-reject it. A delta continues a group the - // break interrupted (a stale marker or a max-delay hole), so it keeps the decoder's references; - // a reset would make WebCodecs require a key chunk and throw. `reset()` cannot cancel an - // output callback that already holds a frame; the clock epoch it captured rejects that one. + // Re-prime at a keyframe after a hole or restart, dropping pictures still queued in the codec. + // A delta may continue a group that a max-delay hole interrupted and still need its references; + // resetting before it would make WebCodecs require a key chunk. An output callback that already + // holds a frame stays parked on Sync; only a new timeline invalidates its clock epoch. #resetAtKeyframe(decoder: VideoDecoder, keyframe: boolean): void { if (!this.#resetPending) return; this.#resetPending = false; diff --git a/quest/m1/cut-audit/README.md b/quest/m1/cut-audit/README.md index 0542968a292..256de29e29e 100644 --- a/quest/m1/cut-audit/README.md +++ b/quest/m1/cut-audit/README.md @@ -60,7 +60,6 @@ leaving JS-only redesigns of `js/net` to [rs2ts](/quest/m2/rs2ts/README.md). - [A reserved group survives a route returning](/quest/m1/cut-audit/reserved-route-return.md) - a route flip A to B to A no longer panics a peeked reader - [JS tracks end clean only on a deliberate finish](/quest/m1/cut-audit/js-close-cause.md) - local close, violations, and unanswered requests end with errors - [Withdrawing a moq-transport FETCH or TRACK_STATUS ends it](/quest/m1/cut-audit/ietf-withdraw-resolve.md) - withdrawal during resolution releases the request -- [A media hole in the player keeps the timeline](/quest/m1/cut-audit/watch-audio-hole.md) - a skipped group re-primes decoders without flushing audio or re-anchoring sync - [A description-less AAC rendition means one thing](/quest/m1/cut-audit/aac-description-less.md) - every player and exporter agrees on the frame format - [RTMP sessions follow one strict path](/quest/m1/cut-audit/rtmp-session-strict.md) - idle sockets, repeated requests, rewinds, and stalled servers are handled - [The export jitter buffer admits only frames due within its window](/quest/m1/cut-audit/jitter-admission.md) - a fast track is backpressured, not buffered without bound diff --git a/quest/m1/cut-audit/watch-audio-hole.md b/quest/m1/cut-audit/watch-audio-hole.md deleted file mode 100644 index feed0d322f0..00000000000 --- a/quest/m1/cut-audit/watch-audio-hole.md +++ /dev/null @@ -1,49 +0,0 @@ -# [M] A media hole in the player keeps the timeline - -## Goal - -When `` skips a group (a max-delay skip or a hole timeout), audio -and video treat it as a hole on the same timeline: the decoder re-primes, -the audio ring keeps the PCM already decoded, and the shared `Sync` clock -keeps its anchor. Only a real new timeline (a declared end followed by a -later group, or a new instance) re-anchors, as Rust's `moq-audio` already -behaves. - -## Plan - -Found by the 2026-10-24 pre-cut audit at 151a529670. Source-inferred, -inherited (release resets on container `discontinuity` the same way). - -With 20 ms audio groups under congestion, one lost group runs -`#onRead` (`js/watch/src/audio/decoder.ts:891`) -> `Terminal` gap -> -`#restart()` (`audio/terminal.ts:57`): `this.sync.reset()` bumps the epoch, -dropping every decoded video frame parked in `Sync.wait` (`sync.ts:304`), -and `this.reset()` zeroes the worklet ring (`ring-buffer.ts:216`). 20 ms of -loss throws away a floor of valid PCM and re-stalls playback; video skips its -in-flight frames. Rust's `rs/moq-audio/src/decode/consumer.rs:513` resets -codec prediction only and plays silence through the hole. -`decoder.test.ts:1182` asserts the flush (with an 840 ms gap) and changes -with the fix. - -- Make the container read distinguish a hole from a new timeline, in - `@moq/hang`'s consumer if that is where the distinction is known. -- On a hole: `decoder.reset()`/`configure()` (Chrome stamps output from its - first input) and re-prime the `Terminal` epoch; keep the ring (it is - timestamp-indexed) and `Sync`. Apply the same to video's `#onBreak` - (`video/decoder.ts:606`, `:691`). -- Each decoder has near-identical legacy and CMAF read loops - (`audio/decoder.ts:522-664`, `video/decoder.ts:589-728`); fold each into - one loop over the container format first if that makes the change land - once instead of twice. - -The audit traced about a dozen watch fixes to resets of the shared clock and -ring (8b6abc0a13, 69b0538326, 1696858a0f, 177e92c755, 1a98a3aea7, -076cfb7500, c09e14995a, 4311a7c9a7, 8b83bb5f8a, d6e4f47238, 41906d97e0); -with only a new timeline re-anchoring, most reset call sites should go. - -Public API: none. Wire: none. - -## Related - -- [A/V clock](/quest/m1/av-clock.md) - the video-skip half, and the audio playhead driving `Sync` -- [Buffered playback subscribes in ascending order](/quest/m1/buffered-ascending.md) - the buffered-mode half of the same timeline