From 054c9bd96ca3e0cea89df04f1fad146198be9a39 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 26 Oct 2026 18:25:19 -0700 Subject: [PATCH 1/4] chore(quest): claim quest/m1/cut-audit/watch-audio-hole.md Quest-Claim: 13972d370d5d5bee From aeda750103e2dd4f6baa2c3599715bed5077337d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 26 Oct 2026 18:47:28 -0700 Subject: [PATCH 2/4] fix(watch): keep media holes on the current timeline Co-Authored-By: GPT-6 --- js/watch/src/audio/decoder.test.ts | 153 +++++++++++++++++++------ js/watch/src/audio/decoder.ts | 7 +- js/watch/src/audio/terminal.test.ts | 12 ++ js/watch/src/audio/terminal.ts | 15 +-- js/watch/src/video/decoder.test.ts | 46 ++++++-- js/watch/src/video/decoder.ts | 50 ++++---- quest/m1/cut-audit/README.md | 1 - quest/m1/cut-audit/watch-audio-hole.md | 49 -------- 8 files changed, 209 insertions(+), 124 deletions(-) delete mode 100644 quest/m1/cut-audit/watch-audio-hole.md diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index f01bae01c47..0e7e14dfb9a 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -683,20 +683,42 @@ describe("Decoder across a rendition's absence", () => { } }); - it("keeps the graph when only the container changes", async () => { - const playback = await play(Milli(100)); - try { - await playback.play(); - playback.restore({ container: { kind: "loc" } }); - await microtasks(); - await playback.play(); - - expect(playback.subscriptions()).toBe(2); - expect(contexts).toHaveLength(1); - } finally { - playback.close(); - } - }); + for (const [name, patch] of Object.entries({ + container: { container: { kind: "loc" } }, + codec: { codec: "mp4a.40.2" }, + })) { + it(`keeps compatible PCM and Sync when the ${name} changes on the same instance`, async () => { + const clock = new Manual(); + const installed = clock.install(); + const playback = await play(Milli(100), true); + const reset = spyOn(playback.sync, "reset"); + try { + await playback.play(7); + const ring = playback.rings().at(-1); + if (!ring) throw new Error("missing audio ring"); + const held = ring.length; + const reference = playback.sync.out.reference.peek(); + expect(held).toBeGreaterThan(0); + playback.restore(patch); + await microtasks(); + expect(playback.subscriptions()).toBe(2); + expect(playback.codecs()).toBe(2); + expect(contexts).toHaveLength(1); + expect(playback.rings()).toEqual([ring]); + expect(ring.length).toBe(held); + expect(playback.resets()).toBe(0); + expect(reset).not.toHaveBeenCalled(); + expect(playback.sync.out.reference.peek()).toBe(reference); + expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(960); + await playback.play(); + expect(reset).not.toHaveBeenCalled(); + } finally { + playback.close(); + await microtasks(); + installed.close(); + } + }); + } it("rebuilds the graph on a new channel count", async () => { const playback = await play(Milli(100)); @@ -930,6 +952,14 @@ async function livePlayback(burst: number) { return { sync, error: decoder.out.error, + arrivals: () => decoder.arrivals(), + // Lose one whole group without changing the following media timestamps or arrival cadence. + async hole() { + if (held.length) throw new Error("a hole needs an empty delivery burst"); + sequence++; + frame++; + await clock.advance(FRAME); + }, counts, ring, get burst() { @@ -993,6 +1023,36 @@ async function livePlayback(burst: number) { }; } +it("a missing live group re-primes without clearing decoded PCM or resetting Sync", async () => { + const playback = await livePlayback(1); + const reset = spyOn(playback.sync, "reset"); + const prime = spyOn(AudioDecoder.prototype, "reset"); + const arrivals = playback.arrivals(); + const received: { kind: string }[] = []; + const reading = (async () => { + for await (const arrival of arrivals) received.push(arrival); + })(); + try { + await playback.run(1000); + const ring = playback.ring(); + expect(ring?.stalled).toBe(false); + const buffered = ring?.length; + await playback.hole(); + expect(ring?.length).toBe(buffered); + await playback.run(2000); + expect(received.some((arrival) => arrival.kind === "gap")).toBe(true); + expect(prime).toHaveBeenCalledTimes(1); + expect(reset).not.toHaveBeenCalled(); + expect(playback.ring()).toBe(ring); + expect(ring?.stalled).toBe(false); + expect(playback.error.peek()).toBeUndefined(); + } finally { + arrivals.close(); + await reading; + await playback.close(); + } +}); + describe("Decoder across a rename", () => { // A codec change renames the audio track, and the replacement's consumer reads the same path. // Clearing the estimate at teardown and restarting it from the cold-start prior dropped the @@ -1179,30 +1239,47 @@ it("waits for video support before allowing audio-only startup", async () => { } }); -it("does not count flushed audio toward a new startup floor after a gap", async () => { - const playback = await play(Milli(100), true); - try { - await playback.play(7); - expect(playback.sync.playing.peek()).toBe(true); - const openingRing = playback.rings().at(-1); - if (!openingRing) throw new Error("missing audio ring"); - openingRing.read([new Float32Array(960), new Float32Array(960)]); - await microtasks(); - playback.push({ kind: "gap", timestamp: Micro(160_000), duration: Micro(840_000) }); - await microtasks(); - frameTimestamp = 1_000_000; - await playback.play(1); - expect(playback.sync.playing.peek()).toBe(false); - const ring = playback.rings().at(-1); - if (!ring) throw new Error("missing audio ring"); - expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(0); - await playback.play(5); - expect(playback.sync.playing.peek()).toBe(true); - } finally { - playback.close(); - await microtasks(); - } -}); +for (const cmaf of [false, true]) { + it(`a ${cmaf ? "CMAF" : "legacy"} hole re-primes the codec while keeping PCM and the clock`, async () => { + const clock = new Manual(); + const installed = clock.install(); + const playback = await play(Milli(100), true, cmaf); + const reset = spyOn(playback.sync, "reset"); + const prime = spyOn(AudioDecoder.prototype, "reset"); + try { + await playback.play(7); + expect(playback.sync.playing.peek()).toBe(true); + const ring = playback.rings().at(-1); + if (!ring) throw new Error("missing audio ring"); + ring.read([new Float32Array(960), new Float32Array(960)]); + await microtasks(); + const held = ring.length; + const reference = playback.sync.out.reference.peek(); + const resets = playback.resets(); + const parked = playback.sync.wait(Milli(1000)); + expect(held).toBeGreaterThan(0); + playback.push({ kind: "gap", timestamp: Micro(160_000), duration: Micro(20_000) }); + await microtasks(); + expect(prime).toHaveBeenCalledTimes(1); + expect(reset).not.toHaveBeenCalled(); + expect(playback.resets()).toBe(resets); + expect(ring.length).toBe(held); + expect(playback.sync.out.reference.peek()).toBe(reference); + expect(playback.sync.playing.peek()).toBe(true); + expect(Bun.peek.status(parked)).toBe("pending"); + frameTimestamp = 160_000; + await playback.play(1); + expect(playback.sync.playing.peek()).toBe(true); + expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(960); + await clock.advance(Nano.fromMilli(Milli(1200))); + expect(await parked).toBe(true); + } finally { + playback.close(); + await microtasks(); + installed.close(); + } + }); +} it.each([100, 1000])( "closing during shared startup at %dms releases the audio graph and backpressured reader", diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 1a6c8792cee..8ae3fb90d7a 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -891,8 +891,11 @@ export class Decoder { // a gap, or a later group after a declared end. #onRead(read: Container.Read): boolean { if (!this.#terminal.update(read)) return false; - this.sync.reset(); - this.reset(); + // A hole restarts codec prediction on the same timeline, keeping decoded PCM and parked video. + if (read.kind === "frame") { + this.sync.reset(); + this.reset(); + } return true; } diff --git a/js/watch/src/audio/terminal.test.ts b/js/watch/src/audio/terminal.test.ts index fab05c830cc..e858994beb6 100644 --- a/js/watch/src/audio/terminal.test.ts +++ b/js/watch/src/audio/terminal.test.ts @@ -100,3 +100,15 @@ test("an endpoint does not trim media resumed in a later group", () => { expect(terminal.end).toBeUndefined(); expect(terminal.span({ timestamp: 5_000_000, sampleRate: 48_000, numberOfFrames: 960 }).frames).toBe(960); }); + +test("a hole preserves a declared endpoint until a later group starts a new timeline", () => { + const terminal = new Terminal(); + terminal.update(frame(0, 0)); + terminal.update(end(1, 20_000)); + expect(terminal.update(gap)).toBe(true); + expect(terminal.end).toBe(20_000 as Micro); + expect(terminal.update(frame(1, 10_000))).toBe(false); + expect(terminal.span({ timestamp: 10_000, sampleRate: 48_000, numberOfFrames: 960 }).frames).toBe(480); + expect(terminal.update(frame(2, 1_000_000))).toBe(true); + expect(terminal.end).toBeUndefined(); +}); diff --git a/js/watch/src/audio/terminal.ts b/js/watch/src/audio/terminal.ts index 39aaf26ec9c..aade858314d 100644 --- a/js/watch/src/audio/terminal.ts +++ b/js/watch/src/audio/terminal.ts @@ -24,7 +24,7 @@ export interface DecodedSpan { /** Track codec epochs and map decoded output onto the source timeline. */ export class Terminal { - // Whether a frame arrived since the last reset, so there is an epoch to end. + // Whether media has started on this timeline. #started = false; #end?: Micro; // The group that carried `#end`. It bounds only the terminal packets that follow it there. @@ -57,7 +57,9 @@ export class Terminal { update(read: Update): boolean { if (read.kind === "gap") { this.#prime = true; - return this.#restart(); + this.#before = undefined; + this.#resetEpoch(); + return this.#started; } if (read.kind === "end") { this.#end = read.end; @@ -66,7 +68,8 @@ export class Terminal { } // A later group after a declared end comes from a fresh encoder; the end's own group carries // the stopped encoder's final packets. - const reset = this.#end !== undefined && read.group !== this.#endGroup && this.#restart(); + const reset = this.#endGroup !== undefined && read.group > this.#endGroup; + if (reset) this.#restart(); this.#started = true; this.#epoch ??= read.frame.timestamp; if (this.#prime) { @@ -76,15 +79,13 @@ export class Terminal { return reset; } - // Start a new epoch, reporting whether one was running. - #restart(): boolean { - const started = this.#started; + // Start a fresh timeline after its declared end. + #restart(): void { this.#started = false; this.#end = undefined; this.#endGroup = undefined; this.#before = undefined; this.#resetEpoch(); - return started; } /** Remove codec pre-skip and terminal padding from one decoded sample. */ diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index e69bbdc6963..f9df88ff538 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -497,7 +497,7 @@ it("clears the picture once no rendition is selectable", async () => { } }); -// A late marker from older history breaks the timeline without making older groups newer. +// A marker from older history cannot make older groups newer. function marker(group: number, end: number): Read { return { kind: "end", group, end: Micro(end) }; } @@ -584,11 +584,15 @@ for (const kind of ["legacy", "cmaf"] as const) { } }); - // A group may restart at its own start, so a picture already queued from later in that - // group is above the new keyframe. It must not come out afterwards and late-reject it. - it(`${kind} drops a picture queued before a gap`, async () => { + // A declared end permits the next group to start a fresh timeline. Pictures still inside + // the codec must not come out above that new keyframe and late-reject it. + it.each([false, true])(`${kind} drops codec-queued pictures at a declared restart (hole=%s)`, async (hole) => { const queued = 60_000_000; - const playback = await guardedPlayback(kind, [sample(1, 0), sample(1, queued), gap, sample(2, 0)], [queued]); + const playback = await guardedPlayback( + kind, + [sample(1, 0), sample(1, queued), marker(1, queued), ...(hole ? [gap] : []), sample(2, 0)], + [queued], + ); try { expect(playback.submitted).toEqual([0, queued, 0]); expect(playback.outputs).toEqual([0, 0]); @@ -599,8 +603,36 @@ for (const kind of ["legacy", "cmaf"] as const) { } }); - // A stale marker can break the timeline while the live group is only partly delivered, so the - // next frame is a delta that still needs the decoder's references. + for (const boundary of ["hole", "end"] as const) { + it(`${kind} keeps parked pictures and the anchor across a ${boundary}`, async () => { + const clock = manualAt(1000); + const reads: Read[] = []; + const playback = await guardedPlayback(kind, reads); + const reset = spyOn(playback.sync, "reset"); + try { + playback.enabled.set(false); + playback.delay.set(Milli(1000)); + await microtasks(); + reads.push(sample(0, 0), sample(0, 100_000, false), boundary === "hole" ? gap : marker(0, 200_000)); + if (boundary === "hole") reads.push(sample(2, 200_000)); + playback.enabled.set(true); + await microtasks(); + expect(reset).not.toHaveBeenCalled(); + expect(playback.pictures.find((picture) => picture.timestamp === 100_000)?.closed).toBe(false); + expect(playback.resets).toBe(boundary === "hole" ? 1 : 0); + await clock.advance(1200); + expect(playback.decoder.out.timestamp.peek()).toBe(Milli(boundary === "hole" ? 200 : 100)); + } finally { + reset.mockRestore(); + playback.close(); + await microtasks(); + clock.installed.close(); + } + }); + } + + // A stale marker can arrive while the live group is only partly delivered. Its next delta + // still needs the decoder's references. it(`${kind} keeps decoding a delta that follows a stale marker`, async () => { const playback = await guardedPlayback(kind, [ sample(10, 1000), diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 943cc72f494..a37b535606b 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -452,8 +452,10 @@ class DecoderTrack { // Codec description from the last configure, reused after a playhead reset. #description?: Uint8Array; - // The playhead moved and the next decoded frame decides whether the decoder resets. + // A hole or timeline restart makes the next decoded frame decide whether the codec resets. #resetPending = false; + // A declared endpoint stays in force through holes until a later group starts a new timeline. + #endGroup?: number; // The newest consumer reading this track, whose estimate a replacement continues. The predecessor // is only held until the first one opens. @@ -599,20 +601,19 @@ class DecoderTrack { for (;;) { const next = await nextMedia(consumer); if (!next) break; + this.#onRead(next, latest); - // A gap or a declared end breaks the timeline: re-anchor before decoding further, and - // never bridge the buffer across it. + // A gap or declared end cuts the decoded span; neither bridges buffered ranges. if (next.kind !== "frame") { if (next.kind === "gap") recovery.gap(); - if (latest !== undefined) this.#onBreak(); previous = undefined; continue; } const { frame } = next; // An older keyframe would replace the codec references needed by live deltas. Group - // sequences only grow within a subscription, so a break (which a stale marker can - // raise) never resets this guard. Rejected payloads still count as received bytes. + // sequences only grow within a subscription, so neither a hole nor an endpoint + // resets this guard. Rejected payloads still count as received bytes. const stale = latest !== undefined && next.group < latest; const accepted = !stale && recovery.accept(frame); this.stats.update((current) => ({ @@ -685,19 +686,19 @@ class DecoderTrack { for (;;) { const next = await nextMedia(consumer); if (!next) break; + this.#onRead(next, latest); - // A gap or a declared end breaks the timeline (see #runLegacy). + // Do not bridge buffered ranges across a gap or end (see #runLegacy). if (next.kind !== "frame") { if (next.kind === "gap") recovery.gap(); - if (latest !== undefined) this.#onBreak(); previous = undefined; continue; } const { frame } = next; // An older keyframe would replace the codec references needed by live deltas. Group - // sequences only grow within a subscription, so a break (which a stale marker can - // raise) never resets this guard. Rejected payloads still count as received bytes. + // sequences only grow within a subscription, so neither a hole nor an endpoint + // resets this guard. Rejected payloads still count as received bytes. const stale = latest !== undefined && next.group < latest; const accepted = !stale && recovery.accept(frame); this.stats.update((current) => ({ @@ -756,10 +757,21 @@ class DecoderTrack { } } - // The container read a gap or a declared end after media. Re-anchor and let the next decoded - // frame decide the decoder reset (see #resetAtKeyframe). Clearing `timestamp` keeps a stale - // high value from late-rejecting the following frames, which may overlap the previous group's - // tail. The held picture stays until the new keyframe renders. + // Container gaps restart prediction on the existing clock. Only a declared end followed by + // a later group establishes another timeline; a marker from older history cannot end live media. + #onRead(read: Container.Read, latest: number | undefined): void { + if (read.kind === "gap") { + if (latest !== undefined) this.#resetPending = true; + } else if (read.kind === "end") { + if (latest === undefined || read.group >= latest) this.#endGroup = read.group; + } else if (this.#endGroup !== undefined && read.group > this.#endGroup) { + this.#endGroup = undefined; + this.#onBreak(); + } + } + + // A fresh timeline invalidates parked pictures and its previous decoded span. The held picture + // stays until the new keyframe renders; the codec resets before that keyframe is decoded. #onBreak(): void { this.first.set(undefined); this.timestamp.set(undefined); @@ -768,12 +780,10 @@ class DecoderTrack { this.#resetPending = true; } - // Call before each decode. After a break, a keyframe starts a new timeline, so drop the - // pictures still queued in the decoder: one from later in a group that restarts would - // otherwise come out above the new keyframe and late-reject it. A delta continues a group the - // break interrupted (a stale marker or a max-delay hole), so it keeps the decoder's references; - // a reset would make WebCodecs require a key chunk and throw. `reset()` cannot cancel an - // output callback that already holds a frame; the clock epoch it captured rejects that one. + // Re-prime at a keyframe after a hole or restart, dropping pictures still queued in the codec. + // A delta may continue a group that a max-delay hole interrupted and still need its references; + // resetting before it would make WebCodecs require a key chunk. An output callback that already + // holds a frame stays parked on Sync; only a new timeline invalidates its clock epoch. #resetAtKeyframe(decoder: VideoDecoder, keyframe: boolean): void { if (!this.#resetPending) return; this.#resetPending = false; diff --git a/quest/m1/cut-audit/README.md b/quest/m1/cut-audit/README.md index a84b26c68c0..09009169e73 100644 --- a/quest/m1/cut-audit/README.md +++ b/quest/m1/cut-audit/README.md @@ -62,7 +62,6 @@ leaving JS-only redesigns of `js/net` to [rs2ts](/quest/m2/rs2ts/README.md). - [A malformed lite message closes the session](/quest/m1/cut-audit/announce-decode-complete.md) - a bad ANNOUNCE body no longer jams its stream - [Withdrawing a moq-transport FETCH or TRACK_STATUS ends it](/quest/m1/cut-audit/ietf-withdraw-resolve.md) - withdrawal during resolution releases the request - [JS moq-transport sessions close on peer violations](/quest/m1/cut-audit/js-ietf-violations.md) - right close codes, no unhandled rejection, bounded reflected namespaces -- [A media hole in the player keeps the timeline](/quest/m1/cut-audit/watch-audio-hole.md) - a skipped group re-primes decoders without flushing audio or re-anchoring sync - [A camera broadcast stays announced across unmute](/quest/m1/cut-audit/publish-announce-latch.md) - viewers do not restart on every device flap - [Muted and volume are independent](/quest/m1/cut-audit/watch-volume-mute.md) - `` starts muted at 0.8 - [A description-less AAC rendition means one thing](/quest/m1/cut-audit/aac-description-less.md) - every player and exporter agrees on the frame format diff --git a/quest/m1/cut-audit/watch-audio-hole.md b/quest/m1/cut-audit/watch-audio-hole.md deleted file mode 100644 index feed0d322f0..00000000000 --- a/quest/m1/cut-audit/watch-audio-hole.md +++ /dev/null @@ -1,49 +0,0 @@ -# [M] A media hole in the player keeps the timeline - -## Goal - -When `` skips a group (a max-delay skip or a hole timeout), audio -and video treat it as a hole on the same timeline: the decoder re-primes, -the audio ring keeps the PCM already decoded, and the shared `Sync` clock -keeps its anchor. Only a real new timeline (a declared end followed by a -later group, or a new instance) re-anchors, as Rust's `moq-audio` already -behaves. - -## Plan - -Found by the 2026-10-24 pre-cut audit at 151a529670. Source-inferred, -inherited (release resets on container `discontinuity` the same way). - -With 20 ms audio groups under congestion, one lost group runs -`#onRead` (`js/watch/src/audio/decoder.ts:891`) -> `Terminal` gap -> -`#restart()` (`audio/terminal.ts:57`): `this.sync.reset()` bumps the epoch, -dropping every decoded video frame parked in `Sync.wait` (`sync.ts:304`), -and `this.reset()` zeroes the worklet ring (`ring-buffer.ts:216`). 20 ms of -loss throws away a floor of valid PCM and re-stalls playback; video skips its -in-flight frames. Rust's `rs/moq-audio/src/decode/consumer.rs:513` resets -codec prediction only and plays silence through the hole. -`decoder.test.ts:1182` asserts the flush (with an 840 ms gap) and changes -with the fix. - -- Make the container read distinguish a hole from a new timeline, in - `@moq/hang`'s consumer if that is where the distinction is known. -- On a hole: `decoder.reset()`/`configure()` (Chrome stamps output from its - first input) and re-prime the `Terminal` epoch; keep the ring (it is - timestamp-indexed) and `Sync`. Apply the same to video's `#onBreak` - (`video/decoder.ts:606`, `:691`). -- Each decoder has near-identical legacy and CMAF read loops - (`audio/decoder.ts:522-664`, `video/decoder.ts:589-728`); fold each into - one loop over the container format first if that makes the change land - once instead of twice. - -The audit traced about a dozen watch fixes to resets of the shared clock and -ring (8b6abc0a13, 69b0538326, 1696858a0f, 177e92c755, 1a98a3aea7, -076cfb7500, c09e14995a, 4311a7c9a7, 8b83bb5f8a, d6e4f47238, 41906d97e0); -with only a new timeline re-anchoring, most reset call sites should go. - -Public API: none. Wire: none. - -## Related - -- [A/V clock](/quest/m1/av-clock.md) - the video-skip half, and the audio playhead driving `Sync` -- [Buffered playback subscribes in ascending order](/quest/m1/buffered-ascending.md) - the buffered-mode half of the same timeline From 80a2534c298da04e2d37c205129fcae239c4cd00 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 26 Oct 2026 19:23:26 -0700 Subject: [PATCH 3/4] fix(watch): checkpoint buffered startup accounting and ownership decision Co-Authored-By: GPT-6 --- js/watch/src/audio/buffer.ts | 4 +- js/watch/src/audio/decoder.test.ts | 46 ++++++++- js/watch/src/audio/decoder.ts | 20 ++-- js/watch/src/video/decoder.test.ts | 40 ++++++++ js/watch/src/video/decoder.ts | 20 ++-- quest/m1/cut-audit/README.md | 1 + quest/m1/cut-audit/watch-audio-hole.md | 131 +++++++++++++++++++++++++ 7 files changed, 244 insertions(+), 18 deletions(-) create mode 100644 quest/m1/cut-audit/watch-audio-hole.md diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index c629b3c0f7e..1bd9e2641bf 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -133,7 +133,7 @@ export class Backpressure { this.#release(); } - /** Note that the decoder closed, dropping every frame still decoding. */ + /** Note that the decoder closed or reset, dropping every frame still decoding. */ dropped(): void { this.#decoding = []; this.#release(); @@ -344,7 +344,7 @@ export class AudioBuffer { this.#backpressure.decoded(end, landed); } - /** Note that the decoder closed, dropping every frame still decoding. */ + /** Note that the decoder closed or reset, dropping every frame still decoding. */ dropped(): void { this.#backpressure.dropped(); } diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 0e7e14dfb9a..b0610eac127 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -154,12 +154,18 @@ let codecs = 0; let codecErrors: AudioDecoderInit["error"][] = []; // Thrown by the next `decode`, as WebCodecs does synchronously for a chunk it refuses. let decodeThrows: Error | undefined; +let deferAudioOutput = false; +let pendingAudio: (() => void)[] = []; +let audioInputs: number[] = []; beforeEach(() => { frameTimestamp = 0; codecs = 0; codecErrors = []; decodeThrows = undefined; + deferAudioOutput = false; + pendingAudio = []; + audioInputs = []; contexts = []; worklets = []; moduleLoaded = Promise.resolve(); @@ -179,12 +185,16 @@ beforeEach(() => { } reset() { this.needsKey = true; + pendingAudio = []; } decode(chunk: { timestamp: number; type: EncodedAudioChunkType }) { if (decodeThrows) throw decodeThrows; if (this.needsKey && chunk.type !== "key") throw new DOMException("key chunk required", "DataError"); this.needsKey = false; - this.init.output(new FakeData(chunk.timestamp) as unknown as AudioData); + audioInputs.push(chunk.timestamp); + const emit = () => this.init.output(new FakeData(chunk.timestamp) as unknown as AudioData); + if (deferAudioOutput) pendingAudio.push(emit); + else emit(); } close() { this.state = "closed"; @@ -380,6 +390,40 @@ async function play(initial: Delay, buffered = false, cmaf = false) { }; } +for (const cmaf of [false, true]) { + it(`a ${cmaf ? "CMAF" : "legacy"} gap releases accounting for codec outputs it discarded`, async () => { + const clock = new Manual(); + const installed = clock.install(); + deferAudioOutput = true; + const playback = await play(Milli(100), true, cmaf); + try { + for (let n = 0; n < 6; n++) { + playback.push({ ...frame(), group: n === 0 ? 0 : 1 }); + await microtasks(); + } + expect(audioInputs).toHaveLength(6); + expect(pendingAudio).toHaveLength(6); + expect(playback.inserted()).toHaveLength(0); + playback.push({ kind: "gap", timestamp: Micro(140_000), duration: Micro(20_000) }); + await microtasks(); + expect(pendingAudio).toHaveLength(0); + deferAudioOutput = false; + frameTimestamp = 160_000; + const resumed = frame(); + playback.push({ ...resumed, group: 2 }); + await microtasks(); + expect(audioInputs).toHaveLength(7); + expect(playback.inserted()).toContain(resumed.frame.timestamp); + await playback.play(5); + expect(playback.sync.playing.peek()).toBe(true); + } finally { + playback.close(); + await microtasks(); + installed.close(); + } + }); +} + it("decodes an audio packet inside a group after a gap", async () => { const playback = await play(Milli(100)); try { diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 8ae3fb90d7a..b1ccf7149dd 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -512,9 +512,7 @@ export class Decoder { if (!(await play(order.flush()))) return; if (!next) break; if (this.#onRead(next)) { - decoder.reset(); - decoder.configure(decoderConfig); - order.reset(); + this.#reprime(decoder, decoderConfig, order); } if (next.kind === "gap") this.#arrivals.write(gap(next)); continue; @@ -601,9 +599,7 @@ export class Decoder { if (!(await play(order.flush()))) return; if (!next) break; if (this.#onRead(next)) { - decoder.reset(); - decoder.configure(decoderConfig); - order.reset(); + this.#reprime(decoder, decoderConfig, order); } if (next.kind === "gap") this.#arrivals.write(gap(next)); continue; @@ -639,9 +635,7 @@ export class Decoder { ): Promise { for (const read of reads) { if (this.#onRead(read)) { - decoder.reset(); - decoder.configure(config); - order.reset(); + this.#reprime(decoder, config, order); } if (!order.admit(read)) continue; if (!(await this.#decode(effect, decoder, read.frame))) return false; @@ -649,6 +643,14 @@ export class Decoder { return true; } + // A reset discards queued codec output, so release its accounting without clearing retained PCM. + #reprime(decoder: AudioDecoder, config: AudioDecoderConfig, order: Order): void { + decoder.reset(); + this.#ring.peek()?.dropped(); + decoder.configure(config); + order.reset(); + } + // Decode one frame once backpressure admits it, reporting whether to keep going. async #decode(effect: Effect, decoder: AudioDecoder, frame: Container.ReceivedFrame): Promise { const ring = await this.#ready(effect); diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index f9df88ff538..1aa24d800f7 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -1070,6 +1070,46 @@ it("buffered video holds its poster until its own span reaches the floor", async } }); +for (const kind of ["legacy", "cmaf"] as const) { + it(`${kind} bounds startup across disconnected ranges, then starts on the retained timeline`, async () => { + const clock = manualAt(0); + const reads: Read[] = []; + const playback = await guardedPlayback(kind, reads, [], false, 1000); + const audio = new Signal<{ start: Milli; end: Milli } | null>(null); + const remove = playback.sync.register(new Signal(Milli(100)), audio); + const reset = spyOn(playback.sync, "reset"); + try { + playback.enabled.set(false); + await microtasks(); + for (let group = 0; group < 10; group++) { + if (group) reads.push(gap); + reads.push(sample(group, group * 100_000), sample(group, group * 100_000 + 40_000, false)); + } + playback.enabled.set(true); + await microtasks(); + await clock.advance(0); + // Three received 40ms spans cover the 100ms floor; their holes count for nothing. + expect(playback.submitted).toEqual([0, 40_000, 100_000, 140_000, 200_000, 240_000]); + expect(playback.sync.playing.peek()).toBe(false); + expect(reset).not.toHaveBeenCalled(); + audio.set({ start: Milli(0), end: Milli(100) }); + await microtasks(); + await clock.advance(0); + expect(playback.sync.playing.peek()).toBe(true); + expect(playback.submitted).toHaveLength(20); + await clock.advance(1000); + expect(playback.decoder.out.timestamp.peek()).toBe(Milli(940)); + expect(reset).not.toHaveBeenCalled(); + } finally { + remove(); + reset.mockRestore(); + playback.close(); + await microtasks(); + clock.installed.close(); + } + }); +} + it("bounds startup decoding while another track is missing and releases it on close", async () => { const clock = manualAt(0); const playback = await guardedPlayback("legacy", [], [], true, 1000); diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index a37b535606b..4fb52880f92 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -32,6 +32,13 @@ import type { Source } from "./source"; // The amount of time to wait before considering the video to be buffering. const BUFFERING = Milli(500); +// The startup floor counts received media across holes, without counting the missing spans. +function startupSpan(ranges: Container.BufferedRanges, first: Milli): { start: Milli; end: Milli } { + let duration = 0; + for (const range of ranges) duration += Math.max(0, range.end - Math.max(range.start, first)); + return { start: first, end: Milli(first + duration) }; +} + /** Signals the decoder reads. */ export type DecoderInput = { /** Whether to download the video track. Defaults to true; the parent may wire it from the renderer's output. */ @@ -157,8 +164,7 @@ export class Decoder { const timestamp = effect.get(active.first); if (timestamp === undefined) return null; const ranges = effect.get(active.buffered); - const span = ranges.find((range) => range.start <= timestamp && range.end >= timestamp); - return span ? { ...span } : { start: timestamp, end: timestamp }; + return startupSpan(ranges, timestamp); }); this.#signals.cleanup(this.sync.register(target, startup)); this.#identity = this.#signals.computed((effect) => { @@ -738,15 +744,17 @@ class DecoderTrack { async #waitStartup(effect: Effect): Promise { // Some codecs need more input before their first output. Do not park that input // behind a gate that needs the very picture the decoder has not produced yet. - if (this.first.peek() === undefined) return; - const span = this.#buffered.peek()[0]; - if (!span || span.end - span.start < this.sync.out.delay.peek()) return; + const first = this.first.peek(); + if (first === undefined) return; + const span = startupSpan(this.#buffered.peek(), first); + if (span.end - span.start < this.sync.out.delay.peek()) return; const waiting = new Effect(); const refill = new Once(); waiting.run((inner) => { // An adaptive floor can rise while another track fills. Resume decoding to reach it, // rather than waiting on a clock whose startup now needs more video. - if (span.end - span.start < inner.get(this.sync.out.delay)) refill.set(true); + const current = startupSpan(inner.get(this.#buffered), first); + if (current.end - current.start < inner.get(this.sync.out.delay)) refill.set(true); }); try { // Wait on the span's first picture, which is due the moment startup releases, so decoding diff --git a/quest/m1/cut-audit/README.md b/quest/m1/cut-audit/README.md index 09009169e73..a84b26c68c0 100644 --- a/quest/m1/cut-audit/README.md +++ b/quest/m1/cut-audit/README.md @@ -62,6 +62,7 @@ leaving JS-only redesigns of `js/net` to [rs2ts](/quest/m2/rs2ts/README.md). - [A malformed lite message closes the session](/quest/m1/cut-audit/announce-decode-complete.md) - a bad ANNOUNCE body no longer jams its stream - [Withdrawing a moq-transport FETCH or TRACK_STATUS ends it](/quest/m1/cut-audit/ietf-withdraw-resolve.md) - withdrawal during resolution releases the request - [JS moq-transport sessions close on peer violations](/quest/m1/cut-audit/js-ietf-violations.md) - right close codes, no unhandled rejection, bounded reflected namespaces +- [A media hole in the player keeps the timeline](/quest/m1/cut-audit/watch-audio-hole.md) - a skipped group re-primes decoders without flushing audio or re-anchoring sync - [A camera broadcast stays announced across unmute](/quest/m1/cut-audit/publish-announce-latch.md) - viewers do not restart on every device flap - [Muted and volume are independent](/quest/m1/cut-audit/watch-volume-mute.md) - `` starts muted at 0.8 - [A description-less AAC rendition means one thing](/quest/m1/cut-audit/aac-description-less.md) - every player and exporter agrees on the frame format diff --git a/quest/m1/cut-audit/watch-audio-hole.md b/quest/m1/cut-audit/watch-audio-hole.md new file mode 100644 index 00000000000..b5d4254db4a --- /dev/null +++ b/quest/m1/cut-audit/watch-audio-hole.md @@ -0,0 +1,131 @@ +# [M] A media hole in the player keeps the timeline + +## Goal + +When `` skips a group (a max-delay skip or a hole timeout), audio +and video treat it as a hole on the same timeline: the decoder re-primes, +the audio ring keeps the PCM already decoded, and the shared `Sync` clock +keeps its anchor. Only a real new timeline (a declared end followed by a +later group, or a new instance) re-anchors, as Rust's `moq-audio` already +behaves. + +## Plan + +### Checkpoint: timeline preserved, video ownership unresolved + +PR [moq-dev/moq#6903](https://github.com/moq-dev/moq/pull/6903) remains a draft. +The settled rule is unchanged: a hole re-primes codecs on the existing timeline, +keeping decoded PCM and Sync. A declared end followed by a later group, or a new +media instance, re-anchors. Existing `Container.Read` already distinguishes these; +no public API or wire change is needed. + +Implemented and reproduced in both legacy and CMAF paths: + +- Preserve PCM and Sync through holes, including parked decoded video. +- Preserve an audio endpoint across a hole and clip its own terminal group. +- Keep compatible PCM/Sync through same-instance codec/container changes. Genuine + declared restarts/new instances still reset; explicit caller reset is unchanged. +- Clear discarded audio decode accounting through `ring.dropped()` at every codec + reset. Six delayed 20 ms inputs otherwise fill the 100 ms floor after reset has + discarded their output, blocking the resumed input forever. +- Sum disconnected received/decoded video spans consistently for startup and its + decode gate, excluding the holes. Three 40 ms spans now stop decoding at six + pictures while another track is pending, then release on that track's readiness. + +The video ownership P1 is **not solved**. Legacy single-picture groups can have +unknown/zero duration. Their buffered ranges remain empty across holes, so no +amount of such received media satisfies the duration floor. The sum-only decode +gate therefore keeps decoding and parking GPU pictures indefinitely. A decoded +storage bound cannot depend only on that same unavailable duration. + +### Decision needed + +1. Separate decoded-storage ownership/backpressure from startup coverage, and + choose an explicit startup policy. Recommended: preserve the current rule + that missing media is not counted as received buffered media. +2. Change startup coverage semantics to count inferred holes as playable silence + or held pictures. This changes the existing media-coverage rule. +3. Defer the larger design and retain this draft until a bounded design is chosen. + +No storage budget, inferred duration, public shape, or coverage-policy change has +been selected while the maintainer is away. Keep this quest and PR draft until +both retained decoded state and startup progress are bounded by the chosen design. + +### Portable zero-duration counterexample + +In this PR checkout, append the following fixture case to +`js/watch/src/video/decoder.test.ts`, then run: + +```bash +nix develop --command bun test js/watch/src/video/decoder.test.ts --test-name-pattern 'diagnostic:' +``` + +It uses that tracked file's existing `manualAt`, `guardedPlayback`, and +`legacyFrame` helpers, real MoQ Track/Container reads, and the configured max-delay +deadline. The failing assertion is intentional evidence, not retained CI coverage. +Close/reopen producer APIs should follow any upstream lifecycle migration when +replaying on a newer base. + +```typescript +// BEGIN REAL SINGLE-FRAME DIAGNOSTIC +for (const count of [8, 16]) { + it(`diagnostic: ${count} real single-picture groups retain startup output`, async () => { + const clock = manualAt(0); + const playback = await guardedPlayback("legacy", [], [], true, 1000); + const remove = playback.sync.register(new Signal(Milli(100)), new Signal(null)); + if (!playback.track) throw new Error("missing video track"); + try { + for (let n = 0; n < count; n++) { + const group = new Moq.Group.Producer(n * 2); + group.writeFrame(legacyFrame(Micro(n * 40_000))); + group.close(); + playback.track.writeGroup(group); + await microtasks(); + } + await clock.advance(playback.sync.out.maxDelay.peek()); + await microtasks(); + console.log("real-single-picture-startup", JSON.stringify({input:count,submitted:playback.submitted.length,held:playback.pictures.filter(frame=>!frame.closed).length,playing:playback.sync.playing.peek(),buffered:playback.decoder.out.buffered.peek(),codecResets:playback.resets})); + expect(playback.submitted.length).toBeLessThan(count); + } finally { + remove();playback.close();await microtasks();clock.installed.close(); + } + }); +} +// END REAL SINGLE-FRAME DIAGNOSTIC +``` + +Observed after the safe accounting/sum fixes, with a 100 ms floor and another +track pending: + +```json +{"input":8,"submitted":8,"held":8,"playing":false,"buffered":[],"codecResets":7} +{"input":16,"submitted":16,"held":16,"playing":false,"buffered":[],"codecResets":15} +``` + +The first-to-last PTS spans are only 280/600 ms, within the configured 1100 ms +max delay. Each missing group expires under manual time. This is retained output +growth, not a timing/performance threshold. The fixture's pictures stand in for +native decoded GPU frames; it does not measure physical GPU allocation. + +### Verification recorded so far + +At `aeda750103e2dd4f6baa2c3599715bed5077337d`, `just fix`, clean-head `just ci` +(409 watch tests and affected dependents/API Extractor/tooling), and trusted +`quest check` passed. Chromium 156 native Opus/AudioWorklet proof kept the 40–120 ms +PCM span through an inferred 40 ms hole: codec resets 1, ring resets 0, Sync +resets 0, reference -100 ms; resumed PCM started at 140000 µs with nonzero tone +energy, and the parked Sync wait later presented. The dev player painted BBB. +The focused real Chromium media republish control passed at 29.9 fps over 118 +presented frames with synchronized tone output; its children and ports were reaped. + +The review-accounting regressions failed before the retained fixes and passed +after them (107 focused decoder tests). The final checkpoint's clean-head gates +and native surface result are recorded in the PR body before handoff. Passing +ordinary ranges and that browser proof do not solve the zero-duration ownership +counterexample above. No retry, sleep, larger timeout, deployment, or publication +is part of this checkpoint. + +## Related + +- [A/V clock](/quest/m1/av-clock.md) - the video-skip half, and the audio playhead driving `Sync` +- [Buffered playback subscribes in ascending order](/quest/m1/buffered-ascending.md) - the buffered-mode half of the same timeline From f7dfc4aac6b001007859e60a1c1bc8b0bcf012e8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 27 Oct 2026 12:47:35 -0700 Subject: [PATCH 4/4] fix(watch): bound parked startup pictures by the max delay Holes keep single-picture groups from building a received span, so the coverage gate alone never stopped startup decoding. Also stop once the input sits the max delay past the opening picture, the span every other stage already bounds. The quest is complete. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/video/decoder.test.ts | 28 ++++++ js/watch/src/video/decoder.ts | 29 +++--- quest/m1/cut-audit/README.md | 1 - quest/m1/cut-audit/watch-audio-hole.md | 131 ------------------------- 4 files changed, 46 insertions(+), 143 deletions(-) delete mode 100644 quest/m1/cut-audit/watch-audio-hole.md diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index 4ccbc414e5d..a984114ebe6 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -1109,6 +1109,34 @@ for (const kind of ["legacy", "cmaf"] as const) { }); } +// Single-picture groups split by holes never build a received span, so coverage alone never stops +// startup decoding. Parked pictures must stay within the max delay of the opening one instead. +for (const kind of ["legacy", "cmaf"] as const) { + it(`${kind} bounds startup pictures by the max delay when holes keep coverage empty`, async () => { + const clock = manualAt(0); + const reads: Read[] = []; + for (let group = 0; group < 64; group++) { + if (group) reads.push(gap); + reads.push(sample(group * 2, group * 40_000)); + } + const playback = await guardedPlayback(kind, reads, [], false, 1000); + const remove = playback.sync.register(new Signal(Milli(100)), new Signal(null)); + try { + await microtasks(); + await clock.advance(0); + // The 1100ms max delay holds the opening picture through the first one at or past it. + expect(playback.submitted).toHaveLength(29); + expect(playback.submitted.at(-1)).toBe(1_120_000); + expect(playback.sync.playing.peek()).toBe(false); + } finally { + remove(); + playback.close(); + await microtasks(); + clock.installed.close(); + } + }); +} + it("bounds startup decoding while another track is missing and releases it on close", async () => { const clock = manualAt(0); const playback = await guardedPlayback("legacy", [], [], true, 1000); diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 4fb52880f92..4d0454eea9f 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -651,7 +651,8 @@ class DecoderTrack { latest = next.group; this.#resetAtKeyframe(decoder, frame.keyframe); decoder.decode(chunk); - if (!this.sync.playing.peek()) await this.#waitStartup(effect); + if (!this.sync.playing.peek()) + await this.#waitStartup(effect, Milli.fromMicro(frame.timestamp as Micro)); } }); } @@ -734,32 +735,38 @@ class DecoderTrack { timestamp: frame.timestamp, }), ); - if (!this.sync.playing.peek()) await this.#waitStartup(effect); + if (!this.sync.playing.peek()) await this.#waitStartup(effect, Milli.fromMicro(frame.timestamp)); } }); } - // Once this track has filled its floor, leave further media encoded upstream while the other - // track fills. Otherwise a missing audio track could retain unbounded decoded GPU frames. - async #waitStartup(effect: Effect): Promise { + // Decoded pictures park until startup releases, so leave further media encoded upstream once this + // track covers its floor, or once `timestamp` sits the max delay past the opening picture: the + // most any held frame may lead the playhead, which startup anchors there. Holes keep coverage + // short without bound, so only the second limit bounds the parked pictures. + async #waitStartup(effect: Effect, timestamp: Milli): Promise { // Some codecs need more input before their first output. Do not park that input // behind a gate that needs the very picture the decoder has not produced yet. const first = this.first.peek(); if (first === undefined) return; - const span = startupSpan(this.#buffered.peek(), first); - if (span.end - span.start < this.sync.out.delay.peek()) return; + const filled = (get: (signal: Getter) => T) => { + const span = startupSpan(get(this.#buffered), first); + return ( + span.end - span.start >= get(this.sync.out.delay) || timestamp - first >= get(this.sync.out.maxDelay) + ); + }; + if (!filled((signal) => signal.peek())) return; const waiting = new Effect(); const refill = new Once(); waiting.run((inner) => { // An adaptive floor can rise while another track fills. Resume decoding to reach it, // rather than waiting on a clock whose startup now needs more video. - const current = startupSpan(inner.get(this.#buffered), first); - if (current.end - current.start < inner.get(this.sync.out.delay)) refill.set(true); + if (!filled((signal) => inner.get(signal))) refill.set(true); }); try { - // Wait on the span's first picture, which is due the moment startup releases, so decoding + // Wait on the first picture, which is due the moment startup releases, so decoding // resumes with the whole floor still ahead instead of once it has played out. - await effect.race(refill, this.sync.wait(span.start, waiting.abort)); + await effect.race(refill, this.sync.wait(first, waiting.abort)); } finally { waiting.close(); } diff --git a/quest/m1/cut-audit/README.md b/quest/m1/cut-audit/README.md index 0542968a292..256de29e29e 100644 --- a/quest/m1/cut-audit/README.md +++ b/quest/m1/cut-audit/README.md @@ -60,7 +60,6 @@ leaving JS-only redesigns of `js/net` to [rs2ts](/quest/m2/rs2ts/README.md). - [A reserved group survives a route returning](/quest/m1/cut-audit/reserved-route-return.md) - a route flip A to B to A no longer panics a peeked reader - [JS tracks end clean only on a deliberate finish](/quest/m1/cut-audit/js-close-cause.md) - local close, violations, and unanswered requests end with errors - [Withdrawing a moq-transport FETCH or TRACK_STATUS ends it](/quest/m1/cut-audit/ietf-withdraw-resolve.md) - withdrawal during resolution releases the request -- [A media hole in the player keeps the timeline](/quest/m1/cut-audit/watch-audio-hole.md) - a skipped group re-primes decoders without flushing audio or re-anchoring sync - [A description-less AAC rendition means one thing](/quest/m1/cut-audit/aac-description-less.md) - every player and exporter agrees on the frame format - [RTMP sessions follow one strict path](/quest/m1/cut-audit/rtmp-session-strict.md) - idle sockets, repeated requests, rewinds, and stalled servers are handled - [The export jitter buffer admits only frames due within its window](/quest/m1/cut-audit/jitter-admission.md) - a fast track is backpressured, not buffered without bound diff --git a/quest/m1/cut-audit/watch-audio-hole.md b/quest/m1/cut-audit/watch-audio-hole.md deleted file mode 100644 index b5d4254db4a..00000000000 --- a/quest/m1/cut-audit/watch-audio-hole.md +++ /dev/null @@ -1,131 +0,0 @@ -# [M] A media hole in the player keeps the timeline - -## Goal - -When `` skips a group (a max-delay skip or a hole timeout), audio -and video treat it as a hole on the same timeline: the decoder re-primes, -the audio ring keeps the PCM already decoded, and the shared `Sync` clock -keeps its anchor. Only a real new timeline (a declared end followed by a -later group, or a new instance) re-anchors, as Rust's `moq-audio` already -behaves. - -## Plan - -### Checkpoint: timeline preserved, video ownership unresolved - -PR [moq-dev/moq#6903](https://github.com/moq-dev/moq/pull/6903) remains a draft. -The settled rule is unchanged: a hole re-primes codecs on the existing timeline, -keeping decoded PCM and Sync. A declared end followed by a later group, or a new -media instance, re-anchors. Existing `Container.Read` already distinguishes these; -no public API or wire change is needed. - -Implemented and reproduced in both legacy and CMAF paths: - -- Preserve PCM and Sync through holes, including parked decoded video. -- Preserve an audio endpoint across a hole and clip its own terminal group. -- Keep compatible PCM/Sync through same-instance codec/container changes. Genuine - declared restarts/new instances still reset; explicit caller reset is unchanged. -- Clear discarded audio decode accounting through `ring.dropped()` at every codec - reset. Six delayed 20 ms inputs otherwise fill the 100 ms floor after reset has - discarded their output, blocking the resumed input forever. -- Sum disconnected received/decoded video spans consistently for startup and its - decode gate, excluding the holes. Three 40 ms spans now stop decoding at six - pictures while another track is pending, then release on that track's readiness. - -The video ownership P1 is **not solved**. Legacy single-picture groups can have -unknown/zero duration. Their buffered ranges remain empty across holes, so no -amount of such received media satisfies the duration floor. The sum-only decode -gate therefore keeps decoding and parking GPU pictures indefinitely. A decoded -storage bound cannot depend only on that same unavailable duration. - -### Decision needed - -1. Separate decoded-storage ownership/backpressure from startup coverage, and - choose an explicit startup policy. Recommended: preserve the current rule - that missing media is not counted as received buffered media. -2. Change startup coverage semantics to count inferred holes as playable silence - or held pictures. This changes the existing media-coverage rule. -3. Defer the larger design and retain this draft until a bounded design is chosen. - -No storage budget, inferred duration, public shape, or coverage-policy change has -been selected while the maintainer is away. Keep this quest and PR draft until -both retained decoded state and startup progress are bounded by the chosen design. - -### Portable zero-duration counterexample - -In this PR checkout, append the following fixture case to -`js/watch/src/video/decoder.test.ts`, then run: - -```bash -nix develop --command bun test js/watch/src/video/decoder.test.ts --test-name-pattern 'diagnostic:' -``` - -It uses that tracked file's existing `manualAt`, `guardedPlayback`, and -`legacyFrame` helpers, real MoQ Track/Container reads, and the configured max-delay -deadline. The failing assertion is intentional evidence, not retained CI coverage. -Close/reopen producer APIs should follow any upstream lifecycle migration when -replaying on a newer base. - -```typescript -// BEGIN REAL SINGLE-FRAME DIAGNOSTIC -for (const count of [8, 16]) { - it(`diagnostic: ${count} real single-picture groups retain startup output`, async () => { - const clock = manualAt(0); - const playback = await guardedPlayback("legacy", [], [], true, 1000); - const remove = playback.sync.register(new Signal(Milli(100)), new Signal(null)); - if (!playback.track) throw new Error("missing video track"); - try { - for (let n = 0; n < count; n++) { - const group = new Moq.Group.Producer(n * 2); - group.writeFrame(legacyFrame(Micro(n * 40_000))); - group.close(); - playback.track.writeGroup(group); - await microtasks(); - } - await clock.advance(playback.sync.out.maxDelay.peek()); - await microtasks(); - console.log("real-single-picture-startup", JSON.stringify({input:count,submitted:playback.submitted.length,held:playback.pictures.filter(frame=>!frame.closed).length,playing:playback.sync.playing.peek(),buffered:playback.decoder.out.buffered.peek(),codecResets:playback.resets})); - expect(playback.submitted.length).toBeLessThan(count); - } finally { - remove();playback.close();await microtasks();clock.installed.close(); - } - }); -} -// END REAL SINGLE-FRAME DIAGNOSTIC -``` - -Observed after the safe accounting/sum fixes, with a 100 ms floor and another -track pending: - -```json -{"input":8,"submitted":8,"held":8,"playing":false,"buffered":[],"codecResets":7} -{"input":16,"submitted":16,"held":16,"playing":false,"buffered":[],"codecResets":15} -``` - -The first-to-last PTS spans are only 280/600 ms, within the configured 1100 ms -max delay. Each missing group expires under manual time. This is retained output -growth, not a timing/performance threshold. The fixture's pictures stand in for -native decoded GPU frames; it does not measure physical GPU allocation. - -### Verification recorded so far - -At `aeda750103e2dd4f6baa2c3599715bed5077337d`, `just fix`, clean-head `just ci` -(409 watch tests and affected dependents/API Extractor/tooling), and trusted -`quest check` passed. Chromium 156 native Opus/AudioWorklet proof kept the 40–120 ms -PCM span through an inferred 40 ms hole: codec resets 1, ring resets 0, Sync -resets 0, reference -100 ms; resumed PCM started at 140000 µs with nonzero tone -energy, and the parked Sync wait later presented. The dev player painted BBB. -The focused real Chromium media republish control passed at 29.9 fps over 118 -presented frames with synchronized tone output; its children and ports were reaped. - -The review-accounting regressions failed before the retained fixes and passed -after them (107 focused decoder tests). The final checkpoint's clean-head gates -and native surface result are recorded in the PR body before handoff. Passing -ordinary ranges and that browser proof do not solve the zero-duration ownership -counterexample above. No retry, sleep, larger timeout, deployment, or publication -is part of this checkpoint. - -## Related - -- [A/V clock](/quest/m1/av-clock.md) - the video-skip half, and the audio playhead driving `Sync` -- [Buffered playback subscribes in ascending order](/quest/m1/buffered-ascending.md) - the buffered-mode half of the same timeline