From bc202d9bb2722e6ffc269dbf3b3a36a227084c97 Mon Sep 17 00:00:00 2001 From: shermerL <86936692+shermerL@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:55:05 +0800 Subject: [PATCH 1/4] fix(watch): gate buffered playback on the shared startup floor --- .github/workflows/video.yml | 5 +- js/watch/src/audio/buffer.ts | 29 +++- js/watch/src/audio/decoder.test.ts | 185 +++++++++++++++++++++- js/watch/src/audio/decoder.ts | 16 +- js/watch/src/audio/render-worklet.test.ts | 16 ++ js/watch/src/audio/render-worklet.ts | 9 +- js/watch/src/audio/render.ts | 11 +- js/watch/src/audio/source.ts | 34 ++-- js/watch/src/sync.test.ts | 97 ++++++++++++ js/watch/src/sync.ts | 75 ++++++++- js/watch/src/video/decoder.test.ts | 166 ++++++++++++++++++- js/watch/src/video/decoder.ts | 44 ++++- js/watch/src/video/source.ts | 5 + test/video/renderer.browser.ts | 85 ++++++++++ test/video/src/main.ts | 1 + test/video/src/poster.ts | 144 +++++++++++++++++ 16 files changed, 879 insertions(+), 43 deletions(-) create mode 100644 test/video/src/poster.ts diff --git a/.github/workflows/video.yml b/.github/workflows/video.yml index 7830b2b5a50..1290fb11d4e 100644 --- a/.github/workflows/video.yml +++ b/.github/workflows/video.yml @@ -5,9 +5,8 @@ name: Video # browser. SwiftShader stands in for the GPU, so no GPU runner is needed. # # Narrow trigger, like wasm.yml: the renderer, the harness, and the two files that -# wire the renderer into `` (which the harness drives). `@moq/watch` -# wires it the same way but needs a relay to drive, so its regressions surface -# nightly instead. +# wire the renderer into `` (which the harness drives). Buffered +# `@moq/watch` startup is driven from an in-memory origin, without a relay. permissions: contents: read diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index 31bd9cbe4dd..ddfb1e3b4da 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -1,6 +1,6 @@ import { Micro, Milli, type Second } from "@moq/clock"; import { Effect, type Getter, Signal } from "@moq/signals"; -import type { Data, Init, Latency, Reset, State, Truncate } from "./render"; +import type { Data, Init, Latency, Playing, Reset, State, Truncate } from "./render"; /** * Timestamp-based backpressure for buffered playback. The decoded PCM ring only holds the latency @@ -144,6 +144,7 @@ export class AudioBuffer { // Bumped per reset. A state message posted before the worklet saw the reset reports the old // timeline's playhead, which would open the gate for the new one. #epoch = 0; + #playing?: Getter; #signals = new Effect(); @@ -156,9 +157,12 @@ export class AudioBuffer { latency: number; /** Anchor to the first frame and play through, holding the lookahead upstream. */ buffered: boolean; + /** Hold consumption until the shared startup floor is ready. */ + playing?: Getter; }, ) { - const { channels, rate, buffered } = props; + const { channels, rate, buffered, playing } = props; + this.#playing = props.playing; this.#worklet = worklet; this.channels = channels; this.rate = rate; @@ -166,8 +170,21 @@ export class AudioBuffer { this.#backpressure = new Backpressure(buffered, samplesToMicro(props.latency, rate)); const latency = Milli.fromSecond((props.latency / rate) as Second); - const msg: Init = { type: "init", channels, rate, latency, buffered }; + const msg: Init = { + type: "init", + channels, + rate, + latency, + buffered, + ...(props.playing && { playing: props.playing.peek() }), + }; worklet.port.postMessage(msg); + if (playing) { + this.#signals.run((effect) => { + const msg: Playing = { type: "playing", playing: effect.get(playing), epoch: this.#epoch }; + worklet.port.postMessage(msg); + }); + } this.#signals.event(worklet.port, "message", (ev: Event) => { const data = (ev as MessageEvent).data; @@ -218,7 +235,11 @@ export class AudioBuffer { /** Flush buffered samples and re-stall, ready to anchor the next utterance (buffered mode). */ reset(): void { this.#epoch++; - const msg: Reset = { type: "reset", epoch: this.#epoch }; + const msg: Reset = { + type: "reset", + epoch: this.#epoch, + ...(this.#playing && { playing: this.#playing.peek() }), + }; this.#worklet.port.postMessage(msg); this.#backpressure.reset(); } diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 3bb299bc601..5e916c88cd0 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -7,6 +7,8 @@ import * as Moq from "@moq/net"; import { Effect, Signal } from "@moq/signals"; import type { Broadcast } from "../broadcast"; import { type Delay, Sync } from "../sync"; +import { Decoder as VideoDecoder } from "../video/decoder"; +import { Source as VideoSource } from "../video/source"; import type { Message, State } from "./render"; import { AudioRingBuffer } from "./ring-buffer"; import { Source } from "./source"; @@ -60,16 +62,20 @@ let worklets: FakeWorklet[] = []; class FakeWorklet { ring?: AudioRingBuffer; #epoch = 0; + #playing = true; readonly port = Object.assign(new EventTarget(), { postMessage: (msg: Message) => { - if (msg.type === "init") this.ring = new AudioRingBuffer(msg); - else if (msg.type === "data") this.ring?.write(msg.timestamp, msg.data); + if (msg.type === "init") { + this.ring = new AudioRingBuffer(msg); + this.#playing = msg.playing ?? true; + } else if (msg.type === "data") this.ring?.write(msg.timestamp, msg.data); else if (msg.type === "latency") this.ring?.resize(msg.latency); else if (msg.type === "truncate") this.ring?.truncate(msg.timestamp); else if (msg.type === "reset") { this.ring?.reset(); this.#epoch = msg.epoch; - } + this.#playing = msg.playing ?? true; + } else if (msg.type === "playing" && msg.epoch === this.#epoch) this.#playing = msg.playing; }, start() {}, }); @@ -86,7 +92,7 @@ class FakeWorklet { read(output: Float32Array[]): number { const ring = this.ring; if (!ring) return 0; - const read = ring.read(output); + const read = this.#playing ? ring.read(output) : 0; const state: State = { type: "state", timestamp: ring.timestamp, @@ -306,6 +312,7 @@ async function play(initial: Delay, buffered = false, cmaf = false) { return { delay, + sync, push, enabled, decoding, @@ -1016,3 +1023,173 @@ describe("Decoder across a rename", () => { } }); }); + +it("keeps audio's opening buffered while an enabled video track fills", async () => { + const clock = new Manual(); + const installed = clock.install(); + const playback = await play(Milli(100), true); + const video = new Signal<{ start: Milli; end: Milli } | null>(null); + const remove = playback.sync.register(new Signal(Milli(100)), video); + try { + await playback.play(8); + await clock.advance(Nano.zero); + const ring = playback.rings()[0]; + expect(ring).toBeDefined(); + const opening = ring.ring?.timestamp; + for (let i = 0; i < 5; i++) expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(0); + await clock.advance(Nano.fromMilli(Milli(500))); + expect(ring.ring?.timestamp).toBe(opening); + video.set({ start: Milli(20), end: Milli(120) }); + await clock.advance(Nano.zero); + expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(960); + expect(ring.ring?.timestamp).toBe(Micro((opening ?? 0) + 20_000)); + } finally { + remove(); + playback.close(); + await microtasks(); + installed.close(); + } +}); + +it("waits for video support before allowing audio-only startup", async () => { + const playback = await play(Milli(100), true); + let resolve!: (supported: boolean) => void; + const source = new VideoSource({ + broadcast: { + out: { + catalog: new Signal({ + video: { + renditions: { + video: { codec: "vp8", codedWidth: 16, codedHeight: 16, container: { kind: "legacy" } }, + }, + }, + }), + }, + } as unknown as Broadcast, + supported: () => + new Promise((done) => { + resolve = done; + }), + }); + const video = new VideoDecoder({ source, sync: playback.sync }); + try { + await microtasks(); + await playback.play(8); + const ring = playback.rings()[0]; + expect(ring).toBeDefined(); + expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(0); + resolve(false); + await microtasks(); + expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(960); + } finally { + video.close(); + source.close(); + playback.close(); + await microtasks(); + } +}); + +it("does not count flushed audio toward a new startup floor after a gap", async () => { + const playback = await play(Milli(100), true); + try { + await playback.play(7); + expect(playback.sync.playing.peek()).toBe(true); + const openingRing = playback.rings().at(-1); + if (!openingRing) throw new Error("missing audio ring"); + openingRing.read([new Float32Array(960), new Float32Array(960)]); + await microtasks(); + playback.push({ kind: "gap", timestamp: Micro(160_000), duration: Micro(840_000) }); + await microtasks(); + frameTimestamp = 1_000_000; + await playback.play(1); + expect(playback.sync.playing.peek()).toBe(false); + const ring = playback.rings().at(-1); + if (!ring) throw new Error("missing audio ring"); + expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(0); + await playback.play(5); + expect(playback.sync.playing.peek()).toBe(true); + } finally { + playback.close(); + await microtasks(); + } +}); + +it("closing during shared startup releases the audio graph and backpressured reader", async () => { + const playback = await play(Milli(100), true); + const video = new Signal<{ start: Milli; end: Milli } | null>(null); + const remove = playback.sync.register(new Signal(Milli(100)), video); + const disconnect = spyOn(FakeWorklet.prototype, "disconnect"); + const closeTrack = spyOn(Moq.Track.Subscriber.prototype, "close"); + try { + await playback.play(20); + expect(playback.sync.playing.peek()).toBe(false); + const context = playback.context.peek() as unknown as FakeContext; + expect(context).toBeDefined(); + const decoded = playback.inserted().length; + playback.close(); + await microtasks(); + expect(context.calls.close).toBe(1); + expect(disconnect).toHaveBeenCalled(); + expect(closeTrack).toHaveBeenCalled(); + // The other track becoming ready cannot revive the cancelled audio reader. + video.set({ start: Milli(0), end: Milli(100) }); + await microtasks(); + expect(playback.inserted()).toHaveLength(decoded); + } finally { + remove(); + playback.close(); + await microtasks(); + disconnect.mockRestore(); + closeTrack.mockRestore(); + } +}); + +it("a rejected audio support probe does not leave buffered video waiting on a pending track", async () => { + const sync = new Sync({ delay: Milli(100), buffer: Milli(1000) }); + const remove = sync.register(new Signal(Milli(100)), new Signal({ start: Milli(0), end: Milli(100) })); + let reject!: (error: Error) => void; + const failure = new Error("audio support probe failed"); + const logged = spyOn(console, "error").mockImplementation(() => {}); + const source = new Source({ + broadcast: { + in: { enabled: new Signal(true) }, + out: { + catalog: new Signal({ + audio: { + renditions: { + audio: Catalog.AudioConfigSchema.parse({ + codec: "opus", + container: { kind: "legacy" }, + sampleRate: 48000, + numberOfChannels: 2, + }), + }, + }, + }), + }, + } as unknown as Broadcast, + supported: () => + new Promise((_resolve, fail) => { + reject = fail; + }), + }); + const decoder = new Decoder({ source, sync }); + try { + await microtasks(); + sync.received(Milli(0)); + await microtasks(); + expect(sync.playing.peek()).toBe(false); + reject(failure); + await microtasks(); + expect(logged).toHaveBeenCalledWith("spawn error", failure); + expect(source.pending.peek()).toBe(false); + expect(source.out.config.peek()).toBeUndefined(); + expect(sync.playing.peek()).toBe(true); + } finally { + decoder.close(); + source.close(); + remove(); + sync.close(); + logged.mockRestore(); + } +}); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 4dd5d68f7a8..fbf62ba4bb4 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -173,7 +173,14 @@ export class Decoder { delay: delay !== undefined ? Milli(delay) : undefined, }); }); - this.#signals.cleanup(this.sync.register(playout)); + const startup = this.#signals.computed((effect) => { + if (!effect.get(this.in.enabled)) return undefined; + if (effect.get(this.source.pending)) return null; + if (!effect.get(this.source.out.config)) return undefined; + const span = effect.get(this.#decodeBuffered)[0]; + return span ? { ...span } : null; + }); + this.#signals.cleanup(this.sync.register(playout, startup)); this.#identity = this.#signals.computed((effect) => { const config = effect.get(this.source.out.config); return config ? playbackIdentity(config) : undefined; @@ -275,6 +282,7 @@ export class Decoder { rate: sampleRate, latency: latencySamples, buffered, + playing: this.sync.playing, }); effect.cleanup(() => ring.close()); effect.set(this.#ring, ring); @@ -405,9 +413,8 @@ export class Decoder { // Every handle to one instance shares `closed`, so it identifies the instance where the // handle itself does not. if (this.#instance !== undefined && this.#instance !== active.closed) { - this.#ring.peek()?.reset(); - this.#decodeBuffered.set([]); this.sync.reset(); + this.reset(); } this.#instance = active.closed; @@ -820,6 +827,7 @@ export class Decoder { * Use in buffered mode at an utterance boundary (see Sync.reset). */ reset(): void { + this.#decodeBuffered.set([]); this.#ring.peek()?.reset(); } @@ -827,8 +835,8 @@ export class Decoder { // a gap, or a later group after a declared end. #onRead(read: Container.Read): boolean { if (!this.#terminal.update(read)) return false; - this.#ring.peek()?.reset(); this.sync.reset(); + this.reset(); return true; } diff --git a/js/watch/src/audio/render-worklet.test.ts b/js/watch/src/audio/render-worklet.test.ts index 6fd19d9fdbd..16a5697f491 100644 --- a/js/watch/src/audio/render-worklet.test.ts +++ b/js/watch/src/audio/render-worklet.test.ts @@ -94,3 +94,19 @@ describe("render dispatch", () => { expect([...output.slice(80)]).toEqual(Array(20).fill(2)); }); }); + +it("holds PCM without consuming it until the shared startup gate opens", () => { + const { send, read, sent } = renderer(); + send({ type: "playing", playing: false, epoch: 0 }); + send({ type: "data", timestamp: Micro(500_000), data: [new Float32Array(200).fill(1)] }); + for (let i = 0; i < 5; i++) expect([...read(20)]).toEqual(Array(20).fill(0)); + expect(sent[0]).toMatchObject({ timestamp: Micro(500_000), underruns: 0 }); + send({ type: "playing", playing: true, epoch: 0 }); + expect(read(100)[99]).toBe(1); + send({ type: "reset", epoch: 1, playing: false }); + send({ type: "data", timestamp: Micro(0), data: [new Float32Array(200).fill(2)] }); + send({ type: "playing", playing: true, epoch: 0 }); + expect([...read(20)]).toEqual(Array(20).fill(0)); + send({ type: "playing", playing: true, epoch: 1 }); + expect(read(100)[99]).toBe(2); +}); diff --git a/js/watch/src/audio/render-worklet.ts b/js/watch/src/audio/render-worklet.ts index d202f006d77..3d29e22128e 100644 --- a/js/watch/src/audio/render-worklet.ts +++ b/js/watch/src/audio/render-worklet.ts @@ -11,6 +11,7 @@ class Render extends AudioWorkletProcessor { #underflow = 0; #stateCounter = 0; #epoch = 0; + #playing = true; // Whether the previous quantum ended short, so the next one fades back in. #short = false; @@ -21,6 +22,7 @@ class Render extends AudioWorkletProcessor { const msg = event.data; if (msg.type === "init") { this.#ring = new AudioRingBuffer(msg); + this.#playing = msg.playing ?? true; this.#underflow = 0; this.#short = false; } else if (msg.type === "data") { @@ -32,6 +34,9 @@ class Render extends AudioWorkletProcessor { } else if (msg.type === "reset") { this.#ring?.reset(); this.#epoch = msg.epoch; + this.#playing = msg.playing ?? true; + } else if (msg.type === "playing" && msg.epoch === this.#epoch) { + this.#playing = msg.playing; } }; } @@ -39,7 +44,7 @@ class Render extends AudioWorkletProcessor { process(_inputs: Float32Array[][], outputs: Float32Array[][], _parameters: Record) { const output = outputs[0]; const ring = this.#ring; - const samplesRead = ring?.read(output) ?? 0; + const samplesRead = this.#playing ? (ring?.read(output) ?? 0) : 0; if (samplesRead < output[0].length) { // Fade the tail of what we did read down to the silence that follows it. @@ -49,7 +54,7 @@ class Render extends AudioWorkletProcessor { channel[samplesRead - ramp + i] *= 1 - (i + 1) / ramp; } } - this.#underflow += output[0].length - samplesRead; + if (this.#playing) this.#underflow += output[0].length - samplesRead; this.#short = true; } else { if (this.#short) { diff --git a/js/watch/src/audio/render.ts b/js/watch/src/audio/render.ts index 31002e63a48..16956374fdd 100644 --- a/js/watch/src/audio/render.ts +++ b/js/watch/src/audio/render.ts @@ -1,7 +1,7 @@ import type { Micro, Milli } from "@moq/clock"; /** Everything the main thread sends the render worklet over its port. */ -export type Message = Init | Data | Latency | Reset | Truncate; +export type Message = Init | Data | Latency | Reset | Truncate | Playing; /** Create the worklet's ring. */ export interface Init { @@ -12,6 +12,7 @@ export interface Init { // Buffered mode: anchor to the first frame and play through; the ring is sized to the floor and // the lookahead above it is held back upstream (the main thread applies the backpressure). buffered: boolean; + playing?: boolean; } /** Flush the buffer and re-stall. */ @@ -19,6 +20,7 @@ export interface Reset { type: "reset"; // Echoed in every later state message, so the main thread can tell state from before the reset. epoch: number; + playing?: boolean; } /** Drop buffered samples at or after `timestamp`, keeping what is already due. */ @@ -52,3 +54,10 @@ export interface State { // The epoch of the last reset the worklet applied. epoch: number; } + +/** Release or hold consumption without discarding the queued opening. */ +export interface Playing { + type: "playing"; + playing: boolean; + epoch: number; +} diff --git a/js/watch/src/audio/source.ts b/js/watch/src/audio/source.ts index b54a18d605c..fb2d0027b9f 100644 --- a/js/watch/src/audio/source.ts +++ b/js/watch/src/audio/source.ts @@ -55,6 +55,9 @@ export class Source { }; /** The derived outputs, read-only. */ readonly out = readonlys(this.#out); + #pending = new Signal(false); + /** Whether rendition support is still being resolved. @internal */ + readonly pending: Getter = this.#pending; #signals = new Effect(); @@ -85,25 +88,30 @@ export class Source { const supported = effect.get(this.in.supported); if (!supported) return; + effect.set(this.#pending, true, false); effect.spawn(async () => { - const available: Record = {}; + try { + const available: Record = {}; - for (const [name, config] of Object.entries(renditions)) { - // `supported` comes from the consumer, so we cannot assume it ever settles. A rerun - // waits for the tasks it spawned, so an unraced probe would hold the next run shut. - const isSupported = await effect.race(supported(config)); + for (const [name, config] of Object.entries(renditions)) { + // `supported` comes from the consumer, so we cannot assume it ever settles. A rerun + // waits for the tasks it spawned, so an unraced probe would hold the next run shut. + const isSupported = await effect.race(supported(config)); - // Torn down: stop probing and publish nothing, since the rerun redoes this. - if (effect.abort.aborted) return; + // Torn down: stop probing and publish nothing, since the rerun redoes this. + if (effect.abort.aborted) return; - if (isSupported) available[name] = config; - } + if (isSupported) available[name] = config; + } - if (Object.keys(available).length === 0 && Object.keys(renditions).length > 0) { - console.warn("no supported audio renditions found:", renditions); - } + if (Object.keys(available).length === 0 && Object.keys(renditions).length > 0) { + console.warn("no supported audio renditions found:", renditions); + } - this.#out.available.set(available); + this.#out.available.set(available); + } finally { + this.#pending.set(false); + } }); } diff --git a/js/watch/src/sync.test.ts b/js/watch/src/sync.test.ts index dd50caff3b4..84a00faa505 100644 --- a/js/watch/src/sync.test.ts +++ b/js/watch/src/sync.test.ts @@ -326,3 +326,100 @@ describe("skip floor", () => { sync.close(); }); }); + +describe("buffered startup", () => { + for (const first of ["audio", "video"] as const) { + it(`waits for both spans when ${first} fills first`, async () => { + const clock = new Manual(); + const installed = clock.install(); + const sync = new Sync({ delay: Milli(100), buffer: Milli(1000) }); + const spans = { + audio: new Signal<{ start: Milli; end: Milli } | null>(null), + video: new Signal<{ start: Milli; end: Milli } | null>(null), + }; + const dispose = Object.values(spans).map((span) => sync.register(new Signal(Milli(100)), span)); + try { + await clock.advance(Nano.zero); + sync.received(Milli(500)); + spans[first].set({ start: Milli(500), end: Milli(600) }); + await clock.advance(Nano.zero); + let delivered = false; + const waiting = sync.wait(Milli(500)).then((ok) => { + delivered = ok; + }); + await clock.advance(Nano.fromMilli(Milli(1000))); + expect(delivered).toBe(false); + expect(sync.now()).toBeUndefined(); + spans[first === "audio" ? "video" : "audio"].set({ start: Milli(500), end: Milli(599) }); + await clock.advance(Nano.zero); + expect(delivered).toBe(false); + spans[first === "audio" ? "video" : "audio"].set({ start: Milli(500), end: Milli(600) }); + await clock.advance(Nano.zero); + await waiting; + expect(delivered).toBe(true); + expect(sync.now()).toBe(Milli(500)); + } finally { + for (const fn of dispose) fn(); + sync.close(); + installed.close(); + } + }); + } + it("releases startup waits on abort, reset, and close without starting playback", async () => { + const clock = new Manual(); + const installed = clock.install(); + const sync = new Sync({ delay: Milli(100), buffer: Milli(1000) }); + const dispose = sync.register(new Signal(Milli(100)), new Signal(null)); + try { + await clock.advance(Nano.zero); + sync.received(Milli(0)); + await clock.advance(Nano.zero); + const abort = new AbortController(); + const cancelled = sync.wait(Milli(0), abort.signal); + abort.abort(); + expect(await cancelled).toBe(false); + const reset = sync.wait(Milli(0)); + sync.reset(); + sync.received(Milli(0)); + expect(await reset).toBe(false); + const closed = sync.wait(Milli(0)); + sync.close(); + expect(await closed).toBe(false); + } finally { + dispose(); + sync.close(); + installed.close(); + } + }); + it("removes a disabled track from the floor and leaves live pacing unchanged", async () => { + const clock = new Manual(); + const installed = clock.install(); + const buffer = new Signal(Milli(1000)); + const sync = new Sync({ delay: Milli(100), buffer }); + const absent = new Signal<{ start: Milli; end: Milli } | null | undefined>(null); + const remove = sync.register(new Signal(Milli(100)), absent); + const video = sync.register(new Signal(Milli(100)), new Signal({ start: Milli(0), end: Milli(100) })); + try { + await clock.advance(Nano.zero); + sync.received(Milli(0)); + await clock.advance(Nano.zero); + expect(sync.playing.peek()).toBe(false); + absent.set(undefined); + await clock.advance(Nano.zero); + expect(await sync.wait(Milli(0))).toBe(true); + buffer.set(Milli.zero); + sync.reset(); + absent.set(null); + await clock.advance(Nano.zero); + sync.received(Milli(0)); + const waiting = sync.wait(Milli(0)); + await clock.advance(Nano.fromMilli(Milli(100))); + expect(await waiting).toBe(true); + } finally { + remove(); + video(); + sync.close(); + installed.close(); + } + }); +}); diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index ea6218a0763..f0aafdbf54d 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -35,7 +35,9 @@ export type SyncInput = { * Zero minimizes latency, the live default: playback re-anchors as soon as anything arrives * early. A larger value lets faster-than-real-time frames (a TTS response with future * timestamps) build up instead of being skipped. Always finite, so worst case the audio ring - * drops its oldest samples rather than exhausting memory. + * drops its oldest samples rather than exhausting memory. With a nonzero buffer, the first video + * frame is a poster until every enabled media track holds the resolved delay; `buffer` is + * lookahead, not the amount required to start. */ buffer: Getter; }; @@ -83,7 +85,17 @@ export class Sync { /** The derived outputs, read-only. */ readonly out = readonlys(this.#out); - #media = new Signal<{ target: Getter }[]>([]); + #media = new Signal< + { + target: Getter; + // undefined is inactive, null is selected but has no playable first span yet. + startup?: Getter<{ start: Milli; end: Milli } | null | undefined>; + }[] + >([]); + #started = false; + #playing = new Signal(true); + /** Whether buffered startup has released the media consumers. @internal */ + readonly playing: Getter = this.#playing; // Bumped by every `reset()`. A frame that waited across a bump belongs to the previous timeline, // even when the next frame restores the same reference value. @@ -106,18 +118,54 @@ export class Sync { this.#signals.run(this.#runInstant.bind(this)); this.#signals.run(this.#runDelay.bind(this)); this.#signals.run(this.#runMaxDelay.bind(this)); + this.#signals.run(this.#runStartup.bind(this)); } /** * Hold a decoder's "auto" playout target in the shared playback clock until disposed. * @internal */ - register(target: Getter): Dispose { - const registered = { target }; + register( + target: Getter, + startup?: Getter<{ start: Milli; end: Milli } | null | undefined>, + ): Dispose { + const registered = { target, startup }; this.#media.update((media) => [...media, registered]); return () => this.#media.update((media) => media.filter((candidate) => candidate !== registered)); } + // Startup is data-driven: time passing cannot fill a missing track. The first picture still + // travels through the ordinary renderer while the consumers wait for this shared gate. + #runStartup(effect: Effect): void { + effect.get(this.#epoch); + const buffered = effect.get(this.#out.buffered); + const delay = effect.get(this.#out.delay); + const reference = effect.get(this.#out.reference); + const spans = effect.get(this.#media).flatMap(({ startup }) => { + const span = startup && effect.get(startup); + return span === undefined ? [] : [span]; + }); + if (!buffered || spans.length === 0) { + this.#playing.set(true); + return; + } + if (this.#started) return; + const ready = spans.filter((span) => span !== null); + if ( + reference === undefined || + ready.length !== spans.length || + ready.some((span) => span.end - span.start < delay) + ) { + this.#playing.set(false); + return; + } + const first = Math.min(...ready.map((span) => span.start)); + this.#started = true; + // Anchor at the retained opening, not at its arrival: startup may have waited on another track. + this.#out.reference.set(Nano.toMilli(Nano(now())) - first - delay); + this.#playing.set(true); + } + #runInstant(effect: Effect): void { this.#out.instant.set(effect.get(this.in.delay) === "instant"); } @@ -166,6 +214,10 @@ export class Sync { return false; } + // Retain the opening while another track fills; the ordinary max-delay bounds still own + // storage, but a later arrival must not advance the shared playhead before startup. + if (!this.#playing.peek()) return false; + // Check if `wait()` would not sleep at all. // NOTE: We check here instead of in `wait()` so we can identify when frames are received late. // Otherwise, chained `wait()` calls would cause a false-positive during CPU starvation. @@ -203,6 +255,8 @@ export class Sync { reset(): void { this.#epoch.update((epoch) => epoch + 1); this.#floor = undefined; + this.#started = false; + this.#playing.set(!this.#out.buffered.peek()); this.#out.reference.set(undefined); } @@ -222,9 +276,10 @@ export class Sync { /** * The PTS that should be rendering right now, derived from the reference + delay; negative - * before the first frame's turn. Undefined if no frames have been received yet. + * before the first frame's turn. Undefined before ingress or while buffered startup is filling. */ now(): Milli | undefined { + if (!this.#playing.peek()) return undefined; const reference = this.#out.reference.peek(); if (reference === undefined) return undefined; return Milli(Nano.toMilli(Nano(now())) - reference - this.#out.delay.peek()); @@ -253,6 +308,11 @@ export class Sync { // Peek the input, not `out.instant`, which lags a same-turn change until effects flush. if (this.in.delay.peek() === "instant") return true; + if (!this.#playing.peek()) { + await this.#sleep(undefined, signal); + continue; + } + // NOTE: This function runs in parallel for each frame. const sleep = reference + timestamp - Nano.toMilli(Nano(now())) + this.#out.delay.peek(); @@ -266,7 +326,7 @@ export class Sync { // Sleeps for `ms`, or wakes early once anything the sleep was computed from changes or `signal` // aborts. Releases every listener either way: frames sleep once each, so a listener left on a // signal that never changes would pile up for the life of the player. - #sleep(ms: number, signal?: AbortSignal): Promise { + #sleep(ms: number | undefined, signal?: AbortSignal): Promise { return new Promise((resolve) => { const timer = new AbortController(); const wake = () => { @@ -274,13 +334,14 @@ export class Sync { for (const dispose of disposes) dispose(); resolve(); }; - void sleep(Nano.fromMilli(Milli(ms)), { signal: timer.signal }).then(wake, () => {}); + if (ms !== undefined) void sleep(Nano.fromMilli(Milli(ms)), { signal: timer.signal }).then(wake, () => {}); const disposes = [ this.in.delay.changed(wake), this.#out.delay.changed(wake), this.#out.reference.changed(wake), this.#epoch.changed(wake), this.#closed.changed(wake), + this.#playing.changed(wake), ]; if (signal) { signal.addEventListener("abort", wake, { once: true }); diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index f9663810325..1d0468fea39 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -200,13 +200,20 @@ const gap: Read = { kind: "gap", timestamp: Micro(0), duration: Micro(0) }; // `reads` stands in for the container consumer, unless `live`, which reads what tests publish into // `producer` through real consumers instead. -async function guardedPlayback(kind: "legacy" | "cmaf", reads: Read[], hold: readonly number[] = [], live = false) { +async function guardedPlayback( + kind: "legacy" | "cmaf", + reads: Read[], + hold: readonly number[] = [], + live = false, + buffer = 0, +) { // Load the optional container before the fixture drains only reactive microtasks. if (kind === "cmaf") await import("@moq/hang/cmaf"); const originalDecoder = Object.getOwnPropertyDescriptor(globalThis, "VideoDecoder"); const originalChunk = Object.getOwnPropertyDescriptor(globalThis, "EncodedVideoChunk"); const submitted: number[] = []; const outputs: number[] = []; + const pictures: Picture[] = []; const owed = new Set(hold); const pending: Array<() => void> = []; let resets = 0; @@ -245,7 +252,9 @@ async function guardedPlayback(kind: "legacy" | "cmaf", reads: Read[], hold: rea submitted.push(chunk.timestamp); const emit = () => { outputs.push(chunk.timestamp); - void this.callbacks.output(new Picture(chunk.timestamp) as unknown as VideoFrame); + const picture = new Picture(chunk.timestamp); + pictures.push(picture); + void this.callbacks.output(picture as unknown as VideoFrame); }; if (owed.delete(chunk.timestamp)) { pending.push(emit); @@ -301,8 +310,8 @@ async function guardedPlayback(kind: "legacy" | "cmaf", reads: Read[], hold: rea mediaProducer.close(); }; const source = new Source({ broadcast, supported: async () => true }); - const delay = new Signal("instant"); - const sync = new Sync({ delay }); + const delay = new Signal(buffer ? Milli(100) : "instant"); + const sync = new Sync({ delay, buffer: Milli(buffer) }); const enabled = new Signal(true); // Published before the decoder subscribes, since a broadcast with no request handler refuses // an unknown track. @@ -330,6 +339,7 @@ async function guardedPlayback(kind: "legacy" | "cmaf", reads: Read[], hold: rea track, submitted, outputs, + pictures, sync, get resets() { return resets; @@ -1001,3 +1011,151 @@ it("a video track read from another path does not inherit the outgoing delay", a clock.installed.close(); } }); + +// The first picture is useful before the startup floor, but elapsed wall time is not buffered media. +it("buffered video holds its poster until its own span reaches the floor", async () => { + const clock = manualAt(0); + const playback = await guardedPlayback("legacy", [], [], true, 1000); + const group = new Moq.Group.Producer(0); + if (!playback.track) throw new Error("missing video track"); + playback.track.writeGroup(group); + try { + group.writeFrame(legacyFrame(Micro(0))); + group.writeFrame(legacyFrame(Micro(40_000))); + await microtasks(); + expect(playback.decoder.out.frame.peek()?.timestamp).toBe(0); + await clock.advance(500); + expect(playback.decoder.out.frame.peek()?.timestamp).toBe(0); + group.writeFrame(legacyFrame(Micro(100_000))); + await microtasks(); + await clock.advance(40); + expect(playback.decoder.out.frame.peek()?.timestamp).toBe(40_000); + } finally { + group.close(); + playback.close(); + await microtasks(); + clock.installed.close(); + } +}); + +it("bounds startup decoding while another track is missing and releases it on close", async () => { + const clock = manualAt(0); + const playback = await guardedPlayback("legacy", [], [], true, 1000); + const remove = playback.sync.register(new Signal(Milli(100)), new Signal(null)); + if (!playback.track) throw new Error("missing video track"); + const group = playback.track.appendGroup(); + try { + for (let i = 0; i < 20; i++) group.writeFrame(legacyFrame(Micro(i * 40_000))); + await microtasks(); + await clock.advance(1000); + expect(playback.submitted).toEqual([0, 40_000, 80_000, 120_000]); + expect(playback.decoder.out.frame.peek()?.timestamp).toBe(0); + playback.close(); + await microtasks(); + expect(playback.pictures.every((frame) => frame.closed)).toBe(true); + } finally { + remove(); + group.close(); + playback.close(); + await microtasks(); + clock.installed.close(); + } +}); + +it("refills the video floor after a discontinuity instead of waiting on the previous poster timestamp", async () => { + const clock = manualAt(0); + const reads: Read[] = [ + sample(0, 0), + sample(0, 100_000), + marker(0, 100_000), + sample(1, 1_000_000), + sample(1, 1_100_000), + ]; + const playback = await guardedPlayback("legacy", reads, [], false, 1000); + try { + await clock.advance(100); + await microtasks(); + await clock.advance(100); + await microtasks(); + expect(playback.decoder.out.frame.peek()?.timestamp).toBeGreaterThanOrEqual(1_000_000); + } finally { + playback.close(); + await microtasks(); + clock.installed.close(); + } +}); + +it("continues filling when the shared floor rises while video waits on audio", async () => { + const clock = manualAt(0); + const playback = await guardedPlayback("legacy", [], [], true, 1000); + const audio = new Signal<{ start: Milli; end: Milli } | null>(null); + const remove = playback.sync.register(new Signal(Milli(100)), audio); + if (!playback.track) throw new Error("missing video track"); + const group = playback.track.appendGroup(); + try { + for (let i = 0; i < 7; i++) group.writeFrame(legacyFrame(Micro(i * 40_000))); + await microtasks(); + await clock.advance(0); + expect(playback.submitted).toEqual([0, 40_000, 80_000, 120_000]); + playback.delay.set(Milli(200)); + audio.set({ start: Milli(0), end: Milli(200) }); + await microtasks(); + await clock.advance(240); + expect(playback.decoder.out.frame.peek()?.timestamp).toBeGreaterThanOrEqual(200_000); + } finally { + remove(); + group.close(); + playback.close(); + await microtasks(); + clock.installed.close(); + } +}); + +it("keeps feeding a decoder that has not output its first picture yet", async () => { + const clock = manualAt(0); + const playback = await guardedPlayback("legacy", [], [0, 40_000, 80_000, 120_000], true, 1000); + if (!playback.track) throw new Error("missing video track"); + const group = playback.track.appendGroup(); + try { + for (let i = 0; i < 5; i++) group.writeFrame(legacyFrame(Micro(i * 40_000))); + await microtasks(); + await clock.advance(160); + expect(playback.decoder.out.frame.peek()?.timestamp).toBe(160_000); + } finally { + group.close(); + playback.close(); + await microtasks(); + clock.installed.close(); + } +}); + +for (const kind of ["legacy", "cmaf"] as const) { + it(`cancels a ${kind} startup wait on disable and can enable a fresh decoder`, async () => { + const clock = manualAt(0); + const playback = await guardedPlayback(kind, [sample(0, 0), sample(0, 100_000)], [0, 100_000], false, 1000); + const other = new Signal<{ start: Milli; end: Milli } | null>(null); + const remove = playback.sync.register(new Signal(Milli(100)), other); + try { + playback.release(); + await microtasks(); + expect(playback.sync.playing.peek()).toBe(false); + expect(playback.decoder.out.frame.peek()?.timestamp).toBe(0); + playback.enabled.set(false); + await microtasks(); + expect(playback.pictures.every((frame) => frame.closed)).toBe(true); + const codecs = playback.codecs; + playback.enabled.set(true); + await microtasks(); + expect(playback.codecs).toBe(codecs + 1); + // A late output from the cancelled codec must not replace the retained poster. + playback.emit(900_000, 0); + await microtasks(); + expect(playback.decoder.out.frame.peek()?.timestamp).toBe(0); + } finally { + remove(); + playback.close(); + await microtasks(); + clock.installed.close(); + } + }); +} diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index b3e3e609e0f..390040147cb 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -9,6 +9,7 @@ import { type Getter, getter, type Inputs, + Once, type Readonlys, readonlys, Signal, @@ -146,7 +147,19 @@ export class Decoder { const delay = active && effect.get(active.catalog)?.video?.renditions?.[active.track]?.delay; return Milli(Math.max(spread + (delay ?? 0), jitter ?? Milli.zero)); }); - this.#signals.cleanup(this.sync.register(target)); + const startup = this.#signals.computed((effect) => { + if (!effect.get(this.in.enabled)) return undefined; + if (effect.get(this.source.pending)) return null; + if (!effect.get(this.source.out.config)) return undefined; + const active = effect.get(this.#active); + if (!active) return null; + const timestamp = effect.get(active.first); + if (timestamp === undefined) return null; + const ranges = effect.get(active.buffered); + const span = ranges.find((range) => range.start <= timestamp && range.end >= timestamp); + return span ? { ...span } : { start: timestamp, end: timestamp }; + }); + this.#signals.cleanup(this.sync.register(target, startup)); this.#identity = this.#signals.computed((effect) => { const config = effect.get(this.source.out.config); return config ? playbackIdentity(config) : undefined; @@ -426,6 +439,8 @@ class DecoderTrack { // Network jitter + decode buffer. buffered = new Signal([]); + // First decoded picture on this timeline, independent of the previously held poster. + readonly first = new Signal(undefined); // How late frames arrive relative to the earliest one, from the container consumer. spread = new Signal(undefined); @@ -492,6 +507,8 @@ class DecoderTrack { return; } + if (this.first.peek() === undefined) this.first.set(timestamp); + // Render something while we wait for the sync to catch up, unless the frame is // already off the timeline: WebCodecs can hold a picture past a reset or a skip. if (this.frame.peek() === undefined && this.sync.presentable(timestamp)) { @@ -626,6 +643,7 @@ class DecoderTrack { latest = next.group; this.#resetAtKeyframe(decoder, frame.keyframe); decoder.decode(chunk); + if (!this.sync.playing.peek()) await this.#waitStartup(effect, Milli.fromMicro(frame.timestamp)); } }); } @@ -708,15 +726,39 @@ class DecoderTrack { timestamp: frame.timestamp, }), ); + if (!this.sync.playing.peek()) await this.#waitStartup(effect, Milli.fromMicro(frame.timestamp)); } }); } + // Once this track has filled its floor, leave further media encoded upstream while the other + // track fills. Otherwise a missing audio track could retain unbounded decoded GPU frames. + async #waitStartup(effect: Effect, timestamp: Milli): Promise { + // Some codecs need more input before their first output. Do not park that input + // behind a gate that needs the very picture the decoder has not produced yet. + if (this.first.peek() === undefined) return; + const span = this.#buffered.peek()[0]; + if (!span || span.end - span.start < this.sync.out.delay.peek()) return; + const waiting = new Effect(); + const refill = new Once(); + waiting.run((inner) => { + // An adaptive floor can rise while another track fills. Resume decoding to reach it, + // rather than waiting on a clock whose startup now needs more video. + if (span.end - span.start < inner.get(this.sync.out.delay)) refill.set(true); + }); + try { + await effect.race(refill, this.sync.wait(timestamp, waiting.abort)); + } finally { + waiting.close(); + } + } + // The container read a gap or a declared end after media. Re-anchor and let the next decoded // frame decide the decoder reset (see #resetAtKeyframe). Clearing `timestamp` keeps a stale // high value from late-rejecting the following frames, which may overlap the previous group's // tail. The held picture stays until the new keyframe renders. #onBreak(): void { + this.first.set(undefined); this.timestamp.set(undefined); this.#buffered.set([]); this.sync.reset(); diff --git a/js/watch/src/video/source.ts b/js/watch/src/video/source.ts index c79042917dc..3c4fa40cd75 100644 --- a/js/watch/src/video/source.ts +++ b/js/watch/src/video/source.ts @@ -214,6 +214,9 @@ export class Source { }; /** The derived outputs, read-only. */ readonly out = readonlys(this.#out); + #pending = new Signal(false); + /** Whether rendition support is still being resolved. @internal */ + readonly pending: Getter = this.#pending; #signals = new Effect(); #supportCache = new WeakMap>(); @@ -260,6 +263,7 @@ export class Source { if (!names.has(name)) cache.delete(name); } + effect.set(this.#pending, true, false); effect.spawn(async () => { const available: Record = {}; @@ -304,6 +308,7 @@ export class Source { this.#out.error.set(undefined); } this.#out.available.set(available); + this.#pending.set(false); }); } diff --git a/test/video/renderer.browser.ts b/test/video/renderer.browser.ts index 8e10ff5f818..652cd005dec 100644 --- a/test/video/renderer.browser.ts +++ b/test/video/renderer.browser.ts @@ -221,3 +221,88 @@ test("closing while the caption chunk loads installs no stale overlay", async () await page.close(); } }); + +for (const audio of [false, true]) { + test(`buffered ${audio ? "A/V" : "video"} paints a poster before the common floor`, async () => { + const page = await open(); + try { + await page.evaluate((audio) => window.startPoster(audio, true), audio); + if (audio) { + await page.locator("body").click(); + await page.waitForFunction(() => window.poster.state().audioState === "running"); + } + await page.evaluate(() => { + window.poster.push(0); + window.poster.push(40_000); + }); + await page.waitForFunction(() => window.poster.state().color === "red"); + // Browser paint observation spans several floor intervals, so a time-only gate fails. + await page.waitForTimeout(300); + expect(await page.evaluate(() => window.poster.state())).toMatchObject({ + timestamp: 0, + color: "red", + playing: false, + }); + await page.evaluate(() => window.poster.push(100_000)); + if (audio) { + await page.waitForTimeout(300); + expect(await page.evaluate(() => window.poster.state())).toMatchObject({ + timestamp: 0, + color: "red", + playing: false, + }); + await page.evaluate(() => window.poster.audio()); + } + await page.waitForFunction(() => window.poster.state().color === "blue"); + expect(await page.evaluate(() => window.poster.state().playing)).toBe(true); + } finally { + await page.evaluate(() => window.poster?.close()); + await page.close(); + } + }, 30_000); +} + +test("live video does not wait to fill the buffered startup floor", async () => { + const page = await open(); + try { + await page.evaluate(() => window.startPoster(false, false)); + await page.evaluate(() => { + window.poster.push(0); + window.poster.push(40_000); + }); + await page.waitForFunction(() => window.poster.state().color === "blue"); + expect(await page.evaluate(() => window.poster.state().timestamp)).toBe(40_000); + } finally { + await page.evaluate(() => window.poster?.close()); + await page.close(); + } +}, 30_000); + +test("buffered audio keeps its opening while video fills, with a running audio context", async () => { + const page = await open(); + try { + await page.evaluate(() => window.startPoster(true, true)); + await page.locator("body").click(); + await page.waitForFunction(() => window.poster.state().audioState === "running"); + await page.evaluate(() => window.poster.audio()); + const initial = await page.evaluate(() => window.poster.state().audioTimestamp); + await page.waitForTimeout(300); + expect(await page.evaluate(() => window.poster.state())).toMatchObject({ + audioTimestamp: initial, + playing: false, + }); + await page.evaluate(() => { + window.poster.push(0); + window.poster.push(40_000); + }); + await page.waitForFunction(() => window.poster.state().color === "red"); + expect(await page.evaluate(() => window.poster.state().playing)).toBe(false); + await page.evaluate(() => window.poster.push(100_000)); + await page.waitForFunction( + () => window.poster.state().color === "blue" && (window.poster.state().audioTimestamp ?? 0) > 0, + ); + } finally { + await page.evaluate(() => window.poster?.close()); + await page.close(); + } +}, 30_000); diff --git a/test/video/src/main.ts b/test/video/src/main.ts index aa3c7214dba..c88b3a22354 100644 --- a/test/video/src/main.ts +++ b/test/video/src/main.ts @@ -8,6 +8,7 @@ */ import "./gpu.ts"; import "./captions.ts"; +import "./poster.ts"; import "@moq/publish/element"; import "@moq/watch/element"; import { Signal } from "@moq/signals"; diff --git a/test/video/src/poster.ts b/test/video/src/poster.ts new file mode 100644 index 00000000000..a1f28584f39 --- /dev/null +++ b/test/video/src/poster.ts @@ -0,0 +1,144 @@ +import { Micro, Milli } from "@moq/clock"; +import * as Catalog from "@moq/hang/catalog"; +import { Legacy } from "@moq/hang/container"; +import { Origin, Path } from "@moq/net"; +import { Player } from "@moq/watch"; + +// Real WebCodecs and rendering, with publication controlled independently of wall time. +async function poster(audio: boolean, buffered: boolean) { + const origin = new Origin.Producer(); + const name = Path.from("poster"); + const producer = origin.createBroadcast(name); + const track = producer.createTrack("video"); + const group = track.appendGroup(); + const audioTrack = audio ? producer.createTrack("audio") : undefined; + const audioGroup = audioTrack?.appendGroup(); + const catalog: Catalog.Root = { + video: { + renditions: { + video: Catalog.VideoConfigSchema.parse({ + codec: "vp8", + codedWidth: 64, + codedHeight: 32, + container: { kind: "legacy" }, + }), + }, + }, + ...(audio && { + audio: { + renditions: { + audio: Catalog.AudioConfigSchema.parse({ + codec: "opus", + sampleRate: 48000, + numberOfChannels: 1, + container: { kind: "legacy" }, + }), + }, + }, + }), + }; + producer.announce(); + const canvas = document.createElement("canvas"); + document.body.append(canvas); + const player = new Player({ + origin, + name, + catalogFormat: "manual", + catalog, + canvas, + visible: "always", + backend: "2d", + delay: Milli(100), + buffer: Milli(buffered ? 1000 : 0), + }); + const chunks = new Map(); + const encoder = new VideoEncoder({ + output(chunk) { + const data = new Uint8Array(chunk.byteLength); + chunk.copyTo(data); + chunks.set(chunk.timestamp, data); + }, + error(error) { + throw error; + }, + }); + encoder.configure({ codec: "vp8", width: 64, height: 32, bitrate: 100_000, latencyMode: "realtime" }); + const picture = new OffscreenCanvas(64, 32); + const ctx = picture.getContext("2d"); + if (!ctx) throw new Error("no encoder canvas context"); + for (const timestamp of [0, 40_000, 100_000, 140_000]) { + ctx.fillStyle = timestamp === 0 ? "red" : "blue"; + ctx.fillRect(0, 0, 64, 32); + const frame = new VideoFrame(picture, { timestamp }); + encoder.encode(frame, { keyFrame: true }); + frame.close(); + } + await encoder.flush(); + encoder.close(); + const push = (timestamp: number) => { + const payload = chunks.get(timestamp); + if (!payload) throw new Error("missing encoded frame"); + group.writeFrame({ payload: Legacy.encodeFrame(payload, Micro(timestamp)) }); + }; + return { + push, + async audio() { + if (!audioGroup) throw new Error("no audio track"); + const encode = new AudioEncoder({ + output(chunk) { + const data = new Uint8Array(chunk.byteLength); + chunk.copyTo(data); + audioGroup.writeFrame({ payload: Legacy.encodeFrame(data, Micro(chunk.timestamp)) }); + }, + error(error) { + throw error; + }, + }); + encode.configure({ codec: "opus", sampleRate: 48000, numberOfChannels: 1, bitrate: 64_000 }); + for (let i = 0; i < 12; i++) { + const frame = new AudioData({ + format: "f32-planar", + sampleRate: 48000, + numberOfFrames: 960, + numberOfChannels: 1, + timestamp: i * 20_000, + data: new Float32Array(960), + }); + encode.encode(frame); + frame.close(); + } + await encode.flush(); + encode.close(); + }, + state() { + const context = canvas.getContext("2d"); + if (!context) throw new Error("no renderer canvas context"); + const rgba = context.getImageData(32, 16, 1, 1).data; + return { + timestamp: player.renderer.out.frame.peek()?.timestamp, + playing: player.sync.playing.peek(), + color: rgba[0] > 160 ? "red" : rgba[2] > 160 ? "blue" : "black", + audioTimestamp: player.audio.out.timestamp.peek(), + audioState: player.audio.out.context.peek()?.state, + }; + }, + close() { + player.close(); + group.close(); + audioGroup?.close(); + producer.close(); + origin.close(); + canvas.remove(); + }, + }; +} + +declare global { + interface Window { + poster: Awaited>; + startPoster(audio: boolean, buffered: boolean): Promise; + } +} +window.startPoster = async (audio, buffered) => { + window.poster = await poster(audio, buffered); +}; From cd800008e90664fe3809d8a715f5cf89d9f5575a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 22:39:05 -0700 Subject: [PATCH 2/4] fix(watch): resume video decoding when buffered startup releases Video parked at its startup floor now waits on the span's first picture, which is due the moment startup releases, rather than on the last parked frame, which left no decoded lookahead right after startup. Also adapt the poster harness to the nested Player props from main, run the Video gate for the watch playback path it drives, document the poster, and delete the finished quest. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release-gate.yml | 4 +++ .github/workflows/video.yml | 6 ++--- doc/lib/js/watch.md | 6 +++-- js/watch/src/video/decoder.test.ts | 26 ++++++++++++++++++++ js/watch/src/video/decoder.ts | 10 +++++--- quest/m1/README.md | 1 - quest/m1/buffered-ascending.md | 3 +-- quest/m1/buffered-video-poster.md | 39 ------------------------------ test/video/src/poster.ts | 4 +-- 9 files changed, 45 insertions(+), 54 deletions(-) delete mode 100644 quest/m1/buffered-video-poster.md diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml index 9b863fcbbd4..290a53ce219 100644 --- a/.github/workflows/release-gate.yml +++ b/.github/workflows/release-gate.yml @@ -256,6 +256,10 @@ jobs: - "js/video/**" - "js/publish/src/preview.ts" - "js/publish/src/element.ts" + - "js/watch/src/player.ts" + - "js/watch/src/sync.ts" + - "js/watch/src/audio/**" + - "js/watch/src/video/**" - "test/video/**" - "test/justfile" - "package.json" diff --git a/.github/workflows/video.yml b/.github/workflows/video.yml index 1290fb11d4e..a9287a7fa3a 100644 --- a/.github/workflows/video.yml +++ b/.github/workflows/video.yml @@ -4,9 +4,9 @@ name: Video # the WebGPU path, a lost device, and the Canvas2D fallback only exist in a real # browser. SwiftShader stands in for the GPU, so no GPU runner is needed. # -# Narrow trigger, like wasm.yml: the renderer, the harness, and the two files that -# wire the renderer into `` (which the harness drives). Buffered -# `@moq/watch` startup is driven from an in-memory origin, without a relay. +# Narrow trigger, like wasm.yml: the renderer, the harness, and what the harness +# drives: the two files that wire the renderer into ``, and the +# `@moq/watch` playback path, fed from an in-memory origin so no relay is needed. permissions: contents: read diff --git a/doc/lib/js/watch.md b/doc/lib/js/watch.md index ba819962dbe..baffe6c1dbd 100644 --- a/doc/lib/js/watch.md +++ b/doc/lib/js/watch.md @@ -188,8 +188,10 @@ response emitted in one burst with future timestamps, wants the opposite. Set Durations need a unit; a bare number is rejected. Audio holds the buffer as encoded frames, so a large one is cheap; video waits as decoded pictures, which -hold decoder memory, so a long video buffer is not. `el.reset()` flushes and re-anchors at the -next frame, which is how a producer interrupts an utterance. +hold decoder memory, so a long video buffer is not. The first video frame shows +as a poster at once, and playback starts once every enabled track has buffered +the delay. `el.reset()` flushes and re-anchors at the next frame, which is how a +producer interrupts an utterance. ## Strict CSP diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index 1d0468fea39..6860a8cf4cc 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -1062,6 +1062,32 @@ it("bounds startup decoding while another track is missing and releases it on cl } }); +// Resuming only once the held floor has played out would leave no decoded lookahead. +it("resumes decoding the moment shared startup releases", async () => { + const clock = manualAt(0); + const playback = await guardedPlayback("legacy", [], [], true, 1000); + const audio = new Signal<{ start: Milli; end: Milli } | null>(null); + const remove = playback.sync.register(new Signal(Milli(100)), audio); + if (!playback.track) throw new Error("missing video track"); + const group = playback.track.appendGroup(); + try { + for (let i = 0; i < 8; i++) group.writeFrame(legacyFrame(Micro(i * 40_000))); + await microtasks(); + await clock.advance(0); + expect(playback.submitted).toEqual([0, 40_000, 80_000, 120_000]); + audio.set({ start: Milli(0), end: Milli(100) }); + await microtasks(); + await clock.advance(0); + expect(playback.submitted).toHaveLength(8); + } finally { + remove(); + group.close(); + playback.close(); + await microtasks(); + clock.installed.close(); + } +}); + it("refills the video floor after a discontinuity instead of waiting on the previous poster timestamp", async () => { const clock = manualAt(0); const reads: Read[] = [ diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 390040147cb..76b186bd2e3 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -643,7 +643,7 @@ class DecoderTrack { latest = next.group; this.#resetAtKeyframe(decoder, frame.keyframe); decoder.decode(chunk); - if (!this.sync.playing.peek()) await this.#waitStartup(effect, Milli.fromMicro(frame.timestamp)); + if (!this.sync.playing.peek()) await this.#waitStartup(effect); } }); } @@ -726,14 +726,14 @@ class DecoderTrack { timestamp: frame.timestamp, }), ); - if (!this.sync.playing.peek()) await this.#waitStartup(effect, Milli.fromMicro(frame.timestamp)); + if (!this.sync.playing.peek()) await this.#waitStartup(effect); } }); } // Once this track has filled its floor, leave further media encoded upstream while the other // track fills. Otherwise a missing audio track could retain unbounded decoded GPU frames. - async #waitStartup(effect: Effect, timestamp: Milli): Promise { + async #waitStartup(effect: Effect): Promise { // Some codecs need more input before their first output. Do not park that input // behind a gate that needs the very picture the decoder has not produced yet. if (this.first.peek() === undefined) return; @@ -747,7 +747,9 @@ class DecoderTrack { if (span.end - span.start < inner.get(this.sync.out.delay)) refill.set(true); }); try { - await effect.race(refill, this.sync.wait(timestamp, waiting.abort)); + // Wait on the span's first picture, which is due the moment startup releases, so decoding + // resumes with the whole floor still ahead instead of once it has played out. + await effect.race(refill, this.sync.wait(span.start, waiting.abort)); } finally { waiting.close(); } diff --git a/quest/m1/README.md b/quest/m1/README.md index 63d7a3a8c71..1fd98e9aaf0 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -134,7 +134,6 @@ audit's receive-memory, priority, and lease bugs rank ahead of that reshape. - [A/V clock](/quest/m1/av-clock.md) - the audio playhead drives Sync.reference while audio plays, through per-track sync handles - [Container reads drop backfill](/quest/m1/read-backfill.md) - the JS container consumer drops groups below its read cursor like Rust - [Buffered playback subscribes in ascending order](/quest/m1/buffered-ascending.md) - a buffered `` asks for the oldest group first, so a voice reply's opening arrives first, and audio's head hold shrinks to the floor -- [Buffered video poster](/quest/m1/buffered-video-poster.md) - buffered startup paints the first video frame at once and starts A/V together at a shared floor - [qmux local lifecycle](/quest/m1/qmux-local-lifecycle.md) - an abort blocked on send credit resets at once, and a close behind a backed-up socket still sends APPLICATION_CLOSE - [JS reload config](/quest/m1/js-reload-config.md) - `@moq/net` reload takes Rust's `healthy` backoff and no ignored `extensions` - [JS release preflight](/quest/m1/js-publish-preflight.md) - `release.ts` refuses a package whose `@moq/*` dependency is not on npm, so no dependent publishes uninstallable diff --git a/quest/m1/buffered-ascending.md b/quest/m1/buffered-ascending.md index b47f4748e1d..7edeec56866 100644 --- a/quest/m1/buffered-ascending.md +++ b/quest/m1/buffered-ascending.md @@ -11,7 +11,7 @@ sorted. Live playback (`buffer` of zero) stays newest-first. Not here: the protocol's `Group Order` (landed in [moq-dev/moq#5385](https://github.com/moq-dev/moq/pull/5385)), the buffered -video poster ([Buffered video poster](/quest/m1/buffered-video-poster.md)), and +video poster (landed in [moq-dev/moq#6609](https://github.com/moq-dev/moq/pull/6609)), and ordering for the Python or FFI audio consumers. ## Plan @@ -78,6 +78,5 @@ Implementation: ## Related -- [Buffered video poster](/quest/m1/buffered-video-poster.md) - buffered startup paints the first video frame and holds video until the shared floor - [Container reads drop backfill](/quest/m1/read-backfill.md) - the JS consumer stops delivering groups below its cursor - [Subscribe ranges](/quest/m1/subscribe-ranges/README.md) - owns the wire's `Group Order` and the ordered cursor's floor diff --git a/quest/m1/buffered-video-poster.md b/quest/m1/buffered-video-poster.md deleted file mode 100644 index 24bc8d282e0..00000000000 --- a/quest/m1/buffered-video-poster.md +++ /dev/null @@ -1,39 +0,0 @@ -# [S] Buffered video paints a poster while it fills - -## Goal - -A `` with a nonzero `buffer` renders its first decoded video frame -as soon as it decodes, as a poster, and holds video there until playback -starts. Synchronized audio and video share one floor: playback starts once -every enabled track has buffered the floor, and a video-only broadcast waits -on video's own span. Live playback is unchanged. - -## Plan - -Decided 2026-10-23 by the maintainer, while planning -[Buffered playback subscribes in ascending order](/quest/m1/buffered-ascending.md): - -- ✅ Video holds before playback starts too, but the first frame renders right - away as a poster while the rest loads. -- ✅ Synchronized A/V shares one floor. Not chosen: video starting on its own - floor; audio alone deciding the start. -- ✅ Its own quest, ready now: it has no protocol dependency. Not chosen: one - quest with the ascending work, blocked on lite-07. - -Implementation: - -- First record what buffered video does at startup today: when the first - frame paints, and whether it moves before the shared clock starts. -- Reuse the paused poster path (`renderer.out.frame`, read by `Player`) - rather than adding a second one. -- Gate the shared clock's start on every enabled track reaching the floor, - consistent with [A/V clock](/quest/m1/av-clock.md), where the audio - playhead drives `Sync.reference` once audio plays. -- Tests: the poster renders before the floor; video does not advance until - every enabled track has buffered the floor; video-only starts on its own - span; live mode is unchanged. - -## Related - -- [Buffered playback subscribes in ascending order](/quest/m1/buffered-ascending.md) - buffered subscriptions ask for the oldest group first -- [A/V clock](/quest/m1/av-clock.md) - the audio playhead drives `Sync.reference` while audio plays diff --git a/test/video/src/poster.ts b/test/video/src/poster.ts index a1f28584f39..18a50b0bd11 100644 --- a/test/video/src/poster.ts +++ b/test/video/src/poster.ts @@ -45,9 +45,7 @@ async function poster(audio: boolean, buffered: boolean) { name, catalogFormat: "manual", catalog, - canvas, - visible: "always", - backend: "2d", + video: { canvas, visible: "always", backend: "2d" }, delay: Milli(100), buffer: Milli(buffered ? 1000 : 0), }); From 4c01f0cf9c426a48b12ed940d96f010c3c4bba3e Mon Sep 17 00:00:00 2001 From: shermerL <86936692+shermerL@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:31:47 +0800 Subject: [PATCH 3/4] fix(watch): require contiguous audio coverage before startup --- js/watch/src/audio/decoder.test.ts | 112 ++++++++++++++++++++++++++++- js/watch/src/audio/decoder.ts | 52 ++++++++++---- 2 files changed, 149 insertions(+), 15 deletions(-) diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index cd420ce91ee..cbe2ebd3f8e 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -223,6 +223,7 @@ afterEach(() => { // Hands frames to whichever container consumer is reading, and ends a consumer's reads on close. An // Error stands in for a malformed read. function feed() { + let currentConsumer: Container.Consumer | undefined; const closed = new WeakSet(); const queue: (Read | Error)[] = []; let waiter: @@ -230,6 +231,7 @@ function feed() { | undefined; spyOn(Container.Consumer.prototype, "next").mockImplementation(function (this: Container.Consumer) { + currentConsumer = this; if (closed.has(this)) return Promise.resolve(undefined); const read = queue.shift(); if (read instanceof Error) return Promise.reject(read); @@ -249,19 +251,27 @@ function feed() { close.call(this); }); - return (read: Read | Error) => { + const push = (read: Read | Error) => { const current = waiter; waiter = undefined; if (!current) queue.push(read); else if (read instanceof Error) current.reject(read); else current.resolve(read); }; + return { + push, + // Network arrivals can advance while next() is held behind decoder backpressure. + buffered: (ranges: Container.BufferedRanges) => { + if (!currentConsumer) throw new Error("consumer has not started"); + (currentConsumer.buffered as Signal).set(ranges); + }, + }; } async function play(initial: Delay, buffered = false, cmaf = false) { // These cases test timeline handover after the optional decoder has loaded. if (cmaf) await import("@moq/hang/cmaf"); - const push = feed(); + const { push, buffered: queued } = feed(); const truncate = spyOn(AudioRingBuffer.prototype, "truncate"); const reset = spyOn(AudioRingBuffer.prototype, "reset"); const insert = spyOn(AudioRingBuffer.prototype, "write"); @@ -319,6 +329,7 @@ async function play(initial: Delay, buffered = false, cmaf = false) { delay, sync, push, + queued, enabled, decoding, context: decoder.out.context, @@ -1275,3 +1286,100 @@ it("a rejected audio support probe does not leave buffered video waiting on a pe logged.mockRestore(); } }); + +it.each([false, true])("does not bridge a missing network range to release shared startup (CMAF=%s)", async (cmaf) => { + const clock = new Manual(); + const installed = clock.install(); + const playback = await play(Milli(1500), true, cmaf); + const video = new Signal<{ start: Milli; end: Milli } | null>({ start: Milli(20), end: Milli(1520) }); + const remove = playback.sync.register(new Signal(Milli(1500)), video); + try { + await playback.play(25); + playback.queued([ + { start: Milli(520), end: Milli(620) }, + { start: Milli(1500), end: Milli(3020) }, + ]); + const next = frame(); + next.group = 1; + playback.push(next); + await microtasks(); + await clock.advance(Nano.zero); + const ring = playback.rings()[0]; + expect(ring.ring?.length).toBe(24000); + expect(playback.sync.playing.peek()).toBe(false); + expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(0); + // The gap fills without another frame being pulled out of the consumer. + playback.queued([{ start: Milli(520), end: Milli(3020) }]); + await clock.advance(Nano.zero); + expect(playback.sync.playing.peek()).toBe(true); + expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(960); + } finally { + remove(); + playback.close(); + await microtasks(); + installed.close(); + } +}); + +it("does not bridge a hole in a released encoded head", async () => { + const clock = new Manual(); + const installed = clock.install(); + const playback = await play(Milli(1500), true); + try { + await playback.play(30); + frameTimestamp = 2_000_000; + await playback.play(30); + await clock.advance(Nano.zero); + expect(playback.rings()[0].ring?.length).toBe(24000); + expect(playback.sync.playing.peek()).toBe(false); + } finally { + playback.close(); + await microtasks(); + installed.close(); + } +}); + +it("closing with a hole pending cannot release startup on a late arrival", async () => { + const playback = await play(Milli(1500), true); + const disconnect = spyOn(FakeWorklet.prototype, "disconnect"); + const closeTrack = spyOn(Moq.Track.Subscriber.prototype, "close"); + try { + await playback.play(25); + playback.queued([{ start: Milli(1500), end: Milli(3020) }]); + const next = frame(); + next.group = 1; + playback.push(next); + await microtasks(); + expect(playback.sync.playing.peek()).toBe(false); + const context = playback.context.peek() as unknown as FakeContext; + const decoded = playback.inserted().length; + playback.close(); + await microtasks(); + playback.queued([{ start: Milli(520), end: Milli(3020) }]); + await microtasks(); + expect(context.calls.close).toBe(1); + expect(disconnect).toHaveBeenCalled(); + expect(closeTrack).toHaveBeenCalled(); + expect(playback.inserted()).toHaveLength(decoded); + } finally { + playback.close(); + await microtasks(); + disconnect.mockRestore(); + closeTrack.mockRestore(); + } +}); + +it("keeps rounded packet endpoints contiguous above the PCM floor", async () => { + const playback = await play(Milli(1500), true); + try { + await playback.play(30); + // Container timestamps are rounded to microseconds independently of sample durations. + frameTimestamp += 1; + await playback.play(70); + expect(playback.rings()[0].ring?.length).toBe(24000); + expect(playback.sync.playing.peek()).toBe(true); + } finally { + playback.close(); + await microtasks(); + } +}); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index f995fdfdb5d..2520250c901 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -30,6 +30,11 @@ import { Terminal } from "./terminal"; type FrameRead = Extract; +type Queued = { + newest: Micro | undefined; + buffered: Container.BufferedRanges; +}; + /** Signals the decoder reads. */ export type DecoderInput = { // Whether to download the audio track. Defaults to true. @@ -111,7 +116,7 @@ export class Decoder { // Decode buffer: audio sent to worklet but not yet played #decodeBuffered = new Signal([]); - #queued = new Signal(undefined); + #queued = new Signal(undefined); // Audio ring bridging the main thread and the worklet. #ring = new Signal(undefined); @@ -180,12 +185,16 @@ export class Decoder { if (!effect.get(this.source.out.config)) return undefined; const span = effect.get(this.#decodeBuffered)[0]; if (!span) return null; - // The target above the PCM floor stays encoded. Count only the hold already queued - // behind decoded output, so startup neither demands unbounded PCM nor skips its floor. - const queued = effect.get(this.#queued); + // Deep targets stay encoded, but a later range cannot fill a hole after decoded audio. + const queued = effect.get(this.#queued) ?? []; const hold = Milli.fromMicro(ringHold(effect.get(this.sync.out.delay))); - const end = Math.max(span.end, Math.min(span.end + hold, Milli.fromMicro(queued ?? Micro.zero))); - return { start: span.start, end: Milli(end) }; + let end = span.end; + for (const range of queued) { + // Match decoded-span rounding slack; do not bridge a missing packet. + if (range.start > end + 1) break; + end = Milli(Math.max(end, range.end)); + } + return { start: span.start, end: Milli(Math.min(end, span.end + hold)) }; }); this.#signals.cleanup(this.sync.register(playout, startup)); this.#identity = this.#signals.computed((effect) => { @@ -494,7 +503,7 @@ export class Decoder { const order = this.#order(); const play = (reads: FrameRead[]) => { - read.set(order.newest(reads) as Micro | undefined); + read.set({ newest: order.newest(reads) as Micro | undefined, buffered: queuedRanges(reads, config) }); return this.#play(effect, decoder, decoderConfig, order, reads); }; for (;;) { @@ -583,7 +592,7 @@ export class Decoder { const order = this.#order(); const play = (reads: FrameRead[]) => { - read.set(order.newest(reads) as Micro | undefined); + read.set({ newest: order.newest(reads) as Micro | undefined, buffered: queuedRanges(reads, config) }); return this.#play(effect, decoder, decoderConfig, order, reads); }; for (;;) { @@ -684,7 +693,7 @@ export class Decoder { sub: Moq.Track.Subscriber, format: Container.Format, { config, path }: { config: DecoderConfig; path: Moq.Path.Valid }, - ): { consumer: Container.Consumer; read: Signal } { + ): { consumer: Container.Consumer; read: Signal } { const outgoing = this.#consumer; const continues = outgoing?.path === path; const consumer = new Container.Consumer(sub, { @@ -711,16 +720,18 @@ export class Decoder { // the decode loop's held frame is released once enough is queued behind it. That includes the // frames the loop has read but not decoded yet: a buffered head waits in the reorder buffer // until it spans the whole target, leaving the consumer empty. - const read = new Signal(undefined); + const read = new Signal({ newest: undefined, buffered: [] }); effect.run((inner) => { const ring = inner.get(this.#ring); if (!ring) return; - let newest = inner.get(read); - for (const { end } of inner.get(consumer.buffered)) { + const pending = inner.get(read); + const network = inner.get(consumer.buffered); + let newest = pending.newest; + for (const { end } of network) { const micro = Micro.fromMilli(end); if (newest === undefined || micro > newest) newest = micro; } - inner.set(this.#queued, newest); + inner.set(this.#queued, Container.mergeBufferedRanges(network, pending.buffered)); ring.queued(newest); }); @@ -961,3 +972,18 @@ function spawnFatal(effect: Effect, failure: Signal, task: () } }); } + +// Released reads are timestamp ordered and remain owned by the decode loop while it waits. Count +// their actual duration, not the distance to the next packet, which could include missing audio. +function queuedRanges(reads: FrameRead[], config: DecoderConfig): Container.BufferedRanges { + const ranges: Container.BufferedRanges = []; + for (const { frame } of reads) { + const start = Milli.fromMicro(frame.timestamp); + const duration = packetDuration(config.codec, frame) ?? frameDuration(config) ?? Milli.zero; + const end = Milli(start + duration); + const last = ranges.at(-1); + if (last && start <= last.end + 1) last.end = Milli(Math.max(last.end, end)); + else ranges.push({ start, end }); + } + return ranges; +} From 74c06fd57464c8c65ef5cfaf5291ad09530ff09b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 24 Oct 2026 12:37:24 -0700 Subject: [PATCH 4/4] fix(watch): count queued audio across holes for buffered startup A lost packet must never hold playback startup. Sum the audio queued past the decoded span across holes instead of stopping at the first gap, so missing audio is not counted as buffered and a hole cannot stall startup. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/decoder.test.ts | 75 ++++-------------------------- js/watch/src/audio/decoder.ts | 16 +++---- 2 files changed, 15 insertions(+), 76 deletions(-) diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index cbe2ebd3f8e..1c449b169a0 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -1287,7 +1287,7 @@ it("a rejected audio support probe does not leave buffered video waiting on a pe } }); -it.each([false, true])("does not bridge a missing network range to release shared startup (CMAF=%s)", async (cmaf) => { +it.each([false, true])("releases shared startup across a hole in queued audio (CMAF=%s)", async (cmaf) => { const clock = new Manual(); const installed = clock.install(); const playback = await play(Milli(1500), true, cmaf); @@ -1295,9 +1295,10 @@ it.each([false, true])("does not bridge a missing network range to release share const remove = playback.sync.register(new Signal(Milli(1500)), video); try { await playback.play(25); + // The audio after the hole counts, the hole itself does not. playback.queued([ { start: Milli(520), end: Milli(620) }, - { start: Milli(1500), end: Milli(3020) }, + { start: Milli(1500), end: Milli(1600) }, ]); const next = frame(); next.group = 1; @@ -1308,8 +1309,11 @@ it.each([false, true])("does not bridge a missing network range to release share expect(ring.ring?.length).toBe(24000); expect(playback.sync.playing.peek()).toBe(false); expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(0); - // The gap fills without another frame being pulled out of the consumer. - playback.queued([{ start: Milli(520), end: Milli(3020) }]); + // Enough arrives past the hole while it stays open. + playback.queued([ + { start: Milli(520), end: Milli(620) }, + { start: Milli(1500), end: Milli(2500) }, + ]); await clock.advance(Nano.zero); expect(playback.sync.playing.peek()).toBe(true); expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(960); @@ -1320,66 +1324,3 @@ it.each([false, true])("does not bridge a missing network range to release share installed.close(); } }); - -it("does not bridge a hole in a released encoded head", async () => { - const clock = new Manual(); - const installed = clock.install(); - const playback = await play(Milli(1500), true); - try { - await playback.play(30); - frameTimestamp = 2_000_000; - await playback.play(30); - await clock.advance(Nano.zero); - expect(playback.rings()[0].ring?.length).toBe(24000); - expect(playback.sync.playing.peek()).toBe(false); - } finally { - playback.close(); - await microtasks(); - installed.close(); - } -}); - -it("closing with a hole pending cannot release startup on a late arrival", async () => { - const playback = await play(Milli(1500), true); - const disconnect = spyOn(FakeWorklet.prototype, "disconnect"); - const closeTrack = spyOn(Moq.Track.Subscriber.prototype, "close"); - try { - await playback.play(25); - playback.queued([{ start: Milli(1500), end: Milli(3020) }]); - const next = frame(); - next.group = 1; - playback.push(next); - await microtasks(); - expect(playback.sync.playing.peek()).toBe(false); - const context = playback.context.peek() as unknown as FakeContext; - const decoded = playback.inserted().length; - playback.close(); - await microtasks(); - playback.queued([{ start: Milli(520), end: Milli(3020) }]); - await microtasks(); - expect(context.calls.close).toBe(1); - expect(disconnect).toHaveBeenCalled(); - expect(closeTrack).toHaveBeenCalled(); - expect(playback.inserted()).toHaveLength(decoded); - } finally { - playback.close(); - await microtasks(); - disconnect.mockRestore(); - closeTrack.mockRestore(); - } -}); - -it("keeps rounded packet endpoints contiguous above the PCM floor", async () => { - const playback = await play(Milli(1500), true); - try { - await playback.play(30); - // Container timestamps are rounded to microseconds independently of sample durations. - frameTimestamp += 1; - await playback.play(70); - expect(playback.rings()[0].ring?.length).toBe(24000); - expect(playback.sync.playing.peek()).toBe(true); - } finally { - playback.close(); - await microtasks(); - } -}); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 2520250c901..98c5dceb039 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -185,16 +185,14 @@ export class Decoder { if (!effect.get(this.source.out.config)) return undefined; const span = effect.get(this.#decodeBuffered)[0]; if (!span) return null; - // Deep targets stay encoded, but a later range cannot fill a hole after decoded audio. - const queued = effect.get(this.#queued) ?? []; - const hold = Milli.fromMicro(ringHold(effect.get(this.sync.out.delay))); - let end = span.end; - for (const range of queued) { - // Match decoded-span rounding slack; do not bridge a missing packet. - if (range.start > end + 1) break; - end = Milli(Math.max(end, range.end)); + // Deep targets stay encoded, so count the audio queued past the decoded span. Sum it across + // holes: a lost packet must not hold startup, and the missing audio is not counted as buffered. + let queued = 0; + for (const range of effect.get(this.#queued) ?? []) { + queued += Math.max(0, range.end - Math.max(range.start, span.end)); } - return { start: span.start, end: Milli(Math.min(end, span.end + hold)) }; + const hold = Milli.fromMicro(ringHold(effect.get(this.sync.out.delay))); + return { start: span.start, end: Milli(span.end + Math.min(queued, hold)) }; }); this.#signals.cleanup(this.sync.register(playout, startup)); this.#identity = this.#signals.computed((effect) => {