diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml index 9b863fcbbd4..290a53ce219 100644 --- a/.github/workflows/release-gate.yml +++ b/.github/workflows/release-gate.yml @@ -256,6 +256,10 @@ jobs: - "js/video/**" - "js/publish/src/preview.ts" - "js/publish/src/element.ts" + - "js/watch/src/player.ts" + - "js/watch/src/sync.ts" + - "js/watch/src/audio/**" + - "js/watch/src/video/**" - "test/video/**" - "test/justfile" - "package.json" diff --git a/.github/workflows/video.yml b/.github/workflows/video.yml index 7830b2b5a50..a9287a7fa3a 100644 --- a/.github/workflows/video.yml +++ b/.github/workflows/video.yml @@ -4,10 +4,9 @@ name: Video # the WebGPU path, a lost device, and the Canvas2D fallback only exist in a real # browser. SwiftShader stands in for the GPU, so no GPU runner is needed. # -# Narrow trigger, like wasm.yml: the renderer, the harness, and the two files that -# wire the renderer into `` (which the harness drives). `@moq/watch` -# wires it the same way but needs a relay to drive, so its regressions surface -# nightly instead. +# Narrow trigger, like wasm.yml: the renderer, the harness, and what the harness +# drives: the two files that wire the renderer into ``, and the +# `@moq/watch` playback path, fed from an in-memory origin so no relay is needed. permissions: contents: read diff --git a/doc/lib/js/watch.md b/doc/lib/js/watch.md index ba819962dbe..baffe6c1dbd 100644 --- a/doc/lib/js/watch.md +++ b/doc/lib/js/watch.md @@ -188,8 +188,10 @@ response emitted in one burst with future timestamps, wants the opposite. Set Durations need a unit; a bare number is rejected. Audio holds the buffer as encoded frames, so a large one is cheap; video waits as decoded pictures, which -hold decoder memory, so a long video buffer is not. `el.reset()` flushes and re-anchors at the -next frame, which is how a producer interrupts an utterance. +hold decoder memory, so a long video buffer is not. The first video frame shows +as a poster at once, and playback starts once every enabled track has buffered +the delay. `el.reset()` flushes and re-anchors at the next frame, which is how a +producer interrupts an utterance. ## Strict CSP diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index 7ab8c89eccf..c629b3c0f7e 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -1,7 +1,7 @@ import { Micro, Milli, type Second } from "@moq/clock"; import { Effect, type Getter, Signal } from "@moq/signals"; import { ringHold, ringSamples } from "./latency"; -import type { Data, Init, Latency, Reset, State, Truncate } from "./render"; +import type { Data, Init, Latency, Playing, Reset, State, Truncate } from "./render"; /** * Decides when the decode loop may decode a frame, which is how the target delay is split between @@ -275,6 +275,7 @@ export class AudioBuffer { // Bumped per reset or park. A state message posted before the worklet saw it reports the ring // playing on the old terms, which would open the gate. #epoch = 0; + #playing?: Getter; #signals = new Effect(); @@ -287,17 +288,33 @@ export class AudioBuffer { delay: Milli; /** Anchor to the first frame and play through, holding the lookahead upstream. */ buffered: boolean; + /** Hold consumption until the shared startup floor is ready. */ + playing?: Getter; }, ) { - const { channels, rate, buffered } = props; + const { channels, rate, buffered, playing } = props; + this.#playing = props.playing; this.#worklet = worklet; this.channels = channels; this.rate = rate; this.#backpressure = new Backpressure(rate, buffered, props.delay); - const msg: Init = { type: "init", channels, rate, latency: this.#latency(), buffered }; + const msg: Init = { + type: "init", + channels, + rate, + latency: this.#latency(), + buffered, + ...(playing && { playing: playing.peek() }), + }; worklet.port.postMessage(msg); + if (playing) { + this.#signals.run((effect) => { + const msg: Playing = { type: "playing", playing: effect.get(playing), epoch: this.#epoch }; + worklet.port.postMessage(msg); + }); + } this.#signals.event(worklet.port, "message", (ev: Event) => { const data = (ev as MessageEvent).data; @@ -382,7 +399,11 @@ export class AudioBuffer { /** Flush buffered samples and re-stall, ready to anchor the next utterance (buffered mode). */ reset(): void { this.#epoch++; - const msg: Reset = { type: "reset", epoch: this.#epoch }; + const msg: Reset = { + type: "reset", + epoch: this.#epoch, + ...(this.#playing && { playing: this.#playing.peek() }), + }; this.#worklet.port.postMessage(msg); this.#backpressure.reset(); } diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 344182defb1..1c449b169a0 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -7,6 +7,8 @@ import * as Moq from "@moq/net"; import { Effect, Signal } from "@moq/signals"; import type { Broadcast } from "../broadcast"; import { type Delay, Sync } from "../sync"; +import { Decoder as VideoDecoder } from "../video/decoder"; +import { Source as VideoSource } from "../video/source"; import type { Message, State } from "./render"; import { AudioRingBuffer } from "./ring-buffer"; import { Source } from "./source"; @@ -60,10 +62,13 @@ let worklets: FakeWorklet[] = []; class FakeWorklet { ring?: AudioRingBuffer; #epoch = 0; + #playing = true; readonly port = Object.assign(new EventTarget(), { postMessage: (msg: Message) => { - if (msg.type === "init") this.ring = new AudioRingBuffer(msg); - else if (msg.type === "data") this.ring?.write(msg.timestamp, msg.data); + if (msg.type === "init") { + this.ring = new AudioRingBuffer(msg); + this.#playing = msg.playing ?? true; + } else if (msg.type === "data") this.ring?.write(msg.timestamp, msg.data); else if (msg.type === "latency") { this.ring?.setLatency(msg.latency); if (msg.park !== undefined) { @@ -74,7 +79,8 @@ class FakeWorklet { else if (msg.type === "reset") { this.ring?.reset(); this.#epoch = msg.epoch; - } + this.#playing = msg.playing ?? true; + } else if (msg.type === "playing" && msg.epoch === this.#epoch) this.#playing = msg.playing; }, start() {}, }); @@ -91,7 +97,7 @@ class FakeWorklet { read(output: Float32Array[]): number { const ring = this.ring; if (!ring) return 0; - const read = ring.read(output); + const read = this.#playing ? ring.read(output) : 0; const state: State = { type: "state", timestamp: ring.timestamp, @@ -217,6 +223,7 @@ afterEach(() => { // Hands frames to whichever container consumer is reading, and ends a consumer's reads on close. An // Error stands in for a malformed read. function feed() { + let currentConsumer: Container.Consumer | undefined; const closed = new WeakSet(); const queue: (Read | Error)[] = []; let waiter: @@ -224,6 +231,7 @@ function feed() { | undefined; spyOn(Container.Consumer.prototype, "next").mockImplementation(function (this: Container.Consumer) { + currentConsumer = this; if (closed.has(this)) return Promise.resolve(undefined); const read = queue.shift(); if (read instanceof Error) return Promise.reject(read); @@ -243,19 +251,27 @@ function feed() { close.call(this); }); - return (read: Read | Error) => { + const push = (read: Read | Error) => { const current = waiter; waiter = undefined; if (!current) queue.push(read); else if (read instanceof Error) current.reject(read); else current.resolve(read); }; + return { + push, + // Network arrivals can advance while next() is held behind decoder backpressure. + buffered: (ranges: Container.BufferedRanges) => { + if (!currentConsumer) throw new Error("consumer has not started"); + (currentConsumer.buffered as Signal).set(ranges); + }, + }; } async function play(initial: Delay, buffered = false, cmaf = false) { // These cases test timeline handover after the optional decoder has loaded. if (cmaf) await import("@moq/hang/cmaf"); - const push = feed(); + const { push, buffered: queued } = feed(); const truncate = spyOn(AudioRingBuffer.prototype, "truncate"); const reset = spyOn(AudioRingBuffer.prototype, "reset"); const insert = spyOn(AudioRingBuffer.prototype, "write"); @@ -311,7 +327,9 @@ async function play(initial: Delay, buffered = false, cmaf = false) { return { delay, + sync, push, + queued, enabled, decoding, context: decoder.out.context, @@ -1066,3 +1084,243 @@ describe("Decoder across a rename", () => { } }); }); + +it("keeps audio's opening buffered while an enabled video track fills", async () => { + const clock = new Manual(); + const installed = clock.install(); + const playback = await play(Milli(100), true); + const video = new Signal<{ start: Milli; end: Milli } | null>(null); + const remove = playback.sync.register(new Signal(Milli(100)), video); + try { + await playback.play(8); + await clock.advance(Nano.zero); + const ring = playback.rings()[0]; + expect(ring).toBeDefined(); + const opening = ring.ring?.timestamp; + for (let i = 0; i < 5; i++) expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(0); + await clock.advance(Nano.fromMilli(Milli(500))); + expect(ring.ring?.timestamp).toBe(opening); + video.set({ start: Milli(20), end: Milli(120) }); + await clock.advance(Nano.zero); + expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(960); + expect(ring.ring?.timestamp).toBe(Micro((opening ?? 0) + 20_000)); + } finally { + remove(); + playback.close(); + await microtasks(); + installed.close(); + } +}); + +it("starts above the decoded audio floor without expanding PCM storage", async () => { + const clock = new Manual(); + const installed = clock.install(); + const playback = await play(Milli(1000), true); + const video = new Signal<{ start: Milli; end: Milli } | null>(null); + const remove = playback.sync.register(new Signal(Milli(1000)), video); + try { + await playback.play(80); + await clock.advance(Nano.zero); + const ring = playback.rings()[0]; + expect(ring).toBeDefined(); + expect(ring.ring?.length).toBeLessThanOrEqual(24_000); + expect(playback.sync.playing.peek()).toBe(false); + const opening = ring.ring?.timestamp; + for (let i = 0; i < 5; i++) expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(0); + await clock.advance(Nano.fromMilli(Milli(500))); + expect(ring.ring?.timestamp).toBe(opening); + video.set({ start: Milli(20), end: Milli(1020) }); + await clock.advance(Nano.zero); + expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(960); + expect(ring.ring?.timestamp).toBe(Micro((opening ?? 0) + 20_000)); + } finally { + remove(); + playback.close(); + await microtasks(); + installed.close(); + } +}); + +it("waits for video support before allowing audio-only startup", async () => { + const playback = await play(Milli(100), true); + let resolve!: (supported: boolean) => void; + const source = new VideoSource({ + broadcast: { + out: { + catalog: new Signal({ + video: { + renditions: { + video: { codec: "vp8", codedWidth: 16, codedHeight: 16, container: { kind: "legacy" } }, + }, + }, + }), + }, + } as unknown as Broadcast, + supported: () => + new Promise((done) => { + resolve = done; + }), + }); + const video = new VideoDecoder({ source, sync: playback.sync }); + try { + await microtasks(); + await playback.play(8); + const ring = playback.rings()[0]; + expect(ring).toBeDefined(); + expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(0); + resolve(false); + await microtasks(); + expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(960); + } finally { + video.close(); + source.close(); + playback.close(); + await microtasks(); + } +}); + +it("does not count flushed audio toward a new startup floor after a gap", async () => { + const playback = await play(Milli(100), true); + try { + await playback.play(7); + expect(playback.sync.playing.peek()).toBe(true); + const openingRing = playback.rings().at(-1); + if (!openingRing) throw new Error("missing audio ring"); + openingRing.read([new Float32Array(960), new Float32Array(960)]); + await microtasks(); + playback.push({ kind: "gap", timestamp: Micro(160_000), duration: Micro(840_000) }); + await microtasks(); + frameTimestamp = 1_000_000; + await playback.play(1); + expect(playback.sync.playing.peek()).toBe(false); + const ring = playback.rings().at(-1); + if (!ring) throw new Error("missing audio ring"); + expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(0); + await playback.play(5); + expect(playback.sync.playing.peek()).toBe(true); + } finally { + playback.close(); + await microtasks(); + } +}); + +it.each([100, 1000])( + "closing during shared startup at %dms releases the audio graph and backpressured reader", + async (delay) => { + const playback = await play(Milli(delay), true); + const video = new Signal<{ start: Milli; end: Milli } | null>(null); + const remove = playback.sync.register(new Signal(Milli(delay)), video); + const disconnect = spyOn(FakeWorklet.prototype, "disconnect"); + const closeTrack = spyOn(Moq.Track.Subscriber.prototype, "close"); + try { + await playback.play(delay / 20 + 30); + expect(playback.sync.playing.peek()).toBe(false); + const context = playback.context.peek() as unknown as FakeContext; + expect(context).toBeDefined(); + const decoded = playback.inserted().length; + playback.close(); + await microtasks(); + expect(context.calls.close).toBe(1); + expect(disconnect).toHaveBeenCalled(); + expect(closeTrack).toHaveBeenCalled(); + // The other track becoming ready cannot revive the cancelled audio reader. + video.set({ start: Milli(0), end: Milli(delay) }); + await microtasks(); + expect(playback.inserted()).toHaveLength(decoded); + } finally { + remove(); + playback.close(); + await microtasks(); + disconnect.mockRestore(); + closeTrack.mockRestore(); + } + }, +); + +it("a rejected audio support probe does not leave buffered video waiting on a pending track", async () => { + const sync = new Sync({ delay: Milli(100), buffer: Milli(1000) }); + const remove = sync.register(new Signal(Milli(100)), new Signal({ start: Milli(0), end: Milli(100) })); + let reject!: (error: Error) => void; + const failure = new Error("audio support probe failed"); + const logged = spyOn(console, "error").mockImplementation(() => {}); + const source = new Source({ + broadcast: { + in: { enabled: new Signal(true) }, + out: { + catalog: new Signal({ + audio: { + renditions: { + audio: Catalog.AudioConfigSchema.parse({ + codec: "opus", + container: { kind: "legacy" }, + sampleRate: 48000, + numberOfChannels: 2, + }), + }, + }, + }), + }, + } as unknown as Broadcast, + supported: () => + new Promise((_resolve, fail) => { + reject = fail; + }), + }); + const decoder = new Decoder({ source, sync }); + try { + await microtasks(); + sync.received(Milli(0)); + await microtasks(); + expect(sync.playing.peek()).toBe(false); + reject(failure); + await microtasks(); + expect(logged).toHaveBeenCalledWith("spawn error", failure); + expect(source.pending.peek()).toBe(false); + expect(source.out.config.peek()).toBeUndefined(); + expect(sync.playing.peek()).toBe(true); + } finally { + decoder.close(); + source.close(); + remove(); + sync.close(); + logged.mockRestore(); + } +}); + +it.each([false, true])("releases shared startup across a hole in queued audio (CMAF=%s)", async (cmaf) => { + const clock = new Manual(); + const installed = clock.install(); + const playback = await play(Milli(1500), true, cmaf); + const video = new Signal<{ start: Milli; end: Milli } | null>({ start: Milli(20), end: Milli(1520) }); + const remove = playback.sync.register(new Signal(Milli(1500)), video); + try { + await playback.play(25); + // The audio after the hole counts, the hole itself does not. + playback.queued([ + { start: Milli(520), end: Milli(620) }, + { start: Milli(1500), end: Milli(1600) }, + ]); + const next = frame(); + next.group = 1; + playback.push(next); + await microtasks(); + await clock.advance(Nano.zero); + const ring = playback.rings()[0]; + expect(ring.ring?.length).toBe(24000); + expect(playback.sync.playing.peek()).toBe(false); + expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(0); + // Enough arrives past the hole while it stays open. + playback.queued([ + { start: Milli(520), end: Milli(620) }, + { start: Milli(1500), end: Milli(2500) }, + ]); + await clock.advance(Nano.zero); + expect(playback.sync.playing.peek()).toBe(true); + expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(960); + } finally { + remove(); + playback.close(); + await microtasks(); + installed.close(); + } +}); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index dc0cd40bd08..98c5dceb039 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -21,7 +21,7 @@ import type { Sync } from "../sync"; import { AudioBuffer } from "./buffer"; import { type DecoderConfig, frameDuration, type PlaybackIdentity, packetDuration, playbackIdentity } from "./config"; import { Handover } from "./handover"; -import { AUTO_MAX_DELAY, target } from "./latency"; +import { AUTO_MAX_DELAY, ringHold, target } from "./latency"; import { Order } from "./order"; // A blob: URL, or a hosted file when assets() is set; see vite-plugin-worklet. import RenderWorklet from "./render-worklet.ts?worklet"; @@ -30,6 +30,11 @@ import { Terminal } from "./terminal"; type FrameRead = Extract; +type Queued = { + newest: Micro | undefined; + buffered: Container.BufferedRanges; +}; + /** Signals the decoder reads. */ export type DecoderInput = { // Whether to download the audio track. Defaults to true. @@ -111,6 +116,7 @@ export class Decoder { // Decode buffer: audio sent to worklet but not yet played #decodeBuffered = new Signal([]); + #queued = new Signal(undefined); // Audio ring bridging the main thread and the worklet. #ring = new Signal(undefined); @@ -173,7 +179,22 @@ export class Decoder { delay: delay !== undefined ? Milli(delay) : undefined, }); }); - this.#signals.cleanup(this.sync.register(playout)); + const startup = this.#signals.computed((effect) => { + if (!effect.get(this.in.enabled)) return undefined; + if (effect.get(this.source.pending)) return null; + if (!effect.get(this.source.out.config)) return undefined; + const span = effect.get(this.#decodeBuffered)[0]; + if (!span) return null; + // Deep targets stay encoded, so count the audio queued past the decoded span. Sum it across + // holes: a lost packet must not hold startup, and the missing audio is not counted as buffered. + let queued = 0; + for (const range of effect.get(this.#queued) ?? []) { + queued += Math.max(0, range.end - Math.max(range.start, span.end)); + } + const hold = Milli.fromMicro(ringHold(effect.get(this.sync.out.delay))); + return { start: span.start, end: Milli(span.end + Math.min(queued, hold)) }; + }); + this.#signals.cleanup(this.sync.register(playout, startup)); this.#identity = this.#signals.computed((effect) => { const config = effect.get(this.source.out.config); return config ? playbackIdentity(config) : undefined; @@ -270,6 +291,7 @@ export class Decoder { rate: sampleRate, delay: this.sync.out.delay.peek(), buffered: this.sync.out.buffered.peek(), + playing: this.sync.playing, }); effect.cleanup(() => ring.close()); effect.set(this.#ring, ring); @@ -399,9 +421,8 @@ export class Decoder { // Every handle to one instance shares `closed`, so it identifies the instance where the // handle itself does not. if (this.#instance !== undefined && this.#instance !== active.closed) { - this.#ring.peek()?.reset(); - this.#decodeBuffered.set([]); this.sync.reset(); + this.reset(); } this.#instance = active.closed; @@ -480,7 +501,7 @@ export class Decoder { const order = this.#order(); const play = (reads: FrameRead[]) => { - read.set(order.newest(reads) as Micro | undefined); + read.set({ newest: order.newest(reads) as Micro | undefined, buffered: queuedRanges(reads, config) }); return this.#play(effect, decoder, decoderConfig, order, reads); }; for (;;) { @@ -569,7 +590,7 @@ export class Decoder { const order = this.#order(); const play = (reads: FrameRead[]) => { - read.set(order.newest(reads) as Micro | undefined); + read.set({ newest: order.newest(reads) as Micro | undefined, buffered: queuedRanges(reads, config) }); return this.#play(effect, decoder, decoderConfig, order, reads); }; for (;;) { @@ -670,7 +691,7 @@ export class Decoder { sub: Moq.Track.Subscriber, format: Container.Format, { config, path }: { config: DecoderConfig; path: Moq.Path.Valid }, - ): { consumer: Container.Consumer; read: Signal } { + ): { consumer: Container.Consumer; read: Signal } { const outgoing = this.#consumer; const continues = outgoing?.path === path; const consumer = new Container.Consumer(sub, { @@ -697,15 +718,18 @@ export class Decoder { // the decode loop's held frame is released once enough is queued behind it. That includes the // frames the loop has read but not decoded yet: a buffered head waits in the reorder buffer // until it spans the whole target, leaving the consumer empty. - const read = new Signal(undefined); + const read = new Signal({ newest: undefined, buffered: [] }); effect.run((inner) => { const ring = inner.get(this.#ring); if (!ring) return; - let newest = inner.get(read); - for (const { end } of inner.get(consumer.buffered)) { + const pending = inner.get(read); + const network = inner.get(consumer.buffered); + let newest = pending.newest; + for (const { end } of network) { const micro = Micro.fromMilli(end); if (newest === undefined || micro > newest) newest = micro; } + inner.set(this.#queued, Container.mergeBufferedRanges(network, pending.buffered)); ring.queued(newest); }); @@ -857,6 +881,8 @@ export class Decoder { * Use in buffered mode at an utterance boundary (see Sync.reset). */ reset(): void { + this.#decodeBuffered.set([]); + this.#queued.set(undefined); this.#ring.peek()?.reset(); } @@ -864,8 +890,8 @@ export class Decoder { // a gap, or a later group after a declared end. #onRead(read: Container.Read): boolean { if (!this.#terminal.update(read)) return false; - this.#ring.peek()?.reset(); this.sync.reset(); + this.reset(); return true; } @@ -944,3 +970,18 @@ function spawnFatal(effect: Effect, failure: Signal, task: () } }); } + +// Released reads are timestamp ordered and remain owned by the decode loop while it waits. Count +// their actual duration, not the distance to the next packet, which could include missing audio. +function queuedRanges(reads: FrameRead[], config: DecoderConfig): Container.BufferedRanges { + const ranges: Container.BufferedRanges = []; + for (const { frame } of reads) { + const start = Milli.fromMicro(frame.timestamp); + const duration = packetDuration(config.codec, frame) ?? frameDuration(config) ?? Milli.zero; + const end = Milli(start + duration); + const last = ranges.at(-1); + if (last && start <= last.end + 1) last.end = Milli(Math.max(last.end, end)); + else ranges.push({ start, end }); + } + return ranges; +} diff --git a/js/watch/src/audio/render-worklet.test.ts b/js/watch/src/audio/render-worklet.test.ts index 48eef2ecb3f..b6afeadb67e 100644 --- a/js/watch/src/audio/render-worklet.test.ts +++ b/js/watch/src/audio/render-worklet.test.ts @@ -114,3 +114,19 @@ describe("render dispatch", () => { expect([...output.slice(80)]).toEqual(Array(20).fill(2)); }); }); + +it("holds PCM without consuming it until the shared startup gate opens", () => { + const { send, read, sent } = renderer(); + send({ type: "playing", playing: false, epoch: 0 }); + send({ type: "data", timestamp: Micro(500_000), data: [new Float32Array(200).fill(1)] }); + for (let i = 0; i < 5; i++) expect([...read(20)]).toEqual(Array(20).fill(0)); + expect(sent[0]).toMatchObject({ timestamp: Micro(500_000), underruns: 0 }); + send({ type: "playing", playing: true, epoch: 0 }); + expect(read(100)[99]).toBe(1); + send({ type: "reset", epoch: 1, playing: false }); + send({ type: "data", timestamp: Micro(0), data: [new Float32Array(200).fill(2)] }); + send({ type: "playing", playing: true, epoch: 0 }); + expect([...read(20)]).toEqual(Array(20).fill(0)); + send({ type: "playing", playing: true, epoch: 1 }); + expect(read(100)[99]).toBe(2); +}); diff --git a/js/watch/src/audio/render-worklet.ts b/js/watch/src/audio/render-worklet.ts index 59475f162b8..6cafe639a57 100644 --- a/js/watch/src/audio/render-worklet.ts +++ b/js/watch/src/audio/render-worklet.ts @@ -11,6 +11,7 @@ class Render extends AudioWorkletProcessor { #underflow = 0; #stateCounter = 0; #epoch = 0; + #playing = true; // Whether the previous quantum ended short, so the next one fades back in. #short = false; @@ -21,6 +22,7 @@ class Render extends AudioWorkletProcessor { const msg = event.data; if (msg.type === "init") { this.#ring = new AudioRingBuffer(msg); + this.#playing = msg.playing ?? true; this.#underflow = 0; this.#short = false; } else if (msg.type === "data") { @@ -36,6 +38,9 @@ class Render extends AudioWorkletProcessor { } else if (msg.type === "reset") { this.#ring?.reset(); this.#epoch = msg.epoch; + this.#playing = msg.playing ?? true; + } else if (msg.type === "playing" && msg.epoch === this.#epoch) { + this.#playing = msg.playing; } }; } @@ -43,7 +48,7 @@ class Render extends AudioWorkletProcessor { process(_inputs: Float32Array[][], outputs: Float32Array[][], _parameters: Record) { const output = outputs[0]; const ring = this.#ring; - const samplesRead = ring?.read(output) ?? 0; + const samplesRead = this.#playing ? (ring?.read(output) ?? 0) : 0; if (samplesRead < output[0].length) { // Fade the tail of what we did read down to the silence that follows it. @@ -53,7 +58,7 @@ class Render extends AudioWorkletProcessor { channel[samplesRead - ramp + i] *= 1 - (i + 1) / ramp; } } - this.#underflow += output[0].length - samplesRead; + if (this.#playing) this.#underflow += output[0].length - samplesRead; this.#short = true; } else { if (this.#short) { diff --git a/js/watch/src/audio/render.ts b/js/watch/src/audio/render.ts index 77ecb70fa22..13689ab7f76 100644 --- a/js/watch/src/audio/render.ts +++ b/js/watch/src/audio/render.ts @@ -1,7 +1,7 @@ import type { Micro, Milli } from "@moq/clock"; /** Everything the main thread sends the render worklet over its port. */ -export type Message = Init | Data | Latency | Reset | Truncate; +export type Message = Init | Data | Latency | Reset | Truncate | Playing; /** Create the worklet's ring. */ export interface Init { @@ -12,6 +12,7 @@ export interface Init { // Buffered mode: anchor to the first frame and play through. The ring holds the floor; the // lookahead above it is held back upstream (the main thread applies the backpressure). buffered: boolean; + playing?: boolean; } /** Flush the buffer and re-stall. */ @@ -19,6 +20,7 @@ export interface Reset { type: "reset"; // Echoed in every later state message, so the main thread can tell state from before the reset. epoch: number; + playing?: boolean; } /** Drop buffered samples at or after `timestamp`, keeping what is already due. */ @@ -55,3 +57,10 @@ export interface State { // The epoch of the last reset or park the worklet applied. epoch: number; } + +/** Release or hold consumption without discarding the queued opening. */ +export interface Playing { + type: "playing"; + playing: boolean; + epoch: number; +} diff --git a/js/watch/src/audio/source.ts b/js/watch/src/audio/source.ts index b54a18d605c..fb2d0027b9f 100644 --- a/js/watch/src/audio/source.ts +++ b/js/watch/src/audio/source.ts @@ -55,6 +55,9 @@ export class Source { }; /** The derived outputs, read-only. */ readonly out = readonlys(this.#out); + #pending = new Signal(false); + /** Whether rendition support is still being resolved. @internal */ + readonly pending: Getter = this.#pending; #signals = new Effect(); @@ -85,25 +88,30 @@ export class Source { const supported = effect.get(this.in.supported); if (!supported) return; + effect.set(this.#pending, true, false); effect.spawn(async () => { - const available: Record = {}; + try { + const available: Record = {}; - for (const [name, config] of Object.entries(renditions)) { - // `supported` comes from the consumer, so we cannot assume it ever settles. A rerun - // waits for the tasks it spawned, so an unraced probe would hold the next run shut. - const isSupported = await effect.race(supported(config)); + for (const [name, config] of Object.entries(renditions)) { + // `supported` comes from the consumer, so we cannot assume it ever settles. A rerun + // waits for the tasks it spawned, so an unraced probe would hold the next run shut. + const isSupported = await effect.race(supported(config)); - // Torn down: stop probing and publish nothing, since the rerun redoes this. - if (effect.abort.aborted) return; + // Torn down: stop probing and publish nothing, since the rerun redoes this. + if (effect.abort.aborted) return; - if (isSupported) available[name] = config; - } + if (isSupported) available[name] = config; + } - if (Object.keys(available).length === 0 && Object.keys(renditions).length > 0) { - console.warn("no supported audio renditions found:", renditions); - } + if (Object.keys(available).length === 0 && Object.keys(renditions).length > 0) { + console.warn("no supported audio renditions found:", renditions); + } - this.#out.available.set(available); + this.#out.available.set(available); + } finally { + this.#pending.set(false); + } }); } diff --git a/js/watch/src/sync.test.ts b/js/watch/src/sync.test.ts index dd50caff3b4..84a00faa505 100644 --- a/js/watch/src/sync.test.ts +++ b/js/watch/src/sync.test.ts @@ -326,3 +326,100 @@ describe("skip floor", () => { sync.close(); }); }); + +describe("buffered startup", () => { + for (const first of ["audio", "video"] as const) { + it(`waits for both spans when ${first} fills first`, async () => { + const clock = new Manual(); + const installed = clock.install(); + const sync = new Sync({ delay: Milli(100), buffer: Milli(1000) }); + const spans = { + audio: new Signal<{ start: Milli; end: Milli } | null>(null), + video: new Signal<{ start: Milli; end: Milli } | null>(null), + }; + const dispose = Object.values(spans).map((span) => sync.register(new Signal(Milli(100)), span)); + try { + await clock.advance(Nano.zero); + sync.received(Milli(500)); + spans[first].set({ start: Milli(500), end: Milli(600) }); + await clock.advance(Nano.zero); + let delivered = false; + const waiting = sync.wait(Milli(500)).then((ok) => { + delivered = ok; + }); + await clock.advance(Nano.fromMilli(Milli(1000))); + expect(delivered).toBe(false); + expect(sync.now()).toBeUndefined(); + spans[first === "audio" ? "video" : "audio"].set({ start: Milli(500), end: Milli(599) }); + await clock.advance(Nano.zero); + expect(delivered).toBe(false); + spans[first === "audio" ? "video" : "audio"].set({ start: Milli(500), end: Milli(600) }); + await clock.advance(Nano.zero); + await waiting; + expect(delivered).toBe(true); + expect(sync.now()).toBe(Milli(500)); + } finally { + for (const fn of dispose) fn(); + sync.close(); + installed.close(); + } + }); + } + it("releases startup waits on abort, reset, and close without starting playback", async () => { + const clock = new Manual(); + const installed = clock.install(); + const sync = new Sync({ delay: Milli(100), buffer: Milli(1000) }); + const dispose = sync.register(new Signal(Milli(100)), new Signal(null)); + try { + await clock.advance(Nano.zero); + sync.received(Milli(0)); + await clock.advance(Nano.zero); + const abort = new AbortController(); + const cancelled = sync.wait(Milli(0), abort.signal); + abort.abort(); + expect(await cancelled).toBe(false); + const reset = sync.wait(Milli(0)); + sync.reset(); + sync.received(Milli(0)); + expect(await reset).toBe(false); + const closed = sync.wait(Milli(0)); + sync.close(); + expect(await closed).toBe(false); + } finally { + dispose(); + sync.close(); + installed.close(); + } + }); + it("removes a disabled track from the floor and leaves live pacing unchanged", async () => { + const clock = new Manual(); + const installed = clock.install(); + const buffer = new Signal(Milli(1000)); + const sync = new Sync({ delay: Milli(100), buffer }); + const absent = new Signal<{ start: Milli; end: Milli } | null | undefined>(null); + const remove = sync.register(new Signal(Milli(100)), absent); + const video = sync.register(new Signal(Milli(100)), new Signal({ start: Milli(0), end: Milli(100) })); + try { + await clock.advance(Nano.zero); + sync.received(Milli(0)); + await clock.advance(Nano.zero); + expect(sync.playing.peek()).toBe(false); + absent.set(undefined); + await clock.advance(Nano.zero); + expect(await sync.wait(Milli(0))).toBe(true); + buffer.set(Milli.zero); + sync.reset(); + absent.set(null); + await clock.advance(Nano.zero); + sync.received(Milli(0)); + const waiting = sync.wait(Milli(0)); + await clock.advance(Nano.fromMilli(Milli(100))); + expect(await waiting).toBe(true); + } finally { + remove(); + video(); + sync.close(); + installed.close(); + } + }); +}); diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index ea6218a0763..f0aafdbf54d 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -35,7 +35,9 @@ export type SyncInput = { * Zero minimizes latency, the live default: playback re-anchors as soon as anything arrives * early. A larger value lets faster-than-real-time frames (a TTS response with future * timestamps) build up instead of being skipped. Always finite, so worst case the audio ring - * drops its oldest samples rather than exhausting memory. + * drops its oldest samples rather than exhausting memory. With a nonzero buffer, the first video + * frame is a poster until every enabled media track holds the resolved delay; `buffer` is + * lookahead, not the amount required to start. */ buffer: Getter; }; @@ -83,7 +85,17 @@ export class Sync { /** The derived outputs, read-only. */ readonly out = readonlys(this.#out); - #media = new Signal<{ target: Getter }[]>([]); + #media = new Signal< + { + target: Getter; + // undefined is inactive, null is selected but has no playable first span yet. + startup?: Getter<{ start: Milli; end: Milli } | null | undefined>; + }[] + >([]); + #started = false; + #playing = new Signal(true); + /** Whether buffered startup has released the media consumers. @internal */ + readonly playing: Getter = this.#playing; // Bumped by every `reset()`. A frame that waited across a bump belongs to the previous timeline, // even when the next frame restores the same reference value. @@ -106,18 +118,54 @@ export class Sync { this.#signals.run(this.#runInstant.bind(this)); this.#signals.run(this.#runDelay.bind(this)); this.#signals.run(this.#runMaxDelay.bind(this)); + this.#signals.run(this.#runStartup.bind(this)); } /** * Hold a decoder's "auto" playout target in the shared playback clock until disposed. * @internal */ - register(target: Getter): Dispose { - const registered = { target }; + register( + target: Getter, + startup?: Getter<{ start: Milli; end: Milli } | null | undefined>, + ): Dispose { + const registered = { target, startup }; this.#media.update((media) => [...media, registered]); return () => this.#media.update((media) => media.filter((candidate) => candidate !== registered)); } + // Startup is data-driven: time passing cannot fill a missing track. The first picture still + // travels through the ordinary renderer while the consumers wait for this shared gate. + #runStartup(effect: Effect): void { + effect.get(this.#epoch); + const buffered = effect.get(this.#out.buffered); + const delay = effect.get(this.#out.delay); + const reference = effect.get(this.#out.reference); + const spans = effect.get(this.#media).flatMap(({ startup }) => { + const span = startup && effect.get(startup); + return span === undefined ? [] : [span]; + }); + if (!buffered || spans.length === 0) { + this.#playing.set(true); + return; + } + if (this.#started) return; + const ready = spans.filter((span) => span !== null); + if ( + reference === undefined || + ready.length !== spans.length || + ready.some((span) => span.end - span.start < delay) + ) { + this.#playing.set(false); + return; + } + const first = Math.min(...ready.map((span) => span.start)); + this.#started = true; + // Anchor at the retained opening, not at its arrival: startup may have waited on another track. + this.#out.reference.set(Nano.toMilli(Nano(now())) - first - delay); + this.#playing.set(true); + } + #runInstant(effect: Effect): void { this.#out.instant.set(effect.get(this.in.delay) === "instant"); } @@ -166,6 +214,10 @@ export class Sync { return false; } + // Retain the opening while another track fills; the ordinary max-delay bounds still own + // storage, but a later arrival must not advance the shared playhead before startup. + if (!this.#playing.peek()) return false; + // Check if `wait()` would not sleep at all. // NOTE: We check here instead of in `wait()` so we can identify when frames are received late. // Otherwise, chained `wait()` calls would cause a false-positive during CPU starvation. @@ -203,6 +255,8 @@ export class Sync { reset(): void { this.#epoch.update((epoch) => epoch + 1); this.#floor = undefined; + this.#started = false; + this.#playing.set(!this.#out.buffered.peek()); this.#out.reference.set(undefined); } @@ -222,9 +276,10 @@ export class Sync { /** * The PTS that should be rendering right now, derived from the reference + delay; negative - * before the first frame's turn. Undefined if no frames have been received yet. + * before the first frame's turn. Undefined before ingress or while buffered startup is filling. */ now(): Milli | undefined { + if (!this.#playing.peek()) return undefined; const reference = this.#out.reference.peek(); if (reference === undefined) return undefined; return Milli(Nano.toMilli(Nano(now())) - reference - this.#out.delay.peek()); @@ -253,6 +308,11 @@ export class Sync { // Peek the input, not `out.instant`, which lags a same-turn change until effects flush. if (this.in.delay.peek() === "instant") return true; + if (!this.#playing.peek()) { + await this.#sleep(undefined, signal); + continue; + } + // NOTE: This function runs in parallel for each frame. const sleep = reference + timestamp - Nano.toMilli(Nano(now())) + this.#out.delay.peek(); @@ -266,7 +326,7 @@ export class Sync { // Sleeps for `ms`, or wakes early once anything the sleep was computed from changes or `signal` // aborts. Releases every listener either way: frames sleep once each, so a listener left on a // signal that never changes would pile up for the life of the player. - #sleep(ms: number, signal?: AbortSignal): Promise { + #sleep(ms: number | undefined, signal?: AbortSignal): Promise { return new Promise((resolve) => { const timer = new AbortController(); const wake = () => { @@ -274,13 +334,14 @@ export class Sync { for (const dispose of disposes) dispose(); resolve(); }; - void sleep(Nano.fromMilli(Milli(ms)), { signal: timer.signal }).then(wake, () => {}); + if (ms !== undefined) void sleep(Nano.fromMilli(Milli(ms)), { signal: timer.signal }).then(wake, () => {}); const disposes = [ this.in.delay.changed(wake), this.#out.delay.changed(wake), this.#out.reference.changed(wake), this.#epoch.changed(wake), this.#closed.changed(wake), + this.#playing.changed(wake), ]; if (signal) { signal.addEventListener("abort", wake, { once: true }); diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index 116168613f4..e9927a0d5f7 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -200,13 +200,20 @@ const gap: Read = { kind: "gap", timestamp: Micro(0), duration: Micro(0) }; // `reads` stands in for the container consumer, unless `live`, which reads what tests publish into // `producer` through real consumers instead. -async function guardedPlayback(kind: "legacy" | "cmaf", reads: Read[], hold: readonly number[] = [], live = false) { +async function guardedPlayback( + kind: "legacy" | "cmaf", + reads: Read[], + hold: readonly number[] = [], + live = false, + buffer = 0, +) { // Load the optional container before the fixture drains only reactive microtasks. if (kind === "cmaf") await import("@moq/hang/cmaf"); const originalDecoder = Object.getOwnPropertyDescriptor(globalThis, "VideoDecoder"); const originalChunk = Object.getOwnPropertyDescriptor(globalThis, "EncodedVideoChunk"); const submitted: number[] = []; const outputs: number[] = []; + const pictures: Picture[] = []; const owed = new Set(hold); const pending: Array<() => void> = []; let resets = 0; @@ -245,7 +252,9 @@ async function guardedPlayback(kind: "legacy" | "cmaf", reads: Read[], hold: rea submitted.push(chunk.timestamp); const emit = () => { outputs.push(chunk.timestamp); - void this.callbacks.output(new Picture(chunk.timestamp) as unknown as VideoFrame); + const picture = new Picture(chunk.timestamp); + pictures.push(picture); + void this.callbacks.output(picture as unknown as VideoFrame); }; if (owed.delete(chunk.timestamp)) { pending.push(emit); @@ -301,8 +310,8 @@ async function guardedPlayback(kind: "legacy" | "cmaf", reads: Read[], hold: rea mediaProducer.close(); }; const source = new Source({ broadcast, supported: async () => true }); - const delay = new Signal("instant"); - const sync = new Sync({ delay }); + const delay = new Signal(buffer ? Milli(100) : "instant"); + const sync = new Sync({ delay, buffer: Milli(buffer) }); const enabled = new Signal(true); // Published before the decoder subscribes, since a broadcast with no request handler refuses // an unknown track. @@ -330,6 +339,7 @@ async function guardedPlayback(kind: "legacy" | "cmaf", reads: Read[], hold: rea track, submitted, outputs, + pictures, sync, get resets() { return resets; @@ -1001,3 +1011,177 @@ it("a video track read from another path does not inherit the outgoing delay", a clock.installed.close(); } }); + +// The first picture is useful before the startup floor, but elapsed wall time is not buffered media. +it("buffered video holds its poster until its own span reaches the floor", async () => { + const clock = manualAt(0); + const playback = await guardedPlayback("legacy", [], [], true, 1000); + const group = new Moq.Group.Producer(0); + if (!playback.track) throw new Error("missing video track"); + playback.track.writeGroup(group); + try { + group.writeFrame(legacyFrame(Micro(0))); + group.writeFrame(legacyFrame(Micro(40_000))); + await microtasks(); + expect(playback.decoder.out.frame.peek()?.timestamp).toBe(0); + await clock.advance(500); + expect(playback.decoder.out.frame.peek()?.timestamp).toBe(0); + group.writeFrame(legacyFrame(Micro(100_000))); + await microtasks(); + await clock.advance(40); + expect(playback.decoder.out.frame.peek()?.timestamp).toBe(40_000); + } finally { + group.close(); + playback.close(); + await microtasks(); + clock.installed.close(); + } +}); + +it("bounds startup decoding while another track is missing and releases it on close", async () => { + const clock = manualAt(0); + const playback = await guardedPlayback("legacy", [], [], true, 1000); + const remove = playback.sync.register(new Signal(Milli(100)), new Signal(null)); + if (!playback.track) throw new Error("missing video track"); + const group = playback.track.appendGroup(); + try { + for (let i = 0; i < 20; i++) group.writeFrame(legacyFrame(Micro(i * 40_000))); + await microtasks(); + await clock.advance(1000); + expect(playback.submitted).toEqual([0, 40_000, 80_000, 120_000]); + expect(playback.decoder.out.frame.peek()?.timestamp).toBe(0); + playback.close(); + await microtasks(); + expect(playback.pictures.every((frame) => frame.closed)).toBe(true); + } finally { + remove(); + group.close(); + playback.close(); + await microtasks(); + clock.installed.close(); + } +}); + +// Resuming only once the held floor has played out would leave no decoded lookahead. +it("resumes decoding the moment shared startup releases", async () => { + const clock = manualAt(0); + const playback = await guardedPlayback("legacy", [], [], true, 1000); + const audio = new Signal<{ start: Milli; end: Milli } | null>(null); + const remove = playback.sync.register(new Signal(Milli(100)), audio); + if (!playback.track) throw new Error("missing video track"); + const group = playback.track.appendGroup(); + try { + for (let i = 0; i < 8; i++) group.writeFrame(legacyFrame(Micro(i * 40_000))); + await microtasks(); + await clock.advance(0); + expect(playback.submitted).toEqual([0, 40_000, 80_000, 120_000]); + audio.set({ start: Milli(0), end: Milli(100) }); + await microtasks(); + await clock.advance(0); + expect(playback.submitted).toHaveLength(8); + } finally { + remove(); + group.close(); + playback.close(); + await microtasks(); + clock.installed.close(); + } +}); + +it("refills the video floor after a discontinuity instead of waiting on the previous poster timestamp", async () => { + const clock = manualAt(0); + const reads: Read[] = [ + sample(0, 0), + sample(0, 100_000), + marker(0, 100_000), + sample(1, 1_000_000), + sample(1, 1_100_000), + ]; + const playback = await guardedPlayback("legacy", reads, [], false, 1000); + try { + await clock.advance(100); + await microtasks(); + await clock.advance(100); + await microtasks(); + expect(playback.decoder.out.frame.peek()?.timestamp).toBeGreaterThanOrEqual(1_000_000); + } finally { + playback.close(); + await microtasks(); + clock.installed.close(); + } +}); + +it("continues filling when the shared floor rises while video waits on audio", async () => { + const clock = manualAt(0); + const playback = await guardedPlayback("legacy", [], [], true, 1000); + const audio = new Signal<{ start: Milli; end: Milli } | null>(null); + const remove = playback.sync.register(new Signal(Milli(100)), audio); + if (!playback.track) throw new Error("missing video track"); + const group = playback.track.appendGroup(); + try { + for (let i = 0; i < 7; i++) group.writeFrame(legacyFrame(Micro(i * 40_000))); + await microtasks(); + await clock.advance(0); + expect(playback.submitted).toEqual([0, 40_000, 80_000, 120_000]); + playback.delay.set(Milli(200)); + audio.set({ start: Milli(0), end: Milli(200) }); + await microtasks(); + await clock.advance(240); + expect(playback.decoder.out.frame.peek()?.timestamp).toBeGreaterThanOrEqual(200_000); + } finally { + remove(); + group.close(); + playback.close(); + await microtasks(); + clock.installed.close(); + } +}); + +it("keeps feeding a decoder that has not output its first picture yet", async () => { + const clock = manualAt(0); + const playback = await guardedPlayback("legacy", [], [0, 40_000, 80_000, 120_000], true, 1000); + if (!playback.track) throw new Error("missing video track"); + const group = playback.track.appendGroup(); + try { + for (let i = 0; i < 5; i++) group.writeFrame(legacyFrame(Micro(i * 40_000))); + await microtasks(); + await clock.advance(160); + expect(playback.decoder.out.frame.peek()?.timestamp).toBe(160_000); + } finally { + group.close(); + playback.close(); + await microtasks(); + clock.installed.close(); + } +}); + +for (const kind of ["legacy", "cmaf"] as const) { + it(`cancels a ${kind} startup wait on disable and can enable a fresh decoder`, async () => { + const clock = manualAt(0); + const playback = await guardedPlayback(kind, [sample(0, 0), sample(0, 100_000)], [0, 100_000], false, 1000); + const other = new Signal<{ start: Milli; end: Milli } | null>(null); + const remove = playback.sync.register(new Signal(Milli(100)), other); + try { + playback.release(); + await microtasks(); + expect(playback.sync.playing.peek()).toBe(false); + expect(playback.decoder.out.frame.peek()?.timestamp).toBe(0); + playback.enabled.set(false); + await microtasks(); + expect(playback.pictures.every((frame) => frame.closed)).toBe(true); + const codecs = playback.codecs; + playback.enabled.set(true); + await microtasks(); + expect(playback.codecs).toBe(codecs + 1); + // A late output from the cancelled codec must not replace the retained poster. + playback.emit(900_000, 0); + await microtasks(); + expect(playback.decoder.out.frame.peek()?.timestamp).toBe(0); + } finally { + remove(); + playback.close(); + await microtasks(); + clock.installed.close(); + } + }); +} diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index b3e3e609e0f..76b186bd2e3 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -9,6 +9,7 @@ import { type Getter, getter, type Inputs, + Once, type Readonlys, readonlys, Signal, @@ -146,7 +147,19 @@ export class Decoder { const delay = active && effect.get(active.catalog)?.video?.renditions?.[active.track]?.delay; return Milli(Math.max(spread + (delay ?? 0), jitter ?? Milli.zero)); }); - this.#signals.cleanup(this.sync.register(target)); + const startup = this.#signals.computed((effect) => { + if (!effect.get(this.in.enabled)) return undefined; + if (effect.get(this.source.pending)) return null; + if (!effect.get(this.source.out.config)) return undefined; + const active = effect.get(this.#active); + if (!active) return null; + const timestamp = effect.get(active.first); + if (timestamp === undefined) return null; + const ranges = effect.get(active.buffered); + const span = ranges.find((range) => range.start <= timestamp && range.end >= timestamp); + return span ? { ...span } : { start: timestamp, end: timestamp }; + }); + this.#signals.cleanup(this.sync.register(target, startup)); this.#identity = this.#signals.computed((effect) => { const config = effect.get(this.source.out.config); return config ? playbackIdentity(config) : undefined; @@ -426,6 +439,8 @@ class DecoderTrack { // Network jitter + decode buffer. buffered = new Signal([]); + // First decoded picture on this timeline, independent of the previously held poster. + readonly first = new Signal(undefined); // How late frames arrive relative to the earliest one, from the container consumer. spread = new Signal(undefined); @@ -492,6 +507,8 @@ class DecoderTrack { return; } + if (this.first.peek() === undefined) this.first.set(timestamp); + // Render something while we wait for the sync to catch up, unless the frame is // already off the timeline: WebCodecs can hold a picture past a reset or a skip. if (this.frame.peek() === undefined && this.sync.presentable(timestamp)) { @@ -626,6 +643,7 @@ class DecoderTrack { latest = next.group; this.#resetAtKeyframe(decoder, frame.keyframe); decoder.decode(chunk); + if (!this.sync.playing.peek()) await this.#waitStartup(effect); } }); } @@ -708,15 +726,41 @@ class DecoderTrack { timestamp: frame.timestamp, }), ); + if (!this.sync.playing.peek()) await this.#waitStartup(effect); } }); } + // Once this track has filled its floor, leave further media encoded upstream while the other + // track fills. Otherwise a missing audio track could retain unbounded decoded GPU frames. + async #waitStartup(effect: Effect): Promise { + // Some codecs need more input before their first output. Do not park that input + // behind a gate that needs the very picture the decoder has not produced yet. + if (this.first.peek() === undefined) return; + const span = this.#buffered.peek()[0]; + if (!span || span.end - span.start < this.sync.out.delay.peek()) return; + const waiting = new Effect(); + const refill = new Once(); + waiting.run((inner) => { + // An adaptive floor can rise while another track fills. Resume decoding to reach it, + // rather than waiting on a clock whose startup now needs more video. + if (span.end - span.start < inner.get(this.sync.out.delay)) refill.set(true); + }); + try { + // Wait on the span's first picture, which is due the moment startup releases, so decoding + // resumes with the whole floor still ahead instead of once it has played out. + await effect.race(refill, this.sync.wait(span.start, waiting.abort)); + } finally { + waiting.close(); + } + } + // The container read a gap or a declared end after media. Re-anchor and let the next decoded // frame decide the decoder reset (see #resetAtKeyframe). Clearing `timestamp` keeps a stale // high value from late-rejecting the following frames, which may overlap the previous group's // tail. The held picture stays until the new keyframe renders. #onBreak(): void { + this.first.set(undefined); this.timestamp.set(undefined); this.#buffered.set([]); this.sync.reset(); diff --git a/js/watch/src/video/source.ts b/js/watch/src/video/source.ts index c79042917dc..3c4fa40cd75 100644 --- a/js/watch/src/video/source.ts +++ b/js/watch/src/video/source.ts @@ -214,6 +214,9 @@ export class Source { }; /** The derived outputs, read-only. */ readonly out = readonlys(this.#out); + #pending = new Signal(false); + /** Whether rendition support is still being resolved. @internal */ + readonly pending: Getter = this.#pending; #signals = new Effect(); #supportCache = new WeakMap>(); @@ -260,6 +263,7 @@ export class Source { if (!names.has(name)) cache.delete(name); } + effect.set(this.#pending, true, false); effect.spawn(async () => { const available: Record = {}; @@ -304,6 +308,7 @@ export class Source { this.#out.error.set(undefined); } this.#out.available.set(available); + this.#pending.set(false); }); } diff --git a/quest/m1/README.md b/quest/m1/README.md index c08df9ec713..c368765d6d2 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -139,7 +139,6 @@ Binding durations wait on Duration support in the generated C. - [A/V clock](/quest/m1/av-clock.md) - the audio playhead drives Sync.reference while audio plays, through per-track sync handles - [Container reads drop backfill](/quest/m1/read-backfill.md) - the JS container consumer drops groups below its read cursor like Rust - [Buffered playback subscribes in ascending order](/quest/m1/buffered-ascending.md) - a buffered `` asks for the oldest group first, so a voice reply's opening arrives first, and audio's head hold shrinks to the floor -- [Buffered video poster](/quest/m1/buffered-video-poster.md) - buffered startup paints the first video frame at once and starts A/V together at a shared floor - [JS reload config](/quest/m1/js-reload-config.md) - `@moq/net` reload takes Rust's `healthy` backoff and no ignored `extensions` - [JS release preflight](/quest/m1/js-publish-preflight.md) - `release.ts` refuses a package whose `@moq/*` dependency is not on npm, so no dependent publishes uninstallable - [TS fidelity and recovery](/quest/m1/ts-reliability/README.md) - both modes have explicit profiles, automated validation, loss tests, and a dedicated guide diff --git a/quest/m1/buffered-ascending.md b/quest/m1/buffered-ascending.md index b47f4748e1d..7edeec56866 100644 --- a/quest/m1/buffered-ascending.md +++ b/quest/m1/buffered-ascending.md @@ -11,7 +11,7 @@ sorted. Live playback (`buffer` of zero) stays newest-first. Not here: the protocol's `Group Order` (landed in [moq-dev/moq#5385](https://github.com/moq-dev/moq/pull/5385)), the buffered -video poster ([Buffered video poster](/quest/m1/buffered-video-poster.md)), and +video poster (landed in [moq-dev/moq#6609](https://github.com/moq-dev/moq/pull/6609)), and ordering for the Python or FFI audio consumers. ## Plan @@ -78,6 +78,5 @@ Implementation: ## Related -- [Buffered video poster](/quest/m1/buffered-video-poster.md) - buffered startup paints the first video frame and holds video until the shared floor - [Container reads drop backfill](/quest/m1/read-backfill.md) - the JS consumer stops delivering groups below its cursor - [Subscribe ranges](/quest/m1/subscribe-ranges/README.md) - owns the wire's `Group Order` and the ordered cursor's floor diff --git a/quest/m1/buffered-video-poster.md b/quest/m1/buffered-video-poster.md deleted file mode 100644 index 24bc8d282e0..00000000000 --- a/quest/m1/buffered-video-poster.md +++ /dev/null @@ -1,39 +0,0 @@ -# [S] Buffered video paints a poster while it fills - -## Goal - -A `` with a nonzero `buffer` renders its first decoded video frame -as soon as it decodes, as a poster, and holds video there until playback -starts. Synchronized audio and video share one floor: playback starts once -every enabled track has buffered the floor, and a video-only broadcast waits -on video's own span. Live playback is unchanged. - -## Plan - -Decided 2026-10-23 by the maintainer, while planning -[Buffered playback subscribes in ascending order](/quest/m1/buffered-ascending.md): - -- ✅ Video holds before playback starts too, but the first frame renders right - away as a poster while the rest loads. -- ✅ Synchronized A/V shares one floor. Not chosen: video starting on its own - floor; audio alone deciding the start. -- ✅ Its own quest, ready now: it has no protocol dependency. Not chosen: one - quest with the ascending work, blocked on lite-07. - -Implementation: - -- First record what buffered video does at startup today: when the first - frame paints, and whether it moves before the shared clock starts. -- Reuse the paused poster path (`renderer.out.frame`, read by `Player`) - rather than adding a second one. -- Gate the shared clock's start on every enabled track reaching the floor, - consistent with [A/V clock](/quest/m1/av-clock.md), where the audio - playhead drives `Sync.reference` once audio plays. -- Tests: the poster renders before the floor; video does not advance until - every enabled track has buffered the floor; video-only starts on its own - span; live mode is unchanged. - -## Related - -- [Buffered playback subscribes in ascending order](/quest/m1/buffered-ascending.md) - buffered subscriptions ask for the oldest group first -- [A/V clock](/quest/m1/av-clock.md) - the audio playhead drives `Sync.reference` while audio plays diff --git a/test/video/renderer.browser.ts b/test/video/renderer.browser.ts index 8e10ff5f818..652cd005dec 100644 --- a/test/video/renderer.browser.ts +++ b/test/video/renderer.browser.ts @@ -221,3 +221,88 @@ test("closing while the caption chunk loads installs no stale overlay", async () await page.close(); } }); + +for (const audio of [false, true]) { + test(`buffered ${audio ? "A/V" : "video"} paints a poster before the common floor`, async () => { + const page = await open(); + try { + await page.evaluate((audio) => window.startPoster(audio, true), audio); + if (audio) { + await page.locator("body").click(); + await page.waitForFunction(() => window.poster.state().audioState === "running"); + } + await page.evaluate(() => { + window.poster.push(0); + window.poster.push(40_000); + }); + await page.waitForFunction(() => window.poster.state().color === "red"); + // Browser paint observation spans several floor intervals, so a time-only gate fails. + await page.waitForTimeout(300); + expect(await page.evaluate(() => window.poster.state())).toMatchObject({ + timestamp: 0, + color: "red", + playing: false, + }); + await page.evaluate(() => window.poster.push(100_000)); + if (audio) { + await page.waitForTimeout(300); + expect(await page.evaluate(() => window.poster.state())).toMatchObject({ + timestamp: 0, + color: "red", + playing: false, + }); + await page.evaluate(() => window.poster.audio()); + } + await page.waitForFunction(() => window.poster.state().color === "blue"); + expect(await page.evaluate(() => window.poster.state().playing)).toBe(true); + } finally { + await page.evaluate(() => window.poster?.close()); + await page.close(); + } + }, 30_000); +} + +test("live video does not wait to fill the buffered startup floor", async () => { + const page = await open(); + try { + await page.evaluate(() => window.startPoster(false, false)); + await page.evaluate(() => { + window.poster.push(0); + window.poster.push(40_000); + }); + await page.waitForFunction(() => window.poster.state().color === "blue"); + expect(await page.evaluate(() => window.poster.state().timestamp)).toBe(40_000); + } finally { + await page.evaluate(() => window.poster?.close()); + await page.close(); + } +}, 30_000); + +test("buffered audio keeps its opening while video fills, with a running audio context", async () => { + const page = await open(); + try { + await page.evaluate(() => window.startPoster(true, true)); + await page.locator("body").click(); + await page.waitForFunction(() => window.poster.state().audioState === "running"); + await page.evaluate(() => window.poster.audio()); + const initial = await page.evaluate(() => window.poster.state().audioTimestamp); + await page.waitForTimeout(300); + expect(await page.evaluate(() => window.poster.state())).toMatchObject({ + audioTimestamp: initial, + playing: false, + }); + await page.evaluate(() => { + window.poster.push(0); + window.poster.push(40_000); + }); + await page.waitForFunction(() => window.poster.state().color === "red"); + expect(await page.evaluate(() => window.poster.state().playing)).toBe(false); + await page.evaluate(() => window.poster.push(100_000)); + await page.waitForFunction( + () => window.poster.state().color === "blue" && (window.poster.state().audioTimestamp ?? 0) > 0, + ); + } finally { + await page.evaluate(() => window.poster?.close()); + await page.close(); + } +}, 30_000); diff --git a/test/video/src/main.ts b/test/video/src/main.ts index aa3c7214dba..c88b3a22354 100644 --- a/test/video/src/main.ts +++ b/test/video/src/main.ts @@ -8,6 +8,7 @@ */ import "./gpu.ts"; import "./captions.ts"; +import "./poster.ts"; import "@moq/publish/element"; import "@moq/watch/element"; import { Signal } from "@moq/signals"; diff --git a/test/video/src/poster.ts b/test/video/src/poster.ts new file mode 100644 index 00000000000..18a50b0bd11 --- /dev/null +++ b/test/video/src/poster.ts @@ -0,0 +1,142 @@ +import { Micro, Milli } from "@moq/clock"; +import * as Catalog from "@moq/hang/catalog"; +import { Legacy } from "@moq/hang/container"; +import { Origin, Path } from "@moq/net"; +import { Player } from "@moq/watch"; + +// Real WebCodecs and rendering, with publication controlled independently of wall time. +async function poster(audio: boolean, buffered: boolean) { + const origin = new Origin.Producer(); + const name = Path.from("poster"); + const producer = origin.createBroadcast(name); + const track = producer.createTrack("video"); + const group = track.appendGroup(); + const audioTrack = audio ? producer.createTrack("audio") : undefined; + const audioGroup = audioTrack?.appendGroup(); + const catalog: Catalog.Root = { + video: { + renditions: { + video: Catalog.VideoConfigSchema.parse({ + codec: "vp8", + codedWidth: 64, + codedHeight: 32, + container: { kind: "legacy" }, + }), + }, + }, + ...(audio && { + audio: { + renditions: { + audio: Catalog.AudioConfigSchema.parse({ + codec: "opus", + sampleRate: 48000, + numberOfChannels: 1, + container: { kind: "legacy" }, + }), + }, + }, + }), + }; + producer.announce(); + const canvas = document.createElement("canvas"); + document.body.append(canvas); + const player = new Player({ + origin, + name, + catalogFormat: "manual", + catalog, + video: { canvas, visible: "always", backend: "2d" }, + delay: Milli(100), + buffer: Milli(buffered ? 1000 : 0), + }); + const chunks = new Map(); + const encoder = new VideoEncoder({ + output(chunk) { + const data = new Uint8Array(chunk.byteLength); + chunk.copyTo(data); + chunks.set(chunk.timestamp, data); + }, + error(error) { + throw error; + }, + }); + encoder.configure({ codec: "vp8", width: 64, height: 32, bitrate: 100_000, latencyMode: "realtime" }); + const picture = new OffscreenCanvas(64, 32); + const ctx = picture.getContext("2d"); + if (!ctx) throw new Error("no encoder canvas context"); + for (const timestamp of [0, 40_000, 100_000, 140_000]) { + ctx.fillStyle = timestamp === 0 ? "red" : "blue"; + ctx.fillRect(0, 0, 64, 32); + const frame = new VideoFrame(picture, { timestamp }); + encoder.encode(frame, { keyFrame: true }); + frame.close(); + } + await encoder.flush(); + encoder.close(); + const push = (timestamp: number) => { + const payload = chunks.get(timestamp); + if (!payload) throw new Error("missing encoded frame"); + group.writeFrame({ payload: Legacy.encodeFrame(payload, Micro(timestamp)) }); + }; + return { + push, + async audio() { + if (!audioGroup) throw new Error("no audio track"); + const encode = new AudioEncoder({ + output(chunk) { + const data = new Uint8Array(chunk.byteLength); + chunk.copyTo(data); + audioGroup.writeFrame({ payload: Legacy.encodeFrame(data, Micro(chunk.timestamp)) }); + }, + error(error) { + throw error; + }, + }); + encode.configure({ codec: "opus", sampleRate: 48000, numberOfChannels: 1, bitrate: 64_000 }); + for (let i = 0; i < 12; i++) { + const frame = new AudioData({ + format: "f32-planar", + sampleRate: 48000, + numberOfFrames: 960, + numberOfChannels: 1, + timestamp: i * 20_000, + data: new Float32Array(960), + }); + encode.encode(frame); + frame.close(); + } + await encode.flush(); + encode.close(); + }, + state() { + const context = canvas.getContext("2d"); + if (!context) throw new Error("no renderer canvas context"); + const rgba = context.getImageData(32, 16, 1, 1).data; + return { + timestamp: player.renderer.out.frame.peek()?.timestamp, + playing: player.sync.playing.peek(), + color: rgba[0] > 160 ? "red" : rgba[2] > 160 ? "blue" : "black", + audioTimestamp: player.audio.out.timestamp.peek(), + audioState: player.audio.out.context.peek()?.state, + }; + }, + close() { + player.close(); + group.close(); + audioGroup?.close(); + producer.close(); + origin.close(); + canvas.remove(); + }, + }; +} + +declare global { + interface Window { + poster: Awaited>; + startPoster(audio: boolean, buffered: boolean): Promise; + } +} +window.startPoster = async (audio, buffered) => { + window.poster = await poster(audio, buffered); +};