diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 17f97a060fa..295121dad69 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -326,15 +326,55 @@ it("decodes an audio packet inside a group after a gap", async () => { } }); -// A subscription's head arrives newest-first. Chrome's AudioDecoder stamps every output from its -// first input, so decoding the older frames after it shifts the rest of the track. +// A subscription's head arrives newest-first, one group per frame. Chrome's AudioDecoder stamps every +// output from its first input, so decoding the older frames after it shifts the rest of the track. for (const cmaf of [false, true]) { - it(`decodes a ${cmaf ? "CMAF" : "legacy"} subscription head in timestamp order`, async () => { + it(`decodes a ${cmaf ? "CMAF" : "legacy"} buffered subscription head in timestamp order`, async () => { const playback = await play(Milli(100), true, cmaf); + try { + for (const ms of [140, 120, 100, 80, 60, 40, 20, 0]) { + frameTimestamp = (ms - 20) * 1000; + playback.push({ ...frame(), group: ms / 20 }); + await microtasks(); + } + expect(playback.inserted()).toEqual([]); + + // The live edge moves on, so the head is complete and plays from its oldest frame. + frameTimestamp = 140_000; + playback.push({ ...frame(), group: 8 }); + await microtasks(); + expect(playback.inserted()).toEqual([0, 20_000, 40_000, 60_000, 80_000, 100_000]); + } finally { + playback.close(); + await microtasks(); + } + }); + + // A fresh encoder after a declared end may restart its timeline below the old one, and the decoder + // resets for it before anything is judged out of order. + it(`decodes a ${cmaf ? "CMAF" : "legacy"} fresh encoder below the timeline it replaced`, async () => { + const playback = await play(Milli(100), false, cmaf); + try { + frameTimestamp = 480_000; + playback.push({ ...frame(), group: 0 }); + await microtasks(); + playback.push({ kind: "end", group: 0, end: Micro(520_000) }); + frameTimestamp = 0; + playback.push({ ...frame(), group: 1 }); + await microtasks(); + expect(playback.inserted()).toEqual([500_000, 20_000]); + } finally { + playback.close(); + await microtasks(); + } + }); + + it(`skips a ${cmaf ? "CMAF" : "legacy"} live subscription head older than its live edge`, async () => { + const playback = await play(Milli(100), false, cmaf); try { for (const ms of [140, 120, 100, 80, 60, 40, 20, 0, 160]) { frameTimestamp = (ms - 20) * 1000; - playback.push(frame()); + playback.push({ ...frame(), group: ms / 20 }); await microtasks(); } expect(playback.inserted()).toEqual([140_000, 160_000]); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 2d7ab72c5ac..b4454809eeb 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -22,11 +22,14 @@ import { type AudioBuffer, createAudioBuffer } from "./buffer"; import { type DecoderConfig, frameDuration, type PlaybackIdentity, packetDuration, playbackIdentity } from "./config"; import { Handover } from "./handover"; import { AUTO_MAX_DELAY, ringSamples, target } from "./latency"; +import { Order } from "./order"; // A blob: URL, or a hosted file when assets() is set; see vite-plugin-worklet. import RenderWorklet from "./render-worklet.ts?worklet"; import type { Source } from "./source"; import { Terminal } from "./terminal"; +type FrameRead = Extract; + /** Signals the decoder reads. */ export type DecoderInput = { // Whether to download the audio track. Defaults to true. @@ -476,18 +479,22 @@ export class Decoder { }; decoder.configure(decoderConfig); - // The newest frame sent to the decoder since it was last reset. - let newest: Micro | undefined; + const order = this.#order(); + const play = (reads: FrameRead[]) => this.#play(effect, decoder, decoderConfig, order, reads); for (;;) { const next = await nextMedia(consumer); - if (!next) break; - if (this.#onRead(next)) { - decoder.reset(); - decoder.configure(decoderConfig); - newest = undefined; + if (next?.kind !== "frame") { + // The track ending, or a gap or end, cuts a held head short. + if (!(await play(order.flush()))) return; + if (!next) break; + if (this.#onRead(next)) { + decoder.reset(); + decoder.configure(decoderConfig); + order.reset(); + } + if (next.kind === "gap") this.#arrivals.write(gap(next)); + continue; } - if (next.kind === "gap") this.#arrivals.write(gap(next)); - if (next.kind !== "frame") continue; const { frame } = next; // Classify ingress before decoder backpressure changes the delivery clock. @@ -502,32 +509,7 @@ export class Decoder { bytesReceived: (stats?.bytesReceived ?? 0) + frame.payload.byteLength, })); - // Decode in timestamp order. A subscription's head groups arrive after the live edge served - // alongside them, and the ring, anchored at the first decoded frame, cannot place audio - // before it. Chrome's AudioDecoder would also stamp every later output from its first - // input, shifting the rest of the track by the reorder and overflowing a buffered ring. - if (newest !== undefined && frame.timestamp < newest) continue; - newest = frame.timestamp; - - const ring = await this.#ready(effect); - if (!ring) break; - - // Backpressure: in buffered mode this holds the encoded frame until the playhead nears - // it, keeping the lookahead above the floor as Opus instead of decoded PCM. No-op live. - await ring.wait(frame.timestamp as Micro, effect.abort); - if (effect.abort.aborted) break; - - const chunk = new EncodedAudioChunk({ - // WebCodecs audio chunks are key even inside a MoQ group. - type: "key", - data: frame.payload, - timestamp: frame.timestamp, - }); - - // A fatal decode error closes the decoder, so decoding again throws InvalidStateError out - // of this loop. Stop instead: the error callback already reported the real failure. - if (decoder.state === "closed") break; - decoder.decode(chunk); + if (!(await play(order.push(next)))) return; } }); } @@ -580,20 +562,22 @@ export class Decoder { }; decoder.configure(decoderConfig); - // The newest frame sent to the decoder since it was last reset. - let newest: Micro | undefined; + const order = this.#order(); + const play = (reads: FrameRead[]) => this.#play(effect, decoder, decoderConfig, order, reads); for (;;) { const next = await nextMedia(consumer); - if (!next) break; - - // Reset and re-anchor before decoding the first frame of a new codec epoch. - if (this.#onRead(next)) { - decoder.reset(); - decoder.configure(decoderConfig); - newest = undefined; + if (next?.kind !== "frame") { + // The track ending, or a gap or end, cuts a held head short. + if (!(await play(order.flush()))) return; + if (!next) break; + if (this.#onRead(next)) { + decoder.reset(); + decoder.configure(decoderConfig); + order.reset(); + } + if (next.kind === "gap") this.#arrivals.write(gap(next)); + continue; } - if (next.kind === "gap") this.#arrivals.write(gap(next)); - if (next.kind !== "frame") continue; const { frame } = next; const timestamp = Milli.fromMicro(frame.timestamp); @@ -607,34 +591,67 @@ export class Decoder { bytesReceived: (stats?.bytesReceived ?? 0) + frame.payload.byteLength, })); - // Decode in timestamp order. A subscription's head groups arrive after the live edge served - // alongside them, and the ring, anchored at the first decoded frame, cannot place audio - // before it. Chrome's AudioDecoder would also stamp every later output from its first - // input, shifting the rest of the track by the reorder and overflowing a buffered ring. - if (newest !== undefined && frame.timestamp < newest) continue; - newest = frame.timestamp; - - const ring = await this.#ready(effect); - if (!ring) break; - - // Backpressure: in buffered mode this holds the encoded frame until the playhead nears - // it, keeping the lookahead above the floor as Opus instead of decoded PCM. No-op live. - await ring.wait(frame.timestamp, effect.abort); - if (effect.abort.aborted) break; - - if (decoder.state === "closed") break; - decoder.decode( - new EncodedAudioChunk({ - // WebCodecs audio chunks are key even inside a MoQ group. - type: "key", - data: frame.payload, - timestamp: frame.timestamp, - }), - ); + if (!(await play(order.push(next)))) return; } }); } + // Decode released frames, resetting the decoder before the first frame of a new codec epoch and + // only then dropping one that is out of order. The terminal sees them in decode order, so it + // anchors on the oldest. Reports whether to keep going. + async #play( + effect: Effect, + decoder: AudioDecoder, + config: AudioDecoderConfig, + order: Order, + reads: FrameRead[], + ): Promise { + for (const read of reads) { + if (this.#onRead(read)) { + decoder.reset(); + decoder.configure(config); + order.reset(); + } + if (!order.admit(read)) continue; + if (!(await this.#decode(effect, decoder, read.frame))) return false; + } + return true; + } + + // Decode one frame once backpressure admits it, reporting whether to keep going. + async #decode(effect: Effect, decoder: AudioDecoder, frame: Container.ReceivedFrame): Promise { + const ring = await this.#ready(effect); + if (!ring) return false; + + // Backpressure: in buffered mode this holds the encoded frame until the playhead nears + // it, keeping the lookahead above the floor as Opus instead of decoded PCM. No-op live. + await ring.wait(frame.timestamp, effect.abort); + if (effect.abort.aborted) return false; + + // A fatal decode error closes the decoder, so decoding again throws InvalidStateError out + // of the loop. Stop instead: the error callback already reported the real failure. + if (decoder.state === "closed") return false; + decoder.decode( + new EncodedAudioChunk({ + // WebCodecs audio chunks are key even inside a MoQ group. + type: "key", + data: frame.payload, + timestamp: frame.timestamp, + }), + ); + return true; + } + + // A buffered subscription holds its head until it runs past the floor, which playback waits for + // anyway, or spans the window the relay serves. + #order(): Order { + if (!this.sync.out.buffered.peek()) return new Order(undefined); + return new Order(() => ({ + floor: Micro.fromMilli(this.sync.out.delay.peek()), + maxDelay: Micro.fromMilli(this.#subscribeMaxDelay.peek()), + })); + } + // Reorders groups and frames up to the shared max delay. A rename is not a cold start: the path's // arrivals are what they were, so the estimate continues from whatever read it before. Another // path's arrivals have nothing to do with what this one learned. diff --git a/js/watch/src/audio/order.test.ts b/js/watch/src/audio/order.test.ts new file mode 100644 index 00000000000..d8269b7e03e --- /dev/null +++ b/js/watch/src/audio/order.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from "bun:test"; +import { Order } from "./order"; + +type Read = { group: number; frame: { timestamp: number } }; +const read = (group: number, timestamp: number): Read => ({ group, frame: { timestamp } }); +const timestamps = (reads: Read[]) => reads.map(({ frame }) => frame.timestamp); + +describe("Order", () => { + it("releases a held head sorted once a newer group arrives", () => { + const order = new Order(() => ({ floor: 100, maxDelay: 1000 })); + for (const t of [40, 20, 0]) expect(order.push(read(t / 20, t))).toEqual([]); + expect(timestamps(order.push(read(3, 60)))).toEqual([0, 20, 40, 60]); + expect(timestamps(order.push(read(4, 80)))).toEqual([80]); + }); + + // A publisher may keep a long run of audio in one group, which never brings a newer one. + it("releases a head once its first group runs past the floor", () => { + const order = new Order(() => ({ floor: 100, maxDelay: 1000 })); + expect(order.push(read(5, 0))).toEqual([]); + expect(order.push(read(4, -20))).toEqual([]); + expect(order.push(read(5, 100))).toEqual([]); + expect(timestamps(order.push(read(5, 120)))).toEqual([-20, 0, 100, 120]); + }); + + // With no floor, playback starts on the first frame, but the head still has to arrive first. + it("holds the head past its first frame with a zero floor", () => { + const order = new Order(() => ({ floor: 0, maxDelay: 1000 })); + expect(order.push(read(1, 20))).toEqual([]); + expect(order.push(read(0, 0))).toEqual([]); + expect(timestamps(order.push(read(1, 40)))).toEqual([0, 20, 40]); + }); + + // Nothing older than the subscription's max delay is served, so a longer head is complete. + it("releases a head once it spans the max delay", () => { + const order = new Order(() => ({ floor: 100, maxDelay: 60 })); + for (const t of [60, 40, 20]) expect(order.push(read(t / 20, t))).toEqual([]); + expect(timestamps(order.push(read(0, 0)))).toEqual([0, 20, 40, 60]); + }); + + it("releases the head as it stands on a flush", () => { + const order = new Order(() => ({ floor: 100, maxDelay: 1000 })); + order.push(read(1, 20)); + order.push(read(0, 0)); + expect(timestamps(order.flush())).toEqual([0, 20]); + expect(order.flush()).toEqual([]); + }); + + it("admits a frame only if it is no older than one already admitted", () => { + const order = new Order(() => ({ floor: 100, maxDelay: 1000 })); + order.push(read(1, 20)); + const released = order.push(read(2, 40)); + expect(released.every((r) => order.admit(r))).toBe(true); + expect(order.push(read(0, 0))).toEqual([read(0, 0)]); + expect(order.admit(read(0, 0))).toBe(false); + + // A reset decoder restamps from its next input, so older frames are welcome again. + order.reset(); + expect(order.admit(read(0, 0))).toBe(true); + }); + + // A rise in the delay while the head arrives must not release it at the old floor. + it("reads its bounds as the head arrives", () => { + let floor = 20; + const order = new Order(() => ({ floor, maxDelay: 1000 })); + expect(order.push(read(5, 0))).toEqual([]); + floor = 100; + expect(order.push(read(5, 40))).toEqual([]); + expect(order.push(read(4, -20))).toEqual([]); + expect(timestamps(order.push(read(5, 120)))).toEqual([-20, 0, 40, 120]); + }); + + it("holds nothing live", () => { + const order = new Order(undefined); + expect(timestamps(order.push(read(1, 20)))).toEqual([20]); + expect(order.admit(read(1, 20))).toBe(true); + expect(timestamps(order.push(read(0, 0)))).toEqual([0]); + expect(order.admit(read(0, 0))).toBe(false); + }); +}); diff --git a/js/watch/src/audio/order.ts b/js/watch/src/audio/order.ts new file mode 100644 index 00000000000..098ba25e402 --- /dev/null +++ b/js/watch/src/audio/order.ts @@ -0,0 +1,72 @@ +/** + * Puts one subscription's frames in timestamp order for the audio decoder. + * + * Groups go out newest-first, so a subscription's head (the groups the publisher still holds) arrives + * after the live edge it was served alongside. Chrome's AudioDecoder stamps every output from its + * first input, so decoding the head after the live edge shifts the rest of the track, and the ring, + * anchored at the first decoded frame, cannot place the head anyway. + * + * A buffered subscription holds its head and releases it sorted. The head is complete once a newer + * group than the first arrives, or once the first group alone runs past the floor: the ring could not + * start playing before holding that much, so waiting for the head costs no startup latency. It is + * also released once it spans the subscription's max delay, since nothing older is served. After + * that, and throughout when not holding, a frame older than one already decoded is not admitted. + */ +export class Order { + // The current bounds in microseconds, while the head is still arriving. + #hold: (() => { floor: number; maxDelay: number }) | undefined; + // The head, in arrival order. + #held: T[] = []; + // The group the head started with, and its own first frame. + #first?: { group: number; timestamp: number }; + // The oldest and newest held timestamps. + #span?: { oldest: number; newest: number }; + // The newest frame admitted since the decoder was last reset. + #newest?: number; + + /** Hold the head within the bounds `hold` reports, or pass each frame straight through. */ + constructor(hold: (() => { floor: number; maxDelay: number }) | undefined) { + this.#hold = hold; + } + + /** Accept a frame, returning the frames released for decoding, oldest first. */ + push(read: T): T[] { + const hold = this.#hold?.(); + if (!hold) return [read]; + + const { group, frame } = read; + this.#first ??= { group, timestamp: frame.timestamp }; + if (group > this.#first.group) return [...this.flush(), read]; + + this.#held.push(read); + const oldest = Math.min(this.#span?.oldest ?? frame.timestamp, frame.timestamp); + const newest = Math.max(this.#span?.newest ?? frame.timestamp, frame.timestamp); + this.#span = { oldest, newest }; + + const ran = group === this.#first.group && frame.timestamp - this.#first.timestamp > hold.floor; + if (ran || newest - oldest >= hold.maxDelay) return this.flush(); + return []; + } + + /** Release the head as it stands, ending the hold: the track ended, or a gap or end cut it short. */ + flush(): T[] { + this.#hold = undefined; + const held = this.#held; + this.#held = []; + // Stable, so frames sharing a timestamp keep their arrival order. + return held.sort((a, b) => a.frame.timestamp - b.frame.timestamp); + } + + /** Whether a released frame is still in order for the decoder, recording it if so. */ + admit(read: T): boolean { + const { timestamp } = read.frame; + if (this.#newest !== undefined && timestamp < this.#newest) return false; + this.#newest = timestamp; + return true; + } + + /** Start the order over after the decoder resets, which restarts its output timestamps. */ + reset(): void { + this.#newest = undefined; + } +}