diff --git a/bun.lock b/bun.lock index d10f5e5b029..718fd80494e 100644 --- a/bun.lock +++ b/bun.lock @@ -324,7 +324,7 @@ }, "js/watch": { "name": "@moq/watch", - "version": "0.6.5", + "version": "0.6.6", "dependencies": { "@moq/hang": "workspace:^", "@moq/json": "workspace:^", diff --git a/js/watch/package.json b/js/watch/package.json index 5eee56083f8..5f3e8e4528e 100644 --- a/js/watch/package.json +++ b/js/watch/package.json @@ -1,7 +1,7 @@ { "name": "@moq/watch", "type": "module", - "version": "0.6.5", + "version": "0.6.6", "jsr": false, "description": "Watch Media over QUIC broadcasts", "license": "(MIT OR Apache-2.0)", diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index ce24b928a18..706044c389a 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -21,12 +21,16 @@ import { type AudioBuffer, createAudioBuffer } from "./buffer"; import { type DecoderConfig, decoderConfig, type PlaybackIdentity, playbackIdentity } from "./config"; import { Handover } from "./handover"; import { ringSamples } from "./latency"; +import { Order } from "./order"; // Compiled and inlined as a blob URL via vite-plugin-worklet. import RenderWorklet from "./render-worklet.ts?worklet"; import type { Source } from "./source"; import { type DecodedSpan, Terminal } from "./terminal"; import { Warmup } from "./warmup"; +type Next = NonNullable>>; +type FrameNext = Next & { frame: NonNullable }; + const LEGACY_WARMUP_CALLBACKS = 3; export type DecoderInput = { @@ -345,22 +349,30 @@ export class Decoder { }; decoder.configure(decoderConfig); - // The newest frame sent to the decoder since it was last reset. - let newest: Time.Micro | undefined; + const order = this.#order(); + let discontinuity: number | undefined; for (;;) { const next = await nextMedia(consumer); - if (!next) break; - if (this.#onNext(next)) { - decoder.reset(); - decoder.configure(decoderConfig); - newest = undefined; + // The track ending, a marker, or a discontinuity cuts a held head short. A group merely + // completing does not. + if ( + next === undefined || + next.end !== undefined || + next.discontinuity !== (discontinuity ?? next.discontinuity) + ) { + if (!(await this.#play(effect, decoder, decoderConfig, order, order.flush()))) return; } - if (next.end !== undefined) { + if (!next) break; + discontinuity = next.discontinuity; + if (next.frame === undefined || next.end !== undefined) { + if (this.#onNext(next)) { + decoder.reset(); + decoder.configure(decoderConfig); + order.reset(); + } continue; } - const { frame } = next; - if (!frame) continue; // Mark that we received this frame right now. const timestamp = Time.Milli.fromMicro(frame.timestamp as Time.Micro); @@ -370,27 +382,7 @@ export class Decoder { bytesReceived: (stats?.bytesReceived ?? 0) + frame.payload.byteLength, })); - // Decode in timestamp order. A subscription's head groups arrive after the live edge served - // alongside them, and the ring, anchored at the first decoded frame, cannot place audio - // before it. Chrome's AudioDecoder would also stamp every later output from its first - // input, shifting the rest of the track by the reorder and overflowing a buffered ring. - if (newest !== undefined && (frame.timestamp as Time.Micro) < newest) continue; - newest = frame.timestamp as Time.Micro; - - // Backpressure: in buffered mode this holds the encoded frame until the playhead nears - // it, keeping the lookahead above the floor as Opus instead of decoded PCM. No-op live. - await this.#ring?.wait(frame.timestamp as Time.Micro, effect.abort); - - const chunk = new EncodedAudioChunk({ - type: frame.keyframe ? "key" : "delta", - data: frame.payload, - timestamp: frame.timestamp, - }); - - // A fatal decode error closes the decoder, so decoding again throws InvalidStateError out - // of this loop. Stop instead: the error callback already reported the real failure. - if (decoder.state === "closed") break; - decoder.decode(chunk); + if (!(await this.#play(effect, decoder, decoderConfig, order, order.push({ ...next, frame })))) return; } }); } @@ -446,21 +438,31 @@ export class Decoder { }; decoder.configure(decoderConfig); - // The newest frame sent to the decoder since it was last reset. - let newest: Time.Micro | undefined; + const order = this.#order(); + let discontinuity: number | undefined; for (;;) { const next = await nextMedia(consumer); + // The track ending, a marker, or a discontinuity cuts a held head short. A group merely + // completing does not. + if ( + next === undefined || + next.end !== undefined || + next.discontinuity !== (discontinuity ?? next.discontinuity) + ) { + if (!(await this.#play(effect, decoder, decoderConfig, order, order.flush()))) return; + } if (!next) break; - - // Reset and re-anchor before decoding the first frame of a new codec epoch. - if (this.#onNext(next)) { - decoder.reset(); - decoder.configure(decoderConfig); - newest = undefined; + discontinuity = next.discontinuity; + if (next.frame === undefined) { + // Reset and re-anchor before decoding the first frame of a new codec epoch. + if (this.#onNext(next)) { + decoder.reset(); + decoder.configure(decoderConfig); + order.reset(); + } + continue; } - const { frame } = next; - if (!frame) continue; const timestamp = Time.Milli.fromMicro(frame.timestamp); this.sync.received(timestamp, "audio"); @@ -469,29 +471,58 @@ export class Decoder { bytesReceived: (stats?.bytesReceived ?? 0) + frame.payload.byteLength, })); - // Decode in timestamp order. A subscription's head groups arrive after the live edge served - // alongside them, and the ring, anchored at the first decoded frame, cannot place audio - // before it. Chrome's AudioDecoder would also stamp every later output from its first - // input, shifting the rest of the track by the reorder and overflowing a buffered ring. - if (newest !== undefined && frame.timestamp < newest) continue; - newest = frame.timestamp; - - // Backpressure: in buffered mode this holds the encoded frame until the playhead nears - // it, keeping the lookahead above the floor as Opus instead of decoded PCM. No-op live. - await this.#ring?.wait(frame.timestamp, effect.abort); - - if (decoder.state === "closed") break; - decoder.decode( - new EncodedAudioChunk({ - type: frame.keyframe ? "key" : "delta", - data: frame.payload, - timestamp: frame.timestamp, - }), - ); + if (!(await this.#play(effect, decoder, decoderConfig, order, order.push({ ...next, frame })))) return; } }); } + // A buffered subscription holds its head until it runs past the floor, which playback waits for + // anyway, or spans the window the relay serves. + #order(): Order { + if (!this.sync.out.buffered.peek()) return new Order(undefined); + return new Order(() => ({ + floor: Time.Micro.fromMilli(this.sync.out.delay.peek()), + maxDelay: Time.Micro.fromMilli(this.sync.out.maxAge.peek()), + })); + } + + // Decode released frames, resetting the decoder before the first frame of a new codec epoch and + // only then dropping one that is out of order. The terminal sees them in decode order, so it + // anchors on the oldest. Reports whether to keep going. + async #play( + effect: Effect, + decoder: AudioDecoder, + config: AudioDecoderConfig, + order: Order, + reads: FrameNext[], + ): Promise { + for (const next of reads) { + if (this.#onNext(next)) { + decoder.reset(); + decoder.configure(config); + order.reset(); + } + if (!order.admit(next)) continue; + const { frame } = next; + + // Backpressure: in buffered mode this holds the encoded frame until the playhead nears + // it, keeping the lookahead above the floor as Opus instead of decoded PCM. No-op live. + await this.#ring?.wait(frame.timestamp as Time.Micro, effect.abort); + + // A fatal decode error closes the decoder, so decoding again throws InvalidStateError out + // of the loop. Stop instead: the error callback already reported the real failure. + if (decoder.state === "closed") return false; + decoder.decode( + new EncodedAudioChunk({ + type: frame.keyframe ? "key" : "delta", + data: frame.payload, + timestamp: frame.timestamp, + }), + ); + } + return true; + } + #emit(sample: AudioData, decoded: DecodedSpan = this.#terminal.span(sample)) { const { timestamp, frameOffset, frames } = decoded; const timestampMilli = Time.Milli.fromMicro(timestamp); diff --git a/js/watch/src/audio/order.test.ts b/js/watch/src/audio/order.test.ts new file mode 100644 index 00000000000..d8269b7e03e --- /dev/null +++ b/js/watch/src/audio/order.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from "bun:test"; +import { Order } from "./order"; + +type Read = { group: number; frame: { timestamp: number } }; +const read = (group: number, timestamp: number): Read => ({ group, frame: { timestamp } }); +const timestamps = (reads: Read[]) => reads.map(({ frame }) => frame.timestamp); + +describe("Order", () => { + it("releases a held head sorted once a newer group arrives", () => { + const order = new Order(() => ({ floor: 100, maxDelay: 1000 })); + for (const t of [40, 20, 0]) expect(order.push(read(t / 20, t))).toEqual([]); + expect(timestamps(order.push(read(3, 60)))).toEqual([0, 20, 40, 60]); + expect(timestamps(order.push(read(4, 80)))).toEqual([80]); + }); + + // A publisher may keep a long run of audio in one group, which never brings a newer one. + it("releases a head once its first group runs past the floor", () => { + const order = new Order(() => ({ floor: 100, maxDelay: 1000 })); + expect(order.push(read(5, 0))).toEqual([]); + expect(order.push(read(4, -20))).toEqual([]); + expect(order.push(read(5, 100))).toEqual([]); + expect(timestamps(order.push(read(5, 120)))).toEqual([-20, 0, 100, 120]); + }); + + // With no floor, playback starts on the first frame, but the head still has to arrive first. + it("holds the head past its first frame with a zero floor", () => { + const order = new Order(() => ({ floor: 0, maxDelay: 1000 })); + expect(order.push(read(1, 20))).toEqual([]); + expect(order.push(read(0, 0))).toEqual([]); + expect(timestamps(order.push(read(1, 40)))).toEqual([0, 20, 40]); + }); + + // Nothing older than the subscription's max delay is served, so a longer head is complete. + it("releases a head once it spans the max delay", () => { + const order = new Order(() => ({ floor: 100, maxDelay: 60 })); + for (const t of [60, 40, 20]) expect(order.push(read(t / 20, t))).toEqual([]); + expect(timestamps(order.push(read(0, 0)))).toEqual([0, 20, 40, 60]); + }); + + it("releases the head as it stands on a flush", () => { + const order = new Order(() => ({ floor: 100, maxDelay: 1000 })); + order.push(read(1, 20)); + order.push(read(0, 0)); + expect(timestamps(order.flush())).toEqual([0, 20]); + expect(order.flush()).toEqual([]); + }); + + it("admits a frame only if it is no older than one already admitted", () => { + const order = new Order(() => ({ floor: 100, maxDelay: 1000 })); + order.push(read(1, 20)); + const released = order.push(read(2, 40)); + expect(released.every((r) => order.admit(r))).toBe(true); + expect(order.push(read(0, 0))).toEqual([read(0, 0)]); + expect(order.admit(read(0, 0))).toBe(false); + + // A reset decoder restamps from its next input, so older frames are welcome again. + order.reset(); + expect(order.admit(read(0, 0))).toBe(true); + }); + + // A rise in the delay while the head arrives must not release it at the old floor. + it("reads its bounds as the head arrives", () => { + let floor = 20; + const order = new Order(() => ({ floor, maxDelay: 1000 })); + expect(order.push(read(5, 0))).toEqual([]); + floor = 100; + expect(order.push(read(5, 40))).toEqual([]); + expect(order.push(read(4, -20))).toEqual([]); + expect(timestamps(order.push(read(5, 120)))).toEqual([-20, 0, 40, 120]); + }); + + it("holds nothing live", () => { + const order = new Order(undefined); + expect(timestamps(order.push(read(1, 20)))).toEqual([20]); + expect(order.admit(read(1, 20))).toBe(true); + expect(timestamps(order.push(read(0, 0)))).toEqual([0]); + expect(order.admit(read(0, 0))).toBe(false); + }); +}); diff --git a/js/watch/src/audio/order.ts b/js/watch/src/audio/order.ts new file mode 100644 index 00000000000..098ba25e402 --- /dev/null +++ b/js/watch/src/audio/order.ts @@ -0,0 +1,72 @@ +/** + * Puts one subscription's frames in timestamp order for the audio decoder. + * + * Groups go out newest-first, so a subscription's head (the groups the publisher still holds) arrives + * after the live edge it was served alongside. Chrome's AudioDecoder stamps every output from its + * first input, so decoding the head after the live edge shifts the rest of the track, and the ring, + * anchored at the first decoded frame, cannot place the head anyway. + * + * A buffered subscription holds its head and releases it sorted. The head is complete once a newer + * group than the first arrives, or once the first group alone runs past the floor: the ring could not + * start playing before holding that much, so waiting for the head costs no startup latency. It is + * also released once it spans the subscription's max delay, since nothing older is served. After + * that, and throughout when not holding, a frame older than one already decoded is not admitted. + */ +export class Order { + // The current bounds in microseconds, while the head is still arriving. + #hold: (() => { floor: number; maxDelay: number }) | undefined; + // The head, in arrival order. + #held: T[] = []; + // The group the head started with, and its own first frame. + #first?: { group: number; timestamp: number }; + // The oldest and newest held timestamps. + #span?: { oldest: number; newest: number }; + // The newest frame admitted since the decoder was last reset. + #newest?: number; + + /** Hold the head within the bounds `hold` reports, or pass each frame straight through. */ + constructor(hold: (() => { floor: number; maxDelay: number }) | undefined) { + this.#hold = hold; + } + + /** Accept a frame, returning the frames released for decoding, oldest first. */ + push(read: T): T[] { + const hold = this.#hold?.(); + if (!hold) return [read]; + + const { group, frame } = read; + this.#first ??= { group, timestamp: frame.timestamp }; + if (group > this.#first.group) return [...this.flush(), read]; + + this.#held.push(read); + const oldest = Math.min(this.#span?.oldest ?? frame.timestamp, frame.timestamp); + const newest = Math.max(this.#span?.newest ?? frame.timestamp, frame.timestamp); + this.#span = { oldest, newest }; + + const ran = group === this.#first.group && frame.timestamp - this.#first.timestamp > hold.floor; + if (ran || newest - oldest >= hold.maxDelay) return this.flush(); + return []; + } + + /** Release the head as it stands, ending the hold: the track ended, or a gap or end cut it short. */ + flush(): T[] { + this.#hold = undefined; + const held = this.#held; + this.#held = []; + // Stable, so frames sharing a timestamp keep their arrival order. + return held.sort((a, b) => a.frame.timestamp - b.frame.timestamp); + } + + /** Whether a released frame is still in order for the decoder, recording it if so. */ + admit(read: T): boolean { + const { timestamp } = read.frame; + if (this.#newest !== undefined && timestamp < this.#newest) return false; + this.#newest = timestamp; + return true; + } + + /** Start the order over after the decoder resets, which restarts its output timestamps. */ + reset(): void { + this.#newest = undefined; + } +}