diff --git a/packages/spf/src/playback/actors/dom/audio-renderer.ts b/packages/spf/src/playback/actors/dom/audio-renderer.ts index aa73951532..d870d131c4 100644 --- a/packages/spf/src/playback/actors/dom/audio-renderer.ts +++ b/packages/spf/src/playback/actors/dom/audio-renderer.ts @@ -36,6 +36,15 @@ export interface AudioFrameSource { export interface AudioContextLike { readonly currentTime: number; readonly destination: AudioNode; + /** Seconds between the graph and the audio subsystem; the worklet renderer places its clock behind it. */ + readonly baseLatency?: number; + /** Seconds from the audio subsystem to the speaker; read fresh, as it changes with the device. */ + readonly outputLatency?: number; + /** + * The real context behind a wrapping view such as the adapter's, for APIs that only accept one — an + * `AudioWorkletNode` cannot be constructed on a plain object. + */ + readonly baseContext?: BaseAudioContext; createBuffer(numberOfChannels: number, length: number, sampleRate: number): AudioBuffer; createBufferSource(): AudioBufferSourceNode; } diff --git a/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts b/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts new file mode 100644 index 0000000000..3931fc9e15 --- /dev/null +++ b/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts @@ -0,0 +1,546 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vite-plus/test'; + +import type { JitterFrame } from '../../track-subscriber'; +import type { AudioContextLike, AudioFrameSource } from '../audio-renderer'; +import { + type AudioPlayer, + type AudioPlayerInput, + type AudioPlayerInputOptions, + createWorkletAudioRendererActor, +} from '../worklet-audio-renderer'; + +const FRAME_US = 20_000; +const JITTER_TARGET_US = 60_000; +const TICK_MS = 10; +const CONFIG: AudioDecoderConfig = { codec: 'opus', sampleRate: 48_000, numberOfChannels: 1 }; + +/** An Opus TOC byte for one 20 ms CELT frame (config 19, code 0). */ +const OPUS_20MS_TOC = 19 << 3; + +function frameAt(timestampUs: number): JitterFrame { + return { groupId: timestampUs, objectId: 0, timestampUs, isKey: true, payload: new Uint8Array([OPUS_20MS_TOC]) }; +} + +/** Frames every 20 ms over `[fromUs, untilUs)`. */ +function framesBetween(fromUs: number, untilUs: number): JitterFrame[] { + const frames: JitterFrame[] = []; + + for (let timestampUs = fromUs; timestampUs < untilUs; timestampUs += FRAME_US) frames.push(frameAt(timestampUs)); + + return frames; +} + +function queueSource(frames: JitterFrame[] = []): AudioFrameSource & { queue: JitterFrame[] } { + const queue = [...frames]; + + return { queue, peek: () => queue[0], dequeue: () => queue.shift() }; +} + +interface FakeInput extends AudioPlayerInput { + options: AudioPlayerInputOptions; + pushed: number[]; + cuts: number; + removed: boolean; + stats: { playing: boolean; bufferedUs: number; jitterTargetUs: number }; +} + +/** The core's surface with no audio behind it: pushes are recorded, and the test plays the worklet's reports. */ +function createFakePlayer() { + const inputs: FakeInput[] = []; + const connected: AudioNode[] = []; + let closed = false; + const player: AudioPlayer = { + output: { connect: (destination) => connected.push(destination) }, + addInput(options = {}) { + const input: FakeInput = { + options, + pushed: [], + cuts: 0, + removed: false, + stats: { playing: true, bufferedUs: 0, jitterTargetUs: JITTER_TARGET_US }, + push(frame) { + input.pushed.push(frame.timestampUs); + }, + cut() { + input.cuts++; + }, + remove() { + input.removed = true; + }, + }; + + inputs.push(input); + return input; + }, + close() { + closed = true; + }, + }; + + return { + player, + inputs, + connected, + get closed() { + return closed; + }, + /** The worklet reporting that the audio at `timestampUs` is audible now. */ + report(timestampUs: number, atUs: number) { + inputs[inputs.length - 1]!.options.clock!.reportAudio(timestampUs, atUs); + }, + }; +} + +// SAFETY: the renderer only hands `destination` to `output.connect`, which the fake records by identity. +const destination = {} as AudioNode; + +/** Context time follows the test's clock; `suspended` freezes it, as `AudioContext.suspend()` does. */ +let contextUs = 0; +let suspended = false; + +function fakeAudioContext(outputLatency?: number): AudioContextLike { + return { + get currentTime() { + return contextUs / 1_000_000; + }, + destination, + outputLatency, + createBuffer: () => { + throw new Error('unused'); + }, + createBufferSource: () => { + throw new Error('unused'); + }, + }; +} + +describe('createWorkletAudioRendererActor', () => { + let nowUs = 0; + + /** Move wall time, and context time with it unless the context is suspended. */ + const at = (us: number) => { + if (!suspended) contextUs += us - nowUs; + + nowUs = us; + }; + + beforeEach(() => { + vi.useFakeTimers(); + at(0); + contextUs = 0; + suspended = false; + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + /** A renderer whose player is ready and whose first input exists. */ + async function setup(options: { getJoinAnchorUs?: () => number | undefined; outputLatency?: number } = {}) { + const { outputLatency, ...rendererOptions } = options; + const fake = createFakePlayer(); + const renderer = createWorkletAudioRendererActor({ + audioContext: fakeAudioContext(outputLatency), + createAudioPlayer: () => Promise.resolve(fake.player), + tickIntervalMs: TICK_MS, + ...rendererOptions, + }); + + await vi.advanceTimersByTimeAsync(0); + return { fake, renderer }; + } + + const tick = () => vi.advanceTimersByTime(TICK_MS); + + it('connects the player output to the context destination', async () => { + const { fake, renderer } = await setup(); + + expect(fake.connected).toEqual([destination]); + expect(renderer.getClockTimeUs()).toBeUndefined(); + + renderer.destroy(); + expect(fake.closed).toBe(true); + }); + + it('fills to the jitter target plus a frame, then meters by the clock', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 1_000_000)); + + renderer.setTrack(source, CONFIG); + tick(); + + const input = fake.inputs[0]!; + + expect(input.options.decoderConfig).toEqual(CONFIG); + expect(input.pushed).toEqual([0, 20_000, 40_000, 60_000]); + expect(renderer.getClockTimeUs()).toBeUndefined(); + + // Playing: the worklet should hold about its target ahead of what renders. + fake.report(0, 0); + at(30_000); + tick(); + + expect(input.pushed.at(-1)).toBe(80_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(30_000, 0); + expect(renderer.snapshot.get().context.scheduledUntilUs).toBe(100_000); + }); + + it('joins at the edge, dropping buffered audio behind the anchor unheard', async () => { + const { fake, renderer } = await setup({ getJoinAnchorUs: () => 200_000 }); + + renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); + tick(); + + expect(fake.inputs[0]!.pushed[0]).toBe(200_000); + }); + + it('holds the clock where the worklet ran dry, through the refill, until audio past it plays', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + + at(100_000); + tick(); + expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); + + // Audio comes back: the refill moves the pushed end, not the clock. + fake.inputs[0]!.stats.playing = false; + source.queue.push(...framesBetween(80_000, 400_000)); + at(300_000); + tick(); + expect(fake.inputs[0]!.pushed.slice(4)).toEqual([80_000, 100_000, 120_000, 140_000]); + expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); + + fake.report(80_000, 320_000); + at(330_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(90_000, 0); + }); + + it('does not step back when the refill is reported ahead of being audible', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + at(100_000); + tick(); + + fake.inputs[0]!.stats.playing = false; + source.queue.push(...framesBetween(80_000, 400_000)); + at(300_000); + tick(); + + // Rendered now, audible 100 ms from now: the anchor alone reads 80 ms behind the hold. + fake.report(80_000, 400_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); + + at(410_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(90_000, 0); + }); + + it('resumes from the audible position when audio returns before the device drained', async () => { + const { fake, renderer } = await setup({ outputLatency: 0.05 }); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + + // Rendering passed the last pushed sample; 50 ms of it is still in the device. + at(30_000); + tick(); + expect(renderer.getClockTimeUs()).toBeCloseTo(30_000, 0); + + fake.inputs[0]!.stats.playing = false; + source.queue.push(...framesBetween(80_000, 400_000)); + at(40_000); + tick(); + fake.report(80_000, 90_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(40_000, 0); + }); + + it('ends pushed audio where the last packet ends, not across a gap of lost packets', async () => { + const { fake, renderer } = await setup(); + + renderer.setTrack(queueSource([0, 20_000, 40_000, 100_000].map(frameAt)), CONFIG); + tick(); + fake.report(0, 0); + at(60_000); + tick(); + + expect(fake.inputs[0]!.pushed).toEqual([0, 20_000, 40_000, 100_000]); + expect(renderer.snapshot.get().context.scheduledUntilUs).toBe(120_000); + }); + + it('keeps the hold through a late report of the audio before the underrun', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + at(100_000); + tick(); + + // The refill is pushed before the core has said it stopped; then the + // tail's last report, still in flight, arrives. + source.queue.push(...framesBetween(80_000, 400_000)); + tick(); + fake.report(77_333, 77_333); + at(200_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); + }); + + it('holds a placed re-join clock at the audio pushed while the fill is short', async () => { + let anchorUs: number | undefined; + const { fake, renderer } = await setup({ getJoinAnchorUs: () => anchorUs }); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + + anchorUs = 4_900_000; + source.queue.push(frameAt(5_000_000)); + tick(); + at(100_000); + tick(); + expect(fake.inputs[0]!.pushed.at(-1)).toBe(5_000_000); + + // One packet cannot start the worklet; the clock stops at its end. + at(2_100_000); + tick(); + expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); + + // The refill plays, reported ahead of being audible: no step back. + source.queue.push(...framesBetween(5_020_000, 5_200_000)); + tick(); + fake.report(5_000_000, 2_150_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); + }); + + it('gives the core a report time source that stops while the context is suspended', async () => { + const { fake, renderer } = await setup(); + + renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); + tick(); + + const clock = fake.inputs[0]!.options.clock!; + + at(40_000); + expect(clock.localNow()).toBeCloseTo(40_000, 0); + + suspended = true; + at(540_000); + expect(clock.localNow()).toBeCloseTo(40_000, 0); + + // The first report after resume, timed on that source, lands where audio is. + suspended = false; + at(560_000); + fake.report(60_000, clock.localNow()); + expect(renderer.getClockTimeUs()).toBeCloseTo(60_000, 0); + }); + + it('drops what the core holds at a timeline reset and ignores reports from before it', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + + // A publisher re-anchor: the timeline steps back by far more than a gap. + source.queue.push(...framesBetween(-5_000_000, -4_000_000)); + tick(); + + const input = fake.inputs[0]!; + + expect(input.cuts).toBe(1); + expect(input.pushed.at(4)).toBe(-5_000_000); + expect(renderer.getClockTimeUs()).toBeUndefined(); + + // Still in flight from the old timeline when the cut landed. + fake.report(40_000, 0); + expect(renderer.getClockTimeUs()).toBeUndefined(); + + fake.report(-5_000_000, 0); + expect(renderer.getClockTimeUs()).toBeCloseTo(-5_000_000, 0); + }); + + it('places a re-join whose whole buffer is past the anchor at its distance from it', async () => { + let anchorUs: number | undefined; + const { fake, renderer } = await setup({ getJoinAnchorUs: () => anchorUs }); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + + // A catch-up skip landed on a group 100 ms past where the anchor falls. + anchorUs = 4_900_000; + source.queue.push(...framesBetween(5_000_000, 6_000_000)); + tick(); + + const input = fake.inputs[0]!; + + expect(input.pushed).toHaveLength(4); + expect(renderer.getClockTimeUs()).toBeCloseTo(4_900_000, 0); + + at(50_000); + tick(); + expect(input.pushed.at(4)).toBe(5_000_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(4_950_000, 0); + }); + + it('freezes the clock and stops metering while the context is suspended', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 1_000_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + at(30_000); + tick(); + + const pushed = fake.inputs[0]!.pushed.length; + + // Paused for two seconds: nothing plays, so nothing is due and nothing ran dry. + suspended = true; + at(2_030_000); + tick(); + expect(renderer.getClockTimeUs()).toBeCloseTo(30_000, 0); + expect(fake.inputs[0]!.pushed).toHaveLength(pushed); + + suspended = false; + at(2_040_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(40_000, 0); + }); + + it('ignores reports from an input that a track change removed', async () => { + const { fake, renderer } = await setup(); + + renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); + tick(); + + const stale = fake.inputs[0]!.options.clock!; + + renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); + tick(); + stale.reportAudio(40_000, 0); + expect(renderer.getClockTimeUs()).toBeUndefined(); + + fake.report(0, 0); + expect(renderer.getClockTimeUs()).toBeCloseTo(0, 0); + }); + + it('counts every time the worklet runs dry, across track changes', async () => { + const { fake, renderer } = await setup(); + + renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); + tick(); + fake.report(0, 0); + + // A stall far longer than any jitter ceiling still counts, once. + at(100_000); + tick(); + at(3_000_000); + tick(); + expect(renderer.snapshot.get().context.underruns).toBe(1); + + renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); + tick(); + fake.report(0, 3_000_000); + at(3_100_000); + tick(); + expect(renderer.snapshot.get().context.underruns).toBe(2); + }); + + it('counts a dry edge that a track change lands on before the next tick', async () => { + const { fake, renderer } = await setup(); + + renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); + tick(); + fake.report(0, 0); + + at(100_000); + renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); + expect(renderer.snapshot.get().context.underruns).toBe(1); + }); + + it('rescues a frozen re-join placement whose next packets are past the fill horizon', async () => { + let anchorUs: number | undefined; + const { fake, renderer } = await setup({ getJoinAnchorUs: () => anchorUs }); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + + anchorUs = 4_900_000; + source.queue.push(frameAt(5_000_000)); + tick(); + at(200_000); + tick(); + expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); + + // Packets resume 80 ms after the lone one ends: beyond the fill horizon. + fake.inputs[0]!.stats.playing = false; + source.queue.push(...framesBetween(5_100_000, 5_400_000)); + at(400_000); + tick(); + expect(fake.inputs[0]!.pushed).toContain(5_100_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); + }); + + it('caps the core jitter target at the target latency', async () => { + const fake = createFakePlayer(); + const renderer = createWorkletAudioRendererActor({ + audioContext: fakeAudioContext(), + createAudioPlayer: () => Promise.resolve(fake.player), + getTargetLatencyUs: () => 250_000, + tickIntervalMs: TICK_MS, + }); + + await vi.advanceTimersByTimeAsync(0); + renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); + tick(); + + expect(fake.inputs[0]!.options.clock!.targetLatencyUs).toBe(250_000); + }); + + it('fails a track with no decoder config, and a player that cannot be created', async () => { + const { renderer } = await setup(); + + renderer.setTrack(queueSource(), null); + expect(renderer.snapshot.get().context.status).toBe('error'); + + const failing = createWorkletAudioRendererActor({ + audioContext: fakeAudioContext(), + createAudioPlayer: () => Promise.reject(new Error('no worklet')), + }); + + await vi.advanceTimersByTimeAsync(0); + expect(failing.snapshot.get().context.status).toBe('error'); + failing.destroy(); + + const unconnectable = createFakePlayer(); + + unconnectable.player.output.connect = () => { + throw new Error('cross-context connect'); + }; + + const disconnected = createWorkletAudioRendererActor({ + audioContext: fakeAudioContext(), + createAudioPlayer: () => Promise.resolve(unconnectable.player), + }); + + await vi.advanceTimersByTimeAsync(0); + expect(disconnected.snapshot.get().context.status).toBe('error'); + expect(unconnectable.closed).toBe(true); + disconnected.destroy(); + }); +}); diff --git a/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts b/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts new file mode 100644 index 0000000000..d92ef39dc7 --- /dev/null +++ b/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts @@ -0,0 +1,571 @@ +/** + * AudioWorklet renderer actor over the shared audio player core: `@mux/moq-media`'s `createAudioPlayer`, injected as + * `createAudioPlayer`. Same surface as `createAudioRendererActor`, frames in from the subscriber's jitter buffer and + * the **master clock** out, so the engine can run either behind one seam. + * + * The core decodes with WebCodecs, holds an adaptive jitter buffer in an `AudioWorklet`, and reports the audible + * timestamp about every 20 ms. **Latency stays v10's.** The renderer meters frames into the core so the worklet holds + * about its own jitter target, and the rest of the target latency stays in the subscriber's buffer, where `syncLatency` + * measures and steers it. The core's adaptive target is left alone: under metering its growth only moves depth from one + * buffer to the other. + * + * The clock the core reports into is the renderer's own, not the SDK's `PlayoutClock`, so `getClockTimeUs` keeps the + * contract the video renderer and `trackPlayoutTime` rely on: `undefined` before any audio, and **held where the + * worklet ran dry** rather than paced on by wall time — a wall-paced clock steps back when audio resumes, which holds + * slaved video on its last frame for the whole step. + * + * TODO(mmcc/v10#66): apply `getPlaybackRate` once the core has a per-input rate (muxinc/relay-sdks#329). Until then + * audio ignores the latency controller's nudges, which is why the engine only uses this renderer when a + * `createAudioPlayer` is configured. + */ +import { createTransitionActor } from '../../../core/actors/create-transition-actor'; +import { TIMELINE_DISCONTINUITY_US } from '../../../media/moq/timeline'; +import type { + AudioContextLike, + AudioFrameSource, + AudioRendererActor, + AudioRendererContext, + AudioRendererStatus, +} from './audio-renderer'; + +// ============================================================================= +// Types +// ============================================================================= + +/** + * The part of `@mux/moq-media`'s `PlayoutClock` an audio player input calls. The renderer implements it, so the input + * reports into the renderer's clock. + */ +export interface AudioPlayerClock { + /** Caps the input's jitter target: the core keeps it under this less the output delay and 40 ms. */ + readonly targetLatencyUs: number; + /** The time source reports are measured on, in microseconds. */ + localNow(): number; + /** The audio with this timestamp is audible at local time `atUs`. */ + reportAudio(timestampUs: number, atUs?: number): void; + /** The input stopped. */ + stopAudio(): void; +} + +/** The part of `@mux/moq-media`'s `AudioInputOptions` the renderer passes. */ +export interface AudioPlayerInputOptions { + clock?: AudioPlayerClock; + decoderConfig?: AudioDecoderConfig; + /** A decoder error. The core decodes again from the next frame, with a new decoder. */ + onError?(cause: unknown): void; +} + +/** The part of `@mux/moq-media`'s `AudioInputStats` the renderer reads. Durations are microseconds. */ +export interface AudioPlayerInputStats { + /** Whether the worklet is playing, rather than filling to its target. */ + readonly playing: boolean; + /** Decoded audio the worklet holds. */ + readonly bufferedUs: number; + /** How much the worklet holds before it plays, at the start and after running dry. */ + readonly jitterTargetUs: number; +} + +/** The part of `@mux/moq-media`'s `AudioPlayerInput` the renderer uses: one track. */ +export interface AudioPlayerInput { + /** Decode and hold a frame, in decode order. */ + push(frame: { timestampUs: number; payload: Uint8Array }): void; + /** Without a range: drop everything held or still decoding. */ + cut(): void; + readonly stats: AudioPlayerInputStats; + remove(): void; +} + +/** The part of `@mux/moq-media`'s `AudioPlayer` the renderer uses. */ +export interface AudioPlayer { + /** The mix of every input; the renderer connects it to its context's `destination`. */ + readonly output: { connect(destination: AudioNode): void }; + addInput(options?: AudioPlayerInputOptions): AudioPlayerInput; + /** Remove every input and disconnect the output. */ + close(): void; +} + +/** + * Create the player on the renderer's context. `@mux/moq-media`'s `createAudioPlayer` qualifies once given a real + * `AudioContext`, which a wrapping context exposes as `baseContext`: + * + * ```ts + * createAudioPlayer: (context) => createAudioPlayer({ context: context.baseContext as AudioContext }); + * ``` + */ +export type CreateAudioPlayer = (context: AudioContextLike) => Promise; + +export interface CreateWorkletAudioRendererOptions { + /** Where the player's output connects (`destination`), and the output latencies the clock is placed behind. */ + audioContext: AudioContextLike; + createAudioPlayer: CreateAudioPlayer; + /** + * **Join at the live edge**, as `createAudioRendererActor`'s option of the same name: buffered audio older than this + * is discarded before the first frame of a (re)start reaches the core. + */ + getJoinAnchorUs?: () => number | undefined; + /** + * The target latency in microseconds, read on every clock report. It caps the core's jitter target, so the worklet + * never holds more than the latency the controller is aiming at. Default 2 s, where the core's own 400 ms ceiling + * applies instead. + */ + getTargetLatencyUs?: () => number | undefined; + /** Metering cadence in ms. Default 10. */ + tickIntervalMs?: number; +} + +type RendererMessage = + | { type: 'status'; status: AudioRendererStatus; error?: unknown } + | { type: 'decoder-error'; error: unknown } + | { type: 'scheduled'; untilUs: number } + | { type: 'underrun' }; + +// ============================================================================= +// Implementation +// ============================================================================= + +const DEFAULT_TICK_INTERVAL_MS = 10; +const DEFAULT_TARGET_LATENCY_US = 2_000_000; +/** The frame duration assumed before a frame has shown one: Opus's usual 20 ms. */ +const DEFAULT_FRAME_DURATION_US = 20_000; +/** Reports and pushed spans may disagree by sample rounding and resampling offsets. */ +const REPORT_TOLERANCE_US = 5_000; +/** + * How long a fill may sit short of playback before one more frame is pushed. The fill pushes exactly the jitter target + * plus a frame, so audio the core drops on arrival (behind what it already decoded) could otherwise leave it waiting + * forever. + */ +const FILL_STALL_US = 100_000; +/** See `audio-renderer`'s threshold of the same name: a jump this large, either way, is a new timeline. */ +const DISCONTINUITY_THRESHOLD_US = TIMELINE_DISCONTINUITY_US; + +/** + * How long an Opus packet plays, from its TOC byte (RFC 6716 §3.1); `undefined` when it cannot be read. The same rule + * as `@mux/moq-media`'s `opusPacketDurationUs`. + */ +function opusPacketDurationUs(packet: Uint8Array): number | undefined { + const toc = packet[0]; + if (toc === undefined) return undefined; + + const config = toc >> 3; + // SILK: 10, 20, 40, 60 ms; hybrid: 10, 20 ms; CELT: 2.5, 5, 10, 20 ms. + const frameUs = + config < 12 + ? [10_000, 20_000, 40_000, 60_000][config % 4]! + : config < 16 + ? [10_000, 20_000][config % 2]! + : [2_500, 5_000, 10_000, 20_000][config % 4]!; + const code = toc & 3; + if (code === 0) return frameUs; + + if (code !== 3) return 2 * frameUs; + + const count = packet[1]; + + return count === undefined ? undefined : (count & 0x3f) * frameUs; +} + +export function createWorkletAudioRendererActor(options: CreateWorkletAudioRendererOptions): AudioRendererActor { + const { audioContext } = options; + + const inner = createTransitionActor( + { status: 'idle', framesScheduled: 0, underruns: 0 }, + (context, message) => { + switch (message.type) { + case 'status': + return { ...context, status: message.status, error: message.error }; + case 'decoder-error': + return { ...context, error: message.error }; + case 'scheduled': + return { + ...context, + status: 'rendering', + framesScheduled: context.framesScheduled + 1, + scheduledUntilUs: message.untilUs, + }; + case 'underrun': + return { ...context, underruns: context.underruns + 1 }; + } + } + ); + + let destroyed = false; + let player: AudioPlayer | undefined; + let input: AudioPlayerInput | undefined; + let source: AudioFrameSource | null = null; + let decoderConfig: AudioDecoderConfig | null = null; + + /** The schedule has no anchor yet; see `audio-renderer`'s flag of the same name. */ + let needsJoinAnchor = true; + /** The pending anchor follows a mid-stream discontinuity; see `audio-renderer`'s flag of the same name. */ + let rejoinAnchorArmed = false; + + // What this timeline has pushed into the core. Reset with the timeline. + let firstPushedUs: number | undefined; + let lastPushedUs: number | undefined; + /** Known once a packet's duration or two frames' spacing has shown it. */ + let frameDurationUs: number | undefined; + /** Where the newest pushed frame ends: the furthest the clock can read. */ + let pushedEndUs: number | undefined; + /** First timestamp pushed in the current fill, and the context time of the latest push. */ + let fillStartUs: number | undefined; + let lastPushAtUs = 0; + + /** + * The clock: timeline position `mediaUs` is audible at context time `contextUs`, advancing with the context clock + * from there. Set by every core report, or **placed** at a re-join anchor before the core plays anything (see + * `dropToJoinAnchor`). + * + * Context time, not `performance.now()`, because it is what stops when the adapter suspends the context on pause: a + * wall-paced clock would run on through the pause, read the frozen worklet as dry and refill it, and step back on the + * first report after resume. The core times its reports on the same source (`localNow`), so its render clock's + * calibration does not carry a pause into the first reports after it either. + */ + let anchor: { mediaUs: number; contextUs: number; placed: boolean; frozen?: boolean } | undefined; + /** Bumped per input, so a report from a removed input cannot anchor its replacement. */ + let inputGeneration = 0; + /** + * Where the worklet ran dry. The clock holds there until a report of audio past it arrives: frames pushed to refill + * the worklet move `pushedEndUs` before they play, and must not drag the clock forward with them. + */ + let dryLimitUs: number | undefined; + /** + * The position the clock held at through the latest underrun, kept as a floor after it ends. The first report of the + * refill is dated ahead by the output latency — the sample is not audible yet — so its anchor extrapolates to before + * the hold, and the clock would otherwise step back by that much. + */ + let holdFloorUs: number | undefined; + /** + * The core has reported itself stopped since the dry point was set. Until it has, a report near the dry point is the + * tail of the audio before it, still in flight, and must not end the hold. + */ + let stoppedSinceDry = false; + + const outputLatencyUs = (): number => + ((audioContext.baseLatency ?? 0) + (audioContext.outputLatency ?? 0)) * 1_000_000; + + const contextNowUs = (): number => audioContext.currentTime * 1_000_000; + + /** The audible position extrapolated from the anchor, ignoring every limit. */ + const extrapolatedUs = (): number | undefined => + anchor && (anchor.frozen ? anchor.mediaUs : anchor.mediaUs + (contextNowUs() - anchor.contextUs)); + + /** The clock one input reports into; it goes deaf once that input is replaced. */ + const createClock = (): AudioPlayerClock => { + const generation = ++inputGeneration; + + return { + get targetLatencyUs() { + return options.getTargetLatencyUs?.() ?? DEFAULT_TARGET_LATENCY_US; + }, + localNow: contextNowUs, + reportAudio(timestampUs, atUs) { + if (generation !== inputGeneration) return; + + reportAudio(timestampUs, atUs); + }, + // The renderer resets the clock itself wherever a track or timeline ends. + stopAudio() {}, + }; + }; + + const raiseHoldFloor = (heldUs: number): void => { + holdFloorUs = Math.max(holdFloorUs ?? heldUs, heldUs); + }; + + /** + * A placed clock runs from the anchor until the core plays, but never past the audio pushed: a fill that stays short + * of the jitter target cannot start the worklet, and the clock would otherwise run on through audio not there. + */ + const placedPositionUs = (): number | undefined => { + const positionUs = extrapolatedUs(); + + return positionUs === undefined || pushedEndUs === undefined ? positionUs : Math.min(positionUs, pushedEndUs); + }; + + /** `timestampUs` is audible at context time `atUs` (default now). */ + const reportAudio = (timestampUs: number, atUs: number | undefined): void => { + // Only audio pushed on this timeline: a report already in flight when + // the timeline restarted describes audio the cut dropped. + if ( + firstPushedUs === undefined || + pushedEndUs === undefined || + timestampUs < firstPushedUs - REPORT_TOLERANCE_US || + timestampUs > pushedEndUs + REPORT_TOLERANCE_US + ) { + return; + } + + // The refill's first report ends the hold. Its floor is what the clock + // reads at that moment — the audible position, which can still be short + // of the dry point while the device plays out what it had queued. + if ( + dryLimitUs !== undefined && + ((stoppedSinceDry && timestampUs >= dryLimitUs - REPORT_TOLERANCE_US) || + timestampUs > dryLimitUs + REPORT_TOLERANCE_US) + ) { + raiseHoldFloor(Math.min(extrapolatedUs() ?? dryLimitUs, dryLimitUs)); + dryLimitUs = undefined; + } + + // Likewise the first report after a placed re-join, which is dated ahead + // of the placed clock by the output latency. + if (anchor?.placed) raiseHoldFloor(placedPositionUs()!); + + anchor = { mediaUs: timestampUs, contextUs: atUs ?? contextNowUs(), placed: false }; + }; + + /** Forget everything pushed on the current timeline, and the clock placed on it. */ + const resetTimeline = (): void => { + firstPushedUs = undefined; + lastPushedUs = undefined; + frameDurationUs = undefined; + pushedEndUs = undefined; + fillStartUs = undefined; + anchor = undefined; + dryLimitUs = undefined; + holdFloorUs = undefined; + stoppedSinceDry = false; + }; + + const removeInput = (): void => { + if (!input) return; + + inputGeneration++; + input.remove(); + input = undefined; + }; + + /** + * Discard buffered audio older than the join anchor before the first push of a (re)start, so playout starts at the + * live edge rather than at the oldest group the relay replayed. Mirrors `audio-renderer`'s function of the same name, + * including landing a re-join anchor by **placement** when the whole buffer is already past it: the clock is placed + * at the anchor and runs from there, and metering by timeline holds the head back until it is due, so playout resumes + * at the target depth rather than at the live edge. + */ + const dropToJoinAnchor = (): void => { + const anchorUs = options.getJoinAnchorUs?.(); + if (anchorUs === undefined || !source) return; + + let dropped = 0; + + for (let next = source.peek(); next && next.timestampUs < anchorUs; next = source.peek()) { + source.dequeue(); + dropped++; + } + + if (dropped > 0) { + needsJoinAnchor = false; + rejoinAnchorArmed = false; + return; + } + + // A fresh join stays armed: a relay's group replay arrives as a burst, + // oldest first, so the buffer's live edge may still be moving. + if (!rejoinAnchorArmed || !source.peek()) return; + + anchor = { mediaUs: anchorUs, contextUs: contextNowUs(), placed: true }; + rejoinAnchorArmed = false; + needsJoinAnchor = false; + }; + + /** + * Whether `timestampUs` should reach the core now. Filling — before the first report, or after running dry — pushes + * the jitter target plus a frame, which is what the worklet waits for before playing. Playing pushes what will be + * rendered within the jitter target, so the worklet holds about that much and the rest waits in the subscriber. + */ + const isDue = (timestampUs: number, stats: AudioPlayerInputStats): boolean => { + const positionUs = anchor?.placed ? placedPositionUs() : extrapolatedUs(); + + // A frozen placement is a fill too: its clock waits for the worklet to + // start, so only the fill rules (including the stall rescue) can move it. + if (positionUs === undefined || dryLimitUs !== undefined || anchor?.frozen) { + if ( + fillStartUs === undefined || + timestampUs < fillStartUs + stats.jitterTargetUs + (frameDurationUs ?? DEFAULT_FRAME_DURATION_US) + ) { + return true; + } + + return contextNowUs() - lastPushAtUs > FILL_STALL_US && stats.bufferedUs < stats.jitterTargetUs; + } + + return timestampUs < positionUs + outputLatencyUs() + stats.jitterTargetUs; + }; + + const push = (active: AudioPlayerInput, timestampUs: number, payload: Uint8Array): void => { + active.push({ timestampUs, payload }); + + // Opus says how long each packet is. Otherwise the shortest step between + // pushed frames is the frame length: a step over lost packets is longer, + // and must not extend the audio past where it really ends. + const opusUs = decoderConfig?.codec === 'opus' ? opusPacketDurationUs(payload) : undefined; + const stepUs = lastPushedUs === undefined ? undefined : timestampUs - lastPushedUs; + + if (opusUs !== undefined) frameDurationUs = opusUs; + else if (stepUs !== undefined && stepUs > 0) frameDurationUs = Math.min(frameDurationUs ?? stepUs, stepUs); + + firstPushedUs ??= timestampUs; + fillStartUs ??= timestampUs; + lastPushedUs = timestampUs; + lastPushAtUs = contextNowUs(); + pushedEndUs = timestampUs + (frameDurationUs ?? DEFAULT_FRAME_DURATION_US); + inner.send({ type: 'scheduled', untilUs: pushedEndUs }); + }; + + /** + * The worklet runs dry once rendering passes everything pushed: metering pushes every frame as soon as it is due, so + * nothing was left to push. Each such edge is an underrun, as `audio-renderer` counts them. The core's own counter + * skips dry spells longer than its jitter ceiling, which `adaptLatencyTarget` still needs as failure evidence. + */ + const detectDry = (): void => { + const positionUs = extrapolatedUs(); + + if ( + !anchor || + anchor.placed || + dryLimitUs !== undefined || + pushedEndUs === undefined || + positionUs === undefined || + positionUs + outputLatencyUs() < pushedEndUs + ) { + return; + } + + dryLimitUs = pushedEndUs; + stoppedSinceDry = false; + fillStartUs = undefined; + inner.send({ type: 'underrun' }); + }; + + const tick = (): void => { + if (destroyed || !player || !source || !decoderConfig) return; + + try { + const active = (input ??= player.addInput({ + clock: createClock(), + decoderConfig, + onError: (error) => inner.send({ type: 'decoder-error', error }), + })); + const { stats } = active; + + detectDry(); + + const positionUs = extrapolatedUs(); + + if (dryLimitUs !== undefined && !stats.playing) stoppedSinceDry = true; + + // A placed clock that reaches the end of the audio pushed stops there + // until the core plays, so refill frames cannot drag it forward. + if (anchor?.placed && !anchor.frozen && pushedEndUs !== undefined && positionUs! >= pushedEndUs) { + anchor = { mediaUs: pushedEndUs, contextUs: contextNowUs(), placed: true, frozen: true }; + } + + for (;;) { + if (needsJoinAnchor && lastPushedUs === undefined) dropToJoinAnchor(); + + const next = source.peek(); + if (!next) return; + + // A timeline reset, either way: drop what the core holds and re-place + // the anchor on the jumped-to timeline (see `audio-renderer`). + if (lastPushedUs !== undefined && Math.abs(next.timestampUs - lastPushedUs) > DISCONTINUITY_THRESHOLD_US) { + active.cut(); + resetTimeline(); + needsJoinAnchor = true; + rejoinAnchorArmed = true; + continue; + } + + if (!isDue(next.timestampUs, stats)) return; + + push(active, next.timestampUs, next.payload); + source.dequeue(); + } + } catch (error) { + inner.send({ type: 'status', status: 'error', error }); + } + }; + const timer = setInterval(tick, options.tickIntervalMs ?? DEFAULT_TICK_INTERVAL_MS); + + options.createAudioPlayer(audioContext).then( + (created) => { + if (destroyed) { + created.close(); + return; + } + + try { + created.output.connect(audioContext.destination); + player = created; + } catch (error) { + created.close(); + inner.send({ type: 'status', status: 'error', error }); + } + }, + (error) => { + if (!destroyed) inner.send({ type: 'status', status: 'error', error }); + } + ); + + return { + get snapshot() { + return inner.snapshot; + }, + + setTrack(nextSource, nextConfig): void { + // See `audio-renderer`: a source with no config could never play. + const missingConfig = nextSource !== null && nextConfig === null; + + source = missingConfig ? null : nextSource; + decoderConfig = nextConfig; + needsJoinAnchor = true; + rejoinAnchorArmed = false; + // An edge since the last tick still counts: the reset below forgets it. + detectDry(); + removeInput(); + resetTimeline(); + + if (missingConfig) { + inner.send({ + type: 'status', + status: 'error', + error: new Error('audio renderer track has no decoder config'), + }); + return; + } + + inner.send({ type: 'status', status: source ? 'rendering' : 'idle' }); + }, + + getClockTimeUs(): number | undefined { + const positionUs = extrapolatedUs(); + if (positionUs === undefined) return undefined; + + if (anchor?.placed) return placedPositionUs(); + + // Never past the audio pushed, and once the worklet ran dry never past + // where it did: the clock holds through an underrun and resumes when + // audio does, as the source-node renderer's clamped segments did. + const limitUs = dryLimitUs ?? pushedEndUs; + const clampedUs = limitUs === undefined ? positionUs : Math.min(positionUs, limitUs); + + if (holdFloorUs === undefined) return clampedUs; + + if (clampedUs < holdFloorUs) return holdFloorUs; + + holdFloorUs = undefined; + return clampedUs; + }, + + destroy(): void { + if (destroyed) return; + + destroyed = true; + clearInterval(timer); + removeInput(); + player?.close(); + player = undefined; + inner.destroy(); + }, + }; +} diff --git a/packages/spf/src/playback/behaviors/dom/setup-moq-renderers.ts b/packages/spf/src/playback/behaviors/dom/setup-moq-renderers.ts index c3d224c6b2..3613b4988d 100644 --- a/packages/spf/src/playback/behaviors/dom/setup-moq-renderers.ts +++ b/packages/spf/src/playback/behaviors/dom/setup-moq-renderers.ts @@ -46,6 +46,7 @@ import { createAudioRendererActor, } from '../../actors/dom/audio-renderer'; import { createVideoRendererActor, type VideoRendererActor } from '../../actors/dom/video-renderer'; +import { type CreateAudioPlayer, createWorkletAudioRendererActor } from '../../actors/dom/worklet-audio-renderer'; import type { TrackSubscriberActor } from '../../actors/track-subscriber'; import { type LatencyControlConfig, type PlayoutClockOwner, resolveLatencyControlConfig } from '../sync-latency'; @@ -93,6 +94,25 @@ const CLOCK_PUBLISH_INTERVAL_MS = 100; export interface MoqRendererConfig { /** Shared with `syncLatency`: `joinAtEdge`, the target the clocks aim at, and the slew bounds. */ latency?: Partial; + /** + * Render audio through the shared AudioWorklet player core instead of scheduled `AudioBufferSourceNode`s (see + * `worklet-audio-renderer`). Opt-in until the core can apply `playoutRate` nudges (mmcc/v10#66). + */ + createAudioPlayer?: CreateAudioPlayer; +} + +/** The target latency `syncLatency` resolves for `subscriber`'s track, in seconds. */ +function resolveTrackTargetSeconds( + subscriber: TrackSubscriberActor | undefined, + targetLatencySignal: ReadonlySignal, + adaptiveTargetLatencySignal: ReadonlySignal, + latency: LatencyControlConfig +): number { + return resolveTargetLatencySeconds( + preferredTargetLatencySeconds(peek(targetLatencySignal), peek(adaptiveTargetLatencySignal)), + subscriber?.track.moq.targetLatency, + latency.defaultTargetLatency + ); } /** @@ -116,10 +136,11 @@ function makeEdgeTargetUs( // slots — the clocks and the controller must aim at one number, and // the adaptive proposal has to reach both or the slew would chase an // edge the controller is not steering toward. - const targetSeconds = resolveTargetLatencySeconds( - preferredTargetLatencySeconds(peek(targetLatencySignal), peek(adaptiveTargetLatencySignal)), - subscriber?.track.moq.targetLatency, - latency.defaultTargetLatency + const targetSeconds = resolveTrackTargetSeconds( + subscriber, + targetLatencySignal, + adaptiveTargetLatencySignal, + latency ); return joinAnchorUs(newestTimestampUs, targetSeconds); @@ -174,10 +195,11 @@ function makeAudioJoinAnchor( // Same resolution `syncLatency` and `makeEdgeTargetUs` perform, from // the same input slots — the clocks and the controller aim at one number. - const targetSeconds = resolveTargetLatencySeconds( - preferredTargetLatencySeconds(peek(targetLatencySignal), peek(adaptiveTargetLatencySignal)), - subscriber?.track.moq.targetLatency, - latency.defaultTargetLatency + const targetSeconds = resolveTrackTargetSeconds( + subscriber, + targetLatencySignal, + adaptiveTargetLatencySignal, + latency ); const anchorUs = joinAnchorUs(newestTimestampUs, targetSeconds); @@ -224,22 +246,39 @@ function setupAudioRendererSetup({ 'preconditions-unmet': {}, 'renderer-active': { entry: () => { - const renderer = createAudioRendererActor({ - audioContext: context.audioContext.get()!, - // No paused gating here: the adapter suspends the - // AudioContext on pause, which freezes the hardware clock and - // every scheduled source. Rate 0 would instead produce an - // infinite clock segment (duration ÷ 0) and sources whose - // `playbackRate` stays 0 after resume — a permanent stall. - getPlaybackRate: () => peek(state.playoutRate) ?? 1, - getJoinAnchorUs: makeAudioJoinAnchor( - context.audioSubscriberActor, - state.targetLatency, - state.adaptiveTargetLatency, - latencyConfig, - context.videoRendererActor - ), - }); + const audioContext = context.audioContext.get()!; + const getJoinAnchorUs = makeAudioJoinAnchor( + context.audioSubscriberActor, + state.targetLatency, + state.adaptiveTargetLatency, + latencyConfig, + context.videoRendererActor + ); + const renderer = config?.createAudioPlayer + ? createWorkletAudioRendererActor({ + audioContext, + createAudioPlayer: config.createAudioPlayer, + getJoinAnchorUs, + // Caps the worklet's jitter target, so the core never holds + // more than the latency the controller is aiming at. + getTargetLatencyUs: () => + resolveTrackTargetSeconds( + peek(context.audioSubscriberActor), + state.targetLatency, + state.adaptiveTargetLatency, + latencyConfig + ) * 1_000_000, + }) + : createAudioRendererActor({ + audioContext, + // No paused gating here: the adapter suspends the + // AudioContext on pause, which freezes the hardware clock and + // every scheduled source. Rate 0 would instead produce an + // infinite clock segment (duration ÷ 0) and sources whose + // `playbackRate` stays 0 after resume — a permanent stall. + getPlaybackRate: () => peek(state.playoutRate) ?? 1, + getJoinAnchorUs, + }); context.audioRendererActor.set(renderer); return () => { diff --git a/packages/spf/src/playback/behaviors/dom/tests/setup-moq-renderers.test.ts b/packages/spf/src/playback/behaviors/dom/tests/setup-moq-renderers.test.ts index 136ea9229e..f3bb440edb 100644 --- a/packages/spf/src/playback/behaviors/dom/tests/setup-moq-renderers.test.ts +++ b/packages/spf/src/playback/behaviors/dom/tests/setup-moq-renderers.test.ts @@ -5,6 +5,7 @@ import type { AudioContextLike, AudioRendererActor } from '../../../actors/dom/a import { createAudioRendererActor } from '../../../actors/dom/audio-renderer'; import type { VideoRendererActor } from '../../../actors/dom/video-renderer'; import { createVideoRendererActor } from '../../../actors/dom/video-renderer'; +import { createWorkletAudioRendererActor } from '../../../actors/dom/worklet-audio-renderer'; import type { TrackSubscriberActor } from '../../../actors/track-subscriber'; import type { PlayoutClockOwner } from '../../sync-latency'; import { @@ -23,6 +24,9 @@ vi.mock('../../../actors/dom/audio-renderer', () => ({ vi.mock('../../../actors/dom/video-renderer', () => ({ createVideoRendererActor: vi.fn(), })); +vi.mock('../../../actors/dom/worklet-audio-renderer', () => ({ + createWorkletAudioRendererActor: vi.fn(), +})); function makeFakeAudioRenderer(): AudioRendererActor { return { @@ -91,6 +95,29 @@ describe('setupAudioRenderer', () => { reactor.destroy(); }); + it('renders through the worklet core when a createAudioPlayer is configured', async () => { + vi.mocked(createWorkletAudioRendererActor).mockImplementation(() => makeFakeAudioRenderer()); + const createAudioPlayer = vi.fn(); + const { state, context, reactor } = setupSetupAudioRenderer({ createAudioPlayer }); + + await vi.waitFor(() => expect(createWorkletAudioRendererActor).toHaveBeenCalledTimes(1)); + expect(createAudioRendererActor).not.toHaveBeenCalled(); + + const options = vi.mocked(createWorkletAudioRendererActor).mock.calls[0]![0]; + + expect(options.createAudioPlayer).toBe(createAudioPlayer); + expect(options.audioContext).toBe(context.audioContext.get()); + + // The core's jitter ceiling follows the same target the controller holds. + context.audioSubscriberActor.set(makeFakeSubscriber(10_000_000)); + expect(options.getTargetLatencyUs!()).toBe(500_000); + state.targetLatency.set(2); + expect(options.getTargetLatencyUs!()).toBe(2_000_000); + expect(options.getJoinAnchorUs!()).toBe(8_000_000); + + reactor.destroy(); + }); + it('anchors the renderer at the live edge of its own jitter buffer', async () => { vi.mocked(createAudioRendererActor).mockImplementation(() => makeFakeAudioRenderer()); const { state, context, reactor } = setupSetupAudioRenderer(); diff --git a/packages/spf/src/playback/engines/moq/adapter.ts b/packages/spf/src/playback/engines/moq/adapter.ts index 87574d5cbc..9ffaf64dfa 100644 --- a/packages/spf/src/playback/engines/moq/adapter.ts +++ b/packages/spf/src/playback/engines/moq/adapter.ts @@ -330,6 +330,12 @@ export function MoqMediaMixin>( gain.connect(audioContext.destination); gain.gain.value = this.#muted ? 0 : this.#volume; + // The worklet renderer's core constructs an `AudioWorkletNode`, which + // only a real context can host; an injected fake has none to offer. + const baseContext = + audioContext.baseContext ?? + (globalThis.BaseAudioContext && audioContext instanceof globalThis.BaseAudioContext ? audioContext : undefined); + return { get currentTime() { return audioContext.currentTime; @@ -337,6 +343,13 @@ export function MoqMediaMixin>( get destination() { return gain; }, + get baseLatency() { + return audioContext.baseLatency; + }, + get outputLatency() { + return audioContext.outputLatency; + }, + baseContext, createBuffer: (numberOfChannels, length, sampleRate) => audioContext.createBuffer(numberOfChannels, length, sampleRate), createBufferSource: () => audioContext.createBufferSource(), diff --git a/packages/spf/src/playback/engines/moq/engine.ts b/packages/spf/src/playback/engines/moq/engine.ts index b6660e0a53..bc64fc0787 100644 --- a/packages/spf/src/playback/engines/moq/engine.ts +++ b/packages/spf/src/playback/engines/moq/engine.ts @@ -11,6 +11,7 @@ import type { BandwidthConfig, BandwidthState } from '../../../network/bandwidth import type { RetryBackoffConfig } from '../../../network/retry-backoff'; import type { AudioContextLike, AudioRendererActor } from '../../actors/dom/audio-renderer'; import type { VideoRendererActor } from '../../actors/dom/video-renderer'; +import type { CreateAudioPlayer } from '../../actors/dom/worklet-audio-renderer'; import type { CreateMoqTransport, MoqAuthProvider, MoqSessionActor } from '../../actors/moq-session'; import type { TrackSubscriberActor } from '../../actors/track-subscriber'; import { type AdaptiveLatencyConfig, adaptLatencyTarget } from '../../behaviors/adapt-latency-target'; @@ -153,6 +154,11 @@ export interface MoqEngineConfig extends ShareSignalsConfig; /** Latency tuning: the controller (`syncLatency`) plus the renderers' join-at-edge anchor. */ latency?: Partial; + /** + * Render audio through the shared AudioWorklet player core (`@mux/moq-media`'s `createAudioPlayer`) instead of + * scheduled `AudioBufferSourceNode`s. Opt-in: the core cannot apply `playoutRate` nudges yet (mmcc/v10#66). + */ + createAudioPlayer?: CreateAudioPlayer; /** * Adaptive-latency tuning (`adaptLatencyTarget`), including its `enabled` switch — **off by default**, so an * untouched engine holds the fixed setpoint `latency` describes and this behavior registers no timer. Deliberately