From 31f33919fd31163546129b13654f2826f1a4131a Mon Sep 17 00:00:00 2001 From: Matt McClure Date: Mon, 19 Oct 2026 18:07:23 +0000 Subject: [PATCH 1/7] feat(spf): opt-in MoQ audio renderer over the shared worklet audio core Adds createWorkletAudioRendererActor behind the AudioRendererActor seam, driving @mux/moq-media's createAudioPlayer through a structural interface injected as createAudioPlayer (engine config). Keeps the join-at-edge anchor (including re-join placement), a clock that holds where the worklet ran dry, and underrun counting from input stats; meters frames so the worklet holds its own jitter target and v10's latency stays in the subscriber buffer. Opt-in: the core has no playback rate yet (muxinc/relay-sdks#329), so playoutRate nudges do not reach audio on this path. Refs #66 --- .../src/playback/actors/dom/audio-renderer.ts | 9 + .../dom/tests/worklet-audio-renderer.test.ts | 302 ++++++++++++ .../actors/dom/worklet-audio-renderer.ts | 446 ++++++++++++++++++ .../behaviors/dom/setup-moq-renderers.ts | 87 +++- .../dom/tests/setup-moq-renderers.test.ts | 27 ++ .../spf/src/playback/engines/moq/adapter.ts | 12 + .../spf/src/playback/engines/moq/engine.ts | 6 + 7 files changed, 865 insertions(+), 24 deletions(-) create mode 100644 packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts create mode 100644 packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts diff --git a/packages/spf/src/playback/actors/dom/audio-renderer.ts b/packages/spf/src/playback/actors/dom/audio-renderer.ts index aa73951532..d870d131c4 100644 --- a/packages/spf/src/playback/actors/dom/audio-renderer.ts +++ b/packages/spf/src/playback/actors/dom/audio-renderer.ts @@ -36,6 +36,15 @@ export interface AudioFrameSource { export interface AudioContextLike { readonly currentTime: number; readonly destination: AudioNode; + /** Seconds between the graph and the audio subsystem; the worklet renderer places its clock behind it. */ + readonly baseLatency?: number; + /** Seconds from the audio subsystem to the speaker; read fresh, as it changes with the device. */ + readonly outputLatency?: number; + /** + * The real context behind a wrapping view such as the adapter's, for APIs that only accept one — an + * `AudioWorkletNode` cannot be constructed on a plain object. + */ + readonly baseContext?: BaseAudioContext; createBuffer(numberOfChannels: number, length: number, sampleRate: number): AudioBuffer; createBufferSource(): AudioBufferSourceNode; } diff --git a/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts b/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts new file mode 100644 index 0000000000..7213484dfc --- /dev/null +++ b/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts @@ -0,0 +1,302 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vite-plus/test'; + +import type { JitterFrame } from '../../track-subscriber'; +import type { AudioContextLike, AudioFrameSource } from '../audio-renderer'; +import { + type AudioPlayer, + type AudioPlayerInput, + type AudioPlayerInputOptions, + createWorkletAudioRendererActor, +} from '../worklet-audio-renderer'; + +const FRAME_US = 20_000; +const JITTER_TARGET_US = 60_000; +const TICK_MS = 10; +const CONFIG: AudioDecoderConfig = { codec: 'opus', sampleRate: 48_000, numberOfChannels: 1 }; + +function frameAt(timestampUs: number): JitterFrame { + return { groupId: timestampUs, objectId: 0, timestampUs, isKey: true, payload: new Uint8Array([timestampUs & 0xff]) }; +} + +/** Frames every 20 ms over `[fromUs, untilUs)`. */ +function framesBetween(fromUs: number, untilUs: number): JitterFrame[] { + const frames: JitterFrame[] = []; + + for (let timestampUs = fromUs; timestampUs < untilUs; timestampUs += FRAME_US) frames.push(frameAt(timestampUs)); + + return frames; +} + +function queueSource(frames: JitterFrame[] = []): AudioFrameSource & { queue: JitterFrame[] } { + const queue = [...frames]; + + return { queue, peek: () => queue[0], dequeue: () => queue.shift() }; +} + +interface FakeInput extends AudioPlayerInput { + options: AudioPlayerInputOptions; + pushed: number[]; + cuts: number; + removed: boolean; + stats: { underruns: number; bufferedUs: number; jitterTargetUs: number }; +} + +/** The core's surface with no audio behind it: pushes are recorded, and the test plays the worklet's reports. */ +function createFakePlayer() { + const inputs: FakeInput[] = []; + const connected: AudioNode[] = []; + let closed = false; + const player: AudioPlayer = { + output: { connect: (destination) => connected.push(destination) }, + addInput(options = {}) { + const input: FakeInput = { + options, + pushed: [], + cuts: 0, + removed: false, + stats: { underruns: 0, bufferedUs: 0, jitterTargetUs: JITTER_TARGET_US }, + push(frame) { + input.pushed.push(frame.timestampUs); + }, + cut() { + input.cuts++; + }, + remove() { + input.removed = true; + }, + }; + + inputs.push(input); + return input; + }, + close() { + closed = true; + }, + }; + + return { + player, + inputs, + connected, + get closed() { + return closed; + }, + /** The worklet reporting that the audio at `timestampUs` is audible now. */ + report(timestampUs: number, atUs: number) { + inputs[inputs.length - 1]!.options.clock!.reportAudio(timestampUs, atUs); + }, + }; +} + +// SAFETY: the renderer only hands `destination` to `output.connect`, which the fake records by identity. +const destination = {} as AudioNode; + +function fakeAudioContext(): AudioContextLike { + return { + currentTime: 0, + destination, + createBuffer: () => { + throw new Error('unused'); + }, + createBufferSource: () => { + throw new Error('unused'); + }, + }; +} + +describe('createWorkletAudioRendererActor', () => { + let nowUs = 0; + + beforeEach(() => { + vi.useFakeTimers(); + nowUs = 0; + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + /** A renderer whose player is ready and whose first input exists. */ + async function setup(options: { getJoinAnchorUs?: () => number | undefined } = {}) { + const fake = createFakePlayer(); + const renderer = createWorkletAudioRendererActor({ + audioContext: fakeAudioContext(), + createAudioPlayer: () => Promise.resolve(fake.player), + tickIntervalMs: TICK_MS, + now: () => nowUs, + ...options, + }); + + await vi.advanceTimersByTimeAsync(0); + return { fake, renderer }; + } + + const tick = () => vi.advanceTimersByTime(TICK_MS); + + it('connects the player output to the context destination', async () => { + const { fake, renderer } = await setup(); + + expect(fake.connected).toEqual([destination]); + expect(renderer.getClockTimeUs()).toBeUndefined(); + + renderer.destroy(); + expect(fake.closed).toBe(true); + }); + + it('fills to the jitter target plus a frame, then meters by the clock', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 1_000_000)); + + renderer.setTrack(source, CONFIG); + tick(); + + const input = fake.inputs[0]!; + + expect(input.options.decoderConfig).toEqual(CONFIG); + expect(input.pushed).toEqual([0, 20_000, 40_000, 60_000]); + expect(renderer.getClockTimeUs()).toBeUndefined(); + + // Playing: the worklet should hold about its target ahead of what renders. + fake.report(0, 0); + nowUs = 30_000; + tick(); + + expect(input.pushed.at(-1)).toBe(80_000); + expect(renderer.getClockTimeUs()).toBe(30_000); + expect(renderer.snapshot.get().context.scheduledUntilUs).toBe(100_000); + }); + + it('joins at the edge, dropping buffered audio behind the anchor unheard', async () => { + const { fake, renderer } = await setup({ getJoinAnchorUs: () => 200_000 }); + + renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); + tick(); + + expect(fake.inputs[0]!.pushed[0]).toBe(200_000); + }); + + it('holds the clock where the worklet ran dry, through the refill, until audio past it plays', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + + nowUs = 100_000; + tick(); + expect(renderer.getClockTimeUs()).toBe(80_000); + + // Audio comes back: the refill moves the pushed end, not the clock. + source.queue.push(...framesBetween(80_000, 400_000)); + nowUs = 300_000; + tick(); + expect(fake.inputs[0]!.pushed.slice(4)).toEqual([80_000, 100_000, 120_000, 140_000]); + expect(renderer.getClockTimeUs()).toBe(80_000); + + fake.report(80_000, 320_000); + nowUs = 330_000; + expect(renderer.getClockTimeUs()).toBe(90_000); + }); + + it('drops what the core holds at a timeline reset and ignores reports from before it', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + + // A publisher re-anchor: the timeline steps back by far more than a gap. + source.queue.push(...framesBetween(-5_000_000, -4_000_000)); + tick(); + + const input = fake.inputs[0]!; + + expect(input.cuts).toBe(1); + expect(input.pushed.at(4)).toBe(-5_000_000); + expect(renderer.getClockTimeUs()).toBeUndefined(); + + // Still in flight from the old timeline when the cut landed. + fake.report(40_000, 0); + expect(renderer.getClockTimeUs()).toBeUndefined(); + + fake.report(-5_000_000, 0); + expect(renderer.getClockTimeUs()).toBe(-5_000_000); + }); + + it('places a re-join whose whole buffer is past the anchor at its distance from it', async () => { + let anchorUs: number | undefined; + const { fake, renderer } = await setup({ getJoinAnchorUs: () => anchorUs }); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + + // A catch-up skip landed on a group 100 ms past where the anchor falls. + anchorUs = 4_900_000; + source.queue.push(...framesBetween(5_000_000, 6_000_000)); + tick(); + + const input = fake.inputs[0]!; + + expect(input.pushed).toHaveLength(4); + expect(renderer.getClockTimeUs()).toBe(4_900_000); + + nowUs = 50_000; + tick(); + expect(input.pushed.at(4)).toBe(5_000_000); + expect(renderer.getClockTimeUs()).toBe(4_950_000); + }); + + it('counts underruns across track changes', async () => { + const { fake, renderer } = await setup(); + + renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); + tick(); + fake.inputs[0]!.stats.underruns = 2; + tick(); + expect(renderer.snapshot.get().context.underruns).toBe(2); + + renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); + expect(fake.inputs[0]!.removed).toBe(true); + + tick(); + fake.inputs[1]!.stats.underruns = 1; + tick(); + expect(renderer.snapshot.get().context.underruns).toBe(3); + }); + + it('caps the core jitter target at the target latency', async () => { + const fake = createFakePlayer(); + const renderer = createWorkletAudioRendererActor({ + audioContext: fakeAudioContext(), + createAudioPlayer: () => Promise.resolve(fake.player), + getTargetLatencyUs: () => 250_000, + tickIntervalMs: TICK_MS, + }); + + await vi.advanceTimersByTimeAsync(0); + renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); + tick(); + + expect(fake.inputs[0]!.options.clock!.targetLatencyUs).toBe(250_000); + }); + + it('fails a track with no decoder config, and a player that cannot be created', async () => { + const { renderer } = await setup(); + + renderer.setTrack(queueSource(), null); + expect(renderer.snapshot.get().context.status).toBe('error'); + + const failing = createWorkletAudioRendererActor({ + audioContext: fakeAudioContext(), + createAudioPlayer: () => Promise.reject(new Error('no worklet')), + }); + + await vi.advanceTimersByTimeAsync(0); + expect(failing.snapshot.get().context.status).toBe('error'); + failing.destroy(); + }); +}); diff --git a/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts b/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts new file mode 100644 index 0000000000..0858355079 --- /dev/null +++ b/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts @@ -0,0 +1,446 @@ +/** + * AudioWorklet renderer actor over the shared audio player core: `@mux/moq-media`'s `createAudioPlayer`, injected as + * `createAudioPlayer`. Same surface as `createAudioRendererActor`, frames in from the subscriber's jitter buffer and + * the **master clock** out, so the engine can run either behind one seam. + * + * The core decodes with WebCodecs, holds an adaptive jitter buffer in an `AudioWorklet`, and reports the audible + * timestamp about every 20 ms. **Latency stays v10's.** The renderer meters frames into the core so the worklet holds + * about its own jitter target, and the rest of the target latency stays in the subscriber's buffer, where `syncLatency` + * measures and steers it. The core's adaptive target is left alone: under metering its growth only moves depth from one + * buffer to the other. + * + * The clock the core reports into is the renderer's own, not the SDK's `PlayoutClock`, so `getClockTimeUs` keeps the + * contract the video renderer and `trackPlayoutTime` rely on: `undefined` before any audio, and **held where the + * worklet ran dry** rather than paced on by wall time — a wall-paced clock steps back when audio resumes, which holds + * slaved video on its last frame for the whole step. + * + * TODO(mmcc/v10#66): apply `getPlaybackRate` once the core has a per-input rate (muxinc/relay-sdks#329). Until then + * audio ignores the latency controller's nudges, which is why the engine only uses this renderer when a + * `createAudioPlayer` is configured. + */ +import { createTransitionActor } from '../../../core/actors/create-transition-actor'; +import { TIMELINE_DISCONTINUITY_US } from '../../../media/moq/timeline'; +import type { + AudioContextLike, + AudioFrameSource, + AudioRendererActor, + AudioRendererContext, + AudioRendererStatus, +} from './audio-renderer'; + +// ============================================================================= +// Types +// ============================================================================= + +/** + * The part of `@mux/moq-media`'s `PlayoutClock` an audio player input calls. The renderer implements it, so the input + * reports into the renderer's clock. + */ +export interface AudioPlayerClock { + /** Caps the input's jitter target: the core keeps it under this less the output delay and 40 ms. */ + readonly targetLatencyUs: number; + /** The time source reports are measured on, in microseconds. */ + localNow(): number; + /** The audio with this timestamp is audible at local time `atUs`. */ + reportAudio(timestampUs: number, atUs?: number): void; + /** The input stopped. */ + stopAudio(): void; +} + +/** The part of `@mux/moq-media`'s `AudioInputOptions` the renderer passes. */ +export interface AudioPlayerInputOptions { + clock?: AudioPlayerClock; + decoderConfig?: AudioDecoderConfig; + /** A decoder error. The core decodes again from the next frame, with a new decoder. */ + onError?(cause: unknown): void; +} + +/** The part of `@mux/moq-media`'s `AudioInputStats` the renderer reads. Durations are microseconds. */ +export interface AudioPlayerInputStats { + /** Times the worklet ran dry while playing and audio came back within the jitter ceiling. Cumulative. */ + readonly underruns: number; + /** Decoded audio the worklet holds. */ + readonly bufferedUs: number; + /** How much the worklet holds before it plays, at the start and after running dry. */ + readonly jitterTargetUs: number; +} + +/** The part of `@mux/moq-media`'s `AudioPlayerInput` the renderer uses: one track. */ +export interface AudioPlayerInput { + /** Decode and hold a frame, in decode order. */ + push(frame: { timestampUs: number; payload: Uint8Array }): void; + /** Without a range: drop everything held or still decoding. */ + cut(): void; + readonly stats: AudioPlayerInputStats; + remove(): void; +} + +/** The part of `@mux/moq-media`'s `AudioPlayer` the renderer uses. */ +export interface AudioPlayer { + /** The mix of every input; the renderer connects it to its context's `destination`. */ + readonly output: { connect(destination: AudioNode): void }; + addInput(options?: AudioPlayerInputOptions): AudioPlayerInput; + /** Remove every input and disconnect the output. */ + close(): void; +} + +/** + * Create the player on the renderer's context. `@mux/moq-media`'s `createAudioPlayer` qualifies once given a real + * `AudioContext`, which a wrapping context exposes as `baseContext`: + * + * ```ts + * createAudioPlayer: (context) => createAudioPlayer({ context: context.baseContext as AudioContext }); + * ``` + */ +export type CreateAudioPlayer = (context: AudioContextLike) => Promise; + +export interface CreateWorkletAudioRendererOptions { + /** Where the player's output connects (`destination`), and the output latencies the clock is placed behind. */ + audioContext: AudioContextLike; + createAudioPlayer: CreateAudioPlayer; + /** + * **Join at the live edge**, as `createAudioRendererActor`'s option of the same name: buffered audio older than this + * is discarded before the first frame of a (re)start reaches the core. + */ + getJoinAnchorUs?: () => number | undefined; + /** + * The target latency in microseconds, read on every clock report. It caps the core's jitter target, so the worklet + * never holds more than the latency the controller is aiming at. Default 2 s, where the core's own 400 ms ceiling + * applies instead. + */ + getTargetLatencyUs?: () => number | undefined; + /** Metering cadence in ms. Default 10. */ + tickIntervalMs?: number; + /** Microsecond time source for the clock. Default `performance.now()`. */ + now?: () => number; +} + +type RendererMessage = + | { type: 'status'; status: AudioRendererStatus; error?: unknown } + | { type: 'decoder-error'; error: unknown } + | { type: 'scheduled'; untilUs: number } + | { type: 'underruns'; count: number }; + +// ============================================================================= +// Implementation +// ============================================================================= + +const DEFAULT_TICK_INTERVAL_MS = 10; +const DEFAULT_TARGET_LATENCY_US = 2_000_000; +/** The frame duration assumed before two contiguous frames have shown one: Opus's usual 20 ms. */ +const DEFAULT_FRAME_DURATION_US = 20_000; +/** Longest gap between two pushed frames that still measures a frame duration rather than a loss. */ +const MAX_FRAME_DURATION_US = 120_000; +/** Reports and pushed spans may disagree by sample rounding and resampling offsets. */ +const REPORT_TOLERANCE_US = 5_000; +/** + * How long a fill may sit short of playback before one more frame is pushed. The fill pushes exactly the jitter target + * plus a frame, so audio the core drops on arrival (behind what it already decoded) could otherwise leave it waiting + * forever. + */ +const FILL_STALL_US = 100_000; +/** See `audio-renderer`'s threshold of the same name: a jump this large, either way, is a new timeline. */ +const DISCONTINUITY_THRESHOLD_US = TIMELINE_DISCONTINUITY_US; + +const performanceNowUs = (): number => performance.now() * 1000; + +export function createWorkletAudioRendererActor(options: CreateWorkletAudioRendererOptions): AudioRendererActor { + const { audioContext } = options; + const now = options.now ?? performanceNowUs; + + const inner = createTransitionActor( + { status: 'idle', framesScheduled: 0, underruns: 0 }, + (context, message) => { + switch (message.type) { + case 'status': + return { ...context, status: message.status, error: message.error }; + case 'decoder-error': + return { ...context, error: message.error }; + case 'scheduled': + return { + ...context, + status: 'rendering', + framesScheduled: context.framesScheduled + 1, + scheduledUntilUs: message.untilUs, + }; + case 'underruns': + return { ...context, underruns: message.count }; + } + } + ); + + let destroyed = false; + let player: AudioPlayer | undefined; + let input: AudioPlayerInput | undefined; + let source: AudioFrameSource | null = null; + let decoderConfig: AudioDecoderConfig | null = null; + /** Underruns counted by inputs already removed: the context's counter spans the actor's life, not one track's. */ + let underrunsBefore = 0; + + /** The schedule has no anchor yet; see `audio-renderer`'s flag of the same name. */ + let needsJoinAnchor = true; + /** The pending anchor follows a mid-stream discontinuity; see `audio-renderer`'s flag of the same name. */ + let rejoinAnchorArmed = false; + + // What this timeline has pushed into the core. Reset with the timeline. + let firstPushedUs: number | undefined; + let lastPushedUs: number | undefined; + let frameDurationUs = DEFAULT_FRAME_DURATION_US; + /** Where the newest pushed frame ends: the furthest the clock can read. */ + let pushedEndUs: number | undefined; + /** First timestamp pushed in the current fill, and when the latest push was. */ + let fillStartUs: number | undefined; + let lastPushAtUs = 0; + + /** + * The clock: timeline position `mediaUs` was audible at local time `atUs`, advancing in real time from there. Set by + * every core report, or **placed** at a re-join anchor before the core plays anything (see `dropToJoinAnchor`). + */ + let anchor: { mediaUs: number; atUs: number; placed: boolean } | undefined; + /** + * Where the worklet ran dry. The clock holds there until a report of audio past it arrives: frames pushed to refill + * the worklet move `pushedEndUs` before they play, and must not drag the clock forward with them. + */ + let dryLimitUs: number | undefined; + + const outputLatencyUs = (): number => + ((audioContext.baseLatency ?? 0) + (audioContext.outputLatency ?? 0)) * 1_000_000; + + /** The audible position extrapolated from the anchor, ignoring every limit. */ + const extrapolatedUs = (): number | undefined => anchor && anchor.mediaUs + (now() - anchor.atUs); + + const clock: AudioPlayerClock = { + get targetLatencyUs() { + return options.getTargetLatencyUs?.() ?? DEFAULT_TARGET_LATENCY_US; + }, + localNow: now, + reportAudio(timestampUs, atUs) { + // Only audio pushed on this timeline: a report already in flight when + // the timeline restarted describes audio the cut dropped. + if ( + firstPushedUs === undefined || + pushedEndUs === undefined || + timestampUs < firstPushedUs - REPORT_TOLERANCE_US || + timestampUs > pushedEndUs + REPORT_TOLERANCE_US + ) { + return; + } + + anchor = { mediaUs: timestampUs, atUs: atUs ?? now(), placed: false }; + + if (dryLimitUs !== undefined && timestampUs >= dryLimitUs - REPORT_TOLERANCE_US) dryLimitUs = undefined; + }, + // The renderer resets the clock itself wherever a track or timeline ends. + stopAudio() {}, + }; + + /** Forget everything pushed on the current timeline, and the clock placed on it. */ + const resetTimeline = (): void => { + firstPushedUs = undefined; + lastPushedUs = undefined; + frameDurationUs = DEFAULT_FRAME_DURATION_US; + pushedEndUs = undefined; + fillStartUs = undefined; + anchor = undefined; + dryLimitUs = undefined; + }; + + const removeInput = (): void => { + if (!input) return; + + underrunsBefore += input.stats.underruns; + input.remove(); + input = undefined; + }; + + /** + * Discard buffered audio older than the join anchor before the first push of a (re)start, so playout starts at the + * live edge rather than at the oldest group the relay replayed. Mirrors `audio-renderer`'s function of the same name, + * including landing a re-join anchor by **placement** when the whole buffer is already past it: the clock is placed + * at the anchor and runs from there, and metering by timeline holds the head back until it is due, so playout resumes + * at the target depth rather than at the live edge. + */ + const dropToJoinAnchor = (): void => { + const anchorUs = options.getJoinAnchorUs?.(); + if (anchorUs === undefined || !source) return; + + let dropped = 0; + + for (let next = source.peek(); next && next.timestampUs < anchorUs; next = source.peek()) { + source.dequeue(); + dropped++; + } + + if (dropped > 0) { + needsJoinAnchor = false; + rejoinAnchorArmed = false; + return; + } + + // A fresh join stays armed: a relay's group replay arrives as a burst, + // oldest first, so the buffer's live edge may still be moving. + if (!rejoinAnchorArmed || !source.peek()) return; + + anchor = { mediaUs: anchorUs, atUs: now(), placed: true }; + rejoinAnchorArmed = false; + needsJoinAnchor = false; + }; + + /** + * Whether `timestampUs` should reach the core now. Filling — before the first report, or after running dry — pushes + * the jitter target plus a frame, which is what the worklet waits for before playing. Playing pushes what will be + * rendered within the jitter target, so the worklet holds about that much and the rest waits in the subscriber. + */ + const isDue = (timestampUs: number, stats: AudioPlayerInputStats): boolean => { + const positionUs = extrapolatedUs(); + + if (positionUs === undefined || dryLimitUs !== undefined) { + if (fillStartUs === undefined || timestampUs < fillStartUs + stats.jitterTargetUs + frameDurationUs) return true; + + return now() - lastPushAtUs > FILL_STALL_US && stats.bufferedUs < stats.jitterTargetUs; + } + + return timestampUs < positionUs + outputLatencyUs() + stats.jitterTargetUs; + }; + + const push = (active: AudioPlayerInput, timestampUs: number, payload: Uint8Array): void => { + active.push({ timestampUs, payload }); + + if (lastPushedUs !== undefined) { + const deltaUs = timestampUs - lastPushedUs; + + if (deltaUs > 0 && deltaUs <= MAX_FRAME_DURATION_US) frameDurationUs = deltaUs; + } + + firstPushedUs ??= timestampUs; + fillStartUs ??= timestampUs; + lastPushedUs = timestampUs; + lastPushAtUs = now(); + pushedEndUs = timestampUs + frameDurationUs; + inner.send({ type: 'scheduled', untilUs: pushedEndUs }); + }; + + const tick = (): void => { + if (destroyed || !player || !source || !decoderConfig) return; + + try { + const active = (input ??= player.addInput({ + clock, + decoderConfig, + onError: (error) => inner.send({ type: 'decoder-error', error }), + })); + const { stats } = active; + const underruns = underrunsBefore + stats.underruns; + + if (underruns !== inner.snapshot.get().context.underruns) inner.send({ type: 'underruns', count: underruns }); + + // The worklet runs dry once rendering passes everything pushed: metering + // pushes every frame as soon as it is due, so nothing was left to push. + const positionUs = extrapolatedUs(); + + if ( + anchor && + !anchor.placed && + dryLimitUs === undefined && + pushedEndUs !== undefined && + positionUs !== undefined && + positionUs + outputLatencyUs() >= pushedEndUs + ) { + dryLimitUs = pushedEndUs; + fillStartUs = undefined; + } + + for (;;) { + if (needsJoinAnchor && lastPushedUs === undefined) dropToJoinAnchor(); + + const next = source.peek(); + if (!next) return; + + // A timeline reset, either way: drop what the core holds and re-place + // the anchor on the jumped-to timeline (see `audio-renderer`). + if (lastPushedUs !== undefined && Math.abs(next.timestampUs - lastPushedUs) > DISCONTINUITY_THRESHOLD_US) { + active.cut(); + resetTimeline(); + needsJoinAnchor = true; + rejoinAnchorArmed = true; + continue; + } + + if (!isDue(next.timestampUs, stats)) return; + + push(active, next.timestampUs, next.payload); + source.dequeue(); + } + } catch (error) { + inner.send({ type: 'status', status: 'error', error }); + } + }; + const timer = setInterval(tick, options.tickIntervalMs ?? DEFAULT_TICK_INTERVAL_MS); + + options.createAudioPlayer(audioContext).then( + (created) => { + if (destroyed) { + created.close(); + return; + } + + created.output.connect(audioContext.destination); + player = created; + }, + (error) => { + if (!destroyed) inner.send({ type: 'status', status: 'error', error }); + } + ); + + return { + get snapshot() { + return inner.snapshot; + }, + + setTrack(nextSource, nextConfig): void { + // See `audio-renderer`: a source with no config could never play. + const missingConfig = nextSource !== null && nextConfig === null; + + source = missingConfig ? null : nextSource; + decoderConfig = nextConfig; + needsJoinAnchor = true; + rejoinAnchorArmed = false; + removeInput(); + resetTimeline(); + + if (missingConfig) { + inner.send({ + type: 'status', + status: 'error', + error: new Error('audio renderer track has no decoder config'), + }); + return; + } + + inner.send({ type: 'status', status: source ? 'rendering' : 'idle' }); + }, + + getClockTimeUs(): number | undefined { + const positionUs = extrapolatedUs(); + if (positionUs === undefined || anchor?.placed) return positionUs; + + // Never past the audio pushed, and once the worklet ran dry never past + // where it did: the clock holds through an underrun and resumes when + // audio does, as the source-node renderer's clamped segments did. + const limitUs = dryLimitUs ?? pushedEndUs; + + return limitUs === undefined ? positionUs : Math.min(positionUs, limitUs); + }, + + destroy(): void { + if (destroyed) return; + + destroyed = true; + clearInterval(timer); + removeInput(); + player?.close(); + player = undefined; + inner.destroy(); + }, + }; +} diff --git a/packages/spf/src/playback/behaviors/dom/setup-moq-renderers.ts b/packages/spf/src/playback/behaviors/dom/setup-moq-renderers.ts index c3d224c6b2..3613b4988d 100644 --- a/packages/spf/src/playback/behaviors/dom/setup-moq-renderers.ts +++ b/packages/spf/src/playback/behaviors/dom/setup-moq-renderers.ts @@ -46,6 +46,7 @@ import { createAudioRendererActor, } from '../../actors/dom/audio-renderer'; import { createVideoRendererActor, type VideoRendererActor } from '../../actors/dom/video-renderer'; +import { type CreateAudioPlayer, createWorkletAudioRendererActor } from '../../actors/dom/worklet-audio-renderer'; import type { TrackSubscriberActor } from '../../actors/track-subscriber'; import { type LatencyControlConfig, type PlayoutClockOwner, resolveLatencyControlConfig } from '../sync-latency'; @@ -93,6 +94,25 @@ const CLOCK_PUBLISH_INTERVAL_MS = 100; export interface MoqRendererConfig { /** Shared with `syncLatency`: `joinAtEdge`, the target the clocks aim at, and the slew bounds. */ latency?: Partial; + /** + * Render audio through the shared AudioWorklet player core instead of scheduled `AudioBufferSourceNode`s (see + * `worklet-audio-renderer`). Opt-in until the core can apply `playoutRate` nudges (mmcc/v10#66). + */ + createAudioPlayer?: CreateAudioPlayer; +} + +/** The target latency `syncLatency` resolves for `subscriber`'s track, in seconds. */ +function resolveTrackTargetSeconds( + subscriber: TrackSubscriberActor | undefined, + targetLatencySignal: ReadonlySignal, + adaptiveTargetLatencySignal: ReadonlySignal, + latency: LatencyControlConfig +): number { + return resolveTargetLatencySeconds( + preferredTargetLatencySeconds(peek(targetLatencySignal), peek(adaptiveTargetLatencySignal)), + subscriber?.track.moq.targetLatency, + latency.defaultTargetLatency + ); } /** @@ -116,10 +136,11 @@ function makeEdgeTargetUs( // slots — the clocks and the controller must aim at one number, and // the adaptive proposal has to reach both or the slew would chase an // edge the controller is not steering toward. - const targetSeconds = resolveTargetLatencySeconds( - preferredTargetLatencySeconds(peek(targetLatencySignal), peek(adaptiveTargetLatencySignal)), - subscriber?.track.moq.targetLatency, - latency.defaultTargetLatency + const targetSeconds = resolveTrackTargetSeconds( + subscriber, + targetLatencySignal, + adaptiveTargetLatencySignal, + latency ); return joinAnchorUs(newestTimestampUs, targetSeconds); @@ -174,10 +195,11 @@ function makeAudioJoinAnchor( // Same resolution `syncLatency` and `makeEdgeTargetUs` perform, from // the same input slots — the clocks and the controller aim at one number. - const targetSeconds = resolveTargetLatencySeconds( - preferredTargetLatencySeconds(peek(targetLatencySignal), peek(adaptiveTargetLatencySignal)), - subscriber?.track.moq.targetLatency, - latency.defaultTargetLatency + const targetSeconds = resolveTrackTargetSeconds( + subscriber, + targetLatencySignal, + adaptiveTargetLatencySignal, + latency ); const anchorUs = joinAnchorUs(newestTimestampUs, targetSeconds); @@ -224,22 +246,39 @@ function setupAudioRendererSetup({ 'preconditions-unmet': {}, 'renderer-active': { entry: () => { - const renderer = createAudioRendererActor({ - audioContext: context.audioContext.get()!, - // No paused gating here: the adapter suspends the - // AudioContext on pause, which freezes the hardware clock and - // every scheduled source. Rate 0 would instead produce an - // infinite clock segment (duration ÷ 0) and sources whose - // `playbackRate` stays 0 after resume — a permanent stall. - getPlaybackRate: () => peek(state.playoutRate) ?? 1, - getJoinAnchorUs: makeAudioJoinAnchor( - context.audioSubscriberActor, - state.targetLatency, - state.adaptiveTargetLatency, - latencyConfig, - context.videoRendererActor - ), - }); + const audioContext = context.audioContext.get()!; + const getJoinAnchorUs = makeAudioJoinAnchor( + context.audioSubscriberActor, + state.targetLatency, + state.adaptiveTargetLatency, + latencyConfig, + context.videoRendererActor + ); + const renderer = config?.createAudioPlayer + ? createWorkletAudioRendererActor({ + audioContext, + createAudioPlayer: config.createAudioPlayer, + getJoinAnchorUs, + // Caps the worklet's jitter target, so the core never holds + // more than the latency the controller is aiming at. + getTargetLatencyUs: () => + resolveTrackTargetSeconds( + peek(context.audioSubscriberActor), + state.targetLatency, + state.adaptiveTargetLatency, + latencyConfig + ) * 1_000_000, + }) + : createAudioRendererActor({ + audioContext, + // No paused gating here: the adapter suspends the + // AudioContext on pause, which freezes the hardware clock and + // every scheduled source. Rate 0 would instead produce an + // infinite clock segment (duration ÷ 0) and sources whose + // `playbackRate` stays 0 after resume — a permanent stall. + getPlaybackRate: () => peek(state.playoutRate) ?? 1, + getJoinAnchorUs, + }); context.audioRendererActor.set(renderer); return () => { diff --git a/packages/spf/src/playback/behaviors/dom/tests/setup-moq-renderers.test.ts b/packages/spf/src/playback/behaviors/dom/tests/setup-moq-renderers.test.ts index 136ea9229e..f3bb440edb 100644 --- a/packages/spf/src/playback/behaviors/dom/tests/setup-moq-renderers.test.ts +++ b/packages/spf/src/playback/behaviors/dom/tests/setup-moq-renderers.test.ts @@ -5,6 +5,7 @@ import type { AudioContextLike, AudioRendererActor } from '../../../actors/dom/a import { createAudioRendererActor } from '../../../actors/dom/audio-renderer'; import type { VideoRendererActor } from '../../../actors/dom/video-renderer'; import { createVideoRendererActor } from '../../../actors/dom/video-renderer'; +import { createWorkletAudioRendererActor } from '../../../actors/dom/worklet-audio-renderer'; import type { TrackSubscriberActor } from '../../../actors/track-subscriber'; import type { PlayoutClockOwner } from '../../sync-latency'; import { @@ -23,6 +24,9 @@ vi.mock('../../../actors/dom/audio-renderer', () => ({ vi.mock('../../../actors/dom/video-renderer', () => ({ createVideoRendererActor: vi.fn(), })); +vi.mock('../../../actors/dom/worklet-audio-renderer', () => ({ + createWorkletAudioRendererActor: vi.fn(), +})); function makeFakeAudioRenderer(): AudioRendererActor { return { @@ -91,6 +95,29 @@ describe('setupAudioRenderer', () => { reactor.destroy(); }); + it('renders through the worklet core when a createAudioPlayer is configured', async () => { + vi.mocked(createWorkletAudioRendererActor).mockImplementation(() => makeFakeAudioRenderer()); + const createAudioPlayer = vi.fn(); + const { state, context, reactor } = setupSetupAudioRenderer({ createAudioPlayer }); + + await vi.waitFor(() => expect(createWorkletAudioRendererActor).toHaveBeenCalledTimes(1)); + expect(createAudioRendererActor).not.toHaveBeenCalled(); + + const options = vi.mocked(createWorkletAudioRendererActor).mock.calls[0]![0]; + + expect(options.createAudioPlayer).toBe(createAudioPlayer); + expect(options.audioContext).toBe(context.audioContext.get()); + + // The core's jitter ceiling follows the same target the controller holds. + context.audioSubscriberActor.set(makeFakeSubscriber(10_000_000)); + expect(options.getTargetLatencyUs!()).toBe(500_000); + state.targetLatency.set(2); + expect(options.getTargetLatencyUs!()).toBe(2_000_000); + expect(options.getJoinAnchorUs!()).toBe(8_000_000); + + reactor.destroy(); + }); + it('anchors the renderer at the live edge of its own jitter buffer', async () => { vi.mocked(createAudioRendererActor).mockImplementation(() => makeFakeAudioRenderer()); const { state, context, reactor } = setupSetupAudioRenderer(); diff --git a/packages/spf/src/playback/engines/moq/adapter.ts b/packages/spf/src/playback/engines/moq/adapter.ts index 87574d5cbc..e271d986bf 100644 --- a/packages/spf/src/playback/engines/moq/adapter.ts +++ b/packages/spf/src/playback/engines/moq/adapter.ts @@ -330,6 +330,11 @@ export function MoqMediaMixin>( gain.connect(audioContext.destination); gain.gain.value = this.#muted ? 0 : this.#volume; + // The worklet renderer's core constructs an `AudioWorkletNode`, which + // only a real context can host; an injected fake has none to offer. + const baseContext = + audioContext.baseContext ?? (audioContext instanceof BaseAudioContext ? audioContext : undefined); + return { get currentTime() { return audioContext.currentTime; @@ -337,6 +342,13 @@ export function MoqMediaMixin>( get destination() { return gain; }, + get baseLatency() { + return audioContext.baseLatency; + }, + get outputLatency() { + return audioContext.outputLatency; + }, + baseContext, createBuffer: (numberOfChannels, length, sampleRate) => audioContext.createBuffer(numberOfChannels, length, sampleRate), createBufferSource: () => audioContext.createBufferSource(), diff --git a/packages/spf/src/playback/engines/moq/engine.ts b/packages/spf/src/playback/engines/moq/engine.ts index b6660e0a53..bc64fc0787 100644 --- a/packages/spf/src/playback/engines/moq/engine.ts +++ b/packages/spf/src/playback/engines/moq/engine.ts @@ -11,6 +11,7 @@ import type { BandwidthConfig, BandwidthState } from '../../../network/bandwidth import type { RetryBackoffConfig } from '../../../network/retry-backoff'; import type { AudioContextLike, AudioRendererActor } from '../../actors/dom/audio-renderer'; import type { VideoRendererActor } from '../../actors/dom/video-renderer'; +import type { CreateAudioPlayer } from '../../actors/dom/worklet-audio-renderer'; import type { CreateMoqTransport, MoqAuthProvider, MoqSessionActor } from '../../actors/moq-session'; import type { TrackSubscriberActor } from '../../actors/track-subscriber'; import { type AdaptiveLatencyConfig, adaptLatencyTarget } from '../../behaviors/adapt-latency-target'; @@ -153,6 +154,11 @@ export interface MoqEngineConfig extends ShareSignalsConfig; /** Latency tuning: the controller (`syncLatency`) plus the renderers' join-at-edge anchor. */ latency?: Partial; + /** + * Render audio through the shared AudioWorklet player core (`@mux/moq-media`'s `createAudioPlayer`) instead of + * scheduled `AudioBufferSourceNode`s. Opt-in: the core cannot apply `playoutRate` nudges yet (mmcc/v10#66). + */ + createAudioPlayer?: CreateAudioPlayer; /** * Adaptive-latency tuning (`adaptLatencyTarget`), including its `enabled` switch — **off by default**, so an * untouched engine holds the fixed setpoint `latency` describes and this behavior registers no timer. Deliberately From a38555ec1e0c24ef9daf197346a36da57ade492a Mon Sep 17 00:00:00 2001 From: Matt McClure Date: Mon, 19 Oct 2026 18:22:16 +0000 Subject: [PATCH 2/7] fix(spf): worklet audio renderer clock follows context time through pause Run the renderer's clock and fill-stall timer on AudioContext.currentTime, which stops when the adapter suspends the context, instead of performance.now(); scope each input's clock so reports from a removed input are ignored; report output.connect failures; guard the BaseAudioContext global in the adapter. Refs #66 --- .../dom/tests/worklet-audio-renderer.test.ts | 100 +++++++++++++++--- .../actors/dom/worklet-audio-renderer.ts | 96 +++++++++++------ .../spf/src/playback/engines/moq/adapter.ts | 3 +- 3 files changed, 151 insertions(+), 48 deletions(-) diff --git a/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts b/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts index 7213484dfc..757bbc8573 100644 --- a/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts +++ b/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts @@ -91,9 +91,15 @@ function createFakePlayer() { // SAFETY: the renderer only hands `destination` to `output.connect`, which the fake records by identity. const destination = {} as AudioNode; +/** Context time follows the test's clock; `suspended` freezes it, as `AudioContext.suspend()` does. */ +let contextUs = 0; +let suspended = false; + function fakeAudioContext(): AudioContextLike { return { - currentTime: 0, + get currentTime() { + return contextUs / 1_000_000; + }, destination, createBuffer: () => { throw new Error('unused'); @@ -107,9 +113,18 @@ function fakeAudioContext(): AudioContextLike { describe('createWorkletAudioRendererActor', () => { let nowUs = 0; + /** Move wall time, and context time with it unless the context is suspended. */ + const at = (us: number) => { + if (!suspended) contextUs += us - nowUs; + + nowUs = us; + }; + beforeEach(() => { vi.useFakeTimers(); - nowUs = 0; + at(0); + contextUs = 0; + suspended = false; }); afterEach(() => { @@ -158,11 +173,11 @@ describe('createWorkletAudioRendererActor', () => { // Playing: the worklet should hold about its target ahead of what renders. fake.report(0, 0); - nowUs = 30_000; + at(30_000); tick(); expect(input.pushed.at(-1)).toBe(80_000); - expect(renderer.getClockTimeUs()).toBe(30_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(30_000, 0); expect(renderer.snapshot.get().context.scheduledUntilUs).toBe(100_000); }); @@ -183,20 +198,20 @@ describe('createWorkletAudioRendererActor', () => { tick(); fake.report(0, 0); - nowUs = 100_000; + at(100_000); tick(); - expect(renderer.getClockTimeUs()).toBe(80_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); // Audio comes back: the refill moves the pushed end, not the clock. source.queue.push(...framesBetween(80_000, 400_000)); - nowUs = 300_000; + at(300_000); tick(); expect(fake.inputs[0]!.pushed.slice(4)).toEqual([80_000, 100_000, 120_000, 140_000]); - expect(renderer.getClockTimeUs()).toBe(80_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); fake.report(80_000, 320_000); - nowUs = 330_000; - expect(renderer.getClockTimeUs()).toBe(90_000); + at(330_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(90_000, 0); }); it('drops what the core holds at a timeline reset and ignores reports from before it', async () => { @@ -222,7 +237,7 @@ describe('createWorkletAudioRendererActor', () => { expect(renderer.getClockTimeUs()).toBeUndefined(); fake.report(-5_000_000, 0); - expect(renderer.getClockTimeUs()).toBe(-5_000_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(-5_000_000, 0); }); it('places a re-join whose whole buffer is past the anchor at its distance from it', async () => { @@ -242,12 +257,53 @@ describe('createWorkletAudioRendererActor', () => { const input = fake.inputs[0]!; expect(input.pushed).toHaveLength(4); - expect(renderer.getClockTimeUs()).toBe(4_900_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(4_900_000, 0); - nowUs = 50_000; + at(50_000); tick(); expect(input.pushed.at(4)).toBe(5_000_000); - expect(renderer.getClockTimeUs()).toBe(4_950_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(4_950_000, 0); + }); + + it('freezes the clock and stops metering while the context is suspended', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 1_000_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + at(30_000); + tick(); + + const pushed = fake.inputs[0]!.pushed.length; + + // Paused for two seconds: nothing plays, so nothing is due and nothing ran dry. + suspended = true; + at(2_030_000); + tick(); + expect(renderer.getClockTimeUs()).toBeCloseTo(30_000, 0); + expect(fake.inputs[0]!.pushed).toHaveLength(pushed); + + suspended = false; + at(2_040_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(40_000, 0); + }); + + it('ignores reports from an input that a track change removed', async () => { + const { fake, renderer } = await setup(); + + renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); + tick(); + + const stale = fake.inputs[0]!.options.clock!; + + renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); + tick(); + stale.reportAudio(40_000, 0); + expect(renderer.getClockTimeUs()).toBeUndefined(); + + fake.report(0, 0); + expect(renderer.getClockTimeUs()).toBeCloseTo(0, 0); }); it('counts underruns across track changes', async () => { @@ -298,5 +354,21 @@ describe('createWorkletAudioRendererActor', () => { await vi.advanceTimersByTimeAsync(0); expect(failing.snapshot.get().context.status).toBe('error'); failing.destroy(); + + const unconnectable = createFakePlayer(); + + unconnectable.player.output.connect = () => { + throw new Error('cross-context connect'); + }; + + const disconnected = createWorkletAudioRendererActor({ + audioContext: fakeAudioContext(), + createAudioPlayer: () => Promise.resolve(unconnectable.player), + }); + + await vi.advanceTimersByTimeAsync(0); + expect(disconnected.snapshot.get().context.status).toBe('error'); + expect(unconnectable.closed).toBe(true); + disconnected.destroy(); }); }); diff --git a/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts b/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts index 0858355079..77c8b80ba7 100644 --- a/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts +++ b/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts @@ -111,7 +111,10 @@ export interface CreateWorkletAudioRendererOptions { getTargetLatencyUs?: () => number | undefined; /** Metering cadence in ms. Default 10. */ tickIntervalMs?: number; - /** Microsecond time source for the clock. Default `performance.now()`. */ + /** + * The microsecond time source the core times its reports on (`localNow`). Default `performance.now()`. The renderer's + * own clock runs on `audioContext.currentTime`. + */ now?: () => number; } @@ -188,15 +191,22 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende let frameDurationUs = DEFAULT_FRAME_DURATION_US; /** Where the newest pushed frame ends: the furthest the clock can read. */ let pushedEndUs: number | undefined; - /** First timestamp pushed in the current fill, and when the latest push was. */ + /** First timestamp pushed in the current fill, and the context time of the latest push. */ let fillStartUs: number | undefined; let lastPushAtUs = 0; /** - * The clock: timeline position `mediaUs` was audible at local time `atUs`, advancing in real time from there. Set by - * every core report, or **placed** at a re-join anchor before the core plays anything (see `dropToJoinAnchor`). + * The clock: timeline position `mediaUs` is audible at context time `contextUs`, advancing with the context clock + * from there. Set by every core report, or **placed** at a re-join anchor before the core plays anything (see + * `dropToJoinAnchor`). + * + * Context time, not `performance.now()`, because it is what stops when the adapter suspends the context on pause: a + * wall-paced clock would run on through the pause, read the frozen worklet as dry and refill it, and step back on the + * first report after resume. */ - let anchor: { mediaUs: number; atUs: number; placed: boolean } | undefined; + let anchor: { mediaUs: number; contextUs: number; placed: boolean } | undefined; + /** Bumped per input, so a report from a removed input cannot anchor its replacement. */ + let inputGeneration = 0; /** * Where the worklet ran dry. The clock holds there until a report of audio past it arrives: frames pushed to refill * the worklet move `pushedEndUs` before they play, and must not drag the clock forward with them. @@ -206,32 +216,46 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende const outputLatencyUs = (): number => ((audioContext.baseLatency ?? 0) + (audioContext.outputLatency ?? 0)) * 1_000_000; + const contextNowUs = (): number => audioContext.currentTime * 1_000_000; + /** The audible position extrapolated from the anchor, ignoring every limit. */ - const extrapolatedUs = (): number | undefined => anchor && anchor.mediaUs + (now() - anchor.atUs); + const extrapolatedUs = (): number | undefined => anchor && anchor.mediaUs + (contextNowUs() - anchor.contextUs); + + /** The clock one input reports into; it goes deaf once that input is replaced. */ + const createClock = (): AudioPlayerClock => { + const generation = ++inputGeneration; + + return { + get targetLatencyUs() { + return options.getTargetLatencyUs?.() ?? DEFAULT_TARGET_LATENCY_US; + }, + localNow: now, + reportAudio(timestampUs, atUs) { + if (generation !== inputGeneration) return; + + reportAudio(timestampUs, atUs); + }, + // The renderer resets the clock itself wherever a track or timeline ends. + stopAudio() {}, + }; + }; - const clock: AudioPlayerClock = { - get targetLatencyUs() { - return options.getTargetLatencyUs?.() ?? DEFAULT_TARGET_LATENCY_US; - }, - localNow: now, - reportAudio(timestampUs, atUs) { - // Only audio pushed on this timeline: a report already in flight when - // the timeline restarted describes audio the cut dropped. - if ( - firstPushedUs === undefined || - pushedEndUs === undefined || - timestampUs < firstPushedUs - REPORT_TOLERANCE_US || - timestampUs > pushedEndUs + REPORT_TOLERANCE_US - ) { - return; - } + /** `timestampUs` is audible at local time `atUs` (default now), which the anchor keeps as context time. */ + const reportAudio = (timestampUs: number, atUs: number | undefined): void => { + // Only audio pushed on this timeline: a report already in flight when + // the timeline restarted describes audio the cut dropped. + if ( + firstPushedUs === undefined || + pushedEndUs === undefined || + timestampUs < firstPushedUs - REPORT_TOLERANCE_US || + timestampUs > pushedEndUs + REPORT_TOLERANCE_US + ) { + return; + } - anchor = { mediaUs: timestampUs, atUs: atUs ?? now(), placed: false }; + anchor = { mediaUs: timestampUs, contextUs: contextNowUs() + ((atUs ?? now()) - now()), placed: false }; - if (dryLimitUs !== undefined && timestampUs >= dryLimitUs - REPORT_TOLERANCE_US) dryLimitUs = undefined; - }, - // The renderer resets the clock itself wherever a track or timeline ends. - stopAudio() {}, + if (dryLimitUs !== undefined && timestampUs >= dryLimitUs - REPORT_TOLERANCE_US) dryLimitUs = undefined; }; /** Forget everything pushed on the current timeline, and the clock placed on it. */ @@ -249,6 +273,7 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende if (!input) return; underrunsBefore += input.stats.underruns; + inputGeneration++; input.remove(); input = undefined; }; @@ -281,7 +306,7 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende // oldest first, so the buffer's live edge may still be moving. if (!rejoinAnchorArmed || !source.peek()) return; - anchor = { mediaUs: anchorUs, atUs: now(), placed: true }; + anchor = { mediaUs: anchorUs, contextUs: contextNowUs(), placed: true }; rejoinAnchorArmed = false; needsJoinAnchor = false; }; @@ -297,7 +322,7 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende if (positionUs === undefined || dryLimitUs !== undefined) { if (fillStartUs === undefined || timestampUs < fillStartUs + stats.jitterTargetUs + frameDurationUs) return true; - return now() - lastPushAtUs > FILL_STALL_US && stats.bufferedUs < stats.jitterTargetUs; + return contextNowUs() - lastPushAtUs > FILL_STALL_US && stats.bufferedUs < stats.jitterTargetUs; } return timestampUs < positionUs + outputLatencyUs() + stats.jitterTargetUs; @@ -315,7 +340,7 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende firstPushedUs ??= timestampUs; fillStartUs ??= timestampUs; lastPushedUs = timestampUs; - lastPushAtUs = now(); + lastPushAtUs = contextNowUs(); pushedEndUs = timestampUs + frameDurationUs; inner.send({ type: 'scheduled', untilUs: pushedEndUs }); }; @@ -325,7 +350,7 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende try { const active = (input ??= player.addInput({ - clock, + clock: createClock(), decoderConfig, onError: (error) => inner.send({ type: 'decoder-error', error }), })); @@ -384,8 +409,13 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende return; } - created.output.connect(audioContext.destination); - player = created; + try { + created.output.connect(audioContext.destination); + player = created; + } catch (error) { + created.close(); + inner.send({ type: 'status', status: 'error', error }); + } }, (error) => { if (!destroyed) inner.send({ type: 'status', status: 'error', error }); diff --git a/packages/spf/src/playback/engines/moq/adapter.ts b/packages/spf/src/playback/engines/moq/adapter.ts index e271d986bf..9ffaf64dfa 100644 --- a/packages/spf/src/playback/engines/moq/adapter.ts +++ b/packages/spf/src/playback/engines/moq/adapter.ts @@ -333,7 +333,8 @@ export function MoqMediaMixin>( // The worklet renderer's core constructs an `AudioWorkletNode`, which // only a real context can host; an injected fake has none to offer. const baseContext = - audioContext.baseContext ?? (audioContext instanceof BaseAudioContext ? audioContext : undefined); + audioContext.baseContext ?? + (globalThis.BaseAudioContext && audioContext instanceof globalThis.BaseAudioContext ? audioContext : undefined); return { get currentTime() { From 1a7945d99ff720498025e34210e15279a9e02557 Mon Sep 17 00:00:00 2001 From: Matt McClure Date: Mon, 19 Oct 2026 18:27:30 +0000 Subject: [PATCH 3/7] fix(spf): keep the worklet clock monotonic across underrun recovery and pause Hold the underrun position as a floor until the refill's future-dated anchor reaches it, and time the core's reports on context time (localNow) so its render-clock calibration stops across a suspend instead of jumping the clock by the pause on resume. Refs #66 --- .../dom/tests/worklet-audio-renderer.test.ts | 45 ++++++++++++++++++- .../actors/dom/worklet-audio-renderer.ts | 37 +++++++++------ 2 files changed, 67 insertions(+), 15 deletions(-) diff --git a/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts b/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts index 757bbc8573..2efc767f2d 100644 --- a/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts +++ b/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts @@ -138,7 +138,6 @@ describe('createWorkletAudioRendererActor', () => { audioContext: fakeAudioContext(), createAudioPlayer: () => Promise.resolve(fake.player), tickIntervalMs: TICK_MS, - now: () => nowUs, ...options, }); @@ -214,6 +213,50 @@ describe('createWorkletAudioRendererActor', () => { expect(renderer.getClockTimeUs()).toBeCloseTo(90_000, 0); }); + it('does not step back when the refill is reported ahead of being audible', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + at(100_000); + tick(); + + source.queue.push(...framesBetween(80_000, 400_000)); + at(300_000); + tick(); + + // Rendered now, audible 100 ms from now: the anchor alone reads 80 ms behind the hold. + fake.report(80_000, 400_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); + + at(410_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(90_000, 0); + }); + + it('gives the core a report time source that stops while the context is suspended', async () => { + const { fake, renderer } = await setup(); + + renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); + tick(); + + const clock = fake.inputs[0]!.options.clock!; + + at(40_000); + expect(clock.localNow()).toBeCloseTo(40_000, 0); + + suspended = true; + at(540_000); + expect(clock.localNow()).toBeCloseTo(40_000, 0); + + // The first report after resume, timed on that source, lands where audio is. + suspended = false; + at(560_000); + fake.report(60_000, clock.localNow()); + expect(renderer.getClockTimeUs()).toBeCloseTo(60_000, 0); + }); + it('drops what the core holds at a timeline reset and ignores reports from before it', async () => { const { fake, renderer } = await setup(); const source = queueSource(framesBetween(0, 80_000)); diff --git a/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts b/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts index 77c8b80ba7..d5a05044de 100644 --- a/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts +++ b/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts @@ -111,11 +111,6 @@ export interface CreateWorkletAudioRendererOptions { getTargetLatencyUs?: () => number | undefined; /** Metering cadence in ms. Default 10. */ tickIntervalMs?: number; - /** - * The microsecond time source the core times its reports on (`localNow`). Default `performance.now()`. The renderer's - * own clock runs on `audioContext.currentTime`. - */ - now?: () => number; } type RendererMessage = @@ -145,11 +140,8 @@ const FILL_STALL_US = 100_000; /** See `audio-renderer`'s threshold of the same name: a jump this large, either way, is a new timeline. */ const DISCONTINUITY_THRESHOLD_US = TIMELINE_DISCONTINUITY_US; -const performanceNowUs = (): number => performance.now() * 1000; - export function createWorkletAudioRendererActor(options: CreateWorkletAudioRendererOptions): AudioRendererActor { const { audioContext } = options; - const now = options.now ?? performanceNowUs; const inner = createTransitionActor( { status: 'idle', framesScheduled: 0, underruns: 0 }, @@ -202,7 +194,8 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende * * Context time, not `performance.now()`, because it is what stops when the adapter suspends the context on pause: a * wall-paced clock would run on through the pause, read the frozen worklet as dry and refill it, and step back on the - * first report after resume. + * first report after resume. The core times its reports on the same source (`localNow`), so its render clock's + * calibration does not carry a pause into the first reports after it either. */ let anchor: { mediaUs: number; contextUs: number; placed: boolean } | undefined; /** Bumped per input, so a report from a removed input cannot anchor its replacement. */ @@ -212,6 +205,12 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende * the worklet move `pushedEndUs` before they play, and must not drag the clock forward with them. */ let dryLimitUs: number | undefined; + /** + * The position the clock held at through the latest underrun, kept as a floor after it ends. The first report of the + * refill is dated ahead by the output latency — the sample is not audible yet — so its anchor extrapolates to before + * the hold, and the clock would otherwise step back by that much. + */ + let holdFloorUs: number | undefined; const outputLatencyUs = (): number => ((audioContext.baseLatency ?? 0) + (audioContext.outputLatency ?? 0)) * 1_000_000; @@ -229,7 +228,7 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende get targetLatencyUs() { return options.getTargetLatencyUs?.() ?? DEFAULT_TARGET_LATENCY_US; }, - localNow: now, + localNow: contextNowUs, reportAudio(timestampUs, atUs) { if (generation !== inputGeneration) return; @@ -240,7 +239,7 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende }; }; - /** `timestampUs` is audible at local time `atUs` (default now), which the anchor keeps as context time. */ + /** `timestampUs` is audible at context time `atUs` (default now). */ const reportAudio = (timestampUs: number, atUs: number | undefined): void => { // Only audio pushed on this timeline: a report already in flight when // the timeline restarted describes audio the cut dropped. @@ -253,9 +252,12 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende return; } - anchor = { mediaUs: timestampUs, contextUs: contextNowUs() + ((atUs ?? now()) - now()), placed: false }; + anchor = { mediaUs: timestampUs, contextUs: atUs ?? contextNowUs(), placed: false }; - if (dryLimitUs !== undefined && timestampUs >= dryLimitUs - REPORT_TOLERANCE_US) dryLimitUs = undefined; + if (dryLimitUs !== undefined && timestampUs >= dryLimitUs - REPORT_TOLERANCE_US) { + holdFloorUs = dryLimitUs; + dryLimitUs = undefined; + } }; /** Forget everything pushed on the current timeline, and the clock placed on it. */ @@ -267,6 +269,7 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende fillStartUs = undefined; anchor = undefined; dryLimitUs = undefined; + holdFloorUs = undefined; }; const removeInput = (): void => { @@ -458,8 +461,14 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende // where it did: the clock holds through an underrun and resumes when // audio does, as the source-node renderer's clamped segments did. const limitUs = dryLimitUs ?? pushedEndUs; + const clampedUs = limitUs === undefined ? positionUs : Math.min(positionUs, limitUs); + + if (holdFloorUs === undefined) return clampedUs; + + if (clampedUs < holdFloorUs) return holdFloorUs; - return limitUs === undefined ? positionUs : Math.min(positionUs, limitUs); + holdFloorUs = undefined; + return clampedUs; }, destroy(): void { From f7d1672f324bb5b85d1e8cfde1d8c218a56defdd Mon Sep 17 00:00:00 2001 From: Matt McClure Date: Mon, 19 Oct 2026 18:32:21 +0000 Subject: [PATCH 4/7] fix(spf): worklet clock floor follows the audible position; frame length ignores loss gaps Take the recovery floor from what the clock read when the hold released, not the render-side dry point, and read Opus packet durations from the TOC byte (else the shortest step between frames) so a gap of lost packets does not extend pushed audio past where it ends. Refs #66 --- .../dom/tests/worklet-audio-renderer.test.ts | 48 ++++++++++++-- .../actors/dom/worklet-audio-renderer.ts | 66 +++++++++++++++---- 2 files changed, 95 insertions(+), 19 deletions(-) diff --git a/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts b/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts index 2efc767f2d..36ddc7b2ee 100644 --- a/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts +++ b/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts @@ -14,8 +14,11 @@ const JITTER_TARGET_US = 60_000; const TICK_MS = 10; const CONFIG: AudioDecoderConfig = { codec: 'opus', sampleRate: 48_000, numberOfChannels: 1 }; +/** An Opus TOC byte for one 20 ms CELT frame (config 19, code 0). */ +const OPUS_20MS_TOC = 19 << 3; + function frameAt(timestampUs: number): JitterFrame { - return { groupId: timestampUs, objectId: 0, timestampUs, isKey: true, payload: new Uint8Array([timestampUs & 0xff]) }; + return { groupId: timestampUs, objectId: 0, timestampUs, isKey: true, payload: new Uint8Array([OPUS_20MS_TOC]) }; } /** Frames every 20 ms over `[fromUs, untilUs)`. */ @@ -95,12 +98,13 @@ const destination = {} as AudioNode; let contextUs = 0; let suspended = false; -function fakeAudioContext(): AudioContextLike { +function fakeAudioContext(outputLatency?: number): AudioContextLike { return { get currentTime() { return contextUs / 1_000_000; }, destination, + outputLatency, createBuffer: () => { throw new Error('unused'); }, @@ -132,13 +136,14 @@ describe('createWorkletAudioRendererActor', () => { }); /** A renderer whose player is ready and whose first input exists. */ - async function setup(options: { getJoinAnchorUs?: () => number | undefined } = {}) { + async function setup(options: { getJoinAnchorUs?: () => number | undefined; outputLatency?: number } = {}) { + const { outputLatency, ...rendererOptions } = options; const fake = createFakePlayer(); const renderer = createWorkletAudioRendererActor({ - audioContext: fakeAudioContext(), + audioContext: fakeAudioContext(outputLatency), createAudioPlayer: () => Promise.resolve(fake.player), tickIntervalMs: TICK_MS, - ...options, + ...rendererOptions, }); await vi.advanceTimersByTimeAsync(0); @@ -235,6 +240,39 @@ describe('createWorkletAudioRendererActor', () => { expect(renderer.getClockTimeUs()).toBeCloseTo(90_000, 0); }); + it('resumes from the audible position when audio returns before the device drained', async () => { + const { fake, renderer } = await setup({ outputLatency: 0.05 }); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + + // Rendering passed the last pushed sample; 50 ms of it is still in the device. + at(30_000); + tick(); + expect(renderer.getClockTimeUs()).toBeCloseTo(30_000, 0); + + source.queue.push(...framesBetween(80_000, 400_000)); + at(40_000); + tick(); + fake.report(80_000, 90_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(40_000, 0); + }); + + it('ends pushed audio where the last packet ends, not across a gap of lost packets', async () => { + const { fake, renderer } = await setup(); + + renderer.setTrack(queueSource([0, 20_000, 40_000, 100_000].map(frameAt)), CONFIG); + tick(); + fake.report(0, 0); + at(60_000); + tick(); + + expect(fake.inputs[0]!.pushed).toEqual([0, 20_000, 40_000, 100_000]); + expect(renderer.snapshot.get().context.scheduledUntilUs).toBe(120_000); + }); + it('gives the core a report time source that stops while the context is suspended', async () => { const { fake, renderer } = await setup(); diff --git a/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts b/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts index d5a05044de..68bc90303e 100644 --- a/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts +++ b/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts @@ -125,10 +125,8 @@ type RendererMessage = const DEFAULT_TICK_INTERVAL_MS = 10; const DEFAULT_TARGET_LATENCY_US = 2_000_000; -/** The frame duration assumed before two contiguous frames have shown one: Opus's usual 20 ms. */ +/** The frame duration assumed before a frame has shown one: Opus's usual 20 ms. */ const DEFAULT_FRAME_DURATION_US = 20_000; -/** Longest gap between two pushed frames that still measures a frame duration rather than a loss. */ -const MAX_FRAME_DURATION_US = 120_000; /** Reports and pushed spans may disagree by sample rounding and resampling offsets. */ const REPORT_TOLERANCE_US = 5_000; /** @@ -140,6 +138,32 @@ const FILL_STALL_US = 100_000; /** See `audio-renderer`'s threshold of the same name: a jump this large, either way, is a new timeline. */ const DISCONTINUITY_THRESHOLD_US = TIMELINE_DISCONTINUITY_US; +/** + * How long an Opus packet plays, from its TOC byte (RFC 6716 §3.1); `undefined` when it cannot be read. The same rule + * as `@mux/moq-media`'s `opusPacketDurationUs`. + */ +function opusPacketDurationUs(packet: Uint8Array): number | undefined { + const toc = packet[0]; + if (toc === undefined) return undefined; + + const config = toc >> 3; + // SILK: 10, 20, 40, 60 ms; hybrid: 10, 20 ms; CELT: 2.5, 5, 10, 20 ms. + const frameUs = + config < 12 + ? [10_000, 20_000, 40_000, 60_000][config % 4]! + : config < 16 + ? [10_000, 20_000][config % 2]! + : [2_500, 5_000, 10_000, 20_000][config % 4]!; + const code = toc & 3; + if (code === 0) return frameUs; + + if (code !== 3) return 2 * frameUs; + + const count = packet[1]; + + return count === undefined ? undefined : (count & 0x3f) * frameUs; +} + export function createWorkletAudioRendererActor(options: CreateWorkletAudioRendererOptions): AudioRendererActor { const { audioContext } = options; @@ -180,7 +204,8 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende // What this timeline has pushed into the core. Reset with the timeline. let firstPushedUs: number | undefined; let lastPushedUs: number | undefined; - let frameDurationUs = DEFAULT_FRAME_DURATION_US; + /** Known once a packet's duration or two frames' spacing has shown it. */ + let frameDurationUs: number | undefined; /** Where the newest pushed frame ends: the furthest the clock can read. */ let pushedEndUs: number | undefined; /** First timestamp pushed in the current fill, and the context time of the latest push. */ @@ -252,19 +277,24 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende return; } - anchor = { mediaUs: timestampUs, contextUs: atUs ?? contextNowUs(), placed: false }; - + // The refill's first report ends the hold. Its floor is what the clock + // reads at that moment — the audible position, which can still be short + // of the dry point while the device plays out what it had queued. if (dryLimitUs !== undefined && timestampUs >= dryLimitUs - REPORT_TOLERANCE_US) { - holdFloorUs = dryLimitUs; + const heldUs = Math.min(extrapolatedUs() ?? dryLimitUs, dryLimitUs); + + holdFloorUs = Math.max(holdFloorUs ?? heldUs, heldUs); dryLimitUs = undefined; } + + anchor = { mediaUs: timestampUs, contextUs: atUs ?? contextNowUs(), placed: false }; }; /** Forget everything pushed on the current timeline, and the clock placed on it. */ const resetTimeline = (): void => { firstPushedUs = undefined; lastPushedUs = undefined; - frameDurationUs = DEFAULT_FRAME_DURATION_US; + frameDurationUs = undefined; pushedEndUs = undefined; fillStartUs = undefined; anchor = undefined; @@ -323,7 +353,12 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende const positionUs = extrapolatedUs(); if (positionUs === undefined || dryLimitUs !== undefined) { - if (fillStartUs === undefined || timestampUs < fillStartUs + stats.jitterTargetUs + frameDurationUs) return true; + if ( + fillStartUs === undefined || + timestampUs < fillStartUs + stats.jitterTargetUs + (frameDurationUs ?? DEFAULT_FRAME_DURATION_US) + ) { + return true; + } return contextNowUs() - lastPushAtUs > FILL_STALL_US && stats.bufferedUs < stats.jitterTargetUs; } @@ -334,17 +369,20 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende const push = (active: AudioPlayerInput, timestampUs: number, payload: Uint8Array): void => { active.push({ timestampUs, payload }); - if (lastPushedUs !== undefined) { - const deltaUs = timestampUs - lastPushedUs; + // Opus says how long each packet is. Otherwise the shortest step between + // pushed frames is the frame length: a step over lost packets is longer, + // and must not extend the audio past where it really ends. + const opusUs = decoderConfig?.codec === 'opus' ? opusPacketDurationUs(payload) : undefined; + const stepUs = lastPushedUs === undefined ? undefined : timestampUs - lastPushedUs; - if (deltaUs > 0 && deltaUs <= MAX_FRAME_DURATION_US) frameDurationUs = deltaUs; - } + if (opusUs !== undefined) frameDurationUs = opusUs; + else if (stepUs !== undefined && stepUs > 0) frameDurationUs = Math.min(frameDurationUs ?? stepUs, stepUs); firstPushedUs ??= timestampUs; fillStartUs ??= timestampUs; lastPushedUs = timestampUs; lastPushAtUs = contextNowUs(); - pushedEndUs = timestampUs + frameDurationUs; + pushedEndUs = timestampUs + (frameDurationUs ?? DEFAULT_FRAME_DURATION_US); inner.send({ type: 'scheduled', untilUs: pushedEndUs }); }; From dddb2fedfa6cce829c99ff6b3c5d0c7b2aa6f453 Mon Sep 17 00:00:00 2001 From: Matt McClure Date: Mon, 19 Oct 2026 18:37:36 +0000 Subject: [PATCH 5/7] fix(spf): bound the placed re-join clock and ignore pre-underrun tail reports A placed clock now stops at the end of the audio pushed and holds there until the core plays, with the first real report floored like an underrun recovery. The underrun hold ends only once the core has reported itself stopped (or on a report clearly past the dry point), so a late report of the old tail no longer releases it. Refs #66 --- .../dom/tests/worklet-audio-renderer.test.ts | 54 +++++++++++++++++- .../actors/dom/worklet-audio-renderer.ts | 56 ++++++++++++++++--- 2 files changed, 100 insertions(+), 10 deletions(-) diff --git a/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts b/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts index 36ddc7b2ee..16db54cecd 100644 --- a/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts +++ b/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts @@ -41,7 +41,7 @@ interface FakeInput extends AudioPlayerInput { pushed: number[]; cuts: number; removed: boolean; - stats: { underruns: number; bufferedUs: number; jitterTargetUs: number }; + stats: { underruns: number; playing: boolean; bufferedUs: number; jitterTargetUs: number }; } /** The core's surface with no audio behind it: pushes are recorded, and the test plays the worklet's reports. */ @@ -57,7 +57,7 @@ function createFakePlayer() { pushed: [], cuts: 0, removed: false, - stats: { underruns: 0, bufferedUs: 0, jitterTargetUs: JITTER_TARGET_US }, + stats: { underruns: 0, playing: true, bufferedUs: 0, jitterTargetUs: JITTER_TARGET_US }, push(frame) { input.pushed.push(frame.timestampUs); }, @@ -207,6 +207,7 @@ describe('createWorkletAudioRendererActor', () => { expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); // Audio comes back: the refill moves the pushed end, not the clock. + fake.inputs[0]!.stats.playing = false; source.queue.push(...framesBetween(80_000, 400_000)); at(300_000); tick(); @@ -228,6 +229,7 @@ describe('createWorkletAudioRendererActor', () => { at(100_000); tick(); + fake.inputs[0]!.stats.playing = false; source.queue.push(...framesBetween(80_000, 400_000)); at(300_000); tick(); @@ -253,6 +255,7 @@ describe('createWorkletAudioRendererActor', () => { tick(); expect(renderer.getClockTimeUs()).toBeCloseTo(30_000, 0); + fake.inputs[0]!.stats.playing = false; source.queue.push(...framesBetween(80_000, 400_000)); at(40_000); tick(); @@ -273,6 +276,53 @@ describe('createWorkletAudioRendererActor', () => { expect(renderer.snapshot.get().context.scheduledUntilUs).toBe(120_000); }); + it('keeps the hold through a late report of the audio before the underrun', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + at(100_000); + tick(); + + // The refill is pushed before the core has said it stopped; then the + // tail's last report, still in flight, arrives. + source.queue.push(...framesBetween(80_000, 400_000)); + tick(); + fake.report(77_333, 77_333); + at(200_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); + }); + + it('holds a placed re-join clock at the audio pushed while the fill is short', async () => { + let anchorUs: number | undefined; + const { fake, renderer } = await setup({ getJoinAnchorUs: () => anchorUs }); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + + anchorUs = 4_900_000; + source.queue.push(frameAt(5_000_000)); + tick(); + at(100_000); + tick(); + expect(fake.inputs[0]!.pushed.at(-1)).toBe(5_000_000); + + // One packet cannot start the worklet; the clock stops at its end. + at(2_100_000); + tick(); + expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); + + // The refill plays, reported ahead of being audible: no step back. + source.queue.push(...framesBetween(5_020_000, 5_200_000)); + tick(); + fake.report(5_000_000, 2_150_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); + }); + it('gives the core a report time source that stops while the context is suspended', async () => { const { fake, renderer } = await setup(); diff --git a/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts b/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts index 68bc90303e..0fdb02a9fb 100644 --- a/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts +++ b/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts @@ -59,6 +59,8 @@ export interface AudioPlayerInputOptions { export interface AudioPlayerInputStats { /** Times the worklet ran dry while playing and audio came back within the jitter ceiling. Cumulative. */ readonly underruns: number; + /** Whether the worklet is playing, rather than filling to its target. */ + readonly playing: boolean; /** Decoded audio the worklet holds. */ readonly bufferedUs: number; /** How much the worklet holds before it plays, at the start and after running dry. */ @@ -222,7 +224,7 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende * first report after resume. The core times its reports on the same source (`localNow`), so its render clock's * calibration does not carry a pause into the first reports after it either. */ - let anchor: { mediaUs: number; contextUs: number; placed: boolean } | undefined; + let anchor: { mediaUs: number; contextUs: number; placed: boolean; frozen?: boolean } | undefined; /** Bumped per input, so a report from a removed input cannot anchor its replacement. */ let inputGeneration = 0; /** @@ -236,6 +238,11 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende * the hold, and the clock would otherwise step back by that much. */ let holdFloorUs: number | undefined; + /** + * The core has reported itself stopped since the dry point was set. Until it has, a report near the dry point is the + * tail of the audio before it, still in flight, and must not end the hold. + */ + let stoppedSinceDry = false; const outputLatencyUs = (): number => ((audioContext.baseLatency ?? 0) + (audioContext.outputLatency ?? 0)) * 1_000_000; @@ -243,7 +250,8 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende const contextNowUs = (): number => audioContext.currentTime * 1_000_000; /** The audible position extrapolated from the anchor, ignoring every limit. */ - const extrapolatedUs = (): number | undefined => anchor && anchor.mediaUs + (contextNowUs() - anchor.contextUs); + const extrapolatedUs = (): number | undefined => + anchor && (anchor.frozen ? anchor.mediaUs : anchor.mediaUs + (contextNowUs() - anchor.contextUs)); /** The clock one input reports into; it goes deaf once that input is replaced. */ const createClock = (): AudioPlayerClock => { @@ -264,6 +272,20 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende }; }; + const raiseHoldFloor = (heldUs: number): void => { + holdFloorUs = Math.max(holdFloorUs ?? heldUs, heldUs); + }; + + /** + * A placed clock runs from the anchor until the core plays, but never past the audio pushed: a fill that stays short + * of the jitter target cannot start the worklet, and the clock would otherwise run on through audio not there. + */ + const placedPositionUs = (): number | undefined => { + const positionUs = extrapolatedUs(); + + return positionUs === undefined || pushedEndUs === undefined ? positionUs : Math.min(positionUs, pushedEndUs); + }; + /** `timestampUs` is audible at context time `atUs` (default now). */ const reportAudio = (timestampUs: number, atUs: number | undefined): void => { // Only audio pushed on this timeline: a report already in flight when @@ -280,13 +302,19 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende // The refill's first report ends the hold. Its floor is what the clock // reads at that moment — the audible position, which can still be short // of the dry point while the device plays out what it had queued. - if (dryLimitUs !== undefined && timestampUs >= dryLimitUs - REPORT_TOLERANCE_US) { - const heldUs = Math.min(extrapolatedUs() ?? dryLimitUs, dryLimitUs); - - holdFloorUs = Math.max(holdFloorUs ?? heldUs, heldUs); + if ( + dryLimitUs !== undefined && + ((stoppedSinceDry && timestampUs >= dryLimitUs - REPORT_TOLERANCE_US) || + timestampUs > dryLimitUs + REPORT_TOLERANCE_US) + ) { + raiseHoldFloor(Math.min(extrapolatedUs() ?? dryLimitUs, dryLimitUs)); dryLimitUs = undefined; } + // Likewise the first report after a placed re-join, which is dated ahead + // of the placed clock by the output latency. + if (anchor?.placed) raiseHoldFloor(placedPositionUs()!); + anchor = { mediaUs: timestampUs, contextUs: atUs ?? contextNowUs(), placed: false }; }; @@ -300,6 +328,7 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende anchor = undefined; dryLimitUs = undefined; holdFloorUs = undefined; + stoppedSinceDry = false; }; const removeInput = (): void => { @@ -350,7 +379,7 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende * rendered within the jitter target, so the worklet holds about that much and the rest waits in the subscriber. */ const isDue = (timestampUs: number, stats: AudioPlayerInputStats): boolean => { - const positionUs = extrapolatedUs(); + const positionUs = anchor?.placed ? placedPositionUs() : extrapolatedUs(); if (positionUs === undefined || dryLimitUs !== undefined) { if ( @@ -413,9 +442,18 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende positionUs + outputLatencyUs() >= pushedEndUs ) { dryLimitUs = pushedEndUs; + stoppedSinceDry = false; fillStartUs = undefined; } + if (dryLimitUs !== undefined && !stats.playing) stoppedSinceDry = true; + + // A placed clock that reaches the end of the audio pushed stops there + // until the core plays, so refill frames cannot drag it forward. + if (anchor?.placed && !anchor.frozen && pushedEndUs !== undefined && positionUs! >= pushedEndUs) { + anchor = { mediaUs: pushedEndUs, contextUs: contextNowUs(), placed: true, frozen: true }; + } + for (;;) { if (needsJoinAnchor && lastPushedUs === undefined) dropToJoinAnchor(); @@ -493,7 +531,9 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende getClockTimeUs(): number | undefined { const positionUs = extrapolatedUs(); - if (positionUs === undefined || anchor?.placed) return positionUs; + if (positionUs === undefined) return undefined; + + if (anchor?.placed) return placedPositionUs(); // Never past the audio pushed, and once the worklet ran dry never past // where it did: the clock holds through an underrun and resumes when From 0e5fcb3d88ec5333f55e80fb5a81ecc6bcc2b159 Mon Sep 17 00:00:00 2001 From: Matt McClure Date: Mon, 19 Oct 2026 18:45:45 +0000 Subject: [PATCH 6/7] fix(spf): count every worklet starvation edge; let a frozen re-join refill Count underruns on the renderer's own dry edges, as audio-renderer does, instead of the core's counter, which skips stalls longer than its jitter ceiling that adaptLatencyTarget still needs. Treat a frozen placed clock as filling so the fill-stall rescue can start the worklet across a packet gap. Refs #66 --- .../dom/tests/worklet-audio-renderer.test.ts | 45 +++++++++++++++---- .../actors/dom/worklet-audio-renderer.ts | 22 +++++---- 2 files changed, 47 insertions(+), 20 deletions(-) diff --git a/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts b/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts index 16db54cecd..5e452b5360 100644 --- a/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts +++ b/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts @@ -41,7 +41,7 @@ interface FakeInput extends AudioPlayerInput { pushed: number[]; cuts: number; removed: boolean; - stats: { underruns: number; playing: boolean; bufferedUs: number; jitterTargetUs: number }; + stats: { playing: boolean; bufferedUs: number; jitterTargetUs: number }; } /** The core's surface with no audio behind it: pushes are recorded, and the test plays the worklet's reports. */ @@ -57,7 +57,7 @@ function createFakePlayer() { pushed: [], cuts: 0, removed: false, - stats: { underruns: 0, playing: true, bufferedUs: 0, jitterTargetUs: JITTER_TARGET_US }, + stats: { playing: true, bufferedUs: 0, jitterTargetUs: JITTER_TARGET_US }, push(frame) { input.pushed.push(frame.timestampUs); }, @@ -437,22 +437,51 @@ describe('createWorkletAudioRendererActor', () => { expect(renderer.getClockTimeUs()).toBeCloseTo(0, 0); }); - it('counts underruns across track changes', async () => { + it('counts every time the worklet runs dry, across track changes', async () => { const { fake, renderer } = await setup(); renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); tick(); - fake.inputs[0]!.stats.underruns = 2; + fake.report(0, 0); + + // A stall far longer than any jitter ceiling still counts, once. + at(100_000); tick(); - expect(renderer.snapshot.get().context.underruns).toBe(2); + at(3_000_000); + tick(); + expect(renderer.snapshot.get().context.underruns).toBe(1); renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); - expect(fake.inputs[0]!.removed).toBe(true); + tick(); + fake.report(0, 3_000_000); + at(3_100_000); + tick(); + expect(renderer.snapshot.get().context.underruns).toBe(2); + }); + + it('rescues a frozen re-join placement whose next packets are past the fill horizon', async () => { + let anchorUs: number | undefined; + const { fake, renderer } = await setup({ getJoinAnchorUs: () => anchorUs }); + const source = queueSource(framesBetween(0, 80_000)); + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + + anchorUs = 4_900_000; + source.queue.push(frameAt(5_000_000)); tick(); - fake.inputs[1]!.stats.underruns = 1; + at(200_000); + tick(); + expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); + + // Packets resume 80 ms after the lone one ends: beyond the fill horizon. + fake.inputs[0]!.stats.playing = false; + source.queue.push(...framesBetween(5_100_000, 5_400_000)); + at(400_000); tick(); - expect(renderer.snapshot.get().context.underruns).toBe(3); + expect(fake.inputs[0]!.pushed).toContain(5_100_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); }); it('caps the core jitter target at the target latency', async () => { diff --git a/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts b/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts index 0fdb02a9fb..c23f9fc2a8 100644 --- a/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts +++ b/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts @@ -57,8 +57,6 @@ export interface AudioPlayerInputOptions { /** The part of `@mux/moq-media`'s `AudioInputStats` the renderer reads. Durations are microseconds. */ export interface AudioPlayerInputStats { - /** Times the worklet ran dry while playing and audio came back within the jitter ceiling. Cumulative. */ - readonly underruns: number; /** Whether the worklet is playing, rather than filling to its target. */ readonly playing: boolean; /** Decoded audio the worklet holds. */ @@ -119,7 +117,7 @@ type RendererMessage = | { type: 'status'; status: AudioRendererStatus; error?: unknown } | { type: 'decoder-error'; error: unknown } | { type: 'scheduled'; untilUs: number } - | { type: 'underruns'; count: number }; + | { type: 'underrun' }; // ============================================================================= // Implementation @@ -184,8 +182,8 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende framesScheduled: context.framesScheduled + 1, scheduledUntilUs: message.untilUs, }; - case 'underruns': - return { ...context, underruns: message.count }; + case 'underrun': + return { ...context, underruns: context.underruns + 1 }; } } ); @@ -195,8 +193,6 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende let input: AudioPlayerInput | undefined; let source: AudioFrameSource | null = null; let decoderConfig: AudioDecoderConfig | null = null; - /** Underruns counted by inputs already removed: the context's counter spans the actor's life, not one track's. */ - let underrunsBefore = 0; /** The schedule has no anchor yet; see `audio-renderer`'s flag of the same name. */ let needsJoinAnchor = true; @@ -334,7 +330,6 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende const removeInput = (): void => { if (!input) return; - underrunsBefore += input.stats.underruns; inputGeneration++; input.remove(); input = undefined; @@ -381,7 +376,9 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende const isDue = (timestampUs: number, stats: AudioPlayerInputStats): boolean => { const positionUs = anchor?.placed ? placedPositionUs() : extrapolatedUs(); - if (positionUs === undefined || dryLimitUs !== undefined) { + // A frozen placement is a fill too: its clock waits for the worklet to + // start, so only the fill rules (including the stall rescue) can move it. + if (positionUs === undefined || dryLimitUs !== undefined || anchor?.frozen) { if ( fillStartUs === undefined || timestampUs < fillStartUs + stats.jitterTargetUs + (frameDurationUs ?? DEFAULT_FRAME_DURATION_US) @@ -425,12 +422,12 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende onError: (error) => inner.send({ type: 'decoder-error', error }), })); const { stats } = active; - const underruns = underrunsBefore + stats.underruns; - - if (underruns !== inner.snapshot.get().context.underruns) inner.send({ type: 'underruns', count: underruns }); // The worklet runs dry once rendering passes everything pushed: metering // pushes every frame as soon as it is due, so nothing was left to push. + // Each such edge is an underrun, as `audio-renderer` counts them. The + // core's own counter skips dry spells longer than its jitter ceiling, + // which `adaptLatencyTarget` still needs as failure evidence. const positionUs = extrapolatedUs(); if ( @@ -444,6 +441,7 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende dryLimitUs = pushedEndUs; stoppedSinceDry = false; fillStartUs = undefined; + inner.send({ type: 'underrun' }); } if (dryLimitUs !== undefined && !stats.playing) stoppedSinceDry = true; From c2beabf5c33aff1e094bc838b51acd8dffc4b6ce Mon Sep 17 00:00:00 2001 From: Matt McClure Date: Mon, 19 Oct 2026 20:48:13 +0000 Subject: [PATCH 7/7] fix(spf): count a worklet dry edge that a track change lands on Detect a pending dry edge in setTrack before the input and timeline are reset, so an underrun between the last tick and the switch still counts. Refs #66 --- .../dom/tests/worklet-audio-renderer.test.ts | 12 +++++ .../actors/dom/worklet-audio-renderer.ts | 48 +++++++++++-------- 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts b/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts index 5e452b5360..3931fc9e15 100644 --- a/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts +++ b/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts @@ -459,6 +459,18 @@ describe('createWorkletAudioRendererActor', () => { expect(renderer.snapshot.get().context.underruns).toBe(2); }); + it('counts a dry edge that a track change lands on before the next tick', async () => { + const { fake, renderer } = await setup(); + + renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); + tick(); + fake.report(0, 0); + + at(100_000); + renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); + expect(renderer.snapshot.get().context.underruns).toBe(1); + }); + it('rescues a frozen re-join placement whose next packets are past the fill horizon', async () => { let anchorUs: number | undefined; const { fake, renderer } = await setup({ getJoinAnchorUs: () => anchorUs }); diff --git a/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts b/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts index c23f9fc2a8..d92ef39dc7 100644 --- a/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts +++ b/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts @@ -412,6 +412,31 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende inner.send({ type: 'scheduled', untilUs: pushedEndUs }); }; + /** + * The worklet runs dry once rendering passes everything pushed: metering pushes every frame as soon as it is due, so + * nothing was left to push. Each such edge is an underrun, as `audio-renderer` counts them. The core's own counter + * skips dry spells longer than its jitter ceiling, which `adaptLatencyTarget` still needs as failure evidence. + */ + const detectDry = (): void => { + const positionUs = extrapolatedUs(); + + if ( + !anchor || + anchor.placed || + dryLimitUs !== undefined || + pushedEndUs === undefined || + positionUs === undefined || + positionUs + outputLatencyUs() < pushedEndUs + ) { + return; + } + + dryLimitUs = pushedEndUs; + stoppedSinceDry = false; + fillStartUs = undefined; + inner.send({ type: 'underrun' }); + }; + const tick = (): void => { if (destroyed || !player || !source || !decoderConfig) return; @@ -423,26 +448,9 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende })); const { stats } = active; - // The worklet runs dry once rendering passes everything pushed: metering - // pushes every frame as soon as it is due, so nothing was left to push. - // Each such edge is an underrun, as `audio-renderer` counts them. The - // core's own counter skips dry spells longer than its jitter ceiling, - // which `adaptLatencyTarget` still needs as failure evidence. - const positionUs = extrapolatedUs(); + detectDry(); - if ( - anchor && - !anchor.placed && - dryLimitUs === undefined && - pushedEndUs !== undefined && - positionUs !== undefined && - positionUs + outputLatencyUs() >= pushedEndUs - ) { - dryLimitUs = pushedEndUs; - stoppedSinceDry = false; - fillStartUs = undefined; - inner.send({ type: 'underrun' }); - } + const positionUs = extrapolatedUs(); if (dryLimitUs !== undefined && !stats.playing) stoppedSinceDry = true; @@ -512,6 +520,8 @@ export function createWorkletAudioRendererActor(options: CreateWorkletAudioRende decoderConfig = nextConfig; needsJoinAnchor = true; rejoinAnchorArmed = false; + // An edge since the last tick still counts: the reset below forgets it. + detectDry(); removeInput(); resetTimeline();