From b835d98be507c783979931f5760d2eb27dee2ba6 Mon Sep 17 00:00:00 2001 From: Matt McClure Date: Tue, 27 Oct 2026 18:10:27 +0000 Subject: [PATCH 1/3] docs(spf): name @caprid/relay as the source of the subscriber priorities Claude-Session: https://claude.ai/code/session_01DaH4ihf1LLPqmXShYkvBat --- packages/spf/src/playback/actors/track-subscriber.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/spf/src/playback/actors/track-subscriber.ts b/packages/spf/src/playback/actors/track-subscriber.ts index 3c05161cf4..e22a21d0eb 100644 --- a/packages/spf/src/playback/actors/track-subscriber.ts +++ b/packages/spf/src/playback/actors/track-subscriber.ts @@ -164,7 +164,7 @@ export const DEFAULT_STALL_TIMEOUT_MS = 10_000; * SUBSCRIBER_PRIORITY by catalog track kind, lower being more urgent: audio ahead of video on a congested downlink. * moq-relay 0.17 orders what it sends a viewer by subscriber priority alone and ignores the publisher's, and the value * cannot change later — any REQUEST_UPDATE ends the subscription there — so it rides every SUBSCRIBE. Same values as - * the relay SDKs' `mediaSubscribeParameters` (muxinc/relay-sdks#337). Other kinds send none, leaving the peer's + * `@caprid/relay`'s `mediaSubscribeParameters` (muxinc/relay-sdks#337). Other kinds send none, leaving the peer's * default. */ export const MEDIA_SUBSCRIBER_PRIORITY: Readonly>> = { From 1da4b561b264bc7fbb92b8b0ff825f3140d7f1c9 Mon Sep 17 00:00:00 2001 From: Matt McClure Date: Tue, 27 Oct 2026 18:10:27 +0000 Subject: [PATCH 2/3] feat(spf): render MoQ audio through the @caprid/moq-media worklet core The AudioWorklet player core is now published, so the worklet renderer becomes the only audio renderer and the AudioBufferSourceNode path is removed. The latency controller's playout rate reaches the core as the input's playbackRate (clamped to its 0.5-2 range), and the clock is rebased on each change so it advances at the rate in effect. The renderer defaults to the core on the render context's baseContext; the engine's createAudioPlayer option stays as an override. @caprid/* joins minimumReleaseAgeExclude. Closes mmcc/v10#66 Claude-Session: https://claude.ai/code/session_01DaH4ihf1LLPqmXShYkvBat --- .agents/plans/moq-engine-implementation.md | 3 - .../simple-moq-video/tests/index.test.ts | 6 - packages/spf/package.json | 1 + .../src/playback/actors/dom/audio-renderer.ts | 774 ++++++------ .../actors/dom/tests/audio-renderer.test.ts | 1044 ++++++++--------- .../dom/tests/worklet-audio-renderer.test.ts | 546 --------- .../actors/dom/worklet-audio-renderer.ts | 571 --------- .../behaviors/dom/setup-moq-renderers.ts | 50 +- .../dom/tests/setup-moq-renderers.test.ts | 17 +- .../spf/src/playback/engines/moq/adapter.ts | 12 +- .../spf/src/playback/engines/moq/engine.ts | 7 +- .../engines/moq/tests/adapter.test.ts | 12 +- pnpm-lock.yaml | 97 ++ pnpm-workspace.yaml | 1 + 14 files changed, 1045 insertions(+), 2096 deletions(-) delete mode 100644 packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts delete mode 100644 packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts diff --git a/.agents/plans/moq-engine-implementation.md b/.agents/plans/moq-engine-implementation.md index 7085c1f851..f4792bd739 100644 --- a/.agents/plans/moq-engine-implementation.md +++ b/.agents/plans/moq-engine-implementation.md @@ -41,9 +41,6 @@ beside this file. Notable corrections to the plan's assumptions: - `syncPreload` / `trackLoadTriggers` are not composed into the engine: both are `HTMLMediaElement`-bound and there is no element; the adapter writes `state.preload` / `state.loadActivated` directly. -- Audio renderer uses `AudioBufferSourceNode` scheduling, not the - AudioWorklet ring buffer, with the clock contract already shaped for - the swap (TODO(audio-worklet) in `audio-renderer.ts`). - No text-track subscription behavior yet (`switchTextTrack` composes and selects, but nothing renders text). - No WebCodecs `canPlayTrack` probe is injected into selection — diff --git a/packages/html/src/media/simple-moq-video/tests/index.test.ts b/packages/html/src/media/simple-moq-video/tests/index.test.ts index 83028dcf31..8aa3ac42ae 100644 --- a/packages/html/src/media/simple-moq-video/tests/index.test.ts +++ b/packages/html/src/media/simple-moq-video/tests/index.test.ts @@ -13,12 +13,6 @@ function createFakeAudioContext(initialState: AudioContextState) { connect: () => {}, gain: { value: 1 }, }) as unknown as GainNode, - createBuffer: (() => { - throw new Error('unused in wrapper tests'); - }) as MoqAudioContext['createBuffer'], - createBufferSource: (() => { - throw new Error('unused in wrapper tests'); - }) as MoqAudioContext['createBufferSource'], resume: vi.fn(async () => { fake.state = 'running'; }), diff --git a/packages/spf/package.json b/packages/spf/package.json index a3862f5c76..0383c79e0f 100644 --- a/packages/spf/package.json +++ b/packages/spf/package.json @@ -79,6 +79,7 @@ "size:all": "node scripts/measure-size.js --all" }, "dependencies": { + "@caprid/moq-media": "^0.1.0", "@videojs/media": "workspace:*", "@videojs/utils": "workspace:*", "signal-polyfill": "^0.2.2" diff --git a/packages/spf/src/playback/actors/dom/audio-renderer.ts b/packages/spf/src/playback/actors/dom/audio-renderer.ts index d870d131c4..66f8ed1a6d 100644 --- a/packages/spf/src/playback/actors/dom/audio-renderer.ts +++ b/packages/spf/src/playback/actors/dom/audio-renderer.ts @@ -1,20 +1,22 @@ /** - * AudioDecoder → Web Audio renderer actor. **Owns the master clock**: once audio is scheduled, `getClockTimeUs()` maps - * the AudioContext's hardware clock back to media time, and the video renderer (and `currentTime` derivation) follow - * it. + * Audio renderer actor over the shared AudioWorklet player core, `@caprid/moq-media`'s `createAudioPlayer`. Frames in + * from the subscriber's jitter buffer, the **master clock** out: the video renderer and `currentTime` follow it. * - * Decoded `AudioData` is copied into `AudioBuffer`s and scheduled gaplessly with `AudioBufferSourceNode`s. Each - * scheduled buffer records a clock segment (context-time span ↔ media time × rate), and the clock reads the segment - * containing `currentTime` — so it reports what was _actually_ scheduled: late arrivals and rate nudges shift the - * timeline forward from where they happen instead of rescaling elapsed time. Rate nudges from the latency controller - * apply as `playbackRate` on the scheduled sources (a ±5% nudge is a barely audible pitch shift). + * The core decodes with WebCodecs, holds an adaptive jitter buffer in an `AudioWorklet`, and reports the audible + * timestamp about every 20 ms. **Latency stays v10's.** The renderer meters frames into the core so the worklet holds + * about its own jitter target, and the rest of the target latency stays in the subscriber's buffer, where `syncLatency` + * measures and steers it. The core's adaptive target is left alone: under metering its growth only moves depth from one + * buffer to the other. The controller's rate nudges apply as the input's `playbackRate` (a ±5% nudge is a barely + * audible pitch shift). * - * Where the schedule _starts_ is `getJoinAnchorUs`: with one supplied the clock anchors at the live edge and the - * buffered backlog is discarded, instead of anchoring at the oldest replayed group and playing all of it. - * - * TODO(audio-worklet): replace source-node scheduling with an AudioWorklet ring buffer for tighter jitter control and - * clean rate adjustment; the actor surface (frame source in, master clock out) is designed so that swap is internal. + * The clock the core reports into is the renderer's own, not the SDK's `PlayoutClock`, so `getClockTimeUs` keeps the + * contract the video renderer and `trackPlayoutTime` rely on: `undefined` before any audio, and **held where the + * worklet ran dry** rather than paced on by wall time — a wall-paced clock steps back when audio resumes, which holds + * slaved video on its last frame for the whole step. */ +import { createAudioPlayer as createCoreAudioPlayer, opusPacketDurationUs } from '@caprid/moq-media'; +import { clamp } from '@videojs/utils/number'; + import { createTransitionActor, type TransitionActor } from '../../../core/actors/create-transition-actor'; import { TIMELINE_DISCONTINUITY_US } from '../../../media/moq/timeline'; import type { JitterFrame } from '../track-subscriber'; @@ -36,7 +38,7 @@ export interface AudioFrameSource { export interface AudioContextLike { readonly currentTime: number; readonly destination: AudioNode; - /** Seconds between the graph and the audio subsystem; the worklet renderer places its clock behind it. */ + /** Seconds between the graph and the audio subsystem; the clock is placed behind it. */ readonly baseLatency?: number; /** Seconds from the audio subsystem to the speaker; read fresh, as it changes with the device. */ readonly outputLatency?: number; @@ -45,8 +47,6 @@ export interface AudioContextLike { * `AudioWorkletNode` cannot be constructed on a plain object. */ readonly baseContext?: BaseAudioContext; - createBuffer(numberOfChannels: number, length: number, sampleRate: number): AudioBuffer; - createBufferSource(): AudioBufferSourceNode; } export type AudioRendererStatus = 'idle' | 'rendering' | 'error'; @@ -54,13 +54,13 @@ export type AudioRendererStatus = 'idle' | 'rendering' | 'error'; export interface AudioRendererContext { status: AudioRendererStatus; framesScheduled: number; - /** Media time of the newest scheduled audio, in microseconds. */ + /** Media time of the newest audio handed to the core, in microseconds. */ scheduledUntilUs?: number; /** - * **Times the schedule ran dry** — the context clock passed the end of everything scheduled while a track was still - * attached. Counted per rising edge, not per tick, so one starvation is one increment. + * **Times the worklet ran dry** — rendering passed the end of everything pushed while a track was still attached. + * Counted per rising edge, not per tick, so one starvation is one increment. * - * The clock itself hides this: `getClockTimeUs` clamps to the segment end and resumes seamlessly, which is right for + * The clock itself hides this: `getClockTimeUs` holds at the dry point and resumes seamlessly, which is right for * presentation and useless as a signal. This is the direct "the target latency is below what the path can sustain" * evidence the adaptive controller has no other way to obtain — a shallow buffer looks identical to a well-behaved * one right up until it is empty. @@ -69,63 +69,140 @@ export interface AudioRendererContext { error?: unknown; } +export interface AudioRendererActor extends Pick, 'snapshot'> { + /** Point the renderer at a (new) frame source + decoder config. */ + setTrack(source: AudioFrameSource | null, config: AudioDecoderConfig | null): void; + /** Master playout clock: current media time in microseconds, or `undefined` before any audio is audible. */ + getClockTimeUs(): number | undefined; + destroy(): void; +} + +/** + * The part of `@caprid/moq-media`'s `PlayoutClock` an audio player input calls. The renderer implements it, so the + * input reports into the renderer's clock. + */ +export interface AudioPlayerClock { + /** Caps the input's jitter target: the core keeps it under this less the output delay and 40 ms. */ + readonly targetLatencyUs: number; + /** The time source reports are measured on, in microseconds. */ + localNow(): number; + /** The audio with this timestamp is audible at local time `atUs`. */ + reportAudio(timestampUs: number, atUs?: number): void; + /** The input stopped. */ + stopAudio(): void; +} + +/** The part of `@caprid/moq-media`'s `AudioInputOptions` the renderer passes. */ +export interface AudioPlayerInputOptions { + clock?: AudioPlayerClock; + decoderConfig?: AudioDecoderConfig; + /** See `AudioPlayerInput.playbackRate`. */ + playbackRate?: number; + /** A decoder error. The core decodes again from the next frame, with a new decoder. */ + onError?(cause: unknown): void; +} + +/** The part of `@caprid/moq-media`'s `AudioInputStats` the renderer reads. Durations are microseconds. */ +export interface AudioPlayerInputStats { + /** Whether the worklet is playing, rather than filling to its target. */ + readonly playing: boolean; + /** Decoded audio the worklet holds. */ + readonly bufferedUs: number; + /** How much the worklet holds before it plays, at the start and after running dry. */ + readonly jitterTargetUs: number; +} + +/** The part of `@caprid/moq-media`'s `AudioPlayerInput` the renderer uses: one track. */ +export interface AudioPlayerInput { + /** Decode and hold a frame, in decode order. */ + push(frame: { timestampUs: number; payload: Uint8Array }): void; + /** Without a range: drop everything held or still decoding. */ + cut(): void; + /** Held audio played per output frame, from the next render quantum; 0.5–2, or the core throws. */ + playbackRate: number; + readonly stats: AudioPlayerInputStats; + remove(): void; +} + +/** The part of `@caprid/moq-media`'s `AudioPlayer` the renderer uses. */ +export interface AudioPlayer { + /** The mix of every input; the renderer connects it to its context's `destination`. */ + readonly output: { connect(destination: AudioNode): void }; + addInput(options?: AudioPlayerInputOptions): AudioPlayerInput; + /** Remove every input and disconnect the output. */ + close(): void; +} + +/** Create the player on the renderer's context. Default: the core on the context's `baseContext`. */ +export type CreateAudioPlayer = (context: AudioContextLike) => Promise; + export interface CreateAudioRendererOptions { + /** Where the player's output connects (`destination`), and the output latencies the clock is placed behind. */ audioContext: AudioContextLike; - /** Playout rate (latency nudges). Default 1. */ + createAudioPlayer?: CreateAudioPlayer; + /** Playout rate (latency nudges), clamped to the core's 0.5–2. Default 1. */ getPlaybackRate?: () => number; - /** Scheduling safety margin in seconds ahead of `currentTime`. Default 0.05. */ - scheduleMargin?: number; - /** Decode/schedule cadence in ms. Default 10. */ - tickIntervalMs?: number; /** - * **Join at the live edge.** Media timestamp the schedule should start at while it has no anchor — at join, and again - * after a catch-up skip. Buffered audio older than it is discarded rather than played: audio cannot be fast-forwarded - * the way video can without pitch artifacts. `undefined` (the default, and whatever the caller returns before any - * frame has arrived) anchors at the oldest buffered frame instead. + * **Join at the live edge.** Media timestamp the playout should start at while it has no anchor — at join, and again + * after a catch-up skip. Buffered audio older than it is discarded before it reaches the core: audio cannot be + * fast-forwarded the way video can without pitch artifacts. `undefined` (the default, and whatever the caller returns + * before any frame has arrived) starts at the oldest buffered frame instead. */ getJoinAnchorUs?: () => number | undefined; + /** + * The target latency in microseconds, read on every clock report. It caps the core's jitter target, so the worklet + * never holds more than the latency the controller is aiming at. Default 2 s, where the core's own 400 ms ceiling + * applies instead. + */ + getTargetLatencyUs?: () => number | undefined; + /** Metering cadence in ms. Default 10. */ + tickIntervalMs?: number; } type RendererMessage = | { type: 'status'; status: AudioRendererStatus; error?: unknown } + | { type: 'decoder-error'; error: unknown } | { type: 'scheduled'; untilUs: number } | { type: 'underrun' }; -export interface AudioRendererActor extends Pick, 'snapshot'> { - /** Point the renderer at a (new) frame source + decoder config. */ - setTrack(source: AudioFrameSource | null, config: AudioDecoderConfig | null): void; - /** Master playout clock: current media time in microseconds, or `undefined` before any audio has been scheduled. */ - getClockTimeUs(): number | undefined; - destroy(): void; -} - // ============================================================================= // Implementation // ============================================================================= -const DEFAULT_SCHEDULE_MARGIN_S = 0.05; const DEFAULT_TICK_INTERVAL_MS = 10; +const DEFAULT_TARGET_LATENCY_US = 2_000_000; +/** The frame duration assumed before a frame has shown one: Opus's usual 20 ms. */ +const DEFAULT_FRAME_DURATION_US = 20_000; +/** Reports and pushed spans may disagree by sample rounding and resampling offsets. */ +const REPORT_TOLERANCE_US = 5_000; /** - * Max chunks in flight inside the decoder. The schedule horizon only moves when decoder _outputs_ arrive (async), so - * without this bound a single tick would dequeue an entire backlog — scheduling seconds of audio at once and - * sawtoothing the jitter-buffer depth the latency controller measures. + * How long a fill may sit short of playback before one more frame is pushed. The fill pushes exactly the jitter target + * plus a frame, so audio the core drops on arrival (behind what it already decoded) could otherwise leave it waiting + * forever. */ -const MAX_PENDING_DECODES = 4; -/** Allow sample rounding without treating a missing audio packet as contiguous. */ -const AUDIO_TIMESTAMP_TOLERANCE_US = 1_000; +const FILL_STALL_US = 100_000; /** * Media-time jump beyond which incoming audio is a timeline reset rather than a gap or reorder within the same timeline * (see `TIMELINE_DISCONTINUITY_US`). **Both directions.** Forward: a latency catch-up skipped groups, or the - * publisher's capture gapped — scheduling the jump as silence would keep the latency it was meant to shed. Backward: - * the publisher re-anchored the track's timeline (a capture source switch) — butt-joining it would keep audio playing + * publisher's capture gapped — playing the jump as silence would keep the latency it was meant to shed. Backward: the + * publisher re-anchored the track's timeline (a capture source switch) — butt-joining it would keep audio playing * seamlessly while the master clock silently steps backwards by the jump, holding the slaved video renderer "early" on - * every frame it ever receives again. Either way the schedule re-anchors on the jumped-to timeline. + * every frame it ever receives again. Either way playout re-anchors on the jumped-to timeline. */ const DISCONTINUITY_THRESHOLD_US = TIMELINE_DISCONTINUITY_US; +/** The core's `playbackRate` range; it throws outside it. */ +const MIN_PLAYBACK_RATE = 0.5; +const MAX_PLAYBACK_RATE = 2; + +/** The core on the real context behind `context`: an `AudioWorkletNode` needs one. */ +const createDefaultAudioPlayer: CreateAudioPlayer = async (context) => { + if (!context.baseContext) throw new Error('audio renderer context has no baseContext to host the AudioWorklet'); + + return createCoreAudioPlayer({ context: context.baseContext }); +}; export function createAudioRendererActor(options: CreateAudioRendererOptions): AudioRendererActor { const { audioContext } = options; - const scheduleMargin = options.scheduleMargin ?? DEFAULT_SCHEDULE_MARGIN_S; const inner = createTransitionActor( { status: 'idle', framesScheduled: 0, underruns: 0 }, @@ -133,6 +210,8 @@ export function createAudioRendererActor(options: CreateAudioRendererOptions): A switch (message.type) { case 'status': return { ...context, status: message.status, error: message.error }; + case 'decoder-error': + return { ...context, error: message.error }; case 'scheduled': return { ...context, @@ -146,16 +225,13 @@ export function createAudioRendererActor(options: CreateAudioRendererOptions): A } ); + let destroyed = false; + let player: AudioPlayer | undefined; + let input: AudioPlayerInput | undefined; let source: AudioFrameSource | null = null; let decoderConfig: AudioDecoderConfig | null = null; - let decoder: AudioDecoder | null = null; - let destroyed = false; - /** Timestamp of the last chunk fed to the decoder, for discontinuity detection. */ - let lastEnqueuedUs: number | undefined; - let lastDecodedDurationUs: number | undefined; - let lastDecodedEndUs: number | undefined; - let drainingDecoder: AudioDecoder | undefined; - /** The schedule has no anchor yet: the next one should be placed at the live edge. */ + + /** Playout has no anchor yet: the next one should be placed at the live edge. */ let needsJoinAnchor = true; /** * The pending anchor placement follows a mid-stream discontinuity rather than a track (re)start. The distinction @@ -165,103 +241,166 @@ export function createAudioRendererActor(options: CreateAudioRendererOptions): A * pitch-shifted seconds rebuilding. */ let rejoinAnchorArmed = false; + + // What this timeline has pushed into the core. Reset with the timeline. + let firstPushedUs: number | undefined; + let lastPushedUs: number | undefined; + /** Known once a packet's duration or two frames' spacing has shown it. */ + let frameDurationUs: number | undefined; + /** Where the newest pushed frame ends: the furthest the clock can read. */ + let pushedEndUs: number | undefined; + /** First timestamp pushed in the current fill, and the context time of the latest push. */ + let fillStartUs: number | undefined; + let lastPushAtUs = 0; + /** - * Media time the next scheduled buffer should be placed relative to: the landed re-join anchor. Consumed by the first - * buffer scheduled onto an empty schedule — it starts `(timestamp − scheduleFromUs)` later than arrival, with a - * bookkeeping segment spanning the gap so the clock advances smoothly from the anchor instead of stepping to the - * buffer. + * The clock: timeline position `mediaUs` is audible at context time `contextUs`, advancing with the context clock + * times `playbackRate` from there. Set by every core report, or **placed** at a re-join anchor before the core plays + * anything (see `dropToJoinAnchor`). + * + * Context time, not `performance.now()`, because it is what stops when the adapter suspends the context on pause: a + * wall-paced clock would run on through the pause, read the frozen worklet as dry and refill it, and step back on the + * first report after resume. The core times its reports on the same source (`localNow`), so its render clock's + * calibration does not carry a pause into the first reports after it either. */ - let scheduleFromUs: number | undefined; - + let anchor: { mediaUs: number; contextUs: number; placed: boolean; frozen?: boolean } | undefined; + /** Bumped per input, so a report from a removed input cannot anchor its replacement. */ + let inputGeneration = 0; /** - * One scheduled buffer's context-time span mapped to media time. The media time at context time `t` inside the span - * is `mediaUs + (t - startCtx) * 1e6 * rate`. + * Where the worklet ran dry. The clock holds there until a report of audio past it arrives: frames pushed to refill + * the worklet move `pushedEndUs` before they play, and must not drag the clock forward with them. */ - interface ClockSegment { - startCtx: number; - endCtx: number; - mediaUs: number; - rate: number; - } - + let dryLimitUs: number | undefined; /** - * The scheduled timeline, in start order. Bounded by the schedule horizon (`scheduleMargin * 4` of audio ≈ a dozen - * 20ms buffers) plus whatever `pruneSegments` hasn't collected yet. + * The position the clock held at through the latest underrun, kept as a floor after it ends. The first report of the + * refill is dated ahead by the output latency — the sample is not audible yet — so its anchor extrapolates to before + * the hold, and the clock would otherwise step back by that much. */ - const segments: ClockSegment[] = []; - const activeSources = new Set(); + let holdFloorUs: number | undefined; + /** + * The core has reported itself stopped since the dry point was set. Until it has, a report near the dry point is the + * tail of the audio before it, still in flight, and must not end the hold. + */ + let stoppedSinceDry = false; + /** The rate the current input plays at, and the clock advances at. */ + let playbackRate = 1; - const rate = (): number => options.getPlaybackRate?.() ?? 1; + /** The audio between rendering and audible, in media time. */ + const outputLatencyUs = (): number => + ((audioContext.baseLatency ?? 0) + (audioContext.outputLatency ?? 0)) * 1_000_000 * playbackRate; - const segmentEndMediaUs = (segment: ClockSegment): number => - segment.mediaUs + (segment.endCtx - segment.startCtx) * 1_000_000 * segment.rate; + const contextNowUs = (): number => audioContext.currentTime * 1_000_000; - /** True while the context clock is past everything scheduled (see `underruns`). */ - let starved = false; + /** The audible position extrapolated from the anchor, ignoring every limit. */ + const extrapolatedUs = (): number | undefined => + anchor && (anchor.frozen ? anchor.mediaUs : anchor.mediaUs + (contextNowUs() - anchor.contextUs) * playbackRate); /** - * Count the rising edge of a schedule that has run dry. An empty schedule is not an underrun — at join, and after - * every re-anchor that clears it, there was no schedule to starve. + * Apply the controller's rate from the next render quantum. The clock is rebased first, so the time before the change + * keeps the old pace; the audio still in the output path plays out at it too, which the next report corrects. */ - const detectUnderrun = (): void => { - const last = segments[segments.length - 1]; + const applyPlaybackRate = (active: AudioPlayerInput): void => { + const rate = clamp(options.getPlaybackRate?.() ?? 1, MIN_PLAYBACK_RATE, MAX_PLAYBACK_RATE); + if (rate === playbackRate) return; - if (!last) { - starved = false; - return; - } + if (anchor && !anchor.frozen) anchor = { ...anchor, mediaUs: extrapolatedUs()!, contextUs: contextNowUs() }; - const dry = last.endCtx <= audioContext.currentTime; + active.playbackRate = rate; + playbackRate = rate; + }; - if (dry && !starved) inner.send({ type: 'underrun' }); + /** The clock one input reports into; it goes deaf once that input is replaced. */ + const createClock = (): AudioPlayerClock => { + const generation = ++inputGeneration; + + return { + get targetLatencyUs() { + return options.getTargetLatencyUs?.() ?? DEFAULT_TARGET_LATENCY_US; + }, + localNow: contextNowUs, + reportAudio(timestampUs, atUs) { + if (generation !== inputGeneration) return; + + reportAudio(timestampUs, atUs); + }, + // The renderer resets the clock itself wherever a track or timeline ends. + stopAudio() {}, + }; + }; - starved = dry; + const raiseHoldFloor = (heldUs: number): void => { + holdFloorUs = Math.max(holdFloorUs ?? heldUs, heldUs); }; - /** Drop segments that finished playing; keep the current/newest one so the clock can hold on underrun. */ - const pruneSegments = (): void => { - const now = audioContext.currentTime; + /** + * A placed clock runs from the anchor until the core plays, but never past the audio pushed: a fill that stays short + * of the jitter target cannot start the worklet, and the clock would otherwise run on through audio not there. + */ + const placedPositionUs = (): number | undefined => { + const positionUs = extrapolatedUs(); - while (segments.length > 1 && now >= segments[1]!.startCtx) segments.shift(); + return positionUs === undefined || pushedEndUs === undefined ? positionUs : Math.min(positionUs, pushedEndUs); }; - const stopAll = (): void => { - for (const node of activeSources) { - try { - node.stop(); - } catch { - // never started or already stopped - } + /** `timestampUs` is audible at context time `atUs` (default now). */ + const reportAudio = (timestampUs: number, atUs: number | undefined): void => { + // Only audio pushed on this timeline: a report already in flight when + // the timeline restarted describes audio the cut dropped. + if ( + firstPushedUs === undefined || + pushedEndUs === undefined || + timestampUs < firstPushedUs - REPORT_TOLERANCE_US || + timestampUs > pushedEndUs + REPORT_TOLERANCE_US + ) { + return; + } + + // The refill's first report ends the hold. Its floor is what the clock + // reads at that moment — the audible position, which can still be short + // of the dry point while the device plays out what it had queued. + if ( + dryLimitUs !== undefined && + ((stoppedSinceDry && timestampUs >= dryLimitUs - REPORT_TOLERANCE_US) || + timestampUs > dryLimitUs + REPORT_TOLERANCE_US) + ) { + raiseHoldFloor(Math.min(extrapolatedUs() ?? dryLimitUs, dryLimitUs)); + dryLimitUs = undefined; } - activeSources.clear(); - segments.length = 0; + // Likewise the first report after a placed re-join, which is dated ahead + // of the placed clock by the output latency. + if (anchor?.placed) raiseHoldFloor(placedPositionUs()!); + + anchor = { mediaUs: timestampUs, contextUs: atUs ?? contextNowUs(), placed: false }; }; - const closeDecoder = (): void => { - if (decoder && decoder.state !== 'closed') decoder.close(); - - decoder = null; - drainingDecoder = undefined; - lastDecodedDurationUs = undefined; - lastDecodedEndUs = undefined; - // A fresh decoder starts a fresh input timeline: the next chunk fed is - // not a jump within the old one, so it must not re-trigger the - // discontinuity restart that closed this decoder — and a gap placement - // landed against the old timeline must not span into the new one. - lastEnqueuedUs = undefined; - scheduleFromUs = undefined; - stopAll(); + /** Forget everything pushed on the current timeline, and the clock placed on it. */ + const resetTimeline = (): void => { + firstPushedUs = undefined; + lastPushedUs = undefined; + frameDurationUs = undefined; + pushedEndUs = undefined; + fillStartUs = undefined; + anchor = undefined; + dryLimitUs = undefined; + holdFloorUs = undefined; + stoppedSinceDry = false; + }; + + const removeInput = (): void => { + if (!input) return; + + inputGeneration++; + input.remove(); + input = undefined; }; /** - * Discard buffered audio older than the join anchor, so the schedule starts at the live edge rather than at the - * oldest group the relay replayed. Video decode-forwards its backlog; audio cannot (pitch), so the backlog is dropped - * unheard. - * - * Anything already scheduled belongs to the pre-anchor timeline, so a landed drop closes the decoder — the jumped-to - * audio then anchors fresh instead of being butt-joined after a matching stretch of scheduled silence, which would - * keep exactly the latency the drop shed. + * Discard buffered audio older than the join anchor before the first push of a (re)start, so playout starts at the + * live edge rather than at the oldest group the relay replayed. A re-join anchor lands by **placement** when the + * whole buffer is already past it (see `rejoinAnchorArmed`): the clock is placed at the anchor and runs from there, + * and metering by timeline holds the head back until it is due, so playout resumes at the target depth rather than at + * the live edge. */ const dropToJoinAnchor = (): void => { const anchorUs = options.getJoinAnchorUs?.(); @@ -277,266 +416,132 @@ export function createAudioRendererActor(options: CreateAudioRendererOptions): A if (dropped > 0) { needsJoinAnchor = false; rejoinAnchorArmed = false; - // The buffer head now sits at the anchor, so scheduling it on - // arrival lands the clock there with the target depth behind it. - scheduleFromUs = undefined; - closeDecoder(); return; } - // Nothing behind the edge (yet). At a fresh join, stay armed: a relay's - // group replay arrives as a burst (oldest first), so the buffer's live - // edge may still be moving. - if (!rejoinAnchorArmed) return; - - // A discontinuity re-join whose whole buffer is already ahead of the - // anchor — no replay is coming (bursts arrive oldest-first, and the - // oldest is already past the anchor). Land the anchor by *placement* - // instead of by trimming: the head schedules at its distance from the - // anchor, so playout resumes at the target depth rather than racing to - // the live edge and pitch-rebuilding from zero. - if (!source.peek()) return; + // A fresh join stays armed: a relay's group replay arrives as a burst, + // oldest first, so the buffer's live edge may still be moving. + if (!rejoinAnchorArmed || !source.peek()) return; - scheduleFromUs = anchorUs; + anchor = { mediaUs: anchorUs, contextUs: contextNowUs(), placed: true }; rejoinAnchorArmed = false; needsJoinAnchor = false; }; - const scheduleAudioData = (data: AudioData): void => { - try { - if (destroyed || data.numberOfFrames === 0) return; - - lastDecodedDurationUs = data.duration; - lastDecodedEndUs = data.timestamp + data.duration; - - const buffer = audioContext.createBuffer(data.numberOfChannels, data.numberOfFrames, data.sampleRate); - - for (let channel = 0; channel < data.numberOfChannels; channel++) { - const channelData = new Float32Array(data.numberOfFrames); - - data.copyTo(channelData, { planeIndex: channel, format: 'f32-planar' }); - buffer.copyToChannel(channelData, channel); - } - - const currentRate = rate(); - // A jump past the discontinuity threshold in either direction means - // the source is on a different timeline — skipped ahead (latency - // catch-up: scheduling it as silence would keep the latency it was - // meant to shed, and re-trigger the catch-up loop forever) or - // re-anchored behind (a publisher source switch: butt-joining it - // would step the master clock backwards under the slaved video). - // Drop the stale schedule and anchor fresh. - // - // "Fresh" has to mean *through the join anchor*, the same way the dequeue - // path above re-arms it. Clearing the schedule alone left `needsJoinAnchor` - // false — and nothing on this path ever set it — so the next buffer took the - // no-previous-segment branch below and started at - // `currentTime + scheduleMargin`, keeping whatever latency the skip was meant - // to shed, with no pending re-anchor left to correct it. - // - // Returning rather than scheduling this buffer is what lets the re-anchor - // see an empty schedule (`tick` only drops while `segments.length === 0`); - // `finally` closes the AudioData, and one dropped buffer is cheaper than a - // timeline anchored at the wrong place. - // - // **Kept as a guard against a decoder, not as a live path.** The dequeue - // path's own check fires first on every jump that reaches this one: it is - // armed exactly when `previous` exists (only `closeDecoder` clears - // `lastEnqueuedUs`, and it clears `segments` in the same breath), it - // compares the same threshold, and `AudioDecoder.close()` discards every - // in-flight decode — measured, not assumed — so no pre-jump output can - // arrive after it either. What is left for this branch is a decoder whose - // *outputs* jump where its inputs did not, the mirror image of the - // timestamp rebasing the dequeue check exists for. Anything relying on it - // firing has no coverage; treat it as unexercised. - const previous = segments[segments.length - 1]; - - if (previous && Math.abs(data.timestamp - segmentEndMediaUs(previous)) > DISCONTINUITY_THRESHOLD_US) { - stopAll(); - needsJoinAnchor = true; - rejoinAnchorArmed = true; - return; + /** + * Whether `timestampUs` should reach the core now. Filling — before the first report, or after running dry — pushes + * the jitter target plus a frame, which is what the worklet waits for before playing. Playing pushes what will be + * rendered within the jitter target, so the worklet holds about that much and the rest waits in the subscriber. + */ + const isDue = (timestampUs: number, stats: AudioPlayerInputStats): boolean => { + const positionUs = anchor?.placed ? placedPositionUs() : extrapolatedUs(); + + // A frozen placement is a fill too: its clock waits for the worklet to + // start, so only the fill rules (including the stall rescue) can move it. + if (positionUs === undefined || dryLimitUs !== undefined || anchor?.frozen) { + if ( + fillStartUs === undefined || + timestampUs < fillStartUs + stats.jitterTargetUs + (frameDurationUs ?? DEFAULT_FRAME_DURATION_US) + ) { + return true; } - const last = segments[segments.length - 1]; - // Continue the scheduled timeline: media-contiguous data butt-joins - // the previous buffer; a media gap inserts a matching stretch of - // context-time silence. Late arrivals (and the first buffer) start - // no earlier than the context clock — the timeline shifts forward - // from there rather than trying to make up lost time. - let idealStart: number; - - if (last) { - const mediaEndUs = segmentEndMediaUs(last); - const gapS = Math.max(0, data.timestamp - mediaEndUs) / 1_000_000 / currentRate; - - idealStart = last.endCtx + gapS; - - if (gapS > 0) { - // The next packet establishes the gap's duration. Advance the - // master clock through that known silence so video need not - // freeze with each missing audio packet. - segments.push({ startCtx: last.endCtx, endCtx: idealStart, mediaUs: mediaEndUs, rate: currentRate }); - } - } else { - idealStart = audioContext.currentTime + scheduleMargin; - - // A landed re-join anchor places this first buffer at its distance - // from the anchor rather than at arrival, resuming playout at the - // target depth. The bookkeeping segment spanning the gap plays - // nothing (nothing is scheduled over it) but lets the clock advance - // smoothly from the anchor — so the slaved video renderer plays - // through the gap instead of seeing the clock step to the buffer. - if (scheduleFromUs !== undefined) { - const gapS = Math.max(0, data.timestamp - scheduleFromUs) / 1_000_000 / currentRate; - - if (gapS > 0) { - segments.push({ - startCtx: idealStart, - endCtx: idealStart + gapS, - mediaUs: scheduleFromUs, - rate: currentRate, - }); - idealStart += gapS; - } - - scheduleFromUs = undefined; - } - } + return contextNowUs() - lastPushAtUs > FILL_STALL_US && stats.bufferedUs < stats.jitterTargetUs; + } - const startAt = Math.max(idealStart, audioContext.currentTime); + return timestampUs < positionUs + outputLatencyUs() + stats.jitterTargetUs; + }; - const node = audioContext.createBufferSource(); + const push = (active: AudioPlayerInput, timestampUs: number, payload: Uint8Array): void => { + active.push({ timestampUs, payload }); - node.buffer = buffer; - node.playbackRate.value = currentRate; - node.connect(audioContext.destination); - node.onended = () => activeSources.delete(node); - node.start(startAt); - activeSources.add(node); + // Opus says how long each packet is. Otherwise the shortest step between + // pushed frames is the frame length: a step over lost packets is longer, + // and must not extend the audio past where it really ends. + const opusUs = decoderConfig?.codec === 'opus' ? opusPacketDurationUs(payload) : undefined; + const stepUs = lastPushedUs === undefined ? undefined : timestampUs - lastPushedUs; - const durationS = data.numberOfFrames / data.sampleRate / currentRate; + if (opusUs !== undefined) frameDurationUs = opusUs; + else if (stepUs !== undefined && stepUs > 0) frameDurationUs = Math.min(frameDurationUs ?? stepUs, stepUs); - segments.push({ startCtx: startAt, endCtx: startAt + durationS, mediaUs: data.timestamp, rate: currentRate }); - inner.send({ type: 'scheduled', untilUs: data.timestamp + data.duration }); - } finally { - data.close(); - } + firstPushedUs ??= timestampUs; + fillStartUs ??= timestampUs; + lastPushedUs = timestampUs; + lastPushAtUs = contextNowUs(); + pushedEndUs = timestampUs + (frameDurationUs ?? DEFAULT_FRAME_DURATION_US); + inner.send({ type: 'scheduled', untilUs: pushedEndUs }); }; - const ensureDecoder = (): AudioDecoder | null => { - if (decoder || !decoderConfig) return decoder; + /** + * The worklet runs dry once rendering passes everything pushed: metering pushes every frame as soon as it is due, so + * nothing was left to push. Each such edge is an underrun (`AudioRendererContext.underruns`). The core's own counter + * skips dry spells longer than its jitter ceiling, which `adaptLatencyTarget` still needs as failure evidence. + */ + const detectDry = (): void => { + const positionUs = extrapolatedUs(); + + if ( + !anchor || + anchor.placed || + dryLimitUs !== undefined || + pushedEndUs === undefined || + positionUs === undefined || + positionUs + outputLatencyUs() < pushedEndUs + ) { + return; + } - decoder = new AudioDecoder({ - output: scheduleAudioData, - error: (error) => inner.send({ type: 'status', status: 'error', error }), - }); - decoder.configure(decoderConfig); - return decoder; + dryLimitUs = pushedEndUs; + stoppedSinceDry = false; + fillStartUs = undefined; + inner.send({ type: 'underrun' }); }; const tick = (): void => { - // After a decoder error there is nothing productive to pull into — - // draining would silently strip the jitter buffer at tick rate. - if (destroyed || !source || drainingDecoder || inner.snapshot.get().context.status === 'error') return; + if (destroyed || !player || !source || !decoderConfig) return; try { - pruneSegments(); - detectUnderrun(); - - // Keep the schedule topped up to the margin horizon; every audio - // frame is independently decodable, so no keyframe gating. - while ( - (decoder?.decodeQueueSize ?? 0) < MAX_PENDING_DECODES && - (segments[segments.length - 1]?.endCtx ?? 0) - audioContext.currentTime < scheduleMargin * 4 - ) { - // Only while the schedule has no anchor to disturb: in steady state - // the latency controller owns depth corrections (rate nudges inside - // its deadband), and dropping audio here would preempt them. - if (needsJoinAnchor && segments.length === 0) dropToJoinAnchor(); + const active = (input ??= player.addInput({ + clock: createClock(), + decoderConfig, + playbackRate, + onError: (error) => inner.send({ type: 'decoder-error', error }), + })); + const { stats } = active; + + detectDry(); + applyPlaybackRate(active); + + const positionUs = extrapolatedUs(); + + if (dryLimitUs !== undefined && !stats.playing) stoppedSinceDry = true; + + // A placed clock that reaches the end of the audio pushed stops there + // until the core plays, so refill frames cannot drag it forward. + if (anchor?.placed && !anchor.frozen && pushedEndUs !== undefined && positionUs! >= pushedEndUs) { + anchor = { mediaUs: pushedEndUs, contextUs: contextNowUs(), placed: true, frozen: true }; + } + + for (;;) { + if (needsJoinAnchor && lastPushedUs === undefined) dropToJoinAnchor(); const next = source.peek(); if (!next) return; - // FFmpeg-backed audio decoders rebase output timestamps by frame - // accumulation from the first input, hiding an input-side jump - // from the scheduler. Restart the decoder at the discontinuity so - // outputs re-base to the jumped-to timeline (every audio frame is - // independently decodable, so a restart is glitch-free). - if (lastEnqueuedUs !== undefined && Math.abs(next.timestampUs - lastEnqueuedUs) > DISCONTINUITY_THRESHOLD_US) { - closeDecoder(); - // Re-place the anchor before feeding the jumped-to timeline: a - // forward jump (catch-up skip) landed on a group start still up - // to one GOP behind the live edge, and a backward jump (publisher - // re-anchor) is a whole new timeline whose edge the anchor must - // be re-read against. + // A timeline reset, either way: drop what the core holds and re-place + // the anchor on the jumped-to timeline (see `DISCONTINUITY_THRESHOLD_US`). + if (lastPushedUs !== undefined && Math.abs(next.timestampUs - lastPushedUs) > DISCONTINUITY_THRESHOLD_US) { + active.cut(); + resetTimeline(); needsJoinAnchor = true; rejoinAnchorArmed = true; continue; } - // Some audio decoders accumulate output timestamps across missing - // packets. Even a single short gap then permanently shifts the - // audio master clock behind the wire timeline. Drain before - // restarting so pending outputs and scheduled audio survive. The - // output-end check also handles codecs that split one input into - // several outputs: packet duration is only a prompt to check. - if ( - decoder && - lastEnqueuedUs !== undefined && - (lastDecodedDurationUs === undefined || - next.timestampUs - lastEnqueuedUs > lastDecodedDurationUs + AUDIO_TIMESTAMP_TOLERANCE_US) - ) { - const current = decoder; - const timestampUs = next.timestampUs; - - drainingDecoder = current; - void current.flush().then( - () => { - if (decoder !== current || destroyed) return; - - if (lastDecodedEndUs !== undefined && timestampUs - lastDecodedEndUs > AUDIO_TIMESTAMP_TOLERANCE_US) { - current.close(); - decoder = null; - lastDecodedDurationUs = undefined; - lastDecodedEndUs = undefined; - } - - lastEnqueuedUs = undefined; - drainingDecoder = undefined; - }, - (error) => { - if (decoder !== current || destroyed) return; - - drainingDecoder = undefined; - inner.send({ type: 'status', status: 'error', error }); - } - ); - return; - } + if (!isDue(next.timestampUs, stats)) return; - const active = ensureDecoder(); - // An errored decoder is closed but still referenced — stop pulling - // instead of feeding it doomed decode calls. - if (!active || active.state === 'closed') return; - - // Decode before dequeue: if decode throws, the frame stays - // buffered for a later decoder instead of being silently dropped. - active.decode( - new EncodedAudioChunk({ - // Every audio sample is independently decodable — `isKey` is a - // LOC group-boundary marker (meaningful for video's GOP - // gating), not a decodability signal here. A live-edge - // (`next-object`) join almost never lands on group object 0, - // and WebCodecs requires the first post-configure chunk to be - // 'key', so gating on `next.isKey` would reject it. - type: 'key', - timestamp: next.timestampUs, - data: next.payload, - }) - ); + push(active, next.timestampUs, next.payload); source.dequeue(); - lastEnqueuedUs = next.timestampUs; } } catch (error) { inner.send({ type: 'status', status: 'error', error }); @@ -544,24 +549,45 @@ export function createAudioRendererActor(options: CreateAudioRendererOptions): A }; const timer = setInterval(tick, options.tickIntervalMs ?? DEFAULT_TICK_INTERVAL_MS); + (options.createAudioPlayer ?? createDefaultAudioPlayer)(audioContext).then( + (created) => { + if (destroyed) { + created.close(); + return; + } + + try { + created.output.connect(audioContext.destination); + player = created; + } catch (error) { + created.close(); + inner.send({ type: 'status', status: 'error', error }); + } + }, + (error) => { + if (!destroyed) inner.send({ type: 'status', status: 'error', error }); + } + ); + return { get snapshot() { return inner.snapshot; }, setTrack(nextSource, nextConfig): void { - // A source without a decodable config can never schedule audio — it - // would spin every tick while the subscriber's jitter buffer grows - // unbounded. Fail fast instead. + // A source without a decodable config can never play — it would spin + // every tick while the subscriber's jitter buffer grows unbounded. + // Fail fast instead. const missingConfig = nextSource !== null && nextConfig === null; source = missingConfig ? null : nextSource; decoderConfig = nextConfig; needsJoinAnchor = true; - // A track (re)start is a fresh join, not a re-join: time-to-first-audio - // wins over landing at depth (see `rejoinAnchorArmed`). rejoinAnchorArmed = false; - closeDecoder(); + // An edge since the last tick still counts: the reset below forgets it. + detectDry(); + removeInput(); + resetTimeline(); if (missingConfig) { inner.send({ @@ -576,19 +602,23 @@ export function createAudioRendererActor(options: CreateAudioRendererOptions): A }, getClockTimeUs(): number | undefined { - pruneSegments(); - const segment = segments[0]; - if (!segment) return undefined; - - // Clamp into the segment: before the first sample plays the clock - // holds at its start; past the last scheduled sample (underrun) it - // holds at the end and resumes seamlessly when audio is scheduled. - const elapsed = Math.min( - Math.max(0, audioContext.currentTime - segment.startCtx), - segment.endCtx - segment.startCtx - ); - - return segment.mediaUs + elapsed * 1_000_000 * segment.rate; + const positionUs = extrapolatedUs(); + if (positionUs === undefined) return undefined; + + if (anchor?.placed) return placedPositionUs(); + + // Never past the audio pushed, and once the worklet ran dry never past + // where it did: the clock holds through an underrun and resumes when + // audio does. + const limitUs = dryLimitUs ?? pushedEndUs; + const clampedUs = limitUs === undefined ? positionUs : Math.min(positionUs, limitUs); + + if (holdFloorUs === undefined) return clampedUs; + + if (clampedUs < holdFloorUs) return holdFloorUs; + + holdFloorUs = undefined; + return clampedUs; }, destroy(): void { @@ -596,7 +626,9 @@ export function createAudioRendererActor(options: CreateAudioRendererOptions): A destroyed = true; clearInterval(timer); - closeDecoder(); + removeInput(); + player?.close(); + player = undefined; inner.destroy(); }, }; diff --git a/packages/spf/src/playback/actors/dom/tests/audio-renderer.test.ts b/packages/spf/src/playback/actors/dom/tests/audio-renderer.test.ts index 1d0cb2b02b..3c16007c2f 100644 --- a/packages/spf/src/playback/actors/dom/tests/audio-renderer.test.ts +++ b/packages/spf/src/playback/actors/dom/tests/audio-renderer.test.ts @@ -1,645 +1,615 @@ -import { describe, expect, it, vi } from 'vite-plus/test'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vite-plus/test'; import type { JitterFrame } from '../../track-subscriber'; -import { type AudioContextLike, type AudioFrameSource, createAudioRendererActor } from '../audio-renderer'; - -const SAMPLE_RATE = 48_000; -const FRAME_SAMPLES = 960; // 20ms opus frames -const FRAME_DURATION_US = 20_000; +import { + type AudioContextLike, + type AudioFrameSource, + type AudioPlayer, + type AudioPlayerInput, + type AudioPlayerInputOptions, + createAudioRendererActor, +} from '../audio-renderer'; + +const FRAME_US = 20_000; +const JITTER_TARGET_US = 60_000; +const TICK_MS = 10; +const CONFIG: AudioDecoderConfig = { codec: 'opus', sampleRate: 48_000, numberOfChannels: 1 }; + +/** An Opus TOC byte for one 20 ms CELT frame (config 19, code 0). */ +const OPUS_20MS_TOC = 19 << 3; + +function frameAt(timestampUs: number): JitterFrame { + return { groupId: timestampUs, objectId: 0, timestampUs, isKey: true, payload: new Uint8Array([OPUS_20MS_TOC]) }; +} -/** Encode a short opus sequence and return LOC-shaped jitter frames. */ -async function encodeTestFrames(count: number): Promise { +/** Frames every 20 ms over `[fromUs, untilUs)`. */ +function framesBetween(fromUs: number, untilUs: number): JitterFrame[] { const frames: JitterFrame[] = []; - const encoder = new AudioEncoder({ - output: (chunk) => { - const payload = new Uint8Array(chunk.byteLength); - - chunk.copyTo(payload); - frames.push({ - groupId: 0, - objectId: frames.length, - timestampUs: frames.length * FRAME_DURATION_US, - isKey: true, - payload, - }); - }, - error: (error) => { - throw error; - }, - }); - - encoder.configure({ codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1, bitrate: 32_000 }); - for (let i = 0; i < count; i++) { - const samples = new Float32Array(FRAME_SAMPLES); + for (let timestampUs = fromUs; timestampUs < untilUs; timestampUs += FRAME_US) frames.push(frameAt(timestampUs)); - for (let s = 0; s < FRAME_SAMPLES; s++) samples[s] = Math.sin((2 * Math.PI * 440 * s) / SAMPLE_RATE) * 0.5; + return frames; +} - const data = new AudioData({ - format: 'f32', - sampleRate: SAMPLE_RATE, - numberOfFrames: FRAME_SAMPLES, - numberOfChannels: 1, - timestamp: i * FRAME_DURATION_US, - data: samples, - }); +function queueSource(frames: JitterFrame[] = []): AudioFrameSource & { queue: JitterFrame[] } { + const queue = [...frames]; - encoder.encode(data); - data.close(); - } + return { queue, peek: () => queue[0], dequeue: () => queue.shift() }; +} - await encoder.flush(); - encoder.close(); - return frames; +interface FakeInput extends AudioPlayerInput { + options: AudioPlayerInputOptions; + pushed: number[]; + cuts: number; + removed: boolean; + stats: { playing: boolean; bufferedUs: number; jitterTargetUs: number }; } -function arraySource(frames: JitterFrame[]): AudioFrameSource { - const queue = [...frames]; +/** The core's surface with no audio behind it: pushes are recorded, and the test plays the worklet's reports. */ +function createFakePlayer() { + const inputs: FakeInput[] = []; + const connected: AudioNode[] = []; + let closed = false; + const player: AudioPlayer = { + output: { connect: (destination) => connected.push(destination) }, + addInput(options = {}) { + const input: FakeInput = { + options, + pushed: [], + cuts: 0, + removed: false, + playbackRate: options.playbackRate ?? 1, + stats: { playing: true, bufferedUs: 0, jitterTargetUs: JITTER_TARGET_US }, + push(frame) { + input.pushed.push(frame.timestampUs); + }, + cut() { + input.cuts++; + }, + remove() { + input.removed = true; + }, + }; + + inputs.push(input); + return input; + }, + close() { + closed = true; + }, + }; return { - peek: () => queue[0], - dequeue: () => queue.shift(), + player, + inputs, + connected, + get closed() { + return closed; + }, + /** The worklet reporting that the audio at `timestampUs` is audible now. */ + report(timestampUs: number, atUs: number) { + inputs[inputs.length - 1]!.options.clock!.reportAudio(timestampUs, atUs); + }, }; } -/** - * `AudioContextLike` with a hand-cranked clock: scheduling is a no-op, so tests can place `currentTime` anywhere and - * read the media clock back. - */ -function createFakeAudioContext(): AudioContextLike & { currentTime: number } { +// SAFETY: the renderer only hands `destination` to `output.connect`, which the fake records by identity. +const destination = {} as AudioNode; + +/** Context time follows the test's clock; `suspended` freezes it, as `AudioContext.suspend()` does. */ +let contextUs = 0; +let suspended = false; + +function fakeAudioContext(outputLatency?: number): AudioContextLike { return { - currentTime: 0, - destination: {} as AudioNode, - createBuffer: (_channels, length, sampleRate) => - ({ copyToChannel: () => {}, duration: length / sampleRate }) as unknown as AudioBuffer, - createBufferSource: () => - ({ - buffer: null, - playbackRate: { value: 1 }, - connect: () => {}, - start: () => {}, - stop: () => {}, - onended: null, - }) as unknown as AudioBufferSourceNode, + get currentTime() { + return contextUs / 1_000_000; + }, + destination, + outputLatency, }; } describe('createAudioRendererActor', () => { - it('decodes and schedules audio, exposing the master clock', async () => { - const frames = await encodeTestFrames(5); - // OfflineAudioContext satisfies AudioContextLike; its clock stays at 0, - // so the media clock holds at the anchor — deterministic for a test. - const audioContext = new OfflineAudioContext(1, SAMPLE_RATE, SAMPLE_RATE); - const renderer = createAudioRendererActor({ audioContext, scheduleMargin: 0.05 }); - - expect(renderer.getClockTimeUs()).toBeUndefined(); + let nowUs = 0; - renderer.setTrack(arraySource(frames), { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); + /** Move wall time, and context time with it unless the context is suspended. */ + const at = (us: number) => { + if (!suspended) contextUs += us - nowUs; - await vi.waitFor(() => expect(renderer.snapshot.get().context.framesScheduled).toBeGreaterThan(0), { - timeout: 5000, - }); - const context = renderer.snapshot.get().context; - - expect(context.status).toBe('rendering'); - expect(context.scheduledUntilUs).toBeGreaterThan(0); + nowUs = us; + }; - // Clock anchored to the first scheduled frame's media time (0), and it - // holds there while the offline context clock is not advancing. - expect(renderer.getClockTimeUs()).toBe(0); + beforeEach(() => { + vi.useFakeTimers(); + at(0); + contextUs = 0; + suspended = false; + }); - renderer.destroy(); + afterEach(() => { + vi.useRealTimers(); }); - // The clock deliberately hides an underrun — it clamps to the segment - // end and resumes seamlessly, which is right for presentation and - // useless as a signal. This counter is the only direct evidence that a - // target latency is below what the path sustains. - it('counts the rising edge of a schedule that ran dry', async () => { - const frames = await encodeTestFrames(3); - const audioContext = createFakeAudioContext(); - const renderer = createAudioRendererActor({ audioContext, scheduleMargin: 0.05, tickIntervalMs: 1 }); - - // Nothing scheduled is not an underrun: at join there is no schedule - // to starve. - await new Promise((resolve) => setTimeout(resolve, 30)); - expect(renderer.snapshot.get().context.underruns).toBe(0); - - renderer.setTrack(arraySource(frames), { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - await vi.waitFor(() => expect(renderer.snapshot.get().context.framesScheduled).toBeGreaterThan(0), { - timeout: 5000, + /** A renderer whose player is ready and whose first input exists. */ + async function setup( + options: { + getJoinAnchorUs?: () => number | undefined; + getPlaybackRate?: () => number; + outputLatency?: number; + } = {} + ) { + const { outputLatency, ...rendererOptions } = options; + const fake = createFakePlayer(); + const renderer = createAudioRendererActor({ + audioContext: fakeAudioContext(outputLatency), + createAudioPlayer: () => Promise.resolve(fake.player), + tickIntervalMs: TICK_MS, + ...rendererOptions, }); - expect(renderer.snapshot.get().context.underruns).toBe(0); - // The source is exhausted; walk the hardware clock past everything - // scheduled and the schedule is dry. - audioContext.currentTime = 60; - await vi.waitFor(() => expect(renderer.snapshot.get().context.underruns).toBe(1), { timeout: 5000 }); + await vi.advanceTimersByTimeAsync(0); + return { fake, renderer }; + } - // Still dry several ticks later — one starvation, one increment. - await new Promise((resolve) => setTimeout(resolve, 50)); - expect(renderer.snapshot.get().context.underruns).toBe(1); + const tick = () => vi.advanceTimersByTime(TICK_MS); + + it('connects the player output to the context destination', async () => { + const { fake, renderer } = await setup(); + + expect(fake.connected).toEqual([destination]); + expect(renderer.getClockTimeUs()).toBeUndefined(); renderer.destroy(); + expect(fake.closed).toBe(true); }); - it('clearing the track resets to idle and drops the anchor', async () => { - const frames = await encodeTestFrames(2); - const audioContext = new OfflineAudioContext(1, SAMPLE_RATE, SAMPLE_RATE); - const renderer = createAudioRendererActor({ audioContext }); + it('fills to the jitter target plus a frame, then meters by the clock', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 1_000_000)); - renderer.setTrack(arraySource(frames), { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - await vi.waitFor(() => expect(renderer.snapshot.get().context.framesScheduled).toBeGreaterThan(0), { - timeout: 5000, - }); + renderer.setTrack(source, CONFIG); + tick(); + + const input = fake.inputs[0]!; - renderer.setTrack(null, null); - expect(renderer.snapshot.get().context.status).toBe('idle'); + expect(input.options.decoderConfig).toEqual(CONFIG); + expect(input.pushed).toEqual([0, 20_000, 40_000, 60_000]); expect(renderer.getClockTimeUs()).toBeUndefined(); - renderer.destroy(); + // Playing: the worklet should hold about its target ahead of what renders. + fake.report(0, 0); + at(30_000); + tick(); + + expect(input.pushed.at(-1)).toBe(80_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(30_000, 0); + expect(renderer.snapshot.get().context.scheduledUntilUs).toBe(100_000); }); - it('errors when a source is set without a decoder config', async () => { - const audioContext = createFakeAudioContext(); - const renderer = createAudioRendererActor({ audioContext, tickIntervalMs: 5 }); - const queue: JitterFrame[] = [ - { groupId: 0, objectId: 0, timestampUs: 0, isKey: true, payload: new Uint8Array([1]) }, - { groupId: 0, objectId: 1, timestampUs: 20_000, isKey: true, payload: new Uint8Array([2]) }, - ]; - const source: AudioFrameSource = { peek: () => queue[0], dequeue: () => queue.shift() }; + it('joins at the edge, dropping buffered audio behind the anchor unheard', async () => { + const { fake, renderer } = await setup({ getJoinAnchorUs: () => 200_000 }); - renderer.setTrack(source, null); + renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); + tick(); - expect(renderer.snapshot.get().context.status).toBe('error'); - expect(renderer.snapshot.get().context.error).toBeInstanceOf(Error); - // The rejected source must not be drained (or retained) by the tick loop. - await new Promise((resolve) => setTimeout(resolve, 50)); - expect(queue).toHaveLength(2); + expect(fake.inputs[0]!.pushed[0]).toBe(200_000); + }); - renderer.destroy(); + it('holds the clock where the worklet ran dry, through the refill, until audio past it plays', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + + at(100_000); + tick(); + expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); + + // Audio comes back: the refill moves the pushed end, not the clock. + fake.inputs[0]!.stats.playing = false; + source.queue.push(...framesBetween(80_000, 400_000)); + at(300_000); + tick(); + expect(fake.inputs[0]!.pushed.slice(4)).toEqual([80_000, 100_000, 120_000, 140_000]); + expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); + + fake.report(80_000, 320_000); + at(330_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(90_000, 0); }); - it('stops draining the jitter buffer after a decoder error', async () => { - const frames = await encodeTestFrames(10); - const audioContext = createFakeAudioContext(); - const queue = [...frames]; - const source: AudioFrameSource = { peek: () => queue[0], dequeue: () => queue.shift() }; - const renderer = createAudioRendererActor({ audioContext, tickIntervalMs: 5 }); + it('does not step back when the refill is reported ahead of being audible', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 80_000)); - // A well-formed but undecodable config: the decoder rejects either at - // configure or on first decode, and the pull loop must stop instead of - // stripping the queue one frame per tick. - renderer.setTrack(source, { codec: 'bogus-codec', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + at(100_000); + tick(); - await vi.waitFor(() => expect(renderer.snapshot.get().context.status).toBe('error'), { timeout: 5000 }); - const remaining = queue.length; + fake.inputs[0]!.stats.playing = false; + source.queue.push(...framesBetween(80_000, 400_000)); + at(300_000); + tick(); - expect(remaining).toBeGreaterThan(0); - await new Promise((resolve) => setTimeout(resolve, 100)); - expect(queue).toHaveLength(remaining); + // Rendered now, audible 100 ms from now: the anchor alone reads 80 ms behind the hold. + fake.report(80_000, 400_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); - renderer.destroy(); + at(410_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(90_000, 0); }); - it('decodes a live-edge join whose first frame is mid-group (isKey: false)', async () => { - const frames = await encodeTestFrames(3); + it('resumes from the audible position when audio returns before the device drained', async () => { + const { fake, renderer } = await setup({ outputLatency: 0.05 }); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + + // Rendering passed the last pushed sample; 50 ms of it is still in the device. + at(30_000); + tick(); + expect(renderer.getClockTimeUs()).toBeCloseTo(30_000, 0); + + fake.inputs[0]!.stats.playing = false; + source.queue.push(...framesBetween(80_000, 400_000)); + at(40_000); + tick(); + fake.report(80_000, 90_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(40_000, 0); + }); - // A `next-object` subscribe joins live, so the first delivered frame - // is whatever object is newest right now — essentially never a LOC - // group's object 0 (`isKey: true`). The regression: decode() gated the - // WebCodecs chunk type on `isKey`, and WebCodecs rejects a 'delta' - // chunk as the first one fed to a fresh decoder. - frames[0] = { ...frames[0]!, groupId: 3, objectId: 7, isKey: false }; - const audioContext = new OfflineAudioContext(1, SAMPLE_RATE, SAMPLE_RATE); - const renderer = createAudioRendererActor({ audioContext, scheduleMargin: 0.05 }); + it('ends pushed audio where the last packet ends, not across a gap of lost packets', async () => { + const { fake, renderer } = await setup(); - renderer.setTrack(arraySource(frames), { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); + renderer.setTrack(queueSource([0, 20_000, 40_000, 100_000].map(frameAt)), CONFIG); + tick(); + fake.report(0, 0); + at(60_000); + tick(); - await vi.waitFor(() => expect(renderer.snapshot.get().context.framesScheduled).toBeGreaterThan(0), { - timeout: 5000, - }); - expect(renderer.snapshot.get().context.status).toBe('rendering'); + expect(fake.inputs[0]!.pushed).toEqual([0, 20_000, 40_000, 100_000]); + expect(renderer.snapshot.get().context.scheduledUntilUs).toBe(120_000); + }); - renderer.destroy(); + it('keeps the hold through a late report of the audio before the underrun', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + at(100_000); + tick(); + + // The refill is pushed before the core has said it stopped; then the + // tail's last report, still in flight, arrives. + source.queue.push(...framesBetween(80_000, 400_000)); + tick(); + fake.report(77_333, 77_333); + at(200_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); }); - it('applies rate changes forward-only: already-scheduled audio keeps its clock mapping', async () => { - const frames = await encodeTestFrames(5); - const audioContext = createFakeAudioContext(); - let playbackRate = 1; - const renderer = createAudioRendererActor({ - audioContext, - scheduleMargin: 0.05, - getPlaybackRate: () => playbackRate, - }); + it('holds a placed re-join clock at the audio pushed while the fill is short', async () => { + let anchorUs: number | undefined; + const { fake, renderer } = await setup({ getJoinAnchorUs: () => anchorUs }); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + + anchorUs = 4_900_000; + source.queue.push(frameAt(5_000_000)); + tick(); + at(100_000); + tick(); + expect(fake.inputs[0]!.pushed.at(-1)).toBe(5_000_000); + + // One packet cannot start the worklet; the clock stops at its end. + at(2_100_000); + tick(); + expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); + + // The refill plays, reported ahead of being audible: no step back. + source.queue.push(...framesBetween(5_020_000, 5_200_000)); + tick(); + fake.report(5_000_000, 2_150_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); + }); - renderer.setTrack(arraySource(frames), { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - // The decoder may split outputs, so wait on scheduled media time (the - // last input chunk spans 80–100ms) rather than an output count. - await vi.waitFor( - () => expect(renderer.snapshot.get().context.scheduledUntilUs).toBeGreaterThanOrEqual(4.5 * FRAME_DURATION_US), - { timeout: 5000 } - ); + it('gives the core a report time source that stops while the context is suspended', async () => { + const { fake, renderer } = await setup(); - // 30ms into playback (the timeline starts at the 50ms schedule margin). - audioContext.currentTime = 0.08; - const before = renderer.getClockTimeUs()!; + renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); + tick(); - expect(before).toBeGreaterThan(0); - expect(before).toBeLessThan(5 * FRAME_DURATION_US); + const clock = fake.inputs[0]!.options.clock!; - // A rate nudge must not rescale time already scheduled at the old - // rate — the regression here was `clock = anchor + elapsed * newRate`, - // which jumped by 5% of the total elapsed interval on every nudge. - playbackRate = 1.5; - expect(renderer.getClockTimeUs()).toBe(before); + at(40_000); + expect(clock.localNow()).toBeCloseTo(40_000, 0); - // The clock advances with the context clock through the scheduled - // segments regardless of the current rate setting. - audioContext.currentTime = 0.09; - const later = renderer.getClockTimeUs()!; + suspended = true; + at(540_000); + expect(clock.localNow()).toBeCloseTo(40_000, 0); - expect(later).toBeGreaterThan(before); - expect(later).toBeLessThanOrEqual(before + 10_001); + // The first report after resume, timed on that source, lands where audio is. + suspended = false; + at(560_000); + fake.report(60_000, clock.localNow()); + expect(renderer.getClockTimeUs()).toBeCloseTo(60_000, 0); + }); - renderer.destroy(); + it('drops what the core holds at a timeline reset and ignores reports from before it', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + + // A publisher re-anchor: the timeline steps back by far more than a gap. + source.queue.push(...framesBetween(-5_000_000, -4_000_000)); + tick(); + + const input = fake.inputs[0]!; + + expect(input.cuts).toBe(1); + expect(input.pushed.at(4)).toBe(-5_000_000); + expect(renderer.getClockTimeUs()).toBeUndefined(); + + // Still in flight from the old timeline when the cut landed. + fake.report(40_000, 0); + expect(renderer.getClockTimeUs()).toBeUndefined(); + + fake.report(-5_000_000, 0); + expect(renderer.getClockTimeUs()).toBeCloseTo(-5_000_000, 0); }); - it('holds on underrun and resumes from late-scheduled audio, not the ideal timeline', async () => { - const frames = await encodeTestFrames(3); - const audioContext = createFakeAudioContext(); - const queue = [...frames.slice(0, 2)]; - const source: AudioFrameSource = { peek: () => queue[0], dequeue: () => queue.shift() }; - const renderer = createAudioRendererActor({ audioContext, scheduleMargin: 0.05 }); - - renderer.setTrack(source, { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - await vi.waitFor( - () => expect(renderer.snapshot.get().context.scheduledUntilUs).toBeGreaterThanOrEqual(1.5 * FRAME_DURATION_US), - { timeout: 5000 } - ); - - // Context clock runs past everything scheduled: the media clock holds - // at the end of scheduled audio instead of drifting ahead. - audioContext.currentTime = 0.5; - const held = renderer.getClockTimeUs()!; - - audioContext.currentTime = 0.6; - expect(renderer.getClockTimeUs()).toBe(held); - expect(held).toBeGreaterThanOrEqual(FRAME_DURATION_US); - expect(held).toBeLessThanOrEqual(2 * FRAME_DURATION_US); - - // The late frame schedules at the context clock; the media clock - // resumes seamlessly from where it held — not from where the original - // anchor says it "should" be (~550ms in). - queue.push(frames[2]!); - await vi.waitFor( - () => expect(renderer.snapshot.get().context.scheduledUntilUs).toBeGreaterThanOrEqual(2.5 * FRAME_DURATION_US), - { timeout: 5000 } - ); - audioContext.currentTime = 0.61; - const resumed = renderer.getClockTimeUs()!; - - expect(resumed).toBeGreaterThanOrEqual(held); - expect(resumed).toBeLessThan(held + FRAME_DURATION_US); + it('places a re-join whose whole buffer is past the anchor at its distance from it', async () => { + let anchorUs: number | undefined; + const { fake, renderer } = await setup({ getJoinAnchorUs: () => anchorUs }); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); - renderer.destroy(); + // A catch-up skip landed on a group 100 ms past where the anchor falls. + anchorUs = 4_900_000; + source.queue.push(...framesBetween(5_000_000, 6_000_000)); + tick(); + + const input = fake.inputs[0]!; + + expect(input.pushed).toHaveLength(4); + expect(renderer.getClockTimeUs()).toBeCloseTo(4_900_000, 0); + + at(50_000); + tick(); + expect(input.pushed.at(4)).toBe(5_000_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(4_950_000, 0); }); - it.each([20_000, 200_000])('preserves a %iµs gap in the incoming audio timeline', async (gapUs) => { - const frames = (await encodeTestFrames(8)).map((frame, index) => ({ - ...frame, - timestampUs: frame.timestampUs + (index >= 2 ? gapUs : 0), - })); - const audioContext = createFakeAudioContext(); - const renderer = createAudioRendererActor({ audioContext, scheduleMargin: 0.2 }); - - try { - renderer.setTrack(arraySource(frames), { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - - await vi.waitFor( - () => expect(renderer.snapshot.get().context.scheduledUntilUs).toBeGreaterThanOrEqual(150_000 + gapUs), - { - timeout: 2000, - } - ); - // A known media gap is scheduled silence, not a stalled clock: - // video can keep advancing while no audio samples are available. - audioContext.currentTime = 0.24 + gapUs / 2_000_000; - expect(renderer.getClockTimeUs()).toBeGreaterThanOrEqual(30_000 + gapUs / 2); - expect(renderer.getClockTimeUs()).toBeLessThanOrEqual(50_000 + gapUs / 2); - audioContext.currentTime = 0.3 + gapUs / 1_000_000; - expect(renderer.getClockTimeUs()).toBeGreaterThanOrEqual(90_000 + gapUs); - expect(renderer.getClockTimeUs()).toBeLessThanOrEqual(110_000 + gapUs); - } finally { - renderer.destroy(); - } + it('freezes the clock and stops metering while the context is suspended', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 1_000_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + at(30_000); + tick(); + + const pushed = fake.inputs[0]!.pushed.length; + + // Paused for two seconds: nothing plays, so nothing is due and nothing ran dry. + suspended = true; + at(2_030_000); + tick(); + expect(renderer.getClockTimeUs()).toBeCloseTo(30_000, 0); + expect(fake.inputs[0]!.pushed).toHaveLength(pushed); + + suspended = false; + at(2_040_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(40_000, 0); }); - it('does not revive a decoder whose track was cleared while it drained', async () => { - const frames = await encodeTestFrames(5); - const audioContext = createFakeAudioContext(); - const createSource = vi.spyOn(audioContext, 'createBufferSource'); - const flush = AudioDecoder.prototype.flush; - let finishDrain!: () => void; - const drained = new Promise((resolve) => { - finishDrain = resolve; - }); - const pendingFlush = vi.spyOn(AudioDecoder.prototype, 'flush').mockImplementation(async function ( - this: AudioDecoder - ) { - await flush.call(this); - await drained; - }); - const renderer = createAudioRendererActor({ audioContext }); - - try { - renderer.setTrack(arraySource(frames), { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - await vi.waitFor(() => expect(pendingFlush).toHaveBeenCalled()); - renderer.setTrack(null, null); - const scheduled = createSource.mock.calls.length; - - finishDrain(); - await new Promise((resolve) => setTimeout(resolve, 50)); - expect(renderer.snapshot.get().context.status).toBe('idle'); - expect(renderer.getClockTimeUs()).toBeUndefined(); - expect(createSource).toHaveBeenCalledTimes(scheduled); - } finally { - finishDrain(); - renderer.destroy(); - pendingFlush.mockRestore(); - createSource.mockRestore(); - } + it('ignores reports from an input that a track change removed', async () => { + const { fake, renderer } = await setup(); + + renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); + tick(); + + const stale = fake.inputs[0]!.options.clock!; + + renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); + tick(); + stale.reportAudio(40_000, 0); + expect(renderer.getClockTimeUs()).toBeUndefined(); + + fake.report(0, 0); + expect(renderer.getClockTimeUs()).toBeCloseTo(0, 0); }); - it('re-anchors instead of inserting silence on a large timestamp jump', async () => { - const frames = await encodeTestFrames(3); - // Latency catch-up: everything from the third frame on arrives from far - // ahead of the scheduled timeline. The whole tail moves (the encoder's - // flush can emit one more output than was fed) — a real skip relocates - // the stream, it does not interleave two timelines. - const JUMP_US = 5_000_000; + it('counts every time the worklet runs dry, across track changes', async () => { + const { fake, renderer } = await setup(); - for (let i = 2; i < frames.length; i++) { - frames[i] = { ...frames[i]!, timestampUs: JUMP_US + (i - 2) * FRAME_DURATION_US }; - } + renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); + tick(); + fake.report(0, 0); - const audioContext = createFakeAudioContext(); - const renderer = createAudioRendererActor({ audioContext, scheduleMargin: 0.05 }); + // A stall far longer than any jitter ceiling still counts, once. + at(100_000); + tick(); + at(3_000_000); + tick(); + expect(renderer.snapshot.get().context.underruns).toBe(1); - renderer.setTrack(arraySource(frames), { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - await vi.waitFor(() => expect(renderer.snapshot.get().context.scheduledUntilUs).toBeGreaterThanOrEqual(JUMP_US), { - timeout: 5000, - }); + renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); + tick(); + fake.report(0, 3_000_000); + at(3_100_000); + tick(); + expect(renderer.snapshot.get().context.underruns).toBe(2); + }); - // Without re-anchoring the jump maps to ~5s of scheduled silence, so - // shortly into playback the clock would still sit in the pre-jump - // timeline; the reset schedules the jumped audio right at the context - // clock (+margin) and the media clock lands past the jump immediately. - audioContext.currentTime = 0.06; - expect(renderer.getClockTimeUs()).toBeGreaterThanOrEqual(JUMP_US); + it('counts a dry edge that a track change lands on before the next tick', async () => { + const { fake, renderer } = await setup(); - renderer.destroy(); + renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); + tick(); + fake.report(0, 0); + + at(100_000); + renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); + expect(renderer.snapshot.get().context.underruns).toBe(1); }); - it('anchors at the live edge, discarding the replayed backlog unheard', async () => { - // The relay replayed 1s of audio before the first tick runs. - const frames = await encodeTestFrames(50); - const NEWEST_US = 49 * FRAME_DURATION_US; - const TARGET_S = 0.2; - const ANCHOR_US = NEWEST_US - TARGET_S * 1_000_000; - const audioContext = createFakeAudioContext(); - const queue = [...frames]; - const source: AudioFrameSource = { peek: () => queue[0], dequeue: () => queue.shift() }; - const renderer = createAudioRendererActor({ - audioContext, - scheduleMargin: 0.05, - getJoinAnchorUs: () => ANCHOR_US, - }); + it('rescues a frozen re-join placement whose next packets are past the fill horizon', async () => { + let anchorUs: number | undefined; + const { fake, renderer } = await setup({ getJoinAnchorUs: () => anchorUs }); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + + anchorUs = 4_900_000; + source.queue.push(frameAt(5_000_000)); + tick(); + at(200_000); + tick(); + expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); + + // Packets resume 80 ms after the lone one ends: beyond the fill horizon. + fake.inputs[0]!.stats.playing = false; + source.queue.push(...framesBetween(5_100_000, 5_400_000)); + at(400_000); + tick(); + expect(fake.inputs[0]!.pushed).toContain(5_100_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); + }); - renderer.setTrack(source, { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - await vi.waitFor(() => expect(renderer.snapshot.get().context.framesScheduled).toBeGreaterThan(0), { - timeout: 5000, - }); + it('applies the playout rate to the input and paces the clock with it', async () => { + let rate = 1; + const { fake, renderer } = await setup({ getPlaybackRate: () => rate }); + + renderer.setTrack(queueSource(framesBetween(0, 2_000_000)), CONFIG); + tick(); + fake.report(0, 0); - // The 39 pre-anchor frames left the buffer without being decoded: none - // remain, and nothing like that many were scheduled. - expect(queue.every((frame) => frame.timestampUs >= ANCHOR_US)).toBe(true); - expect(renderer.snapshot.get().context.framesScheduled).toBeLessThan(15); - expect(renderer.snapshot.get().context.scheduledUntilUs).toBeGreaterThan(ANCHOR_US); + for (let us = 10_000; us <= 100_000; us += 10_000) { + at(us); + tick(); + } - // The clock reads the anchor, not 0 — and the backlog did not become - // ~800ms of scheduled silence in front of it, which would put the - // clock back at the start of the timeline this far into playback. - expect(renderer.getClockTimeUs()).toBeGreaterThanOrEqual(ANCHOR_US); - audioContext.currentTime = 0.06; - expect(renderer.getClockTimeUs()).toBeGreaterThanOrEqual(ANCHOR_US); + expect(renderer.getClockTimeUs()).toBeCloseTo(100_000, 0); - renderer.destroy(); + rate = 1.05; + tick(); + expect(fake.inputs[0]!.playbackRate).toBe(1.05); + + // The time before the change keeps its pace; only what follows speeds up. + for (let us = 110_000; us <= 200_000; us += 10_000) { + at(us); + tick(); + } + + expect(renderer.getClockTimeUs()).toBeCloseTo(205_000, 0); }); - it('leaves a buffer shallower than the target alone', async () => { - const frames = await encodeTestFrames(5); - const audioContext = createFakeAudioContext(); - const queue = [...frames]; - const source: AudioFrameSource = { peek: () => queue[0], dequeue: () => queue.shift() }; - // A live join with no replay: the edge is inside the target, so the - // anchor sits behind the buffer head and must not move playout. - const renderer = createAudioRendererActor({ - audioContext, - scheduleMargin: 0.05, - getJoinAnchorUs: () => -400_000, - }); + it('clamps the playout rate to the range the core accepts, and starts a new input at it', async () => { + let rate = 3; + const { fake, renderer } = await setup({ getPlaybackRate: () => rate }); - renderer.setTrack(source, { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - await vi.waitFor(() => expect(renderer.snapshot.get().context.framesScheduled).toBeGreaterThan(0), { - timeout: 5000, - }); + renderer.setTrack(queueSource(framesBetween(0, 200_000)), CONFIG); + tick(); + expect(fake.inputs[0]!.playbackRate).toBe(2); - expect(renderer.getClockTimeUs()).toBe(0); + rate = 0; + tick(); + expect(fake.inputs[0]!.playbackRate).toBe(0.5); - renderer.destroy(); + renderer.setTrack(queueSource(framesBetween(0, 200_000)), CONFIG); + tick(); + expect(fake.inputs[1]!.options.playbackRate).toBe(0.5); + expect(renderer.snapshot.get().context.status).toBe('rendering'); }); - it('re-anchors at the live edge after a catch-up skip, not at the group start', async () => { - const frames = await encodeTestFrames(30); - const audioContext = createFakeAudioContext(); - // Two 300ms stretches either side of a catch-up skip: the controller - // drops the buffer to the newest keyframe-led group, so the surviving - // group still starts ~5s of media time behind its own newest frame. - const SKIP_US = 5_000_000; - const queue = frames.slice(0, 15); - const source: AudioFrameSource = { peek: () => queue[0], dequeue: () => queue.shift() }; - let anchorUs: number | undefined; + it('caps the core jitter target at the target latency', async () => { + const fake = createFakePlayer(); const renderer = createAudioRendererActor({ - audioContext, - scheduleMargin: 0.05, - getJoinAnchorUs: () => anchorUs, + audioContext: fakeAudioContext(), + createAudioPlayer: () => Promise.resolve(fake.player), + getTargetLatencyUs: () => 250_000, + tickIntervalMs: TICK_MS, }); - renderer.setTrack(source, { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - await vi.waitFor(() => expect(renderer.snapshot.get().context.framesScheduled).toBeGreaterThan(0), { - timeout: 5000, - }); - expect(renderer.getClockTimeUs()).toBe(0); - - // The skip lands: the surviving group runs SKIP_US … SKIP_US + 300ms. - const survivingGroup = frames - .slice(15) - .map((frame, i) => ({ ...frame, timestampUs: SKIP_US + i * FRAME_DURATION_US })); - const newestUs = survivingGroup[survivingGroup.length - 1]!.timestampUs; - - anchorUs = newestUs - 100_000; - queue.length = 0; - queue.push(...survivingGroup); - // Play out what was scheduled before the skip, so the schedule horizon - // reopens and the renderer pulls the jumped-to timeline. - audioContext.currentTime = 0.25; - - // Playout resumes within the target of the edge, rather than at the - // group start SKIP_US — which is what the plain discontinuity - // re-anchor would have done. - await vi.waitFor(() => expect(renderer.getClockTimeUs()).toBeGreaterThanOrEqual(anchorUs!), { timeout: 5000 }); - expect(renderer.getClockTimeUs()).toBeLessThanOrEqual(newestUs + FRAME_DURATION_US); + await vi.advanceTimersByTimeAsync(0); + renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); + tick(); - renderer.destroy(); + expect(fake.inputs[0]!.options.clock!.targetLatencyUs).toBe(250_000); }); - // A publisher that replaces its capture source re-anchors that track's - // timeline; landing *behind* the old one must be treated as the timeline - // reset it is. Butt-joining it (the backward jump slips past a - // forward-only check) keeps audio playing seamlessly while the master - // clock silently steps backwards by the jump — and the slaved video - // renderer then holds every frame it ever receives "early", frozen - // exactly the step behind for the rest of the stream. - it('re-anchors onto a timeline that stepped backwards instead of continuing the old one', async () => { - const frames = await encodeTestFrames(6); - const BASE_US = 10_000_000; - const STEP_BACK_US = 5_000_000; - const TARGET_US = 100_000; - const old = frames.slice(0, 3).map((frame, i) => ({ ...frame, timestampUs: BASE_US + i * FRAME_DURATION_US })); - const fresh = frames - .slice(3) - .map((frame, i) => ({ ...frame, timestampUs: BASE_US - STEP_BACK_US + i * FRAME_DURATION_US })); - const audioContext = createFakeAudioContext(); - const queue = [...old]; - const source: AudioFrameSource = { peek: () => queue[0], dequeue: () => queue.shift() }; - let edgeUs = old[old.length - 1]!.timestampUs; - const renderer = createAudioRendererActor({ - audioContext, - scheduleMargin: 0.05, - getJoinAnchorUs: () => edgeUs - TARGET_US, - }); + it('defaults to the core on the context baseContext, and fails without one', async () => { + // The core's first act is loading its worklet onto the context; failing + // that with a marker shows the default reached the core on this context. + const workletUnavailable = new Error('worklet unavailable'); + const addModule = vi.fn(() => Promise.reject(workletUnavailable)); + // SAFETY: the core touches only `audioWorklet` before the rejected load. + const baseContext = { audioWorklet: { addModule } } as unknown as BaseAudioContext; + const renderer = createAudioRendererActor({ audioContext: { ...fakeAudioContext(), baseContext } }); + + await vi.advanceTimersByTimeAsync(0); + expect(addModule).toHaveBeenCalledTimes(1); + expect(renderer.snapshot.get().context.error).toBe(workletUnavailable); + renderer.destroy(); - renderer.setTrack(source, { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - await vi.waitFor( - () => - expect(renderer.snapshot.get().context.scheduledUntilUs).toBeGreaterThanOrEqual(BASE_US + FRAME_DURATION_US), - { timeout: 5000 } - ); - audioContext.currentTime = 0.06; - expect(renderer.getClockTimeUs()).toBeGreaterThanOrEqual(BASE_US); - - // The switched source's timeline arrives a whole step behind, delivered - // continuously; the subscriber's edge has already reset onto it. - queue.push(...fresh); - edgeUs = fresh[fresh.length - 1]!.timestampUs; - - // The clock lands on the new timeline at the join anchor — with the fake - // context clock frozen inside the old schedule, a butt-join would keep - // reading the old timeline here instead. - await vi.waitFor( - () => { - const clock = renderer.getClockTimeUs(); - - expect(clock).toBeDefined(); - expect(clock!).toBeLessThan(BASE_US); - expect(clock!).toBeGreaterThanOrEqual(edgeUs - TARGET_US); - }, - { timeout: 5000 } - ); - - // …and playout proceeds through the anchor gap into the new audio. - audioContext.currentTime = 0.2; - await vi.waitFor(() => expect(renderer.getClockTimeUs()).toBeGreaterThanOrEqual(fresh[0]!.timestampUs), { - timeout: 5000, - }); + const bare = createAudioRendererActor({ audioContext: fakeAudioContext() }); - renderer.destroy(); + await vi.advanceTimersByTimeAsync(0); + expect(bare.snapshot.get().context.status).toBe('error'); + bare.destroy(); }); - // A discontinuity interrupts playout that already ran at depth, so the - // re-join lands *at the join anchor* (the target latency) rather than at - // arrival. Landing at arrival collapses playout to the live edge — the - // reported "audio jumps to real-time" — which the latency controller then - // spends tens of pitch-shifted seconds rebuilding at its ±5% nudge. - it('lands a discontinuity re-join at the join anchor instead of racing to the arrival edge', async () => { - const frames = await encodeTestFrames(6); - const JUMP_US = 5_000_000; - const TARGET_US = 200_000; - const old = frames.slice(0, 3).map((frame, i) => ({ ...frame, timestampUs: i * FRAME_DURATION_US })); - const fresh = frames.slice(3).map((frame, i) => ({ ...frame, timestampUs: JUMP_US + i * FRAME_DURATION_US })); - const audioContext = createFakeAudioContext(); - const queue = [...old]; - const source: AudioFrameSource = { peek: () => queue[0], dequeue: () => queue.shift() }; - let edgeUs = old[old.length - 1]!.timestampUs; - const renderer = createAudioRendererActor({ - audioContext, - scheduleMargin: 0.05, - getJoinAnchorUs: () => edgeUs - TARGET_US, - }); + it('fails a track with no decoder config, and a player that cannot be created', async () => { + const { renderer } = await setup(); - renderer.setTrack(source, { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - await vi.waitFor( - () => expect(renderer.snapshot.get().context.scheduledUntilUs).toBeGreaterThanOrEqual(FRAME_DURATION_US), - { timeout: 5000 } - ); - - queue.push(...fresh); - edgeUs = fresh[fresh.length - 1]!.timestampUs; - const anchorUs = edgeUs - TARGET_US; - - // The clock resumes at the anchor — target depth behind the jumped-to - // edge — not at the buffer head the way an at-arrival landing would. - await vi.waitFor( - () => { - const clock = renderer.getClockTimeUs(); - - expect(clock).toBeDefined(); - expect(clock!).toBeGreaterThanOrEqual(anchorUs); - expect(clock!).toBeLessThan(fresh[0]!.timestampUs); - }, - { timeout: 5000 } - ); - - // Advancing the context clock through the anchor gap reaches the head. - audioContext.currentTime = 0.3; - await vi.waitFor(() => expect(renderer.getClockTimeUs()).toBeGreaterThanOrEqual(fresh[0]!.timestampUs), { - timeout: 5000, + renderer.setTrack(queueSource(), null); + expect(renderer.snapshot.get().context.status).toBe('error'); + + const failing = createAudioRendererActor({ + audioContext: fakeAudioContext(), + createAudioPlayer: () => Promise.reject(new Error('no worklet')), }); - renderer.destroy(); - }); + await vi.advanceTimersByTimeAsync(0); + expect(failing.snapshot.get().context.status).toBe('error'); + failing.destroy(); - it('schedules only up to the horizon instead of draining a backlog in one tick', async () => { - const frames = await encodeTestFrames(50); // 1s of audio, all buffered up-front - const audioContext = createFakeAudioContext(); // context clock frozen at 0 - const queue = [...frames]; - const source: AudioFrameSource = { peek: () => queue[0], dequeue: () => queue.shift() }; - const renderer = createAudioRendererActor({ audioContext, scheduleMargin: 0.05 }); - - renderer.setTrack(source, { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - await vi.waitFor( - () => expect(renderer.snapshot.get().context.scheduledUntilUs).toBeGreaterThanOrEqual(7 * FRAME_DURATION_US), - { timeout: 5000 } - ); - // Let a few more ticks run: with the clock frozen, scheduling must - // stop at the horizon (margin * 4 plus in-flight decodes), leaving - // the rest of the backlog in the source's jitter buffer. - await new Promise((resolve) => setTimeout(resolve, 150)); - expect(renderer.snapshot.get().context.scheduledUntilUs).toBeLessThan(20 * FRAME_DURATION_US); - expect(queue.length).toBeGreaterThan(30); + const unconnectable = createFakePlayer(); - renderer.destroy(); + unconnectable.player.output.connect = () => { + throw new Error('cross-context connect'); + }; + + const disconnected = createAudioRendererActor({ + audioContext: fakeAudioContext(), + createAudioPlayer: () => Promise.resolve(unconnectable.player), + }); + + await vi.advanceTimersByTimeAsync(0); + expect(disconnected.snapshot.get().context.status).toBe('error'); + expect(unconnectable.closed).toBe(true); + disconnected.destroy(); }); }); diff --git a/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts b/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts deleted file mode 100644 index 3931fc9e15..0000000000 --- a/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts +++ /dev/null @@ -1,546 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vite-plus/test'; - -import type { JitterFrame } from '../../track-subscriber'; -import type { AudioContextLike, AudioFrameSource } from '../audio-renderer'; -import { - type AudioPlayer, - type AudioPlayerInput, - type AudioPlayerInputOptions, - createWorkletAudioRendererActor, -} from '../worklet-audio-renderer'; - -const FRAME_US = 20_000; -const JITTER_TARGET_US = 60_000; -const TICK_MS = 10; -const CONFIG: AudioDecoderConfig = { codec: 'opus', sampleRate: 48_000, numberOfChannels: 1 }; - -/** An Opus TOC byte for one 20 ms CELT frame (config 19, code 0). */ -const OPUS_20MS_TOC = 19 << 3; - -function frameAt(timestampUs: number): JitterFrame { - return { groupId: timestampUs, objectId: 0, timestampUs, isKey: true, payload: new Uint8Array([OPUS_20MS_TOC]) }; -} - -/** Frames every 20 ms over `[fromUs, untilUs)`. */ -function framesBetween(fromUs: number, untilUs: number): JitterFrame[] { - const frames: JitterFrame[] = []; - - for (let timestampUs = fromUs; timestampUs < untilUs; timestampUs += FRAME_US) frames.push(frameAt(timestampUs)); - - return frames; -} - -function queueSource(frames: JitterFrame[] = []): AudioFrameSource & { queue: JitterFrame[] } { - const queue = [...frames]; - - return { queue, peek: () => queue[0], dequeue: () => queue.shift() }; -} - -interface FakeInput extends AudioPlayerInput { - options: AudioPlayerInputOptions; - pushed: number[]; - cuts: number; - removed: boolean; - stats: { playing: boolean; bufferedUs: number; jitterTargetUs: number }; -} - -/** The core's surface with no audio behind it: pushes are recorded, and the test plays the worklet's reports. */ -function createFakePlayer() { - const inputs: FakeInput[] = []; - const connected: AudioNode[] = []; - let closed = false; - const player: AudioPlayer = { - output: { connect: (destination) => connected.push(destination) }, - addInput(options = {}) { - const input: FakeInput = { - options, - pushed: [], - cuts: 0, - removed: false, - stats: { playing: true, bufferedUs: 0, jitterTargetUs: JITTER_TARGET_US }, - push(frame) { - input.pushed.push(frame.timestampUs); - }, - cut() { - input.cuts++; - }, - remove() { - input.removed = true; - }, - }; - - inputs.push(input); - return input; - }, - close() { - closed = true; - }, - }; - - return { - player, - inputs, - connected, - get closed() { - return closed; - }, - /** The worklet reporting that the audio at `timestampUs` is audible now. */ - report(timestampUs: number, atUs: number) { - inputs[inputs.length - 1]!.options.clock!.reportAudio(timestampUs, atUs); - }, - }; -} - -// SAFETY: the renderer only hands `destination` to `output.connect`, which the fake records by identity. -const destination = {} as AudioNode; - -/** Context time follows the test's clock; `suspended` freezes it, as `AudioContext.suspend()` does. */ -let contextUs = 0; -let suspended = false; - -function fakeAudioContext(outputLatency?: number): AudioContextLike { - return { - get currentTime() { - return contextUs / 1_000_000; - }, - destination, - outputLatency, - createBuffer: () => { - throw new Error('unused'); - }, - createBufferSource: () => { - throw new Error('unused'); - }, - }; -} - -describe('createWorkletAudioRendererActor', () => { - let nowUs = 0; - - /** Move wall time, and context time with it unless the context is suspended. */ - const at = (us: number) => { - if (!suspended) contextUs += us - nowUs; - - nowUs = us; - }; - - beforeEach(() => { - vi.useFakeTimers(); - at(0); - contextUs = 0; - suspended = false; - }); - - afterEach(() => { - vi.useRealTimers(); - }); - - /** A renderer whose player is ready and whose first input exists. */ - async function setup(options: { getJoinAnchorUs?: () => number | undefined; outputLatency?: number } = {}) { - const { outputLatency, ...rendererOptions } = options; - const fake = createFakePlayer(); - const renderer = createWorkletAudioRendererActor({ - audioContext: fakeAudioContext(outputLatency), - createAudioPlayer: () => Promise.resolve(fake.player), - tickIntervalMs: TICK_MS, - ...rendererOptions, - }); - - await vi.advanceTimersByTimeAsync(0); - return { fake, renderer }; - } - - const tick = () => vi.advanceTimersByTime(TICK_MS); - - it('connects the player output to the context destination', async () => { - const { fake, renderer } = await setup(); - - expect(fake.connected).toEqual([destination]); - expect(renderer.getClockTimeUs()).toBeUndefined(); - - renderer.destroy(); - expect(fake.closed).toBe(true); - }); - - it('fills to the jitter target plus a frame, then meters by the clock', async () => { - const { fake, renderer } = await setup(); - const source = queueSource(framesBetween(0, 1_000_000)); - - renderer.setTrack(source, CONFIG); - tick(); - - const input = fake.inputs[0]!; - - expect(input.options.decoderConfig).toEqual(CONFIG); - expect(input.pushed).toEqual([0, 20_000, 40_000, 60_000]); - expect(renderer.getClockTimeUs()).toBeUndefined(); - - // Playing: the worklet should hold about its target ahead of what renders. - fake.report(0, 0); - at(30_000); - tick(); - - expect(input.pushed.at(-1)).toBe(80_000); - expect(renderer.getClockTimeUs()).toBeCloseTo(30_000, 0); - expect(renderer.snapshot.get().context.scheduledUntilUs).toBe(100_000); - }); - - it('joins at the edge, dropping buffered audio behind the anchor unheard', async () => { - const { fake, renderer } = await setup({ getJoinAnchorUs: () => 200_000 }); - - renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); - tick(); - - expect(fake.inputs[0]!.pushed[0]).toBe(200_000); - }); - - it('holds the clock where the worklet ran dry, through the refill, until audio past it plays', async () => { - const { fake, renderer } = await setup(); - const source = queueSource(framesBetween(0, 80_000)); - - renderer.setTrack(source, CONFIG); - tick(); - fake.report(0, 0); - - at(100_000); - tick(); - expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); - - // Audio comes back: the refill moves the pushed end, not the clock. - fake.inputs[0]!.stats.playing = false; - source.queue.push(...framesBetween(80_000, 400_000)); - at(300_000); - tick(); - expect(fake.inputs[0]!.pushed.slice(4)).toEqual([80_000, 100_000, 120_000, 140_000]); - expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); - - fake.report(80_000, 320_000); - at(330_000); - expect(renderer.getClockTimeUs()).toBeCloseTo(90_000, 0); - }); - - it('does not step back when the refill is reported ahead of being audible', async () => { - const { fake, renderer } = await setup(); - const source = queueSource(framesBetween(0, 80_000)); - - renderer.setTrack(source, CONFIG); - tick(); - fake.report(0, 0); - at(100_000); - tick(); - - fake.inputs[0]!.stats.playing = false; - source.queue.push(...framesBetween(80_000, 400_000)); - at(300_000); - tick(); - - // Rendered now, audible 100 ms from now: the anchor alone reads 80 ms behind the hold. - fake.report(80_000, 400_000); - expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); - - at(410_000); - expect(renderer.getClockTimeUs()).toBeCloseTo(90_000, 0); - }); - - it('resumes from the audible position when audio returns before the device drained', async () => { - const { fake, renderer } = await setup({ outputLatency: 0.05 }); - const source = queueSource(framesBetween(0, 80_000)); - - renderer.setTrack(source, CONFIG); - tick(); - fake.report(0, 0); - - // Rendering passed the last pushed sample; 50 ms of it is still in the device. - at(30_000); - tick(); - expect(renderer.getClockTimeUs()).toBeCloseTo(30_000, 0); - - fake.inputs[0]!.stats.playing = false; - source.queue.push(...framesBetween(80_000, 400_000)); - at(40_000); - tick(); - fake.report(80_000, 90_000); - expect(renderer.getClockTimeUs()).toBeCloseTo(40_000, 0); - }); - - it('ends pushed audio where the last packet ends, not across a gap of lost packets', async () => { - const { fake, renderer } = await setup(); - - renderer.setTrack(queueSource([0, 20_000, 40_000, 100_000].map(frameAt)), CONFIG); - tick(); - fake.report(0, 0); - at(60_000); - tick(); - - expect(fake.inputs[0]!.pushed).toEqual([0, 20_000, 40_000, 100_000]); - expect(renderer.snapshot.get().context.scheduledUntilUs).toBe(120_000); - }); - - it('keeps the hold through a late report of the audio before the underrun', async () => { - const { fake, renderer } = await setup(); - const source = queueSource(framesBetween(0, 80_000)); - - renderer.setTrack(source, CONFIG); - tick(); - fake.report(0, 0); - at(100_000); - tick(); - - // The refill is pushed before the core has said it stopped; then the - // tail's last report, still in flight, arrives. - source.queue.push(...framesBetween(80_000, 400_000)); - tick(); - fake.report(77_333, 77_333); - at(200_000); - expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); - }); - - it('holds a placed re-join clock at the audio pushed while the fill is short', async () => { - let anchorUs: number | undefined; - const { fake, renderer } = await setup({ getJoinAnchorUs: () => anchorUs }); - const source = queueSource(framesBetween(0, 80_000)); - - renderer.setTrack(source, CONFIG); - tick(); - fake.report(0, 0); - - anchorUs = 4_900_000; - source.queue.push(frameAt(5_000_000)); - tick(); - at(100_000); - tick(); - expect(fake.inputs[0]!.pushed.at(-1)).toBe(5_000_000); - - // One packet cannot start the worklet; the clock stops at its end. - at(2_100_000); - tick(); - expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); - - // The refill plays, reported ahead of being audible: no step back. - source.queue.push(...framesBetween(5_020_000, 5_200_000)); - tick(); - fake.report(5_000_000, 2_150_000); - expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); - }); - - it('gives the core a report time source that stops while the context is suspended', async () => { - const { fake, renderer } = await setup(); - - renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); - tick(); - - const clock = fake.inputs[0]!.options.clock!; - - at(40_000); - expect(clock.localNow()).toBeCloseTo(40_000, 0); - - suspended = true; - at(540_000); - expect(clock.localNow()).toBeCloseTo(40_000, 0); - - // The first report after resume, timed on that source, lands where audio is. - suspended = false; - at(560_000); - fake.report(60_000, clock.localNow()); - expect(renderer.getClockTimeUs()).toBeCloseTo(60_000, 0); - }); - - it('drops what the core holds at a timeline reset and ignores reports from before it', async () => { - const { fake, renderer } = await setup(); - const source = queueSource(framesBetween(0, 80_000)); - - renderer.setTrack(source, CONFIG); - tick(); - fake.report(0, 0); - - // A publisher re-anchor: the timeline steps back by far more than a gap. - source.queue.push(...framesBetween(-5_000_000, -4_000_000)); - tick(); - - const input = fake.inputs[0]!; - - expect(input.cuts).toBe(1); - expect(input.pushed.at(4)).toBe(-5_000_000); - expect(renderer.getClockTimeUs()).toBeUndefined(); - - // Still in flight from the old timeline when the cut landed. - fake.report(40_000, 0); - expect(renderer.getClockTimeUs()).toBeUndefined(); - - fake.report(-5_000_000, 0); - expect(renderer.getClockTimeUs()).toBeCloseTo(-5_000_000, 0); - }); - - it('places a re-join whose whole buffer is past the anchor at its distance from it', async () => { - let anchorUs: number | undefined; - const { fake, renderer } = await setup({ getJoinAnchorUs: () => anchorUs }); - const source = queueSource(framesBetween(0, 80_000)); - - renderer.setTrack(source, CONFIG); - tick(); - fake.report(0, 0); - - // A catch-up skip landed on a group 100 ms past where the anchor falls. - anchorUs = 4_900_000; - source.queue.push(...framesBetween(5_000_000, 6_000_000)); - tick(); - - const input = fake.inputs[0]!; - - expect(input.pushed).toHaveLength(4); - expect(renderer.getClockTimeUs()).toBeCloseTo(4_900_000, 0); - - at(50_000); - tick(); - expect(input.pushed.at(4)).toBe(5_000_000); - expect(renderer.getClockTimeUs()).toBeCloseTo(4_950_000, 0); - }); - - it('freezes the clock and stops metering while the context is suspended', async () => { - const { fake, renderer } = await setup(); - const source = queueSource(framesBetween(0, 1_000_000)); - - renderer.setTrack(source, CONFIG); - tick(); - fake.report(0, 0); - at(30_000); - tick(); - - const pushed = fake.inputs[0]!.pushed.length; - - // Paused for two seconds: nothing plays, so nothing is due and nothing ran dry. - suspended = true; - at(2_030_000); - tick(); - expect(renderer.getClockTimeUs()).toBeCloseTo(30_000, 0); - expect(fake.inputs[0]!.pushed).toHaveLength(pushed); - - suspended = false; - at(2_040_000); - expect(renderer.getClockTimeUs()).toBeCloseTo(40_000, 0); - }); - - it('ignores reports from an input that a track change removed', async () => { - const { fake, renderer } = await setup(); - - renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); - tick(); - - const stale = fake.inputs[0]!.options.clock!; - - renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); - tick(); - stale.reportAudio(40_000, 0); - expect(renderer.getClockTimeUs()).toBeUndefined(); - - fake.report(0, 0); - expect(renderer.getClockTimeUs()).toBeCloseTo(0, 0); - }); - - it('counts every time the worklet runs dry, across track changes', async () => { - const { fake, renderer } = await setup(); - - renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); - tick(); - fake.report(0, 0); - - // A stall far longer than any jitter ceiling still counts, once. - at(100_000); - tick(); - at(3_000_000); - tick(); - expect(renderer.snapshot.get().context.underruns).toBe(1); - - renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); - tick(); - fake.report(0, 3_000_000); - at(3_100_000); - tick(); - expect(renderer.snapshot.get().context.underruns).toBe(2); - }); - - it('counts a dry edge that a track change lands on before the next tick', async () => { - const { fake, renderer } = await setup(); - - renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); - tick(); - fake.report(0, 0); - - at(100_000); - renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); - expect(renderer.snapshot.get().context.underruns).toBe(1); - }); - - it('rescues a frozen re-join placement whose next packets are past the fill horizon', async () => { - let anchorUs: number | undefined; - const { fake, renderer } = await setup({ getJoinAnchorUs: () => anchorUs }); - const source = queueSource(framesBetween(0, 80_000)); - - renderer.setTrack(source, CONFIG); - tick(); - fake.report(0, 0); - - anchorUs = 4_900_000; - source.queue.push(frameAt(5_000_000)); - tick(); - at(200_000); - tick(); - expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); - - // Packets resume 80 ms after the lone one ends: beyond the fill horizon. - fake.inputs[0]!.stats.playing = false; - source.queue.push(...framesBetween(5_100_000, 5_400_000)); - at(400_000); - tick(); - expect(fake.inputs[0]!.pushed).toContain(5_100_000); - expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); - }); - - it('caps the core jitter target at the target latency', async () => { - const fake = createFakePlayer(); - const renderer = createWorkletAudioRendererActor({ - audioContext: fakeAudioContext(), - createAudioPlayer: () => Promise.resolve(fake.player), - getTargetLatencyUs: () => 250_000, - tickIntervalMs: TICK_MS, - }); - - await vi.advanceTimersByTimeAsync(0); - renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); - tick(); - - expect(fake.inputs[0]!.options.clock!.targetLatencyUs).toBe(250_000); - }); - - it('fails a track with no decoder config, and a player that cannot be created', async () => { - const { renderer } = await setup(); - - renderer.setTrack(queueSource(), null); - expect(renderer.snapshot.get().context.status).toBe('error'); - - const failing = createWorkletAudioRendererActor({ - audioContext: fakeAudioContext(), - createAudioPlayer: () => Promise.reject(new Error('no worklet')), - }); - - await vi.advanceTimersByTimeAsync(0); - expect(failing.snapshot.get().context.status).toBe('error'); - failing.destroy(); - - const unconnectable = createFakePlayer(); - - unconnectable.player.output.connect = () => { - throw new Error('cross-context connect'); - }; - - const disconnected = createWorkletAudioRendererActor({ - audioContext: fakeAudioContext(), - createAudioPlayer: () => Promise.resolve(unconnectable.player), - }); - - await vi.advanceTimersByTimeAsync(0); - expect(disconnected.snapshot.get().context.status).toBe('error'); - expect(unconnectable.closed).toBe(true); - disconnected.destroy(); - }); -}); diff --git a/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts b/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts deleted file mode 100644 index d92ef39dc7..0000000000 --- a/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts +++ /dev/null @@ -1,571 +0,0 @@ -/** - * AudioWorklet renderer actor over the shared audio player core: `@mux/moq-media`'s `createAudioPlayer`, injected as - * `createAudioPlayer`. Same surface as `createAudioRendererActor`, frames in from the subscriber's jitter buffer and - * the **master clock** out, so the engine can run either behind one seam. - * - * The core decodes with WebCodecs, holds an adaptive jitter buffer in an `AudioWorklet`, and reports the audible - * timestamp about every 20 ms. **Latency stays v10's.** The renderer meters frames into the core so the worklet holds - * about its own jitter target, and the rest of the target latency stays in the subscriber's buffer, where `syncLatency` - * measures and steers it. The core's adaptive target is left alone: under metering its growth only moves depth from one - * buffer to the other. - * - * The clock the core reports into is the renderer's own, not the SDK's `PlayoutClock`, so `getClockTimeUs` keeps the - * contract the video renderer and `trackPlayoutTime` rely on: `undefined` before any audio, and **held where the - * worklet ran dry** rather than paced on by wall time — a wall-paced clock steps back when audio resumes, which holds - * slaved video on its last frame for the whole step. - * - * TODO(mmcc/v10#66): apply `getPlaybackRate` once the core has a per-input rate (muxinc/relay-sdks#329). Until then - * audio ignores the latency controller's nudges, which is why the engine only uses this renderer when a - * `createAudioPlayer` is configured. - */ -import { createTransitionActor } from '../../../core/actors/create-transition-actor'; -import { TIMELINE_DISCONTINUITY_US } from '../../../media/moq/timeline'; -import type { - AudioContextLike, - AudioFrameSource, - AudioRendererActor, - AudioRendererContext, - AudioRendererStatus, -} from './audio-renderer'; - -// ============================================================================= -// Types -// ============================================================================= - -/** - * The part of `@mux/moq-media`'s `PlayoutClock` an audio player input calls. The renderer implements it, so the input - * reports into the renderer's clock. - */ -export interface AudioPlayerClock { - /** Caps the input's jitter target: the core keeps it under this less the output delay and 40 ms. */ - readonly targetLatencyUs: number; - /** The time source reports are measured on, in microseconds. */ - localNow(): number; - /** The audio with this timestamp is audible at local time `atUs`. */ - reportAudio(timestampUs: number, atUs?: number): void; - /** The input stopped. */ - stopAudio(): void; -} - -/** The part of `@mux/moq-media`'s `AudioInputOptions` the renderer passes. */ -export interface AudioPlayerInputOptions { - clock?: AudioPlayerClock; - decoderConfig?: AudioDecoderConfig; - /** A decoder error. The core decodes again from the next frame, with a new decoder. */ - onError?(cause: unknown): void; -} - -/** The part of `@mux/moq-media`'s `AudioInputStats` the renderer reads. Durations are microseconds. */ -export interface AudioPlayerInputStats { - /** Whether the worklet is playing, rather than filling to its target. */ - readonly playing: boolean; - /** Decoded audio the worklet holds. */ - readonly bufferedUs: number; - /** How much the worklet holds before it plays, at the start and after running dry. */ - readonly jitterTargetUs: number; -} - -/** The part of `@mux/moq-media`'s `AudioPlayerInput` the renderer uses: one track. */ -export interface AudioPlayerInput { - /** Decode and hold a frame, in decode order. */ - push(frame: { timestampUs: number; payload: Uint8Array }): void; - /** Without a range: drop everything held or still decoding. */ - cut(): void; - readonly stats: AudioPlayerInputStats; - remove(): void; -} - -/** The part of `@mux/moq-media`'s `AudioPlayer` the renderer uses. */ -export interface AudioPlayer { - /** The mix of every input; the renderer connects it to its context's `destination`. */ - readonly output: { connect(destination: AudioNode): void }; - addInput(options?: AudioPlayerInputOptions): AudioPlayerInput; - /** Remove every input and disconnect the output. */ - close(): void; -} - -/** - * Create the player on the renderer's context. `@mux/moq-media`'s `createAudioPlayer` qualifies once given a real - * `AudioContext`, which a wrapping context exposes as `baseContext`: - * - * ```ts - * createAudioPlayer: (context) => createAudioPlayer({ context: context.baseContext as AudioContext }); - * ``` - */ -export type CreateAudioPlayer = (context: AudioContextLike) => Promise; - -export interface CreateWorkletAudioRendererOptions { - /** Where the player's output connects (`destination`), and the output latencies the clock is placed behind. */ - audioContext: AudioContextLike; - createAudioPlayer: CreateAudioPlayer; - /** - * **Join at the live edge**, as `createAudioRendererActor`'s option of the same name: buffered audio older than this - * is discarded before the first frame of a (re)start reaches the core. - */ - getJoinAnchorUs?: () => number | undefined; - /** - * The target latency in microseconds, read on every clock report. It caps the core's jitter target, so the worklet - * never holds more than the latency the controller is aiming at. Default 2 s, where the core's own 400 ms ceiling - * applies instead. - */ - getTargetLatencyUs?: () => number | undefined; - /** Metering cadence in ms. Default 10. */ - tickIntervalMs?: number; -} - -type RendererMessage = - | { type: 'status'; status: AudioRendererStatus; error?: unknown } - | { type: 'decoder-error'; error: unknown } - | { type: 'scheduled'; untilUs: number } - | { type: 'underrun' }; - -// ============================================================================= -// Implementation -// ============================================================================= - -const DEFAULT_TICK_INTERVAL_MS = 10; -const DEFAULT_TARGET_LATENCY_US = 2_000_000; -/** The frame duration assumed before a frame has shown one: Opus's usual 20 ms. */ -const DEFAULT_FRAME_DURATION_US = 20_000; -/** Reports and pushed spans may disagree by sample rounding and resampling offsets. */ -const REPORT_TOLERANCE_US = 5_000; -/** - * How long a fill may sit short of playback before one more frame is pushed. The fill pushes exactly the jitter target - * plus a frame, so audio the core drops on arrival (behind what it already decoded) could otherwise leave it waiting - * forever. - */ -const FILL_STALL_US = 100_000; -/** See `audio-renderer`'s threshold of the same name: a jump this large, either way, is a new timeline. */ -const DISCONTINUITY_THRESHOLD_US = TIMELINE_DISCONTINUITY_US; - -/** - * How long an Opus packet plays, from its TOC byte (RFC 6716 §3.1); `undefined` when it cannot be read. The same rule - * as `@mux/moq-media`'s `opusPacketDurationUs`. - */ -function opusPacketDurationUs(packet: Uint8Array): number | undefined { - const toc = packet[0]; - if (toc === undefined) return undefined; - - const config = toc >> 3; - // SILK: 10, 20, 40, 60 ms; hybrid: 10, 20 ms; CELT: 2.5, 5, 10, 20 ms. - const frameUs = - config < 12 - ? [10_000, 20_000, 40_000, 60_000][config % 4]! - : config < 16 - ? [10_000, 20_000][config % 2]! - : [2_500, 5_000, 10_000, 20_000][config % 4]!; - const code = toc & 3; - if (code === 0) return frameUs; - - if (code !== 3) return 2 * frameUs; - - const count = packet[1]; - - return count === undefined ? undefined : (count & 0x3f) * frameUs; -} - -export function createWorkletAudioRendererActor(options: CreateWorkletAudioRendererOptions): AudioRendererActor { - const { audioContext } = options; - - const inner = createTransitionActor( - { status: 'idle', framesScheduled: 0, underruns: 0 }, - (context, message) => { - switch (message.type) { - case 'status': - return { ...context, status: message.status, error: message.error }; - case 'decoder-error': - return { ...context, error: message.error }; - case 'scheduled': - return { - ...context, - status: 'rendering', - framesScheduled: context.framesScheduled + 1, - scheduledUntilUs: message.untilUs, - }; - case 'underrun': - return { ...context, underruns: context.underruns + 1 }; - } - } - ); - - let destroyed = false; - let player: AudioPlayer | undefined; - let input: AudioPlayerInput | undefined; - let source: AudioFrameSource | null = null; - let decoderConfig: AudioDecoderConfig | null = null; - - /** The schedule has no anchor yet; see `audio-renderer`'s flag of the same name. */ - let needsJoinAnchor = true; - /** The pending anchor follows a mid-stream discontinuity; see `audio-renderer`'s flag of the same name. */ - let rejoinAnchorArmed = false; - - // What this timeline has pushed into the core. Reset with the timeline. - let firstPushedUs: number | undefined; - let lastPushedUs: number | undefined; - /** Known once a packet's duration or two frames' spacing has shown it. */ - let frameDurationUs: number | undefined; - /** Where the newest pushed frame ends: the furthest the clock can read. */ - let pushedEndUs: number | undefined; - /** First timestamp pushed in the current fill, and the context time of the latest push. */ - let fillStartUs: number | undefined; - let lastPushAtUs = 0; - - /** - * The clock: timeline position `mediaUs` is audible at context time `contextUs`, advancing with the context clock - * from there. Set by every core report, or **placed** at a re-join anchor before the core plays anything (see - * `dropToJoinAnchor`). - * - * Context time, not `performance.now()`, because it is what stops when the adapter suspends the context on pause: a - * wall-paced clock would run on through the pause, read the frozen worklet as dry and refill it, and step back on the - * first report after resume. The core times its reports on the same source (`localNow`), so its render clock's - * calibration does not carry a pause into the first reports after it either. - */ - let anchor: { mediaUs: number; contextUs: number; placed: boolean; frozen?: boolean } | undefined; - /** Bumped per input, so a report from a removed input cannot anchor its replacement. */ - let inputGeneration = 0; - /** - * Where the worklet ran dry. The clock holds there until a report of audio past it arrives: frames pushed to refill - * the worklet move `pushedEndUs` before they play, and must not drag the clock forward with them. - */ - let dryLimitUs: number | undefined; - /** - * The position the clock held at through the latest underrun, kept as a floor after it ends. The first report of the - * refill is dated ahead by the output latency — the sample is not audible yet — so its anchor extrapolates to before - * the hold, and the clock would otherwise step back by that much. - */ - let holdFloorUs: number | undefined; - /** - * The core has reported itself stopped since the dry point was set. Until it has, a report near the dry point is the - * tail of the audio before it, still in flight, and must not end the hold. - */ - let stoppedSinceDry = false; - - const outputLatencyUs = (): number => - ((audioContext.baseLatency ?? 0) + (audioContext.outputLatency ?? 0)) * 1_000_000; - - const contextNowUs = (): number => audioContext.currentTime * 1_000_000; - - /** The audible position extrapolated from the anchor, ignoring every limit. */ - const extrapolatedUs = (): number | undefined => - anchor && (anchor.frozen ? anchor.mediaUs : anchor.mediaUs + (contextNowUs() - anchor.contextUs)); - - /** The clock one input reports into; it goes deaf once that input is replaced. */ - const createClock = (): AudioPlayerClock => { - const generation = ++inputGeneration; - - return { - get targetLatencyUs() { - return options.getTargetLatencyUs?.() ?? DEFAULT_TARGET_LATENCY_US; - }, - localNow: contextNowUs, - reportAudio(timestampUs, atUs) { - if (generation !== inputGeneration) return; - - reportAudio(timestampUs, atUs); - }, - // The renderer resets the clock itself wherever a track or timeline ends. - stopAudio() {}, - }; - }; - - const raiseHoldFloor = (heldUs: number): void => { - holdFloorUs = Math.max(holdFloorUs ?? heldUs, heldUs); - }; - - /** - * A placed clock runs from the anchor until the core plays, but never past the audio pushed: a fill that stays short - * of the jitter target cannot start the worklet, and the clock would otherwise run on through audio not there. - */ - const placedPositionUs = (): number | undefined => { - const positionUs = extrapolatedUs(); - - return positionUs === undefined || pushedEndUs === undefined ? positionUs : Math.min(positionUs, pushedEndUs); - }; - - /** `timestampUs` is audible at context time `atUs` (default now). */ - const reportAudio = (timestampUs: number, atUs: number | undefined): void => { - // Only audio pushed on this timeline: a report already in flight when - // the timeline restarted describes audio the cut dropped. - if ( - firstPushedUs === undefined || - pushedEndUs === undefined || - timestampUs < firstPushedUs - REPORT_TOLERANCE_US || - timestampUs > pushedEndUs + REPORT_TOLERANCE_US - ) { - return; - } - - // The refill's first report ends the hold. Its floor is what the clock - // reads at that moment — the audible position, which can still be short - // of the dry point while the device plays out what it had queued. - if ( - dryLimitUs !== undefined && - ((stoppedSinceDry && timestampUs >= dryLimitUs - REPORT_TOLERANCE_US) || - timestampUs > dryLimitUs + REPORT_TOLERANCE_US) - ) { - raiseHoldFloor(Math.min(extrapolatedUs() ?? dryLimitUs, dryLimitUs)); - dryLimitUs = undefined; - } - - // Likewise the first report after a placed re-join, which is dated ahead - // of the placed clock by the output latency. - if (anchor?.placed) raiseHoldFloor(placedPositionUs()!); - - anchor = { mediaUs: timestampUs, contextUs: atUs ?? contextNowUs(), placed: false }; - }; - - /** Forget everything pushed on the current timeline, and the clock placed on it. */ - const resetTimeline = (): void => { - firstPushedUs = undefined; - lastPushedUs = undefined; - frameDurationUs = undefined; - pushedEndUs = undefined; - fillStartUs = undefined; - anchor = undefined; - dryLimitUs = undefined; - holdFloorUs = undefined; - stoppedSinceDry = false; - }; - - const removeInput = (): void => { - if (!input) return; - - inputGeneration++; - input.remove(); - input = undefined; - }; - - /** - * Discard buffered audio older than the join anchor before the first push of a (re)start, so playout starts at the - * live edge rather than at the oldest group the relay replayed. Mirrors `audio-renderer`'s function of the same name, - * including landing a re-join anchor by **placement** when the whole buffer is already past it: the clock is placed - * at the anchor and runs from there, and metering by timeline holds the head back until it is due, so playout resumes - * at the target depth rather than at the live edge. - */ - const dropToJoinAnchor = (): void => { - const anchorUs = options.getJoinAnchorUs?.(); - if (anchorUs === undefined || !source) return; - - let dropped = 0; - - for (let next = source.peek(); next && next.timestampUs < anchorUs; next = source.peek()) { - source.dequeue(); - dropped++; - } - - if (dropped > 0) { - needsJoinAnchor = false; - rejoinAnchorArmed = false; - return; - } - - // A fresh join stays armed: a relay's group replay arrives as a burst, - // oldest first, so the buffer's live edge may still be moving. - if (!rejoinAnchorArmed || !source.peek()) return; - - anchor = { mediaUs: anchorUs, contextUs: contextNowUs(), placed: true }; - rejoinAnchorArmed = false; - needsJoinAnchor = false; - }; - - /** - * Whether `timestampUs` should reach the core now. Filling — before the first report, or after running dry — pushes - * the jitter target plus a frame, which is what the worklet waits for before playing. Playing pushes what will be - * rendered within the jitter target, so the worklet holds about that much and the rest waits in the subscriber. - */ - const isDue = (timestampUs: number, stats: AudioPlayerInputStats): boolean => { - const positionUs = anchor?.placed ? placedPositionUs() : extrapolatedUs(); - - // A frozen placement is a fill too: its clock waits for the worklet to - // start, so only the fill rules (including the stall rescue) can move it. - if (positionUs === undefined || dryLimitUs !== undefined || anchor?.frozen) { - if ( - fillStartUs === undefined || - timestampUs < fillStartUs + stats.jitterTargetUs + (frameDurationUs ?? DEFAULT_FRAME_DURATION_US) - ) { - return true; - } - - return contextNowUs() - lastPushAtUs > FILL_STALL_US && stats.bufferedUs < stats.jitterTargetUs; - } - - return timestampUs < positionUs + outputLatencyUs() + stats.jitterTargetUs; - }; - - const push = (active: AudioPlayerInput, timestampUs: number, payload: Uint8Array): void => { - active.push({ timestampUs, payload }); - - // Opus says how long each packet is. Otherwise the shortest step between - // pushed frames is the frame length: a step over lost packets is longer, - // and must not extend the audio past where it really ends. - const opusUs = decoderConfig?.codec === 'opus' ? opusPacketDurationUs(payload) : undefined; - const stepUs = lastPushedUs === undefined ? undefined : timestampUs - lastPushedUs; - - if (opusUs !== undefined) frameDurationUs = opusUs; - else if (stepUs !== undefined && stepUs > 0) frameDurationUs = Math.min(frameDurationUs ?? stepUs, stepUs); - - firstPushedUs ??= timestampUs; - fillStartUs ??= timestampUs; - lastPushedUs = timestampUs; - lastPushAtUs = contextNowUs(); - pushedEndUs = timestampUs + (frameDurationUs ?? DEFAULT_FRAME_DURATION_US); - inner.send({ type: 'scheduled', untilUs: pushedEndUs }); - }; - - /** - * The worklet runs dry once rendering passes everything pushed: metering pushes every frame as soon as it is due, so - * nothing was left to push. Each such edge is an underrun, as `audio-renderer` counts them. The core's own counter - * skips dry spells longer than its jitter ceiling, which `adaptLatencyTarget` still needs as failure evidence. - */ - const detectDry = (): void => { - const positionUs = extrapolatedUs(); - - if ( - !anchor || - anchor.placed || - dryLimitUs !== undefined || - pushedEndUs === undefined || - positionUs === undefined || - positionUs + outputLatencyUs() < pushedEndUs - ) { - return; - } - - dryLimitUs = pushedEndUs; - stoppedSinceDry = false; - fillStartUs = undefined; - inner.send({ type: 'underrun' }); - }; - - const tick = (): void => { - if (destroyed || !player || !source || !decoderConfig) return; - - try { - const active = (input ??= player.addInput({ - clock: createClock(), - decoderConfig, - onError: (error) => inner.send({ type: 'decoder-error', error }), - })); - const { stats } = active; - - detectDry(); - - const positionUs = extrapolatedUs(); - - if (dryLimitUs !== undefined && !stats.playing) stoppedSinceDry = true; - - // A placed clock that reaches the end of the audio pushed stops there - // until the core plays, so refill frames cannot drag it forward. - if (anchor?.placed && !anchor.frozen && pushedEndUs !== undefined && positionUs! >= pushedEndUs) { - anchor = { mediaUs: pushedEndUs, contextUs: contextNowUs(), placed: true, frozen: true }; - } - - for (;;) { - if (needsJoinAnchor && lastPushedUs === undefined) dropToJoinAnchor(); - - const next = source.peek(); - if (!next) return; - - // A timeline reset, either way: drop what the core holds and re-place - // the anchor on the jumped-to timeline (see `audio-renderer`). - if (lastPushedUs !== undefined && Math.abs(next.timestampUs - lastPushedUs) > DISCONTINUITY_THRESHOLD_US) { - active.cut(); - resetTimeline(); - needsJoinAnchor = true; - rejoinAnchorArmed = true; - continue; - } - - if (!isDue(next.timestampUs, stats)) return; - - push(active, next.timestampUs, next.payload); - source.dequeue(); - } - } catch (error) { - inner.send({ type: 'status', status: 'error', error }); - } - }; - const timer = setInterval(tick, options.tickIntervalMs ?? DEFAULT_TICK_INTERVAL_MS); - - options.createAudioPlayer(audioContext).then( - (created) => { - if (destroyed) { - created.close(); - return; - } - - try { - created.output.connect(audioContext.destination); - player = created; - } catch (error) { - created.close(); - inner.send({ type: 'status', status: 'error', error }); - } - }, - (error) => { - if (!destroyed) inner.send({ type: 'status', status: 'error', error }); - } - ); - - return { - get snapshot() { - return inner.snapshot; - }, - - setTrack(nextSource, nextConfig): void { - // See `audio-renderer`: a source with no config could never play. - const missingConfig = nextSource !== null && nextConfig === null; - - source = missingConfig ? null : nextSource; - decoderConfig = nextConfig; - needsJoinAnchor = true; - rejoinAnchorArmed = false; - // An edge since the last tick still counts: the reset below forgets it. - detectDry(); - removeInput(); - resetTimeline(); - - if (missingConfig) { - inner.send({ - type: 'status', - status: 'error', - error: new Error('audio renderer track has no decoder config'), - }); - return; - } - - inner.send({ type: 'status', status: source ? 'rendering' : 'idle' }); - }, - - getClockTimeUs(): number | undefined { - const positionUs = extrapolatedUs(); - if (positionUs === undefined) return undefined; - - if (anchor?.placed) return placedPositionUs(); - - // Never past the audio pushed, and once the worklet ran dry never past - // where it did: the clock holds through an underrun and resumes when - // audio does, as the source-node renderer's clamped segments did. - const limitUs = dryLimitUs ?? pushedEndUs; - const clampedUs = limitUs === undefined ? positionUs : Math.min(positionUs, limitUs); - - if (holdFloorUs === undefined) return clampedUs; - - if (clampedUs < holdFloorUs) return holdFloorUs; - - holdFloorUs = undefined; - return clampedUs; - }, - - destroy(): void { - if (destroyed) return; - - destroyed = true; - clearInterval(timer); - removeInput(); - player?.close(); - player = undefined; - inner.destroy(); - }, - }; -} diff --git a/packages/spf/src/playback/behaviors/dom/setup-moq-renderers.ts b/packages/spf/src/playback/behaviors/dom/setup-moq-renderers.ts index 3613b4988d..e5a572fb72 100644 --- a/packages/spf/src/playback/behaviors/dom/setup-moq-renderers.ts +++ b/packages/spf/src/playback/behaviors/dom/setup-moq-renderers.ts @@ -43,10 +43,10 @@ import { import { type AudioContextLike, type AudioRendererActor, + type CreateAudioPlayer, createAudioRendererActor, } from '../../actors/dom/audio-renderer'; import { createVideoRendererActor, type VideoRendererActor } from '../../actors/dom/video-renderer'; -import { type CreateAudioPlayer, createWorkletAudioRendererActor } from '../../actors/dom/worklet-audio-renderer'; import type { TrackSubscriberActor } from '../../actors/track-subscriber'; import { type LatencyControlConfig, type PlayoutClockOwner, resolveLatencyControlConfig } from '../sync-latency'; @@ -94,10 +94,7 @@ const CLOCK_PUBLISH_INTERVAL_MS = 100; export interface MoqRendererConfig { /** Shared with `syncLatency`: `joinAtEdge`, the target the clocks aim at, and the slew bounds. */ latency?: Partial; - /** - * Render audio through the shared AudioWorklet player core instead of scheduled `AudioBufferSourceNode`s (see - * `worklet-audio-renderer`). Opt-in until the core can apply `playoutRate` nudges (mmcc/v10#66). - */ + /** Override the audio renderer's player core factory; see `CreateAudioPlayer`. */ createAudioPlayer?: CreateAudioPlayer; } @@ -254,31 +251,24 @@ function setupAudioRendererSetup({ latencyConfig, context.videoRendererActor ); - const renderer = config?.createAudioPlayer - ? createWorkletAudioRendererActor({ - audioContext, - createAudioPlayer: config.createAudioPlayer, - getJoinAnchorUs, - // Caps the worklet's jitter target, so the core never holds - // more than the latency the controller is aiming at. - getTargetLatencyUs: () => - resolveTrackTargetSeconds( - peek(context.audioSubscriberActor), - state.targetLatency, - state.adaptiveTargetLatency, - latencyConfig - ) * 1_000_000, - }) - : createAudioRendererActor({ - audioContext, - // No paused gating here: the adapter suspends the - // AudioContext on pause, which freezes the hardware clock and - // every scheduled source. Rate 0 would instead produce an - // infinite clock segment (duration ÷ 0) and sources whose - // `playbackRate` stays 0 after resume — a permanent stall. - getPlaybackRate: () => peek(state.playoutRate) ?? 1, - getJoinAnchorUs, - }); + const renderer = createAudioRendererActor({ + audioContext, + createAudioPlayer: config?.createAudioPlayer, + // No paused gating here: the adapter suspends the AudioContext on + // pause, which freezes the worklet and the clock with it. The core + // has no rate 0 to pause with anyway. + getPlaybackRate: () => peek(state.playoutRate) ?? 1, + getJoinAnchorUs, + // Caps the worklet's jitter target, so the core never holds + // more than the latency the controller is aiming at. + getTargetLatencyUs: () => + resolveTrackTargetSeconds( + peek(context.audioSubscriberActor), + state.targetLatency, + state.adaptiveTargetLatency, + latencyConfig + ) * 1_000_000, + }); context.audioRendererActor.set(renderer); return () => { diff --git a/packages/spf/src/playback/behaviors/dom/tests/setup-moq-renderers.test.ts b/packages/spf/src/playback/behaviors/dom/tests/setup-moq-renderers.test.ts index f3bb440edb..3bb6c07bd9 100644 --- a/packages/spf/src/playback/behaviors/dom/tests/setup-moq-renderers.test.ts +++ b/packages/spf/src/playback/behaviors/dom/tests/setup-moq-renderers.test.ts @@ -5,7 +5,6 @@ import type { AudioContextLike, AudioRendererActor } from '../../../actors/dom/a import { createAudioRendererActor } from '../../../actors/dom/audio-renderer'; import type { VideoRendererActor } from '../../../actors/dom/video-renderer'; import { createVideoRendererActor } from '../../../actors/dom/video-renderer'; -import { createWorkletAudioRendererActor } from '../../../actors/dom/worklet-audio-renderer'; import type { TrackSubscriberActor } from '../../../actors/track-subscriber'; import type { PlayoutClockOwner } from '../../sync-latency'; import { @@ -24,9 +23,6 @@ vi.mock('../../../actors/dom/audio-renderer', () => ({ vi.mock('../../../actors/dom/video-renderer', () => ({ createVideoRendererActor: vi.fn(), })); -vi.mock('../../../actors/dom/worklet-audio-renderer', () => ({ - createWorkletAudioRendererActor: vi.fn(), -})); function makeFakeAudioRenderer(): AudioRendererActor { return { @@ -87,23 +83,22 @@ describe('setupAudioRenderer', () => { const { getPlaybackRate } = vi.mocked(createAudioRendererActor).mock.calls[0]![0]; expect(getPlaybackRate!()).toBe(1); - // Rate must stay positive even through a pause: a rate of 0 would - // schedule an infinite clock segment (duration ÷ 0) and dead sources. + // Rate must stay positive even through a pause: the core accepts no + // rate below 0.5, and suspending the context already holds the clock. state.playoutRate.set(1.05); expect(getPlaybackRate!()).toBe(1.05); reactor.destroy(); }); - it('renders through the worklet core when a createAudioPlayer is configured', async () => { - vi.mocked(createWorkletAudioRendererActor).mockImplementation(() => makeFakeAudioRenderer()); + it('passes the configured createAudioPlayer and caps the core at the target latency', async () => { + vi.mocked(createAudioRendererActor).mockImplementation(() => makeFakeAudioRenderer()); const createAudioPlayer = vi.fn(); const { state, context, reactor } = setupSetupAudioRenderer({ createAudioPlayer }); - await vi.waitFor(() => expect(createWorkletAudioRendererActor).toHaveBeenCalledTimes(1)); - expect(createAudioRendererActor).not.toHaveBeenCalled(); + await vi.waitFor(() => expect(createAudioRendererActor).toHaveBeenCalledTimes(1)); - const options = vi.mocked(createWorkletAudioRendererActor).mock.calls[0]![0]; + const options = vi.mocked(createAudioRendererActor).mock.calls[0]![0]; expect(options.createAudioPlayer).toBe(createAudioPlayer); expect(options.audioContext).toBe(context.audioContext.get()); diff --git a/packages/spf/src/playback/engines/moq/adapter.ts b/packages/spf/src/playback/engines/moq/adapter.ts index 9ffaf64dfa..45fdd9d35f 100644 --- a/packages/spf/src/playback/engines/moq/adapter.ts +++ b/packages/spf/src/playback/engines/moq/adapter.ts @@ -330,7 +330,7 @@ export function MoqMediaMixin>( gain.connect(audioContext.destination); gain.gain.value = this.#muted ? 0 : this.#volume; - // The worklet renderer's core constructs an `AudioWorkletNode`, which + // The renderer's player core constructs an `AudioWorkletNode`, which // only a real context can host; an injected fake has none to offer. const baseContext = audioContext.baseContext ?? @@ -350,9 +350,6 @@ export function MoqMediaMixin>( return audioContext.outputLatency; }, baseContext, - createBuffer: (numberOfChannels, length, sampleRate) => - audioContext.createBuffer(numberOfChannels, length, sampleRate), - createBufferSource: () => audioContext.createBufferSource(), }; } @@ -464,10 +461,9 @@ export function MoqMediaMixin>( this.#gain = undefined; this.#renderContext = undefined; // Close the context only after the composition has torn the renderers - // down: an in-flight renderer tick still calls `createBuffer` / - // `createBufferSource`, which throw `InvalidStateError` on a closed - // context. Both arms close so a failed teardown still releases the - // audio device. + // down: the renderer's player core still builds and drives nodes on + // it, which throws `InvalidStateError` on a closed context. Both arms + // close so a failed teardown still releases the audio device. const closeAudio = () => { audioContext?.close().catch(() => {}); }; diff --git a/packages/spf/src/playback/engines/moq/engine.ts b/packages/spf/src/playback/engines/moq/engine.ts index bc64fc0787..4b3206203a 100644 --- a/packages/spf/src/playback/engines/moq/engine.ts +++ b/packages/spf/src/playback/engines/moq/engine.ts @@ -9,9 +9,8 @@ import type { QualityConfig } from '../../../media/abr/quality-selection'; import type { AudioTrack, MaybeResolvedPresentation, TextTrack, VideoTrack } from '../../../media/types'; import type { BandwidthConfig, BandwidthState } from '../../../network/bandwidth-estimator'; import type { RetryBackoffConfig } from '../../../network/retry-backoff'; -import type { AudioContextLike, AudioRendererActor } from '../../actors/dom/audio-renderer'; +import type { AudioContextLike, AudioRendererActor, CreateAudioPlayer } from '../../actors/dom/audio-renderer'; import type { VideoRendererActor } from '../../actors/dom/video-renderer'; -import type { CreateAudioPlayer } from '../../actors/dom/worklet-audio-renderer'; import type { CreateMoqTransport, MoqAuthProvider, MoqSessionActor } from '../../actors/moq-session'; import type { TrackSubscriberActor } from '../../actors/track-subscriber'; import { type AdaptiveLatencyConfig, adaptLatencyTarget } from '../../behaviors/adapt-latency-target'; @@ -155,8 +154,8 @@ export interface MoqEngineConfig extends ShareSignalsConfig; /** - * Render audio through the shared AudioWorklet player core (`@mux/moq-media`'s `createAudioPlayer`) instead of - * scheduled `AudioBufferSourceNode`s. Opt-in: the core cannot apply `playoutRate` nudges yet (mmcc/v10#66). + * Audio player core factory override (tests, or a context the default cannot host). Defaults to `@caprid/moq-media`'s + * `createAudioPlayer` on the render context's `baseContext`. */ createAudioPlayer?: CreateAudioPlayer; /** diff --git a/packages/spf/src/playback/engines/moq/tests/adapter.test.ts b/packages/spf/src/playback/engines/moq/tests/adapter.test.ts index 9585d0c3cd..f645790ef6 100644 --- a/packages/spf/src/playback/engines/moq/tests/adapter.test.ts +++ b/packages/spf/src/playback/engines/moq/tests/adapter.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vite-plus/test'; -import { type MoqAudioContext, MoqMediaElement } from '../adapter'; +import { MoqMediaElement } from '../adapter'; /** * Structural `MoqAudioContext` fake: tracks state transitions through resume/suspend spies so tests can assert the @@ -21,12 +21,6 @@ function createFakeAudioContext(initialState: AudioContextState) { destination: {} as AudioNode, gain, createGain: vi.fn(() => gain), - createBuffer: (() => { - throw new Error('unused in adapter tests'); - }) as MoqAudioContext['createBuffer'], - createBufferSource: (() => { - throw new Error('unused in adapter tests'); - }) as MoqAudioContext['createBufferSource'], resume: vi.fn(async () => { fake.state = 'running'; }), @@ -142,8 +136,8 @@ describe('MoqMediaMixin', () => { media.attach(document.createElement('canvas')); media.destroy(); - // Renderer ticks still call createBuffer/createBufferSource until the - // composition has stopped them; closing first throws InvalidStateError. + // The renderer's player core still drives nodes on the context until + // the composition has stopped it; closing first throws InvalidStateError. expect(audioContext.close).not.toHaveBeenCalled(); await vi.waitFor(() => expect(audioContext.close).toHaveBeenCalledTimes(1)); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 81eb089a32..a408954d35 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1247,6 +1247,9 @@ importers: packages/spf: dependencies: + '@caprid/moq-media': + specifier: ^0.1.0 + version: 0.1.0 '@videojs/media': specifier: workspace:* version: link:../media @@ -2103,6 +2106,14 @@ packages: cpu: [x64] os: [win32] + '@caprid/moq-core@0.1.0': + resolution: {integrity: sha512-GWNqRU+XODpG/4fCUQVXYZtONVibYOY4BwYH4rt7lQuvBJ0sh49W4gw+1jAhY+a924bMGSSGzJ1shr8G5xMMmQ==} + engines: {node: '>=24'} + + '@caprid/moq-media@0.1.0': + resolution: {integrity: sha512-Iu2+r7JYZoRYzoM9Xf3aK2KqBH8x26NGpKhdI3jFICoSr3IeAIXPYaad46HkHq68ottv5LwV4cxyVaFGrQfbZg==} + engines: {node: '>=24'} + '@capsizecss/unpack@4.0.0': resolution: {integrity: sha512-VERIM64vtTP1C4mxQ5thVT9fK0apjPFobqybMtA1UdUujWka24ERHbRHFGmpbbhp73MhV+KSsHQH9C6uOTdEQA==} engines: {node: '>=18'} @@ -5580,133 +5591,157 @@ packages: resolution: {integrity: sha512-jrOY5WM+AaAqkv51fHP1x28ifto4WgcZVzodLUyMU1jMWn5Sq+VciRdk/n8E0Ey5w4p4cWWE+m5GfXWOYh7Kzw==} cpu: [arm64] os: [android] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-codegen/binding-darwin-arm64@0.9.5': resolution: {integrity: sha512-4O4lkCQIzPZGjiNB1GORSI6hBECbiiSBS/APZXPvNKYH+nhY4uuqv03LNXA+SET3hoBjvr95P5rIhY8KQaQUBA==} cpu: [arm64] os: [darwin] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-codegen/binding-darwin-x64@0.9.5': resolution: {integrity: sha512-9EvoUO0SEhD6/d8VHdWuPerepPMSR1y84+UEgz8Un1Ope14Oe7xMvePpEQLTLovoIFZ8Zg3iZ8z8E11pZMqC3g==} cpu: [x64] os: [darwin] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-codegen/binding-freebsd-x64@0.9.5': resolution: {integrity: sha512-HqT78WwgHTmp8lwujoUa9CrIortX4DdpuiVC18ZPSGvuuJf4ylpIEI6QrQEM78Zwz3muhAWAOXZ5irdiYY+AyA==} cpu: [x64] os: [freebsd] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-codegen/binding-linux-arm-gnu@0.9.5': resolution: {integrity: sha512-QJXwIW6Ms3QIawbcBIedmrmXnfdpuAOjZJ/eAABq5XTzWvSxZ7lutu9W5yIHahlaTaFnBo7ikrVMD1szLTpPUw==} cpu: [arm] os: [linux] libc: [glibc] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-codegen/binding-linux-arm-musl@0.9.5': resolution: {integrity: sha512-eHbFy3IHGb+IYarrYwAo0yWSEQV3eAmEn6VrsKA6I1Wy1YPJxWqSJlV69JhqsHtJuJlljUIF3ex0y46yaKIu9w==} cpu: [arm] os: [linux] libc: [musl] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-codegen/binding-linux-arm64-gnu@0.9.5': resolution: {integrity: sha512-ByoJMbySTaDhjAXSu8q6Lh7HKg3YoesXpcT72aYk0Aiw4PCznmY4ybpLTq0RCvp0RIPhFm/6yECFiGBwyCC1nw==} cpu: [arm64] os: [linux] libc: [glibc] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-codegen/binding-linux-arm64-musl@0.9.5': resolution: {integrity: sha512-gD9vfXIoBw1toSxhO9rgmSu/FEfy3PMznJAxVjIuH8DrWEiDKXmJO0pJKfj1Ltbe/TmtWVZR+TjISqeSIGQzMg==} cpu: [arm64] os: [linux] libc: [musl] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-codegen/binding-linux-x64-gnu@0.9.5': resolution: {integrity: sha512-BARvdnvqMGjOr5Iel2JH+9H5vAIE0R6H0Z2fsT02xrvmI/1ZXNB8lkuX+ZGevPhZb3zJ9WeC/R8JZstrsUOK5g==} cpu: [x64] os: [linux] libc: [glibc] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-codegen/binding-linux-x64-musl@0.9.5': resolution: {integrity: sha512-x8flcevS1fbb7ESrmpOY/pON4eInSaE/7Ktjqx2udOE2W33BNSZmJuwpyUgo54AoHNqrcaM7szqydyJn5fNvew==} cpu: [x64] os: [linux] libc: [musl] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-codegen/binding-win32-arm64@0.9.5': resolution: {integrity: sha512-KOL/rBatWqH4ZpCNoF8ZtSNdOJbAxBJLmk/VeRciepGROPTbPum1A9t67GpFgOU7qrkUWlPJdJ+KHKbvbmOt+w==} cpu: [arm64] os: [win32] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-codegen/binding-win32-x64@0.9.5': resolution: {integrity: sha512-FxENahEjWSan59Syh/us/Kf4wNq8qrJLFJ3R2N4Oiwtb6yNdz/rWLNTIoNSssnsp7IoWJdUP6o/Z8ppg7lXMcg==} cpu: [x64] os: [win32] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-parser/binding-android-arm64@0.9.5': resolution: {integrity: sha512-A2JCFCSHfnficqYEw4Iujpx7XrkMM3UfFcgJFmYpZSo9zM4nyhmdIIE0FogSduuU60lhM0/UcfuUBXIVNBMlGQ==} cpu: [arm64] os: [android] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-parser/binding-darwin-arm64@0.9.5': resolution: {integrity: sha512-3PiyU+Eare4YuKaQ22N98/yAiROPY5o/NQJHraICzDvk4pgS+m+bgLKOvkGBRn33OnV95Vdv1Mn38b+MQHpULQ==} cpu: [arm64] os: [darwin] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-parser/binding-darwin-x64@0.9.5': resolution: {integrity: sha512-blFMAFI7AInI83XaiOF8cIeiRM46Nz9EfpZtZPRLbKxSA9aAr5v8aHYpmfN6NoyXxAv/10pwS4gsAuc1W6fy5Q==} cpu: [x64] os: [darwin] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-parser/binding-freebsd-x64@0.9.5': resolution: {integrity: sha512-TsNuL4qsZdO0tHp5GW43Y5fHeLPpPHA675wdcPNTcdq2ZO5AvsQWFFoiDg8CH4oBktfsQ9tdvKhjLnLYwKvp4g==} cpu: [x64] os: [freebsd] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-parser/binding-linux-arm-gnu@0.9.5': resolution: {integrity: sha512-b0afYK5gHeV8RdmOcqAlgM8ONsye4cax4DMnIBaoJyPMd1UTGvTbpkqQcPVdhmHlcfcGVGMV1laeVovlr/dscw==} cpu: [arm] os: [linux] libc: [glibc] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-parser/binding-linux-arm-musl@0.9.5': resolution: {integrity: sha512-fD3lKzl+r6j6n8DwiMY53qnh5DLqD8KJjCp+NbVud54Nnh3Q9Wprqss3YzyMHP3NSJk663Wlg1E1X5qOuFVFig==} cpu: [arm] os: [linux] libc: [musl] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-parser/binding-linux-arm64-gnu@0.9.5': resolution: {integrity: sha512-aRU/aCphV1MWCl/lvI6NX8LgucHQ68Fx+vz7NBb/5MEr2EuzCxiPOQqeFcMdWh+2AED/p0AvAREch0c+cSnlhA==} cpu: [arm64] os: [linux] libc: [glibc] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-parser/binding-linux-arm64-musl@0.9.5': resolution: {integrity: sha512-5+Guro0l8H473YXlEjVNBRLN/IbPbJdnQh1zo0OLt49xxnON+XMJRRaEyTOTfkn9QSRg0+BhEKGR4W9Gu2uZJQ==} cpu: [arm64] os: [linux] libc: [musl] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-parser/binding-linux-x64-gnu@0.9.5': resolution: {integrity: sha512-pwwSyV9q+GlvzSXlsMZBMlgk22L5bud714/vqZCdeUTivzeUVltQzUaf3IQXOGXdpc59JYTeuMCtbGquaAUoCA==} cpu: [x64] os: [linux] libc: [glibc] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-parser/binding-linux-x64-musl@0.9.5': resolution: {integrity: sha512-z18j6JN3lBHH8vzN7gG1M8fI0nlgItSfnC9PfbmUbt97iHViKfw2iA2G9fGwRMe5LyPRZBrejIlapbygPbpdaw==} cpu: [x64] os: [linux] libc: [musl] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-parser/binding-win32-arm64@0.9.5': resolution: {integrity: sha512-s6Gwttb1dQvtPX6Bgkw+UPC9IO3UBDXc6Zezog8MMgvu3UfJBBB9TQqKBX/2quu0Eyh+lLSAyNlIyyecaagUPg==} cpu: [arm64] os: [win32] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-parser/binding-win32-x64@0.9.5': resolution: {integrity: sha512-FrERt9YWatY3bJfSUdi4YWY+6iQcyo3MzCC821BxlWM5TFZEHijnpz/bknR79VHlXY/9CvXz8ZlaAGPsTtN3nw==} cpu: [x64] os: [win32] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-toolchain/types@0.9.5': resolution: {integrity: sha512-KiuLNNgX9uNealaWAR+G3/cMXnRk9x4TY2EkYe/KIag+UPdwiA0RRf1hr1WAxzTP8KGzCTkqUdLPvqh32sEO3w==} @@ -10980,6 +11015,12 @@ snapshots: '@bruits/satteri-win32-x64-msvc@0.10.5': optional: true + '@caprid/moq-core@0.1.0': {} + + '@caprid/moq-media@0.1.0': + dependencies: + '@caprid/moq-core': 0.1.0 + '@capsizecss/unpack@4.0.0': dependencies: fontkitten: 1.0.3 @@ -19251,6 +19292,62 @@ snapshots: transitivePeerDependencies: - msw + vitest@5.0.1(@opentelemetry/api@1.9.1)(@types/node@22.19.15)(@vitest/browser-playwright@5.0.1)(@vitest/browser-preview@5.0.1)(@vitest/coverage-v8@5.0.1)(@vitest/ui@5.0.1)(@voidzero-dev/vite-plus-core@1.0.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.1)(typescript@6.0.2)(unrun@0.2.39)(yaml@2.9.0))(happy-dom@20.11.1)(jsdom@26.1.0(supports-color@8.1.1)): + dependencies: + '@types/chai': 5.2.3 + '@vitest/mocker': 5.0.1(@voidzero-dev/vite-plus-core@1.0.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.1)(typescript@6.0.2)(unrun@0.2.39)(yaml@2.9.0)) + chai: 6.2.2 + es-module-lexer: 2.3.2 + expect-type: 1.4.0 + magic-string: 1.4.2 + obug: 2.1.4 + picomatch: 4.0.7 + std-env: 4.2.0 + tinybench: 6.1.4 + tinyexec: 1.3.0 + tinyglobby: 0.2.17 + vite: '@voidzero-dev/vite-plus-core@1.0.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.1)(typescript@6.0.2)(unrun@0.2.39)(yaml@2.9.0)' + why-is-node-running: 2.3.0 + optionalDependencies: + '@opentelemetry/api': 1.9.1 + '@types/node': 22.19.15 + '@vitest/browser-playwright': 5.0.1(@voidzero-dev/vite-plus-core@1.0.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.1)(typescript@6.0.2)(unrun@0.2.39)(yaml@2.9.0))(playwright@1.59.1)(vitest@5.0.1) + '@vitest/browser-preview': 5.0.1(@voidzero-dev/vite-plus-core@1.0.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.1)(typescript@6.0.2)(unrun@0.2.39)(yaml@2.9.0))(vitest@5.0.1) + '@vitest/coverage-v8': 5.0.1(@vitest/browser@5.0.1)(vitest@5.0.1) + '@vitest/ui': 5.0.1(vitest@5.0.1) + happy-dom: 20.11.1 + jsdom: 26.1.0(supports-color@8.1.1) + transitivePeerDependencies: + - msw + + vitest@5.0.1(@opentelemetry/api@1.9.1)(@types/node@22.19.15)(@vitest/browser-playwright@5.0.1)(@vitest/browser-preview@5.0.1)(@vitest/coverage-v8@5.0.1)(@vitest/ui@5.0.1)(@voidzero-dev/vite-plus-core@1.0.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.1)(typescript@6.0.2)(unrun@0.2.39)(yaml@2.9.0))(happy-dom@20.11.1)(jsdom@27.4.0(supports-color@8.1.1)): + dependencies: + '@types/chai': 5.2.3 + '@vitest/mocker': 5.0.1(@voidzero-dev/vite-plus-core@1.0.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.1)(typescript@6.0.2)(unrun@0.2.39)(yaml@2.9.0)) + chai: 6.2.2 + es-module-lexer: 2.3.2 + expect-type: 1.4.0 + magic-string: 1.4.2 + obug: 2.1.4 + picomatch: 4.0.7 + std-env: 4.2.0 + tinybench: 6.1.4 + tinyexec: 1.3.0 + tinyglobby: 0.2.17 + vite: '@voidzero-dev/vite-plus-core@1.0.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.1)(typescript@6.0.2)(unrun@0.2.39)(yaml@2.9.0)' + why-is-node-running: 2.3.0 + optionalDependencies: + '@opentelemetry/api': 1.9.1 + '@types/node': 22.19.15 + '@vitest/browser-playwright': 5.0.1(@voidzero-dev/vite-plus-core@1.0.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.1)(typescript@6.0.2)(unrun@0.2.39)(yaml@2.9.0))(playwright@1.59.1)(vitest@5.0.1) + '@vitest/browser-preview': 5.0.1(@voidzero-dev/vite-plus-core@1.0.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.1)(typescript@6.0.2)(unrun@0.2.39)(yaml@2.9.0))(vitest@5.0.1) + '@vitest/coverage-v8': 5.0.1(@vitest/browser@5.0.1)(vitest@5.0.1) + '@vitest/ui': 5.0.1(vitest@5.0.1) + happy-dom: 20.11.1 + jsdom: 27.4.0(supports-color@8.1.1) + transitivePeerDependencies: + - msw + vitest@5.0.1(@opentelemetry/api@1.9.1)(@types/node@22.20.1)(@vitest/browser-playwright@5.0.1)(@vitest/browser-preview@5.0.1)(@vitest/coverage-v8@5.0.1)(@vitest/ui@5.0.1)(@voidzero-dev/vite-plus-core@1.0.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.1)(typescript@6.0.2)(unrun@0.2.39)(yaml@2.9.0))(happy-dom@20.11.1)(jsdom@27.4.0(supports-color@8.1.1)): dependencies: '@types/chai': 5.2.3 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index ec937b5110..ae5b9b3691 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -77,3 +77,4 @@ minimumReleaseAgeExclude: - '@oxfmt/*' - vitest - '@vitest/*' + - '@caprid/*' From d75c1648b8ccaf12465def103458301f74ff7f5f Mon Sep 17 00:00:00 2001 From: Matt McClure Date: Tue, 27 Oct 2026 18:34:31 +0000 Subject: [PATCH 3/3] fix(spf): start a new fill when a placed re-join clock freezes Measured from the timeline's first frame, packets that resume past a gap landed beyond the fill horizon and reached the core one per 100 ms stall rescue, holding the clock for up to the core's jitter ceiling over again. Claude-Session: https://claude.ai/code/session_01DaH4ihf1LLPqmXShYkvBat --- packages/spf/src/playback/actors/dom/audio-renderer.ts | 5 ++++- .../src/playback/actors/dom/tests/audio-renderer.test.ts | 7 ++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/spf/src/playback/actors/dom/audio-renderer.ts b/packages/spf/src/playback/actors/dom/audio-renderer.ts index 66f8ed1a6d..1cb1dd9773 100644 --- a/packages/spf/src/playback/actors/dom/audio-renderer.ts +++ b/packages/spf/src/playback/actors/dom/audio-renderer.ts @@ -517,9 +517,12 @@ export function createAudioRendererActor(options: CreateAudioRendererOptions): A if (dryLimitUs !== undefined && !stats.playing) stoppedSinceDry = true; // A placed clock that reaches the end of the audio pushed stops there - // until the core plays, so refill frames cannot drag it forward. + // until the core plays, so refill frames cannot drag it forward. The + // refill is a new fill, as after a dry edge: measured from the timeline's + // first frame, packets after a gap would land past its horizon. if (anchor?.placed && !anchor.frozen && pushedEndUs !== undefined && positionUs! >= pushedEndUs) { anchor = { mediaUs: pushedEndUs, contextUs: contextNowUs(), placed: true, frozen: true }; + fillStartUs = undefined; } for (;;) { diff --git a/packages/spf/src/playback/actors/dom/tests/audio-renderer.test.ts b/packages/spf/src/playback/actors/dom/tests/audio-renderer.test.ts index 3c16007c2f..dac483314a 100644 --- a/packages/spf/src/playback/actors/dom/tests/audio-renderer.test.ts +++ b/packages/spf/src/playback/actors/dom/tests/audio-renderer.test.ts @@ -473,7 +473,7 @@ describe('createAudioRendererActor', () => { expect(renderer.snapshot.get().context.underruns).toBe(1); }); - it('rescues a frozen re-join placement whose next packets are past the fill horizon', async () => { + it('refills a frozen re-join placement in one go when packets resume past a gap', async () => { let anchorUs: number | undefined; const { fake, renderer } = await setup({ getJoinAnchorUs: () => anchorUs }); const source = queueSource(framesBetween(0, 80_000)); @@ -489,12 +489,13 @@ describe('createAudioRendererActor', () => { tick(); expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); - // Packets resume 80 ms after the lone one ends: beyond the fill horizon. + // Packets resume 80 ms after the lone one ends: past the first fill's + // horizon, so the refill measures from where they start. fake.inputs[0]!.stats.playing = false; source.queue.push(...framesBetween(5_100_000, 5_400_000)); at(400_000); tick(); - expect(fake.inputs[0]!.pushed).toContain(5_100_000); + expect(fake.inputs[0]!.pushed.slice(-4)).toEqual([5_100_000, 5_120_000, 5_140_000, 5_160_000]); expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); });