diff --git a/.agents/plans/moq-engine-implementation.md b/.agents/plans/moq-engine-implementation.md index 7085c1f851..f4792bd739 100644 --- a/.agents/plans/moq-engine-implementation.md +++ b/.agents/plans/moq-engine-implementation.md @@ -41,9 +41,6 @@ beside this file. Notable corrections to the plan's assumptions: - `syncPreload` / `trackLoadTriggers` are not composed into the engine: both are `HTMLMediaElement`-bound and there is no element; the adapter writes `state.preload` / `state.loadActivated` directly. -- Audio renderer uses `AudioBufferSourceNode` scheduling, not the - AudioWorklet ring buffer, with the clock contract already shaped for - the swap (TODO(audio-worklet) in `audio-renderer.ts`). - No text-track subscription behavior yet (`switchTextTrack` composes and selects, but nothing renders text). - No WebCodecs `canPlayTrack` probe is injected into selection — diff --git a/packages/html/src/media/simple-moq-video/tests/index.test.ts b/packages/html/src/media/simple-moq-video/tests/index.test.ts index 83028dcf31..8aa3ac42ae 100644 --- a/packages/html/src/media/simple-moq-video/tests/index.test.ts +++ b/packages/html/src/media/simple-moq-video/tests/index.test.ts @@ -13,12 +13,6 @@ function createFakeAudioContext(initialState: AudioContextState) { connect: () => {}, gain: { value: 1 }, }) as unknown as GainNode, - createBuffer: (() => { - throw new Error('unused in wrapper tests'); - }) as MoqAudioContext['createBuffer'], - createBufferSource: (() => { - throw new Error('unused in wrapper tests'); - }) as MoqAudioContext['createBufferSource'], resume: vi.fn(async () => { fake.state = 'running'; }), diff --git a/packages/spf/package.json b/packages/spf/package.json index a3862f5c76..0383c79e0f 100644 --- a/packages/spf/package.json +++ b/packages/spf/package.json @@ -79,6 +79,7 @@ "size:all": "node scripts/measure-size.js --all" }, "dependencies": { + "@caprid/moq-media": "^0.1.0", "@videojs/media": "workspace:*", "@videojs/utils": "workspace:*", "signal-polyfill": "^0.2.2" diff --git a/packages/spf/src/playback/actors/dom/audio-renderer.ts b/packages/spf/src/playback/actors/dom/audio-renderer.ts index d870d131c4..1cb1dd9773 100644 --- a/packages/spf/src/playback/actors/dom/audio-renderer.ts +++ b/packages/spf/src/playback/actors/dom/audio-renderer.ts @@ -1,20 +1,22 @@ /** - * AudioDecoder → Web Audio renderer actor. **Owns the master clock**: once audio is scheduled, `getClockTimeUs()` maps - * the AudioContext's hardware clock back to media time, and the video renderer (and `currentTime` derivation) follow - * it. + * Audio renderer actor over the shared AudioWorklet player core, `@caprid/moq-media`'s `createAudioPlayer`. Frames in + * from the subscriber's jitter buffer, the **master clock** out: the video renderer and `currentTime` follow it. * - * Decoded `AudioData` is copied into `AudioBuffer`s and scheduled gaplessly with `AudioBufferSourceNode`s. Each - * scheduled buffer records a clock segment (context-time span ↔ media time × rate), and the clock reads the segment - * containing `currentTime` — so it reports what was _actually_ scheduled: late arrivals and rate nudges shift the - * timeline forward from where they happen instead of rescaling elapsed time. Rate nudges from the latency controller - * apply as `playbackRate` on the scheduled sources (a ±5% nudge is a barely audible pitch shift). + * The core decodes with WebCodecs, holds an adaptive jitter buffer in an `AudioWorklet`, and reports the audible + * timestamp about every 20 ms. **Latency stays v10's.** The renderer meters frames into the core so the worklet holds + * about its own jitter target, and the rest of the target latency stays in the subscriber's buffer, where `syncLatency` + * measures and steers it. The core's adaptive target is left alone: under metering its growth only moves depth from one + * buffer to the other. The controller's rate nudges apply as the input's `playbackRate` (a ±5% nudge is a barely + * audible pitch shift). * - * Where the schedule _starts_ is `getJoinAnchorUs`: with one supplied the clock anchors at the live edge and the - * buffered backlog is discarded, instead of anchoring at the oldest replayed group and playing all of it. - * - * TODO(audio-worklet): replace source-node scheduling with an AudioWorklet ring buffer for tighter jitter control and - * clean rate adjustment; the actor surface (frame source in, master clock out) is designed so that swap is internal. + * The clock the core reports into is the renderer's own, not the SDK's `PlayoutClock`, so `getClockTimeUs` keeps the + * contract the video renderer and `trackPlayoutTime` rely on: `undefined` before any audio, and **held where the + * worklet ran dry** rather than paced on by wall time — a wall-paced clock steps back when audio resumes, which holds + * slaved video on its last frame for the whole step. */ +import { createAudioPlayer as createCoreAudioPlayer, opusPacketDurationUs } from '@caprid/moq-media'; +import { clamp } from '@videojs/utils/number'; + import { createTransitionActor, type TransitionActor } from '../../../core/actors/create-transition-actor'; import { TIMELINE_DISCONTINUITY_US } from '../../../media/moq/timeline'; import type { JitterFrame } from '../track-subscriber'; @@ -36,7 +38,7 @@ export interface AudioFrameSource { export interface AudioContextLike { readonly currentTime: number; readonly destination: AudioNode; - /** Seconds between the graph and the audio subsystem; the worklet renderer places its clock behind it. */ + /** Seconds between the graph and the audio subsystem; the clock is placed behind it. */ readonly baseLatency?: number; /** Seconds from the audio subsystem to the speaker; read fresh, as it changes with the device. */ readonly outputLatency?: number; @@ -45,8 +47,6 @@ export interface AudioContextLike { * `AudioWorkletNode` cannot be constructed on a plain object. */ readonly baseContext?: BaseAudioContext; - createBuffer(numberOfChannels: number, length: number, sampleRate: number): AudioBuffer; - createBufferSource(): AudioBufferSourceNode; } export type AudioRendererStatus = 'idle' | 'rendering' | 'error'; @@ -54,13 +54,13 @@ export type AudioRendererStatus = 'idle' | 'rendering' | 'error'; export interface AudioRendererContext { status: AudioRendererStatus; framesScheduled: number; - /** Media time of the newest scheduled audio, in microseconds. */ + /** Media time of the newest audio handed to the core, in microseconds. */ scheduledUntilUs?: number; /** - * **Times the schedule ran dry** — the context clock passed the end of everything scheduled while a track was still - * attached. Counted per rising edge, not per tick, so one starvation is one increment. + * **Times the worklet ran dry** — rendering passed the end of everything pushed while a track was still attached. + * Counted per rising edge, not per tick, so one starvation is one increment. * - * The clock itself hides this: `getClockTimeUs` clamps to the segment end and resumes seamlessly, which is right for + * The clock itself hides this: `getClockTimeUs` holds at the dry point and resumes seamlessly, which is right for * presentation and useless as a signal. This is the direct "the target latency is below what the path can sustain" * evidence the adaptive controller has no other way to obtain — a shallow buffer looks identical to a well-behaved * one right up until it is empty. @@ -69,63 +69,140 @@ export interface AudioRendererContext { error?: unknown; } +export interface AudioRendererActor extends Pick, 'snapshot'> { + /** Point the renderer at a (new) frame source + decoder config. */ + setTrack(source: AudioFrameSource | null, config: AudioDecoderConfig | null): void; + /** Master playout clock: current media time in microseconds, or `undefined` before any audio is audible. */ + getClockTimeUs(): number | undefined; + destroy(): void; +} + +/** + * The part of `@caprid/moq-media`'s `PlayoutClock` an audio player input calls. The renderer implements it, so the + * input reports into the renderer's clock. + */ +export interface AudioPlayerClock { + /** Caps the input's jitter target: the core keeps it under this less the output delay and 40 ms. */ + readonly targetLatencyUs: number; + /** The time source reports are measured on, in microseconds. */ + localNow(): number; + /** The audio with this timestamp is audible at local time `atUs`. */ + reportAudio(timestampUs: number, atUs?: number): void; + /** The input stopped. */ + stopAudio(): void; +} + +/** The part of `@caprid/moq-media`'s `AudioInputOptions` the renderer passes. */ +export interface AudioPlayerInputOptions { + clock?: AudioPlayerClock; + decoderConfig?: AudioDecoderConfig; + /** See `AudioPlayerInput.playbackRate`. */ + playbackRate?: number; + /** A decoder error. The core decodes again from the next frame, with a new decoder. */ + onError?(cause: unknown): void; +} + +/** The part of `@caprid/moq-media`'s `AudioInputStats` the renderer reads. Durations are microseconds. */ +export interface AudioPlayerInputStats { + /** Whether the worklet is playing, rather than filling to its target. */ + readonly playing: boolean; + /** Decoded audio the worklet holds. */ + readonly bufferedUs: number; + /** How much the worklet holds before it plays, at the start and after running dry. */ + readonly jitterTargetUs: number; +} + +/** The part of `@caprid/moq-media`'s `AudioPlayerInput` the renderer uses: one track. */ +export interface AudioPlayerInput { + /** Decode and hold a frame, in decode order. */ + push(frame: { timestampUs: number; payload: Uint8Array }): void; + /** Without a range: drop everything held or still decoding. */ + cut(): void; + /** Held audio played per output frame, from the next render quantum; 0.5–2, or the core throws. */ + playbackRate: number; + readonly stats: AudioPlayerInputStats; + remove(): void; +} + +/** The part of `@caprid/moq-media`'s `AudioPlayer` the renderer uses. */ +export interface AudioPlayer { + /** The mix of every input; the renderer connects it to its context's `destination`. */ + readonly output: { connect(destination: AudioNode): void }; + addInput(options?: AudioPlayerInputOptions): AudioPlayerInput; + /** Remove every input and disconnect the output. */ + close(): void; +} + +/** Create the player on the renderer's context. Default: the core on the context's `baseContext`. */ +export type CreateAudioPlayer = (context: AudioContextLike) => Promise; + export interface CreateAudioRendererOptions { + /** Where the player's output connects (`destination`), and the output latencies the clock is placed behind. */ audioContext: AudioContextLike; - /** Playout rate (latency nudges). Default 1. */ + createAudioPlayer?: CreateAudioPlayer; + /** Playout rate (latency nudges), clamped to the core's 0.5–2. Default 1. */ getPlaybackRate?: () => number; - /** Scheduling safety margin in seconds ahead of `currentTime`. Default 0.05. */ - scheduleMargin?: number; - /** Decode/schedule cadence in ms. Default 10. */ - tickIntervalMs?: number; /** - * **Join at the live edge.** Media timestamp the schedule should start at while it has no anchor — at join, and again - * after a catch-up skip. Buffered audio older than it is discarded rather than played: audio cannot be fast-forwarded - * the way video can without pitch artifacts. `undefined` (the default, and whatever the caller returns before any - * frame has arrived) anchors at the oldest buffered frame instead. + * **Join at the live edge.** Media timestamp the playout should start at while it has no anchor — at join, and again + * after a catch-up skip. Buffered audio older than it is discarded before it reaches the core: audio cannot be + * fast-forwarded the way video can without pitch artifacts. `undefined` (the default, and whatever the caller returns + * before any frame has arrived) starts at the oldest buffered frame instead. */ getJoinAnchorUs?: () => number | undefined; + /** + * The target latency in microseconds, read on every clock report. It caps the core's jitter target, so the worklet + * never holds more than the latency the controller is aiming at. Default 2 s, where the core's own 400 ms ceiling + * applies instead. + */ + getTargetLatencyUs?: () => number | undefined; + /** Metering cadence in ms. Default 10. */ + tickIntervalMs?: number; } type RendererMessage = | { type: 'status'; status: AudioRendererStatus; error?: unknown } + | { type: 'decoder-error'; error: unknown } | { type: 'scheduled'; untilUs: number } | { type: 'underrun' }; -export interface AudioRendererActor extends Pick, 'snapshot'> { - /** Point the renderer at a (new) frame source + decoder config. */ - setTrack(source: AudioFrameSource | null, config: AudioDecoderConfig | null): void; - /** Master playout clock: current media time in microseconds, or `undefined` before any audio has been scheduled. */ - getClockTimeUs(): number | undefined; - destroy(): void; -} - // ============================================================================= // Implementation // ============================================================================= -const DEFAULT_SCHEDULE_MARGIN_S = 0.05; const DEFAULT_TICK_INTERVAL_MS = 10; +const DEFAULT_TARGET_LATENCY_US = 2_000_000; +/** The frame duration assumed before a frame has shown one: Opus's usual 20 ms. */ +const DEFAULT_FRAME_DURATION_US = 20_000; +/** Reports and pushed spans may disagree by sample rounding and resampling offsets. */ +const REPORT_TOLERANCE_US = 5_000; /** - * Max chunks in flight inside the decoder. The schedule horizon only moves when decoder _outputs_ arrive (async), so - * without this bound a single tick would dequeue an entire backlog — scheduling seconds of audio at once and - * sawtoothing the jitter-buffer depth the latency controller measures. + * How long a fill may sit short of playback before one more frame is pushed. The fill pushes exactly the jitter target + * plus a frame, so audio the core drops on arrival (behind what it already decoded) could otherwise leave it waiting + * forever. */ -const MAX_PENDING_DECODES = 4; -/** Allow sample rounding without treating a missing audio packet as contiguous. */ -const AUDIO_TIMESTAMP_TOLERANCE_US = 1_000; +const FILL_STALL_US = 100_000; /** * Media-time jump beyond which incoming audio is a timeline reset rather than a gap or reorder within the same timeline * (see `TIMELINE_DISCONTINUITY_US`). **Both directions.** Forward: a latency catch-up skipped groups, or the - * publisher's capture gapped — scheduling the jump as silence would keep the latency it was meant to shed. Backward: - * the publisher re-anchored the track's timeline (a capture source switch) — butt-joining it would keep audio playing + * publisher's capture gapped — playing the jump as silence would keep the latency it was meant to shed. Backward: the + * publisher re-anchored the track's timeline (a capture source switch) — butt-joining it would keep audio playing * seamlessly while the master clock silently steps backwards by the jump, holding the slaved video renderer "early" on - * every frame it ever receives again. Either way the schedule re-anchors on the jumped-to timeline. + * every frame it ever receives again. Either way playout re-anchors on the jumped-to timeline. */ const DISCONTINUITY_THRESHOLD_US = TIMELINE_DISCONTINUITY_US; +/** The core's `playbackRate` range; it throws outside it. */ +const MIN_PLAYBACK_RATE = 0.5; +const MAX_PLAYBACK_RATE = 2; + +/** The core on the real context behind `context`: an `AudioWorkletNode` needs one. */ +const createDefaultAudioPlayer: CreateAudioPlayer = async (context) => { + if (!context.baseContext) throw new Error('audio renderer context has no baseContext to host the AudioWorklet'); + + return createCoreAudioPlayer({ context: context.baseContext }); +}; export function createAudioRendererActor(options: CreateAudioRendererOptions): AudioRendererActor { const { audioContext } = options; - const scheduleMargin = options.scheduleMargin ?? DEFAULT_SCHEDULE_MARGIN_S; const inner = createTransitionActor( { status: 'idle', framesScheduled: 0, underruns: 0 }, @@ -133,6 +210,8 @@ export function createAudioRendererActor(options: CreateAudioRendererOptions): A switch (message.type) { case 'status': return { ...context, status: message.status, error: message.error }; + case 'decoder-error': + return { ...context, error: message.error }; case 'scheduled': return { ...context, @@ -146,16 +225,13 @@ export function createAudioRendererActor(options: CreateAudioRendererOptions): A } ); + let destroyed = false; + let player: AudioPlayer | undefined; + let input: AudioPlayerInput | undefined; let source: AudioFrameSource | null = null; let decoderConfig: AudioDecoderConfig | null = null; - let decoder: AudioDecoder | null = null; - let destroyed = false; - /** Timestamp of the last chunk fed to the decoder, for discontinuity detection. */ - let lastEnqueuedUs: number | undefined; - let lastDecodedDurationUs: number | undefined; - let lastDecodedEndUs: number | undefined; - let drainingDecoder: AudioDecoder | undefined; - /** The schedule has no anchor yet: the next one should be placed at the live edge. */ + + /** Playout has no anchor yet: the next one should be placed at the live edge. */ let needsJoinAnchor = true; /** * The pending anchor placement follows a mid-stream discontinuity rather than a track (re)start. The distinction @@ -165,103 +241,166 @@ export function createAudioRendererActor(options: CreateAudioRendererOptions): A * pitch-shifted seconds rebuilding. */ let rejoinAnchorArmed = false; + + // What this timeline has pushed into the core. Reset with the timeline. + let firstPushedUs: number | undefined; + let lastPushedUs: number | undefined; + /** Known once a packet's duration or two frames' spacing has shown it. */ + let frameDurationUs: number | undefined; + /** Where the newest pushed frame ends: the furthest the clock can read. */ + let pushedEndUs: number | undefined; + /** First timestamp pushed in the current fill, and the context time of the latest push. */ + let fillStartUs: number | undefined; + let lastPushAtUs = 0; + /** - * Media time the next scheduled buffer should be placed relative to: the landed re-join anchor. Consumed by the first - * buffer scheduled onto an empty schedule — it starts `(timestamp − scheduleFromUs)` later than arrival, with a - * bookkeeping segment spanning the gap so the clock advances smoothly from the anchor instead of stepping to the - * buffer. + * The clock: timeline position `mediaUs` is audible at context time `contextUs`, advancing with the context clock + * times `playbackRate` from there. Set by every core report, or **placed** at a re-join anchor before the core plays + * anything (see `dropToJoinAnchor`). + * + * Context time, not `performance.now()`, because it is what stops when the adapter suspends the context on pause: a + * wall-paced clock would run on through the pause, read the frozen worklet as dry and refill it, and step back on the + * first report after resume. The core times its reports on the same source (`localNow`), so its render clock's + * calibration does not carry a pause into the first reports after it either. */ - let scheduleFromUs: number | undefined; - + let anchor: { mediaUs: number; contextUs: number; placed: boolean; frozen?: boolean } | undefined; + /** Bumped per input, so a report from a removed input cannot anchor its replacement. */ + let inputGeneration = 0; /** - * One scheduled buffer's context-time span mapped to media time. The media time at context time `t` inside the span - * is `mediaUs + (t - startCtx) * 1e6 * rate`. + * Where the worklet ran dry. The clock holds there until a report of audio past it arrives: frames pushed to refill + * the worklet move `pushedEndUs` before they play, and must not drag the clock forward with them. */ - interface ClockSegment { - startCtx: number; - endCtx: number; - mediaUs: number; - rate: number; - } - + let dryLimitUs: number | undefined; + /** + * The position the clock held at through the latest underrun, kept as a floor after it ends. The first report of the + * refill is dated ahead by the output latency — the sample is not audible yet — so its anchor extrapolates to before + * the hold, and the clock would otherwise step back by that much. + */ + let holdFloorUs: number | undefined; /** - * The scheduled timeline, in start order. Bounded by the schedule horizon (`scheduleMargin * 4` of audio ≈ a dozen - * 20ms buffers) plus whatever `pruneSegments` hasn't collected yet. + * The core has reported itself stopped since the dry point was set. Until it has, a report near the dry point is the + * tail of the audio before it, still in flight, and must not end the hold. */ - const segments: ClockSegment[] = []; - const activeSources = new Set(); + let stoppedSinceDry = false; + /** The rate the current input plays at, and the clock advances at. */ + let playbackRate = 1; - const rate = (): number => options.getPlaybackRate?.() ?? 1; + /** The audio between rendering and audible, in media time. */ + const outputLatencyUs = (): number => + ((audioContext.baseLatency ?? 0) + (audioContext.outputLatency ?? 0)) * 1_000_000 * playbackRate; - const segmentEndMediaUs = (segment: ClockSegment): number => - segment.mediaUs + (segment.endCtx - segment.startCtx) * 1_000_000 * segment.rate; + const contextNowUs = (): number => audioContext.currentTime * 1_000_000; - /** True while the context clock is past everything scheduled (see `underruns`). */ - let starved = false; + /** The audible position extrapolated from the anchor, ignoring every limit. */ + const extrapolatedUs = (): number | undefined => + anchor && (anchor.frozen ? anchor.mediaUs : anchor.mediaUs + (contextNowUs() - anchor.contextUs) * playbackRate); /** - * Count the rising edge of a schedule that has run dry. An empty schedule is not an underrun — at join, and after - * every re-anchor that clears it, there was no schedule to starve. + * Apply the controller's rate from the next render quantum. The clock is rebased first, so the time before the change + * keeps the old pace; the audio still in the output path plays out at it too, which the next report corrects. */ - const detectUnderrun = (): void => { - const last = segments[segments.length - 1]; + const applyPlaybackRate = (active: AudioPlayerInput): void => { + const rate = clamp(options.getPlaybackRate?.() ?? 1, MIN_PLAYBACK_RATE, MAX_PLAYBACK_RATE); + if (rate === playbackRate) return; - if (!last) { - starved = false; - return; - } + if (anchor && !anchor.frozen) anchor = { ...anchor, mediaUs: extrapolatedUs()!, contextUs: contextNowUs() }; - const dry = last.endCtx <= audioContext.currentTime; + active.playbackRate = rate; + playbackRate = rate; + }; - if (dry && !starved) inner.send({ type: 'underrun' }); + /** The clock one input reports into; it goes deaf once that input is replaced. */ + const createClock = (): AudioPlayerClock => { + const generation = ++inputGeneration; + + return { + get targetLatencyUs() { + return options.getTargetLatencyUs?.() ?? DEFAULT_TARGET_LATENCY_US; + }, + localNow: contextNowUs, + reportAudio(timestampUs, atUs) { + if (generation !== inputGeneration) return; + + reportAudio(timestampUs, atUs); + }, + // The renderer resets the clock itself wherever a track or timeline ends. + stopAudio() {}, + }; + }; - starved = dry; + const raiseHoldFloor = (heldUs: number): void => { + holdFloorUs = Math.max(holdFloorUs ?? heldUs, heldUs); }; - /** Drop segments that finished playing; keep the current/newest one so the clock can hold on underrun. */ - const pruneSegments = (): void => { - const now = audioContext.currentTime; + /** + * A placed clock runs from the anchor until the core plays, but never past the audio pushed: a fill that stays short + * of the jitter target cannot start the worklet, and the clock would otherwise run on through audio not there. + */ + const placedPositionUs = (): number | undefined => { + const positionUs = extrapolatedUs(); - while (segments.length > 1 && now >= segments[1]!.startCtx) segments.shift(); + return positionUs === undefined || pushedEndUs === undefined ? positionUs : Math.min(positionUs, pushedEndUs); }; - const stopAll = (): void => { - for (const node of activeSources) { - try { - node.stop(); - } catch { - // never started or already stopped - } + /** `timestampUs` is audible at context time `atUs` (default now). */ + const reportAudio = (timestampUs: number, atUs: number | undefined): void => { + // Only audio pushed on this timeline: a report already in flight when + // the timeline restarted describes audio the cut dropped. + if ( + firstPushedUs === undefined || + pushedEndUs === undefined || + timestampUs < firstPushedUs - REPORT_TOLERANCE_US || + timestampUs > pushedEndUs + REPORT_TOLERANCE_US + ) { + return; } - activeSources.clear(); - segments.length = 0; + // The refill's first report ends the hold. Its floor is what the clock + // reads at that moment — the audible position, which can still be short + // of the dry point while the device plays out what it had queued. + if ( + dryLimitUs !== undefined && + ((stoppedSinceDry && timestampUs >= dryLimitUs - REPORT_TOLERANCE_US) || + timestampUs > dryLimitUs + REPORT_TOLERANCE_US) + ) { + raiseHoldFloor(Math.min(extrapolatedUs() ?? dryLimitUs, dryLimitUs)); + dryLimitUs = undefined; + } + + // Likewise the first report after a placed re-join, which is dated ahead + // of the placed clock by the output latency. + if (anchor?.placed) raiseHoldFloor(placedPositionUs()!); + + anchor = { mediaUs: timestampUs, contextUs: atUs ?? contextNowUs(), placed: false }; }; - const closeDecoder = (): void => { - if (decoder && decoder.state !== 'closed') decoder.close(); - - decoder = null; - drainingDecoder = undefined; - lastDecodedDurationUs = undefined; - lastDecodedEndUs = undefined; - // A fresh decoder starts a fresh input timeline: the next chunk fed is - // not a jump within the old one, so it must not re-trigger the - // discontinuity restart that closed this decoder — and a gap placement - // landed against the old timeline must not span into the new one. - lastEnqueuedUs = undefined; - scheduleFromUs = undefined; - stopAll(); + /** Forget everything pushed on the current timeline, and the clock placed on it. */ + const resetTimeline = (): void => { + firstPushedUs = undefined; + lastPushedUs = undefined; + frameDurationUs = undefined; + pushedEndUs = undefined; + fillStartUs = undefined; + anchor = undefined; + dryLimitUs = undefined; + holdFloorUs = undefined; + stoppedSinceDry = false; + }; + + const removeInput = (): void => { + if (!input) return; + + inputGeneration++; + input.remove(); + input = undefined; }; /** - * Discard buffered audio older than the join anchor, so the schedule starts at the live edge rather than at the - * oldest group the relay replayed. Video decode-forwards its backlog; audio cannot (pitch), so the backlog is dropped - * unheard. - * - * Anything already scheduled belongs to the pre-anchor timeline, so a landed drop closes the decoder — the jumped-to - * audio then anchors fresh instead of being butt-joined after a matching stretch of scheduled silence, which would - * keep exactly the latency the drop shed. + * Discard buffered audio older than the join anchor before the first push of a (re)start, so playout starts at the + * live edge rather than at the oldest group the relay replayed. A re-join anchor lands by **placement** when the + * whole buffer is already past it (see `rejoinAnchorArmed`): the clock is placed at the anchor and runs from there, + * and metering by timeline holds the head back until it is due, so playout resumes at the target depth rather than at + * the live edge. */ const dropToJoinAnchor = (): void => { const anchorUs = options.getJoinAnchorUs?.(); @@ -277,266 +416,135 @@ export function createAudioRendererActor(options: CreateAudioRendererOptions): A if (dropped > 0) { needsJoinAnchor = false; rejoinAnchorArmed = false; - // The buffer head now sits at the anchor, so scheduling it on - // arrival lands the clock there with the target depth behind it. - scheduleFromUs = undefined; - closeDecoder(); return; } - // Nothing behind the edge (yet). At a fresh join, stay armed: a relay's - // group replay arrives as a burst (oldest first), so the buffer's live - // edge may still be moving. - if (!rejoinAnchorArmed) return; + // A fresh join stays armed: a relay's group replay arrives as a burst, + // oldest first, so the buffer's live edge may still be moving. + if (!rejoinAnchorArmed || !source.peek()) return; - // A discontinuity re-join whose whole buffer is already ahead of the - // anchor — no replay is coming (bursts arrive oldest-first, and the - // oldest is already past the anchor). Land the anchor by *placement* - // instead of by trimming: the head schedules at its distance from the - // anchor, so playout resumes at the target depth rather than racing to - // the live edge and pitch-rebuilding from zero. - if (!source.peek()) return; - - scheduleFromUs = anchorUs; + anchor = { mediaUs: anchorUs, contextUs: contextNowUs(), placed: true }; rejoinAnchorArmed = false; needsJoinAnchor = false; }; - const scheduleAudioData = (data: AudioData): void => { - try { - if (destroyed || data.numberOfFrames === 0) return; - - lastDecodedDurationUs = data.duration; - lastDecodedEndUs = data.timestamp + data.duration; - - const buffer = audioContext.createBuffer(data.numberOfChannels, data.numberOfFrames, data.sampleRate); - - for (let channel = 0; channel < data.numberOfChannels; channel++) { - const channelData = new Float32Array(data.numberOfFrames); - - data.copyTo(channelData, { planeIndex: channel, format: 'f32-planar' }); - buffer.copyToChannel(channelData, channel); - } - - const currentRate = rate(); - // A jump past the discontinuity threshold in either direction means - // the source is on a different timeline — skipped ahead (latency - // catch-up: scheduling it as silence would keep the latency it was - // meant to shed, and re-trigger the catch-up loop forever) or - // re-anchored behind (a publisher source switch: butt-joining it - // would step the master clock backwards under the slaved video). - // Drop the stale schedule and anchor fresh. - // - // "Fresh" has to mean *through the join anchor*, the same way the dequeue - // path above re-arms it. Clearing the schedule alone left `needsJoinAnchor` - // false — and nothing on this path ever set it — so the next buffer took the - // no-previous-segment branch below and started at - // `currentTime + scheduleMargin`, keeping whatever latency the skip was meant - // to shed, with no pending re-anchor left to correct it. - // - // Returning rather than scheduling this buffer is what lets the re-anchor - // see an empty schedule (`tick` only drops while `segments.length === 0`); - // `finally` closes the AudioData, and one dropped buffer is cheaper than a - // timeline anchored at the wrong place. - // - // **Kept as a guard against a decoder, not as a live path.** The dequeue - // path's own check fires first on every jump that reaches this one: it is - // armed exactly when `previous` exists (only `closeDecoder` clears - // `lastEnqueuedUs`, and it clears `segments` in the same breath), it - // compares the same threshold, and `AudioDecoder.close()` discards every - // in-flight decode — measured, not assumed — so no pre-jump output can - // arrive after it either. What is left for this branch is a decoder whose - // *outputs* jump where its inputs did not, the mirror image of the - // timestamp rebasing the dequeue check exists for. Anything relying on it - // firing has no coverage; treat it as unexercised. - const previous = segments[segments.length - 1]; - - if (previous && Math.abs(data.timestamp - segmentEndMediaUs(previous)) > DISCONTINUITY_THRESHOLD_US) { - stopAll(); - needsJoinAnchor = true; - rejoinAnchorArmed = true; - return; + /** + * Whether `timestampUs` should reach the core now. Filling — before the first report, or after running dry — pushes + * the jitter target plus a frame, which is what the worklet waits for before playing. Playing pushes what will be + * rendered within the jitter target, so the worklet holds about that much and the rest waits in the subscriber. + */ + const isDue = (timestampUs: number, stats: AudioPlayerInputStats): boolean => { + const positionUs = anchor?.placed ? placedPositionUs() : extrapolatedUs(); + + // A frozen placement is a fill too: its clock waits for the worklet to + // start, so only the fill rules (including the stall rescue) can move it. + if (positionUs === undefined || dryLimitUs !== undefined || anchor?.frozen) { + if ( + fillStartUs === undefined || + timestampUs < fillStartUs + stats.jitterTargetUs + (frameDurationUs ?? DEFAULT_FRAME_DURATION_US) + ) { + return true; } - const last = segments[segments.length - 1]; - // Continue the scheduled timeline: media-contiguous data butt-joins - // the previous buffer; a media gap inserts a matching stretch of - // context-time silence. Late arrivals (and the first buffer) start - // no earlier than the context clock — the timeline shifts forward - // from there rather than trying to make up lost time. - let idealStart: number; - - if (last) { - const mediaEndUs = segmentEndMediaUs(last); - const gapS = Math.max(0, data.timestamp - mediaEndUs) / 1_000_000 / currentRate; - - idealStart = last.endCtx + gapS; - - if (gapS > 0) { - // The next packet establishes the gap's duration. Advance the - // master clock through that known silence so video need not - // freeze with each missing audio packet. - segments.push({ startCtx: last.endCtx, endCtx: idealStart, mediaUs: mediaEndUs, rate: currentRate }); - } - } else { - idealStart = audioContext.currentTime + scheduleMargin; - - // A landed re-join anchor places this first buffer at its distance - // from the anchor rather than at arrival, resuming playout at the - // target depth. The bookkeeping segment spanning the gap plays - // nothing (nothing is scheduled over it) but lets the clock advance - // smoothly from the anchor — so the slaved video renderer plays - // through the gap instead of seeing the clock step to the buffer. - if (scheduleFromUs !== undefined) { - const gapS = Math.max(0, data.timestamp - scheduleFromUs) / 1_000_000 / currentRate; - - if (gapS > 0) { - segments.push({ - startCtx: idealStart, - endCtx: idealStart + gapS, - mediaUs: scheduleFromUs, - rate: currentRate, - }); - idealStart += gapS; - } - - scheduleFromUs = undefined; - } - } + return contextNowUs() - lastPushAtUs > FILL_STALL_US && stats.bufferedUs < stats.jitterTargetUs; + } - const startAt = Math.max(idealStart, audioContext.currentTime); + return timestampUs < positionUs + outputLatencyUs() + stats.jitterTargetUs; + }; - const node = audioContext.createBufferSource(); + const push = (active: AudioPlayerInput, timestampUs: number, payload: Uint8Array): void => { + active.push({ timestampUs, payload }); - node.buffer = buffer; - node.playbackRate.value = currentRate; - node.connect(audioContext.destination); - node.onended = () => activeSources.delete(node); - node.start(startAt); - activeSources.add(node); + // Opus says how long each packet is. Otherwise the shortest step between + // pushed frames is the frame length: a step over lost packets is longer, + // and must not extend the audio past where it really ends. + const opusUs = decoderConfig?.codec === 'opus' ? opusPacketDurationUs(payload) : undefined; + const stepUs = lastPushedUs === undefined ? undefined : timestampUs - lastPushedUs; - const durationS = data.numberOfFrames / data.sampleRate / currentRate; + if (opusUs !== undefined) frameDurationUs = opusUs; + else if (stepUs !== undefined && stepUs > 0) frameDurationUs = Math.min(frameDurationUs ?? stepUs, stepUs); - segments.push({ startCtx: startAt, endCtx: startAt + durationS, mediaUs: data.timestamp, rate: currentRate }); - inner.send({ type: 'scheduled', untilUs: data.timestamp + data.duration }); - } finally { - data.close(); - } + firstPushedUs ??= timestampUs; + fillStartUs ??= timestampUs; + lastPushedUs = timestampUs; + lastPushAtUs = contextNowUs(); + pushedEndUs = timestampUs + (frameDurationUs ?? DEFAULT_FRAME_DURATION_US); + inner.send({ type: 'scheduled', untilUs: pushedEndUs }); }; - const ensureDecoder = (): AudioDecoder | null => { - if (decoder || !decoderConfig) return decoder; + /** + * The worklet runs dry once rendering passes everything pushed: metering pushes every frame as soon as it is due, so + * nothing was left to push. Each such edge is an underrun (`AudioRendererContext.underruns`). The core's own counter + * skips dry spells longer than its jitter ceiling, which `adaptLatencyTarget` still needs as failure evidence. + */ + const detectDry = (): void => { + const positionUs = extrapolatedUs(); + + if ( + !anchor || + anchor.placed || + dryLimitUs !== undefined || + pushedEndUs === undefined || + positionUs === undefined || + positionUs + outputLatencyUs() < pushedEndUs + ) { + return; + } - decoder = new AudioDecoder({ - output: scheduleAudioData, - error: (error) => inner.send({ type: 'status', status: 'error', error }), - }); - decoder.configure(decoderConfig); - return decoder; + dryLimitUs = pushedEndUs; + stoppedSinceDry = false; + fillStartUs = undefined; + inner.send({ type: 'underrun' }); }; const tick = (): void => { - // After a decoder error there is nothing productive to pull into — - // draining would silently strip the jitter buffer at tick rate. - if (destroyed || !source || drainingDecoder || inner.snapshot.get().context.status === 'error') return; + if (destroyed || !player || !source || !decoderConfig) return; try { - pruneSegments(); - detectUnderrun(); - - // Keep the schedule topped up to the margin horizon; every audio - // frame is independently decodable, so no keyframe gating. - while ( - (decoder?.decodeQueueSize ?? 0) < MAX_PENDING_DECODES && - (segments[segments.length - 1]?.endCtx ?? 0) - audioContext.currentTime < scheduleMargin * 4 - ) { - // Only while the schedule has no anchor to disturb: in steady state - // the latency controller owns depth corrections (rate nudges inside - // its deadband), and dropping audio here would preempt them. - if (needsJoinAnchor && segments.length === 0) dropToJoinAnchor(); + const active = (input ??= player.addInput({ + clock: createClock(), + decoderConfig, + playbackRate, + onError: (error) => inner.send({ type: 'decoder-error', error }), + })); + const { stats } = active; + + detectDry(); + applyPlaybackRate(active); + + const positionUs = extrapolatedUs(); + + if (dryLimitUs !== undefined && !stats.playing) stoppedSinceDry = true; + + // A placed clock that reaches the end of the audio pushed stops there + // until the core plays, so refill frames cannot drag it forward. The + // refill is a new fill, as after a dry edge: measured from the timeline's + // first frame, packets after a gap would land past its horizon. + if (anchor?.placed && !anchor.frozen && pushedEndUs !== undefined && positionUs! >= pushedEndUs) { + anchor = { mediaUs: pushedEndUs, contextUs: contextNowUs(), placed: true, frozen: true }; + fillStartUs = undefined; + } + + for (;;) { + if (needsJoinAnchor && lastPushedUs === undefined) dropToJoinAnchor(); const next = source.peek(); if (!next) return; - // FFmpeg-backed audio decoders rebase output timestamps by frame - // accumulation from the first input, hiding an input-side jump - // from the scheduler. Restart the decoder at the discontinuity so - // outputs re-base to the jumped-to timeline (every audio frame is - // independently decodable, so a restart is glitch-free). - if (lastEnqueuedUs !== undefined && Math.abs(next.timestampUs - lastEnqueuedUs) > DISCONTINUITY_THRESHOLD_US) { - closeDecoder(); - // Re-place the anchor before feeding the jumped-to timeline: a - // forward jump (catch-up skip) landed on a group start still up - // to one GOP behind the live edge, and a backward jump (publisher - // re-anchor) is a whole new timeline whose edge the anchor must - // be re-read against. + // A timeline reset, either way: drop what the core holds and re-place + // the anchor on the jumped-to timeline (see `DISCONTINUITY_THRESHOLD_US`). + if (lastPushedUs !== undefined && Math.abs(next.timestampUs - lastPushedUs) > DISCONTINUITY_THRESHOLD_US) { + active.cut(); + resetTimeline(); needsJoinAnchor = true; rejoinAnchorArmed = true; continue; } - // Some audio decoders accumulate output timestamps across missing - // packets. Even a single short gap then permanently shifts the - // audio master clock behind the wire timeline. Drain before - // restarting so pending outputs and scheduled audio survive. The - // output-end check also handles codecs that split one input into - // several outputs: packet duration is only a prompt to check. - if ( - decoder && - lastEnqueuedUs !== undefined && - (lastDecodedDurationUs === undefined || - next.timestampUs - lastEnqueuedUs > lastDecodedDurationUs + AUDIO_TIMESTAMP_TOLERANCE_US) - ) { - const current = decoder; - const timestampUs = next.timestampUs; - - drainingDecoder = current; - void current.flush().then( - () => { - if (decoder !== current || destroyed) return; - - if (lastDecodedEndUs !== undefined && timestampUs - lastDecodedEndUs > AUDIO_TIMESTAMP_TOLERANCE_US) { - current.close(); - decoder = null; - lastDecodedDurationUs = undefined; - lastDecodedEndUs = undefined; - } - - lastEnqueuedUs = undefined; - drainingDecoder = undefined; - }, - (error) => { - if (decoder !== current || destroyed) return; - - drainingDecoder = undefined; - inner.send({ type: 'status', status: 'error', error }); - } - ); - return; - } + if (!isDue(next.timestampUs, stats)) return; - const active = ensureDecoder(); - // An errored decoder is closed but still referenced — stop pulling - // instead of feeding it doomed decode calls. - if (!active || active.state === 'closed') return; - - // Decode before dequeue: if decode throws, the frame stays - // buffered for a later decoder instead of being silently dropped. - active.decode( - new EncodedAudioChunk({ - // Every audio sample is independently decodable — `isKey` is a - // LOC group-boundary marker (meaningful for video's GOP - // gating), not a decodability signal here. A live-edge - // (`next-object`) join almost never lands on group object 0, - // and WebCodecs requires the first post-configure chunk to be - // 'key', so gating on `next.isKey` would reject it. - type: 'key', - timestamp: next.timestampUs, - data: next.payload, - }) - ); + push(active, next.timestampUs, next.payload); source.dequeue(); - lastEnqueuedUs = next.timestampUs; } } catch (error) { inner.send({ type: 'status', status: 'error', error }); @@ -544,24 +552,45 @@ export function createAudioRendererActor(options: CreateAudioRendererOptions): A }; const timer = setInterval(tick, options.tickIntervalMs ?? DEFAULT_TICK_INTERVAL_MS); + (options.createAudioPlayer ?? createDefaultAudioPlayer)(audioContext).then( + (created) => { + if (destroyed) { + created.close(); + return; + } + + try { + created.output.connect(audioContext.destination); + player = created; + } catch (error) { + created.close(); + inner.send({ type: 'status', status: 'error', error }); + } + }, + (error) => { + if (!destroyed) inner.send({ type: 'status', status: 'error', error }); + } + ); + return { get snapshot() { return inner.snapshot; }, setTrack(nextSource, nextConfig): void { - // A source without a decodable config can never schedule audio — it - // would spin every tick while the subscriber's jitter buffer grows - // unbounded. Fail fast instead. + // A source without a decodable config can never play — it would spin + // every tick while the subscriber's jitter buffer grows unbounded. + // Fail fast instead. const missingConfig = nextSource !== null && nextConfig === null; source = missingConfig ? null : nextSource; decoderConfig = nextConfig; needsJoinAnchor = true; - // A track (re)start is a fresh join, not a re-join: time-to-first-audio - // wins over landing at depth (see `rejoinAnchorArmed`). rejoinAnchorArmed = false; - closeDecoder(); + // An edge since the last tick still counts: the reset below forgets it. + detectDry(); + removeInput(); + resetTimeline(); if (missingConfig) { inner.send({ @@ -576,19 +605,23 @@ export function createAudioRendererActor(options: CreateAudioRendererOptions): A }, getClockTimeUs(): number | undefined { - pruneSegments(); - const segment = segments[0]; - if (!segment) return undefined; - - // Clamp into the segment: before the first sample plays the clock - // holds at its start; past the last scheduled sample (underrun) it - // holds at the end and resumes seamlessly when audio is scheduled. - const elapsed = Math.min( - Math.max(0, audioContext.currentTime - segment.startCtx), - segment.endCtx - segment.startCtx - ); - - return segment.mediaUs + elapsed * 1_000_000 * segment.rate; + const positionUs = extrapolatedUs(); + if (positionUs === undefined) return undefined; + + if (anchor?.placed) return placedPositionUs(); + + // Never past the audio pushed, and once the worklet ran dry never past + // where it did: the clock holds through an underrun and resumes when + // audio does. + const limitUs = dryLimitUs ?? pushedEndUs; + const clampedUs = limitUs === undefined ? positionUs : Math.min(positionUs, limitUs); + + if (holdFloorUs === undefined) return clampedUs; + + if (clampedUs < holdFloorUs) return holdFloorUs; + + holdFloorUs = undefined; + return clampedUs; }, destroy(): void { @@ -596,7 +629,9 @@ export function createAudioRendererActor(options: CreateAudioRendererOptions): A destroyed = true; clearInterval(timer); - closeDecoder(); + removeInput(); + player?.close(); + player = undefined; inner.destroy(); }, }; diff --git a/packages/spf/src/playback/actors/dom/tests/audio-renderer.test.ts b/packages/spf/src/playback/actors/dom/tests/audio-renderer.test.ts index 1d0cb2b02b..dac483314a 100644 --- a/packages/spf/src/playback/actors/dom/tests/audio-renderer.test.ts +++ b/packages/spf/src/playback/actors/dom/tests/audio-renderer.test.ts @@ -1,645 +1,616 @@ -import { describe, expect, it, vi } from 'vite-plus/test'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vite-plus/test'; import type { JitterFrame } from '../../track-subscriber'; -import { type AudioContextLike, type AudioFrameSource, createAudioRendererActor } from '../audio-renderer'; - -const SAMPLE_RATE = 48_000; -const FRAME_SAMPLES = 960; // 20ms opus frames -const FRAME_DURATION_US = 20_000; +import { + type AudioContextLike, + type AudioFrameSource, + type AudioPlayer, + type AudioPlayerInput, + type AudioPlayerInputOptions, + createAudioRendererActor, +} from '../audio-renderer'; + +const FRAME_US = 20_000; +const JITTER_TARGET_US = 60_000; +const TICK_MS = 10; +const CONFIG: AudioDecoderConfig = { codec: 'opus', sampleRate: 48_000, numberOfChannels: 1 }; + +/** An Opus TOC byte for one 20 ms CELT frame (config 19, code 0). */ +const OPUS_20MS_TOC = 19 << 3; + +function frameAt(timestampUs: number): JitterFrame { + return { groupId: timestampUs, objectId: 0, timestampUs, isKey: true, payload: new Uint8Array([OPUS_20MS_TOC]) }; +} -/** Encode a short opus sequence and return LOC-shaped jitter frames. */ -async function encodeTestFrames(count: number): Promise { +/** Frames every 20 ms over `[fromUs, untilUs)`. */ +function framesBetween(fromUs: number, untilUs: number): JitterFrame[] { const frames: JitterFrame[] = []; - const encoder = new AudioEncoder({ - output: (chunk) => { - const payload = new Uint8Array(chunk.byteLength); - - chunk.copyTo(payload); - frames.push({ - groupId: 0, - objectId: frames.length, - timestampUs: frames.length * FRAME_DURATION_US, - isKey: true, - payload, - }); - }, - error: (error) => { - throw error; - }, - }); - - encoder.configure({ codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1, bitrate: 32_000 }); - for (let i = 0; i < count; i++) { - const samples = new Float32Array(FRAME_SAMPLES); + for (let timestampUs = fromUs; timestampUs < untilUs; timestampUs += FRAME_US) frames.push(frameAt(timestampUs)); - for (let s = 0; s < FRAME_SAMPLES; s++) samples[s] = Math.sin((2 * Math.PI * 440 * s) / SAMPLE_RATE) * 0.5; + return frames; +} - const data = new AudioData({ - format: 'f32', - sampleRate: SAMPLE_RATE, - numberOfFrames: FRAME_SAMPLES, - numberOfChannels: 1, - timestamp: i * FRAME_DURATION_US, - data: samples, - }); +function queueSource(frames: JitterFrame[] = []): AudioFrameSource & { queue: JitterFrame[] } { + const queue = [...frames]; - encoder.encode(data); - data.close(); - } + return { queue, peek: () => queue[0], dequeue: () => queue.shift() }; +} - await encoder.flush(); - encoder.close(); - return frames; +interface FakeInput extends AudioPlayerInput { + options: AudioPlayerInputOptions; + pushed: number[]; + cuts: number; + removed: boolean; + stats: { playing: boolean; bufferedUs: number; jitterTargetUs: number }; } -function arraySource(frames: JitterFrame[]): AudioFrameSource { - const queue = [...frames]; +/** The core's surface with no audio behind it: pushes are recorded, and the test plays the worklet's reports. */ +function createFakePlayer() { + const inputs: FakeInput[] = []; + const connected: AudioNode[] = []; + let closed = false; + const player: AudioPlayer = { + output: { connect: (destination) => connected.push(destination) }, + addInput(options = {}) { + const input: FakeInput = { + options, + pushed: [], + cuts: 0, + removed: false, + playbackRate: options.playbackRate ?? 1, + stats: { playing: true, bufferedUs: 0, jitterTargetUs: JITTER_TARGET_US }, + push(frame) { + input.pushed.push(frame.timestampUs); + }, + cut() { + input.cuts++; + }, + remove() { + input.removed = true; + }, + }; + + inputs.push(input); + return input; + }, + close() { + closed = true; + }, + }; return { - peek: () => queue[0], - dequeue: () => queue.shift(), + player, + inputs, + connected, + get closed() { + return closed; + }, + /** The worklet reporting that the audio at `timestampUs` is audible now. */ + report(timestampUs: number, atUs: number) { + inputs[inputs.length - 1]!.options.clock!.reportAudio(timestampUs, atUs); + }, }; } -/** - * `AudioContextLike` with a hand-cranked clock: scheduling is a no-op, so tests can place `currentTime` anywhere and - * read the media clock back. - */ -function createFakeAudioContext(): AudioContextLike & { currentTime: number } { +// SAFETY: the renderer only hands `destination` to `output.connect`, which the fake records by identity. +const destination = {} as AudioNode; + +/** Context time follows the test's clock; `suspended` freezes it, as `AudioContext.suspend()` does. */ +let contextUs = 0; +let suspended = false; + +function fakeAudioContext(outputLatency?: number): AudioContextLike { return { - currentTime: 0, - destination: {} as AudioNode, - createBuffer: (_channels, length, sampleRate) => - ({ copyToChannel: () => {}, duration: length / sampleRate }) as unknown as AudioBuffer, - createBufferSource: () => - ({ - buffer: null, - playbackRate: { value: 1 }, - connect: () => {}, - start: () => {}, - stop: () => {}, - onended: null, - }) as unknown as AudioBufferSourceNode, + get currentTime() { + return contextUs / 1_000_000; + }, + destination, + outputLatency, }; } describe('createAudioRendererActor', () => { - it('decodes and schedules audio, exposing the master clock', async () => { - const frames = await encodeTestFrames(5); - // OfflineAudioContext satisfies AudioContextLike; its clock stays at 0, - // so the media clock holds at the anchor — deterministic for a test. - const audioContext = new OfflineAudioContext(1, SAMPLE_RATE, SAMPLE_RATE); - const renderer = createAudioRendererActor({ audioContext, scheduleMargin: 0.05 }); - - expect(renderer.getClockTimeUs()).toBeUndefined(); + let nowUs = 0; - renderer.setTrack(arraySource(frames), { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); + /** Move wall time, and context time with it unless the context is suspended. */ + const at = (us: number) => { + if (!suspended) contextUs += us - nowUs; - await vi.waitFor(() => expect(renderer.snapshot.get().context.framesScheduled).toBeGreaterThan(0), { - timeout: 5000, - }); - const context = renderer.snapshot.get().context; - - expect(context.status).toBe('rendering'); - expect(context.scheduledUntilUs).toBeGreaterThan(0); + nowUs = us; + }; - // Clock anchored to the first scheduled frame's media time (0), and it - // holds there while the offline context clock is not advancing. - expect(renderer.getClockTimeUs()).toBe(0); + beforeEach(() => { + vi.useFakeTimers(); + at(0); + contextUs = 0; + suspended = false; + }); - renderer.destroy(); + afterEach(() => { + vi.useRealTimers(); }); - // The clock deliberately hides an underrun — it clamps to the segment - // end and resumes seamlessly, which is right for presentation and - // useless as a signal. This counter is the only direct evidence that a - // target latency is below what the path sustains. - it('counts the rising edge of a schedule that ran dry', async () => { - const frames = await encodeTestFrames(3); - const audioContext = createFakeAudioContext(); - const renderer = createAudioRendererActor({ audioContext, scheduleMargin: 0.05, tickIntervalMs: 1 }); - - // Nothing scheduled is not an underrun: at join there is no schedule - // to starve. - await new Promise((resolve) => setTimeout(resolve, 30)); - expect(renderer.snapshot.get().context.underruns).toBe(0); - - renderer.setTrack(arraySource(frames), { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - await vi.waitFor(() => expect(renderer.snapshot.get().context.framesScheduled).toBeGreaterThan(0), { - timeout: 5000, + /** A renderer whose player is ready and whose first input exists. */ + async function setup( + options: { + getJoinAnchorUs?: () => number | undefined; + getPlaybackRate?: () => number; + outputLatency?: number; + } = {} + ) { + const { outputLatency, ...rendererOptions } = options; + const fake = createFakePlayer(); + const renderer = createAudioRendererActor({ + audioContext: fakeAudioContext(outputLatency), + createAudioPlayer: () => Promise.resolve(fake.player), + tickIntervalMs: TICK_MS, + ...rendererOptions, }); - expect(renderer.snapshot.get().context.underruns).toBe(0); - // The source is exhausted; walk the hardware clock past everything - // scheduled and the schedule is dry. - audioContext.currentTime = 60; - await vi.waitFor(() => expect(renderer.snapshot.get().context.underruns).toBe(1), { timeout: 5000 }); + await vi.advanceTimersByTimeAsync(0); + return { fake, renderer }; + } - // Still dry several ticks later — one starvation, one increment. - await new Promise((resolve) => setTimeout(resolve, 50)); - expect(renderer.snapshot.get().context.underruns).toBe(1); + const tick = () => vi.advanceTimersByTime(TICK_MS); + + it('connects the player output to the context destination', async () => { + const { fake, renderer } = await setup(); + + expect(fake.connected).toEqual([destination]); + expect(renderer.getClockTimeUs()).toBeUndefined(); renderer.destroy(); + expect(fake.closed).toBe(true); }); - it('clearing the track resets to idle and drops the anchor', async () => { - const frames = await encodeTestFrames(2); - const audioContext = new OfflineAudioContext(1, SAMPLE_RATE, SAMPLE_RATE); - const renderer = createAudioRendererActor({ audioContext }); + it('fills to the jitter target plus a frame, then meters by the clock', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 1_000_000)); - renderer.setTrack(arraySource(frames), { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - await vi.waitFor(() => expect(renderer.snapshot.get().context.framesScheduled).toBeGreaterThan(0), { - timeout: 5000, - }); + renderer.setTrack(source, CONFIG); + tick(); + + const input = fake.inputs[0]!; - renderer.setTrack(null, null); - expect(renderer.snapshot.get().context.status).toBe('idle'); + expect(input.options.decoderConfig).toEqual(CONFIG); + expect(input.pushed).toEqual([0, 20_000, 40_000, 60_000]); expect(renderer.getClockTimeUs()).toBeUndefined(); - renderer.destroy(); + // Playing: the worklet should hold about its target ahead of what renders. + fake.report(0, 0); + at(30_000); + tick(); + + expect(input.pushed.at(-1)).toBe(80_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(30_000, 0); + expect(renderer.snapshot.get().context.scheduledUntilUs).toBe(100_000); }); - it('errors when a source is set without a decoder config', async () => { - const audioContext = createFakeAudioContext(); - const renderer = createAudioRendererActor({ audioContext, tickIntervalMs: 5 }); - const queue: JitterFrame[] = [ - { groupId: 0, objectId: 0, timestampUs: 0, isKey: true, payload: new Uint8Array([1]) }, - { groupId: 0, objectId: 1, timestampUs: 20_000, isKey: true, payload: new Uint8Array([2]) }, - ]; - const source: AudioFrameSource = { peek: () => queue[0], dequeue: () => queue.shift() }; + it('joins at the edge, dropping buffered audio behind the anchor unheard', async () => { + const { fake, renderer } = await setup({ getJoinAnchorUs: () => 200_000 }); - renderer.setTrack(source, null); + renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); + tick(); - expect(renderer.snapshot.get().context.status).toBe('error'); - expect(renderer.snapshot.get().context.error).toBeInstanceOf(Error); - // The rejected source must not be drained (or retained) by the tick loop. - await new Promise((resolve) => setTimeout(resolve, 50)); - expect(queue).toHaveLength(2); + expect(fake.inputs[0]!.pushed[0]).toBe(200_000); + }); - renderer.destroy(); + it('holds the clock where the worklet ran dry, through the refill, until audio past it plays', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + + at(100_000); + tick(); + expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); + + // Audio comes back: the refill moves the pushed end, not the clock. + fake.inputs[0]!.stats.playing = false; + source.queue.push(...framesBetween(80_000, 400_000)); + at(300_000); + tick(); + expect(fake.inputs[0]!.pushed.slice(4)).toEqual([80_000, 100_000, 120_000, 140_000]); + expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); + + fake.report(80_000, 320_000); + at(330_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(90_000, 0); }); - it('stops draining the jitter buffer after a decoder error', async () => { - const frames = await encodeTestFrames(10); - const audioContext = createFakeAudioContext(); - const queue = [...frames]; - const source: AudioFrameSource = { peek: () => queue[0], dequeue: () => queue.shift() }; - const renderer = createAudioRendererActor({ audioContext, tickIntervalMs: 5 }); + it('does not step back when the refill is reported ahead of being audible', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 80_000)); - // A well-formed but undecodable config: the decoder rejects either at - // configure or on first decode, and the pull loop must stop instead of - // stripping the queue one frame per tick. - renderer.setTrack(source, { codec: 'bogus-codec', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + at(100_000); + tick(); - await vi.waitFor(() => expect(renderer.snapshot.get().context.status).toBe('error'), { timeout: 5000 }); - const remaining = queue.length; + fake.inputs[0]!.stats.playing = false; + source.queue.push(...framesBetween(80_000, 400_000)); + at(300_000); + tick(); - expect(remaining).toBeGreaterThan(0); - await new Promise((resolve) => setTimeout(resolve, 100)); - expect(queue).toHaveLength(remaining); + // Rendered now, audible 100 ms from now: the anchor alone reads 80 ms behind the hold. + fake.report(80_000, 400_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); - renderer.destroy(); + at(410_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(90_000, 0); }); - it('decodes a live-edge join whose first frame is mid-group (isKey: false)', async () => { - const frames = await encodeTestFrames(3); + it('resumes from the audible position when audio returns before the device drained', async () => { + const { fake, renderer } = await setup({ outputLatency: 0.05 }); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + + // Rendering passed the last pushed sample; 50 ms of it is still in the device. + at(30_000); + tick(); + expect(renderer.getClockTimeUs()).toBeCloseTo(30_000, 0); + + fake.inputs[0]!.stats.playing = false; + source.queue.push(...framesBetween(80_000, 400_000)); + at(40_000); + tick(); + fake.report(80_000, 90_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(40_000, 0); + }); - // A `next-object` subscribe joins live, so the first delivered frame - // is whatever object is newest right now — essentially never a LOC - // group's object 0 (`isKey: true`). The regression: decode() gated the - // WebCodecs chunk type on `isKey`, and WebCodecs rejects a 'delta' - // chunk as the first one fed to a fresh decoder. - frames[0] = { ...frames[0]!, groupId: 3, objectId: 7, isKey: false }; - const audioContext = new OfflineAudioContext(1, SAMPLE_RATE, SAMPLE_RATE); - const renderer = createAudioRendererActor({ audioContext, scheduleMargin: 0.05 }); + it('ends pushed audio where the last packet ends, not across a gap of lost packets', async () => { + const { fake, renderer } = await setup(); - renderer.setTrack(arraySource(frames), { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); + renderer.setTrack(queueSource([0, 20_000, 40_000, 100_000].map(frameAt)), CONFIG); + tick(); + fake.report(0, 0); + at(60_000); + tick(); - await vi.waitFor(() => expect(renderer.snapshot.get().context.framesScheduled).toBeGreaterThan(0), { - timeout: 5000, - }); - expect(renderer.snapshot.get().context.status).toBe('rendering'); + expect(fake.inputs[0]!.pushed).toEqual([0, 20_000, 40_000, 100_000]); + expect(renderer.snapshot.get().context.scheduledUntilUs).toBe(120_000); + }); - renderer.destroy(); + it('keeps the hold through a late report of the audio before the underrun', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + at(100_000); + tick(); + + // The refill is pushed before the core has said it stopped; then the + // tail's last report, still in flight, arrives. + source.queue.push(...framesBetween(80_000, 400_000)); + tick(); + fake.report(77_333, 77_333); + at(200_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); }); - it('applies rate changes forward-only: already-scheduled audio keeps its clock mapping', async () => { - const frames = await encodeTestFrames(5); - const audioContext = createFakeAudioContext(); - let playbackRate = 1; - const renderer = createAudioRendererActor({ - audioContext, - scheduleMargin: 0.05, - getPlaybackRate: () => playbackRate, - }); + it('holds a placed re-join clock at the audio pushed while the fill is short', async () => { + let anchorUs: number | undefined; + const { fake, renderer } = await setup({ getJoinAnchorUs: () => anchorUs }); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + + anchorUs = 4_900_000; + source.queue.push(frameAt(5_000_000)); + tick(); + at(100_000); + tick(); + expect(fake.inputs[0]!.pushed.at(-1)).toBe(5_000_000); + + // One packet cannot start the worklet; the clock stops at its end. + at(2_100_000); + tick(); + expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); + + // The refill plays, reported ahead of being audible: no step back. + source.queue.push(...framesBetween(5_020_000, 5_200_000)); + tick(); + fake.report(5_000_000, 2_150_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); + }); - renderer.setTrack(arraySource(frames), { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - // The decoder may split outputs, so wait on scheduled media time (the - // last input chunk spans 80–100ms) rather than an output count. - await vi.waitFor( - () => expect(renderer.snapshot.get().context.scheduledUntilUs).toBeGreaterThanOrEqual(4.5 * FRAME_DURATION_US), - { timeout: 5000 } - ); + it('gives the core a report time source that stops while the context is suspended', async () => { + const { fake, renderer } = await setup(); - // 30ms into playback (the timeline starts at the 50ms schedule margin). - audioContext.currentTime = 0.08; - const before = renderer.getClockTimeUs()!; + renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); + tick(); - expect(before).toBeGreaterThan(0); - expect(before).toBeLessThan(5 * FRAME_DURATION_US); + const clock = fake.inputs[0]!.options.clock!; - // A rate nudge must not rescale time already scheduled at the old - // rate — the regression here was `clock = anchor + elapsed * newRate`, - // which jumped by 5% of the total elapsed interval on every nudge. - playbackRate = 1.5; - expect(renderer.getClockTimeUs()).toBe(before); + at(40_000); + expect(clock.localNow()).toBeCloseTo(40_000, 0); - // The clock advances with the context clock through the scheduled - // segments regardless of the current rate setting. - audioContext.currentTime = 0.09; - const later = renderer.getClockTimeUs()!; + suspended = true; + at(540_000); + expect(clock.localNow()).toBeCloseTo(40_000, 0); - expect(later).toBeGreaterThan(before); - expect(later).toBeLessThanOrEqual(before + 10_001); + // The first report after resume, timed on that source, lands where audio is. + suspended = false; + at(560_000); + fake.report(60_000, clock.localNow()); + expect(renderer.getClockTimeUs()).toBeCloseTo(60_000, 0); + }); - renderer.destroy(); + it('drops what the core holds at a timeline reset and ignores reports from before it', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + + // A publisher re-anchor: the timeline steps back by far more than a gap. + source.queue.push(...framesBetween(-5_000_000, -4_000_000)); + tick(); + + const input = fake.inputs[0]!; + + expect(input.cuts).toBe(1); + expect(input.pushed.at(4)).toBe(-5_000_000); + expect(renderer.getClockTimeUs()).toBeUndefined(); + + // Still in flight from the old timeline when the cut landed. + fake.report(40_000, 0); + expect(renderer.getClockTimeUs()).toBeUndefined(); + + fake.report(-5_000_000, 0); + expect(renderer.getClockTimeUs()).toBeCloseTo(-5_000_000, 0); }); - it('holds on underrun and resumes from late-scheduled audio, not the ideal timeline', async () => { - const frames = await encodeTestFrames(3); - const audioContext = createFakeAudioContext(); - const queue = [...frames.slice(0, 2)]; - const source: AudioFrameSource = { peek: () => queue[0], dequeue: () => queue.shift() }; - const renderer = createAudioRendererActor({ audioContext, scheduleMargin: 0.05 }); - - renderer.setTrack(source, { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - await vi.waitFor( - () => expect(renderer.snapshot.get().context.scheduledUntilUs).toBeGreaterThanOrEqual(1.5 * FRAME_DURATION_US), - { timeout: 5000 } - ); - - // Context clock runs past everything scheduled: the media clock holds - // at the end of scheduled audio instead of drifting ahead. - audioContext.currentTime = 0.5; - const held = renderer.getClockTimeUs()!; - - audioContext.currentTime = 0.6; - expect(renderer.getClockTimeUs()).toBe(held); - expect(held).toBeGreaterThanOrEqual(FRAME_DURATION_US); - expect(held).toBeLessThanOrEqual(2 * FRAME_DURATION_US); - - // The late frame schedules at the context clock; the media clock - // resumes seamlessly from where it held — not from where the original - // anchor says it "should" be (~550ms in). - queue.push(frames[2]!); - await vi.waitFor( - () => expect(renderer.snapshot.get().context.scheduledUntilUs).toBeGreaterThanOrEqual(2.5 * FRAME_DURATION_US), - { timeout: 5000 } - ); - audioContext.currentTime = 0.61; - const resumed = renderer.getClockTimeUs()!; - - expect(resumed).toBeGreaterThanOrEqual(held); - expect(resumed).toBeLessThan(held + FRAME_DURATION_US); + it('places a re-join whose whole buffer is past the anchor at its distance from it', async () => { + let anchorUs: number | undefined; + const { fake, renderer } = await setup({ getJoinAnchorUs: () => anchorUs }); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); - renderer.destroy(); + // A catch-up skip landed on a group 100 ms past where the anchor falls. + anchorUs = 4_900_000; + source.queue.push(...framesBetween(5_000_000, 6_000_000)); + tick(); + + const input = fake.inputs[0]!; + + expect(input.pushed).toHaveLength(4); + expect(renderer.getClockTimeUs()).toBeCloseTo(4_900_000, 0); + + at(50_000); + tick(); + expect(input.pushed.at(4)).toBe(5_000_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(4_950_000, 0); }); - it.each([20_000, 200_000])('preserves a %iµs gap in the incoming audio timeline', async (gapUs) => { - const frames = (await encodeTestFrames(8)).map((frame, index) => ({ - ...frame, - timestampUs: frame.timestampUs + (index >= 2 ? gapUs : 0), - })); - const audioContext = createFakeAudioContext(); - const renderer = createAudioRendererActor({ audioContext, scheduleMargin: 0.2 }); - - try { - renderer.setTrack(arraySource(frames), { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - - await vi.waitFor( - () => expect(renderer.snapshot.get().context.scheduledUntilUs).toBeGreaterThanOrEqual(150_000 + gapUs), - { - timeout: 2000, - } - ); - // A known media gap is scheduled silence, not a stalled clock: - // video can keep advancing while no audio samples are available. - audioContext.currentTime = 0.24 + gapUs / 2_000_000; - expect(renderer.getClockTimeUs()).toBeGreaterThanOrEqual(30_000 + gapUs / 2); - expect(renderer.getClockTimeUs()).toBeLessThanOrEqual(50_000 + gapUs / 2); - audioContext.currentTime = 0.3 + gapUs / 1_000_000; - expect(renderer.getClockTimeUs()).toBeGreaterThanOrEqual(90_000 + gapUs); - expect(renderer.getClockTimeUs()).toBeLessThanOrEqual(110_000 + gapUs); - } finally { - renderer.destroy(); - } + it('freezes the clock and stops metering while the context is suspended', async () => { + const { fake, renderer } = await setup(); + const source = queueSource(framesBetween(0, 1_000_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + at(30_000); + tick(); + + const pushed = fake.inputs[0]!.pushed.length; + + // Paused for two seconds: nothing plays, so nothing is due and nothing ran dry. + suspended = true; + at(2_030_000); + tick(); + expect(renderer.getClockTimeUs()).toBeCloseTo(30_000, 0); + expect(fake.inputs[0]!.pushed).toHaveLength(pushed); + + suspended = false; + at(2_040_000); + expect(renderer.getClockTimeUs()).toBeCloseTo(40_000, 0); }); - it('does not revive a decoder whose track was cleared while it drained', async () => { - const frames = await encodeTestFrames(5); - const audioContext = createFakeAudioContext(); - const createSource = vi.spyOn(audioContext, 'createBufferSource'); - const flush = AudioDecoder.prototype.flush; - let finishDrain!: () => void; - const drained = new Promise((resolve) => { - finishDrain = resolve; - }); - const pendingFlush = vi.spyOn(AudioDecoder.prototype, 'flush').mockImplementation(async function ( - this: AudioDecoder - ) { - await flush.call(this); - await drained; - }); - const renderer = createAudioRendererActor({ audioContext }); - - try { - renderer.setTrack(arraySource(frames), { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - await vi.waitFor(() => expect(pendingFlush).toHaveBeenCalled()); - renderer.setTrack(null, null); - const scheduled = createSource.mock.calls.length; - - finishDrain(); - await new Promise((resolve) => setTimeout(resolve, 50)); - expect(renderer.snapshot.get().context.status).toBe('idle'); - expect(renderer.getClockTimeUs()).toBeUndefined(); - expect(createSource).toHaveBeenCalledTimes(scheduled); - } finally { - finishDrain(); - renderer.destroy(); - pendingFlush.mockRestore(); - createSource.mockRestore(); - } + it('ignores reports from an input that a track change removed', async () => { + const { fake, renderer } = await setup(); + + renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); + tick(); + + const stale = fake.inputs[0]!.options.clock!; + + renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); + tick(); + stale.reportAudio(40_000, 0); + expect(renderer.getClockTimeUs()).toBeUndefined(); + + fake.report(0, 0); + expect(renderer.getClockTimeUs()).toBeCloseTo(0, 0); }); - it('re-anchors instead of inserting silence on a large timestamp jump', async () => { - const frames = await encodeTestFrames(3); - // Latency catch-up: everything from the third frame on arrives from far - // ahead of the scheduled timeline. The whole tail moves (the encoder's - // flush can emit one more output than was fed) — a real skip relocates - // the stream, it does not interleave two timelines. - const JUMP_US = 5_000_000; + it('counts every time the worklet runs dry, across track changes', async () => { + const { fake, renderer } = await setup(); - for (let i = 2; i < frames.length; i++) { - frames[i] = { ...frames[i]!, timestampUs: JUMP_US + (i - 2) * FRAME_DURATION_US }; - } + renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); + tick(); + fake.report(0, 0); - const audioContext = createFakeAudioContext(); - const renderer = createAudioRendererActor({ audioContext, scheduleMargin: 0.05 }); + // A stall far longer than any jitter ceiling still counts, once. + at(100_000); + tick(); + at(3_000_000); + tick(); + expect(renderer.snapshot.get().context.underruns).toBe(1); - renderer.setTrack(arraySource(frames), { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - await vi.waitFor(() => expect(renderer.snapshot.get().context.scheduledUntilUs).toBeGreaterThanOrEqual(JUMP_US), { - timeout: 5000, - }); + renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); + tick(); + fake.report(0, 3_000_000); + at(3_100_000); + tick(); + expect(renderer.snapshot.get().context.underruns).toBe(2); + }); - // Without re-anchoring the jump maps to ~5s of scheduled silence, so - // shortly into playback the clock would still sit in the pre-jump - // timeline; the reset schedules the jumped audio right at the context - // clock (+margin) and the media clock lands past the jump immediately. - audioContext.currentTime = 0.06; - expect(renderer.getClockTimeUs()).toBeGreaterThanOrEqual(JUMP_US); + it('counts a dry edge that a track change lands on before the next tick', async () => { + const { fake, renderer } = await setup(); - renderer.destroy(); + renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); + tick(); + fake.report(0, 0); + + at(100_000); + renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); + expect(renderer.snapshot.get().context.underruns).toBe(1); }); - it('anchors at the live edge, discarding the replayed backlog unheard', async () => { - // The relay replayed 1s of audio before the first tick runs. - const frames = await encodeTestFrames(50); - const NEWEST_US = 49 * FRAME_DURATION_US; - const TARGET_S = 0.2; - const ANCHOR_US = NEWEST_US - TARGET_S * 1_000_000; - const audioContext = createFakeAudioContext(); - const queue = [...frames]; - const source: AudioFrameSource = { peek: () => queue[0], dequeue: () => queue.shift() }; - const renderer = createAudioRendererActor({ - audioContext, - scheduleMargin: 0.05, - getJoinAnchorUs: () => ANCHOR_US, - }); + it('refills a frozen re-join placement in one go when packets resume past a gap', async () => { + let anchorUs: number | undefined; + const { fake, renderer } = await setup({ getJoinAnchorUs: () => anchorUs }); + const source = queueSource(framesBetween(0, 80_000)); + + renderer.setTrack(source, CONFIG); + tick(); + fake.report(0, 0); + + anchorUs = 4_900_000; + source.queue.push(frameAt(5_000_000)); + tick(); + at(200_000); + tick(); + expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); + + // Packets resume 80 ms after the lone one ends: past the first fill's + // horizon, so the refill measures from where they start. + fake.inputs[0]!.stats.playing = false; + source.queue.push(...framesBetween(5_100_000, 5_400_000)); + at(400_000); + tick(); + expect(fake.inputs[0]!.pushed.slice(-4)).toEqual([5_100_000, 5_120_000, 5_140_000, 5_160_000]); + expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); + }); - renderer.setTrack(source, { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - await vi.waitFor(() => expect(renderer.snapshot.get().context.framesScheduled).toBeGreaterThan(0), { - timeout: 5000, - }); + it('applies the playout rate to the input and paces the clock with it', async () => { + let rate = 1; + const { fake, renderer } = await setup({ getPlaybackRate: () => rate }); + + renderer.setTrack(queueSource(framesBetween(0, 2_000_000)), CONFIG); + tick(); + fake.report(0, 0); - // The 39 pre-anchor frames left the buffer without being decoded: none - // remain, and nothing like that many were scheduled. - expect(queue.every((frame) => frame.timestampUs >= ANCHOR_US)).toBe(true); - expect(renderer.snapshot.get().context.framesScheduled).toBeLessThan(15); - expect(renderer.snapshot.get().context.scheduledUntilUs).toBeGreaterThan(ANCHOR_US); + for (let us = 10_000; us <= 100_000; us += 10_000) { + at(us); + tick(); + } - // The clock reads the anchor, not 0 — and the backlog did not become - // ~800ms of scheduled silence in front of it, which would put the - // clock back at the start of the timeline this far into playback. - expect(renderer.getClockTimeUs()).toBeGreaterThanOrEqual(ANCHOR_US); - audioContext.currentTime = 0.06; - expect(renderer.getClockTimeUs()).toBeGreaterThanOrEqual(ANCHOR_US); + expect(renderer.getClockTimeUs()).toBeCloseTo(100_000, 0); - renderer.destroy(); + rate = 1.05; + tick(); + expect(fake.inputs[0]!.playbackRate).toBe(1.05); + + // The time before the change keeps its pace; only what follows speeds up. + for (let us = 110_000; us <= 200_000; us += 10_000) { + at(us); + tick(); + } + + expect(renderer.getClockTimeUs()).toBeCloseTo(205_000, 0); }); - it('leaves a buffer shallower than the target alone', async () => { - const frames = await encodeTestFrames(5); - const audioContext = createFakeAudioContext(); - const queue = [...frames]; - const source: AudioFrameSource = { peek: () => queue[0], dequeue: () => queue.shift() }; - // A live join with no replay: the edge is inside the target, so the - // anchor sits behind the buffer head and must not move playout. - const renderer = createAudioRendererActor({ - audioContext, - scheduleMargin: 0.05, - getJoinAnchorUs: () => -400_000, - }); + it('clamps the playout rate to the range the core accepts, and starts a new input at it', async () => { + let rate = 3; + const { fake, renderer } = await setup({ getPlaybackRate: () => rate }); - renderer.setTrack(source, { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - await vi.waitFor(() => expect(renderer.snapshot.get().context.framesScheduled).toBeGreaterThan(0), { - timeout: 5000, - }); + renderer.setTrack(queueSource(framesBetween(0, 200_000)), CONFIG); + tick(); + expect(fake.inputs[0]!.playbackRate).toBe(2); - expect(renderer.getClockTimeUs()).toBe(0); + rate = 0; + tick(); + expect(fake.inputs[0]!.playbackRate).toBe(0.5); - renderer.destroy(); + renderer.setTrack(queueSource(framesBetween(0, 200_000)), CONFIG); + tick(); + expect(fake.inputs[1]!.options.playbackRate).toBe(0.5); + expect(renderer.snapshot.get().context.status).toBe('rendering'); }); - it('re-anchors at the live edge after a catch-up skip, not at the group start', async () => { - const frames = await encodeTestFrames(30); - const audioContext = createFakeAudioContext(); - // Two 300ms stretches either side of a catch-up skip: the controller - // drops the buffer to the newest keyframe-led group, so the surviving - // group still starts ~5s of media time behind its own newest frame. - const SKIP_US = 5_000_000; - const queue = frames.slice(0, 15); - const source: AudioFrameSource = { peek: () => queue[0], dequeue: () => queue.shift() }; - let anchorUs: number | undefined; + it('caps the core jitter target at the target latency', async () => { + const fake = createFakePlayer(); const renderer = createAudioRendererActor({ - audioContext, - scheduleMargin: 0.05, - getJoinAnchorUs: () => anchorUs, + audioContext: fakeAudioContext(), + createAudioPlayer: () => Promise.resolve(fake.player), + getTargetLatencyUs: () => 250_000, + tickIntervalMs: TICK_MS, }); - renderer.setTrack(source, { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - await vi.waitFor(() => expect(renderer.snapshot.get().context.framesScheduled).toBeGreaterThan(0), { - timeout: 5000, - }); - expect(renderer.getClockTimeUs()).toBe(0); - - // The skip lands: the surviving group runs SKIP_US … SKIP_US + 300ms. - const survivingGroup = frames - .slice(15) - .map((frame, i) => ({ ...frame, timestampUs: SKIP_US + i * FRAME_DURATION_US })); - const newestUs = survivingGroup[survivingGroup.length - 1]!.timestampUs; - - anchorUs = newestUs - 100_000; - queue.length = 0; - queue.push(...survivingGroup); - // Play out what was scheduled before the skip, so the schedule horizon - // reopens and the renderer pulls the jumped-to timeline. - audioContext.currentTime = 0.25; - - // Playout resumes within the target of the edge, rather than at the - // group start SKIP_US — which is what the plain discontinuity - // re-anchor would have done. - await vi.waitFor(() => expect(renderer.getClockTimeUs()).toBeGreaterThanOrEqual(anchorUs!), { timeout: 5000 }); - expect(renderer.getClockTimeUs()).toBeLessThanOrEqual(newestUs + FRAME_DURATION_US); + await vi.advanceTimersByTimeAsync(0); + renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); + tick(); - renderer.destroy(); + expect(fake.inputs[0]!.options.clock!.targetLatencyUs).toBe(250_000); }); - // A publisher that replaces its capture source re-anchors that track's - // timeline; landing *behind* the old one must be treated as the timeline - // reset it is. Butt-joining it (the backward jump slips past a - // forward-only check) keeps audio playing seamlessly while the master - // clock silently steps backwards by the jump — and the slaved video - // renderer then holds every frame it ever receives "early", frozen - // exactly the step behind for the rest of the stream. - it('re-anchors onto a timeline that stepped backwards instead of continuing the old one', async () => { - const frames = await encodeTestFrames(6); - const BASE_US = 10_000_000; - const STEP_BACK_US = 5_000_000; - const TARGET_US = 100_000; - const old = frames.slice(0, 3).map((frame, i) => ({ ...frame, timestampUs: BASE_US + i * FRAME_DURATION_US })); - const fresh = frames - .slice(3) - .map((frame, i) => ({ ...frame, timestampUs: BASE_US - STEP_BACK_US + i * FRAME_DURATION_US })); - const audioContext = createFakeAudioContext(); - const queue = [...old]; - const source: AudioFrameSource = { peek: () => queue[0], dequeue: () => queue.shift() }; - let edgeUs = old[old.length - 1]!.timestampUs; - const renderer = createAudioRendererActor({ - audioContext, - scheduleMargin: 0.05, - getJoinAnchorUs: () => edgeUs - TARGET_US, - }); + it('defaults to the core on the context baseContext, and fails without one', async () => { + // The core's first act is loading its worklet onto the context; failing + // that with a marker shows the default reached the core on this context. + const workletUnavailable = new Error('worklet unavailable'); + const addModule = vi.fn(() => Promise.reject(workletUnavailable)); + // SAFETY: the core touches only `audioWorklet` before the rejected load. + const baseContext = { audioWorklet: { addModule } } as unknown as BaseAudioContext; + const renderer = createAudioRendererActor({ audioContext: { ...fakeAudioContext(), baseContext } }); + + await vi.advanceTimersByTimeAsync(0); + expect(addModule).toHaveBeenCalledTimes(1); + expect(renderer.snapshot.get().context.error).toBe(workletUnavailable); + renderer.destroy(); - renderer.setTrack(source, { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - await vi.waitFor( - () => - expect(renderer.snapshot.get().context.scheduledUntilUs).toBeGreaterThanOrEqual(BASE_US + FRAME_DURATION_US), - { timeout: 5000 } - ); - audioContext.currentTime = 0.06; - expect(renderer.getClockTimeUs()).toBeGreaterThanOrEqual(BASE_US); - - // The switched source's timeline arrives a whole step behind, delivered - // continuously; the subscriber's edge has already reset onto it. - queue.push(...fresh); - edgeUs = fresh[fresh.length - 1]!.timestampUs; - - // The clock lands on the new timeline at the join anchor — with the fake - // context clock frozen inside the old schedule, a butt-join would keep - // reading the old timeline here instead. - await vi.waitFor( - () => { - const clock = renderer.getClockTimeUs(); - - expect(clock).toBeDefined(); - expect(clock!).toBeLessThan(BASE_US); - expect(clock!).toBeGreaterThanOrEqual(edgeUs - TARGET_US); - }, - { timeout: 5000 } - ); - - // …and playout proceeds through the anchor gap into the new audio. - audioContext.currentTime = 0.2; - await vi.waitFor(() => expect(renderer.getClockTimeUs()).toBeGreaterThanOrEqual(fresh[0]!.timestampUs), { - timeout: 5000, - }); + const bare = createAudioRendererActor({ audioContext: fakeAudioContext() }); - renderer.destroy(); + await vi.advanceTimersByTimeAsync(0); + expect(bare.snapshot.get().context.status).toBe('error'); + bare.destroy(); }); - // A discontinuity interrupts playout that already ran at depth, so the - // re-join lands *at the join anchor* (the target latency) rather than at - // arrival. Landing at arrival collapses playout to the live edge — the - // reported "audio jumps to real-time" — which the latency controller then - // spends tens of pitch-shifted seconds rebuilding at its ±5% nudge. - it('lands a discontinuity re-join at the join anchor instead of racing to the arrival edge', async () => { - const frames = await encodeTestFrames(6); - const JUMP_US = 5_000_000; - const TARGET_US = 200_000; - const old = frames.slice(0, 3).map((frame, i) => ({ ...frame, timestampUs: i * FRAME_DURATION_US })); - const fresh = frames.slice(3).map((frame, i) => ({ ...frame, timestampUs: JUMP_US + i * FRAME_DURATION_US })); - const audioContext = createFakeAudioContext(); - const queue = [...old]; - const source: AudioFrameSource = { peek: () => queue[0], dequeue: () => queue.shift() }; - let edgeUs = old[old.length - 1]!.timestampUs; - const renderer = createAudioRendererActor({ - audioContext, - scheduleMargin: 0.05, - getJoinAnchorUs: () => edgeUs - TARGET_US, - }); + it('fails a track with no decoder config, and a player that cannot be created', async () => { + const { renderer } = await setup(); - renderer.setTrack(source, { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - await vi.waitFor( - () => expect(renderer.snapshot.get().context.scheduledUntilUs).toBeGreaterThanOrEqual(FRAME_DURATION_US), - { timeout: 5000 } - ); - - queue.push(...fresh); - edgeUs = fresh[fresh.length - 1]!.timestampUs; - const anchorUs = edgeUs - TARGET_US; - - // The clock resumes at the anchor — target depth behind the jumped-to - // edge — not at the buffer head the way an at-arrival landing would. - await vi.waitFor( - () => { - const clock = renderer.getClockTimeUs(); - - expect(clock).toBeDefined(); - expect(clock!).toBeGreaterThanOrEqual(anchorUs); - expect(clock!).toBeLessThan(fresh[0]!.timestampUs); - }, - { timeout: 5000 } - ); - - // Advancing the context clock through the anchor gap reaches the head. - audioContext.currentTime = 0.3; - await vi.waitFor(() => expect(renderer.getClockTimeUs()).toBeGreaterThanOrEqual(fresh[0]!.timestampUs), { - timeout: 5000, + renderer.setTrack(queueSource(), null); + expect(renderer.snapshot.get().context.status).toBe('error'); + + const failing = createAudioRendererActor({ + audioContext: fakeAudioContext(), + createAudioPlayer: () => Promise.reject(new Error('no worklet')), }); - renderer.destroy(); - }); + await vi.advanceTimersByTimeAsync(0); + expect(failing.snapshot.get().context.status).toBe('error'); + failing.destroy(); - it('schedules only up to the horizon instead of draining a backlog in one tick', async () => { - const frames = await encodeTestFrames(50); // 1s of audio, all buffered up-front - const audioContext = createFakeAudioContext(); // context clock frozen at 0 - const queue = [...frames]; - const source: AudioFrameSource = { peek: () => queue[0], dequeue: () => queue.shift() }; - const renderer = createAudioRendererActor({ audioContext, scheduleMargin: 0.05 }); - - renderer.setTrack(source, { codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 }); - await vi.waitFor( - () => expect(renderer.snapshot.get().context.scheduledUntilUs).toBeGreaterThanOrEqual(7 * FRAME_DURATION_US), - { timeout: 5000 } - ); - // Let a few more ticks run: with the clock frozen, scheduling must - // stop at the horizon (margin * 4 plus in-flight decodes), leaving - // the rest of the backlog in the source's jitter buffer. - await new Promise((resolve) => setTimeout(resolve, 150)); - expect(renderer.snapshot.get().context.scheduledUntilUs).toBeLessThan(20 * FRAME_DURATION_US); - expect(queue.length).toBeGreaterThan(30); + const unconnectable = createFakePlayer(); - renderer.destroy(); + unconnectable.player.output.connect = () => { + throw new Error('cross-context connect'); + }; + + const disconnected = createAudioRendererActor({ + audioContext: fakeAudioContext(), + createAudioPlayer: () => Promise.resolve(unconnectable.player), + }); + + await vi.advanceTimersByTimeAsync(0); + expect(disconnected.snapshot.get().context.status).toBe('error'); + expect(unconnectable.closed).toBe(true); + disconnected.destroy(); }); }); diff --git a/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts b/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts deleted file mode 100644 index 3931fc9e15..0000000000 --- a/packages/spf/src/playback/actors/dom/tests/worklet-audio-renderer.test.ts +++ /dev/null @@ -1,546 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vite-plus/test'; - -import type { JitterFrame } from '../../track-subscriber'; -import type { AudioContextLike, AudioFrameSource } from '../audio-renderer'; -import { - type AudioPlayer, - type AudioPlayerInput, - type AudioPlayerInputOptions, - createWorkletAudioRendererActor, -} from '../worklet-audio-renderer'; - -const FRAME_US = 20_000; -const JITTER_TARGET_US = 60_000; -const TICK_MS = 10; -const CONFIG: AudioDecoderConfig = { codec: 'opus', sampleRate: 48_000, numberOfChannels: 1 }; - -/** An Opus TOC byte for one 20 ms CELT frame (config 19, code 0). */ -const OPUS_20MS_TOC = 19 << 3; - -function frameAt(timestampUs: number): JitterFrame { - return { groupId: timestampUs, objectId: 0, timestampUs, isKey: true, payload: new Uint8Array([OPUS_20MS_TOC]) }; -} - -/** Frames every 20 ms over `[fromUs, untilUs)`. */ -function framesBetween(fromUs: number, untilUs: number): JitterFrame[] { - const frames: JitterFrame[] = []; - - for (let timestampUs = fromUs; timestampUs < untilUs; timestampUs += FRAME_US) frames.push(frameAt(timestampUs)); - - return frames; -} - -function queueSource(frames: JitterFrame[] = []): AudioFrameSource & { queue: JitterFrame[] } { - const queue = [...frames]; - - return { queue, peek: () => queue[0], dequeue: () => queue.shift() }; -} - -interface FakeInput extends AudioPlayerInput { - options: AudioPlayerInputOptions; - pushed: number[]; - cuts: number; - removed: boolean; - stats: { playing: boolean; bufferedUs: number; jitterTargetUs: number }; -} - -/** The core's surface with no audio behind it: pushes are recorded, and the test plays the worklet's reports. */ -function createFakePlayer() { - const inputs: FakeInput[] = []; - const connected: AudioNode[] = []; - let closed = false; - const player: AudioPlayer = { - output: { connect: (destination) => connected.push(destination) }, - addInput(options = {}) { - const input: FakeInput = { - options, - pushed: [], - cuts: 0, - removed: false, - stats: { playing: true, bufferedUs: 0, jitterTargetUs: JITTER_TARGET_US }, - push(frame) { - input.pushed.push(frame.timestampUs); - }, - cut() { - input.cuts++; - }, - remove() { - input.removed = true; - }, - }; - - inputs.push(input); - return input; - }, - close() { - closed = true; - }, - }; - - return { - player, - inputs, - connected, - get closed() { - return closed; - }, - /** The worklet reporting that the audio at `timestampUs` is audible now. */ - report(timestampUs: number, atUs: number) { - inputs[inputs.length - 1]!.options.clock!.reportAudio(timestampUs, atUs); - }, - }; -} - -// SAFETY: the renderer only hands `destination` to `output.connect`, which the fake records by identity. -const destination = {} as AudioNode; - -/** Context time follows the test's clock; `suspended` freezes it, as `AudioContext.suspend()` does. */ -let contextUs = 0; -let suspended = false; - -function fakeAudioContext(outputLatency?: number): AudioContextLike { - return { - get currentTime() { - return contextUs / 1_000_000; - }, - destination, - outputLatency, - createBuffer: () => { - throw new Error('unused'); - }, - createBufferSource: () => { - throw new Error('unused'); - }, - }; -} - -describe('createWorkletAudioRendererActor', () => { - let nowUs = 0; - - /** Move wall time, and context time with it unless the context is suspended. */ - const at = (us: number) => { - if (!suspended) contextUs += us - nowUs; - - nowUs = us; - }; - - beforeEach(() => { - vi.useFakeTimers(); - at(0); - contextUs = 0; - suspended = false; - }); - - afterEach(() => { - vi.useRealTimers(); - }); - - /** A renderer whose player is ready and whose first input exists. */ - async function setup(options: { getJoinAnchorUs?: () => number | undefined; outputLatency?: number } = {}) { - const { outputLatency, ...rendererOptions } = options; - const fake = createFakePlayer(); - const renderer = createWorkletAudioRendererActor({ - audioContext: fakeAudioContext(outputLatency), - createAudioPlayer: () => Promise.resolve(fake.player), - tickIntervalMs: TICK_MS, - ...rendererOptions, - }); - - await vi.advanceTimersByTimeAsync(0); - return { fake, renderer }; - } - - const tick = () => vi.advanceTimersByTime(TICK_MS); - - it('connects the player output to the context destination', async () => { - const { fake, renderer } = await setup(); - - expect(fake.connected).toEqual([destination]); - expect(renderer.getClockTimeUs()).toBeUndefined(); - - renderer.destroy(); - expect(fake.closed).toBe(true); - }); - - it('fills to the jitter target plus a frame, then meters by the clock', async () => { - const { fake, renderer } = await setup(); - const source = queueSource(framesBetween(0, 1_000_000)); - - renderer.setTrack(source, CONFIG); - tick(); - - const input = fake.inputs[0]!; - - expect(input.options.decoderConfig).toEqual(CONFIG); - expect(input.pushed).toEqual([0, 20_000, 40_000, 60_000]); - expect(renderer.getClockTimeUs()).toBeUndefined(); - - // Playing: the worklet should hold about its target ahead of what renders. - fake.report(0, 0); - at(30_000); - tick(); - - expect(input.pushed.at(-1)).toBe(80_000); - expect(renderer.getClockTimeUs()).toBeCloseTo(30_000, 0); - expect(renderer.snapshot.get().context.scheduledUntilUs).toBe(100_000); - }); - - it('joins at the edge, dropping buffered audio behind the anchor unheard', async () => { - const { fake, renderer } = await setup({ getJoinAnchorUs: () => 200_000 }); - - renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); - tick(); - - expect(fake.inputs[0]!.pushed[0]).toBe(200_000); - }); - - it('holds the clock where the worklet ran dry, through the refill, until audio past it plays', async () => { - const { fake, renderer } = await setup(); - const source = queueSource(framesBetween(0, 80_000)); - - renderer.setTrack(source, CONFIG); - tick(); - fake.report(0, 0); - - at(100_000); - tick(); - expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); - - // Audio comes back: the refill moves the pushed end, not the clock. - fake.inputs[0]!.stats.playing = false; - source.queue.push(...framesBetween(80_000, 400_000)); - at(300_000); - tick(); - expect(fake.inputs[0]!.pushed.slice(4)).toEqual([80_000, 100_000, 120_000, 140_000]); - expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); - - fake.report(80_000, 320_000); - at(330_000); - expect(renderer.getClockTimeUs()).toBeCloseTo(90_000, 0); - }); - - it('does not step back when the refill is reported ahead of being audible', async () => { - const { fake, renderer } = await setup(); - const source = queueSource(framesBetween(0, 80_000)); - - renderer.setTrack(source, CONFIG); - tick(); - fake.report(0, 0); - at(100_000); - tick(); - - fake.inputs[0]!.stats.playing = false; - source.queue.push(...framesBetween(80_000, 400_000)); - at(300_000); - tick(); - - // Rendered now, audible 100 ms from now: the anchor alone reads 80 ms behind the hold. - fake.report(80_000, 400_000); - expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); - - at(410_000); - expect(renderer.getClockTimeUs()).toBeCloseTo(90_000, 0); - }); - - it('resumes from the audible position when audio returns before the device drained', async () => { - const { fake, renderer } = await setup({ outputLatency: 0.05 }); - const source = queueSource(framesBetween(0, 80_000)); - - renderer.setTrack(source, CONFIG); - tick(); - fake.report(0, 0); - - // Rendering passed the last pushed sample; 50 ms of it is still in the device. - at(30_000); - tick(); - expect(renderer.getClockTimeUs()).toBeCloseTo(30_000, 0); - - fake.inputs[0]!.stats.playing = false; - source.queue.push(...framesBetween(80_000, 400_000)); - at(40_000); - tick(); - fake.report(80_000, 90_000); - expect(renderer.getClockTimeUs()).toBeCloseTo(40_000, 0); - }); - - it('ends pushed audio where the last packet ends, not across a gap of lost packets', async () => { - const { fake, renderer } = await setup(); - - renderer.setTrack(queueSource([0, 20_000, 40_000, 100_000].map(frameAt)), CONFIG); - tick(); - fake.report(0, 0); - at(60_000); - tick(); - - expect(fake.inputs[0]!.pushed).toEqual([0, 20_000, 40_000, 100_000]); - expect(renderer.snapshot.get().context.scheduledUntilUs).toBe(120_000); - }); - - it('keeps the hold through a late report of the audio before the underrun', async () => { - const { fake, renderer } = await setup(); - const source = queueSource(framesBetween(0, 80_000)); - - renderer.setTrack(source, CONFIG); - tick(); - fake.report(0, 0); - at(100_000); - tick(); - - // The refill is pushed before the core has said it stopped; then the - // tail's last report, still in flight, arrives. - source.queue.push(...framesBetween(80_000, 400_000)); - tick(); - fake.report(77_333, 77_333); - at(200_000); - expect(renderer.getClockTimeUs()).toBeCloseTo(80_000, 0); - }); - - it('holds a placed re-join clock at the audio pushed while the fill is short', async () => { - let anchorUs: number | undefined; - const { fake, renderer } = await setup({ getJoinAnchorUs: () => anchorUs }); - const source = queueSource(framesBetween(0, 80_000)); - - renderer.setTrack(source, CONFIG); - tick(); - fake.report(0, 0); - - anchorUs = 4_900_000; - source.queue.push(frameAt(5_000_000)); - tick(); - at(100_000); - tick(); - expect(fake.inputs[0]!.pushed.at(-1)).toBe(5_000_000); - - // One packet cannot start the worklet; the clock stops at its end. - at(2_100_000); - tick(); - expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); - - // The refill plays, reported ahead of being audible: no step back. - source.queue.push(...framesBetween(5_020_000, 5_200_000)); - tick(); - fake.report(5_000_000, 2_150_000); - expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); - }); - - it('gives the core a report time source that stops while the context is suspended', async () => { - const { fake, renderer } = await setup(); - - renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); - tick(); - - const clock = fake.inputs[0]!.options.clock!; - - at(40_000); - expect(clock.localNow()).toBeCloseTo(40_000, 0); - - suspended = true; - at(540_000); - expect(clock.localNow()).toBeCloseTo(40_000, 0); - - // The first report after resume, timed on that source, lands where audio is. - suspended = false; - at(560_000); - fake.report(60_000, clock.localNow()); - expect(renderer.getClockTimeUs()).toBeCloseTo(60_000, 0); - }); - - it('drops what the core holds at a timeline reset and ignores reports from before it', async () => { - const { fake, renderer } = await setup(); - const source = queueSource(framesBetween(0, 80_000)); - - renderer.setTrack(source, CONFIG); - tick(); - fake.report(0, 0); - - // A publisher re-anchor: the timeline steps back by far more than a gap. - source.queue.push(...framesBetween(-5_000_000, -4_000_000)); - tick(); - - const input = fake.inputs[0]!; - - expect(input.cuts).toBe(1); - expect(input.pushed.at(4)).toBe(-5_000_000); - expect(renderer.getClockTimeUs()).toBeUndefined(); - - // Still in flight from the old timeline when the cut landed. - fake.report(40_000, 0); - expect(renderer.getClockTimeUs()).toBeUndefined(); - - fake.report(-5_000_000, 0); - expect(renderer.getClockTimeUs()).toBeCloseTo(-5_000_000, 0); - }); - - it('places a re-join whose whole buffer is past the anchor at its distance from it', async () => { - let anchorUs: number | undefined; - const { fake, renderer } = await setup({ getJoinAnchorUs: () => anchorUs }); - const source = queueSource(framesBetween(0, 80_000)); - - renderer.setTrack(source, CONFIG); - tick(); - fake.report(0, 0); - - // A catch-up skip landed on a group 100 ms past where the anchor falls. - anchorUs = 4_900_000; - source.queue.push(...framesBetween(5_000_000, 6_000_000)); - tick(); - - const input = fake.inputs[0]!; - - expect(input.pushed).toHaveLength(4); - expect(renderer.getClockTimeUs()).toBeCloseTo(4_900_000, 0); - - at(50_000); - tick(); - expect(input.pushed.at(4)).toBe(5_000_000); - expect(renderer.getClockTimeUs()).toBeCloseTo(4_950_000, 0); - }); - - it('freezes the clock and stops metering while the context is suspended', async () => { - const { fake, renderer } = await setup(); - const source = queueSource(framesBetween(0, 1_000_000)); - - renderer.setTrack(source, CONFIG); - tick(); - fake.report(0, 0); - at(30_000); - tick(); - - const pushed = fake.inputs[0]!.pushed.length; - - // Paused for two seconds: nothing plays, so nothing is due and nothing ran dry. - suspended = true; - at(2_030_000); - tick(); - expect(renderer.getClockTimeUs()).toBeCloseTo(30_000, 0); - expect(fake.inputs[0]!.pushed).toHaveLength(pushed); - - suspended = false; - at(2_040_000); - expect(renderer.getClockTimeUs()).toBeCloseTo(40_000, 0); - }); - - it('ignores reports from an input that a track change removed', async () => { - const { fake, renderer } = await setup(); - - renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); - tick(); - - const stale = fake.inputs[0]!.options.clock!; - - renderer.setTrack(queueSource(framesBetween(0, 1_000_000)), CONFIG); - tick(); - stale.reportAudio(40_000, 0); - expect(renderer.getClockTimeUs()).toBeUndefined(); - - fake.report(0, 0); - expect(renderer.getClockTimeUs()).toBeCloseTo(0, 0); - }); - - it('counts every time the worklet runs dry, across track changes', async () => { - const { fake, renderer } = await setup(); - - renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); - tick(); - fake.report(0, 0); - - // A stall far longer than any jitter ceiling still counts, once. - at(100_000); - tick(); - at(3_000_000); - tick(); - expect(renderer.snapshot.get().context.underruns).toBe(1); - - renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); - tick(); - fake.report(0, 3_000_000); - at(3_100_000); - tick(); - expect(renderer.snapshot.get().context.underruns).toBe(2); - }); - - it('counts a dry edge that a track change lands on before the next tick', async () => { - const { fake, renderer } = await setup(); - - renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); - tick(); - fake.report(0, 0); - - at(100_000); - renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); - expect(renderer.snapshot.get().context.underruns).toBe(1); - }); - - it('rescues a frozen re-join placement whose next packets are past the fill horizon', async () => { - let anchorUs: number | undefined; - const { fake, renderer } = await setup({ getJoinAnchorUs: () => anchorUs }); - const source = queueSource(framesBetween(0, 80_000)); - - renderer.setTrack(source, CONFIG); - tick(); - fake.report(0, 0); - - anchorUs = 4_900_000; - source.queue.push(frameAt(5_000_000)); - tick(); - at(200_000); - tick(); - expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); - - // Packets resume 80 ms after the lone one ends: beyond the fill horizon. - fake.inputs[0]!.stats.playing = false; - source.queue.push(...framesBetween(5_100_000, 5_400_000)); - at(400_000); - tick(); - expect(fake.inputs[0]!.pushed).toContain(5_100_000); - expect(renderer.getClockTimeUs()).toBeCloseTo(5_020_000, 0); - }); - - it('caps the core jitter target at the target latency', async () => { - const fake = createFakePlayer(); - const renderer = createWorkletAudioRendererActor({ - audioContext: fakeAudioContext(), - createAudioPlayer: () => Promise.resolve(fake.player), - getTargetLatencyUs: () => 250_000, - tickIntervalMs: TICK_MS, - }); - - await vi.advanceTimersByTimeAsync(0); - renderer.setTrack(queueSource(framesBetween(0, 80_000)), CONFIG); - tick(); - - expect(fake.inputs[0]!.options.clock!.targetLatencyUs).toBe(250_000); - }); - - it('fails a track with no decoder config, and a player that cannot be created', async () => { - const { renderer } = await setup(); - - renderer.setTrack(queueSource(), null); - expect(renderer.snapshot.get().context.status).toBe('error'); - - const failing = createWorkletAudioRendererActor({ - audioContext: fakeAudioContext(), - createAudioPlayer: () => Promise.reject(new Error('no worklet')), - }); - - await vi.advanceTimersByTimeAsync(0); - expect(failing.snapshot.get().context.status).toBe('error'); - failing.destroy(); - - const unconnectable = createFakePlayer(); - - unconnectable.player.output.connect = () => { - throw new Error('cross-context connect'); - }; - - const disconnected = createWorkletAudioRendererActor({ - audioContext: fakeAudioContext(), - createAudioPlayer: () => Promise.resolve(unconnectable.player), - }); - - await vi.advanceTimersByTimeAsync(0); - expect(disconnected.snapshot.get().context.status).toBe('error'); - expect(unconnectable.closed).toBe(true); - disconnected.destroy(); - }); -}); diff --git a/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts b/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts deleted file mode 100644 index d92ef39dc7..0000000000 --- a/packages/spf/src/playback/actors/dom/worklet-audio-renderer.ts +++ /dev/null @@ -1,571 +0,0 @@ -/** - * AudioWorklet renderer actor over the shared audio player core: `@mux/moq-media`'s `createAudioPlayer`, injected as - * `createAudioPlayer`. Same surface as `createAudioRendererActor`, frames in from the subscriber's jitter buffer and - * the **master clock** out, so the engine can run either behind one seam. - * - * The core decodes with WebCodecs, holds an adaptive jitter buffer in an `AudioWorklet`, and reports the audible - * timestamp about every 20 ms. **Latency stays v10's.** The renderer meters frames into the core so the worklet holds - * about its own jitter target, and the rest of the target latency stays in the subscriber's buffer, where `syncLatency` - * measures and steers it. The core's adaptive target is left alone: under metering its growth only moves depth from one - * buffer to the other. - * - * The clock the core reports into is the renderer's own, not the SDK's `PlayoutClock`, so `getClockTimeUs` keeps the - * contract the video renderer and `trackPlayoutTime` rely on: `undefined` before any audio, and **held where the - * worklet ran dry** rather than paced on by wall time — a wall-paced clock steps back when audio resumes, which holds - * slaved video on its last frame for the whole step. - * - * TODO(mmcc/v10#66): apply `getPlaybackRate` once the core has a per-input rate (muxinc/relay-sdks#329). Until then - * audio ignores the latency controller's nudges, which is why the engine only uses this renderer when a - * `createAudioPlayer` is configured. - */ -import { createTransitionActor } from '../../../core/actors/create-transition-actor'; -import { TIMELINE_DISCONTINUITY_US } from '../../../media/moq/timeline'; -import type { - AudioContextLike, - AudioFrameSource, - AudioRendererActor, - AudioRendererContext, - AudioRendererStatus, -} from './audio-renderer'; - -// ============================================================================= -// Types -// ============================================================================= - -/** - * The part of `@mux/moq-media`'s `PlayoutClock` an audio player input calls. The renderer implements it, so the input - * reports into the renderer's clock. - */ -export interface AudioPlayerClock { - /** Caps the input's jitter target: the core keeps it under this less the output delay and 40 ms. */ - readonly targetLatencyUs: number; - /** The time source reports are measured on, in microseconds. */ - localNow(): number; - /** The audio with this timestamp is audible at local time `atUs`. */ - reportAudio(timestampUs: number, atUs?: number): void; - /** The input stopped. */ - stopAudio(): void; -} - -/** The part of `@mux/moq-media`'s `AudioInputOptions` the renderer passes. */ -export interface AudioPlayerInputOptions { - clock?: AudioPlayerClock; - decoderConfig?: AudioDecoderConfig; - /** A decoder error. The core decodes again from the next frame, with a new decoder. */ - onError?(cause: unknown): void; -} - -/** The part of `@mux/moq-media`'s `AudioInputStats` the renderer reads. Durations are microseconds. */ -export interface AudioPlayerInputStats { - /** Whether the worklet is playing, rather than filling to its target. */ - readonly playing: boolean; - /** Decoded audio the worklet holds. */ - readonly bufferedUs: number; - /** How much the worklet holds before it plays, at the start and after running dry. */ - readonly jitterTargetUs: number; -} - -/** The part of `@mux/moq-media`'s `AudioPlayerInput` the renderer uses: one track. */ -export interface AudioPlayerInput { - /** Decode and hold a frame, in decode order. */ - push(frame: { timestampUs: number; payload: Uint8Array }): void; - /** Without a range: drop everything held or still decoding. */ - cut(): void; - readonly stats: AudioPlayerInputStats; - remove(): void; -} - -/** The part of `@mux/moq-media`'s `AudioPlayer` the renderer uses. */ -export interface AudioPlayer { - /** The mix of every input; the renderer connects it to its context's `destination`. */ - readonly output: { connect(destination: AudioNode): void }; - addInput(options?: AudioPlayerInputOptions): AudioPlayerInput; - /** Remove every input and disconnect the output. */ - close(): void; -} - -/** - * Create the player on the renderer's context. `@mux/moq-media`'s `createAudioPlayer` qualifies once given a real - * `AudioContext`, which a wrapping context exposes as `baseContext`: - * - * ```ts - * createAudioPlayer: (context) => createAudioPlayer({ context: context.baseContext as AudioContext }); - * ``` - */ -export type CreateAudioPlayer = (context: AudioContextLike) => Promise; - -export interface CreateWorkletAudioRendererOptions { - /** Where the player's output connects (`destination`), and the output latencies the clock is placed behind. */ - audioContext: AudioContextLike; - createAudioPlayer: CreateAudioPlayer; - /** - * **Join at the live edge**, as `createAudioRendererActor`'s option of the same name: buffered audio older than this - * is discarded before the first frame of a (re)start reaches the core. - */ - getJoinAnchorUs?: () => number | undefined; - /** - * The target latency in microseconds, read on every clock report. It caps the core's jitter target, so the worklet - * never holds more than the latency the controller is aiming at. Default 2 s, where the core's own 400 ms ceiling - * applies instead. - */ - getTargetLatencyUs?: () => number | undefined; - /** Metering cadence in ms. Default 10. */ - tickIntervalMs?: number; -} - -type RendererMessage = - | { type: 'status'; status: AudioRendererStatus; error?: unknown } - | { type: 'decoder-error'; error: unknown } - | { type: 'scheduled'; untilUs: number } - | { type: 'underrun' }; - -// ============================================================================= -// Implementation -// ============================================================================= - -const DEFAULT_TICK_INTERVAL_MS = 10; -const DEFAULT_TARGET_LATENCY_US = 2_000_000; -/** The frame duration assumed before a frame has shown one: Opus's usual 20 ms. */ -const DEFAULT_FRAME_DURATION_US = 20_000; -/** Reports and pushed spans may disagree by sample rounding and resampling offsets. */ -const REPORT_TOLERANCE_US = 5_000; -/** - * How long a fill may sit short of playback before one more frame is pushed. The fill pushes exactly the jitter target - * plus a frame, so audio the core drops on arrival (behind what it already decoded) could otherwise leave it waiting - * forever. - */ -const FILL_STALL_US = 100_000; -/** See `audio-renderer`'s threshold of the same name: a jump this large, either way, is a new timeline. */ -const DISCONTINUITY_THRESHOLD_US = TIMELINE_DISCONTINUITY_US; - -/** - * How long an Opus packet plays, from its TOC byte (RFC 6716 §3.1); `undefined` when it cannot be read. The same rule - * as `@mux/moq-media`'s `opusPacketDurationUs`. - */ -function opusPacketDurationUs(packet: Uint8Array): number | undefined { - const toc = packet[0]; - if (toc === undefined) return undefined; - - const config = toc >> 3; - // SILK: 10, 20, 40, 60 ms; hybrid: 10, 20 ms; CELT: 2.5, 5, 10, 20 ms. - const frameUs = - config < 12 - ? [10_000, 20_000, 40_000, 60_000][config % 4]! - : config < 16 - ? [10_000, 20_000][config % 2]! - : [2_500, 5_000, 10_000, 20_000][config % 4]!; - const code = toc & 3; - if (code === 0) return frameUs; - - if (code !== 3) return 2 * frameUs; - - const count = packet[1]; - - return count === undefined ? undefined : (count & 0x3f) * frameUs; -} - -export function createWorkletAudioRendererActor(options: CreateWorkletAudioRendererOptions): AudioRendererActor { - const { audioContext } = options; - - const inner = createTransitionActor( - { status: 'idle', framesScheduled: 0, underruns: 0 }, - (context, message) => { - switch (message.type) { - case 'status': - return { ...context, status: message.status, error: message.error }; - case 'decoder-error': - return { ...context, error: message.error }; - case 'scheduled': - return { - ...context, - status: 'rendering', - framesScheduled: context.framesScheduled + 1, - scheduledUntilUs: message.untilUs, - }; - case 'underrun': - return { ...context, underruns: context.underruns + 1 }; - } - } - ); - - let destroyed = false; - let player: AudioPlayer | undefined; - let input: AudioPlayerInput | undefined; - let source: AudioFrameSource | null = null; - let decoderConfig: AudioDecoderConfig | null = null; - - /** The schedule has no anchor yet; see `audio-renderer`'s flag of the same name. */ - let needsJoinAnchor = true; - /** The pending anchor follows a mid-stream discontinuity; see `audio-renderer`'s flag of the same name. */ - let rejoinAnchorArmed = false; - - // What this timeline has pushed into the core. Reset with the timeline. - let firstPushedUs: number | undefined; - let lastPushedUs: number | undefined; - /** Known once a packet's duration or two frames' spacing has shown it. */ - let frameDurationUs: number | undefined; - /** Where the newest pushed frame ends: the furthest the clock can read. */ - let pushedEndUs: number | undefined; - /** First timestamp pushed in the current fill, and the context time of the latest push. */ - let fillStartUs: number | undefined; - let lastPushAtUs = 0; - - /** - * The clock: timeline position `mediaUs` is audible at context time `contextUs`, advancing with the context clock - * from there. Set by every core report, or **placed** at a re-join anchor before the core plays anything (see - * `dropToJoinAnchor`). - * - * Context time, not `performance.now()`, because it is what stops when the adapter suspends the context on pause: a - * wall-paced clock would run on through the pause, read the frozen worklet as dry and refill it, and step back on the - * first report after resume. The core times its reports on the same source (`localNow`), so its render clock's - * calibration does not carry a pause into the first reports after it either. - */ - let anchor: { mediaUs: number; contextUs: number; placed: boolean; frozen?: boolean } | undefined; - /** Bumped per input, so a report from a removed input cannot anchor its replacement. */ - let inputGeneration = 0; - /** - * Where the worklet ran dry. The clock holds there until a report of audio past it arrives: frames pushed to refill - * the worklet move `pushedEndUs` before they play, and must not drag the clock forward with them. - */ - let dryLimitUs: number | undefined; - /** - * The position the clock held at through the latest underrun, kept as a floor after it ends. The first report of the - * refill is dated ahead by the output latency — the sample is not audible yet — so its anchor extrapolates to before - * the hold, and the clock would otherwise step back by that much. - */ - let holdFloorUs: number | undefined; - /** - * The core has reported itself stopped since the dry point was set. Until it has, a report near the dry point is the - * tail of the audio before it, still in flight, and must not end the hold. - */ - let stoppedSinceDry = false; - - const outputLatencyUs = (): number => - ((audioContext.baseLatency ?? 0) + (audioContext.outputLatency ?? 0)) * 1_000_000; - - const contextNowUs = (): number => audioContext.currentTime * 1_000_000; - - /** The audible position extrapolated from the anchor, ignoring every limit. */ - const extrapolatedUs = (): number | undefined => - anchor && (anchor.frozen ? anchor.mediaUs : anchor.mediaUs + (contextNowUs() - anchor.contextUs)); - - /** The clock one input reports into; it goes deaf once that input is replaced. */ - const createClock = (): AudioPlayerClock => { - const generation = ++inputGeneration; - - return { - get targetLatencyUs() { - return options.getTargetLatencyUs?.() ?? DEFAULT_TARGET_LATENCY_US; - }, - localNow: contextNowUs, - reportAudio(timestampUs, atUs) { - if (generation !== inputGeneration) return; - - reportAudio(timestampUs, atUs); - }, - // The renderer resets the clock itself wherever a track or timeline ends. - stopAudio() {}, - }; - }; - - const raiseHoldFloor = (heldUs: number): void => { - holdFloorUs = Math.max(holdFloorUs ?? heldUs, heldUs); - }; - - /** - * A placed clock runs from the anchor until the core plays, but never past the audio pushed: a fill that stays short - * of the jitter target cannot start the worklet, and the clock would otherwise run on through audio not there. - */ - const placedPositionUs = (): number | undefined => { - const positionUs = extrapolatedUs(); - - return positionUs === undefined || pushedEndUs === undefined ? positionUs : Math.min(positionUs, pushedEndUs); - }; - - /** `timestampUs` is audible at context time `atUs` (default now). */ - const reportAudio = (timestampUs: number, atUs: number | undefined): void => { - // Only audio pushed on this timeline: a report already in flight when - // the timeline restarted describes audio the cut dropped. - if ( - firstPushedUs === undefined || - pushedEndUs === undefined || - timestampUs < firstPushedUs - REPORT_TOLERANCE_US || - timestampUs > pushedEndUs + REPORT_TOLERANCE_US - ) { - return; - } - - // The refill's first report ends the hold. Its floor is what the clock - // reads at that moment — the audible position, which can still be short - // of the dry point while the device plays out what it had queued. - if ( - dryLimitUs !== undefined && - ((stoppedSinceDry && timestampUs >= dryLimitUs - REPORT_TOLERANCE_US) || - timestampUs > dryLimitUs + REPORT_TOLERANCE_US) - ) { - raiseHoldFloor(Math.min(extrapolatedUs() ?? dryLimitUs, dryLimitUs)); - dryLimitUs = undefined; - } - - // Likewise the first report after a placed re-join, which is dated ahead - // of the placed clock by the output latency. - if (anchor?.placed) raiseHoldFloor(placedPositionUs()!); - - anchor = { mediaUs: timestampUs, contextUs: atUs ?? contextNowUs(), placed: false }; - }; - - /** Forget everything pushed on the current timeline, and the clock placed on it. */ - const resetTimeline = (): void => { - firstPushedUs = undefined; - lastPushedUs = undefined; - frameDurationUs = undefined; - pushedEndUs = undefined; - fillStartUs = undefined; - anchor = undefined; - dryLimitUs = undefined; - holdFloorUs = undefined; - stoppedSinceDry = false; - }; - - const removeInput = (): void => { - if (!input) return; - - inputGeneration++; - input.remove(); - input = undefined; - }; - - /** - * Discard buffered audio older than the join anchor before the first push of a (re)start, so playout starts at the - * live edge rather than at the oldest group the relay replayed. Mirrors `audio-renderer`'s function of the same name, - * including landing a re-join anchor by **placement** when the whole buffer is already past it: the clock is placed - * at the anchor and runs from there, and metering by timeline holds the head back until it is due, so playout resumes - * at the target depth rather than at the live edge. - */ - const dropToJoinAnchor = (): void => { - const anchorUs = options.getJoinAnchorUs?.(); - if (anchorUs === undefined || !source) return; - - let dropped = 0; - - for (let next = source.peek(); next && next.timestampUs < anchorUs; next = source.peek()) { - source.dequeue(); - dropped++; - } - - if (dropped > 0) { - needsJoinAnchor = false; - rejoinAnchorArmed = false; - return; - } - - // A fresh join stays armed: a relay's group replay arrives as a burst, - // oldest first, so the buffer's live edge may still be moving. - if (!rejoinAnchorArmed || !source.peek()) return; - - anchor = { mediaUs: anchorUs, contextUs: contextNowUs(), placed: true }; - rejoinAnchorArmed = false; - needsJoinAnchor = false; - }; - - /** - * Whether `timestampUs` should reach the core now. Filling — before the first report, or after running dry — pushes - * the jitter target plus a frame, which is what the worklet waits for before playing. Playing pushes what will be - * rendered within the jitter target, so the worklet holds about that much and the rest waits in the subscriber. - */ - const isDue = (timestampUs: number, stats: AudioPlayerInputStats): boolean => { - const positionUs = anchor?.placed ? placedPositionUs() : extrapolatedUs(); - - // A frozen placement is a fill too: its clock waits for the worklet to - // start, so only the fill rules (including the stall rescue) can move it. - if (positionUs === undefined || dryLimitUs !== undefined || anchor?.frozen) { - if ( - fillStartUs === undefined || - timestampUs < fillStartUs + stats.jitterTargetUs + (frameDurationUs ?? DEFAULT_FRAME_DURATION_US) - ) { - return true; - } - - return contextNowUs() - lastPushAtUs > FILL_STALL_US && stats.bufferedUs < stats.jitterTargetUs; - } - - return timestampUs < positionUs + outputLatencyUs() + stats.jitterTargetUs; - }; - - const push = (active: AudioPlayerInput, timestampUs: number, payload: Uint8Array): void => { - active.push({ timestampUs, payload }); - - // Opus says how long each packet is. Otherwise the shortest step between - // pushed frames is the frame length: a step over lost packets is longer, - // and must not extend the audio past where it really ends. - const opusUs = decoderConfig?.codec === 'opus' ? opusPacketDurationUs(payload) : undefined; - const stepUs = lastPushedUs === undefined ? undefined : timestampUs - lastPushedUs; - - if (opusUs !== undefined) frameDurationUs = opusUs; - else if (stepUs !== undefined && stepUs > 0) frameDurationUs = Math.min(frameDurationUs ?? stepUs, stepUs); - - firstPushedUs ??= timestampUs; - fillStartUs ??= timestampUs; - lastPushedUs = timestampUs; - lastPushAtUs = contextNowUs(); - pushedEndUs = timestampUs + (frameDurationUs ?? DEFAULT_FRAME_DURATION_US); - inner.send({ type: 'scheduled', untilUs: pushedEndUs }); - }; - - /** - * The worklet runs dry once rendering passes everything pushed: metering pushes every frame as soon as it is due, so - * nothing was left to push. Each such edge is an underrun, as `audio-renderer` counts them. The core's own counter - * skips dry spells longer than its jitter ceiling, which `adaptLatencyTarget` still needs as failure evidence. - */ - const detectDry = (): void => { - const positionUs = extrapolatedUs(); - - if ( - !anchor || - anchor.placed || - dryLimitUs !== undefined || - pushedEndUs === undefined || - positionUs === undefined || - positionUs + outputLatencyUs() < pushedEndUs - ) { - return; - } - - dryLimitUs = pushedEndUs; - stoppedSinceDry = false; - fillStartUs = undefined; - inner.send({ type: 'underrun' }); - }; - - const tick = (): void => { - if (destroyed || !player || !source || !decoderConfig) return; - - try { - const active = (input ??= player.addInput({ - clock: createClock(), - decoderConfig, - onError: (error) => inner.send({ type: 'decoder-error', error }), - })); - const { stats } = active; - - detectDry(); - - const positionUs = extrapolatedUs(); - - if (dryLimitUs !== undefined && !stats.playing) stoppedSinceDry = true; - - // A placed clock that reaches the end of the audio pushed stops there - // until the core plays, so refill frames cannot drag it forward. - if (anchor?.placed && !anchor.frozen && pushedEndUs !== undefined && positionUs! >= pushedEndUs) { - anchor = { mediaUs: pushedEndUs, contextUs: contextNowUs(), placed: true, frozen: true }; - } - - for (;;) { - if (needsJoinAnchor && lastPushedUs === undefined) dropToJoinAnchor(); - - const next = source.peek(); - if (!next) return; - - // A timeline reset, either way: drop what the core holds and re-place - // the anchor on the jumped-to timeline (see `audio-renderer`). - if (lastPushedUs !== undefined && Math.abs(next.timestampUs - lastPushedUs) > DISCONTINUITY_THRESHOLD_US) { - active.cut(); - resetTimeline(); - needsJoinAnchor = true; - rejoinAnchorArmed = true; - continue; - } - - if (!isDue(next.timestampUs, stats)) return; - - push(active, next.timestampUs, next.payload); - source.dequeue(); - } - } catch (error) { - inner.send({ type: 'status', status: 'error', error }); - } - }; - const timer = setInterval(tick, options.tickIntervalMs ?? DEFAULT_TICK_INTERVAL_MS); - - options.createAudioPlayer(audioContext).then( - (created) => { - if (destroyed) { - created.close(); - return; - } - - try { - created.output.connect(audioContext.destination); - player = created; - } catch (error) { - created.close(); - inner.send({ type: 'status', status: 'error', error }); - } - }, - (error) => { - if (!destroyed) inner.send({ type: 'status', status: 'error', error }); - } - ); - - return { - get snapshot() { - return inner.snapshot; - }, - - setTrack(nextSource, nextConfig): void { - // See `audio-renderer`: a source with no config could never play. - const missingConfig = nextSource !== null && nextConfig === null; - - source = missingConfig ? null : nextSource; - decoderConfig = nextConfig; - needsJoinAnchor = true; - rejoinAnchorArmed = false; - // An edge since the last tick still counts: the reset below forgets it. - detectDry(); - removeInput(); - resetTimeline(); - - if (missingConfig) { - inner.send({ - type: 'status', - status: 'error', - error: new Error('audio renderer track has no decoder config'), - }); - return; - } - - inner.send({ type: 'status', status: source ? 'rendering' : 'idle' }); - }, - - getClockTimeUs(): number | undefined { - const positionUs = extrapolatedUs(); - if (positionUs === undefined) return undefined; - - if (anchor?.placed) return placedPositionUs(); - - // Never past the audio pushed, and once the worklet ran dry never past - // where it did: the clock holds through an underrun and resumes when - // audio does, as the source-node renderer's clamped segments did. - const limitUs = dryLimitUs ?? pushedEndUs; - const clampedUs = limitUs === undefined ? positionUs : Math.min(positionUs, limitUs); - - if (holdFloorUs === undefined) return clampedUs; - - if (clampedUs < holdFloorUs) return holdFloorUs; - - holdFloorUs = undefined; - return clampedUs; - }, - - destroy(): void { - if (destroyed) return; - - destroyed = true; - clearInterval(timer); - removeInput(); - player?.close(); - player = undefined; - inner.destroy(); - }, - }; -} diff --git a/packages/spf/src/playback/actors/track-subscriber.ts b/packages/spf/src/playback/actors/track-subscriber.ts index 3c05161cf4..e22a21d0eb 100644 --- a/packages/spf/src/playback/actors/track-subscriber.ts +++ b/packages/spf/src/playback/actors/track-subscriber.ts @@ -164,7 +164,7 @@ export const DEFAULT_STALL_TIMEOUT_MS = 10_000; * SUBSCRIBER_PRIORITY by catalog track kind, lower being more urgent: audio ahead of video on a congested downlink. * moq-relay 0.17 orders what it sends a viewer by subscriber priority alone and ignores the publisher's, and the value * cannot change later — any REQUEST_UPDATE ends the subscription there — so it rides every SUBSCRIBE. Same values as - * the relay SDKs' `mediaSubscribeParameters` (muxinc/relay-sdks#337). Other kinds send none, leaving the peer's + * `@caprid/relay`'s `mediaSubscribeParameters` (muxinc/relay-sdks#337). Other kinds send none, leaving the peer's * default. */ export const MEDIA_SUBSCRIBER_PRIORITY: Readonly>> = { diff --git a/packages/spf/src/playback/behaviors/dom/setup-moq-renderers.ts b/packages/spf/src/playback/behaviors/dom/setup-moq-renderers.ts index 3613b4988d..e5a572fb72 100644 --- a/packages/spf/src/playback/behaviors/dom/setup-moq-renderers.ts +++ b/packages/spf/src/playback/behaviors/dom/setup-moq-renderers.ts @@ -43,10 +43,10 @@ import { import { type AudioContextLike, type AudioRendererActor, + type CreateAudioPlayer, createAudioRendererActor, } from '../../actors/dom/audio-renderer'; import { createVideoRendererActor, type VideoRendererActor } from '../../actors/dom/video-renderer'; -import { type CreateAudioPlayer, createWorkletAudioRendererActor } from '../../actors/dom/worklet-audio-renderer'; import type { TrackSubscriberActor } from '../../actors/track-subscriber'; import { type LatencyControlConfig, type PlayoutClockOwner, resolveLatencyControlConfig } from '../sync-latency'; @@ -94,10 +94,7 @@ const CLOCK_PUBLISH_INTERVAL_MS = 100; export interface MoqRendererConfig { /** Shared with `syncLatency`: `joinAtEdge`, the target the clocks aim at, and the slew bounds. */ latency?: Partial; - /** - * Render audio through the shared AudioWorklet player core instead of scheduled `AudioBufferSourceNode`s (see - * `worklet-audio-renderer`). Opt-in until the core can apply `playoutRate` nudges (mmcc/v10#66). - */ + /** Override the audio renderer's player core factory; see `CreateAudioPlayer`. */ createAudioPlayer?: CreateAudioPlayer; } @@ -254,31 +251,24 @@ function setupAudioRendererSetup({ latencyConfig, context.videoRendererActor ); - const renderer = config?.createAudioPlayer - ? createWorkletAudioRendererActor({ - audioContext, - createAudioPlayer: config.createAudioPlayer, - getJoinAnchorUs, - // Caps the worklet's jitter target, so the core never holds - // more than the latency the controller is aiming at. - getTargetLatencyUs: () => - resolveTrackTargetSeconds( - peek(context.audioSubscriberActor), - state.targetLatency, - state.adaptiveTargetLatency, - latencyConfig - ) * 1_000_000, - }) - : createAudioRendererActor({ - audioContext, - // No paused gating here: the adapter suspends the - // AudioContext on pause, which freezes the hardware clock and - // every scheduled source. Rate 0 would instead produce an - // infinite clock segment (duration ÷ 0) and sources whose - // `playbackRate` stays 0 after resume — a permanent stall. - getPlaybackRate: () => peek(state.playoutRate) ?? 1, - getJoinAnchorUs, - }); + const renderer = createAudioRendererActor({ + audioContext, + createAudioPlayer: config?.createAudioPlayer, + // No paused gating here: the adapter suspends the AudioContext on + // pause, which freezes the worklet and the clock with it. The core + // has no rate 0 to pause with anyway. + getPlaybackRate: () => peek(state.playoutRate) ?? 1, + getJoinAnchorUs, + // Caps the worklet's jitter target, so the core never holds + // more than the latency the controller is aiming at. + getTargetLatencyUs: () => + resolveTrackTargetSeconds( + peek(context.audioSubscriberActor), + state.targetLatency, + state.adaptiveTargetLatency, + latencyConfig + ) * 1_000_000, + }); context.audioRendererActor.set(renderer); return () => { diff --git a/packages/spf/src/playback/behaviors/dom/tests/setup-moq-renderers.test.ts b/packages/spf/src/playback/behaviors/dom/tests/setup-moq-renderers.test.ts index f3bb440edb..3bb6c07bd9 100644 --- a/packages/spf/src/playback/behaviors/dom/tests/setup-moq-renderers.test.ts +++ b/packages/spf/src/playback/behaviors/dom/tests/setup-moq-renderers.test.ts @@ -5,7 +5,6 @@ import type { AudioContextLike, AudioRendererActor } from '../../../actors/dom/a import { createAudioRendererActor } from '../../../actors/dom/audio-renderer'; import type { VideoRendererActor } from '../../../actors/dom/video-renderer'; import { createVideoRendererActor } from '../../../actors/dom/video-renderer'; -import { createWorkletAudioRendererActor } from '../../../actors/dom/worklet-audio-renderer'; import type { TrackSubscriberActor } from '../../../actors/track-subscriber'; import type { PlayoutClockOwner } from '../../sync-latency'; import { @@ -24,9 +23,6 @@ vi.mock('../../../actors/dom/audio-renderer', () => ({ vi.mock('../../../actors/dom/video-renderer', () => ({ createVideoRendererActor: vi.fn(), })); -vi.mock('../../../actors/dom/worklet-audio-renderer', () => ({ - createWorkletAudioRendererActor: vi.fn(), -})); function makeFakeAudioRenderer(): AudioRendererActor { return { @@ -87,23 +83,22 @@ describe('setupAudioRenderer', () => { const { getPlaybackRate } = vi.mocked(createAudioRendererActor).mock.calls[0]![0]; expect(getPlaybackRate!()).toBe(1); - // Rate must stay positive even through a pause: a rate of 0 would - // schedule an infinite clock segment (duration ÷ 0) and dead sources. + // Rate must stay positive even through a pause: the core accepts no + // rate below 0.5, and suspending the context already holds the clock. state.playoutRate.set(1.05); expect(getPlaybackRate!()).toBe(1.05); reactor.destroy(); }); - it('renders through the worklet core when a createAudioPlayer is configured', async () => { - vi.mocked(createWorkletAudioRendererActor).mockImplementation(() => makeFakeAudioRenderer()); + it('passes the configured createAudioPlayer and caps the core at the target latency', async () => { + vi.mocked(createAudioRendererActor).mockImplementation(() => makeFakeAudioRenderer()); const createAudioPlayer = vi.fn(); const { state, context, reactor } = setupSetupAudioRenderer({ createAudioPlayer }); - await vi.waitFor(() => expect(createWorkletAudioRendererActor).toHaveBeenCalledTimes(1)); - expect(createAudioRendererActor).not.toHaveBeenCalled(); + await vi.waitFor(() => expect(createAudioRendererActor).toHaveBeenCalledTimes(1)); - const options = vi.mocked(createWorkletAudioRendererActor).mock.calls[0]![0]; + const options = vi.mocked(createAudioRendererActor).mock.calls[0]![0]; expect(options.createAudioPlayer).toBe(createAudioPlayer); expect(options.audioContext).toBe(context.audioContext.get()); diff --git a/packages/spf/src/playback/engines/moq/adapter.ts b/packages/spf/src/playback/engines/moq/adapter.ts index 9ffaf64dfa..45fdd9d35f 100644 --- a/packages/spf/src/playback/engines/moq/adapter.ts +++ b/packages/spf/src/playback/engines/moq/adapter.ts @@ -330,7 +330,7 @@ export function MoqMediaMixin>( gain.connect(audioContext.destination); gain.gain.value = this.#muted ? 0 : this.#volume; - // The worklet renderer's core constructs an `AudioWorkletNode`, which + // The renderer's player core constructs an `AudioWorkletNode`, which // only a real context can host; an injected fake has none to offer. const baseContext = audioContext.baseContext ?? @@ -350,9 +350,6 @@ export function MoqMediaMixin>( return audioContext.outputLatency; }, baseContext, - createBuffer: (numberOfChannels, length, sampleRate) => - audioContext.createBuffer(numberOfChannels, length, sampleRate), - createBufferSource: () => audioContext.createBufferSource(), }; } @@ -464,10 +461,9 @@ export function MoqMediaMixin>( this.#gain = undefined; this.#renderContext = undefined; // Close the context only after the composition has torn the renderers - // down: an in-flight renderer tick still calls `createBuffer` / - // `createBufferSource`, which throw `InvalidStateError` on a closed - // context. Both arms close so a failed teardown still releases the - // audio device. + // down: the renderer's player core still builds and drives nodes on + // it, which throws `InvalidStateError` on a closed context. Both arms + // close so a failed teardown still releases the audio device. const closeAudio = () => { audioContext?.close().catch(() => {}); }; diff --git a/packages/spf/src/playback/engines/moq/engine.ts b/packages/spf/src/playback/engines/moq/engine.ts index bc64fc0787..4b3206203a 100644 --- a/packages/spf/src/playback/engines/moq/engine.ts +++ b/packages/spf/src/playback/engines/moq/engine.ts @@ -9,9 +9,8 @@ import type { QualityConfig } from '../../../media/abr/quality-selection'; import type { AudioTrack, MaybeResolvedPresentation, TextTrack, VideoTrack } from '../../../media/types'; import type { BandwidthConfig, BandwidthState } from '../../../network/bandwidth-estimator'; import type { RetryBackoffConfig } from '../../../network/retry-backoff'; -import type { AudioContextLike, AudioRendererActor } from '../../actors/dom/audio-renderer'; +import type { AudioContextLike, AudioRendererActor, CreateAudioPlayer } from '../../actors/dom/audio-renderer'; import type { VideoRendererActor } from '../../actors/dom/video-renderer'; -import type { CreateAudioPlayer } from '../../actors/dom/worklet-audio-renderer'; import type { CreateMoqTransport, MoqAuthProvider, MoqSessionActor } from '../../actors/moq-session'; import type { TrackSubscriberActor } from '../../actors/track-subscriber'; import { type AdaptiveLatencyConfig, adaptLatencyTarget } from '../../behaviors/adapt-latency-target'; @@ -155,8 +154,8 @@ export interface MoqEngineConfig extends ShareSignalsConfig; /** - * Render audio through the shared AudioWorklet player core (`@mux/moq-media`'s `createAudioPlayer`) instead of - * scheduled `AudioBufferSourceNode`s. Opt-in: the core cannot apply `playoutRate` nudges yet (mmcc/v10#66). + * Audio player core factory override (tests, or a context the default cannot host). Defaults to `@caprid/moq-media`'s + * `createAudioPlayer` on the render context's `baseContext`. */ createAudioPlayer?: CreateAudioPlayer; /** diff --git a/packages/spf/src/playback/engines/moq/tests/adapter.test.ts b/packages/spf/src/playback/engines/moq/tests/adapter.test.ts index 9585d0c3cd..f645790ef6 100644 --- a/packages/spf/src/playback/engines/moq/tests/adapter.test.ts +++ b/packages/spf/src/playback/engines/moq/tests/adapter.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vite-plus/test'; -import { type MoqAudioContext, MoqMediaElement } from '../adapter'; +import { MoqMediaElement } from '../adapter'; /** * Structural `MoqAudioContext` fake: tracks state transitions through resume/suspend spies so tests can assert the @@ -21,12 +21,6 @@ function createFakeAudioContext(initialState: AudioContextState) { destination: {} as AudioNode, gain, createGain: vi.fn(() => gain), - createBuffer: (() => { - throw new Error('unused in adapter tests'); - }) as MoqAudioContext['createBuffer'], - createBufferSource: (() => { - throw new Error('unused in adapter tests'); - }) as MoqAudioContext['createBufferSource'], resume: vi.fn(async () => { fake.state = 'running'; }), @@ -142,8 +136,8 @@ describe('MoqMediaMixin', () => { media.attach(document.createElement('canvas')); media.destroy(); - // Renderer ticks still call createBuffer/createBufferSource until the - // composition has stopped them; closing first throws InvalidStateError. + // The renderer's player core still drives nodes on the context until + // the composition has stopped it; closing first throws InvalidStateError. expect(audioContext.close).not.toHaveBeenCalled(); await vi.waitFor(() => expect(audioContext.close).toHaveBeenCalledTimes(1)); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 81eb089a32..a408954d35 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1247,6 +1247,9 @@ importers: packages/spf: dependencies: + '@caprid/moq-media': + specifier: ^0.1.0 + version: 0.1.0 '@videojs/media': specifier: workspace:* version: link:../media @@ -2103,6 +2106,14 @@ packages: cpu: [x64] os: [win32] + '@caprid/moq-core@0.1.0': + resolution: {integrity: sha512-GWNqRU+XODpG/4fCUQVXYZtONVibYOY4BwYH4rt7lQuvBJ0sh49W4gw+1jAhY+a924bMGSSGzJ1shr8G5xMMmQ==} + engines: {node: '>=24'} + + '@caprid/moq-media@0.1.0': + resolution: {integrity: sha512-Iu2+r7JYZoRYzoM9Xf3aK2KqBH8x26NGpKhdI3jFICoSr3IeAIXPYaad46HkHq68ottv5LwV4cxyVaFGrQfbZg==} + engines: {node: '>=24'} + '@capsizecss/unpack@4.0.0': resolution: {integrity: sha512-VERIM64vtTP1C4mxQ5thVT9fK0apjPFobqybMtA1UdUujWka24ERHbRHFGmpbbhp73MhV+KSsHQH9C6uOTdEQA==} engines: {node: '>=18'} @@ -5580,133 +5591,157 @@ packages: resolution: {integrity: sha512-jrOY5WM+AaAqkv51fHP1x28ifto4WgcZVzodLUyMU1jMWn5Sq+VciRdk/n8E0Ey5w4p4cWWE+m5GfXWOYh7Kzw==} cpu: [arm64] os: [android] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-codegen/binding-darwin-arm64@0.9.5': resolution: {integrity: sha512-4O4lkCQIzPZGjiNB1GORSI6hBECbiiSBS/APZXPvNKYH+nhY4uuqv03LNXA+SET3hoBjvr95P5rIhY8KQaQUBA==} cpu: [arm64] os: [darwin] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-codegen/binding-darwin-x64@0.9.5': resolution: {integrity: sha512-9EvoUO0SEhD6/d8VHdWuPerepPMSR1y84+UEgz8Un1Ope14Oe7xMvePpEQLTLovoIFZ8Zg3iZ8z8E11pZMqC3g==} cpu: [x64] os: [darwin] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-codegen/binding-freebsd-x64@0.9.5': resolution: {integrity: sha512-HqT78WwgHTmp8lwujoUa9CrIortX4DdpuiVC18ZPSGvuuJf4ylpIEI6QrQEM78Zwz3muhAWAOXZ5irdiYY+AyA==} cpu: [x64] os: [freebsd] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-codegen/binding-linux-arm-gnu@0.9.5': resolution: {integrity: sha512-QJXwIW6Ms3QIawbcBIedmrmXnfdpuAOjZJ/eAABq5XTzWvSxZ7lutu9W5yIHahlaTaFnBo7ikrVMD1szLTpPUw==} cpu: [arm] os: [linux] libc: [glibc] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-codegen/binding-linux-arm-musl@0.9.5': resolution: {integrity: sha512-eHbFy3IHGb+IYarrYwAo0yWSEQV3eAmEn6VrsKA6I1Wy1YPJxWqSJlV69JhqsHtJuJlljUIF3ex0y46yaKIu9w==} cpu: [arm] os: [linux] libc: [musl] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-codegen/binding-linux-arm64-gnu@0.9.5': resolution: {integrity: sha512-ByoJMbySTaDhjAXSu8q6Lh7HKg3YoesXpcT72aYk0Aiw4PCznmY4ybpLTq0RCvp0RIPhFm/6yECFiGBwyCC1nw==} cpu: [arm64] os: [linux] libc: [glibc] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-codegen/binding-linux-arm64-musl@0.9.5': resolution: {integrity: sha512-gD9vfXIoBw1toSxhO9rgmSu/FEfy3PMznJAxVjIuH8DrWEiDKXmJO0pJKfj1Ltbe/TmtWVZR+TjISqeSIGQzMg==} cpu: [arm64] os: [linux] libc: [musl] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-codegen/binding-linux-x64-gnu@0.9.5': resolution: {integrity: sha512-BARvdnvqMGjOr5Iel2JH+9H5vAIE0R6H0Z2fsT02xrvmI/1ZXNB8lkuX+ZGevPhZb3zJ9WeC/R8JZstrsUOK5g==} cpu: [x64] os: [linux] libc: [glibc] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-codegen/binding-linux-x64-musl@0.9.5': resolution: {integrity: sha512-x8flcevS1fbb7ESrmpOY/pON4eInSaE/7Ktjqx2udOE2W33BNSZmJuwpyUgo54AoHNqrcaM7szqydyJn5fNvew==} cpu: [x64] os: [linux] libc: [musl] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-codegen/binding-win32-arm64@0.9.5': resolution: {integrity: sha512-KOL/rBatWqH4ZpCNoF8ZtSNdOJbAxBJLmk/VeRciepGROPTbPum1A9t67GpFgOU7qrkUWlPJdJ+KHKbvbmOt+w==} cpu: [arm64] os: [win32] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-codegen/binding-win32-x64@0.9.5': resolution: {integrity: sha512-FxENahEjWSan59Syh/us/Kf4wNq8qrJLFJ3R2N4Oiwtb6yNdz/rWLNTIoNSssnsp7IoWJdUP6o/Z8ppg7lXMcg==} cpu: [x64] os: [win32] + deprecated: yuku-codegen is pure JavaScript since 0.14 '@yuku-parser/binding-android-arm64@0.9.5': resolution: {integrity: sha512-A2JCFCSHfnficqYEw4Iujpx7XrkMM3UfFcgJFmYpZSo9zM4nyhmdIIE0FogSduuU60lhM0/UcfuUBXIVNBMlGQ==} cpu: [arm64] os: [android] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-parser/binding-darwin-arm64@0.9.5': resolution: {integrity: sha512-3PiyU+Eare4YuKaQ22N98/yAiROPY5o/NQJHraICzDvk4pgS+m+bgLKOvkGBRn33OnV95Vdv1Mn38b+MQHpULQ==} cpu: [arm64] os: [darwin] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-parser/binding-darwin-x64@0.9.5': resolution: {integrity: sha512-blFMAFI7AInI83XaiOF8cIeiRM46Nz9EfpZtZPRLbKxSA9aAr5v8aHYpmfN6NoyXxAv/10pwS4gsAuc1W6fy5Q==} cpu: [x64] os: [darwin] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-parser/binding-freebsd-x64@0.9.5': resolution: {integrity: sha512-TsNuL4qsZdO0tHp5GW43Y5fHeLPpPHA675wdcPNTcdq2ZO5AvsQWFFoiDg8CH4oBktfsQ9tdvKhjLnLYwKvp4g==} cpu: [x64] os: [freebsd] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-parser/binding-linux-arm-gnu@0.9.5': resolution: {integrity: sha512-b0afYK5gHeV8RdmOcqAlgM8ONsye4cax4DMnIBaoJyPMd1UTGvTbpkqQcPVdhmHlcfcGVGMV1laeVovlr/dscw==} cpu: [arm] os: [linux] libc: [glibc] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-parser/binding-linux-arm-musl@0.9.5': resolution: {integrity: sha512-fD3lKzl+r6j6n8DwiMY53qnh5DLqD8KJjCp+NbVud54Nnh3Q9Wprqss3YzyMHP3NSJk663Wlg1E1X5qOuFVFig==} cpu: [arm] os: [linux] libc: [musl] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-parser/binding-linux-arm64-gnu@0.9.5': resolution: {integrity: sha512-aRU/aCphV1MWCl/lvI6NX8LgucHQ68Fx+vz7NBb/5MEr2EuzCxiPOQqeFcMdWh+2AED/p0AvAREch0c+cSnlhA==} cpu: [arm64] os: [linux] libc: [glibc] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-parser/binding-linux-arm64-musl@0.9.5': resolution: {integrity: sha512-5+Guro0l8H473YXlEjVNBRLN/IbPbJdnQh1zo0OLt49xxnON+XMJRRaEyTOTfkn9QSRg0+BhEKGR4W9Gu2uZJQ==} cpu: [arm64] os: [linux] libc: [musl] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-parser/binding-linux-x64-gnu@0.9.5': resolution: {integrity: sha512-pwwSyV9q+GlvzSXlsMZBMlgk22L5bud714/vqZCdeUTivzeUVltQzUaf3IQXOGXdpc59JYTeuMCtbGquaAUoCA==} cpu: [x64] os: [linux] libc: [glibc] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-parser/binding-linux-x64-musl@0.9.5': resolution: {integrity: sha512-z18j6JN3lBHH8vzN7gG1M8fI0nlgItSfnC9PfbmUbt97iHViKfw2iA2G9fGwRMe5LyPRZBrejIlapbygPbpdaw==} cpu: [x64] os: [linux] libc: [musl] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-parser/binding-win32-arm64@0.9.5': resolution: {integrity: sha512-s6Gwttb1dQvtPX6Bgkw+UPC9IO3UBDXc6Zezog8MMgvu3UfJBBB9TQqKBX/2quu0Eyh+lLSAyNlIyyecaagUPg==} cpu: [arm64] os: [win32] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-parser/binding-win32-x64@0.9.5': resolution: {integrity: sha512-FrERt9YWatY3bJfSUdi4YWY+6iQcyo3MzCC821BxlWM5TFZEHijnpz/bknR79VHlXY/9CvXz8ZlaAGPsTtN3nw==} cpu: [x64] os: [win32] + deprecated: yuku-parser runs on yuku-core since 0.14 '@yuku-toolchain/types@0.9.5': resolution: {integrity: sha512-KiuLNNgX9uNealaWAR+G3/cMXnRk9x4TY2EkYe/KIag+UPdwiA0RRf1hr1WAxzTP8KGzCTkqUdLPvqh32sEO3w==} @@ -10980,6 +11015,12 @@ snapshots: '@bruits/satteri-win32-x64-msvc@0.10.5': optional: true + '@caprid/moq-core@0.1.0': {} + + '@caprid/moq-media@0.1.0': + dependencies: + '@caprid/moq-core': 0.1.0 + '@capsizecss/unpack@4.0.0': dependencies: fontkitten: 1.0.3 @@ -19251,6 +19292,62 @@ snapshots: transitivePeerDependencies: - msw + vitest@5.0.1(@opentelemetry/api@1.9.1)(@types/node@22.19.15)(@vitest/browser-playwright@5.0.1)(@vitest/browser-preview@5.0.1)(@vitest/coverage-v8@5.0.1)(@vitest/ui@5.0.1)(@voidzero-dev/vite-plus-core@1.0.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.1)(typescript@6.0.2)(unrun@0.2.39)(yaml@2.9.0))(happy-dom@20.11.1)(jsdom@26.1.0(supports-color@8.1.1)): + dependencies: + '@types/chai': 5.2.3 + '@vitest/mocker': 5.0.1(@voidzero-dev/vite-plus-core@1.0.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.1)(typescript@6.0.2)(unrun@0.2.39)(yaml@2.9.0)) + chai: 6.2.2 + es-module-lexer: 2.3.2 + expect-type: 1.4.0 + magic-string: 1.4.2 + obug: 2.1.4 + picomatch: 4.0.7 + std-env: 4.2.0 + tinybench: 6.1.4 + tinyexec: 1.3.0 + tinyglobby: 0.2.17 + vite: '@voidzero-dev/vite-plus-core@1.0.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.1)(typescript@6.0.2)(unrun@0.2.39)(yaml@2.9.0)' + why-is-node-running: 2.3.0 + optionalDependencies: + '@opentelemetry/api': 1.9.1 + '@types/node': 22.19.15 + '@vitest/browser-playwright': 5.0.1(@voidzero-dev/vite-plus-core@1.0.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.1)(typescript@6.0.2)(unrun@0.2.39)(yaml@2.9.0))(playwright@1.59.1)(vitest@5.0.1) + '@vitest/browser-preview': 5.0.1(@voidzero-dev/vite-plus-core@1.0.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.1)(typescript@6.0.2)(unrun@0.2.39)(yaml@2.9.0))(vitest@5.0.1) + '@vitest/coverage-v8': 5.0.1(@vitest/browser@5.0.1)(vitest@5.0.1) + '@vitest/ui': 5.0.1(vitest@5.0.1) + happy-dom: 20.11.1 + jsdom: 26.1.0(supports-color@8.1.1) + transitivePeerDependencies: + - msw + + vitest@5.0.1(@opentelemetry/api@1.9.1)(@types/node@22.19.15)(@vitest/browser-playwright@5.0.1)(@vitest/browser-preview@5.0.1)(@vitest/coverage-v8@5.0.1)(@vitest/ui@5.0.1)(@voidzero-dev/vite-plus-core@1.0.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.1)(typescript@6.0.2)(unrun@0.2.39)(yaml@2.9.0))(happy-dom@20.11.1)(jsdom@27.4.0(supports-color@8.1.1)): + dependencies: + '@types/chai': 5.2.3 + '@vitest/mocker': 5.0.1(@voidzero-dev/vite-plus-core@1.0.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.1)(typescript@6.0.2)(unrun@0.2.39)(yaml@2.9.0)) + chai: 6.2.2 + es-module-lexer: 2.3.2 + expect-type: 1.4.0 + magic-string: 1.4.2 + obug: 2.1.4 + picomatch: 4.0.7 + std-env: 4.2.0 + tinybench: 6.1.4 + tinyexec: 1.3.0 + tinyglobby: 0.2.17 + vite: '@voidzero-dev/vite-plus-core@1.0.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.1)(typescript@6.0.2)(unrun@0.2.39)(yaml@2.9.0)' + why-is-node-running: 2.3.0 + optionalDependencies: + '@opentelemetry/api': 1.9.1 + '@types/node': 22.19.15 + '@vitest/browser-playwright': 5.0.1(@voidzero-dev/vite-plus-core@1.0.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.1)(typescript@6.0.2)(unrun@0.2.39)(yaml@2.9.0))(playwright@1.59.1)(vitest@5.0.1) + '@vitest/browser-preview': 5.0.1(@voidzero-dev/vite-plus-core@1.0.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.1)(typescript@6.0.2)(unrun@0.2.39)(yaml@2.9.0))(vitest@5.0.1) + '@vitest/coverage-v8': 5.0.1(@vitest/browser@5.0.1)(vitest@5.0.1) + '@vitest/ui': 5.0.1(vitest@5.0.1) + happy-dom: 20.11.1 + jsdom: 27.4.0(supports-color@8.1.1) + transitivePeerDependencies: + - msw + vitest@5.0.1(@opentelemetry/api@1.9.1)(@types/node@22.20.1)(@vitest/browser-playwright@5.0.1)(@vitest/browser-preview@5.0.1)(@vitest/coverage-v8@5.0.1)(@vitest/ui@5.0.1)(@voidzero-dev/vite-plus-core@1.0.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.1)(typescript@6.0.2)(unrun@0.2.39)(yaml@2.9.0))(happy-dom@20.11.1)(jsdom@27.4.0(supports-color@8.1.1)): dependencies: '@types/chai': 5.2.3 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index ec937b5110..ae5b9b3691 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -77,3 +77,4 @@ minimumReleaseAgeExclude: - '@oxfmt/*' - vitest - '@vitest/*' + - '@caprid/*'