From 5c3dd02a63e6316c3836cddb6d86ef5ee0e0ac8a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 7 Sep 2026 20:58:36 -0700 Subject: [PATCH 1/2] fix(watch): size the auto delay from measured arrivals and re-buffer the audio ring "auto" derived its jitter buffer from the round trip: max(20ms, 1.25 x minRtt), falling back to a flat 100ms. The round trip says nothing about how evenly a publisher emits frames, so a sender flushing a hundred milliseconds of media at once got a target sized for a retransmit and the ring never held enough to play through the next flush. Measure the arrival spread instead, at container frame arrival and before the latency budget can skip a group. Each arrival contributes `now - timestamp` relative to the running minimum of that difference into a decaying histogram, and the estimate is its 95th percentile plus one frame, the shape NetEq uses. It rises as soon as a run of late frames proves the buffer is too shallow and lowers by at most one frame per second. Sync takes the largest across audio and video, held above the per-rendition catalog floor. Both rings hold a chunk of slack above the target, land back on the target when they skip, re-stall when they run dry, and gain `stall()` so a deepening target parks the playhead until the ring holds it again. A shortfall is ramped rather than stepped, and an underrun counter reaches the stats panel, the buffering indicator, and the demo's audio bar. Co-Authored-By: Claude Opus 5 --- demo/web/src/viz.ts | 2 +- doc/lib/js/watch.md | 4 +- js/hang/src/container/consumer.test.ts | 18 ++ js/hang/src/container/consumer.ts | 25 +++ js/hang/src/container/jitter.test.ts | 111 +++++++++++ js/hang/src/container/jitter.ts | 156 +++++++++++++++ js/moq-boy/src/game.ts | 11 +- js/watch/src/audio/buffer.ts | 45 ++++- js/watch/src/audio/decoder.ts | 68 ++++--- js/watch/src/audio/latency.test.ts | 13 +- js/watch/src/audio/latency.ts | 22 --- js/watch/src/audio/render-worklet.ts | 37 +++- js/watch/src/audio/render.ts | 12 +- js/watch/src/audio/replay.test.ts | 183 ++++++++++++++++++ js/watch/src/audio/ring-buffer.test.ts | 162 +++++++++++----- js/watch/src/audio/ring-buffer.ts | 67 +++++-- js/watch/src/audio/shared-ring-buffer.test.ts | 106 +++++++++- js/watch/src/audio/shared-ring-buffer.ts | 61 +++++- js/watch/src/element.ts | 12 +- js/watch/src/sync.test.ts | 45 +++++ js/watch/src/sync.ts | 79 ++++---- .../src/ui/components/buffering-indicator.ts | 7 +- js/watch/src/ui/components/latency.ts | 2 +- js/watch/src/ui/stats.ts | 5 + js/watch/src/video/decoder.ts | 16 ++ 25 files changed, 1081 insertions(+), 188 deletions(-) create mode 100644 js/hang/src/container/jitter.test.ts create mode 100644 js/hang/src/container/jitter.ts create mode 100644 js/watch/src/audio/replay.test.ts diff --git a/demo/web/src/viz.ts b/demo/web/src/viz.ts index abe1a5a4969..827b83eda46 100644 --- a/demo/web/src/viz.ts +++ b/demo/web/src/viz.ts @@ -307,7 +307,7 @@ export function bufferBars(parent: Signals.Effect, watch: MoqWatch): HTMLElement const timestamp = watch.sync.now() as number | undefined; const stalled = watch.video.out.stalled.peek(); drawRanges(video.canvas, watch.video.out.buffered.peek(), timestamp, stalled); - drawRanges(audio.canvas, watch.audio.out.buffered.peek(), timestamp, false); + drawRanges(audio.canvas, watch.audio.out.buffered.peek(), timestamp, watch.audio.out.stalled.peek()); parent.animate(draw); }; parent.animate(draw); diff --git a/doc/lib/js/watch.md b/doc/lib/js/watch.md index 6a49a8b3982..2ff883e4f08 100644 --- a/doc/lib/js/watch.md +++ b/doc/lib/js/watch.md @@ -30,10 +30,10 @@ in sync at the latency you ask for. | --- | --- | | `url`, `name` | Relay URL (with `?jwt=` if needed) and broadcast name. | | `paused`, `muted`, `volume` | The usual player controls, mirrored as reactive properties. | -| `delay` | How far playback trails the live edge: `"auto"` (derived from RTT, the default), a duration like `"300ms"`, or `"instant"` to paint frames as they decode with no pacing at all. | +| `delay` | How far playback trails the live edge: `"auto"` (the default, sized from how late frames actually arrive), a duration like `"300ms"`, or `"instant"` to paint frames as they decode with no pacing at all. | | `buffer` | Future-dated media held beyond the live edge before playback skips ahead, e.g. `"30s"`. Defaults to none. | | `captions` | The caption track to show, or absent for off. `el.text.out.available` lists the renditions for a picker. | -| `jitter` | The jitter buffer in ms. | +| `jitter` | The jitter buffer in ms: in `"auto"` this is the measured arrival spread, read-only in practice. | | `visible` | Only subscribe to video while the element is on screen: a margin (`"20%"` default, `"200px"`), `"always"`, or `"never"`. | | `reload` | Wait for the broadcast to be announced before subscribing (default on), so a player can be mounted before the stream exists. | | `catalog-format` | `hang` (default, from the `.hang` suffix), `hangz` (compressed), `msf`, or `manual` to supply the catalog yourself. | diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index 2da4b48da6c..905f8853760 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -323,6 +323,24 @@ test("Consumer skips groups via PTS-span when over the max age", async () => { consumer.close(); }); +test("Consumer measures how late frames arrive", async () => { + const track = new Track.Producer("test"); + const consumer = new Consumer(track.subscribe(), { format: new LegacyFormat(), latency: 500 as Time.Milli }); + + // A prompt first group sets the arrival baseline. + writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); + await settle(150); + + // The next group carries 20ms of media but shows up 150ms later, so a player needs the + // difference in its buffer to render it on time. Nothing about the round trip says that. + writeGroupWithLegacyFrames(track, 1, [20_000 as Time.Micro]); + track.close(); + + await drainFrames(consumer, 300); + expect(consumer.spread.peek()).toBeGreaterThanOrEqual(50 as Time.Milli); + consumer.close(); +}); + // --- Ordering --- test("Consumer delivers groups in sequence order regardless of arrival order", async () => { diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 26b3bae2cae..fd976b5548c 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -3,6 +3,7 @@ import * as Moq from "@moq/net"; import { Effect, type Getter, type GetterInit, getter, Signal } from "@moq/signals"; import type { Format } from "./format"; +import { Jitter } from "./jitter"; import type { BufferedRanges, Frame } from "./types"; /** Options for constructing a {@link Consumer}. */ @@ -143,6 +144,18 @@ export class Consumer { /** The time ranges currently buffered and ready to play. */ readonly buffered: Getter = this.#buffered; + // Measured at arrival, before any group is skipped: a target derived from what survives the + // age budget would only ever confirm the budget it was cut to. + #spread = new Jitter(); + + /** + * How late frames arrive relative to the earliest one, measured as they land. + * + * Size the playback buffer with this rather than with the round trip, which says nothing about + * how evenly a publisher emits frames. + */ + readonly spread: Getter = this.#spread.value; + #signals = new Effect(); /** Start consuming the given track, decoding frames with `props.format`. */ @@ -256,11 +269,21 @@ export class Consumer { // max age budget is what eventually breaks the stall. if (this.#classifyStale(group)) return; this.#checkReset(group); + + // Measured once the timeline is settled and before the age budget can skip + // the group: a reneged straggler is already gone, a rewinding group is read + // against the baseline `#checkReset` just re-anchored rather than against + // the epoch it replaced, and a target derived from what survives the budget + // would only ever confirm the budget it was cut to. + if (!marker) this.#spread.observe(frame.timestamp, Moq.Time.Milli.now()); this.#checkMaxAge(); // A newer group reaching back to where the stalled active group has // already presented means we can advance now instead of waiting. skipped = this.#tryDurationSkip(); + } else if (!marker) { + // The active group needs no classification, so its arrival counts as is. + this.#spread.observe(frame.timestamp, Moq.Time.Milli.now()); } // Wake next() for the current delivery head so its frames surface as they @@ -438,6 +461,7 @@ export class Consumer { this.#rewind.boundary = reset; this.#rewind.discontinuity++; this.#gap = true; + this.#spread.reanchor(); // Drop buffered groups the boundary can already prove stale; keep ambiguous ones. this.#groups = this.#groups.filter((g) => { @@ -661,6 +685,7 @@ export class Consumer { this.#presentedEnd = undefined; this.#deliveredGroup = undefined; this.#gap = true; + this.#spread.reanchor(); } #updateBuffered(): void { diff --git a/js/hang/src/container/jitter.test.ts b/js/hang/src/container/jitter.test.ts new file mode 100644 index 00000000000..34196c3f447 --- /dev/null +++ b/js/hang/src/container/jitter.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from "bun:test"; +import { Time } from "@moq/net"; +import { Jitter } from "./jitter"; + +const CHUNK_MS = 20; + +/** Feed one arrival, in milliseconds on both axes. */ +function observe(jitter: Jitter, media: number, arrival: number): void { + jitter.observe(Time.Micro.fromMilli(media as Time.Milli), arrival as Time.Milli); +} + +/** + * A sender emitting one chunk every `CHUNK_MS`, holding `burst - 1` of them back and flushing the + * whole run at once. `burst` of 1 is an evenly paced sender. + */ +function flush(jitter: Jitter, chunks: number, burst: number, base = 50): void { + let held: number[] = []; + for (let i = 0; i < chunks; i++) { + const media = i * CHUNK_MS; + held.push(media); + if (held.length < burst) continue; + const arrival = media + base; + for (const m of held) observe(jitter, m, arrival); + held = []; + } +} + +describe("arrival jitter", () => { + it("is one frame for an evenly paced sender", () => { + // Nothing arrives late, so all the buffer needs to hold is the frame being played. + const jitter = new Jitter(); + flush(jitter, 500, 1); + expect(jitter.value.peek()).toBe(CHUNK_MS as Time.Milli); + }); + + it("does not follow the absolute delay", () => { + // A sender on the other side of the planet is not a jittery sender: only the spread of + // arrivals matters, which is what makes the round trip the wrong term to size a buffer from. + const near = new Jitter(); + flush(near, 500, 1, 10); + const far = new Jitter(); + flush(far, 500, 1, 2000); + expect(far.value.peek()).toBe(near.value.peek()); + }); + + it("converges on the flush span of a bursty sender", () => { + // Three chunks land at once, so the oldest is two chunk durations late. + const jitter = new Jitter(); + flush(jitter, 600, 3); + expect(jitter.value.peek()).toBeGreaterThanOrEqual(3 * CHUNK_MS); + expect(jitter.value.peek()).toBeLessThanOrEqual(4 * CHUNK_MS); + }); + + it("rises as soon as one arrival proves the buffer is too shallow", () => { + const jitter = new Jitter(); + flush(jitter, 100, 1); + expect(jitter.value.peek()).toBe(CHUNK_MS as Time.Milli); + + // A single 250ms straggler is only 1% of the samples, so the percentile ignores it. It costs + // one bucket of histogram resolution, not a quarter second of buffer. + observe(jitter, 100 * CHUNK_MS, 100 * CHUNK_MS + 250); + expect(jitter.value.peek()).toBeLessThan(2 * CHUNK_MS); + + // A run of them is the network, not an outlier, and the estimate follows within a second. + for (let i = 0; i < 20; i++) { + observe(jitter, (101 + i) * CHUNK_MS, (101 + i) * CHUNK_MS + 250); + } + expect(jitter.value.peek()).toBeGreaterThan(150 as Time.Milli); + }); + + it("lowers by at most one chunk per interval", () => { + const jitter = new Jitter(); + flush(jitter, 600, 3); + const peak = jitter.value.peek(); + expect(peak).toBeGreaterThan(0 as Time.Milli); + + // Arrivals settle. Sample the estimate every second of the recovery: a step larger than one + // chunk would drop a viewer's playhead onto a cushion that isn't there yet. + let previous = peak; + let last = 0; + const base = 600 * CHUNK_MS; + for (let i = 0; i < 6000; i++) { + const media = base + i * CHUNK_MS; + observe(jitter, media, media + 50); + const now = media + 50; + if (now - last < 1000) continue; + last = now; + const current = jitter.value.peek(); + expect(previous - current).toBeLessThanOrEqual(CHUNK_MS); + previous = current; + } + + // And it does come back down to a frame plus the histogram's resolution once arrivals have + // been even for a while. + expect(jitter.value.peek()).toBeLessThan(2 * CHUNK_MS); + }); + + it("forgets the baseline across a discontinuity", () => { + const jitter = new Jitter(); + flush(jitter, 200, 1); + + // The publisher restarts its timeline an hour in the past. Without a re-anchor every arrival + // after it reads as an hour late and the estimate saturates. + jitter.reanchor(); + const base = 3_600_000; + for (let i = 0; i < 200; i++) { + observe(jitter, base + i * CHUNK_MS, i * CHUNK_MS + 50); + } + expect(jitter.value.peek()).toBe(CHUNK_MS as Time.Milli); + }); +}); diff --git a/js/hang/src/container/jitter.ts b/js/hang/src/container/jitter.ts new file mode 100644 index 00000000000..497630b0983 --- /dev/null +++ b/js/hang/src/container/jitter.ts @@ -0,0 +1,156 @@ +import { Time } from "@moq/net"; +import { type Getter, Signal } from "@moq/signals"; + +/** Width of one histogram bucket, which is also the resolution of the estimate. */ +const BUCKET = 5; + +/** Number of buckets, so the estimate saturates at `BUCKET * BUCKETS`. */ +const BUCKETS = 200; + +/** + * How much an arrival still counts once another arrives. 0.9993 halves a sample's weight after + * ~1000 arrivals, roughly 20s of 50/s audio: long enough to hold a rare burst, short enough to + * forget a network that has since settled. + */ +const FORGET = 0.9993; + +/** The fraction of arrivals the estimate covers. */ +const PERCENTILE = 0.95; + +/** How long the estimate holds its value before it may step down again. */ +const LOWER_INTERVAL = 1000; + +/** + * How long a minimum stays authoritative. The minimum is tracked over two windows and read as the + * smaller of the two, so it survives at least this long and at most twice it. Without an expiry a + * single early arrival would anchor the spread for the life of the stream, and a sender clock + * drifting away from the receiver's would inflate it without bound. + */ +const WINDOW = 30_000; + +/** + * How much buffer late arrivals need, measured at arrival rather than derived from the round trip. + * + * Each arrival contributes `now - timestamp` relative to the running minimum of that difference, + * which is the delay a player has to absorb to render the frame on time: zero for an evenly paced + * sender, one flush span for a sender that emits a burst of frames at once. Samples land in a + * histogram that forgets old arrivals, and the estimate is a high percentile of it plus one frame, + * the shape WebRTC's NetEq uses. The extra frame is what keeps audio in the buffer at the bottom of + * the swing the spread describes, rather than landing it on exactly zero. + * + * The estimate rises as soon as a late frame proves the buffer is too shallow, and lowers by at + * most one frame per interval, so a refinement shrinks a viewer's buffer in steps it can absorb. + */ +export class Jitter { + // Weighted count of arrivals per spread bucket, and their sum. + #buckets = new Float64Array(BUCKETS); + #total = 0; + + // Smallest arrival delay in the current window and in the one before it. See WINDOW. + #current?: number; + #previous?: number; + #windowStart?: number; + + // Largest spread ever measured. Buckets are read at their upper edge, which covers the whole + // spread the bucket might hold; this caps that at a spread actually seen, so an evenly paced + // sender reads as zero rather than as one bucket. + #max = 0; + + // Smallest positive gap between consecutive timestamps, i.e. the frame duration. It bounds how + // fast the estimate steps down. + #spacing?: number; + #latest?: Time.Micro; + + // When the estimate last moved, so a step down waits out LOWER_INTERVAL. + #lowered?: number; + + #value = new Signal(Time.Milli.zero); + + /** The current estimate: enough buffer to render `PERCENTILE` of arrivals on time. */ + readonly value: Getter = this.#value; + + /** Fold one frame into the estimate, given its media timestamp and the wall time it arrived. */ + observe(timestamp: Time.Micro, now: Time.Milli): void { + const delay = now - Time.Milli.fromMicro(timestamp); + + // Roll the minimum window forward so a stale baseline expires. + this.#windowStart ??= now; + if (now - this.#windowStart >= WINDOW) { + this.#previous = this.#current; + this.#current = undefined; + this.#windowStart = now; + } + this.#current = this.#current === undefined ? delay : Math.min(this.#current, delay); + const min = this.#previous === undefined ? this.#current : Math.min(this.#current, this.#previous); + + // Learn the frame duration from the timeline itself; it is the step size for lowering. + if (this.#latest !== undefined && timestamp > this.#latest) { + const gap = Time.Milli.fromMicro((timestamp - this.#latest) as Time.Micro); + this.#spacing = this.#spacing === undefined ? gap : Math.min(this.#spacing, gap); + } + if (this.#latest === undefined || timestamp > this.#latest) this.#latest = timestamp; + + const spread = Math.max(0, delay - min); + this.#max = Math.max(this.#max, spread); + const bucket = Math.min(BUCKETS - 1, Math.floor(spread / BUCKET)); + + // Decay everything, then count this arrival, so a sample's weight is relative to how many + // have arrived since rather than to how long ago it was. + for (let i = 0; i < BUCKETS; i++) this.#buckets[i] *= FORGET; + this.#total = this.#total * FORGET + 1; + this.#buckets[bucket] += 1; + + this.#publish(now); + } + + /** + * Forget the arrival baseline, keeping the measured spread. + * + * A discontinuity moves the media timeline underneath the measurement, so `now - timestamp` + * jumps by the size of the jump and the old minimum describes a timeline that no longer + * exists. The histogram is in spread space, which the jump does not move, so it survives. + */ + reanchor(): void { + this.#current = undefined; + this.#previous = undefined; + this.#windowStart = undefined; + this.#latest = undefined; + } + + #publish(now: Time.Milli): void { + // One frame on top of the percentile: the buffer swings by the spread the percentile + // measures, so without it the bottom of that swing lands on an empty ring. + const step = this.#spacing ?? BUCKET; + const measured = this.#percentile() + step; + const current = this.#value.peek(); + + if (measured >= current) { + this.#lowered = now; + if (measured > current) this.#value.set(Time.Milli(measured)); + return; + } + + // Lower gradually: the buffer shrinks by at most one frame per interval, so a refined + // estimate never drops the playhead onto an empty ring. + this.#lowered ??= now; + if (now - this.#lowered < LOWER_INTERVAL) return; + this.#lowered = now; + + this.#value.set(Time.Milli(Math.max(measured, current - step))); + } + + #percentile(): Time.Milli { + if (this.#total <= 0) return Time.Milli.zero; + + const want = this.#total * PERCENTILE; + let sum = 0; + for (let i = 0; i < BUCKETS; i++) { + sum += this.#buckets[i]; + // The upper edge of the bucket: the spread it holds is somewhere inside it, so covering + // the whole bucket is what actually covers the percentile. + if (sum >= want) return Time.Milli(Math.min((i + 1) * BUCKET, this.#max)); + } + + return Time.Milli(Math.min(BUCKETS * BUCKET, this.#max)); + } +} diff --git a/js/moq-boy/src/game.ts b/js/moq-boy/src/game.ts index 9c881808c2b..1f3e7fc743a 100644 --- a/js/moq-boy/src/game.ts +++ b/js/moq-boy/src/game.ts @@ -136,14 +136,17 @@ export class Game { }); this.#signals.cleanup(() => this.audioSource.close()); - // The decoder owns rendition handoffs but also needs Sync. Bridge its jitter output through a - // local signal so Sync can be constructed first. + // The decoders own rendition handoffs and measure how late frames arrive, but they need Sync + // to exist first. Bridge their outputs through local signals. const videoJitter = new Moq.Signals.Signal(undefined); + const videoSpread = new Moq.Signals.Signal(undefined); + const audioSpread = new Moq.Signals.Signal(undefined); this.sync = new Watch.Sync({ delay: this.delay, - probe: connection.probe, video: videoJitter, audio: this.audioSource.out.jitter, + videoSpread, + audioSpread, }); this.#signals.cleanup(() => this.sync.close()); @@ -152,6 +155,7 @@ export class Game { const videoEnabled = new Moq.Signals.Signal(true); this.videoDecoder = new Watch.Video.Decoder(this.videoSource, this.sync, { enabled: videoEnabled }); this.#signals.proxy(videoJitter, this.videoDecoder.out.jitter); + this.#signals.proxy(videoSpread, this.videoDecoder.out.spread); this.#signals.cleanup(() => this.videoDecoder.close()); // Renderer needs a canvas created by the UI layer, set via `canvas`. @@ -165,6 +169,7 @@ export class Game { // Audio pipeline. The emitter stops the download when muted or paused. const audioEnabled = new Moq.Signals.Signal(false); this.audioDecoder = new Watch.Audio.Decoder(this.audioSource, this.sync, { enabled: audioEnabled }); + this.#signals.proxy(audioSpread, this.audioDecoder.out.spread); this.#signals.cleanup(() => this.audioDecoder.close()); const audioPaused = new Moq.Signals.Signal(true); diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index 4c3a0263e5d..504cd5ee14d 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -1,6 +1,6 @@ import { Time } from "@moq/net"; import { Effect, type Getter, Signal } from "@moq/signals"; -import type { Data, InitPost, InitShared, Latency, Reset, State, Truncate } from "./render"; +import type { Data, InitPost, InitShared, Latency, Reset, Stall, State, Truncate } from "./render"; import { allocSharedRingBuffer, SharedRingBuffer } from "./shared-ring-buffer"; /** @@ -21,7 +21,7 @@ class Backpressure { this.#headroom = headroom; } - // Move the gate as the floor changes (e.g. "real-time" jitter tracking RTT). + // Move the gate as the floor changes (e.g. "auto" tracking the measured arrival spread). setHeadroom(headroom: Time.Micro): void { this.#headroom = headroom; } @@ -77,6 +77,14 @@ export interface AudioBuffer { /** Flush buffered samples and re-stall, ready to anchor the next utterance (buffered mode). */ reset(): void; + /** + * Hold playback until the ring holds the target again, keeping everything buffered. + * + * Used when the target deepens: `setLatency` alone only raises the bar a future refill has to + * clear, so a ring already playing keeps draining at its old depth. + */ + stall(): void; + /** * Drop buffered samples at or after `timestamp`, keeping what is already due. * @@ -100,6 +108,9 @@ export interface AudioBuffer { /** Whether the buffer is stalled (waiting to fill). */ readonly stalled: Getter; + /** How many times the ring has run dry mid-playback, cumulative. */ + readonly underruns: Getter; + /** Release any resources (event listeners, intervals, etc.). */ close(): void; } @@ -148,6 +159,9 @@ class SharedAudioBuffer implements AudioBuffer { readonly #stalled = new Signal(true); readonly stalled: Getter = this.#stalled; + readonly #underruns = new Signal(0); + readonly underruns: Getter = this.#underruns; + #backpressure: Backpressure; #signals = new Effect(); @@ -174,6 +188,7 @@ class SharedAudioBuffer implements AudioBuffer { const stalled = this.#ring.stalled; this.#timestamp.set(this.#ring.timestamp); this.#stalled.set(stalled); + this.#underruns.set(this.#ring.underruns); // While stalled the playhead is parked, so release the decode loop to refill the floor; // once playing, hold it to ~the floor ahead. if (stalled) this.#backpressure.flush(); @@ -210,8 +225,16 @@ class SharedAudioBuffer implements AudioBuffer { this.#backpressure.flush(); // the old timeline is gone; let the decode loop re-anchor } + stall(): void { + this.#ring.stall(); + this.#backpressure.flush(); // let the decode loop fill the deeper target + } + wait(timestamp: Time.Micro): Promise { - // Stalled = still filling the floor (bootstrap or underflow): let frames through to refill. + // Stalled = still filling the floor (bootstrap, an underrun the reader re-stalled on, or an + // explicit reset): let frames through so the ring refills to the floor. This is the single + // re-buffer path; the ring un-stalls itself on the insert that reaches the floor, so the + // gate closes again within one frame rather than draining the whole lookahead. if (this.#ring.stalled) return Promise.resolve(); return this.#backpressure.wait(timestamp, this.#ring.timestamp); } @@ -234,6 +257,9 @@ class PostAudioBuffer implements AudioBuffer { readonly #stalled = new Signal(true); readonly stalled: Getter = this.#stalled; + readonly #underruns = new Signal(0); + readonly underruns: Getter = this.#underruns; + // Backpressure runs off the playhead the worklet reports in its state messages. #backpressure: Backpressure; @@ -256,6 +282,7 @@ class PostAudioBuffer implements AudioBuffer { if (data?.type === "state") { this.#timestamp.set(data.timestamp); this.#stalled.set(data.stalled); + this.#underruns.set(data.underruns); // While stalled the playhead is parked, so release the decode loop to refill the floor; // once playing, hold it to ~the floor ahead. if (data.stalled) this.#backpressure.flush(); @@ -295,8 +322,18 @@ class PostAudioBuffer implements AudioBuffer { this.#backpressure.flush(); // the old timeline is gone; let the decode loop re-anchor } + stall(): void { + const msg: Stall = { type: "stall" }; + this.#worklet.port.postMessage(msg); + // Mirror it locally rather than waiting for the worklet's next state message, so `wait()` + // releases the decode loop now. + this.#stalled.set(true); + this.#backpressure.flush(); // let the decode loop fill the deeper target + } + wait(timestamp: Time.Micro): Promise { - // Stalled = still filling the floor (bootstrap or underflow): let frames through to refill. + // Stalled = still filling the floor (bootstrap, an underrun the reader re-stalled on, or an + // explicit reset): let frames through so the ring refills to the floor. See SharedAudioBuffer. if (this.#stalled.peek()) return Promise.resolve(); // Uses the worklet-reported playhead, which lags by a state-message interval; the floor's // headroom covers that. The worklet still drops the oldest if a frame slips through. diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 91a93de9fd6..f7889d8309f 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -10,7 +10,7 @@ import { subscribeMedia } from "../media"; import type { Sync } from "../sync"; import { type AudioBuffer, createAudioBuffer } from "./buffer"; import { Handover } from "./handover"; -import { reanchorFloor, ringSamples } from "./latency"; +import { ringSamples } from "./latency"; // Compiled and inlined as a blob URL via vite-plugin-worklet. import RenderWorklet from "./render-worklet.ts?worklet"; import type { Source } from "./source"; @@ -45,8 +45,15 @@ type DecoderOutput = { // Whether the audio buffer is stalled (waiting to fill) stalled: Signal; + // How many times the ring ran dry mid-playback, so the UI can show that the target is too low. + underruns: Signal; + // Combined buffered ranges (network jitter + decode buffer) buffered: Signal; + + // How late audio frames arrive relative to the earliest one, measured by the container + // consumer. Wired into Sync by the parent, which sizes the "auto" delay from it. + spread: Signal; }; /** Cumulative audio statistics since the decoder started. */ @@ -72,7 +79,9 @@ export class Decoder { stats: new Signal(undefined), timestamp: new Signal(undefined), stalled: new Signal(true), + underruns: new Signal(0), buffered: new Signal([]), + spread: new Signal(undefined), }; readonly out = readonlys(this.#out); @@ -91,9 +100,9 @@ export class Decoder { // Ordered discontinuity and endpoint state from the container consumer. #terminal = new Terminal(); - // The latency floor as of the last settled change, to detect a floor *increase* (needs a deeper - // cushion) versus a decrease or a real-time RTT wiggle. See #runLatencyReanchor. - #prevFloor?: Time.Milli; + // The derived target as of the last settled change, to detect a *deepening* (which needs the + // ring to refill) versus a decrease. See #runLatencyReanchor. + #prevTarget?: Time.Milli; // Which subscription the ring's buffered samples came from. See #runDecoder. #handover = new Handover(); @@ -186,6 +195,9 @@ export class Decoder { effect.run((inner) => { this.#out.stalled.set(inner.get(ring.stalled)); }); + effect.run((inner) => { + this.#out.underruns.set(inner.get(ring.underruns)); + }); effect.set(this.#out.root, worklet); }); @@ -217,31 +229,31 @@ export class Decoder { ring.setLatency(ringSamples(ring.rate, delay)); } - // Re-anchor when the delay floor *increases*. A larger floor needs a deeper cushion: video - // rebuilds it implicitly (its per-frame sync.wait() reads the live buffer, so it just holds - // longer), but the audio ring keeps draining at its old depth -- resize() (via setLatency) only - // re-stalls an *empty* ring, so a mid-playback ring never refills to the new floor and audio runs - // ahead of video (the "raise latency, only video re-buffers" desync). reset() re-stalls the ring - // so it refills to the new floor. Watch the latency target and media delay, excluding adaptive - // RTT jitter, and debounce so a slider drag coalesces into one re-anchor. Decreases are left to - // natural catch-up. + // Park playback when the target *deepens*, so the ring refills to it. Video rebuilds a deeper + // cushion implicitly (its per-frame sync.wait() reads the live buffer, so it just holds longer), + // but the audio ring keeps draining at its old depth: setLatency only raises the bar a future + // refill has to clear, so a ring already playing never gets deeper and audio runs ahead of video + // (the "raise latency, only video re-buffers" desync). Stalling spends the deficit as silence, + // once, instead of leaving it to the underrun that the shallow buffer eventually causes anyway. + // + // The derived delay, not the user's setting, since the arrival estimator moves it too. Only a + // deepening worth more than a frame counts, so the estimator's small refinements ride through, + // and the debounce coalesces a slider drag or a converging estimate into a single stall. + // Decreases are left to natural catch-up. #runLatencyReanchor(effect: Effect): void { - const floor = reanchorFloor({ - delay: effect.get(this.sync.in.delay), - audio: effect.get(this.sync.in.audio), - video: effect.get(this.sync.in.video), - }); - if (this.#prevFloor === undefined) { + const target = effect.get(this.sync.out.delay); + const step = effect.get(this.source.out.jitter) ?? Time.Milli.zero; + if (this.#prevTarget === undefined) { // Startup: the initial fill already builds the cushion; just record the baseline. - this.#prevFloor = floor; + this.#prevTarget = target; return; } - // When the timer fires, the floor read above is still current: any change would have rerun + // When the timer fires, the target read above is still current: any change would have rerun // this effect (tearing down the timer), so compare it against the pre-change baseline directly. - const baseline = this.#prevFloor; + const baseline = this.#prevTarget; effect.timer(() => { - if (floor > baseline) this.reset(); - this.#prevFloor = floor; + if (target - baseline > step) this.#ring?.stall(); + this.#prevTarget = target; }, LATENCY_REANCHOR_DEBOUNCE_MS); } @@ -304,6 +316,11 @@ export class Decoder { this.#out.buffered.update(() => Container.mergeBufferedRanges(network, decode)); }); + // Publish the measured arrival spread for Sync. Cleared on teardown so a departed track + // stops holding the buffer open. + effect.run((inner) => this.#out.spread.set(inner.get(consumer.spread))); + effect.cleanup(() => this.#out.spread.set(undefined)); + effect.spawn(async () => { const loaded = await Util.Libav.polyfill(); if (!loaded) return; // cancelled @@ -409,6 +426,11 @@ export class Decoder { this.#out.buffered.update(() => Container.mergeBufferedRanges(network, decode)); }); + // Publish the measured arrival spread for Sync. Cleared on teardown so a departed track + // stops holding the buffer open. + effect.run((inner) => this.#out.spread.set(inner.get(consumer.spread))); + effect.cleanup(() => this.#out.spread.set(undefined)); + effect.spawn(async () => { const loaded = await Util.Libav.polyfill(); if (!loaded) return; // cancelled diff --git a/js/watch/src/audio/latency.test.ts b/js/watch/src/audio/latency.test.ts index 39cade4ba05..e6918210456 100644 --- a/js/watch/src/audio/latency.test.ts +++ b/js/watch/src/audio/latency.test.ts @@ -1,20 +1,9 @@ import { describe, expect, it } from "bun:test"; import type { Time } from "@moq/net"; -import { reanchorFloor, ringSamples } from "./latency"; +import { ringSamples } from "./latency"; const ms = (value: number) => value as Time.Milli; -describe("reanchorFloor", () => { - it("includes the fixed delay and the largest media delay", () => { - expect(reanchorFloor({ delay: ms(100), audio: ms(20), video: ms(80) })).toBe(ms(180)); - }); - - it("tracks rendition delay without adaptive RTT jitter", () => { - expect(reanchorFloor({ delay: "auto", audio: ms(20), video: ms(80) })).toBe(ms(80)); - expect(reanchorFloor({ delay: "auto", audio: ms(20), video: ms(200) })).toBe(ms(200)); - }); -}); - describe("ringSamples", () => { // `delay="instant"` reports a zero buffer. Passed through, the ring rejects it and the // worklet is left with no backend that any later resize can revive. diff --git a/js/watch/src/audio/latency.ts b/js/watch/src/audio/latency.ts index e45485bdc8d..1885d28fd9f 100644 --- a/js/watch/src/audio/latency.ts +++ b/js/watch/src/audio/latency.ts @@ -1,26 +1,4 @@ import { Time } from "@moq/net"; -import type { Delay } from "../sync"; - -/** The inputs that determine whether audio needs a deeper playback cushion. */ -export interface ReanchorFloor { - /** How far playback trails the live edge. */ - delay: Delay; - - /** Additional delay required by the selected audio rendition. */ - audio?: Time.Milli; - - /** Additional delay required by the active or pending video rendition. */ - video?: Time.Milli; -} - -/** The stable delay floor whose increase requires the audio ring to refill. */ -export function reanchorFloor(props: ReanchorFloor): Time.Milli { - // "auto" and "instant" contribute nothing: the adaptive RTT component is deliberately excluded - // so an RTT wiggle doesn't re-anchor, and "instant" holds nothing at all. - const target = typeof props.delay === "number" ? props.delay : Time.Milli.zero; - const media = Time.Milli.max(props.audio ?? Time.Milli.zero, props.video ?? Time.Milli.zero); - return Time.Milli.add(target, media); -} // An AudioWorkletProcessor renders in fixed 128-sample quanta, so a ring shallower than one can // never be read from. diff --git a/js/watch/src/audio/render-worklet.ts b/js/watch/src/audio/render-worklet.ts index 723dfcfa993..313cb67f311 100644 --- a/js/watch/src/audio/render-worklet.ts +++ b/js/watch/src/audio/render-worklet.ts @@ -2,11 +2,17 @@ import type { Message, State } from "./render"; import { AudioRingBuffer } from "./ring-buffer"; import { SharedRingBuffer } from "./shared-ring-buffer"; +// Samples to fade over when playback stops or resumes short of a full quantum. A shortfall +// otherwise steps straight to silence and back, which is an audible click on every underrun. +const RAMP = 64; + class Render extends AudioWorkletProcessor { // Set after init, depending on which path the main thread chose. #backend?: SharedRingBuffer | AudioRingBuffer; #underflow = 0; #stateCounter = 0; + // Whether the previous quantum ended short, so the next one fades back in. + #short = false; constructor() { super(); @@ -18,10 +24,12 @@ class Render extends AudioWorkletProcessor { const previous = this.#backend instanceof SharedRingBuffer ? this.#backend : undefined; this.#backend = new SharedRingBuffer(msg, previous); this.#underflow = 0; + this.#short = false; } else if (msg.type === "init-post") { console.log("[audio-worklet] init-post: using postMessage path"); this.#backend = new AudioRingBuffer(msg); this.#underflow = 0; + this.#short = false; } else if (msg.type === "data") { // Only meaningful in post mode. if (this.#backend instanceof AudioRingBuffer) this.#backend.write(msg.timestamp, msg.data); @@ -34,6 +42,9 @@ class Render extends AudioWorkletProcessor { } else if (msg.type === "reset") { // Only meaningful in post mode; shared mode resets via the control array. if (this.#backend instanceof AudioRingBuffer) this.#backend.reset(); + } else if (msg.type === "stall") { + // Only meaningful in post mode; shared mode stalls via the control array. + if (this.#backend instanceof AudioRingBuffer) this.#backend.stall(); } }; } @@ -44,10 +55,29 @@ class Render extends AudioWorkletProcessor { const samplesRead = backend?.read(output) ?? 0; if (samplesRead < output[0].length) { + // Fade the tail of what we did read down to the silence that follows it. + for (const channel of output) { + const ramp = Math.min(RAMP, samplesRead); + for (let i = 0; i < ramp; i++) { + channel[samplesRead - ramp + i] *= 1 - (i + 1) / ramp; + } + } this.#underflow += output[0].length - samplesRead; - } else if (this.#underflow > 0 && backend) { - console.debug(`audio underflow: ${Math.round((1000 * this.#underflow) / backend.rate)}ms`); - this.#underflow = 0; + this.#short = true; + } else { + if (this.#short) { + // Fade back in from the silence the shortfall left behind. + for (const channel of output) { + for (let i = 0; i < Math.min(RAMP, samplesRead); i++) { + channel[i] *= (i + 1) / RAMP; + } + } + this.#short = false; + } + if (this.#underflow > 0 && backend) { + console.debug(`audio underflow: ${Math.round((1000 * this.#underflow) / backend.rate)}ms`); + this.#underflow = 0; + } } // In post mode the main thread can't read worklet state directly, so we @@ -61,6 +91,7 @@ class Render extends AudioWorkletProcessor { type: "state", timestamp: backend.timestamp, stalled: backend.stalled, + underruns: backend.underruns, }; this.port.postMessage(state); } diff --git a/js/watch/src/audio/render.ts b/js/watch/src/audio/render.ts index e0c82524687..a537ac1bbc2 100644 --- a/js/watch/src/audio/render.ts +++ b/js/watch/src/audio/render.ts @@ -2,7 +2,7 @@ import type { Time } from "@moq/net"; import type { SharedRingBufferInit } from "./shared-ring-buffer"; /** Everything the main thread sends the render worklet over its port. */ -export type Message = InitShared | InitPost | Data | Latency | Reset | Truncate; +export type Message = InitShared | InitPost | Data | Latency | Reset | Stall | Truncate; export type ToMain = State; /** Init message when SharedArrayBuffer is available. */ @@ -26,6 +26,14 @@ export interface Reset { type: "reset"; } +/** + * Hold playback until the buffer holds the target again, keeping what is buffered (fallback path + * only; the shared path stalls via Atomics). + */ +export interface Stall { + type: "stall"; +} + /** * Drop buffered samples at or after `timestamp`, keeping what is already due (fallback path only; * the shared path truncates via Atomics). @@ -53,4 +61,6 @@ export interface State { type: "state"; timestamp: Time.Micro; stalled: boolean; + // How many times the ring has run dry mid-playback, cumulative. + underruns: number; } diff --git a/js/watch/src/audio/replay.test.ts b/js/watch/src/audio/replay.test.ts new file mode 100644 index 00000000000..617a57194a7 --- /dev/null +++ b/js/watch/src/audio/replay.test.ts @@ -0,0 +1,183 @@ +import { describe, expect, it } from "bun:test"; +import { Time } from "@moq/net"; +import { AudioRingBuffer } from "./ring-buffer"; +import { allocSharedRingBuffer, SharedRingBuffer } from "./shared-ring-buffer"; + +// Replay recorded-shape arrival traces through both rings and count what a listener would hear: +// quanta rendered short (an underrun) and samples the ring threw away (a skip). Both rings are +// driven through the same harness, since the postMessage fallback is the path every page without +// cross-origin isolation takes. + +const RATE = 48000; +const QUANTUM = 128; // an AudioWorklet render quantum +const CHUNK_MS = 20; // one Opus frame +const CHUNK = (RATE * CHUNK_MS) / 1000; +// What the catalog advertises: one frame plus the render quantum. Sync holds this as a floor under +// the measured target. +const FLOOR = CHUNK_MS + Math.ceil((QUANTUM / RATE) * 1000); + +/** Deterministic PRNG, so a trace is the same on every machine. */ +function rng(seed: number): () => number { + let a = seed >>> 0; + return () => { + a = (a + 0x6d2b79f5) >>> 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +/** One frame of a trace: when it was captured, and the wall time it reached us. */ +interface Arrival { + media: number; + arrival: number; +} + +/** + * A sender emitting a frame every `CHUNK_MS`, holding `burst - 1` of them back and flushing the + * run at once, plus `spread` ms of per-flush network jitter. `burst` of 1 is an evenly paced + * sender; anything above it is the PES packing an importer produces. + */ +function trace(frames: number, burst: number, spread: number, seed = 7): Arrival[] { + const rand = rng(seed); + const out: Arrival[] = []; + let held: number[] = []; + for (let i = 0; i < frames; i++) { + const media = i * CHUNK_MS; + held.push(media); + if (held.length < burst) continue; + // Every frame keeps its own capture time; the whole run shares one arrival. + const arrival = media + 50 + rand() * spread; + for (const m of held) out.push({ media: m, arrival }); + held = []; + } + return out; +} + +/** + * What the estimator converges on for a trace: the p95 arrival spread plus a frame of cushion, + * held above the catalog floor. + */ +function target(t: Arrival[]): number { + let min = Number.POSITIVE_INFINITY; + for (const { media, arrival } of t) min = Math.min(min, arrival - media); + const spreads = t.map(({ media, arrival }) => arrival - media - min).sort((a, b) => a - b); + const p95 = Math.ceil(spreads[Math.floor(spreads.length * 0.95)]); + return Math.max(FLOOR, p95 + CHUNK_MS); +} + +/** The two rings behind one interface, since the harness drives them identically. */ +interface Ring { + insert(timestamp: Time.Micro, data: Float32Array[]): void; + read(output: Float32Array[]): number; + readonly length: number; +} + +function shared(latencyMs: number): Ring { + const samples = Math.ceil((RATE * latencyMs) / 1000); + const ring = new SharedRingBuffer(allocSharedRingBuffer(1, Math.max(RATE, samples * 2), RATE)); + ring.setLatency(samples); + return ring; +} + +function post(latencyMs: number): Ring { + const ring = new AudioRingBuffer({ rate: RATE, channels: 1, latency: latencyMs as Time.Milli }); + return { + insert: (timestamp, data) => ring.write(timestamp, data), + read: (output) => ring.read(output), + get length() { + return ring.length; + }, + }; +} + +interface Result { + /** Quanta rendered short of a full block after the warmup, i.e. audible underruns. */ + underruns: number; + /** Samples inserted after the warmup that were never played: skipped or overflow-dropped. */ + skipped: number; +} + +/** + * Play `t` through `ring` in real time: insert every frame the moment it arrives, and pull one + * render quantum every quantum's worth of wall time, which is what the AudioWorklet does. + */ +function replay(ring: Ring, t: Arrival[], warmupMs: number): Result { + const step = (QUANTUM / RATE) * 1000; + const output = [new Float32Array(QUANTUM)]; + const end = t[t.length - 1].arrival; + + let next = 0; + let played = 0; + let inserted = 0; + let short = 0; + let started = false; + let mark: { played: number; inserted: number; buffered: number; short: number } | undefined; + + for (let now = 0; now < end; now += step) { + while (next < t.length && t[next].arrival <= now) { + ring.insert(Time.Micro.fromMilli(t[next].media as Time.Milli), [new Float32Array(CHUNK).fill(0.5)]); + inserted += CHUNK; + next++; + } + + const count = ring.read(output); + played += count; + if (started && count < QUANTUM) short++; + if (count > 0) started = true; + + if (now >= warmupMs && !mark) mark = { played, inserted, buffered: ring.length, short }; + } + + const from = mark ?? { played, inserted, buffered: ring.length, short }; + return { + underruns: short - from.short, + skipped: inserted - from.inserted - (played - from.played) - (ring.length - from.buffered), + }; +} + +const RINGS: Array<[string, (latencyMs: number) => Ring]> = [ + ["shared", shared], + ["post", post], +]; + +describe.each(RINGS)("%s ring replay", (_name, build) => { + it("plays an evenly paced sender without underruns or skips", () => { + const t = trace(600, 1, 10); + expect(replay(build(target(t)), t, 2000)).toEqual({ underruns: 0, skipped: 0 }); + }); + + it("plays a bursty sender at the measured target", () => { + // Three frames flushed at once, which is what an importer packing PES payloads produces. + const t = trace(600, 3, 5); + expect(target(t)).toBeGreaterThan(2 * CHUNK_MS); + const result = replay(build(target(t)), t, 2000); + expect(result.underruns).toBeLessThanOrEqual(2); + expect(result.skipped).toBe(0); + }); + + it("keeps a five frame flush span playing", () => { + // A hundred milliseconds of arrivals landing at once. The target covers the trough and the + // slack above it absorbs the peak, so the flush plays instead of being cut down to the + // target on arrival. + // + // A 95th percentile target sits by construction at the edge of what arrives, so the tail + // beyond it can still land a couple of times across the ten second window; closing that + // without a deeper buffer is what the time-stretch quest is for. Two orders of magnitude + // below the round-trip target below, and the postMessage ring bounds its refill at capacity + // rather than in read(), so it can also land one flush past the band. + const t = trace(600, 5, 5); + const result = replay(build(target(t)), t, 2000); + expect(result.underruns).toBeLessThanOrEqual(2); + expect(result.skipped).toBeLessThanOrEqual(2 * CHUNK); + }); + + it("underruns constantly when the target ignores the arrival spread", () => { + // 46ms is what the round-trip formula produced on the connection this was measured on: it + // describes the network and says nothing about a sender that flushes five frames at once. + const t = trace(600, 5, 5); + const result = replay(build(46), t, 2000); + expect(result.underruns).toBeGreaterThan(100); + expect(result.skipped).toBeGreaterThan(10 * CHUNK); + }); +}); diff --git a/js/watch/src/audio/ring-buffer.test.ts b/js/watch/src/audio/ring-buffer.test.ts index 5d1fb2ceba0..86188d5973e 100644 --- a/js/watch/src/audio/ring-buffer.test.ts +++ b/js/watch/src/audio/ring-buffer.test.ts @@ -35,7 +35,9 @@ describe("initialization", () => { it("should initialize with valid parameters", () => { const buffer = new AudioRingBuffer({ rate: 48000, channels: 2, latency: 100 as Time.Milli }); - expect(buffer.capacity).toBe(4800); // 48000 * 0.1 + // Capacity is twice the target: the ring has to be able to hold the slack the overflow band + // tolerates, or every chunk arriving on a full ring would drop the oldest samples. + expect(buffer.capacity).toBe(9600); // 48000 * 0.1 * 2 expect(buffer.length).toBe(0); }); @@ -281,9 +283,10 @@ describe("stall behavior", () => { describe("ring buffer wrapping", () => { it("should wrap around when buffer is full", () => { + // 100 sample target, so 200 samples of capacity. const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Time.Milli }); - // Fill the buffer + // Fill to the target write(buffer, 0 as Time.Milli, 100, { channels: 1, value: 1.0 }); expect(buffer.stalled).toBe(false); @@ -297,22 +300,23 @@ describe("ring buffer wrapping", () => { // Now we have 100 samples available (50-149) expect(buffer.length).toBe(100); - // Write 50 more samples at timestamp 150, this will wrap around + // One chunk above the target is tolerated rather than dropped (50-199). write(buffer, 150 as Time.Milli, 50, { channels: 1, value: 3.0 }); + expect(buffer.length).toBe(150); - // Should still have 100 samples (buffer is at capacity) + // Past the band: drop back to the target, wrapping abs 200-249 onto slots 0-49. + write(buffer, 200 as Time.Milli, 50, { channels: 1, value: 4.0 }); expect(buffer.length).toBe(100); - // Read all 100 samples + // Read all 100 samples: abs 150-199 = 3.0, abs 200-249 = 4.0. const output2 = read(buffer, 100, 1); expect(output2[0].length).toBe(100); - // First 50 should be 2.0, next 50 should be 3.0 for (let i = 0; i < 50; i++) { - expect(output2[0][i]).toBe(2.0); + expect(output2[0][i]).toBe(3.0); } for (let i = 50; i < 100; i++) { - expect(output2[0][i]).toBe(3.0); + expect(output2[0][i]).toBe(4.0); } }); }); @@ -390,7 +394,7 @@ describe("edge cases", () => { describe("resize", () => { it("should resize to a larger buffer", () => { const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Time.Milli }); - expect(buffer.capacity).toBe(100); + expect(buffer.capacity).toBe(200); // Write 50 samples write(buffer, 0 as Time.Milli, 50, { channels: 1, value: 1.0 }); @@ -399,27 +403,27 @@ describe("resize", () => { // Resize to larger buffer (200ms = 200 samples) buffer.resize(200 as Time.Milli); - expect(buffer.capacity).toBe(200); + expect(buffer.capacity).toBe(400); expect(buffer.length).toBe(50); // Samples preserved expect(buffer.stalled).toBe(true); // Should trigger stall }); it("should resize to a smaller buffer and keep the most recent samples", () => { const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Time.Milli }); - expect(buffer.capacity).toBe(100); + expect(buffer.capacity).toBe(200); // Write 80 samples: first 40 with value 1.0, next 40 with value 2.0 write(buffer, 0 as Time.Milli, 40, { channels: 1, value: 1.0 }); write(buffer, 40 as Time.Milli, 40, { channels: 1, value: 2.0 }); expect(buffer.length).toBe(80); - // Resize to smaller buffer (50ms = 50 samples) - // Should keep the most recent 50 samples (samples 30-79) - buffer.resize(50 as Time.Milli); + // Resize to a 30 sample target (60 samples of capacity). + // Should keep the most recent 60 samples (samples 20-79) + buffer.resize(30 as Time.Milli); - expect(buffer.capacity).toBe(50); - expect(buffer.length).toBe(50); // Truncated to new capacity - expect(buffer.stalled).toBe(true); // Should trigger stall + expect(buffer.capacity).toBe(60); + expect(buffer.length).toBe(60); // Truncated to new capacity + expect(buffer.stalled).toBe(true); // Never reached the old 100 sample target }); it("should be a no-op when capacity is unchanged", () => { @@ -434,7 +438,7 @@ describe("resize", () => { // Should still not be stalled (no-op) expect(buffer.stalled).toBe(false); - expect(buffer.capacity).toBe(100); + expect(buffer.capacity).toBe(200); }); it("should throw on zero latency", () => { @@ -456,8 +460,8 @@ describe("resize", () => { // Resize to smaller buffer buffer.resize(50 as Time.Milli); - expect(buffer.capacity).toBe(50); - expect(buffer.length).toBe(50); + expect(buffer.capacity).toBe(100); + expect(buffer.length).toBe(60); expect(buffer.stalled).toBe(true); }); @@ -468,7 +472,7 @@ describe("resize", () => { // Resize empty buffer buffer.resize(200 as Time.Milli); - expect(buffer.capacity).toBe(200); + expect(buffer.capacity).toBe(400); expect(buffer.length).toBe(0); expect(buffer.stalled).toBe(true); }); @@ -488,10 +492,10 @@ describe("resize", () => { write(buffer, 100 as Time.Milli, 30, { channels: 1, value: 2.0 }); expect(buffer.length).toBe(70); // 130 - 60 - // Resize to 50 samples - should keep most recent 50 (samples 80-129) + // Resize to a 50 sample target - 100 of capacity, so all 70 survive (samples 60-129) buffer.resize(50 as Time.Milli); - expect(buffer.capacity).toBe(50); - expect(buffer.length).toBe(50); + expect(buffer.capacity).toBe(100); + expect(buffer.length).toBe(70); expect(buffer.stalled).toBe(false); }); @@ -505,22 +509,26 @@ describe("resize", () => { buffer.resize(50 as Time.Milli); expect(buffer.stalled).toBe(true); - // Write new data to fill the buffer and exit stall - // The overflow will discard preserved samples and advance readIndex + // Write new data to reach the new target and exit stall. Nothing is dropped: the ring has + // room for the preserved samples plus a chunk above the target. write(buffer, 50 as Time.Milli, 50, { channels: 1, value: 2.0 }); expect(buffer.stalled).toBe(false); + expect(buffer.length).toBe(100); - // Read should return the new data (value 2.0) - const output = read(buffer, 50, 1); - expect(output[0].length).toBe(50); + // Read plays the preserved samples first, then the new ones. + const output = read(buffer, 100, 1); + expect(output[0].length).toBe(100); for (let i = 0; i < 50; i++) { + expect(output[0][i]).toBe(1.0); + } + for (let i = 50; i < 100; i++) { expect(output[0][i]).toBe(2.0); } }); it("should preserve samples correctly when resizing larger then filling", () => { const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 50 as Time.Milli }); - expect(buffer.capacity).toBe(50); + expect(buffer.capacity).toBe(100); // Write 30 samples write(buffer, 0 as Time.Milli, 30, { channels: 1, value: 1.0 }); @@ -528,7 +536,7 @@ describe("resize", () => { // Resize to larger buffer (100ms = 100 samples) buffer.resize(100 as Time.Milli); - expect(buffer.capacity).toBe(100); + expect(buffer.capacity).toBe(200); expect(buffer.length).toBe(30); // All samples preserved expect(buffer.stalled).toBe(true); @@ -569,26 +577,26 @@ describe("resize", () => { // abs 110-119 → slots 10-19, value 3.0. write(buffer, 110 as Time.Milli, 10, { channels: 1, value: 3.0 }); - // Preserved range after resize: most-recent 50 samples = abs 70..119. - // copyStart = 120 - 50 = 70, and 70 % 50 = 20 (non-zero → triggers the bug). + // Preserved range after resize: all 100 buffered samples = abs 20..119. + // copyStart = 120 - 100 = 20, and 20 % 100 = 20 (non-zero → triggers the bug). buffer.resize(50 as Time.Milli); - expect(buffer.capacity).toBe(50); - expect(buffer.length).toBe(50); + expect(buffer.capacity).toBe(100); + expect(buffer.length).toBe(100); expect(buffer.stalled).toBe(false); // Read the preserved samples. Expected layout by absolute index: - // abs 70..99 = 1.0 (30 samples, from the initial fill) + // abs 20..99 = 1.0 (80 samples, from the initial fill) // abs 100..109 = 2.0 (10 samples) // abs 110..119 = 3.0 (10 samples) - const output = read(buffer, 50, 1); - expect(output[0].length).toBe(50); - for (let i = 0; i < 30; i++) { + const output = read(buffer, 100, 1); + expect(output[0].length).toBe(100); + for (let i = 0; i < 80; i++) { expect(output[0][i]).toBe(1.0); } - for (let i = 30; i < 40; i++) { + for (let i = 80; i < 90; i++) { expect(output[0][i]).toBe(2.0); } - for (let i = 40; i < 50; i++) { + for (let i = 90; i < 100; i++) { expect(output[0][i]).toBe(3.0); } }); @@ -607,16 +615,16 @@ describe("resize", () => { // Wrap the writer over slots 0-29 with value 2.0 (abs 100-129). write(buffer, 100 as Time.Milli, 30, { channels: 1, value: 2.0 }); - // Resize smaller. Preserved = last 50 samples = abs 80..129. + // Resize smaller. All 90 buffered samples fit the new 100 sample capacity: abs 40..129. buffer.resize(50 as Time.Milli); - expect(buffer.length).toBe(50); + expect(buffer.length).toBe(90); expect(buffer.stalled).toBe(false); // Write the next 10 samples at their real timestamp. This should append, // not create a gap or be discarded as "too old". write(buffer, 130 as Time.Milli, 10, { channels: 1, value: 3.0 }); - // Buffer capacity is 50, so the oldest 10 samples (abs 80..89) drop out. + // That put the ring past the band, so it drops back to the 50 sample target. // Remaining: abs 90..99 = 1.0 (10), abs 100..129 = 2.0 (30), abs 130..139 = 3.0 (10). expect(buffer.length).toBe(50); @@ -667,7 +675,7 @@ describe("live anchoring", () => { // with zeros, un-stalled on that overflow, and left the playhead a floor before the frame. it("anchors a live ring to the first frame instead of gap-filling from zero", () => { const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Time.Milli }); - expect(buffer.capacity).toBe(100); + expect(buffer.capacity).toBe(200); write(buffer, 2000 as Time.Milli, 40, { channels: 1, value: 0.5 }); // Still short of the floor, so nothing plays yet and the ring holds only real samples. @@ -685,6 +693,70 @@ describe("live anchoring", () => { }); }); +describe("re-buffer", () => { + it("re-stalls and refills to the target after running dry", () => { + const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 40 as Time.Milli }); + + write(buffer, 0 as Time.Milli, 20, { channels: 1, value: 1.0 }); + write(buffer, 20 as Time.Milli, 20, { channels: 1, value: 1.0 }); + expect(buffer.stalled).toBe(false); + + // Drain it. + expect(read(buffer, 40, 1)[0].length).toBe(40); + expect(buffer.length).toBe(0); + + // The next read finds nothing and parks playback rather than handing the worklet a quantum + // of silence and trying again on the next one. + expect(read(buffer, 20, 1)[0].length).toBe(0); + expect(buffer.stalled).toBe(true); + expect(buffer.underruns).toBe(1); + + // A single chunk is not the target, so playback stays parked. + write(buffer, 40 as Time.Milli, 20, { channels: 1, value: 2.0 }); + expect(buffer.stalled).toBe(true); + expect(read(buffer, 20, 1)[0].length).toBe(0); + + // Reaching the target resumes it, and nothing buffered in the meantime was lost. + write(buffer, 60 as Time.Milli, 20, { channels: 1, value: 2.0 }); + expect(buffer.stalled).toBe(false); + expect(read(buffer, 40, 1)[0].length).toBe(40); + }); + + it("stall() parks playback without discarding what is buffered", () => { + const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 40 as Time.Milli }); + + write(buffer, 0 as Time.Milli, 20, { channels: 1, value: 1.0 }); + write(buffer, 20 as Time.Milli, 20, { channels: 1, value: 1.0 }); + expect(read(buffer, 20, 1)[0].length).toBe(20); + + // The target deepens, so playback parks until the ring holds the new depth. + buffer.resize(60 as Time.Milli); + buffer.stall(); + expect(buffer.stalled).toBe(true); + expect(read(buffer, 20, 1)[0].length).toBe(0); + expect(buffer.length).toBe(20); // nothing was thrown away, unlike reset() + + write(buffer, 40 as Time.Milli, 20, { channels: 1, value: 2.0 }); + expect(buffer.stalled).toBe(true); // 40 buffered, target is 60 + + write(buffer, 60 as Time.Milli, 20, { channels: 1, value: 2.0 }); + expect(buffer.stalled).toBe(false); + + // Playback resumes on the same timeline: the samples buffered before the stall play first. + const output = read(buffer, 20, 1); + expect(output[0].length).toBe(20); + expect(output[0][0]).toBe(1.0); + expect(buffer.underruns).toBe(0); + }); + + it("does not count an underrun while parked", () => { + const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 40 as Time.Milli }); + read(buffer, 20, 1); + read(buffer, 20, 1); + expect(buffer.underruns).toBe(0); + }); +}); + describe("buffered mode", () => { function createBuffered(latency: number) { return new AudioRingBuffer({ rate: 1000, channels: 1, latency: latency as Time.Milli, buffered: true }); diff --git a/js/watch/src/audio/ring-buffer.ts b/js/watch/src/audio/ring-buffer.ts index fb94f036928..58660435428 100644 --- a/js/watch/src/audio/ring-buffer.ts +++ b/js/watch/src/audio/ring-buffer.ts @@ -8,6 +8,13 @@ export class AudioRingBuffer { readonly rate: number; readonly channels: number; #stalled = true; + #underruns = 0; + + // Samples in the most recent write, i.e. one decoded chunk. The overflow band tolerates this + // much above the target before dropping, so a ring sitting exactly on the target doesn't drop + // audio the moment the next chunk lands. The most recent write rather than a running maximum: + // one oversized decode would otherwise widen the band for the life of the ring. + #chunk = 0; // Buffered mode: play through everything buffered without skipping ahead. readonly #buffered: boolean; @@ -33,9 +40,11 @@ export class AudioRingBuffer { this.channels = props.channels; this.#buffered = props.buffered ?? false; - // The ring holds the latency floor as PCM. Buffered mode gives it headroom above the floor - // so the backpressure-paced decode loop (on the main thread) doesn't overflow-drop; the rest - // of the lookahead stays encoded upstream. + // The ring holds the latency floor as PCM, with headroom above it. Sizing capacity to the + // floor exactly makes the ring physically incapable of holding the slack the overflow band + // wants, so every chunk that arrives while the ring is on target drops the oldest samples. + // In buffered mode the headroom is also what keeps the backpressure-paced decode loop (on + // the main thread) from overflow-dropping; the rest of the lookahead stays encoded upstream. const capacity = this.#capacityFor(this.#latencySamples); this.#buffer = []; @@ -45,13 +54,18 @@ export class AudioRingBuffer { } #capacityFor(latencySamples: number): number { - return this.#buffered ? latencySamples * 2 : latencySamples; + return latencySamples * 2; } get stalled(): boolean { return this.#stalled; } + /** How many times the reader has run dry mid-playback. */ + get underruns(): number { + return this.#underruns; + } + get timestamp(): Time.Micro { return Time.Micro.fromSecond((this.#readIndex / this.rate) as Time.Second); } @@ -127,13 +141,18 @@ export class AudioRingBuffer { } const end = start + samples; - - // Check if we need to discard old samples to prevent overflow - const overflow = end - this.#readIndex - this.#buffer[0].length; - if (overflow >= 0) { - // Discard old samples and exit stalled mode - this.#stalled = false; - this.#readIndex += overflow; + this.#chunk = data[0].length; + + // Bound the ring. While playing, drop the oldest once it holds a whole chunk more than the + // target and land back on the target: frames arrive one chunk at a time, so a ring sitting + // exactly on the target is a chunk above it the moment the next one lands, and dropping on + // that overshoot discards audio on every single write. While stalled the reader is not + // consuming, so only the hard capacity applies; the band would throw away the very audio the + // refill is accumulating. Buffered mode plays through everything, so it is capacity-bound too. + const playing = !this.#stalled && !this.#buffered; + const limit = playing ? Math.min(this.#latencySamples + this.#chunk, this.capacity) : this.capacity; + if (end - this.#readIndex > limit) { + this.#readIndex = end - (playing ? Math.min(this.#latencySamples, this.capacity) : this.capacity); } // Fill gaps with zeros if there's a discontinuity @@ -171,9 +190,10 @@ export class AudioRingBuffer { this.#writeIndex = end; } - // Start playback once we've buffered the latency target. In buffered mode the cap - // is large, so we usually un-stall here rather than via the overflow path above. - if (this.#buffered && this.length >= this.#latencySamples) { + // Start playback once we've buffered the latency target. This is the only way out of a + // stall, so an underrun mid-playback refills to the target before resuming rather than + // playing the next chunk on an empty cushion. + if (this.length >= this.#latencySamples) { this.#stalled = false; } } @@ -191,6 +211,18 @@ export class AudioRingBuffer { this.#writeIndex = Math.max(target, this.#readIndex); } + /** + * Hold playback until the ring holds the target again, keeping everything buffered. + * + * Used when the target deepens: `resize` alone only raises the bar a future refill has to clear, + * so a ring already playing keeps draining at its old depth and audio runs that much ahead of + * video. Parking the playhead spends exactly the deficit as silence and resumes on the same + * timeline, where `reset` would throw the buffer away and re-anchor. + */ + stall(): void { + this.#stalled = true; + } + // Flush all buffered samples and re-stall, ready to anchor the next utterance. reset(): void { this.#readIndex = 0; @@ -204,7 +236,12 @@ export class AudioRingBuffer { if (this.#stalled) return 0; const samples = Math.min(this.#writeIndex - this.#readIndex, output[0].length); - if (samples === 0) return 0; + if (samples <= 0) { + // Ran dry mid-playback: re-stall so write() refills to the target before resuming. + this.#stalled = true; + this.#underruns++; + return 0; + } for (let channel = 0; channel < this.channels; channel++) { const dst = output[channel]; diff --git a/js/watch/src/audio/shared-ring-buffer.test.ts b/js/watch/src/audio/shared-ring-buffer.test.ts index 2647ea3cc7e..9ee8f3ac2bf 100644 --- a/js/watch/src/audio/shared-ring-buffer.test.ts +++ b/js/watch/src/audio/shared-ring-buffer.test.ts @@ -32,6 +32,24 @@ function insert( buffer.insert(Time.Micro.fromMilli(timestampMs as Time.Milli), data); } +/** + * Insert `samples` as a run of `chunk`-sized inserts starting at `timestampMs`. + * + * The skip band is one chunk wide, so a test that wants a skip has to arrive the way a decoder + * does: many small chunks, not one big one. The rate is 1000 in these tests, so a sample is a ms. + */ +function insertChunks( + buffer: SharedRingBuffer, + timestampMs: number, + samples: number, + chunk: number, + opts?: { channels?: number; value?: number }, +): void { + for (let i = 0; i < samples; i += chunk) { + insert(buffer, timestampMs + i, Math.min(chunk, samples - i), opts); + } +} + function read(buffer: SharedRingBuffer, samples: number, channelCount?: number): Float32Array[] { const ch = channelCount ?? buffer.channels; const output: Float32Array[] = []; @@ -52,7 +70,7 @@ describe("initialization", () => { expect(init.capacity).toBe(128); expect(init.rate).toBe(1000); expect(init.samples.byteLength).toBe(2 * 128 * 4); // 2 channels * 128 samples * Float32 - expect(init.control.byteLength).toBe(4 * 4); // 4 control slots * Int32 + expect(init.control.byteLength).toBe(6 * 4); // 6 control slots * Int32 expect(init.state.byteLength).toBe(8); // packed epoch + read cursor }); @@ -407,11 +425,11 @@ describe("overflow", () => { }); describe("latency skip", () => { - it("should skip READ when buffered exceeds LATENCY", () => { + it("should skip READ when buffered exceeds LATENCY plus a chunk", () => { const buffer = create({ rate: 1000, channels: 1, capacity: 100, latency: 20 }); - // Fill 60 samples — exceeds LATENCY of 20 - insert(buffer, 0, 60, { channels: 1, value: 1.0 }); + // Fill 60 samples in 10-sample chunks — a whole chunk past the 20 sample LATENCY. + insertChunks(buffer, 0, 60, 10, { channels: 1, value: 1.0 }); expect(buffer.stalled).toBe(false); // Read should skip ahead to maintain LATENCY distance from WRITE @@ -421,6 +439,18 @@ describe("latency skip", () => { expect(output[0].length).toBe(20); }); + it("should tolerate one chunk above LATENCY", () => { + // A ring sitting on the target is a chunk above it the moment the next chunk lands, so + // skipping on that overshoot would discard audio on every single insert. + const buffer = create({ rate: 1000, channels: 1, capacity: 100, latency: 20 }); + + insertChunks(buffer, 0, 40, 20, { channels: 1, value: 1.0 }); + expect(buffer.stalled).toBe(false); + + const output = read(buffer, 128, 1); + expect(output[0].length).toBe(40); + }); + it("should not skip when buffered is within LATENCY", () => { const buffer = create({ rate: 1000, channels: 1, capacity: 100, latency: 50 }); @@ -458,8 +488,8 @@ describe("setLatency", () => { it("should dynamically change latency affecting skip behavior", () => { const buffer = create({ rate: 1000, channels: 1, capacity: 100, latency: 50 }); - // Fill 80 samples - insert(buffer, 0, 80, { channels: 1, value: 1.0 }); + // Fill 80 samples, in chunks so the skip band stays narrow. + insertChunks(buffer, 0, 80, 10, { channels: 1, value: 1.0 }); expect(buffer.stalled).toBe(false); // With LATENCY=50, reading should skip to 30 (80-50) @@ -467,7 +497,7 @@ describe("setLatency", () => { expect(output1[0].length).toBe(50); // Write more - insert(buffer, 80, 80, { channels: 1, value: 2.0 }); + insertChunks(buffer, 80, 80, 10, { channels: 1, value: 2.0 }); // Change latency to 20 buffer.setLatency(20); @@ -478,6 +508,68 @@ describe("setLatency", () => { }); }); +describe("re-buffer", () => { + it("re-stalls and refills to the target after running dry", () => { + const buffer = create({ rate: 1000, channels: 1, capacity: 256, latency: 40 }); + + insertChunks(buffer, 0, 40, 20, { channels: 1, value: 1.0 }); + expect(buffer.stalled).toBe(false); + + // Drain it. + expect(read(buffer, 40, 1)[0].length).toBe(40); + expect(buffer.length).toBe(0); + + // The next read finds nothing and parks playback rather than handing the worklet a quantum + // of silence and trying again on the next one. + expect(read(buffer, 20, 1)[0].length).toBe(0); + expect(buffer.stalled).toBe(true); + expect(buffer.underruns).toBe(1); + + // A single chunk is not the target, so playback stays parked. + insert(buffer, 40, 20, { channels: 1, value: 2.0 }); + expect(buffer.stalled).toBe(true); + expect(read(buffer, 20, 1)[0].length).toBe(0); + + // Reaching the target resumes it, and nothing buffered in the meantime was lost. + insert(buffer, 60, 20, { channels: 1, value: 2.0 }); + expect(buffer.stalled).toBe(false); + expect(read(buffer, 40, 1)[0].length).toBe(40); + }); + + it("stall() parks playback without discarding what is buffered", () => { + const buffer = create({ rate: 1000, channels: 1, capacity: 256, latency: 40 }); + + insertChunks(buffer, 0, 40, 20, { channels: 1, value: 1.0 }); + expect(read(buffer, 20, 1)[0].length).toBe(20); + + // The target deepens, so playback parks until the ring holds the new depth. + buffer.setLatency(60); + buffer.stall(); + expect(buffer.stalled).toBe(true); + expect(read(buffer, 20, 1)[0].length).toBe(0); + expect(buffer.length).toBe(20); // nothing was thrown away, unlike reset() + + insert(buffer, 40, 20, { channels: 1, value: 2.0 }); + expect(buffer.stalled).toBe(true); // 40 buffered, target is 60 + + insert(buffer, 60, 20, { channels: 1, value: 2.0 }); + expect(buffer.stalled).toBe(false); + + // Playback resumes on the same timeline: the samples buffered before the stall play first. + const output = read(buffer, 20, 1); + expect(output[0].length).toBe(20); + expect(output[0][0]).toBe(1.0); + expect(buffer.underruns).toBe(0); + }); + + it("does not count an underrun while parked", () => { + const buffer = create({ rate: 1000, channels: 1, capacity: 256, latency: 40 }); + read(buffer, 20, 1); + read(buffer, 20, 1); + expect(buffer.underruns).toBe(0); + }); +}); + describe("stalled getter", () => { it("should reflect STALLED flag", () => { const buffer = create({ rate: 1000, channels: 1, capacity: 100, latency: 50 }); diff --git a/js/watch/src/audio/shared-ring-buffer.ts b/js/watch/src/audio/shared-ring-buffer.ts index ce790948547..a431a8a15cf 100644 --- a/js/watch/src/audio/shared-ring-buffer.ts +++ b/js/watch/src/audio/shared-ring-buffer.ts @@ -3,10 +3,24 @@ import { Time } from "@moq/net"; // Control array slot indices. The playhead is not here: see `state`. const WRITE = 0; const LATENCY = 1; +/** + * Whether playback is held while the ring refills. The one slot both threads write. + * + * The writer clears it once the ring holds the target, the reader raises it when the ring runs + * dry. Neither needs a compare-exchange. The reader only raises it having observed an empty ring, + * which means it already drained everything the writer published, and the writer only clears it + * having published enough to cover the target. A raise landing after a clear costs one quantum of + * silence and the next insert undoes it, since every insert re-checks. A clear landing after a + * raise is correct on its face: the samples are there. + */ const STALLED = 2; // Timeline identity changes only on re-anchor, independently of the packed mutation epoch. const TIMELINE = 3; -const CONTROL_SLOTS = 4; +// The size of the most recent insert, i.e. one decoded chunk. See `read`. +const CHUNK = 4; +// How many times the reader ran dry mid-playback. Diagnostics only. +const UNDERRUN = 5; +const CONTROL_SLOTS = 6; /** * The playhead and its mutation epoch, packed into one 64-bit word: epoch in the high @@ -297,6 +311,15 @@ export class SharedRingBuffer { } } + // Publish the chunk size for the reader's skip band before the samples it describes become + // visible. The two stores are not one transaction, so the reader can land between them: + // this way it sees the wider band with the old cursor, which only makes it skip less, where + // the other order would have it measure a longer span against a narrower band and cut audio + // that is not actually late. The most recent insert, not a running maximum: one oversized + // decode would otherwise widen the band for the life of the ring, and a publisher that + // changes its frame duration is described within one insert. + Atomics.store(this.#control, CHUNK, originalLength); + // Advance WRITE (only forward) Atomics.store(this.#control, WRITE, i32Max(Atomics.load(this.#control, WRITE), end)); @@ -329,17 +352,29 @@ export class SharedRingBuffer { const write = Atomics.load(this.#control, WRITE); const latency = Atomics.load(this.#control, LATENCY); - // Latency skip: if buffered data exceeds LATENCY, skip ahead. + // Latency skip: skip ahead only once the ring holds a whole chunk more than the target, + // landing back on the target. Frames arrive one chunk at a time, so a ring sitting exactly + // on the target is a chunk above it the moment the next one lands; skipping on that + // overshoot discards audio on every single insert. The chunk of slack is the difference + // between tolerating normal arrival and cutting it. // CAS ensures we never step backward relative to a concurrent writer advance. // Disabled in buffered mode, where we deliberately play through the whole buffer. const buffered = (write - read) | 0; - if (!this.buffered && latency > 0 && buffered > latency) { + const slack = Atomics.load(this.#control, CHUNK); + if (!this.buffered && latency > 0 && buffered > ((latency + slack) | 0)) { const skipTo = (write - latency) | 0; if (((skipTo - read) | 0) > 0) read = skipTo; } const available = (write - read) | 0; const count = Math.min(available, output[0].length); + if (available <= 0) { + // Ran dry mid-playback. Re-stall so `insert` refills to the target before playback + // resumes: the alternative is playing the next chunk on an empty cushion, which + // underruns again on the very next quantum. See STALLED for the write discipline. + Atomics.store(this.#control, STALLED, 1); + Atomics.add(this.#control, UNDERRUN, 1); + } if (count <= 0) { // A latency skip still has to be published, and still only if nothing moved. if (((read - readOf(state)) | 0) > 0) { @@ -404,6 +439,19 @@ export class SharedRingBuffer { this.#step(); } + /** + * Hold playback until the ring holds the target again, keeping everything buffered. + * Main thread only. + * + * Used when the target deepens: `setLatency` alone only raises the bar a future refill has to + * clear, so a ring already playing keeps draining at its old depth and audio runs that much + * ahead of video. Parking the playhead spends exactly the deficit as silence and resumes on the + * same timeline, where `reset` would throw the buffer away and re-anchor. + */ + stall(): void { + Atomics.store(this.#control, STALLED, 1); + } + /** * Flush buffered samples and re-stall, ready to anchor the next utterance (buffered mode). * Main thread only. The worklet reader sees STALLED and stops until the next insert. @@ -458,6 +506,8 @@ export class SharedRingBuffer { Atomics.store(dst.#control, WRITE, write); Atomics.store(dst.#control, LATENCY, latency); Atomics.store(dst.#control, STALLED, stalled); + Atomics.store(dst.#control, CHUNK, Atomics.load(this.#control, CHUNK)); + Atomics.store(dst.#control, UNDERRUN, Atomics.load(this.#control, UNDERRUN)); // Carry the unwrapped playhead over, rebased onto dst's READ. Fold the same `read` // snapshot the copy used so both sides agree on one observation; `copyStart` is at or @@ -498,6 +548,11 @@ export class SharedRingBuffer { return Atomics.load(this.#control, STALLED) === 1; } + /** How many times the reader has run dry mid-playback. */ + get underruns(): number { + return Atomics.load(this.#control, UNDERRUN); + } + /** * Number of buffered samples (WRITE - READ). * diff --git a/js/watch/src/element.ts b/js/watch/src/element.ts index e5e01f413a8..0d45b20cb66 100644 --- a/js/watch/src/element.ts +++ b/js/watch/src/element.ts @@ -238,22 +238,28 @@ export default class MoqWatch extends HTMLElement { }); this.signals.cleanup(() => this.text.close()); - // The video decoder owns rendition handoffs but also needs Sync. Bridge its output through a - // parent-owned signal so Sync can be constructed first without exposing mutable wiring. + // The decoders own rendition handoffs and measure how late frames arrive, but they need Sync + // to exist first. Bridge their outputs through parent-owned signals rather than exposing + // mutable wiring. const videoJitter = new Signal(undefined); + const videoSpread = new Signal(undefined); + const audioSpread = new Signal(undefined); this.sync = new Sync({ delay: this.controls.delay, buffer: this.controls.buffer, - probe: this.connection.probe, video: videoJitter, audio: audioSource.out.jitter, + videoSpread, + audioSpread, }); this.signals.cleanup(() => this.sync.close()); this.video = new Video.Decoder(videoSource, this.sync, { enabled: this.#videoEnabled }); this.signals.proxy(videoJitter, this.video.out.jitter); + this.signals.proxy(videoSpread, this.video.out.spread); this.audio = new Audio.Decoder(audioSource, this.sync, { enabled: this.#audioEnabled }); + this.signals.proxy(audioSpread, this.audio.out.spread); this.signals.cleanup(() => { this.video.close(); this.audio.close(); diff --git a/js/watch/src/sync.test.ts b/js/watch/src/sync.test.ts index ca8c8209b1f..79fbbc395e3 100644 --- a/js/watch/src/sync.test.ts +++ b/js/watch/src/sync.test.ts @@ -63,3 +63,48 @@ describe("delay and buffer", () => { sync.close(); }); }); + +describe("auto delay", () => { + it("starts at the advertised delay and follows the measured arrivals", async () => { + const audioSpread = new Signal(undefined); + const sync = new Sync({ audio: 20 as Time.Milli, video: 33 as Time.Milli, audioSpread }); + await flush(); + + // Nothing measured yet, so the delay is what the catalog advertises. + expect(sync.out.jitter.peek()).toBe(0 as Time.Milli); + expect(sync.out.delay.peek()).toBe(33 as Time.Milli); + + // A publisher flushing 250ms of audio at once needs 250ms of buffer, whatever the RTT is. + // The measured spread already covers what the catalog advertises, so it replaces the + // advertised delay rather than stacking on it. + audioSpread.set(250 as Time.Milli); + await flush(); + expect(sync.out.jitter.peek()).toBe(250 as Time.Milli); + expect(sync.out.delay.peek()).toBe(250 as Time.Milli); + + sync.close(); + }); + + it("takes the largest spread across tracks", async () => { + const audioSpread = new Signal(40 as Time.Milli); + const videoSpread = new Signal(120 as Time.Milli); + const sync = new Sync({ audioSpread, videoSpread }); + await flush(); + expect(sync.out.jitter.peek()).toBe(120 as Time.Milli); + + audioSpread.set(200 as Time.Milli); + await flush(); + expect(sync.out.jitter.peek()).toBe(200 as Time.Milli); + + sync.close(); + }); + + it("ignores the measured spread when the delay is a fixed number", async () => { + const audioSpread = new Signal(250 as Time.Milli); + const sync = new Sync({ delay: 500 as Time.Milli, audio: 20 as Time.Milli, audioSpread }); + await flush(); + expect(sync.out.jitter.peek()).toBe(500 as Time.Milli); + expect(sync.out.delay.peek()).toBe(520 as Time.Milli); + sync.close(); + }); +}); diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index 3f0c2a82fcb..fc03ad4d3a0 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -1,11 +1,11 @@ -import type * as Moq from "@moq/net"; import { Time } from "@moq/net"; import { Effect, type Getter, getter, type Inputs, type Readonlys, readonlys, Signal } from "@moq/signals"; /** * How far playback trails the live edge. * - * `"auto"` (the default) sizes the jitter buffer from the connection RTT; a `Time.Milli` fixes it. + * `"auto"` (the default) sizes the jitter buffer from how late frames actually arrive; a + * `Time.Milli` fixes it. * * `"instant"` drops the buffer and the pacing together: nothing is held, and {@link Sync.wait} * returns without sleeping, so a frame presents as soon as it exists. It also overrides @@ -15,9 +15,6 @@ import { Effect, type Getter, getter, type Inputs, type Readonlys, readonlys, Si */ export type Delay = "instant" | "auto" | Time.Milli; -const MIN_JITTER = Time.Milli(20); -const FALLBACK_JITTER = Time.Milli(100); - export type SyncInput = { /** How far playback trails the live edge. See {@link Delay}. */ delay: Getter; @@ -32,17 +29,22 @@ export type SyncInput = { */ buffer: Getter; - /** - * The connection's PROBE estimates, whose RTT drives "auto" jitter. Usually wired - * from a `Connection.Shared`'s or `Reload`'s `probe`. - */ - probe: Getter; - - /** Any additional delay required for audio (wired from the per-rendition source). */ + /** The delay the selected audio rendition advertises, wired from the per-rendition source. */ audio: Getter; - /** Any additional delay required for video (wired from the per-rendition source). */ + /** The delay the selected video rendition advertises, wired from the per-rendition source. */ video: Getter; + + /** + * How late audio frames arrive relative to the earliest one, from the container consumer. + * + * This is what `"auto"` sizes the jitter buffer from: it measures what the publisher and the + * network actually deliver, which the round trip does not describe. + */ + audioSpread: Getter; + + /** How late video frames arrive relative to the earliest one, from the container consumer. */ + videoSpread: Getter; }; type SyncOutput = { @@ -50,11 +52,11 @@ type SyncOutput = { // This will keep being updated as we catch up to the live playhead then will be relatively static. reference: Signal; - // The resolved delay from the live edge to the playhead: jitter + max(audio, video). + // The resolved delay from the live edge to the playhead. See `#runDelay` for how the terms combine. delay: Signal; // The jitter component of `delay` (always numeric). - // In "auto" mode this is updated automatically from RTT. + // In "auto" mode this follows the measured arrival spread, the largest across tracks. // When the delay is a number, jitter equals that number. jitter: Signal; @@ -77,7 +79,7 @@ export class Sync { readonly #out: SyncOutput = { reference: new Signal(undefined), delay: new Signal(Time.Milli.zero), - jitter: new Signal(FALLBACK_JITTER), + jitter: new Signal(Time.Milli.zero), timestamp: new Signal(undefined), buffered: new Signal(false), maxAge: new Signal(Time.Milli.zero), @@ -91,19 +93,16 @@ export class Sync { // Per-label late-frame tracking: accumulate count and max lateness, flush on recovery. #late = new Map(); - // Minimum RTT seen, used as the baseline for jitter calculation. - // Avoids inflating jitter due to bufferbloat. - #minRtt: number | undefined; - #signals = new Effect(); constructor(props?: Inputs) { this.in = { delay: getter(props?.delay ?? ("auto" as Delay)), buffer: getter(props?.buffer ?? Time.Milli.zero), - probe: getter(props?.probe), audio: getter(props?.audio), video: getter(props?.video), + audioSpread: getter(props?.audioSpread), + videoSpread: getter(props?.videoSpread), }; this.#update = Promise.withResolvers(); @@ -123,49 +122,45 @@ export class Sync { this.#out.maxAge.set(Time.Milli.add(delay, buffer)); } + // "auto" sizes the buffer from what arrives, not from the round trip. A retransmit costs an RTT, + // but a publisher flushing a second of media at once costs a second, and a well-paced publisher + // across the world costs nothing above its frame duration: the arrival spread measures both, so + // it is the only term here. #runJitter(effect: Effect): void { const delay = effect.get(this.in.delay); if (delay === "instant") { // Holds nothing at all. - this.#minRtt = undefined; this.#out.jitter.set(Time.Milli.zero); return; } if (typeof delay === "number") { // Fixed mode: the configured delay is the jitter. - this.#minRtt = undefined; this.#out.jitter.set(delay); return; } - // "auto" mode: compute jitter from the connection's RTT estimate. - const rtt = effect.get(this.in.probe)?.rtt; - if (rtt !== undefined) { - // Track minimum RTT as baseline, ignoring bufferbloat. - this.#minRtt = this.#minRtt !== undefined ? Math.min(this.#minRtt, rtt) : rtt; - - // Buffer enough for a retransmit (1 RTT for ACK + retransmit). - const jitter = Time.Milli(Math.max(MIN_JITTER, this.#minRtt * 1.25)); - this.#out.jitter.set(jitter); - return; - } - - // No RTT available: fall back to static default. - this.#minRtt = undefined; - this.#out.jitter.set(FALLBACK_JITTER); + const audio = effect.get(this.in.audioSpread) ?? Time.Milli.zero; + const video = effect.get(this.in.videoSpread) ?? Time.Milli.zero; + this.#out.jitter.set(Time.Milli.max(audio, video)); } + // A fixed delay is what the viewer asked for, held on top of whatever the renditions advertise. + // "auto" takes the larger of the two terms rather than their sum: the advertised delay and the + // measured spread describe the same thing, a publisher that emits in bursts, so adding them + // would double the buffer for exactly the publishers that need it most. "instant" holds nothing. #runDelay(effect: Effect): void { + const mode = effect.get(this.in.delay); const jitter = effect.get(this.#out.jitter); const video = effect.get(this.in.video) ?? Time.Milli.zero; const audio = effect.get(this.in.audio) ?? Time.Milli.zero; + const advertised = Time.Milli.max(video, audio); - // A zero delay still holds the rendition's own delay, which is a frame interval at 60fps. - // "instant" holds nothing at all. - const instant = effect.get(this.in.delay) === "instant"; - const delay = instant ? Time.Milli.zero : Time.Milli.add(Time.Milli.max(video, audio), jitter); + let delay: Time.Milli; + if (mode === "instant") delay = Time.Milli.zero; + else if (typeof mode === "number") delay = Time.Milli.add(advertised, jitter); + else delay = Time.Milli.max(advertised, jitter); this.#out.delay.set(delay); this.#update.resolve(); diff --git a/js/watch/src/ui/components/buffering-indicator.ts b/js/watch/src/ui/components/buffering-indicator.ts index 432b0e4b37f..0901ac0eebf 100644 --- a/js/watch/src/ui/components/buffering-indicator.ts +++ b/js/watch/src/ui/components/buffering-indicator.ts @@ -9,7 +9,12 @@ export function bufferingIndicator(parent: Effect, watch: MoqWatch): HTMLElement container.appendChild(spinner); parent.run((effect) => { - const buffering = effect.get(watch.video.out.stalled); + // Audio re-buffers on its own once the ring runs dry, which is just as much a stall as a + // video one. Gate it on audio actually being downloaded: the ring reports stalled from + // construction and drains once the download stops, so a video-only broadcast or a muted + // player would otherwise show the spinner forever. + const audio = effect.get(watch.audio.in.enabled) && effect.get(watch.audio.source.out.config) !== undefined; + const buffering = effect.get(watch.video.out.stalled) || (audio && effect.get(watch.audio.out.stalled)); const offline = effect.get(watch.broadcast.out.status) === "offline"; const unsupported = effect.get(watch.video.source.out.error) === "unsupported"; container.style.display = buffering && !offline && !unsupported ? "" : "none"; diff --git a/js/watch/src/ui/components/latency.ts b/js/watch/src/ui/components/latency.ts index 57428989c6d..43d5c00fa46 100644 --- a/js/watch/src/ui/components/latency.ts +++ b/js/watch/src/ui/components/latency.ts @@ -63,7 +63,7 @@ export function latencyTab(parent: Effect, watch: MoqWatch): HTMLElement { const hint = DOM.create( "div", { className: "tab-hint" }, - "A larger delay smooths over network jitter at the cost of latency. Auto tracks the connection RTT. Drag the timeline to fine-tune.", + "A larger delay smooths over network jitter at the cost of latency. Auto follows how late frames actually arrive. Drag the timeline to fine-tune.", ); container.append(chips, timeline, readout, hint); diff --git a/js/watch/src/ui/stats.ts b/js/watch/src/ui/stats.ts index bbb18199ba0..2e80e25f862 100644 --- a/js/watch/src/ui/stats.ts +++ b/js/watch/src/ui/stats.ts @@ -86,6 +86,7 @@ export function statsTab(parent: Effect, watch: MoqWatch): HTMLElement { const aRate2 = line(audioCard.grid, "Sample rate"); const aChannels = line(audioCard.grid, "Channels"); const aBitrate = line(audioCard.grid, "Bitrate"); + const aUnderruns = line(audioCard.grid, "Underruns"); track(parent, audioCard, { catalog: watch.audio.source.out.catalog, flag: watch.controls.muted, @@ -137,6 +138,10 @@ export function statsTab(parent: Effect, watch: MoqWatch): HTMLElement { aBitrate.textContent = aBitrate2 !== undefined ? formatBitrate(aBitrate2) : "—"; if (aStats) aPrev = { bytes: aStats.bytesReceived, when: now }; + // A non-zero count means the target is below what arrivals actually need. + const aUnder = watch.audio.out.underruns.peek(); + aUnderruns.textContent = watch.audio.out.stalled.peek() ? `${aUnder} (buffering)` : `${aUnder}`; + // Network. "Estimated max" is the congestion controller / PROBE estimate; // "Actual" is the goodput we measure from the video + audio byte counters. const probe = watch.connection.probe.peek(); diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 684ae821b24..ec2b0aa4275 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -63,6 +63,10 @@ type DecoderOutput = { // Combined buffered ranges (network jitter + decode buffer) buffered: Signal; + + // How late video frames arrive relative to the earliest one, measured by the container + // consumer. Wired into Sync by the parent, which sizes the "auto" delay from it. + spread: Signal; }; /** Downloads video from a track and decodes it into {@link VideoFrame}s with WebCodecs. */ @@ -79,6 +83,7 @@ export class Decoder { stats: new Signal(undefined), jitter: new Signal(undefined), buffered: new Signal([]), + spread: new Signal(undefined), }; readonly out = readonlys(this.#out); @@ -189,6 +194,7 @@ export class Decoder { if (!active) { // Clear stale data when disabled (e.g. paused or not visible). this.#out.buffered.set([]); + this.#out.spread.set(undefined); return; } @@ -205,6 +211,7 @@ export class Decoder { }); effect.proxy(this.#out.timestamp, active.timestamp); effect.proxy(this.#out.buffered, active.buffered); + effect.proxy(this.#out.spread, active.spread); } #runDisplay(effect: Effect): void { @@ -276,6 +283,9 @@ class DecoderTrack { // Network jitter + decode buffer. buffered = new Signal([]); + // How late frames arrive relative to the earliest one, from the container consumer. + spread = new Signal(undefined); + // Decoded frames waiting to be rendered. #buffered = new Signal([]); @@ -388,6 +398,9 @@ class DecoderTrack { this.buffered.update(() => Container.mergeBufferedRanges(network, decode)); }); + // Publish the measured arrival spread for Sync. + effect.run((inner) => this.spread.set(inner.get(consumer.spread))); + decoder.configure({ codec: this.config.codec, description: this.config.description ? Util.Hex.toBytes(this.config.description) : undefined, @@ -464,6 +477,9 @@ class DecoderTrack { this.buffered.update(() => Container.mergeBufferedRanges(network, decode)); }); + // Publish the measured arrival spread for Sync. + effect.run((inner) => this.spread.set(inner.get(consumer.spread))); + // Configure decoder with description from catalog decoder.configure({ codec: this.config.codec, From 85e9c5df31e62a8762e2eefd4a86ab04a79f7372 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 7 Sep 2026 21:27:23 -0700 Subject: [PATCH 2/2] fix(watch): count a short quantum as an underrun and expire a stale arrival minimum Two findings from the automated review. A quantum the ring can only partly fill ends in silence just as an empty one does, but only the empty case was counted, so a chunk landing before the next read hid the gap from the stats panel. Count it, without parking playback: the shortfall is under one quantum, and a refill would spend the whole target as silence to cover it. Replaying the recorded flush spans with a re-stall on every short quantum raised the underrun count from two to two dozen per run. The estimator kept the minimum from before a gap longer than both of its windows authoritative for another window, so a stream resuming on a higher path delay read every punctual frame as late. Expire both minima after such a gap. Also the dev API: the consumer test used the old `latency` prop name. Co-Authored-By: Claude Fable 5.1 --- js/hang/src/container/consumer.test.ts | 2 +- js/hang/src/container/jitter.test.ts | 14 +++++++++++++ js/hang/src/container/jitter.ts | 12 +++++++++-- js/watch/src/audio/ring-buffer.test.ts | 20 +++++++++++++++++++ js/watch/src/audio/ring-buffer.ts | 6 ++++++ js/watch/src/audio/shared-ring-buffer.test.ts | 19 ++++++++++++++++++ js/watch/src/audio/shared-ring-buffer.ts | 5 +++++ js/watch/src/element.ts | 2 +- 8 files changed, 76 insertions(+), 4 deletions(-) diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index 905f8853760..1e2cab3c2f3 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -325,7 +325,7 @@ test("Consumer skips groups via PTS-span when over the max age", async () => { test("Consumer measures how late frames arrive", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(track.subscribe(), { format: new LegacyFormat(), latency: 500 as Time.Milli }); + const consumer = new Consumer(track.subscribe(), { format: new LegacyFormat(), maxAge: 500 as Time.Milli }); // A prompt first group sets the arrival baseline. writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); diff --git a/js/hang/src/container/jitter.test.ts b/js/hang/src/container/jitter.test.ts index 34196c3f447..cb108bcd542 100644 --- a/js/hang/src/container/jitter.test.ts +++ b/js/hang/src/container/jitter.test.ts @@ -95,6 +95,20 @@ describe("arrival jitter", () => { expect(jitter.value.peek()).toBeLessThan(2 * CHUNK_MS); }); + it("expires the baseline after a long gap", () => { + const jitter = new Jitter(); + flush(jitter, 200, 1); + + // Nothing arrives for over a minute, and the path delay is higher when the stream resumes. + // Frames still land evenly, so they are punctual, not late: a minimum from before the gap + // would read the whole difference as jitter. + const base = 200 * CHUNK_MS + 90_000; + for (let i = 0; i < 200; i++) { + observe(jitter, base + i * CHUNK_MS, base + i * CHUNK_MS + 400); + } + expect(jitter.value.peek()).toBe(CHUNK_MS as Time.Milli); + }); + it("forgets the baseline across a discontinuity", () => { const jitter = new Jitter(); flush(jitter, 200, 1); diff --git a/js/hang/src/container/jitter.ts b/js/hang/src/container/jitter.ts index 497630b0983..d94f11efb4d 100644 --- a/js/hang/src/container/jitter.ts +++ b/js/hang/src/container/jitter.ts @@ -73,9 +73,17 @@ export class Jitter { observe(timestamp: Time.Micro, now: Time.Milli): void { const delay = now - Time.Milli.fromMicro(timestamp); - // Roll the minimum window forward so a stale baseline expires. + // Roll the minimum window forward so a stale baseline expires. A gap longer than both + // windows expires both minima: promoting the one from before the gap would hold a path + // delay that no longer exists authoritative for another window, reading every punctual + // frame after the gap as late. this.#windowStart ??= now; - if (now - this.#windowStart >= WINDOW) { + const elapsed = now - this.#windowStart; + if (elapsed >= 2 * WINDOW) { + this.#previous = undefined; + this.#current = undefined; + this.#windowStart = now; + } else if (elapsed >= WINDOW) { this.#previous = this.#current; this.#current = undefined; this.#windowStart = now; diff --git a/js/watch/src/audio/ring-buffer.test.ts b/js/watch/src/audio/ring-buffer.test.ts index 86188d5973e..b282bb28927 100644 --- a/js/watch/src/audio/ring-buffer.test.ts +++ b/js/watch/src/audio/ring-buffer.test.ts @@ -722,6 +722,26 @@ describe("re-buffer", () => { expect(read(buffer, 40, 1)[0].length).toBe(40); }); + it("counts a quantum it could only partly fill as an underrun", () => { + const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 40 as Time.Milli }); + + write(buffer, 0 as Time.Milli, 20, { channels: 1, value: 1.0 }); + write(buffer, 20 as Time.Milli, 20, { channels: 1, value: 1.0 }); + expect(read(buffer, 30, 1)[0].length).toBe(30); + + // Ten samples remain against a quantum of twenty: the rest of the quantum is silence, which + // is an underrun whether or not a chunk lands before the next read. It is not a stall: a + // refill would cost the whole target to cover a gap shorter than one quantum. + expect(read(buffer, 20, 1)[0].length).toBe(10); + expect(buffer.stalled).toBe(false); + expect(buffer.underruns).toBe(1); + + // A chunk landing before the next read keeps playback going. + write(buffer, 40 as Time.Milli, 20, { channels: 1, value: 2.0 }); + expect(read(buffer, 20, 1)[0].length).toBe(20); + expect(buffer.underruns).toBe(1); + }); + it("stall() parks playback without discarding what is buffered", () => { const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 40 as Time.Milli }); diff --git a/js/watch/src/audio/ring-buffer.ts b/js/watch/src/audio/ring-buffer.ts index 58660435428..2a84080845b 100644 --- a/js/watch/src/audio/ring-buffer.ts +++ b/js/watch/src/audio/ring-buffer.ts @@ -256,6 +256,12 @@ export class AudioRingBuffer { } this.#readIndex += samples; + + // A short quantum ends in silence, so it counts as an underrun even if a chunk lands before + // the next read. It does not park playback: the shortfall is under one quantum, and a + // refill would spend the whole target as silence to cover it. + if (samples < output[0].length) this.#underruns++; + return samples; } } diff --git a/js/watch/src/audio/shared-ring-buffer.test.ts b/js/watch/src/audio/shared-ring-buffer.test.ts index 9ee8f3ac2bf..0a3af36a21c 100644 --- a/js/watch/src/audio/shared-ring-buffer.test.ts +++ b/js/watch/src/audio/shared-ring-buffer.test.ts @@ -536,6 +536,25 @@ describe("re-buffer", () => { expect(read(buffer, 40, 1)[0].length).toBe(40); }); + it("counts a quantum it could only partly fill as an underrun", () => { + const buffer = create({ rate: 1000, channels: 1, capacity: 256, latency: 40 }); + + insertChunks(buffer, 0, 40, 20, { channels: 1, value: 1.0 }); + expect(read(buffer, 30, 1)[0].length).toBe(30); + + // Ten samples remain against a quantum of twenty: the rest of the quantum is silence, which + // is an underrun whether or not a chunk lands before the next read. It is not a stall: a + // refill would cost the whole target to cover a gap shorter than one quantum. + expect(read(buffer, 20, 1)[0].length).toBe(10); + expect(buffer.stalled).toBe(false); + expect(buffer.underruns).toBe(1); + + // A chunk landing before the next read keeps playback going. + insert(buffer, 40, 20, { channels: 1, value: 2.0 }); + expect(read(buffer, 20, 1)[0].length).toBe(20); + expect(buffer.underruns).toBe(1); + }); + it("stall() parks playback without discarding what is buffered", () => { const buffer = create({ rate: 1000, channels: 1, capacity: 256, latency: 40 }); diff --git a/js/watch/src/audio/shared-ring-buffer.ts b/js/watch/src/audio/shared-ring-buffer.ts index a431a8a15cf..e15f5ccd2cf 100644 --- a/js/watch/src/audio/shared-ring-buffer.ts +++ b/js/watch/src/audio/shared-ring-buffer.ts @@ -403,6 +403,11 @@ export class SharedRingBuffer { return 0; } + // A short quantum ends in silence, so it counts as an underrun even if a chunk lands before + // the next read. It does not park playback: the shortfall is under one quantum, and a + // refill would spend the whole target as silence to cover it. + if (count < output[0].length) Atomics.add(this.#control, UNDERRUN, 1); + return count; } diff --git a/js/watch/src/element.ts b/js/watch/src/element.ts index 0d45b20cb66..d6b14253c49 100644 --- a/js/watch/src/element.ts +++ b/js/watch/src/element.ts @@ -561,7 +561,7 @@ export default class MoqWatch extends HTMLElement { /** * How far playback trails the live edge, in milliseconds. See {@link Delay}. * - * `"auto"` (the default) sizes the jitter buffer from the connection RTT. `"instant"` drops the + * `"auto"` (the default) sizes the jitter buffer from how late frames arrive. `"instant"` drops the * clock instead: video paints the moment it decodes and audio is disabled. */ get delay(): Delay {