diff --git a/demo/web/src/viz.ts b/demo/web/src/viz.ts index abe1a5a4969..827b83eda46 100644 --- a/demo/web/src/viz.ts +++ b/demo/web/src/viz.ts @@ -307,7 +307,7 @@ export function bufferBars(parent: Signals.Effect, watch: MoqWatch): HTMLElement const timestamp = watch.sync.now() as number | undefined; const stalled = watch.video.out.stalled.peek(); drawRanges(video.canvas, watch.video.out.buffered.peek(), timestamp, stalled); - drawRanges(audio.canvas, watch.audio.out.buffered.peek(), timestamp, false); + drawRanges(audio.canvas, watch.audio.out.buffered.peek(), timestamp, watch.audio.out.stalled.peek()); parent.animate(draw); }; parent.animate(draw); diff --git a/doc/lib/js/watch.md b/doc/lib/js/watch.md index 6a49a8b3982..2ff883e4f08 100644 --- a/doc/lib/js/watch.md +++ b/doc/lib/js/watch.md @@ -30,10 +30,10 @@ in sync at the latency you ask for. | --- | --- | | `url`, `name` | Relay URL (with `?jwt=` if needed) and broadcast name. | | `paused`, `muted`, `volume` | The usual player controls, mirrored as reactive properties. | -| `delay` | How far playback trails the live edge: `"auto"` (derived from RTT, the default), a duration like `"300ms"`, or `"instant"` to paint frames as they decode with no pacing at all. | +| `delay` | How far playback trails the live edge: `"auto"` (the default, sized from how late frames actually arrive), a duration like `"300ms"`, or `"instant"` to paint frames as they decode with no pacing at all. | | `buffer` | Future-dated media held beyond the live edge before playback skips ahead, e.g. `"30s"`. Defaults to none. | | `captions` | The caption track to show, or absent for off. `el.text.out.available` lists the renditions for a picker. | -| `jitter` | The jitter buffer in ms. | +| `jitter` | The jitter buffer in ms: in `"auto"` this is the measured arrival spread, read-only in practice. | | `visible` | Only subscribe to video while the element is on screen: a margin (`"20%"` default, `"200px"`), `"always"`, or `"never"`. | | `reload` | Wait for the broadcast to be announced before subscribing (default on), so a player can be mounted before the stream exists. | | `catalog-format` | `hang` (default, from the `.hang` suffix), `hangz` (compressed), `msf`, or `manual` to supply the catalog yourself. | diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index 2da4b48da6c..1e2cab3c2f3 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -323,6 +323,24 @@ test("Consumer skips groups via PTS-span when over the max age", async () => { consumer.close(); }); +test("Consumer measures how late frames arrive", async () => { + const track = new Track.Producer("test"); + const consumer = new Consumer(track.subscribe(), { format: new LegacyFormat(), maxAge: 500 as Time.Milli }); + + // A prompt first group sets the arrival baseline. + writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); + await settle(150); + + // The next group carries 20ms of media but shows up 150ms later, so a player needs the + // difference in its buffer to render it on time. Nothing about the round trip says that. + writeGroupWithLegacyFrames(track, 1, [20_000 as Time.Micro]); + track.close(); + + await drainFrames(consumer, 300); + expect(consumer.spread.peek()).toBeGreaterThanOrEqual(50 as Time.Milli); + consumer.close(); +}); + // --- Ordering --- test("Consumer delivers groups in sequence order regardless of arrival order", async () => { diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 26b3bae2cae..fd976b5548c 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -3,6 +3,7 @@ import * as Moq from "@moq/net"; import { Effect, type Getter, type GetterInit, getter, Signal } from "@moq/signals"; import type { Format } from "./format"; +import { Jitter } from "./jitter"; import type { BufferedRanges, Frame } from "./types"; /** Options for constructing a {@link Consumer}. */ @@ -143,6 +144,18 @@ export class Consumer { /** The time ranges currently buffered and ready to play. */ readonly buffered: Getter = this.#buffered; + // Measured at arrival, before any group is skipped: a target derived from what survives the + // age budget would only ever confirm the budget it was cut to. + #spread = new Jitter(); + + /** + * How late frames arrive relative to the earliest one, measured as they land. + * + * Size the playback buffer with this rather than with the round trip, which says nothing about + * how evenly a publisher emits frames. + */ + readonly spread: Getter = this.#spread.value; + #signals = new Effect(); /** Start consuming the given track, decoding frames with `props.format`. */ @@ -256,11 +269,21 @@ export class Consumer { // max age budget is what eventually breaks the stall. if (this.#classifyStale(group)) return; this.#checkReset(group); + + // Measured once the timeline is settled and before the age budget can skip + // the group: a reneged straggler is already gone, a rewinding group is read + // against the baseline `#checkReset` just re-anchored rather than against + // the epoch it replaced, and a target derived from what survives the budget + // would only ever confirm the budget it was cut to. + if (!marker) this.#spread.observe(frame.timestamp, Moq.Time.Milli.now()); this.#checkMaxAge(); // A newer group reaching back to where the stalled active group has // already presented means we can advance now instead of waiting. skipped = this.#tryDurationSkip(); + } else if (!marker) { + // The active group needs no classification, so its arrival counts as is. + this.#spread.observe(frame.timestamp, Moq.Time.Milli.now()); } // Wake next() for the current delivery head so its frames surface as they @@ -438,6 +461,7 @@ export class Consumer { this.#rewind.boundary = reset; this.#rewind.discontinuity++; this.#gap = true; + this.#spread.reanchor(); // Drop buffered groups the boundary can already prove stale; keep ambiguous ones. this.#groups = this.#groups.filter((g) => { @@ -661,6 +685,7 @@ export class Consumer { this.#presentedEnd = undefined; this.#deliveredGroup = undefined; this.#gap = true; + this.#spread.reanchor(); } #updateBuffered(): void { diff --git a/js/hang/src/container/jitter.test.ts b/js/hang/src/container/jitter.test.ts new file mode 100644 index 00000000000..cb108bcd542 --- /dev/null +++ b/js/hang/src/container/jitter.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it } from "bun:test"; +import { Time } from "@moq/net"; +import { Jitter } from "./jitter"; + +const CHUNK_MS = 20; + +/** Feed one arrival, in milliseconds on both axes. */ +function observe(jitter: Jitter, media: number, arrival: number): void { + jitter.observe(Time.Micro.fromMilli(media as Time.Milli), arrival as Time.Milli); +} + +/** + * A sender emitting one chunk every `CHUNK_MS`, holding `burst - 1` of them back and flushing the + * whole run at once. `burst` of 1 is an evenly paced sender. + */ +function flush(jitter: Jitter, chunks: number, burst: number, base = 50): void { + let held: number[] = []; + for (let i = 0; i < chunks; i++) { + const media = i * CHUNK_MS; + held.push(media); + if (held.length < burst) continue; + const arrival = media + base; + for (const m of held) observe(jitter, m, arrival); + held = []; + } +} + +describe("arrival jitter", () => { + it("is one frame for an evenly paced sender", () => { + // Nothing arrives late, so all the buffer needs to hold is the frame being played. + const jitter = new Jitter(); + flush(jitter, 500, 1); + expect(jitter.value.peek()).toBe(CHUNK_MS as Time.Milli); + }); + + it("does not follow the absolute delay", () => { + // A sender on the other side of the planet is not a jittery sender: only the spread of + // arrivals matters, which is what makes the round trip the wrong term to size a buffer from. + const near = new Jitter(); + flush(near, 500, 1, 10); + const far = new Jitter(); + flush(far, 500, 1, 2000); + expect(far.value.peek()).toBe(near.value.peek()); + }); + + it("converges on the flush span of a bursty sender", () => { + // Three chunks land at once, so the oldest is two chunk durations late. + const jitter = new Jitter(); + flush(jitter, 600, 3); + expect(jitter.value.peek()).toBeGreaterThanOrEqual(3 * CHUNK_MS); + expect(jitter.value.peek()).toBeLessThanOrEqual(4 * CHUNK_MS); + }); + + it("rises as soon as one arrival proves the buffer is too shallow", () => { + const jitter = new Jitter(); + flush(jitter, 100, 1); + expect(jitter.value.peek()).toBe(CHUNK_MS as Time.Milli); + + // A single 250ms straggler is only 1% of the samples, so the percentile ignores it. It costs + // one bucket of histogram resolution, not a quarter second of buffer. + observe(jitter, 100 * CHUNK_MS, 100 * CHUNK_MS + 250); + expect(jitter.value.peek()).toBeLessThan(2 * CHUNK_MS); + + // A run of them is the network, not an outlier, and the estimate follows within a second. + for (let i = 0; i < 20; i++) { + observe(jitter, (101 + i) * CHUNK_MS, (101 + i) * CHUNK_MS + 250); + } + expect(jitter.value.peek()).toBeGreaterThan(150 as Time.Milli); + }); + + it("lowers by at most one chunk per interval", () => { + const jitter = new Jitter(); + flush(jitter, 600, 3); + const peak = jitter.value.peek(); + expect(peak).toBeGreaterThan(0 as Time.Milli); + + // Arrivals settle. Sample the estimate every second of the recovery: a step larger than one + // chunk would drop a viewer's playhead onto a cushion that isn't there yet. + let previous = peak; + let last = 0; + const base = 600 * CHUNK_MS; + for (let i = 0; i < 6000; i++) { + const media = base + i * CHUNK_MS; + observe(jitter, media, media + 50); + const now = media + 50; + if (now - last < 1000) continue; + last = now; + const current = jitter.value.peek(); + expect(previous - current).toBeLessThanOrEqual(CHUNK_MS); + previous = current; + } + + // And it does come back down to a frame plus the histogram's resolution once arrivals have + // been even for a while. + expect(jitter.value.peek()).toBeLessThan(2 * CHUNK_MS); + }); + + it("expires the baseline after a long gap", () => { + const jitter = new Jitter(); + flush(jitter, 200, 1); + + // Nothing arrives for over a minute, and the path delay is higher when the stream resumes. + // Frames still land evenly, so they are punctual, not late: a minimum from before the gap + // would read the whole difference as jitter. + const base = 200 * CHUNK_MS + 90_000; + for (let i = 0; i < 200; i++) { + observe(jitter, base + i * CHUNK_MS, base + i * CHUNK_MS + 400); + } + expect(jitter.value.peek()).toBe(CHUNK_MS as Time.Milli); + }); + + it("forgets the baseline across a discontinuity", () => { + const jitter = new Jitter(); + flush(jitter, 200, 1); + + // The publisher restarts its timeline an hour in the past. Without a re-anchor every arrival + // after it reads as an hour late and the estimate saturates. + jitter.reanchor(); + const base = 3_600_000; + for (let i = 0; i < 200; i++) { + observe(jitter, base + i * CHUNK_MS, i * CHUNK_MS + 50); + } + expect(jitter.value.peek()).toBe(CHUNK_MS as Time.Milli); + }); +}); diff --git a/js/hang/src/container/jitter.ts b/js/hang/src/container/jitter.ts new file mode 100644 index 00000000000..d94f11efb4d --- /dev/null +++ b/js/hang/src/container/jitter.ts @@ -0,0 +1,164 @@ +import { Time } from "@moq/net"; +import { type Getter, Signal } from "@moq/signals"; + +/** Width of one histogram bucket, which is also the resolution of the estimate. */ +const BUCKET = 5; + +/** Number of buckets, so the estimate saturates at `BUCKET * BUCKETS`. */ +const BUCKETS = 200; + +/** + * How much an arrival still counts once another arrives. 0.9993 halves a sample's weight after + * ~1000 arrivals, roughly 20s of 50/s audio: long enough to hold a rare burst, short enough to + * forget a network that has since settled. + */ +const FORGET = 0.9993; + +/** The fraction of arrivals the estimate covers. */ +const PERCENTILE = 0.95; + +/** How long the estimate holds its value before it may step down again. */ +const LOWER_INTERVAL = 1000; + +/** + * How long a minimum stays authoritative. The minimum is tracked over two windows and read as the + * smaller of the two, so it survives at least this long and at most twice it. Without an expiry a + * single early arrival would anchor the spread for the life of the stream, and a sender clock + * drifting away from the receiver's would inflate it without bound. + */ +const WINDOW = 30_000; + +/** + * How much buffer late arrivals need, measured at arrival rather than derived from the round trip. + * + * Each arrival contributes `now - timestamp` relative to the running minimum of that difference, + * which is the delay a player has to absorb to render the frame on time: zero for an evenly paced + * sender, one flush span for a sender that emits a burst of frames at once. Samples land in a + * histogram that forgets old arrivals, and the estimate is a high percentile of it plus one frame, + * the shape WebRTC's NetEq uses. The extra frame is what keeps audio in the buffer at the bottom of + * the swing the spread describes, rather than landing it on exactly zero. + * + * The estimate rises as soon as a late frame proves the buffer is too shallow, and lowers by at + * most one frame per interval, so a refinement shrinks a viewer's buffer in steps it can absorb. + */ +export class Jitter { + // Weighted count of arrivals per spread bucket, and their sum. + #buckets = new Float64Array(BUCKETS); + #total = 0; + + // Smallest arrival delay in the current window and in the one before it. See WINDOW. + #current?: number; + #previous?: number; + #windowStart?: number; + + // Largest spread ever measured. Buckets are read at their upper edge, which covers the whole + // spread the bucket might hold; this caps that at a spread actually seen, so an evenly paced + // sender reads as zero rather than as one bucket. + #max = 0; + + // Smallest positive gap between consecutive timestamps, i.e. the frame duration. It bounds how + // fast the estimate steps down. + #spacing?: number; + #latest?: Time.Micro; + + // When the estimate last moved, so a step down waits out LOWER_INTERVAL. + #lowered?: number; + + #value = new Signal(Time.Milli.zero); + + /** The current estimate: enough buffer to render `PERCENTILE` of arrivals on time. */ + readonly value: Getter = this.#value; + + /** Fold one frame into the estimate, given its media timestamp and the wall time it arrived. */ + observe(timestamp: Time.Micro, now: Time.Milli): void { + const delay = now - Time.Milli.fromMicro(timestamp); + + // Roll the minimum window forward so a stale baseline expires. A gap longer than both + // windows expires both minima: promoting the one from before the gap would hold a path + // delay that no longer exists authoritative for another window, reading every punctual + // frame after the gap as late. + this.#windowStart ??= now; + const elapsed = now - this.#windowStart; + if (elapsed >= 2 * WINDOW) { + this.#previous = undefined; + this.#current = undefined; + this.#windowStart = now; + } else if (elapsed >= WINDOW) { + this.#previous = this.#current; + this.#current = undefined; + this.#windowStart = now; + } + this.#current = this.#current === undefined ? delay : Math.min(this.#current, delay); + const min = this.#previous === undefined ? this.#current : Math.min(this.#current, this.#previous); + + // Learn the frame duration from the timeline itself; it is the step size for lowering. + if (this.#latest !== undefined && timestamp > this.#latest) { + const gap = Time.Milli.fromMicro((timestamp - this.#latest) as Time.Micro); + this.#spacing = this.#spacing === undefined ? gap : Math.min(this.#spacing, gap); + } + if (this.#latest === undefined || timestamp > this.#latest) this.#latest = timestamp; + + const spread = Math.max(0, delay - min); + this.#max = Math.max(this.#max, spread); + const bucket = Math.min(BUCKETS - 1, Math.floor(spread / BUCKET)); + + // Decay everything, then count this arrival, so a sample's weight is relative to how many + // have arrived since rather than to how long ago it was. + for (let i = 0; i < BUCKETS; i++) this.#buckets[i] *= FORGET; + this.#total = this.#total * FORGET + 1; + this.#buckets[bucket] += 1; + + this.#publish(now); + } + + /** + * Forget the arrival baseline, keeping the measured spread. + * + * A discontinuity moves the media timeline underneath the measurement, so `now - timestamp` + * jumps by the size of the jump and the old minimum describes a timeline that no longer + * exists. The histogram is in spread space, which the jump does not move, so it survives. + */ + reanchor(): void { + this.#current = undefined; + this.#previous = undefined; + this.#windowStart = undefined; + this.#latest = undefined; + } + + #publish(now: Time.Milli): void { + // One frame on top of the percentile: the buffer swings by the spread the percentile + // measures, so without it the bottom of that swing lands on an empty ring. + const step = this.#spacing ?? BUCKET; + const measured = this.#percentile() + step; + const current = this.#value.peek(); + + if (measured >= current) { + this.#lowered = now; + if (measured > current) this.#value.set(Time.Milli(measured)); + return; + } + + // Lower gradually: the buffer shrinks by at most one frame per interval, so a refined + // estimate never drops the playhead onto an empty ring. + this.#lowered ??= now; + if (now - this.#lowered < LOWER_INTERVAL) return; + this.#lowered = now; + + this.#value.set(Time.Milli(Math.max(measured, current - step))); + } + + #percentile(): Time.Milli { + if (this.#total <= 0) return Time.Milli.zero; + + const want = this.#total * PERCENTILE; + let sum = 0; + for (let i = 0; i < BUCKETS; i++) { + sum += this.#buckets[i]; + // The upper edge of the bucket: the spread it holds is somewhere inside it, so covering + // the whole bucket is what actually covers the percentile. + if (sum >= want) return Time.Milli(Math.min((i + 1) * BUCKET, this.#max)); + } + + return Time.Milli(Math.min(BUCKETS * BUCKET, this.#max)); + } +} diff --git a/js/moq-boy/src/game.ts b/js/moq-boy/src/game.ts index 9c881808c2b..1f3e7fc743a 100644 --- a/js/moq-boy/src/game.ts +++ b/js/moq-boy/src/game.ts @@ -136,14 +136,17 @@ export class Game { }); this.#signals.cleanup(() => this.audioSource.close()); - // The decoder owns rendition handoffs but also needs Sync. Bridge its jitter output through a - // local signal so Sync can be constructed first. + // The decoders own rendition handoffs and measure how late frames arrive, but they need Sync + // to exist first. Bridge their outputs through local signals. const videoJitter = new Moq.Signals.Signal(undefined); + const videoSpread = new Moq.Signals.Signal(undefined); + const audioSpread = new Moq.Signals.Signal(undefined); this.sync = new Watch.Sync({ delay: this.delay, - probe: connection.probe, video: videoJitter, audio: this.audioSource.out.jitter, + videoSpread, + audioSpread, }); this.#signals.cleanup(() => this.sync.close()); @@ -152,6 +155,7 @@ export class Game { const videoEnabled = new Moq.Signals.Signal(true); this.videoDecoder = new Watch.Video.Decoder(this.videoSource, this.sync, { enabled: videoEnabled }); this.#signals.proxy(videoJitter, this.videoDecoder.out.jitter); + this.#signals.proxy(videoSpread, this.videoDecoder.out.spread); this.#signals.cleanup(() => this.videoDecoder.close()); // Renderer needs a canvas created by the UI layer, set via `canvas`. @@ -165,6 +169,7 @@ export class Game { // Audio pipeline. The emitter stops the download when muted or paused. const audioEnabled = new Moq.Signals.Signal(false); this.audioDecoder = new Watch.Audio.Decoder(this.audioSource, this.sync, { enabled: audioEnabled }); + this.#signals.proxy(audioSpread, this.audioDecoder.out.spread); this.#signals.cleanup(() => this.audioDecoder.close()); const audioPaused = new Moq.Signals.Signal(true); diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index 4c3a0263e5d..504cd5ee14d 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -1,6 +1,6 @@ import { Time } from "@moq/net"; import { Effect, type Getter, Signal } from "@moq/signals"; -import type { Data, InitPost, InitShared, Latency, Reset, State, Truncate } from "./render"; +import type { Data, InitPost, InitShared, Latency, Reset, Stall, State, Truncate } from "./render"; import { allocSharedRingBuffer, SharedRingBuffer } from "./shared-ring-buffer"; /** @@ -21,7 +21,7 @@ class Backpressure { this.#headroom = headroom; } - // Move the gate as the floor changes (e.g. "real-time" jitter tracking RTT). + // Move the gate as the floor changes (e.g. "auto" tracking the measured arrival spread). setHeadroom(headroom: Time.Micro): void { this.#headroom = headroom; } @@ -77,6 +77,14 @@ export interface AudioBuffer { /** Flush buffered samples and re-stall, ready to anchor the next utterance (buffered mode). */ reset(): void; + /** + * Hold playback until the ring holds the target again, keeping everything buffered. + * + * Used when the target deepens: `setLatency` alone only raises the bar a future refill has to + * clear, so a ring already playing keeps draining at its old depth. + */ + stall(): void; + /** * Drop buffered samples at or after `timestamp`, keeping what is already due. * @@ -100,6 +108,9 @@ export interface AudioBuffer { /** Whether the buffer is stalled (waiting to fill). */ readonly stalled: Getter; + /** How many times the ring has run dry mid-playback, cumulative. */ + readonly underruns: Getter; + /** Release any resources (event listeners, intervals, etc.). */ close(): void; } @@ -148,6 +159,9 @@ class SharedAudioBuffer implements AudioBuffer { readonly #stalled = new Signal(true); readonly stalled: Getter = this.#stalled; + readonly #underruns = new Signal(0); + readonly underruns: Getter = this.#underruns; + #backpressure: Backpressure; #signals = new Effect(); @@ -174,6 +188,7 @@ class SharedAudioBuffer implements AudioBuffer { const stalled = this.#ring.stalled; this.#timestamp.set(this.#ring.timestamp); this.#stalled.set(stalled); + this.#underruns.set(this.#ring.underruns); // While stalled the playhead is parked, so release the decode loop to refill the floor; // once playing, hold it to ~the floor ahead. if (stalled) this.#backpressure.flush(); @@ -210,8 +225,16 @@ class SharedAudioBuffer implements AudioBuffer { this.#backpressure.flush(); // the old timeline is gone; let the decode loop re-anchor } + stall(): void { + this.#ring.stall(); + this.#backpressure.flush(); // let the decode loop fill the deeper target + } + wait(timestamp: Time.Micro): Promise { - // Stalled = still filling the floor (bootstrap or underflow): let frames through to refill. + // Stalled = still filling the floor (bootstrap, an underrun the reader re-stalled on, or an + // explicit reset): let frames through so the ring refills to the floor. This is the single + // re-buffer path; the ring un-stalls itself on the insert that reaches the floor, so the + // gate closes again within one frame rather than draining the whole lookahead. if (this.#ring.stalled) return Promise.resolve(); return this.#backpressure.wait(timestamp, this.#ring.timestamp); } @@ -234,6 +257,9 @@ class PostAudioBuffer implements AudioBuffer { readonly #stalled = new Signal(true); readonly stalled: Getter = this.#stalled; + readonly #underruns = new Signal(0); + readonly underruns: Getter = this.#underruns; + // Backpressure runs off the playhead the worklet reports in its state messages. #backpressure: Backpressure; @@ -256,6 +282,7 @@ class PostAudioBuffer implements AudioBuffer { if (data?.type === "state") { this.#timestamp.set(data.timestamp); this.#stalled.set(data.stalled); + this.#underruns.set(data.underruns); // While stalled the playhead is parked, so release the decode loop to refill the floor; // once playing, hold it to ~the floor ahead. if (data.stalled) this.#backpressure.flush(); @@ -295,8 +322,18 @@ class PostAudioBuffer implements AudioBuffer { this.#backpressure.flush(); // the old timeline is gone; let the decode loop re-anchor } + stall(): void { + const msg: Stall = { type: "stall" }; + this.#worklet.port.postMessage(msg); + // Mirror it locally rather than waiting for the worklet's next state message, so `wait()` + // releases the decode loop now. + this.#stalled.set(true); + this.#backpressure.flush(); // let the decode loop fill the deeper target + } + wait(timestamp: Time.Micro): Promise { - // Stalled = still filling the floor (bootstrap or underflow): let frames through to refill. + // Stalled = still filling the floor (bootstrap, an underrun the reader re-stalled on, or an + // explicit reset): let frames through so the ring refills to the floor. See SharedAudioBuffer. if (this.#stalled.peek()) return Promise.resolve(); // Uses the worklet-reported playhead, which lags by a state-message interval; the floor's // headroom covers that. The worklet still drops the oldest if a frame slips through. diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 91a93de9fd6..f7889d8309f 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -10,7 +10,7 @@ import { subscribeMedia } from "../media"; import type { Sync } from "../sync"; import { type AudioBuffer, createAudioBuffer } from "./buffer"; import { Handover } from "./handover"; -import { reanchorFloor, ringSamples } from "./latency"; +import { ringSamples } from "./latency"; // Compiled and inlined as a blob URL via vite-plugin-worklet. import RenderWorklet from "./render-worklet.ts?worklet"; import type { Source } from "./source"; @@ -45,8 +45,15 @@ type DecoderOutput = { // Whether the audio buffer is stalled (waiting to fill) stalled: Signal; + // How many times the ring ran dry mid-playback, so the UI can show that the target is too low. + underruns: Signal; + // Combined buffered ranges (network jitter + decode buffer) buffered: Signal; + + // How late audio frames arrive relative to the earliest one, measured by the container + // consumer. Wired into Sync by the parent, which sizes the "auto" delay from it. + spread: Signal; }; /** Cumulative audio statistics since the decoder started. */ @@ -72,7 +79,9 @@ export class Decoder { stats: new Signal(undefined), timestamp: new Signal(undefined), stalled: new Signal(true), + underruns: new Signal(0), buffered: new Signal([]), + spread: new Signal(undefined), }; readonly out = readonlys(this.#out); @@ -91,9 +100,9 @@ export class Decoder { // Ordered discontinuity and endpoint state from the container consumer. #terminal = new Terminal(); - // The latency floor as of the last settled change, to detect a floor *increase* (needs a deeper - // cushion) versus a decrease or a real-time RTT wiggle. See #runLatencyReanchor. - #prevFloor?: Time.Milli; + // The derived target as of the last settled change, to detect a *deepening* (which needs the + // ring to refill) versus a decrease. See #runLatencyReanchor. + #prevTarget?: Time.Milli; // Which subscription the ring's buffered samples came from. See #runDecoder. #handover = new Handover(); @@ -186,6 +195,9 @@ export class Decoder { effect.run((inner) => { this.#out.stalled.set(inner.get(ring.stalled)); }); + effect.run((inner) => { + this.#out.underruns.set(inner.get(ring.underruns)); + }); effect.set(this.#out.root, worklet); }); @@ -217,31 +229,31 @@ export class Decoder { ring.setLatency(ringSamples(ring.rate, delay)); } - // Re-anchor when the delay floor *increases*. A larger floor needs a deeper cushion: video - // rebuilds it implicitly (its per-frame sync.wait() reads the live buffer, so it just holds - // longer), but the audio ring keeps draining at its old depth -- resize() (via setLatency) only - // re-stalls an *empty* ring, so a mid-playback ring never refills to the new floor and audio runs - // ahead of video (the "raise latency, only video re-buffers" desync). reset() re-stalls the ring - // so it refills to the new floor. Watch the latency target and media delay, excluding adaptive - // RTT jitter, and debounce so a slider drag coalesces into one re-anchor. Decreases are left to - // natural catch-up. + // Park playback when the target *deepens*, so the ring refills to it. Video rebuilds a deeper + // cushion implicitly (its per-frame sync.wait() reads the live buffer, so it just holds longer), + // but the audio ring keeps draining at its old depth: setLatency only raises the bar a future + // refill has to clear, so a ring already playing never gets deeper and audio runs ahead of video + // (the "raise latency, only video re-buffers" desync). Stalling spends the deficit as silence, + // once, instead of leaving it to the underrun that the shallow buffer eventually causes anyway. + // + // The derived delay, not the user's setting, since the arrival estimator moves it too. Only a + // deepening worth more than a frame counts, so the estimator's small refinements ride through, + // and the debounce coalesces a slider drag or a converging estimate into a single stall. + // Decreases are left to natural catch-up. #runLatencyReanchor(effect: Effect): void { - const floor = reanchorFloor({ - delay: effect.get(this.sync.in.delay), - audio: effect.get(this.sync.in.audio), - video: effect.get(this.sync.in.video), - }); - if (this.#prevFloor === undefined) { + const target = effect.get(this.sync.out.delay); + const step = effect.get(this.source.out.jitter) ?? Time.Milli.zero; + if (this.#prevTarget === undefined) { // Startup: the initial fill already builds the cushion; just record the baseline. - this.#prevFloor = floor; + this.#prevTarget = target; return; } - // When the timer fires, the floor read above is still current: any change would have rerun + // When the timer fires, the target read above is still current: any change would have rerun // this effect (tearing down the timer), so compare it against the pre-change baseline directly. - const baseline = this.#prevFloor; + const baseline = this.#prevTarget; effect.timer(() => { - if (floor > baseline) this.reset(); - this.#prevFloor = floor; + if (target - baseline > step) this.#ring?.stall(); + this.#prevTarget = target; }, LATENCY_REANCHOR_DEBOUNCE_MS); } @@ -304,6 +316,11 @@ export class Decoder { this.#out.buffered.update(() => Container.mergeBufferedRanges(network, decode)); }); + // Publish the measured arrival spread for Sync. Cleared on teardown so a departed track + // stops holding the buffer open. + effect.run((inner) => this.#out.spread.set(inner.get(consumer.spread))); + effect.cleanup(() => this.#out.spread.set(undefined)); + effect.spawn(async () => { const loaded = await Util.Libav.polyfill(); if (!loaded) return; // cancelled @@ -409,6 +426,11 @@ export class Decoder { this.#out.buffered.update(() => Container.mergeBufferedRanges(network, decode)); }); + // Publish the measured arrival spread for Sync. Cleared on teardown so a departed track + // stops holding the buffer open. + effect.run((inner) => this.#out.spread.set(inner.get(consumer.spread))); + effect.cleanup(() => this.#out.spread.set(undefined)); + effect.spawn(async () => { const loaded = await Util.Libav.polyfill(); if (!loaded) return; // cancelled diff --git a/js/watch/src/audio/latency.test.ts b/js/watch/src/audio/latency.test.ts index 39cade4ba05..e6918210456 100644 --- a/js/watch/src/audio/latency.test.ts +++ b/js/watch/src/audio/latency.test.ts @@ -1,20 +1,9 @@ import { describe, expect, it } from "bun:test"; import type { Time } from "@moq/net"; -import { reanchorFloor, ringSamples } from "./latency"; +import { ringSamples } from "./latency"; const ms = (value: number) => value as Time.Milli; -describe("reanchorFloor", () => { - it("includes the fixed delay and the largest media delay", () => { - expect(reanchorFloor({ delay: ms(100), audio: ms(20), video: ms(80) })).toBe(ms(180)); - }); - - it("tracks rendition delay without adaptive RTT jitter", () => { - expect(reanchorFloor({ delay: "auto", audio: ms(20), video: ms(80) })).toBe(ms(80)); - expect(reanchorFloor({ delay: "auto", audio: ms(20), video: ms(200) })).toBe(ms(200)); - }); -}); - describe("ringSamples", () => { // `delay="instant"` reports a zero buffer. Passed through, the ring rejects it and the // worklet is left with no backend that any later resize can revive. diff --git a/js/watch/src/audio/latency.ts b/js/watch/src/audio/latency.ts index e45485bdc8d..1885d28fd9f 100644 --- a/js/watch/src/audio/latency.ts +++ b/js/watch/src/audio/latency.ts @@ -1,26 +1,4 @@ import { Time } from "@moq/net"; -import type { Delay } from "../sync"; - -/** The inputs that determine whether audio needs a deeper playback cushion. */ -export interface ReanchorFloor { - /** How far playback trails the live edge. */ - delay: Delay; - - /** Additional delay required by the selected audio rendition. */ - audio?: Time.Milli; - - /** Additional delay required by the active or pending video rendition. */ - video?: Time.Milli; -} - -/** The stable delay floor whose increase requires the audio ring to refill. */ -export function reanchorFloor(props: ReanchorFloor): Time.Milli { - // "auto" and "instant" contribute nothing: the adaptive RTT component is deliberately excluded - // so an RTT wiggle doesn't re-anchor, and "instant" holds nothing at all. - const target = typeof props.delay === "number" ? props.delay : Time.Milli.zero; - const media = Time.Milli.max(props.audio ?? Time.Milli.zero, props.video ?? Time.Milli.zero); - return Time.Milli.add(target, media); -} // An AudioWorkletProcessor renders in fixed 128-sample quanta, so a ring shallower than one can // never be read from. diff --git a/js/watch/src/audio/render-worklet.ts b/js/watch/src/audio/render-worklet.ts index 723dfcfa993..313cb67f311 100644 --- a/js/watch/src/audio/render-worklet.ts +++ b/js/watch/src/audio/render-worklet.ts @@ -2,11 +2,17 @@ import type { Message, State } from "./render"; import { AudioRingBuffer } from "./ring-buffer"; import { SharedRingBuffer } from "./shared-ring-buffer"; +// Samples to fade over when playback stops or resumes short of a full quantum. A shortfall +// otherwise steps straight to silence and back, which is an audible click on every underrun. +const RAMP = 64; + class Render extends AudioWorkletProcessor { // Set after init, depending on which path the main thread chose. #backend?: SharedRingBuffer | AudioRingBuffer; #underflow = 0; #stateCounter = 0; + // Whether the previous quantum ended short, so the next one fades back in. + #short = false; constructor() { super(); @@ -18,10 +24,12 @@ class Render extends AudioWorkletProcessor { const previous = this.#backend instanceof SharedRingBuffer ? this.#backend : undefined; this.#backend = new SharedRingBuffer(msg, previous); this.#underflow = 0; + this.#short = false; } else if (msg.type === "init-post") { console.log("[audio-worklet] init-post: using postMessage path"); this.#backend = new AudioRingBuffer(msg); this.#underflow = 0; + this.#short = false; } else if (msg.type === "data") { // Only meaningful in post mode. if (this.#backend instanceof AudioRingBuffer) this.#backend.write(msg.timestamp, msg.data); @@ -34,6 +42,9 @@ class Render extends AudioWorkletProcessor { } else if (msg.type === "reset") { // Only meaningful in post mode; shared mode resets via the control array. if (this.#backend instanceof AudioRingBuffer) this.#backend.reset(); + } else if (msg.type === "stall") { + // Only meaningful in post mode; shared mode stalls via the control array. + if (this.#backend instanceof AudioRingBuffer) this.#backend.stall(); } }; } @@ -44,10 +55,29 @@ class Render extends AudioWorkletProcessor { const samplesRead = backend?.read(output) ?? 0; if (samplesRead < output[0].length) { + // Fade the tail of what we did read down to the silence that follows it. + for (const channel of output) { + const ramp = Math.min(RAMP, samplesRead); + for (let i = 0; i < ramp; i++) { + channel[samplesRead - ramp + i] *= 1 - (i + 1) / ramp; + } + } this.#underflow += output[0].length - samplesRead; - } else if (this.#underflow > 0 && backend) { - console.debug(`audio underflow: ${Math.round((1000 * this.#underflow) / backend.rate)}ms`); - this.#underflow = 0; + this.#short = true; + } else { + if (this.#short) { + // Fade back in from the silence the shortfall left behind. + for (const channel of output) { + for (let i = 0; i < Math.min(RAMP, samplesRead); i++) { + channel[i] *= (i + 1) / RAMP; + } + } + this.#short = false; + } + if (this.#underflow > 0 && backend) { + console.debug(`audio underflow: ${Math.round((1000 * this.#underflow) / backend.rate)}ms`); + this.#underflow = 0; + } } // In post mode the main thread can't read worklet state directly, so we @@ -61,6 +91,7 @@ class Render extends AudioWorkletProcessor { type: "state", timestamp: backend.timestamp, stalled: backend.stalled, + underruns: backend.underruns, }; this.port.postMessage(state); } diff --git a/js/watch/src/audio/render.ts b/js/watch/src/audio/render.ts index e0c82524687..a537ac1bbc2 100644 --- a/js/watch/src/audio/render.ts +++ b/js/watch/src/audio/render.ts @@ -2,7 +2,7 @@ import type { Time } from "@moq/net"; import type { SharedRingBufferInit } from "./shared-ring-buffer"; /** Everything the main thread sends the render worklet over its port. */ -export type Message = InitShared | InitPost | Data | Latency | Reset | Truncate; +export type Message = InitShared | InitPost | Data | Latency | Reset | Stall | Truncate; export type ToMain = State; /** Init message when SharedArrayBuffer is available. */ @@ -26,6 +26,14 @@ export interface Reset { type: "reset"; } +/** + * Hold playback until the buffer holds the target again, keeping what is buffered (fallback path + * only; the shared path stalls via Atomics). + */ +export interface Stall { + type: "stall"; +} + /** * Drop buffered samples at or after `timestamp`, keeping what is already due (fallback path only; * the shared path truncates via Atomics). @@ -53,4 +61,6 @@ export interface State { type: "state"; timestamp: Time.Micro; stalled: boolean; + // How many times the ring has run dry mid-playback, cumulative. + underruns: number; } diff --git a/js/watch/src/audio/replay.test.ts b/js/watch/src/audio/replay.test.ts new file mode 100644 index 00000000000..617a57194a7 --- /dev/null +++ b/js/watch/src/audio/replay.test.ts @@ -0,0 +1,183 @@ +import { describe, expect, it } from "bun:test"; +import { Time } from "@moq/net"; +import { AudioRingBuffer } from "./ring-buffer"; +import { allocSharedRingBuffer, SharedRingBuffer } from "./shared-ring-buffer"; + +// Replay recorded-shape arrival traces through both rings and count what a listener would hear: +// quanta rendered short (an underrun) and samples the ring threw away (a skip). Both rings are +// driven through the same harness, since the postMessage fallback is the path every page without +// cross-origin isolation takes. + +const RATE = 48000; +const QUANTUM = 128; // an AudioWorklet render quantum +const CHUNK_MS = 20; // one Opus frame +const CHUNK = (RATE * CHUNK_MS) / 1000; +// What the catalog advertises: one frame plus the render quantum. Sync holds this as a floor under +// the measured target. +const FLOOR = CHUNK_MS + Math.ceil((QUANTUM / RATE) * 1000); + +/** Deterministic PRNG, so a trace is the same on every machine. */ +function rng(seed: number): () => number { + let a = seed >>> 0; + return () => { + a = (a + 0x6d2b79f5) >>> 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +/** One frame of a trace: when it was captured, and the wall time it reached us. */ +interface Arrival { + media: number; + arrival: number; +} + +/** + * A sender emitting a frame every `CHUNK_MS`, holding `burst - 1` of them back and flushing the + * run at once, plus `spread` ms of per-flush network jitter. `burst` of 1 is an evenly paced + * sender; anything above it is the PES packing an importer produces. + */ +function trace(frames: number, burst: number, spread: number, seed = 7): Arrival[] { + const rand = rng(seed); + const out: Arrival[] = []; + let held: number[] = []; + for (let i = 0; i < frames; i++) { + const media = i * CHUNK_MS; + held.push(media); + if (held.length < burst) continue; + // Every frame keeps its own capture time; the whole run shares one arrival. + const arrival = media + 50 + rand() * spread; + for (const m of held) out.push({ media: m, arrival }); + held = []; + } + return out; +} + +/** + * What the estimator converges on for a trace: the p95 arrival spread plus a frame of cushion, + * held above the catalog floor. + */ +function target(t: Arrival[]): number { + let min = Number.POSITIVE_INFINITY; + for (const { media, arrival } of t) min = Math.min(min, arrival - media); + const spreads = t.map(({ media, arrival }) => arrival - media - min).sort((a, b) => a - b); + const p95 = Math.ceil(spreads[Math.floor(spreads.length * 0.95)]); + return Math.max(FLOOR, p95 + CHUNK_MS); +} + +/** The two rings behind one interface, since the harness drives them identically. */ +interface Ring { + insert(timestamp: Time.Micro, data: Float32Array[]): void; + read(output: Float32Array[]): number; + readonly length: number; +} + +function shared(latencyMs: number): Ring { + const samples = Math.ceil((RATE * latencyMs) / 1000); + const ring = new SharedRingBuffer(allocSharedRingBuffer(1, Math.max(RATE, samples * 2), RATE)); + ring.setLatency(samples); + return ring; +} + +function post(latencyMs: number): Ring { + const ring = new AudioRingBuffer({ rate: RATE, channels: 1, latency: latencyMs as Time.Milli }); + return { + insert: (timestamp, data) => ring.write(timestamp, data), + read: (output) => ring.read(output), + get length() { + return ring.length; + }, + }; +} + +interface Result { + /** Quanta rendered short of a full block after the warmup, i.e. audible underruns. */ + underruns: number; + /** Samples inserted after the warmup that were never played: skipped or overflow-dropped. */ + skipped: number; +} + +/** + * Play `t` through `ring` in real time: insert every frame the moment it arrives, and pull one + * render quantum every quantum's worth of wall time, which is what the AudioWorklet does. + */ +function replay(ring: Ring, t: Arrival[], warmupMs: number): Result { + const step = (QUANTUM / RATE) * 1000; + const output = [new Float32Array(QUANTUM)]; + const end = t[t.length - 1].arrival; + + let next = 0; + let played = 0; + let inserted = 0; + let short = 0; + let started = false; + let mark: { played: number; inserted: number; buffered: number; short: number } | undefined; + + for (let now = 0; now < end; now += step) { + while (next < t.length && t[next].arrival <= now) { + ring.insert(Time.Micro.fromMilli(t[next].media as Time.Milli), [new Float32Array(CHUNK).fill(0.5)]); + inserted += CHUNK; + next++; + } + + const count = ring.read(output); + played += count; + if (started && count < QUANTUM) short++; + if (count > 0) started = true; + + if (now >= warmupMs && !mark) mark = { played, inserted, buffered: ring.length, short }; + } + + const from = mark ?? { played, inserted, buffered: ring.length, short }; + return { + underruns: short - from.short, + skipped: inserted - from.inserted - (played - from.played) - (ring.length - from.buffered), + }; +} + +const RINGS: Array<[string, (latencyMs: number) => Ring]> = [ + ["shared", shared], + ["post", post], +]; + +describe.each(RINGS)("%s ring replay", (_name, build) => { + it("plays an evenly paced sender without underruns or skips", () => { + const t = trace(600, 1, 10); + expect(replay(build(target(t)), t, 2000)).toEqual({ underruns: 0, skipped: 0 }); + }); + + it("plays a bursty sender at the measured target", () => { + // Three frames flushed at once, which is what an importer packing PES payloads produces. + const t = trace(600, 3, 5); + expect(target(t)).toBeGreaterThan(2 * CHUNK_MS); + const result = replay(build(target(t)), t, 2000); + expect(result.underruns).toBeLessThanOrEqual(2); + expect(result.skipped).toBe(0); + }); + + it("keeps a five frame flush span playing", () => { + // A hundred milliseconds of arrivals landing at once. The target covers the trough and the + // slack above it absorbs the peak, so the flush plays instead of being cut down to the + // target on arrival. + // + // A 95th percentile target sits by construction at the edge of what arrives, so the tail + // beyond it can still land a couple of times across the ten second window; closing that + // without a deeper buffer is what the time-stretch quest is for. Two orders of magnitude + // below the round-trip target below, and the postMessage ring bounds its refill at capacity + // rather than in read(), so it can also land one flush past the band. + const t = trace(600, 5, 5); + const result = replay(build(target(t)), t, 2000); + expect(result.underruns).toBeLessThanOrEqual(2); + expect(result.skipped).toBeLessThanOrEqual(2 * CHUNK); + }); + + it("underruns constantly when the target ignores the arrival spread", () => { + // 46ms is what the round-trip formula produced on the connection this was measured on: it + // describes the network and says nothing about a sender that flushes five frames at once. + const t = trace(600, 5, 5); + const result = replay(build(46), t, 2000); + expect(result.underruns).toBeGreaterThan(100); + expect(result.skipped).toBeGreaterThan(10 * CHUNK); + }); +}); diff --git a/js/watch/src/audio/ring-buffer.test.ts b/js/watch/src/audio/ring-buffer.test.ts index 5d1fb2ceba0..b282bb28927 100644 --- a/js/watch/src/audio/ring-buffer.test.ts +++ b/js/watch/src/audio/ring-buffer.test.ts @@ -35,7 +35,9 @@ describe("initialization", () => { it("should initialize with valid parameters", () => { const buffer = new AudioRingBuffer({ rate: 48000, channels: 2, latency: 100 as Time.Milli }); - expect(buffer.capacity).toBe(4800); // 48000 * 0.1 + // Capacity is twice the target: the ring has to be able to hold the slack the overflow band + // tolerates, or every chunk arriving on a full ring would drop the oldest samples. + expect(buffer.capacity).toBe(9600); // 48000 * 0.1 * 2 expect(buffer.length).toBe(0); }); @@ -281,9 +283,10 @@ describe("stall behavior", () => { describe("ring buffer wrapping", () => { it("should wrap around when buffer is full", () => { + // 100 sample target, so 200 samples of capacity. const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Time.Milli }); - // Fill the buffer + // Fill to the target write(buffer, 0 as Time.Milli, 100, { channels: 1, value: 1.0 }); expect(buffer.stalled).toBe(false); @@ -297,22 +300,23 @@ describe("ring buffer wrapping", () => { // Now we have 100 samples available (50-149) expect(buffer.length).toBe(100); - // Write 50 more samples at timestamp 150, this will wrap around + // One chunk above the target is tolerated rather than dropped (50-199). write(buffer, 150 as Time.Milli, 50, { channels: 1, value: 3.0 }); + expect(buffer.length).toBe(150); - // Should still have 100 samples (buffer is at capacity) + // Past the band: drop back to the target, wrapping abs 200-249 onto slots 0-49. + write(buffer, 200 as Time.Milli, 50, { channels: 1, value: 4.0 }); expect(buffer.length).toBe(100); - // Read all 100 samples + // Read all 100 samples: abs 150-199 = 3.0, abs 200-249 = 4.0. const output2 = read(buffer, 100, 1); expect(output2[0].length).toBe(100); - // First 50 should be 2.0, next 50 should be 3.0 for (let i = 0; i < 50; i++) { - expect(output2[0][i]).toBe(2.0); + expect(output2[0][i]).toBe(3.0); } for (let i = 50; i < 100; i++) { - expect(output2[0][i]).toBe(3.0); + expect(output2[0][i]).toBe(4.0); } }); }); @@ -390,7 +394,7 @@ describe("edge cases", () => { describe("resize", () => { it("should resize to a larger buffer", () => { const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Time.Milli }); - expect(buffer.capacity).toBe(100); + expect(buffer.capacity).toBe(200); // Write 50 samples write(buffer, 0 as Time.Milli, 50, { channels: 1, value: 1.0 }); @@ -399,27 +403,27 @@ describe("resize", () => { // Resize to larger buffer (200ms = 200 samples) buffer.resize(200 as Time.Milli); - expect(buffer.capacity).toBe(200); + expect(buffer.capacity).toBe(400); expect(buffer.length).toBe(50); // Samples preserved expect(buffer.stalled).toBe(true); // Should trigger stall }); it("should resize to a smaller buffer and keep the most recent samples", () => { const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Time.Milli }); - expect(buffer.capacity).toBe(100); + expect(buffer.capacity).toBe(200); // Write 80 samples: first 40 with value 1.0, next 40 with value 2.0 write(buffer, 0 as Time.Milli, 40, { channels: 1, value: 1.0 }); write(buffer, 40 as Time.Milli, 40, { channels: 1, value: 2.0 }); expect(buffer.length).toBe(80); - // Resize to smaller buffer (50ms = 50 samples) - // Should keep the most recent 50 samples (samples 30-79) - buffer.resize(50 as Time.Milli); + // Resize to a 30 sample target (60 samples of capacity). + // Should keep the most recent 60 samples (samples 20-79) + buffer.resize(30 as Time.Milli); - expect(buffer.capacity).toBe(50); - expect(buffer.length).toBe(50); // Truncated to new capacity - expect(buffer.stalled).toBe(true); // Should trigger stall + expect(buffer.capacity).toBe(60); + expect(buffer.length).toBe(60); // Truncated to new capacity + expect(buffer.stalled).toBe(true); // Never reached the old 100 sample target }); it("should be a no-op when capacity is unchanged", () => { @@ -434,7 +438,7 @@ describe("resize", () => { // Should still not be stalled (no-op) expect(buffer.stalled).toBe(false); - expect(buffer.capacity).toBe(100); + expect(buffer.capacity).toBe(200); }); it("should throw on zero latency", () => { @@ -456,8 +460,8 @@ describe("resize", () => { // Resize to smaller buffer buffer.resize(50 as Time.Milli); - expect(buffer.capacity).toBe(50); - expect(buffer.length).toBe(50); + expect(buffer.capacity).toBe(100); + expect(buffer.length).toBe(60); expect(buffer.stalled).toBe(true); }); @@ -468,7 +472,7 @@ describe("resize", () => { // Resize empty buffer buffer.resize(200 as Time.Milli); - expect(buffer.capacity).toBe(200); + expect(buffer.capacity).toBe(400); expect(buffer.length).toBe(0); expect(buffer.stalled).toBe(true); }); @@ -488,10 +492,10 @@ describe("resize", () => { write(buffer, 100 as Time.Milli, 30, { channels: 1, value: 2.0 }); expect(buffer.length).toBe(70); // 130 - 60 - // Resize to 50 samples - should keep most recent 50 (samples 80-129) + // Resize to a 50 sample target - 100 of capacity, so all 70 survive (samples 60-129) buffer.resize(50 as Time.Milli); - expect(buffer.capacity).toBe(50); - expect(buffer.length).toBe(50); + expect(buffer.capacity).toBe(100); + expect(buffer.length).toBe(70); expect(buffer.stalled).toBe(false); }); @@ -505,22 +509,26 @@ describe("resize", () => { buffer.resize(50 as Time.Milli); expect(buffer.stalled).toBe(true); - // Write new data to fill the buffer and exit stall - // The overflow will discard preserved samples and advance readIndex + // Write new data to reach the new target and exit stall. Nothing is dropped: the ring has + // room for the preserved samples plus a chunk above the target. write(buffer, 50 as Time.Milli, 50, { channels: 1, value: 2.0 }); expect(buffer.stalled).toBe(false); + expect(buffer.length).toBe(100); - // Read should return the new data (value 2.0) - const output = read(buffer, 50, 1); - expect(output[0].length).toBe(50); + // Read plays the preserved samples first, then the new ones. + const output = read(buffer, 100, 1); + expect(output[0].length).toBe(100); for (let i = 0; i < 50; i++) { + expect(output[0][i]).toBe(1.0); + } + for (let i = 50; i < 100; i++) { expect(output[0][i]).toBe(2.0); } }); it("should preserve samples correctly when resizing larger then filling", () => { const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 50 as Time.Milli }); - expect(buffer.capacity).toBe(50); + expect(buffer.capacity).toBe(100); // Write 30 samples write(buffer, 0 as Time.Milli, 30, { channels: 1, value: 1.0 }); @@ -528,7 +536,7 @@ describe("resize", () => { // Resize to larger buffer (100ms = 100 samples) buffer.resize(100 as Time.Milli); - expect(buffer.capacity).toBe(100); + expect(buffer.capacity).toBe(200); expect(buffer.length).toBe(30); // All samples preserved expect(buffer.stalled).toBe(true); @@ -569,26 +577,26 @@ describe("resize", () => { // abs 110-119 → slots 10-19, value 3.0. write(buffer, 110 as Time.Milli, 10, { channels: 1, value: 3.0 }); - // Preserved range after resize: most-recent 50 samples = abs 70..119. - // copyStart = 120 - 50 = 70, and 70 % 50 = 20 (non-zero → triggers the bug). + // Preserved range after resize: all 100 buffered samples = abs 20..119. + // copyStart = 120 - 100 = 20, and 20 % 100 = 20 (non-zero → triggers the bug). buffer.resize(50 as Time.Milli); - expect(buffer.capacity).toBe(50); - expect(buffer.length).toBe(50); + expect(buffer.capacity).toBe(100); + expect(buffer.length).toBe(100); expect(buffer.stalled).toBe(false); // Read the preserved samples. Expected layout by absolute index: - // abs 70..99 = 1.0 (30 samples, from the initial fill) + // abs 20..99 = 1.0 (80 samples, from the initial fill) // abs 100..109 = 2.0 (10 samples) // abs 110..119 = 3.0 (10 samples) - const output = read(buffer, 50, 1); - expect(output[0].length).toBe(50); - for (let i = 0; i < 30; i++) { + const output = read(buffer, 100, 1); + expect(output[0].length).toBe(100); + for (let i = 0; i < 80; i++) { expect(output[0][i]).toBe(1.0); } - for (let i = 30; i < 40; i++) { + for (let i = 80; i < 90; i++) { expect(output[0][i]).toBe(2.0); } - for (let i = 40; i < 50; i++) { + for (let i = 90; i < 100; i++) { expect(output[0][i]).toBe(3.0); } }); @@ -607,16 +615,16 @@ describe("resize", () => { // Wrap the writer over slots 0-29 with value 2.0 (abs 100-129). write(buffer, 100 as Time.Milli, 30, { channels: 1, value: 2.0 }); - // Resize smaller. Preserved = last 50 samples = abs 80..129. + // Resize smaller. All 90 buffered samples fit the new 100 sample capacity: abs 40..129. buffer.resize(50 as Time.Milli); - expect(buffer.length).toBe(50); + expect(buffer.length).toBe(90); expect(buffer.stalled).toBe(false); // Write the next 10 samples at their real timestamp. This should append, // not create a gap or be discarded as "too old". write(buffer, 130 as Time.Milli, 10, { channels: 1, value: 3.0 }); - // Buffer capacity is 50, so the oldest 10 samples (abs 80..89) drop out. + // That put the ring past the band, so it drops back to the 50 sample target. // Remaining: abs 90..99 = 1.0 (10), abs 100..129 = 2.0 (30), abs 130..139 = 3.0 (10). expect(buffer.length).toBe(50); @@ -667,7 +675,7 @@ describe("live anchoring", () => { // with zeros, un-stalled on that overflow, and left the playhead a floor before the frame. it("anchors a live ring to the first frame instead of gap-filling from zero", () => { const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Time.Milli }); - expect(buffer.capacity).toBe(100); + expect(buffer.capacity).toBe(200); write(buffer, 2000 as Time.Milli, 40, { channels: 1, value: 0.5 }); // Still short of the floor, so nothing plays yet and the ring holds only real samples. @@ -685,6 +693,90 @@ describe("live anchoring", () => { }); }); +describe("re-buffer", () => { + it("re-stalls and refills to the target after running dry", () => { + const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 40 as Time.Milli }); + + write(buffer, 0 as Time.Milli, 20, { channels: 1, value: 1.0 }); + write(buffer, 20 as Time.Milli, 20, { channels: 1, value: 1.0 }); + expect(buffer.stalled).toBe(false); + + // Drain it. + expect(read(buffer, 40, 1)[0].length).toBe(40); + expect(buffer.length).toBe(0); + + // The next read finds nothing and parks playback rather than handing the worklet a quantum + // of silence and trying again on the next one. + expect(read(buffer, 20, 1)[0].length).toBe(0); + expect(buffer.stalled).toBe(true); + expect(buffer.underruns).toBe(1); + + // A single chunk is not the target, so playback stays parked. + write(buffer, 40 as Time.Milli, 20, { channels: 1, value: 2.0 }); + expect(buffer.stalled).toBe(true); + expect(read(buffer, 20, 1)[0].length).toBe(0); + + // Reaching the target resumes it, and nothing buffered in the meantime was lost. + write(buffer, 60 as Time.Milli, 20, { channels: 1, value: 2.0 }); + expect(buffer.stalled).toBe(false); + expect(read(buffer, 40, 1)[0].length).toBe(40); + }); + + it("counts a quantum it could only partly fill as an underrun", () => { + const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 40 as Time.Milli }); + + write(buffer, 0 as Time.Milli, 20, { channels: 1, value: 1.0 }); + write(buffer, 20 as Time.Milli, 20, { channels: 1, value: 1.0 }); + expect(read(buffer, 30, 1)[0].length).toBe(30); + + // Ten samples remain against a quantum of twenty: the rest of the quantum is silence, which + // is an underrun whether or not a chunk lands before the next read. It is not a stall: a + // refill would cost the whole target to cover a gap shorter than one quantum. + expect(read(buffer, 20, 1)[0].length).toBe(10); + expect(buffer.stalled).toBe(false); + expect(buffer.underruns).toBe(1); + + // A chunk landing before the next read keeps playback going. + write(buffer, 40 as Time.Milli, 20, { channels: 1, value: 2.0 }); + expect(read(buffer, 20, 1)[0].length).toBe(20); + expect(buffer.underruns).toBe(1); + }); + + it("stall() parks playback without discarding what is buffered", () => { + const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 40 as Time.Milli }); + + write(buffer, 0 as Time.Milli, 20, { channels: 1, value: 1.0 }); + write(buffer, 20 as Time.Milli, 20, { channels: 1, value: 1.0 }); + expect(read(buffer, 20, 1)[0].length).toBe(20); + + // The target deepens, so playback parks until the ring holds the new depth. + buffer.resize(60 as Time.Milli); + buffer.stall(); + expect(buffer.stalled).toBe(true); + expect(read(buffer, 20, 1)[0].length).toBe(0); + expect(buffer.length).toBe(20); // nothing was thrown away, unlike reset() + + write(buffer, 40 as Time.Milli, 20, { channels: 1, value: 2.0 }); + expect(buffer.stalled).toBe(true); // 40 buffered, target is 60 + + write(buffer, 60 as Time.Milli, 20, { channels: 1, value: 2.0 }); + expect(buffer.stalled).toBe(false); + + // Playback resumes on the same timeline: the samples buffered before the stall play first. + const output = read(buffer, 20, 1); + expect(output[0].length).toBe(20); + expect(output[0][0]).toBe(1.0); + expect(buffer.underruns).toBe(0); + }); + + it("does not count an underrun while parked", () => { + const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 40 as Time.Milli }); + read(buffer, 20, 1); + read(buffer, 20, 1); + expect(buffer.underruns).toBe(0); + }); +}); + describe("buffered mode", () => { function createBuffered(latency: number) { return new AudioRingBuffer({ rate: 1000, channels: 1, latency: latency as Time.Milli, buffered: true }); diff --git a/js/watch/src/audio/ring-buffer.ts b/js/watch/src/audio/ring-buffer.ts index fb94f036928..2a84080845b 100644 --- a/js/watch/src/audio/ring-buffer.ts +++ b/js/watch/src/audio/ring-buffer.ts @@ -8,6 +8,13 @@ export class AudioRingBuffer { readonly rate: number; readonly channels: number; #stalled = true; + #underruns = 0; + + // Samples in the most recent write, i.e. one decoded chunk. The overflow band tolerates this + // much above the target before dropping, so a ring sitting exactly on the target doesn't drop + // audio the moment the next chunk lands. The most recent write rather than a running maximum: + // one oversized decode would otherwise widen the band for the life of the ring. + #chunk = 0; // Buffered mode: play through everything buffered without skipping ahead. readonly #buffered: boolean; @@ -33,9 +40,11 @@ export class AudioRingBuffer { this.channels = props.channels; this.#buffered = props.buffered ?? false; - // The ring holds the latency floor as PCM. Buffered mode gives it headroom above the floor - // so the backpressure-paced decode loop (on the main thread) doesn't overflow-drop; the rest - // of the lookahead stays encoded upstream. + // The ring holds the latency floor as PCM, with headroom above it. Sizing capacity to the + // floor exactly makes the ring physically incapable of holding the slack the overflow band + // wants, so every chunk that arrives while the ring is on target drops the oldest samples. + // In buffered mode the headroom is also what keeps the backpressure-paced decode loop (on + // the main thread) from overflow-dropping; the rest of the lookahead stays encoded upstream. const capacity = this.#capacityFor(this.#latencySamples); this.#buffer = []; @@ -45,13 +54,18 @@ export class AudioRingBuffer { } #capacityFor(latencySamples: number): number { - return this.#buffered ? latencySamples * 2 : latencySamples; + return latencySamples * 2; } get stalled(): boolean { return this.#stalled; } + /** How many times the reader has run dry mid-playback. */ + get underruns(): number { + return this.#underruns; + } + get timestamp(): Time.Micro { return Time.Micro.fromSecond((this.#readIndex / this.rate) as Time.Second); } @@ -127,13 +141,18 @@ export class AudioRingBuffer { } const end = start + samples; - - // Check if we need to discard old samples to prevent overflow - const overflow = end - this.#readIndex - this.#buffer[0].length; - if (overflow >= 0) { - // Discard old samples and exit stalled mode - this.#stalled = false; - this.#readIndex += overflow; + this.#chunk = data[0].length; + + // Bound the ring. While playing, drop the oldest once it holds a whole chunk more than the + // target and land back on the target: frames arrive one chunk at a time, so a ring sitting + // exactly on the target is a chunk above it the moment the next one lands, and dropping on + // that overshoot discards audio on every single write. While stalled the reader is not + // consuming, so only the hard capacity applies; the band would throw away the very audio the + // refill is accumulating. Buffered mode plays through everything, so it is capacity-bound too. + const playing = !this.#stalled && !this.#buffered; + const limit = playing ? Math.min(this.#latencySamples + this.#chunk, this.capacity) : this.capacity; + if (end - this.#readIndex > limit) { + this.#readIndex = end - (playing ? Math.min(this.#latencySamples, this.capacity) : this.capacity); } // Fill gaps with zeros if there's a discontinuity @@ -171,9 +190,10 @@ export class AudioRingBuffer { this.#writeIndex = end; } - // Start playback once we've buffered the latency target. In buffered mode the cap - // is large, so we usually un-stall here rather than via the overflow path above. - if (this.#buffered && this.length >= this.#latencySamples) { + // Start playback once we've buffered the latency target. This is the only way out of a + // stall, so an underrun mid-playback refills to the target before resuming rather than + // playing the next chunk on an empty cushion. + if (this.length >= this.#latencySamples) { this.#stalled = false; } } @@ -191,6 +211,18 @@ export class AudioRingBuffer { this.#writeIndex = Math.max(target, this.#readIndex); } + /** + * Hold playback until the ring holds the target again, keeping everything buffered. + * + * Used when the target deepens: `resize` alone only raises the bar a future refill has to clear, + * so a ring already playing keeps draining at its old depth and audio runs that much ahead of + * video. Parking the playhead spends exactly the deficit as silence and resumes on the same + * timeline, where `reset` would throw the buffer away and re-anchor. + */ + stall(): void { + this.#stalled = true; + } + // Flush all buffered samples and re-stall, ready to anchor the next utterance. reset(): void { this.#readIndex = 0; @@ -204,7 +236,12 @@ export class AudioRingBuffer { if (this.#stalled) return 0; const samples = Math.min(this.#writeIndex - this.#readIndex, output[0].length); - if (samples === 0) return 0; + if (samples <= 0) { + // Ran dry mid-playback: re-stall so write() refills to the target before resuming. + this.#stalled = true; + this.#underruns++; + return 0; + } for (let channel = 0; channel < this.channels; channel++) { const dst = output[channel]; @@ -219,6 +256,12 @@ export class AudioRingBuffer { } this.#readIndex += samples; + + // A short quantum ends in silence, so it counts as an underrun even if a chunk lands before + // the next read. It does not park playback: the shortfall is under one quantum, and a + // refill would spend the whole target as silence to cover it. + if (samples < output[0].length) this.#underruns++; + return samples; } } diff --git a/js/watch/src/audio/shared-ring-buffer.test.ts b/js/watch/src/audio/shared-ring-buffer.test.ts index 2647ea3cc7e..0a3af36a21c 100644 --- a/js/watch/src/audio/shared-ring-buffer.test.ts +++ b/js/watch/src/audio/shared-ring-buffer.test.ts @@ -32,6 +32,24 @@ function insert( buffer.insert(Time.Micro.fromMilli(timestampMs as Time.Milli), data); } +/** + * Insert `samples` as a run of `chunk`-sized inserts starting at `timestampMs`. + * + * The skip band is one chunk wide, so a test that wants a skip has to arrive the way a decoder + * does: many small chunks, not one big one. The rate is 1000 in these tests, so a sample is a ms. + */ +function insertChunks( + buffer: SharedRingBuffer, + timestampMs: number, + samples: number, + chunk: number, + opts?: { channels?: number; value?: number }, +): void { + for (let i = 0; i < samples; i += chunk) { + insert(buffer, timestampMs + i, Math.min(chunk, samples - i), opts); + } +} + function read(buffer: SharedRingBuffer, samples: number, channelCount?: number): Float32Array[] { const ch = channelCount ?? buffer.channels; const output: Float32Array[] = []; @@ -52,7 +70,7 @@ describe("initialization", () => { expect(init.capacity).toBe(128); expect(init.rate).toBe(1000); expect(init.samples.byteLength).toBe(2 * 128 * 4); // 2 channels * 128 samples * Float32 - expect(init.control.byteLength).toBe(4 * 4); // 4 control slots * Int32 + expect(init.control.byteLength).toBe(6 * 4); // 6 control slots * Int32 expect(init.state.byteLength).toBe(8); // packed epoch + read cursor }); @@ -407,11 +425,11 @@ describe("overflow", () => { }); describe("latency skip", () => { - it("should skip READ when buffered exceeds LATENCY", () => { + it("should skip READ when buffered exceeds LATENCY plus a chunk", () => { const buffer = create({ rate: 1000, channels: 1, capacity: 100, latency: 20 }); - // Fill 60 samples — exceeds LATENCY of 20 - insert(buffer, 0, 60, { channels: 1, value: 1.0 }); + // Fill 60 samples in 10-sample chunks — a whole chunk past the 20 sample LATENCY. + insertChunks(buffer, 0, 60, 10, { channels: 1, value: 1.0 }); expect(buffer.stalled).toBe(false); // Read should skip ahead to maintain LATENCY distance from WRITE @@ -421,6 +439,18 @@ describe("latency skip", () => { expect(output[0].length).toBe(20); }); + it("should tolerate one chunk above LATENCY", () => { + // A ring sitting on the target is a chunk above it the moment the next chunk lands, so + // skipping on that overshoot would discard audio on every single insert. + const buffer = create({ rate: 1000, channels: 1, capacity: 100, latency: 20 }); + + insertChunks(buffer, 0, 40, 20, { channels: 1, value: 1.0 }); + expect(buffer.stalled).toBe(false); + + const output = read(buffer, 128, 1); + expect(output[0].length).toBe(40); + }); + it("should not skip when buffered is within LATENCY", () => { const buffer = create({ rate: 1000, channels: 1, capacity: 100, latency: 50 }); @@ -458,8 +488,8 @@ describe("setLatency", () => { it("should dynamically change latency affecting skip behavior", () => { const buffer = create({ rate: 1000, channels: 1, capacity: 100, latency: 50 }); - // Fill 80 samples - insert(buffer, 0, 80, { channels: 1, value: 1.0 }); + // Fill 80 samples, in chunks so the skip band stays narrow. + insertChunks(buffer, 0, 80, 10, { channels: 1, value: 1.0 }); expect(buffer.stalled).toBe(false); // With LATENCY=50, reading should skip to 30 (80-50) @@ -467,7 +497,7 @@ describe("setLatency", () => { expect(output1[0].length).toBe(50); // Write more - insert(buffer, 80, 80, { channels: 1, value: 2.0 }); + insertChunks(buffer, 80, 80, 10, { channels: 1, value: 2.0 }); // Change latency to 20 buffer.setLatency(20); @@ -478,6 +508,87 @@ describe("setLatency", () => { }); }); +describe("re-buffer", () => { + it("re-stalls and refills to the target after running dry", () => { + const buffer = create({ rate: 1000, channels: 1, capacity: 256, latency: 40 }); + + insertChunks(buffer, 0, 40, 20, { channels: 1, value: 1.0 }); + expect(buffer.stalled).toBe(false); + + // Drain it. + expect(read(buffer, 40, 1)[0].length).toBe(40); + expect(buffer.length).toBe(0); + + // The next read finds nothing and parks playback rather than handing the worklet a quantum + // of silence and trying again on the next one. + expect(read(buffer, 20, 1)[0].length).toBe(0); + expect(buffer.stalled).toBe(true); + expect(buffer.underruns).toBe(1); + + // A single chunk is not the target, so playback stays parked. + insert(buffer, 40, 20, { channels: 1, value: 2.0 }); + expect(buffer.stalled).toBe(true); + expect(read(buffer, 20, 1)[0].length).toBe(0); + + // Reaching the target resumes it, and nothing buffered in the meantime was lost. + insert(buffer, 60, 20, { channels: 1, value: 2.0 }); + expect(buffer.stalled).toBe(false); + expect(read(buffer, 40, 1)[0].length).toBe(40); + }); + + it("counts a quantum it could only partly fill as an underrun", () => { + const buffer = create({ rate: 1000, channels: 1, capacity: 256, latency: 40 }); + + insertChunks(buffer, 0, 40, 20, { channels: 1, value: 1.0 }); + expect(read(buffer, 30, 1)[0].length).toBe(30); + + // Ten samples remain against a quantum of twenty: the rest of the quantum is silence, which + // is an underrun whether or not a chunk lands before the next read. It is not a stall: a + // refill would cost the whole target to cover a gap shorter than one quantum. + expect(read(buffer, 20, 1)[0].length).toBe(10); + expect(buffer.stalled).toBe(false); + expect(buffer.underruns).toBe(1); + + // A chunk landing before the next read keeps playback going. + insert(buffer, 40, 20, { channels: 1, value: 2.0 }); + expect(read(buffer, 20, 1)[0].length).toBe(20); + expect(buffer.underruns).toBe(1); + }); + + it("stall() parks playback without discarding what is buffered", () => { + const buffer = create({ rate: 1000, channels: 1, capacity: 256, latency: 40 }); + + insertChunks(buffer, 0, 40, 20, { channels: 1, value: 1.0 }); + expect(read(buffer, 20, 1)[0].length).toBe(20); + + // The target deepens, so playback parks until the ring holds the new depth. + buffer.setLatency(60); + buffer.stall(); + expect(buffer.stalled).toBe(true); + expect(read(buffer, 20, 1)[0].length).toBe(0); + expect(buffer.length).toBe(20); // nothing was thrown away, unlike reset() + + insert(buffer, 40, 20, { channels: 1, value: 2.0 }); + expect(buffer.stalled).toBe(true); // 40 buffered, target is 60 + + insert(buffer, 60, 20, { channels: 1, value: 2.0 }); + expect(buffer.stalled).toBe(false); + + // Playback resumes on the same timeline: the samples buffered before the stall play first. + const output = read(buffer, 20, 1); + expect(output[0].length).toBe(20); + expect(output[0][0]).toBe(1.0); + expect(buffer.underruns).toBe(0); + }); + + it("does not count an underrun while parked", () => { + const buffer = create({ rate: 1000, channels: 1, capacity: 256, latency: 40 }); + read(buffer, 20, 1); + read(buffer, 20, 1); + expect(buffer.underruns).toBe(0); + }); +}); + describe("stalled getter", () => { it("should reflect STALLED flag", () => { const buffer = create({ rate: 1000, channels: 1, capacity: 100, latency: 50 }); diff --git a/js/watch/src/audio/shared-ring-buffer.ts b/js/watch/src/audio/shared-ring-buffer.ts index ce790948547..e15f5ccd2cf 100644 --- a/js/watch/src/audio/shared-ring-buffer.ts +++ b/js/watch/src/audio/shared-ring-buffer.ts @@ -3,10 +3,24 @@ import { Time } from "@moq/net"; // Control array slot indices. The playhead is not here: see `state`. const WRITE = 0; const LATENCY = 1; +/** + * Whether playback is held while the ring refills. The one slot both threads write. + * + * The writer clears it once the ring holds the target, the reader raises it when the ring runs + * dry. Neither needs a compare-exchange. The reader only raises it having observed an empty ring, + * which means it already drained everything the writer published, and the writer only clears it + * having published enough to cover the target. A raise landing after a clear costs one quantum of + * silence and the next insert undoes it, since every insert re-checks. A clear landing after a + * raise is correct on its face: the samples are there. + */ const STALLED = 2; // Timeline identity changes only on re-anchor, independently of the packed mutation epoch. const TIMELINE = 3; -const CONTROL_SLOTS = 4; +// The size of the most recent insert, i.e. one decoded chunk. See `read`. +const CHUNK = 4; +// How many times the reader ran dry mid-playback. Diagnostics only. +const UNDERRUN = 5; +const CONTROL_SLOTS = 6; /** * The playhead and its mutation epoch, packed into one 64-bit word: epoch in the high @@ -297,6 +311,15 @@ export class SharedRingBuffer { } } + // Publish the chunk size for the reader's skip band before the samples it describes become + // visible. The two stores are not one transaction, so the reader can land between them: + // this way it sees the wider band with the old cursor, which only makes it skip less, where + // the other order would have it measure a longer span against a narrower band and cut audio + // that is not actually late. The most recent insert, not a running maximum: one oversized + // decode would otherwise widen the band for the life of the ring, and a publisher that + // changes its frame duration is described within one insert. + Atomics.store(this.#control, CHUNK, originalLength); + // Advance WRITE (only forward) Atomics.store(this.#control, WRITE, i32Max(Atomics.load(this.#control, WRITE), end)); @@ -329,17 +352,29 @@ export class SharedRingBuffer { const write = Atomics.load(this.#control, WRITE); const latency = Atomics.load(this.#control, LATENCY); - // Latency skip: if buffered data exceeds LATENCY, skip ahead. + // Latency skip: skip ahead only once the ring holds a whole chunk more than the target, + // landing back on the target. Frames arrive one chunk at a time, so a ring sitting exactly + // on the target is a chunk above it the moment the next one lands; skipping on that + // overshoot discards audio on every single insert. The chunk of slack is the difference + // between tolerating normal arrival and cutting it. // CAS ensures we never step backward relative to a concurrent writer advance. // Disabled in buffered mode, where we deliberately play through the whole buffer. const buffered = (write - read) | 0; - if (!this.buffered && latency > 0 && buffered > latency) { + const slack = Atomics.load(this.#control, CHUNK); + if (!this.buffered && latency > 0 && buffered > ((latency + slack) | 0)) { const skipTo = (write - latency) | 0; if (((skipTo - read) | 0) > 0) read = skipTo; } const available = (write - read) | 0; const count = Math.min(available, output[0].length); + if (available <= 0) { + // Ran dry mid-playback. Re-stall so `insert` refills to the target before playback + // resumes: the alternative is playing the next chunk on an empty cushion, which + // underruns again on the very next quantum. See STALLED for the write discipline. + Atomics.store(this.#control, STALLED, 1); + Atomics.add(this.#control, UNDERRUN, 1); + } if (count <= 0) { // A latency skip still has to be published, and still only if nothing moved. if (((read - readOf(state)) | 0) > 0) { @@ -368,6 +403,11 @@ export class SharedRingBuffer { return 0; } + // A short quantum ends in silence, so it counts as an underrun even if a chunk lands before + // the next read. It does not park playback: the shortfall is under one quantum, and a + // refill would spend the whole target as silence to cover it. + if (count < output[0].length) Atomics.add(this.#control, UNDERRUN, 1); + return count; } @@ -404,6 +444,19 @@ export class SharedRingBuffer { this.#step(); } + /** + * Hold playback until the ring holds the target again, keeping everything buffered. + * Main thread only. + * + * Used when the target deepens: `setLatency` alone only raises the bar a future refill has to + * clear, so a ring already playing keeps draining at its old depth and audio runs that much + * ahead of video. Parking the playhead spends exactly the deficit as silence and resumes on the + * same timeline, where `reset` would throw the buffer away and re-anchor. + */ + stall(): void { + Atomics.store(this.#control, STALLED, 1); + } + /** * Flush buffered samples and re-stall, ready to anchor the next utterance (buffered mode). * Main thread only. The worklet reader sees STALLED and stops until the next insert. @@ -458,6 +511,8 @@ export class SharedRingBuffer { Atomics.store(dst.#control, WRITE, write); Atomics.store(dst.#control, LATENCY, latency); Atomics.store(dst.#control, STALLED, stalled); + Atomics.store(dst.#control, CHUNK, Atomics.load(this.#control, CHUNK)); + Atomics.store(dst.#control, UNDERRUN, Atomics.load(this.#control, UNDERRUN)); // Carry the unwrapped playhead over, rebased onto dst's READ. Fold the same `read` // snapshot the copy used so both sides agree on one observation; `copyStart` is at or @@ -498,6 +553,11 @@ export class SharedRingBuffer { return Atomics.load(this.#control, STALLED) === 1; } + /** How many times the reader has run dry mid-playback. */ + get underruns(): number { + return Atomics.load(this.#control, UNDERRUN); + } + /** * Number of buffered samples (WRITE - READ). * diff --git a/js/watch/src/element.ts b/js/watch/src/element.ts index e5e01f413a8..d6b14253c49 100644 --- a/js/watch/src/element.ts +++ b/js/watch/src/element.ts @@ -238,22 +238,28 @@ export default class MoqWatch extends HTMLElement { }); this.signals.cleanup(() => this.text.close()); - // The video decoder owns rendition handoffs but also needs Sync. Bridge its output through a - // parent-owned signal so Sync can be constructed first without exposing mutable wiring. + // The decoders own rendition handoffs and measure how late frames arrive, but they need Sync + // to exist first. Bridge their outputs through parent-owned signals rather than exposing + // mutable wiring. const videoJitter = new Signal(undefined); + const videoSpread = new Signal(undefined); + const audioSpread = new Signal(undefined); this.sync = new Sync({ delay: this.controls.delay, buffer: this.controls.buffer, - probe: this.connection.probe, video: videoJitter, audio: audioSource.out.jitter, + videoSpread, + audioSpread, }); this.signals.cleanup(() => this.sync.close()); this.video = new Video.Decoder(videoSource, this.sync, { enabled: this.#videoEnabled }); this.signals.proxy(videoJitter, this.video.out.jitter); + this.signals.proxy(videoSpread, this.video.out.spread); this.audio = new Audio.Decoder(audioSource, this.sync, { enabled: this.#audioEnabled }); + this.signals.proxy(audioSpread, this.audio.out.spread); this.signals.cleanup(() => { this.video.close(); this.audio.close(); @@ -555,7 +561,7 @@ export default class MoqWatch extends HTMLElement { /** * How far playback trails the live edge, in milliseconds. See {@link Delay}. * - * `"auto"` (the default) sizes the jitter buffer from the connection RTT. `"instant"` drops the + * `"auto"` (the default) sizes the jitter buffer from how late frames arrive. `"instant"` drops the * clock instead: video paints the moment it decodes and audio is disabled. */ get delay(): Delay { diff --git a/js/watch/src/sync.test.ts b/js/watch/src/sync.test.ts index ca8c8209b1f..79fbbc395e3 100644 --- a/js/watch/src/sync.test.ts +++ b/js/watch/src/sync.test.ts @@ -63,3 +63,48 @@ describe("delay and buffer", () => { sync.close(); }); }); + +describe("auto delay", () => { + it("starts at the advertised delay and follows the measured arrivals", async () => { + const audioSpread = new Signal(undefined); + const sync = new Sync({ audio: 20 as Time.Milli, video: 33 as Time.Milli, audioSpread }); + await flush(); + + // Nothing measured yet, so the delay is what the catalog advertises. + expect(sync.out.jitter.peek()).toBe(0 as Time.Milli); + expect(sync.out.delay.peek()).toBe(33 as Time.Milli); + + // A publisher flushing 250ms of audio at once needs 250ms of buffer, whatever the RTT is. + // The measured spread already covers what the catalog advertises, so it replaces the + // advertised delay rather than stacking on it. + audioSpread.set(250 as Time.Milli); + await flush(); + expect(sync.out.jitter.peek()).toBe(250 as Time.Milli); + expect(sync.out.delay.peek()).toBe(250 as Time.Milli); + + sync.close(); + }); + + it("takes the largest spread across tracks", async () => { + const audioSpread = new Signal(40 as Time.Milli); + const videoSpread = new Signal(120 as Time.Milli); + const sync = new Sync({ audioSpread, videoSpread }); + await flush(); + expect(sync.out.jitter.peek()).toBe(120 as Time.Milli); + + audioSpread.set(200 as Time.Milli); + await flush(); + expect(sync.out.jitter.peek()).toBe(200 as Time.Milli); + + sync.close(); + }); + + it("ignores the measured spread when the delay is a fixed number", async () => { + const audioSpread = new Signal(250 as Time.Milli); + const sync = new Sync({ delay: 500 as Time.Milli, audio: 20 as Time.Milli, audioSpread }); + await flush(); + expect(sync.out.jitter.peek()).toBe(500 as Time.Milli); + expect(sync.out.delay.peek()).toBe(520 as Time.Milli); + sync.close(); + }); +}); diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index 3f0c2a82fcb..fc03ad4d3a0 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -1,11 +1,11 @@ -import type * as Moq from "@moq/net"; import { Time } from "@moq/net"; import { Effect, type Getter, getter, type Inputs, type Readonlys, readonlys, Signal } from "@moq/signals"; /** * How far playback trails the live edge. * - * `"auto"` (the default) sizes the jitter buffer from the connection RTT; a `Time.Milli` fixes it. + * `"auto"` (the default) sizes the jitter buffer from how late frames actually arrive; a + * `Time.Milli` fixes it. * * `"instant"` drops the buffer and the pacing together: nothing is held, and {@link Sync.wait} * returns without sleeping, so a frame presents as soon as it exists. It also overrides @@ -15,9 +15,6 @@ import { Effect, type Getter, getter, type Inputs, type Readonlys, readonlys, Si */ export type Delay = "instant" | "auto" | Time.Milli; -const MIN_JITTER = Time.Milli(20); -const FALLBACK_JITTER = Time.Milli(100); - export type SyncInput = { /** How far playback trails the live edge. See {@link Delay}. */ delay: Getter; @@ -32,17 +29,22 @@ export type SyncInput = { */ buffer: Getter; - /** - * The connection's PROBE estimates, whose RTT drives "auto" jitter. Usually wired - * from a `Connection.Shared`'s or `Reload`'s `probe`. - */ - probe: Getter; - - /** Any additional delay required for audio (wired from the per-rendition source). */ + /** The delay the selected audio rendition advertises, wired from the per-rendition source. */ audio: Getter; - /** Any additional delay required for video (wired from the per-rendition source). */ + /** The delay the selected video rendition advertises, wired from the per-rendition source. */ video: Getter; + + /** + * How late audio frames arrive relative to the earliest one, from the container consumer. + * + * This is what `"auto"` sizes the jitter buffer from: it measures what the publisher and the + * network actually deliver, which the round trip does not describe. + */ + audioSpread: Getter; + + /** How late video frames arrive relative to the earliest one, from the container consumer. */ + videoSpread: Getter; }; type SyncOutput = { @@ -50,11 +52,11 @@ type SyncOutput = { // This will keep being updated as we catch up to the live playhead then will be relatively static. reference: Signal; - // The resolved delay from the live edge to the playhead: jitter + max(audio, video). + // The resolved delay from the live edge to the playhead. See `#runDelay` for how the terms combine. delay: Signal; // The jitter component of `delay` (always numeric). - // In "auto" mode this is updated automatically from RTT. + // In "auto" mode this follows the measured arrival spread, the largest across tracks. // When the delay is a number, jitter equals that number. jitter: Signal; @@ -77,7 +79,7 @@ export class Sync { readonly #out: SyncOutput = { reference: new Signal(undefined), delay: new Signal(Time.Milli.zero), - jitter: new Signal(FALLBACK_JITTER), + jitter: new Signal(Time.Milli.zero), timestamp: new Signal(undefined), buffered: new Signal(false), maxAge: new Signal(Time.Milli.zero), @@ -91,19 +93,16 @@ export class Sync { // Per-label late-frame tracking: accumulate count and max lateness, flush on recovery. #late = new Map(); - // Minimum RTT seen, used as the baseline for jitter calculation. - // Avoids inflating jitter due to bufferbloat. - #minRtt: number | undefined; - #signals = new Effect(); constructor(props?: Inputs) { this.in = { delay: getter(props?.delay ?? ("auto" as Delay)), buffer: getter(props?.buffer ?? Time.Milli.zero), - probe: getter(props?.probe), audio: getter(props?.audio), video: getter(props?.video), + audioSpread: getter(props?.audioSpread), + videoSpread: getter(props?.videoSpread), }; this.#update = Promise.withResolvers(); @@ -123,49 +122,45 @@ export class Sync { this.#out.maxAge.set(Time.Milli.add(delay, buffer)); } + // "auto" sizes the buffer from what arrives, not from the round trip. A retransmit costs an RTT, + // but a publisher flushing a second of media at once costs a second, and a well-paced publisher + // across the world costs nothing above its frame duration: the arrival spread measures both, so + // it is the only term here. #runJitter(effect: Effect): void { const delay = effect.get(this.in.delay); if (delay === "instant") { // Holds nothing at all. - this.#minRtt = undefined; this.#out.jitter.set(Time.Milli.zero); return; } if (typeof delay === "number") { // Fixed mode: the configured delay is the jitter. - this.#minRtt = undefined; this.#out.jitter.set(delay); return; } - // "auto" mode: compute jitter from the connection's RTT estimate. - const rtt = effect.get(this.in.probe)?.rtt; - if (rtt !== undefined) { - // Track minimum RTT as baseline, ignoring bufferbloat. - this.#minRtt = this.#minRtt !== undefined ? Math.min(this.#minRtt, rtt) : rtt; - - // Buffer enough for a retransmit (1 RTT for ACK + retransmit). - const jitter = Time.Milli(Math.max(MIN_JITTER, this.#minRtt * 1.25)); - this.#out.jitter.set(jitter); - return; - } - - // No RTT available: fall back to static default. - this.#minRtt = undefined; - this.#out.jitter.set(FALLBACK_JITTER); + const audio = effect.get(this.in.audioSpread) ?? Time.Milli.zero; + const video = effect.get(this.in.videoSpread) ?? Time.Milli.zero; + this.#out.jitter.set(Time.Milli.max(audio, video)); } + // A fixed delay is what the viewer asked for, held on top of whatever the renditions advertise. + // "auto" takes the larger of the two terms rather than their sum: the advertised delay and the + // measured spread describe the same thing, a publisher that emits in bursts, so adding them + // would double the buffer for exactly the publishers that need it most. "instant" holds nothing. #runDelay(effect: Effect): void { + const mode = effect.get(this.in.delay); const jitter = effect.get(this.#out.jitter); const video = effect.get(this.in.video) ?? Time.Milli.zero; const audio = effect.get(this.in.audio) ?? Time.Milli.zero; + const advertised = Time.Milli.max(video, audio); - // A zero delay still holds the rendition's own delay, which is a frame interval at 60fps. - // "instant" holds nothing at all. - const instant = effect.get(this.in.delay) === "instant"; - const delay = instant ? Time.Milli.zero : Time.Milli.add(Time.Milli.max(video, audio), jitter); + let delay: Time.Milli; + if (mode === "instant") delay = Time.Milli.zero; + else if (typeof mode === "number") delay = Time.Milli.add(advertised, jitter); + else delay = Time.Milli.max(advertised, jitter); this.#out.delay.set(delay); this.#update.resolve(); diff --git a/js/watch/src/ui/components/buffering-indicator.ts b/js/watch/src/ui/components/buffering-indicator.ts index 432b0e4b37f..0901ac0eebf 100644 --- a/js/watch/src/ui/components/buffering-indicator.ts +++ b/js/watch/src/ui/components/buffering-indicator.ts @@ -9,7 +9,12 @@ export function bufferingIndicator(parent: Effect, watch: MoqWatch): HTMLElement container.appendChild(spinner); parent.run((effect) => { - const buffering = effect.get(watch.video.out.stalled); + // Audio re-buffers on its own once the ring runs dry, which is just as much a stall as a + // video one. Gate it on audio actually being downloaded: the ring reports stalled from + // construction and drains once the download stops, so a video-only broadcast or a muted + // player would otherwise show the spinner forever. + const audio = effect.get(watch.audio.in.enabled) && effect.get(watch.audio.source.out.config) !== undefined; + const buffering = effect.get(watch.video.out.stalled) || (audio && effect.get(watch.audio.out.stalled)); const offline = effect.get(watch.broadcast.out.status) === "offline"; const unsupported = effect.get(watch.video.source.out.error) === "unsupported"; container.style.display = buffering && !offline && !unsupported ? "" : "none"; diff --git a/js/watch/src/ui/components/latency.ts b/js/watch/src/ui/components/latency.ts index 57428989c6d..43d5c00fa46 100644 --- a/js/watch/src/ui/components/latency.ts +++ b/js/watch/src/ui/components/latency.ts @@ -63,7 +63,7 @@ export function latencyTab(parent: Effect, watch: MoqWatch): HTMLElement { const hint = DOM.create( "div", { className: "tab-hint" }, - "A larger delay smooths over network jitter at the cost of latency. Auto tracks the connection RTT. Drag the timeline to fine-tune.", + "A larger delay smooths over network jitter at the cost of latency. Auto follows how late frames actually arrive. Drag the timeline to fine-tune.", ); container.append(chips, timeline, readout, hint); diff --git a/js/watch/src/ui/stats.ts b/js/watch/src/ui/stats.ts index bbb18199ba0..2e80e25f862 100644 --- a/js/watch/src/ui/stats.ts +++ b/js/watch/src/ui/stats.ts @@ -86,6 +86,7 @@ export function statsTab(parent: Effect, watch: MoqWatch): HTMLElement { const aRate2 = line(audioCard.grid, "Sample rate"); const aChannels = line(audioCard.grid, "Channels"); const aBitrate = line(audioCard.grid, "Bitrate"); + const aUnderruns = line(audioCard.grid, "Underruns"); track(parent, audioCard, { catalog: watch.audio.source.out.catalog, flag: watch.controls.muted, @@ -137,6 +138,10 @@ export function statsTab(parent: Effect, watch: MoqWatch): HTMLElement { aBitrate.textContent = aBitrate2 !== undefined ? formatBitrate(aBitrate2) : "—"; if (aStats) aPrev = { bytes: aStats.bytesReceived, when: now }; + // A non-zero count means the target is below what arrivals actually need. + const aUnder = watch.audio.out.underruns.peek(); + aUnderruns.textContent = watch.audio.out.stalled.peek() ? `${aUnder} (buffering)` : `${aUnder}`; + // Network. "Estimated max" is the congestion controller / PROBE estimate; // "Actual" is the goodput we measure from the video + audio byte counters. const probe = watch.connection.probe.peek(); diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 684ae821b24..ec2b0aa4275 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -63,6 +63,10 @@ type DecoderOutput = { // Combined buffered ranges (network jitter + decode buffer) buffered: Signal; + + // How late video frames arrive relative to the earliest one, measured by the container + // consumer. Wired into Sync by the parent, which sizes the "auto" delay from it. + spread: Signal; }; /** Downloads video from a track and decodes it into {@link VideoFrame}s with WebCodecs. */ @@ -79,6 +83,7 @@ export class Decoder { stats: new Signal(undefined), jitter: new Signal(undefined), buffered: new Signal([]), + spread: new Signal(undefined), }; readonly out = readonlys(this.#out); @@ -189,6 +194,7 @@ export class Decoder { if (!active) { // Clear stale data when disabled (e.g. paused or not visible). this.#out.buffered.set([]); + this.#out.spread.set(undefined); return; } @@ -205,6 +211,7 @@ export class Decoder { }); effect.proxy(this.#out.timestamp, active.timestamp); effect.proxy(this.#out.buffered, active.buffered); + effect.proxy(this.#out.spread, active.spread); } #runDisplay(effect: Effect): void { @@ -276,6 +283,9 @@ class DecoderTrack { // Network jitter + decode buffer. buffered = new Signal([]); + // How late frames arrive relative to the earliest one, from the container consumer. + spread = new Signal(undefined); + // Decoded frames waiting to be rendered. #buffered = new Signal([]); @@ -388,6 +398,9 @@ class DecoderTrack { this.buffered.update(() => Container.mergeBufferedRanges(network, decode)); }); + // Publish the measured arrival spread for Sync. + effect.run((inner) => this.spread.set(inner.get(consumer.spread))); + decoder.configure({ codec: this.config.codec, description: this.config.description ? Util.Hex.toBytes(this.config.description) : undefined, @@ -464,6 +477,9 @@ class DecoderTrack { this.buffered.update(() => Container.mergeBufferedRanges(network, decode)); }); + // Publish the measured arrival spread for Sync. + effect.run((inner) => this.spread.set(inner.get(consumer.spread))); + // Configure decoder with description from catalog decoder.configure({ codec: this.config.codec,