diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index de9fc3010e4..79c5e2383bb 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -1,9 +1,9 @@ import { afterEach, beforeEach, describe, expect, it, jest, mock, spyOn } from "bun:test"; -import { Micro, Milli, Nano, now } from "@moq/clock"; +import { Manual, Micro, Milli, Nano, now } from "@moq/clock"; import * as Catalog from "@moq/hang/catalog"; import * as Container from "@moq/hang/container"; import * as Moq from "@moq/net"; -import { Signal } from "@moq/signals"; +import { Effect, Signal } from "@moq/signals"; import type { Broadcast } from "../broadcast"; import { type Delay, Sync } from "../sync"; import { SharedRingBuffer } from "./shared-ring-buffer"; @@ -663,3 +663,215 @@ describe("fatal decoder errors", () => { } }); }); + +// A manual clock installed at `ms`, so arrivals are stamped only as fast as a test advances it. +function manualAt(ms: number) { + const clock = new Manual(); + clock.advance(Nano.fromMilli(Milli(ms))); + const restore = clock.install(); + return { advance: (by: number) => clock.advanceAsync(Nano.fromMilli(Milli(by))), restore }; +} + +// A legacy container frame: the media timestamp, then one payload byte. +function legacyFrame(media: Micro): Moq.Group.Frame { + const timestamp = Moq.Varint.encode(media); + const payload = new Uint8Array(timestamp.byteLength + 1); + payload.set(timestamp, 0); + return { payload, timestamp: Moq.Time.Timestamp.fromMicros(media) }; +} + +// A live "auto" playback reading real tracks through real container consumers on a manual clock. +// One 20ms Opus frame per group, flushed in runs of `burst`, so each run's first frame lands that +// much late. Each tick renders 20ms from the decoder's ring the way the worklet does. +async function livePlayback(burst: number) { + const FRAME = 20; + const clock = manualAt(0); + const insert = spyOn(SharedRingBuffer.prototype, "insert"); + + const audio = Catalog.AudioConfigSchema.parse({ + codec: "opus", + container: { kind: "legacy" }, + sampleRate: 48_000, + numberOfChannels: 2, + }); + const catalog = new Signal({ audio: { renditions: { audio } } }); + // The broadcast the rendition reads, which a `broadcast` override can move to another path. + let producer = new Moq.Broadcast.Producer(); + const media = new Signal(producer.consume()); + const broadcast = new Signal({ + in: { enabled: new Signal(true) }, + out: { catalog, active: new Signal({ closed: {} }) }, + relativeBroadcast: (effect: Effect) => effect.get(media), + } as unknown as Broadcast); + const source = new Source({ broadcast, supported: async () => true }); + const sync = new Sync({ delay: new Signal("auto") }); + // Published before the decoder subscribes, since a broadcast with no request handler refuses + // an unknown track. + let track = producer.createTrack("audio", { timescale: Moq.Time.Timescale.MILLI }); + const decoder = new Decoder({ source, sync }); + + const samples = (48_000 * FRAME) / 1000; + const output = [new Float32Array(samples), new Float32Array(samples)]; + const counts = { inserted: 0, played: 0, silent: 0 }; + let frame = 0; + let sequence = 0; + let held: number[] = []; + + const ring = () => insert.mock.contexts.at(-1) as SharedRingBuffer | undefined; + return { + sync, + error: decoder.out.error, + counts, + ring, + get burst() { + return burst; + }, + set burst(value: number) { + burst = value; + }, + // Rename the rendition, publishing the rest of the timeline on the new track. + rename(name: string, patch: Record = {}) { + track.close(); + track = producer.createTrack(name, { timescale: Moq.Time.Timescale.MILLI }); + sequence = 0; + catalog.set({ audio: { renditions: { [name]: Catalog.AudioConfigSchema.parse({ ...audio, ...patch }) } } }); + }, + // Serve the same rendition from another path, as a `broadcast` override does. + move(path: string) { + media.peek().close(); + producer = new Moq.Origin.Producer().createBroadcast(Moq.Path.from(path)); + track = producer.createTrack("audio", { timescale: Moq.Time.Timescale.MILLI }); + sequence = 0; + media.set(producer.consume()); + }, + // Remove the rendition from the catalog. + remove: () => catalog.set({}), + // Publish every frame due through `until`, rendering 20ms per tick. + async run(until: number) { + for (let t = frame * FRAME; t <= until; t += FRAME) { + await clock.advance(FRAME); + for (; frame * FRAME <= t; frame++) { + held.push(frame * FRAME); + if (held.length < burst) continue; + for (const timestamp of held) { + const group = new Moq.Group.Producer(sequence++); + group.writeFrame(legacyFrame(Micro.fromMilli(Milli(timestamp)))); + group.close(); + track.writeGroup(group); + } + held = []; + } + await new Promise((resolve) => setImmediate(resolve)); + await microtasks(); + + const current = ring(); + if (!current || current.stalled) continue; + const read = current.read(output); + counts.played += read; + if (read < samples) counts.silent++; + } + counts.inserted = insert.mock.calls.reduce((sum, [, data]) => sum + (data as Float32Array[])[0].length, 0); + }, + async close() { + decoder.close(); + source.close(); + sync.close(); + media.peek().close(); + producer.close(); + await microtasks(); + clock.restore(); + }, + }; +} + +describe("Decoder across a rename", () => { + // A codec change renames the audio track, and the replacement's consumer reads the same path. + // Clearing the estimate at teardown and restarting it from the cold-start prior dropped the + // shared "auto" delay and climbed back as the new track relearned the lateness, so the ring + // skipped ahead on the drop and went silent refilling on each rise. + it("keeps the delay, so the ring neither skips nor goes silent", async () => { + // Flushed in runs of 12, so each run's first frame lands ~220ms late. + const playback = await livePlayback(12); + try { + // Long enough for the estimate to settle on the lateness this path shows. + await playback.run(20_000); + const delay = playback.sync.out.delay.peek(); + expect(delay).toBeGreaterThan(Milli(200)); + const ring = playback.ring(); + const before = { ...playback.counts, buffered: ring?.length ?? 0 }; + let lowest = delay; + const watch = new Effect(); + watch.subscribe(playback.sync.out.delay, (value) => { + lowest = Milli(Math.min(lowest, value)); + }); + + playback.rename("audio/1"); + await playback.run(26_000); + watch.close(); + + const { inserted, played, silent } = playback.counts; + const buffered = playback.ring()?.length ?? 0; + const skipped = inserted - before.inserted - (played - before.played) - (buffered - before.buffered); + expect({ skipped, silent: silent - before.silent }).toEqual({ skipped: 0, silent: 0 }); + expect(playback.ring()).toBe(ring); + expect(lowest).toBeGreaterThanOrEqual(delay); + } finally { + await playback.close(); + } + }); + + // Another path's arrivals say nothing about this one's, so a rendition read from elsewhere + // learns its own estimate instead of inheriting the outgoing one. + it("does not carry the delay to another path", async () => { + const playback = await livePlayback(12); + try { + await playback.run(20_000); + const warm = playback.sync.out.delay.peek(); + expect(warm).toBeGreaterThan(Milli(200)); + + // The same media from another path, where every frame lands on time. + playback.move("elsewhere"); + playback.burst = 1; + await playback.run(22_000); + + expect(playback.sync.out.delay.peek()).toBeLessThan(warm); + } finally { + await playback.close(); + } + }); + + // The probe accepts a CMAF init segment it cannot parse for Opus, so setup throws before the + // replacement's consumer exists. Nothing replaces the outgoing estimate, so it must not linger. + it("reports a replacement that fails to set up and stops holding the delay", async () => { + const playback = await livePlayback(12); + try { + await playback.run(20_000); + const warm = playback.sync.out.delay.peek(); + expect(warm).toBeGreaterThan(Milli(200)); + + playback.rename("audio/1", { container: { kind: "cmaf", init: "AAAA" } }); + await microtasks(); + + expect(playback.error.peek()).toBeInstanceOf(Error); + expect(playback.sync.out.delay.peek()).toBeLessThan(warm); + } finally { + await playback.close(); + } + }); + + it("stops holding the delay once the rendition is removed", async () => { + const playback = await livePlayback(12); + try { + await playback.run(20_000); + const warm = playback.sync.out.delay.peek(); + expect(warm).toBeGreaterThan(Milli(200)); + + playback.remove(); + await microtasks(); + + expect(playback.sync.out.delay.peek()).toBeLessThan(warm); + } finally { + await playback.close(); + } + }); +}); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index a1bdd14258d..0642da2453e 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -122,6 +122,10 @@ export class Decoder { // The container consumer's arrival estimate, unset while nothing is subscribed. #measured = new Signal(undefined); + // The newest container consumer, the path it reads, and its codec, whose estimate a replacement + // continues. + #consumer?: { path: Moq.Path.Valid; codec: string; consumer: Container.Consumer }; + // The subscription's max delay: the shared budget, raised to the estimator's ceiling in "auto". #subscribeMaxDelay = new Signal(Milli.zero); #maxDelay = new Signal(Milli.zero); @@ -342,15 +346,23 @@ export class Decoder { this.#out.error.set(undefined); } - if (!running || !broadcast || !track || !identity) return; + if (!running || !broadcast || !track || !identity) { + this.#retire(); + return; + } const config = identity.decoder; - this.#frame.set(frameDuration(config)); - if (!active || !key) return; + if (!active || !key) { + this.#retire(); + return; + } // This source already failed; wait for another. - if (this.#failed) return; + if (this.#failed) { + this.#retire(); + return; + } // Closing this effect releases both its container reader and track subscription. const failure = new Signal(undefined); let attempt: Effect | undefined; @@ -365,6 +377,7 @@ export class Decoder { }; this.#out.error.set(error); attempt?.close(); + this.#retire(); }); // Another broadcast (a new name, or a republish) brings its own timeline, which a ring and a @@ -394,12 +407,22 @@ export class Decoder { priority: Catalog.PRIORITY.audio, maxDelay: this.#subscribeMaxDelay, }); - if (!sub) return; + if (!sub) { + this.#retire(); + return; + } - if (config.container.kind === "cmaf") { - this.#runCmafDecoder(run, sub, { config, failure }); - } else { - this.#runLegacyDecoder(run, sub, { config, failure }); + // A config the probe accepted can still fail to parse (a malformed init segment or Opus + // header). Report it like a decode failure, which also retires the outgoing estimate. + const props = { config, failure, path: active.path }; + try { + if (config.container.kind === "cmaf") { + this.#runCmafDecoder(run, sub, props); + } else { + this.#runLegacyDecoder(run, sub, props); + } + } catch (error) { + failure.set(error instanceof Error ? error : new Error(String(error))); } }); } @@ -407,7 +430,7 @@ export class Decoder { #runLegacyDecoder( effect: Effect, sub: Moq.Track.Subscriber, - { config, failure }: { config: DecoderConfig; failure: Signal }, + { config, failure, path }: { config: DecoderConfig; failure: Signal; path: Moq.Path.Valid }, ): void { const preSkip = config.codec === "opus" && config.description ? Util.Opus.preSkip(Util.Hex.toBytes(config.description)) : 0; @@ -416,23 +439,7 @@ export class Decoder { config.container.kind === "loc" ? new Container.Loc.Format("audio") : new Container.Legacy.Format(config); // Create consumer with slightly less latency than the render worklet to avoid underflowing. // TODO include JITTER_UNDERHEAD - const consumer = new Container.Consumer(sub, { - format, - maxDelay: this.#maxDelay, - }); - effect.cleanup(() => consumer.close()); - - // Combine network jitter buffer with decode buffer - effect.run((inner) => { - const network = inner.get(consumer.buffered); - const decode = inner.get(this.#decodeBuffered); - this.#out.buffered.update(() => Container.mergeBufferedRanges(network, decode)); - }); - - // Feed the arrival estimate into the playout target. Cleared on teardown so a departed track - // stops holding the buffer open. - effect.run((inner) => this.#measured.set(inner.get(consumer.spread))); - effect.cleanup(() => this.#measured.set(undefined)); + const consumer = this.#consume(effect, sub, format, { config, path }); effect.spawn(async () => { const loaded = await Util.Libav.polyfill(); @@ -510,7 +517,7 @@ export class Decoder { #runCmafDecoder( effect: Effect, sub: Moq.Track.Subscriber, - { config, failure }: { config: DecoderConfig; failure: Signal }, + { config, failure, path }: { config: DecoderConfig; failure: Signal; path: Moq.Path.Valid }, ): void { if (config.container.kind !== "cmaf") return; // just to help typescript @@ -528,23 +535,7 @@ export class Decoder { ? Util.Hex.toBytes(config.description) : init.description; - const consumer = new Container.Consumer(sub, { - format: new Container.Cmaf.Format(init), - maxDelay: this.#maxDelay, - }); - effect.cleanup(() => consumer.close()); - - // Combine network jitter buffer with decode buffer - effect.run((inner) => { - const network = inner.get(consumer.buffered); - const decode = inner.get(this.#decodeBuffered); - this.#out.buffered.update(() => Container.mergeBufferedRanges(network, decode)); - }); - - // Feed the arrival estimate into the playout target. Cleared on teardown so a departed track - // stops holding the buffer open. - effect.run((inner) => this.#measured.set(inner.get(consumer.spread))); - effect.cleanup(() => this.#measured.set(undefined)); + const consumer = this.#consume(effect, sub, new Container.Cmaf.Format(init), { config, path }); effect.spawn(async () => { const loaded = await Util.Libav.polyfill(); @@ -612,6 +603,49 @@ export class Decoder { }); } + // Reorders groups and frames up to the shared max delay. A rename is not a cold start: the path's + // arrivals are what they were, so the estimate continues from whatever read it before. Another + // path's arrivals have nothing to do with what this one learned. + #consume( + effect: Effect, + sub: Moq.Track.Subscriber, + format: Container.Format, + { config, path }: { config: DecoderConfig; path: Moq.Path.Valid }, + ): Container.Consumer { + const outgoing = this.#consumer; + const continues = outgoing?.path === path; + const consumer = new Container.Consumer(sub, { + format, + maxDelay: this.#maxDelay, + predecessor: continues ? outgoing.consumer : undefined, + }); + effect.cleanup(() => consumer.close()); + this.#consumer = { path, codec: config.codec, consumer }; + + // Opus states its duration only per packet, so the same codec continuing keeps the outgoing + // one until the first packet refines it. Anything else starts from the codec's own constant. + const frame = frameDuration(config); + if (frame !== undefined || !continues || outgoing.codec !== config.codec) this.#frame.set(frame); + + // Combine network jitter buffer with decode buffer + effect.run((inner) => { + const network = inner.get(consumer.buffered); + const decode = inner.get(this.#decodeBuffered); + this.#out.buffered.update(() => Container.mergeBufferedRanges(network, decode)); + }); + + // Feed the arrival estimate into the playout target. It outlives this consumer, so the target + // holds until the replacement takes over; see #retire. + effect.run((inner) => this.#measured.set(inner.get(consumer.spread))); + return consumer; + } + + // No consumer replaces the outgoing one, so its estimate stops holding the buffer open. + #retire(): void { + this.#consumer = undefined; + this.#measured.set(undefined); + } + // The ring, once the worklet has loaded. Decoding waits for it, so the frames that arrive first // queue in the consumer instead of being decoded into nowhere. Undefined once `effect` is torn down. async #ready(effect: Effect): Promise { diff --git a/quest/m1/audio-jitter-target/README.md b/quest/m1/audio-jitter-target/README.md index 0d7975602a1..67b8fe24fbb 100644 --- a/quest/m1/audio-jitter-target/README.md +++ b/quest/m1/audio-jitter-target/README.md @@ -36,7 +36,6 @@ the native decode path too, asserting the same target series the browser's ## Required -- [Audio rename](/quest/m1/audio-jitter-target/audio-rename.md) - a renamed audio track continues the outgoing arrival estimate, so playback neither skips nor stalls - [Watch](/quest/m1/audio-jitter-target/watch.md) - the browser's measured target, proven on Chrome and Safari against the public relay - [AAC bursty budgets](/quest/m1/audio-jitter-target/aac-bursty-budgets.md) - post-fix nightlies confirm AAC no longer re-stalls, and its budgets are tightened from them - [Opus near-zero budgets](/quest/m1/audio-jitter-target/opus-near-zero-budgets.md) - post-flush nightlies confirm the first row no longer stalls, and its budgets are tightened from them diff --git a/quest/m1/audio-jitter-target/audio-rename.md b/quest/m1/audio-jitter-target/audio-rename.md deleted file mode 100644 index b1942936ebd..00000000000 --- a/quest/m1/audio-jitter-target/audio-rename.md +++ /dev/null @@ -1,47 +0,0 @@ -# [S] An audio rename keeps the arrival estimate - -## Goal - -When `@moq/publish` renames an audio track after a decoder identity change, -`@moq/watch` keeps playing audio without skipping or stalling: the replacement -audio consumer continues the outgoing track's arrival estimate, as video does. -A track read from another broadcast path still starts from the cold-start -prior, as `doc/concept/audio-jitter.md` ("Across renditions") requires. - -## Plan - -#5635 added `ConsumerProps.predecessor` to `@moq/hang` and the video hand-off -this quest mirrors; this quest adds no API of its own. - -Today the audio decoder's `#runLegacyDecoder` and `#runCmafDecoder` clear -`#measured` on teardown, before the replacement subscription opens, and the -new `Container.Consumer` starts from the prior. At a rename, the audio target -drops out of the shared delay, then returns at the cold 100 ms and climbs as -the new track relearns the path. The ring skips on the drop and parks on each -rise, the same failure #5635 fixed for video. - -Decided 2026-10-16 (recommended options; the paper trail is in the plan PR): - -- Hand the outgoing `Container.Consumer` to the replacement through - `ConsumerProps.predecessor`, only when both read the same broadcast path, - mirroring the video decoder. No new API. -- Keep the outgoing estimate in `#measured` until the replacement's consumer - exists. Clear it only when no replacement follows (disabled, muted, or the - rendition removed), so a departed track still stops holding the buffer open. -- Prove it with a unit test in `js/watch/src/audio/decoder.test.ts` on a - mocked clock: warm a bursty audio track, rename it, and assert the ring - neither skips nor goes silent and the delay never drops. A second case moves - the media to another path and asserts it does not inherit. No new - real-browser media case: the unit test reproduces the mechanism - deterministically, and the fixture would need a second audio codec - configuration to rename audio at all. - -Out of scope: `rs/moq-audio` builds a fresh `Target` per decode consumer, so a -native player that follows a rename restarts cold too. That needs its own -quest if native playback follows renames. - -Public API: none. Wire: none. - -## Related - -- [Browser follows a renamed track](/quest/m1/browser-rename-check.md) - the browser check that found the video stall