diff --git a/js/justfile b/js/justfile index c69c42d0672..4957055f73d 100644 --- a/js/justfile +++ b/js/justfile @@ -156,6 +156,11 @@ ring-bench: bun install --frozen-lockfile bun watch/bench/ring.ts +# Sweep audio rendition count against repeated mute/unmute catalog updates. +audio-support-bench: + bun install --frozen-lockfile + bun watch/bench/support.ts + # Sweep decoder count against PCM callback size during audio warmup. audio-warmup-bench: bun install --frozen-lockfile diff --git a/js/watch/bench/support.ts b/js/watch/bench/support.ts new file mode 100644 index 00000000000..0e72f646481 --- /dev/null +++ b/js/watch/bench/support.ts @@ -0,0 +1,81 @@ +// Catalog-update costs with a warmed audio support verdict. Native probes run in browsers; +// Bun's promise probe isolates selection overhead and probe counts for the nightly series. +import type * as Catalog from "@moq/hang/catalog"; +import { Signal } from "@moq/signals"; +import { decoderConfigKey } from "../src/audio/config"; +import { Source } from "../src/audio/source"; +import type { Broadcast } from "../src/broadcast"; +import { supportCacheKey } from "../src/support-probe"; + +/** Measure metadata updates over rendition count, checking every published config. */ +export async function run(): Promise { + const native = typeof AudioDecoder !== "undefined"; + const probe = native + ? async (config: Catalog.AudioConfig) => + ( + await AudioDecoder.isConfigSupported({ + codec: config.codec, + sampleRate: config.sampleRate, + numberOfChannels: config.numberOfChannels, + }) + ).supported ?? false + : async () => true; + console.log("probe,renditions,updates,probes,us_per_update"); + for (const count of [1, 8, 32]) { + let probes = 0; + const supported = Object.assign( + async (config: Catalog.AudioConfig) => { + probes++; + return probe(config); + }, + { [supportCacheKey]: decoderConfigKey }, + ); + const renditions: Record = Object.fromEntries( + Array.from({ length: count }, (_, i) => [ + `audio${i}`, + { + codec: "opus", + container: { kind: "legacy" }, + sampleRate: 48_000, + numberOfChannels: 2, + } as Catalog.AudioConfig, + ]), + ); + const catalog = new Signal({ audio: { renditions } }); + const source = new Source({ broadcast: { out: { catalog } } as unknown as Broadcast, supported }); + try { + await source.out.available.changed(); + if (Object.keys(source.out.available.peek()).length !== count) throw new Error("unsupported fixture"); + for (const [measured, updates] of [ + [false, 100], + [true, 20], + [true, 200], + ] as const) { + const before = probes; + const start = performance.now(); + for (let update = 0; update < updates; update++) { + const enabled = update % 2 === 0; + const changed = source.out.available.changed(); + catalog.set({ + audio: { + renditions: Object.fromEntries( + Object.entries(renditions).map(([name, c]) => [name, { ...c, enabled }]), + ), + }, + }); + await changed; + const available = Object.values(source.out.available.peek()); + if (available.length !== count || available.some((c) => c.enabled !== enabled)) + throw new Error("stale rendition metadata"); + } + const us = ((performance.now() - start) * 1000) / updates; + if (!measured) continue; + console.log(`${native ? "native" : "promise"},${count},${updates},${probes - before},${us.toFixed(3)}`); + } + } finally { + source.close(); + } + } +} + +await run(); diff --git a/js/watch/src/audio/config.ts b/js/watch/src/audio/config.ts index ba4d7e02e47..103a4839f61 100644 --- a/js/watch/src/audio/config.ts +++ b/js/watch/src/audio/config.ts @@ -36,6 +36,11 @@ export function decoderConfig(config: Catalog.AudioConfig): DecoderConfig { }; } +/** Return a stable cache key for the fields used by the decoder support probe. */ +export function decoderConfigKey(config: Catalog.AudioConfig): string { + return JSON.stringify(decoderConfig(config)); +} + /** Reduce a rendition config to the fields that require a replacement subscription. */ export function playbackIdentity(config: Catalog.AudioConfig): PlaybackIdentity { return { diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index c8a006b4d4e..f4fbfb6116b 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -6,9 +6,11 @@ import * as Container from "@moq/hang/container"; import * as Moq from "@moq/net"; import { Effect, Signal } from "@moq/signals"; import type { Broadcast } from "../broadcast"; +import { supportCacheKey } from "../support-probe"; import { type Delay, Sync } from "../sync"; import { Decoder as VideoDecoder } from "../video/decoder"; import { Source as VideoSource } from "../video/source"; +import { decoderConfigKey } from "./config"; import type { Message, State } from "./render"; import { AudioRingBuffer } from "./ring-buffer"; import { Source } from "./source"; @@ -322,7 +324,9 @@ async function play(initial: Delay, buffered = false, cmaf = false) { } as unknown as Broadcast); const delay = new Signal(initial); - const source = new Source({ broadcast, supported: async () => true }); + const supportProbes = mock(async () => true); + Object.assign(supportProbes, { [supportCacheKey]: decoderConfigKey }); + const source = new Source({ broadcast, supported: supportProbes }); const buffer = new Signal(Milli(buffered ? 1000 : 0)); const sync = new Sync({ delay, buffer }); // The decoder's own switch, which a mute or pause turns off. @@ -343,6 +347,7 @@ async function play(initial: Delay, buffered = false, cmaf = false) { }; return { + supportProbes, delay, buffer, sync, @@ -718,10 +723,15 @@ describe("Decoder across a rendition's absence", () => { await microtasks(); expect(playback.track()).toBe("audio"); expect(contexts).toHaveLength(1); + drain(playback.rings()[0]); + await microtasks(); + expect(contexts[0].state).toBe("suspended"); playback.restore(); await microtasks(); expect(playback.track()).toBe("audio"); + expect(contexts[0].state).toBe("running"); + expect(playback.supportProbes).toHaveBeenCalledTimes(1); const before = playback.inserted().length; const timestamps = await playback.play(6); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 3373fc636f4..8bca327e8ad 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -17,9 +17,17 @@ import { Producer as ArrivalProducer, type Arrivals, gap } from "../arrivals"; import { hostedAssets } from "../assets"; import { base64ToBytes } from "../base64"; import { type Failure, failureMoved, nextMedia, subscribeMedia } from "../media"; +import { supportCacheKey } from "../support-probe"; import type { Sync } from "../sync"; import { AudioBuffer } from "./buffer"; -import { type DecoderConfig, frameDuration, type PlaybackIdentity, packetDuration, playbackIdentity } from "./config"; +import { + type DecoderConfig, + decoderConfigKey, + frameDuration, + type PlaybackIdentity, + packetDuration, + playbackIdentity, +} from "./config"; import { Handover } from "./handover"; import { AUTO_MAX_DELAY, ringHold, target } from "./latency"; import { Order } from "./order"; @@ -977,6 +985,8 @@ async function supported(config: Catalog.AudioConfig): Promise { return res.supported ?? false; } +Object.assign(supported, { [supportCacheKey]: decoderConfigKey }); + // Run `task` for one decode attempt. A setup, read, or decode failure that is not the attempt's // cancellation fails its source through `failure`, which latches it and tears the attempt down. function spawnFatal(effect: Effect, failure: Signal, task: () => Promise): void { diff --git a/js/watch/src/audio/source.test.ts b/js/watch/src/audio/source.test.ts index 2a3dfc28ddf..0f97b2ef69d 100644 --- a/js/watch/src/audio/source.test.ts +++ b/js/watch/src/audio/source.test.ts @@ -3,6 +3,8 @@ import { yieldNow } from "@moq/clock"; import * as Catalog from "@moq/hang/catalog"; import { Signal } from "@moq/signals"; import type { Broadcast } from "../broadcast"; +import { supportCacheKey } from "../support-probe"; +import { decoderConfigKey } from "./config"; import { Source } from "./source"; function config(codec: string, fields: Record = {}): Catalog.AudioConfig { @@ -310,3 +312,37 @@ test("serves unchanged configs from the cache when the catalog changes", async ( source.close(); } }); + +test("mute and unmute reuse the decoder support verdict and publish the new metadata", async () => { + const initial = config("opus"); + const state = broadcast({ mic: initial }); + let probes = 0; + const supported = Object.assign( + async () => { + probes++; + return true; + }, + { [supportCacheKey]: decoderConfigKey }, + ); + const source = new Source({ broadcast: state.broadcast, supported }); + try { + await settled(source); + for (const enabled of [false, true, false, true]) { + state.catalog.set({ + audio: { renditions: { mic: config("opus", { enabled, jitter: 20, bitrate: 96_000 }) } }, + }); + await settled(source); + expect(source.out.available.peek().mic.enabled).toBe(enabled); + expect(source.out.track.peek()).toBe("mic"); + expect(source.out.config.peek()?.enabled).toBe(enabled); + expect(probes).toBe(1); + } + + // A field the decoder depends on still invalidates the verdict. + state.catalog.set({ audio: { renditions: { mic: config("opus", { sampleRate: 44100 }) } } }); + await settled(source); + expect(probes).toBe(2); + } finally { + source.close(); + } +}); diff --git a/js/watch/src/audio/supported.test.ts b/js/watch/src/audio/supported.test.ts index 0505ae3fe27..5022d0448e4 100644 --- a/js/watch/src/audio/supported.test.ts +++ b/js/watch/src/audio/supported.test.ts @@ -1,5 +1,8 @@ -import { expect, mock, test } from "bun:test"; +import { expect, mock, spyOn, test } from "bun:test"; import * as Catalog from "@moq/hang/catalog"; +import { Signal } from "@moq/signals"; +import type { Broadcast } from "../broadcast"; +import { Source } from "./source"; // Bun cannot load the blob-URL worklet import, so mock it before the decoder imports it. mock.module("./render-worklet.ts?worklet", () => ({ default: async () => "blob:fake-render" })); @@ -22,3 +25,32 @@ test("renditions are probed through the libav fallback without a native AudioDec expect(await Decoder.supported(config({ codec: "opus" }))).toBe(true); expect(await Decoder.supported(config({ codec: "flac" }))).toBe(false); }); + +test("the decoder support probe is reused through mute and unmute", async () => { + const config = Catalog.AudioConfigSchema.parse({ + codec: "opus", + container: { kind: "legacy" }, + sampleRate: 48000, + numberOfChannels: 2, + }); + // Load the real libav decoder before counting its WebCodecs support queries. + await Decoder.supported(config); + using queries = spyOn(AudioDecoder, "isConfigSupported"); + const catalog = new Signal({ audio: { renditions: { audio: config } } }); + const source = new Source({ + broadcast: { out: { catalog } } as unknown as Broadcast, + supported: Decoder.supported, + }); + try { + await source.out.available.changed(); + for (const enabled of [false, true]) { + const changed = source.out.available.changed(); + catalog.set({ audio: { renditions: { audio: { ...config, enabled } } } }); + await changed; + expect(source.out.available.peek().audio.enabled).toBe(enabled); + } + expect(queries).toHaveBeenCalledTimes(1); + } finally { + source.close(); + } +}); diff --git a/quest/m1/push-to-talk/README.md b/quest/m1/push-to-talk/README.md index e754b81ebc0..ce53ae84400 100644 --- a/quest/m1/push-to-talk/README.md +++ b/quest/m1/push-to-talk/README.md @@ -1,4 +1,4 @@ -# Push-to-talk loses no speech +# [M] Push-to-talk loses no speech ## Goal @@ -53,10 +53,6 @@ encoder. Capture now stays open, so only a device swap, a settings change or a new `source` recovers one. Decide whether re-enabling should retry a failed encoder, and record why. -## Required - -- [Watch caches audio codec support](/quest/m1/push-to-talk/audio-support-cache.md) - an unmute resumes audio without waiting on a support check - ## Closes - [#7101](https://github.com/moq-dev/moq/issues/7101) - a disabled audio rendition is deselected, so the start of the next utterance is lost diff --git a/quest/m1/push-to-talk/audio-support-cache.md b/quest/m1/push-to-talk/audio-support-cache.md deleted file mode 100644 index 7f19ada9777..00000000000 --- a/quest/m1/push-to-talk/audio-support-cache.md +++ /dev/null @@ -1,17 +0,0 @@ -# [XS] Watch caches audio codec support - -## Goal - -`js/watch`'s audio source checks codec support once per rendition config, as -video does, so an unmute resumes the AudioContext without waiting on a support -check. - -## Plan - -Found 2026-10-28 by [moq-dev/moq#7155](https://github.com/moq-dev/moq/pull/7155): -the audio source re-checks codec support on every catalog change, while video -caches the result. So on unmute the AudioContext resumes only after a support -check of a few milliseconds, even though frames are already arriving. Share -the cache with video if that keeps it simpler. - -Public API: none. Wire: none. diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index d682cd95a3c..a46a4b12484 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -156,6 +156,7 @@ case "$1" in javascript slow-group just js slow-group-bench javascript jitter just js jitter-bench javascript audio-warmup just js audio-warmup-bench + javascript audio-support just js audio-support-bench javascript clock just js clock-bench ;; # The binding benchmark's runners at a token budget, so they keep working at this