Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions js/justfile
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,11 @@ ring-bench:
bun install --frozen-lockfile
bun watch/bench/ring.ts

# Sweep audio rendition count against repeated mute/unmute catalog updates.
audio-support-bench:
bun install --frozen-lockfile
bun watch/bench/support.ts

# Sweep decoder count against PCM callback size during audio warmup.
audio-warmup-bench:
bun install --frozen-lockfile
Expand Down
81 changes: 81 additions & 0 deletions js/watch/bench/support.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
// Catalog-update costs with a warmed audio support verdict. Native probes run in browsers;
// Bun's promise probe isolates selection overhead and probe counts for the nightly series.
import type * as Catalog from "@moq/hang/catalog";
import { Signal } from "@moq/signals";
import { decoderConfigKey } from "../src/audio/config";
import { Source } from "../src/audio/source";
import type { Broadcast } from "../src/broadcast";
import { supportCacheKey } from "../src/support-probe";

/** Measure metadata updates over rendition count, checking every published config. */
export async function run(): Promise<void> {
const native = typeof AudioDecoder !== "undefined";
const probe = native
? async (config: Catalog.AudioConfig) =>
(
await AudioDecoder.isConfigSupported({
codec: config.codec,
sampleRate: config.sampleRate,
numberOfChannels: config.numberOfChannels,
})
).supported ?? false
: async () => true;
console.log("probe,renditions,updates,probes,us_per_update");
for (const count of [1, 8, 32]) {
let probes = 0;
const supported = Object.assign(
async (config: Catalog.AudioConfig) => {
probes++;
return probe(config);
},
{ [supportCacheKey]: decoderConfigKey },
);
const renditions: Record<string, Catalog.AudioConfig> = Object.fromEntries(
Array.from({ length: count }, (_, i) => [
`audio${i}`,
{
codec: "opus",
container: { kind: "legacy" },
sampleRate: 48_000,
numberOfChannels: 2,
} as Catalog.AudioConfig,
]),
);
const catalog = new Signal<Catalog.Root>({ audio: { renditions } });
const source = new Source({ broadcast: { out: { catalog } } as unknown as Broadcast, supported });
try {
await source.out.available.changed();
if (Object.keys(source.out.available.peek()).length !== count) throw new Error("unsupported fixture");
for (const [measured, updates] of [
[false, 100],
[true, 20],
[true, 200],
] as const) {
const before = probes;
const start = performance.now();
for (let update = 0; update < updates; update++) {
const enabled = update % 2 === 0;
const changed = source.out.available.changed();
catalog.set({
audio: {
renditions: Object.fromEntries(
Object.entries(renditions).map(([name, c]) => [name, { ...c, enabled }]),
),
},
});
await changed;
const available = Object.values(source.out.available.peek());
if (available.length !== count || available.some((c) => c.enabled !== enabled))
throw new Error("stale rendition metadata");
}
const us = ((performance.now() - start) * 1000) / updates;
if (!measured) continue;
console.log(`${native ? "native" : "promise"},${count},${updates},${probes - before},${us.toFixed(3)}`);
}
} finally {
source.close();
}
}
}

await run();
5 changes: 5 additions & 0 deletions js/watch/src/audio/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,11 @@ export function decoderConfig(config: Catalog.AudioConfig): DecoderConfig {
};
}

/** Return a stable cache key for the fields used by the decoder support probe. */
export function decoderConfigKey(config: Catalog.AudioConfig): string {
return JSON.stringify(decoderConfig(config));
}

/** Reduce a rendition config to the fields that require a replacement subscription. */
export function playbackIdentity(config: Catalog.AudioConfig): PlaybackIdentity {
return {
Expand Down
12 changes: 11 additions & 1 deletion js/watch/src/audio/decoder.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@ import * as Container from "@moq/hang/container";
import * as Moq from "@moq/net";
import { Effect, Signal } from "@moq/signals";
import type { Broadcast } from "../broadcast";
import { supportCacheKey } from "../support-probe";
import { type Delay, Sync } from "../sync";
import { Decoder as VideoDecoder } from "../video/decoder";
import { Source as VideoSource } from "../video/source";
import { decoderConfigKey } from "./config";
import type { Message, State } from "./render";
import { AudioRingBuffer } from "./ring-buffer";
import { Source } from "./source";
Expand Down Expand Up @@ -322,7 +324,9 @@ async function play(initial: Delay, buffered = false, cmaf = false) {
} as unknown as Broadcast);

const delay = new Signal<Delay>(initial);
const source = new Source({ broadcast, supported: async () => true });
const supportProbes = mock(async () => true);
Object.assign(supportProbes, { [supportCacheKey]: decoderConfigKey });
const source = new Source({ broadcast, supported: supportProbes });
const buffer = new Signal(Milli(buffered ? 1000 : 0));
const sync = new Sync({ delay, buffer });
// The decoder's own switch, which a mute or pause turns off.
Expand All @@ -343,6 +347,7 @@ async function play(initial: Delay, buffered = false, cmaf = false) {
};

return {
supportProbes,
delay,
buffer,
sync,
Expand Down Expand Up @@ -718,10 +723,15 @@ describe("Decoder across a rendition's absence", () => {
await microtasks();
expect(playback.track()).toBe("audio");
expect(contexts).toHaveLength(1);
drain(playback.rings()[0]);
await microtasks();
expect(contexts[0].state).toBe("suspended");

playback.restore();
await microtasks();
expect(playback.track()).toBe("audio");
expect(contexts[0].state).toBe("running");
expect(playback.supportProbes).toHaveBeenCalledTimes(1);
const before = playback.inserted().length;
const timestamps = await playback.play(6);

Expand Down
12 changes: 11 additions & 1 deletion js/watch/src/audio/decoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,17 @@ import { Producer as ArrivalProducer, type Arrivals, gap } from "../arrivals";
import { hostedAssets } from "../assets";
import { base64ToBytes } from "../base64";
import { type Failure, failureMoved, nextMedia, subscribeMedia } from "../media";
import { supportCacheKey } from "../support-probe";
import type { Sync } from "../sync";
import { AudioBuffer } from "./buffer";
import { type DecoderConfig, frameDuration, type PlaybackIdentity, packetDuration, playbackIdentity } from "./config";
import {
type DecoderConfig,
decoderConfigKey,
frameDuration,
type PlaybackIdentity,
packetDuration,
playbackIdentity,
} from "./config";
import { Handover } from "./handover";
import { AUTO_MAX_DELAY, ringHold, target } from "./latency";
import { Order } from "./order";
Expand Down Expand Up @@ -977,6 +985,8 @@ async function supported(config: Catalog.AudioConfig): Promise<boolean> {
return res.supported ?? false;
}

Object.assign(supported, { [supportCacheKey]: decoderConfigKey });

// Run `task` for one decode attempt. A setup, read, or decode failure that is not the attempt's
// cancellation fails its source through `failure`, which latches it and tears the attempt down.
function spawnFatal(effect: Effect, failure: Signal<Error | undefined>, task: () => Promise<void>): void {
Expand Down
36 changes: 36 additions & 0 deletions js/watch/src/audio/source.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ import { yieldNow } from "@moq/clock";
import * as Catalog from "@moq/hang/catalog";
import { Signal } from "@moq/signals";
import type { Broadcast } from "../broadcast";
import { supportCacheKey } from "../support-probe";
import { decoderConfigKey } from "./config";
import { Source } from "./source";

function config(codec: string, fields: Record<string, unknown> = {}): Catalog.AudioConfig {
Expand Down Expand Up @@ -310,3 +312,37 @@ test("serves unchanged configs from the cache when the catalog changes", async (
source.close();
}
});

test("mute and unmute reuse the decoder support verdict and publish the new metadata", async () => {
const initial = config("opus");
const state = broadcast({ mic: initial });
let probes = 0;
const supported = Object.assign(
async () => {
probes++;
return true;
},
{ [supportCacheKey]: decoderConfigKey },
);
const source = new Source({ broadcast: state.broadcast, supported });
try {
await settled(source);
for (const enabled of [false, true, false, true]) {
state.catalog.set({
audio: { renditions: { mic: config("opus", { enabled, jitter: 20, bitrate: 96_000 }) } },
});
await settled(source);
expect(source.out.available.peek().mic.enabled).toBe(enabled);
expect(source.out.track.peek()).toBe("mic");
expect(source.out.config.peek()?.enabled).toBe(enabled);
expect(probes).toBe(1);
}

// A field the decoder depends on still invalidates the verdict.
state.catalog.set({ audio: { renditions: { mic: config("opus", { sampleRate: 44100 }) } } });
await settled(source);
expect(probes).toBe(2);
} finally {
source.close();
}
});
34 changes: 33 additions & 1 deletion js/watch/src/audio/supported.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import { expect, mock, test } from "bun:test";
import { expect, mock, spyOn, test } from "bun:test";
import * as Catalog from "@moq/hang/catalog";
import { Signal } from "@moq/signals";
import type { Broadcast } from "../broadcast";
import { Source } from "./source";

// Bun cannot load the blob-URL worklet import, so mock it before the decoder imports it.
mock.module("./render-worklet.ts?worklet", () => ({ default: async () => "blob:fake-render" }));
Expand All @@ -22,3 +25,32 @@ test("renditions are probed through the libav fallback without a native AudioDec
expect(await Decoder.supported(config({ codec: "opus" }))).toBe(true);
expect(await Decoder.supported(config({ codec: "flac" }))).toBe(false);
});

test("the decoder support probe is reused through mute and unmute", async () => {
const config = Catalog.AudioConfigSchema.parse({
codec: "opus",
container: { kind: "legacy" },
sampleRate: 48000,
numberOfChannels: 2,
});
// Load the real libav decoder before counting its WebCodecs support queries.
await Decoder.supported(config);
using queries = spyOn(AudioDecoder, "isConfigSupported");
const catalog = new Signal<Catalog.Root>({ audio: { renditions: { audio: config } } });
const source = new Source({
broadcast: { out: { catalog } } as unknown as Broadcast,
supported: Decoder.supported,
});
try {
await source.out.available.changed();
for (const enabled of [false, true]) {
const changed = source.out.available.changed();
catalog.set({ audio: { renditions: { audio: { ...config, enabled } } } });
await changed;
expect(source.out.available.peek().audio.enabled).toBe(enabled);
}
expect(queries).toHaveBeenCalledTimes(1);
} finally {
source.close();
}
});
6 changes: 1 addition & 5 deletions quest/m1/push-to-talk/README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Push-to-talk loses no speech
# [M] Push-to-talk loses no speech

## Goal

Expand Down Expand Up @@ -53,10 +53,6 @@ encoder. Capture now stays open, so only a device swap, a settings change or a
new `source` recovers one. Decide whether re-enabling should retry a failed
encoder, and record why.

## Required

- [Watch caches audio codec support](/quest/m1/push-to-talk/audio-support-cache.md) - an unmute resumes audio without waiting on a support check

## Closes

- [#7101](https://github.com/moq-dev/moq/issues/7101) - a disabled audio rendition is deselected, so the start of the next utterance is lost
17 changes: 0 additions & 17 deletions quest/m1/push-to-talk/audio-support-cache.md

This file was deleted.

1 change: 1 addition & 0 deletions sh/gh/nightly.sh
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ case "$1" in
javascript slow-group just js slow-group-bench
javascript jitter just js jitter-bench
javascript audio-warmup just js audio-warmup-bench
javascript audio-support just js audio-support-bench
javascript clock just js clock-bench
;;
# The binding benchmark's runners at a token budget, so they keep working at this
Expand Down
Loading