Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 22 additions & 7 deletions js/publish/src/audio/encoder.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { Time } from "@moq/net";
import { Signal } from "@moq/signals";
import { Baseline } from "../jitter";
import type { AudioFrame, Format } from "./capture";
import { Encoder, resolve } from "./encoder";
import { type Codec, Encoder, resolve, toEncoderConfig } from "./encoder";

// Bun does not load Vite's worklet URL imports from the public audio entrypoint.
mock.module("./capture-worklet.ts?worklet", () => ({ default: "blob:fake-capture" }));
Expand Down Expand Up @@ -55,14 +55,21 @@ describe("resolve", () => {
});
});

describe("toEncoderConfig", () => {
test("configures voice without DTX", () => {
const config = toEncoderConfig(resolve(captured, "opus"), "voice", {});
expect(config.opus).toEqual({ application: "voip", signal: "voice", frameDuration: 20_000 } as never);
});
});

// Like Chrome's Opus encoder, it holds the newest chunks until later input pushes them out, and
// stamps each chunk from the first input's timestamp plus the audio encoded since, so a jump in
// input timestamps never reaches the output.
class LaggingAudioEncoder {
static readonly LAG = 2;

// Called on configure; the encoder publishes its pipeline synchronously right after.
static onConfigure: (() => void) | undefined;
static onConfigure: ((config: AudioEncoderConfig) => void) | undefined;

state: CodecState = "unconfigured";
#output: EncodedAudioChunkOutputCallback;
Expand All @@ -74,9 +81,9 @@ class LaggingAudioEncoder {
this.#output = init.output;
}

configure(): void {
configure(config: AudioEncoderConfig): void {
this.state = "configured";
LaggingAudioEncoder.onConfigure?.();
LaggingAudioEncoder.onConfigure?.(config);
}

encode(data: AudioData): void {
Expand Down Expand Up @@ -186,8 +193,8 @@ class Feed {
}

// An Encoder wired to a fake capture feed, recording each written frame as [timestamp, payload bytes].
async function setup(baseline = new Baseline()) {
const configured = new Promise<void>((resolve) => {
async function setup(baseline = new Baseline(), codec?: Codec) {
const configured = new Promise<AudioEncoderConfig>((resolve) => {
LaggingAudioEncoder.onConfigure = resolve;
});

Expand Down Expand Up @@ -222,12 +229,14 @@ async function setup(baseline = new Baseline()) {
const encoder = new Encoder("audio", {
broadcast: { audio: () => rendition, baseline } as never,
capture: capture as never,
codec,
});

await configured;
const config = await configured;
LaggingAudioEncoder.onConfigure = undefined;

return {
config,
encoder,
track,
rendition,
Expand Down Expand Up @@ -301,6 +310,12 @@ test("a push completing several frames keeps the encoder running", async () => {
]);
});

test("passes an explicit Opus DTX request to the encoder", async () => {
using _webcodecs = installFakeWebCodecs();
using env = await setup(new Baseline(), { mime: "opus", usedtx: true });
expect(env.config.opus?.usedtx).toBe(true);
});

// Another rendition on the same broadcast flushing with far less lateness leaves this one trailing
// it, which the catalog advertises as `delay`.
test("a rendition trailing the broadcast's earliest advertises delay", async () => {
Expand Down
24 changes: 14 additions & 10 deletions js/publish/src/audio/encoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,10 @@ export type OpusConfig = {
complexity?: number; // 0-10, higher is better quality but more CPU
packetlossperc?: number; // 0-100, expected loss the encoder optimizes for
useinbandfec?: boolean; // in-band forward error correction
usedtx?: boolean; // discontinuous transmission (silence suppression)
// Discontinuous transmission (silence suppression), off by default. Chromium stamps Opus output by
// counting the samples emitted, so each frame DTX suppresses pulls later audio earlier and the
// published timeline drifts from the capture clock until that is fixed.
usedtx?: boolean;
};

/** Cumulative encoder output totals, measured from the chunks the encoder produces. */
Expand Down Expand Up @@ -669,25 +672,26 @@ interface OpusEncoderConfigExt extends OpusEncoderConfig {
}

// Opus settings implied by the audio kind. These are only defaults: any field set explicitly via
// OpusConfig (carried in opusOptions) overrides them, so a caller can always opt out. DTX (silence
// suppression) is enabled for voice, where speech has natural gaps that collapse to tiny
// comfort-noise packets. Music has no useful silence to suppress, and "auto" leaves every knob to
// the browser.
// OpusConfig (carried in opusOptions) overrides them, so a caller can always opt out. "auto" leaves
// every knob to the browser.
function opusKindDefaults(kind: Kind): OpusEncoderConfigExt {
switch (kind) {
case "voice":
return { application: "voip", signal: "voice", usedtx: true };
return { application: "voip", signal: "voice" };
case "music":
return { application: "audio", signal: "music" };
default:
return {};
}
}

// Build the WebCodecs encoder config from the catalog (decoder) config, a Kind hint, and any
// Opus-only knobs. Those knobs are kept out of the catalog since they only affect encoding. AAC has
// no such knobs, so it just uses the shared base fields (codec/sampleRate/channels/bitrate).
function toEncoderConfig(resolved: Resolved, kind: Kind, opusOptions: OpusEncoderConfigExt): AudioEncoderConfig {
/**
* Build the WebCodecs encoder config from the catalog (decoder) config, a Kind hint, and any
* Opus-only knobs. Those knobs are kept out of the catalog since they only affect encoding. AAC has
* no such knobs, so it just uses the shared base fields (codec/sampleRate/channels/bitrate).
* @internal
*/
export function toEncoderConfig(resolved: Resolved, kind: Kind, opusOptions: OpusEncoderConfigExt): AudioEncoderConfig {
const config = resolved.catalog;
const encoderConfig: AudioEncoderConfig = {
codec: config.codec,
Expand Down
4 changes: 2 additions & 2 deletions js/publish/src/audio/types.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// The kind of audio being encoded. Drives the default Opus application/signal/DTX settings on the
// The kind of audio being encoded. Drives the default Opus application/signal settings on the
// encoder, each of which an explicit OpusConfig can still override.
// - "voice": speech (microphone). Opus application=voip, signal=voice, and DTX on (speech has gaps).
// - "voice": speech (microphone). Opus application=voip + signal=voice.
// - "music": music or mixed content (screen/tab capture). Opus application=audio + signal=music.
// - "auto": let the encoder decide. Opus defaults (good for unknown sources like file playback).
export type Kind = "voice" | "music" | "auto";
Expand Down
Loading