From 5e5b614dc018f6c66577ec5cbfba1376f8fcac62 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 23:13:30 -0800 Subject: [PATCH 01/19] chore(quest): claim quest/m1/push-to-talk/dtx.md Quest-Claim: 5e2c4ac25ce7cfa9 From 766cb0ade4a6fd013f12112d57b1b296f714c095 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 23:28:06 -0800 Subject: [PATCH 02/19] feat(publish): muting suppresses audio instead of disabling the rendition The audio encoder gains a muted knob: it feeds the codec nothing, stamps chunks from a queue of the inputs it fed, and marks the end once the codec has emitted them, so a break needs neither a flush nor a codec restart. Co-Authored-By: Claude Opus 5.5 --- js/publish/src/audio/encoder.test.ts | 253 ++++++++++++++++++--------- js/publish/src/audio/encoder.ts | 213 +++++++++++----------- js/publish/src/element.ts | 11 +- 3 files changed, 283 insertions(+), 194 deletions(-) diff --git a/js/publish/src/audio/encoder.test.ts b/js/publish/src/audio/encoder.test.ts index f31e1f2ffe5..578b318b146 100644 --- a/js/publish/src/audio/encoder.test.ts +++ b/js/publish/src/audio/encoder.test.ts @@ -96,14 +96,15 @@ describe("toEncoderConfig", () => { }); }); -// Like Chrome's Opus encoder, it holds the newest chunks until later input pushes them out, and -// stamps each chunk from the first input's timestamp plus the audio encoded since, so a jump in -// input timestamps never reaches the output. -class LaggingAudioEncoder { - static instances: LaggingAudioEncoder[] = []; +// Like Chrome's Opus encoder, it emits one chunk per input and stamps each from the first input's +// timestamp plus the audio encoded since, so a jump in input timestamps never reaches the output. +// `LAG` holds that many of the newest chunks until later input pushes them out, a slower codec than +// any browser measured, for the tests that need one. +class FakeAudioEncoder { + static instances: FakeAudioEncoder[] = []; static probes: AudioEncoderConfig[] = []; static description: ((config: AudioEncoderConfig) => Uint8Array) | undefined; - static readonly LAG = 2; + static LAG = 0; // Called on configure; the encoder publishes its pipeline synchronously right after. static onConfigure: ((config: AudioEncoderConfig) => void) | undefined; @@ -122,7 +123,7 @@ class LaggingAudioEncoder { constructor(init: AudioEncoderInit) { this.#output = init.output; this.#error = init.error; - LaggingAudioEncoder.instances.push(this); + FakeAudioEncoder.instances.push(this); } fail(error: Error): void { @@ -136,7 +137,7 @@ class LaggingAudioEncoder { this.#fresh = true; queueMicrotask(() => { this.#fresh = false; - if (!this.#probe) LaggingAudioEncoder.onConfigure?.(config); + if (!this.#probe) FakeAudioEncoder.onConfigure?.(config); }); } @@ -145,7 +146,7 @@ class LaggingAudioEncoder { this.#base ??= data.timestamp; this.#held.push({ timestamp: this.#base + this.#encoded, duration }); this.#encoded += duration; - while (this.#held.length > LaggingAudioEncoder.LAG) { + while (this.#held.length > FakeAudioEncoder.LAG) { this.#emit(this.#held.shift() as { timestamp: number; duration: number }); } } @@ -153,7 +154,7 @@ class LaggingAudioEncoder { #emit({ timestamp, duration }: { timestamp: number; duration: number }): void { const chunk = { type: "key", timestamp, duration, byteLength: 1, copyTo: (dest: Uint8Array) => dest.set([1]) }; const config = this.#config; - const description = config && LaggingAudioEncoder.description?.(config); + const description = config && FakeAudioEncoder.description?.(config); const metadata = config && description ? { @@ -171,9 +172,9 @@ class LaggingAudioEncoder { async flush(): Promise { if (this.#fresh) { this.#probe = true; - if (this.#config) LaggingAudioEncoder.probes.push(structuredClone(this.#config)); - const index = LaggingAudioEncoder.instances.indexOf(this); - if (index !== -1) LaggingAudioEncoder.instances.splice(index, 1); + if (this.#config) FakeAudioEncoder.probes.push(structuredClone(this.#config)); + const index = FakeAudioEncoder.instances.indexOf(this); + if (index !== -1) FakeAudioEncoder.instances.splice(index, 1); } for (const chunk of this.#held.splice(0)) this.#emit(chunk); } @@ -206,12 +207,13 @@ class FakeAudioData { } function installFakeWebCodecs() { - LaggingAudioEncoder.instances = []; - LaggingAudioEncoder.probes = []; - LaggingAudioEncoder.description = undefined; + FakeAudioEncoder.instances = []; + FakeAudioEncoder.probes = []; + FakeAudioEncoder.LAG = 0; + FakeAudioEncoder.description = undefined; const names = ["AudioEncoder", "AudioDecoder", "AudioData"] as const; const originals = names.map((name) => Object.getOwnPropertyDescriptor(globalThis, name)); - const fakes = [LaggingAudioEncoder, class {}, FakeAudioData]; + const fakes = [FakeAudioEncoder, class {}, FakeAudioData]; names.forEach((name, i) => { Object.defineProperty(globalThis, name, { configurable: true, writable: true, value: fakes[i] }); }); @@ -284,8 +286,8 @@ function fakeCapture(feed: Feed) { // Resolves on the next AudioEncoder configure; the encoder publishes its pipeline synchronously after. function configured(): Promise { return new Promise((resolve) => { - LaggingAudioEncoder.onConfigure = (config) => { - LaggingAudioEncoder.onConfigure = undefined; + FakeAudioEncoder.onConfigure = (config) => { + FakeAudioEncoder.onConfigure = undefined; resolve(config); }; }); @@ -371,6 +373,7 @@ async function setup(order = new Order(), codec?: Codec, groupDuration?: Milli) // not the encoder's gap-blind one, or they trail the next gap's marker and are dropped as pre-gap. test("an on-demand gap marks where submitted audio ends and drops the chunks held across it", async () => { using _webcodecs = installFakeWebCodecs(); + FakeAudioEncoder.LAG = 2; using env = await setup(); const { track, rendition, feed, written, writes } = env; @@ -407,8 +410,8 @@ test("an on-demand gap marks where submitted audio ends and drops the chunks hel // A pause with a subscriber attached breaks the timeline just as losing demand does: a subscriber // that stays across it, or joins during it, must not read the audio before it as live. The codec -// keeps its configuration, so the audio it still held is flushed out before the marker. -test("disabling with a subscriber attached flushes the held audio, then marks where it ends", async () => { +// keeps its configuration and is not flushed. +test("disabling with a subscriber attached marks where the submitted audio ends", async () => { using _webcodecs = installFakeWebCodecs(); using env = await setup(); const { enabled, feed, written } = env; @@ -421,14 +424,14 @@ test("disabling with a subscriber attached flushes the held audio, then marks wh await feed.drain(); }; - await push(4); // two written, two held + await push(4); enabled.set(false); await settle(); await push(2); // nothing publishing enabled.set(true); - await push(3); // the restarted codec, so one written and two held + await push(3); expect(written).toEqual([ [20_000, 1], @@ -437,14 +440,18 @@ test("disabling with a subscriber attached flushes the held audio, then marks wh [80_000, 1], [100_000, 0], [140_000, 1], + [160_000, 1], + [180_000, 1], ]); - expect(LaggingAudioEncoder.instances).toHaveLength(1); + expect(FakeAudioEncoder.instances).toHaveLength(1); }); -// The codec has to restart across a capture gap to re-base its output clock, but the audio it still -// held predates the gap: it is flushed out first, and the marker follows it. -test("a capture gap flushes the held audio, then marks where it ends", async () => { +// A codec that still holds chunks at a capture gap stamps them, and the input after the gap, as one +// contiguous run. Stamping from what was submitted keeps each on its capture clock, and the marker +// waits behind the held chunks rather than dropping them as pre-gap. +test("a capture gap marks where the audio before it ends, after the chunks the codec holds", async () => { using _webcodecs = installFakeWebCodecs(); + FakeAudioEncoder.LAG = 2; using _now = manualAt(0); using env = await setup(); const { feed, written } = env; @@ -457,7 +464,7 @@ test("a capture gap flushes the held audio, then marks where it ends", async () }; await push(20_000, 4); // two written, two held - await push(200_000, 4); // the restarted codec, so two written and two held + await push(200_000, 4); // the held two, the marker, then two more written and two held expect(written).toEqual([ [20_000, 1], @@ -468,13 +475,14 @@ test("a capture gap flushes the held audio, then marks where it ends", async () [200_000, 1], [220_000, 1], ]); - expect(LaggingAudioEncoder.instances).toHaveLength(1); + expect(FakeAudioEncoder.instances).toHaveLength(1); }); // A lost track already marked where its audio ends, so what the codec held for it must not open the // track that replaces it. test("a closed track's held audio stays off its replacement", async () => { using _webcodecs = installFakeWebCodecs(); + FakeAudioEncoder.LAG = 2; using _now = manualAt(0); using env = await setup(); const { track, rendition, feed } = env; @@ -542,7 +550,7 @@ test("re-enabling with an unchanged capture reuses the probe", async () => { const { encoder, enabled } = env; const before = encoder.out.catalog.peek(); - expect(LaggingAudioEncoder.probes).toHaveLength(1); + expect(FakeAudioEncoder.probes).toHaveLength(1); enabled.set(false); await settle(); @@ -551,7 +559,7 @@ test("re-enabling with an unchanged capture reuses the probe", async () => { enabled.set(true); await settle(2); expect(encoder.out.catalog.peek()).toEqual(before); - expect(LaggingAudioEncoder.probes).toHaveLength(1); + expect(FakeAudioEncoder.probes).toHaveLength(1); }); test("re-enabling with a changed sample rate probes again", async () => { @@ -568,7 +576,7 @@ test("re-enabling with a changed sample rate probes again", async () => { for (let config = await configuring; config.sampleRate !== 24_000; config = await configuring) { configuring = configured(); } - expect(LaggingAudioEncoder.probes.map((probe) => probe.sampleRate)).toEqual([48_000, 24_000]); + expect(FakeAudioEncoder.probes.map((probe) => probe.sampleRate)).toEqual([48_000, 24_000]); expect(encoder.out.catalog.peek()?.enabled).toBe(false); enabled.set(true); @@ -577,6 +585,71 @@ test("re-enabling with a changed sample rate probes again", async () => { expect(encoder.out.catalog.peek()?.enabled).toBeUndefined(); }); +// Push-to-talk: a mute sends nothing and ends the utterance with a marker, while the rendition stays +// enabled and its track stays the same, so a listener hears the next press with no catalog update. +test("muting sends nothing and marks where the audio ends, keeping the rendition enabled", async () => { + using _webcodecs = installFakeWebCodecs(); + using env = await setup(); + const { encoder, feed, written, rendition } = env; + + const push = async (start: number, count: number) => { + for (let i = 0; i < count; i++) { + await feed.push({ timestamp: Micro(start + i * 20_000), channels: [new Float32Array(960)] }); + } + await feed.drain(); + }; + + const before = encoder.out.catalog.peek(); + expect(before?.enabled).toBeUndefined(); + await push(20_000, 2); + + encoder.muted.set(true); + await settle(); + expect(encoder.out.active.peek()).toBe(false); + await push(60_000, 3); // sent nowhere + expect(rendition.config.peek()).toEqual(before as never); + + encoder.muted.set(false); + await settle(); + expect(encoder.out.active.peek()).toBe(true); + await push(120_000, 2); + + expect(written).toEqual([ + [20_000, 1], + [40_000, 1], + [60_000, 0], + [120_000, 1], + [140_000, 1], + ]); + expect(rendition.config.peek()).toEqual(before as never); + expect(FakeAudioEncoder.instances).toHaveLength(1); + expect(FakeAudioEncoder.probes).toHaveLength(1); +}); + +// With `capture="release"` a mute closes the microphone, taking its format away. The rendition stays +// advertised, still enabled, through the mute and while the reopened microphone resolves. +test("a muted rendition keeps its config enabled while its capture is released", async () => { + using _webcodecs = installFakeWebCodecs(); + using env = await setup(); + const { encoder, capture } = env; + + const before = encoder.out.catalog.peek(); + encoder.muted.set(true); + const format = capture.out.format.peek(); + capture.out.format.set(undefined as never); + await settle(); + expect(encoder.out.catalog.peek()).toEqual(before); + + encoder.muted.set(false); + await settle(); + expect(encoder.out.catalog.peek()).toEqual(before); + + capture.out.format.set(format); + await settle(10); + expect(encoder.out.catalog.peek()).toEqual(before); + expect(FakeAudioEncoder.probes).toHaveLength(1); +}); + // Chrome stamps encoder output from its first input, ignoring later jumps, so audio resumed after a // long mute must still carry its capture timestamps rather than trail them by the mute. test("audio resumed after a multi-second mute carries its capture timestamps", async () => { @@ -591,7 +664,7 @@ test("audio resumed after a multi-second mute carries its capture timestamps", a await feed.drain(); }; - await push(20_000, 4); // two written, two held + await push(20_000, 4); enabled.set(false); await settle(); @@ -599,7 +672,7 @@ test("audio resumed after a multi-second mute carries its capture timestamps", a // Capture keeps running through a five-second mute. await push(100_000, 250); enabled.set(true); - await push(5_100_000, 4); // two written, two held + await push(5_100_000, 4); expect(written).toEqual([ [20_000, 1], @@ -609,8 +682,10 @@ test("audio resumed after a multi-second mute carries its capture timestamps", a [100_000, 0], [5_100_000, 1], [5_120_000, 1], + [5_140_000, 1], + [5_160_000, 1], ]); - expect(LaggingAudioEncoder.probes).toHaveLength(1); + expect(FakeAudioEncoder.probes).toHaveLength(1); }); // Closing tears down the subscription and the pipeline, which both end the epoch; cleanups run @@ -623,21 +698,23 @@ test("closing with a subscriber attached marks the end once before the track clo for (let i = 0; i < 4; i++) { await feed.push({ timestamp: Micro(20_000 + i * 20_000), channels: [new Float32Array(960)] }); } - await feed.drain(); // two written, two held + await feed.drain(); encoder.close(); expect(written).toEqual([ [20_000, 1], [40_000, 1], + [60_000, 1], + [80_000, 1], [100_000, 0], ]); expect(track.closed.peek()).toBeDefined(); }); -// A push that completes several frames is still one continuous stream, so it must not restart the -// encoder and drop the chunks it holds. -test("a push completing several frames keeps the encoder running", async () => { +// A push that completes several frames is still one continuous stream, so it must not break the +// timeline between them. +test("a push completing several frames stays one stream", async () => { using _webcodecs = installFakeWebCodecs(); using env = await setup(); const { feed, written } = env; @@ -653,6 +730,8 @@ test("a push completing several frames keeps the encoder running", async () => { [38_700, 1], [58_700, 1], [78_700, 1], + [98_700, 1], + [118_700, 1], ]); }); @@ -671,7 +750,7 @@ test("a group duration packs frames until the minimum and restarts after a break await feed.drain(); }; - await push(7); // five written, two held + await push(5); expect(groups).toEqual([5]); expect(appended[0].closed.peek()).toBeUndefined(); await push(2); // the next frame opens a new group and closes the first @@ -681,15 +760,16 @@ test("a group duration packs frames until the minimum and restarts after a break enabled.set(false); await settle(); + await push(2); // nothing publishing enabled.set(true); - await push(4); // the restarted codec, so two written and two held + await push(2); expect(written.map(([timestamp]) => timestamp)).toEqual([ - 20_000, 40_000, 60_000, 80_000, 100_000, 120_000, 140_000, 160_000, 180_000, 200_000, 200_000, 220_000, + 20_000, 40_000, 60_000, 80_000, 100_000, 120_000, 140_000, 160_000, 200_000, 220_000, ]); - // 120ms opens the next group, closing the five frames at 20-100ms. The break flushes 160-180ms - // into that group and ends it early, so resumed frames start a fresh group after the marker. - expect(groups).toEqual([5, 4, 1, 2]); + // 120ms opens the next group, closing the five frames at 20-100ms. The break ends that group + // early, so resumed frames start a fresh group after the marker. + expect(groups).toEqual([5, 2, 1, 2]); }); test("default audio groups carry twenty milliseconds", async () => { @@ -705,7 +785,7 @@ test("default audio groups carry twenty milliseconds", async () => { { mime: "opus", frameDuration: Milli(frameMs) }, groupMs === undefined ? undefined : Milli(groupMs), ); - for (let index = 0; index < packets * 2 + LaggingAudioEncoder.LAG; index++) { + for (let index = 0; index < packets * 2 + FakeAudioEncoder.LAG; index++) { await env.feed.push({ timestamp: Micro(20_000 + index * frameMs * 1000), channels: [new Float32Array(frameMs * 48)], @@ -750,6 +830,8 @@ test("the catalog bitrate only rises", async () => { // this one trailing it, which the catalog advertises as `delay` once a second of it has been measured. test("a rendition written behind another advertises delay", async () => { using _webcodecs = installFakeWebCodecs(); + // Each push writes the frame captured two before it, so the sibling stays ahead. + FakeAudioEncoder.LAG = 2; const clock = manualAt(200); try { @@ -760,8 +842,7 @@ test("a rendition written behind another advertises delay", async () => { expect(encoder.out.catalog.peek()?.delay).toBeUndefined(); // A sibling that stays 100ms ahead of the frames written, with a clock origin that keeps - // timestamps nonnegative. The encoder holds two frames, so each push writes the frame captured - // two before it. + // timestamps nonnegative. const sibling = new Estimator(); const start = Nano.toMicro(Nano(now())) - 100_000; const push = async (index: number) => { @@ -853,7 +934,7 @@ test("a failed audio encoder keeps its last catalog config disabled", async () = using _now = manualAt(0); using _error = spyOn(console, "error").mockImplementation(() => {}); using env = await setup(); - const live = LaggingAudioEncoder.instances.at(-1); + const live = FakeAudioEncoder.instances.at(-1); if (!live) throw new Error("expected live audio encoder"); const failure = new Error("audio encoder failed"); live.fail(failure); @@ -862,7 +943,7 @@ test("a failed audio encoder keeps its last catalog config disabled", async () = expect(env.encoder.out.error.peek()).toBe(failure); expect(env.encoder.out.catalog.peek()).toMatchObject({ codec: "opus", enabled: false }); expect(env.encoder.out.active.peek()).toBe(false); - expect(LaggingAudioEncoder.instances).toHaveLength(1); + expect(FakeAudioEncoder.instances).toHaveLength(1); }); test("audio failure stays local through demand changes and capture format re-arms a fresh track", async () => { @@ -910,11 +991,11 @@ test("audio failure stays local through demand changes and capture format re-arm const siblingTrack = sibling.track.peek(); if (!siblingTrack) throw new Error("expected demanded sibling track"); expect(siblingTrack.appendGroup().sequence).toBe(0n); - await push(0, 1); - expect((await subscriber.largest())?.group).toBe(0n); + await push(0, 1); // a group per frame + expect((await subscriber.largest())?.group).toBe(2n); catalog = new Json.Snapshot.Consumer({ track: net.track(Broadcast.CATALOG_TRACK) }); expect((await catalog.latest())?.value.audio?.renditions.audio?.enabled).not.toBe(false); - const live = LaggingAudioEncoder.instances.at(-1); + const live = FakeAudioEncoder.instances.at(-1); if (!live) throw new Error("expected live audio encoder"); const failure = new Error("audio codec failed"); live.fail(failure); @@ -923,7 +1004,7 @@ test("audio failure stays local through demand changes and capture format re-arm expect(encoder.out.error.peek()).toBe(failure); expect((await catalog.latest())?.value.audio?.renditions.audio?.enabled).toBe(false); expect(encoder.out.active.peek()).toBe(false); - const instances = LaggingAudioEncoder.instances.length; + const instances = FakeAudioEncoder.instances.length; subscriber.close(); subscriber = net.track("audio").subscribe(); broadcast.audio("audio/new"); @@ -935,7 +1016,7 @@ test("audio failure stays local through demand changes and capture format re-arm expect(subscriber.closed.peek()).toBeDefined(); expect(encoder.out.error.peek()).toBe(failure); expect(encoder.out.catalog.peek()?.enabled).toBe(false); - expect(LaggingAudioEncoder.instances).toHaveLength(instances); + expect(FakeAudioEncoder.instances).toHaveLength(instances); expect(broadcast.net.peek()).toBe(net); expect(announced?.active.peek()?.instance).toBe(epoch); expect(siblingTrack.appendGroup().sequence).toBe(1n); @@ -953,7 +1034,7 @@ test("audio failure stays local through demand changes and capture format re-arm recovered = net.track(config.track).subscribe(); await settle(); await push(3, 2); - expect((await recovered.largest())?.group).toBe(0n); + expect((await recovered.largest())?.group).toBe(2n); expect(encoder.out.active.peek()).toBe(true); expect(recovered.closed.peek()).toBeUndefined(); expect(broadcast.net.peek()).toBe(net); @@ -983,7 +1064,7 @@ test("an audio failure without demand still recovers under a fresh track name", await settle(); const net = broadcast.net.peek(); if (!net) throw new Error("expected network broadcast"); - const live = LaggingAudioEncoder.instances.at(-1); + const live = FakeAudioEncoder.instances.at(-1); if (!live) throw new Error("expected audio encoder without demand"); const failure = new Error("idle audio encoder failed"); live.fail(failure); @@ -1028,7 +1109,7 @@ test.each(["failure first", "config first"])( await settle(); const net = broadcast.net.peek(); if (!net) throw new Error("expected network broadcast"); - const live = LaggingAudioEncoder.instances.at(-1); + const live = FakeAudioEncoder.instances.at(-1); if (!live) throw new Error("expected idle audio encoder"); const reconfiguring = configured(); if (order === "config first") encoder.codec.set({ mime: "opus", bitrate: 64_000 }); @@ -1084,7 +1165,7 @@ test.each([undefined, false, true])( }; await push(4); - expect(encoder.out.stats.peek().frames).toBe(onDemand === false ? 2 : 0); + expect(encoder.out.stats.peek().frames).toBe(onDemand === false ? 4 : 0); late = net.track("audio").subscribe({ delay: { max: Milli(10_000) } }); await settle(); expect(encoder.out.active.peek()).toBe(true); @@ -1155,7 +1236,7 @@ test("a backlog encode refusal aborts its rendition when demand arrives", async await settle(); await feed.push({ timestamp: Micro(0), channels: [new Float32Array(960)] }); await feed.drain(); - const live = LaggingAudioEncoder.instances.at(-1); + const live = FakeAudioEncoder.instances.at(-1); if (!live) throw new Error("expected configured encoder"); const failure = new Error("codec refused buffered input"); using _encode = spyOn(live, "encode").mockImplementation(() => { @@ -1203,13 +1284,14 @@ test("default audio replays the last second after three seconds without a subscr if (!net) throw new Error("expected network broadcast"); late = net.track("audio").subscribe({ delay: { max: Milli(1000) } }); await settle(); - expect(encoder.out.stats.peek().frames).toBe(50 - LaggingAudioEncoder.LAG); + expect(encoder.out.stats.peek().frames).toBe(50); for (let index = 150; index < 153; index++) { await feed.push({ timestamp: Micro(index * 20_000), channels: [new Float32Array(960)] }); } await feed.drain(); + // The subscription's one second reaches back from the newest frame, 152, to 102. const reader = late.ordered(); - for (let index = 100; index < 151; index++) { + for (let index = 102; index < 153; index++) { const group = await reader.nextGroup(); if (!group) throw new Error("expected buffered group"); const frame = await group.readFrame(); @@ -1324,7 +1406,7 @@ test("expired raw audio ends the epoch before the replay resumes", async () => { test("Opus description is probed with the exact live config before advertising", async () => { using _codecs = installFakeWebCodecs(); - LaggingAudioEncoder.description = (config) => new TextEncoder().encode(JSON.stringify(config)); + FakeAudioEncoder.description = (config) => new TextEncoder().encode(JSON.stringify(config)); using env = await setup(new Order(), { mime: "opus", frameDuration: Milli(2.5), @@ -1334,14 +1416,14 @@ test("Opus description is probed with the exact live config before advertising", bitrate: 64000, }); expect(env.encoder.out.catalog.peek()?.description).toBe(Buffer.from(JSON.stringify(env.config)).toString("hex")); - expect(LaggingAudioEncoder.probes).toEqual([env.config]); - expect(LaggingAudioEncoder.instances).toHaveLength(1); + expect(FakeAudioEncoder.probes).toEqual([env.config]); + expect(FakeAudioEncoder.instances).toHaveLength(1); expect(env.written).toHaveLength(0); const configuring = configured(); env.capture.in.source.set({ track: {}, kind: "voice" } as never); const voice = await configuring; expect(voice.opus).toMatchObject({ application: "voip", signal: "voice" }); - expect(LaggingAudioEncoder.probes.at(-1)).toEqual(voice); + expect(FakeAudioEncoder.probes.at(-1)).toEqual(voice); expect(env.encoder.out.catalog.peek()?.description).toBe(Buffer.from(JSON.stringify(voice)).toString("hex")); }); @@ -1385,7 +1467,7 @@ test.each([false, true])("audio retained before a disable is encoded on demand ( expect(encoder.out.stats.peek().frames).toBe(0); return; } - // The codec is flushed after the history, since nothing follows it while disabled. + // The history is encoded and its marker follows at once, since nothing else is owed while disabled. expect(encoder.out.stats.peek().frames).toBe(4); const reader = late.ordered(); for (let index = 0; index < 4; index++) { @@ -1404,7 +1486,7 @@ test.each([false, true])("audio retained before a disable is encoded on demand ( }); // Re-enabled before anyone wanted it, the retained audio stays ahead of the break: all of it is -// encoded and flushed before the marker, then the resumed audio starts a fresh epoch. +// encoded before the marker, then the resumed audio starts a fresh epoch. test("audio retained across a disable and re-enable is served before the break", async () => { using _webcodecs = installFakeWebCodecs(); const clock = new Manual(); @@ -1443,8 +1525,8 @@ test("audio retained across a disable and re-enable is served before the break", if (!net) throw new Error("expected network broadcast"); late = net.track("audio").subscribe({ delay: { max: Milli(1000) } }); await settle(20); - // The four retained frames are flushed before the marker, then the restarted codec holds two. - expect(encoder.out.stats.peek().frames).toBe(5); + // The four retained frames, then the marker, then the three resumed. + expect(encoder.out.stats.peek().frames).toBe(7); expect(discontinuity.mock.calls.map(([end]) => end)).toEqual([Micro(80_000)]); const reader = late.ordered(); const read: [number, number][] = []; @@ -1473,8 +1555,8 @@ test("audio retained across a disable and re-enable is served before the break", } }); -// A replay submits the whole backlog at once, so restarting the codec at a gap inside it would drop -// every frame of that drain still queued in a real codec. The gap flushes them out first instead. +// A replay submits the whole backlog at once, so a gap inside it must break the timeline without +// dropping any frame of that drain still owed by the codec. test("a capture gap inside replayed audio keeps every frame on both sides of it", async () => { using _webcodecs = installFakeWebCodecs(); const clock = new Manual(); @@ -1507,8 +1589,8 @@ test("a capture gap inside replayed audio keeps every frame on both sides of it" if (!net) throw new Error("expected network broadcast"); late = net.track("audio").subscribe({ delay: { max: Milli(1000) } }); await settle(20); - // Every frame before the gap, then the restarted codec holds two of the four after it. - expect(encoder.out.stats.peek().frames).toBe(6); + // Every frame on both sides of the gap. + expect(encoder.out.stats.peek().frames).toBe(8); expect(discontinuity.mock.calls.map(([end]) => end)).toEqual([Micro(80_000)]); const reader = late.ordered(); const read: [number, number][] = []; @@ -1538,8 +1620,8 @@ test("a capture gap inside replayed audio keeps every frame on both sides of it" } }); -// A zero max age retains nothing, not even the frame waiting behind a gap's flush, so the gap must not -// depend on the backlog to be noticed, or every frame after it would wait forever. +// A zero max age retains nothing, so a gap must not depend on the backlog to be noticed, or every +// frame after it would wait forever. test("a capture gap without retention resumes the audio after it", async () => { using _webcodecs = installFakeWebCodecs(); const clock = new Manual(); @@ -1570,12 +1652,11 @@ test("a capture gap without retention resumes the audio after it", async () => { subscriber = net.track("audio").subscribe(); await settle(); - await push(0, 4); // two written, two held - // The first frame after the gap waits behind the flush, where nothing is retained. + await push(0, 4); await push(200_000, 4); expect(encode.mock.calls.map(([, timestamp]) => timestamp)).toEqual( - [0, 20_000, 40_000, 60_000, 220_000].map((t) => Micro(t)), + [0, 20_000, 40_000, 60_000, 200_000, 220_000, 240_000, 260_000].map((t) => Micro(t)), ); expect(discontinuity.mock.calls.map(([end]) => end)).toEqual([Micro(80_000)]); } finally { @@ -1588,7 +1669,7 @@ test("a capture gap without retention resumes the audio after it", async () => { } }); -// Two disables before anyone listens leave two breaks in the backlog: each flushes the audio before it. +// Two disables before anyone listens leave two breaks in the backlog: each follows the audio before it. test("every disable retained before demand gets its own break", async () => { using _webcodecs = installFakeWebCodecs(); const clock = new Manual(); @@ -1676,10 +1757,9 @@ test("a break after expired history keeps the resumed audio flowing", async () = if (!net) throw new Error("expected network broadcast"); late = net.track("audio").subscribe({ delay: { max: Milli(1000) } }); await settle(20); - // The codec holds two of the three resumed frames. - expect(encoder.out.stats.peek().frames).toBe(1); - await push(2_060_000, 2); expect(encoder.out.stats.peek().frames).toBe(3); + await push(2_060_000, 2); + expect(encoder.out.stats.peek().frames).toBe(5); } finally { late?.close(); encoder.close(); @@ -1689,8 +1769,8 @@ test("a break after expired history keeps the resumed audio flowing", async () = }); // Demand that arrives after a re-enable but before any resumed frame drains the retained audio first, -// so the break still flushes it rather than cutting it off with the codec mid-flight. -test("demand between a re-enable and the resumed audio still flushes the break", async () => { +// so the break follows it rather than cutting it off with the codec mid-flight. +test("demand between a re-enable and the resumed audio still breaks after the retained audio", async () => { using _webcodecs = installFakeWebCodecs(); const clock = new Manual(); const installed = clock.install(); @@ -1729,8 +1809,7 @@ test("demand between a re-enable and the resumed audio still flushes the break", expect(discontinuity.mock.calls.map(([end]) => end)).toEqual([Micro(80_000)]); await push(200_000, 3); - // The restarted codec holds two of the three resumed frames. - expect(encoder.out.stats.peek().frames).toBe(5); + expect(encoder.out.stats.peek().frames).toBe(7); expect(discontinuity).toHaveBeenCalledTimes(1); } finally { discontinuity.mockRestore(); diff --git a/js/publish/src/audio/encoder.ts b/js/publish/src/audio/encoder.ts index c1203ad7bd7..30f89dbfe42 100644 --- a/js/publish/src/audio/encoder.ts +++ b/js/publish/src/audio/encoder.ts @@ -91,11 +91,12 @@ export interface Stats { /** Signals the encoder reads. */ export type EncoderInput = { - // Whether to encode this rendition. Defaults to true. When false it stops taking capture, ending the + // Whether to serve this rendition. Defaults to true. When false it stops taking capture, ending the // epoch, but stays in the catalog with `enabled: false` and its last config, and stays registered so a // subscriber still gets an idle track. Audio retained before the disable stays too: a subscriber // still gets it encoded, and it expires on the broadcast's `maxAge`. The codec stays configured, so - // re-enabling resumes at once. + // re-enabling resumes at once. For a pause the rendition comes back from, such as push-to-talk, use + // `muted` instead, which costs subscribers no catalog update. enabled: Getter; // The broadcast to register the rendition on. Undefined resolves the config but has nowhere to publish. @@ -116,6 +117,9 @@ export type EncoderProps = Inputs & { // User tuning knobs. Seed a value or wire a Signal; also live-editable via the matching field. volume?: number | Signal; + // Send no audio while true, keeping the rendition enabled and its subscribers attached. Defaults to false. + muted?: boolean | Signal; + // Codec selection plus encoder settings. Defaults to "opus". codec?: Codec | Signal; @@ -135,9 +139,9 @@ export type EncoderOutput = { // browser supports `sinkId`, so nothing connected to its destination is heard. Undefined for a // source that isn't a track. root: Signal; - // True while encoding capture: enabled with a captured format and a track, which also takes a - // subscriber unless the broadcast's `onDemand` is false. Retained audio encoded for a subscriber - // while disabled does not count. + // True while encoding capture: enabled and unmuted with a captured format and a track, which also + // takes a subscriber unless the broadcast's `onDemand` is false. Retained audio encoded for a + // subscriber while paused does not count. active: Signal; // Cumulative output totals (frames, bytes) measured while serving, and the target bitrate. stats: Signal; @@ -168,6 +172,12 @@ export class Encoder { /** Linear gain applied before encoding, where 1 is unity. Each change ramps over 50 ms, so a mute is silent once it passes. */ volume: Signal; + /** + * Whether to send nothing, as push-to-talk does between presses. Unlike `enabled`, the rendition + * stays enabled in the catalog and its subscribers stay attached, so the next audio reaches them + * at once; each muted span is a declared end, which a player fills as intentional silence. + */ + muted: Signal; /** The live-editable codec selection plus its encoder settings. */ codec: Signal; /** @@ -261,6 +271,9 @@ export class Encoder { // config as disabled until the re-enabled capture resolves a new one. A capture that never reopens // leaves it disabled, which is accurate: no frames are coming. #paused = false; + // Whether it was muted since a config last resolved, so it keeps advertising its last config, + // enabled, while a released capture is closed and until the reopened one resolves. + #muted = false; constructor(name: string, props?: EncoderProps) { // `source` moved to Audio.Capture, which renditions share. TypeScript catches this, but a @@ -278,6 +291,7 @@ export class Encoder { bandwidth: getter(props?.bandwidth), }; this.volume = Signal.from(props?.volume ?? 1); + this.muted = Signal.from(props?.muted ?? false); this.codec = Signal.from(props?.codec ?? "opus"); this.groupDuration = Signal.from(props?.groupDuration ?? Milli(20)); @@ -463,7 +477,7 @@ export class Encoder { }); effect.run((effect) => { - const enabled = effect.get(this.in.enabled); + const enabled = effect.get(this.in.enabled) && !effect.get(this.muted); const capture = effect.get(this.in.capture); const format = capture ? effect.get(capture.out.format) : undefined; const track = effect.get(rendition.track); @@ -473,7 +487,7 @@ export class Encoder { const onDemand = effect.get(broadcast.in.onDemand) ?? true; const demanded = !!track && effect.get(track.demand().wanted) !== undefined; - // Not gated on `enabled`: a disabled rendition takes no capture, but still encodes what it retained. + // Not gated on `enabled` or `muted`: a paused rendition takes no capture, but still encodes what it retained. this.#encoding = !!format && !!track && !fatal && (!onDemand || demanded); effect.set(this.#out.active, enabled && this.#encoding, false); try { @@ -608,21 +622,23 @@ export class Encoder { effect.cleanup(() => this.#failures.update((n) => n - 1)); } - // Publish the probed config before the first frame reaches the track. A - // disabled rendition keeps its last config, since muting also releases the capture, and stays - // disabled after re-enabling until the reopened capture reports its format. + // Publish the probed config before the first frame reaches the track. A disabled rendition keeps its + // last config, since a disable may also release the capture, and stays disabled after re-enabling + // until the reopened capture reports its format. A muted one keeps it enabled throughout. #runCatalog(effect: Effect): void { const estimate = effect.get(this.#estimate); const enabled = effect.get(this.in.enabled) && !effect.get(this.#out.error); const slot = this.#current(effect); if (!enabled) this.#paused = true; + if (effect.get(this.muted)) this.#muted = true; const config = enabled ? effect.get(this.#config)?.catalog : undefined; if (!config) { - const last = this.#paused ? slot.last : undefined; - effect.set(this.#out.catalog, last && { ...last, ...estimate, enabled: false }); + const last = this.#paused || this.#muted ? slot.last : undefined; + effect.set(this.#out.catalog, last && { ...last, ...estimate, ...(this.#paused && { enabled: false }) }); return; } this.#paused = false; + this.#muted = false; // The catalog bitrate is the most this track has been configured for, so lowering the knob // retunes the encoder without lowering what the catalog advertises. @@ -646,15 +662,19 @@ export class Encoder { return opus; } - // End the epoch with a discontinuity marker (see Container.Legacy.Producer.discontinuity) where the - // submitted media ends, so a subscriber that stays across the break, or joins during it, doesn't - // read the audio before it as live. Called whenever the track goes away or the pipeline stops. - #cut(): void { - const end = this.#next; - this.#next = undefined; - const live = this.#live; - if (end === undefined || !live || live.track.closed.peek() !== undefined) return; + // End the epoch with a discontinuity marker (see Container.Legacy.Producer.discontinuity) at `end`, + // by default where the submitted media ends, so a subscriber that stays across the break, or joins + // during it, doesn't read the audio before it as live. A marker at or below the newest one is + // already covered by it. + #cut(end = this.#next): void { + if (end === undefined) return; + if (this.#next !== undefined && this.#next <= end) this.#next = undefined; + if (this.#floor !== undefined && end <= this.#floor) return; + // The epoch ends here even when no track is left to mark, so whatever the codec still owes for + // it stays off the next track. this.#floor = end; + const live = this.#live; + if (!live || live.track.closed.peek() !== undefined) return; live.producer.discontinuity(end); live.start = undefined; } @@ -692,11 +712,31 @@ export class Encoder { let resampler = createResampler(); let framer = createFramer(resolved, config.sampleRate); + // What the codec still owes, in submission order: each input's timestamp, or a break whose + // marker goes down once every chunk before it is out. Chunks are stamped from here, never + // by the codec: Chrome and Firefox stamp input after a gap as if it were contiguous. + // Every browser emits one chunk per input, in order, so a codec that holds a chunk only + // delays its stamp, and a break needs no flush. + const owed: ({ timestamp: Micro } | { cut: Micro })[] = []; + // Write the marker of every break the codec has caught up with. + const settle = () => { + for (let next = owed[0]; next && "cut" in next; next = owed[0]) { + owed.shift(); + this.#cut(next.cut); + } + }; + const encoder = new AudioEncoder({ output: (frame) => { if (frame.type !== "key") { throw new Error("only key frames are supported"); } + const input = owed.shift(); + if (!input || "cut" in input) { + this.#abort(new Error("the audio encoder emitted a chunk for no input")); + return; + } + const timestamp = input.timestamp; this.#out.stats.update((stats) => ({ ...stats, @@ -705,21 +745,20 @@ export class Encoder { })); const live = this.#live; - if (!live) return; - const timestamp = frame.timestamp as Micro; - if (this.#floor !== undefined && timestamp < this.#floor) return; - - // Every audio frame decodes on its own, so any of them can open a group: the - // first one at or past the minimum after the open group's start. Frames forward - // as they are written rather than waiting for the group to fill. A dropped - // group leaves a gap for the codec's PLC. - const keyframe = live.start === undefined || timestamp - live.start >= this.#groupDuration; - if (keyframe) live.start = timestamp; - live.producer.encode(frame, timestamp, keyframe); - const history = this.#history !== undefined && timestamp < this.#history; - if (!history && this.#estimator.write(frame.timestamp, order, keyframe)) { - this.#estimate.set({ ...this.#estimator.estimate }); + if (live && (this.#floor === undefined || timestamp >= this.#floor)) { + // Every audio frame decodes on its own, so any of them can open a group: the + // first one at or past the minimum after the open group's start. Frames forward + // as they are written rather than waiting for the group to fill. A dropped + // group leaves a gap for the codec's PLC. + const keyframe = live.start === undefined || timestamp - live.start >= this.#groupDuration; + if (keyframe) live.start = timestamp; + live.producer.encode(frame, timestamp, keyframe); + const history = this.#history !== undefined && timestamp < this.#history; + if (!history && this.#estimator.write(timestamp, order, keyframe)) { + this.#estimate.set({ ...this.#estimator.estimate }); + } } + settle(); }, error: (err) => { if (!aborted.aborted) this.#abort(err); @@ -738,15 +777,13 @@ export class Encoder { return; } - // Where the next frame starts if it continues the last one encoded. + // Where the next frame starts if it continues the last one submitted, unset after a break. let contiguous: Micro | undefined; - // Disabled: capture is dropped, while the retained audio stays reachable. + // Disabled or muted: capture is dropped, while the retained audio stays reachable. let paused = false; - // A disable left retained audio, so the timeline breaks after it rather than at once. + // A pause left retained audio, so the timeline breaks after it rather than at once. let breakPending = false; - // The codec is flushing before a break, so nothing new reaches it until the marker is down. - let flushing = false; const backlog = new Backlog(retention, config.sampleRate); const deadline = new Signal(undefined); @@ -763,18 +800,22 @@ export class Encoder { ); }); - // Whether a gap (a lost track, or a capture discontinuity) separates `data` from the last - // frame the codec took. Chrome stamps encoder output from the first input's timestamp - // plus the samples encoded since, ignoring any later jump, so across a gap the output - // would trail the capture clock by the gap, and the next gap's marker would then sit - // ahead of everything encoded after it. So a gap restarts the codec, which re-bases it. - // Timestamps round to whole microseconds, so a chunk's timestamp and the marker placed - // at the next frame's start agree exactly. - const gap = (data: AudioFrame) => contiguous !== undefined && Math.round(data.timestamp) !== contiguous; + // End the epoch where the submitted audio ends, once the codec has emitted all of it: a + // marker placed while chunks are still owed would drop them as pre-gap. + const cut = () => { + if (contiguous === undefined) return; + owed.push({ cut: contiguous }); + contiguous = undefined; + settle(); + }; - // Encode one frame, which must not follow a gap: the gap's flush and restart come first. + // Encode one frame, breaking the timeline first if it does not continue the last one. + // Timestamps round to whole microseconds, so a chunk's timestamp and the marker placed at + // the next frame's start agree exactly. const submit = (data: AudioFrame) => { const timestamp = Math.round(data.timestamp) as Micro; + if (contiguous !== undefined && timestamp !== contiguous) cut(); + const joinedLength = data.channels.reduce((total, channel) => total + channel.length, 0); const joined = new Float32Array(joinedLength); @@ -793,6 +834,7 @@ export class Encoder { transfer: [joined.buffer], }); + owed.push({ timestamp }); encoder.encode(frame); frame.close(); contiguous = Math.round( @@ -800,64 +842,29 @@ export class Encoder { ) as Micro; this.#next = contiguous; }; - // The next frame starts a new epoch, so it starts the codec afresh too. - const restart = () => { - encoder.reset(); - encoder.configure(encoderConfig); - contiguous = undefined; - }; - // End the epoch where the submitted audio ends, once the codec has emitted all of it: a - // marker placed while chunks are still in flight would drop them as pre-gap. - const cutAfterFlush = () => { - if (flushing) return; - // Nothing reached the codec since it last started, so nothing is in flight. - if (contiguous === undefined) return this.#cut(); - // The epoch already ended without a flush (a lost track, or expired history), so what - // the codec holds sits below that marker and has nowhere to go. - if (this.#next === undefined) return restart(); - flushing = true; - encoder - .flush() - .then(() => { - if (aborted.aborted) return; - flushing = false; - this.#cut(); - restart(); - if (this.#encoding) pipeline.drain(); - }) - .catch((error) => { - if (!aborted.aborted) this.#abort(error); - }); - }; const pipeline: Pipeline = { channelCount: config.numberOfChannels, drain: () => { - if (flushing) return; backlog.expire(clock()); - // Expired history is a real gap, unlike a loss of demand: end the epoch where - // the encoded audio stops, so its last frame isn't stretched across the gap. - // What the codec still holds is older than the expired audio, so it is dropped. - if (backlog.takeEvicted()) this.#cut(); + // Expired history is a real gap, unlike a loss of demand: end the epoch where the + // encoded audio stops, so its last frame isn't stretched across the gap. + if (backlog.takeEvicted()) cut(); for (;;) { const frame = backlog.peek(); if (!frame) break; - // A disable or a gap broke the timeline here; the rest follows once the marker - // is down. A cut with nothing in flight lands at once. - if (backlog.takeCut() || gap(frame)) { - cutAfterFlush(); - if (flushing) break; - } + // A pause broke the timeline here. + if (backlog.takeCut()) cut(); backlog.shift(); submit(frame); this.#history = this.#next; } deadline.set(backlog.deadline); - // The audio retained before a disable is all encoded, so the break follows it, even - // when re-enabled since: resumed audio then waits behind the flush. + // The audio retained before a pause is all encoded, so the break follows it, even + // when resumed since. if (breakPending && backlog.empty) { breakPending = false; - cutAfterFlush(); + cut(); } }, push: (captured: AudioFrame) => { @@ -867,28 +874,28 @@ export class Encoder { const frames = framer.push(input); for (const [i, data] of frames.entries()) { if (!this.#live) continue; - const cut = breakPending; + const pending = breakPending; breakPending = false; - const live = this.#encoding && !flushing && backlog.empty; - // Started here rather than from the backlog, which may not keep the frame: a - // zero max age retains nothing. - if (live && (cut || gap(data))) cutAfterFlush(); - if (live && !flushing) { + if (this.#encoding && backlog.empty) { + if (pending) cut(); submit(data); continue; } - // It waits for a subscriber, or behind the flush before a break. + // It waits for a subscriber. Started here rather than from the backlog, which + // may not keep the frame: a zero max age retains nothing. const end = frames[i + 1]?.timestamp ?? framer.next; - backlog.push(data, end, clock(), cut && !live); + backlog.push(data, end, clock(), pending); } deadline.set(backlog.deadline); }, }; - // A disable stops taking capture but keeps the codec and retained audio, so a subscriber - // still gets that history and re-enabling resumes without reconfiguring. + // A disable or a mute stops taking capture but keeps the codec and retained audio, so a + // subscriber still gets that history and resuming needs no reconfigure. Neither flushes: + // the codec emits what it owes on its own, and a flush would send its lookahead tail at + // once, a frame ahead of its media. effect.run((toggle) => { - if (toggle.get(this.in.enabled)) { + if (toggle.get(this.in.enabled) && !toggle.get(this.muted)) { paused = false; return; } @@ -896,7 +903,7 @@ export class Encoder { // The capture resumes on a later timeline, so drop the partial frame and filter state. resampler = createResampler(); framer = createFramer(resolved, config.sampleRate); - if (backlog.empty) cutAfterFlush(); + if (backlog.empty) cut(); else breakPending = true; }); diff --git a/js/publish/src/element.ts b/js/publish/src/element.ts index 98f97576abd..af7218577c9 100644 --- a/js/publish/src/element.ts +++ b/js/publish/src/element.ts @@ -196,7 +196,8 @@ export default class MoqPublish extends HTMLElement { // The preview element, either a