From bcd6040fd68121c8e2d6d7701dfe6d8964bc23bf Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 29 Oct 2026 23:30:53 -0700 Subject: [PATCH] feat(publish): pack several audio frames per group on release (backport #4910) Port Audio.Encoder's groupDuration from main onto release's encoder. The default of 0 keeps one closed group per frame. A positive duration holds the group open until a frame at least that far past its start, a demand-gap marker, or the encode pipeline stopping. Co-Authored-By: Claude Opus 5.5 --- doc/lib/js/publish.md | 11 ++++ js/publish/api/element.api.md | 2 + js/publish/api/index.api.md | 2 + js/publish/src/audio/encoder.test.ts | 90 +++++++++++++++++++++++++--- js/publish/src/audio/encoder.ts | 66 +++++++++++++++++--- 5 files changed, 154 insertions(+), 17 deletions(-) diff --git a/doc/lib/js/publish.md b/doc/lib/js/publish.md index 7bfe1a28a89..cbf96714f27 100644 --- a/doc/lib/js/publish.md +++ b/doc/lib/js/publish.md @@ -54,6 +54,17 @@ framerate, and bitrate are tunable through `el.video.config`; the audio encoder exposes its codec and volume. For simulcast or several renditions, drop the element and register your own encoders on a `Publish.Broadcast`. +Each audio frame is its own group by default, so a relay pays a stream and a +group's bookkeeping per 20ms frame. `el.audio.groupDuration` sets a minimum +per group instead, such as `Time.Milli(100)`: the first frame at least 100ms +after the group's first timestamp opens the next group and closes the previous +one. If encoding pauses, the current group stays open until the next frame, a +timeline marker, or the encoder stopping. Frames still forward as they are +encoded, so grouping does not buffer them, but loss gets coarser: a viewer that +falls behind skips a whole group, and a lost frame holds back the rest of its +group until it is retransmitted. A 60ms Opus `frameDuration` also cuts the +group rate, without code, at the cost of encoder latency. + The video and audio encoders measure how far their output falls behind the media clock when they flush frames. Catalog jitter is the spread above each rendition's own recent minimum lateness, so a constant encoder delay is not jitter. diff --git a/js/publish/api/element.api.md b/js/publish/api/element.api.md index 1fcfb66a7a4..c05b59bb294 100644 --- a/js/publish/api/element.api.md +++ b/js/publish/api/element.api.md @@ -218,6 +218,7 @@ class Encoder_2 { get capture(): Capture_2 | undefined; close(): void; codec: Signal; + groupDuration: Signal; readonly in: Readonlys; readonly name: string; readonly out: Readonlys; @@ -259,6 +260,7 @@ type EncoderProps = Inputs & { type EncoderProps_2 = Inputs & { volume?: number | Signal; codec?: Codec | Signal; + groupDuration?: Time.Milli | Signal; }; class Fanout { diff --git a/js/publish/api/index.api.md b/js/publish/api/index.api.md index 556120bb486..442f8bdc484 100644 --- a/js/publish/api/index.api.md +++ b/js/publish/api/index.api.md @@ -239,6 +239,7 @@ class Encoder { get capture(): Capture | undefined; close(): void; codec: Signal; + groupDuration: Signal; readonly in: Readonlys; readonly name: string; readonly out: Readonlys; @@ -286,6 +287,7 @@ type EncoderOutput_2 = { type EncoderProps = Inputs & { volume?: number | Signal; codec?: Codec | Signal; + groupDuration?: Time.Milli | Signal; }; type EncoderProps_2 = Inputs & { diff --git a/js/publish/src/audio/encoder.test.ts b/js/publish/src/audio/encoder.test.ts index 86f21216f22..4aa29c4b1ad 100644 --- a/js/publish/src/audio/encoder.test.ts +++ b/js/publish/src/audio/encoder.test.ts @@ -192,21 +192,32 @@ class Feed { } } -// An Encoder wired to a fake capture feed, recording each written frame as [timestamp, payload bytes]. -async function setup(baseline = new Baseline(), codec?: Codec) { +// An Encoder wired to a fake capture feed, recording each written frame as [timestamp, payload bytes] +// and how many frames each group carries. +async function setup(baseline = new Baseline(), codec?: Codec, groupDuration?: Time.Milli) { const configured = new Promise((resolve) => { LaggingAudioEncoder.onConfigure = resolve; }); const track = new Moq.Track.Producer("audio").accept(); const written: [number, number][] = []; + const groups: number[] = []; + const appended: ReturnType[] = []; const writes = { onWrite: undefined as (() => void) | undefined }; - const writeFrame = track.writeFrame.bind(track); - track.writeFrame = (frame) => { - const [timestamp, payload] = Moq.Varint.decode(frame.payload); - written.push([timestamp, payload.byteLength]); - writeFrame(frame); - writes.onWrite?.(); + const appendGroup = track.appendGroup.bind(track); + track.appendGroup = () => { + const group = appendGroup(); + appended.push(group); + const index = groups.push(0) - 1; + const writeFrame = group.writeFrame.bind(group); + group.writeFrame = (frame) => { + const [timestamp, payload] = Moq.Varint.decode(frame.payload); + written.push([timestamp, payload.byteLength]); + groups[index]++; + writeFrame(frame); + writes.onWrite?.(); + }; + return group; }; const rendition = { @@ -230,6 +241,7 @@ async function setup(baseline = new Baseline(), codec?: Codec) { broadcast: { audio: () => rendition, baseline } as never, capture: capture as never, codec, + groupDuration, }); const config = await configured; @@ -242,6 +254,8 @@ async function setup(baseline = new Baseline(), codec?: Codec) { rendition, feed, written, + groups, + appended, writes, [Symbol.dispose]() { encoder.close(); @@ -294,7 +308,7 @@ test("a demand gap marks where submitted audio ends and drops the chunks held ac test("a push completing several frames keeps the encoder running", async () => { using _webcodecs = installFakeWebCodecs(); using env = await setup(); - const { feed, written } = env; + const { feed, written, appended } = env; // Two 20ms Opus frames per push. for (let index = 0; index < 3; index++) { @@ -308,6 +322,64 @@ test("a push completing several frames keeps the encoder running", async () => { [58_700, 1], [78_700, 1], ]); + // Without a group duration, each frame's group closes as soon as it is written. + expect(appended.map((group) => group.closed.peek())).toEqual([null, null, null, null]); +}); + +// The first frame at the minimum opens the next group. A demand gap closes the group early, so the +// first frame after it opens a fresh one. +test("a group duration packs frames until the minimum and restarts after a demand gap", async () => { + using _webcodecs = installFakeWebCodecs(); + using env = await setup(new Baseline(), undefined, Time.Milli(100)); + const { track, rendition, feed, written, groups, appended, writes } = env; + + let index = 0; + const push = async (count: number) => { + for (let i = 0; i < count; i++, index++) { + await feed.push({ timestamp: Time.Micro(20_000 + index * 20_000), channels: [new Float32Array(960)] }); + } + await feed.drain(); + }; + + await push(7); // five written, two held + expect(groups).toEqual([5]); + expect(appended[0].closed.peek()).toBeUndefined(); + await push(2); // the next frame opens a new group and closes the first + expect(groups).toEqual([5, 2]); + expect(appended[0].closed.peek()).toBeNull(); + + const marked = new Promise((resolve) => { + writes.onWrite = resolve; + }); + rendition.track.set(undefined); + await marked; + writes.onWrite = undefined; + expect(appended[1].closed.peek()).toBeNull(); + + rendition.track.set(track); + await push(4); // releases the two held pre-gap chunks, which are dropped, then two resumed ones + + expect(written.map(([timestamp]) => timestamp)).toEqual([ + 20_000, 40_000, 60_000, 80_000, 100_000, 120_000, 140_000, 200_000, 200_000, 220_000, + ]); + // 120ms opens the next group, closing the five frames at 20-100ms. The gap ends the 120-140ms + // group early, so resumed frames start a fresh group after the marker. + expect(groups).toEqual([5, 2, 1, 2]); +}); + +// A negative or non-finite duration has no meaning, so the rendition drops out of the catalog. +test("an invalid group duration refuses the rendition", async () => { + using _webcodecs = installFakeWebCodecs(); + using env = await setup(new Baseline(), undefined, Time.Milli(100)); + const { encoder } = env; + + expect(encoder.out.catalog.peek()).toBeDefined(); + encoder.groupDuration.set(Time.Milli(-1)); + await settle(); + expect(encoder.out.catalog.peek()).toBeUndefined(); + encoder.groupDuration.set(Time.Milli(0)); + await settle(); + expect(encoder.out.catalog.peek()).toBeDefined(); }); test("passes an explicit Opus DTX request to the encoder", async () => { diff --git a/js/publish/src/audio/encoder.ts b/js/publish/src/audio/encoder.ts index dcbf1984706..ed0119afd7e 100644 --- a/js/publish/src/audio/encoder.ts +++ b/js/publish/src/audio/encoder.ts @@ -101,6 +101,9 @@ export type EncoderProps = Inputs & { // Codec selection plus encoder settings. Defaults to "opus". codec?: Codec | Signal; + + // The minimum audio carried by each group. Defaults to 0, a group per frame. + groupDuration?: Time.Milli | Signal; }; type EncoderOutput = { @@ -139,6 +142,12 @@ export class Encoder { volume: Signal; /** The live-editable codec selection plus its encoder settings. */ codec: Signal; + /** + * The minimum timestamp span before a frame opens the next group, closing the previous one. 0 puts every frame in its own group. A longer group costs the + * relay fewer streams but makes loss coarser: a viewer that falls behind skips a whole group. A + * negative or non-finite duration refuses the rendition. + */ + groupDuration: Signal; /** * The capture supplying this rendition, or undefined while none is wired. @@ -177,6 +186,14 @@ export class Encoder { // marker and are dropped. #floor: Time.Micro | undefined; + // The group still accepting frames and the track it belongs to, with its first timestamp. Only + // held while groupDuration is positive; otherwise each frame's group closes once written. + #group: { track: Moq.Track.Producer; group: Moq.Group.Producer; start: Time.Micro } | undefined; + + // The last valid group duration. #runConfig refuses the rendition on an invalid one, so a bad + // value never reaches the writer. + #groupDuration = Time.Micro(0); + // The fatal error an AudioEncoder reported, if any. That instance can never encode again and // reconfiguring it would be a retry, so the rendition stays down for the life of this encoder. #fatal = new Signal(undefined); @@ -212,6 +229,7 @@ export class Encoder { }; this.volume = Signal.from(props?.volume ?? 1); this.codec = Signal.from(props?.codec ?? "opus"); + this.groupDuration = Signal.from(props?.groupDuration ?? Time.Milli(0)); // Only the capture graph has a node to expose. this.#signals.run((effect) => { @@ -318,6 +336,8 @@ export class Encoder { const track = effect.get(rendition.track); if (!track) return; effect.cleanup(() => { + // The marker's group follows, so the open one ends here and resumed frames open a fresh one. + this.#closeGroup(); const end = this.#next; this.#next = undefined; if (end === undefined || track.closed.peek() !== undefined) return; @@ -369,6 +389,11 @@ export class Encoder { // Gated on `enabled` the same way the video encoder is: a disabled rendition has to drop out of // the catalog, and a sample source keeps its format while muted rather than tearing down. #runConfig(effect: Effect): void { + const groupDuration = effect.get(this.groupDuration); + if (!Number.isFinite(groupDuration) || groupDuration < 0) + throw new Error(`audio group duration must be a finite, non-negative number of ms: ${groupDuration}`); + this.#groupDuration = Time.Micro.fromMilli(groupDuration); + const capture = effect.get(this.in.capture); const captured = capture ? effect.get(capture.out.format) : undefined; if (!effect.get(this.in.enabled) || !captured) { @@ -456,15 +481,11 @@ export class Encoder { bytes: stats.bytes + frame.byteLength, })); - // Each audio frame is its own group so the relay can forward it without - // waiting for a group boundary. Loss is handled by the codec's PLC. const live = track.peek(); if (!live) return; - if (this.#floor !== undefined && frame.timestamp < this.#floor) return; - live.writeFrame({ - payload: Container.Legacy.encodeFrame(frame, frame.timestamp as Time.Micro), - timestamp: Time.Timestamp.fromMicros(frame.timestamp as Time.Micro), - }); + const timestamp = frame.timestamp as Time.Micro; + if (this.#floor !== undefined && timestamp < this.#floor) return; + this.#write(live, frame, timestamp); if (this.#estimator.flush(frame.timestamp, baseline)) { const catalog = this.#out.catalog.peek(); if (catalog) this.#out.catalog.set({ ...catalog, ...this.#estimator.estimate }); @@ -477,9 +498,11 @@ export class Encoder { this.#fatal.set(err); }, }); - // A fatal error already closed the codec, and closing it twice throws. + // A fatal error already closed the codec, and closing it twice throws. A rebuilt encoder + // restarts its output clock, so its first frame opens a fresh group. effect.cleanup(() => { if (encoder.state !== "closed") encoder.close(); + this.#closeGroup(); }); console.debug("encoding audio", encoderConfig); @@ -550,6 +573,33 @@ export class Encoder { }); } + // Every audio frame decodes on its own, so any of them can open a group: the first one at or past + // the minimum after the open group's start. Frames forward as they are written rather than + // waiting for the group to fill. A dropped group leaves a gap for the codec's PLC. + #write(track: Moq.Track.Producer, data: EncodedAudioChunk, timestamp: Time.Micro): void { + const frame = { + payload: Container.Legacy.encodeFrame(data, timestamp), + timestamp: Time.Timestamp.fromMicros(timestamp), + }; + + const open = this.#group; + if (open?.track === track && timestamp - open.start < this.#groupDuration) { + open.group.writeFrame(frame); + return; + } + + this.#closeGroup(); + const group = track.appendGroup(); + group.writeFrame(frame); + if (this.#groupDuration > 0) this.#group = { track, group, start: timestamp }; + else group.close(); + } + + #closeGroup(): void { + this.#group?.group.close(); + this.#group = undefined; + } + #setDecoderDescription(config: Catalog.AudioConfig, source: AllowSharedBufferSource | undefined): void { if (config.codec !== "opus" || !source) return;