diff --git a/doc/lib/js/publish.md b/doc/lib/js/publish.md index cbf96714f27..e2714a6fbdb 100644 --- a/doc/lib/js/publish.md +++ b/doc/lib/js/publish.md @@ -58,8 +58,10 @@ Each audio frame is its own group by default, so a relay pays a stream and a group's bookkeeping per 20ms frame. `el.audio.groupDuration` sets a minimum per group instead, such as `Time.Milli(100)`: the first frame at least 100ms after the group's first timestamp opens the next group and closes the previous -one. If encoding pauses, the current group stays open until the next frame, a -timeline marker, or the encoder stopping. Frames still forward as they are +one. A group also ends early at the per-group cap of 8192 frames or 32 MiB, so +a long duration with short frames splits into several groups. If encoding +pauses, the current group stays open until the next frame, a timeline marker, +or the encoder stopping. Frames still forward as they are encoded, so grouping does not buffer them, but loss gets coarser: a viewer that falls behind skips a whole group, and a lost frame holds back the rest of its group until it is retransmitted. A 60ms Opus `frameDuration` also cuts the diff --git a/js/publish/src/audio/encoder.test.ts b/js/publish/src/audio/encoder.test.ts index 4aa29c4b1ad..c465fddad38 100644 --- a/js/publish/src/audio/encoder.test.ts +++ b/js/publish/src/audio/encoder.test.ts @@ -70,6 +70,8 @@ class LaggingAudioEncoder { // Called on configure; the encoder publishes its pipeline synchronously right after. static onConfigure: ((config: AudioEncoderConfig) => void) | undefined; + // The size of every chunk it outputs. + static bytes = 1; state: CodecState = "unconfigured"; #output: EncodedAudioChunkOutputCallback; @@ -97,8 +99,8 @@ class LaggingAudioEncoder { type: "key", timestamp, duration, - byteLength: 1, - copyTo: (dest: Uint8Array) => dest.set([1]), + byteLength: LaggingAudioEncoder.bytes, + copyTo: (dest: Uint8Array) => dest.fill(1), }; this.#output(chunk as unknown as EncodedAudioChunk); } @@ -367,6 +369,47 @@ test("a group duration packs frames until the minimum and restarts after a deman expect(groups).toEqual([5, 2, 1, 2]); }); +// A long group duration with short frames would pass the net per-group caps, which abort the +// group with GroupTooLarge, so the writer rolls to a new group first. +test("a long group duration rolls the group at the frame cap", async () => { + using _webcodecs = installFakeWebCodecs(); + using env = await setup(new Baseline(), { mime: "opus", frameDuration: Time.Milli(2.5) }, Time.Milli(60_000)); + const { feed, groups, appended } = env; + + const frames = Moq.Group.MAX_GROUP_FRAMES + 3; + for (let i = 0; i < frames; i++) { + await feed.push({ timestamp: Time.Micro(i * 2_500), channels: [new Float32Array(120)] }); + } + await feed.drain(); + + // The fake encoder holds two frames back. + expect(groups).toEqual([Moq.Group.MAX_GROUP_FRAMES, 1]); + expect(appended[0].closed.peek()).toBeNull(); + expect(appended[1].closed.peek()).toBeUndefined(); +}); + +test("a long group duration rolls the group at the byte cap", async () => { + using _webcodecs = installFakeWebCodecs(); + const bytes = LaggingAudioEncoder.bytes; + LaggingAudioEncoder.bytes = 1024 * 1024; + try { + using env = await setup(new Baseline(), undefined, Time.Milli(60_000)); + const { feed, groups, appended } = env; + + for (let i = 0; i < 40; i++) { + await feed.push({ timestamp: Time.Micro(i * 20_000), channels: [new Float32Array(960)] }); + } + await feed.drain(); + + // Each frame carries a timestamp header, so 31 full MiB chunks fit under the 32 MiB cap. + expect(groups).toEqual([31, 7]); + expect(appended[0].closed.peek()).toBeNull(); + expect(appended[1].closed.peek()).toBeUndefined(); + } finally { + LaggingAudioEncoder.bytes = bytes; + } +}); + // A negative or non-finite duration has no meaning, so the rendition drops out of the catalog. test("an invalid group duration refuses the rendition", async () => { using _webcodecs = installFakeWebCodecs(); diff --git a/js/publish/src/audio/encoder.ts b/js/publish/src/audio/encoder.ts index ed0119afd7e..322212a023a 100644 --- a/js/publish/src/audio/encoder.ts +++ b/js/publish/src/audio/encoder.ts @@ -1,7 +1,7 @@ import * as Catalog from "@moq/hang/catalog"; import * as Container from "@moq/hang/container"; import * as Util from "@moq/hang/util"; -import type * as Moq from "@moq/net"; +import * as Moq from "@moq/net"; import { Time } from "@moq/net"; import { type Computed, @@ -145,7 +145,8 @@ export class Encoder { /** * The minimum timestamp span before a frame opens the next group, closing the previous one. 0 puts every frame in its own group. A longer group costs the * relay fewer streams but makes loss coarser: a viewer that falls behind skips a whole group. A - * negative or non-finite duration refuses the rendition. + * group also ends early at the per-group frame or byte cap. A negative or non-finite duration + * refuses the rendition. */ groupDuration: Signal; @@ -186,9 +187,12 @@ export class Encoder { // marker and are dropped. #floor: Time.Micro | undefined; - // The group still accepting frames and the track it belongs to, with its first timestamp. Only - // held while groupDuration is positive; otherwise each frame's group closes once written. - #group: { track: Moq.Track.Producer; group: Moq.Group.Producer; start: Time.Micro } | undefined; + // The group still accepting frames and the track it belongs to, with its first timestamp and the + // frames and bytes written so far. Only held while groupDuration is positive; otherwise each + // frame's group closes once written. + #group: + | { track: Moq.Track.Producer; group: Moq.Group.Producer; start: Time.Micro; frames: number; bytes: number } + | undefined; // The last valid group duration. #runConfig refuses the rendition on an invalid one, so a bad // value never reaches the writer. @@ -575,23 +579,32 @@ export class Encoder { // Every audio frame decodes on its own, so any of them can open a group: the first one at or past // the minimum after the open group's start. Frames forward as they are written rather than - // waiting for the group to fill. A dropped group leaves a gap for the codec's PLC. + // waiting for the group to fill. A dropped group leaves a gap for the codec's PLC. A group also + // ends before it would pass the net per-group caps, which abort it with GroupTooLarge. #write(track: Moq.Track.Producer, data: EncodedAudioChunk, timestamp: Time.Micro): void { const frame = { payload: Container.Legacy.encodeFrame(data, timestamp), timestamp: Time.Timestamp.fromMicros(timestamp), }; + const bytes = frame.payload.byteLength; const open = this.#group; - if (open?.track === track && timestamp - open.start < this.#groupDuration) { + if ( + open?.track === track && + timestamp - open.start < this.#groupDuration && + open.frames < Moq.Group.MAX_GROUP_FRAMES && + open.bytes + bytes <= Moq.Group.MAX_GROUP_CACHE_BYTES + ) { open.group.writeFrame(frame); + open.frames += 1; + open.bytes += bytes; return; } this.#closeGroup(); const group = track.appendGroup(); group.writeFrame(frame); - if (this.#groupDuration > 0) this.#group = { track, group, start: timestamp }; + if (this.#groupDuration > 0) this.#group = { track, group, start: timestamp, frames: 1, bytes }; else group.close(); }