Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions doc/lib/js/publish.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,10 @@ Each audio frame is its own group by default, so a relay pays a stream and a
group's bookkeeping per 20ms frame. `el.audio.groupDuration` sets a minimum
per group instead, such as `Time.Milli(100)`: the first frame at least 100ms
after the group's first timestamp opens the next group and closes the previous
one. If encoding pauses, the current group stays open until the next frame, a
timeline marker, or the encoder stopping. Frames still forward as they are
one. A group also ends early at the per-group cap of 8192 frames or 32 MiB, so
a long duration with short frames splits into several groups. If encoding
pauses, the current group stays open until the next frame, a timeline marker,
or the encoder stopping. Frames still forward as they are
encoded, so grouping does not buffer them, but loss gets coarser: a viewer that
falls behind skips a whole group, and a lost frame holds back the rest of its
group until it is retransmitted. A 60ms Opus `frameDuration` also cuts the
Expand Down
47 changes: 45 additions & 2 deletions js/publish/src/audio/encoder.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ class LaggingAudioEncoder {

// Called on configure; the encoder publishes its pipeline synchronously right after.
static onConfigure: ((config: AudioEncoderConfig) => void) | undefined;
// The size of every chunk it outputs.
static bytes = 1;

state: CodecState = "unconfigured";
#output: EncodedAudioChunkOutputCallback;
Expand Down Expand Up @@ -97,8 +99,8 @@ class LaggingAudioEncoder {
type: "key",
timestamp,
duration,
byteLength: 1,
copyTo: (dest: Uint8Array) => dest.set([1]),
byteLength: LaggingAudioEncoder.bytes,
copyTo: (dest: Uint8Array) => dest.fill(1),
};
this.#output(chunk as unknown as EncodedAudioChunk);
}
Expand Down Expand Up @@ -367,6 +369,47 @@ test("a group duration packs frames until the minimum and restarts after a deman
expect(groups).toEqual([5, 2, 1, 2]);
});

// A long group duration with short frames would pass the net per-group caps, which abort the
// group with GroupTooLarge, so the writer rolls to a new group first.
test("a long group duration rolls the group at the frame cap", async () => {
using _webcodecs = installFakeWebCodecs();
using env = await setup(new Baseline(), { mime: "opus", frameDuration: Time.Milli(2.5) }, Time.Milli(60_000));
const { feed, groups, appended } = env;

const frames = Moq.Group.MAX_GROUP_FRAMES + 3;
for (let i = 0; i < frames; i++) {
await feed.push({ timestamp: Time.Micro(i * 2_500), channels: [new Float32Array(120)] });
}
await feed.drain();

// The fake encoder holds two frames back.
expect(groups).toEqual([Moq.Group.MAX_GROUP_FRAMES, 1]);
expect(appended[0].closed.peek()).toBeNull();
expect(appended[1].closed.peek()).toBeUndefined();
});

test("a long group duration rolls the group at the byte cap", async () => {
using _webcodecs = installFakeWebCodecs();
const bytes = LaggingAudioEncoder.bytes;
LaggingAudioEncoder.bytes = 1024 * 1024;
try {
using env = await setup(new Baseline(), undefined, Time.Milli(60_000));
const { feed, groups, appended } = env;

for (let i = 0; i < 40; i++) {
await feed.push({ timestamp: Time.Micro(i * 20_000), channels: [new Float32Array(960)] });
}
await feed.drain();

// Each frame carries a timestamp header, so 31 full MiB chunks fit under the 32 MiB cap.
expect(groups).toEqual([31, 7]);
expect(appended[0].closed.peek()).toBeNull();
expect(appended[1].closed.peek()).toBeUndefined();
} finally {
LaggingAudioEncoder.bytes = bytes;
}
});

// A negative or non-finite duration has no meaning, so the rendition drops out of the catalog.
test("an invalid group duration refuses the rendition", async () => {
using _webcodecs = installFakeWebCodecs();
Expand Down
29 changes: 21 additions & 8 deletions js/publish/src/audio/encoder.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import * as Catalog from "@moq/hang/catalog";
import * as Container from "@moq/hang/container";
import * as Util from "@moq/hang/util";
import type * as Moq from "@moq/net";
import * as Moq from "@moq/net";
import { Time } from "@moq/net";
import {
type Computed,
Expand Down Expand Up @@ -145,7 +145,8 @@ export class Encoder {
/**
* The minimum timestamp span before a frame opens the next group, closing the previous one. 0 puts every frame in its own group. A longer group costs the
* relay fewer streams but makes loss coarser: a viewer that falls behind skips a whole group. A
* negative or non-finite duration refuses the rendition.
* group also ends early at the per-group frame or byte cap. A negative or non-finite duration
* refuses the rendition.
*/
groupDuration: Signal<Time.Milli>;

Expand Down Expand Up @@ -186,9 +187,12 @@ export class Encoder {
// marker and are dropped.
#floor: Time.Micro | undefined;

// The group still accepting frames and the track it belongs to, with its first timestamp. Only
// held while groupDuration is positive; otherwise each frame's group closes once written.
#group: { track: Moq.Track.Producer; group: Moq.Group.Producer; start: Time.Micro } | undefined;
// The group still accepting frames and the track it belongs to, with its first timestamp and the
// frames and bytes written so far. Only held while groupDuration is positive; otherwise each
// frame's group closes once written.
#group:
| { track: Moq.Track.Producer; group: Moq.Group.Producer; start: Time.Micro; frames: number; bytes: number }
| undefined;

// The last valid group duration. #runConfig refuses the rendition on an invalid one, so a bad
// value never reaches the writer.
Expand Down Expand Up @@ -575,23 +579,32 @@ export class Encoder {

// Every audio frame decodes on its own, so any of them can open a group: the first one at or past
// the minimum after the open group's start. Frames forward as they are written rather than
// waiting for the group to fill. A dropped group leaves a gap for the codec's PLC.
// waiting for the group to fill. A dropped group leaves a gap for the codec's PLC. A group also
// ends before it would pass the net per-group caps, which abort it with GroupTooLarge.
#write(track: Moq.Track.Producer, data: EncodedAudioChunk, timestamp: Time.Micro): void {
const frame = {
payload: Container.Legacy.encodeFrame(data, timestamp),
timestamp: Time.Timestamp.fromMicros(timestamp),
};

const bytes = frame.payload.byteLength;
const open = this.#group;
if (open?.track === track && timestamp - open.start < this.#groupDuration) {
if (
open?.track === track &&
timestamp - open.start < this.#groupDuration &&
open.frames < Moq.Group.MAX_GROUP_FRAMES &&
open.bytes + bytes <= Moq.Group.MAX_GROUP_CACHE_BYTES
) {
open.group.writeFrame(frame);
open.frames += 1;
open.bytes += bytes;
return;
}

this.#closeGroup();
const group = track.appendGroup();
group.writeFrame(frame);
if (this.#groupDuration > 0) this.#group = { track, group, start: timestamp };
if (this.#groupDuration > 0) this.#group = { track, group, start: timestamp, frames: 1, bytes };
else group.close();
}

Expand Down
Loading