Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions doc/lib/js/publish.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,17 @@ framerate, and bitrate are tunable through `el.video.config`; the audio
encoder exposes its codec and volume. For simulcast or several renditions,
drop the element and register your own encoders on a `Publish.Broadcast`.

Each audio frame is its own group by default, so a relay pays a stream and a
group's bookkeeping per 20ms frame. `el.audio.groupDuration` sets a minimum
per group instead, such as `Time.Milli(100)`: the first frame at least 100ms
after the group's first timestamp opens the next group and closes the previous
one. If encoding pauses, the current group stays open until the next frame, a
timeline marker, or the encoder stopping. Frames still forward as they are
encoded, so grouping does not buffer them, but loss gets coarser: a viewer that
falls behind skips a whole group, and a lost frame holds back the rest of its
group until it is retransmitted. A 60ms Opus `frameDuration` also cuts the
group rate, without code, at the cost of encoder latency.

The video and audio encoders measure how far their output falls behind the media
clock when they flush frames. Catalog jitter is the spread above each
rendition's own recent minimum lateness, so a constant encoder delay is not jitter.
Expand Down
2 changes: 2 additions & 0 deletions js/publish/api/element.api.md
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,7 @@ class Encoder_2 {
get capture(): Capture_2 | undefined;
close(): void;
codec: Signal<Codec>;
groupDuration: Signal<Time.Milli>;
readonly in: Readonlys<EncoderInput_2>;
readonly name: string;
readonly out: Readonlys<EncoderOutput_2>;
Expand Down Expand Up @@ -259,6 +260,7 @@ type EncoderProps = Inputs<EncoderInput> & {
type EncoderProps_2 = Inputs<EncoderInput_2> & {
volume?: number | Signal<number>;
codec?: Codec | Signal<Codec>;
groupDuration?: Time.Milli | Signal<Time.Milli>;
};

class Fanout<T> {
Expand Down
2 changes: 2 additions & 0 deletions js/publish/api/index.api.md
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,7 @@ class Encoder {
get capture(): Capture | undefined;
close(): void;
codec: Signal<Codec>;
groupDuration: Signal<Time.Milli>;
readonly in: Readonlys<EncoderInput>;
readonly name: string;
readonly out: Readonlys<EncoderOutput>;
Expand Down Expand Up @@ -286,6 +287,7 @@ type EncoderOutput_2 = {
type EncoderProps = Inputs<EncoderInput> & {
volume?: number | Signal<number>;
codec?: Codec | Signal<Codec>;
groupDuration?: Time.Milli | Signal<Time.Milli>;
};

type EncoderProps_2 = Inputs<EncoderInput_2> & {
Expand Down
90 changes: 81 additions & 9 deletions js/publish/src/audio/encoder.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -192,21 +192,32 @@ class Feed {
}
}

// An Encoder wired to a fake capture feed, recording each written frame as [timestamp, payload bytes].
async function setup(baseline = new Baseline(), codec?: Codec) {
// An Encoder wired to a fake capture feed, recording each written frame as [timestamp, payload bytes]
// and how many frames each group carries.
async function setup(baseline = new Baseline(), codec?: Codec, groupDuration?: Time.Milli) {
const configured = new Promise<AudioEncoderConfig>((resolve) => {
LaggingAudioEncoder.onConfigure = resolve;
});

const track = new Moq.Track.Producer("audio").accept();
const written: [number, number][] = [];
const groups: number[] = [];
const appended: ReturnType<typeof track.appendGroup>[] = [];
const writes = { onWrite: undefined as (() => void) | undefined };
const writeFrame = track.writeFrame.bind(track);
track.writeFrame = (frame) => {
const [timestamp, payload] = Moq.Varint.decode(frame.payload);
written.push([timestamp, payload.byteLength]);
writeFrame(frame);
writes.onWrite?.();
const appendGroup = track.appendGroup.bind(track);
track.appendGroup = () => {
const group = appendGroup();
appended.push(group);
const index = groups.push(0) - 1;
const writeFrame = group.writeFrame.bind(group);
group.writeFrame = (frame) => {
const [timestamp, payload] = Moq.Varint.decode(frame.payload);
written.push([timestamp, payload.byteLength]);
groups[index]++;
writeFrame(frame);
writes.onWrite?.();
};
return group;
};

const rendition = {
Expand All @@ -230,6 +241,7 @@ async function setup(baseline = new Baseline(), codec?: Codec) {
broadcast: { audio: () => rendition, baseline } as never,
capture: capture as never,
codec,
groupDuration,
});

const config = await configured;
Expand All @@ -242,6 +254,8 @@ async function setup(baseline = new Baseline(), codec?: Codec) {
rendition,
feed,
written,
groups,
appended,
writes,
[Symbol.dispose]() {
encoder.close();
Expand Down Expand Up @@ -294,7 +308,7 @@ test("a demand gap marks where submitted audio ends and drops the chunks held ac
test("a push completing several frames keeps the encoder running", async () => {
using _webcodecs = installFakeWebCodecs();
using env = await setup();
const { feed, written } = env;
const { feed, written, appended } = env;

// Two 20ms Opus frames per push.
for (let index = 0; index < 3; index++) {
Expand All @@ -308,6 +322,64 @@ test("a push completing several frames keeps the encoder running", async () => {
[58_700, 1],
[78_700, 1],
]);
// Without a group duration, each frame's group closes as soon as it is written.
expect(appended.map((group) => group.closed.peek())).toEqual([null, null, null, null]);
});

// The first frame at the minimum opens the next group. A demand gap closes the group early, so the
// first frame after it opens a fresh one.
test("a group duration packs frames until the minimum and restarts after a demand gap", async () => {
using _webcodecs = installFakeWebCodecs();
using env = await setup(new Baseline(), undefined, Time.Milli(100));
const { track, rendition, feed, written, groups, appended, writes } = env;

let index = 0;
const push = async (count: number) => {
for (let i = 0; i < count; i++, index++) {
await feed.push({ timestamp: Time.Micro(20_000 + index * 20_000), channels: [new Float32Array(960)] });
}
await feed.drain();
};

await push(7); // five written, two held
expect(groups).toEqual([5]);
expect(appended[0].closed.peek()).toBeUndefined();
await push(2); // the next frame opens a new group and closes the first
expect(groups).toEqual([5, 2]);
expect(appended[0].closed.peek()).toBeNull();

const marked = new Promise<void>((resolve) => {
writes.onWrite = resolve;
});
rendition.track.set(undefined);
await marked;
writes.onWrite = undefined;
expect(appended[1].closed.peek()).toBeNull();

rendition.track.set(track);
await push(4); // releases the two held pre-gap chunks, which are dropped, then two resumed ones

expect(written.map(([timestamp]) => timestamp)).toEqual([
20_000, 40_000, 60_000, 80_000, 100_000, 120_000, 140_000, 200_000, 200_000, 220_000,
]);
// 120ms opens the next group, closing the five frames at 20-100ms. The gap ends the 120-140ms
// group early, so resumed frames start a fresh group after the marker.
expect(groups).toEqual([5, 2, 1, 2]);
});

// A negative or non-finite duration has no meaning, so the rendition drops out of the catalog.
test("an invalid group duration refuses the rendition", async () => {
using _webcodecs = installFakeWebCodecs();
using env = await setup(new Baseline(), undefined, Time.Milli(100));
const { encoder } = env;

expect(encoder.out.catalog.peek()).toBeDefined();
encoder.groupDuration.set(Time.Milli(-1));
await settle();
expect(encoder.out.catalog.peek()).toBeUndefined();
encoder.groupDuration.set(Time.Milli(0));
await settle();
expect(encoder.out.catalog.peek()).toBeDefined();
});

test("passes an explicit Opus DTX request to the encoder", async () => {
Expand Down
66 changes: 58 additions & 8 deletions js/publish/src/audio/encoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,9 @@ export type EncoderProps = Inputs<EncoderInput> & {

// Codec selection plus encoder settings. Defaults to "opus".
codec?: Codec | Signal<Codec>;

// The minimum audio carried by each group. Defaults to 0, a group per frame.
groupDuration?: Time.Milli | Signal<Time.Milli>;
};

type EncoderOutput = {
Expand Down Expand Up @@ -139,6 +142,12 @@ export class Encoder {
volume: Signal<number>;
/** The live-editable codec selection plus its encoder settings. */
codec: Signal<Codec>;
/**
* The minimum timestamp span before a frame opens the next group, closing the previous one. 0 puts every frame in its own group. A longer group costs the
* relay fewer streams but makes loss coarser: a viewer that falls behind skips a whole group. A
* negative or non-finite duration refuses the rendition.
*/
groupDuration: Signal<Time.Milli>;

/**
* The capture supplying this rendition, or undefined while none is wired.
Expand Down Expand Up @@ -177,6 +186,14 @@ export class Encoder {
// marker and are dropped.
#floor: Time.Micro | undefined;

// The group still accepting frames and the track it belongs to, with its first timestamp. Only
// held while groupDuration is positive; otherwise each frame's group closes once written.
#group: { track: Moq.Track.Producer; group: Moq.Group.Producer; start: Time.Micro } | undefined;

// The last valid group duration. #runConfig refuses the rendition on an invalid one, so a bad
// value never reaches the writer.
#groupDuration = Time.Micro(0);

// The fatal error an AudioEncoder reported, if any. That instance can never encode again and
// reconfiguring it would be a retry, so the rendition stays down for the life of this encoder.
#fatal = new Signal<Error | undefined>(undefined);
Expand Down Expand Up @@ -212,6 +229,7 @@ export class Encoder {
};
this.volume = Signal.from(props?.volume ?? 1);
this.codec = Signal.from<Codec>(props?.codec ?? "opus");
this.groupDuration = Signal.from(props?.groupDuration ?? Time.Milli(0));

// Only the capture graph has a node to expose.
this.#signals.run((effect) => {
Expand Down Expand Up @@ -318,6 +336,8 @@ export class Encoder {
const track = effect.get(rendition.track);
if (!track) return;
effect.cleanup(() => {
// The marker's group follows, so the open one ends here and resumed frames open a fresh one.
this.#closeGroup();
const end = this.#next;
this.#next = undefined;
if (end === undefined || track.closed.peek() !== undefined) return;
Expand Down Expand Up @@ -369,6 +389,11 @@ export class Encoder {
// Gated on `enabled` the same way the video encoder is: a disabled rendition has to drop out of
// the catalog, and a sample source keeps its format while muted rather than tearing down.
#runConfig(effect: Effect): void {
const groupDuration = effect.get(this.groupDuration);
if (!Number.isFinite(groupDuration) || groupDuration < 0)
throw new Error(`audio group duration must be a finite, non-negative number of ms: ${groupDuration}`);
this.#groupDuration = Time.Micro.fromMilli(groupDuration);

const capture = effect.get(this.in.capture);
const captured = capture ? effect.get(capture.out.format) : undefined;
if (!effect.get(this.in.enabled) || !captured) {
Expand Down Expand Up @@ -456,15 +481,11 @@ export class Encoder {
bytes: stats.bytes + frame.byteLength,
}));

// Each audio frame is its own group so the relay can forward it without
// waiting for a group boundary. Loss is handled by the codec's PLC.
const live = track.peek();
if (!live) return;
if (this.#floor !== undefined && frame.timestamp < this.#floor) return;
live.writeFrame({
payload: Container.Legacy.encodeFrame(frame, frame.timestamp as Time.Micro),
timestamp: Time.Timestamp.fromMicros(frame.timestamp as Time.Micro),
});
const timestamp = frame.timestamp as Time.Micro;
if (this.#floor !== undefined && timestamp < this.#floor) return;
this.#write(live, frame, timestamp);
if (this.#estimator.flush(frame.timestamp, baseline)) {
const catalog = this.#out.catalog.peek();
if (catalog) this.#out.catalog.set({ ...catalog, ...this.#estimator.estimate });
Expand All @@ -477,9 +498,11 @@ export class Encoder {
this.#fatal.set(err);
},
});
// A fatal error already closed the codec, and closing it twice throws.
// A fatal error already closed the codec, and closing it twice throws. A rebuilt encoder
// restarts its output clock, so its first frame opens a fresh group.
effect.cleanup(() => {
if (encoder.state !== "closed") encoder.close();
this.#closeGroup();
});

console.debug("encoding audio", encoderConfig);
Expand Down Expand Up @@ -550,6 +573,33 @@ export class Encoder {
});
}

// Every audio frame decodes on its own, so any of them can open a group: the first one at or past
// the minimum after the open group's start. Frames forward as they are written rather than
// waiting for the group to fill. A dropped group leaves a gap for the codec's PLC.
#write(track: Moq.Track.Producer, data: EncodedAudioChunk, timestamp: Time.Micro): void {
const frame = {
payload: Container.Legacy.encodeFrame(data, timestamp),
timestamp: Time.Timestamp.fromMicros(timestamp),
};

const open = this.#group;
if (open?.track === track && timestamp - open.start < this.#groupDuration) {
open.group.writeFrame(frame);
return;
}

this.#closeGroup();
const group = track.appendGroup();
group.writeFrame(frame);
if (this.#groupDuration > 0) this.#group = { track, group, start: timestamp };
else group.close();
}

#closeGroup(): void {
this.#group?.group.close();
this.#group = undefined;
}

#setDecoderDescription(config: Catalog.AudioConfig, source: AllowSharedBufferSource | undefined): void {
if (config.codec !== "opus" || !source) return;

Expand Down
Loading