Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 34 additions & 7 deletions js/publish/src/audio/encoder.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,9 @@ describe("resolve", () => {
});
});

// Like Chrome's Opus encoder, it holds the newest chunks until later input pushes them out.
// Like Chrome's Opus encoder, it holds the newest chunks until later input pushes them out, and it
// numbers its output from its own sample count since the first input after a (re)configure, so a
// hole in the input timestamps never reaches the output.
class LaggingAudioEncoder {
static readonly LAG = 2;

Expand All @@ -64,6 +66,7 @@ class LaggingAudioEncoder {
state: CodecState = "unconfigured";
#output: EncodedAudioChunkOutputCallback;
#held: { timestamp: number; duration: number }[] = [];
#next: number | undefined;

constructor(init: AudioEncoderInit) {
this.#output = init.output;
Expand All @@ -74,9 +77,17 @@ class LaggingAudioEncoder {
LaggingAudioEncoder.onConfigure?.();
}

reset(): void {
this.state = "unconfigured";
this.#held = [];
this.#next = undefined;
}

encode(data: AudioData): void {
const duration = Math.round((data.numberOfFrames / data.sampleRate) * 1_000_000);
this.#held.push({ timestamp: data.timestamp, duration });
const timestamp = this.#next ?? data.timestamp;
this.#next = timestamp + duration;
this.#held.push({ timestamp, duration });
while (this.#held.length > LaggingAudioEncoder.LAG) {
const { timestamp, duration } = this.#held.shift() as { timestamp: number; duration: number };
const chunk = {
Expand Down Expand Up @@ -171,10 +182,12 @@ class Feed {
}
}

// The encoder outlives a demand gap, so chunks it held when demand disappeared surface after the
// resume. Written after the marker, they would put pre-gap media on the live edge, and a rounding
// step below the marker aborts every subscriber.
test("a demand gap marks where submitted audio ends and drops the chunks held across it", async () => {
// Chunks the encoder held when demand disappeared must not surface after the marker: they would put
// pre-gap media on the live edge, and a rounding step below the marker aborts every subscriber. And
// since the encoder numbers its output from its own sample count, the resumed audio has to restart
// on the capture clock: otherwise it reads as older than it is by the gap, and the next gap's marker
// drops that much of it (seconds of silence after a pause and a rejoin).
test("a demand gap marks where submitted audio ends and resumes on the capture clock", async () => {
using _webcodecs = installFakeWebCodecs();
const configured = new Promise<void>((resolve) => {
LaggingAudioEncoder.onConfigure = resolve;
Expand Down Expand Up @@ -234,14 +247,28 @@ test("a demand gap marks where submitted audio ends and drops the chunks held ac

await push(2); // gated
rendition.track.set(track);
await push(4); // releases the two held pre-gap chunks, then two resumed ones
await push(4); // two resumed chunks written, two held

const remarked = new Promise<void>((resolve) => {
onWrite = resolve;
});
rendition.track.set(undefined);
await remarked;
onWrite = undefined;

await push(3); // gated
rendition.track.set(track);
await push(4);

expect(written).toEqual([
[18_700, 1],
[38_700, 1],
[98_700, 0],
[138_700, 1],
[158_700, 1],
[218_700, 0],
[278_700, 1],
[298_700, 1],
]);
} finally {
LaggingAudioEncoder.onConfigure = undefined;
Expand Down
24 changes: 21 additions & 3 deletions js/publish/src/audio/encoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -160,8 +160,9 @@ export class Encoder {
// newest one, where a demand gap's discontinuity marker goes. Cleared once the marker is written.
#next: Time.Micro | undefined;

// The newest demand gap's marker. The AudioEncoder outlives the gap, so chunks it still held
// when demand disappeared surface after the resume; they sit below the marker and are dropped.
// The newest demand gap's marker. Chunks the AudioEncoder still held when demand disappeared can
// surface once it returns (before the pipeline resets the encoder, or when no frame was skipped at
// all); they sit below the marker and are dropped.
#floor: Time.Micro | undefined;

// The fatal error an AudioEncoder reported, if any. That instance can never encode again and
Expand Down Expand Up @@ -443,6 +444,9 @@ export class Encoder {
console.debug("encoding audio", encoderConfig);
encoder.configure(encoderConfig);

// Whether the demand gate skipped a frame since the last one submitted.
let skipped = false;

const pipeline: Pipeline = {
channelCount: config.numberOfChannels,
push: (captured: AudioFrame) => {
Expand All @@ -452,7 +456,21 @@ export class Encoder {
for (const data of framer.push(input)) {
// The demand gate. The framer still consumes every sample so its timestamps stay
// on the capture clock, but there is nowhere to send a chunk with no subscriber.
if (!track.peek()) continue;
if (!track.peek()) {
skipped = true;
continue;
}

// The AudioEncoder numbers its output from its own sample count (Chrome does), so
// it never sees the frames the gate skipped: every chunk after the gap would carry
// a timestamp that far behind the capture clock, below the gap's marker. Restart its
// timeline at this frame. The chunks it still held from before the gap go with it,
// and they sit below the marker anyway.
if (skipped) {
skipped = false;
encoder.reset();
encoder.configure(encoderConfig);
}

// Round to whole microseconds once, here, so a chunk's timestamp and the marker
// placed at the next frame's start agree exactly.
Expand Down
Loading