Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions js/watch/src/audio/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ export function decoderDescription(codec: string, description: Uint8Array | unde
* Undefined for Opus, whose packets state their own duration (see `Util.Opus.packetSamples`), and
* for codecs with no constant. Never learned from observed timestamps.
*/
export function frameDuration(config: Pick<Catalog.AudioConfig, "codec" | "sampleRate">): Duration | undefined {
export function frameDuration(config: { readonly codec: string; readonly sampleRate: number }): Duration | undefined {
const samples = frameSamples(config);
if (samples === undefined) return undefined;
return Duration.millis((samples * 1000) / config.sampleRate);
Expand All @@ -96,7 +96,7 @@ export function packetDuration(codec: string, frame: Container.Frame): Duration
return samples === undefined ? undefined : Duration.millis((samples * 1000) / OPUS_RATE);
}

function frameSamples(config: Pick<Catalog.AudioConfig, "codec" | "sampleRate">): number | undefined {
function frameSamples(config: { readonly codec: string; readonly sampleRate: number }): number | undefined {
switch (Util.Aac.objectType(config.codec)) {
case 2:
return AAC_LC_FRAME_SAMPLES;
Expand Down
75 changes: 66 additions & 9 deletions js/watch/src/audio/decoder.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -481,7 +481,7 @@ for (const description of [undefined, "1190"]) {
}

for (const cmaf of [false, true]) {
it(`a ${cmaf ? "CMAF" : "legacy"} gap releases accounting for codec outputs it discarded`, async () => {
it(`a ${cmaf ? "CMAF" : "legacy"} gap lands the codec outputs still pending before restarting it`, async () => {
const clock = new Manual();
const installed = clock.install();
deferAudioOutput = true;
Expand All @@ -496,15 +496,9 @@ for (const cmaf of [false, true]) {
expect(playback.inserted()).toHaveLength(0);
playback.push({ kind: "gap", timestamp: Duration.micros(140_000), duration: Duration.micros(20_000) });
await microtasks();
// A starved decoder can hold everything since the last gap, which a reset alone would drop.
expect(pendingAudio).toHaveLength(0);
deferAudioOutput = false;
frameTimestamp = 160_000;
const resumed = frame();
playback.push({ ...resumed, group: 2n });
await microtasks();
expect(audioInputs).toHaveLength(7);
expect(playback.inserted()).toContain(resumed.frame.timestamp.asMicros());
await playback.play(5);
expect(playback.inserted()).toHaveLength(6);
expect(playback.sync.playing.peek()).toBe(true);
} finally {
playback.close();
Expand All @@ -530,6 +524,69 @@ it("decodes an audio packet inside a group after a gap", async () => {
}
});

// The warmup the restarted decoder needs is the last frames before the gap, restamped to end where
// the frame after it starts, so it trims their output instead of the frame's.
it("re-primes the decoder on the warmup before a gap, so the frame after it plays", async () => {
const playback = await play(Duration.millis(100));
try {
playback.restore({ warmup: 80 });
await microtasks();
await playback.play();
const played = audioChunks.length;
const decoded = playback.feedback().audio?.decoded ?? 0;
playback.push({
kind: "gap",
timestamp: Duration.micros(frameTimestamp + 20_000),
duration: Duration.millis(100),
});
frameTimestamp += 100_000;
const next = frame();
playback.push({ ...next, group: 1n });
await microtasks();
const at = next.frame.timestamp.asMicros();
expect(audioChunks.slice(played).map((chunk) => chunk.timestamp)).toEqual([
at - 80_000,
at - 60_000,
at - 40_000,
at - 20_000,
at,
]);
expect(playback.inserted()).toContain(at);
// The pre-roll's output is not counted as decoded again; only the frame after the gap is.
expect(playback.feedback().audio?.decoded).toBe(decoded + 1);
} finally {
playback.close();
await microtasks();
}
});

// AAC frames state no duration, so the codec's frame length (1024 samples) is what the pre-roll covers.
it("re-primes an AAC decoder on frames that state no duration", async () => {
const playback = await play(Duration.millis(100));
try {
playback.restore({ codec: "mp4a.40.2", sampleRate: 48_000, numberOfChannels: 2, warmup: 80 });
await microtasks();
await playback.play();
const played = audioChunks.length;
playback.push({
kind: "gap",
timestamp: Duration.micros(frameTimestamp + 20_000),
duration: Duration.millis(100),
});
frameTimestamp += 100_000;
const next = frame();
playback.push({ ...next, group: 1n });
await microtasks();
// Four 21.3 ms frames cover the 80 ms warmup, ahead of the frame after the gap.
const chunks = audioChunks.slice(played).map((chunk) => chunk.timestamp);
expect(chunks).toHaveLength(5);
expect(chunks.at(-1)).toBe(next.frame.timestamp.asMicros());
} finally {
playback.close();
await microtasks();
}
});

// A subscription's head arrives newest-first, one group per frame. Chrome's AudioDecoder stamps every
// output from its first input, so decoding the older frames after it shifts the rest of the track.
for (const cmaf of [false, true]) {
Expand Down
65 changes: 46 additions & 19 deletions js/watch/src/audio/decoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -686,8 +686,7 @@ export class Decoder {
let written: number | undefined;
const decoder = new AudioDecoder({
output: (data) => {
this.#count(key, { decoded: 1 });
const landed = this.#emit(data);
const landed = this.#emit(data, key);
if (landed !== undefined) written = Math.max(written ?? landed, landed);
},
error: (error) => failure.set(error),
Expand Down Expand Up @@ -725,9 +724,7 @@ export class Decoder {
await this.#onFinish(effect, decoder, () => written);
break;
}
if (this.#onRead(next)) {
this.#reprime(decoder, decoderConfig, order);
}
if (!(await this.#apply(effect, decoder, decoderConfig, order, next))) return;
continue;
}
const { frame } = next;
Expand Down Expand Up @@ -783,8 +780,7 @@ export class Decoder {
let written: number | undefined;
const decoder = new AudioDecoder({
output: (data) => {
this.#count(key, { decoded: 1 });
const landed = this.#emit(data);
const landed = this.#emit(data, key);
if (landed !== undefined) written = Math.max(written ?? landed, landed);
},
error: (error) => failure.set(error),
Expand Down Expand Up @@ -819,9 +815,7 @@ export class Decoder {
await this.#onFinish(effect, decoder, () => written);
break;
}
if (this.#onRead(next)) {
this.#reprime(decoder, decoderConfig, order);
}
if (!(await this.#apply(effect, decoder, decoderConfig, order, next))) return;
continue;
}
const { frame } = next;
Expand Down Expand Up @@ -854,12 +848,31 @@ export class Decoder {
reads: FrameRead[],
): Promise<boolean> {
for (const read of reads) {
if (this.#onRead(read)) {
this.#reprime(decoder, config, order);
}
if (!(await this.#apply(effect, decoder, config, order, read))) return false;
if (!order.admit(read)) continue;
if (!(await this.#decode(effect, decoder, read.frame))) return false;
// A frame stating no duration, such as AAC's, runs the codec's frame length, as queuedRanges
// counts it.
const duration = packetDuration(config.codec, read.frame) ?? frameDuration(config);
if (!(await this.#decode(effect, decoder, read.frame, duration))) return false;
}
return true;
}

// Apply an ordered read, restarting the codec when it ends an epoch, and report whether to keep
// going. The decoder first hands over what it holds, mapped on the epoch it was decoded in: a reset
// would drop it, and a starved decoder skipping groups can hold everything since the last skip.
async #apply(
effect: Effect,
decoder: AudioDecoder,
config: AudioDecoderConfig,
order: Order<FrameRead>,
read: Container.Read,
): Promise<boolean> {
if (this.#terminal.ends(read) && decoder.state === "configured") {
await decoder.flush();
if (effect.abort.aborted) return false;
}
if (this.#onRead(read)) this.#reprime(decoder, config, order);
return true;
}

Expand All @@ -871,8 +884,14 @@ export class Decoder {
order.reset();
}

// Decode one frame once backpressure admits it, reporting whether to keep going.
async #decode(effect: Effect, decoder: AudioDecoder, frame: Container.ReceivedFrame): Promise<boolean> {
// Decode one frame once backpressure admits it, behind any pre-roll a gap left, reporting whether
// to keep going.
async #decode(
effect: Effect,
decoder: AudioDecoder,
frame: Container.ReceivedFrame,
duration: Duration | undefined,
): Promise<boolean> {
const ring = await this.#ready(effect);
if (!ring) return false;

Expand All @@ -885,6 +904,9 @@ export class Decoder {
// A fatal decode error closes the decoder, so decoding again throws InvalidStateError out
// of the loop. Stop instead: the error callback already reported the real failure.
if (decoder.state === "closed") return false;
for (const preroll of this.#terminal.preroll()) {
decoder.decode(new EncodedAudioChunk({ type: "key", data: preroll.payload, timestamp: preroll.timestamp }));
}
decoder.decode(
new EncodedAudioChunk({
// WebCodecs audio chunks are key even inside a MoQ group.
Expand All @@ -893,6 +915,7 @@ export class Decoder {
timestamp: frame.timestamp.asMicros(),
}),
);
this.#terminal.decoded(frame, duration);
return true;
}

Expand Down Expand Up @@ -1028,17 +1051,21 @@ export class Decoder {
this.#handover.declared(end);
}

// Place decoded audio, returning where the PCM written ends, in microseconds.
#emit(sample: AudioData): number | undefined {
// Count and place decoded audio, returning where the PCM written ends, in microseconds.
#emit(sample: AudioData, key: Key | undefined): number | undefined {
// Whatever becomes of it, the output answers the frames it was decoded from, so the gate counts
// only what still decodes. Reported after any insert, so the gate always counts the frame as
// either decoding or in the ring.
// Pre-roll output answers no frame: it only primes the decoder, and the warmup trims it. It is
// not counted as decoded either, since the frames it came from already were.
const preroll = this.#terminal.primes(sample);
Comment thread
kixelated marked this conversation as resolved.
Resolved
if (!preroll) this.#count(key, { decoded: 1 });
const decoded = sample.timestamp + (sample.numberOfFrames / sample.sampleRate) * 1_000_000;
let landed: number | undefined;
try {
landed = this.#place(sample);
} finally {
this.#ring.peek()?.decoded(decoded, landed);
if (!preroll) this.#ring.peek()?.decoded(decoded, landed);
}
return landed;
}
Expand Down
103 changes: 103 additions & 0 deletions js/watch/src/audio/terminal.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -112,3 +112,106 @@ test("a hole preserves a declared endpoint until a later group starts a new time
expect(terminal.update(frame(2n, 1_000_000))).toBe(true);
expect(terminal.end).toBeUndefined();
});

const packet = (timestamp: number) => ({
timestamp: Duration.micros(timestamp),
payload: new Uint8Array([timestamp / 20_000]),
});
const opus = (timestamp: number) => ({ timestamp, sampleRate: 48_000, numberOfFrames: 960 });

test("a gap re-primes the decoder on the warmup before it, so the audio after it plays whole", () => {
const terminal = new Terminal();
terminal.clear(312, 80_000);
for (let i = 0; i < 6; i++) {
terminal.update(frame(0n, i * 20_000));
terminal.decoded(packet(i * 20_000), Duration.micros(20_000));
}
expect(terminal.update(gap)).toBe(true);
terminal.update(frame(2n, 1_000_000));

// The newest 80 ms, back to back and ending where the frame after the gap starts.
const preroll = terminal.preroll();
expect(preroll.map((p) => p.timestamp)).toEqual([920_000, 940_000, 960_000, 980_000]);
expect(preroll.map((p) => p.payload[0])).toEqual([2, 3, 4, 5]);
expect(terminal.preroll()).toEqual([]);

// The pre-roll's output takes the pre-skip and the warmup, and answers no frame.
for (const { timestamp } of preroll) {
expect(terminal.primes(opus(timestamp))).toBe(true);
expect(terminal.span(opus(timestamp)).frames).toBe(0);
}
// The overlap the pre-skip carries over the restart starts before the frame, so only it is trimmed.
expect(terminal.primes(opus(1_000_000))).toBe(false);
expect(terminal.span(opus(1_000_000))).toEqual({ timestamp: 1_000_000, frameOffset: 312, frames: 648 });
expect(terminal.span(opus(1_020_000))).toEqual({ timestamp: 1_013_500, frameOffset: 0, frames: 960 });
});

// A decoder that coalesces outputs can hand back the pre-roll and the frame after the gap together.
test("output reaching past the pre-roll holds the audio after the gap", () => {
const terminal = new Terminal();
terminal.clear(0, 80_000);
for (let i = 0; i < 4; i++) {
terminal.update(frame(0n, i * 20_000));
terminal.decoded(packet(i * 20_000), Duration.micros(20_000));
}
terminal.update(gap);
terminal.update(frame(1n, 1_000_000));
expect(terminal.preroll()).toHaveLength(4);

// 20 ms of pre-roll alone, then the other 60 ms of it with the frame after the gap.
expect(terminal.primes(opus(920_000))).toBe(true);
const coalesced = { timestamp: 940_000, sampleRate: 48_000, numberOfFrames: 4 * 960 };
expect(terminal.primes(coalesced)).toBe(false);
expect(terminal.span(coalesced)).toEqual({ timestamp: 1_000_000, frameOffset: 3 * 960, frames: 960 });
});

test("gaps closer than the warmup still play, the pre-roll carrying over the last one", () => {
const terminal = new Terminal();
terminal.clear(0, 80_000);
for (let i = 0; i < 4; i++) {
terminal.update(frame(0n, i * 20_000));
terminal.decoded(packet(i * 20_000), Duration.micros(20_000));
}
for (const start of [1_000_000, 2_000_000]) {
terminal.update(gap);
terminal.update(frame(1n, start));
expect(terminal.preroll().map((p) => p.timestamp)).toEqual([
start - 80_000,
start - 60_000,
start - 40_000,
start - 20_000,
]);
terminal.decoded(packet(start), Duration.micros(20_000));
expect(terminal.span(opus(start)).frames).toBe(960);
}
});

test("a short history trims only the warmup it leaves uncovered", () => {
const terminal = new Terminal();
terminal.clear(0, 80_000);
terminal.update(frame(0n, 0));
terminal.decoded(packet(0), Duration.micros(20_000));
terminal.update(frame(0n, 20_000));
// A frame of unknown duration cannot be restamped, so the pre-roll starts after it.
terminal.decoded(packet(20_000), undefined);
terminal.update(frame(0n, 40_000));
terminal.decoded(packet(40_000), Duration.micros(20_000));
terminal.update(gap);
terminal.update(frame(1n, 1_000_000));
expect(terminal.preroll().map((p) => p.timestamp)).toEqual([980_000]);
const span = terminal.span({ timestamp: 1_000_000, sampleRate: 48_000, numberOfFrames: 4_800 });
expect(span).toEqual({ timestamp: 1_060_000, frameOffset: 2_880, frames: 1_920 });
});

test("a fresh encoder after a declared end takes no pre-roll from the stopped one", () => {
const terminal = new Terminal();
terminal.clear(0, 80_000);
terminal.update(frame(0n, 0));
terminal.decoded(packet(0), Duration.micros(20_000));
terminal.update(end(0n, 20_000));
expect(terminal.update(frame(1n, 1_000_000))).toBe(true);
terminal.update(gap);
terminal.update(frame(2n, 2_000_000));
expect(terminal.preroll()).toEqual([]);
expect(terminal.span(opus(2_000_000)).frames).toBe(0);
});
Loading