Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 44 additions & 4 deletions js/watch/src/audio/decoder.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -326,15 +326,55 @@ it("decodes an audio packet inside a group after a gap", async () => {
}
});

// A subscription's head arrives newest-first. Chrome's AudioDecoder stamps every output from its
// first input, so decoding the older frames after it shifts the rest of the track.
// A subscription's head arrives newest-first, one group per frame. Chrome's AudioDecoder stamps every
// output from its first input, so decoding the older frames after it shifts the rest of the track.
for (const cmaf of [false, true]) {
it(`decodes a ${cmaf ? "CMAF" : "legacy"} subscription head in timestamp order`, async () => {
it(`decodes a ${cmaf ? "CMAF" : "legacy"} buffered subscription head in timestamp order`, async () => {
const playback = await play(Milli(100), true, cmaf);
try {
for (const ms of [140, 120, 100, 80, 60, 40, 20, 0]) {
frameTimestamp = (ms - 20) * 1000;
playback.push({ ...frame(), group: ms / 20 });
await microtasks();
}
expect(playback.inserted()).toEqual([]);

// The live edge moves on, so the head is complete and plays from its oldest frame.
frameTimestamp = 140_000;
playback.push({ ...frame(), group: 8 });
await microtasks();
expect(playback.inserted()).toEqual([0, 20_000, 40_000, 60_000, 80_000, 100_000]);
} finally {
playback.close();
await microtasks();
}
});

// A fresh encoder after a declared end may restart its timeline below the old one, and the decoder
// resets for it before anything is judged out of order.
it(`decodes a ${cmaf ? "CMAF" : "legacy"} fresh encoder below the timeline it replaced`, async () => {
const playback = await play(Milli(100), false, cmaf);
try {
frameTimestamp = 480_000;
playback.push({ ...frame(), group: 0 });
await microtasks();
playback.push({ kind: "end", group: 0, end: Micro(520_000) });
frameTimestamp = 0;
playback.push({ ...frame(), group: 1 });
await microtasks();
expect(playback.inserted()).toEqual([500_000, 20_000]);
} finally {
playback.close();
await microtasks();
}
});

it(`skips a ${cmaf ? "CMAF" : "legacy"} live subscription head older than its live edge`, async () => {
const playback = await play(Milli(100), false, cmaf);
try {
for (const ms of [140, 120, 100, 80, 60, 40, 20, 0, 160]) {
frameTimestamp = (ms - 20) * 1000;
playback.push(frame());
playback.push({ ...frame(), group: ms / 20 });
await microtasks();
}
expect(playback.inserted()).toEqual([140_000, 160_000]);
Expand Down
157 changes: 87 additions & 70 deletions js/watch/src/audio/decoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,14 @@ import { type AudioBuffer, createAudioBuffer } from "./buffer";
import { type DecoderConfig, frameDuration, type PlaybackIdentity, packetDuration, playbackIdentity } from "./config";
import { Handover } from "./handover";
import { AUTO_MAX_DELAY, ringSamples, target } from "./latency";
import { Order } from "./order";
// A blob: URL, or a hosted file when assets() is set; see vite-plugin-worklet.
import RenderWorklet from "./render-worklet.ts?worklet";
import type { Source } from "./source";
import { Terminal } from "./terminal";

type FrameRead = Extract<Container.Read, { kind: "frame" }>;

/** Signals the decoder reads. */
export type DecoderInput = {
// Whether to download the audio track. Defaults to true.
Expand Down Expand Up @@ -476,18 +479,22 @@ export class Decoder {
};
decoder.configure(decoderConfig);

// The newest frame sent to the decoder since it was last reset.
let newest: Micro | undefined;
const order = this.#order();
const play = (reads: FrameRead[]) => this.#play(effect, decoder, decoderConfig, order, reads);
for (;;) {
const next = await nextMedia(consumer);
if (!next) break;
if (this.#onRead(next)) {
decoder.reset();
decoder.configure(decoderConfig);
newest = undefined;
if (next?.kind !== "frame") {
// The track ending, or a gap or end, cuts a held head short.
if (!(await play(order.flush()))) return;
if (!next) break;
if (this.#onRead(next)) {
decoder.reset();
decoder.configure(decoderConfig);
order.reset();
}
if (next.kind === "gap") this.#arrivals.write(gap(next));
continue;
}
if (next.kind === "gap") this.#arrivals.write(gap(next));
if (next.kind !== "frame") continue;
const { frame } = next;

// Classify ingress before decoder backpressure changes the delivery clock.
Expand All @@ -502,32 +509,7 @@ export class Decoder {
bytesReceived: (stats?.bytesReceived ?? 0) + frame.payload.byteLength,
}));

// Decode in timestamp order. A subscription's head groups arrive after the live edge served
// alongside them, and the ring, anchored at the first decoded frame, cannot place audio
// before it. Chrome's AudioDecoder would also stamp every later output from its first
// input, shifting the rest of the track by the reorder and overflowing a buffered ring.
if (newest !== undefined && frame.timestamp < newest) continue;
newest = frame.timestamp;

const ring = await this.#ready(effect);
if (!ring) break;

// Backpressure: in buffered mode this holds the encoded frame until the playhead nears
// it, keeping the lookahead above the floor as Opus instead of decoded PCM. No-op live.
await ring.wait(frame.timestamp as Micro, effect.abort);
if (effect.abort.aborted) break;

const chunk = new EncodedAudioChunk({
// WebCodecs audio chunks are key even inside a MoQ group.
type: "key",
data: frame.payload,
timestamp: frame.timestamp,
});

// A fatal decode error closes the decoder, so decoding again throws InvalidStateError out
// of this loop. Stop instead: the error callback already reported the real failure.
if (decoder.state === "closed") break;
decoder.decode(chunk);
if (!(await play(order.push(next)))) return;
}
});
}
Expand Down Expand Up @@ -580,20 +562,22 @@ export class Decoder {
};
decoder.configure(decoderConfig);

// The newest frame sent to the decoder since it was last reset.
let newest: Micro | undefined;
const order = this.#order();
const play = (reads: FrameRead[]) => this.#play(effect, decoder, decoderConfig, order, reads);
for (;;) {
const next = await nextMedia(consumer);
if (!next) break;

// Reset and re-anchor before decoding the first frame of a new codec epoch.
if (this.#onRead(next)) {
decoder.reset();
decoder.configure(decoderConfig);
newest = undefined;
if (next?.kind !== "frame") {
// The track ending, or a gap or end, cuts a held head short.
if (!(await play(order.flush()))) return;
if (!next) break;
if (this.#onRead(next)) {
decoder.reset();
decoder.configure(decoderConfig);
order.reset();
}
if (next.kind === "gap") this.#arrivals.write(gap(next));
continue;
}
if (next.kind === "gap") this.#arrivals.write(gap(next));
if (next.kind !== "frame") continue;
const { frame } = next;

const timestamp = Milli.fromMicro(frame.timestamp);
Expand All @@ -607,34 +591,67 @@ export class Decoder {
bytesReceived: (stats?.bytesReceived ?? 0) + frame.payload.byteLength,
}));

// Decode in timestamp order. A subscription's head groups arrive after the live edge served
// alongside them, and the ring, anchored at the first decoded frame, cannot place audio
// before it. Chrome's AudioDecoder would also stamp every later output from its first
// input, shifting the rest of the track by the reorder and overflowing a buffered ring.
if (newest !== undefined && frame.timestamp < newest) continue;
newest = frame.timestamp;

const ring = await this.#ready(effect);
if (!ring) break;

// Backpressure: in buffered mode this holds the encoded frame until the playhead nears
// it, keeping the lookahead above the floor as Opus instead of decoded PCM. No-op live.
await ring.wait(frame.timestamp, effect.abort);
if (effect.abort.aborted) break;

if (decoder.state === "closed") break;
decoder.decode(
new EncodedAudioChunk({
// WebCodecs audio chunks are key even inside a MoQ group.
type: "key",
data: frame.payload,
timestamp: frame.timestamp,
}),
);
if (!(await play(order.push(next)))) return;
}
});
}

// Decode released frames, resetting the decoder before the first frame of a new codec epoch and
// only then dropping one that is out of order. The terminal sees them in decode order, so it
// anchors on the oldest. Reports whether to keep going.
async #play(
effect: Effect,
decoder: AudioDecoder,
config: AudioDecoderConfig,
order: Order<FrameRead>,
reads: FrameRead[],
): Promise<boolean> {
for (const read of reads) {
if (this.#onRead(read)) {
decoder.reset();
decoder.configure(config);
order.reset();
}
if (!order.admit(read)) continue;
if (!(await this.#decode(effect, decoder, read.frame))) return false;
}
return true;
}

// Decode one frame once backpressure admits it, reporting whether to keep going.
async #decode(effect: Effect, decoder: AudioDecoder, frame: Container.ReceivedFrame): Promise<boolean> {
const ring = await this.#ready(effect);
if (!ring) return false;

// Backpressure: in buffered mode this holds the encoded frame until the playhead nears
// it, keeping the lookahead above the floor as Opus instead of decoded PCM. No-op live.
await ring.wait(frame.timestamp, effect.abort);
if (effect.abort.aborted) return false;

// A fatal decode error closes the decoder, so decoding again throws InvalidStateError out
// of the loop. Stop instead: the error callback already reported the real failure.
if (decoder.state === "closed") return false;
decoder.decode(
new EncodedAudioChunk({
// WebCodecs audio chunks are key even inside a MoQ group.
type: "key",
data: frame.payload,
timestamp: frame.timestamp,
}),
);
return true;
}

// A buffered subscription holds its head until it runs past the floor, which playback waits for
// anyway, or spans the window the relay serves.
#order(): Order<FrameRead> {
if (!this.sync.out.buffered.peek()) return new Order(undefined);
return new Order(() => ({
floor: Micro.fromMilli(this.sync.out.delay.peek()),
maxDelay: Micro.fromMilli(this.#subscribeMaxDelay.peek()),
Comment on lines +650 to +651

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Keep ordering bounds synchronized with delay updates

If the numeric or auto delay rises while the subscription head is still being collected, these one-time peek() values remain at their initial settings even though the existing subscription, consumer, and ring all react to the new delay. A same-group frame can therefore satisfy the stale, smaller floor and release the head; older groups subsequently admitted by the enlarged maxDelay are then discarded by Order as older than its released frame, truncating the start of buffered audio. Update the orderer's bounds when the delay changes or derive the release decision from the current signals.

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in f59e54f: Order reads its floor and max delay through a getter on every held frame, so a delay change while the head arrives applies to the next release decision. Covered by "reads its bounds as the head arrives".

(Written by Claude Opus 5.5)

}));
}

// Reorders groups and frames up to the shared max delay. A rename is not a cold start: the path's
// arrivals are what they were, so the estimate continues from whatever read it before. Another
// path's arrivals have nothing to do with what this one learned.
Expand Down
79 changes: 79 additions & 0 deletions js/watch/src/audio/order.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
import { describe, expect, it } from "bun:test";
import { Order } from "./order";

type Read = { group: number; frame: { timestamp: number } };
const read = (group: number, timestamp: number): Read => ({ group, frame: { timestamp } });
const timestamps = (reads: Read[]) => reads.map(({ frame }) => frame.timestamp);

describe("Order", () => {
it("releases a held head sorted once a newer group arrives", () => {
const order = new Order<Read>(() => ({ floor: 100, maxDelay: 1000 }));
for (const t of [40, 20, 0]) expect(order.push(read(t / 20, t))).toEqual([]);
expect(timestamps(order.push(read(3, 60)))).toEqual([0, 20, 40, 60]);
expect(timestamps(order.push(read(4, 80)))).toEqual([80]);
});

// A publisher may keep a long run of audio in one group, which never brings a newer one.
it("releases a head once its first group runs past the floor", () => {
const order = new Order<Read>(() => ({ floor: 100, maxDelay: 1000 }));
expect(order.push(read(5, 0))).toEqual([]);
expect(order.push(read(4, -20))).toEqual([]);
expect(order.push(read(5, 100))).toEqual([]);
expect(timestamps(order.push(read(5, 120)))).toEqual([-20, 0, 100, 120]);
});

// With no floor, playback starts on the first frame, but the head still has to arrive first.
it("holds the head past its first frame with a zero floor", () => {
const order = new Order<Read>(() => ({ floor: 0, maxDelay: 1000 }));
expect(order.push(read(1, 20))).toEqual([]);
expect(order.push(read(0, 0))).toEqual([]);
expect(timestamps(order.push(read(1, 40)))).toEqual([0, 20, 40]);
});

// Nothing older than the subscription's max delay is served, so a longer head is complete.
it("releases a head once it spans the max delay", () => {
const order = new Order<Read>(() => ({ floor: 100, maxDelay: 60 }));
for (const t of [60, 40, 20]) expect(order.push(read(t / 20, t))).toEqual([]);
expect(timestamps(order.push(read(0, 0)))).toEqual([0, 20, 40, 60]);
});

it("releases the head as it stands on a flush", () => {
const order = new Order<Read>(() => ({ floor: 100, maxDelay: 1000 }));
order.push(read(1, 20));
order.push(read(0, 0));
expect(timestamps(order.flush())).toEqual([0, 20]);
expect(order.flush()).toEqual([]);
});

it("admits a frame only if it is no older than one already admitted", () => {
const order = new Order<Read>(() => ({ floor: 100, maxDelay: 1000 }));
order.push(read(1, 20));
const released = order.push(read(2, 40));
expect(released.every((r) => order.admit(r))).toBe(true);
expect(order.push(read(0, 0))).toEqual([read(0, 0)]);
expect(order.admit(read(0, 0))).toBe(false);

// A reset decoder restamps from its next input, so older frames are welcome again.
order.reset();
expect(order.admit(read(0, 0))).toBe(true);
});

// A rise in the delay while the head arrives must not release it at the old floor.
it("reads its bounds as the head arrives", () => {
let floor = 20;
const order = new Order<Read>(() => ({ floor, maxDelay: 1000 }));
expect(order.push(read(5, 0))).toEqual([]);
floor = 100;
expect(order.push(read(5, 40))).toEqual([]);
expect(order.push(read(4, -20))).toEqual([]);
expect(timestamps(order.push(read(5, 120)))).toEqual([-20, 0, 40, 120]);
});

it("holds nothing live", () => {
const order = new Order<Read>(undefined);
expect(timestamps(order.push(read(1, 20)))).toEqual([20]);
expect(order.admit(read(1, 20))).toBe(true);
expect(timestamps(order.push(read(0, 0)))).toEqual([0]);
expect(order.admit(read(0, 0))).toBe(false);
});
});
Loading
Loading