Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion bun.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion js/watch/package.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "@moq/watch",
"type": "module",
"version": "0.6.5",
"version": "0.6.6",
"jsr": false,
"description": "Watch Media over QUIC broadcasts",
"license": "(MIT OR Apache-2.0)",
Expand Down
151 changes: 91 additions & 60 deletions js/watch/src/audio/decoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,16 @@ import { type AudioBuffer, createAudioBuffer } from "./buffer";
import { type DecoderConfig, decoderConfig, type PlaybackIdentity, playbackIdentity } from "./config";
import { Handover } from "./handover";
import { ringSamples } from "./latency";
import { Order } from "./order";
// Compiled and inlined as a blob URL via vite-plugin-worklet.
import RenderWorklet from "./render-worklet.ts?worklet";
import type { Source } from "./source";
import { type DecodedSpan, Terminal } from "./terminal";
import { Warmup } from "./warmup";

type Next = NonNullable<Awaited<ReturnType<Container.Consumer["next"]>>>;
type FrameNext = Next & { frame: NonNullable<Next["frame"]> };

const LEGACY_WARMUP_CALLBACKS = 3;

export type DecoderInput = {
Expand Down Expand Up @@ -345,22 +349,30 @@ export class Decoder {
};
decoder.configure(decoderConfig);

// The newest frame sent to the decoder since it was last reset.
let newest: Time.Micro | undefined;
const order = this.#order();
let discontinuity: number | undefined;
for (;;) {
const next = await nextMedia(consumer);
if (!next) break;
if (this.#onNext(next)) {
decoder.reset();
decoder.configure(decoderConfig);
newest = undefined;
// The track ending, a marker, or a discontinuity cuts a held head short. A group merely
// completing does not.
if (
next === undefined ||
next.end !== undefined ||
next.discontinuity !== (discontinuity ?? next.discontinuity)
) {
if (!(await this.#play(effect, decoder, decoderConfig, order, order.flush()))) return;
}
if (next.end !== undefined) {
if (!next) break;
discontinuity = next.discontinuity;
if (next.frame === undefined || next.end !== undefined) {
if (this.#onNext(next)) {
decoder.reset();
decoder.configure(decoderConfig);
order.reset();
}
continue;
}

const { frame } = next;
if (!frame) continue;

// Mark that we received this frame right now.
const timestamp = Time.Milli.fromMicro(frame.timestamp as Time.Micro);
Expand All @@ -370,27 +382,7 @@ export class Decoder {
bytesReceived: (stats?.bytesReceived ?? 0) + frame.payload.byteLength,
}));

// Decode in timestamp order. A subscription's head groups arrive after the live edge served
// alongside them, and the ring, anchored at the first decoded frame, cannot place audio
// before it. Chrome's AudioDecoder would also stamp every later output from its first
// input, shifting the rest of the track by the reorder and overflowing a buffered ring.
if (newest !== undefined && (frame.timestamp as Time.Micro) < newest) continue;
newest = frame.timestamp as Time.Micro;

// Backpressure: in buffered mode this holds the encoded frame until the playhead nears
// it, keeping the lookahead above the floor as Opus instead of decoded PCM. No-op live.
await this.#ring?.wait(frame.timestamp as Time.Micro, effect.abort);

const chunk = new EncodedAudioChunk({
type: frame.keyframe ? "key" : "delta",
data: frame.payload,
timestamp: frame.timestamp,
});

// A fatal decode error closes the decoder, so decoding again throws InvalidStateError out
// of this loop. Stop instead: the error callback already reported the real failure.
if (decoder.state === "closed") break;
decoder.decode(chunk);
if (!(await this.#play(effect, decoder, decoderConfig, order, order.push({ ...next, frame })))) return;
}
});
}
Expand Down Expand Up @@ -446,21 +438,31 @@ export class Decoder {
};
decoder.configure(decoderConfig);

// The newest frame sent to the decoder since it was last reset.
let newest: Time.Micro | undefined;
const order = this.#order();
let discontinuity: number | undefined;
for (;;) {
const next = await nextMedia(consumer);
// The track ending, a marker, or a discontinuity cuts a held head short. A group merely
// completing does not.
if (
next === undefined ||
next.end !== undefined ||
next.discontinuity !== (discontinuity ?? next.discontinuity)
) {
if (!(await this.#play(effect, decoder, decoderConfig, order, order.flush()))) return;
}
if (!next) break;

// Reset and re-anchor before decoding the first frame of a new codec epoch.
if (this.#onNext(next)) {
decoder.reset();
decoder.configure(decoderConfig);
newest = undefined;
discontinuity = next.discontinuity;
if (next.frame === undefined) {
// Reset and re-anchor before decoding the first frame of a new codec epoch.
if (this.#onNext(next)) {
decoder.reset();
decoder.configure(decoderConfig);
order.reset();
}
continue;
}

const { frame } = next;
if (!frame) continue;

const timestamp = Time.Milli.fromMicro(frame.timestamp);
this.sync.received(timestamp, "audio");
Expand All @@ -469,29 +471,58 @@ export class Decoder {
bytesReceived: (stats?.bytesReceived ?? 0) + frame.payload.byteLength,
}));

// Decode in timestamp order. A subscription's head groups arrive after the live edge served
// alongside them, and the ring, anchored at the first decoded frame, cannot place audio
// before it. Chrome's AudioDecoder would also stamp every later output from its first
// input, shifting the rest of the track by the reorder and overflowing a buffered ring.
if (newest !== undefined && frame.timestamp < newest) continue;
newest = frame.timestamp;

// Backpressure: in buffered mode this holds the encoded frame until the playhead nears
// it, keeping the lookahead above the floor as Opus instead of decoded PCM. No-op live.
await this.#ring?.wait(frame.timestamp, effect.abort);

if (decoder.state === "closed") break;
decoder.decode(
new EncodedAudioChunk({
type: frame.keyframe ? "key" : "delta",
data: frame.payload,
timestamp: frame.timestamp,
}),
);
if (!(await this.#play(effect, decoder, decoderConfig, order, order.push({ ...next, frame })))) return;

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Reset ordering before admitting a discontinuity frame

When a buffered consumer reports a discontinuity on a frame whose timestamp precedes a previously held live-edge frame, such as after Consumer drops a missing head group for exceeding maxAge, the preceding flush sets Order.#newest to the live-edge timestamp and this call filters out the discontinuity frame before #play can invoke #onNext and order.reset(). The decoder and ring therefore do not reset at the gap, and this frame plus subsequent frames up to the old timestamp are silently dropped; handle the discontinuity reset before applying the previous epoch's ordering bound.

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in fbbce7e, with the same change on main in f59e54f. Order no longer filters on release: #play runs #onNext (main: #onRead) first, resets the decoder and the order for a new epoch, and only then asks order.admit() whether the frame is out of order. A discontinuity frame below the old live edge now resets and decodes. Main's decoder.test.ts covers the equivalent case, a fresh encoder after an end restarting below the old timeline. It fails on the previous commit.

(Written by Claude Opus 5.5)

}
});
}

// A buffered subscription holds its head until it runs past the floor, which playback waits for
// anyway, or spans the window the relay serves.
#order(): Order<FrameNext> {
if (!this.sync.out.buffered.peek()) return new Order(undefined);
return new Order(() => ({
floor: Time.Micro.fromMilli(this.sync.out.delay.peek()),
maxDelay: Time.Micro.fromMilli(this.sync.out.maxAge.peek()),
Comment on lines +484 to +485

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Keep ordering bounds synchronized with live delay

When automatic delay changes after #order() runs, these peek() calls permanently retain the old floor and maximum age even though #runLatency updates the ring and Container.Consumer receives a live maxAge getter. If the delay rises while the subscription head is arriving, Order can release at the smaller old floor and then reject a subsequently delivered older group via #newest, clipping the head; if it falls, startup is unnecessarily held to the old floor. Pass live getters into Order or recreate/update its bounds when these signals change.

AGENTS.md reference: js/AGENTS.md:L13-L13

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in fbbce7e: the bounds come from a getter read on every held frame, so they follow delay and maxAge.

(Written by Claude Opus 5.5)

}));
}

// Decode released frames, resetting the decoder before the first frame of a new codec epoch and
// only then dropping one that is out of order. The terminal sees them in decode order, so it
// anchors on the oldest. Reports whether to keep going.
async #play(
effect: Effect,
decoder: AudioDecoder,
config: AudioDecoderConfig,
order: Order<FrameNext>,
reads: FrameNext[],
): Promise<boolean> {
for (const next of reads) {
if (this.#onNext(next)) {
decoder.reset();
decoder.configure(config);
order.reset();
}
if (!order.admit(next)) continue;
const { frame } = next;

// Backpressure: in buffered mode this holds the encoded frame until the playhead nears
// it, keeping the lookahead above the floor as Opus instead of decoded PCM. No-op live.
await this.#ring?.wait(frame.timestamp as Time.Micro, effect.abort);

// A fatal decode error closes the decoder, so decoding again throws InvalidStateError out
// of the loop. Stop instead: the error callback already reported the real failure.
if (decoder.state === "closed") return false;
decoder.decode(
new EncodedAudioChunk({
type: frame.keyframe ? "key" : "delta",
data: frame.payload,
timestamp: frame.timestamp,
}),
);
}
return true;
}

#emit(sample: AudioData, decoded: DecodedSpan = this.#terminal.span(sample)) {
const { timestamp, frameOffset, frames } = decoded;
const timestampMilli = Time.Milli.fromMicro(timestamp);
Expand Down
79 changes: 79 additions & 0 deletions js/watch/src/audio/order.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
import { describe, expect, it } from "bun:test";
import { Order } from "./order";

type Read = { group: number; frame: { timestamp: number } };
const read = (group: number, timestamp: number): Read => ({ group, frame: { timestamp } });
const timestamps = (reads: Read[]) => reads.map(({ frame }) => frame.timestamp);

describe("Order", () => {
it("releases a held head sorted once a newer group arrives", () => {
const order = new Order<Read>(() => ({ floor: 100, maxDelay: 1000 }));
for (const t of [40, 20, 0]) expect(order.push(read(t / 20, t))).toEqual([]);
expect(timestamps(order.push(read(3, 60)))).toEqual([0, 20, 40, 60]);
expect(timestamps(order.push(read(4, 80)))).toEqual([80]);
});

// A publisher may keep a long run of audio in one group, which never brings a newer one.
it("releases a head once its first group runs past the floor", () => {
const order = new Order<Read>(() => ({ floor: 100, maxDelay: 1000 }));
expect(order.push(read(5, 0))).toEqual([]);
expect(order.push(read(4, -20))).toEqual([]);
expect(order.push(read(5, 100))).toEqual([]);
expect(timestamps(order.push(read(5, 120)))).toEqual([-20, 0, 100, 120]);
});

// With no floor, playback starts on the first frame, but the head still has to arrive first.
it("holds the head past its first frame with a zero floor", () => {
const order = new Order<Read>(() => ({ floor: 0, maxDelay: 1000 }));
expect(order.push(read(1, 20))).toEqual([]);
expect(order.push(read(0, 0))).toEqual([]);
expect(timestamps(order.push(read(1, 40)))).toEqual([0, 20, 40]);
});

// Nothing older than the subscription's max delay is served, so a longer head is complete.
it("releases a head once it spans the max delay", () => {
const order = new Order<Read>(() => ({ floor: 100, maxDelay: 60 }));
for (const t of [60, 40, 20]) expect(order.push(read(t / 20, t))).toEqual([]);
expect(timestamps(order.push(read(0, 0)))).toEqual([0, 20, 40, 60]);
});

it("releases the head as it stands on a flush", () => {
const order = new Order<Read>(() => ({ floor: 100, maxDelay: 1000 }));
order.push(read(1, 20));
order.push(read(0, 0));
expect(timestamps(order.flush())).toEqual([0, 20]);
expect(order.flush()).toEqual([]);
});

it("admits a frame only if it is no older than one already admitted", () => {
const order = new Order<Read>(() => ({ floor: 100, maxDelay: 1000 }));
order.push(read(1, 20));
const released = order.push(read(2, 40));
expect(released.every((r) => order.admit(r))).toBe(true);
expect(order.push(read(0, 0))).toEqual([read(0, 0)]);
expect(order.admit(read(0, 0))).toBe(false);

// A reset decoder restamps from its next input, so older frames are welcome again.
order.reset();
expect(order.admit(read(0, 0))).toBe(true);
});

// A rise in the delay while the head arrives must not release it at the old floor.
it("reads its bounds as the head arrives", () => {
let floor = 20;
const order = new Order<Read>(() => ({ floor, maxDelay: 1000 }));
expect(order.push(read(5, 0))).toEqual([]);
floor = 100;
expect(order.push(read(5, 40))).toEqual([]);
expect(order.push(read(4, -20))).toEqual([]);
expect(timestamps(order.push(read(5, 120)))).toEqual([-20, 0, 40, 120]);
});

it("holds nothing live", () => {
const order = new Order<Read>(undefined);
expect(timestamps(order.push(read(1, 20)))).toEqual([20]);
expect(order.admit(read(1, 20))).toBe(true);
expect(timestamps(order.push(read(0, 0)))).toEqual([0]);
expect(order.admit(read(0, 0))).toBe(false);
});
});
72 changes: 72 additions & 0 deletions js/watch/src/audio/order.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
/**
* Puts one subscription's frames in timestamp order for the audio decoder.
*
* Groups go out newest-first, so a subscription's head (the groups the publisher still holds) arrives
* after the live edge it was served alongside. Chrome's AudioDecoder stamps every output from its
* first input, so decoding the head after the live edge shifts the rest of the track, and the ring,
* anchored at the first decoded frame, cannot place the head anyway.
*
* A buffered subscription holds its head and releases it sorted. The head is complete once a newer
* group than the first arrives, or once the first group alone runs past the floor: the ring could not
* start playing before holding that much, so waiting for the head costs no startup latency. It is
* also released once it spans the subscription's max delay, since nothing older is served. After
* that, and throughout when not holding, a frame older than one already decoded is not admitted.
*/
export class Order<T extends { group: number; frame: { timestamp: number } }> {
// The current bounds in microseconds, while the head is still arriving.
#hold: (() => { floor: number; maxDelay: number }) | undefined;
// The head, in arrival order.
#held: T[] = [];
// The group the head started with, and its own first frame.
#first?: { group: number; timestamp: number };
// The oldest and newest held timestamps.
#span?: { oldest: number; newest: number };
// The newest frame admitted since the decoder was last reset.
#newest?: number;

/** Hold the head within the bounds `hold` reports, or pass each frame straight through. */
constructor(hold: (() => { floor: number; maxDelay: number }) | undefined) {
this.#hold = hold;
}

/** Accept a frame, returning the frames released for decoding, oldest first. */
push(read: T): T[] {
const hold = this.#hold?.();
if (!hold) return [read];

const { group, frame } = read;
this.#first ??= { group, timestamp: frame.timestamp };
if (group > this.#first.group) return [...this.flush(), read];

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Keep holding when a newer group overtakes the backlog

When a subscription starts on group N and older cached groups are still in flight, a newly published N+1 can arrive first because js/net/src/lite/priority.ts deliberately ranks newer streams ahead of every older in-flight stream. This branch therefore flushes N and N+1 and permanently disables holding even though N-1 may still arrive; admit() then rejects N-1 by timestamp, so the buffered subscription still clips its audio head under normal ongoing publication. A higher group number cannot be treated as proof that the initial backlog is complete.

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same limit as #6549 (discussion): without a declared start, "a group newer than the first" is a heuristic, and a newer stream overtaking the backlog defeats it. When that happens, the frames released early decode in order and the late backlog is dropped as older, which is 0.6.5's behavior and never worse.

The usual case is fixed: across 10 local Voice demo calls on this build (plain and barge-in), every reply decoded from 0 ms with its first word whole, against 4 of 10 clipped before. The complete fix is flooring the ordered read at the subscription's resolved start, per quest/m1/read-backfill.md. That doesn't fit a patch release.

(Written by Claude Opus 5.5)


this.#held.push(read);
const oldest = Math.min(this.#span?.oldest ?? frame.timestamp, frame.timestamp);
const newest = Math.max(this.#span?.newest ?? frame.timestamp, frame.timestamp);
this.#span = { oldest, newest };

const ran = group === this.#first.group && frame.timestamp - this.#first.timestamp > hold.floor;
if (ran || newest - oldest >= hold.maxDelay) return this.flush();
return [];
}

/** Release the head as it stands, ending the hold: the track ended, or a gap or end cut it short. */
flush(): T[] {
this.#hold = undefined;
const held = this.#held;
this.#held = [];
// Stable, so frames sharing a timestamp keep their arrival order.
return held.sort((a, b) => a.frame.timestamp - b.frame.timestamp);
}

/** Whether a released frame is still in order for the decoder, recording it if so. */
admit(read: T): boolean {
const { timestamp } = read.frame;
if (this.#newest !== undefined && timestamp < this.#newest) return false;
this.#newest = timestamp;
return true;
}

/** Start the order over after the decoder resets, which restarts its output timestamps. */
reset(): void {
this.#newest = undefined;
}
}
Loading