From bb1ba84fbebcebb6636c00ff4985f09ce079e5e7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 08:01:58 -0700 Subject: [PATCH 1/3] fix(watch): hold a buffered subscription's audio head and decode it in order Groups go out newest-first, so a subscription's head reaches the decoder after its live edge. moq-dev/moq#6523 skipped those older frames to keep Chrome's AudioDecoder timeline straight, which cut the first 40-160 ms of every Voice demo reply. A buffered subscription now holds its head and releases it sorted once a newer group arrives, or once the first group alone spans the floor that playback waits for anyway. The terminal sees frames in release order, so it anchors on the oldest. Live subscriptions keep skipping, and so does any frame still older than one already released. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/decoder.test.ts | 29 +++++- js/watch/src/audio/decoder.ts | 151 ++++++++++++++++------------- js/watch/src/audio/order.test.ts | 48 +++++++++ js/watch/src/audio/order.ts | 69 +++++++++++++ 4 files changed, 223 insertions(+), 74 deletions(-) create mode 100644 js/watch/src/audio/order.test.ts create mode 100644 js/watch/src/audio/order.ts diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 17f97a060fa..70f2098670a 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -326,15 +326,36 @@ it("decodes an audio packet inside a group after a gap", async () => { } }); -// A subscription's head arrives newest-first. Chrome's AudioDecoder stamps every output from its -// first input, so decoding the older frames after it shifts the rest of the track. +// A subscription's head arrives newest-first, one group per frame. Chrome's AudioDecoder stamps every +// output from its first input, so decoding the older frames after it shifts the rest of the track. for (const cmaf of [false, true]) { - it(`decodes a ${cmaf ? "CMAF" : "legacy"} subscription head in timestamp order`, async () => { + it(`decodes a ${cmaf ? "CMAF" : "legacy"} buffered subscription head in timestamp order`, async () => { const playback = await play(Milli(100), true, cmaf); + try { + for (const ms of [140, 120, 100, 80, 60, 40, 20, 0]) { + frameTimestamp = (ms - 20) * 1000; + playback.push({ ...frame(), group: ms / 20 }); + await microtasks(); + } + expect(playback.inserted()).toEqual([]); + + // The live edge moves on, so the head is complete and plays from its oldest frame. + frameTimestamp = 140_000; + playback.push({ ...frame(), group: 8 }); + await microtasks(); + expect(playback.inserted()).toEqual([0, 20_000, 40_000, 60_000, 80_000, 100_000]); + } finally { + playback.close(); + await microtasks(); + } + }); + + it(`skips a ${cmaf ? "CMAF" : "legacy"} live subscription head older than its live edge`, async () => { + const playback = await play(Milli(100), false, cmaf); try { for (const ms of [140, 120, 100, 80, 60, 40, 20, 0, 160]) { frameTimestamp = (ms - 20) * 1000; - playback.push(frame()); + playback.push({ ...frame(), group: ms / 20 }); await microtasks(); } expect(playback.inserted()).toEqual([140_000, 160_000]); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 2d7ab72c5ac..71bc6386ecf 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -22,11 +22,14 @@ import { type AudioBuffer, createAudioBuffer } from "./buffer"; import { type DecoderConfig, frameDuration, type PlaybackIdentity, packetDuration, playbackIdentity } from "./config"; import { Handover } from "./handover"; import { AUTO_MAX_DELAY, ringSamples, target } from "./latency"; +import { Order } from "./order"; // A blob: URL, or a hosted file when assets() is set; see vite-plugin-worklet. import RenderWorklet from "./render-worklet.ts?worklet"; import type { Source } from "./source"; import { Terminal } from "./terminal"; +type FrameRead = Extract; + /** Signals the decoder reads. */ export type DecoderInput = { // Whether to download the audio track. Defaults to true. @@ -476,18 +479,22 @@ export class Decoder { }; decoder.configure(decoderConfig); - // The newest frame sent to the decoder since it was last reset. - let newest: Micro | undefined; + const order = this.#order(); + const play = (reads: FrameRead[]) => this.#play(effect, decoder, decoderConfig, order, reads); for (;;) { const next = await nextMedia(consumer); - if (!next) break; - if (this.#onRead(next)) { - decoder.reset(); - decoder.configure(decoderConfig); - newest = undefined; + if (next?.kind !== "frame") { + // The track ending, or a gap or end, cuts a held head short. + if (!(await play(order.flush()))) return; + if (!next) break; + if (this.#onRead(next)) { + decoder.reset(); + decoder.configure(decoderConfig); + order.reset(); + } + if (next.kind === "gap") this.#arrivals.write(gap(next)); + continue; } - if (next.kind === "gap") this.#arrivals.write(gap(next)); - if (next.kind !== "frame") continue; const { frame } = next; // Classify ingress before decoder backpressure changes the delivery clock. @@ -502,32 +509,7 @@ export class Decoder { bytesReceived: (stats?.bytesReceived ?? 0) + frame.payload.byteLength, })); - // Decode in timestamp order. A subscription's head groups arrive after the live edge served - // alongside them, and the ring, anchored at the first decoded frame, cannot place audio - // before it. Chrome's AudioDecoder would also stamp every later output from its first - // input, shifting the rest of the track by the reorder and overflowing a buffered ring. - if (newest !== undefined && frame.timestamp < newest) continue; - newest = frame.timestamp; - - const ring = await this.#ready(effect); - if (!ring) break; - - // Backpressure: in buffered mode this holds the encoded frame until the playhead nears - // it, keeping the lookahead above the floor as Opus instead of decoded PCM. No-op live. - await ring.wait(frame.timestamp as Micro, effect.abort); - if (effect.abort.aborted) break; - - const chunk = new EncodedAudioChunk({ - // WebCodecs audio chunks are key even inside a MoQ group. - type: "key", - data: frame.payload, - timestamp: frame.timestamp, - }); - - // A fatal decode error closes the decoder, so decoding again throws InvalidStateError out - // of this loop. Stop instead: the error callback already reported the real failure. - if (decoder.state === "closed") break; - decoder.decode(chunk); + if (!(await play(order.push(next)))) return; } }); } @@ -580,20 +562,22 @@ export class Decoder { }; decoder.configure(decoderConfig); - // The newest frame sent to the decoder since it was last reset. - let newest: Micro | undefined; + const order = this.#order(); + const play = (reads: FrameRead[]) => this.#play(effect, decoder, decoderConfig, order, reads); for (;;) { const next = await nextMedia(consumer); - if (!next) break; - - // Reset and re-anchor before decoding the first frame of a new codec epoch. - if (this.#onRead(next)) { - decoder.reset(); - decoder.configure(decoderConfig); - newest = undefined; + if (next?.kind !== "frame") { + // The track ending, or a gap or end, cuts a held head short. + if (!(await play(order.flush()))) return; + if (!next) break; + if (this.#onRead(next)) { + decoder.reset(); + decoder.configure(decoderConfig); + order.reset(); + } + if (next.kind === "gap") this.#arrivals.write(gap(next)); + continue; } - if (next.kind === "gap") this.#arrivals.write(gap(next)); - if (next.kind !== "frame") continue; const { frame } = next; const timestamp = Milli.fromMicro(frame.timestamp); @@ -607,34 +591,61 @@ export class Decoder { bytesReceived: (stats?.bytesReceived ?? 0) + frame.payload.byteLength, })); - // Decode in timestamp order. A subscription's head groups arrive after the live edge served - // alongside them, and the ring, anchored at the first decoded frame, cannot place audio - // before it. Chrome's AudioDecoder would also stamp every later output from its first - // input, shifting the rest of the track by the reorder and overflowing a buffered ring. - if (newest !== undefined && frame.timestamp < newest) continue; - newest = frame.timestamp; - - const ring = await this.#ready(effect); - if (!ring) break; - - // Backpressure: in buffered mode this holds the encoded frame until the playhead nears - // it, keeping the lookahead above the floor as Opus instead of decoded PCM. No-op live. - await ring.wait(frame.timestamp, effect.abort); - if (effect.abort.aborted) break; - - if (decoder.state === "closed") break; - decoder.decode( - new EncodedAudioChunk({ - // WebCodecs audio chunks are key even inside a MoQ group. - type: "key", - data: frame.payload, - timestamp: frame.timestamp, - }), - ); + if (!(await play(order.push(next)))) return; } }); } + // Decode released frames in order, resetting the decoder before the first frame of a new codec + // epoch. The terminal sees them in decode order, so it anchors on the oldest. Reports whether to keep going. + async #play( + effect: Effect, + decoder: AudioDecoder, + config: AudioDecoderConfig, + order: Order, + reads: FrameRead[], + ): Promise { + for (const read of reads) { + if (this.#onRead(read)) { + decoder.reset(); + decoder.configure(config); + order.reset(); + } + if (!(await this.#decode(effect, decoder, read.frame))) return false; + } + return true; + } + + // Decode one frame once backpressure admits it, reporting whether to keep going. + async #decode(effect: Effect, decoder: AudioDecoder, frame: Container.ReceivedFrame): Promise { + const ring = await this.#ready(effect); + if (!ring) return false; + + // Backpressure: in buffered mode this holds the encoded frame until the playhead nears + // it, keeping the lookahead above the floor as Opus instead of decoded PCM. No-op live. + await ring.wait(frame.timestamp, effect.abort); + if (effect.abort.aborted) return false; + + // A fatal decode error closes the decoder, so decoding again throws InvalidStateError out + // of the loop. Stop instead: the error callback already reported the real failure. + if (decoder.state === "closed") return false; + decoder.decode( + new EncodedAudioChunk({ + // WebCodecs audio chunks are key even inside a MoQ group. + type: "key", + data: frame.payload, + timestamp: frame.timestamp, + }), + ); + return true; + } + + // A buffered subscription holds its head until it spans the floor, which playback waits for anyway. + #order(): Order { + if (!this.sync.out.buffered.peek()) return new Order(undefined); + return new Order({ floor: Micro.fromMilli(this.sync.out.delay.peek()) }); + } + // Reorders groups and frames up to the shared max delay. A rename is not a cold start: the path's // arrivals are what they were, so the estimate continues from whatever read it before. Another // path's arrivals have nothing to do with what this one learned. diff --git a/js/watch/src/audio/order.test.ts b/js/watch/src/audio/order.test.ts new file mode 100644 index 00000000000..1d3b2741daa --- /dev/null +++ b/js/watch/src/audio/order.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "bun:test"; +import { Order } from "./order"; + +const read = (group: number, timestamp: number) => ({ group, frame: { timestamp } }); +const timestamps = (reads: ReturnType[]) => reads.map(({ frame }) => frame.timestamp); + +describe("Order", () => { + it("releases a held head sorted once a newer group arrives", () => { + const order = new Order({ floor: 100 }); + for (const t of [40, 20, 0]) expect(order.push(read(t / 20, t))).toEqual([]); + expect(timestamps(order.push(read(3, 60)))).toEqual([0, 20, 40, 60]); + expect(timestamps(order.push(read(4, 80)))).toEqual([80]); + }); + + // A publisher may keep a long run of audio in one group, which never brings a newer one. + it("releases a head once its first group spans the floor", () => { + const order = new Order({ floor: 100 }); + expect(order.push(read(5, 0))).toEqual([]); + expect(order.push(read(4, -20))).toEqual([]); + expect(order.push(read(5, 80))).toEqual([]); + expect(timestamps(order.push(read(5, 100)))).toEqual([-20, 0, 80, 100]); + }); + + it("releases the head as it stands on a flush", () => { + const order = new Order({ floor: 100 }); + order.push(read(1, 20)); + order.push(read(0, 0)); + expect(timestamps(order.flush())).toEqual([0, 20]); + expect(order.flush()).toEqual([]); + }); + + it("drops a frame older than one already released", () => { + const order = new Order({ floor: 100 }); + order.push(read(1, 20)); + expect(timestamps(order.push(read(2, 40)))).toEqual([20, 40]); + expect(order.push(read(0, 0))).toEqual([]); + + // A reset decoder restamps from its next input, so older frames are welcome again. + order.reset(); + expect(timestamps(order.push(read(0, 0)))).toEqual([0]); + }); + + it("holds nothing live", () => { + const order = new Order(undefined); + expect(timestamps(order.push(read(1, 20)))).toEqual([20]); + expect(order.push(read(0, 0))).toEqual([]); + }); +}); diff --git a/js/watch/src/audio/order.ts b/js/watch/src/audio/order.ts new file mode 100644 index 00000000000..31eabe1d81b --- /dev/null +++ b/js/watch/src/audio/order.ts @@ -0,0 +1,69 @@ +/** + * Puts one subscription's frames in timestamp order for the audio decoder. + * + * Groups go out newest-first, so a subscription's head (the groups the publisher still holds) arrives + * after the live edge it was served alongside. Chrome's AudioDecoder stamps every output from its + * first input, so decoding the head after the live edge shifts the rest of the track, and the ring, + * anchored at the first decoded frame, cannot place the head anyway. + * + * A buffered subscription holds its head and releases it sorted. The head is complete once a newer + * group than the first arrives, or once the first group alone spans the floor: the ring could not + * start playing before holding that much, so waiting for the head costs no startup latency. Any frame + * still older than one already released is dropped, as is every reordered frame when not holding. + */ +export class Order { + // The head, oldest first, while it is still arriving. + #held: T[] | undefined; + // The group the head started with, and its own oldest frame. + #first?: { group: number; timestamp: number }; + // How far the first group may run before the head is released. + #floor: number; + // The newest frame released since the decoder was last reset. + #newest?: number; + + /** Hold the head until it spans `floor` (microseconds), or decode each frame as it arrives. */ + constructor(hold: { floor: number } | undefined) { + this.#held = hold ? [] : undefined; + this.#floor = hold?.floor ?? 0; + } + + /** Accept a frame, returning the frames ready to decode, oldest first. */ + push(read: T): T[] { + const held = this.#held; + if (!held) return this.#release([read]); + + const { group, frame } = read; + this.#first ??= { group, timestamp: frame.timestamp }; + if (group > this.#first.group) return this.#release([...this.#take(), read]); + + held.splice(held.findLastIndex((other) => other.frame.timestamp <= frame.timestamp) + 1, 0, read); + if (group === this.#first.group && frame.timestamp - this.#first.timestamp >= this.#floor) { + return this.#release(this.#take()); + } + return []; + } + + /** Release the head as it stands: the track ended, or a gap or end cut it short. */ + flush(): T[] { + return this.#release(this.#take()); + } + + /** Start the order over after the decoder resets, which restarts its output timestamps. */ + reset(): void { + this.#newest = undefined; + } + + #take(): T[] { + const held = this.#held ?? []; + this.#held = undefined; + return held; + } + + #release(reads: T[]): T[] { + return reads.filter(({ frame }) => { + if (this.#newest !== undefined && frame.timestamp < this.#newest) return false; + this.#newest = frame.timestamp; + return true; + }); + } +} From 823ea577775ce722499ac371a171cda655969919 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 08:08:46 -0700 Subject: [PATCH 2/3] fix(watch): hold the audio head past a zero floor, and bound it by the max delay Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/decoder.ts | 8 ++++++-- js/watch/src/audio/order.test.ts | 29 ++++++++++++++++++++------- js/watch/src/audio/order.ts | 34 +++++++++++++++++++------------- 3 files changed, 48 insertions(+), 23 deletions(-) diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 71bc6386ecf..55ae67fb13d 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -640,10 +640,14 @@ export class Decoder { return true; } - // A buffered subscription holds its head until it spans the floor, which playback waits for anyway. + // A buffered subscription holds its head until it runs past the floor, which playback waits for + // anyway, or spans the window the relay serves. #order(): Order { if (!this.sync.out.buffered.peek()) return new Order(undefined); - return new Order({ floor: Micro.fromMilli(this.sync.out.delay.peek()) }); + return new Order({ + floor: Micro.fromMilli(this.sync.out.delay.peek()), + maxDelay: Micro.fromMilli(this.#subscribeMaxDelay.peek()), + }); } // Reorders groups and frames up to the shared max delay. A rename is not a cold start: the path's diff --git a/js/watch/src/audio/order.test.ts b/js/watch/src/audio/order.test.ts index 1d3b2741daa..00e722e6063 100644 --- a/js/watch/src/audio/order.test.ts +++ b/js/watch/src/audio/order.test.ts @@ -6,23 +6,38 @@ const timestamps = (reads: ReturnType[]) => reads.map(({ frame }) = describe("Order", () => { it("releases a held head sorted once a newer group arrives", () => { - const order = new Order({ floor: 100 }); + const order = new Order({ floor: 100, maxDelay: 1000 }); for (const t of [40, 20, 0]) expect(order.push(read(t / 20, t))).toEqual([]); expect(timestamps(order.push(read(3, 60)))).toEqual([0, 20, 40, 60]); expect(timestamps(order.push(read(4, 80)))).toEqual([80]); }); // A publisher may keep a long run of audio in one group, which never brings a newer one. - it("releases a head once its first group spans the floor", () => { - const order = new Order({ floor: 100 }); + it("releases a head once its first group runs past the floor", () => { + const order = new Order({ floor: 100, maxDelay: 1000 }); expect(order.push(read(5, 0))).toEqual([]); expect(order.push(read(4, -20))).toEqual([]); - expect(order.push(read(5, 80))).toEqual([]); - expect(timestamps(order.push(read(5, 100)))).toEqual([-20, 0, 80, 100]); + expect(order.push(read(5, 100))).toEqual([]); + expect(timestamps(order.push(read(5, 120)))).toEqual([-20, 0, 100, 120]); + }); + + // With no floor, playback starts on the first frame, but the head still has to arrive first. + it("holds the head past its first frame with a zero floor", () => { + const order = new Order({ floor: 0, maxDelay: 1000 }); + expect(order.push(read(1, 20))).toEqual([]); + expect(order.push(read(0, 0))).toEqual([]); + expect(timestamps(order.push(read(1, 40)))).toEqual([0, 20, 40]); + }); + + // Nothing older than the subscription's max delay is served, so a longer head is complete. + it("releases a head once it spans the max delay", () => { + const order = new Order({ floor: 100, maxDelay: 60 }); + for (const t of [60, 40, 20]) expect(order.push(read(t / 20, t))).toEqual([]); + expect(timestamps(order.push(read(0, 0)))).toEqual([0, 20, 40, 60]); }); it("releases the head as it stands on a flush", () => { - const order = new Order({ floor: 100 }); + const order = new Order({ floor: 100, maxDelay: 1000 }); order.push(read(1, 20)); order.push(read(0, 0)); expect(timestamps(order.flush())).toEqual([0, 20]); @@ -30,7 +45,7 @@ describe("Order", () => { }); it("drops a frame older than one already released", () => { - const order = new Order({ floor: 100 }); + const order = new Order({ floor: 100, maxDelay: 1000 }); order.push(read(1, 20)); expect(timestamps(order.push(read(2, 40)))).toEqual([20, 40]); expect(order.push(read(0, 0))).toEqual([]); diff --git a/js/watch/src/audio/order.ts b/js/watch/src/audio/order.ts index 31eabe1d81b..1290d9a72b0 100644 --- a/js/watch/src/audio/order.ts +++ b/js/watch/src/audio/order.ts @@ -7,24 +7,26 @@ * anchored at the first decoded frame, cannot place the head anyway. * * A buffered subscription holds its head and releases it sorted. The head is complete once a newer - * group than the first arrives, or once the first group alone spans the floor: the ring could not - * start playing before holding that much, so waiting for the head costs no startup latency. Any frame + * group than the first arrives, or once the first group alone runs past the floor: the ring could not + * start playing before holding that much, so waiting for the head costs no startup latency. It is + * also released once it spans the subscription's max delay, since nothing older is served. Any frame * still older than one already released is dropped, as is every reordered frame when not holding. */ export class Order { - // The head, oldest first, while it is still arriving. + // The head, in arrival order, while it is still arriving. #held: T[] | undefined; - // The group the head started with, and its own oldest frame. + // The group the head started with, and its own first frame. #first?: { group: number; timestamp: number }; - // How far the first group may run before the head is released. - #floor: number; + // The oldest and newest held timestamps. + #span?: { oldest: number; newest: number }; + #hold: { floor: number; maxDelay: number }; // The newest frame released since the decoder was last reset. #newest?: number; - /** Hold the head until it spans `floor` (microseconds), or decode each frame as it arrives. */ - constructor(hold: { floor: number } | undefined) { + /** Hold the head (bounds in microseconds), or decode each frame as it arrives. */ + constructor(hold: { floor: number; maxDelay: number } | undefined) { this.#held = hold ? [] : undefined; - this.#floor = hold?.floor ?? 0; + this.#hold = hold ?? { floor: 0, maxDelay: 0 }; } /** Accept a frame, returning the frames ready to decode, oldest first. */ @@ -36,10 +38,13 @@ export class Order { this.#first ??= { group, timestamp: frame.timestamp }; if (group > this.#first.group) return this.#release([...this.#take(), read]); - held.splice(held.findLastIndex((other) => other.frame.timestamp <= frame.timestamp) + 1, 0, read); - if (group === this.#first.group && frame.timestamp - this.#first.timestamp >= this.#floor) { - return this.#release(this.#take()); - } + held.push(read); + const oldest = Math.min(this.#span?.oldest ?? frame.timestamp, frame.timestamp); + const newest = Math.max(this.#span?.newest ?? frame.timestamp, frame.timestamp); + this.#span = { oldest, newest }; + + const ran = group === this.#first.group && frame.timestamp - this.#first.timestamp > this.#hold.floor; + if (ran || newest - oldest >= this.#hold.maxDelay) return this.#release(this.#take()); return []; } @@ -56,7 +61,8 @@ export class Order { #take(): T[] { const held = this.#held ?? []; this.#held = undefined; - return held; + // Stable, so frames sharing a timestamp keep their arrival order. + return held.sort((a, b) => a.frame.timestamp - b.frame.timestamp); } #release(reads: T[]): T[] { From f59e54fb52c06fdf00ac04cc26c7fc745b0aa4fb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 08:20:00 -0700 Subject: [PATCH 3/3] fix(watch): reset for a new codec epoch before judging audio order, and read the hold bounds live Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/decoder.test.ts | 19 +++++++++ js/watch/src/audio/decoder.ts | 10 +++-- js/watch/src/audio/order.test.ts | 44 +++++++++++++------- js/watch/src/audio/order.ts | 65 ++++++++++++++---------------- 4 files changed, 86 insertions(+), 52 deletions(-) diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 70f2098670a..295121dad69 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -350,6 +350,25 @@ for (const cmaf of [false, true]) { } }); + // A fresh encoder after a declared end may restart its timeline below the old one, and the decoder + // resets for it before anything is judged out of order. + it(`decodes a ${cmaf ? "CMAF" : "legacy"} fresh encoder below the timeline it replaced`, async () => { + const playback = await play(Milli(100), false, cmaf); + try { + frameTimestamp = 480_000; + playback.push({ ...frame(), group: 0 }); + await microtasks(); + playback.push({ kind: "end", group: 0, end: Micro(520_000) }); + frameTimestamp = 0; + playback.push({ ...frame(), group: 1 }); + await microtasks(); + expect(playback.inserted()).toEqual([500_000, 20_000]); + } finally { + playback.close(); + await microtasks(); + } + }); + it(`skips a ${cmaf ? "CMAF" : "legacy"} live subscription head older than its live edge`, async () => { const playback = await play(Milli(100), false, cmaf); try { diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 55ae67fb13d..b4454809eeb 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -596,8 +596,9 @@ export class Decoder { }); } - // Decode released frames in order, resetting the decoder before the first frame of a new codec - // epoch. The terminal sees them in decode order, so it anchors on the oldest. Reports whether to keep going. + // Decode released frames, resetting the decoder before the first frame of a new codec epoch and + // only then dropping one that is out of order. The terminal sees them in decode order, so it + // anchors on the oldest. Reports whether to keep going. async #play( effect: Effect, decoder: AudioDecoder, @@ -611,6 +612,7 @@ export class Decoder { decoder.configure(config); order.reset(); } + if (!order.admit(read)) continue; if (!(await this.#decode(effect, decoder, read.frame))) return false; } return true; @@ -644,10 +646,10 @@ export class Decoder { // anyway, or spans the window the relay serves. #order(): Order { if (!this.sync.out.buffered.peek()) return new Order(undefined); - return new Order({ + return new Order(() => ({ floor: Micro.fromMilli(this.sync.out.delay.peek()), maxDelay: Micro.fromMilli(this.#subscribeMaxDelay.peek()), - }); + })); } // Reorders groups and frames up to the shared max delay. A rename is not a cold start: the path's diff --git a/js/watch/src/audio/order.test.ts b/js/watch/src/audio/order.test.ts index 00e722e6063..d8269b7e03e 100644 --- a/js/watch/src/audio/order.test.ts +++ b/js/watch/src/audio/order.test.ts @@ -1,12 +1,13 @@ import { describe, expect, it } from "bun:test"; import { Order } from "./order"; -const read = (group: number, timestamp: number) => ({ group, frame: { timestamp } }); -const timestamps = (reads: ReturnType[]) => reads.map(({ frame }) => frame.timestamp); +type Read = { group: number; frame: { timestamp: number } }; +const read = (group: number, timestamp: number): Read => ({ group, frame: { timestamp } }); +const timestamps = (reads: Read[]) => reads.map(({ frame }) => frame.timestamp); describe("Order", () => { it("releases a held head sorted once a newer group arrives", () => { - const order = new Order({ floor: 100, maxDelay: 1000 }); + const order = new Order(() => ({ floor: 100, maxDelay: 1000 })); for (const t of [40, 20, 0]) expect(order.push(read(t / 20, t))).toEqual([]); expect(timestamps(order.push(read(3, 60)))).toEqual([0, 20, 40, 60]); expect(timestamps(order.push(read(4, 80)))).toEqual([80]); @@ -14,7 +15,7 @@ describe("Order", () => { // A publisher may keep a long run of audio in one group, which never brings a newer one. it("releases a head once its first group runs past the floor", () => { - const order = new Order({ floor: 100, maxDelay: 1000 }); + const order = new Order(() => ({ floor: 100, maxDelay: 1000 })); expect(order.push(read(5, 0))).toEqual([]); expect(order.push(read(4, -20))).toEqual([]); expect(order.push(read(5, 100))).toEqual([]); @@ -23,7 +24,7 @@ describe("Order", () => { // With no floor, playback starts on the first frame, but the head still has to arrive first. it("holds the head past its first frame with a zero floor", () => { - const order = new Order({ floor: 0, maxDelay: 1000 }); + const order = new Order(() => ({ floor: 0, maxDelay: 1000 })); expect(order.push(read(1, 20))).toEqual([]); expect(order.push(read(0, 0))).toEqual([]); expect(timestamps(order.push(read(1, 40)))).toEqual([0, 20, 40]); @@ -31,33 +32,48 @@ describe("Order", () => { // Nothing older than the subscription's max delay is served, so a longer head is complete. it("releases a head once it spans the max delay", () => { - const order = new Order({ floor: 100, maxDelay: 60 }); + const order = new Order(() => ({ floor: 100, maxDelay: 60 })); for (const t of [60, 40, 20]) expect(order.push(read(t / 20, t))).toEqual([]); expect(timestamps(order.push(read(0, 0)))).toEqual([0, 20, 40, 60]); }); it("releases the head as it stands on a flush", () => { - const order = new Order({ floor: 100, maxDelay: 1000 }); + const order = new Order(() => ({ floor: 100, maxDelay: 1000 })); order.push(read(1, 20)); order.push(read(0, 0)); expect(timestamps(order.flush())).toEqual([0, 20]); expect(order.flush()).toEqual([]); }); - it("drops a frame older than one already released", () => { - const order = new Order({ floor: 100, maxDelay: 1000 }); + it("admits a frame only if it is no older than one already admitted", () => { + const order = new Order(() => ({ floor: 100, maxDelay: 1000 })); order.push(read(1, 20)); - expect(timestamps(order.push(read(2, 40)))).toEqual([20, 40]); - expect(order.push(read(0, 0))).toEqual([]); + const released = order.push(read(2, 40)); + expect(released.every((r) => order.admit(r))).toBe(true); + expect(order.push(read(0, 0))).toEqual([read(0, 0)]); + expect(order.admit(read(0, 0))).toBe(false); // A reset decoder restamps from its next input, so older frames are welcome again. order.reset(); - expect(timestamps(order.push(read(0, 0)))).toEqual([0]); + expect(order.admit(read(0, 0))).toBe(true); + }); + + // A rise in the delay while the head arrives must not release it at the old floor. + it("reads its bounds as the head arrives", () => { + let floor = 20; + const order = new Order(() => ({ floor, maxDelay: 1000 })); + expect(order.push(read(5, 0))).toEqual([]); + floor = 100; + expect(order.push(read(5, 40))).toEqual([]); + expect(order.push(read(4, -20))).toEqual([]); + expect(timestamps(order.push(read(5, 120)))).toEqual([-20, 0, 40, 120]); }); it("holds nothing live", () => { - const order = new Order(undefined); + const order = new Order(undefined); expect(timestamps(order.push(read(1, 20)))).toEqual([20]); - expect(order.push(read(0, 0))).toEqual([]); + expect(order.admit(read(1, 20))).toBe(true); + expect(timestamps(order.push(read(0, 0)))).toEqual([0]); + expect(order.admit(read(0, 0))).toBe(false); }); }); diff --git a/js/watch/src/audio/order.ts b/js/watch/src/audio/order.ts index 1290d9a72b0..098ba25e402 100644 --- a/js/watch/src/audio/order.ts +++ b/js/watch/src/audio/order.ts @@ -9,67 +9,64 @@ * A buffered subscription holds its head and releases it sorted. The head is complete once a newer * group than the first arrives, or once the first group alone runs past the floor: the ring could not * start playing before holding that much, so waiting for the head costs no startup latency. It is - * also released once it spans the subscription's max delay, since nothing older is served. Any frame - * still older than one already released is dropped, as is every reordered frame when not holding. + * also released once it spans the subscription's max delay, since nothing older is served. After + * that, and throughout when not holding, a frame older than one already decoded is not admitted. */ export class Order { - // The head, in arrival order, while it is still arriving. - #held: T[] | undefined; + // The current bounds in microseconds, while the head is still arriving. + #hold: (() => { floor: number; maxDelay: number }) | undefined; + // The head, in arrival order. + #held: T[] = []; // The group the head started with, and its own first frame. #first?: { group: number; timestamp: number }; // The oldest and newest held timestamps. #span?: { oldest: number; newest: number }; - #hold: { floor: number; maxDelay: number }; - // The newest frame released since the decoder was last reset. + // The newest frame admitted since the decoder was last reset. #newest?: number; - /** Hold the head (bounds in microseconds), or decode each frame as it arrives. */ - constructor(hold: { floor: number; maxDelay: number } | undefined) { - this.#held = hold ? [] : undefined; - this.#hold = hold ?? { floor: 0, maxDelay: 0 }; + /** Hold the head within the bounds `hold` reports, or pass each frame straight through. */ + constructor(hold: (() => { floor: number; maxDelay: number }) | undefined) { + this.#hold = hold; } - /** Accept a frame, returning the frames ready to decode, oldest first. */ + /** Accept a frame, returning the frames released for decoding, oldest first. */ push(read: T): T[] { - const held = this.#held; - if (!held) return this.#release([read]); + const hold = this.#hold?.(); + if (!hold) return [read]; const { group, frame } = read; this.#first ??= { group, timestamp: frame.timestamp }; - if (group > this.#first.group) return this.#release([...this.#take(), read]); + if (group > this.#first.group) return [...this.flush(), read]; - held.push(read); + this.#held.push(read); const oldest = Math.min(this.#span?.oldest ?? frame.timestamp, frame.timestamp); const newest = Math.max(this.#span?.newest ?? frame.timestamp, frame.timestamp); this.#span = { oldest, newest }; - const ran = group === this.#first.group && frame.timestamp - this.#first.timestamp > this.#hold.floor; - if (ran || newest - oldest >= this.#hold.maxDelay) return this.#release(this.#take()); + const ran = group === this.#first.group && frame.timestamp - this.#first.timestamp > hold.floor; + if (ran || newest - oldest >= hold.maxDelay) return this.flush(); return []; } - /** Release the head as it stands: the track ended, or a gap or end cut it short. */ + /** Release the head as it stands, ending the hold: the track ended, or a gap or end cut it short. */ flush(): T[] { - return this.#release(this.#take()); + this.#hold = undefined; + const held = this.#held; + this.#held = []; + // Stable, so frames sharing a timestamp keep their arrival order. + return held.sort((a, b) => a.frame.timestamp - b.frame.timestamp); + } + + /** Whether a released frame is still in order for the decoder, recording it if so. */ + admit(read: T): boolean { + const { timestamp } = read.frame; + if (this.#newest !== undefined && timestamp < this.#newest) return false; + this.#newest = timestamp; + return true; } /** Start the order over after the decoder resets, which restarts its output timestamps. */ reset(): void { this.#newest = undefined; } - - #take(): T[] { - const held = this.#held ?? []; - this.#held = undefined; - // Stable, so frames sharing a timestamp keep their arrival order. - return held.sort((a, b) => a.frame.timestamp - b.frame.timestamp); - } - - #release(reads: T[]): T[] { - return reads.filter(({ frame }) => { - if (this.#newest !== undefined && frame.timestamp < this.#newest) return false; - this.#newest = frame.timestamp; - return true; - }); - } }