Repository navigation
fix(watch): hold a buffered subscription's audio head and decode it in order (backport, 0.6.6) #6550
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
fix(watch): hold a buffered subscription's audio head and decode it in order (backport, 0.6.6) #6550
Changes from all commits
9be54fe
08586f5
7a9b203
fbbce7e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -21,12 +21,16 @@ import { type AudioBuffer, createAudioBuffer } from "./buffer"; | |
| import { type DecoderConfig, decoderConfig, type PlaybackIdentity, playbackIdentity } from "./config"; | ||
| import { Handover } from "./handover"; | ||
| import { ringSamples } from "./latency"; | ||
| import { Order } from "./order"; | ||
| // Compiled and inlined as a blob URL via vite-plugin-worklet. | ||
| import RenderWorklet from "./render-worklet.ts?worklet"; | ||
| import type { Source } from "./source"; | ||
| import { type DecodedSpan, Terminal } from "./terminal"; | ||
| import { Warmup } from "./warmup"; | ||
|
|
||
| type Next = NonNullable<Awaited<ReturnType<Container.Consumer["next"]>>>; | ||
| type FrameNext = Next & { frame: NonNullable<Next["frame"]> }; | ||
|
|
||
| const LEGACY_WARMUP_CALLBACKS = 3; | ||
|
|
||
| export type DecoderInput = { | ||
|
|
@@ -345,22 +349,30 @@ export class Decoder { | |
| }; | ||
| decoder.configure(decoderConfig); | ||
|
|
||
| // The newest frame sent to the decoder since it was last reset. | ||
| let newest: Time.Micro | undefined; | ||
| const order = this.#order(); | ||
| let discontinuity: number | undefined; | ||
| for (;;) { | ||
| const next = await nextMedia(consumer); | ||
| if (!next) break; | ||
| if (this.#onNext(next)) { | ||
| decoder.reset(); | ||
| decoder.configure(decoderConfig); | ||
| newest = undefined; | ||
| // The track ending, a marker, or a discontinuity cuts a held head short. A group merely | ||
| // completing does not. | ||
| if ( | ||
| next === undefined || | ||
| next.end !== undefined || | ||
| next.discontinuity !== (discontinuity ?? next.discontinuity) | ||
| ) { | ||
| if (!(await this.#play(effect, decoder, decoderConfig, order, order.flush()))) return; | ||
| } | ||
| if (next.end !== undefined) { | ||
| if (!next) break; | ||
| discontinuity = next.discontinuity; | ||
| if (next.frame === undefined || next.end !== undefined) { | ||
| if (this.#onNext(next)) { | ||
| decoder.reset(); | ||
| decoder.configure(decoderConfig); | ||
| order.reset(); | ||
| } | ||
| continue; | ||
| } | ||
|
|
||
| const { frame } = next; | ||
| if (!frame) continue; | ||
|
|
||
| // Mark that we received this frame right now. | ||
| const timestamp = Time.Milli.fromMicro(frame.timestamp as Time.Micro); | ||
|
|
@@ -370,27 +382,7 @@ export class Decoder { | |
| bytesReceived: (stats?.bytesReceived ?? 0) + frame.payload.byteLength, | ||
| })); | ||
|
|
||
| // Decode in timestamp order. A subscription's head groups arrive after the live edge served | ||
| // alongside them, and the ring, anchored at the first decoded frame, cannot place audio | ||
| // before it. Chrome's AudioDecoder would also stamp every later output from its first | ||
| // input, shifting the rest of the track by the reorder and overflowing a buffered ring. | ||
| if (newest !== undefined && (frame.timestamp as Time.Micro) < newest) continue; | ||
| newest = frame.timestamp as Time.Micro; | ||
|
|
||
| // Backpressure: in buffered mode this holds the encoded frame until the playhead nears | ||
| // it, keeping the lookahead above the floor as Opus instead of decoded PCM. No-op live. | ||
| await this.#ring?.wait(frame.timestamp as Time.Micro, effect.abort); | ||
|
|
||
| const chunk = new EncodedAudioChunk({ | ||
| type: frame.keyframe ? "key" : "delta", | ||
| data: frame.payload, | ||
| timestamp: frame.timestamp, | ||
| }); | ||
|
|
||
| // A fatal decode error closes the decoder, so decoding again throws InvalidStateError out | ||
| // of this loop. Stop instead: the error callback already reported the real failure. | ||
| if (decoder.state === "closed") break; | ||
| decoder.decode(chunk); | ||
| if (!(await this.#play(effect, decoder, decoderConfig, order, order.push({ ...next, frame })))) return; | ||
| } | ||
| }); | ||
| } | ||
|
|
@@ -446,21 +438,31 @@ export class Decoder { | |
| }; | ||
| decoder.configure(decoderConfig); | ||
|
|
||
| // The newest frame sent to the decoder since it was last reset. | ||
| let newest: Time.Micro | undefined; | ||
| const order = this.#order(); | ||
| let discontinuity: number | undefined; | ||
| for (;;) { | ||
| const next = await nextMedia(consumer); | ||
| // The track ending, a marker, or a discontinuity cuts a held head short. A group merely | ||
| // completing does not. | ||
| if ( | ||
| next === undefined || | ||
| next.end !== undefined || | ||
| next.discontinuity !== (discontinuity ?? next.discontinuity) | ||
| ) { | ||
| if (!(await this.#play(effect, decoder, decoderConfig, order, order.flush()))) return; | ||
| } | ||
| if (!next) break; | ||
|
|
||
| // Reset and re-anchor before decoding the first frame of a new codec epoch. | ||
| if (this.#onNext(next)) { | ||
| decoder.reset(); | ||
| decoder.configure(decoderConfig); | ||
| newest = undefined; | ||
| discontinuity = next.discontinuity; | ||
| if (next.frame === undefined) { | ||
| // Reset and re-anchor before decoding the first frame of a new codec epoch. | ||
| if (this.#onNext(next)) { | ||
| decoder.reset(); | ||
| decoder.configure(decoderConfig); | ||
| order.reset(); | ||
| } | ||
| continue; | ||
| } | ||
|
|
||
| const { frame } = next; | ||
| if (!frame) continue; | ||
|
|
||
| const timestamp = Time.Milli.fromMicro(frame.timestamp); | ||
| this.sync.received(timestamp, "audio"); | ||
|
|
@@ -469,29 +471,58 @@ export class Decoder { | |
| bytesReceived: (stats?.bytesReceived ?? 0) + frame.payload.byteLength, | ||
| })); | ||
|
|
||
| // Decode in timestamp order. A subscription's head groups arrive after the live edge served | ||
| // alongside them, and the ring, anchored at the first decoded frame, cannot place audio | ||
| // before it. Chrome's AudioDecoder would also stamp every later output from its first | ||
| // input, shifting the rest of the track by the reorder and overflowing a buffered ring. | ||
| if (newest !== undefined && frame.timestamp < newest) continue; | ||
| newest = frame.timestamp; | ||
|
|
||
| // Backpressure: in buffered mode this holds the encoded frame until the playhead nears | ||
| // it, keeping the lookahead above the floor as Opus instead of decoded PCM. No-op live. | ||
| await this.#ring?.wait(frame.timestamp, effect.abort); | ||
|
|
||
| if (decoder.state === "closed") break; | ||
| decoder.decode( | ||
| new EncodedAudioChunk({ | ||
| type: frame.keyframe ? "key" : "delta", | ||
| data: frame.payload, | ||
| timestamp: frame.timestamp, | ||
| }), | ||
| ); | ||
| if (!(await this.#play(effect, decoder, decoderConfig, order, order.push({ ...next, frame })))) return; | ||
| } | ||
| }); | ||
| } | ||
|
|
||
| // A buffered subscription holds its head until it runs past the floor, which playback waits for | ||
| // anyway, or spans the window the relay serves. | ||
| #order(): Order<FrameNext> { | ||
| if (!this.sync.out.buffered.peek()) return new Order(undefined); | ||
| return new Order(() => ({ | ||
| floor: Time.Micro.fromMilli(this.sync.out.delay.peek()), | ||
| maxDelay: Time.Micro.fromMilli(this.sync.out.maxAge.peek()), | ||
|
Comment on lines
+484
to
+485
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When automatic delay changes after AGENTS.md reference: js/AGENTS.md:L13-L13 Useful? React with 👍 / 👎.
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed in fbbce7e: the bounds come from a getter read on every held frame, so they follow (Written by Claude Opus 5.5) |
||
| })); | ||
| } | ||
|
|
||
| // Decode released frames, resetting the decoder before the first frame of a new codec epoch and | ||
| // only then dropping one that is out of order. The terminal sees them in decode order, so it | ||
| // anchors on the oldest. Reports whether to keep going. | ||
| async #play( | ||
| effect: Effect, | ||
| decoder: AudioDecoder, | ||
| config: AudioDecoderConfig, | ||
| order: Order<FrameNext>, | ||
| reads: FrameNext[], | ||
| ): Promise<boolean> { | ||
| for (const next of reads) { | ||
| if (this.#onNext(next)) { | ||
| decoder.reset(); | ||
| decoder.configure(config); | ||
| order.reset(); | ||
| } | ||
| if (!order.admit(next)) continue; | ||
| const { frame } = next; | ||
|
|
||
| // Backpressure: in buffered mode this holds the encoded frame until the playhead nears | ||
| // it, keeping the lookahead above the floor as Opus instead of decoded PCM. No-op live. | ||
| await this.#ring?.wait(frame.timestamp as Time.Micro, effect.abort); | ||
|
|
||
| // A fatal decode error closes the decoder, so decoding again throws InvalidStateError out | ||
| // of the loop. Stop instead: the error callback already reported the real failure. | ||
| if (decoder.state === "closed") return false; | ||
| decoder.decode( | ||
| new EncodedAudioChunk({ | ||
| type: frame.keyframe ? "key" : "delta", | ||
| data: frame.payload, | ||
| timestamp: frame.timestamp, | ||
| }), | ||
| ); | ||
| } | ||
| return true; | ||
| } | ||
|
|
||
| #emit(sample: AudioData, decoded: DecodedSpan = this.#terminal.span(sample)) { | ||
| const { timestamp, frameOffset, frames } = decoded; | ||
| const timestampMilli = Time.Milli.fromMicro(timestamp); | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| import { describe, expect, it } from "bun:test"; | ||
| import { Order } from "./order"; | ||
|
|
||
| type Read = { group: number; frame: { timestamp: number } }; | ||
| const read = (group: number, timestamp: number): Read => ({ group, frame: { timestamp } }); | ||
| const timestamps = (reads: Read[]) => reads.map(({ frame }) => frame.timestamp); | ||
|
|
||
| describe("Order", () => { | ||
| it("releases a held head sorted once a newer group arrives", () => { | ||
| const order = new Order<Read>(() => ({ floor: 100, maxDelay: 1000 })); | ||
| for (const t of [40, 20, 0]) expect(order.push(read(t / 20, t))).toEqual([]); | ||
| expect(timestamps(order.push(read(3, 60)))).toEqual([0, 20, 40, 60]); | ||
| expect(timestamps(order.push(read(4, 80)))).toEqual([80]); | ||
| }); | ||
|
|
||
| // A publisher may keep a long run of audio in one group, which never brings a newer one. | ||
| it("releases a head once its first group runs past the floor", () => { | ||
| const order = new Order<Read>(() => ({ floor: 100, maxDelay: 1000 })); | ||
| expect(order.push(read(5, 0))).toEqual([]); | ||
| expect(order.push(read(4, -20))).toEqual([]); | ||
| expect(order.push(read(5, 100))).toEqual([]); | ||
| expect(timestamps(order.push(read(5, 120)))).toEqual([-20, 0, 100, 120]); | ||
| }); | ||
|
|
||
| // With no floor, playback starts on the first frame, but the head still has to arrive first. | ||
| it("holds the head past its first frame with a zero floor", () => { | ||
| const order = new Order<Read>(() => ({ floor: 0, maxDelay: 1000 })); | ||
| expect(order.push(read(1, 20))).toEqual([]); | ||
| expect(order.push(read(0, 0))).toEqual([]); | ||
| expect(timestamps(order.push(read(1, 40)))).toEqual([0, 20, 40]); | ||
| }); | ||
|
|
||
| // Nothing older than the subscription's max delay is served, so a longer head is complete. | ||
| it("releases a head once it spans the max delay", () => { | ||
| const order = new Order<Read>(() => ({ floor: 100, maxDelay: 60 })); | ||
| for (const t of [60, 40, 20]) expect(order.push(read(t / 20, t))).toEqual([]); | ||
| expect(timestamps(order.push(read(0, 0)))).toEqual([0, 20, 40, 60]); | ||
| }); | ||
|
|
||
| it("releases the head as it stands on a flush", () => { | ||
| const order = new Order<Read>(() => ({ floor: 100, maxDelay: 1000 })); | ||
| order.push(read(1, 20)); | ||
| order.push(read(0, 0)); | ||
| expect(timestamps(order.flush())).toEqual([0, 20]); | ||
| expect(order.flush()).toEqual([]); | ||
| }); | ||
|
|
||
| it("admits a frame only if it is no older than one already admitted", () => { | ||
| const order = new Order<Read>(() => ({ floor: 100, maxDelay: 1000 })); | ||
| order.push(read(1, 20)); | ||
| const released = order.push(read(2, 40)); | ||
| expect(released.every((r) => order.admit(r))).toBe(true); | ||
| expect(order.push(read(0, 0))).toEqual([read(0, 0)]); | ||
| expect(order.admit(read(0, 0))).toBe(false); | ||
|
|
||
| // A reset decoder restamps from its next input, so older frames are welcome again. | ||
| order.reset(); | ||
| expect(order.admit(read(0, 0))).toBe(true); | ||
| }); | ||
|
|
||
| // A rise in the delay while the head arrives must not release it at the old floor. | ||
| it("reads its bounds as the head arrives", () => { | ||
| let floor = 20; | ||
| const order = new Order<Read>(() => ({ floor, maxDelay: 1000 })); | ||
| expect(order.push(read(5, 0))).toEqual([]); | ||
| floor = 100; | ||
| expect(order.push(read(5, 40))).toEqual([]); | ||
| expect(order.push(read(4, -20))).toEqual([]); | ||
| expect(timestamps(order.push(read(5, 120)))).toEqual([-20, 0, 40, 120]); | ||
| }); | ||
|
|
||
| it("holds nothing live", () => { | ||
| const order = new Order<Read>(undefined); | ||
| expect(timestamps(order.push(read(1, 20)))).toEqual([20]); | ||
| expect(order.admit(read(1, 20))).toBe(true); | ||
| expect(timestamps(order.push(read(0, 0)))).toEqual([0]); | ||
| expect(order.admit(read(0, 0))).toBe(false); | ||
| }); | ||
| }); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| /** | ||
| * Puts one subscription's frames in timestamp order for the audio decoder. | ||
| * | ||
| * Groups go out newest-first, so a subscription's head (the groups the publisher still holds) arrives | ||
| * after the live edge it was served alongside. Chrome's AudioDecoder stamps every output from its | ||
| * first input, so decoding the head after the live edge shifts the rest of the track, and the ring, | ||
| * anchored at the first decoded frame, cannot place the head anyway. | ||
| * | ||
| * A buffered subscription holds its head and releases it sorted. The head is complete once a newer | ||
| * group than the first arrives, or once the first group alone runs past the floor: the ring could not | ||
| * start playing before holding that much, so waiting for the head costs no startup latency. It is | ||
| * also released once it spans the subscription's max delay, since nothing older is served. After | ||
| * that, and throughout when not holding, a frame older than one already decoded is not admitted. | ||
| */ | ||
| export class Order<T extends { group: number; frame: { timestamp: number } }> { | ||
| // The current bounds in microseconds, while the head is still arriving. | ||
| #hold: (() => { floor: number; maxDelay: number }) | undefined; | ||
| // The head, in arrival order. | ||
| #held: T[] = []; | ||
| // The group the head started with, and its own first frame. | ||
| #first?: { group: number; timestamp: number }; | ||
| // The oldest and newest held timestamps. | ||
| #span?: { oldest: number; newest: number }; | ||
| // The newest frame admitted since the decoder was last reset. | ||
| #newest?: number; | ||
|
|
||
| /** Hold the head within the bounds `hold` reports, or pass each frame straight through. */ | ||
| constructor(hold: (() => { floor: number; maxDelay: number }) | undefined) { | ||
| this.#hold = hold; | ||
| } | ||
|
|
||
| /** Accept a frame, returning the frames released for decoding, oldest first. */ | ||
| push(read: T): T[] { | ||
| const hold = this.#hold?.(); | ||
| if (!hold) return [read]; | ||
|
|
||
| const { group, frame } = read; | ||
| this.#first ??= { group, timestamp: frame.timestamp }; | ||
| if (group > this.#first.group) return [...this.flush(), read]; | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When a subscription starts on group N and older cached groups are still in flight, a newly published N+1 can arrive first because Useful? React with 👍 / 👎.
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same limit as #6549 (discussion): without a declared start, "a group newer than the first" is a heuristic, and a newer stream overtaking the backlog defeats it. When that happens, the frames released early decode in order and the late backlog is dropped as older, which is 0.6.5's behavior and never worse. The usual case is fixed: across 10 local Voice demo calls on this build (plain and barge-in), every reply decoded from 0 ms with its first word whole, against 4 of 10 clipped before. The complete fix is flooring the ordered read at the subscription's resolved start, per quest/m1/read-backfill.md. That doesn't fit a patch release. (Written by Claude Opus 5.5) |
||
|
|
||
| this.#held.push(read); | ||
| const oldest = Math.min(this.#span?.oldest ?? frame.timestamp, frame.timestamp); | ||
| const newest = Math.max(this.#span?.newest ?? frame.timestamp, frame.timestamp); | ||
| this.#span = { oldest, newest }; | ||
|
|
||
| const ran = group === this.#first.group && frame.timestamp - this.#first.timestamp > hold.floor; | ||
| if (ran || newest - oldest >= hold.maxDelay) return this.flush(); | ||
| return []; | ||
| } | ||
|
|
||
| /** Release the head as it stands, ending the hold: the track ended, or a gap or end cut it short. */ | ||
| flush(): T[] { | ||
| this.#hold = undefined; | ||
| const held = this.#held; | ||
| this.#held = []; | ||
| // Stable, so frames sharing a timestamp keep their arrival order. | ||
| return held.sort((a, b) => a.frame.timestamp - b.frame.timestamp); | ||
| } | ||
|
|
||
| /** Whether a released frame is still in order for the decoder, recording it if so. */ | ||
| admit(read: T): boolean { | ||
| const { timestamp } = read.frame; | ||
| if (this.#newest !== undefined && timestamp < this.#newest) return false; | ||
| this.#newest = timestamp; | ||
| return true; | ||
| } | ||
|
|
||
| /** Start the order over after the decoder resets, which restarts its output timestamps. */ | ||
| reset(): void { | ||
| this.#newest = undefined; | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
When a buffered consumer reports a discontinuity on a frame whose timestamp precedes a previously held live-edge frame, such as after
Consumerdrops a missing head group for exceedingmaxAge, the preceding flush setsOrder.#newestto the live-edge timestamp and this call filters out the discontinuity frame before#playcan invoke#onNextandorder.reset(). The decoder and ring therefore do not reset at the gap, and this frame plus subsequent frames up to the old timestamp are silently dropped; handle the discontinuity reset before applying the previous epoch's ordering bound.Useful? React with 👍 / 👎.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Fixed in fbbce7e, with the same change on main in f59e54f.
Orderno longer filters on release:#playruns#onNext(main:#onRead) first, resets the decoder and the order for a new epoch, and only then asksorder.admit()whether the frame is out of order. A discontinuity frame below the old live edge now resets and decodes. Main'sdecoder.test.tscovers the equivalent case, a fresh encoder after anendrestarting below the old timeline. It fails on the previous commit.(Written by Claude Opus 5.5)