From 541d5c11cb7aa7f3872b350ffaf5dba574d9e69f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 11:50:25 -0800 Subject: [PATCH 1/4] chore(quest): claim quest/m2/watch-lookahead.md Quest-Claim: 77470ad5c033cf9c From cd755daac97a2f4f9a4cc8c4dc6fa60335d4c443 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 11:52:44 -0800 Subject: [PATCH 2/4] feat(watch)!: rename buffer to lookahead Co-Authored-By: Claude Opus 5.5 --- doc/concept/use-case/ai.md | 4 +- doc/lib/js/watch.md | 24 ++++++---- js/watch/src/audio/decoder.test.ts | 10 ++-- js/watch/src/element.ts | 39 ++++++++------- js/watch/src/player.test.ts | 3 +- js/watch/src/player.ts | 7 +-- js/watch/src/sync.test.ts | 48 ++++++++++--------- js/watch/src/sync.ts | 21 ++++---- js/watch/src/video/decoder.test.ts | 8 ++-- .../audio-jitter-target/buffered-converge.md | 18 +++---- quest/m1/buffered-ascending.md | 12 ++--- quest/m2/README.md | 1 - quest/m2/watch-lookahead.md | 41 ---------------- quest/m4/watch-decode-gate.md | 4 +- test/interop/clients/js/open-gop/main.ts | 2 +- test/video/src/poster.ts | 2 +- 16 files changed, 107 insertions(+), 137 deletions(-) delete mode 100644 quest/m2/watch-lookahead.md diff --git a/doc/concept/use-case/ai.md b/doc/concept/use-case/ai.md index ecc6989799b..8276c7f13dd 100644 --- a/doc/concept/use-case/ai.md +++ b/doc/concept/use-case/ai.md @@ -20,8 +20,8 @@ latency budget, video skippable, prompts fully reliable, all on one connection. A TTS model emits a whole sentence in a burst with timestamps in the future. WebRTC keeps buffering and playout inside its media engine; MoQ exposes the timestamped burst to the application, and the player paces it. In the browser, -set a latency ceiling on [``](/lib/js/watch#buffered-playback) to -buffer ahead and call `reset()` on an interruption. The +set a `lookahead` on [``](/lib/js/watch#buffered-playback) to +play ahead of the live edge and call `reset()` on an interruption. The [Pipecat](https://github.com/pipecat-ai/pipecat) voice-agent framework ships a MoQ transport built on this. diff --git a/doc/lib/js/watch.md b/doc/lib/js/watch.md index e06dec0c09d..2083746cac6 100644 --- a/doc/lib/js/watch.md +++ b/doc/lib/js/watch.md @@ -33,7 +33,7 @@ in sync at the latency you ask for. | `versions` | Protocol versions to offer, most preferred first, comma-separated like `moq-transport-18,moq-lite-06`. Every stable version by default. | | `paused`, `muted`, `volume` | The usual player controls. | | `delay` | How far playback trails the live edge: `"auto"` (the default, sized from how late frames arrive; see [audio jitter](/concept/audio-jitter)), a duration like `"300ms"`, or `"instant"` to paint video as it decodes, with no audio. | -| `buffer` | How far ahead of the live edge media may run; see [buffered playback](#buffered-playback). | +| `lookahead` | How far ahead of the live edge media may run; see [buffered playback](#buffered-playback). | | `captions` | The caption track to show. | | `visible` | Only download video while the element is on screen (or near it). | | `backend` | How the canvas is drawn; see [@moq/video](/lib/js/video). `auto` (default), `webgpu`, or `2d`. | @@ -50,7 +50,7 @@ The overlay adds play/pause, volume, fullscreen, a quality selector, a buffering indicator, and a stats panel. Audio, video, and captions play from one clock on local time, anchored at the -earliest arrival (with `buffer`, at the earliest retained frame once startup +earliest arrival (with `lookahead`, at the earliest retained frame once startup fills). A congested track falls behind on its own instead of stalling the others; in `auto` delay, its rising playout target still raises the delay every track plays at. A discontinuity restarts only its own track's timeline, @@ -239,18 +239,22 @@ them; nothing is retained without an open handle. By default the player minimizes latency: it skips ahead whenever media piles up past the delay. Content produced faster than real time, such as a TTS response emitted in one burst with future timestamps, wants the opposite. Set -`buffer` to how far ahead it may run and it plays through at the encoded pace: +`lookahead` to how far ahead it may run and it plays through at the encoded pace: ```html - + ``` -Durations need a unit; a bare number is rejected. Audio holds the buffer as -encoded frames, so a large one is cheap; video waits as decoded pictures, which -hold decoder memory, so a long video buffer is not. The first video frame shows -as a poster at once, and playback starts once every enabled track has buffered -the delay. `el.reset()` flushes and re-anchors at the next frame, which is how a -producer interrupts an utterance. +`lookahead` is not a jitter buffer: a live stream leaves it at zero and tunes +`delay`, since playback only skips ahead once media sits more than +`delay + lookahead` past the playhead. + +Durations need a unit; a bare number is rejected. Audio holds the lookahead +as encoded frames, so a large one is cheap; video waits as decoded pictures, +which hold decoder memory, so a long video lookahead is not. The first video +frame shows as a poster at once, and playback starts once every enabled track +has buffered the delay. `el.reset()` flushes and re-anchors at the next frame, +which is how a producer interrupts an utterance. ## Strict CSP diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 3a15d872cae..3591fc93a72 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -330,8 +330,8 @@ async function play(initial: Delay, buffered = false, cmaf = false) { const supportProbes = mock(async () => true); Object.assign(supportProbes, { [supportCacheKey]: decoderConfigKey }); const source = new Source({ broadcast, supported: supportProbes }); - const buffer = new Signal(Milli(buffered ? 1000 : 0)); - const sync = new Sync({ delay, buffer }); + const lookahead = new Signal(Milli(buffered ? 1000 : 0)); + const sync = new Sync({ delay, lookahead }); // The decoder's own switch, which a mute or pause turns off. const decoding = new Signal(true); const decoder = new Decoder({ source, sync, enabled: decoding }); @@ -352,7 +352,7 @@ async function play(initial: Delay, buffered = false, cmaf = false) { return { supportProbes, delay, - buffer, + lookahead, sync, push, queued, @@ -1517,7 +1517,7 @@ it.each([100, 1000])( ); it("a rejected audio support probe does not leave buffered video waiting on a pending track", async () => { - const sync = new Sync({ delay: Milli(100), buffer: Milli(1000) }); + const sync = new Sync({ delay: Milli(100), lookahead: Milli(1000) }); const remove = sync.track({ target: new Signal(Milli(100)), startup: new Signal({ start: Milli(0), end: Milli(100) }), @@ -1613,7 +1613,7 @@ for (const cmaf of [false, true]) { const playback = await play(Milli(100), !buffered, cmaf); try { await playback.play(12); - playback.buffer.set(Milli(buffered ? 1000 : 0)); + playback.lookahead.set(Milli(buffered ? 1000 : 0)); await microtasks(); const inserted = playback.inserted().length; const start = frameTimestamp; diff --git a/js/watch/src/element.ts b/js/watch/src/element.ts index ef5cb78cb2a..480771d4d3a 100644 --- a/js/watch/src/element.ts +++ b/js/watch/src/element.ts @@ -62,8 +62,10 @@ const OBSERVED = [ "follow", // Renamed to `follow`, observed only to refuse it. "announced", - "delay", + // Renamed to `lookahead`, observed only to refuse it. "buffer", + "delay", + "lookahead", "catalog-format", "captions", "echo", @@ -97,12 +99,12 @@ function parseDelay(value: string | null): Delay { throw invalidAttribute("delay", value, '"auto", "instant", or a duration like "300ms"'); } -// Parse the `buffer` attribute: a duration, or zero when absent. -function parseBuffer(value: string | null): Milli { +// Parse the `lookahead` attribute: a duration, or zero when absent. +function parseLookahead(value: string | null): Milli { if (value === null) return Milli.zero; const parsed = parseDuration(value); if (parsed !== undefined) return parsed; - throw invalidAttribute("buffer", value, 'a duration like "30s"'); + throw invalidAttribute("lookahead", value, 'a duration like "30s"'); } // Parse the `echo` attribute: the viewer's feedback name, one visible path segment, or undefined when absent. @@ -181,7 +183,7 @@ export default class MoqWatch extends HTMLElement { // How far playback trails the live edge. delay: new Signal("auto"), // Future-dated media held beyond the live edge before playback skips ahead. - buffer: new Signal(Milli.zero), + lookahead: new Signal(Milli.zero), video: { // The desired video rendition (resolution/bitrate cap). target: new Signal(undefined), @@ -345,7 +347,7 @@ export default class MoqWatch extends HTMLElement { const delay = effect.get(this.controls.delay); this.#reflect(effect, "delay", typeof delay === "number" ? formatDuration(delay) : delay); }); - this.signals.run((effect) => this.#reflect(effect, "buffer", formatDuration(effect.get(this.controls.buffer)))); + this.signals.run((effect) => this.#reflect(effect, "lookahead", formatDuration(effect.get(this.controls.lookahead)))); // Track the element's rendered size and feed it into the rendition picker, // scaled by devicePixelRatio so high-DPI screens still get sharp renditions. @@ -420,8 +422,11 @@ export default class MoqWatch extends HTMLElement { else if (name === "announced" && newValue !== null) throw new Error("moq-watch: `announced` was renamed to `follow`"); else if (name === "announced") this.#accept("announced"); + else if (name === "buffer" && newValue !== null) + throw new Error("moq-watch: `buffer` was renamed to `lookahead`"); + else if (name === "buffer") this.#accept("buffer"); else if (name === "delay") this.delay = parseDelay(newValue); - else if (name === "buffer") this.buffer = parseBuffer(newValue); + else if (name === "lookahead") this.lookahead = parseLookahead(newValue); else if (name === "catalog-format") this.catalogFormat = parseCatalogFormat(newValue); else if (name === "captions") this.captions = newValue || undefined; else if (name === "echo") this.echo = parseEcho(newValue); @@ -575,19 +580,19 @@ export default class MoqWatch extends HTMLElement { /** * Future-dated media held beyond the live edge before playback skips ahead, in milliseconds. * - * Zero (the default) minimizes latency. A larger value enables buffered playback: build up a - * buffer from future-dated frames (e.g. TTS written faster than real-time) and only skip ahead - * once they would sit further than `delay + buffer` past the playhead. Call `reset()` at each - * utterance boundary. + * Zero (the default) minimizes latency. A larger value enables buffered playback: build up + * future-dated frames (e.g. TTS written faster than real-time) and only skip ahead + * once they would sit further than `delay + lookahead` past the playhead. Call `reset()` at each + * utterance boundary. Not a jitter buffer: live streams leave it at zero and tune `delay`. */ - get buffer(): Milli { - return this.controls.buffer.peek(); + get lookahead(): Milli { + return this.controls.lookahead.peek(); } - set buffer(value: Milli) { - if (typeof value !== "number") throw invalidAttribute("buffer", value, "non-negative milliseconds"); - this.controls.buffer.set(parseBuffer(formatDuration(value))); - this.#accept("buffer"); + set lookahead(value: Milli) { + if (typeof value !== "number") throw invalidAttribute("lookahead", value, "non-negative milliseconds"); + this.controls.lookahead.set(parseLookahead(formatDuration(value))); + this.#accept("lookahead"); } /** diff --git a/js/watch/src/player.test.ts b/js/watch/src/player.test.ts index b0baf5492df..a93862832fa 100644 --- a/js/watch/src/player.test.ts +++ b/js/watch/src/player.test.ts @@ -11,9 +11,10 @@ async function flush() { for (let i = 0; i < 10; i++) await Promise.resolve(); } -test("Player refuses the renamed reload and announced inputs", () => { +test("Player refuses the renamed reload, announced, and buffer inputs", () => { expect(() => new Player({ reload: true } as never)).toThrow("`reload` was renamed to `follow`"); expect(() => new Player({ announced: true } as never)).toThrow("`announced` was renamed to `follow`"); + expect(() => new Player({ buffer: 1000 } as never)).toThrow("`buffer` was renamed to `lookahead`"); }); test.each(["table", "consumer", "structural"] as const)("Player borrows a %s origin", async (role) => { diff --git a/js/watch/src/player.ts b/js/watch/src/player.ts index 2cdb66a3745..16e19c631fd 100644 --- a/js/watch/src/player.ts +++ b/js/watch/src/player.ts @@ -31,7 +31,7 @@ export type PlayerInput = { /** Playback distance from the live edge. */ delay: Getter; /** Future-dated media held beyond the live edge. */ - buffer: Getter; + lookahead: Getter; /** Video rendition and canvas output. */ video: { /** Desired video rendition or quality limits. */ @@ -112,6 +112,7 @@ export class Player { for (const renamed of ["reload", "announced"]) { if (renamed in props) throw new Error(`Watch.Player: \`${renamed}\` was renamed to \`follow\``); } + if ("buffer" in props) throw new Error("Watch.Player: `buffer` was renamed to `lookahead`"); this.in = { origin: getter(props.origin), name: getter(props.name ?? Path.empty()), @@ -122,7 +123,7 @@ export class Player { probe: getter(props.probe), paused: getter(props.paused ?? false), delay: getter(props.delay ?? "auto"), - buffer: getter(props.buffer ?? Milli.zero), + lookahead: getter(props.lookahead ?? Milli.zero), video: { target: getter(props.video?.target), canvas: getter(props.video?.canvas), @@ -166,7 +167,7 @@ export class Player { this.text = new Text.Source({ broadcast: this.broadcast, target: this.in.text.target }); this.#signals.cleanup(() => this.text.close()); - this.sync = new Sync({ delay: this.in.delay, buffer: this.in.buffer }); + this.sync = new Sync({ delay: this.in.delay, lookahead: this.in.lookahead }); this.#signals.cleanup(() => this.sync.close()); this.video = new Video.Decoder({ source: videoSource, sync: this.sync, enabled: this.#videoEnabled }); diff --git a/js/watch/src/sync.test.ts b/js/watch/src/sync.test.ts index 2ba372cb264..18ceb9d7357 100644 --- a/js/watch/src/sync.test.ts +++ b/js/watch/src/sync.test.ts @@ -6,7 +6,11 @@ import { type Delay, Sync } from "./sync"; // Effects in @moq/signals flush on a microtask, so let pending updates drain before asserting. const flush = () => yieldNow(); -describe("delay and buffer", () => { +describe("delay and lookahead", () => { + it("refuses the renamed buffer input", () => { + expect(() => new Sync({ buffer: 1000 } as never)).toThrow("`buffer` was renamed to `lookahead`"); + }); + it("holds no lookahead by default", async () => { const sync = new Sync(); await flush(); @@ -14,7 +18,7 @@ describe("delay and buffer", () => { sync.close(); }); - it("caps maxDelay at the delay when no buffer is configured", async () => { + it("caps maxDelay at the delay when no lookahead is configured", async () => { const sync = new Sync({ delay: 100 as Milli }); await flush(); expect(sync.out.buffered.peek()).toBe(false); @@ -23,39 +27,39 @@ describe("delay and buffer", () => { sync.close(); }); - it("adds the buffer on top of the delay", async () => { - // The buffer is measured from the live edge, so it does not swallow the delay: a frame may - // sit `delay + buffer` ahead of the playhead before playback skips forward. - const sync = new Sync({ delay: 100 as Milli, buffer: 30_000 as Milli }); + it("adds the lookahead on top of the delay", async () => { + // The lookahead is measured from the live edge, so it does not swallow the delay: a frame may + // sit `delay + lookahead` ahead of the playhead before playback skips forward. + const sync = new Sync({ delay: 100 as Milli, lookahead: 30_000 as Milli }); await flush(); expect(sync.out.buffered.peek()).toBe(true); expect(sync.out.maxDelay.peek()).toBe(30_100 as Milli); sync.close(); }); - it("stays unbuffered for a zero buffer", async () => { - const sync = new Sync({ delay: 200 as Milli, buffer: 0 as Milli }); + it("stays unbuffered for a zero lookahead", async () => { + const sync = new Sync({ delay: 200 as Milli, lookahead: 0 as Milli }); await flush(); expect(sync.out.buffered.peek()).toBe(false); expect(sync.out.maxDelay.peek()).toBe(200 as Milli); sync.close(); }); - it("reacts to a buffer set after construction", async () => { - const buffer = new Signal(0 as Milli); - const sync = new Sync({ delay: 100 as Milli, buffer }); + it("reacts to a lookahead set after construction", async () => { + const lookahead = new Signal(0 as Milli); + const sync = new Sync({ delay: 100 as Milli, lookahead }); await flush(); expect(sync.out.buffered.peek()).toBe(false); - buffer.set(30_000 as Milli); + lookahead.set(30_000 as Milli); await flush(); expect(sync.out.buffered.peek()).toBe(true); expect(sync.out.maxDelay.peek()).toBe(30_100 as Milli); sync.close(); }); - it("holds nothing when instant, whatever the buffer says", async () => { - const sync = new Sync({ delay: "instant", buffer: 30_000 as Milli }); + it("holds nothing when instant, whatever the lookahead says", async () => { + const sync = new Sync({ delay: "instant", lookahead: 30_000 as Milli }); await flush(); expect(sync.out.instant.peek()).toBe(true); expect(sync.out.buffered.peek()).toBe(false); @@ -189,7 +193,7 @@ describe("wait", () => { await flush(); expect(Bun.peek.status(parked)).toBe("pending"); - // A frame 10s newer, with no buffer, skips the playhead past the parked one. + // A frame 10s newer, with no lookahead, skips the playhead past the parked one. video.received(Milli(timestamp + 10_000)); expect(await parked).toBe(false); expect(await video.wait(timestamp)).toBe(false); @@ -280,7 +284,7 @@ describe("skip floor", () => { video.received(Milli(1000), Milli(5000)); expect(video.presentable(Milli(0))).toBe(true); - // A startup burst with no buffer: audio 50ms newer skips the playhead to 950ms. + // A startup burst with no lookahead: audio 50ms newer skips the playhead to 950ms. video.received(Milli(1050), Milli(5000)); expect(video.presentable(Milli(1000))).toBe(true); expect(video.presentable(Milli(949))).toBe(false); @@ -345,7 +349,7 @@ describe("buffered startup", () => { it(`waits for both spans when ${first} fills first`, async () => { const clock = new Manual(); const installed = clock.install(); - const sync = new Sync({ delay: Milli(100), buffer: Milli(1000) }); + const sync = new Sync({ delay: Milli(100), lookahead: Milli(1000) }); const spans = { audio: new Signal<{ start: Milli; end: Milli } | null>(null), video: new Signal<{ start: Milli; end: Milli } | null>(null), @@ -384,7 +388,7 @@ describe("buffered startup", () => { it("releases startup waits on abort, reset, and close without starting playback", async () => { const clock = new Manual(); const installed = clock.install(); - const sync = new Sync({ delay: Milli(100), buffer: Milli(1000) }); + const sync = new Sync({ delay: Milli(100), lookahead: Milli(1000) }); const video = sync.track({ target: new Signal(Milli(100)), startup: new Signal(null) }); try { await clock.advance(Nano.zero); @@ -410,8 +414,8 @@ describe("buffered startup", () => { it("removes a disabled track from the floor and leaves live pacing unchanged", async () => { const clock = new Manual(); const installed = clock.install(); - const buffer = new Signal(Milli(1000)); - const sync = new Sync({ delay: Milli(100), buffer }); + const lookahead = new Signal(Milli(1000)); + const sync = new Sync({ delay: Milli(100), lookahead }); const absent = new Signal<{ start: Milli; end: Milli } | null | undefined>(null); const remove = sync.track({ target: new Signal(Milli(100)), startup: absent }); const video = sync.track({ @@ -426,7 +430,7 @@ describe("buffered startup", () => { absent.set(undefined); await clock.advance(Nano.zero); expect(await video.wait(Milli(0))).toBe(true); - buffer.set(Milli.zero); + lookahead.set(Milli.zero); sync.reset(); absent.set(null); await clock.advance(Nano.zero); @@ -541,7 +545,7 @@ describe("tracks", () => { }); it("keeps the playhead where buffered startup put it when the owner leaves", async () => { - const sync = new Sync({ delay: Milli(100), buffer: Milli(1000) }); + const sync = new Sync({ delay: Milli(100), lookahead: Milli(1000) }); const audioSpan = new Signal<{ start: Milli; end: Milli } | null>(null); const videoSpan = new Signal<{ start: Milli; end: Milli } | null>(null); const audio = sync.track({ target: target(), startup: audioSpan }); diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index 06398a92528..f88eb9623fa 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -10,7 +10,7 @@ import { Effect, type Getter, getter, type Inputs, type Readonlys, readonlys, Si * * `"instant"` drops the buffer and the pacing together: nothing is held, and the wait before * presenting a frame never sleeps, so it presents as soon as it exists. It also overrides - * {@link SyncInput.buffer}, since holding a lookahead would contradict holding nothing. Audio is + * {@link SyncInput.lookahead}, since holding a lookahead would contradict holding nothing. Audio is * off: a ring with no depth underruns, so the audio decoder unsubscribes and flushes its ring. */ export type Delay = "instant" | "auto" | Milli; @@ -26,11 +26,11 @@ export type SyncInput = { * Zero minimizes latency, the live default: playback re-anchors as soon as anything arrives * early. A larger value lets faster-than-real-time frames (a TTS response with future * timestamps) build up instead of being skipped. Always finite, so worst case the audio ring - * drops its oldest samples rather than exhausting memory. With a nonzero buffer, the first video - * frame is a poster until every enabled media track holds the resolved delay; `buffer` is - * lookahead, not the amount required to start. + * drops its oldest samples rather than exhausting memory. With a nonzero lookahead, the first video + * frame is a poster until every enabled media track holds the resolved delay; `lookahead` is + * the most held ahead, not the amount required to start. */ - buffer: Getter; + lookahead: Getter; }; /** Signals the sync derives. */ @@ -57,7 +57,7 @@ export type SyncOutput = { // would sit further than `maxDelay` ahead of the playhead. See `reset()`. buffered: Signal; - // Derived: `delay + buffer`, the furthest a held frame may sit ahead of the playhead. Feeds the + // Derived: `delay + lookahead`, the furthest a held frame may sit ahead of the playhead. Feeds the // transport subscription and the container consumer, which bound the same span. Always finite. maxDelay: Signal; }; @@ -116,9 +116,10 @@ export class Clock { owner?: Part; constructor(props?: Inputs) { + if (props && "buffer" in props) throw new Error("Watch.Sync: `buffer` was renamed to `lookahead`"); this.in = { delay: getter(props?.delay ?? ("auto" as Delay)), - buffer: getter(props?.buffer ?? Milli.zero), + lookahead: getter(props?.lookahead ?? Milli.zero), }; } @@ -438,10 +439,10 @@ export class Sync { const out = this.#clock.out; const delay = effect.get(out.delay); // "instant" holds nothing, so a configured lookahead doesn't apply. - const buffer = effect.get(out.instant) ? Milli.zero : effect.get(this.in.buffer); + const lookahead = effect.get(out.instant) ? Milli.zero : effect.get(this.in.lookahead); - out.buffered.set(buffer > 0); - out.maxDelay.set(Milli(delay + buffer)); + out.buffered.set(lookahead > 0); + out.maxDelay.set(Milli(delay + lookahead)); } // A fixed delay is taken literally and "auto" holds the deepest track's target, so every track diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index 05133fb10c7..bbdc36e5084 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -247,7 +247,7 @@ async function guardedPlayback( reads: Read[], hold: readonly number[] = [], live = false, - buffer = 0, + lookahead = 0, { fields = {}, initial, feeds }: { fields?: Partial; initial?: Delay; feeds?: Feeds } = {}, ) { // Load the optional container before the fixture drains only reactive microtasks. @@ -371,8 +371,8 @@ async function guardedPlayback( mediaProducer.close(); }; const source = new Source({ broadcast, supported: async () => true }); - const delay = new Signal(initial ?? (buffer ? Milli(100) : "instant")); - const sync = new Sync({ delay, buffer: Milli(buffer) }); + const delay = new Signal(initial ?? (lookahead ? Milli(100) : "instant")); + const sync = new Sync({ delay, lookahead: Milli(lookahead) }); const enabled = new Signal(true); // Published before the decoder subscribes, since a broadcast with no request handler refuses // an unknown track. @@ -608,7 +608,7 @@ function marker(group: bigint, end: number): Read { for (const kind of ["legacy", "cmaf"] as const) { // Another track skips the shared playhead to `skipTo` while a 1s picture is still inside - // WebCodecs (`queued`) or parked in Sync.wait, with a 100ms delay and no buffer. Reports what + // WebCodecs (`queued`) or parked in Sync.wait, with a 100ms delay and no lookahead. Reports what // was showing before the skip and whether the picture presented once its deadline came due. async function skipDuring(stage: "queued" | "parked", skipTo: number) { const clock = manualAt(5000); diff --git a/quest/m1/audio-jitter-target/buffered-converge.md b/quest/m1/audio-jitter-target/buffered-converge.md index e1878c4de3d..b438cfbfa81 100644 --- a/quest/m1/audio-jitter-target/buffered-converge.md +++ b/quest/m1/audio-jitter-target/buffered-converge.md @@ -2,20 +2,20 @@ ## Goal -A `` with a nonzero `buffer` keeps lip sync in steady state: once +A `` with a nonzero `lookahead` keeps lip sync in steady state: once a stall, a rise of the `"auto"` delay, or publisher clock drift has left the audio playhead behind the shared clock, the ring time-stretches back onto it. It never skips, so a TTS utterance still plays every word. Today buffered mode turns off the ring's catch-up (`playing = !this.#stalled && !this.#buffered` in `js/watch/src/audio/ring-buffer.ts`), so any lag is permanent. Audio trails video for the rest of the stream, and once the lag -passes `delay + buffer` the container consumer drops whole audio groups +passes `delay + lookahead` the container consumer drops whole audio groups ("skipping slow groups"), which is the cut-out reported in [moq-dev/moq#8031](https://github.com/moq-dev/moq/issues/8031). Boundaries: video already follows the shared clock and is unchanged. The clock is not driven by audio, and a slow audio track never holds video. Live -mode (`buffer` of zero) keeps its own catch-up. Native playback has no +mode (`lookahead` of zero) keeps its own catch-up. Native playback has no buffered mode. ## Plan @@ -26,7 +26,7 @@ simulation on main with the real `Sync`, container `Consumer`, `Backpressure` and `AudioRingBuffer` (a copy of `js/watch/src/audio/replay.ts` with buffered mode on) measured audio behind video: about -12 ms live versus +105 ms buffered under `"auto"` with jittery then calm phases; 0 versus +300 ms after -400 ms stalls with `delay=100ms buffer=500ms`; and a publisher clock 1% fast +400 ms stalls with `delay=100ms lookahead=500ms`; and a publisher clock 1% fast climbing to 1.4 s of lag with 2926 slow-group skips. The reporter's other claim, that a delay above 500 ms breaks lip sync, did not reproduce (300, 800 and 1500 ms all held about -22 ms). @@ -37,8 +37,6 @@ Decisions: reply and close / sharpen docs and warn only. Steady-state lip sync comes from each track converging on the shared clock, and buffered mode broke that even for TTS with video. -- Rename `buffer`: keep / ✅ rename in m2, as - [Rename buffer to lookahead](/quest/m2/watch-lookahead.md). - Milestone: ✅ m1, in this epic / m3. Recommendations: @@ -51,10 +49,9 @@ Recommendations: - Extend the replay harness with buffered rows (the stall and the drift scenarios above), budgeting audio-behind-video and slow-group skips, and keep a TTS row asserting no frame is dropped. -- Update the `buffer` docs in `js/watch/src/element.ts`, `js/watch/src/sync.ts` - and `doc/lib/js/watch.md` to say live streams take their slack from - `delay`, and that a buffered player catches up after a stall by playing - slightly faster. +- Update the `lookahead` docs in `js/watch/src/element.ts`, + `js/watch/src/sync.ts` and `doc/lib/js/watch.md` to say that a buffered + player catches up after a stall by playing slightly faster. Public API: none expected. Wire: none. @@ -68,5 +65,4 @@ Public API: none expected. Wire: none. ## Related -- [Rename buffer to lookahead](/quest/m2/watch-lookahead.md) - the name that invited the misuse - [Buffered playback subscribes in ascending order](/quest/m1/buffered-ascending.md) - the other buffered-mode change to the audio path diff --git a/quest/m1/buffered-ascending.md b/quest/m1/buffered-ascending.md index 38e3a4211cf..347d38c109a 100644 --- a/quest/m1/buffered-ascending.md +++ b/quest/m1/buffered-ascending.md @@ -2,12 +2,12 @@ ## Goal -A `` with a nonzero `buffer` subscribes to its audio and video +A `` with a nonzero `lookahead` subscribes to its audio and video tracks with `order: "ascending"`, so the oldest group in its window is sent first and a voice reply's opening never arrives after its live edge. The audio decoder's head hold from [moq-dev/moq#6549](https://github.com/moq-dev/moq/pull/6549) shrinks to one rule: hold the head until it spans the floor, then release it -sorted. Live playback (`buffer` of zero) stays newest-first. +sorted. Live playback (`lookahead` of zero) stays newest-first. Not here: the protocol's `Group Order` (landed in [moq-dev/moq#5385](https://github.com/moq-dev/moq/pull/5385)), the buffered @@ -27,7 +27,7 @@ Decided 2026-10-23 by the maintainer: from #6549 can go. Voice AI motivates it but is not the only user. - ✅ All buffered media goes ascending, audio and video. Not chosen: audio only; an explicit opt-in independent of buffering. -- ✅ The opt-in is buffered mode itself: a nonzero `buffer` makes every media +- ✅ The opt-in is buffered mode itself: a nonzero `lookahead` makes every media subscription ascending. A buffered player always wants its window from the start, and a live one never does, so a second knob adds nothing. - ✅ Keep a head hold, sized by the floor (the minimum latency): hold frames @@ -55,7 +55,7 @@ Implementation: to end. Say in the watch docs that a mixed audience gets newest-first upstream. - The draft says an ascending subscriber SHOULD raise `Subscriber Max Age`. - Buffered mode already subscribes with `delay + buffer`; confirm that covers + Buffered mode already subscribes with `delay + lookahead`; confirm that covers it. - Replace `Order`'s release rules in `js/watch/src/audio/order.ts` with the floor span, keeping `admit()` and the epoch reset ordering from #6549. Keep @@ -65,14 +65,14 @@ Implementation: once the JS consumer drops groups below its cursor, an ascending head arrives above it, so nothing valid is dropped. Whichever lands second re-runs the other's tests. -- Update `doc/concept/use-case/ai.md` and the `buffer` docs in +- Update `doc/concept/use-case/ai.md` and the `lookahead` docs in `js/watch/src/element.ts` and `js/watch/src/sync.ts` in the same PR. - Verify against a voice agent on a lite-07 relay, such as a Pipecat pipeline publishing each TTS reply as a fresh track with one Opus frame per group: every reply decodes from its first frame, plain and barge-in, with no overflow. -Runtime buffer changes switch the audio ring and reader mode. This quest +Runtime lookahead changes switch the audio ring and reader mode. This quest keeps waiting on lite-07 for the head-hold simplification. ## Required diff --git a/quest/m2/README.md b/quest/m2/README.md index a04de05388e..88d6f5f7f1c 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -184,7 +184,6 @@ quests that require them. - [Session policy lives in moq-net](/quest/m1/session-policy/README.md) - accept and connect deadlines, admission, window widening and reconnect live once in moq-net; moq-tokio and moq-relay's io_uring path call them (held in m1 by an open claim) - [Rename moq-rtc to moq-webrtc](/quest/m2/webrtc-rename.md) - one published `moq-webrtc` crate holds the gateway, the ICE driver, and the data-channel transport - [MoQ on rind](/quest/m1/rind/README.md) - every MoQ crate takes its crypto from rind, a pure-Rust graviola fork, so no build compiles C crypto (held in m1 by an open claim) -- [Rename buffer to lookahead](/quest/m2/watch-lookahead.md) - ``'s `buffer` becomes `lookahead`, so it is no longer mistaken for a jitter buffer on live streams - [Cut readiness](/quest/m2/cut-gate/README.md) - the checks the cut needs that break nothing: main no slower than release, vendored fixes ported, the pre-cut audit re-run, and moq-c at parity - [Probe](/quest/m2/probe/README.md) - a session probes up to the bitrate it wants to send or receive, so a stopped simulcast rendition resumes and a viewer up-switches; the browser pads where native sessions retransmit (held in m2 by an open claim; moves to m3) - [Finish the milestone re-cut](/quest/m2/recut-held.md) - claim-held epics move to the milestone their kind and priority give them once their claims merge diff --git a/quest/m2/watch-lookahead.md b/quest/m2/watch-lookahead.md deleted file mode 100644 index 231daf1a020..00000000000 --- a/quest/m2/watch-lookahead.md +++ /dev/null @@ -1,41 +0,0 @@ -# [S] Rename moq-watch's buffer to lookahead - -## Goal - -``'s `buffer` property and attribute, and the `Player`/`Sync` -input behind them, are named `lookahead`. Users read `buffer` as a jitter -buffer and set it on live streams, where it disables catch-up -([moq-dev/moq#8031](https://github.com/moq-dev/moq/issues/8031)). It is -really how far future-dated media, such as TTS written faster than real time, -may run ahead of the live edge. Live slack stays `delay`. - -Boundaries: JavaScript only. Nothing in Rust, Python, or the bindings uses -the name. The buffered-ranges UI (`buffer-control.ts`) shows something else -and keeps its name. - -## Plan - -Planned 2026-11-06 (maintainer, /quest-import of -[moq-dev/moq#8031](https://github.com/moq-dev/moq/issues/8031)). - -Decisions: - -- Rename: keep `buffer` / ✅ rename in m2. Breaking for `@moq/watch`, so it - joins the next breaking release. -- Name: ✅ `lookahead` / `ahead` / `burst`. It says what the value is, and - is already the word the docs use. - -Settled by repository rules: no deprecated alias. `buffer` is removed -outright, and `js/watch/MIGRATE.md` gets an Unreleased entry showing the move. -Internal names derived from it (`buffered`, `parseBuffer`) follow when they -mean the same thing. Update `doc/lib/js/watch.md`, -`doc/concept/use-case/ai.md`, the demos, and quests that name the property, -such as [Buffered playback subscribes in ascending -order](/quest/m1/buffered-ascending.md). - -Public API: breaking, `buffer` becomes `lookahead` on `` and the -`Player`/`Sync` inputs. Wire: none. - -## Related - -- [Buffered audio converges on the shared clock](/quest/m1/audio-jitter-target/buffered-converge.md) - the behavior fix for the same report diff --git a/quest/m4/watch-decode-gate.md b/quest/m4/watch-decode-gate.md index 8a73f039154..bbf97892ec7 100644 --- a/quest/m4/watch-decode-gate.md +++ b/quest/m4/watch-decode-gate.md @@ -4,14 +4,14 @@ `@moq/watch` keeps its video lookahead encoded, as audio already does. Only frames within the decoder's latency of presentation are decoded, so a long -`buffer` holds encoded chunks instead of one decoded `VideoFrame` per frame. +`lookahead` holds encoded chunks instead of one decoded `VideoFrame` per frame. ## Plan Today the video decoder's output callback (`js/watch/src/video/decoder.ts`) awaits `sync.wait` before presenting and closing each frame, so every buffered frame is held decoded. `Sync.received` caps this at -delay + buffer, so it only bites with a large `buffer`, where it can starve +delay + lookahead, so it only bites with a large `lookahead`, where it can starve the hardware decoder's frame pool (2026-10-07 audit). Decided (2026-10-07): wait before decode, like the audio `ring.wait`. Feed a diff --git a/test/interop/clients/js/open-gop/main.ts b/test/interop/clients/js/open-gop/main.ts index ac5a2f4e58d..abca10153dc 100644 --- a/test/interop/clients/js/open-gop/main.ts +++ b/test/interop/clients/js/open-gop/main.ts @@ -92,7 +92,7 @@ window.runRecovery = async (packets) => { catalog: { video: { renditions: { video: config } } }, }); const source = new Source({ broadcast, supported: async () => true }); - const sync = new Sync({ delay: Milli(0), buffer: Milli(10_000) }); + const sync = new Sync({ delay: Milli(0), lookahead: Milli(10_000) }); const decoder = new Decoder({ source, sync }); try { await inputDone; diff --git a/test/video/src/poster.ts b/test/video/src/poster.ts index 5a470095187..95986059a46 100644 --- a/test/video/src/poster.ts +++ b/test/video/src/poster.ts @@ -53,7 +53,7 @@ async function poster(audio: boolean, buffered: boolean) { catalog, video: { canvas, visible: "always", backend: "2d" }, delay: Milli(100), - buffer: Milli(buffered ? 1000 : 0), + lookahead: Milli(buffered ? 1000 : 0), }); const chunks = new Map(); const encoder = new VideoEncoder({ From f62bf37e81defb4ead4e179afd99309976343d93 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 12:05:43 -0800 Subject: [PATCH 3/4] chore(watch): update API report and README for lookahead Co-Authored-By: Claude Opus 5.5 --- js/watch/README.md | 2 +- js/watch/api/element.api.md | 10 +++++----- js/watch/api/index.api.md | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/js/watch/README.md b/js/watch/README.md index 2b3f975cb3c..8225f2cac34 100644 --- a/js/watch/README.md +++ b/js/watch/README.md @@ -73,7 +73,7 @@ The simplest way to watch a stream: | `backend` | auto, webgpu, 2d | `auto` | Graphics API for the canvas. `auto` picks WebGPU where it can render video, else Canvas2D. | | `follow` | boolean | true | Follow the broadcast's announcements: wait for one before subscribing, and move to each restart. Ignored when the relay does not support broadcast discovery. | | `delay` | `auto`, duration, `instant` | `auto` | Distance from the live edge. `instant` paints frames as they decode and disables audio. | -| `buffer` | duration | `0ms` | Future-dated media held before playback skips ahead. | +| `lookahead` | duration | `0ms` | How far future-dated media may run ahead of the live edge. Not a jitter buffer: live streams tune `delay`. | | `captions` | string | off | Text rendition to render. | | `catalog-format` | hang, msf, manual | auto-detected | The catalog format; detected from the name suffix unless set. | | `echo` | path segment | - | The viewer's feedback name: once the catalog solicits feedback, publish `/.echo` reporting what each rendition received and played. | diff --git a/js/watch/api/element.api.md b/js/watch/api/element.api.md index 33095fdae21..5e6963169e7 100644 --- a/js/watch/api/element.api.md +++ b/js/watch/api/element.api.md @@ -16,13 +16,11 @@ import { Video } from "@moq/watch"; export default class extends HTMLElement { constructor(); - attributeChangedCallback(name: "delay" | "buffer" | "url" | "echo" | "name" | "follow" | "volume" | "muted" | "paused" | "visible" | "backend" | "versions" | "announced" | "catalog-format" | "captions", oldValue: string | null, newValue: string | null): void; + attributeChangedCallback(name: "delay" | "lookahead" | "url" | "echo" | "name" | "follow" | "volume" | "muted" | "paused" | "visible" | "backend" | "versions" | "announced" | "buffer" | "catalog-format" | "captions", oldValue: string | null, newValue: string | null): void; readonly audio: Audio.Decoder; get backend(): Video.Backend; set backend(value: Video.Backend); readonly broadcast: Broadcast; - get buffer(): Clock.Milli; - set buffer(value: Clock.Milli); get captions(): string | undefined; set captions(value: string | undefined); get catalog(): Hang.Catalog.Root | undefined; @@ -37,8 +35,8 @@ export default class extends HTMLElement { target: Signals.Signal; volume: Signals.Signal; }; - buffer: Signals.Signal; delay: Signals.Signal; + lookahead: Signals.Signal; paused: Signals.Signal; text: { target: Signals.Signal; @@ -60,11 +58,13 @@ export default class extends HTMLElement { emitter: Audio.Emitter; get follow(): boolean; set follow(value: boolean); + get lookahead(): Clock.Milli; + set lookahead(value: Clock.Milli); get muted(): boolean; set muted(value: boolean); get name(): Net.Path.Valid; set name(value: string | Net.Path.Valid); - static observedAttributes: readonly ["url", "versions", "name", "paused", "volume", "muted", "visible", "backend", "follow", "announced", "delay", "buffer", "catalog-format", "captions", "echo"]; + static observedAttributes: readonly ["url", "versions", "name", "paused", "volume", "muted", "visible", "backend", "follow", "announced", "buffer", "delay", "lookahead", "catalog-format", "captions", "echo"]; get paused(): boolean; set paused(value: boolean); readonly player: Player; diff --git a/js/watch/api/index.api.md b/js/watch/api/index.api.md index 1eea1a447ea..cf9273af40d 100644 --- a/js/watch/api/index.api.md +++ b/js/watch/api/index.api.md @@ -163,12 +163,12 @@ export type PlayerInput = { target: Signals.Getter; volume: Signals.Getter; }; - buffer: Signals.Getter; catalog: Signals.Getter; catalogFormat: Signals.Getter; delay: Signals.Getter; enabled: Signals.Getter; follow: Signals.Getter; + lookahead: Signals.Getter; name: Signals.Getter; origin: Signals.Getter; paused: Signals.Getter; @@ -203,8 +203,8 @@ export class Sync { } export type SyncInput = { - buffer: Signals.Getter; delay: Signals.Getter; + lookahead: Signals.Getter; }; export type SyncOutput = { From 18942711a491386533fdaae69ea47863d8595eeb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 12:13:43 -0800 Subject: [PATCH 4/4] style(watch): format element.ts Co-Authored-By: Claude Opus 5.5 --- js/watch/src/element.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/js/watch/src/element.ts b/js/watch/src/element.ts index 480771d4d3a..81f9d126621 100644 --- a/js/watch/src/element.ts +++ b/js/watch/src/element.ts @@ -347,7 +347,9 @@ export default class MoqWatch extends HTMLElement { const delay = effect.get(this.controls.delay); this.#reflect(effect, "delay", typeof delay === "number" ? formatDuration(delay) : delay); }); - this.signals.run((effect) => this.#reflect(effect, "lookahead", formatDuration(effect.get(this.controls.lookahead)))); + this.signals.run((effect) => + this.#reflect(effect, "lookahead", formatDuration(effect.get(this.controls.lookahead))), + ); // Track the element's rendered size and feed it into the rendition picker, // scaled by devicePixelRatio so high-DPI screens still get sharp renditions.