From f3c1d0d831f172484d747b921f1913cc4fb2b76e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 04:42:49 -0800 Subject: [PATCH 01/10] chore(quest): claim quest/m2/intra-refresh/dual-decoder.md Quest-Claim: fb462e0a241cd5b7 From 1bcd903841f53791486ad26e76ec74887f79b778 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 04:59:04 -0800 Subject: [PATCH 02/10] feat(watch): run a min-delay and a max-delay decoder for warmup renditions A video rendition with a catalog `warmup` (intra refresh) now decodes twice. The min-delay decoder holds the app's latency, drives the shared clock, and may show refresh stripes after a join or a skip. The max-delay decoder starts one group earlier and tolerates the warmup more lateness, so its picture stays clean; it shows whenever its timestamp is at or past the min-delay one's, and nothing shown ever steps back. Co-Authored-By: Claude Opus 5.5 --- doc/lib/js/watch.md | 9 + js/watch/src/video/decoder.test.ts | 427 ++++++++++++++++++++++++++++- js/watch/src/video/decoder.ts | 222 +++++++++++++-- 3 files changed, 632 insertions(+), 26 deletions(-) diff --git a/doc/lib/js/watch.md b/doc/lib/js/watch.md index 38ed468af74..93958feaa18 100644 --- a/doc/lib/js/watch.md +++ b/doc/lib/js/watch.md @@ -53,6 +53,15 @@ The failing track closes its subscription and stops decoding. Select another source or rendition to clear the failure; the player does not resubscribe to broken media automatically. +A video rendition encoded with intra refresh has no keyframes, and its catalog +`warmup` says how long a decoder takes to clean up after joining. The player +decodes such a rendition twice: one decoder holds your delay and may show +refresh stripes after a join or a skip, while a second starts one group earlier +and tolerates `warmup` more lateness, so its picture is clean. The clean +picture shows whenever it has caught up. Each decoder costs its own CPU and +decoder memory, and both share one subscription. A rendition without `warmup` +uses one decoder. + `el.broadcast.out.status` is `offline`, `loading`, `live`, `error`, or `unsupported`. On `unsupported`, update the player to read the broadcast's newer catalog; the overlay shows this notice. On `error` the origin refused the broadcast, `el.broadcast.out.error` says why, diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index e97a3b77f49..edca4eae2a9 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -165,15 +165,18 @@ class Picture { displayWidth = 16; displayHeight = 16; readonly timestamp: number; + // The index of the codec that decoded it, in creation order, or -1 for one a test made. + readonly codec: number; readonly #pictures: Picture[]; - constructor(timestamp: number, pictures: Picture[]) { + constructor(timestamp: number, pictures: Picture[], codec = -1) { this.#pictures = pictures; pictures.push(this); this.timestamp = timestamp; + this.codec = codec; } clone() { if (this.closed) throw new Error("clone of a closed frame"); - return new Picture(this.timestamp, this.#pictures); + return new Picture(this.timestamp, this.#pictures, this.codec); } close() { this.closed = true; @@ -202,20 +205,57 @@ function sample(group: number, timestamp: number, keyframe = true): Read { } const gap: Read = { kind: "gap", timestamp: Micro(0), duration: Micro(0) }; +// One read queue per container consumer, in the order each first reads. An empty queue waits for +// `push` rather than ending the track. +class Feeds { + readonly #queues: Read[][] = []; + readonly #owners = new Map(); + #wake: (() => void)[] = []; + + push(index: number, ...reads: Read[]) { + while (this.#queues.length <= index) this.#queues.push([]); + this.#queues[index].push(...reads); + for (const wake of this.#wake.splice(0)) wake(); + } + + async next(consumer: Container.Consumer): Promise { + let index = this.#owners.get(consumer); + if (index === undefined) { + index = this.#owners.size; + this.#owners.set(consumer, index); + } + for (;;) { + const read = this.#queues[index]?.shift(); + if (read) return read; + await new Promise((resolve) => this.#wake.push(resolve)); + } + } + + // Release every waiting read, ending its track. + close() { + this.#queues.length = 0; + for (const wake of this.#wake.splice(0)) wake(); + } +} + // `reads` stands in for the container consumer, unless `live`, which reads what tests publish into -// `producer` through real consumers instead. +// `producer` through real consumers instead, or `feeds`, which gives each consumer its own queue. +// `fields` add to the rendition's catalog entry, and `initial` is the delay the decoder subscribes with. async function guardedPlayback( kind: "legacy" | "cmaf", reads: Read[], hold: readonly number[] = [], live = false, buffer = 0, + { fields = {}, initial, feeds }: { fields?: Partial; initial?: Delay; feeds?: Feeds } = {}, ) { // Load the optional container before the fixture drains only reactive microtasks. if (kind === "cmaf") await import("@moq/hang/cmaf"); const originalDecoder = Object.getOwnPropertyDescriptor(globalThis, "VideoDecoder"); const originalChunk = Object.getOwnPropertyDescriptor(globalThis, "EncodedVideoChunk"); const submitted: number[] = []; + // What each codec, in creation order, was given. + const decoded: number[][] = []; const outputs: number[] = []; const pictures: Picture[] = []; const owed = new Set(hold); @@ -225,7 +265,10 @@ async function guardedPlayback( const frames: VideoDecoderInit["output"][] = []; const configures: VideoDecoderConfig[] = []; // A read that is an Error stands in for a malformed container read. - const next = spyOn(Container.Consumer.prototype, "next").mockImplementation(async () => { + const next = spyOn(Container.Consumer.prototype, "next").mockImplementation(async function ( + this: Container.Consumer, + ) { + if (feeds) return feeds.next(this); const read = reads.shift() as Read | Error | undefined; if (read instanceof Error) throw read; return read; @@ -236,8 +279,10 @@ async function guardedPlayback( // Like WebCodecs, a configure requires a key chunk before any delta. #keyRequired = true; readonly callbacks: VideoDecoderInit; + readonly index = errors.length; constructor(callbacks: VideoDecoderInit) { this.callbacks = callbacks; + decoded.push([]); errors.push(callbacks.error); frames.push(callbacks.output); } @@ -254,9 +299,10 @@ async function guardedPlayback( if (this.#keyRequired && chunk.type !== "key") throw new Error("a key chunk is required"); this.#keyRequired = false; submitted.push(chunk.timestamp); + decoded[this.index].push(chunk.timestamp); const emit = () => { outputs.push(chunk.timestamp); - const picture = new Picture(chunk.timestamp, pictures); + const picture = new Picture(chunk.timestamp, pictures, this.index); void this.callbacks.output(picture as unknown as VideoFrame); }; if (owed.delete(chunk.timestamp)) { @@ -288,7 +334,12 @@ async function guardedPlayback( } }, }); - const video = config({ codedWidth: 16, codedHeight: 16, description: "01640028ffe100046764002801000268ee" }); + const video = config({ + codedWidth: 16, + codedHeight: 16, + description: "01640028ffe100046764002801000268ee", + ...fields, + }); if (kind === "cmaf") video.container = { kind, @@ -313,7 +364,7 @@ async function guardedPlayback( mediaProducer.close(); }; const source = new Source({ broadcast, supported: async () => true }); - const delay = new Signal(buffer ? Milli(100) : "instant"); + const delay = new Signal(initial ?? (buffer ? Milli(100) : "instant")); const sync = new Sync({ delay, buffer: Milli(buffer) }); const enabled = new Signal(true); // Published before the decoder subscribes, since a broadcast with no request handler refuses @@ -326,7 +377,9 @@ async function guardedPlayback( delay, fail: (error: DOMException, index = -1) => errors.at(index)?.(error), emit: (timestamp: number, index = -1) => - frames.at(index)?.(new Picture(timestamp, pictures) as unknown as VideoFrame), + frames.at(index)?.( + new Picture(timestamp, pictures, (index + frames.length) % frames.length) as unknown as VideoFrame, + ), get codecs() { return errors.length; }, @@ -342,6 +395,7 @@ async function guardedPlayback( }, track, submitted, + decoded, outputs, pictures, sync, @@ -383,6 +437,7 @@ async function guardedPlayback( source.close(); sync.close(); closeAll(); + feeds?.close(); next.mockRestore(); for (const [name, original] of [ ["VideoDecoder", originalDecoder], @@ -1347,3 +1402,359 @@ for (const kind of ["legacy", "cmaf"] as const) { } }); } + +describe("Decoder with a warmup rendition", () => { + // One refresh sweep, which is one group. + const WARMUP = Catalog.u53(300); + // The codecs in creation order: the min-delay decoder opens first. + const MIN = 0; + const MAX = 1; + + // A frame of `group` at `ms`, a recovery point when `key`. + const at = (group: number, ms: number, key = false) => sample(group, ms * 1000, key); + + // A warmup rendition whose two container consumers each read their own feed, presenting at once. + async function refresh(fields: Partial = { warmup: WARMUP }) { + const feeds = new Feeds(); + const playback = await guardedPlayback("legacy", [], [], false, 0, { fields, feeds }); + // Every picture shown, as [ms, codec]. + const shown: [number, number][] = []; + const watch = new Effect(); + watch.subscribe(playback.decoder.out.frame, (frame) => { + const picture = frame as unknown as Picture | undefined; + if (picture) shown.push([picture.timestamp / 1000, picture.codec]); + }); + const current = () => { + const picture = playback.decoder.out.frame.peek() as unknown as Picture | undefined; + return picture && [picture.timestamp / 1000, picture.codec]; + }; + const push = async (lane: number, ...reads: Read[]) => { + feeds.push(lane, ...reads); + await microtasks(); + }; + return { + playback, + shown, + current, + push, + async close() { + watch.close(); + playback.close(); + await microtasks(); + expect(playback.pictures.filter((frame) => !frame.closed)).toEqual([]); + }, + }; + } + + function ascending(shown: [number, number][]) { + for (let i = 1; i < shown.length; i++) { + expect(shown[i][0]).toBeGreaterThanOrEqual(shown[i - 1][0]); + } + } + + it("opens one subscription and one decoder without a warmup", async () => { + for (const [fields, decoders] of [ + [{}, 1], + [{ warmup: Catalog.u53(0) }, 1], + [{ warmup: WARMUP }, 2], + ] as const) { + const subscribe = spyOn(Moq.Track.Consumer.prototype, "subscribe"); + const run = await refresh(fields); + try { + expect(run.playback.codecs).toBe(decoders); + const budgets = subscribe.mock.calls.map(([options]) => options?.maxDelay); + // The max-delay decoder tolerates one sweep more; js/net merges the two upstream. + expect(budgets).toEqual(decoders === 1 ? [Milli(0)] : [Milli(0), Milli(WARMUP)]); + } finally { + subscribe.mockRestore(); + await run.close(); + } + } + }); + + it("shows the min-delay picture on a cold join, then the max-delay one once it catches up", async () => { + const run = await refresh(); + try { + expect(run.playback.codecs).toBe(2); + + // The max-delay decoder joined a group earlier. Before the min-delay decoder anchors the + // clock, its pictures have no place on it. + await run.push(MAX, at(9, 700, true), at(9, 733)); + expect(run.current()).toBeUndefined(); + + // The min-delay decoder's picture shows at once, stripes and all. + await run.push(MIN, at(10, 1000, true)); + expect(run.current()).toEqual([1000, MIN]); + + // Pictures behind it never win, even though they are clean. + await run.push(MAX, at(9, 966)); + expect(run.current()).toEqual([1000, MIN]); + + // Caught up: the max-delay picture wins the tie, and keeps the lead. + await run.push(MAX, at(10, 1000, true)); + expect(run.current()).toEqual([1000, MAX]); + await run.push(MIN, at(10, 1033)); + await run.push(MAX, at(10, 1033)); + expect(run.current()).toEqual([1033, MAX]); + ascending(run.shown); + } finally { + await run.close(); + } + }); + + it("keeps the max-delay picture through a stall shorter than one sweep", async () => { + const run = await refresh(); + try { + for (const lane of [MIN, MAX]) await run.push(lane, at(10, 1000, true), at(10, 1033)); + expect(run.current()).toEqual([1033, MAX]); + + // Group 10 stalls. The min-delay decoder skips to group 11 and shows its stripes. + await run.push(MIN, gap, at(11, 1300, true)); + expect(run.current()).toEqual([1300, MIN]); + + // The max-delay decoder tolerates the stall: it gets the rest of group 10, decodes it + // behind the playhead, and takes over again once it reaches the min-delay picture. + await run.push(MAX, at(10, 1066), at(10, 1100), at(10, 1266)); + expect(run.current()).toEqual([1300, MIN]); + await run.push(MAX, at(11, 1300, true)); + expect(run.current()).toEqual([1300, MAX]); + await run.push(MIN, at(11, 1333)); + await run.push(MAX, at(11, 1333)); + expect(run.current()).toEqual([1333, MAX]); + + // Only the min-delay codec restarted at the skip. + expect(run.playback.resets).toBe(1); + expect(run.playback.decoded[MAX]).toEqual( + [1000, 1033, 1066, 1100, 1266, 1300, 1333].map((ms) => ms * 1000), + ); + ascending(run.shown); + } finally { + await run.close(); + } + }); + + it("skips both decoders on a stall longer than one sweep", async () => { + const run = await refresh(); + try { + for (const lane of [MIN, MAX]) await run.push(lane, at(10, 1000, true), at(10, 1033)); + + // Past the max-delay decoder's budget too: both skip and restart their codecs. + await run.push(MIN, gap, at(12, 1600, true)); + expect(run.current()).toEqual([1600, MIN]); + await run.push(MAX, gap, at(12, 1600, true)); + expect(run.current()).toEqual([1600, MAX]); + expect(run.playback.resets).toBe(2); + ascending(run.shown); + } finally { + await run.close(); + } + }); + + it("moves the shared clock only from the min-delay decoder", async () => { + const run = await refresh(); + const received = spyOn(run.playback.sync, "received"); + const reset = spyOn(run.playback.sync, "reset"); + try { + await run.push(MIN, at(10, 1000, true), at(10, 1033)); + await run.push(MAX, at(9, 700, true), at(9, 733), at(10, 1000, true), at(10, 1033)); + expect(received.mock.calls.map(([timestamp]) => timestamp)).toEqual([Milli(1000), Milli(1033)]); + expect(run.current()).toEqual([1033, MAX]); + + // The publisher restarts its timeline. The min-delay decoder resets the clock and shows the + // new timeline; the max-delay picture still on the old one never wins. + await run.push(MIN, marker(10, 1_066_000), at(11, 0, true)); + expect(reset).toHaveBeenCalledTimes(1); + expect(run.current()).toEqual([0, MIN]); + + // Once the max-delay decoder reads the restart too, it takes over again, without its own reset. + await run.push(MAX, marker(10, 1_066_000), at(11, 0, true)); + expect(reset).toHaveBeenCalledTimes(1); + expect(run.current()).toEqual([0, MAX]); + expect(received.mock.calls.map(([timestamp]) => timestamp)).toEqual([Milli(1000), Milli(1033), Milli(0)]); + } finally { + received.mockRestore(); + reset.mockRestore(); + await run.close(); + } + }); + + it("never shows a lower timestamp across a 100ms trailing gap and a recovery boundary", async () => { + const run = await refresh(); + try { + await run.push(MIN, at(10, 1000, true)); + let min = 1000; + let max = 900; + for (let cycle = 0; cycle < 6; cycle++) { + // Trailing by 100ms, the max-delay decoder catches up at twice the frame rate. + while (max < min) { + min += 33; + run.playback.emit(min * 1000, MIN); + run.playback.emit(max * 1000, MAX); + max = Math.min(max + 66, min); + run.playback.emit(max * 1000, MAX); + await microtasks(); + } + // Level for a few frames, the max-delay picture winning each tie. + for (let i = 0; i < 3; i++) { + min += 33; + max = min; + run.playback.emit(min * 1000, MIN); + await microtasks(); + run.playback.emit(max * 1000, MAX); + await microtasks(); + expect(run.current()).toEqual([min, MAX]); + } + // A skip leaves it 100ms behind again. + min += 100; + } + expect(run.shown.some(([, codec]) => codec === MIN)).toBe(true); + ascending(run.shown); + } finally { + await run.close(); + } + }); + + it("hands the picture back when the max-delay decoder stops producing", async () => { + const run = await refresh(); + try { + for (const lane of [MIN, MAX]) await run.push(lane, at(10, 1000, true)); + expect(run.current()).toEqual([1000, MAX]); + + await run.push(MIN, at(10, 1033)); + expect(run.current()).toEqual([1033, MIN]); + await run.push(MIN, at(10, 1066)); + expect(run.current()).toEqual([1066, MIN]); + } finally { + await run.close(); + } + }); + + // Through real container consumers: the max-delay one reaches back one more sweep for its start. + it("joins the max-delay decoder one group earlier", async () => { + const clock = manualAt(5000); + const playback = await guardedPlayback("legacy", [], [], true, 0, { + fields: { warmup: WARMUP }, + initial: Milli(100), + }); + const track = playback.track as Moq.Track.Producer; + const write = (group: Moq.Group.Producer, from: number, to: number) => { + for (let ms = from; ms <= to; ms += 30) group.writeFrame(legacyFrame(Micro.fromMilli(Milli(ms)))); + }; + const flush = async () => { + await clock.advance(0); + await microtasks(); + }; + try { + // A relay serves a cold join newest first: the live group, then the sweep before it. + write(track.createGroup(5), 1500, 1620); + await flush(); + const live = track.createGroup(4); + write(live, 1200, 1470); + live.finish(); + await flush(); + + // The min-delay decoder starts at the live group, which covers its 100ms reach. The + // max-delay one reaches 400ms back, to the group before. + expect(playback.decoded[MIN][0]).toBe(1_500_000); + expect(playback.decoded[MAX][0]).toBe(1_200_000); + expect(playback.decoded[MAX]).toContain(1_620_000); + + // The poster is the min-delay picture. The burst moved the playhead to 1520, so by the + // time the rest comes due, the max-delay decoder has decoded it too, and its clean + // pictures win. + const shown = playback.decoder.out.frame.peek() as unknown as Picture; + expect([shown.timestamp, shown.codec]).toEqual([1_500_000, MIN]); + await clock.advance(10); + await microtasks(); + const due = playback.decoder.out.frame.peek() as unknown as Picture; + expect([due.timestamp, due.codec]).toEqual([1_530_000, MAX]); + } finally { + playback.close(); + await microtasks(); + clock.installed.close(); + } + }); + + // Through real container consumers, on a live track of 300ms groups at 30ms per frame, 100ms + // behind. Group 2 stalls after 660ms while group 3 keeps arriving, until `release`. + async function stall(release: number) { + const clock = manualAt(0); + const playback = await guardedPlayback("legacy", [], [], true, 0, { + fields: { warmup: WARMUP }, + initial: Milli(100), + }); + const track = playback.track as Moq.Track.Producer; + const shown: [number, number][] = []; + const watch = new Effect(); + watch.subscribe(playback.decoder.out.frame, (frame) => { + const picture = frame as unknown as Picture | undefined; + if (picture) shown.push([picture.timestamp / 1000, picture.codec]); + }); + const groups: Moq.Group.Producer[] = []; + const write = (ms: number) => { + const sequence = Math.floor(ms / 300); + groups[sequence] ??= track.createGroup(sequence); + groups[sequence].writeFrame(legacyFrame(Micro.fromMilli(Milli(ms)))); + if (ms % 300 === 270) groups[sequence].finish(); + }; + const stalled = (ms: number) => ms > 660 && ms < 900; + try { + for (let ms = 0; ms <= 1500; ms += 10) { + if (ms % 30 === 0 && !stalled(ms)) write(ms); + if (ms === release) for (let late = 690; late < 900; late += 30) write(late); + await clock.advance(10); + await microtasks(); + } + return { decoded: playback.decoded, resets: playback.resets, shown }; + } finally { + watch.close(); + playback.close(); + await microtasks(); + clock.installed.close(); + } + } + + it("keeps the max-delay decoder continuous through a live stall shorter than one sweep", async () => { + const { decoded, resets, shown } = await stall(1000); + const stalled = [690, 720, 750, 780, 810, 840, 870].map((ms) => ms * 1000); + // The min-delay decoder skipped the rest of group 2 and restarted at group 3. + for (const ms of stalled) expect(decoded[MIN]).not.toContain(ms); + expect(resets).toBe(1); + // The max-delay decoder waited for it. + for (const ms of stalled) expect(decoded[MAX]).toContain(ms); + // Stripes showed until the max-delay picture caught up, then it took over again. + expect(shown.some(([ms, codec]) => ms >= 900 && codec === MIN)).toBe(true); + expect(shown.at(-1)?.[1]).toBe(MAX); + ascending(shown); + }); + + it("skips both decoders on a live stall longer than one sweep", async () => { + const { decoded, resets, shown } = await stall(1400); + for (const lane of [MIN, MAX]) expect(decoded[lane]).not.toContain(690_000); + expect(resets).toBe(2); + expect(shown.at(-1)?.[1]).toBe(MAX); + ascending(shown); + }); + + it("holds a leading max-delay picture when that decoder closes", async () => { + const run = await refresh(); + try { + for (const lane of [MIN, MAX]) await run.push(lane, at(10, 1000, true)); + // The min-delay decoder is slower to deliver, so the max-delay one leads. + await run.push(MAX, at(10, 1033), at(10, 1066)); + expect(run.current()).toEqual([1066, MAX]); + + // The catalog drops its warmup. The min-delay picture behind does not replace the lead. + run.playback.switchRendition("video", { warmup: Catalog.u53(0) }); + await microtasks(); + expect(run.current()).toEqual([1066, MAX]); + await run.push(MIN, at(10, 1033)); + expect(run.current()).toEqual([1066, MAX]); + await run.push(MIN, at(10, 1066), at(10, 1100)); + expect(run.current()).toEqual([1100, MIN]); + ascending(run.shown); + } finally { + await run.close(); + } + }); +}); diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 0ca3ddda970..5206a56dc1f 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -113,7 +113,7 @@ export class Decoder { readonly out = readonlys(this.#out); // The current track running, held so we can cancel it when the new track is ready. - #active = new Signal(undefined); + #active = new Signal(undefined); // The rendition preparing to replace the active one, with the catalog of the broadcast it // subscribed to. One value so a later source that reuses the track name still notifies: // the name alone would compare equal, and the jitter effect would keep the previous catalog. @@ -126,7 +126,7 @@ export class Decoder { readonly #identity: Computed; #signals = new Effect(); - #selected?: DecoderTrack; + #selected?: Rendition; #failed?: Failure; #clearCurrentFrame(): void { @@ -259,10 +259,10 @@ export class Decoder { // Only a rendition read from the same path continues the outgoing arrival estimate: another // path's arrivals have nothing to do with what this one learned. const outgoing = this.#active.peek(); - const predecessor = outgoing?.broadcast.path === active.path ? outgoing.consumer : undefined; + const predecessor = outgoing?.broadcast.path === active.path ? outgoing : undefined; // Start a new pending effect. - let pending: DecoderTrack | undefined = new DecoderTrack({ + let pending: Rendition | undefined = new Rendition({ sync: this.sync, broadcast: active, track, @@ -357,7 +357,7 @@ export class Decoder { effect.proxy(this.#spread, active.spread); } - #recordError(track: DecoderTrack, error: Error): void { + #recordError(track: Rendition, error: Error): void { // An outgoing failure must not replace the selected rendition's failure latch. if (track !== this.#selected && this.#selected?.error.peek()) return; this.#failed = { @@ -418,6 +418,9 @@ export class Decoder { interface DecoderTrackProps { sync: Sync; + // Set for the max-delay decoder of a rendition with a catalog `warmup`: its start and its + // lateness budget reach this much further, and it leaves the shared clock and stats alone. + warmup?: Getter; broadcast: Moq.Broadcast.Consumer; // The rendition ID; `name` is its track's name in `broadcast`. track: string; @@ -431,7 +434,8 @@ interface DecoderTrackProps { // arrival estimate it continues. predecessor?: Container.Consumer; - stats: Signal; + // Absent for the max-delay decoder: the bytes arrive once and the min-delay decoder counts them. + stats?: Signal; } class DecoderTrack { @@ -443,10 +447,20 @@ class DecoderTrack { config: DecoderConfig; identity: string; catalog: Getter; - stats: Signal; + stats?: Signal; timestamp = new Signal(undefined); frame = new Signal(undefined); + // Counts the timeline restarts this decoder read. A picture decoded before one is from the old + // timeline and never presents after it, even when its clock wait already resolved. + readonly generation = new Signal(0); + + // How far past the shared clock's budgets this decoder reaches. + #warmup: Getter; + // Whether this decoder drives the shared clock: only the min-delay one reports arrivals and + // restarts the timeline. + #drives: boolean; + // Network jitter + decode buffer. buffered = new Signal([]); // First decoded picture on this timeline, independent of the previously held poster. @@ -482,6 +496,8 @@ class DecoderTrack { this.identity = props.identity; this.catalog = props.catalog; this.stats = props.stats; + this.#warmup = props.warmup ?? getter(Milli.zero); + this.#drives = props.warmup === undefined; this.#predecessor = props.predecessor; this.#signals.run(this.#run.bind(this)); @@ -490,10 +506,16 @@ class DecoderTrack { // Reorders groups and frames up to the shared max delay. A rendition switch is not a cold start: // the path's arrivals are what they were, so the estimate continues from whatever read before. #consume(effect: Effect, sub: Moq.Track.Subscriber, format: Container.Format): Container.Consumer { + // The playout delay is where a cold join starts, and the shared max delay how late it may fall. + // The max-delay decoder adds the warmup to both, so it starts at the group before and keeps + // it, as Rust's does, rather than passing the consumer a `warmup` that would reach further. + const delay = new Derived([this.sync.out.delay, this.sync.out.maxDelay, this.#warmup], (min, max, warmup) => ({ + min: Milli(min + warmup), + max: Milli(max + warmup), + })); const consumer = new Container.Consumer(sub, { format, - // The playout delay is where a cold join starts, and the shared max delay how late it may fall. - delay: new Derived([this.sync.out.delay, this.sync.out.maxDelay], (min, max) => ({ min, max })), + delay, predecessor: this.consumer ?? this.#predecessor, }); effect.cleanup(() => consumer.close()); @@ -507,7 +529,8 @@ class DecoderTrack { broadcast: this.broadcast, track: this.name, priority: Catalog.PRIORITY.video, - maxDelay: this.sync.out.maxDelay, + // js/net merges both decoders' subscriptions into one request at the larger max delay. + maxDelay: new Derived([this.sync.out.maxDelay, this.#warmup], (max, warmup) => Milli(max + warmup)), }); if (!sub) return; @@ -519,6 +542,7 @@ class DecoderTrack { // A cancelled codec must not recreate the poster its track already released. Firefox can // still deliver output after close(). if (signal.aborted) return; + const generation = this.generation.peek(); const timestamp = Milli.fromMicro(frame.timestamp as Micro); if (timestamp < (this.timestamp.peek() ?? 0)) { // Late frame, don't render it. @@ -529,13 +553,17 @@ class DecoderTrack { // Render something while we wait for the sync to catch up, unless the frame is // already off the timeline: WebCodecs can hold a picture past a reset or a skip. - if (this.frame.peek() === undefined && this.sync.presentable(timestamp)) { + // Only the min-delay decoder posts one: the max-delay picture shows on the clock. + if (this.#drives && this.frame.peek() === undefined && this.sync.presentable(timestamp)) { this.frame.set(frame.clone()); } // Resolves at once when the delay is "instant". Closing the track releases it. // Check the captured signal again in case cancellation followed the resolved wait. if (!(await this.sync.wait(timestamp, signal)) || signal.aborted) return; + // The max-delay decoder restarts its timeline after the clock did, so the clock's own + // epoch cannot tell its leftover pictures apart. + if (this.generation.peek() !== generation) return; if (timestamp < (this.timestamp.peek() ?? 0)) { // Late frame, don't render it. @@ -629,14 +657,15 @@ class DecoderTrack { // resets this guard. Rejected payloads still count as received bytes. const stale = latest !== undefined && next.group < latest; const accepted = !stale && recovery.accept(frame); - this.stats.update((current) => ({ + this.stats?.update((current) => ({ frameCount: (current?.frameCount ?? 0) + (accepted ? 1 : 0), bytesReceived: (current?.bytesReceived ?? 0) + frame.payload.byteLength, })); if (!accepted) continue; - // Classify ingress before decoder backpressure changes the delivery clock. - this.sync.received(Milli.fromMicro(frame.timestamp as Micro), frame.arrival); + // Classify ingress before decoder backpressure changes the delivery clock. Only the + // min-delay decoder moves the clock: the other reads frames already behind it. + if (this.#drives) this.sync.received(Milli.fromMicro(frame.timestamp as Micro), frame.arrival); const chunk = new EncodedVideoChunk({ type: frame.keyframe ? "key" : "delta", @@ -715,14 +744,14 @@ class DecoderTrack { // resets this guard. Rejected payloads still count as received bytes. const stale = latest !== undefined && next.group < latest; const accepted = !stale && recovery.accept(frame); - this.stats.update((current) => ({ + this.stats?.update((current) => ({ frameCount: (current?.frameCount ?? 0) + (accepted ? 1 : 0), bytesReceived: (current?.bytesReceived ?? 0) + frame.payload.byteLength, })); if (!accepted) continue; - // Classify ingress before decoder backpressure changes the delivery clock. - this.sync.received(Milli.fromMicro(frame.timestamp), frame.arrival); + // Classify ingress before decoder backpressure changes the delivery clock (see #runLegacy). + if (this.#drives) this.sync.received(Milli.fromMicro(frame.timestamp), frame.arrival); // Track decode buffer (see #runLegacy: bridge only when no gap or end came between, // never on group adjacency, which proves nothing about the timeline). @@ -798,7 +827,9 @@ class DecoderTrack { this.first.set(undefined); this.timestamp.set(undefined); this.#buffered.set([]); - this.sync.reset(); + this.generation.update((generation) => generation + 1); + // The min-delay decoder restarts the shared clock. The max-delay one reads the same end later. + if (this.#drives) this.sync.reset(); this.#resetPending = true; } @@ -870,6 +901,161 @@ class DecoderTrack { } } +/** Constructor properties for {@link Rendition}. */ +interface RenditionProps extends Omit { + stats: Signal; + // The rendition on screen when this one was selected from the same path, whose decoders' arrival + // estimates these continue. + predecessor?: Rendition; +} + +/** + * One rendition's playback: a min-delay decoder, plus a max-delay decoder while the catalog gives the + * rendition a nonzero `warmup` (intra refresh, so no keyframes). + * + * The min-delay decoder holds the app's latency and drives the shared clock, so it may show refresh + * stripes after a cold join or a skip. The max-delay decoder starts one group earlier and tolerates + * the warmup more lateness, so its pictures are clean, and it decodes what is already due as fast as + * it can without moving the clock. Its picture is shown whenever its timestamp is at or past the + * min-delay one's, and nothing shown ever steps back in time. Rust players do the same in + * `moq_video::decode::Consumer`. + */ +class Rendition { + readonly error = new Signal(undefined); + readonly broadcast: Moq.Broadcast.Consumer; + readonly track: string; + readonly config: DecoderConfig; + readonly identity: string; + readonly catalog: Getter; + + // The picture chosen between the two decoders, and its timestamp once it presented on the clock: + // undefined while it is only a poster. + readonly frame = new Signal(undefined); + readonly timestamp = new Signal(undefined); + + readonly min: DecoderTrack; + readonly max = new Signal(undefined); + + // The lane picture last shown, its timestamp, and the min-delay timeline it was shown on. Nothing + // below that timestamp shows again until the timeline restarts. + #shown?: VideoFrame; + #last?: Milli; + #generation = 0; + + #signals = new Effect(); + + constructor(props: RenditionProps) { + this.broadcast = props.broadcast; + this.track = props.track; + this.config = props.config; + this.identity = props.identity; + this.catalog = props.catalog; + + const predecessor = props.predecessor; + this.min = new DecoderTrack({ ...props, predecessor: predecessor?.consumer }); + + // A missing or zero `warmup` keeps one decoder. The budgets follow a changed value without + // reopening the max-delay decoder. + const warmup = new Signal(Milli.zero); + const dual = new Signal(false); + this.#signals.run((effect) => { + const config = effect.get(props.catalog)?.video?.renditions?.[props.track]; + warmup.set(Milli(config?.warmup ?? 0)); + dual.set(warmup.peek() > 0); + }); + this.#signals.run((effect) => { + if (!effect.get(dual)) return; + const max = new DecoderTrack({ + ...props, + stats: undefined, + warmup, + predecessor: predecessor?.max.peek()?.consumer ?? predecessor?.consumer, + }); + effect.cleanup(() => max.close()); + effect.set(this.max, max); + }); + + // Either decoder failing fails the rendition, as an HEVC refresh frame sent as `key` does. + this.#signals.run((effect) => { + const max = effect.get(this.max); + const failed = effect.get(this.min.error) ?? (max && effect.get(max.error)); + if (failed) this.error.set(failed); + }); + + this.#signals.run((effect) => { + const max = effect.get(this.max); + this.#select( + { frame: effect.get(this.min.frame), generation: effect.get(this.min.generation), lane: this.min }, + max && { frame: effect.get(max.frame), generation: effect.get(max.generation), lane: max }, + ); + }); + } + + // The min-delay decoder's arrival estimate, which a replacement rendition continues. + get consumer(): Container.Consumer | undefined { + return this.min.consumer; + } + + // Startup reads the clock-driving decoder's first picture and its buffered media. + get first(): Getter { + return this.min.first; + } + + get buffered(): Getter { + return this.min.buffered; + } + + get spread(): Getter { + return this.min.spread; + } + + // Show the max-delay picture when it is at or past the min-delay one, so it wins ties, else the + // min-delay picture. A max-delay picture still on the timeline before the min-delay decoder's last + // restart is never a candidate. Not `caughtUp`: its slack, applied per picture, would alternate + // backwards and forwards. + #select(min: Lane, max: Lane | undefined): void { + // A restarted timeline begins below the previous one's pictures. + if (min.generation !== this.#generation) { + this.#generation = min.generation; + this.#last = undefined; + } + if (!min.frame) return; // Even a clean max-delay picture is a sweep behind until then. + + let pick: Lane & { frame: VideoFrame } = { ...min, frame: min.frame }; + if (max?.frame && max.generation >= min.generation && max.frame.timestamp >= min.frame.timestamp) { + pick = { ...max, frame: max.frame }; + } + if (pick.frame === this.#shown) return; + + const timestamp = Milli.fromMicro(pick.frame.timestamp as Micro); + if (this.#last !== undefined && timestamp < this.#last) return; + this.#shown = pick.frame; + this.#last = timestamp; + + this.timestamp.set(pick.lane.timestamp.peek()); + this.frame.update((prev) => { + prev?.close(); + return pick.frame.clone(); + }); + } + + close(): void { + this.#signals.close(); + this.min.close(); + this.frame.update((prev) => { + prev?.close(); + return undefined; + }); + } +} + +// One decoder's current picture, for the selector. +interface Lane { + frame: VideoFrame | undefined; + generation: number; + lane: DecoderTrack; +} + // Chrome configures a hardware H.264 decoder that then fails the first interlaced picture: field // coding only shows in the SPS, and Chrome falls back to software only when configure() fails. function hardwareAcceleration(config: { interlaced?: boolean }): HardwareAcceleration | undefined { From 38d55dc47a16d30c72f8fcd274e58e54fc44ae43 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 05:37:01 -0800 Subject: [PATCH 03/10] quest: narrow dual-decoder to the H.264 key check each engine applies Chromium accepts an SEI recovery point as a key chunk only with a zero recovery_frame_cnt, so an intra-refresh join throws there. What remains is that, and verifying Firefox and Safari. Co-Authored-By: Claude Opus 5.5 --- quest/m2/intra-refresh/README.md | 2 +- quest/m2/intra-refresh/consumer-warmup.md | 5 +- quest/m2/intra-refresh/dual-decoder.md | 105 +++++----------------- 3 files changed, 26 insertions(+), 86 deletions(-) diff --git a/quest/m2/intra-refresh/README.md b/quest/m2/intra-refresh/README.md index 4030eb12dda..40f00b0336e 100644 --- a/quest/m2/intra-refresh/README.md +++ b/quest/m2/intra-refresh/README.md @@ -49,7 +49,7 @@ make these presentation guarantees true even when sweep lengths change. ## Required -- [Dual decoder](/quest/m2/intra-refresh/dual-decoder.md) - js/watch runs a min-delay and a max-delay decoder and shows the max-delay picture whenever it has caught up +- [Dual decoder](/quest/m2/intra-refresh/dual-decoder.md) - js/watch's two decoders open on an H.264 recovery point in Chromium, Firefox, and Safari - [Plan: recovery boundaries](/quest/m2/intra-refresh/plan-boundary.md) - settle publication ordering for a later, longer sweep - [Consumer warmup](/quest/m2/intra-refresh/consumer-warmup.md) - the transcoder decodes the groups covering `warmup` as history before each group, except at a true IDR - [H.264 import](/quest/m2/intra-refresh/h264-import.md) - the splitter keeps `recovery_frame_cnt` and import publishes `warmup` from it diff --git a/quest/m2/intra-refresh/consumer-warmup.md b/quest/m2/intra-refresh/consumer-warmup.md index 1ffafd36f0d..ea90f7bc11b 100644 --- a/quest/m2/intra-refresh/consumer-warmup.md +++ b/quest/m2/intra-refresh/consumer-warmup.md @@ -6,9 +6,8 @@ The transcoder, given a source rendition with `warmup` set, never encodes a picture its decoder has not fully recovered. Every group it transcodes, whether fetched on demand for a rung or read live after a skip, is preceded by enough earlier groups decoded as history that its first picture is clean. -Players stay live with stripes instead, via the browser -[dual decoder](/quest/m2/intra-refresh/dual-decoder.md) and the native -`moq_video::decode::Consumer` pair. +Players stay live with stripes instead, via the min-delay and max-delay +decoder pairs in `@moq/watch` and the native `moq_video::decode::Consumer`. Renditions without `warmup` behave exactly as today. ## Plan diff --git a/quest/m2/intra-refresh/dual-decoder.md b/quest/m2/intra-refresh/dual-decoder.md index 9ce0326e948..7e3e5cde121 100644 --- a/quest/m2/intra-refresh/dual-decoder.md +++ b/quest/m2/intra-refresh/dual-decoder.md @@ -1,92 +1,33 @@ -# [M] Browser viewers stay live through refresh recovery with two decoders +# [S] Browser decoders join intra-refresh H.264 in every engine ## Goal -A js/watch viewer of a video rendition with `warmup` set never freezes or -waits for a refresh sweep. A min-delay decoder holds the app's latency and -may show refresh stripes right after a cold join or a skip. A max-delay -decoder tolerates one more sweep of delay, so it keeps continuity and its -picture is clean. Whenever both have a frame for the current playback time, -the max-delay picture is shown. Stripes appear only while the max-delay -decoder is catching up and both decoders are unrecovered. A rendition without `warmup` uses -one decoder exactly as today. +A js/watch viewer of an H.264 intra-refresh rendition (catalog `warmup` set) +joins and recovers from skips in Chromium, Firefox, and Safari. Each of its two +decoders (the min-delay and max-delay pair in `js/watch/src/video/decoder.ts`) +can open on a group's first frame, a recovery point that is not an IDR, and +decodes on from there, showing stripes until the sweep completes. ## Plan -Decided in the 2026-10-14 planning interview: +The two decoders, their selector, and their tests landed with this quest's +first PR ([moq-dev/moq#7555](https://github.com/moq-dev/moq/pull/7555)). +What remains is getting the opening chunk past each engine's key check. -- Replaces the freeze rule for players. Live with stripes is better than a - frozen picture. Rust players do the same in `moq_video::decode::Consumer` - (`rs/moq-video/src/decode/select.rs` holds its switch rule); only - the transcoder avoids stripes, by decoding history per - [consumer warmup](/quest/m2/intra-refresh/consumer-warmup.md). -- Budgets: the min-delay decoder skips at the app's latency target (today's - `sync.out.maxDelay`). The max-delay decoder skips at that target plus the - catalog `warmup`, which is one refresh sweep. Two decoders run, both for the - whole session, whenever the budgets differ, so `warmup` gates the feature. - A missing or zero `warmup` keeps one decoder. -- One playback clock. Only the min-delay decoder drives `Sync` (`received`, - `reset` on a break). The max-delay decoder decodes past-due frames as fast - as it can without moving the clock, so it doesn't raise latency. -- No withhold rule and no catalog precision dependency. The max-delay decoder - joins one group earlier, so the frames before its sweep completes are all - behind the playhead. They never win the timestamp comparison. A - rounded-down or stale `warmup` only shrinks its budget and costs extra - stripes, never a dirty frame the min-delay decoder wasn't also showing. - This quest therefore does not wait on - [plan-boundary](/quest/m2/intra-refresh/plan-boundary.md); the warmup draft - amendment it needed landed in [#6494](https://github.com/moq-dev/moq/pull/6494). -- Selection is a small frame selector over the two current candidates: show - the max-delay frame when its timestamp is at or beyond the min-delay frame's, - so it wins ties, else the min-delay frame. Separately, never display a - timestamp below the last one shown. A max-delay decoder that stops producing - therefore loses as soon as the min-delay one advances. Do not reuse `caughtUp` from - `js/watch/src/video/playhead.ts`: its 100 ms slack tolerates one backwards - step per rendition switch, and applied per frame it would alternate - backwards and forwards. Keep the pair per rendition, inside or beside - `DecoderTrack` in `js/watch/src/video/decoder.ts`, so rendition switching - still promotes a whole pair through the existing pending/active path. -- Each decoder opens its own subscription through `subscribeMedia` - (`js/watch/src/media.ts`); a `Container.Consumer` consumes its subscriber. - `js/net` dedups the two into one upstream request with the larger - `maxDelay` (`combineSubscriptions`, `js/net/src/track.ts`), so live groups - arrive once. The second decoder's only extra transfer is the earlier group it - joins on, up to one refresh group per cold join or gap. -- The max-delay decoder reads groups oldest first. Subscriptions default to - descending order and aggregate that way, so a cold join can activate the - newest group in `Container.Consumer` first, and `DecoderTrack` then drops - the earlier group as stale. That would leave the max-delay decoder starting - mid-sweep. Its consumer must wait for the lower group within its budget - before emitting a newer one. ✅ Decided 2026-10-30 (maintainer): a - `Container.Consumer` takes `delay: { min, max }` (ms) and starts at the - group covering the live edge less `min` (the playout delay) plus its - `warmup` prop (the catalog's name and units), waiting for it under that - reach. `max` (`null` for no limit) only bounds lateness and never moves the - start; a `min` past `max` starts at `max`. All default to zero, which - starts at the newest group at once. This decoder passes the playout delay - plus the catalog `warmup` as `min`, and the shared max delay plus the - `warmup` as `max`, so its start reaches the earlier group without the - `warmup` prop, as Rust's max-delay decoder does - ([moq-dev/moq#7311](https://github.com/moq-dev/moq/pull/7311)). -- H.264 recovery-point frames sent as `key` chunks pass Chromium's WebCodecs - keyframe check (`ParseH264KeyFrame` accepts `is_sei_recovery_point`; WPT - `videoDecoder-h264-sei`). Chromium rejects a non-IRAP H.265 frame sent as - `key`, so an HEVC refresh rendition fails through the decoder's existing - `error` signal; no special case. Firefox and Safari behavior is unknown: - check H.264 on both during verification. -- Tests in `js/watch/src/video/decoder.test.ts` with a synthetic `warmup` - catalog and track: a cold join shows the min-delay picture and then switches - to max-delay once it is caught up; a stall shorter than one sweep makes the - min-delay decoder skip while max-delay stays continuous and takes over again; - a stall longer than one sweep skips both; a rendition without `warmup` opens - one subscription and one decoder; only the min-delay decoder moves `Sync`; - a selector regression that repeatedly crosses a 100 ms trailing gap and a - recovery boundary never displays a lower timestamp; a max-delay decoder that - stops producing while the min-delay one continues hands the picture back on - the next min-delay frame. -- Measure the cost of always running two decoders: CPU, decoder memory, and - dropped frames for one versus two decoders on a 1080p rendition, in a - browser, recorded in the PR. +- Chromium (verified on 156): WebCodecs accepts an H.264 chunk sent as `key` + only when it is an IDR or carries an SEI recovery point with + `recovery_frame_cnt == 0` (`media/formats/mp4/avc.cc`, `AnalyzeAnnexB`). An + intra-refresh recovery point counts a whole sweep, so `decode()` throws + "A key frame is required" and the rendition fails through `error`. Adding a + second SEI recovery point with a zero count ahead of the first slice of that + one chunk makes Chromium accept it, and it outputs from that frame on. + Open question for the maintainer: how js/watch handles this (see the PR's + `## Decisions needed`). +- Firefox and Safari: unverified. Playwright's Firefox build reports H.264 + unsupported and its WebKit build would not launch on the Linux host. Check + both with a real browser, joining at a non-IDR recovery point sent as `key`. +- HEVC: Chromium rejects a non-IRAP H.265 frame sent as `key`, so an HEVC + refresh rendition fails through the decoder's existing `error` signal. ## Related From 7775e5a3e92802b2d2564e43a904adaf38731908 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 05:51:12 -0800 Subject: [PATCH 04/10] fix(watch): place each decoder's picture on a timeline by its group A picture the min-delay decoder held from before a publisher restart could seed the no-rewind floor and freeze the new timeline, and a max-delay decoder opened after a restart never matched the min-delay decoder's local restart count. Each picture now carries the group it was read from, and only pictures from the min-delay decoder's current timeline start on are candidates. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/video/decoder.test.ts | 42 +++++++++++++++++++- js/watch/src/video/decoder.ts | 63 +++++++++++++++++++----------- 2 files changed, 80 insertions(+), 25 deletions(-) diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index edca4eae2a9..ee044697547 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -1414,9 +1414,10 @@ describe("Decoder with a warmup rendition", () => { const at = (group: number, ms: number, key = false) => sample(group, ms * 1000, key); // A warmup rendition whose two container consumers each read their own feed, presenting at once. - async function refresh(fields: Partial = { warmup: WARMUP }) { + // `hold` keeps those timestamps in the codec until `release`. + async function refresh(fields: Partial = { warmup: WARMUP }, hold: number[] = []) { const feeds = new Feeds(); - const playback = await guardedPlayback("legacy", [], [], false, 0, { fields, feeds }); + const playback = await guardedPlayback("legacy", [], hold, false, 0, { fields, feeds }); // Every picture shown, as [ms, codec]. const shown: [number, number][] = []; const watch = new Effect(); @@ -1578,6 +1579,43 @@ describe("Decoder with a warmup rendition", () => { } }); + it("keeps a picture held from before a restart out of the new timeline", async () => { + const run = await refresh(undefined, [0]); + try { + for (const lane of [MIN, MAX]) await run.push(lane, at(10, 1000, true)); + expect(run.current()).toEqual([1000, MAX]); + + // The min-delay decoder reads a restart, but its first new picture is still in the codec. + // The picture it holds from the old timeline must not show, nor block the new one. + await run.push(MIN, marker(10, 1_066_000), at(11, 0, true)); + expect(run.current()).toEqual([1000, MAX]); + run.playback.release(); + await microtasks(); + expect(run.current()).toEqual([0, MIN]); + } finally { + await run.close(); + } + }); + + it("takes the max-delay picture when that decoder opens after a restart", async () => { + const run = await refresh({}); + try { + expect(run.playback.codecs).toBe(1); + await run.push(MIN, at(10, 1000, true), marker(10, 1_066_000), at(11, 0, true), at(11, 33)); + expect(run.current()).toEqual([33, MIN]); + + // The catalog gains a warmup. The new decoder never saw the restart, yet reads the same timeline. + run.playback.switchRendition("video", { warmup: WARMUP }); + await microtasks(); + expect(run.playback.codecs).toBe(2); + await run.push(MAX, at(11, 0, true), at(11, 33), at(11, 66)); + await run.push(MIN, at(11, 66)); + expect(run.current()).toEqual([66, MAX]); + } finally { + await run.close(); + } + }); + it("never shows a lower timestamp across a 100ms trailing gap and a recovery boundary", async () => { const run = await refresh(); try { diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 5206a56dc1f..43ae6104059 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -451,9 +451,17 @@ class DecoderTrack { timestamp = new Signal(undefined); frame = new Signal(undefined); + // The group the current `frame` was read from. Group sequences are shared by every subscription to + // the track, so they place a picture on a timeline for either decoder. + group?: number; + // The first group of the timeline this decoder reads, undefined until a publisher restart. + readonly start = new Signal(undefined); + // Counts the timeline restarts this decoder read. A picture decoded before one is from the old // timeline and never presents after it, even when its clock wait already resolved. - readonly generation = new Signal(0); + #generation = 0; + // The group each frame in the codec was read from, by timestamp. + #groups = new Map(); // How far past the shared clock's budgets this decoder reaches. #warmup: Getter; @@ -542,7 +550,9 @@ class DecoderTrack { // A cancelled codec must not recreate the poster its track already released. Firefox can // still deliver output after close(). if (signal.aborted) return; - const generation = this.generation.peek(); + const generation = this.#generation; + const group = this.#groups.get(frame.timestamp); + this.#groups.delete(frame.timestamp); const timestamp = Milli.fromMicro(frame.timestamp as Micro); if (timestamp < (this.timestamp.peek() ?? 0)) { // Late frame, don't render it. @@ -555,6 +565,7 @@ class DecoderTrack { // already off the timeline: WebCodecs can hold a picture past a reset or a skip. // Only the min-delay decoder posts one: the max-delay picture shows on the clock. if (this.#drives && this.frame.peek() === undefined && this.sync.presentable(timestamp)) { + this.group = group; this.frame.set(frame.clone()); } @@ -563,7 +574,7 @@ class DecoderTrack { if (!(await this.sync.wait(timestamp, signal)) || signal.aborted) return; // The max-delay decoder restarts its timeline after the clock did, so the clock's own // epoch cannot tell its leftover pictures apart. - if (this.generation.peek() !== generation) return; + if (this.#generation !== generation) return; if (timestamp < (this.timestamp.peek() ?? 0)) { // Late frame, don't render it. @@ -576,6 +587,7 @@ class DecoderTrack { // Trim the decode buffer as frames are rendered this.#trimBuffered(timestamp); + this.group = group; this.frame.update((prev) => { prev?.close(); return frame.clone(); // avoid closing the frame here @@ -686,6 +698,7 @@ class DecoderTrack { latest = next.group; this.#resetAtKeyframe(decoder, frame.keyframe); + this.#groups.set(chunk.timestamp, next.group); decoder.decode(chunk); if (!this.sync.playing.peek()) await this.#waitStartup(effect, Milli.fromMicro(frame.timestamp as Micro)); @@ -764,6 +777,7 @@ class DecoderTrack { if (decoder.state === "closed") break; latest = next.group; this.#resetAtKeyframe(decoder, frame.keyframe); + this.#groups.set(frame.timestamp, next.group); decoder.decode( new EncodedVideoChunk({ type: frame.keyframe ? "key" : "delta", @@ -817,17 +831,18 @@ class DecoderTrack { if (latest === undefined || read.group >= latest) this.#endGroup = read.group; } else if (this.#endGroup !== undefined && read.group > this.#endGroup) { this.#endGroup = undefined; - this.#onBreak(); + this.#onBreak(read.group); } } // A fresh timeline invalidates parked pictures and its previous decoded span. The held picture // stays until the new keyframe renders; the codec resets before that keyframe is decoded. - #onBreak(): void { + #onBreak(group: number): void { this.first.set(undefined); this.timestamp.set(undefined); this.#buffered.set([]); - this.generation.update((generation) => generation + 1); + this.#generation++; + this.start.set(group); // The min-delay decoder restarts the shared clock. The max-delay one reads the same end later. if (this.#drives) this.sync.reset(); this.#resetPending = true; @@ -843,6 +858,7 @@ class DecoderTrack { // A closed decoder throws on reset. if (!keyframe || decoder.state === "closed") return; decoder.reset(); + this.#groups.clear(); this.#configure(decoder); } @@ -936,11 +952,11 @@ class Rendition { readonly min: DecoderTrack; readonly max = new Signal(undefined); - // The lane picture last shown, its timestamp, and the min-delay timeline it was shown on. Nothing - // below that timestamp shows again until the timeline restarts. + // The lane picture last shown, its timestamp, and the first group of the min-delay timeline it was + // shown on. Nothing below that timestamp shows again until the timeline restarts. #shown?: VideoFrame; #last?: Milli; - #generation = 0; + #start?: number; #signals = new Effect(); @@ -985,8 +1001,9 @@ class Rendition { this.#signals.run((effect) => { const max = effect.get(this.max); this.#select( - { frame: effect.get(this.min.frame), generation: effect.get(this.min.generation), lane: this.min }, - max && { frame: effect.get(max.frame), generation: effect.get(max.generation), lane: max }, + { frame: effect.get(this.min.frame), lane: this.min }, + max && { frame: effect.get(max.frame), lane: max }, + effect.get(this.min.start), ); }); } @@ -1010,21 +1027,22 @@ class Rendition { } // Show the max-delay picture when it is at or past the min-delay one, so it wins ties, else the - // min-delay picture. A max-delay picture still on the timeline before the min-delay decoder's last - // restart is never a candidate. Not `caughtUp`: its slack, applied per picture, would alternate - // backwards and forwards. - #select(min: Lane, max: Lane | undefined): void { + // min-delay picture. Only pictures read from `start` on, the min-delay decoder's timeline, are + // candidates: one held from before a restart neither shows nor sets the floor. Not `caughtUp`: its + // slack, applied per picture, would alternate backwards and forwards. + #select(min: Lane, max: Lane | undefined, start: number | undefined): void { // A restarted timeline begins below the previous one's pictures. - if (min.generation !== this.#generation) { - this.#generation = min.generation; + if (start !== this.#start) { + this.#start = start; this.#last = undefined; } - if (!min.frame) return; // Even a clean max-delay picture is a sweep behind until then. + const current = (lane: Lane | undefined): lane is Lane & { frame: VideoFrame } => + lane?.frame !== undefined && (start === undefined || (lane.lane.group ?? -1) >= start); + // Even a clean max-delay picture is a sweep behind until the min-delay decoder has one. + if (!current(min)) return; - let pick: Lane & { frame: VideoFrame } = { ...min, frame: min.frame }; - if (max?.frame && max.generation >= min.generation && max.frame.timestamp >= min.frame.timestamp) { - pick = { ...max, frame: max.frame }; - } + let pick = min; + if (current(max) && max.frame.timestamp >= min.frame.timestamp) pick = max; if (pick.frame === this.#shown) return; const timestamp = Milli.fromMicro(pick.frame.timestamp as Micro); @@ -1052,7 +1070,6 @@ class Rendition { // One decoder's current picture, for the selector. interface Lane { frame: VideoFrame | undefined; - generation: number; lane: DecoderTrack; } From a003ac15b10f31cd4fd439d56f72a4bfb491165d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 13:41:42 -0800 Subject: [PATCH 05/10] fix(watch): open an intra-refresh H.264 codec past Chromium's key check Chromium's WebCodecs accepts an H.264 chunk that opens a codec only when it is an IDR or carries an SEI recovery point with recovery_frame_cnt == 0. An intra-refresh recovery point counts a whole sweep, so every join and skip of a warmup rendition failed. On the chunk that opens a codec, when the rendition has warmup and the chunk has no IDR, insert a zero-count recovery point before the first slice. Co-Authored-By: Claude Opus 5.5 --- doc/lib/js/watch.md | 7 ++ js/watch/src/video/decoder.test.ts | 148 ++++++++++++++++++++++++- js/watch/src/video/decoder.ts | 122 ++++++++++++++++---- quest/m2/intra-refresh/README.md | 2 +- quest/m2/intra-refresh/dual-decoder.md | 27 +++-- 5 files changed, 270 insertions(+), 36 deletions(-) diff --git a/doc/lib/js/watch.md b/doc/lib/js/watch.md index 93958feaa18..465f73dda1e 100644 --- a/doc/lib/js/watch.md +++ b/doc/lib/js/watch.md @@ -62,6 +62,13 @@ picture shows whenever it has caught up. Each decoder costs its own CPU and decoder memory, and both share one subscription. A rendition without `warmup` uses one decoder. +An H.264 group of such a rendition opens on a recovery point that is not an +IDR. Chromium's WebCodecs refuses to open a decoder on that picture: it wants +an IDR or a recovery point that counts zero frames. So when a decoder opens on +one, the player inserts a second recovery point with a zero count before its +first slice. The slices are untouched; the stripes clear as the sweep +completes. + `el.broadcast.out.status` is `offline`, `loading`, `live`, `error`, or `unsupported`. On `unsupported`, update the player to read the broadcast's newer catalog; the overlay shows this notice. On `error` the origin refused the broadcast, `el.broadcast.out.error` says why, diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index ee044697547..ea4bf4d83f8 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -9,7 +9,7 @@ import { ringSamples } from "../audio/latency"; import { AudioRingBuffer } from "../audio/ring-buffer"; import type { Broadcast } from "../broadcast"; import { type Delay, Sync } from "../sync"; -import { Decoder } from "./decoder"; +import { Decoder, insertRecoveryPoint } from "./decoder"; import { Source } from "./source"; const flush = () => yieldNow(); @@ -191,14 +191,14 @@ function manualAt(ms: number) { clock.jump(Nano.fromMilli(Milli(ms))); return { advance: (by: number) => clock.advance(Nano.fromMilli(Milli(by))), installed: clock.install() }; } -function sample(group: number, timestamp: number, keyframe = true): Read { +function sample(group: number, timestamp: number, keyframe = true, payload: Uint8Array = new Uint8Array([1])): Read { return { kind: "frame", group, frame: { arrival: Nano.toMilli(Nano(now())), timestamp: Micro(timestamp), - payload: new Uint8Array([1]), + payload, keyframe, }, }; @@ -254,8 +254,9 @@ async function guardedPlayback( const originalDecoder = Object.getOwnPropertyDescriptor(globalThis, "VideoDecoder"); const originalChunk = Object.getOwnPropertyDescriptor(globalThis, "EncodedVideoChunk"); const submitted: number[] = []; - // What each codec, in creation order, was given. + // What each codec, in creation order, was given, and the bytes of each chunk. const decoded: number[][] = []; + const payloads: Uint8Array[][] = []; const outputs: number[] = []; const pictures: Picture[] = []; const owed = new Set(hold); @@ -283,6 +284,7 @@ async function guardedPlayback( constructor(callbacks: VideoDecoderInit) { this.callbacks = callbacks; decoded.push([]); + payloads.push([]); errors.push(callbacks.error); frames.push(callbacks.output); } @@ -300,6 +302,7 @@ async function guardedPlayback( this.#keyRequired = false; submitted.push(chunk.timestamp); decoded[this.index].push(chunk.timestamp); + payloads[this.index].push((chunk as unknown as { data: Uint8Array }).data); const emit = () => { outputs.push(chunk.timestamp); const picture = new Picture(chunk.timestamp, pictures, this.index); @@ -328,9 +331,11 @@ async function guardedPlayback( value: class { timestamp: number; type: EncodedVideoChunkType; + data: Uint8Array; constructor(init: EncodedVideoChunkInit) { this.timestamp = init.timestamp; this.type = init.type; + this.data = new Uint8Array(init.data as Uint8Array); } }, }); @@ -396,6 +401,7 @@ async function guardedPlayback( track, submitted, decoded, + payloads, outputs, pictures, sync, @@ -1796,3 +1802,137 @@ describe("Decoder with a warmup rendition", () => { } }); }); + +// H.264 NAL units, without their start code or length prefix. +const NAL = { + aud: [0x09, 0xf0], + sps: [0x67, 0x64, 0x00, 0x28], + pps: [0x68, 0xee], + // An SEI recovery point counting one 30-frame sweep, as an intra-refresh encoder writes it. + sweep: [0x06, 0x06, 0x02, 0x0f, 0x84, 0x80], + // The one the decoder inserts: recovery_frame_cnt = 0. + recovered: [0x06, 0x06, 0x01, 0x84, 0x80], + slice: [0x41, 0x9a, 0x02], + idr: [0x65, 0x88, 0x84], +}; + +// Length-prefixed with `size` bytes, as an avcC `description` declares. +function avcc(nals: number[][], size = 4): Uint8Array { + const out: number[] = []; + for (const nal of nals) { + for (let i = size - 1; i >= 0; i--) out.push((nal.length >> (8 * i)) & 0xff); + out.push(...nal); + } + return new Uint8Array(out); +} + +function annexB(nals: number[][], code = [0, 0, 0, 1]): Uint8Array { + return new Uint8Array(nals.flatMap((nal) => [...code, ...nal])); +} + +// avcC records whose lengthSizeMinusOne is 3 and 1. +const AVCC4 = new Uint8Array([0x01, 0x64, 0x00, 0x28, 0xff]); +const AVCC2 = new Uint8Array([0x01, 0x64, 0x00, 0x28, 0xfd]); + +describe("insertRecoveryPoint", () => { + it("inserts a zero-count recovery point before the first slice of a length-prefixed recovery point", () => { + const payload = avcc([NAL.sweep, NAL.slice, NAL.slice]); + expect(insertRecoveryPoint(payload, AVCC4)).toEqual(avcc([NAL.sweep, NAL.recovered, NAL.slice, NAL.slice])); + expect(insertRecoveryPoint(avcc([NAL.slice], 2), AVCC2)).toEqual(avcc([NAL.recovered, NAL.slice], 2)); + }); + + it("inserts it before the first slice of an Annex B recovery point", () => { + // The slice's 4-byte start code keeps its leading zero byte, which now leads the inserted unit. + expect(insertRecoveryPoint(annexB([NAL.aud, NAL.sweep, NAL.slice]))).toEqual( + new Uint8Array([...annexB([NAL.aud, NAL.sweep]), 0, ...annexB([NAL.recovered, NAL.slice], [0, 0, 1])]), + ); + const short = [0, 0, 1]; + expect(insertRecoveryPoint(annexB([NAL.sps, NAL.pps, NAL.slice], short))).toEqual( + annexB([NAL.sps, NAL.pps, NAL.recovered, NAL.slice], short), + ); + }); + + it("leaves an IDR, a payload without a slice, and a malformed payload unchanged", () => { + for (const [payload, description] of [ + [avcc([NAL.sps, NAL.pps, NAL.idr]), AVCC4], + [annexB([NAL.sps, NAL.pps, NAL.idr]), undefined], + [avcc([NAL.sps, NAL.pps]), AVCC4], + [annexB([NAL.sweep]), undefined], + // A length running past the payload, and a truncated description. + [avcc([NAL.slice]).subarray(0, 6), AVCC4], + [avcc([NAL.slice]), new Uint8Array([0x01])], + ] as const) { + expect(insertRecoveryPoint(payload, description)).toBe(payload); + } + }); +}); + +for (const kind of ["legacy", "cmaf"] as const) { + describe(`Decoder opening a ${kind} warmup rendition`, () => { + const MIN = 0; + const MAX = 1; + const recovery = avcc([NAL.sweep, NAL.slice]); + const patched = avcc([NAL.sweep, NAL.recovered, NAL.slice]); + const delta = avcc([NAL.slice]); + const idr = avcc([NAL.sps, NAL.pps, NAL.idr]); + const at = (group: number, ms: number, key: boolean, payload: Uint8Array) => + sample(group, ms * 1000, key, payload); + + async function open(fields: Partial) { + const feeds = new Feeds(); + const playback = await guardedPlayback(kind, [], [], false, 0, { fields, feeds }); + return { + playback, + push: async (lane: number, ...reads: Read[]) => { + feeds.push(lane, ...reads); + await microtasks(); + }, + }; + } + + it("rewrites only the recovery point that opens each codec", async () => { + const { playback, push } = await open({ warmup: Catalog.u53(300) }); + try { + expect(playback.codecs).toBe(2); + for (const lane of [MIN, MAX]) { + await push(lane, at(10, 1000, true, recovery), at(10, 1033, false, delta)); + // The next group continues the open codec, so Chromium does not check it. + await push(lane, at(11, 1300, true, recovery)); + } + for (const lane of [MIN, MAX]) expect(playback.payloads[lane]).toEqual([patched, delta, recovery]); + + // A skip resets the min-delay codec, so the group it reopens on is rewritten again. + await push(MIN, gap, at(13, 1900, true, recovery)); + expect(playback.resets).toBe(1); + expect(playback.payloads[MIN].at(-1)).toEqual(patched); + } finally { + playback.close(); + await microtasks(); + } + }); + + it("leaves an opening IDR unchanged", async () => { + const { playback, push } = await open({ warmup: Catalog.u53(300) }); + try { + for (const lane of [MIN, MAX]) await push(lane, at(10, 1000, true, idr)); + for (const lane of [MIN, MAX]) expect(playback.payloads[lane]).toEqual([idr]); + } finally { + playback.close(); + await microtasks(); + } + }); + + it("leaves a rendition without a warmup unchanged", async () => { + const { playback, push } = await open({}); + try { + expect(playback.codecs).toBe(1); + await push(MIN, at(10, 1000, true, recovery), gap, at(13, 1900, true, recovery)); + expect(playback.resets).toBe(1); + expect(playback.payloads[MIN]).toEqual([recovery, recovery]); + } finally { + playback.close(); + await microtasks(); + } + }); + }); +} diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 43ae6104059..d437bd6f0ce 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -421,6 +421,9 @@ interface DecoderTrackProps { // Set for the max-delay decoder of a rendition with a catalog `warmup`: its start and its // lateness budget reach this much further, and it leaves the shared clock and stats alone. warmup?: Getter; + // Whether the rendition has a catalog `warmup`, so its groups may open on a recovery point that + // is not an IDR. Set for both decoders. + refresh?: Getter; broadcast: Moq.Broadcast.Consumer; // The rendition ID; `name` is its track's name in `broadcast`. track: string; @@ -468,6 +471,7 @@ class DecoderTrack { // Whether this decoder drives the shared clock: only the min-delay one reports arrivals and // restarts the timeline. #drives: boolean; + #refresh: Getter; // Network jitter + decode buffer. buffered = new Signal([]); @@ -482,6 +486,8 @@ class DecoderTrack { // Codec description from the last configure, reused after a playhead reset. #description?: Uint8Array; + // Whether the next chunk opens the codec, after a configure or a reset. + #opening = false; // A hole or timeline restart makes the next decoded frame decide whether the codec resets. #resetPending = false; @@ -506,6 +512,7 @@ class DecoderTrack { this.stats = props.stats; this.#warmup = props.warmup ?? getter(Milli.zero); this.#drives = props.warmup === undefined; + this.#refresh = props.refresh ?? getter(false); this.#predecessor = props.predecessor; this.#signals.run(this.#run.bind(this)); @@ -679,12 +686,6 @@ class DecoderTrack { // min-delay decoder moves the clock: the other reads frames already behind it. if (this.#drives) this.sync.received(Milli.fromMicro(frame.timestamp as Micro), frame.arrival); - const chunk = new EncodedVideoChunk({ - type: frame.keyframe ? "key" : "delta", - data: frame.payload, - timestamp: frame.timestamp, - }); - // Track decode buffer: frames sent to decoder but not yet rendered. Only bridge from // the previous frame when no gap or end came between them. Group ids can't answer that: // they aren't required to be sequential (some encoders derive them from DTS), so @@ -698,8 +699,8 @@ class DecoderTrack { latest = next.group; this.#resetAtKeyframe(decoder, frame.keyframe); - this.#groups.set(chunk.timestamp, next.group); - decoder.decode(chunk); + this.#groups.set(frame.timestamp, next.group); + decoder.decode(this.#chunk(frame)); if (!this.sync.playing.peek()) await this.#waitStartup(effect, Milli.fromMicro(frame.timestamp as Micro)); } @@ -778,13 +779,7 @@ class DecoderTrack { latest = next.group; this.#resetAtKeyframe(decoder, frame.keyframe); this.#groups.set(frame.timestamp, next.group); - decoder.decode( - new EncodedVideoChunk({ - type: frame.keyframe ? "key" : "delta", - data: frame.payload, - timestamp: frame.timestamp, - }), - ); + decoder.decode(this.#chunk(frame)); if (!this.sync.playing.peek()) await this.#waitStartup(effect, Milli.fromMicro(frame.timestamp)); } }); @@ -862,7 +857,20 @@ class DecoderTrack { this.#configure(decoder); } + // The chunk for `frame`, run through the opening rewrite when it opens the codec. + #chunk(frame: Container.ReceivedFrame): EncodedVideoChunk { + let data = frame.payload; + if (this.#opening) { + this.#opening = false; + if (frame.keyframe && this.#refresh.peek() && isH264(this.config.codec)) { + data = insertRecoveryPoint(data, this.#description); + } + } + return new EncodedVideoChunk({ type: frame.keyframe ? "key" : "delta", data, timestamp: frame.timestamp }); + } + #configure(decoder: VideoDecoder): void { + this.#opening = true; decoder.configure({ codec: this.config.codec, description: this.#description, @@ -918,7 +926,7 @@ class DecoderTrack { } /** Constructor properties for {@link Rendition}. */ -interface RenditionProps extends Omit { +interface RenditionProps extends Omit { stats: Signal; // The rendition on screen when this one was selected from the same path, whose decoders' arrival // estimates these continue. @@ -967,13 +975,14 @@ class Rendition { this.identity = props.identity; this.catalog = props.catalog; - const predecessor = props.predecessor; - this.min = new DecoderTrack({ ...props, predecessor: predecessor?.consumer }); - // A missing or zero `warmup` keeps one decoder. The budgets follow a changed value without // reopening the max-delay decoder. const warmup = new Signal(Milli.zero); const dual = new Signal(false); + + const predecessor = props.predecessor; + this.min = new DecoderTrack({ ...props, refresh: dual, predecessor: predecessor?.consumer }); + this.#signals.run((effect) => { const config = effect.get(props.catalog)?.video?.renditions?.[props.track]; warmup.set(Milli(config?.warmup ?? 0)); @@ -985,6 +994,7 @@ class Rendition { ...props, stats: undefined, warmup, + refresh: dual, predecessor: predecessor?.max.peek()?.consumer ?? predecessor?.consumer, }); effect.cleanup(() => max.close()); @@ -1073,6 +1083,80 @@ interface Lane { lane: DecoderTrack; } +function isH264(codec: string): boolean { + return codec.startsWith("avc1.") || codec.startsWith("avc3."); +} + +// An SEI NAL unit (type 6) holding one recovery point message (payload type 6, one byte): +// recovery_frame_cnt = 0, exact_match_flag = 0, broken_link_flag = 0, changing_slice_group_idc = 0, +// then payload alignment and the RBSP trailing bits. It needs no emulation prevention. +const RECOVERY_SEI = new Uint8Array([0x06, 0x06, 0x01, 0x84, 0x80]); + +/** + * Insert an SEI recovery point with `recovery_frame_cnt = 0` before the first slice of an H.264 + * access unit that has no IDR, so a codec can open on it. `description` is the avcC record when the + * payload is length-prefixed, and absent for Annex B. An IDR, a payload without a slice, or one that + * does not parse comes back unchanged. + * + * Chromium's WebCodecs checks the chunk that opens a codec, after `configure()` or `reset()`: an + * H.264 chunk sent as `key` must hold an IDR, or an SEI recovery point with `recovery_frame_cnt == + * 0` (`AnalyzeAnnexB` in `media/formats/mp4/avc.cc`). Otherwise `decode()` throws "A key frame is + * required". An intra-refresh group opens on a recovery point that counts a whole sweep, so every + * join and skip of a `warmup` rendition would fail. The extra message declares that one picture + * recovered, which is what a `warmup` rendition already accepts: the min-delay decoder shows the + * refresh stripes until the sweep completes, and the max-delay decoder covers them. The slices + * are unchanged; the message goes right before the first one, after any SEI the stream carries. + * @internal + */ +export function insertRecoveryPoint(payload: Uint8Array, description?: Uint8Array): Uint8Array { + if (description) { + // lengthSizeMinusOne, in the low two bits of the avcC record's fifth byte. + if (description.byteLength < 5) return payload; + const size = (description[4] & 0x03) + 1; + for (let offset = 0; offset + size <= payload.byteLength; ) { + let length = 0; + for (let i = 0; i < size; i++) length = length * 256 + payload[offset + i]; + const start = offset + size; + if (length === 0 || start + length > payload.byteLength) return payload; + const type = payload[start] & 0x1f; + if (type === 5) return payload; + if (type >= 1 && type <= 4) { + const prefix = new Uint8Array(size); + for (let i = 0, n = RECOVERY_SEI.byteLength; i < size; i++, n = Math.floor(n / 256)) { + prefix[size - 1 - i] = n % 256; + } + return splice(payload, offset, prefix, RECOVERY_SEI); + } + offset = start + length; + } + return payload; + } + + // Annex B: each NAL unit follows a 00 00 01 start code, with an optional leading zero byte. + for (let i = 0; i + 3 < payload.byteLength; i++) { + if (payload[i] !== 0 || payload[i + 1] !== 0 || payload[i + 2] !== 1) continue; + const type = payload[i + 3] & 0x1f; + if (type === 5) return payload; + if (type >= 1 && type <= 4) return splice(payload, i, new Uint8Array([0, 0, 1]), RECOVERY_SEI); + i += 2; + } + return payload; +} + +// `payload` with `parts` inserted at `at`. +function splice(payload: Uint8Array, at: number, ...parts: Uint8Array[]): Uint8Array { + const length = parts.reduce((sum, part) => sum + part.byteLength, payload.byteLength); + const out = new Uint8Array(length); + out.set(payload.subarray(0, at), 0); + let offset = at; + for (const part of parts) { + out.set(part, offset); + offset += part.byteLength; + } + out.set(payload.subarray(at), offset); + return out; +} + // Chrome configures a hardware H.264 decoder that then fails the first interlaced picture: field // coding only shows in the SPS, and Chrome falls back to software only when configure() fails. function hardwareAcceleration(config: { interlaced?: boolean }): HardwareAcceleration | undefined { diff --git a/quest/m2/intra-refresh/README.md b/quest/m2/intra-refresh/README.md index 40f00b0336e..c921a1a8932 100644 --- a/quest/m2/intra-refresh/README.md +++ b/quest/m2/intra-refresh/README.md @@ -49,7 +49,7 @@ make these presentation guarantees true even when sweep lengths change. ## Required -- [Dual decoder](/quest/m2/intra-refresh/dual-decoder.md) - js/watch's two decoders open on an H.264 recovery point in Chromium, Firefox, and Safari +- [Dual decoder](/quest/m2/intra-refresh/dual-decoder.md) - js/watch's two decoders open on an H.264 recovery point in Firefox and Safari, as they now do in Chromium - [Plan: recovery boundaries](/quest/m2/intra-refresh/plan-boundary.md) - settle publication ordering for a later, longer sweep - [Consumer warmup](/quest/m2/intra-refresh/consumer-warmup.md) - the transcoder decodes the groups covering `warmup` as history before each group, except at a true IDR - [H.264 import](/quest/m2/intra-refresh/h264-import.md) - the splitter keeps `recovery_frame_cnt` and import publishes `warmup` from it diff --git a/quest/m2/intra-refresh/dual-decoder.md b/quest/m2/intra-refresh/dual-decoder.md index 7e3e5cde121..cb551299000 100644 --- a/quest/m2/intra-refresh/dual-decoder.md +++ b/quest/m2/intra-refresh/dual-decoder.md @@ -10,22 +10,25 @@ decodes on from there, showing stripes until the sweep completes. ## Plan -The two decoders, their selector, and their tests landed with this quest's -first PR ([moq-dev/moq#7555](https://github.com/moq-dev/moq/pull/7555)). -What remains is getting the opening chunk past each engine's key check. +The two decoders, their selector, and the Chromium join landed with this +quest's first PR ([moq-dev/moq#7555](https://github.com/moq-dev/moq/pull/7555)). +What remains is checking the other engines. -- Chromium (verified on 156): WebCodecs accepts an H.264 chunk sent as `key` - only when it is an IDR or carries an SEI recovery point with +- Chromium (done, verified on 156): WebCodecs accepts an H.264 chunk that opens + a codec only when it is an IDR or carries an SEI recovery point with `recovery_frame_cnt == 0` (`media/formats/mp4/avc.cc`, `AnalyzeAnnexB`). An - intra-refresh recovery point counts a whole sweep, so `decode()` throws - "A key frame is required" and the rendition fails through `error`. Adding a - second SEI recovery point with a zero count ahead of the first slice of that - one chunk makes Chromium accept it, and it outputs from that frame on. - Open question for the maintainer: how js/watch handles this (see the PR's - `## Decisions needed`). + intra-refresh recovery point counts a whole sweep. + ✅ Decided 2026-11-01 (option A): on the chunk that opens a codec (after + `configure` or `reset`), when the rendition has `warmup` and the chunk is + H.264 with no IDR, `insertRecoveryPoint` in `decoder.ts` inserts an SEI + recovery point with a zero count before the first slice. Rejected: treating + H.264 `warmup` renditions as unsupported in Chromium (B), and asking Chromium + to accept a nonzero count (C), an outside party. - Firefox and Safari: unverified. Playwright's Firefox build reports H.264 unsupported and its WebKit build would not launch on the Linux host. Check - both with a real browser, joining at a non-IDR recovery point sent as `key`. + both with a real browser, joining at a non-IDR recovery point sent as `key`, + with and without the inserted recovery point. If one rejects the join, find + what its key check accepts. - HEVC: Chromium rejects a non-IRAP H.265 frame sent as `key`, so an HEVC refresh rendition fails through the decoder's existing `error` signal. From 6b9f0d54babada4a59547746b64b28f6fae4e2c3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 13:44:15 -0800 Subject: [PATCH 06/10] fix(watch): retire group entries for chunks that produce no picture Co-Authored-By: Claude Opus 5.5 --- js/watch/src/video/decoder.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index d437bd6f0ce..3f5d2ad070b 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -559,7 +559,11 @@ class DecoderTrack { if (signal.aborted) return; const generation = this.#generation; const group = this.#groups.get(frame.timestamp); - this.#groups.delete(frame.timestamp); + // Pictures come out in presentation order, so a chunk at or before this one that + // produced none, such as a hidden VP9 frame, never will. + for (const timestamp of this.#groups.keys()) { + if (timestamp <= frame.timestamp) this.#groups.delete(timestamp); + } const timestamp = Milli.fromMicro(frame.timestamp as Micro); if (timestamp < (this.timestamp.peek() ?? 0)) { // Late frame, don't render it. From 05add87419839627fbf7b2c25a2eb4178c6140b6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 13:46:29 -0800 Subject: [PATCH 07/10] fix(watch): bound the group map by what the codec can still output Co-Authored-By: Claude Opus 5.5 --- js/watch/src/video/decoder.test.ts | 1 + js/watch/src/video/decoder.ts | 26 +++++++++++++++++++------- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index ea4bf4d83f8..dc4cd75a43d 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -277,6 +277,7 @@ async function guardedPlayback( if (live) next.mockRestore(); class Codec { state = "unconfigured"; + decodeQueueSize = 0; // Like WebCodecs, a configure requires a key chunk before any delta. #keyRequired = true; readonly callbacks: VideoDecoderInit; diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 3f5d2ad070b..dd634d4d639 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -30,6 +30,10 @@ import { caughtUp, renditionJitter, switchJitter } from "./playhead"; import { Recovery } from "./recovery"; import type { Source } from "./source"; +// Decoded chunks a codec may hold before outputting their picture: twice the 16-frame decoded +// picture buffer H.264 and H.265 allow, which also covers VP9's and AV1's 8 reference slots. +const REORDER = 32; + // The amount of time to wait before considering the video to be buffering. const BUFFERING = Milli(500); @@ -559,11 +563,7 @@ class DecoderTrack { if (signal.aborted) return; const generation = this.#generation; const group = this.#groups.get(frame.timestamp); - // Pictures come out in presentation order, so a chunk at or before this one that - // produced none, such as a hidden VP9 frame, never will. - for (const timestamp of this.#groups.keys()) { - if (timestamp <= frame.timestamp) this.#groups.delete(timestamp); - } + this.#groups.delete(frame.timestamp); const timestamp = Milli.fromMicro(frame.timestamp as Micro); if (timestamp < (this.timestamp.peek() ?? 0)) { // Late frame, don't render it. @@ -703,7 +703,7 @@ class DecoderTrack { latest = next.group; this.#resetAtKeyframe(decoder, frame.keyframe); - this.#groups.set(frame.timestamp, next.group); + this.#track(decoder, frame.timestamp, next.group); decoder.decode(this.#chunk(frame)); if (!this.sync.playing.peek()) await this.#waitStartup(effect, Milli.fromMicro(frame.timestamp as Micro)); @@ -782,7 +782,7 @@ class DecoderTrack { if (decoder.state === "closed") break; latest = next.group; this.#resetAtKeyframe(decoder, frame.keyframe); - this.#groups.set(frame.timestamp, next.group); + this.#track(decoder, frame.timestamp, next.group); decoder.decode(this.#chunk(frame)); if (!this.sync.playing.peek()) await this.#waitStartup(effect, Milli.fromMicro(frame.timestamp)); } @@ -861,6 +861,18 @@ class DecoderTrack { this.#configure(decoder); } + // Record the group of a chunk about to be decoded. A chunk that produces no picture, such as a + // hidden VP9 frame, leaves its entry behind, so keep only those the codec can still output: the + // chunks still queued, plus a reordering window past the deepest decoded picture buffer. + #track(decoder: VideoDecoder, timestamp: number, group: number): void { + this.#groups.set(timestamp, group); + const limit = decoder.decodeQueueSize + REORDER; + for (const oldest of this.#groups.keys()) { + if (this.#groups.size <= limit) break; + this.#groups.delete(oldest); + } + } + // The chunk for `frame`, run through the opening rewrite when it opens the codec. #chunk(frame: Container.ReceivedFrame): EncodedVideoChunk { let data = frame.payload; From 5a3d950494864dd3708d0de4af26c5554bb1143e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 13:48:46 -0800 Subject: [PATCH 08/10] Revert "fix(watch): bound the group map by what the codec can still output" This reverts commit 05add87419839627fbf7b2c25a2eb4178c6140b6. A 32-chunk window could evict the group of a picture the codec still owes, and WebCodecs gives no proven bound on that pipeline. Whether to cap the map for publishers that never output a picture is left to the maintainer. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/video/decoder.test.ts | 1 - js/watch/src/video/decoder.ts | 26 +++++++------------------- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index dc4cd75a43d..ea4bf4d83f8 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -277,7 +277,6 @@ async function guardedPlayback( if (live) next.mockRestore(); class Codec { state = "unconfigured"; - decodeQueueSize = 0; // Like WebCodecs, a configure requires a key chunk before any delta. #keyRequired = true; readonly callbacks: VideoDecoderInit; diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index dd634d4d639..3f5d2ad070b 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -30,10 +30,6 @@ import { caughtUp, renditionJitter, switchJitter } from "./playhead"; import { Recovery } from "./recovery"; import type { Source } from "./source"; -// Decoded chunks a codec may hold before outputting their picture: twice the 16-frame decoded -// picture buffer H.264 and H.265 allow, which also covers VP9's and AV1's 8 reference slots. -const REORDER = 32; - // The amount of time to wait before considering the video to be buffering. const BUFFERING = Milli(500); @@ -563,7 +559,11 @@ class DecoderTrack { if (signal.aborted) return; const generation = this.#generation; const group = this.#groups.get(frame.timestamp); - this.#groups.delete(frame.timestamp); + // Pictures come out in presentation order, so a chunk at or before this one that + // produced none, such as a hidden VP9 frame, never will. + for (const timestamp of this.#groups.keys()) { + if (timestamp <= frame.timestamp) this.#groups.delete(timestamp); + } const timestamp = Milli.fromMicro(frame.timestamp as Micro); if (timestamp < (this.timestamp.peek() ?? 0)) { // Late frame, don't render it. @@ -703,7 +703,7 @@ class DecoderTrack { latest = next.group; this.#resetAtKeyframe(decoder, frame.keyframe); - this.#track(decoder, frame.timestamp, next.group); + this.#groups.set(frame.timestamp, next.group); decoder.decode(this.#chunk(frame)); if (!this.sync.playing.peek()) await this.#waitStartup(effect, Milli.fromMicro(frame.timestamp as Micro)); @@ -782,7 +782,7 @@ class DecoderTrack { if (decoder.state === "closed") break; latest = next.group; this.#resetAtKeyframe(decoder, frame.keyframe); - this.#track(decoder, frame.timestamp, next.group); + this.#groups.set(frame.timestamp, next.group); decoder.decode(this.#chunk(frame)); if (!this.sync.playing.peek()) await this.#waitStartup(effect, Milli.fromMicro(frame.timestamp)); } @@ -861,18 +861,6 @@ class DecoderTrack { this.#configure(decoder); } - // Record the group of a chunk about to be decoded. A chunk that produces no picture, such as a - // hidden VP9 frame, leaves its entry behind, so keep only those the codec can still output: the - // chunks still queued, plus a reordering window past the deepest decoded picture buffer. - #track(decoder: VideoDecoder, timestamp: number, group: number): void { - this.#groups.set(timestamp, group); - const limit = decoder.decodeQueueSize + REORDER; - for (const oldest of this.#groups.keys()) { - if (this.#groups.size <= limit) break; - this.#groups.delete(oldest); - } - } - // The chunk for `frame`, run through the opening rewrite when it opens the codec. #chunk(frame: Container.ReceivedFrame): EncodedVideoChunk { let data = frame.payload; From b674f14e13182088de42a94f46807902423653d4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 14:03:23 -0800 Subject: [PATCH 09/10] docs(watch): record the unbounded group-map exception Co-Authored-By: Claude Opus 5.5 --- js/watch/src/video/decoder.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 3f5d2ad070b..ab32696d2ae 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -463,7 +463,11 @@ class DecoderTrack { // Counts the timeline restarts this decoder read. A picture decoded before one is from the old // timeline and never presents after it, even when its clock wait already resolved. #generation = 0; - // The group each frame in the codec was read from, by timestamp. + // The group each frame in the codec was read from, by timestamp. Each output retires its entry and + // every one before it. A publisher that never outputs a picture grows this by one entry per chunk; + // the maintainer accepted that as an exception to the bounded-state rule (moq-dev/moq#7555, + // 2026-11-01), since WebCodecs bounds no codec's pending output, so any cap is a guess that can + // evict a picture the codec still owes. #groups = new Map(); // How far past the shared clock's budgets this decoder reaches. From 5620fc283d0e81fc6d3ebdeefbdb34b968fc1ca3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 14:24:08 -0800 Subject: [PATCH 10/10] fix(watch): clear the selection's timestamp on a restart and retire groups only on known outputs A rendition's timestamp kept the old timeline's playhead after a restart, so a pending rendition waited to catch up to it. An output with no group entry, decoded before the last reset, no longer retires entries fed since. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/video/decoder.test.ts | 22 ++++++++++++++++++++++ js/watch/src/video/decoder.ts | 25 ++++++++++++++++--------- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index ea4bf4d83f8..fa32a7cd087 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -1603,6 +1603,28 @@ describe("Decoder with a warmup rendition", () => { } }); + it("promotes a pending rendition once the outgoing one restarts its timeline", async () => { + const run = await refresh({}, [0]); + try { + await run.push(MIN, at(10, 1000, true)); + expect(run.current()).toEqual([1000, MIN]); + + // The outgoing rendition reads a restart whose first picture is still in the codec. It has + // presented nothing on the new timeline, so the replacement need not catch up to 1000. + await run.push(MIN, marker(10, 1_066_000), at(11, 0, true)); + expect(run.current()).toEqual([1000, MIN]); + run.playback.switchRendition(); + await microtasks(); + expect(run.playback.codecs).toBe(2); + // Emitted straight from the replacement codec, since decoding there would flush the held picture. + run.playback.emit(0, 1); + await microtasks(); + expect(run.current()).toEqual([0, 1]); + } finally { + await run.close(); + } + }); + it("takes the max-delay picture when that decoder opens after a restart", async () => { const run = await refresh({}); try { diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index ab32696d2ae..96bddd79f7a 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -463,11 +463,11 @@ class DecoderTrack { // Counts the timeline restarts this decoder read. A picture decoded before one is from the old // timeline and never presents after it, even when its clock wait already resolved. #generation = 0; - // The group each frame in the codec was read from, by timestamp. Each output retires its entry and - // every one before it. A publisher that never outputs a picture grows this by one entry per chunk; - // the maintainer accepted that as an exception to the bounded-state rule (moq-dev/moq#7555, - // 2026-11-01), since WebCodecs bounds no codec's pending output, so any cap is a guess that can - // evict a picture the codec still owes. + // The group each frame in the codec was read from, by timestamp. Each output found here retires its + // entry and every one before it. A publisher that never outputs a picture grows this by one entry + // per chunk; the maintainer accepted that as an exception to the bounded-state rule + // (moq-dev/moq#7555, 2026-11-01), since WebCodecs bounds no codec's pending output, so any cap is + // a guess that can evict a picture the codec still owes. #groups = new Map(); // How far past the shared clock's budgets this decoder reaches. @@ -564,9 +564,13 @@ class DecoderTrack { const generation = this.#generation; const group = this.#groups.get(frame.timestamp); // Pictures come out in presentation order, so a chunk at or before this one that - // produced none, such as a hidden VP9 frame, never will. - for (const timestamp of this.#groups.keys()) { - if (timestamp <= frame.timestamp) this.#groups.delete(timestamp); + // produced none, such as a hidden VP9 frame, never will. Only a picture with an entry + // proves that: one without was decoded before the last reset cleared the map, so its + // timestamp says nothing about the chunks fed since. + if (group !== undefined) { + for (const timestamp of this.#groups.keys()) { + if (timestamp <= frame.timestamp) this.#groups.delete(timestamp); + } } const timestamp = Milli.fromMicro(frame.timestamp as Micro); if (timestamp < (this.timestamp.peek() ?? 0)) { @@ -1049,10 +1053,13 @@ class Rendition { // candidates: one held from before a restart neither shows nor sets the floor. Not `caughtUp`: its // slack, applied per picture, would alternate backwards and forwards. #select(min: Lane, max: Lane | undefined, start: number | undefined): void { - // A restarted timeline begins below the previous one's pictures. + // A restarted timeline begins below the previous one's pictures. Like a decoder's own timestamp + // on a break, the selection's clears until the new timeline presents, so a pending rendition is + // not held to the old timeline's playhead. if (start !== this.#start) { this.#start = start; this.#last = undefined; + this.timestamp.set(undefined); } const current = (lane: Lane | undefined): lane is Lane & { frame: VideoFrame } => lane?.frame !== undefined && (start === undefined || (lane.lane.group ?? -1) >= start);