diff --git a/js/watch/src/sync.test.ts b/js/watch/src/sync.test.ts index d0bef3018d8..2236ee1a91c 100644 --- a/js/watch/src/sync.test.ts +++ b/js/watch/src/sync.test.ts @@ -147,15 +147,55 @@ describe("wait", () => { sync.close(); }); - it("wakes a sleeping wait on reset", async () => { + it("holds a sleeping wait across a reset until the next frame re-anchors", async () => { const sync = new Sync({ delay: 10_000 as Time.Milli }); await flush(); - sync.received(Time.Milli.now()); + const timestamp = Time.Milli.now(); + sync.received(timestamp); + + let woke = false; + const waiting = sync.wait(timestamp).then(() => { + woke = true; + }); + await flush(); + sync.reset(); + await flush(); + expect(woke).toBe(false); + + // The next frame anchors far enough ahead that this one is overdue. + sync.received(Time.Milli.add(timestamp, Time.Milli(20_000))); + await waiting; + expect(woke).toBe(true); + sync.close(); + }); - const waiting = sync.wait(Time.Milli.now()); + it("releases a wait parked across a reset when its signal aborts", async () => { + const sync = new Sync({ delay: 10_000 as Time.Milli }); + await flush(); + const timestamp = Time.Milli.now(); + sync.received(timestamp); + + const controller = new AbortController(); + const waiting = sync.wait(timestamp, controller.signal); await flush(); sync.reset(); + await flush(); + controller.abort(); await waiting; sync.close(); }); + + it("releases a wait parked across a reset when closed", async () => { + const sync = new Sync({ delay: 10_000 as Time.Milli }); + await flush(); + const timestamp = Time.Milli.now(); + sync.received(timestamp); + + const waiting = sync.wait(timestamp); + await flush(); + sync.reset(); + await flush(); + sync.close(); + await waiting; + }); }); diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index 048572cf574..17f3cafbfe9 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -91,6 +91,11 @@ export class Sync { #media = new Signal<{ target: Getter }[]>([]); + #epoch = 0; + + // Releases parked waits, which may otherwise sleep forever on a reference that never returns. + #closed = new Signal(false); + #signals = new Effect(); constructor(props?: Inputs) { @@ -197,10 +202,20 @@ export class Sync { // in buffered mode (typically alongside flushing the audio buffer) so the new content // plays from its own first frame instead of inheriting the previous reference. reset(): void { + this.#epoch++; this.#out.reference.set(undefined); this.#late.clear(); } + /** + * Bumped by every `reset()`. A frame that waited across a bump belongs to the previous + * timeline, even when the next frame restores the same reference value. + * @internal + */ + get epoch(): number { + return this.#epoch; + } + // The PTS that should be rendering right now, derived from the reference + buffer. // Returns undefined if no frames have been received yet. now(): Time.Milli | undefined { @@ -209,8 +224,13 @@ export class Sync { return Time.Milli.sub(Time.Milli.sub(Time.Milli.now(), reference), this.#out.delay.peek()); } - // Sleep until it's time to render this frame. - async wait(timestamp: Time.Milli): Promise { + /** + * Sleep until it's time to render this frame, or until `signal` aborts. + * + * A reset holds the wait until the next frame re-anchors the clock, then measures the frame + * against that new reference. Aborting resolves the wait early and releases it. + */ + async wait(timestamp: Time.Milli, signal?: AbortSignal): Promise { // A zero delay still sleeps: the sleep comes from the reference, which holds an early frame // until its timestamp comes up. "instant" is the only thing that skips the wait itself. // Peek the input, not `out.instant`, which lags a same-turn change until effects flush. @@ -222,8 +242,8 @@ export class Sync { } for (;;) { - // Switching to "instant" wakes the sleep below, so frames parked here leave. - if (this.in.delay.peek() === "instant") return; + // Switching to "instant", closing, or aborting wakes the sleep below, so frames parked here leave. + if (this.in.delay.peek() === "instant" || this.#closed.peek() || signal?.aborted) return; // Sleep until it's time to decode the next frame. // NOTE: This function runs in parallel for each frame. @@ -231,7 +251,10 @@ export class Sync { const ref = Time.Milli.sub(now, timestamp); const currentRef = this.#out.reference.peek(); - if (currentRef === undefined) return; + if (currentRef === undefined) { + await this.#sleep(undefined, signal); + continue; + } const sleep = Time.Milli.add(Time.Milli.sub(currentRef, ref), this.#out.delay.peek()); if (sleep <= 0) return; @@ -239,26 +262,33 @@ export class Sync { // Skip setTimeout for small sleeps; the timer resolution (~4ms) would overshoot. if (sleep < 5) return; - if (await this.#sleep(sleep)) return; + if (await this.#sleep(sleep, signal)) return; } } - // Sleeps for `ms`, or returns false early once anything the sleep was computed from changes. - // Releases every listener either way: frames sleep once each, so a listener left on a signal - // that never changes would pile up for the life of the player. - #sleep(ms: number): Promise { + // Sleeps for `ms` (forever if undefined), or returns false early once anything the sleep was + // computed from changes or `signal` aborts. Releases every listener either way: frames sleep + // once each, so a listener left on a signal that never changes would pile up for the life of + // the player. + #sleep(ms: number | undefined, signal?: AbortSignal): Promise { return new Promise((resolve) => { const wake = (ok: boolean) => { clearTimeout(timer); for (const dispose of disposes) dispose(); resolve(ok); }; - const timer = setTimeout(() => wake(true), ms); + const timer = ms === undefined ? undefined : setTimeout(() => wake(true), ms); const disposes = [ this.in.delay.changed(() => wake(false)), this.#out.delay.changed(() => wake(false)), this.#out.reference.changed(() => wake(false)), + this.#closed.changed(() => wake(false)), ]; + if (signal) { + const abort = () => wake(false); + signal.addEventListener("abort", abort, { once: true }); + disposes.push(() => signal.removeEventListener("abort", abort)); + } }); } @@ -272,6 +302,7 @@ export class Sync { } close() { + this.#closed.set(true); this.#signals.close(); } } diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index af299e9b282..d18bc4775be 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -5,7 +5,7 @@ import * as Moq from "@moq/net"; import { Time } from "@moq/net"; import { type Effect, Signal } from "@moq/signals"; import type { Broadcast } from "../broadcast"; -import { Sync } from "../sync"; +import { type Delay, Sync } from "../sync"; import { Decoder } from "./decoder"; import { Source } from "./source"; @@ -270,12 +270,14 @@ async function guardedPlayback(kind: "legacy" | "cmaf", reads: Read[], hold: rea mediaProducer.close(); }; const source = new Source({ broadcast, supported: async () => true }); - const sync = new Sync({ delay: "instant" }); + const delay = new Signal("instant"); + const sync = new Sync({ delay }); const enabled = new Signal(true); const decoder = new Decoder({ source, sync, enabled }); await microtasks(); return { decoder, + delay, fail: (error: DOMException, index = -1) => errors.at(index)?.(error), emit: (timestamp: number, index = -1) => frames.at(index)?.(new Picture(timestamp) as unknown as VideoFrame), get codecs() { @@ -588,3 +590,112 @@ for (const kind of ["legacy", "cmaf"] as const) { } }); } + +it("a shared audio clock reset cannot present a parked video frame", async () => { + const now = spyOn(Time.Milli, "now").mockReturnValue(Time.Milli(1000)); + const timer = spyOn(globalThis, "setTimeout").mockImplementation( + (() => 1 as unknown as ReturnType) as unknown as typeof setTimeout, + ); + const clear = spyOn(globalThis, "clearTimeout").mockImplementation(() => {}); + const reads: Read[] = []; + const playback = await guardedPlayback("legacy", reads); + try { + playback.enabled.set(false); + playback.delay.set(Time.Milli(1000)); + await microtasks(); + reads.push(sample(0, 0), sample(0, 100_000, 0, false)); + playback.enabled.set(true); + await microtasks(); + expect(playback.submitted).toEqual([0, 100_000]); + expect(playback.decoder.out.timestamp.peek()).toBe(Time.Milli(0)); + // Audio can reset the shared reference while this video's own container + // discontinuity remains unchanged. + playback.sync.reset(); + await microtasks(); + expect(playback.decoder.out.timestamp.peek()).toBe(Time.Milli(0)); + // The next utterance anchors the clock past the parked frame's deadline. + playback.sync.received(Time.Milli(2000)); + await microtasks(); + expect(playback.decoder.out.timestamp.peek()).toBe(Time.Milli(0)); + } finally { + playback.close(); + await microtasks(); + clear.mockRestore(); + timer.mockRestore(); + now.mockRestore(); + } +}); + +it("closing the track releases a video wait parked across a reset", async () => { + const now = spyOn(Time.Milli, "now").mockReturnValue(Time.Milli(1000)); + const timer = spyOn(globalThis, "setTimeout").mockImplementation( + (() => 1 as unknown as ReturnType) as unknown as typeof setTimeout, + ); + const clear = spyOn(globalThis, "clearTimeout").mockImplementation(() => {}); + const reads: Read[] = []; + const playback = await guardedPlayback("legacy", reads); + const waits: Promise[] = []; + const original = playback.sync.wait.bind(playback.sync); + const wait = spyOn(playback.sync, "wait").mockImplementation((timestamp, signal) => { + const waiting = original(timestamp, signal); + waits.push(waiting); + return waiting; + }); + try { + playback.enabled.set(false); + playback.delay.set(Time.Milli(1000)); + await microtasks(); + reads.push(sample(0, 0), sample(0, 100_000, 0, false)); + playback.enabled.set(true); + await microtasks(); + expect(playback.submitted).toEqual([0, 100_000]); + playback.sync.reset(); + await microtasks(); + expect(waits.map((waiting) => Bun.peek.status(waiting))).toContain("pending"); + // No frame re-anchors the shared clock, but the closed track must not leave its wait behind. + playback.enabled.set(false); + await microtasks(); + expect(waits.map((waiting) => Bun.peek.status(waiting))).not.toContain("pending"); + } finally { + wait.mockRestore(); + playback.close(); + await microtasks(); + clear.mockRestore(); + timer.mockRestore(); + now.mockRestore(); + } +}); + +it("a reset drops a parked video frame even when the next anchor restores the reference", async () => { + const now = spyOn(Time.Milli, "now").mockReturnValue(Time.Milli(1000)); + const timers: Array<() => void> = []; + const timer = spyOn(globalThis, "setTimeout").mockImplementation(((fn: () => void) => { + timers.push(fn); + return timers.length as unknown as ReturnType; + }) as unknown as typeof setTimeout); + const clear = spyOn(globalThis, "clearTimeout").mockImplementation(() => {}); + const reads: Read[] = []; + const playback = await guardedPlayback("legacy", reads); + try { + playback.enabled.set(false); + playback.delay.set(Time.Milli(1000)); + await microtasks(); + reads.push(sample(0, 0), sample(0, 100_000, 0, false)); + playback.enabled.set(true); + await microtasks(); + expect(playback.submitted).toEqual([0, 100_000]); + // In one turn, so the reference never visibly changes and the parked sleep keeps its timer. + playback.sync.reset(); + playback.sync.received(Time.Milli(0)); + await microtasks(); + for (const fn of timers.splice(0)) fn(); + await microtasks(); + expect(playback.decoder.out.timestamp.peek()).toBe(Time.Milli(0)); + } finally { + playback.close(); + await microtasks(); + clear.mockRestore(); + timer.mockRestore(); + now.mockRestore(); + } +}); diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 0e47cae0b20..a1f915fee1f 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -410,9 +410,7 @@ class DecoderTrack { // Decoded frames waiting to be rendered. #buffered = new Signal([]); - // The last discontinuity count seen from the container consumer. An output callback - // captures it when the frame comes out, so a picture already parked in sync.wait - // can be dropped after the playhead moves. + // The last discontinuity count seen from the container consumer. #discontinuity = 0; // Codec description from the last configure, reused after a playhead reset. @@ -447,9 +445,10 @@ class DecoderTrack { const decoder = new VideoDecoder({ output: async (frame: VideoFrame) => { try { - // The generation this frame was decoded in. If the playhead bumps while we wait - // below, this frame belongs to the previous generation and must be dropped. - const generation = this.#discontinuity; + // The clock epoch this frame was decoded in. A reset while we wait below (this + // track's playhead moving, or a shared reset such as audio's at an utterance + // boundary) means the frame belongs to the previous timeline and must be dropped. + const epoch = this.sync.epoch; const timestamp = Time.Milli.fromMicro(frame.timestamp as Time.Micro); if (timestamp < (this.timestamp.peek() ?? 0)) { @@ -468,10 +467,12 @@ class DecoderTrack { this.frame.set(frame.clone()); } - // Returns immediately when the latency is "instant". - const ok = await effect.race(this.sync.wait(timestamp).then(() => true)); - if (!ok) return; - if (generation !== this.#discontinuity) return; // the playhead moved while waiting + // Returns immediately when the latency is "instant". Closing the track releases it. + // Capture the signal: a rerun replaces `effect.abort` before this continuation runs. + const signal = effect.abort; + await this.sync.wait(timestamp, signal); + if (signal.aborted) return; + if (epoch !== this.sync.epoch) return; if (timestamp < (this.timestamp.peek() ?? 0)) { // Late frame, don't render it. @@ -684,7 +685,7 @@ class DecoderTrack { // otherwise come out above the new keyframe and late-reject it. A delta continues a group the // bump interrupted (a stale marker or a max-delay hole), so it keeps the decoder's references; // a reset would make WebCodecs require a key chunk and throw. `reset()` cannot cancel an - // output callback that already holds a frame; the generation it captured rejects that one. + // output callback that already holds a frame; the clock epoch it captured rejects that one. #resetAtKeyframe(decoder: VideoDecoder, keyframe: boolean): void { if (!this.#resetPending) return; this.#resetPending = false; diff --git a/quest/m1/watch-break-av-sync.md b/quest/m1/watch-break-av-sync.md index 7983500cdc1..e4e2af33f72 100644 --- a/quest/m1/watch-break-av-sync.md +++ b/quest/m1/watch-break-av-sync.md @@ -20,4 +20,43 @@ the reachable pre-gap GOP by design, per #4914 and #5158) or the player's sync re-anchors one track and not the other. Fix it at that cause, and assert the offset in the media check. -Public API: none expected. Wire: none. +Public API: `Sync.wait` gains an optional `AbortSignal`. Wire: none. + +### Investigation + +On Linux Chromium the unchanged full media fixture passed its positive lifecycle +run and all negative controls. The old reattach case measured sync only after +reaching live, leaving the reported pre-live failure outside its checked window. +The draft moves the resume sync check into that catch-up window. It excludes the +held silent preview and requires each advancing audible pair to stay within the +existing one-tone-step tolerance. The negative unit fixture carries the reported +1.6-second stale-audio offset. + +A passing run does not establish the root cause or fix the player. Retain this +quest until the new early assertion captures a failure with container group and +shared-reference evidence. + +With the early assertion enabled, the positive lifecycle including reattach +passed (30.3 fps, median skew 0 ms, p95 200 ms). The full run then stopped in +the offset negative control on its unrelated audio-tone completeness floor +(68/81 samples, 84% versus 90%). Preserve that validation limit. + +Astra iteration reproduced a related shared-clock defect with mocked time: when +an audio reset cleared the reference, `Sync.wait` returned normally and the video +decoder presented its parked frame. Decided 2026-10-11 (no public API change): +`Sync.wait` stays `Promise` and holds a parked wait across a reset until +the next frame re-anchors the clock. Video drops a frame whose wait spanned a +reset, keyed on an `@internal` reset epoch (stripped from the published types), +which also covers a reset and same-value re-anchor in one turn. `Sync.wait` +gained an optional `AbortSignal` (additive), which the decoder passes so a +closed track releases its parked waits. Focused +regressions fail before and pass after the fix. + +The media harness now keeps early-resume grading active until audio is running +and unstalled. A moving silent preview no longer ends that grading. Existing +skew and timing bounds remain unchanged. + +This does not yet establish the cause of the original 1.6-second reattach sample. +Keep the quest until real-media evidence connects that failure to the fixed path +or independently demonstrates that it is resolved. Frames already waiting for +presentation are covered; queued WebCodecs outputs need separate evidence. \ No newline at end of file diff --git a/test/interop/clients/js/media.ts b/test/interop/clients/js/media.ts index 1cfb045bade..e527cec2609 100644 --- a/test/interop/clients/js/media.ts +++ b/test/interop/clients/js/media.ts @@ -47,6 +47,7 @@ import { } from "./harness"; import { type CaptureState, FAULTS, leakedPlayerStarted, reachesLive, SAMPLE_MS, SAMPLE_RATE } from "./src/contract"; import * as Pattern from "./src/pattern"; +import { resumeSkew } from "./src/resume"; /** Cases beyond the mandatory capability probe, publisher readiness, and cold start. */ const CASES = ["capture-denial", "pause", "disable", "rejoin", "detach", "republish", "late-join"] as const; @@ -194,6 +195,9 @@ const framesToMs = (frames: number) => Math.round((frames * 1000) / Pattern.FPS) * playback runs: a player delayed seconds behind live still shows a near-live preview first. * Audio must also have left preroll, just as in measurePlayback: a new preview can replace an old * one while the audio buffer is still filling, so a changed canvas alone does not prove playback. + * + * A `resumed` player gets the full timeout to start moving again, and the catch-up window opens at + * its first advancing frame. Every audible advancing pair before live is graded for A/V sync. */ async function waitForLive( publisher: Page, @@ -201,22 +205,38 @@ async function waitForLive( player: Page, playerErrors: BrowserErrors, assertion: string, + resumed = false, ): Promise<{ first: number; lag: number; after: number }> { - const since = Date.now(); + let since = Date.now(); + let started = !resumed; + const startup = since + timeoutMs; let first: number | undefined; let lag: number | undefined; let frame: number | undefined; let moved = false; - while (Date.now() < since + LATE_CATCHUP_MS) { + while (Date.now() < (started ? since + LATE_CATCHUP_MS : startup)) { throwPageErrors(publisherErrors); throwPageErrors(playerErrors); const [fixture, state] = await Promise.all([readFixtureState(publisher), readPlayerState(player)]).catch( () => [], ); if (fixture && state?.frameId !== undefined) { + if (resumed) { + const skew = resumeSkew(state, frame); + if (skew !== undefined) + check( + Math.abs(skew) <= MAX_SKEW_STEPS * Pattern.STEP_MS, + "resume audio/video sync", + () => `${assertion}: offset ${skew}ms; ${traceLine(state, since)}`, + ); + } lag = fixture.frameId - state.frameId; first ??= lag; moved = frame !== undefined && state.frameId !== frame; + if (moved && !started) { + started = true; + since = Date.now(); + } const live = reachesLive({ previous: frame, source: fixture.frameId, @@ -233,7 +253,7 @@ async function waitForLive( throw new Failure( assertion, `${moved ? "still" : "held a frame"} ${lag === undefined ? "?" : framesToMs(lag)}ms behind the fixture ` + - `${LATE_CATCHUP_MS}ms after the first frame, against a ${framesToMs(LIVE_LAG_FRAMES)}ms bound for playback`, + `${started ? LATE_CATCHUP_MS : timeoutMs}ms after the first frame, against a ${framesToMs(LIVE_LAG_FRAMES)}ms bound for playback`, ); } @@ -752,13 +772,7 @@ try { console.error(" detach released every session, audio graph, and worker"); await command(player, "reattach"); - await waitForState(player, playerErrors, { - deadline: Date.now() + timeoutMs, - assertion: "reattach resumes playback", - description: `the presented frame to move past the ${busy.frameId} showing before the detach`, - predicate: (state) => (state.frameId ?? 0) > (busy.frameId ?? 0), - }); - await waitForLive(publisher, publisherErrors, player, playerErrors, "reattach reaches live"); + await waitForLive(publisher, publisherErrors, player, playerErrors, "reattach reaches live", true); await measurePlayback(player, playerErrors, "after reattach"); } diff --git a/test/interop/clients/js/src/resume.test.ts b/test/interop/clients/js/src/resume.test.ts new file mode 100644 index 00000000000..75f3ed23535 --- /dev/null +++ b/test/interop/clients/js/src/resume.test.ts @@ -0,0 +1,20 @@ +import { expect, test } from "bun:test"; +import type { Sample } from "./contract"; +import * as Pattern from "./pattern"; +import { resumeSkew } from "./resume"; + +const state = (frameId: number, toneStep: number, audioStalled = false) => + ({ frameId, toneStep, audioStalled, rms: 0.35 }) as Sample; + +test("the first advancing audible picture exposes stale audio", () => { + const sample = state(1, Pattern.expectedStep(1) + 8); + expect(Math.abs(resumeSkew(sample, 0) ?? 0)).toBeGreaterThan(Pattern.STEP_MS); + expect(resumeSkew(state(1, Pattern.expectedStep(1)), 0)).toBe(0); +}); + +test("a held preview and buffering silence are not presented AV pairs", () => { + expect(resumeSkew(state(1, 8), undefined)).toBeUndefined(); + expect(resumeSkew(state(1, 8), 1)).toBeUndefined(); + expect(resumeSkew(state(1, 8, true), 0)).toBeUndefined(); + expect(resumeSkew({ ...state(1, 8), rms: 0 }, 0)).toBeUndefined(); +}); diff --git a/test/interop/clients/js/src/resume.ts b/test/interop/clients/js/src/resume.ts new file mode 100644 index 00000000000..cc71a73c285 --- /dev/null +++ b/test/interop/clients/js/src/resume.ts @@ -0,0 +1,16 @@ +/** The audible offset of an advancing resumed picture, excluding its silent preview. */ +import { AUDIBLE_RMS, type Sample } from "./contract"; +import * as Pattern from "./pattern"; + +export function resumeSkew(sample: Sample, previousFrame: number | undefined): number | undefined { + if ( + previousFrame === undefined || + sample.frameId === undefined || + sample.frameId === previousFrame || + sample.toneStep === undefined || + sample.audioStalled || + (sample.rms ?? 0) < AUDIBLE_RMS + ) + return undefined; + return Pattern.stepSkew(sample.toneStep, Pattern.expectedStep(sample.frameId)) * Pattern.STEP_MS; +} diff --git a/test/justfile b/test/justfile index ad8a62bfa7f..593a4077d68 100644 --- a/test/justfile +++ b/test/justfile @@ -26,7 +26,7 @@ harness: cd .. && bun test test/interop/compat/resolve.test.ts cd .. && bun install --frozen-lockfile cd interop/clients/js && bunx playwright install chromium - cd interop/clients/js && bun test ./harness.browser.ts ./src/contract.test.ts + cd interop/clients/js && bun test ./harness.browser.ts ./src/contract.test.ts ./src/resume.test.ts # Cross-language media interop test, built from this checkout. Stands up a # relay and runs the publisher x subscriber matrix. Default: rust only (fast