diff --git a/doc/lib/js/publish.md b/doc/lib/js/publish.md index 134f5c9f148..fa9f23a1f23 100644 --- a/doc/lib/js/publish.md +++ b/doc/lib/js/publish.md @@ -50,7 +50,8 @@ element. `` shows what the browser can encode. Capture failures, such as `NotAllowedError` when permission is refused, land in `out.error` on the camera or microphone in `el.sources.video` and `el.sources.audio`, and `` shows them. A refused capture waits -for a permission, device, or settings change instead of prompting again. +for a permission, device, or settings change. Apps can toggle the source’s `enabled` +state off and on to retry after a refusal. ## Stats and feedback diff --git a/js/publish/src/source/retry.test.ts b/js/publish/src/source/retry.test.ts index f2487d61b4d..d64cd8ea40e 100644 --- a/js/publish/src/source/retry.test.ts +++ b/js/publish/src/source/retry.test.ts @@ -339,6 +339,29 @@ for (const [kind, create] of [ }); } + test(`${kind} retries a refusal after toggling enabled`, async () => { + using _time = fakeTime(); + using media = install(new FakeMediaDevices()); + media.denied = true; + const enabled = new Signal(true); + const source = create(enabled); + try { + await settle(); + expect(source.out.error.peek()?.name).toBe("NotAllowedError"); + media.denied = false; + enabled.set(false); + await settle(); + enabled.set(true); + await settle(); + expect(enabled.peek()).toBe(true); + expect(source.out.error.peek()).toBeUndefined(); + expect(source.out.source.peek()).toBeDefined(); + expect(media.attempts).toBe(2); + } finally { + source.close(); + } + }); + test(`${kind} reports a refusal and stops retrying until capture is reset`, async () => { using time = fakeTime(); using media = install(new FakeMediaDevices()); diff --git a/js/publish/src/video/encoder.test.ts b/js/publish/src/video/encoder.test.ts index 80e4f33739d..5a3e31d518a 100644 --- a/js/publish/src/video/encoder.test.ts +++ b/js/publish/src/video/encoder.test.ts @@ -140,6 +140,43 @@ function installFakeVideoEncoder() { }; } +for (const stage of ["support", "flush"] as const) { + test(`a cancelled ${stage} probe cannot block the next capture`, async () => { + using _videoEncoder = installFakeVideoEncoder(); + const pending = Promise.withResolvers(); + const held = + stage === "support" + ? spyOn(FakeVideoEncoder, "isConfigSupported").mockImplementationOnce(() => pending.promise) + : spyOn(FakeVideoEncoder.prototype, "flush").mockImplementationOnce(() => pending.promise); + const enabled = new Signal(true); + const encoder = new Encoder("video", { + enabled, + capture: { + in: { source: new Signal({ getSettings: () => ({ frameRate: 30 }), getConstraints: () => ({}) }) }, + out: { display: new Signal({ width: 640, height: 480 }), frames: new Signal(undefined) }, + } as never, + config: { codec: "avc1" }, + }); + try { + await settle(); + expect(held).toHaveBeenCalledTimes(1); + enabled.set(false); + await settle(); + if (stage === "flush") expect(FakeVideoEncoder.instances[0]?.state).toBe("closed"); + enabled.set(true); + await settle(); + expect(encoder.out.catalog.peek()?.codec).toStartWith("avc1"); + // The abandoned browser promise still has not resolved. + expect(held.mock.calls.length).toBeGreaterThan(1); + } finally { + encoder.close(); + held.mockRestore(); + pending.reject(new DOMException("closed", "AbortError")); + await settle(); + } + }); +} + test("encoding tracks encoder config in its child effect", async () => { using _videoEncoder = installFakeVideoEncoder(); const warn = spyOn(console, "warn").mockImplementation(() => {}); @@ -1149,6 +1186,75 @@ async function installRecording() { }; } +for (const stop of ["resume", "close", "detach", "track closed", "track replaced"] as const) { + test(`a stalled final black frame cannot block ${stop}`, async () => { + using recording = await installRecording(); + const enabled = new Signal(true); + const track = new Moq.Track.Producer("video").accept({ timescale: Moq.Time.Timescale.MICRO }); + const rendition = { + slot: new Signal(undefined), + config: new Signal(undefined), + stats: new Signal(undefined), + track: new Signal(track), + join: new Signal(undefined), + close: () => {}, + }; + const order = new Order(); + const broadcast = new Signal({ + video: () => rendition, + order, + join: new Signal(undefined), + } as never); + const encoder = new Encoder("video", { + enabled, + capture: recording.capture as never, + broadcast, + }); + const pending = Promise.withResolvers(); + const closed = spyOn(RecordingVideoEncoder.prototype, "close"); + const leave = spyOn(order, "leave"); + let flush: ReturnType> | undefined; + try { + await settle(); + await recording.send(0); + flush = spyOn(RecordingVideoEncoder.prototype, "flush").mockImplementationOnce(() => pending.promise); + enabled.set(false); + await settle(); + expect(flush).toHaveBeenCalledTimes(1); + // Muting releases sibling catalog delay even while the cosmetic flush is pending. + expect(leave).toHaveBeenCalled(); + const before = closed.mock.calls.length; + enabled.set(false, true); + await settle(); + expect(closed.mock.calls.length).toBe(before); + if (stop === "close") encoder.close(); + else if (stop === "detach") broadcast.set(undefined); + else if (stop === "track closed") track.close(); + else if (stop === "track replaced") + rendition.track.set( + new Moq.Track.Producer("replacement").accept({ timescale: Moq.Time.Timescale.MICRO }), + ); + else enabled.set(true); + await settle(); + expect(closed.mock.calls.length).toBeGreaterThan(before); + expect(leave).toHaveBeenCalled(); + if (stop === "resume") { + await recording.send(300); + expect(recording.keys).toContain(300); + } + } finally { + encoder.close(); + track.finish(); + track.close(); + flush?.mockRestore(); + closed.mockRestore(); + leave.mockRestore(); + pending.resolve(); + await settle(); + } + }); +} + async function recordKeyframes( info: Partial, run: (ctx: { diff --git a/js/publish/src/video/encoder.ts b/js/publish/src/video/encoder.ts index 3e63ac93f3c..0cfe7d173e5 100644 --- a/js/publish/src/video/encoder.ts +++ b/js/publish/src/video/encoder.ts @@ -9,6 +9,7 @@ import { getter, type Inputs, type Readonlys, + race, readonlys, Signal, } from "@moq/signals"; @@ -373,7 +374,7 @@ export class Encoder { effect.set(this.#out.active, enabled && !!track && !error, false); if (!enabled || !track || error) return; - this.#encode(track, broadcast.order, broadcast.join, effect); + this.#encode(rendition.track, broadcast.order, broadcast.join, effect); }); // Reserve against the connection while a subscriber is attached. Wait for a ceiling so we @@ -409,7 +410,14 @@ export class Encoder { } // Encode captured frames into the track producer, reconfiguring when the resolved config changes. - #encode(track: Moq.Track.Producer, order: Order, join: Getter, effect: Effect): void { + #encode( + tracks: Getter, + order: Order, + join: Getter, + effect: Effect, + ): void { + const track = effect.get(tracks); + if (!track) return; const capture = effect.get(this.in.capture); if (!capture) return; @@ -460,7 +468,7 @@ export class Encoder { producer.encode(frame, frame.timestamp as Micro, key); // Including a keyframe the encoder placed on its own, which opened a group too. if (key) answer(); - if (this.#estimator.write(frame.timestamp, order, key)) { + if (!tearingDown && this.#estimator.write(frame.timestamp, order, key)) { this.#estimate.set({ ...this.#estimator.estimate }); } }, @@ -582,6 +590,8 @@ export class Encoder { // and shows black instead of a frozen picture. await new Promise((resolve) => effect.abort.addEventListener("abort", resolve, { once: true })); tearingDown = true; + // A cosmetic flush must not hold sibling renditions to our stale delay. + order.leave(this.#estimator); try { const config = configured; if ( @@ -599,14 +609,26 @@ export class Encoder { } finally { frame.close(); } - // A rejected flush already reported through the error callback. - await encoder.flush().catch(() => {}); + // A cosmetic frame must never hold the next enable or track close behind a codec + // flush. The old codec is closed below even if the browser never settles it. + const flushed = encoder.flush().then( + () => true, + () => true, + ); + do { + if (track.closed.peek() !== undefined || tracks.peek() !== track) break; + const done = await race([ + flushed, + Signal.race(this.in.enabled), + Signal.race(tracks), + track.closed, + effect.closed, + ]); + if (done === true || done === undefined) break; + } while (!this.in.enabled.peek()); } } finally { if (encoder.state !== "closed") encoder.close(); - // A stopped rendition stops holding the others' delay back. Not a cleanup: the black - // keyframe above is written after one would run, and would seat this rendition again. - order.leave(this.#estimator); // The broadcast owns this static track across gaps (a disable, or with `onDemand` a demand // gap). Close the current group, marking the break so a later subscriber resumes on the same // track without the pre-gap group reading as live. A fatal encoder error aborts the track instead. @@ -684,7 +706,9 @@ export class Encoder { try { let detected: Probe; try { - detected = await this.#bestCodec(dimensions, target); + const result = await this.#bestCodec(dimensions, target, effect); + if (!result) return; + detected = result; } catch (err) { // A capture that outgrew the accepted ceiling to a size the hardware refuses encodes scaled // down to fit it, until a smaller capture reruns the probe. Re-probing the ceiling with the @@ -702,7 +726,9 @@ export class Encoder { `video ${dimensions.width}x${dimensions.height} unsupported, keeping ${accepted.width}x${accepted.height}:`, err, ); - detected = await this.#bestCodec(accepted, target); + const result = await this.#bestCodec(accepted, target, effect); + if (!result) return; + detected = result; } if (superseded.aborted) return; @@ -817,7 +843,11 @@ export class Encoder { } // Try to determine the best config for the given settings. - async #bestCodec(dimensions: { width: number; height: number }, target: Target): Promise { + async #bestCodec( + dimensions: { width: number; height: number }, + target: Target, + effect: Effect, + ): Promise { // A list of codecs to try, in order of preference. Only full RFC 6381 strings: Chrome, Firefox, // and Safari all refuse a bare `avc1` or `vp09`, and native players can't decode without the profile. const HARDWARE_CODECS = [ @@ -897,10 +927,12 @@ export class Encoder { hevc: codec.startsWith("hev1") ? { format: "annexb" } : undefined, }; - const { supported } = await VideoEncoder.isConfigSupported(config); - if (!supported) continue; + const support = await effect.race(VideoEncoder.isConfigSupported(config)); + if (!support) return; + if (!support.supported) continue; - const probed = await probeEncoder(config); + const probed = await probeEncoder(config, effect); + if (!probed) return; if ("reported" in probed) return { config, reported: probed.reported }; console.warn(`moq-publish: skipping ${codec}: ${probed.refused}`); } @@ -1022,7 +1054,10 @@ const PROBE_FRAMES = 4; // length-prefixed NAL units with an hvcC description, which no subscriber can decode. A // description means length-prefixed output (per the AVC and HEVC WebCodecs registrations), which // also catches a four-byte length that happens to read as a start code, such as `00 00 01 2c`. -async function probeEncoder(config: VideoEncoderConfig): Promise<{ reported: string } | { refused: string }> { +async function probeEncoder( + config: VideoEncoderConfig, + effect: Effect, +): Promise<{ reported: string } | { refused: string } | undefined> { let reported: string | undefined; let reordered = false; let last: number | undefined; @@ -1043,6 +1078,12 @@ async function probeEncoder(config: VideoEncoderConfig): Promise<{ reported: str error: () => {}, }); + // A superseded probe must release its codec even if the browser never settles flush(). + const close = () => { + if (encoder.state !== "closed") encoder.close(); + }; + effect.cleanup(close); + try { encoder.configure(config); @@ -1063,9 +1104,10 @@ async function probeEncoder(config: VideoEncoderConfig): Promise<{ reported: str } } - await encoder.flush(); + await effect.race(encoder.flush()); + if (effect.abort.aborted) return; } finally { - if (encoder.state !== "closed") encoder.close(); + close(); } if (!reported) throw new Error(`${config.codec} encoder reported no codec string`);