Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion doc/lib/js/publish.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ element. `<moq-publish-support>` shows what the browser can encode.
Capture failures, such as `NotAllowedError` when permission is refused, land in
`out.error` on the camera or microphone in `el.sources.video` and
`el.sources.audio`, and `<moq-publish-ui>` shows them. A refused capture waits
for a permission, device, or settings change instead of prompting again.
for a permission, device, or settings change. Apps can toggle the source’s `enabled`
state off and on to retry after a refusal.

## Stats and feedback

Expand Down
23 changes: 23 additions & 0 deletions js/publish/src/source/retry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,29 @@ for (const [kind, create] of [
});
}

test(`${kind} retries a refusal after toggling enabled`, async () => {
using _time = fakeTime();
using media = install(new FakeMediaDevices());
media.denied = true;
const enabled = new Signal(true);
const source = create(enabled);
try {
await settle();
expect(source.out.error.peek()?.name).toBe("NotAllowedError");
media.denied = false;
enabled.set(false);
await settle();
enabled.set(true);
await settle();
expect(enabled.peek()).toBe(true);
expect(source.out.error.peek()).toBeUndefined();
expect(source.out.source.peek()).toBeDefined();
expect(media.attempts).toBe(2);
} finally {
source.close();
}
});

test(`${kind} reports a refusal and stops retrying until capture is reset`, async () => {
using time = fakeTime();
using media = install(new FakeMediaDevices());
Expand Down
106 changes: 106 additions & 0 deletions js/publish/src/video/encoder.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,43 @@ function installFakeVideoEncoder() {
};
}

for (const stage of ["support", "flush"] as const) {
test(`a cancelled ${stage} probe cannot block the next capture`, async () => {
using _videoEncoder = installFakeVideoEncoder();
const pending = Promise.withResolvers<never>();
const held =
stage === "support"
? spyOn(FakeVideoEncoder, "isConfigSupported").mockImplementationOnce(() => pending.promise)
: spyOn(FakeVideoEncoder.prototype, "flush").mockImplementationOnce(() => pending.promise);
const enabled = new Signal(true);
const encoder = new Encoder("video", {
enabled,
capture: {
in: { source: new Signal({ getSettings: () => ({ frameRate: 30 }), getConstraints: () => ({}) }) },
out: { display: new Signal({ width: 640, height: 480 }), frames: new Signal(undefined) },
} as never,
config: { codec: "avc1" },
});
try {
await settle();
expect(held).toHaveBeenCalledTimes(1);
enabled.set(false);
await settle();
if (stage === "flush") expect(FakeVideoEncoder.instances[0]?.state).toBe("closed");
enabled.set(true);
await settle();
expect(encoder.out.catalog.peek()?.codec).toStartWith("avc1");
// The abandoned browser promise still has not resolved.
expect(held.mock.calls.length).toBeGreaterThan(1);
} finally {
encoder.close();
held.mockRestore();
pending.reject(new DOMException("closed", "AbortError"));
await settle();
}
});
}

test("encoding tracks encoder config in its child effect", async () => {
using _videoEncoder = installFakeVideoEncoder();
const warn = spyOn(console, "warn").mockImplementation(() => {});
Expand Down Expand Up @@ -1149,6 +1186,75 @@ async function installRecording() {
};
}

for (const stop of ["resume", "close", "detach", "track closed", "track replaced"] as const) {
test(`a stalled final black frame cannot block ${stop}`, async () => {
using recording = await installRecording();
const enabled = new Signal(true);
const track = new Moq.Track.Producer("video").accept({ timescale: Moq.Time.Timescale.MICRO });
const rendition = {
slot: new Signal(undefined),
config: new Signal(undefined),
stats: new Signal(undefined),
track: new Signal(track),
join: new Signal(undefined),
close: () => {},
};
const order = new Order();
const broadcast = new Signal<Broadcast | undefined>({
video: () => rendition,
order,
join: new Signal(undefined),
} as never);
const encoder = new Encoder("video", {
enabled,
capture: recording.capture as never,
broadcast,
});
const pending = Promise.withResolvers<void>();
const closed = spyOn(RecordingVideoEncoder.prototype, "close");
const leave = spyOn(order, "leave");
let flush: ReturnType<typeof spyOn<RecordingVideoEncoder, "flush">> | undefined;
try {
await settle();
await recording.send(0);
flush = spyOn(RecordingVideoEncoder.prototype, "flush").mockImplementationOnce(() => pending.promise);
enabled.set(false);
await settle();
expect(flush).toHaveBeenCalledTimes(1);
// Muting releases sibling catalog delay even while the cosmetic flush is pending.
expect(leave).toHaveBeenCalled();
const before = closed.mock.calls.length;
enabled.set(false, true);
await settle();
expect(closed.mock.calls.length).toBe(before);
if (stop === "close") encoder.close();
else if (stop === "detach") broadcast.set(undefined);
else if (stop === "track closed") track.close();
else if (stop === "track replaced")
rendition.track.set(
new Moq.Track.Producer("replacement").accept({ timescale: Moq.Time.Timescale.MICRO }),
);
else enabled.set(true);
await settle();
expect(closed.mock.calls.length).toBeGreaterThan(before);
expect(leave).toHaveBeenCalled();
if (stop === "resume") {
await recording.send(300);
expect(recording.keys).toContain(300);
}
} finally {
encoder.close();
track.finish();
track.close();
flush?.mockRestore();
closed.mockRestore();
leave.mockRestore();
pending.resolve();
await settle();
}
});
}

async function recordKeyframes(
info: Partial<Moq.Track.Info>,
run: (ctx: {
Expand Down
76 changes: 59 additions & 17 deletions js/publish/src/video/encoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {
getter,
type Inputs,
type Readonlys,
race,
readonlys,
Signal,
} from "@moq/signals";
Expand Down Expand Up @@ -373,7 +374,7 @@ export class Encoder {
effect.set(this.#out.active, enabled && !!track && !error, false);
if (!enabled || !track || error) return;

this.#encode(track, broadcast.order, broadcast.join, effect);
this.#encode(rendition.track, broadcast.order, broadcast.join, effect);
});

// Reserve against the connection while a subscriber is attached. Wait for a ceiling so we
Expand Down Expand Up @@ -409,7 +410,14 @@ export class Encoder {
}

// Encode captured frames into the track producer, reconfiguring when the resolved config changes.
#encode(track: Moq.Track.Producer, order: Order, join: Getter<Milli | undefined>, effect: Effect): void {
#encode(
tracks: Getter<Moq.Track.Producer | undefined>,
order: Order,
join: Getter<Milli | undefined>,
effect: Effect,
): void {
const track = effect.get(tracks);
if (!track) return;
const capture = effect.get(this.in.capture);
if (!capture) return;

Expand Down Expand Up @@ -460,7 +468,7 @@ export class Encoder {
producer.encode(frame, frame.timestamp as Micro, key);
// Including a keyframe the encoder placed on its own, which opened a group too.
if (key) answer();
if (this.#estimator.write(frame.timestamp, order, key)) {
if (!tearingDown && this.#estimator.write(frame.timestamp, order, key)) {
this.#estimate.set({ ...this.#estimator.estimate });
}
},
Expand Down Expand Up @@ -582,6 +590,8 @@ export class Encoder {
// and shows black instead of a frozen picture.
await new Promise((resolve) => effect.abort.addEventListener("abort", resolve, { once: true }));
tearingDown = true;
// A cosmetic flush must not hold sibling renditions to our stale delay.
order.leave(this.#estimator);
try {
const config = configured;
if (
Expand All @@ -599,14 +609,26 @@ export class Encoder {
} finally {
frame.close();
}
// A rejected flush already reported through the error callback.
await encoder.flush().catch(() => {});
// A cosmetic frame must never hold the next enable or track close behind a codec
// flush. The old codec is closed below even if the browser never settles it.
const flushed = encoder.flush().then(
() => true,
() => true,
);
do {
if (track.closed.peek() !== undefined || tracks.peek() !== track) break;
const done = await race([
flushed,
Signal.race(this.in.enabled),
Signal.race(tracks),
track.closed,
effect.closed,
]);
if (done === true || done === undefined) break;
} while (!this.in.enabled.peek());
}
} finally {
if (encoder.state !== "closed") encoder.close();
// A stopped rendition stops holding the others' delay back. Not a cleanup: the black
// keyframe above is written after one would run, and would seat this rendition again.
order.leave(this.#estimator);
// The broadcast owns this static track across gaps (a disable, or with `onDemand` a demand
// gap). Close the current group, marking the break so a later subscriber resumes on the same
// track without the pre-gap group reading as live. A fatal encoder error aborts the track instead.
Expand Down Expand Up @@ -684,7 +706,9 @@ export class Encoder {
try {
let detected: Probe;
try {
detected = await this.#bestCodec(dimensions, target);
const result = await this.#bestCodec(dimensions, target, effect);
if (!result) return;
detected = result;
} catch (err) {
// A capture that outgrew the accepted ceiling to a size the hardware refuses encodes scaled
// down to fit it, until a smaller capture reruns the probe. Re-probing the ceiling with the
Expand All @@ -702,7 +726,9 @@ export class Encoder {
`video ${dimensions.width}x${dimensions.height} unsupported, keeping ${accepted.width}x${accepted.height}:`,
err,
);
detected = await this.#bestCodec(accepted, target);
const result = await this.#bestCodec(accepted, target, effect);
if (!result) return;
detected = result;
}
if (superseded.aborted) return;

Expand Down Expand Up @@ -817,7 +843,11 @@ export class Encoder {
}

// Try to determine the best config for the given settings.
async #bestCodec(dimensions: { width: number; height: number }, target: Target): Promise<Probe> {
async #bestCodec(
dimensions: { width: number; height: number },
target: Target,
effect: Effect,
): Promise<Probe | undefined> {
// A list of codecs to try, in order of preference. Only full RFC 6381 strings: Chrome, Firefox,
// and Safari all refuse a bare `avc1` or `vp09`, and native players can't decode without the profile.
const HARDWARE_CODECS = [
Expand Down Expand Up @@ -897,10 +927,12 @@ export class Encoder {
hevc: codec.startsWith("hev1") ? { format: "annexb" } : undefined,
};

const { supported } = await VideoEncoder.isConfigSupported(config);
if (!supported) continue;
const support = await effect.race(VideoEncoder.isConfigSupported(config));
if (!support) return;
if (!support.supported) continue;

const probed = await probeEncoder(config);
const probed = await probeEncoder(config, effect);
if (!probed) return;
if ("reported" in probed) return { config, reported: probed.reported };
console.warn(`moq-publish: skipping ${codec}: ${probed.refused}`);
}
Expand Down Expand Up @@ -1022,7 +1054,10 @@ const PROBE_FRAMES = 4;
// length-prefixed NAL units with an hvcC description, which no subscriber can decode. A
// description means length-prefixed output (per the AVC and HEVC WebCodecs registrations), which
// also catches a four-byte length that happens to read as a start code, such as `00 00 01 2c`.
async function probeEncoder(config: VideoEncoderConfig): Promise<{ reported: string } | { refused: string }> {
async function probeEncoder(
config: VideoEncoderConfig,
effect: Effect,
): Promise<{ reported: string } | { refused: string } | undefined> {
let reported: string | undefined;
let reordered = false;
let last: number | undefined;
Expand All @@ -1043,6 +1078,12 @@ async function probeEncoder(config: VideoEncoderConfig): Promise<{ reported: str
error: () => {},
});

// A superseded probe must release its codec even if the browser never settles flush().
const close = () => {
if (encoder.state !== "closed") encoder.close();
};
effect.cleanup(close);

try {
encoder.configure(config);

Expand All @@ -1063,9 +1104,10 @@ async function probeEncoder(config: VideoEncoderConfig): Promise<{ reported: str
}
}

await encoder.flush();
await effect.race(encoder.flush());
if (effect.abort.aborted) return;
} finally {
if (encoder.state !== "closed") encoder.close();
close();
}

if (!reported) throw new Error(`${config.codec} encoder reported no codec string`);
Expand Down