diff --git a/doc/lib/js/publish.md b/doc/lib/js/publish.md index 051e8b4ab46..f91e8230e3b 100644 --- a/doc/lib/js/publish.md +++ b/doc/lib/js/publish.md @@ -51,6 +51,15 @@ for a permission, device, or settings change instead of prompting again. ## Encoding +An enabled encoder encodes whether or not anyone is subscribed, so its recent +groups wait in the track's cache: a viewer who subscribes late with a +`maxDelay`, or fetches, reaches back as far as `maxAge` keeps. Pass +`onDemand: true` to `Publish.Broadcast` to encode every rendition only while +someone is subscribed instead. That saves the encode while nobody watches, but +a viewer then starts at the keyframe its arrival forces, with no past to reach. +Bandwidth is reserved only while someone is subscribed, so an unwatched +rendition encodes at its configured bitrate. + The video encoder follows its share of the connection's send-rate estimate, so several publishers on one connection do not each target the whole uplink. Codec, resolution, framerate, and bitrate are tunable through @@ -143,7 +152,8 @@ const broadcast = new Publish.Broadcast({ }); // Each encoder registers a rendition on the broadcast (`broadcast.video(name)`) and -// encodes only while someone is subscribed. +// encodes while enabled, subscribed or not. Pass `onDemand: true` to the broadcast +// to encode only while someone is subscribed. new Publish.Video.Encoder("video/hd", { broadcast, capture, enabled: true }); new Publish.Video.Encoder("video/sd", { broadcast, capture, enabled: true, config: { maxScale: 0.25 } }); const audioSource = new Publish.Signals.Computed((effect) => effect.get(microphone.out.source)?.audio); diff --git a/js/publish/README.md b/js/publish/README.md index 3462dd54d3a..65a1c79473d 100644 --- a/js/publish/README.md +++ b/js/publish/README.md @@ -120,8 +120,9 @@ const broadcast = new Publish.Broadcast({ rotation: capture.out.rotation, }); -// Each encoder registers its rendition on the broadcast and encodes only while -// someone is subscribed. +// Each encoder registers its rendition on the broadcast and encodes while enabled, +// subscribed or not, so a late viewer can reach back into the cache. Pass +// `onDemand: true` to the broadcast to encode only while someone is subscribed. const hd = new Publish.Video.Encoder("video/hd", { broadcast, capture, enabled: true }); const sd = new Publish.Video.Encoder("video/sd", { broadcast, capture, enabled: true, config: { maxScale: 0.25 } }); diff --git a/js/publish/api/index.api.md b/js/publish/api/index.api.md index 86d3e05b17e..51ea5be1571 100644 --- a/js/publish/api/index.api.md +++ b/js/publish/api/index.api.md @@ -136,6 +136,7 @@ export type BroadcastInput = { flip: Signals.Getter; maxAge: Signals.Getter; name: Signals.Getter; + onDemand: Signals.Getter; origin: Signals.Getter; rotation: Signals.Getter; }; diff --git a/js/publish/src/audio/encoder.test.ts b/js/publish/src/audio/encoder.test.ts index 879d8684679..b6b7ebf4495 100644 --- a/js/publish/src/audio/encoder.test.ts +++ b/js/publish/src/audio/encoder.test.ts @@ -1,6 +1,7 @@ import { describe, expect, mock, spyOn, test } from "bun:test"; import { Micro, Milli } from "@moq/clock"; import * as Catalog from "@moq/hang/catalog"; +import * as Container from "@moq/hang/container"; import * as Json from "@moq/json"; import * as Moq from "@moq/net"; import { Signal } from "@moq/signals"; @@ -325,7 +326,7 @@ async function setup(order = new Order(), codec?: Codec, groupDuration?: Milli) // resume. Written after the marker, they would put pre-gap media on the live edge, and a rounding // step below the marker aborts every subscriber. The resumed chunks have to carry the capture clock, // not the encoder's gap-blind one, or they trail the next gap's marker and are dropped as pre-gap. -test("a demand gap marks where submitted audio ends and drops the chunks held across it", async () => { +test("an on-demand gap marks where submitted audio ends and drops the chunks held across it", async () => { using _webcodecs = installFakeWebCodecs(); using env = await setup(); const { track, rendition, feed, written, writes } = env; @@ -929,7 +930,8 @@ test.each(["failure first", "config first"])( expect(config?.track).toBeDefined(); expect(config?.track).not.toBe("audio"); expect(encoder.out.error.peek()).toBeUndefined(); - expect(encoder.out.active.peek()).toBe(false); + // Encoding resumes on the fresh track without waiting for a subscriber. + expect(encoder.out.active.peek()).toBe(true); expect(broadcast.net.peek()).toBe(net); } finally { catalogSubscriber?.close(); @@ -939,6 +941,68 @@ test.each(["failure first", "config first"])( }, ); +// Audio encoded with nobody listening waits in the cache for a late subscriber's maxDelay, and the +// last subscriber leaving ends no epoch. On demand, a subscriber finds only what its arrival started. +test.each([false, true])( + "a late subscriber reaches audio encoded before it arrived unless on demand (%p)", + async (onDemand) => { + using _webcodecs = installFakeWebCodecs(); + const broadcast = new Broadcast({ + origin: new Moq.Origin.Producer(), + name: Moq.Path.from("eager.hang"), + onDemand, + }); + const feed = new Feed(); + const configuring = configured(); + const encoder = new Encoder("audio", { broadcast, capture: fakeCapture(feed) as never }); + const discontinuity = spyOn(Container.Legacy.Producer.prototype, "discontinuity"); + let late: Moq.Track.Subscriber | undefined; + try { + await configuring; + await settle(); + const net = broadcast.net.peek(); + if (!net) throw new Error("expected network broadcast"); + expect(encoder.out.active.peek()).toBe(!onDemand); + + // One 20ms Opus frame per push, each its own group. + let index = 0; + const push = async (count: number) => { + for (let i = 0; i < count; i++, index++) { + await feed.push({ timestamp: Micro(index * 20_000), channels: [new Float32Array(960)] }); + } + await feed.drain(); + }; + + await push(4); // unwatched: two written, two held + late = net.track("audio").subscribe({ maxDelay: Milli(10_000) }); + await settle(); + expect(encoder.out.active.peek()).toBe(true); + await push(3); + + const group = await late.ordered().nextGroup(); + if (!group) throw new Error("expected a group"); + const frame = await group.readFrame(); + if (!frame) throw new Error("expected a frame"); + const [timestamp] = Moq.Varint.decode(frame.payload); + // On demand, nothing was submitted before the subscriber arrived. + expect(group.sequence).toBe(0); + expect(timestamp).toBe(onDemand ? 80_000 : 0); + + // Only on demand does the last subscriber leaving end the epoch. + late.close(); + late = undefined; + await settle(); + expect(discontinuity).toHaveBeenCalledTimes(onDemand ? 1 : 0); + expect(encoder.out.active.peek()).toBe(!onDemand); + } finally { + discontinuity.mockRestore(); + late?.close(); + encoder.close(); + broadcast.close(); + } + }, +); + test("Opus description is probed with the exact live config before advertising", async () => { using _codecs = installFakeWebCodecs(); LaggingAudioEncoder.description = (config) => new TextEncoder().encode(JSON.stringify(config)); diff --git a/js/publish/src/audio/encoder.ts b/js/publish/src/audio/encoder.ts index 4a10e868fb9..be0c3a538f7 100644 --- a/js/publish/src/audio/encoder.ts +++ b/js/publish/src/audio/encoder.ts @@ -14,7 +14,7 @@ import { Signal, } from "@moq/signals"; import type { Broadcast } from "../broadcast"; -import { errors } from "../internal"; +import { errors, watched } from "../internal"; import { Estimator, type Order } from "../order"; import type { AudioFrame, Capture, Format } from "./capture"; import { Gain } from "./gain"; @@ -98,8 +98,9 @@ export type EncoderInput = { // yourself and pass the same instance to each. capture: Getter; - // The connection's bandwidth allocator. Audio reserves its configured bitrate so - // video's share is honest, and ignores the grant (Opus is a fixed rate today). + // The connection's bandwidth allocator. While a subscriber is attached, audio reserves + // its configured bitrate so video's share is honest, and ignores the grant (Opus is a + // fixed rate today). bandwidth: Getter; }; @@ -125,7 +126,8 @@ export type EncoderOutput = { // The head of the capture graph, so callers can tap the raw capture. Volume is applied to the // PCM rather than in the graph, so this is pre-gain. Undefined for a source that isn't a track. root: Signal; - // True when a subscriber is attached and we're encoding. + // True while encoding: enabled with a captured format and a track, which with the broadcast's + // `onDemand` also takes a subscriber. active: Signal; // Cumulative output totals (frames, bytes) measured while serving. stats: Signal; @@ -142,8 +144,9 @@ type Pipeline = { * A single audio rendition encoder. * * Registers itself on the {@link Broadcast} under {@link name} (via `broadcast.audio(name)`), pumps PCM - * off the source via a {@link Capture}, and encodes it only while a subscriber is attached (the demand - * gate). Rename by constructing a new encoder; the name is not a signal. + * off the source via a {@link Capture}, and encodes it while enabled, so a late subscriber or fetch + * finds the recent groups in the track's cache. The broadcast's `onDemand` encodes only while a + * subscriber is attached instead. Rename by constructing a new encoder; the name is not a signal. */ export class Encoder { /** The full track name of this rendition, e.g. `"audio/data"`. */ @@ -173,7 +176,7 @@ export class Encoder { return this.in.capture.peek(); } - // The exact encoder settings and probed decoder description advertised before demand arrives. + // The exact encoder settings and probed decoder description, advertised before the first frame. #config = new Signal<(Resolved & { encoder: AudioEncoderConfig }) | undefined>(undefined); // The last Opus probe: the encoder settings it ran with, serialized, and the description it @@ -196,7 +199,8 @@ export class Encoder { // discontinuity and re-anchors on. #pipeline: Pipeline | undefined; - // The current subscription's track and the producer writing into it, or undefined without demand. + // The rendition's track and the producer writing into it, or undefined while the broadcast hands + // over none (with `onDemand`, while nobody subscribes). // `start` is the open group's first timestamp, undefined until a frame opens one. #live: { track: Moq.Track.Producer; producer: Container.Legacy.Producer; start?: Micro } | undefined; @@ -204,8 +208,8 @@ export class Encoder { // newest one, where an epoch's discontinuity marker goes. Cleared once the marker is written. #next: Micro | undefined; - // The newest epoch's marker. The AudioEncoder outlives a demand gap too brief to skip a frame, - // so chunks it still held when demand disappeared surface after the resume; they sit below the + // The newest epoch's marker. The AudioEncoder outlives a track gap too brief to skip a frame, + // so chunks it still held when the track went away surface after the resume; they sit below the // marker and are dropped. #floor: Micro | undefined; @@ -374,8 +378,8 @@ export class Encoder { this.#failure.set(new Once()); } - // Register the rendition on the broadcast, publish its config, and encode only while a subscriber - // is attached (the demand gate). Re-registers cleanly when the broadcast swaps. + // Register the rendition on the broadcast, publish its config, and encode while it hands over a + // track. Re-registers cleanly when the broadcast swaps. #runRegister(effect: Effect): void { const failure = effect.get(this.#failure); const broadcast = effect.get(this.in.broadcast); @@ -403,8 +407,8 @@ export class Encoder { this.#encode(broadcast.order, format, effect); }); - // Each subscription gets its own producer. Losing demand ends the epoch, as stopping the - // pipeline does. + // Each track the broadcast hands over gets its own producer. Losing it (with `onDemand`, the last + // subscriber leaving) ends the epoch, as stopping the pipeline does. effect.run((effect) => { const track = effect.get(rendition.track); if (!track) return; @@ -434,8 +438,8 @@ export class Encoder { effect.set(this.#out.active, enabled && !!format && !!track && !fatal, false); }); - // Claim the configured bitrate so a co-resident video encoder's share is - // honest. Wait for a bitrate so we never claim 0. The grant is ignored: + // Claim the configured bitrate while a subscriber is attached, so a co-resident video + // encoder's share is honest. Wait for a bitrate so we never claim 0. The grant is ignored: // following it for Opus is out of scope. effect.run((effect) => { const enabled = effect.get(this.in.enabled); @@ -443,14 +447,19 @@ export class Encoder { const allocator = effect.get(this.in.bandwidth); if (!enabled || !track || !allocator || effect.get(this.#out.error)) return; - let reservation: Moq.Bandwidth.Reservation | undefined; - effect.subscribe(this.#config, (config) => { - const bitrate = config?.catalog.bitrate; - if (bitrate === undefined) return; - if (!reservation) reservation = allocator.reserve(track.demand(), bitrate); - else reservation.update(bitrate); + const demanded = watched(track, effect); + effect.run((effect) => { + if (!effect.get(demanded)) return; + + let reservation: Moq.Bandwidth.Reservation | undefined; + effect.subscribe(this.#config, (config) => { + const bitrate = config?.catalog.bitrate; + if (bitrate === undefined) return; + if (!reservation) reservation = allocator.reserve(track.demand(), bitrate); + else reservation.update(bitrate); + }); + effect.cleanup(() => reservation?.close()); }); - effect.cleanup(() => reservation?.close()); }); } @@ -548,7 +557,7 @@ export class Encoder { effect.cleanup(() => this.#failures.update((n) => n - 1)); } - // Publish the probed config before a consumer can request the demand-gated track. A + // Publish the probed config before the first frame reaches the track. A // disabled rendition keeps its last config, since muting also releases the capture, and stays // disabled after re-enabling until the reopened capture reports its format. #runCatalog(effect: Effect): void { @@ -593,7 +602,7 @@ export class Encoder { // End the epoch with a discontinuity marker (see Container.Legacy.Producer.discontinuity) where the // submitted media ends, so a subscriber that stays across the break, or joins during it, doesn't - // read the audio before it as live. Called whenever demand disappears or the pipeline stops. + // read the audio before it as live. Called whenever the track goes away or the pipeline stops. #cut(): void { const end = this.#next; this.#next = undefined; @@ -692,8 +701,8 @@ export class Encoder { const frames = framer.push(input); for (const [i, data] of frames.entries()) { - // The demand gate. The framer still consumes every sample so its timestamps stay - // on the capture clock, but there is nowhere to send a chunk with no subscriber. + // No track (with `onDemand`, no subscriber). The framer still consumes every sample + // so its timestamps stay on the capture clock, but there is nowhere to send a chunk. if (!this.#live) continue; // Round to whole microseconds once, here, so a chunk's timestamp and the marker @@ -701,9 +710,9 @@ export class Encoder { const timestamp = Math.round(data.timestamp) as Micro; // Chrome stamps encoder output from the first input's timestamp plus the samples - // encoded since, ignoring any later jump. Across a gap (demand, or a capture + // encoded since, ignoring any later jump. Across a gap (a lost track, or a capture // discontinuity) the output would trail the capture clock by the gap, and the - // next demand gap's marker would then sit ahead of everything encoded after it. + // next gap's marker would then sit ahead of everything encoded after it. // Restarting re-bases the output clock; the chunks it drops predate the gap. if (contiguous !== undefined && timestamp !== contiguous) { encoder.reset(); diff --git a/js/publish/src/broadcast.test.ts b/js/publish/src/broadcast.test.ts index 82c58d1d0a9..bcb192b3736 100644 --- a/js/publish/src/broadcast.test.ts +++ b/js/publish/src/broadcast.test.ts @@ -149,30 +149,58 @@ test("rendition.close() unregisters the name and drops it from the catalog", asy broadcast.close(); }); -test("subscriber demand hands the producer to the rendition and clears it when the track closes", async () => { +test("hands the producer to the rendition without a subscriber and clears it when the track closes", async () => { const broadcast = new Broadcast({ enabled: true, origin: new Origin.Producer(), name: Path.from("test.hang") }); await settle(); + const rendition = broadcast.video("video"); + await settle(); + + // Encoding needs no subscriber, so its groups wait in the cache for a late one. + const track = rendition.track.peek(); + expect(track).toBeDefined(); + + // Closing the producer (encoder error / teardown) clears the signal, with no lingering effect watching it. + track?.finish(); + track?.close(); + await settle(); + expect(rendition.track.peek()).toBeUndefined(); + + broadcast.close(); +}); + +test("on demand hands the producer to the rendition only while a subscriber is attached", async () => { + const broadcast = new Broadcast({ + enabled: true, + origin: new Origin.Producer(), + name: Path.from("test.hang"), + onDemand: true, + }); + await settle(); + const net = broadcast.net.peek(); if (!net) throw new Error("expected a network producer once connected"); const rendition = broadcast.video("video"); await settle(); + expect(rendition.track.peek()).toBeUndefined(); + const subscriber = net.track("video").subscribe(); await settle(); - - // The request loop accepted the subscription and handed the producer to the rendition. const track = rendition.track.peek(); expect(track).toBeDefined(); - // Closing the producer (encoder error / teardown) clears the signal, with no lingering per-subscription - // effect watching it. - track?.finish(); - track?.close(); + // The last subscriber leaving takes it back, but the broadcast keeps the track open for the next one. + subscriber.close(); await settle(); expect(rendition.track.peek()).toBeUndefined(); + expect(track?.closed.peek()).toBeUndefined(); - subscriber.close(); + const again = net.track("video").subscribe(); + await settle(); + expect(rendition.track.peek()).toBe(track); + + again.close(); broadcast.close(); }); @@ -501,7 +529,8 @@ test("a served decoder identity change moves the rendition to a fresh track name expect(config?.track).toBe("video/1"); expect(config?.codec).toBe("vp8"); expect(track.closed.peek()).toBeNull(); - expect(video.track.peek()).toBeUndefined(); + // The replacement is handed over at once, with or without a subscriber. + expect(video.track.peek()?.name).toBe("video/1"); expect(broadcast.net.peek()).toBe(net); expect(announced?.active.peek()?.epoch).toBe(epoch); @@ -690,15 +719,28 @@ test("a name returning with a different identity takes a fresh track name", asyn broadcast.close(); }); -// Nobody has seen an unserved catalog, so its renditions settle without minting names. -test("identity changes before the first announce keep the catalog ID as the track name", async () => { +// Regression: identity was fixed only at the first announce, but an eager encoder caches groups +// before then. A codec change while unannounced kept the name, so a late subscriber or a fetch read +// the old codec's groups under the new catalog config. +test("identity changes before the first announce take a fresh track name", async () => { const announce = new Signal(false); const broadcast = new Broadcast({ origin: new Origin.Producer(), name: Path.from("unserved.hang"), announce }); const video = broadcast.video("video"); video.config.set({ ...videoConfig, bitrate: Catalog.u53(1_000_000) }); await settle(); + // The encoder writes before anyone can find the broadcast. + const cached = video.track.peek(); + if (!cached) throw new Error("expected the track to be handed over before the first announce"); + cached.appendGroup().finish(); + + // Everything except identity still updates the track in place. + video.config.set({ ...videoConfig, bitrate: Catalog.u53(500_000) }); + await settle(); + expect(video.track.peek()).toBe(cached); + video.config.set({ ...videoConfig, codec: "vp8", bitrate: Catalog.u53(500_000) }); await settle(); + expect(cached.closed.peek()).toBeDefined(); announce.set(true); await settle(); @@ -710,12 +752,12 @@ test("identity changes before the first announce keep the catalog ID as the trac const config = (await catalog.latest())?.value.video?.renditions.video; expect(config?.codec).toBe("vp8"); expect(config?.bitrate).toBe(Catalog.u53(500_000)); - expect(config?.track).toBeUndefined(); + expect(config?.track).toBe("video/1"); - // The first snapshot fixed the name's identity. - video.config.set(videoConfig); - await settle(); - expect((await catalog.latest())?.value.video?.renditions.video?.track).toBe("video/1"); + // The replacement starts empty, so no group from the old codec reads under the new config. + const replacement = video.track.peek(); + expect(replacement?.name).toBe("video/1"); + expect(replacement?.appendGroup().sequence).toBe(0); broadcast.close(); }); diff --git a/js/publish/src/broadcast.ts b/js/publish/src/broadcast.ts index 1f835f1d9e9..a790e23ff35 100644 --- a/js/publish/src/broadcast.ts +++ b/js/publish/src/broadcast.ts @@ -58,13 +58,23 @@ export type BroadcastInput = { * so changing this publishes a new network broadcast. */ maxAge: Getter; + + /** + * Whether every rendition encodes only while a subscriber is attached. Defaults to false: an + * enabled rendition encodes with or without a subscriber, so its recent groups sit in the + * track's cache for a late subscription's `maxDelay` or a fetch to reach within `maxAge`. + * + * On demand saves the encode while nobody watches, but a subscriber then finds no past to reach + * back into: each one starts at the keyframe its arrival forces. Capture stays open either way. + */ + onDemand: Getter; }; /** * A published broadcast: the network broadcast plus a catalog producer and its rendition tracks. * * Register renditions with {@link video} / {@link audio}; each returns a {@link Rendition} whose - * producer (usually an encoder) fills the catalog config and encodes into the demand-gated track. + * producer (usually an encoder) fills the catalog config and encodes into its track. * The broadcast owns only its own network/catalog wiring, so {@link close} does not close the * renditions' producers. */ @@ -105,7 +115,8 @@ export class Broadcast { readonly #renditions = new Signal>>({}); // The writable track producer signals backing each Rendition's read-only `track`, keyed by name. - // A static network track is exposed here only while at least one subscriber uses it. + // A static network track is exposed here while it is open, and with `onDemand` only while at + // least one subscriber uses it. readonly #tracks = new Map>(); // The kind of each rendition track in the current network broadcast. A new one starts over. @@ -119,12 +130,14 @@ export class Broadcast { // Every replacement track name minted in the current network broadcast, even one never served. #replacements = new Set(); - // The decoder identity of each track name the current network broadcast's catalog has served, or - // undefined before it serves one. Kept across a rendition's removal, so a name returning with + // The decoder identity of each track name in the current network broadcast, or undefined before + // there is one. Fixed from the broadcast's start rather than its first announce, since an encoder + // may cache groups on a track before anyone can find it, and a late subscriber or a fetch reads + // them under the catalog's config. Kept across a rendition's removal, so a name returning with // different media takes a fresh name too. #served: Map | undefined; - // The identity of each track name in the latest catalog fold, recorded when the catalog is first served. + // The identity of each track name in the latest catalog fold, recorded when a network broadcast starts. #listed = new Map(); #signals = new Effect(); @@ -139,6 +152,7 @@ export class Broadcast { flip: getter(props?.flip ?? false), rotation: getter(props?.rotation ?? 0), maxAge: getter(props?.maxAge), + onDemand: getter(props?.onDemand ?? false), }; this.#signals.run(this.#runCatalog.bind(this)); @@ -361,7 +375,8 @@ export class Broadcast { const tracks = new Map }>(); this.#names.set({}); this.#replacements = new Set(); - this.#served = undefined; + // From here a name keeps the identity it is first listed with. + this.#served = new Map(this.#listed); const kinds = new Map(); this.#kinds = kinds; effect.cleanup(() => { @@ -382,8 +397,6 @@ export class Broadcast { // is whatever the catalog holds at announce time. if (!serving) { serving = true; - // From here a name keeps the identity this first snapshot gives it. - this.#served = new Map(this.#listed); // A catalog may publish once and stay unchanged for the broadcast's whole life. Keep that // sole closed snapshot replayable so a viewer arriving after the ordinary media retention // window can still bootstrap. @@ -405,8 +418,9 @@ export class Broadcast { broadcastPublication.announce({ epoch }); }); - // Static tracks fan out to every subscriber. Keep the encoder-facing handle demand-gated - // so capture and encoding still stop when the final subscriber leaves. + // Static tracks fan out to every subscriber. The encoder-facing handle stays set while the + // track is open, so its groups wait in the cache for a late subscriber. On demand gates it on + // a subscriber too, so encoding stops when the final one leaves. effect.run((inner) => { const renditions = inner.get(this.#renditions); const names = inner.get(this.#names); @@ -452,8 +466,10 @@ export class Broadcast { if (cause) served.abort(cause); }); } - inner.run((demand) => { - demand.set(signal, demand.get(served.demand().wanted) !== undefined ? served : undefined); + inner.run((gate) => { + if (gate.get(served.closed) !== undefined) return; + if (gate.get(this.in.onDemand) && gate.get(served.demand().wanted) === undefined) return; + gate.set(signal, served); }); } }); diff --git a/js/publish/src/internal.ts b/js/publish/src/internal.ts index 0261eb09264..30ef43dc943 100644 --- a/js/publish/src/internal.ts +++ b/js/publish/src/internal.ts @@ -1,4 +1,13 @@ -import type { Getter } from "@moq/signals"; +import type * as Moq from "@moq/net"; +import type { Effect, Getter } from "@moq/signals"; /** @internal Terminal failures of broadcast-owned rendition tracks, including idle tracks. */ export const errors = new WeakMap>(); + +/** + * @internal Whether a subscriber is attached to `track`, scoped to `effect`. Changes only when that + * does, not with every group the subscribers want, so a bandwidth reservation keyed on it holds. + */ +export function watched(track: Moq.Track.Producer, effect: Effect): Getter { + return effect.computed((demand) => demand.get(track.demand().wanted) !== undefined); +} diff --git a/js/publish/src/rendition.ts b/js/publish/src/rendition.ts index 0fe646a6038..264a8cf0d02 100644 --- a/js/publish/src/rendition.ts +++ b/js/publish/src/rendition.ts @@ -5,7 +5,7 @@ import { type Getter, Signal } from "@moq/signals"; export type Kind = "video" | "audio" | "text"; /** - * A registered rendition track on a {@link Broadcast}: the catalog slot plus the demand-gated track. + * A registered rendition track on a {@link Broadcast}: the catalog slot plus its track. * * Constructed only by {@link Broadcast} via `broadcast.video(name)` / `broadcast.audio(name)`; the * producer (usually an encoder) writes {@link config} and encodes into {@link track} while it's set. @@ -28,9 +28,10 @@ export class Rendition { readonly config = new Signal(undefined); /** - * The live track producer while a subscriber is attached, `undefined` otherwise. - * Producers should encode only while this is set (the demand gate), and only media matching - * the current {@link config}: a new identity swaps in a new track. + * The open track producer, or `undefined` while the network broadcast is down, after the track + * closes, and, with the broadcast's `onDemand`, while no subscriber is attached. Producers should + * encode only while this is set, and only media matching the current {@link config}: a new + * identity swaps in a new track. */ readonly track: Getter; diff --git a/js/publish/src/video/encoder.test.ts b/js/publish/src/video/encoder.test.ts index cd809ecd4f4..beeb425b13f 100644 --- a/js/publish/src/video/encoder.test.ts +++ b/js/publish/src/video/encoder.test.ts @@ -148,7 +148,7 @@ test("encoding tracks encoder config in its child effect", async () => { } }); -test("a demand gap marks a discontinuity and leaves the broadcast-owned track open for resume", async () => { +test("an on-demand gap marks a discontinuity and leaves the broadcast-owned track open for resume", async () => { using _videoEncoder = installFakeVideoEncoder(); const discontinuity = spyOn(Container.Legacy.Producer.prototype, "discontinuity"); @@ -889,65 +889,62 @@ test("a re-probe swaps the catalog entry in one update", async () => { // Drive a live encoder over a recording `VideoEncoder` that outputs every frame it is handed, so the // track holds real groups: a keyframe of 100 bytes, each delta 60. `run` sends one frame per call // at `millis` of media time and reads back which frames were forced to be keyframes. -async function recordKeyframes( - info: Partial, - run: (ctx: { - encoder: Encoder; - track: Moq.Track.Producer; - send: (millis: number) => Promise; - }) => Promise, -): Promise { - const keys: number[] = []; - - class Frame { - readonly timestamp: number; - constructor(timestamp: number) { - this.timestamp = timestamp; - } - clone(): Frame { - return new Frame(this.timestamp); - } - close(): void {} +// A captured frame, stamped in microseconds. +class Frame { + readonly timestamp: number; + constructor(timestamp: number) { + this.timestamp = timestamp; } + clone(): Frame { + return new Frame(this.timestamp); + } + close(): void {} +} - class RecordingVideoEncoder { - state: CodecState = "unconfigured"; - #output: VideoEncoderInit["output"]; - #codec?: string; +class RecordingVideoEncoder { + // The timestamps (ms) of the test frames the encoder was asked to make keyframes. + static keys: number[] = []; - constructor(init: VideoEncoderInit) { - this.#output = init.output; - } + state: CodecState = "unconfigured"; + #output: VideoEncoderInit["output"]; + #codec?: string; - static async isConfigSupported(config: VideoEncoderConfig): Promise<{ supported: boolean }> { - return { supported: config.codec.startsWith("avc1") }; - } + constructor(init: VideoEncoderInit) { + this.#output = init.output; + } - configure(config: VideoEncoderConfig): void { - this.state = "configured"; - this.#codec = config.codec; - } + static async isConfigSupported(config: VideoEncoderConfig): Promise<{ supported: boolean }> { + return { supported: config.codec.startsWith("avc1") }; + } - // Records the test's frames only, not the probe's, and outputs each a task later as WebCodecs does. - encode(frame: VideoFrame, options?: VideoEncoderEncodeOptions): void { - if (!(frame instanceof Frame)) return; - const key = !!options?.keyFrame; - if (key) keys.push(frame.timestamp / 1000); - const chunk = { type: key ? "key" : "delta", timestamp: frame.timestamp, byteLength: key ? 100 : 60 }; - queueMicrotask(() => this.#output({ ...chunk, copyTo: () => {} } as never, {} as never)); - } + configure(config: VideoEncoderConfig): void { + this.state = "configured"; + this.#codec = config.codec; + } - // Only the probe flushes; report the configured codec, as Chrome does. - async flush(): Promise { - const chunk = { type: "key", timestamp: 0, byteLength: 1, copyTo: () => {} }; - this.#output(chunk as never, { decoderConfig: { codec: this.#codec } } as never); - } + // Records the test's frames only, not the probe's, and outputs each a task later as WebCodecs does. + encode(frame: VideoFrame, options?: VideoEncoderEncodeOptions): void { + if (!(frame instanceof Frame)) return; + const key = !!options?.keyFrame; + if (key) RecordingVideoEncoder.keys.push(frame.timestamp / 1000); + const chunk = { type: key ? "key" : "delta", timestamp: frame.timestamp, byteLength: key ? 100 : 60 }; + queueMicrotask(() => this.#output({ ...chunk, copyTo: () => {} } as never, {} as never)); + } - close(): void { - this.state = "closed"; - } + // Only the probe flushes; report the configured codec, as Chrome does. + async flush(): Promise { + const chunk = { type: "key", timestamp: 0, byteLength: 1, copyTo: () => {} }; + this.#output(chunk as never, { decoderConfig: { codec: this.#codec } } as never); } + close(): void { + this.state = "closed"; + } +} + +// Install the recording encoder and a capture whose frames `send` feeds, one per call. +async function installRecording() { + RecordingVideoEncoder.keys = []; const original = Object.getOwnPropertyDescriptor(globalThis, "VideoEncoder"); Object.defineProperty(globalThis, "VideoEncoder", { configurable: true, @@ -968,6 +965,40 @@ async function recordKeyframes( { clone: (frame) => frame.clone(), release: (frame) => frame.close() }, ); + const capture = { + in: { source: new Signal({ getSettings: () => ({ frameRate: 30 }), getConstraints: () => ({}) } as never) }, + out: { display: new Signal({ width: 640, height: 480 }), frames: new Signal(fanout) }, + }; + + const send = async (millis: number) => { + controller.enqueue(new Frame(millis * 1000) as unknown as VideoFrame); + await clock.advance(PROBE); + }; + + return { + capture, + send, + keys: RecordingVideoEncoder.keys, + [Symbol.dispose]() { + fanout.close(); + if (original) Object.defineProperty(globalThis, "VideoEncoder", original); + else Reflect.deleteProperty(globalThis, "VideoEncoder"); + if (videoFrame) Object.defineProperty(globalThis, "VideoFrame", videoFrame); + else Reflect.deleteProperty(globalThis, "VideoFrame"); + }, + }; +} + +async function recordKeyframes( + info: Partial, + run: (ctx: { + encoder: Encoder; + track: Moq.Track.Producer; + send: (millis: number) => Promise; + }) => Promise, +): Promise { + using recording = await installRecording(); + const track = new Moq.Track.Producer("video").accept({ timescale: Moq.Time.Timescale.MICRO, ...info }); const rendition = { config: new Signal(undefined), @@ -977,34 +1008,20 @@ async function recordKeyframes( track.close(); }, }; - const capture = { - in: { source: new Signal({ getSettings: () => ({ frameRate: 30 }), getConstraints: () => ({}) } as never) }, - out: { display: new Signal({ width: 640, height: 480 }), frames: new Signal(fanout) }, - }; const encoder = new Encoder("video", { enabled: true, broadcast: { video: () => rendition, order: new Order() } as never, - capture: capture as never, + capture: recording.capture as never, }); - const send = async (millis: number) => { - controller.enqueue(new Frame(millis * 1000) as unknown as VideoFrame); - await clock.advance(PROBE); - }; - try { await settle(); - await run({ encoder, track, send }); - return keys; + await run({ encoder, track, send: recording.send }); + return recording.keys; } finally { encoder.close(); - fanout.close(); track.finish(); track.close(); - if (original) Object.defineProperty(globalThis, "VideoEncoder", original); - else Reflect.deleteProperty(globalThis, "VideoEncoder"); - if (videoFrame) Object.defineProperty(globalThis, "VideoFrame", videoFrame); - else Reflect.deleteProperty(globalThis, "VideoFrame"); } } @@ -1065,6 +1082,120 @@ test("a join over the refresh ratio forces a keyframe on the next frame", async expect(keys).toEqual([0, 250]); }); +test("a join that a written group serves forces no keyframe, even one encoded without a subscriber", async () => { + const keys = await recordKeyframes({ refresh: 2 }, async ({ encoder, track, send }) => { + await send(0); + encoder.cut(); + await send(100); + // One 60 byte delta is under twice the 100 byte keyframe, so the join reads the open group. + await send(150); + const joined = track.subscribe(); + try { + await send(200); + await send(250); + } finally { + joined.close(); + } + }); + expect(keys).toEqual([0, 100]); +}); + +// Groups encoded with nobody watching wait in the cache for a late subscriber's maxDelay, while on +// demand a subscriber finds only what its arrival started. +test.each([false, true])( + "a late subscriber reaches groups encoded before it arrived unless on demand (%p)", + async (onDemand) => { + using recording = await installRecording(); + const broadcast = new Broadcast({ + origin: new Moq.Origin.Producer(), + name: Moq.Path.from("eager.hang"), + onDemand, + }); + const encoder = new Encoder("video", { enabled: true, broadcast, capture: recording.capture as never }); + let late: Moq.Track.Subscriber | undefined; + try { + await settle(); + const net = broadcast.net.peek(); + if (!net) throw new Error("expected network broadcast"); + expect(encoder.out.active.peek()).toBe(!onDemand); + + await recording.send(0); + encoder.cut(); + await recording.send(100); + await recording.send(150); + expect(recording.keys).toEqual(onDemand ? [] : [0, 100]); + + late = net.track("video").subscribe({ maxDelay: Milli(10_000) }); + await settle(); + expect(encoder.out.active.peek()).toBe(true); + await recording.send(200); + + const ordered = late.ordered(); + const groups: number[] = []; + const firsts: number[] = []; + for (let i = 0; i < (onDemand ? 1 : 2); i++) { + const group = await ordered.nextGroup(); + if (!group) throw new Error("expected a group"); + groups.push(group.sequence); + const frame = await group.readFrame(); + if (!frame) throw new Error("expected a frame"); + firsts.push(Moq.Varint.decode(frame.payload)[0] / 1000); + } + if (onDemand) { + // The subscriber's arrival started the encoder, so its first frame opened the only group. + expect(recording.keys).toEqual([200]); + expect(groups).toEqual([0]); + expect(firsts).toEqual([200]); + } else { + // Both groups were cached before it arrived, and its join forced no fresh keyframe. + expect(recording.keys).toEqual([0, 100]); + expect(groups).toEqual([0, 1]); + expect(firsts).toEqual([0, 100]); + } + } finally { + late?.close(); + encoder.close(); + broadcast.close(); + } + }, +); + +test("an unwatched rendition reserves nothing and encodes at its configured bitrate", async () => { + using _videoEncoder = installFakeVideoEncoder(); + const broadcast = new Broadcast({ origin: new Moq.Origin.Producer(), name: Moq.Path.from("unwatched.hang") }); + const capture = { + in: { source: new Signal({ getSettings: () => ({ frameRate: 30 }), getConstraints: () => ({}) } as never) }, + out: { display: new Signal({ width: 640, height: 480 }), frames: new Signal(undefined) }, + }; + const bandwidth = new Moq.Bandwidth.Allocator(new Signal(200_000)); + const encoder = new Encoder("video", { enabled: true, broadcast, capture: capture as never, bandwidth }); + let subscriber: Moq.Track.Subscriber | undefined; + try { + await settle(); + const net = broadcast.net.peek(); + if (!net) throw new Error("expected network broadcast"); + expect(encoder.out.active.peek()).toBe(true); + const configured = encoder.out.resolved.peek()?.bitrate; + if (configured === undefined) throw new Error("expected a resolved config"); + expect(configured).toBeGreaterThan(200_000); + + subscriber = net.track("video").subscribe(); + await settle(); + expect(encoder.out.resolved.peek()?.bitrate).toBe(200_000); + + subscriber.close(); + subscriber = undefined; + await settle(); + expect(encoder.out.active.peek()).toBe(true); + expect(encoder.out.resolved.peek()?.bitrate).toBe(configured); + } finally { + subscriber?.close(); + encoder.close(); + bandwidth.close(); + broadcast.close(); + } +}); + test("a failed video encoder disables only its rendition and recovers on a fresh track", async () => { using _videoEncoder = installFakeVideoEncoder(); using _probe = spyOn(FakeVideoEncoder, "isConfigSupported").mockResolvedValue({ supported: true }); @@ -1097,7 +1228,8 @@ test("a failed video encoder disables only its rendition and recovers on a fresh await flush(); siblingSubscriber = net.track("video/sd").subscribe(); subscriber = net.track("video/hd").subscribe(); - await encoder.out.active.changed(); + // It encodes while enabled, so it was already active before anyone subscribed. + expect(encoder.out.active.peek()).toBe(true); await flush(); const siblingTrack = sibling.track.peek(); if (!siblingTrack) throw new Error("expected demanded sibling track"); diff --git a/js/publish/src/video/encoder.ts b/js/publish/src/video/encoder.ts index 22fb12cb899..ad72d62e27d 100644 --- a/js/publish/src/video/encoder.ts +++ b/js/publish/src/video/encoder.ts @@ -14,7 +14,7 @@ import { Signal, } from "@moq/signals"; import type { Broadcast } from "../broadcast"; -import { errors } from "../internal"; +import { errors, watched } from "../internal"; import { Estimator, type Order } from "../order"; import { hardwareReliable } from "../support/video"; import type { Capture } from "./capture"; @@ -88,8 +88,8 @@ export type EncoderInput = { // The capture pipeline supplying frames and the source track. capture: Getter; - // The connection's bandwidth allocator. The encoder reserves its ceiling and - // follows the grant instead of the whole-session estimate. + // The connection's bandwidth allocator. While a subscriber is attached, the encoder + // reserves its ceiling and follows the grant instead of the whole-session estimate. bandwidth: Getter; }; @@ -109,7 +109,8 @@ export type EncoderOutput = { // The resolved WebCodecs config (codec, bitrate, dimensions), available even with no subscriber. // Exposed so a local preview can re-encode with identical settings to mirror the wire output. resolved: Signal; - // True when a subscriber is attached and we're encoding. + // True while encoding: enabled and the rendition has a track, which with the broadcast's + // `onDemand` also takes a subscriber. active: Signal; // Cumulative output totals (frames, bytes, keyframes) measured while serving. stats: Signal; @@ -119,8 +120,10 @@ export type EncoderOutput = { * A single video rendition encoder. * * Registers itself on the {@link Broadcast} under {@link name} (via `broadcast.video(name)`), resolves - * the best codec/bitrate/dimensions for the capture, and encodes frames only while a subscriber is - * attached (the demand gate). Rename by constructing a new encoder; the name is not a signal. + * the best codec/bitrate/dimensions for the capture, and encodes frames while enabled, so a late + * subscriber or fetch finds the recent groups in the track's cache. The broadcast's `onDemand` encodes + * only while a subscriber is attached instead. A joining subscriber that wants a newer group gets a + * fresh keyframe either way. Rename by constructing a new encoder; the name is not a signal. * * The catalog advertises the largest size encoded so far as the coded-size ceiling, so a capture that * shrinks resizes in band at a keyframe. Outgrowing the ceiling re-probes and advertises a larger one, @@ -338,7 +341,7 @@ export class Encoder { // Publish the resolved catalog config; undefined (until one resolves) leaves it out of the catalog. effect.proxy(rendition.config, this.out.catalog); - // Encode only while enabled and a subscriber is attached (the demand gate). + // Encode while enabled and the broadcast hands over the track (with `onDemand`, a subscriber is attached). effect.run((effect) => { const enabled = effect.get(this.in.enabled); const track = effect.get(rendition.track); @@ -349,29 +352,34 @@ export class Encoder { this.#encode(track, broadcast.order, effect); }); - // Reserve against the connection for as long as this track is live. Wait - // for a ceiling so we never claim 0 and starve siblings for a tick. The - // allocator ignores an idle track, and closing the reservation hands the - // room to siblings. + // Reserve against the connection while a subscriber is attached. Wait for a ceiling so we + // never claim 0 and starve siblings for a tick. Closing the reservation hands the room to + // siblings, and an unwatched rendition encodes at its configured bitrate: its backlog is + // served from the cache rather than paced live. effect.run((effect) => { const enabled = effect.get(this.in.enabled); const track = effect.get(rendition.track); const allocator = effect.get(this.in.bandwidth); if (!enabled || !track || !allocator || effect.get(this.#out.error)) return; - let reservation: Moq.Bandwidth.Reservation | undefined; - effect.subscribe(this.#ceiling, (ceiling) => { - if (ceiling === undefined) return; - if (!reservation) { - reservation = allocator.reserve(track.demand(), ceiling); - this.#reservation.set(reservation); - } else { - reservation.update(ceiling); - } - }); - effect.cleanup(() => { - reservation?.close(); - if (this.#reservation.peek() === reservation) this.#reservation.set(undefined); + const demanded = watched(track, effect); + effect.run((effect) => { + if (!effect.get(demanded)) return; + + let reservation: Moq.Bandwidth.Reservation | undefined; + effect.subscribe(this.#ceiling, (ceiling) => { + if (ceiling === undefined) return; + if (!reservation) { + reservation = allocator.reserve(track.demand(), ceiling); + this.#reservation.set(reservation); + } else { + reservation.update(ceiling); + } + }); + effect.cleanup(() => { + reservation?.close(); + if (this.#reservation.peek() === reservation) this.#reservation.set(undefined); + }); }); }); } @@ -388,13 +396,15 @@ export class Encoder { let lastEncoded: Micro | undefined; // The catalog declares no reordering, so each chunk must present after the one before it. let lastOutput: number | undefined; - // The largest group demand asked for that a keyframe already answers. A join over the refresh - // ratio asks for one past the largest, which the next frame opens. + // The largest group demand asked for that a keyframe already answers, including one still in + // the encoder. A join over the refresh ratio asks for one past the largest, which the next frame + // opens; any other join is served by a group already written, encoded with or without demand. let answered: number | undefined; const answer = () => { const wanted = demand.wanted.peek(); if (wanted !== undefined) answered = Math.max(answered ?? 0, wanted); }; + const covered = () => Math.max(answered ?? -1, track.largest()?.group ?? -1); let tearingDown = false; effect.spawn(async () => { @@ -509,7 +519,7 @@ export class Encoder { since === undefined || since >= Micro.fromMilli(interval) || this.#cut || - (wanted !== undefined && (answered === undefined || wanted > answered)); + (wanted !== undefined && wanted > covered()); if (keyFrame) { lastKeyframe = frame.timestamp as Micro; resized = false; @@ -559,9 +569,9 @@ export class Encoder { // A stopped rendition stops holding the others' delay back. Not a cleanup: the black // keyframe above is written after one would run, and would seat this rendition again. order.leave(this.#estimator); - // The broadcast owns this static track across demand gaps. Close the current group, marking - // the break so a later subscriber resumes on the same track without the pre-gap group reading - // as live. A fatal encoder error aborts the track instead. + // The broadcast owns this static track across gaps (a disable, or with `onDemand` a demand + // gap). Close the current group, marking the break so a later subscriber resumes on the same + // track without the pre-gap group reading as live. A fatal encoder error aborts the track instead. if (track.closed.peek() === undefined) producer.discontinuity(); } }); diff --git a/quest/m1/eager-encode/README.md b/quest/m1/eager-encode/README.md index 64a29d37dd8..78355f4082e 100644 --- a/quest/m1/eager-encode/README.md +++ b/quest/m1/eager-encode/README.md @@ -49,7 +49,6 @@ The epic's own work, once the children land: ## Required -- [@moq/publish encodes while enabled](/quest/m1/eager-encode/js.md) - audio and video encoders drop the demand gate, with an on-demand opt-in - [Raw audio on demand](/quest/m1/eager-encode/raw-audio.md) - audio encodes on demand again, from raw frames kept up to `maxAge` ## Related diff --git a/quest/m1/eager-encode/js.md b/quest/m1/eager-encode/js.md deleted file mode 100644 index 7cf989ef288..00000000000 --- a/quest/m1/eager-encode/js.md +++ /dev/null @@ -1,29 +0,0 @@ -# [M] @moq/publish encodes while enabled - -## Goal - -`@moq/publish` audio and video encoders run whenever their rendition is -enabled, with or without a subscriber. Their groups sit in the track's cache -for a late SUBSCRIBE or FETCH to reach within its `maxDelay`. An `onDemand` -option restores today's gate. - -## Plan - -Decided 2026-10-28 (maintainer); see the [epic](/quest/m1/eager-encode/README.md). - -- The root gate: `js/publish/src/broadcast.ts:413-462` sets `rendition.track` - only while `served.demand().wanted` is defined. Both encoders key off it - (`video/encoder.ts:341-349`, `audio/encoder.ts:401-416` and `:686-688`). -- Losing demand no longer ends an epoch (`#cut()` on demand loss). Disabling - still does. -- Video still answers a join that wants a newer group with a fresh keyframe - (`demand.wanted`). -- `onDemand` is one option on the broadcast, applying to every rendition. - `doc/lib/js/publish.md:146` describes both modes. - -Tests: an enabled encoder with no subscriber writes groups. A subscriber that -arrives later with a covering `maxDelay` receives the earlier groups. -`onDemand` keeps today's behavior. - -Public API: default behavior change, plus an additive `onDemand` option. -Wire: none. diff --git a/quest/m1/eager-encode/raw-audio.md b/quest/m1/eager-encode/raw-audio.md index d82035b62d6..44738aea993 100644 --- a/quest/m1/eager-encode/raw-audio.md +++ b/quest/m1/eager-encode/raw-audio.md @@ -30,7 +30,3 @@ covering the last second receives that second, timestamps matching capture time. Memory stays bounded by `maxAge` or the cap. Public API: default behavior change for audio. Wire: none. - -## Required - -- [@moq/publish encodes while enabled](/quest/m1/eager-encode/js.md) - the JS always-on path this switches from