Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion doc/lib/js/publish.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,15 @@ for a permission, device, or settings change instead of prompting again.

## Encoding

An enabled encoder encodes whether or not anyone is subscribed, so its recent
groups wait in the track's cache: a viewer who subscribes late with a
`maxDelay`, or fetches, reaches back as far as `maxAge` keeps. Pass
`onDemand: true` to `Publish.Broadcast` to encode every rendition only while
someone is subscribed instead. That saves the encode while nobody watches, but
a viewer then starts at the keyframe its arrival forces, with no past to reach.
Bandwidth is reserved only while someone is subscribed, so an unwatched
rendition encodes at its configured bitrate.

The video encoder follows its share of the connection's send-rate estimate, so
several publishers on one connection do not each target the whole uplink.
Codec, resolution, framerate, and bitrate are tunable through
Expand Down Expand Up @@ -143,7 +152,8 @@ const broadcast = new Publish.Broadcast({
});

// Each encoder registers a rendition on the broadcast (`broadcast.video(name)`) and
// encodes only while someone is subscribed.
// encodes while enabled, subscribed or not. Pass `onDemand: true` to the broadcast
// to encode only while someone is subscribed.
new Publish.Video.Encoder("video/hd", { broadcast, capture, enabled: true });
new Publish.Video.Encoder("video/sd", { broadcast, capture, enabled: true, config: { maxScale: 0.25 } });
const audioSource = new Publish.Signals.Computed((effect) => effect.get(microphone.out.source)?.audio);
Expand Down
5 changes: 3 additions & 2 deletions js/publish/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,9 @@ const broadcast = new Publish.Broadcast({
rotation: capture.out.rotation,
});

// Each encoder registers its rendition on the broadcast and encodes only while
// someone is subscribed.
// Each encoder registers its rendition on the broadcast and encodes while enabled,
// subscribed or not, so a late viewer can reach back into the cache. Pass
// `onDemand: true` to the broadcast to encode only while someone is subscribed.

const hd = new Publish.Video.Encoder("video/hd", { broadcast, capture, enabled: true });
const sd = new Publish.Video.Encoder("video/sd", { broadcast, capture, enabled: true, config: { maxScale: 0.25 } });
Expand Down
1 change: 1 addition & 0 deletions js/publish/api/index.api.md
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ export type BroadcastInput = {
flip: Signals.Getter<boolean>;
maxAge: Signals.Getter<Clock.Milli | undefined>;
name: Signals.Getter<Net.Path.Valid>;
onDemand: Signals.Getter<boolean>;
origin: Signals.Getter<Net.Origin.Table | undefined>;
rotation: Signals.Getter<number>;
};
Expand Down
68 changes: 66 additions & 2 deletions js/publish/src/audio/encoder.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { describe, expect, mock, spyOn, test } from "bun:test";
import { Micro, Milli } from "@moq/clock";
import * as Catalog from "@moq/hang/catalog";
import * as Container from "@moq/hang/container";
import * as Json from "@moq/json";
import * as Moq from "@moq/net";
import { Signal } from "@moq/signals";
Expand Down Expand Up @@ -325,7 +326,7 @@ async function setup(order = new Order(), codec?: Codec, groupDuration?: Milli)
// resume. Written after the marker, they would put pre-gap media on the live edge, and a rounding
// step below the marker aborts every subscriber. The resumed chunks have to carry the capture clock,
// not the encoder's gap-blind one, or they trail the next gap's marker and are dropped as pre-gap.
test("a demand gap marks where submitted audio ends and drops the chunks held across it", async () => {
test("an on-demand gap marks where submitted audio ends and drops the chunks held across it", async () => {
using _webcodecs = installFakeWebCodecs();
using env = await setup();
const { track, rendition, feed, written, writes } = env;
Expand Down Expand Up @@ -929,7 +930,8 @@ test.each(["failure first", "config first"])(
expect(config?.track).toBeDefined();
expect(config?.track).not.toBe("audio");
expect(encoder.out.error.peek()).toBeUndefined();
expect(encoder.out.active.peek()).toBe(false);
// Encoding resumes on the fresh track without waiting for a subscriber.
expect(encoder.out.active.peek()).toBe(true);
expect(broadcast.net.peek()).toBe(net);
} finally {
catalogSubscriber?.close();
Expand All @@ -939,6 +941,68 @@ test.each(["failure first", "config first"])(
},
);

// Audio encoded with nobody listening waits in the cache for a late subscriber's maxDelay, and the
// last subscriber leaving ends no epoch. On demand, a subscriber finds only what its arrival started.
test.each([false, true])(
"a late subscriber reaches audio encoded before it arrived unless on demand (%p)",
async (onDemand) => {
using _webcodecs = installFakeWebCodecs();
const broadcast = new Broadcast({
origin: new Moq.Origin.Producer(),
name: Moq.Path.from("eager.hang"),
onDemand,
});
const feed = new Feed();
const configuring = configured();
const encoder = new Encoder("audio", { broadcast, capture: fakeCapture(feed) as never });
const discontinuity = spyOn(Container.Legacy.Producer.prototype, "discontinuity");
let late: Moq.Track.Subscriber | undefined;
try {
await configuring;
await settle();
const net = broadcast.net.peek();
if (!net) throw new Error("expected network broadcast");
expect(encoder.out.active.peek()).toBe(!onDemand);

// One 20ms Opus frame per push, each its own group.
let index = 0;
const push = async (count: number) => {
for (let i = 0; i < count; i++, index++) {
await feed.push({ timestamp: Micro(index * 20_000), channels: [new Float32Array(960)] });
}
await feed.drain();
};

await push(4); // unwatched: two written, two held
late = net.track("audio").subscribe({ maxDelay: Milli(10_000) });
await settle();
expect(encoder.out.active.peek()).toBe(true);
await push(3);

const group = await late.ordered().nextGroup();
if (!group) throw new Error("expected a group");
const frame = await group.readFrame();
if (!frame) throw new Error("expected a frame");
const [timestamp] = Moq.Varint.decode(frame.payload);
// On demand, nothing was submitted before the subscriber arrived.
expect(group.sequence).toBe(0);
expect(timestamp).toBe(onDemand ? 80_000 : 0);

// Only on demand does the last subscriber leaving end the epoch.
late.close();
late = undefined;
await settle();
expect(discontinuity).toHaveBeenCalledTimes(onDemand ? 1 : 0);
expect(encoder.out.active.peek()).toBe(!onDemand);
} finally {
discontinuity.mockRestore();
late?.close();
encoder.close();
broadcast.close();
}
},
);

test("Opus description is probed with the exact live config before advertising", async () => {
using _codecs = installFakeWebCodecs();
LaggingAudioEncoder.description = (config) => new TextEncoder().encode(JSON.stringify(config));
Expand Down
67 changes: 38 additions & 29 deletions js/publish/src/audio/encoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import {
Signal,
} from "@moq/signals";
import type { Broadcast } from "../broadcast";
import { errors } from "../internal";
import { errors, watched } from "../internal";
import { Estimator, type Order } from "../order";
import type { AudioFrame, Capture, Format } from "./capture";
import { Gain } from "./gain";
Expand Down Expand Up @@ -98,8 +98,9 @@ export type EncoderInput = {
// yourself and pass the same instance to each.
capture: Getter<Capture | undefined>;

// The connection's bandwidth allocator. Audio reserves its configured bitrate so
// video's share is honest, and ignores the grant (Opus is a fixed rate today).
// The connection's bandwidth allocator. While a subscriber is attached, audio reserves
// its configured bitrate so video's share is honest, and ignores the grant (Opus is a
// fixed rate today).
bandwidth: Getter<Moq.Bandwidth.Handle | undefined>;
};

Expand All @@ -125,7 +126,8 @@ export type EncoderOutput = {
// The head of the capture graph, so callers can tap the raw capture. Volume is applied to the
// PCM rather than in the graph, so this is pre-gain. Undefined for a source that isn't a track.
root: Signal<AudioNode | undefined>;
// True when a subscriber is attached and we're encoding.
// True while encoding: enabled with a captured format and a track, which with the broadcast's
// `onDemand` also takes a subscriber.
active: Signal<boolean>;
// Cumulative output totals (frames, bytes) measured while serving.
stats: Signal<Stats>;
Expand All @@ -142,8 +144,9 @@ type Pipeline = {
* A single audio rendition encoder.
*
* Registers itself on the {@link Broadcast} under {@link name} (via `broadcast.audio(name)`), pumps PCM
* off the source via a {@link Capture}, and encodes it only while a subscriber is attached (the demand
* gate). Rename by constructing a new encoder; the name is not a signal.
* off the source via a {@link Capture}, and encodes it while enabled, so a late subscriber or fetch
* finds the recent groups in the track's cache. The broadcast's `onDemand` encodes only while a
* subscriber is attached instead. Rename by constructing a new encoder; the name is not a signal.
*/
export class Encoder {
/** The full track name of this rendition, e.g. `"audio/data"`. */
Expand Down Expand Up @@ -173,7 +176,7 @@ export class Encoder {
return this.in.capture.peek();
}

// The exact encoder settings and probed decoder description advertised before demand arrives.
// The exact encoder settings and probed decoder description, advertised before the first frame.
#config = new Signal<(Resolved & { encoder: AudioEncoderConfig }) | undefined>(undefined);

// The last Opus probe: the encoder settings it ran with, serialized, and the description it
Expand All @@ -196,16 +199,17 @@ export class Encoder {
// discontinuity and re-anchors on.
#pipeline: Pipeline | undefined;

// The current subscription's track and the producer writing into it, or undefined without demand.
// The rendition's track and the producer writing into it, or undefined while the broadcast hands
// over none (with `onDemand`, while nobody subscribes).
// `start` is the open group's first timestamp, undefined until a frame opens one.
#live: { track: Moq.Track.Producer; producer: Container.Legacy.Producer; start?: Micro } | undefined;

// Where the next frame submitted to the AudioEncoder starts, i.e. the exclusive end of the
// newest one, where an epoch's discontinuity marker goes. Cleared once the marker is written.
#next: Micro | undefined;

// The newest epoch's marker. The AudioEncoder outlives a demand gap too brief to skip a frame,
// so chunks it still held when demand disappeared surface after the resume; they sit below the
// The newest epoch's marker. The AudioEncoder outlives a track gap too brief to skip a frame,
// so chunks it still held when the track went away surface after the resume; they sit below the
// marker and are dropped.
#floor: Micro | undefined;

Expand Down Expand Up @@ -374,8 +378,8 @@ export class Encoder {
this.#failure.set(new Once<Error>());
}

// Register the rendition on the broadcast, publish its config, and encode only while a subscriber
// is attached (the demand gate). Re-registers cleanly when the broadcast swaps.
// Register the rendition on the broadcast, publish its config, and encode while it hands over a
// track. Re-registers cleanly when the broadcast swaps.
#runRegister(effect: Effect): void {
const failure = effect.get(this.#failure);
const broadcast = effect.get(this.in.broadcast);
Expand Down Expand Up @@ -403,8 +407,8 @@ export class Encoder {
this.#encode(broadcast.order, format, effect);
});

// Each subscription gets its own producer. Losing demand ends the epoch, as stopping the
// pipeline does.
// Each track the broadcast hands over gets its own producer. Losing it (with `onDemand`, the last
// subscriber leaving) ends the epoch, as stopping the pipeline does.
effect.run((effect) => {
const track = effect.get(rendition.track);
if (!track) return;
Expand Down Expand Up @@ -434,23 +438,28 @@ export class Encoder {
effect.set(this.#out.active, enabled && !!format && !!track && !fatal, false);
});

// Claim the configured bitrate so a co-resident video encoder's share is
// honest. Wait for a bitrate so we never claim 0. The grant is ignored:
// Claim the configured bitrate while a subscriber is attached, so a co-resident video
// encoder's share is honest. Wait for a bitrate so we never claim 0. The grant is ignored:
// following it for Opus is out of scope.
effect.run((effect) => {
const enabled = effect.get(this.in.enabled);
const track = effect.get(rendition.track);
const allocator = effect.get(this.in.bandwidth);
if (!enabled || !track || !allocator || effect.get(this.#out.error)) return;

let reservation: Moq.Bandwidth.Reservation | undefined;
effect.subscribe(this.#config, (config) => {
const bitrate = config?.catalog.bitrate;
if (bitrate === undefined) return;
if (!reservation) reservation = allocator.reserve(track.demand(), bitrate);
else reservation.update(bitrate);
const demanded = watched(track, effect);
effect.run((effect) => {
if (!effect.get(demanded)) return;

let reservation: Moq.Bandwidth.Reservation | undefined;
effect.subscribe(this.#config, (config) => {
const bitrate = config?.catalog.bitrate;
if (bitrate === undefined) return;
if (!reservation) reservation = allocator.reserve(track.demand(), bitrate);
else reservation.update(bitrate);
});
effect.cleanup(() => reservation?.close());
});
effect.cleanup(() => reservation?.close());
});
}

Expand Down Expand Up @@ -548,7 +557,7 @@ export class Encoder {
effect.cleanup(() => this.#failures.update((n) => n - 1));
}

// Publish the probed config before a consumer can request the demand-gated track. A
// Publish the probed config before the first frame reaches the track. A
// disabled rendition keeps its last config, since muting also releases the capture, and stays
// disabled after re-enabling until the reopened capture reports its format.
#runCatalog(effect: Effect): void {
Expand Down Expand Up @@ -593,7 +602,7 @@ export class Encoder {

// End the epoch with a discontinuity marker (see Container.Legacy.Producer.discontinuity) where the
// submitted media ends, so a subscriber that stays across the break, or joins during it, doesn't
// read the audio before it as live. Called whenever demand disappears or the pipeline stops.
// read the audio before it as live. Called whenever the track goes away or the pipeline stops.
#cut(): void {
const end = this.#next;
this.#next = undefined;
Expand Down Expand Up @@ -692,18 +701,18 @@ export class Encoder {

const frames = framer.push(input);
for (const [i, data] of frames.entries()) {
// The demand gate. The framer still consumes every sample so its timestamps stay
// on the capture clock, but there is nowhere to send a chunk with no subscriber.
// No track (with `onDemand`, no subscriber). The framer still consumes every sample
// so its timestamps stay on the capture clock, but there is nowhere to send a chunk.
if (!this.#live) continue;

// Round to whole microseconds once, here, so a chunk's timestamp and the marker
// placed at the next frame's start agree exactly.
const timestamp = Math.round(data.timestamp) as Micro;

// Chrome stamps encoder output from the first input's timestamp plus the samples
// encoded since, ignoring any later jump. Across a gap (demand, or a capture
// encoded since, ignoring any later jump. Across a gap (a lost track, or a capture
// discontinuity) the output would trail the capture clock by the gap, and the
// next demand gap's marker would then sit ahead of everything encoded after it.
// next gap's marker would then sit ahead of everything encoded after it.
// Restarting re-bases the output clock; the chunks it drops predate the gap.
if (contiguous !== undefined && timestamp !== contiguous) {
encoder.reset();
Expand Down
Loading
Loading