Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions demo/web/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,9 +119,9 @@ function createTile(name: string): WatchTile {
const watch = document.createElement("moq-watch") as MoqWatch;
watch.name = name;
watch.muted = true; // unmuted only while active (see below)
// Default to a fixed 100ms jitter buffer (instead of adaptive "real-time") so
// the latency visualization has something to show. Drag it in the panel.
watch.setAttribute("latency", "100");
// Default to a fixed 100ms jitter buffer (instead of adaptive "auto") so
// the delay visualization has something to show. Drag it in the panel.
watch.setAttribute("delay", "100ms");
const canvas = document.createElement("canvas");
canvas.style.cssText = "width: 100%; height: auto;";
watch.appendChild(canvas);
Expand Down
22 changes: 11 additions & 11 deletions demo/web/src/viz.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
*
* - `graph()` is a rolling sparkline (used for bitrate / frame rate).
* - `bufferBars()` is an editable view of the video/audio jitter buffer; drag it
* to change the latency target.
* to change the delay.
*/

import type { BufferedRanges, Signals } from "@moq/watch";
Expand Down Expand Up @@ -198,11 +198,11 @@ function drawRanges(
}
}

const STEP = 10; // latency drag/keyboard granularity, in milliseconds
const STEP = 10; // delay drag/keyboard granularity, in milliseconds

/**
* Editable latency visualization: video + audio jitter buffers drawn as bars
* from the playhead (left edge) out to ~4s, with the current latency target
* Editable delay visualization: video + audio jitter buffers drawn as bars
* from the playhead (left edge) out to ~4s, with the current delay
* marked. Drag (or focus + arrow keys) to set the buffer, like `<moq-watch-ui>`.
* Each call binds to one `watch`; close the parent effect to stop it.
*/
Expand All @@ -213,7 +213,7 @@ export function bufferBars(parent: Signals.Effect, watch: MoqWatch): HTMLElement
viz.style.cssText = "position: relative; cursor: ew-resize;";
viz.tabIndex = 0;
viz.setAttribute("role", "slider");
viz.setAttribute("aria-label", "Latency target");
viz.setAttribute("aria-label", "Playback delay");
viz.setAttribute("aria-valuemin", "0");
viz.setAttribute("aria-valuemax", String(BUFFER_MAX));

Expand Down Expand Up @@ -252,22 +252,22 @@ export function bufferBars(parent: Signals.Effect, watch: MoqWatch): HTMLElement

const legend = document.createElement("div");
legend.className = "mt-2 text-[10px] text-neutral-500";
legend.textContent = "buffered ahead of the playhead; drag to change the latency target";
legend.textContent = "buffered ahead of the playhead; drag to change the delay";

root.append(viz, legend);

// Set the latency floor, leaving the ceiling. Cast bypasses the branded ms type.
const setLatency = (ms: number) => {
// Set the playback delay. Cast bypasses the branded ms type.
const setDelay = (ms: number) => {
const clamped = Math.max(0, Math.min(BUFFER_MAX, ms));
watch.latencyMin = clamped as unknown as typeof watch.latencyMin;
watch.delay = clamped as unknown as typeof watch.delay;
};

const setFromX = (clientX: number) => {
const rect = canvasArea.getBoundingClientRect();
if (rect.width <= 0) return;
const x = Math.max(0, Math.min(clientX - rect.left, rect.width));
const ms = (x / rect.width) * BUFFER_MAX;
setLatency(Math.round(ms / STEP) * STEP);
setDelay(Math.round(ms / STEP) * STEP);
};

let dragging = false;
Expand All @@ -290,7 +290,7 @@ export function bufferBars(parent: Signals.Effect, watch: MoqWatch): HTMLElement
: 0;
if (delta === 0) return;
e.preventDefault();
setLatency((watch.sync.out.jitter.peek() as unknown as number) + delta);
setDelay((watch.sync.out.jitter.peek() as unknown as number) + delta);
});

// Position the target line from the live jitter (actual measured buffer in ms).
Expand Down
18 changes: 10 additions & 8 deletions doc/bin/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -215,12 +215,14 @@ it reads without being able to publish it, since that is what an RTMP or SRT
ingest produces. A rendition whose config declares HE-AAC fails when it opens,
either spelling of it; one that carries SBR without declaring it plays as its
AAC-LC core.
`--latency-max` controls how far a stalled media group may lag before it is
skipped and defaults to `500ms`. These flags all follow the `play` verb:
`--max-age` controls how stale a media group may get before it is skipped and
defaults to `500ms`. It is a staleness budget, not a playout delay: raising it
never holds the picture back, it only waits longer on a late group before giving
up. These flags all follow the `play` verb:

```bash
moq --connect https://relay.example.com/anon --broadcast conference.hang \
play --video-name hd --audio-name en --latency-max 250ms
play --video-name hd --audio-name en --max-age 250ms
```

Video decoding prefers the platform hardware backend and falls back to the
Expand Down Expand Up @@ -618,15 +620,15 @@ Import formats:
- `flv` - FLV / RTMP (H.264 video, AAC audio)
- `capture` - capture local devices directly (camera H.264 + microphone Opus; requires the `capture` build feature; does not read stdin)

`import --latency-max <duration>` (default `30s`) declares how long relays keep a
`import --max-age <duration>` (default `30s`) declares how long relays keep a
non-latest group of the published media tracks fetchable. It is a retention
budget, so raising it never makes a subscriber play further behind live: it caps
how far back a fetch can still reach. The default is sized for a segmented
egress (HLS/DASH), which may only advertise segments that are still fetchable;
lower it when nothing reads history and the memory matters:

```bash
moq --connect https://relay.example.com --broadcast my-stream.hang import --latency-max 5s ts
moq --connect https://relay.example.com --broadcast my-stream.hang import --max-age 5s ts
```

It sits on `import` itself rather than the endpoint, so it applies to every
Expand Down Expand Up @@ -672,13 +674,13 @@ for raw elementary streams instead of combining those sinks with a contradictory

Every export sink caps how long a stalled group is waited on before the muxer
skips to a newer one. Each owns the knob so its default fits the transport: the
stdout containers and `rtmp` take `--latency-max` (default `500ms`), and `srt`
stdout containers and `rtmp` take `--max-age` (default `500ms`), and `srt`
reuses its `--latency` (the receive buffer doubles as the skip threshold).
WebRTC (`rtc`) is real-time and doesn't buffer, so it has no such knob. HLS
export doesn't subscribe to media at all (segments are fetched on demand), so
it has no latency knob either. This is the subscriber half of the pair the
protocol names: the export knob is how long *this* consumer waits, while
[`import --latency-max`](#container-formats) is how long the publisher keeps a
[`import --max-age`](#container-formats) is how long the publisher keeps a
group around for anyone to fetch.

### MPEG-TS
Expand All @@ -695,7 +697,7 @@ moq --connect https://relay.example.com --broadcast my-stream.hang export ts | f
```

TS export is paced: bytes leave stdout at the instant the stream's clock (PCR)
asserts, smoothed within the `--latency-max` budget, so the pipe carries a
asserts, smoothed within the `--max-age` budget, so the pipe carries a
real-time transport stream rather than draining as fast as frames arrive.

TS export carries H.264 / H.265 as Annex-B and AAC as ADTS. Both in-band
Expand Down
86 changes: 40 additions & 46 deletions doc/lib/js/@moq/watch.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,11 +71,14 @@ a real bundler (the examples below).
- `muted`: Mute audio (boolean)
- `volume`: Audio volume (0 to 1, default: 1)
- `reload`: Wait for (re)announcement before subscribing (default: true). Ignored when the relay does not support broadcast discovery, which subscribes immediately instead of waiting forever.
- `latency`: Latency target. `"real-time"` (default) derives it from RTT, or a number sets a fixed jitter buffer in ms. Collapses `latency-min` and `latency-max` to one value (minimize latency). `"instant"` stops pacing off the clock, see [instant playback](#instant-playback).
- `latency-min`: Latency floor (the jitter/startup buffer). Same units as `latency`; leaves the ceiling untouched.
- `latency-max`: Latency ceiling. `"real-time"` (default) minimizes latency; a number caps at that many ms. A ceiling above the floor enables [buffered playback](#buffered-playback): build up a buffer from future-dated frames instead of skipping ahead.
- `delay`: How far playback trails the live edge, the jitter/startup buffer. `"auto"` (default) sizes it from the connection RTT, or a duration like `"100ms"` fixes it. `"instant"` stops pacing off the clock, see [instant playback](#instant-playback).
- `buffer`: Future-dated media held beyond the live edge before playback skips ahead, e.g. `"30s"`. Defaults to none, which minimizes latency. Anything above zero enables [buffered playback](#buffered-playback): build up a buffer from future-dated frames instead of skipping ahead.
- `catalog-format`: Catalog format. One of `"hang"`, `"hangz"` (the [DEFLATE-compressed](/concept/layer/hang#compression) `catalog.json.z` track), `"msf"` (see [MSF](/concept/standard/msf)), or `"manual"` (supply the catalog yourself). When omitted, the format is auto-detected from the broadcast `name` extension (`.hang` or `.msf`), falling back to `"hang"`. `"hangz"` is opt-in only and never auto-detected (it shares the `.hang` suffix).

Durations (`delay`, `buffer`) carry a unit, either `ms` or `s`. A bare number is
rejected rather than guessed at, since most players measure these in seconds while
the matching DOM properties are in milliseconds; only `"0"` is accepted unitless.

## Catalog Formats

`@moq/watch` can consume either the default [hang](/concept/layer/hang) catalog
Expand Down Expand Up @@ -148,55 +151,47 @@ el.catalog = myCatalog;

## Buffered playback

Latency is a **range**, `[latency-min, latency-max]`. By default the range is
collapsed (`latency` sets both to one value) and `@moq/watch` minimizes latency:
it anchors playback to the earliest frame seen relative to its timestamp and skips
ahead whenever the buffer grows past the target. That is right for live
conferencing, but wrong for content written *faster than real-time* with
timestamps in the future, such as a text-to-speech response streamed all at once.
`delay` and `buffer` sit on opposite sides of the live edge. `delay` is how far the
playhead trails it, absorbing jitter. `buffer` is how much future-dated media is
held beyond it before playback skips forward.

By default `buffer` is zero and `@moq/watch` minimizes latency: it anchors playback
to the earliest frame seen relative to its timestamp and skips ahead whenever the
buffer grows past the target. That is right for live conferencing, but wrong for
content written *faster than real-time* with timestamps in the future, such as a
text-to-speech response streamed all at once.

Open the range, by setting a `latency-max` above the floor, to anchor playback to
the first frame received and play through at the encoded pace. The buffer is
allowed to float anywhere between the floor and the ceiling without skipping:
Set a `buffer` to anchor playback to the first frame received and play through at
the encoded pace. Media is then allowed to float anywhere up to `delay + buffer`
ahead of the playhead without skipping:

```html
<moq-watch url="https://relay.example.com/anon" name="bot/tts.hang"
latency-min="100" latency-max="30000">
delay="100ms" buffer="30s">
<canvas></canvas>
</moq-watch>
```

In JavaScript, `latency` takes either a scalar (collapsed, minimize) or a range
object. The `latencyMin` / `latencyMax` properties are read-modify-write sugar
over the same `latency` value:
In JavaScript the two are independent properties, both in milliseconds:

```typescript
const el = document.querySelector("moq-watch")!;
el.latency = { min: 100, max: 30_000 }; // floor 100ms, ceiling 30s
// equivalently, set the bounds independently:
el.latencyMin = 100; // floor: start after 100ms buffered
el.latencyMax = 30_000; // ceiling: never skip until 30s buffered
el.delay = 100; // trail the live edge by 100ms
el.buffer = 30_000; // hold up to 30s of future-dated media
```

`latency-min` is the jitter/startup buffer (it can also be `"real-time"` for an
adaptive floor). `latency-max` is the ceiling, and it has two forms:

- a **number** (ms): buffer freely up to the cap, then skip ahead, so latency
stays at most that far behind the newest frame.
- **`"real-time"`** (the default) or any value `<= latency-min`: collapsed, i.e.
today's minimize-latency behavior.
`delay` also takes `"auto"` for a jitter buffer that tracks the connection RTT, which
is the default.

The ceiling is always finite: the buffer is bounded by `latency-max` rather than
growing without limit. The mechanism is the same in every case: the playhead is
anchored on the first frame and only re-anchored (skipped forward) when keeping it
would push latency past `latency-max`. Minimize is just the degenerate case where
the ceiling equals the floor.
The buffer is always finite, so it can't grow without limit. The mechanism is the
same in every case: the playhead is anchored on the first frame and only re-anchored
(skipped forward) when keeping it would leave a frame more than `delay + buffer`
ahead. Minimize is just the degenerate case where `buffer` is zero.

The buffered lookahead is held cheaply. The decoded audio ring only holds the
**floor** (`latency-min`) worth of PCM; everything above it stays upstream as
encoded frames, and the decoder applies backpressure (stops decoding ahead) until
the playhead nears each frame. So a large `latency-max` costs encoded bytes, not
seconds of decoded PCM.
The buffered lookahead is held cheaply. The decoded audio ring only holds `delay`
worth of PCM; everything above it stays upstream as encoded frames, and the decoder
applies backpressure (stops decoding ahead) until the playhead nears each frame. So
a large `buffer` costs encoded bytes, not seconds of decoded PCM.

At each new utterance, call `reset()` to flush the audio buffer and re-anchor
playback to the next frame. The producer can interrupt by writing a new utterance
Expand Down Expand Up @@ -248,42 +243,41 @@ The component exposes everything via its `broadcast` property

## Instant playback

`latency-min="0"` still de-jitters: playback is anchored to the first frame, so a
`delay="0"` still de-jitters: playback is anchored to the first frame, so a
frame that arrives early waits until its timestamp comes up. That's what keeps
motion smooth, and it costs at least a frame interval.

`latency="instant"` empties the buffer and stops pacing off the clock, rather than
`delay="instant"` empties the buffer and stops pacing off the clock, rather than
just shrinking the buffer. Frames paint the moment they decode, so latency drops to whatever the display costs
(about half a refresh interval on average) and the canvas shows the newest frame at
every repaint:

```html
<moq-watch url="https://relay.moq.dev/anon" name="demo/bbb" latency="instant">
<moq-watch url="https://relay.moq.dev/anon" name="demo/bbb" delay="instant">
<canvas></canvas>
</moq-watch>
```

It is a value of `latency`, not a bound, so `latency-min="instant"` is rejected:
there is no range to sit inside. Assigning `latency-min` or `latency-max` writes a
range and therefore leaves the mode.
It is a value of `delay`, and it overrides `buffer`: holding a lookahead would
contradict holding nothing. Assigning a duration to `delay` leaves the mode.

The tradeoff is motion: frames are presented at network cadence rather than at
their timestamps, so delivery jitter shows up as judder. Reach for it when a fresh
picture matters more than a smooth one, such as remote control or a camera
viewfinder, and prefer `latency` otherwise.
viewfinder, and prefer a paced `delay` otherwise.

This disables audio. The audio ring needs a target depth to avoid underrunning,
and unsynced video has nothing pulling it back toward the audio clock, so the two
would drift apart.

Note that `latency-min="0"` is not the same thing, and neither half of this is
Note that `delay="0"` is not the same thing, and neither half of this is
redundant. A zero floor still holds the selected rendition's own delay, a frame
interval at 60fps, and it still sleeps, because the wait comes from the playback
anchor rather than the buffer. `"instant"` drops both.

The clock itself keeps running: video still reports every frame it receives and
still resets it on a rewind. That keeps anything else reading the clock working, and
it means switching back to a paced latency resumes from a current reference instead
it means switching back to a paced delay resumes from a current reference instead
of a stale one.

Captions are one such reader, so they keep working. With nothing buffered they track
Expand Down
6 changes: 3 additions & 3 deletions js/hang/src/container/consumer.nonsequential.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,15 @@ test("Consumer waits on a non-sequential hang group because it has no per-frame
// presentation end is unknown until the next group arrives. With non-sequential group ids we
// therefore cannot prove the timeline is unbroken across a jump: a large id gap is
// indistinguishable from a real missing segment that may still be in transit. So the consumer
// waits (and lets #checkLatency / #tryDurationSkip decide once the latency budget or duration
// waits (and lets #checkMaxAge / #tryDurationSkip decide once the max age budget or duration
// coverage proves the gap too old) rather than eagerly skipping ahead, which would drop an
// in-transit group and wreck audio.
//
// A hang group can only promote early once its end is known, e.g. via an end-of-group signal such
// as a trailing frame carrying the group's end timestamp. Until the container carries that, expect
// the wait. The CMAF path (per-sample duration in the moof) is covered in consumer.test.ts.
const track = new Track.Producer("test");
const consumer = new Consumer(track.subscribe(), { format: new LegacyFormat(), latency: 5000 as Time.Milli });
const consumer = new Consumer(track.subscribe(), { format: new LegacyFormat(), maxAge: 5000 as Time.Milli });

// Group A at a large sequence, completed so the cursor advances past it.
const a = new Group.Producer(1_000_000);
Expand Down Expand Up @@ -55,7 +55,7 @@ test("Consumer waits on a non-sequential hang group because it has no per-frame
timestamp: Time.Timestamp.now(),
});

// Within the latency budget (5s), the parked next() stays parked: no duration means no proof of
// Within the max age budget (5s), the parked next() stays parked: no duration means no proof of
// contiguity, so the consumer waits instead of eagerly delivering across the gap.
const result = await Promise.race([
pending,
Expand Down
4 changes: 2 additions & 2 deletions js/hang/src/container/consumer.outoforder.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ test("out-of-order groups are delivered rather than dropped", async () => {
const track = new Track.Producer("test").accept({ maxAge: 30_000 });
const consumer = new Consumer(track.subscribe({ maxAge: 5000 }), {
format: new LegacyFormat(),
latency: 5000 as Time.Milli,
maxAge: 5000 as Time.Milli,
});

// The live edge lands first, and delivery starts there rather than waiting.
Expand Down Expand Up @@ -80,7 +80,7 @@ test("a below-cursor group still downloading is not truncated", async () => {
const track = new Track.Producer("test").accept({ maxAge: 30_000 });
const consumer = new Consumer(track.subscribe({ maxAge: 5000 }), {
format: new LegacyFormat(),
latency: 5000 as Time.Milli,
maxAge: 5000 as Time.Milli,
});

// The live edge group arrives first and starts delivery (still open).
Expand Down
Loading
Loading