From ab744fc3bac951f987aa37bf1dfffe6cade02378 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 16:03:08 -0700 Subject: [PATCH 1/3] feat(catalog)!: measure jitter and delay from timestamps, not the wall clock The catalog estimator measured each frame's flush time against its media timestamp, so any capture clock drifting from the wall clock read as ever-rising lateness. A physical camera on moq.dev/publish ratcheted audio jitter and delay without bound, republishing the catalog and driving hundreds of SUBSCRIBE_UPDATEs a minute. jitter now covers only reorder delay and batch spans. delay is how far a frame's timestamp trails the highest one another track of the catalog already wrote, so importers measure it too. flush, moq_publish_media_flush, the binding wrappers, and moqsink's encoder pad property are removed. The camera's audio still drifts behind video by PTS; the subscribe update quest becomes the audio capture clock fix. Co-Authored-By: Claude Opus 5.5 --- cpp/obs/src/moq-output.cpp | 11 - dart/moq_ffi/lib/src/moq.dart | 28 +- doc/bin/gstreamer.md | 26 +- doc/concept/audio-jitter.md | 29 +- doc/concept/hang.md | 6 +- doc/lib/c/index.md | 2 +- doc/lib/dart/index.md | 2 +- doc/lib/go/index.md | 2 +- doc/lib/kt/index.md | 2 +- doc/lib/py/index.md | 2 +- doc/lib/swift/index.md | 2 +- drafts/draft-lcurley-moq-hang.md | 37 +- go/wrapper/MIGRATE.md | 5 + go/wrapper/media/media.go | 12 +- js/publish/src/audio/capture.test.ts | 4 +- js/publish/src/audio/encoder.test.ts | 32 +- js/publish/src/audio/encoder.ts | 12 +- js/publish/src/broadcast.ts | 8 +- js/publish/src/jitter.test.ts | 86 ----- js/publish/src/jitter.ts | 72 ---- js/publish/src/order.test.ts | 67 ++++ js/publish/src/order.ts | 52 +++ js/publish/src/video/capture.test.ts | 6 +- js/publish/src/video/encoder-stall.test.ts | 10 +- js/publish/src/video/encoder.test.ts | 26 +- js/publish/src/video/encoder.ts | 12 +- js/watch/src/audio/latency.ts | 6 +- py/moq-rs/MIGRATE.md | 5 + py/moq-rs/README.md | 2 +- py/moq-rs/moq/media.py | 12 +- quest/m1/README.md | 2 +- quest/m1/audio-capture-clock.md | 30 ++ quest/m1/subscribe-update-churn.md | 36 -- rs/moq-audio/src/encode/producer.rs | 5 +- rs/moq-c/MIGRATE.md | 5 + rs/moq-c/README.md | 1 - rs/moq-c/src/api.rs | 18 +- rs/moq-c/src/publish.rs | 8 - rs/moq-c/src/test.rs | 3 - rs/moq-ffi/MIGRATE.md | 5 + rs/moq-ffi/src/media.rs | 15 +- rs/moq-ffi/src/test.rs | 2 - rs/moq-gst/MIGRATE.md | 5 + rs/moq-gst/src/sink/imp.rs | 65 +--- rs/moq-gst/src/sink/pad.rs | 279 ++------------ rs/moq-gst/src/sink/request_pad.rs | 24 +- rs/moq-gst/tests/element.rs | 14 - rs/moq-mux/MIGRATE.md | 6 + rs/moq-mux/benches/catalog.rs | 78 +++- rs/moq-mux/src/binary.rs | 71 +--- rs/moq-mux/src/catalog/data.rs | 26 +- rs/moq-mux/src/catalog/estimate.rs | 427 ++++++++++++--------- rs/moq-mux/src/catalog/producer.rs | 12 +- rs/moq-mux/src/catalog/tracks.rs | 2 +- rs/moq-mux/src/codec/aac/import.rs | 5 - rs/moq-mux/src/codec/av1/import.rs | 5 - rs/moq-mux/src/codec/flac/import.rs | 5 - rs/moq-mux/src/codec/h264/import.rs | 5 - rs/moq-mux/src/codec/h265/import.rs | 5 - rs/moq-mux/src/codec/mp3.rs | 5 - rs/moq-mux/src/codec/opus/import.rs | 5 - rs/moq-mux/src/codec/vp8/import.rs | 5 - rs/moq-mux/src/codec/vp9/import.rs | 5 - rs/moq-mux/src/container/fmp4/import.rs | 2 +- rs/moq-mux/src/container/producer.rs | 39 +- rs/moq-mux/src/import/track.rs | 38 +- rs/moq-mux/src/json.rs | 21 +- rs/moq-video/src/encode/producer.rs | 2 - swift/MIGRATE.md | 5 + swift/Sources/Moq/Media.swift | 8 +- 70 files changed, 722 insertions(+), 1155 deletions(-) delete mode 100644 js/publish/src/jitter.test.ts delete mode 100644 js/publish/src/jitter.ts create mode 100644 js/publish/src/order.test.ts create mode 100644 js/publish/src/order.ts create mode 100644 quest/m1/audio-capture-clock.md delete mode 100644 quest/m1/subscribe-update-churn.md diff --git a/cpp/obs/src/moq-output.cpp b/cpp/obs/src/moq-output.cpp index b8a780f4677..6a73218392e 100644 --- a/cpp/obs/src/moq-output.cpp +++ b/cpp/obs/src/moq-output.cpp @@ -724,17 +724,11 @@ void MoQOutput::AudioData(struct encoder_packet *packet) // Audio has no keyframes, so it has no group boundary of its own: without this the whole // stream is one group. Cut per frame, which is one QUIC stream per packet forwarded without // waiting for the next, the right trade for live. Video groups at its own keyframes. - // Cut before observing the flush so a failed observation never leaves the group open. if (auto result = track.producer->cut(); !result) { LOG_ERROR("Failed to cut audio group: %s", result.error().to_string().c_str()); return; } - if (auto result = track.producer->flush(*timestamp); !result) { - LOG_ERROR("Failed to observe audio encoder flush: %s", result.error().to_string().c_str()); - return; - } - total_bytes_sent += packet->size; } @@ -771,11 +765,6 @@ void MoQOutput::VideoData(struct encoder_packet *packet) return; } - if (auto result = track.producer->flush(*timestamp); !result) { - LOG_ERROR("Failed to observe video encoder flush: %s", result.error().to_string().c_str()); - return; - } - total_bytes_sent += packet->size; } diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index 9f42b179cd6..7f92847848d 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -10205,7 +10205,6 @@ abstract class MoqMediaTrackProducerInterface { MoqTrackDemand demand(); void discontinuity(); void finish(); - void flush({required int timestampUs}); void seek({required int sequence}); void writeFrame({required MoqFrame frame}); } @@ -10313,16 +10312,6 @@ class MoqMediaTrackProducer implements MoqMediaTrackProducerInterface { }, moqExceptionErrorHandler); } - void flush({required int timestampUs}) { - return rustCall((status) { - uniffi_moq_ffi_fn_method_moqmediatrackproducer_flush( - uniffiClonePointer(), - FfiConverterUInt64.lower(timestampUs), - status, - ); - }, moqExceptionErrorHandler); - } - void seek({required int sequence}) { return rustCall((status) { uniffi_moq_ffi_fn_method_moqmediatrackproducer_seek( @@ -16807,15 +16796,6 @@ external void uniffi_moq_ffi_fn_method_moqmediatrackproducer_finish( Pointer uniffiStatus, ); -@Native, Uint64, Pointer)>( - assetId: _uniffiAssetId, -) -external void uniffi_moq_ffi_fn_method_moqmediatrackproducer_flush( - Pointer ptr, - int timestamp_us, - Pointer uniffiStatus, -); - @Native, Uint64, Pointer)>( assetId: _uniffiAssetId, ) @@ -18733,9 +18713,6 @@ uniffi_moq_ffi_checksum_method_moqmediatrackproducer_discontinuity(); @Native(assetId: _uniffiAssetId) external int uniffi_moq_ffi_checksum_method_moqmediatrackproducer_finish(); -@Native(assetId: _uniffiAssetId) -external int uniffi_moq_ffi_checksum_method_moqmediatrackproducer_flush(); - @Native(assetId: _uniffiAssetId) external int uniffi_moq_ffi_checksum_method_moqmediatrackproducer_seek(); @@ -19414,15 +19391,12 @@ void _checkApiChecksums() { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqmediatrackproducer_discontinuity() != - 32931) { + 25213) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqmediatrackproducer_finish() != 45426) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqmediatrackproducer_flush() != 51126) { - throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); - } if (uniffi_moq_ffi_checksum_method_moqmediatrackproducer_seek() != 37244) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } diff --git a/doc/bin/gstreamer.md b/doc/bin/gstreamer.md index b77e58889fb..0a6a0a6841f 100644 --- a/doc/bin/gstreamer.md +++ b/doc/bin/gstreamer.md @@ -21,7 +21,7 @@ nix shell github:moq-dev/moq/release#moq-gst --command gst-launch-1.0 -e \ # Publish a test pattern gst-launch-1.0 -e videotestsrc is-live=true ! x264enc tune=zerolatency ! h264parse \ ! video/x-h264,stream-format=byte-stream,alignment=au ! mux.sink_0 \ - moqsink name=mux url=https://cdn.moq.dev/anon broadcast=.hang sink_0::encoder=true + moqsink name=mux url=https://cdn.moq.dev/anon broadcast=.hang ``` Install via `apt install gstreamer1.0-moq` or `dnf install gstreamer1-moq` @@ -58,8 +58,7 @@ directly. A cue with no duration is dropped rather than left on screen. Audio packets share groups of at least 20 ms while forwarding as they arrive. Each `sink_%u` request pad is one track. Pad properties: `track` names it (default: after the codec), `container=loc` publishes it as -[LOC](/concept/standard#loc) instead of the legacy hang container, -`encoder=true` marks it as fed by a local encoder, and +[LOC](/concept/standard#loc) instead of the legacy hang container, and `track-status`/`track-error` report its lifecycle. Element properties: `url`, `broadcast`, `tls-disable-verify`, `quic-idle-timeout`, `quic-keylog`, `quic-keep-alive`, `linger`, and read-only `status`, `connected`, `moq-version`, and @@ -100,19 +99,14 @@ in the same shape as the relay's sessions track. One read returns both from the same instant, so their difference is 1 while connected. Unlike `status`, a drop you did not poll still moves both counters. -Set `encoder=true` on audio and video pads a local encoder feeds -(`x264enc`, `opusenc`, ...). The pad then measures how late each frame reaches -the sink behind its running time and raises the catalog `jitter` by the spread, -and `delay` by how far it trails the earliest such pad, so players buffer for an -encoder that delivers irregularly or behind the others. Leave it off, the -default, for file, demuxed, and network media: their arrival reflects the disk -or the network, not the original encoder, and a GStreamer segment cannot tell -the two apart. Text and opaque pads refuse it. - -After a pause, flush, or changed TIME segment, the next media buffer resets -the handoff measurement baseline. The pause does not inflate -advertised jitter, and previously measured maxima remain. Resumed timestamps -must continue forward on the broadcast media clock. +The catalog `delay` of each pad is how far its buffers' timestamps trail ones +the other pads already pushed, so players buffer for an encoder that delivers +behind the others. Arrival time is never measured, so file, demuxed, and network +media advertise the same values as a live encoder. + +After a pause, flush, or changed TIME segment, the pad stops holding the others +back until its next buffer, and previously measured maxima remain. Resumed +timestamps must continue forward on the broadcast media clock. A video pad joining mid-GOP drops delta frames until its first keyframe. If a source rewinds below the producer's live edge without signalling a break, the diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index 66388cd34d6..ac4783a87b3 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -304,8 +304,9 @@ business. ## The advertised flush span, and how it composes -A hang rendition may advertise `jitter`: the largest delay the publisher has -seen between a frame being ready and handing it to the transport. It is +A hang rendition may advertise `jitter`: the largest span of media the +publisher writes behind a frame's timestamp, from B-frame reordering or a batch +written together. It is described in [hang](/concept/hang) and specified in [draft-lcurley-moq-hang](/draft/moq-hang). The two quantities share a word and get confused for each other, so plainly: @@ -313,15 +314,15 @@ get confused for each other, so plainly: | | Advertised `jitter` | The estimate here | | --- | --- | --- | | Measured by | the publisher, once | each receiver, continuously | -| Measured at | the transport handoff | the receiver's transport | -| Covers | encoder latency, packet packing, reorder, segmentation | all of that, plus the network | +| Measured from | media timestamps | the receiver's arrival times | +| Covers | packet packing, reorder, segmentation | all of that, plus encoder latency and the network | | Is | a declared lower bound | an observation | -**They compose as a maximum, and the reason matters.** Both are the same -formula, lateness relative to a windowed minimum, evaluated one hop apart. The -receiver measures from the frame's media timestamp to its own arrival, so the -publisher's flush delay is already inside every number on this page: a publisher -flushing 100 ms of media at once produces arrivals whose relative delay reaches +**They compose as a maximum, and the reason matters.** Both are lateness +behind the media timestamp, evaluated one hop apart. The receiver measures from +the frame's media timestamp to its own arrival, so the publisher's batching is +already inside every number on this page: a publisher writing 100 ms of media at +once produces arrivals whose relative delay reaches 100 ms less one frame, on a perfect network, with no help from the catalog. So the advertised value is a **floor**, not an addend: @@ -335,7 +336,7 @@ publishers that need it most. Taking the maximum on the grounds that they "describe the same quantity" reaches the right operation by the wrong argument: they are the same quantity measured at different points, and the receiver's point strictly contains the publisher's. The floor earns its place by being -correct *immediately*, before the estimator has seen a full flush cycle, which +correct *immediately*, before the estimator has seen a full batch, which is the one thing a measurement cannot be. ## Across renditions @@ -346,9 +347,9 @@ constant offset between tracks cancels out of `measured`: a video encoder that flushes 200 ms behind the audio encoder produces the same video target as one that does not, and its frames would all arrive 200 ms late for the shared clock. -The catalog `delay` field carries that offset: how far a rendition's minimum -flush lateness trails the broadcast's earliest rendition, measured by the -publisher and never lowered. It is an addend, not a floor, because nothing on +The catalog `delay` field carries that offset: how far a rendition's +timestamps trail those the broadcast's other renditions already wrote, measured +by the publisher and never lowered. It is an addend, not a floor, because nothing on this page contains it. The shared playout is the largest requirement among the renditions actually subscribed: @@ -357,7 +358,7 @@ playout = max over subscribed renditions of (delay + target) ``` A receiver **must not** subtract one rendition's `delay` from another's. Each is -a lifetime maximum taken against a sliding baseline, so two values need not +a lifetime maximum taken against whichever rendition was ahead, so two values need not share an origin: if the earliest rendition later drifts behind another, their difference understates the real spread. The maximum never under-buffers, because the subscribed renditions' spread is measured from an earliest baseline diff --git a/doc/concept/hang.md b/doc/concept/hang.md index 841f74d2b0b..875e0bc6233 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -65,8 +65,8 @@ A few things the catalog can express beyond decoder config: - **Labels.** Any rendition may carry a human-readable `label` for a track picker. The map key stays the rendition's ID, so labels need not be unique and renaming one doesn't rename the track. - **Renditions in another broadcast.** A rendition may point at a relative broadcast path, so a transcoder can publish a ladder that adds low rungs and references the source's original rendition without re-publishing its bytes. The path resolves against where the consumer found the catalog, so a reference that escapes above the root names nothing and the catalog is rejected. `@moq/watch` and `moqsrc` follow a referenced path's announcements like their own, restarting the rendition on each instance that serves it; every other reader, such as `moq play`, the exporters, and the bindings, waits for the first one announced and keeps it, and an exporter ends the rendition with it. An optional `epoch` beside the path names the [publisher instance](/concept/moq-lite#publisher-epochs) it expects: a route proving another instance ends the rendition instead of falling back to it, while a route without an epoch (an older version) cannot prove one, so there it is best effort. - **Track names.** The map key is a rendition's ID and, by default, its track name. A rendition sets `track` only when the two must differ, such as two renditions of same-named tracks in different broadcasts, or a rendition whose media changed and so needs a new name; readers subscribe to `track` when present. An ID can't name both a video and an audio rendition. -- **Jitter.** A rendition can say how far its frames fell behind the media clock before the publisher flushed them, in whole milliseconds rounded up. Encoders report the spread of lateness above each rendition's own recent minimum, so a constant encoder delay is not jitter; container imports estimate batch spans without counting ingest delay. It describes the publisher, never the network, only grows over the life of a stream, and a player sizes its buffer to at least this much. A `0` is read as absent. -- **Delay.** A rendition can also say how far its frames reach the transport behind the broadcast's earliest rendition, measured the same way from each rendition's minimum lateness, so a video encoder running 200 ms behind audio advertises `delay: 200` on video. It follows the same rules as jitter. A player holds the largest `delay + jitter` among the renditions it subscribes to, and never subtracts one rendition's `delay` from another's. +- **Jitter.** A rendition can say how much media it writes behind a frame's timestamp, in whole milliseconds rounded up: the reorder delay of B-frames and the span of a fragment or batch written together. It is measured from media timestamps alone, never the time a frame was written, so neither a constant encoder delay nor a capture clock counts. It describes the publisher, never the network, only grows over the life of a stream, and a player sizes its buffer to at least this much. A `0` is read as absent. +- **Delay.** A rendition can also say how far its frames' timestamps trail those another rendition already wrote, so a video encoder running 200 ms behind audio advertises `delay: 200` on video. A rendition alone measures none. It follows the same rules as jitter. A player holds the largest `delay + jitter` among the renditions it subscribes to, and never subtracts one rendition's `delay` from another's. - **Colour.** A video rendition may carry `color`: the H.273 primaries, transfer, matrix, and range, plus the HDR10 mastering display and content light levels, so a renderer can set up HDR without parsing the bitstream. The Rust importers fill it from the SPS VUI, AV1 sequence header, and VP9 header, then fill gaps from a container's `colr` box or Matroska `Colour` element; the stream wins a disagreement. The HDR10 values come from H.264 and H.265 SEI or AV1 metadata OBUs in MPEG-TS and raw elementary streams, an AV1 track's av1C, or Matroska, but not yet from H.264 or H.265 in fMP4 or FLV. Nothing in `@moq/watch` reads it yet. - **Spatial media.** A video rendition may carry `projection` (flat `rectilinear`, or `equirectangular` with the degrees of longitude it spans, 180 for VR180) and `stereo` (`mono`, `left-right`, or `top-bottom` frame packing), and an audio rendition may carry `layout: "ambisonic"` for AmbiX channels. Readers treat a value they don't recognize as absent. Nothing in `@moq/watch` renders them yet. - **Disabled renditions.** An audio or video rendition with `enabled: false` has no frames coming, and a player must not select it. Pausing a rendition in `@moq/publish` keeps it in the catalog this way rather than removing it, so a mute is a one-field update; a disabled video rendition encodes one black keyframe first, so a player that predates the field shows black instead of a frozen picture. The field is written only when false. The older video `stalled` field is no longer written, and readers ignore it. @@ -141,7 +141,7 @@ into it on a raw track. The rest is descriptive: `compression` (`deflate`, the same group-scoped `deflate-raw` the catalog uses), `schema` on a JSON track, `mime` on a binary one, `bitrate`, `jitter`, and `delay` with the same meaning as for media, plus -the optional `broadcast` reference and `track` name. A publisher measures `jitter` and `delay` +the optional `broadcast` reference and `track` name. A publisher measures `delay` only from payloads stamped with their capture time on the broadcast clock. A consumer that doesn't recognize a `mode` or `compression` ignores that track and round-trips it verbatim. diff --git a/doc/lib/c/index.md b/doc/lib/c/index.md index db7d564042e..9d1a5cea496 100644 --- a/doc/lib/c/index.md +++ b/doc/lib/c/index.md @@ -66,7 +66,7 @@ inside. The header is the reference: each function carries a doc comment. - **fMP4 import.** `MOQ_CONTAINER_FORMAT_FMP4` splits each fragment into one frame per sample; `MOQ_CONTAINER_FORMAT_CMAF` republishes it whole. - **Audio needs cuts.** Video groups at its keyframes, but audio forms a group only where you call `moq_publish_media_cut`: after every frame, or at a segment cadence to align with video. - **Ending cleanly.** When the input ends, finish every track, then call `moq_publish_finish()`. It consumes the broadcast and returns a finish handle that keeps it announced and serving the finished tracks, writing the linger, their longest max age, to `linger_us`. Keep the handle for that long on your own clock so a subscriber still catching up can drain them, then `moq_publish_finish_close()` it; closing early is the interrupt. `moq_publish_close()` stays the abrupt end. -- **Live encoder timing.** After writing a frame you encoded yourself, call `moq_publish_media_flush` with the same timestamp so the catalog advertises your jitter. Skip it for file and network imports. On a seek or pause, call `moq_publish_media_discontinuity`, then keep timestamps moving forward and resume video on a keyframe. +- **Timeline breaks.** On a seek or pause, call `moq_publish_media_discontinuity`, then keep timestamps moving forward and resume video on a keyframe. - **JSON values.** `moq_consume_json_value` fills `moq_json_value` with the document and its frame's `timestamp_us`, rounded down to a whole microsecond; `timestamp_present` is false on an untimed track. A snapshot consumer's callback fires for each state in order, but a group superseded before it is read is skipped. - **Decoded frames hold decoder buffers.** Free each frame id with `moq_decode_video_frame_free` promptly, or the decoder stalls. - **Stats.** `moq_session_stats()` fills `moq_connection_stats` with `rtt_us`, `estimated_send_rate_bps`, and the byte and packet counters (`bytes_sent`, `bytes_received`, `bytes_lost`, `packets_sent`, `packets_received`, `packets_lost`). Each has a `_valid` flag, false when the transport does not report it, which is not the same as zero. `moq_session_snapshot()` adds the negotiated `protocol`, and while a session reconnects `moq_session_failure()` returns why, with the reason in `moq_error()`. Hold the handle returned by `moq_session_recv_bandwidth()` to request receive estimates. `moq_bandwidth_consumer_peek()` reads the estimate and `moq_bandwidth_consumer_changed()` waits for a change. `moq_bandwidth_consumer_close()` releases demand; the last consumer stops PROBE. Stats snapshots do not request it. diff --git a/doc/lib/dart/index.md b/doc/lib/dart/index.md index 3be8bbfed3a..db32ffaa6f9 100644 --- a/doc/lib/dart/index.md +++ b/doc/lib/dart/index.md @@ -135,7 +135,7 @@ shows which module holds each. - **Names.** Types drop the `Moq` prefix (`Session`, `BroadcastProducer`) as aliases, so the generated names still work. `Container`, `Route`, and the exceptions keep it, since Flutter owns those names. `dynamic` is reserved, so the origin method is `dynamic_`. - **Audio needs cuts.** Video groups at its keyframes, but audio forms a group only where you call `cut()`: after every frame, or at a segment cadence to align with video. -- **Live encoder timing.** After writing a frame you encoded yourself, call `flush(timestampUs: ...)` with the same timestamp so the catalog advertises your jitter. Skip it for file and network imports. On a seek or pause, call `discontinuity()`, then keep timestamps moving forward and resume video on a keyframe. +- **Timeline breaks.** On a seek or pause, call `discontinuity()`, then keep timestamps moving forward and resume video on a keyframe. - **Decoded frames hold decoder buffers.** `dispose()` each frame from a `video.Consumer` promptly rather than waiting for the garbage collector, or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `await moq.close()` or `session.shutdown()` drains for up to one second and reports any error. `session.cancel(code: 0)` closes at once. Finish or abort live tracks first. Awaiting cancellation of an announcement, grant, or request subscription releases its pending native read. Announcements also release their cursor; authorization and server listeners leave their shared owner usable. Application delivery requires explicitly selecting `moq-lite-07-wip` on both peers; older Lite versions (including default lite-06) and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `broadcast.finish()`. The returned `BroadcastFinish` keeps the broadcast announced and serving the finished tracks for `linger`, their longest max age, so a subscriber still catching up can drain them. `await finish.wait()` completes once the linger passes, and `finish.close()` closes the broadcast at once, completing the wait. `broadcast.close()` stays the abrupt end. An unreferenced `BroadcastFinish` closes the broadcast only when the garbage collector reclaims it, possibly long after, so wait on or close it rather than dropping it. diff --git a/doc/lib/go/index.md b/doc/lib/go/index.md index 08e45d39003..5f4b6e97aec 100644 --- a/doc/lib/go/index.md +++ b/doc/lib/go/index.md @@ -125,7 +125,7 @@ shows which module holds each. - **Keep owners reachable.** An origin from `moq.NewOriginProducer` has no `Close`. It ends when the garbage collector reaches its last owner (each producer, published broadcast, and `OriginDynamic`), and its consumers then fail with `moq.ErrClosed`. - **Bandwidth ownership.** An encoder you manage can reserve a share of the session's bandwidth. Reuse its reservation when the ceiling changes; before abandoning or replacing it, call `Update(0)` to return its share and automatic probe ceiling without waiting for garbage collection. The track can stay subscribed and the reservation can be reused. - **Audio needs cuts.** Video groups at its keyframes, but audio forms a group only where you call `Cut()`: after every frame, or at a segment cadence to align with video. -- **Live encoder timing.** After writing a frame you encoded yourself, call `Flush(timestamp)` with the same timestamp so the catalog advertises your jitter. Skip it for file and network imports. On a seek or pause, call `Discontinuity()`, then keep timestamps moving forward and resume video on a keyframe. +- **Timeline breaks.** On a seek or pause, call `Discontinuity()`, then keep timestamps moving forward and resume video on a keyframe. - **Decoded frames hold decoder buffers.** `Close` each `VideoDecodedFrame` promptly, or the decoder stalls. `Resize` is best effort, so read each frame's `Width()` and `Height()`. - **Closing.** `client.Close()` drains for up to one second and returns an error on failure. `session.Shutdown(ctx)` does the same, and `session.Cancel(code)` closes at once. Finish or abort live tracks first. Application delivery requires explicitly selecting `moq-lite-07-wip` on both peers; older Lite versions (including default lite-06) and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `broadcast.Finish()`. The returned `*BroadcastFinish` keeps the broadcast announced and serving the finished tracks for `Linger()`, their longest max age, so a subscriber still catching up can drain them. `finish.Wait(ctx)` returns once the linger passes; ending `ctx`, or `finish.Close()`, closes the broadcast at once. `broadcast.Close()` stays the abrupt end. An unreferenced `*BroadcastFinish` closes the broadcast only when the garbage collector reclaims it, possibly long after, so wait on or close it rather than dropping it. diff --git a/doc/lib/kt/index.md b/doc/lib/kt/index.md index ebcbe2f294c..97bf90c6c62 100644 --- a/doc/lib/kt/index.md +++ b/doc/lib/kt/index.md @@ -96,7 +96,7 @@ shows which module holds each. - **`end`, not `close`.** `broadcast.end()` ends a broadcast for good. `close()`, or leaving `use { }`, releases the handle, which ends the broadcast only once no `dynamic()` handle remains. - **Audio needs cuts.** Video groups at its keyframes, but audio forms a group only where you call `cut()`: after every frame, or at a segment cadence to align with video. -- **Live encoder timing.** After writing a frame you encoded yourself, call `flush(timestampUs)` with the same timestamp so the catalog advertises your jitter. Skip it for file and network imports. On a seek or pause, call `discontinuity()`, then keep timestamps moving forward and resume video on a keyframe. +- **Timeline breaks.** On a seek or pause, call `discontinuity()`, then keep timestamps moving forward and resume video on a keyframe. - **Decoded frames hold decoder buffers.** `close()` each frame from a `dev.moq.video.Consumer` promptly (or `use { }` it), or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** Suspending `moq.shutdown()` drains for up to one second and throws on failure. `Moq.close()`, and so `use { }`, cancels at once. Finish or abort live tracks first. Application delivery requires explicitly selecting `moq-lite-07-wip` on both peers; older Lite versions (including default lite-06) and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `broadcast.finish()`. The returned `BroadcastFinish` keeps the broadcast announced and serving the finished tracks for `linger`, their longest max age, so a subscriber still catching up can drain them. Suspending `finish.wait()` returns once the linger passes; cancelling its coroutine, or `finish.end()`, closes the broadcast at once. `broadcast.end()` stays the abrupt end. An unreferenced `BroadcastFinish` closes the broadcast only when the garbage collector reclaims it, possibly long after, so wait on or close it rather than dropping it. diff --git a/doc/lib/py/index.md b/doc/lib/py/index.md index 4892a91da19..5994a2d434e 100644 --- a/doc/lib/py/index.md +++ b/doc/lib/py/index.md @@ -111,7 +111,7 @@ shows which module holds each. - **Context managers.** `async with` on a client or session awaits a graceful shutdown when the body exits cleanly, and cancels at once when it raises, so your exception survives. - **Audio needs cuts.** Video groups at its keyframes, but audio forms a group only where you call `cut()`: after every frame, or at a segment cadence to align with video. -- **Live encoder timing.** After writing a frame you encoded yourself, call `flush(timestamp)` with the same timestamp so the catalog advertises your jitter. Skip it for file and network imports. On a seek or pause, call `discontinuity()`, then keep timestamps moving forward and resume video on a keyframe. +- **Timeline breaks.** On a seek or pause, call `discontinuity()`, then keep timestamps moving forward and resume video on a keyframe. - **Decoded frames hold decoder buffers.** Drop each frame from a `moq.video.Consumer` promptly, or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `await session.shutdown()` drains for up to one second and raises on failure. `cancel(code)` closes at once. Finish or abort live tracks first. Application delivery requires explicitly selecting `moq-lite-07-wip` on both peers; older Lite versions (including default lite-06) and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `broadcast.finish()`. The returned `BroadcastFinish` keeps the broadcast announced and serving the finished tracks for `linger`, their longest max age, so a subscriber still catching up can drain them. `await finish.wait()` returns once the linger passes; cancelling it, or `finish.close()`, closes the broadcast at once. `broadcast.close()` stays the abrupt end. diff --git a/doc/lib/swift/index.md b/doc/lib/swift/index.md index afc159d2628..f4da66b42d2 100644 --- a/doc/lib/swift/index.md +++ b/doc/lib/swift/index.md @@ -95,7 +95,7 @@ one to one; the API reference has the names, and [Layers](/lib/layers) shows which module holds each. - **Audio needs cuts.** Video groups at its keyframes, but audio forms a group only where you call `cut()`: after every frame, or at a segment cadence to align with video. -- **Live encoder timing.** After writing a frame you encoded yourself, call `flush(timestampUs:)` with the same timestamp so the catalog advertises your jitter. Skip it for file and network imports. On a seek or pause, call `discontinuity()`, then keep timestamps moving forward and resume video on a keyframe. +- **Timeline breaks.** On a seek or pause, call `discontinuity()`, then keep timestamps moving forward and resume video on a keyframe. - **Decoded frames hold decoder buffers.** Release each frame from a `Video.Consumer` promptly, or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `try await session.shutdown()` drains for up to one second and throws on failure. `session.cancel(code:)` closes at once. Finish or abort live tracks first. Application delivery requires explicitly selecting `moq-lite-07-wip` on both peers; older Lite versions (including default lite-06) and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `try broadcast.finish()`. The returned `BroadcastFinish` keeps the broadcast announced and serving the finished tracks for `lingerUs`, their longest max age, so a subscriber still catching up can drain them. `try await finish.wait()` returns once the linger passes; cancelling its task, or `finish.close()`, closes the broadcast at once. `broadcast.close()` stays the abrupt end. diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index fe79214a0f0..90e06796b12 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -630,9 +630,9 @@ The optional `bitrate` field is the track's maximum bitrate in bits per second. The optional `jitter` and `delay` fields carry the same meaning and rules as they do for a media rendition ({{field-jitter}}, {{field-delay}}), with a payload in place of a frame. A payload's presentation timestamp is its capture time on the broadcast's clock, the one its media renditions use, when the publisher knows it (a datagram's arrival, a sensor read), and otherwise the time the publisher wrote it. -A publisher measures `jitter` and `delay` only from payloads that carry a capture time, and MUST omit both for a track written without one. -A publisher MUST NOT use a timestamp on an unrelated clock, such as a device's own uptime, since it would report a meaningless lateness and, through the broadcast-wide minimum, distort every other rendition's `delay`. -A `snapshot` update that writes no frame, such as an unchanged JSON value, is not a flush and is not measured. +A publisher measures `delay` only from payloads that carry a capture time, and MUST omit it for a track written without one. +A publisher MUST NOT use a timestamp on an unrelated clock, such as a device's own uptime, since it would order the payload against the media renditions meaninglessly and distort their `delay`. +A `snapshot` update that writes no frame, such as an unchanged JSON value, is not measured. ## Binary Fields {#binary} A decoder config field carrying raw bytes, notably `description` (an `AllowSharedBufferSource` in WebCodecs), is carried in the catalog as a hex string ({{!RFC4648, Section 8}}). @@ -718,26 +718,24 @@ The container used to frame this rendition's media, as described in {{container} If absent, it defaults to `{ "kind": "legacy" }`. ### jitter {#field-jitter} -The maximum delay, in milliseconds, that a publisher has measured before handing a frame to the transport. +The maximum media span, in milliseconds, that a publisher hands to the transport behind a frame's presentation timestamp. A consumer's jitter buffer SHOULD be at least this large to avoid stalling. If absent, a consumer SHOULD assume no publisher delay has been measured. -An encoder compares each frame's flush time with its media timestamp and subtracts the smallest lateness recently observed on the same rendition. -The field is the spread above that minimum, so neither a constant encoder delay, a fixed clock offset, nor long-term drift counts as jitter. -A container importer instead estimates the delay from the media span of a batch, such as a run of audio PES packets between video packets in TS or an fMP4 fragment, without measuring the time spent waiting for input. -A publisher does not count ingest network delay in this field; a consumer measures network delay separately. +A publisher measures the field from media timestamps alone, never from the time a frame was written. +It covers the reorder delay of a frame presented after frames decoded behind it, and the media span of a batch written together, such as a run of audio PES packets between video packets in TS or an fMP4 fragment. +Neither a constant encoder delay, a capture clock, nor ingest network delay counts; a consumer measures network delay separately. A publisher MUST round the value up to a whole number of milliseconds, so a consumer sizing a buffer against it is never handed a bound below the real one. -A publisher MUST NOT advertise `0`; a track that flushes each frame immediately omits the field instead. +A publisher MUST NOT advertise `0`; a track that writes each frame in presentation order omits the field instead. A consumer receiving `0` SHOULD treat the field as absent, since it is a publisher rounding down rather than a claim of zero delay. A publisher MUST NOT lower a previously advertised value, since a burst it emitted once it may emit again. For example: -- A frame flushed without extra delay contributes no `jitter`, regardless of frame rate. -- An encoder that consistently flushes 200 milliseconds late contributes no `jitter`; only variation above its own minimum counts. -- A fragment or packet batch contributes the media span between its earliest timestamp and flush point. -- Reordered frames contribute the delay they were held before flushing, without treating a decode-order presentation timestamp gap as delay by itself. +- A frame written in presentation order contributes no `jitter`, regardless of frame rate or when it was written. +- A B-frame stream contributes its reorder delay, the presentation timestamp minus the decode timestamp. +- A fragment or packet batch contributes the media span between its earliest and latest timestamps. ### warmup {#field-warmup} @@ -753,17 +751,18 @@ For video, `warmup` can describe a refresh cycle during which decoding converges For audio, it describes decoder pre-roll, for example 80 milliseconds for Opus, and does not change the group-start rule. ### delay {#field-delay} -The maximum amount, in milliseconds, by which a rendition's minimum flush lateness ({{field-jitter}}) has trailed the smallest minimum among the broadcast's renditions that measure it. +The maximum amount, in milliseconds, by which a frame's presentation timestamp has trailed one that another rendition of the broadcast had already written. If absent, a consumer SHOULD assume the rendition does not trail the others. -A publisher measures each rendition's minimum over the same recent window it uses for `jitter`, from one clock shared by every rendition, and advertises the largest difference observed. -A container importer does not measure `delay`. +A publisher compares each frame with the highest timestamp written so far by every other rendition on the same clock, and advertises the largest shortfall observed. +A rendition alone on the broadcast measures no `delay`, and a frame trailing only its own rendition's earlier frames counts toward `jitter` instead. +A rendition that ends, or breaks its timeline, stops holding the others back. The rounding, `0`, and never-lower rules of `jitter` apply unchanged. A consumer SHOULD hold at least the largest `delay` plus `jitter` among the renditions it plays together. A consumer MUST NOT subtract one rendition's `delay` from another's: each is a maximum over the life of the stream, so two values need not share an origin. -For example, a video encoder that flushes 200 milliseconds after the audio encoder for the same media time advertises a video `delay` of 200 and no audio `delay`. +For example, with video at 20 millisecond intervals and audio at 15 written as video 0, video 20, audio 0, audio 15, video 40, audio 30, video 60, audio 45, audio 60, the audio frame at 0 trails video already written at 20, so audio advertises a `delay` of 20 and video none. ### enabled {#field-enabled} The `enabled` field says whether a consumer may select an audio or video rendition; a text rendition does not carry it. @@ -1207,12 +1206,12 @@ This document has no IANA actions. - Specified conversion between relative broadcast references and MSF track namespaces without changing track identity, reading a namespace aligned with the catalog path's trailing segments as the catalog itself. - Added optional audio and video `warmup` durations, rounded up to whole milliseconds, and allowed video groups to start at convergent random access pictures. A consumer also decoding from an earlier group start may present frames before `warmup`, switching to that decoder once it reaches the presented timestamp. -- Defined encoder `jitter` as flush lateness above the rendition's own recent minimum, replacing fixed frame-duration hints; container batches retain media-span estimates. +- Defined `jitter` from media timestamps alone, the reorder delay and batch spans a publisher writes, replacing fixed frame-duration hints and wall-clock flush lateness. - Timed CMAF samples from the frame timestamp, which is the fragment's earliest presentation time; `tfdt` is only relative within the fragment, and places the samples of an untimed frame. A timed CMAF track counts in its `mdhd` ticks. - Allowed a DVR to delete timeline objects no checkpoint recovery needs, oldest first. - Replaced the live-edge floor with monotonic group starts: a group may overlap the previous group's content but not start before it. - Added optional `bitrate` and `jitter` fields to `json` and `binary` track entries. -- Added the optional `delay` rendition field: how far a rendition's minimum flush lateness trails the broadcast's earliest rendition, never lowered once advertised and never subtracted across renditions. +- Added the optional `delay` rendition field: how far a rendition's timestamps trail those other renditions already wrote, never lowered once advertised and never subtracted across renditions. - Recommended namespaced keys for application root sections. - Added the optional `delay` field to `json` and `binary` track entries, measured only from payloads stamped with their capture time on the broadcast clock. - Replaced the video `stalled` field with an optional `enabled` field on audio and video renditions. A consumer MUST NOT select a disabled rendition and ignores `stalled`. A consumer that predates `enabled` keeps selecting a disabled rendition. diff --git a/go/wrapper/MIGRATE.md b/go/wrapper/MIGRATE.md index a181777b005..dbe045b8357 100644 --- a/go/wrapper/MIGRATE.md +++ b/go/wrapper/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `media.TrackProducer.Flush` is gone + +The catalog no longer measures jitter from when a frame was written, so there is nothing to report after writing one. +Delete the call; `delay` is measured from the order of the timestamps you write. + ### Video rendition preference `media.Video` gains `Preference int32`. Update unkeyed struct literals or use keyed fields, leaving `Preference: 0` for the default ranking. Upgrade the matching FFI module too. Higher values are preferred; compatibility transcodes use `-1`. diff --git a/go/wrapper/media/media.go b/go/wrapper/media/media.go index 053b24a367b..f823ced385a 100644 --- a/go/wrapper/media/media.go +++ b/go/wrapper/media/media.go @@ -364,17 +364,7 @@ func (m *TrackProducer) WriteFrame(frame moq.Frame) error { return m.inner.WriteFrame(f) } -// Flush records a local encoder's frame handoff on the broadcast media clock. -// Call after WriteFrame only for local encoder output, not file or network imports. -func (m *TrackProducer) Flush(timestamp time.Duration) error { - us, err := bridge.Duration(timestamp, "timestamp") - if err != nil { - return err - } - return m.inner.Flush(us) -} - -// Discontinuity marks a timeline break and restarts handoff measurement, preserving advertised jitter. +// Discontinuity marks a timeline break, preserving the advertised jitter and delay. func (m *TrackProducer) Discontinuity() error { return m.inner.Discontinuity() } diff --git a/js/publish/src/audio/capture.test.ts b/js/publish/src/audio/capture.test.ts index 9c0aa7e928f..f7ce74a3b3c 100644 --- a/js/publish/src/audio/capture.test.ts +++ b/js/publish/src/audio/capture.test.ts @@ -1,7 +1,7 @@ import { expect, mock, spyOn, test } from "bun:test"; import * as Moq from "@moq/net"; import { Effect, Signal } from "@moq/signals"; -import { Baseline } from "../jitter"; +import { Order } from "../order"; // The capture pulls its processor in as a `?worklet` blob URL, which the bun test loader can't // resolve. Stub it so the module imports; the value is only ever passed to our fake addModule. @@ -563,7 +563,7 @@ test("a failed encoder stays failed until the capture source changes", async () close: () => {}, }; const encoder = new Encoder("audio", { - broadcast: { audio: () => rendition, baseline: new Baseline() } as never, + broadcast: { audio: () => rendition, order: new Order() } as never, capture, enabled, }); diff --git a/js/publish/src/audio/encoder.test.ts b/js/publish/src/audio/encoder.test.ts index d961edd2f2a..2e040d5b01b 100644 --- a/js/publish/src/audio/encoder.test.ts +++ b/js/publish/src/audio/encoder.test.ts @@ -5,7 +5,7 @@ import * as Json from "@moq/json"; import * as Moq from "@moq/net"; import { Signal } from "@moq/signals"; import { Broadcast } from "../broadcast"; -import { Baseline } from "../jitter"; +import { Estimator, Order } from "../order"; import type { AudioFrame, Format } from "./capture"; import { type Codec, Encoder, resolve, toEncoderConfig } from "./encoder"; @@ -27,7 +27,7 @@ describe("resolve", () => { expect(Number(resolve(captured, "opus").catalog.warmup)).toBe(80); }); - // The exact frame duration does not imply encoder flush lateness. + // The exact frame duration is not a jitter hint. test("keeps a 2.5ms Opus frame exact without a catalog hint", () => { const resolved = resolve(captured, { mime: "opus", frameDuration: Milli(2.5) }); expect(resolved.frameDuration).toBe(Micro(2_500)); @@ -256,7 +256,7 @@ function configured(): Promise { // An Encoder wired to a fake capture feed, recording each written frame as [timestamp, payload bytes] // and how many frames each group carries. -async function setup(baseline = new Baseline(), codec?: Codec, groupDuration?: Milli) { +async function setup(order = new Order(), codec?: Codec, groupDuration?: Milli) { const configuring = configured(); const track = new Moq.Track.Producer("audio").accept({ timescale: Moq.Time.Timescale.MILLI }); @@ -291,7 +291,7 @@ async function setup(baseline = new Baseline(), codec?: Codec, groupDuration?: M const enabled = new Signal(true); const encoder = new Encoder("audio", { - broadcast: { audio: () => rendition, baseline } as never, + broadcast: { audio: () => rendition, order } as never, capture: capture as never, enabled, codec, @@ -472,7 +472,7 @@ test("a push completing several frames keeps the encoder running", async () => { // so the first frame after it opens a fresh one. test("a group duration packs frames until the minimum and restarts after a break", async () => { using _webcodecs = installFakeWebCodecs(); - using env = await setup(new Baseline(), undefined, Milli(100)); + using env = await setup(new Order(), undefined, Milli(100)); const { enabled, feed, written, groups, appended, writes } = env; let index = 0; @@ -519,7 +519,7 @@ test("default audio groups carry twenty milliseconds", async () => { [2.5, 1, 0], ] as const) { using env = await setup( - new Baseline(), + new Order(), { mime: "opus", frameDuration: Milli(frameMs) }, groupMs === undefined ? undefined : Milli(groupMs), ); @@ -539,14 +539,14 @@ test("default audio groups carry twenty milliseconds", async () => { // later audio earlier. A plain-JS caller passing the old knob must not reach the encoder. test("never enables Opus DTX", async () => { using _webcodecs = installFakeWebCodecs(); - using env = await setup(new Baseline(), { mime: "opus", usedtx: true } as Codec); + using env = await setup(new Order(), { mime: "opus", usedtx: true } as Codec); expect(env.config.opus?.usedtx).toBeUndefined(); }); // The catalog bitrate is a maximum: lowering the knob retunes the encoder, raising it raises both. test("the catalog bitrate only rises", async () => { using _webcodecs = installFakeWebCodecs(); - using env = await setup(new Baseline(), { mime: "opus", bitrate: 64_000 }); + using env = await setup(new Order(), { mime: "opus", bitrate: 64_000 }); const { encoder } = env; expect(env.config.bitrate).toBe(64_000); expect(encoder.out.catalog.peek()?.bitrate as number | undefined).toBe(64_000); @@ -564,21 +564,21 @@ test("the catalog bitrate only rises", async () => { expect(encoder.out.catalog.peek()?.bitrate as number | undefined).toBe(96_000); }); -// Another rendition on the same broadcast flushing with far less lateness leaves this one trailing -// it, which the catalog advertises as `delay`. -test("a rendition trailing the broadcast's earliest advertises delay", async () => { +// Another rendition on the same broadcast that already wrote media ahead of this one's frames leaves +// this one trailing it, which the catalog advertises as `delay`. +test("a rendition written behind another advertises delay", async () => { using _webcodecs = installFakeWebCodecs(); const clock = spyOn(performance, "now").mockReturnValue(200); try { - const baseline = new Baseline(); - using env = await setup(baseline); + const order = new Order(); + using env = await setup(order); const { encoder, feed } = env; expect(encoder.out.catalog.peek()?.delay).toBeUndefined(); - // A sibling that flushes each frame the instant it is captured. - baseline.observe(0, performance.now() * 1000); + // A sibling that already wrote media up to now. + new Estimator().write(performance.now() * 1000, order); // Captured 100ms ago, with a clock origin that keeps timestamps nonnegative. const start = performance.now() * 1000 - 100_000; @@ -858,7 +858,7 @@ test.each(["failure first", "config first"])( test("Opus description is probed with the exact live config before advertising", async () => { using _codecs = installFakeWebCodecs(); LaggingAudioEncoder.description = (config) => new TextEncoder().encode(JSON.stringify(config)); - using env = await setup(new Baseline(), { + using env = await setup(new Order(), { mime: "opus", frameDuration: Milli(2.5), complexity: 3, diff --git a/js/publish/src/audio/encoder.ts b/js/publish/src/audio/encoder.ts index 12c1fb1a1bd..0342be1de9a 100644 --- a/js/publish/src/audio/encoder.ts +++ b/js/publish/src/audio/encoder.ts @@ -15,7 +15,7 @@ import { } from "@moq/signals"; import type { Broadcast } from "../broadcast"; import { errors } from "../internal"; -import { type Baseline, Estimator } from "../jitter"; +import { Estimator, type Order } from "../order"; import type { AudioFrame, Capture, Format } from "./capture"; import { Gain } from "./gain"; import { Resampler } from "./resampler"; @@ -223,7 +223,7 @@ export class Encoder { #seen?: { capture?: Capture; source?: unknown; format?: Format }; #signals = new Effect(); #estimator = new Estimator(); - // The estimator's jitter and delay, republished whenever either rises. + // The estimator's delay, republished whenever it rises. #estimate = new Signal({}); // What this rendition advertised on each broadcast, keyed by the broadcast or, with none, the // encoder. A broadcast keeps a rendition's track when it re-registers, so returning to one @@ -394,7 +394,7 @@ export class Encoder { const resolved = effect.get(this.#config); if (!enabled || !format || fatal || !resolved) return; - this.#encode(broadcast.baseline, format, effect); + this.#encode(broadcast.order, format, effect); }); // Each subscription gets its own producer. Losing demand ends the epoch, as stopping the @@ -596,10 +596,12 @@ export class Encoder { // Encode captured audio frames into whichever track producer is live. The broadcast owns the // track's lifetime, so this never closes it; a fatal encoder error aborts only the live track. - #encode(baseline: Baseline, format: Format, effect: Effect): void { + #encode(order: Order, format: Format, effect: Effect): void { // The parent owns the config subscription before the async codec scope is opened. const resolved = this.#config.peek(); if (!resolved) return; + // A stopped rendition stops holding the others' delay back. + effect.cleanup(() => order.leave(this.#estimator)); effect.spawn(async () => { // We're using an async polyfill temporarily for Safari support. await Util.Libav.polyfill(); @@ -648,7 +650,7 @@ export class Encoder { const keyframe = live.start === undefined || timestamp - live.start >= this.#groupDuration; if (keyframe) live.start = timestamp; live.producer.encode(frame, timestamp, keyframe); - if (this.#estimator.flush(frame.timestamp, baseline)) { + if (this.#estimator.write(frame.timestamp, order)) { this.#estimate.set({ ...this.#estimator.estimate }); } }, diff --git a/js/publish/src/broadcast.ts b/js/publish/src/broadcast.ts index 80496f9c8c0..c4e8b34d852 100644 --- a/js/publish/src/broadcast.ts +++ b/js/publish/src/broadcast.ts @@ -7,7 +7,7 @@ import { Effect, type Getter, getter, type Inputs, type Readonlys, Signal } from import { rotateVideoDimensions } from "@moq/video"; import { CatalogProducer } from "./catalog"; import { errors } from "./internal"; -import { Baseline } from "./jitter"; +import { Order } from "./order"; import { type Kind, Rendition } from "./rendition"; // The refresh ratio video tracks advertise: a joiner forces a keyframe once a group's deltas @@ -103,10 +103,10 @@ export class Broadcast { readonly net: Getter = this.#net; /** - * @internal The recent minimum flush lateness across every rendition, which each encoder - * measures its catalog `delay` against. Per broadcast, so a swapped one starts fresh. + * @internal The highest timestamp each rendition has written, which each encoder measures its + * catalog `delay` against. Per broadcast, so a swapped one starts fresh. */ - readonly baseline = new Baseline(); + readonly order = new Order(); // The registered renditions keyed by full track name. A plain object so deep-equality detects a // key add/remove; the Rendition values compare by identity, which is stable. diff --git a/js/publish/src/jitter.test.ts b/js/publish/src/jitter.test.ts deleted file mode 100644 index f2970adc854..00000000000 --- a/js/publish/src/jitter.test.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { expect, test } from "bun:test"; -import { u53 } from "@moq/hang/catalog"; -import { Baseline, Estimator } from "./jitter"; - -test("a batch flushed at its end counts its full media span", () => { - const broadcast = new Baseline(); - const estimator = new Estimator(); - estimator.flush(0, broadcast, 0); - expect(estimator.flush(0, broadcast, 120_000)).toBe(true); - expect(estimator.estimate.jitter).toBe(u53(120)); - expect(estimator.flush(40_000, broadcast, 120_000)).toBe(false); -}); - -test("a constant lateness is not jitter", () => { - const broadcast = new Baseline(); - const estimator = new Estimator(); - expect(estimator.flush(0, broadcast, 200_000)).toBe(false); - expect(estimator.flush(240_000, broadcast, 440_000)).toBe(false); - expect(estimator.estimate).toEqual({ jitter: undefined, delay: undefined }); -}); - -test("a sliding minimum bounds slow media clock drift", () => { - const broadcast = new Baseline(); - const estimator = new Estimator(); - for (let second = 0; second < 100; second++) { - estimator.flush(second * 1_000_000, broadcast, second * 1_001_000); - } - expect(estimator.estimate.jitter).toBeLessThanOrEqual(10); - expect(estimator.estimate.jitter).toBeGreaterThan(0); -}); - -test("a faster-than-real-time source keeps lowering the baseline", () => { - const broadcast = new Baseline(); - const estimator = new Estimator(); - for (let second = 0; second < 100; second++) { - expect(estimator.flush(second * 2_000_000, broadcast, second * 1_000_000)).toBe(false); - } -}); - -test("each rendition measures jitter against its own minimum", () => { - const broadcast = new Baseline(); - const audio = new Estimator(); - const video = new Estimator(); - audio.flush(0, broadcast, 0); - // A slower encoder's constant offset is not jitter. - video.flush(0, broadcast, 200_000); - video.flush(40_000, broadcast, 300_000); - video.flush(240_000, broadcast, 440_000); - expect(audio.estimate.jitter).toBeUndefined(); - expect(video.estimate.jitter).toBe(u53(60)); -}); - -test("a rendition trailing the broadcast's earliest advertises the gap as delay", () => { - const broadcast = new Baseline(); - const audio = new Estimator(); - const video = new Estimator(); - for (let frame = 0; frame < 10; frame++) { - const timestamp = frame * 20_000; - audio.flush(timestamp, broadcast, timestamp + 5_000); - video.flush(timestamp, broadcast, timestamp + 205_000); - } - expect(audio.estimate).toEqual({ jitter: undefined, delay: undefined }); - expect(video.estimate).toEqual({ jitter: undefined, delay: u53(200) }); -}); - -test("delay is a lifetime maximum", () => { - const broadcast = new Baseline(); - const audio = new Estimator(); - const video = new Estimator(); - audio.flush(0, broadcast, 0); - expect(video.flush(0, broadcast, 150_000)).toBe(true); - expect(video.estimate.delay).toBe(u53(150)); - - // Video catches up, then audio stops long enough to leave the window: neither lowers it. - expect(video.flush(100_000, broadcast, 100_000)).toBe(false); - expect(video.flush(20_000_000, broadcast, 20_000_000)).toBe(false); - expect(video.estimate.delay).toBe(u53(150)); -}); - -test("broadcasts do not share a baseline", () => { - const audio = new Estimator(); - const video = new Estimator(); - audio.flush(0, new Baseline(), 0); - video.flush(0, new Baseline(), 200_000); - expect(video.estimate.delay).toBeUndefined(); -}); diff --git a/js/publish/src/jitter.ts b/js/publish/src/jitter.ts deleted file mode 100644 index d5fc8c79cd1..00000000000 --- a/js/publish/src/jitter.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { now as clockNow, Nano } from "@moq/clock"; -import * as Catalog from "@moq/hang/catalog"; - -const WINDOW = 10_000_000; // 10 seconds in microseconds. - -type Sample = { at: number; lateness: number }; - -// The minimum flush lateness over the last window, so a media clock drifting against the wall clock -// does not ratchet forever. The queue is ordered by lateness so its head is the minimum, and each -// sample enters/leaves once. -// -// Every js/publish timestamp is the installed clock in microseconds, so lateness compares across renditions and one -// window can be shared by a whole broadcast. -export class Baseline { - #samples: Sample[] = []; - #head = 0; - - // Insert a lateness observed at `now` and return the window's minimum. - observe(lateness: number, now: number): number { - const cutoff = now - WINDOW; - while (this.#head < this.#samples.length && this.#samples[this.#head].at < cutoff) this.#head++; - - while (this.#samples.length > this.#head) { - const last = this.#samples.at(-1); - if (!last || last.lateness < lateness) break; - this.#samples.pop(); - } - if (this.#head === this.#samples.length) { - this.#samples = []; - this.#head = 0; - } else if (this.#head > 1024 && this.#head * 2 > this.#samples.length) { - this.#samples = this.#samples.slice(this.#head); - this.#head = 0; - } - this.#samples.push({ at: now, lateness }); - return this.#samples[this.#head].lateness; - } -} - -// One rendition's catalog `jitter` and `delay`, each a lifetime maximum in whole milliseconds. -// Mirrors `moq_mux::catalog::Estimator`. -export class Estimator { - #baseline = new Baseline(); - #jitter = 0; - #delay = 0; - - // The catalog fields measured so far, each absent until nonzero. - get estimate(): { jitter?: Catalog.U53; delay?: Catalog.U53 } { - return { - jitter: this.#jitter ? Catalog.u53(this.#jitter) : undefined, - delay: this.#delay ? Catalog.u53(this.#delay) : undefined, - }; - } - - // Measure a frame handed to the transport now, against the `broadcast` baseline every rendition - // in the catalog shares. Jitter is the spread above this rendition's own recent minimum lateness, - // so a constant encoder delay is not jitter; delay is how far that minimum trails the broadcast's. - // Returns whether either estimate rose. - flush(timestamp: number, broadcast: Baseline, now: number = Nano.toMicro(Nano(clockNow()))): boolean { - const lateness = now - timestamp; - const earliest = broadcast.observe(lateness, now); - const minimum = this.#baseline.observe(lateness, now); - - const jitter = Math.ceil((lateness - minimum) / 1000); - const delay = Math.ceil((minimum - earliest) / 1000); - if (jitter <= this.#jitter && delay <= this.#delay) return false; - - this.#jitter = Math.max(this.#jitter, jitter); - this.#delay = Math.max(this.#delay, delay); - return true; - } -} diff --git a/js/publish/src/order.test.ts b/js/publish/src/order.test.ts new file mode 100644 index 00000000000..af10e4b2feb --- /dev/null +++ b/js/publish/src/order.test.ts @@ -0,0 +1,67 @@ +import { expect, test } from "bun:test"; +import { u53 } from "@moq/hang/catalog"; +import { Estimator, Order } from "./order"; + +// Video at 20ms and audio at 15ms, written `v0 v20 a0 a15 v40 a30 v60 a45 a60`: audio's first frame +// lands after video has already written 20ms ahead of it. +test("the interleave measures the worst trail", () => { + const order = new Order(); + const video = new Estimator(); + const audio = new Estimator(); + for (const [estimator, millis] of [ + [video, 0], + [video, 20], + [audio, 0], + [audio, 15], + [video, 40], + [audio, 30], + [video, 60], + [audio, 45], + [audio, 60], + ] as const) { + estimator.write(millis * 1000, order); + } + expect(audio.estimate).toEqual({ delay: u53(20) }); + expect(video.estimate).toEqual({ delay: undefined }); +}); + +test("a lone rendition never measures a delay", () => { + const order = new Order(); + const video = new Estimator(); + for (const timestamp of [0, 120_000, 40_000, 80_000]) { + expect(video.write(timestamp, order)).toBe(false); + } + expect(video.estimate.delay).toBeUndefined(); +}); + +test("delay is a lifetime maximum", () => { + const order = new Order(); + const audio = new Estimator(); + const video = new Estimator(); + audio.write(150_000, order); + expect(video.write(0, order)).toBe(true); + expect(video.estimate.delay).toBe(u53(150)); + + // Video catches up and audio leaves: neither lowers it. + expect(video.write(200_000, order)).toBe(false); + order.leave(audio); + expect(video.write(240_000, order)).toBe(false); + expect(video.estimate.delay).toBe(u53(150)); +}); + +test("a rendition that left no longer holds the others back", () => { + const order = new Order(); + const ahead = new Estimator(); + const video = new Estimator(); + ahead.write(10_000_000, order); + order.leave(ahead); + expect(video.write(0, order)).toBe(false); +}); + +test("broadcasts do not share an order", () => { + const audio = new Estimator(); + const video = new Estimator(); + audio.write(200_000, new Order()); + video.write(0, new Order()); + expect(video.estimate.delay).toBeUndefined(); +}); diff --git a/js/publish/src/order.ts b/js/publish/src/order.ts new file mode 100644 index 00000000000..8e6046ba9fe --- /dev/null +++ b/js/publish/src/order.ts @@ -0,0 +1,52 @@ +import * as Catalog from "@moq/hang/catalog"; + +// The highest timestamp each rendition of one broadcast has written. A frame written behind media +// another rendition already wrote has to be waited for by a consumer playing both, by however far it +// trails. Measured from timestamps alone, so neither a capture clock nor a stalled encoder reads as +// delay unless the frames really reach the transport out of order. +// +// Every js/publish timestamp is the installed clock in microseconds, so they compare across +// renditions and one order can be shared by a whole broadcast. +export class Order { + #highs = new Map(); + + // Record `estimator`'s frame at `timestamp` and return how far it trails another rendition's + // highest, or undefined when it trails none. + write(estimator: Estimator, timestamp: number): number | undefined { + let others: number | undefined; + for (const [seat, high] of this.#highs) { + if (seat !== estimator && (others === undefined || high > others)) others = high; + } + const high = this.#highs.get(estimator); + if (high === undefined || timestamp > high) this.#highs.set(estimator, timestamp); + return others !== undefined && others > timestamp ? others - timestamp : undefined; + } + + // Forget `estimator`'s frames, so a rendition that stopped no longer holds the others back. + leave(estimator: Estimator): void { + this.#highs.delete(estimator); + } +} + +// One rendition's catalog `delay`, a lifetime maximum in whole milliseconds. +// Mirrors `moq_mux::catalog::Estimator`. +export class Estimator { + #delay = 0; + + // The catalog field measured so far, absent until nonzero. + get estimate(): { delay?: Catalog.U53 } { + return { delay: this.#delay ? Catalog.u53(this.#delay) : undefined }; + } + + // Place a frame written at `timestamp` in the broadcast's `order`. Returns whether the delay rose. + write(timestamp: number, order: Order): boolean { + const behind = order.write(this, timestamp); + if (behind === undefined) return false; + + const delay = Math.ceil(behind / 1000); + if (delay <= this.#delay) return false; + + this.#delay = delay; + return true; + } +} diff --git a/js/publish/src/video/capture.test.ts b/js/publish/src/video/capture.test.ts index f5ef717ff36..95c6c4c8d1e 100644 --- a/js/publish/src/video/capture.test.ts +++ b/js/publish/src/video/capture.test.ts @@ -2,7 +2,7 @@ import { expect, mock, spyOn, test } from "bun:test"; import { Manual, Milli, Nano } from "@moq/clock"; import * as Moq from "@moq/net"; import { Effect, Signal } from "@moq/signals"; -import { Baseline } from "../jitter"; +import { Order } from "../order"; import type { StreamTrack } from "./types"; mock.module("./capture-worker.ts?worklet", () => ({ default: async () => "blob:fake-worker" })); @@ -437,7 +437,7 @@ test("an encoder resuming on a still source encodes a keyframe of the current pi const rendition = { config: new Signal(undefined), track: live, close: () => {} }; const encoder = new Encoder("video", { enabled: true, - broadcast: { video: () => rendition, baseline: new Baseline() } as never, + broadcast: { video: () => rendition, order: new Order() } as never, capture, }); @@ -543,7 +543,7 @@ test("an encoder torn down while a frame is in flight closes that frame", async const rendition = { config: new Signal(undefined), track: live, close: () => {} }; const encoder = new Encoder("video", { enabled: true, - broadcast: { video: () => rendition, baseline: new Baseline() } as never, + broadcast: { video: () => rendition, order: new Order() } as never, capture, }); diff --git a/js/publish/src/video/encoder-stall.test.ts b/js/publish/src/video/encoder-stall.test.ts index cc31119b82f..4d42671cc09 100644 --- a/js/publish/src/video/encoder-stall.test.ts +++ b/js/publish/src/video/encoder-stall.test.ts @@ -4,7 +4,7 @@ import { u53 } from "@moq/hang/catalog"; import * as Moq from "@moq/net"; import { Signal } from "@moq/signals"; import { Fanout } from "../fanout"; -import { Baseline } from "../jitter"; +import { Estimator, Order } from "../order"; import { Encoder } from "./encoder"; // Model an encoder that cannot accept more work until the test completes its pending input. @@ -106,10 +106,12 @@ test("a slower-than-realtime encoder drops input instead of ratcheting catalog l { clone: (frame) => frame.clone(), release: (frame) => frame.close() }, ); const track = new Moq.Track.Producer("video").accept({ timescale: Moq.Time.Timescale.MICRO }); - const baseline = new Baseline(); + const order = new Order(); + // An on-time audio rendition shares the broadcast, writing each capture as it happens. + const audio = new Estimator(); const rendition = { config: new Signal(undefined), track: new Signal(track), close: () => track.close() }; const encoder = new Encoder("video", { - broadcast: { video: () => rendition, baseline } as never, + broadcast: { video: () => rendition, order } as never, capture: { in: { source: new Signal({ getSettings: () => ({ frameRate: 25 }), getConstraints: () => ({}) }) }, out: { display: new Signal({ width: 640, height: 480 }), frames: new Signal(fanout) }, @@ -119,7 +121,7 @@ test("a slower-than-realtime encoder drops input instead of ratcheting catalog l const send = async (millis = 40) => { await clock.advance(Nano.fromMilli(Milli(millis))); const timestamp = Nano.toMicro(Nano(clock.now())); - baseline.observe(0, timestamp); // An on-time audio rendition shares the broadcast. + audio.write(timestamp, order); controller.enqueue(new Input(timestamp) as never); await clock.advance(Nano.zero); }; diff --git a/js/publish/src/video/encoder.test.ts b/js/publish/src/video/encoder.test.ts index 67217c74fea..b01151f9e62 100644 --- a/js/publish/src/video/encoder.test.ts +++ b/js/publish/src/video/encoder.test.ts @@ -6,7 +6,7 @@ import * as Json from "@moq/json"; import * as Moq from "@moq/net"; import { Signal } from "@moq/signals"; import { Broadcast } from "../broadcast"; -import { Baseline } from "../jitter"; +import { Order } from "../order"; import { Encoder } from "./encoder"; // How long a fake hardware probe takes on the installed clock. @@ -118,7 +118,7 @@ test("encoding tracks encoder config in its child effect", async () => { track: new Signal(track), close: () => track.close(), }; - const broadcast = { video: () => rendition, baseline: new Baseline() }; + const broadcast = { video: () => rendition, order: new Order() }; const capture = { in: { source: new Signal(undefined) }, out: { @@ -165,7 +165,7 @@ test("a demand gap marks a discontinuity and leaves the broadcast-owned track op }; const encoder = new Encoder("video", { enabled: true, - broadcast: { video: () => rendition, baseline: new Baseline() } as never, + broadcast: { video: () => rendition, order: new Order() } as never, capture: capture as never, }); @@ -222,7 +222,7 @@ test("a bandwidth estimate updates the bitrate without blanking the config or re const encoder = new Encoder("video", { enabled: true, - broadcast: { video: () => rendition, baseline: new Baseline() } as never, + broadcast: { video: () => rendition, order: new Order() } as never, capture: capture as never, bandwidth, }); @@ -304,7 +304,7 @@ test("every published config fits a probe for its own codec", async () => { const encoder = new Encoder("video", { enabled: true, - broadcast: { video: () => rendition, baseline: new Baseline() } as never, + broadcast: { video: () => rendition, order: new Order() } as never, capture: capture as never, bandwidth, }); @@ -625,7 +625,7 @@ test("disabling encodes one black keyframe and keeps the rendition with enabled: const enabled = new Signal(true); const encoder = new Encoder("video", { enabled, - broadcast: { video: () => rendition, baseline: new Baseline() } as never, + broadcast: { video: () => rendition, order: new Order() } as never, capture: capture as never, }); @@ -785,7 +785,7 @@ test("the probe encodes with the frame rate and bitrate ceiling", async () => { const bandwidth = new Moq.Bandwidth.Allocator(estimate); const encoder = new Encoder("video", { enabled: true, - broadcast: { video: () => rendition, baseline: new Baseline() } as never, + broadcast: { video: () => rendition, order: new Order() } as never, capture: capture as never, bandwidth, }); @@ -956,7 +956,7 @@ async function recordKeyframes( }; const encoder = new Encoder("video", { enabled: true, - broadcast: { video: () => rendition, baseline: new Baseline() } as never, + broadcast: { video: () => rendition, order: new Order() } as never, capture: capture as never, }); @@ -1211,7 +1211,7 @@ test("a shrinking capture resizes in band below the advertised ceiling", async ( }; const encoder = new Encoder("video", { enabled: true, - broadcast: { video: () => rendition, baseline: new Baseline() } as never, + broadcast: { video: () => rendition, order: new Order() } as never, capture: capture as never, }); const send = async (millis: number) => { @@ -1281,7 +1281,7 @@ test("the catalog bitrate and frame rate only rise", async () => { const bandwidth = new Moq.Bandwidth.Allocator(estimate); const encoder = new Encoder("video", { enabled: true, - broadcast: { video: () => rendition, baseline: new Baseline() } as never, + broadcast: { video: () => rendition, order: new Order() } as never, capture: capture as never, bandwidth, }); @@ -1326,7 +1326,7 @@ test("each broadcast keeps its own ceiling and maxima", async () => { track: new Signal(undefined), close: () => {}, }); - const first = { video: rendition, baseline: new Baseline() }; + const first = { video: rendition, order: new Order() }; const broadcast = new Signal(first); const capture = { in: { source: new Signal({ getSettings: () => ({ frameRate: 30 }), getConstraints: () => ({}) }) }, @@ -1348,7 +1348,7 @@ test("each broadcast keeps its own ceiling and maxima", async () => { expect(encoder.out.catalog.peek()).toMatchObject({ codedWidth: 1920, codedHeight: 1072, framerate: 60 }); const bitrate = encoder.out.catalog.peek()?.bitrate ?? 0; - broadcast.set({ video: rendition, baseline: new Baseline() }); + broadcast.set({ video: rendition, order: new Order() }); // The previous broadcast's probe never describes the new track, even while its own runs. for (let i = 0; i < 5; i++) await Promise.resolve(); expect(encoder.out.catalog.peek()).toBeUndefined(); @@ -1371,7 +1371,7 @@ test("each broadcast keeps its own ceiling and maxima", async () => { enabled.set(false); await settle(); expect(encoder.out.catalog.peek()?.enabled).toBe(false); - broadcast.set({ video: rendition, baseline: new Baseline() }); + broadcast.set({ video: rendition, order: new Order() }); await settle(); expect(encoder.out.catalog.peek()).toBeUndefined(); } finally { diff --git a/js/publish/src/video/encoder.ts b/js/publish/src/video/encoder.ts index 605ab7c27de..5d2328a4908 100644 --- a/js/publish/src/video/encoder.ts +++ b/js/publish/src/video/encoder.ts @@ -15,7 +15,7 @@ import { } from "@moq/signals"; import type { Broadcast } from "../broadcast"; import { errors } from "../internal"; -import { type Baseline, Estimator } from "../jitter"; +import { Estimator, type Order } from "../order"; import { hardwareReliable } from "../support/video"; import type { Capture } from "./capture"; import { normalizeSource, type Source } from "./types"; @@ -194,7 +194,7 @@ export class Encoder { }; #signals = new Effect(); #estimator = new Estimator(); - // The estimator's jitter and delay, republished whenever either rises. + // The estimator's delay, republished whenever it rises. #estimate = new Signal({}); // What this rendition advertised on each broadcast, keyed by the broadcast or, for a local preview, // the encoder. A broadcast keeps a rendition's track when it re-registers, so returning to one @@ -335,7 +335,7 @@ export class Encoder { effect.set(this.#out.active, enabled && !!track && !error, false); if (!enabled || !track || error) return; - this.#encode(track, broadcast.baseline, effect); + this.#encode(track, broadcast.order, effect); }); // Reserve against the connection for as long as this track is live. Wait @@ -366,9 +366,11 @@ export class Encoder { } // Encode captured frames into the track producer, reconfiguring when the resolved config changes. - #encode(track: Moq.Track.Producer, baseline: Baseline, effect: Effect): void { + #encode(track: Moq.Track.Producer, order: Order, effect: Effect): void { const capture = effect.get(this.in.capture); if (!capture) return; + // A stopped rendition stops holding the others' delay back. + effect.cleanup(() => order.leave(this.#estimator)); const producer = new Container.Legacy.Producer(track, new Container.Legacy.Format("video")); const demand = track.demand(); @@ -401,7 +403,7 @@ export class Encoder { producer.encode(frame, frame.timestamp as Micro, key); // Including a keyframe the encoder placed on its own, which opened a group too. if (key) answer(); - if (this.#estimator.flush(frame.timestamp, baseline)) { + if (this.#estimator.write(frame.timestamp, order)) { this.#estimate.set({ ...this.#estimator.estimate }); } }, diff --git a/js/watch/src/audio/latency.ts b/js/watch/src/audio/latency.ts index 8f28a04d515..ebc31fe4e30 100644 --- a/js/watch/src/audio/latency.ts +++ b/js/watch/src/audio/latency.ts @@ -4,11 +4,11 @@ import { Milli, Second } from "@moq/clock"; export interface Target { /** The arrival estimate from the container consumer. */ measured: Milli; - /** The flush span the rendition advertises, if any. */ + /** The batch or reorder span the rendition advertises as `jitter`, if any. */ advertised?: Milli; /** The codec's frame duration, if known. */ frame?: Milli; - /** The catalog `delay`: how far this rendition trails the broadcast's earliest one, if any. */ + /** The catalog `delay`: how far this rendition trails the broadcast's others, if any. */ delay?: Milli; } @@ -17,7 +17,7 @@ export interface Target { * the rendition's catalog delay. * * The advertised span is a floor rather than an addend, because the receiver's measurement already - * contains the publisher's flush delay. The catalog delay is an addend, because the measurement is + * contains the publisher's batching. The catalog delay is an addend, because the measurement is * taken against the track's own fastest frame and cancels any offset between tracks. See * doc/concept/audio-jitter.md. */ diff --git a/py/moq-rs/MIGRATE.md b/py/moq-rs/MIGRATE.md index 6b1c20f173b..b3caf78bdc1 100644 --- a/py/moq-rs/MIGRATE.md +++ b/py/moq-rs/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `media.TrackProducer.flush` is gone + +The catalog no longer measures jitter from when a frame was written, so there is nothing to report after writing one. +Delete the call; `delay` is measured from the order of the timestamps you write. + ### Video rendition preference `moq.media.Video` gains `preference`, defaulting to `0`. Prefer keyword construction and upgrade `moq-ffi` alongside the wrapper. Higher values are preferred; compatibility transcodes use `-1`. diff --git a/py/moq-rs/README.md b/py/moq-rs/README.md index 6cd705eb2f2..a5abdab4398 100644 --- a/py/moq-rs/README.md +++ b/py/moq-rs/README.md @@ -208,7 +208,7 @@ client = moq.Client( as it lands. `media.TrackProducer.write_frame(payload, timestamp)` requires the caller's capture time as -`timedelta`; `.flush(timestamp)` and returned `media.MediaFrame.timestamp` use it too. `demand()` owns the track name and +`timedelta`; returned `media.MediaFrame.timestamp` uses it too. `demand()` owns the track name and subscriber waits. `.cut()` / `.seek(sequence)` draw group boundaries; `.finish()` ends the import. ### Data diff --git a/py/moq-rs/moq/media.py b/py/moq-rs/moq/media.py index 10cd1a55a60..49398cebe43 100644 --- a/py/moq-rs/moq/media.py +++ b/py/moq-rs/moq/media.py @@ -50,7 +50,7 @@ ) from . import subscribe as _subscribe -from ._records import Frame, Subscription, _from_us, _subscription, _to_us +from ._records import Frame, Subscription, _from_us, _subscription from ._stream import _Stream from .publish import BroadcastProducer, TrackDemand, TrackRequest from .subscribe import BroadcastConsumer @@ -166,16 +166,8 @@ def write_frame(self, payload: bytes, timestamp: timedelta) -> None: """Write one encoded frame with a presentation timestamp.""" self._inner.write_frame(Frame(payload, timestamp)._ffi()) - def flush(self, timestamp: timedelta) -> None: - """Record a local encoder's frame handoff on the broadcast media clock. - - Call this after ``write_frame`` only for encoded live output. File, pipe, - and network imports should leave their jitter estimate clock free. - """ - self._inner.flush(_to_us(timestamp, "timestamp")) - def discontinuity(self) -> None: - """Mark a timeline break and restart handoff measurement, preserving advertised jitter.""" + """Mark a timeline break, preserving the advertised jitter and delay.""" self._inner.discontinuity() def cut(self) -> None: diff --git a/quest/m1/README.md b/quest/m1/README.md index 2e0126409dd..ea188f99f5d 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -142,7 +142,7 @@ tracks, so per-group stats stop costing cluster announces. - [Interop pre-skip](/quest/m1/interop-preskip.md) - the interop tone check anchors phase to timestamps, so a misapplied pre-skip fails - [moq-net internals](/quest/m1/net-internals.md) - no per-group session clones, no transport type parameter in lite and IETF internals, and the `Local` check measured - [Main is no slower than release](/quest/m1/release-perf/README.md) - every benchmark where main is slower than release is fixed or accepted with a reason, checked by a final paired run before the cut -- [Subscribe update camera check](/quest/m1/subscribe-update-churn.md) - a person checks a physical-camera publish for SUBSCRIBE_UPDATE chatter; bbb and the fake camera measured quiet +- [Audio capture follows the shared clock](/quest/m1/audio-capture-clock.md) - a physical microphone's audio timestamps stop drifting behind its camera, so the catalog `delay` holds and viewers stop resubscribing - [Unprefixed C++ free functions](/quest/m1/cpp-free-function-prefix.md) - generated free functions drop `moq_`, and `moq.hpp` deletes its five pass-through wrappers before C++ first publishes - [Error cleanups](/quest/m1/error-cleanups.md) - delete the unused `Error::SessionClosed`; raw transport failures already map to `Unroutable` - [API-diff check](/quest/m1/api-diff.md) - a recipe diffs every published package's API against its last release, run on PRs into release and against main before the cut diff --git a/quest/m1/audio-capture-clock.md b/quest/m1/audio-capture-clock.md new file mode 100644 index 00000000000..7f66d1d0437 --- /dev/null +++ b/quest/m1/audio-capture-clock.md @@ -0,0 +1,30 @@ +# [S] Audio capture follows the shared clock + +## Goal + +`js/publish` stamps captured audio on the clock video uses, so a physical +microphone's timestamps stay beside its camera's instead of falling behind them. +A `moq.dev/publish` camera and microphone advertise a steady audio catalog +`delay`, and a viewer sends no steady stream of `subscribe update: id=` lines. + +## Plan + +`js/publish/src/audio/capture-worklet.ts` stamps each quantum as +`sampleCount / sampleRate` past a wall-clock anchor taken once at creation. +A capture device that delivers fewer samples per second than its nominal rate +(or a render thread that drops quanta) loses time against the wall clock that +video frames are stamped on. + +On 2026-10-23 a physical camera and microphone published from `moq.dev/publish` +(release) drifted audio about 0.25 s per second behind video, unbounded. +Each rise republished the catalog, and the viewer pushed a new `maxDelay` to +every subscription: 146 video and 176 audio SUBSCRIBE_UPDATEs in 60 seconds. +The catalog `delay` now comes from timestamp order across renditions, so the +drift still reads as a rising audio `delay`; it is real, and only fixing the +clock removes it. + +Reproduce first and confirm the clock is the cause (log the worklet's stamp +minus `now()` per second). Then tie the stamp to the context's own clock, such +as `currentTime` mapped through `getOutputTimestamp()`, or re-anchor on a +measured gap the way the framer already does, without resampling drift into +the audio itself. diff --git a/quest/m1/subscribe-update-churn.md b/quest/m1/subscribe-update-churn.md deleted file mode 100644 index d879d3fc5f5..00000000000 --- a/quest/m1/subscribe-update-churn.md +++ /dev/null @@ -1,36 +0,0 @@ -# [XS] Check a physical camera for subscription update chatter - -## Goal - -Waiting on a person with a physical camera. A reported `moq.dev/publish` -session logged an endless alternating `subscribe update` for audio and video; -every source measured since then is quiet, and a camera is the one source left. -Delete this quest if the camera publish is quiet too. - -## Plan - -[moq-dev/moq#6305](https://github.com/moq-dev/moq/pull/6305) measured steady -SUBSCRIBE_UPDATEs per track at `5d5974f083` through `cdn.moq.dev`, 60 seconds -after a five-second warmup: - -| Source | Track | Warmup updates | Measured updates | -| --- | --- | --- | --- | -| `demo/bbb.hang` | audio | 0 | 0 | -| `demo/bbb.hang` | video | 1 | 0 | -| Chromium fake camera | audio | 0 | 0 | -| Chromium fake camera | video | 6 | 0 | - -To check, publish a physical camera and microphone from `moq.dev/publish`, then -watch that broadcast from another tab or device. In the viewer tab's DevTools -console, enable the Verbose level and filter on `subscribe update: id=`; -`js/net`'s Lite subscriber logs one line per written update with its track. -The publisher's own `subscribe update:` line can undercount, since it -coalesces updates decoded while its serving loop is busy. After the first few -seconds, count lines per track for a minute. - -- Quiet (no steady stream of lines): delete this quest and its - [m1](/quest/m1/README.md) entry. -- Chatter: the likely cause is `js/watch/src/media.ts` pushing every - `maxDelay` change into `subscriber.update`, compared exactly by - `js/net/src/lite/subscriber.ts`. Replace this check with a fix, such as - rounding the subscription's `maxDelay` up to a power of two ms. diff --git a/rs/moq-audio/src/encode/producer.rs b/rs/moq-audio/src/encode/producer.rs index dab38a56d52..5e299c3f515 100644 --- a/rs/moq-audio/src/encode/producer.rs +++ b/rs/moq-audio/src/encode/producer.rs @@ -443,12 +443,10 @@ impl Producer { self.grouped += self.encoder()?.frame_size() as u64; // Cut as soon as the group is long enough rather than when the next packet arrives, so // the relay sees it finish without waiting. No boundary to give: the next packet bounds - // this one, and codec frames have a deterministic duration anyway. Cut before observing - // the flush so a failed observation never leaves the group open. + // this one, and codec frames have a deterministic duration anyway. if self.grouped >= self.group_samples { self.track.cut(None)?; } - self.track.flush(timestamp, moq_net::time::now())?; Ok(()) } @@ -474,7 +472,6 @@ impl Producer { keyframe: false, duration: None, })?; - track.flush(timestamp, moq_net::time::now())?; } track.cut(Some(terminal.end))?; diff --git a/rs/moq-c/MIGRATE.md b/rs/moq-c/MIGRATE.md index 473549a9132..5fefe5dd27c 100644 --- a/rs/moq-c/MIGRATE.md +++ b/rs/moq-c/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `moq_publish_media_flush` is gone + +The catalog no longer measures jitter from when a frame was written, so there is nothing to report after writing one. +Delete the call; `delay` is measured from the order of the timestamps you write. + ### Video rendition preference `moq_video_config` gains `int32_t preference`. Initialize it to `0` to keep the default ranking; higher values are preferred and compatibility transcodes use `-1`. Rebuild C consumers against the matching header and library because the struct layout changes. diff --git a/rs/moq-c/README.md b/rs/moq-c/README.md index 43d5ee36ff4..24db000ddf6 100644 --- a/rs/moq-c/README.md +++ b/rs/moq-c/README.md @@ -76,7 +76,6 @@ int32_t moq_publish_container_write(uint32_t container, const uint8_t *payload, int32_t moq_publish_container_finish(uint32_t container); int32_t moq_publish_media_finish(uint32_t media); int32_t moq_publish_media_frame(uint32_t media, const uint8_t *payload, uintptr_t payload_size, uint64_t timestamp_us); -int32_t moq_publish_media_flush(uint32_t media, uint64_t timestamp_us); int32_t moq_publish_track(uint32_t broadcast, const char *name, uintptr_t name_len, const moq_track_info *info); int32_t moq_publish_track_group(uint32_t track); int32_t moq_publish_track_frame(uint32_t track, const uint8_t *payload, uintptr_t payload_size, uint64_t timestamp_us); diff --git a/rs/moq-c/src/api.rs b/rs/moq-c/src/api.rs index ee7c75f0f4b..259da3fb841 100644 --- a/rs/moq-c/src/api.rs +++ b/rs/moq-c/src/api.rs @@ -2505,23 +2505,7 @@ pub unsafe extern "C" fn moq_publish_media_frame( }) } -/// Record the transport handoff of one locally encoded frame for catalog jitter. -/// -/// `timestamp_us` is the frame's presentation time on the broadcast media clock. Call this -/// after [moq_publish_media_frame] only for local encoder output; file, pipe, and network imports -/// must remain clock-free. The monotonic handoff time is sampled inside this process. -/// -/// Returns zero on success, or a negative code for an invalid handle or timestamp. -#[unsafe(no_mangle)] -pub extern "C" fn moq_publish_media_flush(media: u32, timestamp_us: u64) -> i32 { - ffi::enter(move || { - let media = ffi::parse_id(media)?; - let timestamp = hang::container::Timestamp::from_micros(timestamp_us)?; - State::lock().publish.media_flush(media, timestamp) - }) -} - -/// Mark a timeline break and restart handoff measurement without lowering advertised jitter. +/// Mark a timeline break without lowering the advertised jitter or delay. /// /// Publishes a discontinuity marker; resumed frames must continue the broadcast media clock, /// and video must resume on a keyframe. diff --git a/rs/moq-c/src/publish.rs b/rs/moq-c/src/publish.rs index 06d5d627905..dbf11e0588d 100644 --- a/rs/moq-c/src/publish.rs +++ b/rs/moq-c/src/publish.rs @@ -400,14 +400,6 @@ impl Publish { Ok(()) } - /// Record a locally encoded frame's transport handoff. Generic imports remain clock-free - /// unless their caller explicitly identifies the frame as encoder output. - pub fn media_flush(&mut self, media: Id, timestamp: hang::container::Timestamp) -> Result<(), Error> { - let track = self.media.get_mut(media).ok_or(Error::MediaNotFound)?; - track.flush(timestamp, moq_net::time::now())?; - Ok(()) - } - /// Mark a timeline break on this media importer. pub fn media_discontinuity(&mut self, media: Id) -> Result<(), Error> { let track = self.media.get_mut(media).ok_or(Error::MediaNotFound)?; diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index 393fc8c9ed6..3504bb91d77 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -2431,7 +2431,6 @@ fn media_cut_bounds_audio_groups() { unsafe { moq_publish_media_frame(media, payload.as_ptr(), payload.len(), i * 20_000) }, 0 ); - assert_eq!(moq_publish_media_flush(media, i * 20_000), 0); assert_eq!(moq_publish_media_cut(media), 0, "each packet is its own group"); } @@ -2444,8 +2443,6 @@ fn media_cut_bounds_audio_groups() { assert_eq!(moq_publish_media_discontinuity(media), 0); // Both report a missing importer rather than panicking on an unknown id. - assert!(moq_publish_media_flush(9999, 0) < 0); - assert!(moq_publish_media_flush(media, u64::MAX) < 0); assert!(moq_publish_media_discontinuity(9999) < 0); assert!(moq_publish_media_discontinuity(0) < 0); assert!(moq_publish_media_cut(9999) < 0); diff --git a/rs/moq-ffi/MIGRATE.md b/rs/moq-ffi/MIGRATE.md index d31ec4e3443..fe153f3a69b 100644 --- a/rs/moq-ffi/MIGRATE.md +++ b/rs/moq-ffi/MIGRATE.md @@ -5,6 +5,11 @@ These are the UniFFI names; each language wrapper has its own `MIGRATE.md` in it ## Unreleased +### `MoqMediaTrackProducer::flush` is gone + +The catalog no longer measures jitter from when a frame was written, so there is nothing to report after writing one. +Delete the call and regenerate bindings; `delay` is measured from the order of the timestamps you write. + ### Video rendition preference `MoqVideo` gains `preference: i32`. Set it to `0` in record literals to keep the default ranking; higher values are preferred and compatibility transcodes use `-1`. Regenerate bindings and rebuild them with the matching native library. diff --git a/rs/moq-ffi/src/media.rs b/rs/moq-ffi/src/media.rs index d13171c216b..7da4049c1bb 100644 --- a/rs/moq-ffi/src/media.rs +++ b/rs/moq-ffi/src/media.rs @@ -985,20 +985,7 @@ impl MoqMediaTrackProducer { Ok(()) } - /// Record a locally encoded frame's handoff for catalog jitter measurement. - /// - /// `timestamp_us` is on the broadcast media clock. Call this after `write_frame` only for - /// encoder output; imported files, pipes, and network media stay clock-free. - pub fn flush(&self, timestamp_us: u64) -> Result<(), MoqError> { - let _guard = crate::ffi::enter(); - let timestamp = moq_net::Timestamp::from_micros(timestamp_us)?; - let mut guard = self.inner.lock().unwrap(); - let media = guard.as_mut().ok_or(MoqError::Closed)?; - media.import.flush(timestamp, moq_clock::Instant::now())?; - Ok(()) - } - - /// Mark a timeline break and restart handoff measurement without lowering advertised jitter. + /// Mark a timeline break without lowering the advertised jitter or delay. /// /// Publishes a discontinuity marker; resumed frames must continue the broadcast media clock, /// and video must resume on a keyframe. diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index c22756bb216..2dc1f80c51f 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -2472,8 +2472,6 @@ async fn dynamic_track_request_can_publish_media() { timestamp_us: Some(20_000), }) .unwrap(); - media.flush(20_000).unwrap(); - assert!(media.flush(u64::MAX).is_err(), "unrepresentable PTS must fail"); let frame = tokio::time::timeout(TIMEOUT, media_consumer.next()) .await diff --git a/rs/moq-gst/MIGRATE.md b/rs/moq-gst/MIGRATE.md index 8da134ac585..e2e27ec56cd 100644 --- a/rs/moq-gst/MIGRATE.md +++ b/rs/moq-gst/MIGRATE.md @@ -4,6 +4,11 @@ How to update your pipelines for each breaking change, newest first. ## Unreleased +### The `encoder` pad property is gone + +`moqsink` no longer measures when buffers arrive, so a pad needs no hint that a local encoder feeds it. +Drop `sink_%u::encoder=true` from pipeline descriptions; every pad now advertises `delay` from the order of its buffer timestamps against the other pads. + ### `moqsrc` audio caps are the standard ones `moqsrc` audio pads now carry `rate` and `channels` as `int`, and Opus pads the mapping fields (`channel-mapping-family`, `stream-count`, `coupled-count`, plus `channel-mapping` beyond stereo) with the OpusHead and an empty OpusTags as `streamheader`, so `decodebin` and `opusdec` accept them and `opusdec` applies the output gain. diff --git a/rs/moq-gst/src/sink/imp.rs b/rs/moq-gst/src/sink/imp.rs index 47fc0e9a306..f878aecd9fc 100644 --- a/rs/moq-gst/src/sink/imp.rs +++ b/rs/moq-gst/src/sink/imp.rs @@ -758,7 +758,7 @@ impl MoqSink { let outcome = lifecycle.media.place_wall(&buffer, sample).and_then(|()| { lifecycle .media - .push_buffer(data, pts, duration, current_running_time, moq_net::time::now()) + .push_buffer(data, pts, duration, current_running_time) .map_err(anyhow::Error::msg) }); let changes = match &outcome { @@ -1236,9 +1236,7 @@ fn observe_caps( ) -> (Option, bool) { lifecycle.media.reserve(catalog, state.wall.clone()); let requested = lifecycle.requested().map(str::to_owned); - let mut options = ProducerOptions::new(caps) - .with_container(lifecycle.container().into()) - .with_encoder(lifecycle.encoder()); + let mut options = ProducerOptions::new(caps).with_container(lifecycle.container().into()); if let Some(track) = requested.as_deref() { options = options.with_track(track); } @@ -1774,65 +1772,6 @@ mod tests { assert!(next.audio.renditions.contains_key("audio")); } - #[test] - fn a_playing_pause_cycle_does_not_raise_encoder_jitter() { - gst::init().unwrap(); - let sink = sink(); - sink.set_property("url", "https://127.0.0.1:1"); - sink.set_property("broadcast", "pause-test"); - let pad = sink - .request_pad_simple("sink_0") - .unwrap() - .downcast::() - .unwrap(); - sink.set_state(gst::State::Paused).unwrap(); - let (broadcast, catalog) = { - let control = sink.imp().control.lock().unwrap(); - let state = control.live.as_ref().unwrap(); - (state.broadcast.clone(), state.catalog.as_ref().unwrap().clone()) - }; - let caps = gst::Caps::builder("audio/x-opus") - .field("rate", 48_000i32) - .field("channels", 2i32) - .build(); - assert!(matches!( - pad.lifecycle().media.observe_caps( - &broadcast, - &catalog, - ProducerOptions::new(&caps).with_track("audio").with_encoder(true) - ), - CapsOutcome::Active(_) - )); - let segment = gst::FormattedSegment::::new(); - pad.lifecycle().media.observe_segment(segment.upcast()); - sink.set_state(gst::State::Playing).unwrap(); - let anchor = moq_net::time::now(); - for (pts, arrival) in [(0, 0), (20, 120), (40, 6_000), (60, 6_020)] { - if pts == 40 { - sink.set_state(gst::State::Paused).unwrap(); - sink.set_state(gst::State::Playing).unwrap(); - } - assert_eq!( - pad.lifecycle() - .media - .push_buffer( - Bytes::from_static(b"opus frame"), - Some(gst::ClockTime::from_mseconds(pts)), - None, - None, - anchor + Duration::from_millis(arrival) - ) - .unwrap(), - PushOutcome::Published - ); - } - assert_eq!( - catalog.snapshot().audio.renditions["audio"].jitter, - Some(Duration::from_millis(100)) - ); - sink.set_state(gst::State::Null).unwrap(); - } - fn spec(element: &super::super::MoqSink, name: &str) -> glib::ParamSpec { element.find_property(name).unwrap() } diff --git a/rs/moq-gst/src/sink/pad.rs b/rs/moq-gst/src/sink/pad.rs index 29267acec99..68108c31856 100644 --- a/rs/moq-gst/src/sink/pad.rs +++ b/rs/moq-gst/src/sink/pad.rs @@ -4,7 +4,6 @@ //! that pad's own streaming thread, so this type is touched from one thread and needs no generation //! tagging or cross-thread failure map. -use moq_net::time::Instant; use std::time::Duration; use anyhow::{Context, Result, ensure}; @@ -51,10 +50,6 @@ struct Media { audio: bool, /// The media time of the first packet in the current audio group. group_start: Option, - /// Records each frame's handoff against the wall clock, raising the catalog jitter by how - /// irregularly a local encoder delivers. Imports leave it off: their arrival times describe the - /// file or network, not the encoder. - encoder: bool, /// Apply a timeline break before the next valid frame, using the normal write error path. discontinuity: bool, /// Video can start or resume mid-GOP, so deltas drop until a keyframe opens a group. Stays set: @@ -64,9 +59,9 @@ struct Media { } impl Media { - /// Publish one frame at `micros` on the media clock, handed over at `now`. Returns false when a - /// frame is dropped while waiting for a keyframe at or beyond the live edge. - fn write(&mut self, data: &Bytes, micros: u64, now: Instant) -> Result { + /// Publish one frame at `micros` on the media clock. Returns false when a frame is dropped while + /// waiting for a keyframe at or beyond the live edge. + fn write(&mut self, data: &Bytes, micros: u64) -> Result { if std::mem::take(&mut self.discontinuity) { self.track.discontinuity()?; self.keyframe = !self.audio; @@ -96,11 +91,6 @@ impl Media { if self.audio { self.group_start.get_or_insert(timestamp); } - if self.encoder { - // Skipping the observation would publish a frame the jitter never saw. - let ts = ts.context("encoder frame timestamp out of range")?; - self.track.flush(ts, now)?; - } Ok(true) } } @@ -110,7 +100,6 @@ pub(super) struct ProducerOptions<'a> { container: hang::catalog::Container, caps: &'a gst::Caps, requested: Option<&'a str>, - encoder: bool, } impl<'a> ProducerOptions<'a> { @@ -119,7 +108,6 @@ impl<'a> ProducerOptions<'a> { container: hang::catalog::Container::default(), caps, requested: None, - encoder: false, } } @@ -132,11 +120,6 @@ impl<'a> ProducerOptions<'a> { self.requested = Some(track); self } - - pub(super) fn with_encoder(mut self, encoder: bool) -> Self { - self.encoder = encoder; - self - } } /// A subtitle pad. GStreamer hands us one decoded cue per buffer (`text/x-raw`, UTF-8) with the @@ -361,15 +344,8 @@ impl Pad { container, caps, requested, - encoder, } = options; let structure = caps.structure(0).context("empty caps")?; - // Only a codec rendition carries the jitter a local encoder's clock would raise. - ensure!( - !encoder || !matches!(structure.name().as_str(), "application/octet-stream" | "text/x-raw"), - "encoder is only supported on audio and video pads, not {}", - structure.name() - ); // Renegotiation: finalize the previous producer before replacing it (closed once, not abandoned). self.finalize()?; // Opaque data has no codec importer and no catalog entry, so it never reaches the codec match. @@ -531,7 +507,6 @@ impl Pad { track, audio, group_start: None, - encoder, discontinuity: false, keyframe: !audio, }))); @@ -746,9 +721,8 @@ impl Pad { /// Import one buffer into the producer. A failed or producer-less pad drops the buffer; a timeline /// drop is logged. Unstamped opaque data on an active timeline uses the element's current running - /// time. An encoder pad records `now` as the frame's handoff once it is published. A bad bitstream - /// (or an oversized frame, rejected by moq-net) invalidates only this pad and says so in the - /// returned outcome. Returns an error when an unstamped opaque buffer has no current + /// time. A bad bitstream (or an oversized frame, rejected by moq-net) invalidates only this pad + /// and says so in the returned outcome. Returns an error when an unstamped opaque buffer has no current /// running time, so the caller fails the flow instead of silently dropping data. pub fn push_buffer( &mut self, @@ -756,7 +730,6 @@ impl Pad { pts: Option, duration: Option, current_running_time: Option, - now: Instant, ) -> std::result::Result { if self.failed { return Ok(PushOutcome::Dropped); @@ -776,7 +749,7 @@ impl Pad { match timestamp { Ok(micros) => { let result: Result<()> = match self.track.as_mut().expect("track present") { - Sink::Media(media) => match media.write(&data, micros, now) { + Sink::Media(media) => match media.write(&data, micros) { Ok(false) => { gst::debug!( CAT, @@ -1187,14 +1160,8 @@ mod tests { "the reserved name is the requested one" ); pad.observe_segment(time_segment()); - pad.push_buffer( - h264_keyframe_au(), - Some(gst::ClockTime::ZERO), - None, - None, - moq_net::time::now(), - ) - .unwrap(); + pad.push_buffer(h264_keyframe_au(), Some(gst::ClockTime::ZERO), None, None) + .unwrap(); let snapshot = catalog.snapshot(); let renditions: Vec = snapshot.video.renditions.keys().map(|name| name.to_string()).collect(); @@ -1330,15 +1297,13 @@ mod tests { .subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(1))) .await .unwrap(); - let anchor = moq_net::time::now(); for index in 0..17 { assert_eq!( pad.push_buffer( Bytes::from_static(&[0x80, 0xff, 0xfe]), Some(gst::ClockTime::from_useconds(100_000 + index * 2_500)), None, - None, - anchor + None ) .unwrap(), PushOutcome::Published @@ -1548,7 +1513,6 @@ mod tests { Some(gst::ClockTime::from_mseconds(20 * index as u64)), None, None, - moq_net::time::now(), ) .unwrap(), PushOutcome::Published @@ -1576,26 +1540,14 @@ mod tests { pad.observe_caps(&broadcast, &catalog, producer_options(&h264_caps(), None)); // No observe_segment: the pad stays in NoSegment. assert_eq!( - pad.push_buffer( - h264_keyframe_au(), - Some(gst::ClockTime::ZERO), - None, - None, - moq_net::time::now() - ) - .unwrap(), + pad.push_buffer(h264_keyframe_au(), Some(gst::ClockTime::ZERO), None, None) + .unwrap(), PushOutcome::NoSegment, "first no-segment buffer is reported" ); assert_eq!( - pad.push_buffer( - h264_keyframe_au(), - Some(gst::ClockTime::ZERO), - None, - None, - moq_net::time::now() - ) - .unwrap(), + pad.push_buffer(h264_keyframe_au(), Some(gst::ClockTime::ZERO), None, None) + .unwrap(), PushOutcome::Dropped, "subsequent no-segment buffers are not re-reported" ); @@ -1636,13 +1588,7 @@ mod tests { assert!(!data.is_failed()); video.observe_segment(time_segment()); video - .push_buffer( - h264_keyframe_au(), - Some(gst::ClockTime::ZERO), - None, - None, - moq_net::time::now(), - ) + .push_buffer(h264_keyframe_au(), Some(gst::ClockTime::ZERO), None, None) .unwrap(); let snapshot = catalog.snapshot(); @@ -1675,7 +1621,6 @@ mod tests { Some(gst::ClockTime::from_mseconds(40)), None, None, - moq_net::time::now(), ) .unwrap(); pad.push_buffer( @@ -1683,7 +1628,6 @@ mod tests { Some(gst::ClockTime::from_mseconds(80)), None, None, - moq_net::time::now(), ) .unwrap(); @@ -1738,14 +1682,8 @@ mod tests { CapsOutcome::Active("camera".to_string()) ); pad.observe_segment(time_segment()); - pad.push_buffer( - h264_keyframe_au(), - Some(gst::ClockTime::ZERO), - None, - None, - moq_net::time::now(), - ) - .unwrap(); + pad.push_buffer(h264_keyframe_au(), Some(gst::ClockTime::ZERO), None, None) + .unwrap(); let config = catalog.snapshot().video.renditions.get("camera").cloned().unwrap(); assert_eq!(config.container, hang::catalog::Container::Loc); @@ -1813,7 +1751,6 @@ mod tests { None, None, Some(gst::ClockTime::from_mseconds(25)), - moq_net::time::now(), ) .unwrap(); assert!(!pad.is_failed(), "a missing PTS uses the supplied running time"); @@ -1848,14 +1785,8 @@ mod tests { pad.observe_segment(time_segment()); assert!( - pad.push_buffer( - Bytes::from_static(b"no timestamp"), - None, - None, - None, - moq_net::time::now() - ) - .is_err(), + pad.push_buffer(Bytes::from_static(b"no timestamp"), None, None, None) + .is_err(), "the caller gets a hard error instead of a silent drop" ); } @@ -1874,14 +1805,8 @@ mod tests { ); assert!(pad.is_failed()); pad.observe_segment(time_segment()); - pad.push_buffer( - Bytes::from_static(b"x"), - Some(gst::ClockTime::ZERO), - None, - None, - moq_net::time::now(), - ) - .unwrap(); + pad.push_buffer(Bytes::from_static(b"x"), Some(gst::ClockTime::ZERO), None, None) + .unwrap(); } // An H.265 camera whose SPS VUI zeroes the colour fields on 4:2:0 (`matrix_coeffs` 0, which @@ -1915,13 +1840,7 @@ mod tests { au.extend_from_slice(nal); } let outcome = pad - .push_buffer( - Bytes::from(au), - Some(gst::ClockTime::ZERO), - None, - None, - moq_net::time::now(), - ) + .push_buffer(Bytes::from(au), Some(gst::ClockTime::ZERO), None, None) .unwrap(); assert_eq!(outcome, PushOutcome::Published); assert!(!pad.is_failed()); @@ -1937,14 +1856,8 @@ mod tests { let mut pad = Pad::new(); pad.observe_caps(&broadcast, &catalog, producer_options(&h264_caps(), None)); pad.observe_segment(time_segment()); - pad.push_buffer( - h264_keyframe_au(), - Some(gst::ClockTime::ZERO), - None, - None, - moq_net::time::now(), - ) - .unwrap(); + pad.push_buffer(h264_keyframe_au(), Some(gst::ClockTime::ZERO), None, None) + .unwrap(); let snapshot = catalog.snapshot(); let track = snapshot.video.renditions.keys().next().expect("a video rendition"); @@ -2131,7 +2044,6 @@ mod tests { Some(gst::ClockTime::ZERO), Some(gst::ClockTime::from_seconds(1)), None, - moq_net::time::now(), ) .unwrap(); assert!(pad.is_failed()); @@ -2158,7 +2070,6 @@ mod tests { Some(gst::ClockTime::from_mseconds(start_ms)), Some(gst::ClockTime::from_mseconds(dur_ms)), None, - moq_net::time::now(), ) .unwrap(); } @@ -2177,14 +2088,8 @@ mod tests { let mut pad = Pad::new(); pad.observe_caps(&broadcast, &catalog, producer_options(&text_caps(), None)); pad.observe_segment(time_segment()); - pad.push_buffer( - Bytes::from_static(b"hello"), - Some(gst::ClockTime::ZERO), - None, - None, - moq_net::time::now(), - ) - .unwrap(); + pad.push_buffer(Bytes::from_static(b"hello"), Some(gst::ClockTime::ZERO), None, None) + .unwrap(); assert!(!pad.is_failed(), "a durationless cue drops the buffer, not the pad"); } @@ -2224,107 +2129,6 @@ mod tests { assert_eq!(emitted, 5, "all five decode-order frames must emit (got {emitted})"); } - // `multifilesrc ! parsebin` and a live encoder deliver identical segments and PTS, so the opt-in is - // the only thing that tells them apart. The same late frame raises only the encoder pad's jitter. - #[test] - fn only_an_encoder_pad_measures_its_handoff() { - gst::init().unwrap(); - let (broadcast, catalog) = producers(); - let mut encoder = Pad::new(); - let mut import = Pad::new(); - encoder.observe_caps( - &broadcast, - &catalog, - producer_options(&h264_caps(), Some("encoder")).with_encoder(true), - ); - import.observe_caps(&broadcast, &catalog, producer_options(&h264_caps(), Some("import"))); - - // The third frame reaches the sink 100ms later than its running time says it should. - let anchor = moq_net::time::now(); - for (pts, arrival) in [(0, 0), (33, 33), (66, 166)] { - for pad in [&mut encoder, &mut import] { - pad.observe_segment(time_segment()); - let outcome = pad - .push_buffer( - h264_keyframe_au(), - Some(gst::ClockTime::from_mseconds(pts)), - None, - None, - anchor + std::time::Duration::from_millis(arrival), - ) - .unwrap(); - assert_eq!(outcome, PushOutcome::Published); - } - } - - let snapshot = catalog.snapshot(); - let jitter = |name: &str| snapshot.video.renditions[name].jitter; - assert_eq!(jitter("encoder"), Some(std::time::Duration::from_millis(100))); - assert_eq!(jitter("import"), None, "an import's arrival never reaches the catalog"); - } - - #[test] - fn encoder_seek_does_not_measure_the_pause_as_jitter() { - gst::init().unwrap(); - for boundary in ["segment", "flush", "pause"] { - let (broadcast, catalog) = producers(); - let mut pad = Pad::new(); - pad.observe_caps( - &broadcast, - &catalog, - producer_options(&h264_caps(), Some("encoder")).with_encoder(true), - ); - let anchor = moq_net::time::now(); - for (pts, arrival) in [(0, 0), (33, 33), (66, 166)] { - pad.observe_segment(time_segment()); - assert_eq!( - pad.push_buffer( - h264_keyframe_au(), - Some(gst::ClockTime::from_mseconds(pts)), - None, - None, - anchor + std::time::Duration::from_millis(arrival) - ) - .unwrap(), - PushOutcome::Published - ); - } - let before = catalog.snapshot().video.renditions["encoder"].jitter; - assert_eq!(before, Some(std::time::Duration::from_millis(100))); - match boundary { - "flush" => pad.flush(), - "pause" => pad.discontinuity(), - _ => {} - } - // Seek in the source while keeping the broadcast media clock moving forward. - for (pts, arrival) in [(30_000, 6_000), (30_033, 6_033)] { - let pts = if boundary == "pause" { - // Pausing preserves the segment while the running clock stops. - pts - 29_000 - } else { - pad.observe_segment(time_segment_at(30_000, 1_000)); - pts - }; - assert_eq!( - pad.push_buffer( - h264_keyframe_au(), - Some(gst::ClockTime::from_mseconds(pts)), - None, - None, - anchor + std::time::Duration::from_millis(arrival) - ) - .unwrap(), - PushOutcome::Published - ); - } - assert_eq!( - catalog.snapshot().video.renditions["encoder"].jitter, - before, - "boundary={boundary}" - ); - } - } - #[test] fn video_start_drops_deltas_until_the_first_keyframe() { gst::init().unwrap(); @@ -2333,9 +2137,8 @@ mod tests { pad.observe_caps(&broadcast, &catalog, producer_options(&h264_caps(), Some("video"))); pad.observe_segment(time_segment()); let delta = Bytes::from_static(&[0, 0, 0, 1, 0x61, 0xe0, 0x12, 0x34]); - let now = moq_net::time::now(); let push = |pad: &mut Pad, data: Bytes, pts: u64| { - pad.push_buffer(data, Some(gst::ClockTime::from_mseconds(pts)), None, None, now) + pad.push_buffer(data, Some(gst::ClockTime::from_mseconds(pts)), None, None) .unwrap() }; assert_eq!(push(&mut pad, delta.clone(), 0), PushOutcome::Dropped); @@ -2354,9 +2157,8 @@ mod tests { let mut pad = Pad::new(); pad.observe_caps(&broadcast, &catalog, producer_options(&h264_caps(), Some("video"))); pad.observe_segment(time_segment()); - let now = moq_net::time::now(); let push = |pad: &mut Pad, data: Bytes, pts: u64| { - pad.push_buffer(data, Some(gst::ClockTime::from_mseconds(pts)), None, None, now) + pad.push_buffer(data, Some(gst::ClockTime::from_mseconds(pts)), None, None) .unwrap() }; assert_eq!(push(&mut pad, h264_keyframe_au(), 1000), PushOutcome::Published); @@ -2383,9 +2185,8 @@ mod tests { pad.observe_caps(&broadcast, &catalog, producer_options(&h264_caps(), Some("video"))); pad.observe_segment(time_segment()); let delta = Bytes::from_static(&[0, 0, 0, 1, 0x61, 0xe0, 0x12, 0x34]); - let now = moq_net::time::now(); let push = |pad: &mut Pad, data: Bytes, pts: u64| { - pad.push_buffer(data, Some(gst::ClockTime::from_mseconds(pts)), None, None, now) + pad.push_buffer(data, Some(gst::ClockTime::from_mseconds(pts)), None, None) .unwrap() }; assert_eq!(push(&mut pad, h264_keyframe_au(), 0), PushOutcome::Published); @@ -2410,9 +2211,8 @@ mod tests { let keyframe = h264_keyframe_au(); let headers = keyframe.slice(..keyframe.len() - 9); let delta = Bytes::from_static(&[0, 0, 0, 1, 0x61, 0xe0, 0x12, 0x34]); - let now = moq_net::time::now(); let push = |pad: &mut Pad, data: Bytes, pts: u64| { - pad.push_buffer(data, Some(gst::ClockTime::from_mseconds(pts)), None, None, now) + pad.push_buffer(data, Some(gst::ClockTime::from_mseconds(pts)), None, None) .unwrap() }; assert_eq!(push(&mut pad, keyframe.clone(), 0), PushOutcome::Published); @@ -2422,25 +2222,4 @@ mod tests { assert_eq!(push(&mut pad, keyframe, 100), PushOutcome::Published); assert_eq!(push(&mut pad, delta, 133), PushOutcome::Published); } - - // Text and opaque tracks carry no codec jitter, so asking them to measure one is a mistake to report - // rather than a setting to ignore. - #[test] - fn an_encoder_pad_must_carry_audio_or_video() { - gst::init().unwrap(); - let (broadcast, catalog) = producers(); - for caps in [text_caps(), opaque_caps()] { - let mut pad = Pad::new(); - let outcome = pad.observe_caps( - &broadcast, - &catalog, - producer_options(&caps, Some("data")).with_encoder(true), - ); - assert!( - matches!(outcome, CapsOutcome::Failed(ref reason) if reason.starts_with("encoder is only supported")), - "{outcome:?}" - ); - assert!(pad.is_failed()); - } - } } diff --git a/rs/moq-gst/src/sink/request_pad.rs b/rs/moq-gst/src/sink/request_pad.rs index 9bc31f19ab2..d8f8f540c48 100644 --- a/rs/moq-gst/src/sink/request_pad.rs +++ b/rs/moq-gst/src/sink/request_pad.rs @@ -41,8 +41,6 @@ struct Settings { effective: Option, /// The wire container selected for this pad's media producer. container: MediaContainer, - /// Whether a local encoder feeds this pad, so its handoff clock may raise the catalog jitter. - encoder: bool, /// What the track is doing, read back through `status`. status: Status, /// The reason the pad was invalidated, read back through `track-error`. @@ -106,11 +104,6 @@ impl PadLifecycle { self.settings.container } - /// Whether a local encoder feeds this pad. - pub(super) fn encoder(&self) -> bool { - self.settings.encoder - } - /// Record the name reserved by a successful CAPS event. pub(super) fn commit(&mut self, track: String) -> Notifications { let before = self @@ -193,19 +186,6 @@ impl ObjectImpl for MoqSinkPadImp { .default_value(MediaContainer::Legacy) .mutable_playing() .build(), - // Provenance is not in the caps: `multifilesrc ! parsebin` hands over the same TIME - // segment and PTS a live encoder does, so only the application can say which it is. - glib::ParamSpecBoolean::builder("encoder") - .nick("Local encoder") - .blurb( - "A local encoder feeds this pad, so the catalog jitter includes how late each \ - frame reaches the sink behind its running time. Leave it off for file, demuxed, \ - and network media, whose arrival says nothing about the original encoder. Audio \ - and video only. Writable in any state until the CAPS event reserves the track", - ) - .default_value(false) - .mutable_playing() - .build(), glib::ParamSpecEnum::builder::("track-status") .nick("Status") .blurb( @@ -239,7 +219,7 @@ impl ObjectImpl for MoqSinkPadImp { ); return; } - // A producer keeps its reserved name, wire container, and provenance for its whole life, so a later write + // A producer keeps its reserved name and wire container for its whole life, so a later write // would read back without ever reaching the broadcast or the catalog. if lifecycle.settings.effective.is_some() { gst::warning!( @@ -256,7 +236,6 @@ impl ObjectImpl for MoqSinkPadImp { lifecycle.settings.requested = value.get::>().unwrap().filter(|name| !name.is_empty()) } "container" => lifecycle.settings.container = value.get().unwrap(), - "encoder" => lifecycle.settings.encoder = value.get().unwrap(), _ => unreachable!(), } } @@ -271,7 +250,6 @@ impl ObjectImpl for MoqSinkPadImp { .or_else(|| lifecycle.settings.requested.clone()) .to_value(), "container" => lifecycle.settings.container.to_value(), - "encoder" => lifecycle.settings.encoder.to_value(), "track-status" => lifecycle.settings.status.to_value(), "track-error" => lifecycle.settings.error.clone().to_value(), _ => unreachable!(), diff --git a/rs/moq-gst/tests/element.rs b/rs/moq-gst/tests/element.rs index 6e884dbd528..9b3151fb4de 100644 --- a/rs/moq-gst/tests/element.rs +++ b/rs/moq-gst/tests/element.rs @@ -622,20 +622,6 @@ fn a_pipeline_description_selects_loc() { ); } -#[test] -fn a_pipeline_description_marks_an_encoder_pad() { - init(); - let sink = gst::parse::launch("moqsink name=publisher url=https://127.0.0.1:1 broadcast=test sink_0::encoder=true") - .expect("parse the description"); - let _pad = sink.request_pad_simple("sink_0").expect("request sink_0"); - assert!(child_of(&sink, "sink_0").property::("encoder")); - let _other = sink.request_pad_simple("sink_1").expect("request sink_1"); - assert!( - !child_of(&sink, "sink_1").property::("encoder"), - "a pad is an import unless it says otherwise" - ); -} - // The acceptance criterion: once CAPS reserves the track, its name and container are fixed; stopping // the element makes both configurable again. #[test] diff --git a/rs/moq-mux/MIGRATE.md b/rs/moq-mux/MIGRATE.md index b787af9368e..f51bc74fb44 100644 --- a/rs/moq-mux/MIGRATE.md +++ b/rs/moq-mux/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `flush` is gone; `jitter` and `delay` come from timestamps + +`catalog::Estimator::flush`, `container::Producer::flush`, `import::Track::flush`, and every codec importer's `flush` are removed: nothing reads the wall clock to measure jitter any more. +Delete the calls. +`jitter` now covers only reorder delay and batch spans, and `delay` is how far a frame's timestamp trails those another track of the same catalog already wrote, so importers advertise it too. + ### Passthrough fMP4 refuses a rewound group as `TimestampRewind` (#6426) A passthrough `container::fmp4::Import` refuses a group that starts below the previous one with `Error::TimestampRewind` before opening it, as an unwrapped track does, instead of `Error::Moq(moq_net::Error::TimestampMismatch)` from a group it already opened. diff --git a/rs/moq-mux/benches/catalog.rs b/rs/moq-mux/benches/catalog.rs index e457aac2cfc..299a650ac13 100644 --- a/rs/moq-mux/benches/catalog.rs +++ b/rs/moq-mux/benches/catalog.rs @@ -1,4 +1,4 @@ -//! Catalog churn from a rising flush estimate, swept over tracks x subscribers. +//! Catalog churn from a rising estimate, swept over tracks x subscribers. //! //! Every track's jitter rises 1 ms per frame for [`RISES`] frames inside one rate window, then one //! more frame lands after the window. Each catalog a subscriber receives makes a player update its @@ -14,6 +14,9 @@ //! end of malformed input. Each drop releases one reservation, so a per-release scan of the //! remaining tracks would show up as a quadratic slope over the track count. //! +//! The write order case interleaves frames across every track of one catalog, each measuring its +//! `delay` against the others, so a per-frame scan of the tracks would show up as a slope. +//! //! Run with `cargo bench -p moq-mux --bench catalog`. use std::hint::black_box; @@ -41,7 +44,6 @@ struct Fanout { tracks: Vec>, subscribers: Vec, waiter: kio::Waiter, - anchor: moq_clock::Instant, } impl Fanout { @@ -78,19 +80,19 @@ impl Fanout { _catalog: catalog, tracks, waiter: kio::Waiter::noop(), - anchor: moq_clock::now(), }; // The initial track set is configuration, not estimate traffic. fanout.receive(); fanout } - /// Every frame lands `frame` ms late, after an on-time first frame, so jitter rises 1 ms each. - fn flush(&mut self, frame: u64) { - let timestamp = Timestamp::from_millis(frame).unwrap(); - let now = self.anchor + Duration::from_millis(2 * frame); + /// Every frame is reordered by `frame` ms, after an in-order first frame, so jitter rises 1 ms + /// each. The cut publishes it. + fn rise(&mut self, frame: u64) { + let delay = Timestamp::from_millis(frame).unwrap(); for track in &mut self.tracks { - track.flush(timestamp, now).unwrap(); + track.reorder(delay); + track.cut(None).unwrap(); } } @@ -130,11 +132,11 @@ impl Fanout { updates += u; }; for frame in 0..=RISES { - self.flush(frame); + self.rise(frame); tally(self.receive()); } tokio::time::advance(Duration::from_secs(1)).await; - self.flush(RISES); + self.rise(RISES); tally(self.receive()); let shape = self.tracks.len() * self.subscribers.len(); @@ -220,5 +222,59 @@ fn bench_unresolved_teardown(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench_estimate_fanout, bench_unresolved_teardown); +/// Tracks interleaving frames on one catalog. +const ORDERED: [usize; 3] = [1, 16, 256]; + +/// Frames each track writes per iteration. +const FRAMES: u64 = 100; + +fn bench_write_order(c: &mut Criterion) { + let mut group = c.benchmark_group("catalog_write_order"); + for tracks in ORDERED { + group.throughput(Throughput::Elements(FRAMES * tracks as u64)); + group.bench_with_input(BenchmarkId::from_parameter(tracks), &tracks, |b, &tracks| { + b.iter_batched( + || { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let producers: Vec<_> = (0..tracks) + .map(|index| { + let info = catalog.track_info(hang::catalog::PRIORITY.video); + let track = broadcast.create_track(format!("v{index}"), info).unwrap(); + let config = hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::VP8); + catalog.video(track, Container::Legacy(Kind::Video), config).unwrap() + }) + .collect(); + (broadcast, catalog, producers) + }, + |(_broadcast, _catalog, mut producers)| { + for frame in 0..FRAMES { + // Each track trails the one before it by a millisecond, so every write measures. + for (index, producer) in producers.iter_mut().enumerate() { + let timestamp = Duration::from_millis(frame * 40 + 1_000) + .saturating_sub(Duration::from_millis(index as u64)); + producer + .write(moq_mux::container::Frame { + timestamp: Timestamp::from_micros(timestamp.as_micros() as u64).unwrap(), + payload: bytes::Bytes::from_static(&[0; 64]), + keyframe: frame == 0, + duration: None, + }) + .unwrap(); + } + } + }, + BatchSize::SmallInput, + ); + }); + } + group.finish(); +} + +criterion_group!( + benches, + bench_estimate_fanout, + bench_unresolved_teardown, + bench_write_order +); criterion_main!(benches); diff --git a/rs/moq-mux/src/binary.rs b/rs/moq-mux/src/binary.rs index 62be7161fa9..95a4402f528 100644 --- a/rs/moq-mux/src/binary.rs +++ b/rs/moq-mux/src/binary.rs @@ -470,7 +470,7 @@ mod test { out } - /// A track whose payloads reach the transport later than another's captures advertises the gap + /// A track whose captures are written after later captures on another track advertises the gap /// as `delay`, while one written without capture times advertises neither `delay` nor `jitter`. #[test] fn a_late_capture_is_delay() { @@ -485,14 +485,15 @@ mod test { .binary_stream(timed_track(&mut broadcast, "bare"), Config::default()) .unwrap(); - let anchor = moq_clock::Instant::now(); - for i in 0..20u64 { + for i in 2..20u64 { let now = moq_net::Timestamp::from_micros(i * 100_000).unwrap(); - let at = |late: u64| anchor + std::time::Duration::from_millis(i * 100 + late); - let fast = fast.listing.as_mut().unwrap(); - fast.record_at(now, 100, Some((now, at(0)))).unwrap(); - let slow = slow.listing.as_mut().unwrap(); - slow.record_at(now, 100, Some((now, at(200)))).unwrap(); + let captured = moq_net::Timestamp::from_micros(i * 100_000 - 200_000).unwrap(); + fast.listing.as_mut().unwrap().record_at(now, 100, Some(now)).unwrap(); + slow.listing + .as_mut() + .unwrap() + .record_at(now, 100, Some(captured)) + .unwrap(); bare.listing.as_mut().unwrap().record_at(now, 100, None).unwrap(); } @@ -510,29 +511,6 @@ mod test { assert_eq!((bare.delay, bare.jitter), (None, None)); } - /// A captured payload measures through the real clock: written right after capture, it - /// advertises no delay beyond the scheduling noise of the test itself. - #[moq_clock::test] - async fn a_capture_measures_through_the_clock() { - moq_clock::sleep(std::time::Duration::from_secs(1)).await; - let (mut broadcast, catalog) = catalog(); - let mut telemetry = catalog - .binary_stream(timed_track(&mut broadcast, "telemetry"), Config::default()) - .unwrap(); - - let captured = moq_clock::Instant::now(); - let at = catalog.clock().capture(captured).unwrap(); - telemetry.append(Timed::from(&b"now"[..]).at(at)).unwrap(); - let late = catalog - .clock() - .capture(captured - std::time::Duration::from_secs(1)) - .unwrap(); - telemetry.append(Timed::from(&b"late"[..]).at(late)).unwrap(); - - let jitter = entry(&catalog, "telemetry").jitter.expect("a late capture is jitter"); - assert!(jitter >= std::time::Duration::from_secs(1), "{jitter:?}"); - } - /// A source's own timestamp, on a catalog clock anchored to that source, is the frame's /// timestamp as given. One ahead of now (a source clock running fast) is published too. #[test] @@ -554,8 +532,8 @@ mod test { } /// A timestamp ahead of now counts as on time: its track advertises no delay, and another track - /// is measured against it as against any on-time track. Measured as given, it would undercut - /// every real flush and inflate the other track's delay by the hour it runs fast. + /// is measured against it as against any on-time track. Ordered as given, it would put the other + /// track behind by the hour it runs fast. #[test] fn an_ahead_timestamp_counts_as_on_time() { let (mut broadcast, catalog) = catalog(); @@ -566,19 +544,16 @@ mod test { .binary_stream(timed_track(&mut broadcast, "ahead"), Config::default()) .unwrap(); - let anchor = moq_clock::Instant::now(); - for i in 0..20u64 { + for i in 2..20u64 { let now = moq_net::Timestamp::from_micros(i * 100_000).unwrap(); - let at = |late: u64| anchor + std::time::Duration::from_millis(i * 100 + late); - let slow = slow.listing.as_mut().unwrap(); - slow.record_at(now, 100, Some((now, at(200)))).unwrap(); - // An hour fast, flushed the moment it was written. + // An hour fast. let fast = moq_net::Timestamp::from_micros(i * 100_000 + 3_600_000_000).unwrap(); - ahead - .listing + ahead.listing.as_mut().unwrap().record_at(now, 100, Some(fast)).unwrap(); + let captured = moq_net::Timestamp::from_micros(i * 100_000 - 200_000).unwrap(); + slow.listing .as_mut() .unwrap() - .record_at(now, 100, Some((fast, at(0)))) + .record_at(now, 100, Some(captured)) .unwrap(); } @@ -959,10 +934,10 @@ mod test { let entry = &catalog.snapshot().ext.mavlink["telemetry"]; assert_eq!(entry.config.bitrate, Some(1_000_000)); - assert_eq!(entry.config.jitter, None, "write spacing is not a flush delay"); + assert_eq!(entry.config.jitter, None, "write spacing is not jitter"); } - /// A supplied bitrate is authoritative, while a capture time still measures jitter and delay. + /// A supplied bitrate is authoritative, however much is written. #[test] fn a_supplied_bitrate_is_kept() { let (mut broadcast, catalog) = catalog(); @@ -973,17 +948,11 @@ mod test { .unwrap(); let listing = telemetry.listing.as_mut().unwrap(); - let anchor = moq_clock::Instant::now(); for i in 0..60u64 { let now = moq_net::Timestamp::from_micros(i * 40_000).unwrap(); - // Every other payload reaches the transport 10ms later than the rest. - let written = anchor + std::time::Duration::from_micros(i * 40_000 + (i % 2) * 10_000); - listing.record_at(now, 5_000, Some((now, written))).unwrap(); + listing.record_at(now, 5_000, Some(now)).unwrap(); } - let entry = &catalog.snapshot().ext.mavlink["telemetry"]; - assert_eq!(entry.config.bitrate, Some(64_000)); - assert_eq!(entry.config.jitter, Some(std::time::Duration::from_millis(10))); assert_eq!(catalog.snapshot().ext.mavlink["telemetry"].config.bitrate, Some(64_000)); } } diff --git a/rs/moq-mux/src/catalog/data.rs b/rs/moq-mux/src/catalog/data.rs index 5d7cf6cc9e9..564caff3b1f 100644 --- a/rs/moq-mux/src/catalog/data.rs +++ b/rs/moq-mux/src/catalog/data.rs @@ -30,7 +30,7 @@ impl + AsMut> IntoRendition for } /// A data track's catalog entry, owned for the life of its producer and kept current with the -/// bitrate, jitter, and delay its writes measure. +/// bitrate and delay its writes measure. /// /// Erases the entry's type, so a data producer's own type doesn't depend on which section lists it. pub(crate) struct Listing { @@ -90,31 +90,27 @@ impl Listing { self.estimator.discontinuity(); } self.clock = clock; - let now = clock.now(); - let flush = captured.map(|captured| (captured, moq_clock::Instant::now())); - self.record_at(now, bytes, flush) + self.record_at(clock.now(), bytes, captured) } /// [`record`](Self::record) at a chosen time, which a test needs since the broadcast clock only - /// moves in real time. `flush` is the capture time and the instant the frame was written. + /// moves in real time. pub(crate) fn record_at( &mut self, now: moq_net::Timestamp, bytes: usize, - flush: Option<(moq_net::Timestamp, moq_clock::Instant)>, + captured: Option, ) -> crate::Result<()> { // Each write is its own span, closed by the next one. self.estimator.cut(Some(now)); - self.estimator.write(now, bytes); + self.estimator.rate(now, bytes); - // The spacing between writes is the application's cadence, not a flush delay, so only a - // capture time measures jitter and delay. Without one neither is advertised. - if let Some((captured, written)) = flush { - // A source clock running slightly fast stamps ahead of now. Measured as given, its - // lateness would undercut every real flush and drag the broadcast's minimum down, - // inflating the delay of every other track. It was flushed no later than now, so it - // counts as zero delay. - self.estimator.flush(captured.min(now), written); + // The write time is the application's cadence, not media time, so only a capture time takes + // a place in the broadcast's write order. Without one no delay is advertised. + if let Some(captured) = captured { + // A source clock running fast stamps ahead of now. Ordered as given, it would put every + // media track behind it by however fast it runs. It was written no later than now. + self.estimator.order(captured.min(now)); } self.rendition.estimate(self.estimator.estimate()) } diff --git a/rs/moq-mux/src/catalog/estimate.rs b/rs/moq-mux/src/catalog/estimate.rs index 56e231a3555..31d5ab74775 100644 --- a/rs/moq-mux/src/catalog/estimate.rs +++ b/rs/moq-mux/src/catalog/estimate.rs @@ -1,5 +1,3 @@ -use moq_clock::Instant; -use std::collections::VecDeque; use std::sync::{Arc, Mutex, PoisonError}; use std::time::Duration; @@ -28,7 +26,6 @@ const FRAMERATES: [f64; 11] = [ /// How far a measurement may sit from the rate it snaps to: half the 0.1% gap between an NTSC /// rate and its integer neighbour, so no measurement is ambiguous between them. const FRAMERATE_TOLERANCE: f64 = 0.0005; -const JITTER_WINDOW: Duration = Duration::from_secs(10); /// The catalog fields an [`Estimator`] can measure from the frames fed to it. /// @@ -38,11 +35,11 @@ const JITTER_WINDOW: Duration = Duration::from_secs(10); #[derive(Clone, Default, Debug, PartialEq)] #[non_exhaustive] pub struct Estimate { - /// The maximum delay between a frame being ready and the publisher flushing it. + /// The largest batch span or reorder delay written. pub jitter: Option, /// The maximum bitrate in bits per second. pub bitrate: Option, - /// The most this track's minimum flush lateness has trailed the broadcast's earliest track. + /// The most a frame's timestamp has trailed one already written by another track. pub delay: Option, /// The maximum frame rate in frames per second, when it is a standard one. pub framerate: Option, @@ -61,7 +58,7 @@ impl Estimate { self } - /// Set the delay behind the broadcast's earliest track (or clear it with `None`). + /// Set the delay behind the other tracks (or clear it with `None`). pub fn with_delay(mut self, delay: impl Into>) -> Self { self.delay = delay.into(); self @@ -74,11 +71,11 @@ impl Estimate { } } -/// Measures the catalog jitter, bitrate, and framerate of one track from the frames written to it. +/// Measures the catalog jitter, bitrate, delay, and framerate of one track from the frames written +/// to it, without reading a clock. /// /// A [`container::Producer`](crate::container::Producer) created through the catalog owns one, -/// feeds it as you write, and publishes the result automatically. Local encoders also call -/// `flush` for each frame they hand to the transport: +/// feeds it as you write, and publishes the result automatically: /// /// ```no_run /// # fn example( @@ -110,18 +107,17 @@ impl Estimate { /// let estimate = estimator.estimate(); /// ``` /// +/// A standalone estimator has no other tracks to order against, so it never measures a delay. +/// /// Move-only (not `Clone`): it owns the running measurement for exactly one track. #[derive(Default)] pub struct Estimator { jitter: Jitter, bitrate: Bitrate, framerate: Framerate, - /// This track's recent minimum flush lateness. - baseline: Window, - /// The recent minimum across every track sharing it. A standalone estimator's is its own, so - /// it never measures a delay. - broadcast: Baseline, - /// The largest amount [`baseline`](Self::baseline) has trailed [`broadcast`](Self::broadcast). + /// This track's place in the write order of every track sharing it. + seat: Seat, + /// The most a written timestamp has trailed the highest one another track had written. delay: Duration, } @@ -131,10 +127,10 @@ impl Estimator { Self::default() } - /// Measure `delay` against the tracks sharing `broadcast`. - pub(crate) fn with_broadcast(broadcast: Baseline) -> Self { + /// Measure `delay` against the other tracks sharing `order`. + pub(crate) fn with_order(order: Order) -> Self { Self { - broadcast, + seat: order.seat(), ..Self::default() } } @@ -142,20 +138,24 @@ impl Estimator { /// Observe a frame of `bytes` encoded bytes at presentation time `timestamp`, as written by /// [`container::Producer::write`](crate::container::Producer::write). pub fn write(&mut self, timestamp: Timestamp, bytes: usize) { + self.rate(timestamp, bytes); + self.order(timestamp); + } + + /// Count a write toward the bitrate and framerate only. + pub(crate) fn rate(&mut self, timestamp: Timestamp, bytes: usize) { let timestamp = nanos(timestamp); self.bitrate.write(timestamp, bytes); self.framerate.write(timestamp); } - /// Measure when an encoder handed a frame to the transport. Only locally encoded frames should - /// call this; imports keep clock-free batch and reorder estimates. Jitter is the spread above - /// this track's own recent minimum lateness, so a constant encoder delay is not jitter. Delay - /// is how far that minimum trails the earliest track on the same catalog. - pub fn flush(&mut self, timestamp: Timestamp, now: Instant) { - let (lateness, earliest) = self.broadcast.observe(timestamp, now); - let minimum = self.baseline.observe(now, lateness); - self.jitter.max = self.jitter.max.max(nanos_between(minimum, lateness)); - self.delay = self.delay.max(nanos_between(earliest, minimum)); + /// Place a frame stamped `timestamp` in the write order. Its delay is how far it trails the + /// highest timestamp another track has already written: a frame the transport carries after + /// later media from another track has to be waited for by a consumer playing both. + pub(crate) fn order(&mut self, timestamp: Timestamp) { + if let Some(behind) = self.seat.write(nanos(timestamp)) { + self.delay = self.delay.max(behind); + } } /// Close the current span at `end`, as [`container::Producer::cut`](crate::container::Producer::cut) @@ -169,16 +169,13 @@ impl Estimator { self.framerate.cut(); } - /// Discard the open bitrate span and the flush baselines, so nothing is measured across a break - /// in the timeline. See [`container::Producer::discontinuity`](crate::container::Producer::discontinuity). - /// - /// The broadcast baseline is cleared too: a pause usually stops every track, and the others' - /// pre-pause minimum would otherwise read as a delay until it left the window. + /// Discard the open bitrate span and this track's place in the write order, so nothing is + /// measured across a break in the timeline. See + /// [`container::Producer::discontinuity`](crate::container::Producer::discontinuity). pub fn discontinuity(&mut self) { self.bitrate.discontinuity(); self.framerate.discontinuity(); - self.baseline = Window::default(); - self.broadcast.discontinuity(); + self.seat.discontinuity(); } /// Observe a frame's reorder delay (`PTS - DTS`), which raises the jitter to the decode buffer a @@ -375,7 +372,7 @@ fn bits_per_second(bytes: u64, duration: Duration) -> u64 { bits_per_second.min(u64::MAX as u128) as u64 } -/// The largest measured flush delay, batch span, or reorder delay. Once advertised it never falls. +/// The largest batch span or reorder delay. Once advertised it never falls. #[derive(Default)] struct Jitter { max: Duration, @@ -387,72 +384,144 @@ impl Jitter { } } -/// The flush clock shared by every track on one catalog: a common epoch, so lateness compares -/// across tracks, and the minimum lateness any of them flushed with over the recent window. +/// The highest timestamp each track on one catalog has written, which every frame's `delay` is +/// measured against. /// -/// An `Instant` has no public mapping to the broadcast's media epoch. The first observation -/// chooses a local origin; its unknown offset cancels when subtracting one minimum from another. +/// A frame is compared with the other tracks only: its own track's earlier frames say nothing +/// about the interleave, and a B-frame trailing its own reference picture is already reorder +/// jitter. #[derive(Clone, Default)] -pub(crate) struct Baseline(Arc>); +pub(crate) struct Order(Arc>); + +impl Order { + /// Take a place in the order for a new track. + fn seat(&self) -> Seat { + let index = self.0.lock().unwrap_or_else(PoisonError::into_inner).join(); + Seat { + order: self.clone(), + index, + } + } +} +/// Every seated track's highest timestamp, plus the two largest, so a write is constant time +/// however many tracks share the catalog. #[derive(Default)] -struct Shared { - epoch: Option, - window: Window, +struct Highs { + /// Indexed by seat; `None` is a free seat, `Some(None)` a track that has written nothing since + /// it joined or broke its timeline. + seats: Vec>>, + /// The largest high and its seat. + first: Option<(usize, u128)>, + /// The largest high on any other seat. + second: Option<(usize, u128)>, } -impl Baseline { - /// Return the frame's lateness on the shared epoch and the broadcast's minimum including it. - fn observe(&self, timestamp: Timestamp, now: Instant) -> (i128, i128) { - let mut shared = self.0.lock().unwrap_or_else(PoisonError::into_inner); - let epoch = *shared.epoch.get_or_insert(now); - let elapsed = match now.checked_duration_since(epoch) { - Some(duration) => duration.as_nanos() as i128, - None => -(epoch.duration_since(now).as_nanos() as i128), +impl Highs { + fn join(&mut self) -> usize { + if let Some(index) = self.seats.iter().position(Option::is_none) { + self.seats[index] = Some(None); + return index; + } + self.seats.push(Some(None)); + self.seats.len() - 1 + } + + /// Raise `index`'s high to `timestamp` and return how far `timestamp` trails the others' highest. + fn write(&mut self, index: usize, timestamp: u128) -> Option { + let others = match self.first { + Some((first, _)) if first == index => self.second, + first => first, }; - let lateness = elapsed - timestamp.as_nanos() as i128; - (lateness, shared.window.observe(now, lateness)) + + let high = self.seats[index] + .as_mut() + .expect("a seated track") + .get_or_insert(timestamp); + *high = (*high).max(timestamp); + let high = *high; + + // Highs only rise, so the top two stay ordered by updating them in place. + match (self.first, self.second) { + (Some((first, _)), _) if first == index => self.first = Some((index, high)), + (Some((_, top)), _) if high > top => { + self.second = self.first; + self.first = Some((index, high)); + } + (None, _) => self.first = Some((index, high)), + (_, Some((second, low))) if second == index || high > low => self.second = Some((index, high)), + (_, None) => self.second = Some((index, high)), + _ => {} + } + + let (_, others) = others?; + elapsed(timestamp, others) } - fn discontinuity(&self) { - self.0.lock().unwrap_or_else(PoisonError::into_inner).window = Window::default(); + /// Forget `index`'s high, freeing its seat when `leave` is set. + fn clear(&mut self, index: usize, leave: bool) { + self.seats[index] = if leave { None } else { Some(None) }; + if [self.first, self.second] + .iter() + .flatten() + .any(|(seat, _)| *seat == index) + { + self.rank(); + } + } + + /// Recompute the top two from every seat. + fn rank(&mut self) { + self.first = None; + self.second = None; + for (index, seat) in self.seats.iter().enumerate() { + let Some(high) = seat.flatten() else { continue }; + match self.first { + Some((_, top)) if high <= top => { + if self.second.is_none_or(|(_, low)| high > low) { + self.second = Some((index, high)); + } + } + first => { + self.second = first; + self.first = Some((index, high)); + } + } + } } } -/// The minimum lateness over the last [`JITTER_WINDOW`], so a media clock drifting against the -/// wall clock does not ratchet forever. A monotonic deque makes insertion and expiry amortized -/// O(1). -#[derive(Default)] -struct Window { - samples: VecDeque, +/// One track's place in an [`Order`], released when the track's estimator drops. +struct Seat { + order: Order, + index: usize, } -struct Sample { - at: Instant, - lateness: i128, +impl Default for Seat { + fn default() -> Self { + Order::default().seat() + } } -impl Window { - /// Insert a lateness observed at `now` and return the window's minimum. - fn observe(&mut self, now: Instant, lateness: i128) -> i128 { - while self - .samples - .front() - .is_some_and(|sample| now.saturating_duration_since(sample.at) > JITTER_WINDOW) - { - self.samples.pop_front(); - } - while self.samples.back().is_some_and(|sample| sample.lateness >= lateness) { - self.samples.pop_back(); - } - self.samples.push_back(Sample { at: now, lateness }); - self.samples.front().expect("the current sample was inserted").lateness +impl Seat { + fn write(&self, timestamp: u128) -> Option { + self.lock().write(self.index, timestamp) + } + + /// A break in the timeline may resume anywhere, so the old high must not hold others back. + fn discontinuity(&self) { + self.lock().clear(self.index, false); + } + + fn lock(&self) -> std::sync::MutexGuard<'_, Highs> { + self.order.0.lock().unwrap_or_else(PoisonError::into_inner) } } -/// `to - from` as a duration, clamped at zero. -fn nanos_between(from: i128, to: i128) -> Duration { - Duration::from_nanos(u64::try_from(to - from).unwrap_or(if to > from { u64::MAX } else { 0 })) +impl Drop for Seat { + fn drop(&mut self) { + self.lock().clear(self.index, true); + } } #[cfg(test)] @@ -474,130 +543,122 @@ mod tests { assert_eq!(estimator.estimate().jitter, Some(Duration::from_millis(80))); } - #[test] - fn batch_flush_at_its_end_reports_its_media_span() { - let mut estimator = Estimator::new(); - let anchor = moq_clock::now(); - estimator.flush(micros(0), anchor); - estimator.flush(micros(0), anchor + Duration::from_millis(120)); - estimator.flush(micros(40_000), anchor + Duration::from_millis(120)); - assert_eq!(estimator.estimate().jitter, Some(Duration::from_millis(120))); + /// Estimators sharing one order, as the tracks of one catalog do. + fn tracks() -> [Estimator; N] { + let order = Order::default(); + std::array::from_fn(|_| Estimator::with_order(order.clone())) } - #[test] - fn constant_encoder_delay_is_not_jitter() { - let mut estimator = Estimator::new(); - let anchor = moq_clock::now(); - estimator.flush(micros(0), anchor + Duration::from_millis(200)); - estimator.flush(micros(240_000), anchor + Duration::from_millis(440)); - assert_eq!(estimator.estimate().jitter, None); + fn ms(value: u64) -> Duration { + Duration::from_millis(value) } + /// Video at 20 ms and audio at 15 ms, written `v0 v20 a0 a15 v40 a30 v60 a45 a60`: audio's + /// first frame lands after video has already written 20 ms ahead of it. #[test] - fn discontinuity_resets_the_flush_baseline() { - let mut estimator = Estimator::new(); - let anchor = moq_clock::now(); - estimator.flush(micros(0), anchor); - estimator.flush(micros(40_000), anchor + Duration::from_millis(50)); - assert_eq!(estimator.estimate().jitter, Some(Duration::from_millis(10))); - - // Resumed at the previous live edge after a 2 s pause: the pause is not jitter. - estimator.discontinuity(); - estimator.flush(micros(40_000), anchor + Duration::from_millis(2_050)); - estimator.flush(micros(80_000), anchor + Duration::from_millis(2_090)); - assert_eq!(estimator.estimate().jitter, Some(Duration::from_millis(10))); + fn interleave_measures_the_worst_trail() { + let [mut video, mut audio] = tracks(); + for (video_frame, pts) in [ + (true, 0), + (true, 20), + (false, 0), + (false, 15), + (true, 40), + (false, 30), + (true, 60), + (false, 45), + (false, 60), + ] { + let track = if video_frame { &mut video } else { &mut audio }; + track.write(micros(pts * 1_000), 1); + } + assert_eq!(audio.estimate().delay, Some(ms(20))); + assert_eq!(video.estimate().delay, None); } #[test] - fn baseline_expires_drift() { - let mut estimator = Estimator::new(); - let anchor = moq_clock::now(); - for second in 0..100u64 { - estimator.flush( - micros(second * 1_000_000), - anchor + Duration::from_millis(second * 1_001), - ); + fn a_lone_track_never_measures_a_delay() { + let [mut video] = tracks(); + // Decode order with B-frames: each trails a reference picture written before it. + for pts in [0, 120_000, 40_000, 80_000] { + video.write(micros(pts), 1); } - let jitter = estimator.estimate().jitter.unwrap(); - assert!(jitter <= Duration::from_millis(10), "{jitter:?}"); + assert_eq!(video.estimate(), Estimate::default()); } + /// A B-frame trailing its own track is reorder jitter, not a delay behind the other tracks. #[test] - fn early_flush_keeps_lowering_the_baseline() { - let mut estimator = Estimator::new(); - let anchor = moq_clock::now(); - for second in 0..100u64 { - estimator.flush(micros(second * 2_000_000), anchor + Duration::from_secs(second)); - } - assert_eq!(estimator.estimate(), Estimate::default()); - } - - /// Two tracks on one broadcast baseline, flushed every 100 ms for `seconds` with the lateness - /// each closure returns (in ms) at a given frame's media time. - fn pair(seconds: u64, fast: impl Fn(u64) -> u64, slow: impl Fn(u64) -> u64) -> (Estimate, Estimate) { - let broadcast = Baseline::default(); - let mut estimators = [ - Estimator::with_broadcast(broadcast.clone()), - Estimator::with_broadcast(broadcast), - ]; - let anchor = moq_clock::now(); - for frame in 0..seconds * 10 { - let pts = frame * 100; - for (estimator, lateness) in estimators.iter_mut().zip([fast(pts), slow(pts)]) { - estimator.flush(micros(pts * 1_000), anchor + Duration::from_millis(pts + lateness)); - } + fn reorder_within_a_track_is_not_delay() { + let [mut video, mut audio] = tracks(); + audio.write(micros(0), 1); + for pts in [0, 120_000, 40_000, 80_000] { + video.write(micros(pts), 1); } - let [fast, slow] = estimators.map(|estimator| estimator.estimate()); - (fast, slow) + assert_eq!(video.estimate().delay, None); } + /// A track that breaks its timeline may resume behind where it was, so its old high no longer + /// holds the others back. #[test] - fn a_constant_offset_is_delay_on_the_slower_track() { - let (fast, slow) = pair(5, |_| 0, |_| 200); - assert_eq!(fast, Estimate::default()); - assert_eq!(slow, Estimate::default().with_delay(Duration::from_millis(200))); + fn a_discontinuity_drops_the_tracks_high() { + let [mut ahead, mut steady] = tracks(); + ahead.write(micros(10_000_000), 1); + ahead.discontinuity(); + ahead.write(micros(1_000_000), 1); + steady.write(micros(1_000_000), 1); + assert_eq!(steady.estimate().delay, None); } #[test] - fn a_common_drift_stays_bounded() { - // Both media clocks run 0.1% slow for 100 s, so each lateness climbs 100 ms together. - let (fast, slow) = pair(100, |pts| pts / 1_000, |pts| 200 + pts / 1_000); - assert_eq!(fast.delay, None); - assert_eq!(slow.delay, Some(Duration::from_millis(200))); - for jitter in [fast.jitter, slow.jitter] { - assert!(jitter.unwrap() <= Duration::from_millis(10), "{jitter:?}"); - } - } + fn a_dropped_track_stops_holding_others_back() { + let order = Order::default(); + let mut ahead = Estimator::with_order(order.clone()); + ahead.write(micros(10_000_000), 1); + drop(ahead); - /// The earliest track starts at 0 and the other at 200 ms, then the first drifts to 500 ms. The - /// delays no longer share an origin, but the drifting track's own `delay` covers the new offset, - /// so the largest `delay + jitter` never falls below the real spread. - #[test] - fn the_earliest_track_changing_does_not_under_buffer() { - let drift = |pts: u64| pts.saturating_sub(10_000).min(50_000) / 100; - let (drifted, steady) = pair(80, drift, |_| 200); - assert_eq!(steady.delay, Some(Duration::from_millis(200))); - assert_eq!(drifted.delay, Some(Duration::from_millis(300))); + let mut steady = Estimator::with_order(order.clone()); + let mut other = Estimator::with_order(order); + other.write(micros(1_000_000), 1); + steady.write(micros(1_000_000), 1); + assert_eq!(steady.estimate().delay, None); } + /// The top two kept incrementally agree with a scan over every seat, through joins, leaves, + /// and breaks. #[test] - fn a_discontinuity_clears_the_broadcast_baseline() { - let broadcast = Baseline::default(); - let mut audio = Estimator::with_broadcast(broadcast.clone()); - let mut video = Estimator::with_broadcast(broadcast); - let anchor = moq_clock::now(); - audio.flush(micros(0), anchor); - video.flush(micros(0), anchor + Duration::from_millis(50)); - assert_eq!(video.estimate().delay, Some(Duration::from_millis(50))); - - // Both resume at the previous live edge after a 2 s pause: the pause is not a delay. - audio.discontinuity(); - video.discontinuity(); - video.flush(micros(40_000), anchor + Duration::from_millis(2_090)); - audio.flush(micros(40_000), anchor + Duration::from_millis(2_040)); - video.flush(micros(80_000), anchor + Duration::from_millis(2_130)); - assert_eq!(video.estimate().delay, Some(Duration::from_millis(50))); - assert_eq!(audio.estimate().delay, None); + fn highs_match_a_full_scan() { + let mut highs = Highs::default(); + let mut live: Vec = Vec::new(); + let mut state = 0x9e37_79b9_7f4a_7c15u64; + let mut next = |bound: u64| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state % bound + }; + + for step in 0..10_000u64 { + match next(10) { + 0 if live.len() < 6 => live.push(highs.join()), + 1 if !live.is_empty() => { + let index = live.swap_remove(next(live.len() as u64) as usize); + highs.clear(index, true); + } + 2 if !live.is_empty() => highs.clear(live[next(live.len() as u64) as usize], false), + _ if !live.is_empty() => { + let index = live[next(live.len() as u64) as usize]; + let timestamp = (step + next(500)) as u128; + let others = live + .iter() + .filter(|seat| **seat != index) + .filter_map(|seat| highs.seats[*seat].flatten()) + .max(); + let expected = others.and_then(|others| elapsed(timestamp, others)); + assert_eq!(highs.write(index, timestamp), expected, "step {step}"); + } + _ => live.push(highs.join()), + } + } } #[test] diff --git a/rs/moq-mux/src/catalog/producer.rs b/rs/moq-mux/src/catalog/producer.rs index 827454590d1..68762bcccd4 100644 --- a/rs/moq-mux/src/catalog/producer.rs +++ b/rs/moq-mux/src/catalog/producer.rs @@ -287,9 +287,9 @@ pub struct Producer { /// Connection allocator passthrough tracks claim their peak-hold bitrate on. /// See [`Config::with_bandwidth`]. bandwidth: moq_net::bandwidth::Allocator, - /// The minimum flush lateness across this catalog's renditions, which each rendition's - /// advertised `delay` is measured against. - baseline: super::estimate::Baseline, + /// The write order across this catalog's renditions, which each rendition's advertised + /// `delay` is measured against. + order: super::estimate::Order, /// What each media track minted under this catalog has written. See [`stats`](Self::stats). sent: Arc>, } @@ -304,7 +304,7 @@ impl Clone for Producer { cutting: self.cutting.clone(), max_age: self.max_age, bandwidth: self.bandwidth.clone(), - baseline: self.baseline.clone(), + order: self.order.clone(), sent: self.sent.clone(), } } @@ -467,7 +467,7 @@ impl Producer { cutting: config.timeline, max_age: config.max_age, bandwidth: config.bandwidth, - baseline: Default::default(), + order: Default::default(), sent: Default::default(), }) } @@ -864,7 +864,7 @@ impl Producer { /// A fresh estimator whose `delay` is measured against this catalog's other renditions. pub(crate) fn estimator(&self) -> super::Estimator { - super::Estimator::with_broadcast(self.baseline.clone()) + super::Estimator::with_order(self.order.clone()) } /// The allocator passthrough tracks claim on. fMP4 writes groups by hand, so it reads this itself. diff --git a/rs/moq-mux/src/catalog/tracks.rs b/rs/moq-mux/src/catalog/tracks.rs index 4c1cb8fd793..eab366a5381 100644 --- a/rs/moq-mux/src/catalog/tracks.rs +++ b/rs/moq-mux/src/catalog/tracks.rs @@ -940,7 +940,7 @@ mod tests { } /// Feed ~40ms 100 kB frames over more than the bitrate window, with a measured - /// 40ms container batch span. PTS spacing alone says nothing about flush delay. + /// 40ms container batch span. PTS spacing alone says nothing about jitter. fn feed>(rendition: &mut Rendition) { let mut estimator = super::super::Estimator::new(); estimator.burst(Duration::from_millis(40)); diff --git a/rs/moq-mux/src/codec/aac/import.rs b/rs/moq-mux/src/codec/aac/import.rs index 78cfd5cffb6..975eb97e808 100644 --- a/rs/moq-mux/src/codec/aac/import.rs +++ b/rs/moq-mux/src/codec/aac/import.rs @@ -49,11 +49,6 @@ impl Import { self.track.track().demand() } - /// Record a locally encoded frame's transport handoff for catalog jitter measurement. - pub fn flush(&mut self, timestamp: moq_net::Timestamp, now: moq_clock::Instant) -> crate::Result<()> { - self.track.flush(timestamp, now) - } - /// Finish the track, flushing the current group. pub fn finish(&mut self) -> crate::Result<()> { self.track.finish()?; diff --git a/rs/moq-mux/src/codec/av1/import.rs b/rs/moq-mux/src/codec/av1/import.rs index 324e33e9a32..78526fe4ec4 100644 --- a/rs/moq-mux/src/codec/av1/import.rs +++ b/rs/moq-mux/src/codec/av1/import.rs @@ -209,11 +209,6 @@ impl Import { self.track.track().demand() } - /// Record a locally encoded frame's transport handoff for catalog jitter measurement. - pub fn flush(&mut self, timestamp: moq_net::Timestamp, now: moq_clock::Instant) -> crate::Result<()> { - self.track.flush(timestamp, now) - } - /// Finish the track, flushing the current group. pub fn finish(&mut self) -> Result<()> { self.track.finish()?; diff --git a/rs/moq-mux/src/codec/flac/import.rs b/rs/moq-mux/src/codec/flac/import.rs index 19fa646ebfb..780c7f740b6 100644 --- a/rs/moq-mux/src/codec/flac/import.rs +++ b/rs/moq-mux/src/codec/flac/import.rs @@ -45,11 +45,6 @@ impl Import { self.track.track().demand() } - /// Record a locally encoded frame's transport handoff for catalog jitter measurement. - pub fn flush(&mut self, timestamp: moq_net::Timestamp, now: moq_clock::Instant) -> crate::Result<()> { - self.track.flush(timestamp, now) - } - /// Finish the track, flushing the current group. pub fn finish(&mut self) -> crate::Result<()> { self.track.finish()?; diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index 1d0a70db968..c54075b810a 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -164,11 +164,6 @@ impl Import { Ok(()) } - /// Record a locally encoded frame's transport handoff for catalog jitter measurement. - pub fn flush(&mut self, timestamp: moq_net::Timestamp, now: moq_clock::Instant) -> crate::Result<()> { - self.track.flush(timestamp, now) - } - /// Record a frame's reorder delay (`PTS - DTS`) so the catalog `jitter` reflects the /// B-frame reorder depth (the decode buffer a transmuxer/player must hold). The /// container supplies this since the elementary stream alone carries no decode time. diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index 60f5256b18c..838cab31ca5 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -169,11 +169,6 @@ impl Import { Ok(()) } - /// Record a locally encoded frame's transport handoff for catalog jitter measurement. - pub fn flush(&mut self, timestamp: moq_net::Timestamp, now: moq_clock::Instant) -> crate::Result<()> { - self.track.flush(timestamp, now) - } - /// Record a frame's reorder delay (`PTS - DTS`) so the catalog `jitter` reflects the /// B-frame reorder depth (the decode buffer a transmuxer/player must hold). The /// container supplies this since the elementary stream alone carries no decode time. diff --git a/rs/moq-mux/src/codec/mp3.rs b/rs/moq-mux/src/codec/mp3.rs index b5d186932e7..17304f4a065 100644 --- a/rs/moq-mux/src/codec/mp3.rs +++ b/rs/moq-mux/src/codec/mp3.rs @@ -134,11 +134,6 @@ impl Import { self.track.track().demand() } - /// Record a locally encoded frame's transport handoff for catalog jitter measurement. - pub fn flush(&mut self, timestamp: moq_net::Timestamp, now: moq_clock::Instant) -> crate::Result<()> { - self.track.flush(timestamp, now) - } - /// Finish the track, flushing the current group. pub fn finish(&mut self) -> crate::Result<()> { self.track.finish()?; diff --git a/rs/moq-mux/src/codec/opus/import.rs b/rs/moq-mux/src/codec/opus/import.rs index b00c4749a57..5d146c5b9dc 100644 --- a/rs/moq-mux/src/codec/opus/import.rs +++ b/rs/moq-mux/src/codec/opus/import.rs @@ -47,11 +47,6 @@ impl Import { self.track.track().demand() } - /// Record a locally encoded frame's transport handoff for catalog jitter measurement. - pub fn flush(&mut self, timestamp: moq_net::Timestamp, now: moq_clock::Instant) -> crate::Result<()> { - self.track.flush(timestamp, now) - } - /// Finish the track, flushing the current group. pub fn finish(&mut self) -> crate::Result<()> { self.track.finish()?; diff --git a/rs/moq-mux/src/codec/vp8/import.rs b/rs/moq-mux/src/codec/vp8/import.rs index c3ff8fbc911..3761f7fadb8 100644 --- a/rs/moq-mux/src/codec/vp8/import.rs +++ b/rs/moq-mux/src/codec/vp8/import.rs @@ -97,11 +97,6 @@ impl Import { self.track.track().demand() } - /// Record a locally encoded frame's transport handoff for catalog jitter measurement. - pub fn flush(&mut self, timestamp: moq_net::Timestamp, now: moq_clock::Instant) -> crate::Result<()> { - self.track.flush(timestamp, now) - } - /// Finish the track, flushing the current group. pub fn finish(&mut self) -> crate::Result<()> { self.track.finish()?; diff --git a/rs/moq-mux/src/codec/vp9/import.rs b/rs/moq-mux/src/codec/vp9/import.rs index 2b78901e75e..fc1f590bd01 100644 --- a/rs/moq-mux/src/codec/vp9/import.rs +++ b/rs/moq-mux/src/codec/vp9/import.rs @@ -98,11 +98,6 @@ impl Import { self.track.track().demand() } - /// Record a locally encoded frame's transport handoff for catalog jitter measurement. - pub fn flush(&mut self, timestamp: moq_net::Timestamp, now: moq_clock::Instant) -> crate::Result<()> { - self.track.flush(timestamp, now) - } - /// Finish the track, flushing the current group. pub fn finish(&mut self) -> crate::Result<()> { self.track.finish()?; diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index af5787ba58c..3624066149f 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -474,7 +474,7 @@ impl Import { start: None, recorder: Some(recorder), pending_sequence: None, - estimator: Estimator::new(), + estimator: self.catalog.estimator(), claim: crate::catalog::Claim::new(self.catalog.bandwidth()), }) } diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index c5e9b11da8e..7b38b682792 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -242,13 +242,6 @@ where Ok(()) } - /// Record when a locally encoded frame reached the transport. Imported media must leave - /// this clock observation out and rely on its container batch and reorder measurements. - pub fn flush(&mut self, timestamp: moq_net::Timestamp, now: moq_clock::Instant) -> crate::Result<()> { - self.estimator.flush(timestamp, now); - self.publish_estimate() - } - /// The catalog key owned by this producer. pub fn name(&self) -> &str { self.rendition @@ -310,7 +303,7 @@ where self.bandwidth.as_ref().and_then(|claim| claim.ceiling()) } - /// The jitter and bitrate measured from the frames written so far. + /// The jitter, bitrate, delay, and framerate measured from the frames written so far. /// /// A catalog-owned producer publishes this automatically. See /// [`Estimator`](crate::catalog::Estimator). @@ -863,12 +856,12 @@ mod tests { producer.finish().unwrap(); let estimate = producer.estimate(); - assert_eq!(estimate.jitter, None, "PTS spacing alone is not flush delay"); + assert_eq!(estimate.jitter, None, "PTS spacing alone is not jitter"); assert_eq!(estimate.bitrate, Some(1_600_000)); } #[test] - fn catalog_flush_measures_delay_across_renditions_and_jitter_within_each() { + fn catalog_write_order_measures_delay_across_renditions() { let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); let mut tracks = Vec::new(); @@ -884,20 +877,20 @@ mod tests { tracks.push(track); } - let anchor = moq_clock::Instant::now(); + // The slow rendition writes each frame after the fast one has written 200ms past it. + for i in 1..10u64 { + tracks[0].write(frame(i * 40_000 + 200_000, true)).unwrap(); + tracks[1].write(frame(i * 40_000, true)).unwrap(); + } + for track in &mut tracks { + track.cut(None).unwrap(); + } + let ms = std::time::Duration::from_millis; - let pts = |millis: u64| Timestamp::from_micros(millis * 1_000).unwrap(); - tracks[0].flush(pts(0), anchor).unwrap(); - // A constant 200ms offset behind the other rendition is delay, not jitter. - tracks[1].flush(pts(0), anchor + ms(200)).unwrap(); - assert_eq!(catalog.snapshot().video.renditions["slow"].jitter, None); - assert_eq!(catalog.snapshot().video.renditions["slow"].delay, Some(ms(200))); - assert_eq!(catalog.snapshot().video.renditions["fast"].delay, None); - - // A frame flushed 60ms later than the slow rendition's own minimum is. - tracks[1].flush(pts(40), anchor + ms(300)).unwrap(); - assert_eq!(catalog.snapshot().video.renditions["fast"].jitter, None); - assert_eq!(catalog.snapshot().video.renditions["slow"].jitter, Some(ms(60))); + let renditions = catalog.snapshot().video.renditions; + assert_eq!(renditions["slow"].delay, Some(ms(200))); + assert_eq!(renditions["fast"].delay, None); + assert_eq!(renditions["slow"].jitter, None, "a constant trail is not jitter"); } /// A passthrough producer claims nothing until the first window closes, then diff --git a/rs/moq-mux/src/import/track.rs b/rs/moq-mux/src/import/track.rs index 582d4275fa1..88eeba0c9cb 100644 --- a/rs/moq-mux/src/import/track.rs +++ b/rs/moq-mux/src/import/track.rs @@ -403,26 +403,6 @@ impl Track { Ok(()) } - /// Record a locally encoded frame's transport handoff. Call this only for an original - /// encoder; file, pipe, and network imports must leave the clock out of the estimate. - pub fn flush(&mut self, timestamp: moq_net::Timestamp, now: moq_clock::Instant) -> Result<()> { - match self.kind { - TrackKind::Avc3 { ref mut import, .. } | TrackKind::Avc1 { ref mut import, .. } => { - import.flush(timestamp, now) - } - TrackKind::Hev1 { ref mut import, .. } | TrackKind::Hvc1 { ref mut import, .. } => { - import.flush(timestamp, now) - } - TrackKind::Av01 { ref mut import, .. } => import.flush(timestamp, now), - TrackKind::Vp8(ref mut import) => import.flush(timestamp, now), - TrackKind::Vp9(ref mut import) => import.flush(timestamp, now), - TrackKind::Aac(ref mut import) => import.flush(timestamp, now), - TrackKind::Opus(ref mut import) => import.flush(timestamp, now), - TrackKind::Mp3(ref mut import) => import.flush(timestamp, now), - TrackKind::Flac(ref mut import) => import.flush(timestamp, now), - } - } - /// Finish the importer, flushing any buffered data. pub fn finish(&mut self) -> Result<()> { match self.kind { @@ -1112,29 +1092,21 @@ mod tests { } #[tokio::test(start_paused = true)] - async fn discontinuity_resets_audio_measurement_and_separates_groups() { + async fn discontinuity_separates_audio_groups() { let (mut broadcast, catalog) = new_broadcast(); let (import, subscriber) = opus_import(&mut broadcast, &catalog); let mut import: Track = import.into(); - let anchor = moq_clock::Instant::now(); - for (pts, arrival) in [(0, 0), (20, 120)] { - let timestamp = Timestamp::from_micros(pts * 1_000).unwrap(); - import.decode(b"a", Some(timestamp)).unwrap(); + for pts in [0, 20] { import - .flush(timestamp, anchor + Duration::from_millis(arrival)) + .decode(b"a", Some(Timestamp::from_micros(pts * 1_000).unwrap())) .unwrap(); } - let before = catalog.snapshot().audio.renditions["audio"].jitter; - assert_eq!(before, Some(Duration::from_millis(100))); import.discontinuity().unwrap(); - for (pts, arrival) in [(100, 6_000), (120, 6_020)] { - let timestamp = Timestamp::from_micros(pts * 1_000).unwrap(); - import.decode(b"b", Some(timestamp)).unwrap(); + for pts in [100, 120] { import - .flush(timestamp, anchor + Duration::from_millis(arrival)) + .decode(b"b", Some(Timestamp::from_micros(pts * 1_000).unwrap())) .unwrap(); } - assert_eq!(catalog.snapshot().audio.renditions["audio"].jitter, before); import.finish().unwrap(); // The middle group is the established empty-payload discontinuity marker. assert_eq!(collect_groups(subscriber).await, vec![2, 1, 2]); diff --git a/rs/moq-mux/src/json.rs b/rs/moq-mux/src/json.rs index 68c7c654663..9dec2faecc9 100644 --- a/rs/moq-mux/src/json.rs +++ b/rs/moq-mux/src/json.rs @@ -757,35 +757,36 @@ mod test { assert_eq!(entry(&catalog, "gps").bitrate, Some(100_000)); assert_eq!(entry(&catalog, "status").bitrate, Some(4_200)); - assert_eq!( - entry(&catalog, "gps").jitter, - None, - "write spacing is not a flush delay" - ); + assert_eq!(entry(&catalog, "gps").jitter, None, "write spacing is not jitter"); } - /// An unchanged snapshot value writes no frame, so its capture time must not measure a flush + /// An unchanged snapshot value writes no frame, so its capture time must not measure a write /// that never happened. #[test] fn an_unchanged_snapshot_measures_nothing() { let (mut broadcast, catalog) = catalog(); + let mut gps = catalog + .json_stream::(timed_track(&mut broadcast, "gps"), Config::default()) + .unwrap(); let mut status = catalog .json_snapshot::(timed_track(&mut broadcast, "status"), Config::default()) .unwrap(); let now = catalog.clock().now(); let value = serde_json::json!({ "armed": true }); status.update(Timed::from(&value).at(now)).unwrap(); + gps.append(Timed::from(&serde_json::json!({ "lat": 0 })).at(now)) + .unwrap(); let stale = now .checked_sub(moq_net::Timestamp::from_micros(1_000_000).unwrap()) .unwrap(); status.update(Timed::from(&value).at(stale)).unwrap(); - assert_eq!(entry(&catalog, "status").jitter, None); + assert_eq!(entry(&catalog, "status").delay, None); - // The same stale capture on a changed value is measured. + // The same stale capture on a changed value is written behind gps. let changed = serde_json::json!({ "armed": false }); status.update(Timed::from(&changed).at(stale)).unwrap(); - let jitter = entry(&catalog, "status").jitter.expect("a late capture is jitter"); - assert!(jitter >= std::time::Duration::from_secs(1), "{jitter:?}"); + let delay = entry(&catalog, "status").delay.expect("a capture behind gps is delay"); + assert!(delay >= std::time::Duration::from_secs(1), "{delay:?}"); } /// The catalog is the only thing that announces a data track, so walking it is the discovery diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index 81e789887d1..bc9062cc310 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -198,13 +198,11 @@ impl Producer { let mut frames = split.decode(&frame.payload, timestamp)?; frames.extend(split.flush(timestamp)?); import.decode(frames)?; - import.flush(frame.timestamp, moq_net::time::now())?; } Codecs::H265 { split, import } => { let mut frames = split.decode(&frame.payload, timestamp)?; frames.extend(split.flush(timestamp)?); import.decode(frames)?; - import.flush(frame.timestamp, moq_net::time::now())?; } } } diff --git a/swift/MIGRATE.md b/swift/MIGRATE.md index b7a77032ef8..96f43e2d968 100644 --- a/swift/MIGRATE.md +++ b/swift/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `Media.TrackProducer.flush(timestampUs:)` is gone + +The catalog no longer measures jitter from when a frame was written, so there is nothing to report after writing one. +Delete the call; `delay` is measured from the order of the timestamps you write. + ### Video rendition preference `Media.Video` gains `preference: Int32`, defaulting to `0`. Rebuild with the matching MoqFFI package. Higher values are preferred; compatibility transcodes use `-1`. diff --git a/swift/Sources/Moq/Media.swift b/swift/Sources/Moq/Media.swift index a2390e3f693..4ddd39209e0 100644 --- a/swift/Sources/Moq/Media.swift +++ b/swift/Sources/Moq/Media.swift @@ -238,13 +238,7 @@ public enum Media { try ffi.writeFrame(frame: Frame(payload: payload, timestampUs: timestampUs)) } - /// Record a local encoder's frame handoff on the broadcast media clock. - /// Call after `writeFrame` only for local encoder output. - public func flush(timestampUs: UInt64) throws { - try ffi.flush(timestampUs: timestampUs) - } - - /// Mark a timeline break and restart handoff measurement, preserving advertised jitter. + /// Mark a timeline break, preserving the advertised jitter and delay. public func discontinuity() throws { try ffi.discontinuity() } From b8fee30cdd0132fad9dde7fd55eef9e7bac9772c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 16:58:15 -0700 Subject: [PATCH 2/3] quest: list both causes of camera audio lag Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 2 +- quest/m1/audio-capture-clock.md | 30 --------------------------- quest/m1/audio-capture-lag.md | 36 +++++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 31 deletions(-) delete mode 100644 quest/m1/audio-capture-clock.md create mode 100644 quest/m1/audio-capture-lag.md diff --git a/quest/m1/README.md b/quest/m1/README.md index ea188f99f5d..62d56690f17 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -142,7 +142,7 @@ tracks, so per-group stats stop costing cluster announces. - [Interop pre-skip](/quest/m1/interop-preskip.md) - the interop tone check anchors phase to timestamps, so a misapplied pre-skip fails - [moq-net internals](/quest/m1/net-internals.md) - no per-group session clones, no transport type parameter in lite and IETF internals, and the `Local` check measured - [Main is no slower than release](/quest/m1/release-perf/README.md) - every benchmark where main is slower than release is fixed or accepted with a reason, checked by a final paired run before the cut -- [Audio capture follows the shared clock](/quest/m1/audio-capture-clock.md) - a physical microphone's audio timestamps stop drifting behind its camera, so the catalog `delay` holds and viewers stop resubscribing +- [Camera audio stops falling behind video](/quest/m1/audio-capture-lag.md) - a physical camera and microphone publish without audio falling behind (lost samples or encoder backlog, not yet told apart), so the catalog `delay` settles and viewers stop resubscribing - [Unprefixed C++ free functions](/quest/m1/cpp-free-function-prefix.md) - generated free functions drop `moq_`, and `moq.hpp` deletes its five pass-through wrappers before C++ first publishes - [Error cleanups](/quest/m1/error-cleanups.md) - delete the unused `Error::SessionClosed`; raw transport failures already map to `Unroutable` - [API-diff check](/quest/m1/api-diff.md) - a recipe diffs every published package's API against its last release, run on PRs into release and against main before the cut diff --git a/quest/m1/audio-capture-clock.md b/quest/m1/audio-capture-clock.md deleted file mode 100644 index 7f66d1d0437..00000000000 --- a/quest/m1/audio-capture-clock.md +++ /dev/null @@ -1,30 +0,0 @@ -# [S] Audio capture follows the shared clock - -## Goal - -`js/publish` stamps captured audio on the clock video uses, so a physical -microphone's timestamps stay beside its camera's instead of falling behind them. -A `moq.dev/publish` camera and microphone advertise a steady audio catalog -`delay`, and a viewer sends no steady stream of `subscribe update: id=` lines. - -## Plan - -`js/publish/src/audio/capture-worklet.ts` stamps each quantum as -`sampleCount / sampleRate` past a wall-clock anchor taken once at creation. -A capture device that delivers fewer samples per second than its nominal rate -(or a render thread that drops quanta) loses time against the wall clock that -video frames are stamped on. - -On 2026-10-23 a physical camera and microphone published from `moq.dev/publish` -(release) drifted audio about 0.25 s per second behind video, unbounded. -Each rise republished the catalog, and the viewer pushed a new `maxDelay` to -every subscription: 146 video and 176 audio SUBSCRIBE_UPDATEs in 60 seconds. -The catalog `delay` now comes from timestamp order across renditions, so the -drift still reads as a rising audio `delay`; it is real, and only fixing the -clock removes it. - -Reproduce first and confirm the clock is the cause (log the worklet's stamp -minus `now()` per second). Then tie the stamp to the context's own clock, such -as `currentTime` mapped through `getOutputTimestamp()`, or re-anchor on a -measured gap the way the framer already does, without resampling drift into -the audio itself. diff --git a/quest/m1/audio-capture-lag.md b/quest/m1/audio-capture-lag.md new file mode 100644 index 00000000000..b124bda0b5e --- /dev/null +++ b/quest/m1/audio-capture-lag.md @@ -0,0 +1,36 @@ +# [S] Camera audio stops falling behind video + +## Goal + +A physical camera and microphone published from `js/publish` keep audio +timestamps beside video's, with no growing backlog. The audio catalog `delay` +settles, and a viewer sends no steady stream of `subscribe update: id=` lines. + +## Plan + +On 2026-10-23 a physical camera and microphone published from `moq.dev/publish` +(release) fell further behind the wall clock at about 0.25 s per second, +unbounded. Each rise republished the catalog, and the viewer pushed a new +`maxDelay` to every subscription: 146 video and 176 audio SUBSCRIBE_UPDATEs in +60 seconds. A 25% shortfall is far beyond device clock drift (parts per +million), and the measurement could not tell which of two causes it was: + +- **Lost samples.** `js/publish/src/audio/capture-worklet.ts` stamps each quantum + as `sampleCount / sampleRate` past one wall-clock anchor. A quantum with no + input channels returns before counting, and a quantum the render thread drops + is never counted, so the timestamps fall behind real time. +- **Encoder backlog.** The timestamps are right, but the Opus encoder runs slower + than real time, so frames leave later and later and the queue grows without + bound. + +Both read as a rising audio `delay` from timestamp order, but they need +different fixes. First tell them apart on a physical camera: log, once a +second, the worklet's latest stamp minus `now()` and the `AudioEncoder`'s +`encodeQueueSize`. A growing stamp gap is lost samples; a growing queue is the +backlog. + +For lost samples, keep the stamp on real time without resampling the drift into +the audio, such as anchoring on the context's clock (`currentTime` through +`getOutputTimestamp()`) or stamping a measured gap the framer already re-anchors +on. For a backlog, find why the encoder falls behind and shed load before the +queue grows, rather than letting latency climb. From 81410254880308183f9cb5cc282b5ac4e79a979b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 17:17:26 -0700 Subject: [PATCH 3/3] quest: drop audio-capture-lag, fixed on main by removing Opus DTX Measured on a physical camera and microphone. Neither quest hypothesis held: worklet stamps track the wall clock and encodeQueueSize stays 0. Release enables Opus DTX for voice, and Chromium stamps Opus output by counting emitted samples, so each suppressed silent frame pulls later audio 20 ms earlier. main already removed DTX (#4808, #4919); a local main build shows no drift. Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 1 - quest/m1/audio-capture-lag.md | 36 ----------------------------------- 2 files changed, 37 deletions(-) delete mode 100644 quest/m1/audio-capture-lag.md diff --git a/quest/m1/README.md b/quest/m1/README.md index 62d56690f17..e05458da183 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -142,7 +142,6 @@ tracks, so per-group stats stop costing cluster announces. - [Interop pre-skip](/quest/m1/interop-preskip.md) - the interop tone check anchors phase to timestamps, so a misapplied pre-skip fails - [moq-net internals](/quest/m1/net-internals.md) - no per-group session clones, no transport type parameter in lite and IETF internals, and the `Local` check measured - [Main is no slower than release](/quest/m1/release-perf/README.md) - every benchmark where main is slower than release is fixed or accepted with a reason, checked by a final paired run before the cut -- [Camera audio stops falling behind video](/quest/m1/audio-capture-lag.md) - a physical camera and microphone publish without audio falling behind (lost samples or encoder backlog, not yet told apart), so the catalog `delay` settles and viewers stop resubscribing - [Unprefixed C++ free functions](/quest/m1/cpp-free-function-prefix.md) - generated free functions drop `moq_`, and `moq.hpp` deletes its five pass-through wrappers before C++ first publishes - [Error cleanups](/quest/m1/error-cleanups.md) - delete the unused `Error::SessionClosed`; raw transport failures already map to `Unroutable` - [API-diff check](/quest/m1/api-diff.md) - a recipe diffs every published package's API against its last release, run on PRs into release and against main before the cut diff --git a/quest/m1/audio-capture-lag.md b/quest/m1/audio-capture-lag.md deleted file mode 100644 index b124bda0b5e..00000000000 --- a/quest/m1/audio-capture-lag.md +++ /dev/null @@ -1,36 +0,0 @@ -# [S] Camera audio stops falling behind video - -## Goal - -A physical camera and microphone published from `js/publish` keep audio -timestamps beside video's, with no growing backlog. The audio catalog `delay` -settles, and a viewer sends no steady stream of `subscribe update: id=` lines. - -## Plan - -On 2026-10-23 a physical camera and microphone published from `moq.dev/publish` -(release) fell further behind the wall clock at about 0.25 s per second, -unbounded. Each rise republished the catalog, and the viewer pushed a new -`maxDelay` to every subscription: 146 video and 176 audio SUBSCRIBE_UPDATEs in -60 seconds. A 25% shortfall is far beyond device clock drift (parts per -million), and the measurement could not tell which of two causes it was: - -- **Lost samples.** `js/publish/src/audio/capture-worklet.ts` stamps each quantum - as `sampleCount / sampleRate` past one wall-clock anchor. A quantum with no - input channels returns before counting, and a quantum the render thread drops - is never counted, so the timestamps fall behind real time. -- **Encoder backlog.** The timestamps are right, but the Opus encoder runs slower - than real time, so frames leave later and later and the queue grows without - bound. - -Both read as a rising audio `delay` from timestamp order, but they need -different fixes. First tell them apart on a physical camera: log, once a -second, the worklet's latest stamp minus `now()` and the `AudioEncoder`'s -`encodeQueueSize`. A growing stamp gap is lost samples; a growing queue is the -backlog. - -For lost samples, keep the stamp on real time without resampling the drift into -the audio, such as anchoring on the context's clock (`currentTime` through -`getOutputTimestamp()`) or stamping a measured gap the framer already re-anchors -on. For a backlog, find why the encoder falls behind and shed load before the -queue grows, rather than letting latency climb.