From a35d54646eadcbecd55e7d5076c631d1864eab26 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 18:59:24 -0800 Subject: [PATCH 1/5] chore(quest): claim quest/m1/api-audit/join.md Quest-Claim: 9702a386fcac10ca From 63f12fd517180d27847cccda56e31a24333f7531 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 19:10:44 -0800 Subject: [PATCH 2/5] refactor!: encoders take a join Video's keyframeInterval/Gop, audio's groupDuration, and the C hints become one join time in JS, Rust, FFI, and C. A zero join is refused. Co-Authored-By: Claude Opus 5.5 --- bench/browser/page/media.ts | 2 +- dart/moq/lib/video.dart | 6 - dart/moq_ffi/lib/src/moq.dart | 149 ++---------------- demo/web/src/publish.ts | 2 +- doc/bin/hls.md | 2 +- doc/concept/hang.md | 2 +- doc/lib/c/index.md | 4 +- doc/lib/js/publish.md | 12 +- doc/lib/kt/index.md | 2 +- doc/lib/rs/moq-audio.md | 5 +- doc/lib/rs/moq-video.md | 2 +- go/wrapper/video/video.go | 8 +- js/publish/src/audio/capture.test.ts | 2 +- js/publish/src/audio/encoder.test.ts | 55 +++++-- js/publish/src/audio/encoder.ts | 46 +++--- js/publish/src/broadcast.test.ts | 20 +-- js/publish/src/broadcast.ts | 4 +- js/publish/src/preview.ts | 2 +- js/publish/src/rendition-internal.ts | 9 +- js/publish/src/video/capture.test.ts | 4 +- js/publish/src/video/encoder-stall.test.ts | 2 +- js/publish/src/video/encoder.test.ts | 113 +++++++------ js/publish/src/video/encoder.ts | 32 ++-- .../kotlin/dev/moq/video/Video.kt | 3 - py/moq-rs/moq/types.py | 4 - py/moq-rs/moq/video.py | 4 +- quest/m1/api-audit/README.md | 4 - quest/m1/api-audit/join.md | 93 ----------- rs/moq-audio/src/encode/producer.rs | 51 ++++-- rs/moq-boy/src/video.rs | 2 +- rs/moq-c/src/api.rs | 61 ++++--- rs/moq-c/src/test.rs | 52 +++--- rs/moq-c/src/video.rs | 29 ++-- rs/moq-cli/src/play/media.rs | 2 +- rs/moq-ffi/src/test.rs | 32 +++- rs/moq-ffi/src/video.rs | 46 ++---- rs/moq-transcode/src/rung.rs | 6 +- rs/moq-video/DESIGN-native-codecs.md | 4 +- rs/moq-video/README.md | 11 +- rs/moq-video/src/capture/wgc.rs | 2 +- rs/moq-video/src/decode/consumer.rs | 2 +- rs/moq-video/src/encode/backend/mediacodec.rs | 5 +- .../src/encode/backend/mediafoundation.rs | 7 +- rs/moq-video/src/encode/backend/mod.rs | 2 +- rs/moq-video/src/encode/backend/nvenc.rs | 8 +- rs/moq-video/src/encode/backend/openh264.rs | 7 +- rs/moq-video/src/encode/backend/v4l2.rs | 8 +- rs/moq-video/src/encode/backend/vaapi.rs | 4 +- .../src/encode/backend/videotoolbox.rs | 5 +- rs/moq-video/src/encode/encoded.rs | 2 +- rs/moq-video/src/encode/encoder.rs | 136 +++++++--------- rs/moq-video/src/encode/mod.rs | 2 +- rs/moq-video/src/encode/producer.rs | 24 +-- rs/moq-video/src/error.rs | 8 +- rs/moq-video/src/frame/cuda_test.rs | 2 +- swift/Sources/Moq/Video.swift | 4 +- test/interop/clients/js/src/fixture.ts | 2 +- 57 files changed, 457 insertions(+), 662 deletions(-) delete mode 100644 quest/m1/api-audit/join.md diff --git a/bench/browser/page/media.ts b/bench/browser/page/media.ts index 3b884fdfd05..f51ede5502b 100644 --- a/bench/browser/page/media.ts +++ b/bench/browser/page/media.ts @@ -70,7 +70,7 @@ async function publish(config: Media): Promise { codec: config.codec, frameRate: config.fps, maxPixels: config.width * config.height, - keyframeInterval: Milli(config.keyframe_ms), + join: Milli(config.keyframe_ms), maxBitrate: config.bitrate, }, }); diff --git a/dart/moq/lib/video.dart b/dart/moq/lib/video.dart index 1de2a0ac6d6..a1c786f93a1 100644 --- a/dart/moq/lib/video.dart +++ b/dart/moq/lib/video.dart @@ -55,12 +55,6 @@ typedef PixelBuffer = PixelBufferMoqVideoSurface; /// One encoded access unit from an [Encoder]. typedef Packet = MoqVideoPacket; -/// How pictures are split into independently decodable groups. -typedef Gop = MoqVideoGop; - -/// A keyframe every `interval` pictures. -typedef Keyframe = KeyframeMoqVideoGop; - /// H.264 or H.265. typedef Codec = MoqVideoCodec; diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index 42518706383..0e3d8f638e7 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -4729,12 +4729,12 @@ class FfiConverterMoqVideoEncoderInput { class MoqVideoEncoderOutput { final MoqVideoCodec codec; final int? bitrate; - final MoqVideoGop? gop; + final Duration? join; final MoqVideoEncoderKind kind; MoqVideoEncoderOutput({ required this.codec, this.bitrate = null, - this.gop = null, + this.join = null, required this.kind, }); } @@ -4756,11 +4756,11 @@ class FfiConverterMoqVideoEncoderOutput { ); final bitrate = bitrate_lifted.value; new_offset += bitrate_lifted.bytesRead; - final gop_lifted = FfiConverterOptionalMoqVideoGop.read( + final join_lifted = FfiConverterOptionalDuration.read( Uint8List.view(buf.buffer, new_offset), ); - final gop = gop_lifted.value; - new_offset += gop_lifted.bytesRead; + final join = join_lifted.value; + new_offset += join_lifted.bytesRead; final kind_lifted = FfiConverterMoqVideoEncoderKind.read( Uint8List.view(buf.buffer, new_offset), ); @@ -4770,7 +4770,7 @@ class FfiConverterMoqVideoEncoderOutput { MoqVideoEncoderOutput( codec: codec, bitrate: bitrate, - gop: gop, + join: join, kind: kind, ), new_offset - buf.offsetInBytes, @@ -4781,7 +4781,7 @@ class FfiConverterMoqVideoEncoderOutput { final total_length = FfiConverterMoqVideoCodec.allocationSize(value.codec) + FfiConverterOptionalUInt64.allocationSize(value.bitrate) + - FfiConverterOptionalMoqVideoGop.allocationSize(value.gop) + + FfiConverterOptionalDuration.allocationSize(value.join) + FfiConverterMoqVideoEncoderKind.allocationSize(value.kind) + 0; final buf = Uint8List(total_length); @@ -4799,8 +4799,8 @@ class FfiConverterMoqVideoEncoderOutput { value.bitrate, Uint8List.view(buf.buffer, new_offset), ); - new_offset += FfiConverterOptionalMoqVideoGop.write( - value.gop, + new_offset += FfiConverterOptionalDuration.write( + value.join, Uint8List.view(buf.buffer, new_offset), ); new_offset += FfiConverterMoqVideoEncoderKind.write( @@ -4813,7 +4813,7 @@ class FfiConverterMoqVideoEncoderOutput { static int allocationSize(MoqVideoEncoderOutput value) { return FfiConverterMoqVideoCodec.allocationSize(value.codec) + FfiConverterOptionalUInt64.allocationSize(value.bitrate) + - FfiConverterOptionalMoqVideoGop.allocationSize(value.gop) + + FfiConverterOptionalDuration.allocationSize(value.join) + FfiConverterMoqVideoEncoderKind.allocationSize(value.kind) + 0; } @@ -8888,86 +8888,6 @@ class NamedMoqVideoEncoderKind extends MoqVideoEncoderKind { } } -abstract class MoqVideoGop { - RustBuffer lower(); - int allocationSize(); - int write(Uint8List buf); -} - -class FfiConverterMoqVideoGop { - static MoqVideoGop lift(RustBuffer buffer) { - return FfiConverterMoqVideoGop.read(buffer.asUint8List()).value; - } - - static LiftRetVal read(Uint8List buf) { - final index = buf.buffer.asByteData(buf.offsetInBytes).getInt32(0); - final subview = Uint8List.view(buf.buffer, buf.offsetInBytes + 4); - switch (index) { - case 1: - final lifted = KeyframeMoqVideoGop.read(subview); - return LiftRetVal( - lifted.value, - lifted.bytesRead - subview.offsetInBytes + 4, - ); - default: - throw UniffiInternalError( - UniffiInternalError.unexpectedEnumCase, - "Unable to determine enum variant", - ); - } - } - - static RustBuffer lower(MoqVideoGop value) { - return value.lower(); - } - - static int allocationSize(MoqVideoGop value) { - return value.allocationSize(); - } - - static int write(MoqVideoGop value, Uint8List buf) { - return value.write(buf) - buf.offsetInBytes; - } -} - -class KeyframeMoqVideoGop extends MoqVideoGop { - final int interval; - KeyframeMoqVideoGop(int this.interval); - KeyframeMoqVideoGop._(int this.interval); - static LiftRetVal read(Uint8List buf) { - int new_offset = buf.offsetInBytes; - final interval_lifted = FfiConverterUInt32.read( - Uint8List.view(buf.buffer, new_offset), - ); - final interval = interval_lifted.value; - new_offset += interval_lifted.bytesRead; - return LiftRetVal(KeyframeMoqVideoGop._(interval), new_offset); - } - - @override - RustBuffer lower() { - final buf = Uint8List(allocationSize()); - write(buf); - return toRustBuffer(buf); - } - - @override - int allocationSize() { - return FfiConverterUInt32.allocationSize(interval) + 4; - } - - @override - int write(Uint8List buf) { - buf.buffer.asByteData(buf.offsetInBytes).setInt32(0, 1); - int new_offset = buf.offsetInBytes + 4; - new_offset += FfiConverterUInt32.write( - interval, - Uint8List.view(buf.buffer, new_offset), - ); - return new_offset; - } -} - enum MoqVideoPixelFormat { i420, rgba } class FfiConverterMoqVideoPixelFormat { @@ -18889,53 +18809,6 @@ class FfiConverterOptionalMoqTrackInfo { } } -class FfiConverterOptionalMoqVideoGop { - static MoqVideoGop? lift(RustBuffer buf) { - return FfiConverterOptionalMoqVideoGop.read(buf.asUint8List()).value; - } - - static LiftRetVal read(Uint8List buf) { - if (ByteData.view(buf.buffer, buf.offsetInBytes).getInt8(0) == 0) { - return LiftRetVal(null, 1); - } - final result = FfiConverterMoqVideoGop.read( - Uint8List.view(buf.buffer, buf.offsetInBytes + 1), - ); - return LiftRetVal(result.value, result.bytesRead + 1); - } - - static int allocationSize([MoqVideoGop? value]) { - if (value == null) { - return 1; - } - return FfiConverterMoqVideoGop.allocationSize(value) + 1; - } - - static RustBuffer lower(MoqVideoGop? value) { - if (value == null) { - return toRustBuffer(Uint8List.fromList([0])); - } - final buf = Uint8List( - FfiConverterOptionalMoqVideoGop.allocationSize(value), - ); - FfiConverterOptionalMoqVideoGop.write(value, buf); - return toRustBuffer(buf); - } - - static int write(MoqVideoGop? value, Uint8List buf) { - if (value == null) { - buf[0] = 0; - return 1; - } - buf[0] = 1; - return FfiConverterMoqVideoGop.write( - value, - Uint8List.view(buf.buffer, buf.offsetInBytes + 1), - ) + - 1; - } -} - class FfiConverterOptionalMoqVideoHint { static MoqVideoHint? lift(RustBuffer buf) { return FfiConverterOptionalMoqVideoHint.read(buf.asUint8List()).value; @@ -24982,7 +24855,7 @@ void _checkApiChecksums() { if (uniffi_moq_ffi_checksum_method_moqvideoencoder_set_bitrate() != 34315) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqvideoproducer_cut() != 18974) { + if (uniffi_moq_ffi_checksum_method_moqvideoproducer_cut() != 56974) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqvideoproducer_demand() != 283) { diff --git a/demo/web/src/publish.ts b/demo/web/src/publish.ts index ed46db5eead..75ac86ded9b 100644 --- a/demo/web/src/publish.ts +++ b/demo/web/src/publish.ts @@ -142,7 +142,7 @@ function encoderConfig(effect: Signals.Effect, target: VideoTarget): Video.Confi codec: effect.get(codec), maxPixels: target.width && target.height ? target.width * target.height : undefined, maxBitrate: br != null ? br * 1000 : undefined, - keyframeInterval: kf != null ? Milli(kf) : undefined, + join: kf != null ? Milli(kf) : undefined, frameRate: target.frameRate, }; } diff --git a/doc/bin/hls.md b/doc/bin/hls.md index 4615307572b..1a7361018f2 100644 --- a/doc/bin/hls.md +++ b/doc/bin/hls.md @@ -54,7 +54,7 @@ support the publisher's codecs. What that means for you: -- **The publisher declares the interval.** Only a broadcast whose catalog carries `media.join.interval` gets playlists. Rust publishers built on `moq-mux` (the CLI, the RTMP, SRT, and WebRTC gateways, and the native bindings) declare it: an encoder that knows its GOP passes it as a hint, and an ingest without one measures each track over its first second or group, whichever is longer, then declares the longest. `@moq/publish` declares its keyframe interval. +- **The publisher declares the interval.** Only a broadcast whose catalog carries `media.join.interval` gets playlists. Rust publishers built on `moq-mux` (the CLI, the RTMP, SRT, and WebRTC gateways, and the native bindings) declare it: an encoder that knows its GOP passes it as a hint, and an ingest without one measures each track over its first second or group, whichever is longer, then declares the longest. `@moq/publish` declares its encoders' `join`. - **Every rendition shares the grid.** A group spans from its first frame's timestamp to the next group's and belongs to the first grid line it contains, or, containing none, the line it starts after. Segment N therefore starts at the group containing N times the segment length, every rendition numbers its segments alike, and each cuts its boundaries at its own keyframes. A grid line no group starts a segment at, because a GOP ran past it or the content paused, is listed as an `EXT-X-GAP` so the numbering never skips: on a 2 s grid, a GOP from 0 to 7 s is segment 0, segments 1 and 2 are gaps, and the next GOP starts segment 3. Timestamps that go back start a discontinuity on that rendition alone; `EXT-X-PROGRAM-DATE-TIME`, written after every discontinuity when the catalog has a clock, lets players resync. - **Include media starts on selection.** The HTTP multivariant playlist reads each root `include` entry's catalog and lists its audio and video beside the broadcast's own. An include shares the broadcast's timeline: its renditions are segmented on the broadcast's join interval and clock, whatever its own catalog declares, so publish it on the same timestamps. One not yet routed is listed once its broadcast is announced. Reading its catalog starts no encoding: only a request to one of its rendition URLs subscribes to that rendition's media. Demand lasts through active responses and, after the last one finishes, for three target durations or the playlist window plus one target duration, whichever is longer (20 s by default); a later request starts it again. The broadcast's own renditions stay subscribed. A segment is servable for as long as the relay's [cache](/bin/relay/config#cache) retains it. Keep `--window` (the playlist length, default 16 s) within that retention and within the media's max age (30 s by default), after which a superseded segment's groups are stale. - **Live joins cost the window.** The exporter subscribes 30 s behind the live edge and lists at most 256 segments. One joining partway through a segment lists nothing before the first segment it sees start. diff --git a/doc/concept/hang.md b/doc/concept/hang.md index 2962d673e7a..c822ac3cda7 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -72,7 +72,7 @@ A few things the catalog can express beyond decoder config: - **Spatial media.** A video rendition may carry `projection` (flat `rectilinear`, or `equirectangular` with the degrees of longitude it spans, 180 for VR180) and `stereo` (`mono`, `left-right`, or `top-bottom` frame packing), and an audio rendition may carry `layout: "ambisonic"` for AmbiX channels. Readers treat a value they don't recognize as absent. Nothing in `@moq/watch` renders them yet. - **Disabled renditions.** An audio or video rendition with `enabled: false` has no new frames coming, and a player must not select it. A subscriber may still get audio retained from before the disable, as when it subscribed a moment earlier. Disabling a rendition in `@moq/publish`, as hiding video does, keeps it in the catalog this way rather than removing it, so it is a one-field update; a disabled video rendition encodes one black keyframe first, so a player that predates the field shows black instead of a frozen picture. The field is written only when false. The older video `stalled` field is no longer written, and readers ignore it. A muted `@moq/publish` microphone is not disabled: it stays enabled and sends nothing, ending each utterance with a discontinuity marker, so listeners stay subscribed and play the gap as intentional silence. - **Preference.** A video rendition may carry a signed `preference`, default `0`. A player keeps only the highest preference it can decode, then picks by size and bitrate within it, as do the single-rendition egresses (RTMP play, WHEP, FLV, the transcoder's source). A publisher that adds its own H.264 copy of an H.265 source marks the copy `-1`, so only a viewer that can't decode H.265 pulls it. Picking a rendition by name still overrides it. The field is written only when nonzero. -- **Join interval.** The root `media.join.interval` promises that every audio and video track starts a group at least that often: the longest GOP any of them publishes. It is fixed for the broadcast, and the [HLS gateway](/bin/hls) numbers every rendition's segments on the grid it fixes, segments the smallest whole multiple of it at least a second long, from each group's first timestamp, so renditions share segment numbers. A publisher that knows its GOP declares it; Rust publishers (`moq-mux` and everything built on it) without one measure each track over its first second or group, whichever is longer, and `@moq/publish` declares its keyframe interval. A broadcast without it gets no HLS playlists. +- **Join interval.** The root `media.join.interval` promises that every audio and video track starts a group at least that often: the longest GOP any of them publishes. It is fixed for the broadcast, and the [HLS gateway](/bin/hls) numbers every rendition's segments on the grid it fixes, segments the smallest whole multiple of it at least a second long, from each group's first timestamp, so renditions share segment numbers. A publisher that knows its GOP declares it; Rust publishers (`moq-mux` and everything built on it) without one measure each track over its first second or group, whichever is longer, and `@moq/publish` declares its encoders' `join`. A broadcast without it gets no HLS playlists. - **Clock.** The optional `media.clock` maps PTS zero to wall time so every media track shares one fixed epoch after timescale conversion. Publish independent sources as sibling broadcasts under one prefix, such as `robot/front.hang` and `robot/rear.hang`; each broadcast's audio and video entries are renditions of that source. In one Rust publisher, pass the same `moq_mux::Clock` to each `catalog::Config::with_clock`; JS publishers on one page already share the page clock. Consumers discover the siblings by prefix and align their timestamps through the advertised wall mapping; `@moq/watch` plays one broadcast at a time. - **Version.** The root `version` is the oldest moq-hang draft that reads the catalog correctly, written only when a field needs one newer than 03: a `track` that differs from its ID, or a video `warmup`. Readers ignore fields they don't know, but refuse a `version` above their own with a typed "unsupported catalog version" error, so a player can say to upgrade instead of misreading the catalog. An additive field never raises it. - **Extensions.** The root is a loose object. Applications add their own sections (`scte35`, for example) next to the ones hang defines, optionally naming a track that carries the data. Every library exposes a way to write your section without clobbering the built-in ones, and readers ignore what they don't know. diff --git a/doc/lib/c/index.md b/doc/lib/c/index.md index ebb435ee7d7..fc6690ca6c5 100644 --- a/doc/lib/c/index.md +++ b/doc/lib/c/index.md @@ -111,8 +111,8 @@ inside. The header is the reference: each function carries a doc comment. - **Subscription defaults.** A zeroed `moq_subscription`, a NULL pointer, and `moq_subscription_defaults()` all mean moq-net's defaults, whose zero `delay` starts at the newest group and skips a superseded one at once. Set `delay.max_unlimited` for no subscriber limit, and `delay.min_unstated` to leave the target latency unstated. - **Compressed tracks.** `moq_track_info.compress` on a raw track sends each group DEFLATE-compressed to opt-in `moq-lite-07-wip` peers; other peers, and every reader here, get the original frames. - **fMP4 import.** `MOQ_CONTAINER_FORMAT_FMP4` splits each fragment into one frame per sample. -- **Audio needs cuts.** Video groups at its keyframes, but audio forms a group only where you call `moq_publish_media_cut`: after every frame, or less often to batch frames into fewer groups. Pass that cadence as `moq_audio_init.hint.group_duration_us`. -- **Catalog hints.** Give the catalog what the encoder knows instead of having it estimated: `moq_video_init.hint` (coded size, bitrate, frame rate, group duration) and `moq_audio_init.hint` (bitrate, group duration). A supplied bitrate is never re-estimated. `group_duration_us` is the longest span between two group starts: the GOP, or your cut cadence. Leave a field's `has_*` flag false to have it measured. A bitrate or `group_duration_us` flag set with a 0 value returns `MOQ_ERROR_INVALID_CONFIG`. +- **Audio needs cuts.** Video groups at its keyframes, but audio forms a group only where you call `moq_publish_media_cut`: after every frame, or less often to batch frames into fewer groups. Pass that cadence as `moq_audio_init.hint.join_us`. +- **Catalog hints.** Give the catalog what the encoder knows instead of having it estimated: `moq_video_init.hint` (coded size, bitrate, frame rate, join) and `moq_audio_init.hint` (bitrate, join). A supplied bitrate is never re-estimated. `join_us` is the time between join points, the longest span between two group starts: the GOP, or your cut cadence. Leave a field's `has_*` flag false to have it measured. A bitrate or `join_us` flag set with a 0 value returns `MOQ_ERROR_INVALID_CONFIG`. - **Discovery.** Announcement captures are always a list of wildcard matches, empty when the filter has no wildcards. - **Ending cleanly.** When the input ends, finish every track, then call `moq_publish_catalog_finish()` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `moq_publish_close()` or `moq_route_free()` takes it offline. How long to keep it up is yours to decide. A track still open at the close ends with `UNROUTABLE` (an internal error over moq-transport, which has no such code), and its handle is released. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). - **Timeline breaks.** On a seek or pause, call `moq_publish_media_discontinuity`, then keep timestamps moving forward and resume video on a keyframe. diff --git a/doc/lib/js/publish.md b/doc/lib/js/publish.md index 1225a30d59c..134f5c9f148 100644 --- a/doc/lib/js/publish.md +++ b/doc/lib/js/publish.md @@ -99,12 +99,12 @@ several publishers on one connection do not each target the whole uplink. Codec, resolution, framerate, and bitrate are tunable through `el.video.config`, and `el.video.cut()` asks for an extra keyframe. A viewer who joins once a group's deltas outweigh twice its keyframe gets a fresh one, -so `keyframeInterval` is only the longest a group runs. Through a moq-lite +so `join` is only the longest a group runs. Through a moq-lite relay, only the join that opens the relay's upstream subscription counts. The broadcast declares the longest group among its renditions as the catalog's `media.join.interval` when it first lists one, which an HLS or DASH export cuts its segments on. It stays fixed for the network broadcast, so raising -`keyframeInterval` later overruns that grid. Size, +`join` later overruns that grid. Size, bitrate, and framerate change in place, while a codec or audio format change keeps the rendition's catalog ID under a fresh `track` name that viewers follow through the catalog. For @@ -112,10 +112,10 @@ simulcast, drop the element and register several encoders on a `Publish.Broadcast`, as below. Audio groups carry at least 20 ms by default, packing shorter codec frames -together. `el.audio.groupDuration` changes that minimum; `Milli(0)` puts -every frame in its own group. A longer minimum lets a relay keep fewer -streams, at the cost of coarser loss: a viewer that falls behind skips a whole -group. Frames still forward as they are encoded. +together. `el.audio.join` changes that minimum; a join shorter than a frame +puts every frame in its own group, and zero is refused. A longer join lets a +relay keep fewer streams, at the cost of coarser loss: a viewer that falls +behind skips a whole group. Frames still forward as they are encoded. Timestamps are `@moq/clock`'s `now()` in microseconds, so every source shares one timeline. The catalog's `media.clock` maps it to wall time. Stamp your own tracks diff --git a/doc/lib/kt/index.md b/doc/lib/kt/index.md index d0892470e57..c67762b7fc9 100644 --- a/doc/lib/kt/index.md +++ b/doc/lib/kt/index.md @@ -82,7 +82,7 @@ Moq.connect("https://relay.example.com").use { moq -> val video = dev.moq.video.Producer( CatalogProducer(broadcast), dev.moq.video.ProducerOptions(input = VideoEncoderInput(format = VideoPixelFormat.RGBA, width = 1280u, height = 720u, framerate = dev.moq.video.FrameRate(30000u, 1001u)), - output = VideoEncoderOutput(codec = VideoCodec.H264, bitrate = null, gop = null, kind = autoEncoder), track = "camera"), + output = VideoEncoderOutput(codec = VideoCodec.H264, bitrate = null, join = null, kind = autoEncoder), track = "camera"), ) video.write(VideoFrame(timestamp = pts, data = rgba)) route.announce() // the route mints its own instance: route.instance() diff --git a/doc/lib/rs/moq-audio.md b/doc/lib/rs/moq-audio.md index a11d6b92068..c53cd28c2b1 100644 --- a/doc/lib/rs/moq-audio.md +++ b/doc/lib/rs/moq-audio.md @@ -43,8 +43,9 @@ history it retained stays on the track until it is subscribed or expires; a devi it the same way before the encoder returns the error. Its consumer can be swapped without changing the track subscribers know. Opus packs 20 ms per packet by default, or 10 ms with a shorter `frame_duration`. Groups carry at least 20 ms by default, -so 10 ms packets pair up. `encode::Options::group_duration` -changes that minimum; zero puts every packet in its own group. Longer groups +so 10 ms packets pair up. `encode::Options::join` +changes that minimum; a join shorter than a packet puts every packet in its own +group, and zero is refused. Longer groups mean fewer streams per relay but coarser loss. There is no AAC encoder, so an AAC encode request is refused; AAC decode is for broadcasts from the ingest gateways. diff --git a/doc/lib/rs/moq-video.md b/doc/lib/rs/moq-video.md index 0faed2db33e..53e5c561794 100644 --- a/doc/lib/rs/moq-video.md +++ b/doc/lib/rs/moq-video.md @@ -31,7 +31,7 @@ open and encoding whether or not anyone subscribes, so a late viewer can fetch the recent past. Set `encode::Capture::on_demand` to open the camera only while someone subscribes, at the cost of that past. A viewer who joins once the current group's deltas outweigh twice its keyframe gets a fresh -keyframe, so the GOP is only the longest a group runs. Through a moq-lite +keyframe, so `encode::Config::join` is only the longest a group runs. Through a moq-lite relay, only the join that opens the relay's upstream subscription counts. A build with the `v4l2` feature keeps every `Auto` or `Hardware` selection to its GOP, whichever encoder it lands on, since a V4L2 driver may refuse the request. diff --git a/go/wrapper/video/video.go b/go/wrapper/video/video.go index b3d75b828af..b8207acb702 100644 --- a/go/wrapper/video/video.go +++ b/go/wrapper/video/video.go @@ -28,15 +28,9 @@ type FrameRate = ffi.MoqVideoFrameRate // EncoderInput declares the pixel layout, resolution, and framerate of written frames. type EncoderInput = ffi.MoqVideoEncoderInput -// EncoderOutput configures the codec, bitrate, group mode, and backend preference. +// EncoderOutput configures the codec, bitrate, join, and backend preference; a nil Join keyframes every 2 seconds. type EncoderOutput = ffi.MoqVideoEncoderOutput -// Gop is how pictures are split into independently decodable groups; a nil EncoderOutput.Gop keyframes every ~2 seconds. -type Gop = ffi.MoqVideoGop - -// Keyframe starts a group with a keyframe every interval frames. -func Keyframe(interval uint32) Gop { return ffi.MoqVideoGopKeyframe{Interval: interval} } - // Frame is one raw picture with its presentation timestamp. type Frame = ffi.MoqVideoFrame diff --git a/js/publish/src/audio/capture.test.ts b/js/publish/src/audio/capture.test.ts index 978ec20974e..d131b693bb7 100644 --- a/js/publish/src/audio/capture.test.ts +++ b/js/publish/src/audio/capture.test.ts @@ -579,7 +579,7 @@ test("a failed encoder stays failed until the capture source changes", async () config: new Signal(undefined), stats: new Signal(undefined), track: new Signal(track), - joinInterval: new Signal(undefined), + join: new Signal(undefined), close: () => {}, }; const encoder = new Encoder("audio", { diff --git a/js/publish/src/audio/encoder.test.ts b/js/publish/src/audio/encoder.test.ts index d4002c4d2e5..1f00740eb94 100644 --- a/js/publish/src/audio/encoder.test.ts +++ b/js/publish/src/audio/encoder.test.ts @@ -59,8 +59,8 @@ describe("resolve", () => { } }); - // A frame opens a group once it is at least the group duration past the open one's start, so - // groups start a whole number of frames apart: that, not the duration, is the join interval. + // A frame opens a group once it is at least `join` past the open one's start, so groups start a + // whole number of frames apart: that, not the requested join, is the effective one. test("groups start a whole number of frames apart", () => { const opus = (frame: number) => resolve(captured, { mime: "opus", frameDuration: Milli(frame) }); for (const [frame, duration, spacing] of [ @@ -68,8 +68,8 @@ describe("resolve", () => { [20, 30, 40], [60, 20, 60], [2.5, 20, 20], - // A zero duration opens a group per frame. - [20, 0, 20], + // A join shorter than a frame opens a group per frame. + [20, 1, 20], ] as const) { expect(groupSpacing(Milli(duration), opus(frame))).toBe(Milli(spacing)); } @@ -295,7 +295,7 @@ function configured(): Promise { // An Encoder wired to a fake capture feed, recording each written frame as [timestamp, payload bytes] // and how many frames each group carries. -async function setup(order = new Order(), codec?: Codec, groupDuration?: Milli) { +async function setup(order = new Order(), codec?: Codec, join?: Milli) { const configuring = configured(); const track = new Moq.Track.Producer("audio").accept({ timescale: Moq.Time.Timescale.MILLI }); @@ -324,7 +324,7 @@ async function setup(order = new Order(), codec?: Codec, groupDuration?: Milli) config: new Signal(undefined), stats: new Signal(undefined), track: new Signal(track), - joinInterval: new Signal(undefined), + join: new Signal(undefined), close: () => { track.finish(); track.close(); @@ -344,7 +344,7 @@ async function setup(order = new Order(), codec?: Codec, groupDuration?: Milli) capture: capture as never, enabled, codec, - groupDuration, + join, }); const config = await configuring; @@ -737,7 +737,7 @@ test("a push completing several frames stays one stream", async () => { // The first frame at the minimum opens the next group. A timeline break closes the group early, // so the first frame after it opens a fresh one. -test("a group duration packs frames until the minimum and restarts after a break", async () => { +test("a join packs frames until the minimum and restarts after a break", async () => { using _webcodecs = installFakeWebCodecs(); using env = await setup(new Order(), undefined, Milli(100)); const { enabled, feed, written, groups, appended } = env; @@ -778,7 +778,7 @@ test("default audio groups carry twenty milliseconds", async () => { [2.5, 8, undefined], [10, 2, undefined], [20, 1, undefined], - [2.5, 1, 0], + [2.5, 1, 1], ] as const) { using env = await setup( new Order(), @@ -904,6 +904,39 @@ test("settles when the codec settings can't resolve", async () => { } }); +// A join of zero would open a group per frame, so it refuses the rendition like any invalid knob, +// settling the gate without a config until a valid join arrives. +test("refuses a join that is not a positive duration", async () => { + using _webcodecs = installFakeWebCodecs(); + const error = spyOn(console, "error").mockImplementation(() => {}); + for (const join of [0, -20, Number.NaN]) { + const capture = { + in: { source: new Signal(undefined) }, + out: { + root: new Signal(undefined), + format: new Signal({ sampleRate: 48_000, channelCount: 1 }), + frames: new Signal(undefined), + }, + blocked: new Signal(false), + }; + const encoder = new Encoder("audio", { capture: capture as never, join: join as Milli }); + try { + await settle(); + expect(encoder.out.catalog.peek()).toBeUndefined(); + expect(encoder.settled.peek()).toBe(true); + + const ready = encoder.out.catalog.changed(); + encoder.join.set(Milli(20)); + await ready; + expect(encoder.out.catalog.peek()).toBeDefined(); + } finally { + encoder.close(); + } + } + expect(error).toHaveBeenCalled(); + error.mockRestore(); +}); + test("settles while the capture waits on a gesture", async () => { const capture = { in: { source: new Signal(undefined) }, @@ -1202,8 +1235,8 @@ test("an audio-only broadcast declares its group spacing as the join interval", const broadcast = new Broadcast({ origin: new Moq.Origin.Producer(), name: Moq.Path.from("join.hang") }); const feed = new Feed(); const configuring = configured(); - // 20 ms Opus frames start a group every two frames for a 30 ms duration. - const encoder = new Encoder("audio", { broadcast, capture: fakeCapture(feed) as never, groupDuration: Milli(30) }); + // 20 ms Opus frames start a group every two frames for a 30 ms join. + const encoder = new Encoder("audio", { broadcast, capture: fakeCapture(feed) as never, join: Milli(30) }); let catalog: Json.Snapshot.Consumer | undefined; try { await configuring; diff --git a/js/publish/src/audio/encoder.ts b/js/publish/src/audio/encoder.ts index 6f90cbc734a..382bed86940 100644 --- a/js/publish/src/audio/encoder.ts +++ b/js/publish/src/audio/encoder.ts @@ -123,8 +123,8 @@ export type EncoderProps = Inputs & { // Codec selection plus encoder settings. Defaults to "opus". codec?: Codec | Signal; - // The minimum audio carried by each group. Defaults to 20 ms; zero puts every frame in its own group. - groupDuration?: Milli | Signal; + // The time between join points: a frame opens a group once it is this far past the last one's start. Defaults to 20 ms. + join?: Milli | Signal; }; /** Signals the encoder derives. */ @@ -181,11 +181,12 @@ export class Encoder { /** The live-editable codec selection plus its encoder settings. */ codec: Signal; /** - * The minimum timestamp span before a frame opens the next group, closing the previous one. Defaults to 20 ms; zero puts every frame in its own group. A longer group costs the + * The time between join points: a frame opens the next group once it is at least this far past + * the open group's start, closing the previous one. Defaults to 20 ms. A longer join costs the * relay fewer streams but makes loss coarser: a viewer that falls behind skips a whole group. A - * negative or non-finite duration refuses the rendition. + * join that is not a positive, finite duration refuses the rendition. */ - groupDuration: Signal; + join: Signal; /** * The capture supplying this rendition, or undefined while none is wired. @@ -242,11 +243,11 @@ export class Encoder { // caught up behind the live edge, not frames written late, so they never raise the catalog delay. #history: Micro | undefined; - // The last valid group duration, which #runConfig validates. - #groupDuration = Micro(20_000); - // How far apart groups start under the resolved config, which the broadcast may declare as the - // catalog's join interval (see groupSpacing). - #joinInterval = new Signal(undefined); + // The last valid join, which #runConfig validates. + #join = Micro(20_000); + // How far apart groups start under the resolved config, a whole number of frames, which the + // broadcast may declare as the catalog's join interval (see groupSpacing). + #spacing = new Signal(undefined); // How many resolution runs threw for their current inputs (e.g. invalid codec settings), so no // config is coming until one reruns. @@ -297,7 +298,7 @@ export class Encoder { this.volume = Signal.from(props?.volume ?? 1); this.muted = Signal.from(props?.muted ?? false); this.codec = Signal.from(props?.codec ?? "opus"); - this.groupDuration = Signal.from(props?.groupDuration ?? Milli(20)); + this.join = Signal.from(props?.join ?? Milli(20)); // Only the capture graph has a node to expose. this.#signals.run((effect) => { @@ -439,7 +440,7 @@ export class Encoder { effect.cleanup(() => rendition.close()); // How far apart groups start, which the broadcast declares as the catalog's join interval when // nothing longer, such as a video GOP, sits beside it. - effect.proxy(rendition.joinInterval, this.#joinInterval); + effect.proxy(rendition.join, this.#spacing); effect.proxy(this.#slot, rendition.slot); effect.cleanup(() => this.#slot.set(undefined)); @@ -540,10 +541,10 @@ export class Encoder { // rather than tearing down. A config probed once is reused, so the catalog flips back without // waiting on a probe. #runConfig(effect: Effect): void { - const groupDuration = effect.get(this.groupDuration); - if (!Number.isFinite(groupDuration) || groupDuration < 0) - throw new Error(`audio group duration must be a finite, non-negative number of ms: ${groupDuration}`); - this.#groupDuration = Micro.fromMilli(groupDuration); + const join = effect.get(this.join); + if (!Number.isFinite(join) || join <= 0) + throw new Error(`audio join must be a finite number of ms greater than 0: ${join}`); + this.#join = Micro.fromMilli(join); const capture = effect.get(this.in.capture); const captured = capture ? effect.get(capture.out.format) : undefined; @@ -554,7 +555,7 @@ export class Encoder { const resolved = resolve(captured, effect.get(this.codec)); // Set before the config is listed, so the catalog's first listing declares it. - this.#joinInterval.set(groupSpacing(groupDuration, resolved)); + this.#spacing.set(groupSpacing(join, resolved)); const source = capture ? effect.get(capture.in.source) : undefined; const kind = source ? sourceKind(source) : "auto"; const encoderConfig = toEncoderConfig(resolved, kind, this.#opusOptions(effect)); @@ -757,7 +758,7 @@ export class Encoder { // first one at or past the minimum after the open group's start. Frames forward // as they are written rather than waiting for the group to fill. A dropped // group leaves a gap for the codec's PLC. - const keyframe = live.start === undefined || timestamp - live.start >= this.#groupDuration; + const keyframe = live.start === undefined || timestamp - live.start >= this.#join; if (keyframe) live.start = timestamp; live.producer.encode(frame, timestamp, keyframe); const history = this.#history !== undefined && timestamp < this.#history; @@ -1008,14 +1009,13 @@ export function resolve(captured: Format, selected: Codec): Resolved { } /** - * How far apart audio groups start: a frame opens one once it is at least `groupDuration` past the - * open group's start, so groups are a whole number of frames apart, and at least one, since a zero - * duration opens a group per frame. + * How far apart audio groups start: a frame opens one once it is at least `join` past the + * open group's start, so groups are a whole number of frames apart, and at least one. * @internal */ -export function groupSpacing(groupDuration: Milli, resolved: Resolved): Milli { +export function groupSpacing(join: Milli, resolved: Resolved): Milli { const frame = resolved.frameDuration ?? (AAC_FRAME_SAMPLES * 1_000_000) / resolved.catalog.sampleRate; - const frames = Math.max(1, Math.ceil(Micro.fromMilli(groupDuration) / frame)); + const frames = Math.max(1, Math.ceil(Micro.fromMilli(join) / frame)); return Milli.fromMicro(Micro(frames * frame)); } diff --git a/js/publish/src/broadcast.test.ts b/js/publish/src/broadcast.test.ts index 4f9e0eb351b..fef92ad70ea 100644 --- a/js/publish/src/broadcast.test.ts +++ b/js/publish/src/broadcast.test.ts @@ -1021,9 +1021,9 @@ test("echo solicits feedback in the catalog unless it walks above the root", asy test("the catalog declares the longest group as the join interval once it lists media", async () => { const broadcast = new Broadcast({ enabled: true, origin: new Origin.Producer(), name: Path.from("join.hang") }); const video = broadcast.video("video/hd"); - video.joinInterval.set(Milli(2000)); + video.join.set(Milli(2000)); const audio = broadcast.audio("audio/data"); - audio.joinInterval.set(Milli(20)); + audio.join.set(Milli(20)); broadcast.text("captions/en").config.set({ format: "vtt", container: { kind: "legacy" } } as Catalog.TextConfig); await settle(); @@ -1049,15 +1049,15 @@ test("the join interval stays fixed for the network broadcast", async () => { const name = new Signal(Path.from("fixed.hang")); const broadcast = new Broadcast({ enabled: true, origin: new Origin.Producer(), name }); const video = broadcast.video("video"); - video.joinInterval.set(Milli(2000)); + video.join.set(Milli(2000)); video.config.set(videoConfig); await settle(); expect((await readCatalog(broadcast))?.media?.join?.interval).toBe(Catalog.u53(2000)); // A longer GOP, from a config change or a later rendition, leaves the declared grid alone. - video.joinInterval.set(Milli(4000)); + video.join.set(Milli(4000)); const later = broadcast.video("video/later"); - later.joinInterval.set(Milli(6000.5)); + later.join.set(Milli(6000.5)); later.config.set(videoConfig); await settle(); expect((await readCatalog(broadcast))?.media?.join?.interval).toBe(Catalog.u53(2000)); @@ -1069,17 +1069,17 @@ test("the join interval stays fixed for the network broadcast", async () => { broadcast.close(); }); -test("an audio-only broadcast declares its group duration, and no duration declares nothing", async () => { +test("an audio-only broadcast declares its join, and no join declares nothing", async () => { const broadcast = new Broadcast({ enabled: true, origin: new Origin.Producer(), name: Path.from("audio.hang") }); const audio = broadcast.audio("audio"); - audio.joinInterval.set(Milli(0)); + audio.join.set(Milli(0)); audio.config.set(audioConfig); await settle(); - // Every frame its own group: the spacing is a frame, unknown up front, so nothing is promised. + // A zero join is no positive spacing, so nothing is promised. expect((await readCatalog(broadcast))?.media?.join).toBeUndefined(); // A later listing tries again once a duration is known. - audio.joinInterval.set(Milli(20)); + audio.join.set(Milli(20)); await settle(); expect((await readCatalog(broadcast))?.media?.join?.interval).toBe(Catalog.u53(20)); broadcast.close(); @@ -1095,7 +1095,7 @@ test("an on-demand rendition declares the interval before encoding, and a media onDemand: true, }); const video = broadcast.video("video"); - video.joinInterval.set(Milli(2000)); + video.join.set(Milli(2000)); video.config.set(videoConfig); await settle(); const net = broadcast.net.peek(); diff --git a/js/publish/src/broadcast.ts b/js/publish/src/broadcast.ts index da6d46ea14a..2a31e199364 100644 --- a/js/publish/src/broadcast.ts +++ b/js/publish/src/broadcast.ts @@ -98,7 +98,7 @@ function longestGroup(effect: Effect, renditions: ReadonlyMap { readonly config = new Signal(undefined); /** - * The longest a video or audio rendition's tracks run between group starts outside a pause, when - * known: a video encoder's keyframe interval, or an audio encoder's group duration. Undefined, or - * not a positive duration, contributes nothing. + * The time between join points the producer holds to, the longest a video or audio rendition's + * tracks run between group starts outside a pause, when known: the effective value of an + * encoder's `join`, after any rounding to whole frames. Undefined, or not a positive duration, + * contributes nothing. * * When the broadcast first lists a video or audio rendition in a network broadcast, it declares * the longest of these as the catalog's `media.join.interval`, rounded up to whole milliseconds, * so an HLS or DASH export can cut its segments before any media is encoded. That declaration * is fixed for the network broadcast, so a later change here does not move it. */ - readonly joinInterval = new Signal(undefined); + readonly join = new Signal(undefined); /** * What the producer has sent under this rendition, written by the producer and reported under diff --git a/js/publish/src/video/capture.test.ts b/js/publish/src/video/capture.test.ts index f3c1b8da3e2..44b3fa4e0e8 100644 --- a/js/publish/src/video/capture.test.ts +++ b/js/publish/src/video/capture.test.ts @@ -465,7 +465,7 @@ test("an encoder resuming on a still source encodes a keyframe of the current pi config: new Signal(undefined), stats: new Signal(undefined), track: live, - joinInterval: new Signal(undefined), + join: new Signal(undefined), close: () => {}, }; const encoder = new Encoder("video", { @@ -579,7 +579,7 @@ test("an encoder torn down while a frame is in flight closes that frame", async config: new Signal(undefined), stats: new Signal(undefined), track: live, - joinInterval: new Signal(undefined), + join: new Signal(undefined), close: () => {}, }; const encoder = new Encoder("video", { diff --git a/js/publish/src/video/encoder-stall.test.ts b/js/publish/src/video/encoder-stall.test.ts index 7558929b7ad..6089ef3855d 100644 --- a/js/publish/src/video/encoder-stall.test.ts +++ b/js/publish/src/video/encoder-stall.test.ts @@ -117,7 +117,7 @@ test("a slower-than-realtime encoder drops input instead of ratcheting catalog l config: new Signal(undefined), stats: new Signal(undefined), track: new Signal(track), - joinInterval: new Signal(undefined), + join: new Signal(undefined), close: () => { track.finish(); track.close(); diff --git a/js/publish/src/video/encoder.test.ts b/js/publish/src/video/encoder.test.ts index 5196eafd431..80e4f33739d 100644 --- a/js/publish/src/video/encoder.test.ts +++ b/js/publish/src/video/encoder.test.ts @@ -150,7 +150,7 @@ test("encoding tracks encoder config in its child effect", async () => { config: new Signal(undefined), stats: new Signal(undefined), track: new Signal(track), - joinInterval: new Signal(undefined), + join: new Signal(undefined), close: () => { track.finish(); track.close(); @@ -194,7 +194,7 @@ test("an on-demand gap marks a discontinuity and leaves the broadcast-owned trac config: new Signal(undefined), stats: new Signal(undefined), track: live, - joinInterval: new Signal(undefined), + join: new Signal(undefined), close: () => { track.finish(); track.close(); @@ -251,7 +251,7 @@ test("a bandwidth estimate updates the bitrate without blanking the config or re config: new Signal(undefined), stats: new Signal(undefined), track: new Signal(track), - joinInterval: new Signal(undefined), + join: new Signal(undefined), close: () => { track.finish(); track.close(); @@ -351,7 +351,7 @@ test("every published config fits a probe for its own codec", async () => { config: new Signal(undefined), stats: new Signal(undefined), track: new Signal(track), - joinInterval: new Signal(undefined), + join: new Signal(undefined), close: () => { track.finish(); track.close(); @@ -573,35 +573,37 @@ test("software-only AV1 is refused even when explicitly requested", async () => // Regression: only the codec probe and `maxScale` settled the encoder on failure, so a knob that broke // resolution later (a negative bitrate) held ``'s announce forever. -test("an invalid knob settles the encoder without a config", async () => { - using _videoEncoder = installFakeVideoEncoder(); - const error = spyOn(console, "error").mockImplementation(() => {}); - const capture = { - in: { source: new Signal({ getSettings: () => ({ frameRate: 30 }), getConstraints: () => ({}) }) }, - out: { display: new Signal({ width: 1920, height: 1080 }) }, - }; - const config = new Signal<{ bitrateScale?: number } | undefined>({ bitrateScale: -1 }); - const probe = spyOn(FakeVideoEncoder, "isConfigSupported"); - const encoder = new Encoder("video", { enabled: true, capture: capture as never, config }); - try { - await settle(); - expect(encoder.out.catalog.peek()).toBeUndefined(); - expect(encoder.settled.peek()).toBe(true); - expect(error).toHaveBeenCalled(); - // Refused before the browser could coerce it into an unsigned bitrate. - expect(probe).not.toHaveBeenCalled(); - - // Fixing the knob clears the failure and resolves a config. - config.set(undefined); - await settle(); - expect(encoder.out.catalog.peek()).toBeDefined(); - expect(encoder.settled.peek()).toBe(true); - } finally { - encoder.close(); - error.mockRestore(); - probe.mockRestore(); - } -}); +for (const invalid of [{ bitrateScale: -1 }, { join: Milli(0) }] satisfies Config[]) { + test(`an invalid knob settles the encoder without a config: ${JSON.stringify(invalid)}`, async () => { + using _videoEncoder = installFakeVideoEncoder(); + const error = spyOn(console, "error").mockImplementation(() => {}); + const capture = { + in: { source: new Signal({ getSettings: () => ({ frameRate: 30 }), getConstraints: () => ({}) }) }, + out: { display: new Signal({ width: 1920, height: 1080 }) }, + }; + const config = new Signal(invalid); + const probe = spyOn(FakeVideoEncoder, "isConfigSupported"); + const encoder = new Encoder("video", { enabled: true, capture: capture as never, config }); + try { + await settle(); + expect(encoder.out.catalog.peek()).toBeUndefined(); + expect(encoder.settled.peek()).toBe(true); + expect(error).toHaveBeenCalled(); + // Refused before the browser could coerce it into an unsigned bitrate. + expect(probe).not.toHaveBeenCalled(); + + // Fixing the knob clears the failure and resolves a config. + config.set(undefined); + await settle(); + expect(encoder.out.catalog.peek()).toBeDefined(); + expect(encoder.settled.peek()).toBe(true); + } finally { + encoder.close(); + error.mockRestore(); + probe.mockRestore(); + } + }); +} test("screen encoders default to logical pixels without scaling an already reduced capture twice", async () => { using _videoEncoder = installFakeVideoEncoder(); @@ -783,7 +785,7 @@ test("disabling encodes one black keyframe and keeps the rendition with enabled: config: new Signal(undefined), stats: new Signal(undefined), track: new Signal(track), - joinInterval: new Signal(undefined), + join: new Signal(undefined), close: () => { track.finish(); track.close(); @@ -953,7 +955,7 @@ test("the probe encodes with the frame rate and bitrate ceiling", async () => { config: new Signal(undefined), stats: new Signal(undefined), track: new Signal(track), - joinInterval: new Signal(undefined), + join: new Signal(undefined), close: () => { track.finish(); track.close(); @@ -1163,7 +1165,7 @@ async function recordKeyframes( config: new Signal(undefined), stats: new Signal(undefined), track: new Signal(track), - joinInterval: new Signal(undefined), + join: new Signal(undefined), close: () => { track.finish(); track.close(); @@ -1322,15 +1324,15 @@ test.each([false, true])( ); // An HLS export cuts segments on the catalog's join interval and reads the media directly, so an -// on-demand rendition declares its keyframe interval before anything is encoded and publishes no records. -test("an on-demand rendition declares its keyframe interval, and a media subscriber starts it", async () => { +// on-demand rendition declares its join before anything is encoded and publishes no records. +test("an on-demand rendition declares its join, and a media subscriber starts it", async () => { using recording = await installRecording(); const broadcast = new Broadcast({ origin: new Moq.Origin.Producer(), name: Moq.Path.from("join.hang"), onDemand: true, }); - const config = new Signal({ keyframeInterval: Milli(3000) }); + const config = new Signal({ join: Milli(3000) }); const encoder = new Encoder("video", { enabled: true, broadcast, @@ -1354,8 +1356,8 @@ test("an on-demand rendition declares its keyframe interval, and a media subscri await settle(); expect(encoder.out.active.peek()).toBe(true); - // A later keyframe interval leaves the declared grid alone for this broadcast. - config.set({ keyframeInterval: Milli(5000) }); + // A later join leaves the declared grid alone for this broadcast. + config.set({ join: Milli(5000) }); await settle(); expect((await readCatalog(broadcast))?.media?.join?.interval).toBe(Catalog.u53(3000)); } finally { @@ -1365,7 +1367,7 @@ test("an on-demand rendition declares its keyframe interval, and a media subscri } }); -test("the default keyframe interval is the declared join interval", async () => { +test("the default join is the declared join interval", async () => { using recording = await installRecording(); const broadcast = new Broadcast({ origin: new Moq.Origin.Producer(), name: Moq.Path.from("default.hang") }); const encoder = new Encoder("video", { enabled: true, broadcast, capture: recording.capture as never }); @@ -1378,7 +1380,7 @@ test("the default keyframe interval is the declared join interval", async () => } }); -// The declared join interval is the longest GOP when media is first listed: a 3000ms interval +// The declared join interval is the longest group when media is first listed: a 3000ms join // declares 3000ms, and a live change at 2500ms, after a keyframe at 0, is held to it. async function liveInterval(changed: Milli): Promise<{ keys: number[]; interval?: number }> { using recording = await installRecording(); @@ -1387,12 +1389,12 @@ async function liveInterval(changed: Milli): Promise<{ keys: number[]; interval? origin: new Moq.Origin.Producer(), name: Moq.Path.from("interval.hang"), }); - const config = new Signal({ keyframeInterval: Milli(3000) }); + const config = new Signal({ join: Milli(3000) }); const encoder = new Encoder("video", { enabled: true, broadcast, capture: recording.capture as never, config }); try { await settle(); for (let ms = 0; ms < 16000; ms += 100) { - if (ms === 2500) config.set({ keyframeInterval: changed }); + if (ms === 2500) config.set({ join: changed }); await recording.send(ms); } return { keys: [...recording.keys], interval: (await readCatalog(broadcast))?.media?.join?.interval }; @@ -1402,7 +1404,7 @@ async function liveInterval(changed: Milli): Promise<{ keys: number[]; interval? } } -test("a live keyframe interval raise is capped at the declared join interval", async () => { +test("a live join raise is capped at the declared join interval", async () => { const capped = { keys: [0, 3000, 6000, 9000, 12000, 15000], interval: 3000 }; expect(await liveInterval(Milli(6000))).toEqual(capped); // Infinite is not a valid Milli, but a config is not checked. @@ -1413,6 +1415,19 @@ test("a live keyframe interval raise is capped at the declared join interval", a expect(cut.keys.slice(0, 4)).toEqual([0, 2500, 3500, 4500]); }); +// A join of zero would keyframe every frame, so it refuses the rendition like any invalid knob. +test("a join that is not a positive duration refuses the rendition", async () => { + const error = spyOn(console, "error").mockImplementation(() => {}); + try { + // A live change at 2500ms stops the encoder already running there. + for (const join of [0, -1, Number.NaN]) { + expect((await liveInterval(join as Milli)).keys).toEqual([0]); + } + } finally { + error.mockRestore(); + } +}); + test("an unwatched rendition reserves nothing and encodes at its configured bitrate", async () => { using _videoEncoder = installFakeVideoEncoder(); const broadcast = new Broadcast({ origin: new Moq.Origin.Producer(), name: Moq.Path.from("unwatched.hang") }); @@ -1676,7 +1691,7 @@ test("a shrinking capture resizes in band below the advertised ceiling", async ( config: new Signal(undefined), stats: new Signal(undefined), track: new Signal(track), - joinInterval: new Signal(undefined), + join: new Signal(undefined), close: () => { track.finish(); track.close(); @@ -1751,7 +1766,7 @@ test("the catalog bitrate and frame rate only rise", async () => { config: new Signal(undefined), stats: new Signal(undefined), track: new Signal(track), - joinInterval: new Signal(undefined), + join: new Signal(undefined), close: () => { track.finish(); track.close(); @@ -1813,7 +1828,7 @@ test("each slot keeps its own ceiling and maxima", async () => { stats: new Signal(undefined), track: new Signal(undefined), slot, - joinInterval: new Signal(undefined), + join: new Signal(undefined), close: () => {}, }); return { video: rendition, order: new Order(), join: new Signal(undefined) }; diff --git a/js/publish/src/video/encoder.ts b/js/publish/src/video/encoder.ts index a8a71e65aec..3e63ac93f3c 100644 --- a/js/publish/src/video/encoder.ts +++ b/js/publish/src/video/encoder.ts @@ -22,7 +22,7 @@ import type { Capture } from "./capture"; import { normalizeSource, type Source } from "./types"; // The keyframe interval when the config sets none. -const DEFAULT_KEYFRAME_INTERVAL = Milli.fromSecond(2 as Second); +const DEFAULT_JOIN = Milli.fromSecond(2 as Second); /** Cumulative encoder output totals, measured from the chunks the encoder produces. */ export interface Stats { @@ -63,13 +63,14 @@ export interface Config { maxScale?: number; /** - * The longest interval between keyframes. A joining subscriber forces one sooner once the - * group's deltas outweigh the track's refresh ratio. The broadcast declares the longest - * interval as its catalog's `media.join.interval` when media is first listed, which then bounds - * every GOP: a live change past it, including to infinite, is capped there until a new network - * broadcast declares again. (default: 2000 milliseconds) + * The time between join points: the longest a group runs before the next keyframe. A joining + * subscriber forces one sooner once the group's deltas outweigh the track's refresh ratio. The + * broadcast declares the longest as its catalog's `media.join.interval` when media is first + * listed, which then bounds every group: a live change past it, including to infinite, is capped + * there until a new network broadcast declares again. Anything but a positive duration refuses + * the rendition. (default: 2000 milliseconds) */ - keyframeInterval?: Milli; + join?: Milli; /** If not provided, the encoder will use the best bitrate for the given width, height, and framerate. */ maxBitrate?: number; @@ -354,10 +355,10 @@ export class Encoder { effect.proxy(this.#slot, rendition.slot); effect.cleanup(() => this.#slot.set(undefined)); - // The longest GOP, which the broadcast declares as the catalog's join interval. + // The longest group, which the broadcast declares as the catalog's join interval. Keyframes + // follow timestamps rather than a frame count, so the requested join is the effective one. effect.run((effect) => { - const interval = effect.get(this.config)?.keyframeInterval ?? DEFAULT_KEYFRAME_INTERVAL; - rendition.joinInterval.set(interval); + rendition.join.set(effect.get(this.config)?.join ?? DEFAULT_JOIN); }); // Publish the resolved catalog config; undefined (until one resolves) leaves it out of the catalog. @@ -539,7 +540,10 @@ export class Encoder { // The catalog's declared join interval is the longest GOP, fixed for the network // broadcast, so a live raise past it stops there. - const requested = config?.keyframeInterval ?? DEFAULT_KEYFRAME_INTERVAL; + const requested = config?.join ?? DEFAULT_JOIN; + // Refused with the config (see #runDimensions), so an encoder already running stops + // rather than keyframing every frame. + if (!(requested > 0)) continue; const declared = join.peek(); const interval = declared === undefined ? requested : Milli(Math.min(requested, declared)); @@ -766,6 +770,10 @@ export class Encoder { if (!source) return; const user = effect.get(this.config); + // Refused here, with the other knobs, so an invalid join settles the encoder without a config. + if (user?.join !== undefined && !(user.join > 0)) { + throw new Error(`join must be greater than 0 ms: ${user.join}`); + } const sourcePixels = display.width * display.height; @@ -903,7 +911,7 @@ export class Encoder { /** * Request a keyframe, opening a new group at a frame no earlier than this call. * - * For a resume, a recording cut, or a known tune-in moment; {@link Config.keyframeInterval} is the + * For a resume, a recording cut, or a known tune-in moment; {@link Config.join} is the * longest a group runs, and a joining subscriber already opens one through the track's refresh * ratio. Requests between two frames coalesce into one keyframe on the next, with no spacing: a * caller in a loop pins the encoder at all-keyframe. A request while not encoding is served by the diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/video/Video.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/video/Video.kt index 19a35130102..a153a6250c7 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/video/Video.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/video/Video.kt @@ -44,9 +44,6 @@ typealias ReadEnd = uniffi.moq.MoqVideoRead.End /** One encoded access unit from an [Encoder]. */ typealias Packet = uniffi.moq.MoqVideoPacket -/** How pictures are split into independently decodable groups. */ -typealias Gop = uniffi.moq.MoqVideoGop - /** H.264 or H.265. */ typealias Codec = uniffi.moq.MoqVideoCodec diff --git a/py/moq-rs/moq/types.py b/py/moq-rs/moq/types.py index c38e9670646..43ce8f7f7d2 100644 --- a/py/moq-rs/moq/types.py +++ b/py/moq-rs/moq/types.py @@ -56,9 +56,6 @@ from moq_ffi import ( MoqVideoEncoderOutput as VideoEncoderOutput, ) -from moq_ffi import ( - MoqVideoGop as VideoGop, -) from moq_ffi import ( MoqVideoPixelFormat as VideoPixelFormat, ) @@ -112,7 +109,6 @@ "VideoCodec", "VideoEncoderInput", "VideoEncoderKind", - "VideoGop", "VideoEncoderOutput", "VideoDecodedFrame", "VideoPixelFormat", diff --git a/py/moq-rs/moq/video.py b/py/moq-rs/moq/video.py index bdc62fe2eca..7eb2ed6b61a 100644 --- a/py/moq-rs/moq/video.py +++ b/py/moq-rs/moq/video.py @@ -27,7 +27,6 @@ from .types import VideoEncoderInput as EncoderInput from .types import VideoEncoderKind as EncoderKind from .types import VideoEncoderOutput as EncoderOutput -from .types import VideoGop as Gop from .types import VideoPixelFormat as PixelFormat Read = VideoDecodedFrame | Gap | End @@ -132,7 +131,7 @@ def write(self, frame: Frame) -> None: def cut(self) -> None: """Start a new group at the next written frame. - Optional: the encoder starts groups on its own per ``output.gop``. Raises + Optional: the encoder starts groups on its own per ``output.join``. Raises if the selected encoder cannot force a keyframe; groups keep their interval. """ self._inner.cut() @@ -229,7 +228,6 @@ def finish(self) -> list[Packet]: "End", "Frame", "Gap", - "Gop", "Packet", "PixelFormat", "Producer", diff --git a/quest/m1/api-audit/README.md b/quest/m1/api-audit/README.md index f3970e1fa02..1faf0a9fb8e 100644 --- a/quest/m1/api-audit/README.md +++ b/quest/m1/api-audit/README.md @@ -75,7 +75,3 @@ Rejected in planning: language's reserved names. - A follow policy enum folding `linger` and `stitch`: one name per meaning (the export wait is `--follow`) fixes the confusion more cheaply. - -## Required - -- [Encoders take a join](/quest/m1/api-audit/join.md) - video's `keyframeInterval`/`Gop`, audio's `groupDuration`, and the C hints become one `join` time in JS, Rust, FFI, and C diff --git a/quest/m1/api-audit/join.md b/quest/m1/api-audit/join.md deleted file mode 100644 index ea5d00439fc..00000000000 --- a/quest/m1/api-audit/join.md +++ /dev/null @@ -1,93 +0,0 @@ -# [M] Encoders take a join - -## Goal - -Every publisher-facing knob that spaces a track's join points is a time -called `join`, after the catalog's `media.join` it feeds, in every language -and for both media: - -- `@moq/publish`: `Video.Encoder`'s `keyframeInterval` and `Audio.Encoder`'s - `groupDuration` (and `el.audio.groupDuration`) become `join`, and - `Rendition.joinInterval` becomes `Rendition.join`. -- `moq-video`: `encode::Config::gop: Gop` becomes `join: Duration` - (default 2 s), and the `Gop` enum is deleted. -- `moq-audio`: `encode::Options::group_duration` becomes `join`. -- `moq-ffi`: `MoqVideoGop` is deleted, and the video encoder output takes - `join: Option`. -- `moq-c`: the video output's `gop` frame count becomes `join_us` (0 keeps - the default), and the video and audio hints' `group_duration_us` / - `has_group_duration` become `join_us` / `has_join`. - -`join` means the time between join points at every level of `@moq/publish`: -an encoder's config `join` is the requested value, `Rendition.join` is the -effective value the encoder holds to, and `Broadcast.join` stays the longest -across renditions, published as `media.join.interval`. - -Defaults stay the same: video still forces an earlier keyframe for a joiner, -and audio still opens a group on the first frame at least `join` past the -last one. Zero changes: every language refuses a `join` of zero (below). -Every rename is a breaking change on `main`, with a migration note per -package in the PR's Impact section. `release` keeps the old names. - -Non-goals: the catalog's own `media.join.interval` field, and -timestamp-driven video joins (the encoder still converts `join` to a frame -count at the configured framerate). - -## Plan - -Decided 2026-11-06 (maintainer, /quest-complete while queueing -[moq-dev/moq#7279](https://github.com/moq-dev/moq/pull/7279)): - -- ✅ **The name is `join`** (`join_us` in C). Audio codecs have no keyframes - and encoders have no groups, but both knobs set how often a viewer can - join, which the catalog already calls `media.join`. "join > keyframe." - The maintainer avoids compound words, and "Interval" reads like a unit. - Its doc comment must say it is the time between join points, since a bare - `join` can read like a method or a boolean. Not chosen: `joinInterval`, - `keyframeInterval` for audio too, `groupInterval`, `interval`, `cadence` - (collides with capture cadence), `period`. -- ✅ **Scope covers video and the C hints**, not only the audio encoder - knobs. Not chosen: audio encoder knobs only; dropping the rename. -- ✅ **Rust video takes a plain `Duration`** (shape A). The encoder converts - it to frames internally, as `Gop::keyframe_every` does today, and - `Gop::duration` goes with the enum, since the catalog reads `join` - directly. Not chosen: a non-exhaustive `Join` enum to keep an extension - point; placing joins by frame timestamp (needs every backend to honor - forced IDRs and an open codec GOP); a frame count in Rust, FFI, and C. -- ✅ Milestone m1, in this epic, since it makes names mirror across Rust, JS, - and the bindings. - -Decided 2026-11-06 (maintainer, /quest-complete, on Pullfrog's review of -[moq-dev/moq#8036](https://github.com/moq-dev/moq/pull/8036)): - -- ✅ **One word at three levels**: `Rendition.joinInterval` becomes - `Rendition.join` too, so the requested, effective, and broadcast-wide - values share one name and meaning. Not chosen: keeping `joinInterval`; - a different name for the encoder knob. -- ✅ **Zero is refused everywhere.** Today JS video keyframes every frame at - `keyframeInterval: 0`, audio opens a group per frame at zero, and Rust video - and FFI already refuse it. Each encoder now rejects a zero `join` at - configuration, which changes behavior for JS and audio callers passing 0; - the PR's Impact section says so. C's zero-initialized video config - keeps 0 as "the default", since C has no `Option`; the C hints already - require a nonzero `join_us` when `has_join` is set. Not chosen: zero means - every frame everywhere; leaving each language's zero as is. - -A positive `join` shorter than one frame rounds up to one frame, as -`Gop::keyframe_every` does today. - -The catalog advertises the effective `join`, not the requested one, since -`media.join.interval` promises an upper bound and sets the HLS/DASH segment -grid. The video encoder keeps an internal effective duration after rounding -to frames (what `Gop::duration` supplies today): the default 2 s at -30000/1001 fps is 60 frames, or 2002 ms, and a 1 ms `join` at 30 fps is one -frame, about 33.3 ms. That effective value is what `Rendition.join` and the -Rust catalog read. Regression tests cover a fractional frame rate and a -positive sub-frame `join`. - -Update `doc/lib/js/publish.md`, `doc/lib/c/index.md`, `doc/lib/rs/moq-audio.md`, -`rs/moq-video/README.md`, and every caller (`moq-cli`, `moq-transcode`, the -backends, the FFI and C wrappers and their language bindings). - -Public API: breaking in `@moq/publish`, `moq-video`, `moq-audio`, `moq-ffi` -and its bindings, and the `moq-c` ABI. Wire: none. diff --git a/rs/moq-audio/src/encode/producer.rs b/rs/moq-audio/src/encode/producer.rs index e06b4881870..1943c94941d 100644 --- a/rs/moq-audio/src/encode/producer.rs +++ b/rs/moq-audio/src/encode/producer.rs @@ -39,10 +39,11 @@ pub struct Options { /// can't at all, and at `hang`'s priorities audio outranks video, so it is only /// ever squeezed on a link that can't carry audio alone. pub bandwidth: moq_net::bandwidth::Allocator, - /// The minimum audio carried by each group. The packet that reaches it closes - /// the group, and the next packet opens a new one. + /// The time between join points: the minimum audio carried by each group. The + /// packet that reaches it closes the group, and the next packet opens a new one, + /// so a positive join shorter than a packet puts every packet in its own group. /// - /// Defaults to 20 ms; zero puts every packet in its own group. Each group costs the relay a stream + /// Defaults to 20 ms; [`Producer::new`] refuses zero. Each group costs the relay a stream /// and its bookkeeping, so raising this trades per-group overhead for coarser /// loss: a viewer that falls behind skips a whole group, and a lost packet /// stalls the rest of its group until retransmitted. Packets still forward as @@ -51,7 +52,7 @@ pub struct Options { /// per group, at the cost of encoder latency. An unfinished group remains open /// across a pause or [`Producer::reanchor`] until the next write; use /// [`Producer::discontinuity`] to close it when capture stops. - pub group_duration: std::time::Duration, + pub join: std::time::Duration, } impl Default for Options { @@ -60,7 +61,7 @@ impl Default for Options { track: None, config: Config::default(), bandwidth: moq_net::bandwidth::Allocator::unlimited(), - group_duration: std::time::Duration::from_millis(20), + join: std::time::Duration::from_millis(20), } } } @@ -131,6 +132,9 @@ impl Reserved { catalog: moq_mux::catalog::Producer, options: &Options, ) -> Result { + if options.join.is_zero() { + return Err(Error::Unsupported("a join of zero opens no group".to_owned())); + } let track = match &options.track { // The catalog's info carries the microsecond timescale audio hang frames stamp, so // Lite05 subscribers know what scale to expect and the model layer accepts @@ -185,7 +189,7 @@ impl Reserved { } self.track.set(catalog)?; - let group_samples = options.group_duration.as_nanos() * u128::from(encoder.codec_rate()); + let group_samples = options.join.as_nanos() * u128::from(encoder.codec_rate()); let group_samples = u64::try_from(group_samples.div_ceil(1_000_000_000)).unwrap_or(u64::MAX); Ok(Registered { @@ -1199,7 +1203,7 @@ mod tests { /// Terminal Opus lookahead samples survive both exact-frame and partial-frame input. #[tokio::test] async fn finish_publishes_the_opus_lookahead_tail() { - for (group_ms, frames) in [(0, 960), (0, 860), (100, 2_880), (100, 2_780)] { + for (group_ms, frames) in [(1, 960), (1, 860), (100, 2_880), (100, 2_780)] { let input = Input { format: Format::F32, sample_rate: 48_000, @@ -1207,7 +1211,7 @@ mod tests { }; let options = Options { track: Some("audio".to_string()), - group_duration: Duration::from_millis(group_ms), + join: Duration::from_millis(group_ms), config: Config { layout: Layout::Mono, bitrate: Some(moq_net::bandwidth::Rate::from_bps(128_000)), @@ -1622,11 +1626,28 @@ mod tests { pts } - /// The packet that reaches the group duration closes its group, so a minimum that - /// isn't a whole number of packets rounds up. + /// A join of zero would open a group per packet, so the producer refuses it + /// before advertising a track that would never carry the requested layout. + #[tokio::test] + async fn a_zero_join_is_refused() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let options = Options { + join: Duration::ZERO, + ..Options::default() + }; + let Err(err) = Producer::new(&mut broadcast, catalog.clone(), Input::default(), &options) else { + panic!("a zero join cannot open"); + }; + assert!(matches!(err, Error::Unsupported(_)), "unexpected error: {err:?}"); + assert!(catalog.snapshot().audio.renditions.is_empty()); + } + + /// The packet that reaches the join closes its group, so a join that isn't a + /// whole number of packets rounds up, and one shorter than a packet groups each. #[tokio::test] - async fn group_duration_packs_packets_per_group() { - for (group_ms, expected) in [(0, [1, 1]), (30, [2, 2]), (100, [5, 5])] { + async fn join_packs_packets_per_group() { + for (group_ms, expected) in [(1, [1, 1]), (30, [2, 2]), (100, [5, 5])] { let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); let consumer = broadcast.consume(); @@ -1637,7 +1658,7 @@ mod tests { }; let options = Options { track: Some("audio".to_string()), - group_duration: Duration::from_millis(group_ms), + join: Duration::from_millis(group_ms), ..Options::default() }; let mut producer = Producer::new(&mut broadcast, catalog, input, &options).unwrap(); @@ -1717,7 +1738,7 @@ mod tests { /// Timeline breaks close a partial group and reset the resumed group's packet count. #[tokio::test] - async fn group_duration_restarts_after_a_timeline_break() { + async fn join_restarts_after_a_timeline_break() { for deferred in [false, true] { let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); @@ -1729,7 +1750,7 @@ mod tests { }; let options = Options { track: Some("audio".to_string()), - group_duration: Duration::from_millis(100), + join: Duration::from_millis(100), ..Options::default() }; let mut producer = Producer::new(&mut broadcast, catalog, input, &options).unwrap(); diff --git a/rs/moq-boy/src/video.rs b/rs/moq-boy/src/video.rs index b8ac1739bf4..93328f1fc16 100644 --- a/rs/moq-boy/src/video.rs +++ b/rs/moq-boy/src/video.rs @@ -57,7 +57,7 @@ impl VideoEncoder { // Probed from a throwaway encoder, before the emulator has run a frame, so a // viewer can discover (and unpause) a moq-boy that has published nothing. let rendition = config.probe().await.expect("failed to probe the H.264 encoder"); - let reserved = catalog.reserve().with_interval(config.gop.duration(config.framerate)); + let reserved = catalog.reserve().with_interval(config.effective_join()); let producer = moq_video::encode::Producer::new(broadcast, reserved, rendition).expect("failed to create avc3 producer"); let demand = producer.demand(); diff --git a/rs/moq-c/src/api.rs b/rs/moq-c/src/api.rs index d8d438b7c50..ad6e7c37c23 100644 --- a/rs/moq-c/src/api.rs +++ b/rs/moq-c/src/api.rs @@ -195,14 +195,14 @@ pub struct moq_audio_hint { /// Whether `bitrate` is present. pub has_bitrate: bool, - /// The longest span between two group starts on this track, in - /// microseconds, when `has_group_duration` is true: a video GOP, or the span - /// between the audio cuts you make. An upper bound, so a broadcast may take - /// the largest across its tracks. Must be at least 1 and at most + /// The time between join points, the longest span between two group starts + /// on this track, in microseconds, when `has_join` is true: a video GOP, or + /// the span between the audio cuts you make. An upper bound, so a broadcast + /// may take the largest across its tracks. Must be at least 1 and at most /// 2^53 - 1 milliseconds. - pub group_duration_us: u64, - /// Whether `group_duration_us` is present. - pub has_group_duration: bool, + pub join_us: u64, + /// Whether `join_us` is present. + pub has_join: bool, } impl moq_audio_hint { @@ -212,26 +212,23 @@ impl moq_audio_hint { Ok(()) } - /// The supplied group duration, for the broadcast's join interval. - fn group_duration(&self) -> Result, Error> { - group_duration(self.has_group_duration, self.group_duration_us) + /// The supplied join, for the broadcast's join interval. + fn join(&self) -> Result, Error> { + join(self.has_join, self.join_us) } } -/// The longest group duration a hint may give: 2^53 - 1 milliseconds, the largest -/// integer a JavaScript reader of the catalog holds exactly. -const MAX_GROUP_DURATION_US: u64 = ((1 << 53) - 1) * 1000; +/// The longest join a hint may give: 2^53 - 1 milliseconds, the largest integer a +/// JavaScript reader of the catalog holds exactly. +const MAX_JOIN_US: u64 = ((1 << 53) - 1) * 1000; -/// A supplied group duration, `None` when absent, or an error for 0 or past -/// [MAX_GROUP_DURATION_US]. -fn group_duration(present: bool, us: u64) -> Result, Error> { +/// A supplied join, `None` when absent, or an error for 0 or past [MAX_JOIN_US]. +fn join(present: bool, us: u64) -> Result, Error> { if !present { return Ok(None); } - if us == 0 || us > MAX_GROUP_DURATION_US { - return Err(Error::InvalidConfig(format!( - "group_duration_us must be 1..={MAX_GROUP_DURATION_US}, got {us}" - ))); + if us == 0 || us > MAX_JOIN_US { + return Err(Error::InvalidConfig(format!("join_us must be 1..={MAX_JOIN_US}, got {us}"))); } Ok(Some(std::time::Duration::from_micros(us))) } @@ -305,14 +302,14 @@ pub struct moq_video_hint { /// Whether `optimize_for_latency` is present. pub has_optimize_for_latency: bool, - /// The longest span between two group starts on this track, in - /// microseconds, when `has_group_duration` is true: a video GOP, or the span - /// between the audio cuts you make. An upper bound, so a broadcast may take - /// the largest across its tracks. Must be at least 1 and at most + /// The time between join points, the longest span between two group starts + /// on this track, in microseconds, when `has_join` is true: a video GOP, or + /// the span between the audio cuts you make. An upper bound, so a broadcast + /// may take the largest across its tracks. Must be at least 1 and at most /// 2^53 - 1 milliseconds. - pub group_duration_us: u64, - /// Whether `group_duration_us` is present. - pub has_group_duration: bool, + pub join_us: u64, + /// Whether `join_us` is present. + pub has_join: bool, } impl moq_video_hint { @@ -333,9 +330,9 @@ impl moq_video_hint { Ok(out) } - /// The supplied group duration, for the broadcast's join interval. - fn group_duration(&self) -> Result, Error> { - group_duration(self.has_group_duration, self.group_duration_us) + /// The supplied join, for the broadcast's join interval. + fn join(&self) -> Result, Error> { + join(self.has_join, self.join_us) } } @@ -2646,7 +2643,7 @@ unsafe fn parse_audio_init( let mut audio = moq_mux::import::AudioInit::new(audio_format_from_u32(config.format)?, init.to_vec()); audio.label = label.map(str::to_string); config.hint.apply(&mut audio)?; - Ok((audio, config.hint.group_duration()?)) + Ok((audio, config.hint.join()?)) } /// Publish one video codec as a new media track. @@ -2687,7 +2684,7 @@ unsafe fn parse_video_init( .media_video_priority(ffi::parse_id(config.lower)?)? .saturating_sub(1); } - Ok((video, config.hint.group_duration()?)) + Ok((video, config.hint.join()?)) } /// Publish a container, which demuxes and publishes its own tracks. diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index 1484f49aaf1..30c52a67fa8 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -5870,7 +5870,7 @@ fn video_raw_publish_consume() { let output = moq_video_encoder_output { codec: moq_video_codec::MOQ_VIDEO_CODEC_H264 as u32, bitrate: 0, - gop: 0, + join_us: 0, kind: moq_video_encoder_kind::MOQ_VIDEO_ENCODER_KIND_SOFTWARE as u32, encoder: std::ptr::null(), encoder_len: 0, @@ -6051,7 +6051,7 @@ fn decode_first_frame(output: &moq_video_decoder_output) -> (u32, u32, usize) { let encode_output = moq_video_encoder_output { codec: moq_video_codec::MOQ_VIDEO_CODEC_H264 as u32, bitrate: 0, - gop: 0, + join_us: 0, kind: moq_video_encoder_kind::MOQ_VIDEO_ENCODER_KIND_SOFTWARE as u32, encoder: std::ptr::null(), encoder_len: 0, @@ -6170,7 +6170,7 @@ fn video_raw_publish_from_many_threads() { // An explicit ceiling so the retunes below stay under the rate the encoder // opened at, which openh264 requires. bitrate: 1_000_000, - gop: 0, + join_us: 0, kind: moq_video_encoder_kind::MOQ_VIDEO_ENCODER_KIND_SOFTWARE as u32, encoder: std::ptr::null(), encoder_len: 0, @@ -6242,7 +6242,7 @@ fn a_stalled_encode_does_not_block_unrelated_calls() { let output = moq_video_encoder_output { codec: moq_video_codec::MOQ_VIDEO_CODEC_H264 as u32, bitrate: 0, - gop: 0, + join_us: 0, kind: moq_video_encoder_kind::MOQ_VIDEO_ENCODER_KIND_SOFTWARE as u32, encoder: std::ptr::null(), encoder_len: 0, @@ -6321,7 +6321,7 @@ fn closing_a_broadcast_does_not_block_unrelated_calls_behind_its_encoder() { let output = moq_video_encoder_output { codec: moq_video_codec::MOQ_VIDEO_CODEC_H264 as u32, bitrate: 0, - gop: 0, + join_us: 0, kind: moq_video_encoder_kind::MOQ_VIDEO_ENCODER_KIND_SOFTWARE as u32, encoder: std::ptr::null(), encoder_len: 0, @@ -6375,7 +6375,7 @@ fn video_raw_publish_rejects_frame_size_mismatch() { let output = moq_video_encoder_output { codec: moq_video_codec::MOQ_VIDEO_CODEC_H264 as u32, bitrate: 0, - gop: 0, + join_us: 0, kind: moq_video_encoder_kind::MOQ_VIDEO_ENCODER_KIND_SOFTWARE as u32, encoder: std::ptr::null(), encoder_len: 0, @@ -6416,7 +6416,7 @@ fn video_raw_publish_rejects_invalid_config() { let valid_output = moq_video_encoder_output { codec: moq_video_codec::MOQ_VIDEO_CODEC_H264 as u32, bitrate: 0, - gop: 0, + join_us: 0, kind: moq_video_encoder_kind::MOQ_VIDEO_ENCODER_KIND_SOFTWARE as u32, encoder: std::ptr::null(), encoder_len: 0, @@ -7508,7 +7508,7 @@ fn encode_video_bitrate_caps_the_reservation() { let output = moq_video_encoder_output { codec: moq_video_codec::MOQ_VIDEO_CODEC_H264 as u32, bitrate: 4_000_000, - gop: 0, + join_us: 0, kind: moq_video_encoder_kind::MOQ_VIDEO_ENCODER_KIND_SOFTWARE as u32, encoder: std::ptr::null(), encoder_len: 0, @@ -7566,7 +7566,7 @@ fn encode_video_rung_needs_a_grant_and_a_band() { let output = |bitrate| moq_video_encoder_output { codec: moq_video_codec::MOQ_VIDEO_CODEC_H264 as u32, bitrate, - gop: 0, + join_us: 0, kind: moq_video_encoder_kind::MOQ_VIDEO_ENCODER_KIND_SOFTWARE as u32, encoder: std::ptr::null(), encoder_len: 0, @@ -7609,7 +7609,7 @@ fn encode_video_ceiling_wakes_a_rung_above() { let output = moq_video_encoder_output { codec: moq_video_codec::MOQ_VIDEO_CODEC_H264 as u32, bitrate: 500_000, - gop: 0, + join_us: 0, kind: moq_video_encoder_kind::MOQ_VIDEO_ENCODER_KIND_SOFTWARE as u32, encoder: std::ptr::null(), encoder_len: 0, @@ -8286,8 +8286,8 @@ fn track_request_media_carries_hints() { hint: moq_audio_hint { bitrate: 96_000, has_bitrate: true, - group_duration_us: 100_000, - has_group_duration: true, + join_us: 100_000, + has_join: true, }, }; let audio = id(unsafe { moq_track_request_audio(request, &audio_config) }); @@ -8306,8 +8306,8 @@ fn track_request_media_carries_hints() { hint: moq_video_hint { bitrate: 2_000_000, has_bitrate: true, - group_duration_us: 3_000_000, - has_group_duration: true, + join_us: 3_000_000, + has_join: true, ..moq_video_hint::default() }, }; @@ -8331,7 +8331,7 @@ fn track_request_media_carries_hints() { assert_eq!(moq_origin_close(origin), 0); } -/// A hint flag set with a 0 value, or a group duration past what milliseconds hold for a +/// A hint flag set with a 0 value, or a join past what milliseconds hold for a /// JavaScript reader, is an invalid config rather than a second encoding of "absent". #[test] fn invalid_hints_are_refused() { @@ -8374,28 +8374,28 @@ fn invalid_hints_are_refused() { ); assert_eq!( audio(moq_audio_hint { - has_group_duration: true, + has_join: true, ..moq_audio_hint::default() }), invalid, - "audio group duration 0" + "audio join 0" ); assert_eq!( audio(moq_audio_hint { - group_duration_us: u64::MAX, - has_group_duration: true, + join_us: u64::MAX, + has_join: true, ..moq_audio_hint::default() }), invalid, - "audio group duration past milliseconds" + "audio join past milliseconds" ); assert_eq!( video(moq_video_hint { - has_group_duration: true, + has_join: true, ..moq_video_hint::default() }), invalid, - "video group duration 0" + "video join 0" ); assert_eq!( video(moq_video_hint { @@ -8413,10 +8413,10 @@ fn invalid_hints_are_refused() { assert_eq!(moq_origin_close(origin), 0); } -/// A group duration hint is the track's join interval, declared once its first window closes, +/// A join hint is the track's join interval, declared once its first window closes, /// rather than the shorter groups it measured. #[test] -fn group_duration_hint_declares_the_join_interval() { +fn join_hint_declares_the_join_interval() { let origin = id(moq_origin_create()); let broadcast = publish_broadcast(origin, b"group-duration"); let init = opus_head(); @@ -8427,8 +8427,8 @@ fn group_duration_hint_declares_the_join_interval() { label: std::ptr::null(), label_len: 0, hint: moq_audio_hint { - group_duration_us: 500_000, - has_group_duration: true, + join_us: 500_000, + has_join: true, ..moq_audio_hint::default() }, }; diff --git a/rs/moq-c/src/video.rs b/rs/moq-c/src/video.rs index 2b5548bcfb4..f6e230e4e15 100644 --- a/rs/moq-c/src/video.rs +++ b/rs/moq-c/src/video.rs @@ -93,8 +93,8 @@ pub struct moq_video_encoder_input { pub width: u32, /// Encoded height in pixels. Must be even. pub height: u32, - /// Nominal frames per second, used for the codec time base and the default - /// bitrate and keyframe interval. Must be non-zero. + /// Nominal frames per second, used for the codec time base, the default + /// bitrate, and rounding the join to whole frames. Must be non-zero. pub framerate: moq_video_frame_rate, } @@ -107,9 +107,10 @@ pub struct moq_video_encoder_output { /// Target bitrate in bits per second. 0 derives one from the resolution and /// framerate. pub bitrate: u64, - /// Keyframe interval in frames: a subscriber joining mid-stream waits at - /// most this many frames before it can decode. 0 uses ~2 seconds. - pub gop: u32, + /// The time between join points, in microseconds: a keyframe opens a group + /// this often, rounded to whole frames, so a subscriber joining mid-stream + /// waits at most this long before it can decode. 0 uses 2 seconds. + pub join_us: u64, /// `moq_video_encoder_kind` discriminant. pub kind: u32, /// Backend name, UTF-8, e.g. `"videotoolbox"`, `"nvenc"`, `"mediafoundation"`, @@ -693,7 +694,7 @@ impl Video { let reserved = target .catalog .reserve() - .with_interval(config.gop.duration(config.framerate)); + .with_interval(config.effective_join()); let ceiling = video_ceiling(config, &rendition); let (lower, priority) = match lower { Some(lower) => { @@ -972,12 +973,10 @@ pub unsafe extern "C" fn moq_encode_video( config.codec = codec_from_u32(raw_output.codec)?; config.kind = unsafe { encoder_kind(raw_output)? }; // The C ABI spells an unset knob as 0, which neither field accepts as a real - // value: a zero bitrate or GOP is the default, not a request. + // value: a zero bitrate or join is the default, not a request. config.bitrate = (raw_output.bitrate != 0).then(|| moq_net::bandwidth::Rate::from_bps(raw_output.bitrate)); - if raw_output.gop != 0 { - config.gop = moq_video::encode::Gop::Keyframe { - interval: raw_output.gop, - }; + if raw_output.join_us != 0 { + config.join = std::time::Duration::from_micros(raw_output.join_us); } // Both before the global lock is taken: bringing up a hardware encoder is slow @@ -1082,8 +1081,8 @@ pub unsafe extern "C" fn moq_encode_video_frame(producer: u32, frame: *const moq /// Cut a new group at the next published frame. /// -/// Optional. The encoder already keyframes every `moq_video_encoder_output.gop` -/// frames, and each of those cuts a group, so a subscriber can always join +/// Optional. The encoder already keyframes every `moq_video_encoder_output.join_us`, +/// and each of those cuts a group, so a subscriber can always join /// without you calling this. Reach for it only to place the boundaries yourself: /// aligning groups with something the encoder cannot see, such as a scene change, /// a source switch, or resuming after an idle gap. @@ -1094,7 +1093,7 @@ pub unsafe extern "C" fn moq_encode_video_frame(producer: u32, frame: *const moq /// /// Fails when the selected encoder cannot force a keyframe (a V4L2 driver /// without the control): nothing is queued, and groups keep falling every -/// `gop` frames. +/// `join_us`. #[unsafe(no_mangle)] pub extern "C" fn moq_encode_video_cut(producer: u32) -> i32 { ffi::enter(move || { @@ -1315,7 +1314,7 @@ mod tests { // Probed rather than hand-built, so the test track carries what a real one would. let encode = moq_video::encode::Config::new(320, 240, moq_video::FrameRate::new(30, 1).unwrap()); let rendition = encode.probe().await.unwrap(); - let reserved = catalog.reserve().with_interval(encode.gop.duration(encode.framerate)); + let reserved = catalog.reserve().with_interval(encode.effective_join()); let producer = moq_video::encode::Producer::new(broadcast, reserved, rendition).unwrap(); let name = producer.demand().name().to_string(); diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 46ea998d8df..402a8bd2eca 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -2217,7 +2217,7 @@ mod tests { let mut producer = moq_mux::container::Producer::new(track, Container::Legacy(moq_mux::container::Kind::Data)); let mut config = moq_video::encode::Config::new(64, 64, moq_video::FrameRate::new(30, 1).unwrap()); config.kind = moq_video::encode::Kind::Software; - config.gop = moq_video::encode::Gop::Keyframe { interval: 120 }; + config.join = Duration::from_secs(4); let mut encoder = moq_video::encode::Encoder::new(&config).unwrap(); for index in 0..=60 { let surface = moq_video::Surface::rgba(&vec![128; 64 * 64 * 4], moq_video::Size::new(64, 64)).unwrap(); diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index 09467397c9f..9971fff550e 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -4021,7 +4021,7 @@ async fn video_raw_publish_consume() { output: MoqVideoEncoderOutput { codec: MoqVideoCodec::H264, bitrate: None, - gop: None, + join: None, // Software so the test is deterministic everywhere: `Auto` would // reach for a hardware backend that CI runners don't have. kind: MoqVideoEncoderKind::Software, @@ -4147,7 +4147,7 @@ async fn video_decode_frame_ownership() { output: MoqVideoEncoderOutput { codec: MoqVideoCodec::H264, bitrate: None, - gop: None, + join: None, // Software so the encode is deterministic everywhere. kind: MoqVideoEncoderKind::Software, }, @@ -4298,7 +4298,7 @@ async fn video_decode_start() { output: MoqVideoEncoderOutput { codec: MoqVideoCodec::H264, bitrate: None, - gop: None, + join: None, kind: MoqVideoEncoderKind::Software, }, track: Some("camera".into()), @@ -4421,7 +4421,7 @@ async fn video_raw_publish_from_many_threads() { // An explicit ceiling so the retunes below stay under the rate the // encoder opened at, which openh264 requires. bitrate: Some(1_000_000), - gop: None, + join: None, kind: MoqVideoEncoderKind::Software, }, track: None, @@ -4499,7 +4499,7 @@ async fn video_raw_publish_rejects_bad_frames() { let output = || MoqVideoEncoderOutput { codec: MoqVideoCodec::H264, bitrate: None, - gop: None, + join: None, kind: MoqVideoEncoderKind::Software, }; @@ -4524,6 +4524,24 @@ async fn video_raw_publish_rejects_bad_frames() { .is_err() ); + // So is a zero join, which would open no group. + assert!( + crate::video::MoqVideoProducer::new( + &crate::media::MoqMediaCatalogProducer::new(broadcast.clone()).unwrap(), + crate::video::MoqVideoProducerOptions { + input: input(320, 240), + output: MoqVideoEncoderOutput { + join: Some(std::time::Duration::ZERO), + ..output() + }, + track: None, + bandwidth: None, + lower: None, + } + ) + .is_err() + ); + let video = crate::video::MoqVideoProducer::new( &crate::media::MoqMediaCatalogProducer::new(broadcast.clone()).unwrap(), crate::video::MoqVideoProducerOptions { @@ -7504,7 +7522,7 @@ async fn video_encoder_follows_a_shrinking_grant() { output: MoqVideoEncoderOutput { codec: MoqVideoCodec::H264, bitrate: Some(4_000_000), - gop: None, + join: None, kind: MoqVideoEncoderKind::Software, }, track: Some("camera".into()), @@ -7563,7 +7581,7 @@ fn software_camera(bitrate: u64) -> (crate::video::MoqVideoEncoderInput, crate:: MoqVideoEncoderOutput { codec: MoqVideoCodec::H264, bitrate: Some(bitrate), - gop: None, + join: None, kind: MoqVideoEncoderKind::Software, }, ) diff --git a/rs/moq-ffi/src/video.rs b/rs/moq-ffi/src/video.rs index 4381221c68e..2a6945dd17a 100644 --- a/rs/moq-ffi/src/video.rs +++ b/rs/moq-ffi/src/video.rs @@ -118,8 +118,8 @@ pub struct MoqVideoEncoderInput { pub width: u32, /// Encoded height in pixels. Must be even. pub height: u32, - /// Nominal frames per second, used for the codec time base and the default - /// bitrate and keyframe interval. Must be non-zero. + /// Nominal frames per second, used for the codec time base, the default + /// bitrate, and rounding the join to whole frames. Must be non-zero. pub framerate: MoqVideoFrameRate, } @@ -132,9 +132,11 @@ pub struct MoqVideoEncoderOutput { /// and framerate. #[uniffi(default = None)] pub bitrate: Option, - /// How encoded pictures are divided into independently decodable groups. + /// The time between join points: a keyframe opens a group this often, + /// rounded to whole frames. `None` keyframes every two seconds; zero is + /// refused. #[uniffi(default = None)] - pub gop: Option, + pub join: Option, /// Encoder implementation preference. Pass /// [`MoqVideoEncoderKind::Auto`] unless you need a specific backend. pub kind: MoqVideoEncoderKind, @@ -164,24 +166,6 @@ pub struct MoqVideoProducerOptions { pub lower: Option>, } -/// How the encoded stream is divided into independently decodable groups. -#[derive(Clone, Copy, uniffi::Enum)] -pub enum MoqVideoGop { - /// Start a group with a keyframe every `interval` frames. - Keyframe { - /// The number of frames between keyframes. - interval: u32, - }, -} - -impl From for moq_video::encode::Gop { - fn from(gop: MoqVideoGop) -> Self { - match gop { - MoqVideoGop::Keyframe { interval } => Self::Keyframe { interval }, - } - } -} - /// One raw video frame: pixels plus a presentation timestamp. /// /// The pixel format and resolution are fixed by [`MoqVideoEncoderInput`] at @@ -226,8 +210,8 @@ fn video_encoder_config( config.codec = output.codec.into(); config.kind = output.kind.clone().into(); config.bitrate = output.bitrate.map(moq_net::bandwidth::Rate::from_bps); - if let Some(gop) = output.gop { - config.gop = gop.into(); + if let Some(join) = output.join { + config.join = join; } Ok(config) } @@ -626,7 +610,7 @@ impl MoqVideoProducer { /// Cut a new group at the next written frame. /// /// Optional. The encoder already keyframes every - /// [`gop`](MoqVideoEncoderOutput::gop) frames, and each of those cuts a + /// [`join`](MoqVideoEncoderOutput::join), and each of those cuts a /// group, so a subscriber can always join without you calling this. Reach for /// it only to place the boundaries yourself: aligning groups with something /// the encoder cannot see, such as a scene change, a source switch, or @@ -785,7 +769,7 @@ impl MoqVideoProducer { // closes its encoder before this one opens, so only one codec session is live. let rendition = block_on(config.probe())?; let encoder = block_on(moq_video::encode::Sink::open(&config))?; - let gop = config.gop.duration(config.framerate); + let join = config.effective_join(); let ceiling = video_ceiling(&config, &rendition); if let Some((_, below, _)) = &lower && *below >= ceiling @@ -818,7 +802,7 @@ impl MoqVideoProducer { }; Ok(moq_video::encode::Producer::with_track( track, - state.catalog.reserve().with_interval(gop), + state.catalog.reserve().with_interval(join), rendition, )?) })?; @@ -1408,7 +1392,7 @@ mod finish_tests { let output = MoqVideoEncoderOutput { codec: MoqVideoCodec::H264, bitrate: None, - gop: None, + join: None, kind: MoqVideoEncoderKind::Software, }; for (numerator, denominator) in [(30_000, 1001), (60_000, 2002), (24_000, 1001)] { @@ -1462,7 +1446,7 @@ mod finish_tests { MoqVideoEncoderOutput { codec: MoqVideoCodec::H264, bitrate: Some(500_000), - gop: None, + join: None, kind: MoqVideoEncoderKind::Software, }, ) @@ -1513,7 +1497,7 @@ mod finish_tests { let output = MoqVideoEncoderOutput { codec: MoqVideoCodec::H264, bitrate: Some(4_000_000), - gop: None, + join: None, kind: MoqVideoEncoderKind::Software, }; let video = crate::video::MoqVideoProducer::new( @@ -1707,7 +1691,7 @@ mod rung_tests { MoqVideoEncoderOutput { codec: MoqVideoCodec::H264, bitrate: Some(bitrate), - gop: None, + join: None, kind: MoqVideoEncoderKind::Software, } } diff --git a/rs/moq-transcode/src/rung.rs b/rs/moq-transcode/src/rung.rs index 098f608ce7a..779f463fd6b 100644 --- a/rs/moq-transcode/src/rung.rs +++ b/rs/moq-transcode/src/rung.rs @@ -126,9 +126,9 @@ impl Rung { config.kind = self.encoder.clone(); config.color = Some(color); config.resize = Some(self.resize); - // Every source group boundary is a cut; the keyframe interval is only a - // backstop against pathologically long source groups. - config.gop = moq_video::encode::Gop::keyframe_every(std::time::Duration::from_secs(8), framerate); + // Every source group boundary is a cut; the join is only a backstop + // against pathologically long source groups. + config.join = std::time::Duration::from_secs(8); Ok(moq_video::encode::Sink::open(&config).await?) } } diff --git a/rs/moq-video/DESIGN-native-codecs.md b/rs/moq-video/DESIGN-native-codecs.md index 47b8f3096e8..14d5dec2022 100644 --- a/rs/moq-video/DESIGN-native-codecs.md +++ b/rs/moq-video/DESIGN-native-codecs.md @@ -344,8 +344,8 @@ Where the implementation differs from the plan above: publish time, so a buffering backend and the `finish()` tail stay in step. The separate `encode_rgba` / `encode_i420` entry points collapsed into `Surface::rgba` plus the single `Encoder::encode`. -- **Group boundaries are the encoder's, not the application's.** `Config::gop` - (a `Gop` enum, keyframes at an interval today) places them on its own and +- **Group boundaries are the encoder's, not the application's.** `Config::join` + (a `Duration` the encoder rounds to a keyframe every so many frames) places them on its own and `encode` takes no per-frame flag; `Encoder::cut()` opens one at the next frame for the callers that genuinely need a decodable starting point (a new output group), and refuses on a backend that cannot force one. diff --git a/rs/moq-video/README.md b/rs/moq-video/README.md index c38a4f02fb7..15565e6b400 100644 --- a/rs/moq-video/README.md +++ b/rs/moq-video/README.md @@ -108,21 +108,20 @@ registration directly. There is no software H.265 encoder (it's hardware-only). `encode::Encoder::encode` takes a raw `Frame` (a timestamp plus a `Surface` holding the pixels) and returns `encode::Encoded`s: one whole access unit each, carrying the timestamp of the picture it was encoded from and whether it is a -keyframe, forced by `cut()` or on the GOP cadence. The timestamp matters for a +keyframe, forced by `cut()` or on the join cadence. The timestamp matters for a backend that buffers, which hands back an earlier frame's access unit while a later one goes in, and for the tail `finish()` drains. Bring your own pixels with `Surface::rgba(...)`, or feed a frame straight from capture or `decode`. -Group boundaries are automatic: `Config::gop` says how the stream is divided -(`Gop::Keyframe { interval }` places a keyframe every so many frames, and an -interval of zero is refused at open), so an application never has to think -about them. `Encoder::cut()` opens a group at the next frame when something +Group boundaries are automatic: `Config::join` is the time between join points +(a keyframe every two seconds by default, rounded to whole frames, and a join of +zero is refused at open), so an application never has to think about them. `Encoder::cut()` opens a group at the next frame when something outside the encoder needs a decodable starting point there: a source group boundary, a scene change, a source switch. The request is held until a frame arrives, so it is safe to call before you have one. It fails with `Error::CutUnsupported` on a backend that cannot force a boundary (a V4L2 driver without the force-keyframe control), and queues nothing then: groups -keep falling where `Config::gop` puts them. `encode::Sink` answers the same +keep falling where `Config::join` puts them. `encode::Sink` answers the same way, awaited. Public entry points: diff --git a/rs/moq-video/src/capture/wgc.rs b/rs/moq-video/src/capture/wgc.rs index af4404a80e3..2961f7a36f0 100644 --- a/rs/moq-video/src/capture/wgc.rs +++ b/rs/moq-video/src/capture/wgc.rs @@ -318,7 +318,7 @@ mod tests { let rendition = config.probe().await.unwrap(); let container = moq_mux::catalog::hang::Container::try_from(&rendition).unwrap(); let mut subscriber = moq_mux::container::Consumer::new(subscriber, container); - let reserved = catalog.reserve().with_interval(config.gop.duration(config.framerate)); + let reserved = catalog.reserve().with_interval(config.effective_join()); let mut producer = Producer::with_track(track, reserved, rendition).unwrap(); assert!(producer.demand().wanted().is_some()); diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index e5e74cd05d0..124db880987 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -489,7 +489,7 @@ mod tests { let rendition = config.probe().await.unwrap(); let reserved = source_catalog .reserve() - .with_interval(config.gop.duration(config.framerate)); + .with_interval(config.effective_join()); let mut producer = EncodeProducer::new(source_broadcast, reserved, rendition).unwrap(); let mut encoder = Encoder::new(&config).unwrap(); let rgba = vec![0x80u8; 320 * 240 * 4]; diff --git a/rs/moq-video/src/encode/backend/mediacodec.rs b/rs/moq-video/src/encode/backend/mediacodec.rs index 85d49e1ff4d..7674a4841e1 100644 --- a/rs/moq-video/src/encode/backend/mediacodec.rs +++ b/rs/moq-video/src/encode/backend/mediacodec.rs @@ -44,7 +44,7 @@ use ndk::media::media_codec::{ use ndk::media::media_format::MediaFormat; use ndk::media_error::MediaError; -use super::super::encoder::{Applied, Codec, Config, Gop}; +use super::super::encoder::{Applied, Codec, Config}; use super::{Backend, Encoded}; use crate::color::{Matrix, Primaries, Range, Transfer}; use crate::{Error, Frame, I420}; @@ -447,8 +447,7 @@ fn encoder_format(config: &Config, mime: &str) -> Result { // MediaCodec takes the keyframe interval in seconds rather than frames, and // reads it as a float, so a sub-second GOP survives instead of rounding to // zero (which would mean an IDR on every frame). - let Gop::Keyframe { interval } = config.gop; - format.set_f32(KEY_I_FRAME_INTERVAL, interval as f32 / config.framerate.as_f64() as f32); + format.set_f32(KEY_I_FRAME_INTERVAL, config.effective_join().as_secs_f32()); // Ask for the shortest pipeline the device offers: output a frame after // input, and no B-frames, whose reorder delay a live track has no use for. // Both are hints an older device drops, which is why `flush` still drains the diff --git a/rs/moq-video/src/encode/backend/mediafoundation.rs b/rs/moq-video/src/encode/backend/mediafoundation.rs index 511a996734f..fa584b0bdd1 100644 --- a/rs/moq-video/src/encode/backend/mediafoundation.rs +++ b/rs/moq-video/src/encode/backend/mediafoundation.rs @@ -43,7 +43,7 @@ use windows::Win32::Media::MediaFoundation::{ use windows::Win32::System::Variant::{VARIANT, VT_BOOL, VT_UI4}; use windows::core::{GUID, Interface}; -use super::super::encoder::{Applied, Codec, Config, Gop}; +use super::super::encoder::{Applied, Codec, Config}; use super::{Backend, Encoded, keyframe_annexb}; use crate::color::{Matrix, Primaries, Range, Transfer}; use crate::frame::{Surface, interleave_uv}; @@ -141,10 +141,7 @@ impl MediaFoundation { height: config.height, framerate: config.framerate, bitrate: clamp_u32(config.resolved_bitrate().as_bps()), - gop: { - let Gop::Keyframe { interval } = config.gop; - interval - }, + gop: config.keyframe_interval(), color: config.resolved_color(), started: false, provides_samples: false, diff --git a/rs/moq-video/src/encode/backend/mod.rs b/rs/moq-video/src/encode/backend/mod.rs index 5f26a522e8d..bae8c8c45a2 100644 --- a/rs/moq-video/src/encode/backend/mod.rs +++ b/rs/moq-video/src/encode/backend/mod.rs @@ -51,7 +51,7 @@ mod vaapi; /// whether forced by `cut` or placed by the backend's own GOP cadence. pub(crate) trait Backend { /// Encode one frame, opening a group at it (an IDR) when `cut` is set. - /// Backends place group boundaries on their own per [`Config::gop`], so this + /// Backends place group boundaries on their own per [`Config::join`], so this /// is only the caller's extra request, arriving via /// [`Encoder::cut`](super::Encoder::cut), and only on a backend whose /// [`can_cut`](Self::can_cut) said yes. diff --git a/rs/moq-video/src/encode/backend/nvenc.rs b/rs/moq-video/src/encode/backend/nvenc.rs index ba60180ff1c..15978cedd2b 100644 --- a/rs/moq-video/src/encode/backend/nvenc.rs +++ b/rs/moq-video/src/encode/backend/nvenc.rs @@ -44,7 +44,7 @@ use moq_nvenc::sys::nvEncodeAPI::{ }; use moq_nvenc::{Encoder, EncoderInitParams, Session}; -use super::super::encoder::{Applied, Codec, Config, Gop, Preset}; +use super::super::encoder::{Applied, Codec, Config, Preset}; use super::{Backend, Encoded, keyframe_annexb}; use crate::color::{Matrix, Primaries, Range, Transfer}; use crate::frame::{Surface, cuda, interleave_uv, vulkan}; @@ -114,7 +114,7 @@ impl Nvenc { .map_err(|e| Error::Codec(anyhow::anyhow!("NVENC preset config: {e}")))?; let cfg = &mut preset.presetCfg; - let Gop::Keyframe { interval } = config.gop; + let interval = config.keyframe_interval(); cfg.gopLength = interval; cfg.frameIntervalP = 1; // no B-frames cfg.rcParams.rateControlMode = NV_ENC_PARAMS_RC_MODE::NV_ENC_PARAMS_RC_CBR; @@ -731,7 +731,7 @@ mod tests { } let mut config = crate::encode::Config::new(320, 240, crate::FrameRate::new(30, 1).unwrap()); config.kind = crate::encode::Kind::Named(NAME.into()); - config.gop = Gop::Keyframe { interval: 3 }; + config.join = std::time::Duration::from_millis(100); let Ok(mut encoder) = crate::encode::Encoder::new(&config) else { return; }; @@ -855,7 +855,7 @@ mod tests { let mut config = crate::encode::Config::new(w, h, crate::FrameRate::new(30, 1).unwrap()); config.kind = crate::encode::Kind::Named(NAME.into()); config.bitrate = Some(moq_net::bandwidth::Rate::from_bps(4_000_000)); - config.gop = Gop::Keyframe { interval: 30 }; + config.join = std::time::Duration::from_secs(1); let mut encoder = crate::encode::Encoder::new(&config).ok()?; let mut idrs = Vec::new(); diff --git a/rs/moq-video/src/encode/backend/openh264.rs b/rs/moq-video/src/encode/backend/openh264.rs index 7806a9833ab..83ffcaaa0db 100644 --- a/rs/moq-video/src/encode/backend/openh264.rs +++ b/rs/moq-video/src/encode/backend/openh264.rs @@ -14,7 +14,7 @@ use openh264_sys2::{ENCODER_OPTION_BITRATE, SBitrateInfo, SPATIAL_LAYER_ALL}; #[cfg(test)] use openh264_sys2::{ENCODER_OPTION_COMPLEXITY, LOW_COMPLEXITY, MEDIUM_COMPLEXITY}; -use super::super::encoder::{Applied, Config, Gop, Preset}; +use super::super::encoder::{Applied, Config, Preset}; use super::{Backend, Encoded}; use crate::color::{Matrix, Primaries, Range, Transfer}; use crate::{Error, Frame}; @@ -38,7 +38,7 @@ impl Openh264 { } fn new(config: &Config) -> Result { - let Gop::Keyframe { interval } = config.gop; + let interval = config.keyframe_interval(); let color = config.resolved_color(); // State the color space in the SPS so a decoder doesn't fall back to // guessing it from the frame height. The SDR curve goes out as BT.709 (1) @@ -368,9 +368,8 @@ mod tests { /// the flag agrees with the bitstream. #[test] fn keyframes_are_flagged_on_the_cadence_and_on_a_cut() { - let framerate = crate::FrameRate::new(30, 1).unwrap(); let config = Config { - gop: Gop::keyframe_every(std::time::Duration::from_secs(1), framerate), + join: std::time::Duration::from_secs(1), ..config() }; let mut enc = Openh264::new(&config).unwrap(); diff --git a/rs/moq-video/src/encode/backend/v4l2.rs b/rs/moq-video/src/encode/backend/v4l2.rs index 4efa14a4e43..42e11ffe1a9 100644 --- a/rs/moq-video/src/encode/backend/v4l2.rs +++ b/rs/moq-video/src/encode/backend/v4l2.rs @@ -64,7 +64,7 @@ use moq_v4l::sys::{ v4l2_mpeg_video_header_mode_V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME, }; -use super::super::encoder::{Applied, Codec, Config, Gop}; +use super::super::encoder::{Applied, Codec, Config}; use super::{Backend, Encoded, keyframe_annexb}; use crate::v4l2::{self, Dequeue, Device, Dir, Planes, Queue, Rect, Request, Role}; use crate::{Error, Frame, Size}; @@ -145,8 +145,10 @@ impl V4l2 { v4l2_mpeg_video_h264_profile_V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE as i32, )?; device.set_control(V4L2_CID_MPEG_VIDEO_H264_LEVEL, h264_level(config)?)?; - let Gop::Keyframe { interval } = config.gop; - device.set_control(V4L2_CID_MPEG_VIDEO_GOP_SIZE, interval.min(i32::MAX as u32) as i32)?; + device.set_control( + V4L2_CID_MPEG_VIDEO_GOP_SIZE, + config.keyframe_interval().min(i32::MAX as u32) as i32, + )?; set_bitrate(&device, config.resolved_bitrate().as_bps())?; // Constant rate is what a live uplink wants: the congestion controller // already owns the rate, and a variable-rate encoder would spend it on the diff --git a/rs/moq-video/src/encode/backend/vaapi.rs b/rs/moq-video/src/encode/backend/vaapi.rs index da17177c435..53bd73ee729 100644 --- a/rs/moq-video/src/encode/backend/vaapi.rs +++ b/rs/moq-video/src/encode/backend/vaapi.rs @@ -39,7 +39,7 @@ use std::path::Path; use bytes::Bytes; use moq_vaapi::encode::{Config as VaapiConfig, Encoder}; -use super::super::encoder::{Applied, Codec, Config, Gop, Preset}; +use super::super::encoder::{Applied, Codec, Config, Preset}; use super::{Backend, Encoded, keyframe_annexb}; use crate::color::Matrix; use crate::frame::{DmaBuf, DrmFormat, I420, vaapi}; @@ -68,7 +68,7 @@ impl Vaapi { fn new(config: &Config) -> Result { let bitrate = config.resolved_bitrate().as_bps().min(u32::MAX as u64) as u32; - let Gop::Keyframe { interval } = config.gop; + let interval = config.keyframe_interval(); // moq-vaapi derives the VUI's primaries and transfer from the matrix, the // way SD and HD video pair them, so any other pairing is refused. let color = config.resolved_color(); diff --git a/rs/moq-video/src/encode/backend/videotoolbox.rs b/rs/moq-video/src/encode/backend/videotoolbox.rs index 099ee768bb0..333bef6cc14 100644 --- a/rs/moq-video/src/encode/backend/videotoolbox.rs +++ b/rs/moq-video/src/encode/backend/videotoolbox.rs @@ -42,7 +42,7 @@ use objc2_video_toolbox::{ kVTProfileLevel_HEVC_Main_AutoLevel, }; -use super::super::encoder::{Applied, Codec, Config, Gop, Preset}; +use super::super::encoder::{Applied, Codec, Config, Preset}; use super::{Backend, Encoded, keyframe_nal}; use crate::color::{Matrix, Primaries, Transfer}; use crate::frame::Surface; @@ -144,11 +144,10 @@ impl VideoToolbox { unsafe { kVTCompressionPropertyKey_AverageBitRate }, clamp_i32(config.resolved_bitrate().as_bps()), )?; - let Gop::Keyframe { interval } = config.gop; set_number( &session, unsafe { kVTCompressionPropertyKey_MaxKeyFrameInterval }, - clamp_i32(interval.into()), + clamp_i32(config.keyframe_interval().into()), )?; set_number( &session, diff --git a/rs/moq-video/src/encode/encoded.rs b/rs/moq-video/src/encode/encoded.rs index a876619c746..68e46411c17 100644 --- a/rs/moq-video/src/encode/encoded.rs +++ b/rs/moq-video/src/encode/encoded.rs @@ -24,7 +24,7 @@ pub struct Encoded { /// Whether this access unit is a keyframe, one a decoder can start from. /// /// Set on every keyframe, whether [`Encoder::cut`](super::Encoder::cut) forced - /// it or the encoder placed it on its own [`Gop`](super::Gop) cadence. + /// it or the encoder placed it on its own [`Config::join`](super::Config::join) cadence. pub keyframe: bool, } diff --git a/rs/moq-video/src/encode/encoder.rs b/rs/moq-video/src/encode/encoder.rs index 4f10f06c8cd..724b3ab8b9b 100644 --- a/rs/moq-video/src/encode/encoder.rs +++ b/rs/moq-video/src/encode/encoder.rs @@ -48,56 +48,6 @@ pub enum Kind { Named(String), } -/// How the encoded stream is divided into groups. -/// -/// Every mode opens a group at a point a subscriber can start decoding, and -/// [`Encoder::cut`] opens one on demand in any of them. `#[non_exhaustive]`: a -/// later mode adds a variant here rather than replacing the field, so match -/// with a fallback arm. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -#[non_exhaustive] -pub enum Gop { - /// A keyframe (an IDR with its parameter sets in band) every `interval` - /// frames, each opening a group. A subscriber joining mid-stream waits at - /// most that many frames for a picture it can decode. - Keyframe { - /// Frames per group. Must be at least 1; [`Encoder::new`] refuses 0. - interval: u32, - }, -} - -impl Gop { - /// Keyframes every `interval` of wall time at `framerate`, rounded down to - /// whole frames and never below one. - pub fn keyframe_every(interval: std::time::Duration, framerate: FrameRate) -> Self { - Self::Keyframe { - interval: framerate.frames(interval).max(1), - } - } - - /// The nominal time between groups at `framerate`. - pub fn duration(self, framerate: FrameRate) -> std::time::Duration { - match self { - Self::Keyframe { interval } => std::time::Duration::from_nanos( - (u128::from(interval) * 1_000_000_000 * u128::from(framerate.denominator()) - / u128::from(framerate.numerator())) - .try_into() - .unwrap_or(u64::MAX), - ), - } - } - - /// Refuse a configuration no backend can honor, before one is opened. - pub(crate) fn validate(&self) -> Result<(), Error> { - match *self { - Self::Keyframe { interval: 0 } => Err(Error::InvalidGop( - "a keyframe interval of 0 frames opens no group".to_owned(), - )), - Self::Keyframe { .. } => Ok(()), - } - } -} - /// How an encoder trades latency for compression at the configured bitrate. /// /// Bitrate is set separately, via [`Config::bitrate`]. Presets differ in the @@ -200,9 +150,13 @@ pub struct Config { /// Target bitrate. `None` derives a sane default /// from resolution and framerate (~0.07 bits per pixel per second). pub bitrate: Option, - /// How the stream is divided into groups. Defaults to a keyframe every two - /// seconds at `framerate`. - pub gop: Gop, + /// The time between join points: a keyframe (an IDR with its parameter sets + /// in band) opens a group this often, so a subscriber joining mid-stream waits + /// at most this long for a picture it can decode. [`Encoder::cut`] opens one + /// sooner. Rounded to whole frames at `framerate`, never below one; see + /// [`effective_join`](Self::effective_join). [`Encoder::new`] refuses zero. + /// Defaults to two seconds. + pub join: std::time::Duration, /// Output codec. Defaults to [`Codec::H264`]. pub codec: Codec, /// Which encoder implementation to use. Defaults to [`Kind::Auto`]. @@ -245,14 +199,14 @@ pub struct Config { impl Config { /// A config encoding `width` x `height` at `framerate`, with the default - /// codec, GOP, and bitrate. + /// codec, join, and bitrate. pub fn new(width: u32, height: u32, framerate: FrameRate) -> Self { Self { width, height, framerate, bitrate: None, - gop: Gop::keyframe_every(std::time::Duration::from_secs(2), framerate), + join: std::time::Duration::from_secs(2), codec: Codec::default(), kind: Kind::Auto, preset: Preset::default(), @@ -268,6 +222,21 @@ impl Config { Size::new(self.width, self.height) } + /// The time between join points the encoder holds to: [`join`](Self::join) + /// rounded to whole frames at `framerate`, the value a catalog declares. + pub fn effective_join(&self) -> std::time::Duration { + let nanos = u128::from(self.keyframe_interval()) * 1_000_000_000 * u128::from(self.framerate.denominator()) + / u128::from(self.framerate.numerator()); + std::time::Duration::from_nanos(nanos.try_into().unwrap_or(u64::MAX)) + } + + /// Frames between keyframes: [`join`](Self::join) rounded to the nearest + /// frame, and never below one, since a positive join shorter than a frame + /// still opens a group per frame. + pub(crate) fn keyframe_interval(&self) -> u32 { + self.framerate.frames(self.join).max(1) + } + /// The catalog rendition an encoder for this config produces. /// /// Resolved by opening a throwaway encoder and encoding one frame, so the profile, level, and @@ -392,7 +361,9 @@ impl Encoder { let size = config.size(); size.validate("encoder")?; size.validate_encodable("encoder", config.framerate)?; - config.gop.validate()?; + if config.join.is_zero() { + return Err(Error::ZeroJoin); + } let backend = backend::open(config)?; let applied = backend.applied(); @@ -466,7 +437,7 @@ impl Encoder { } /// Cut a new group at the next frame, on top of the boundaries - /// [`Config::gop`] places on its own. Today that means encoding it as a + /// [`Config::join`] places on its own. Today that means encoding it as a /// keyframe (an IDR), so a subscriber can start decoding there. /// /// Rarely needed: the encoder opens groups automatically, so reach for this @@ -484,7 +455,7 @@ impl Encoder { /// /// Returns [`Error::CutUnsupported`] when this backend cannot force a group /// boundary (a V4L2 driver without the force-keyframe control), in which case - /// nothing is queued: groups keep falling where [`Config::gop`] puts them, and + /// nothing is queued: groups keep falling where [`Config::join`] puts them, and /// the caller decides whether that layout is acceptable rather than finding /// out from the stream. pub fn cut(&mut self) -> Result<(), Error> { @@ -1253,7 +1224,7 @@ mod tests { }; let mut encoder = encoder_with(Box::new(backend), &config); - // Nothing asked for: `Config::gop` places the boundaries on its own. + // Nothing asked for: `Config::join` places the boundaries on its own. encoder.encode(&gray_frame(320, 240, 0)).unwrap(); // Asked twice before a frame exists: one cut, on the next frame. @@ -1321,41 +1292,40 @@ mod tests { assert_eq!(*log.lock().unwrap(), vec![false]); } - /// An interval of zero opens no group, so it is refused at open rather than + /// A join of zero opens no group, so it is refused at open rather than /// handed to a backend, each of which would do something different with it /// (openh264 keys every frame, V4L2 rejects the control, VideoToolbox reads /// it as unlimited). #[test] - fn a_zero_keyframe_interval_is_refused() { + fn a_zero_join_is_refused() { let config = Config { - gop: Gop::Keyframe { interval: 0 }, + join: std::time::Duration::ZERO, ..Config::new(320, 240, crate::FrameRate::new(30, 1).unwrap()) }; let Err(err) = Encoder::new(&config) else { panic!("an empty group cannot open"); }; - assert!(matches!(err, Error::InvalidGop(_)), "unexpected error: {err:?}"); + assert!(matches!(err, Error::ZeroJoin), "unexpected error: {err:?}"); } - /// The wall-time constructor never produces the interval `Encoder::new` - /// refuses: a window shorter than a frame rounds up to one frame, not down - /// to none. + /// The catalog declares the join after rounding to whole frames, since that + /// is the spacing the stream actually has: a fractional frame rate stretches + /// the default two seconds, and a positive join shorter than a frame rounds + /// up to one frame rather than down to none. #[test] - fn a_sub_frame_keyframe_window_rounds_up_to_one_frame() { - let framerate = crate::FrameRate::new(30, 1).unwrap(); - assert_eq!( - Gop::keyframe_every(std::time::Duration::from_millis(1), framerate), - Gop::Keyframe { interval: 1 } - ); - assert_eq!( - Gop::keyframe_every(std::time::Duration::from_secs(2), framerate), - Gop::Keyframe { interval: 60 } - ); - assert!( - Gop::keyframe_every(std::time::Duration::ZERO, framerate) - .validate() - .is_ok() - ); + fn the_effective_join_is_whole_frames() { + let join = |ms, framerate| { + let config = Config { + join: std::time::Duration::from_millis(ms), + ..Config::new(320, 240, framerate) + }; + (config.keyframe_interval(), config.effective_join()) + }; + let ntsc = crate::FrameRate::new(30_000, 1001).unwrap(); + let thirty = crate::FrameRate::new(30, 1).unwrap(); + assert_eq!(join(2000, ntsc), (60, std::time::Duration::from_millis(2002))); + assert_eq!(join(2000, thirty), (60, std::time::Duration::from_secs(2))); + assert_eq!(join(1, thirty), (1, std::time::Duration::from_nanos(33_333_333))); } /// `cut()` has to reach the codec on a *warm* encoder, which is the case that @@ -1369,10 +1339,10 @@ mod tests { kind: Kind::Software, ..Config::new(320, 240, crate::FrameRate::new(30, 1).unwrap()) }; - // A GOP far longer than the run, so any IDR here was asked for rather than + // A join far longer than the run, so any IDR here was asked for rather than // inserted on schedule. let mut encoder = Encoder::new(&Config { - gop: Gop::Keyframe { interval: 1000 }, + join: std::time::Duration::from_secs(60), ..config }) .unwrap(); diff --git a/rs/moq-video/src/encode/mod.rs b/rs/moq-video/src/encode/mod.rs index 4fbb66d4340..3fe08e2fe47 100644 --- a/rs/moq-video/src/encode/mod.rs +++ b/rs/moq-video/src/encode/mod.rs @@ -44,7 +44,7 @@ mod sink; pub use backend::NAMES; pub use encoded::Encoded; -pub use encoder::{Applied, Codec, Config, Encoder, Gop, Kind, Preset}; +pub use encoder::{Applied, Codec, Config, Encoder, Kind, Preset}; pub use producer::Producer; #[cfg(feature = "capture")] pub use producer::{Capture, publish_capture}; diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index d6cccb83991..be80237a695 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -412,10 +412,10 @@ async fn publish_capture_from( probe_config.kind = options.kind.clone(); probe_config.color = camera.color(); let rendition = probe_config.probe().await?; - // The encoder is built from the same defaults, so the probe's GOP is the one it emits. - Ok::<_, Error>((rendition, probe_config.gop.duration(probe_config.framerate), camera)) + // The encoder is built from the same defaults, so the probe's join is the one it emits. + Ok::<_, Error>((rendition, probe_config.effective_join(), camera)) }; - let (rendition, gop, camera) = match rendition.await { + let (rendition, join, camera) = match rendition.await { Ok(probed) => probed, Err(err) => { // A track that already ended has nobody left to tell. @@ -427,7 +427,7 @@ async fn publish_capture_from( // The importer consumes the track, so keep a handle to abort it if setup fails, // then drop it so it never outlives the importer's. let fallback = track.clone(); - let mut producer = match Producer::with_track(track, catalog.reserve().with_interval(gop), rendition) { + let mut producer = match Producer::with_track(track, catalog.reserve().with_interval(join), rendition) { Ok(producer) => producer, Err(err) => { let _ = fallback.abort(moq_net::Error::Transport(err.to_string())); @@ -893,7 +893,7 @@ mod tests { let mut producer = Producer::new( broadcast, - catalog.reserve().with_interval(config.gop.duration(config.framerate)), + catalog.reserve().with_interval(config.effective_join()), config.probe().await.unwrap(), ) .unwrap(); @@ -966,7 +966,7 @@ mod tests { config.kind = encoder::Kind::Software; let mut producer = Producer::with_track( track, - catalog.reserve().with_interval(config.gop.duration(config.framerate)), + catalog.reserve().with_interval(config.effective_join()), config.probe().await.unwrap(), ) .unwrap(); @@ -996,7 +996,7 @@ mod tests { initial.kind = encoder::Kind::Software; let mut producer = Producer::new( broadcast, - catalog.reserve().with_interval(initial.gop.duration(initial.framerate)), + catalog.reserve().with_interval(initial.effective_join()), initial.probe().await.unwrap(), ) .unwrap(); @@ -1069,7 +1069,7 @@ mod tests { config.kind = encoder::Kind::Software; let _producer = Producer::new( broadcast, - catalog.reserve().with_interval(config.gop.duration(config.framerate)), + catalog.reserve().with_interval(config.effective_join()), config.probe().await.unwrap(), ) .unwrap(); @@ -1115,7 +1115,7 @@ mod tests { let mut subscriber = track.subscribe(None); let mut producer = Producer::with_track( track, - catalog.reserve().with_interval(config.gop.duration(config.framerate)), + catalog.reserve().with_interval(config.effective_join()), config.probe().await.unwrap(), ) .unwrap(); @@ -1223,7 +1223,7 @@ mod tests { probe.kind = encoder::Kind::Software; let mut producer = Producer::with_track( track, - catalog.reserve().with_interval(probe.gop.duration(probe.framerate)), + catalog.reserve().with_interval(probe.effective_join()), probe.probe().await.unwrap(), ) .unwrap(); @@ -1972,7 +1972,7 @@ mod tests { track, catalog .reserve() - .with_interval(rendition.gop.duration(rendition.framerate)), + .with_interval(rendition.effective_join()), rendition.probe().await.unwrap(), ) .unwrap(); @@ -2030,7 +2030,7 @@ mod tests { track, catalog .reserve() - .with_interval(rendition.gop.duration(rendition.framerate)), + .with_interval(rendition.effective_join()), rendition.probe().await.unwrap(), ) .unwrap(); diff --git a/rs/moq-video/src/error.rs b/rs/moq-video/src/error.rs index 8f7cb3967f0..1023a0a404f 100644 --- a/rs/moq-video/src/error.rs +++ b/rs/moq-video/src/error.rs @@ -100,10 +100,10 @@ pub enum Error { encoder: crate::Color, }, - /// The group configuration can't be honored by any backend (the message - /// says why). - #[error("invalid group configuration: {0}")] - InvalidGop(String), + /// [`encode::Config::join`](crate::encode::Config::join) is zero, which opens + /// no group. + #[error("a join of zero opens no group")] + ZeroJoin, /// GPU rendering failure: building the pipeline, importing a frame's surface /// as a texture, or the device itself. diff --git a/rs/moq-video/src/frame/cuda_test.rs b/rs/moq-video/src/frame/cuda_test.rs index 4e5e6127c65..ea037f71115 100644 --- a/rs/moq-video/src/frame/cuda_test.rs +++ b/rs/moq-video/src/frame/cuda_test.rs @@ -95,7 +95,7 @@ fn nvenc(size: Size, color: Color) -> crate::encode::Encoder { let mut config = crate::encode::Config::new(size.width, size.height, crate::FrameRate::new(30, 1).unwrap()); config.kind = crate::encode::Kind::Named("nvenc".into()); config.color = Some(color); - config.gop = crate::encode::Gop::Keyframe { interval: 30 }; + config.join = std::time::Duration::from_secs(1); crate::encode::Encoder::new(&config).expect("open NVENC") } diff --git a/swift/Sources/Moq/Video.swift b/swift/Sources/Moq/Video.swift index 51262a9bbda..560ecbe2683 100644 --- a/swift/Sources/Moq/Video.swift +++ b/swift/Sources/Moq/Video.swift @@ -18,8 +18,6 @@ public enum Video { public typealias Read = MoqVideoRead /// One encoded access unit from an `Encoder`. public typealias Packet = MoqVideoPacket - /// How pictures are split into independently decodable groups. - public typealias Gop = MoqVideoGop /// A video codec identifier (H.264 or H.265). public typealias Codec = VideoCodec /// A CPU pixel layout (I420 or RGBA). @@ -68,7 +66,7 @@ public enum Video { /// Start a new group at the next written frame. /// - /// Optional: the encoder starts groups on its own per `output.gop`, so a + /// Optional: the encoder starts groups on its own per `output.join`, so a /// subscriber can always join without this. Reach for it only to place the /// boundaries yourself, aligning groups with something the encoder can't /// see such as a scene change. Throws if the selected encoder cannot force diff --git a/test/interop/clients/js/src/fixture.ts b/test/interop/clients/js/src/fixture.ts index c27654b1467..fd060dd57ab 100644 --- a/test/interop/clients/js/src/fixture.ts +++ b/test/interop/clients/js/src/fixture.ts @@ -161,7 +161,7 @@ export class Fixture { config: { codec: "avc1", frameRate: Pattern.FPS, - keyframeInterval: KEYFRAME_INTERVAL, + join: KEYFRAME_INTERVAL, maxBitrate: MAX_BITRATE, }, }); From 11f3d1e114f6bce40beb9282ad448ce96145250f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 20:08:29 -0800 Subject: [PATCH 3/5] style: format and refresh the publish API report Co-Authored-By: Claude Opus 5.5 --- js/publish/api/index.api.md | 8 ++++---- rs/moq-c/src/api.rs | 4 +++- rs/moq-c/src/video.rs | 5 +---- rs/moq-video/src/decode/consumer.rs | 4 +--- rs/moq-video/src/encode/producer.rs | 8 ++------ 5 files changed, 11 insertions(+), 18 deletions(-) diff --git a/js/publish/api/index.api.md b/js/publish/api/index.api.md index c1005dd60ac..39b742a0458 100644 --- a/js/publish/api/index.api.md +++ b/js/publish/api/index.api.md @@ -40,8 +40,8 @@ export namespace Audio { get capture(): Audio.Capture | undefined; close(): void; codec: Signals.Signal; - groupDuration: Signals.Signal; readonly in: Signals.Readonlys; + join: Signals.Signal; muted: Signals.Signal; readonly name: string; readonly out: Signals.Readonlys; @@ -62,7 +62,7 @@ export namespace Audio { }; export type EncoderProps = Signals.Inputs & { codec?: Audio.Codec | Signals.Signal; - groupDuration?: Clock.Milli | Signals.Signal; + join?: Clock.Milli | Signals.Signal; muted?: boolean | Signals.Signal; volume?: number | Signals.Signal; }; @@ -221,7 +221,7 @@ export namespace Preview { export class Rendition { close(): void; readonly config: Signals.Signal; - readonly joinInterval: Signals.Signal; + readonly join: Signals.Signal; readonly kind: Kind; readonly name: string; readonly stats: Signals.Signal; @@ -367,7 +367,7 @@ export namespace Video { bitrateScale?: number; codec?: string; frameRate?: number; - keyframeInterval?: Clock.Milli; + join?: Clock.Milli; maxBitrate?: number; maxPixels?: number; maxScale?: number; diff --git a/rs/moq-c/src/api.rs b/rs/moq-c/src/api.rs index ad6e7c37c23..5dbb5f9adf8 100644 --- a/rs/moq-c/src/api.rs +++ b/rs/moq-c/src/api.rs @@ -228,7 +228,9 @@ fn join(present: bool, us: u64) -> Result, Error> { return Ok(None); } if us == 0 || us > MAX_JOIN_US { - return Err(Error::InvalidConfig(format!("join_us must be 1..={MAX_JOIN_US}, got {us}"))); + return Err(Error::InvalidConfig(format!( + "join_us must be 1..={MAX_JOIN_US}, got {us}" + ))); } Ok(Some(std::time::Duration::from_micros(us))) } diff --git a/rs/moq-c/src/video.rs b/rs/moq-c/src/video.rs index f6e230e4e15..0f994f7bd1a 100644 --- a/rs/moq-c/src/video.rs +++ b/rs/moq-c/src/video.rs @@ -691,10 +691,7 @@ impl Video { encoder: moq_video::encode::Sink, lower: Option, ) -> Result { - let reserved = target - .catalog - .reserve() - .with_interval(config.effective_join()); + let reserved = target.catalog.reserve().with_interval(config.effective_join()); let ceiling = video_ceiling(config, &rendition); let (lower, priority) = match lower { Some(lower) => { diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index 124db880987..f0664a66592 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -487,9 +487,7 @@ mod tests { ..EncodeConfig::new(320, 240, crate::FrameRate::new(30, 1).unwrap()) }; let rendition = config.probe().await.unwrap(); - let reserved = source_catalog - .reserve() - .with_interval(config.effective_join()); + let reserved = source_catalog.reserve().with_interval(config.effective_join()); let mut producer = EncodeProducer::new(source_broadcast, reserved, rendition).unwrap(); let mut encoder = Encoder::new(&config).unwrap(); let rgba = vec![0x80u8; 320 * 240 * 4]; diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index be80237a695..e222c58d363 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -1970,9 +1970,7 @@ mod tests { rendition.kind = encoder::Kind::Software; let mut producer = Producer::with_track( track, - catalog - .reserve() - .with_interval(rendition.effective_join()), + catalog.reserve().with_interval(rendition.effective_join()), rendition.probe().await.unwrap(), ) .unwrap(); @@ -2028,9 +2026,7 @@ mod tests { rendition.kind = encoder::Kind::Software; let mut producer = Producer::with_track( track, - catalog - .reserve() - .with_interval(rendition.effective_join()), + catalog.reserve().with_interval(rendition.effective_join()), rendition.probe().await.unwrap(), ) .unwrap(); From 812c045a93f2166bd55fd9c4fd1477ff4d8e7d32 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 20:13:02 -0800 Subject: [PATCH 4/5] docs(video): link Config::join from the crate docs Co-Authored-By: Claude Opus 5.5 --- rs/moq-video/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/moq-video/src/lib.rs b/rs/moq-video/src/lib.rs index 41691fa467a..21a7ed4cff2 100644 --- a/rs/moq-video/src/lib.rs +++ b/rs/moq-video/src/lib.rs @@ -10,7 +10,7 @@ //! [`Surface`] holding the pixels. Capture and [`decode`] produce them, [`encode`] //! consumes them and hands back the compressed [`encode::Encoded`], and [`Size`] //! names a resolution. Group boundaries are the encoder's business: it places -//! them per [`encode::Config::gop`], and [`encode::Encoder::cut`] is there for +//! them per [`encode::Config::join`], and [`encode::Encoder::cut`] is there for //! the rarer case where a caller needs one at a specific frame. //! //! - `capture` describes a frame source and grabs frames per platform: From 0c740bc8ccd14abd508c00e5d020ae302878a511 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 21:31:31 -0800 Subject: [PATCH 5/5] fix: a refused audio join withdraws its spacing, and the effective join rounds up Co-Authored-By: Claude Opus 5.5 --- js/publish/src/audio/encoder.test.ts | 27 +++++++++++++++++++++++++++ js/publish/src/audio/encoder.ts | 5 ++++- rs/moq-video/README.md | 7 ++++--- rs/moq-video/src/encode/encoder.rs | 10 ++++++---- swift/Sources/Moq/Video.swift | 2 +- 5 files changed, 42 insertions(+), 9 deletions(-) diff --git a/js/publish/src/audio/encoder.test.ts b/js/publish/src/audio/encoder.test.ts index 1f00740eb94..4b251bd96ef 100644 --- a/js/publish/src/audio/encoder.test.ts +++ b/js/publish/src/audio/encoder.test.ts @@ -937,6 +937,33 @@ test("refuses a join that is not a positive duration", async () => { error.mockRestore(); }); +// The broadcast reads every registered rendition's spacing, listed or not, so a refused join must not +// leave the old one for a new network broadcast to declare. +test("a refused join withdraws its group spacing", async () => { + using _webcodecs = installFakeWebCodecs(); + const error = spyOn(console, "error").mockImplementation(() => {}); + const name = new Signal(Moq.Path.from("refused.hang")); + const broadcast = new Broadcast({ origin: new Moq.Origin.Producer(), name }); + const video = broadcast.video("video"); + video.join.set(Milli(20)); + video.config.set({ codec: "avc1.640028", container: { kind: "legacy" } }); + // 20 ms Opus frames start a group every two frames for a 30 ms join. + const encoder = new Encoder("audio", { broadcast, capture: fakeCapture(new Feed()) as never, join: Milli(30) }); + try { + await settle(); + expect(broadcast.join.peek()).toBe(Milli(40)); + + encoder.join.set(Milli(0)); + name.set(Moq.Path.from("renamed.hang")); + await settle(); + expect(broadcast.join.peek()).toBe(Milli(20)); + } finally { + encoder.close(); + broadcast.close(); + error.mockRestore(); + } +}); + test("settles while the capture waits on a gesture", async () => { const capture = { in: { source: new Signal(undefined) }, diff --git a/js/publish/src/audio/encoder.ts b/js/publish/src/audio/encoder.ts index 382bed86940..b3b85e2c58c 100644 --- a/js/publish/src/audio/encoder.ts +++ b/js/publish/src/audio/encoder.ts @@ -542,8 +542,11 @@ export class Encoder { // waiting on a probe. #runConfig(effect: Effect): void { const join = effect.get(this.join); - if (!Number.isFinite(join) || join <= 0) + if (!Number.isFinite(join) || join <= 0) { + // The broadcast reads every registered rendition's spacing, listed or not. + this.#spacing.set(undefined); throw new Error(`audio join must be a finite number of ms greater than 0: ${join}`); + } this.#join = Micro.fromMilli(join); const capture = effect.get(this.in.capture); diff --git a/rs/moq-video/README.md b/rs/moq-video/README.md index 15565e6b400..709f369ef5f 100644 --- a/rs/moq-video/README.md +++ b/rs/moq-video/README.md @@ -115,9 +115,10 @@ later one goes in, and for the tail `finish()` drains. Bring your own pixels wit Group boundaries are automatic: `Config::join` is the time between join points (a keyframe every two seconds by default, rounded to whole frames, and a join of -zero is refused at open), so an application never has to think about them. `Encoder::cut()` opens a group at the next frame when something -outside the encoder needs a decodable starting point there: a source group -boundary, a scene change, a source switch. The request is held until a frame +zero is refused at open), so an application never has to think about them. +`Encoder::cut()` opens a group at the next frame when something outside the +encoder needs a decodable starting point there: a source group boundary, a scene +change, a source switch. The request is held until a frame arrives, so it is safe to call before you have one. It fails with `Error::CutUnsupported` on a backend that cannot force a boundary (a V4L2 driver without the force-keyframe control), and queues nothing then: groups diff --git a/rs/moq-video/src/encode/encoder.rs b/rs/moq-video/src/encode/encoder.rs index 724b3ab8b9b..c092a502b46 100644 --- a/rs/moq-video/src/encode/encoder.rs +++ b/rs/moq-video/src/encode/encoder.rs @@ -225,8 +225,9 @@ impl Config { /// The time between join points the encoder holds to: [`join`](Self::join) /// rounded to whole frames at `framerate`, the value a catalog declares. pub fn effective_join(&self) -> std::time::Duration { - let nanos = u128::from(self.keyframe_interval()) * 1_000_000_000 * u128::from(self.framerate.denominator()) - / u128::from(self.framerate.numerator()); + // Rounded up, since a catalog declares it as the longest group. + let nanos = (u128::from(self.keyframe_interval()) * 1_000_000_000 * u128::from(self.framerate.denominator())) + .div_ceil(u128::from(self.framerate.numerator())); std::time::Duration::from_nanos(nanos.try_into().unwrap_or(u64::MAX)) } @@ -1311,7 +1312,8 @@ mod tests { /// The catalog declares the join after rounding to whole frames, since that /// is the spacing the stream actually has: a fractional frame rate stretches /// the default two seconds, and a positive join shorter than a frame rounds - /// up to one frame rather than down to none. + /// up to one frame rather than down to none. Nanoseconds round up too, so + /// the declaration never undercuts the stream. #[test] fn the_effective_join_is_whole_frames() { let join = |ms, framerate| { @@ -1325,7 +1327,7 @@ mod tests { let thirty = crate::FrameRate::new(30, 1).unwrap(); assert_eq!(join(2000, ntsc), (60, std::time::Duration::from_millis(2002))); assert_eq!(join(2000, thirty), (60, std::time::Duration::from_secs(2))); - assert_eq!(join(1, thirty), (1, std::time::Duration::from_nanos(33_333_333))); + assert_eq!(join(1, thirty), (1, std::time::Duration::from_nanos(33_333_334))); } /// `cut()` has to reach the codec on a *warm* encoder, which is the case that diff --git a/swift/Sources/Moq/Video.swift b/swift/Sources/Moq/Video.swift index 560ecbe2683..05686ccbaff 100644 --- a/swift/Sources/Moq/Video.swift +++ b/swift/Sources/Moq/Video.swift @@ -7,7 +7,7 @@ public enum Video { /// The raw frame layout, resolution, and framerate the caller writes. public typealias EncoderInput = VideoEncoderInput - /// The codec, bitrate, group mode, and backend preference. + /// The codec, bitrate, join interval, and backend preference. public typealias EncoderOutput = VideoEncoderOutput /// What a `Consumer` delivers: an optional resize, a delay, and the decoder's surface (macOS only). public typealias DecoderOutput = VideoDecoderOutput