diff --git a/dart/moq/MIGRATE.md b/dart/moq/MIGRATE.md index db6f77ae9d5..e42f8d33b12 100644 --- a/dart/moq/MIGRATE.md +++ b/dart/moq/MIGRATE.md @@ -4,6 +4,10 @@ How to update your code for each breaking change, newest first. ## Unreleased +### Catalog renditions carry their warmup ([#7864](https://github.com/moq-dev/moq/pull/7864)) + +`Video` and `Audio` gain `warmup`, the catalog's decoder warmup; it defaults to `null`, so existing constructors still compile. `TrackConsumer` and the audio and video consumers read it from the rendition and start that much earlier. + ### Separate datagram tracks Use `BroadcastProducer.createDatagrams(name: ..., info: ...)` and `BroadcastConsumer.datagrams(name: ...)` instead of stream-track datagram methods. `DatagramProducer.append(timestamp: ..., payload: ...)` assigns a sequence and `insert(datagram: ...)` preserves one; `DatagramConsumer.recv()` reads its own live cursor. `DatagramInfo` contains priority and optional timescale. Watch `priority()` / `priorityChanged(last: ...)` and serve on-demand tracks through `BroadcastRequests.datagrams()` / `DatagramRequests.requested()`. diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index b2055b918b1..5db3c842715 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -1350,6 +1350,7 @@ class MoqAudio { final bool enabled; final MoqContainer container; final Duration? jitter; + final Duration? warmup; MoqAudio({ this.label = null, this.broadcast = null, @@ -1363,6 +1364,7 @@ class MoqAudio { this.enabled = true, required this.container, this.jitter = null, + this.warmup = null, }); } @@ -1433,6 +1435,11 @@ class FfiConverterMoqAudio { ); final jitter = jitter_lifted.value; new_offset += jitter_lifted.bytesRead; + final warmup_lifted = FfiConverterOptionalDuration.read( + Uint8List.view(buf.buffer, new_offset), + ); + final warmup = warmup_lifted.value; + new_offset += warmup_lifted.bytesRead; return LiftRetVal( MoqAudio( label: label, @@ -1447,6 +1454,7 @@ class FfiConverterMoqAudio { enabled: enabled, container: container, jitter: jitter, + warmup: warmup, ), new_offset - buf.offsetInBytes, ); @@ -1466,6 +1474,7 @@ class FfiConverterMoqAudio { FfiConverterBool.allocationSize(value.enabled) + FfiConverterMoqContainer.allocationSize(value.container) + FfiConverterOptionalDuration.allocationSize(value.jitter) + + FfiConverterOptionalDuration.allocationSize(value.warmup) + 0; final buf = Uint8List(total_length); write(value, buf); @@ -1522,6 +1531,10 @@ class FfiConverterMoqAudio { value.jitter, Uint8List.view(buf.buffer, new_offset), ); + new_offset += FfiConverterOptionalDuration.write( + value.warmup, + Uint8List.view(buf.buffer, new_offset), + ); return new_offset - buf.offsetInBytes; } @@ -1538,6 +1551,7 @@ class FfiConverterMoqAudio { FfiConverterBool.allocationSize(value.enabled) + FfiConverterMoqContainer.allocationSize(value.container) + FfiConverterOptionalDuration.allocationSize(value.jitter) + + FfiConverterOptionalDuration.allocationSize(value.warmup) + 0; } } @@ -2288,6 +2302,7 @@ class MoqVideo { final int preference; final double? framerate; final MoqContainer container; + final Duration? warmup; MoqVideo({ this.label = null, this.broadcast = null, @@ -2302,6 +2317,7 @@ class MoqVideo { this.preference = 0, this.framerate, required this.container, + this.warmup = null, }); } @@ -2377,6 +2393,11 @@ class FfiConverterMoqVideo { ); final container = container_lifted.value; new_offset += container_lifted.bytesRead; + final warmup_lifted = FfiConverterOptionalDuration.read( + Uint8List.view(buf.buffer, new_offset), + ); + final warmup = warmup_lifted.value; + new_offset += warmup_lifted.bytesRead; return LiftRetVal( MoqVideo( label: label, @@ -2392,6 +2413,7 @@ class FfiConverterMoqVideo { preference: preference, framerate: framerate, container: container, + warmup: warmup, ), new_offset - buf.offsetInBytes, ); @@ -2412,6 +2434,7 @@ class FfiConverterMoqVideo { FfiConverterInt32.allocationSize(value.preference) + FfiConverterOptionalDouble64.allocationSize(value.framerate) + FfiConverterMoqContainer.allocationSize(value.container) + + FfiConverterOptionalDuration.allocationSize(value.warmup) + 0; final buf = Uint8List(total_length); write(value, buf); @@ -2472,6 +2495,10 @@ class FfiConverterMoqVideo { value.container, Uint8List.view(buf.buffer, new_offset), ); + new_offset += FfiConverterOptionalDuration.write( + value.warmup, + Uint8List.view(buf.buffer, new_offset), + ); return new_offset - buf.offsetInBytes; } @@ -2489,6 +2516,7 @@ class FfiConverterMoqVideo { FfiConverterInt32.allocationSize(value.preference) + FfiConverterOptionalDouble64.allocationSize(value.framerate) + FfiConverterMoqContainer.allocationSize(value.container) + + FfiConverterOptionalDuration.allocationSize(value.warmup) + 0; } } @@ -24109,7 +24137,7 @@ void _checkApiChecksums() { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_constructor_moqmediatrackconsumer_subscribe() != - 21132) { + 16252) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_constructor_moqmediatrackproducer_audio() != diff --git a/doc/lib/rs/hang.md b/doc/lib/rs/hang.md index 1350f66a161..28e3e127089 100644 --- a/doc/lib/rs/hang.md +++ b/doc/lib/rs/hang.md @@ -25,7 +25,7 @@ catalog for you and decodes any container in group order (`moq_mux::container::Consumer`). Its `Delay { min, max }` says how far behind the live edge to play: it skips groups that fall past `max`, and starts at the group covering the live edge less `min` (your playout delay), plus a rendition's -catalog `warmup` passed to `with_warmup`, waiting up to that long for it when the +catalog `warmup` passed to a `Playout`, waiting up to that long for it when the newest group arrives first. Both default to zero, which starts at the newest group at once. A backlog that landed before the first read starts within `max` of the newest content, keeping the group covering the live edge less `min` and diff --git a/go/wrapper/MIGRATE.md b/go/wrapper/MIGRATE.md index 131c150b070..b335f3ac7f7 100644 --- a/go/wrapper/MIGRATE.md +++ b/go/wrapper/MIGRATE.md @@ -4,6 +4,10 @@ How to update your code for each breaking change, newest first. ## Unreleased +### Catalog renditions carry their warmup ([#7864](https://github.com/moq-dev/moq/pull/7864)) + +`ffi.MoqVideo` and `ffi.MoqAudio` (`media.Video` and `media.Audio`) gain `Warmup`, the catalog's decoder warmup, nil when absent. `media.NewTrackConsumer` and the audio and video `Subscribe` read it from the rendition and start that much earlier; a negative one is refused with an error. Only unkeyed struct literals change. + ### Separate datagram tracks Replace stream-track `AppendDatagram` / `InsertDatagram` / `RecvDatagram` with `BroadcastProducer.CreateDatagrams(name, info)` and `BroadcastConsumer.Datagrams(name)`. `DatagramProducer.Append(timestamp, payload)` assigns a sequence and `Insert(Datagram)` preserves one. `DatagramConsumer.Recv(ctx)` and `All(ctx)` read its independent cursor. `DatagramInfo` carries priority and optional timescale. Watch `Priority()` / `PriorityChanged(ctx, last)` and serve on-demand tracks through `BroadcastRequests.Datagrams()` / `DatagramRequests.Requested(ctx)`. diff --git a/go/wrapper/audio/audio.go b/go/wrapper/audio/audio.go index fe491a1bdb1..307c3bc00fc 100644 --- a/go/wrapper/audio/audio.go +++ b/go/wrapper/audio/audio.go @@ -218,7 +218,7 @@ type Consumer struct{ inner *ffi.MoqAudioConsumer } // Subscribe subscribes to the catalog rendition id, following its broadcast and track. func Subscribe(ctx context.Context, broadcast *moq.BroadcastConsumer, id string, rendition media.Audio, output DecoderOutput) (*Consumer, error) { - if err := bridge.NonNegative(rendition.Jitter, "jitter"); err != nil { + if err := bridge.Audio(rendition); err != nil { return nil, err } out, err := output.ffi() diff --git a/go/wrapper/audio/audio_internal_test.go b/go/wrapper/audio/audio_internal_test.go index c9b7cbe26ac..a447a7c0636 100644 --- a/go/wrapper/audio/audio_internal_test.go +++ b/go/wrapper/audio/audio_internal_test.go @@ -26,9 +26,12 @@ func TestEncoderOutputFrameDuration(t *testing.T) { } } -func TestSubscribeRefusesNegativeJitter(t *testing.T) { +func TestSubscribeRefusesNegativeDurations(t *testing.T) { negative := -time.Millisecond if _, err := Subscribe(context.Background(), nil, "audio", media.Audio{Jitter: &negative}, DecoderOutput{}); err == nil { t.Fatal("a negative jitter was subscribed, want an error") } + if _, err := Subscribe(context.Background(), nil, "audio", media.Audio{Warmup: &negative}, DecoderOutput{}); err == nil { + t.Fatal("a negative warmup was subscribed, want an error") + } } diff --git a/go/wrapper/internal/bridge/handles.go b/go/wrapper/internal/bridge/handles.go index 34f3483b201..48fc0f38573 100644 --- a/go/wrapper/internal/bridge/handles.go +++ b/go/wrapper/internal/bridge/handles.go @@ -53,10 +53,26 @@ func NonNegative(value *time.Duration, name string) error { return nil } -// Rendition refuses an audio rendition's negative jitter, for the same reason. +// Rendition refuses a rendition's negative jitter or warmup, for the same reason. func Rendition(rendition ffi.MoqRendition) error { - if audio, ok := rendition.(ffi.MoqRenditionAudio); ok { - return NonNegative(audio.Config.Jitter, "jitter") + switch rendition := rendition.(type) { + case ffi.MoqRenditionAudio: + return Audio(rendition.Config) + case ffi.MoqRenditionVideo: + return Video(rendition.Config) } return nil } + +// Audio refuses an audio rendition's negative jitter or warmup. +func Audio(audio ffi.MoqAudio) error { + if err := NonNegative(audio.Jitter, "jitter"); err != nil { + return err + } + return NonNegative(audio.Warmup, "warmup") +} + +// Video refuses a video rendition's negative warmup. +func Video(video ffi.MoqVideo) error { + return NonNegative(video.Warmup, "warmup") +} diff --git a/go/wrapper/media/media_test.go b/go/wrapper/media/media_test.go index 94c2460a76e..dceb8f474c5 100644 --- a/go/wrapper/media/media_test.go +++ b/go/wrapper/media/media_test.go @@ -62,6 +62,14 @@ func TestRefusesNegativeDurations(t *testing.T) { if _, err := (&moq.BroadcastConsumer{}).Resolve(ctx, rendition); err == nil { t.Fatal("a negative jitter was resolved, want an error") } + for _, warmup := range []media.Rendition{ + media.AudioRendition("audio", media.Audio{Warmup: &negative}), + media.VideoRendition("video", media.Video{Warmup: &negative}), + } { + if _, err := media.NewTrackConsumer(ctx, nil, media.TrackConfig{Rendition: warmup}); err == nil { + t.Fatal("a negative warmup was subscribed, want an error") + } + } } func TestContainerTimebaseBelongsToOneBroadcast(t *testing.T) { diff --git a/go/wrapper/video/video.go b/go/wrapper/video/video.go index 8111e4e0242..b3d75b828af 100644 --- a/go/wrapper/video/video.go +++ b/go/wrapper/video/video.go @@ -168,6 +168,9 @@ type Consumer struct{ inner *ffi.MoqVideoConsumer } // Subscribe subscribes to the catalog rendition id, following its broadcast and track. func Subscribe(ctx context.Context, broadcast *moq.BroadcastConsumer, id string, rendition media.Video, output DecoderOutput) (*Consumer, error) { + if err := bridge.Video(rendition); err != nil { + return nil, err + } out, err := output.ffi() if err != nil { return nil, err diff --git a/go/wrapper/video/video_internal_test.go b/go/wrapper/video/video_internal_test.go new file mode 100644 index 00000000000..ae52c5cf38e --- /dev/null +++ b/go/wrapper/video/video_internal_test.go @@ -0,0 +1,17 @@ +package video + +import ( + "context" + "testing" + "time" + + "moq.dev/moq/media" +) + +// The generated converter panics on a negative duration, so it is refused first. +func TestSubscribeRefusesNegativeWarmup(t *testing.T) { + negative := -time.Millisecond + if _, err := Subscribe(context.Background(), nil, "video", media.Video{Warmup: &negative}, DecoderOutput{}); err == nil { + t.Fatal("a negative warmup was subscribed, want an error") + } +} diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 99df1a08df1..50c8e5afbca 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -42,7 +42,7 @@ export interface ConsumerProps { * How much further than the min delay the start reaches, in milliseconds (default: 0), so a * decoder that needs that much media before its output is clean starts early enough, such as an * Opus decoder or the max-delay decoder of an intra-refresh rendition. Pass the catalog - * rendition's `warmup`. Mirrors Rust's `Consumer::with_warmup`. + * rendition's `warmup`. Mirrors Rust's `container::Playout`. * * Put the max delay plus the warmup on the subscription, since the publisher skips what falls * outside that budget. The start is no later than the group covering the live edge less diff --git a/kt/moq/MIGRATE.md b/kt/moq/MIGRATE.md index ed062def455..9ef83c33045 100644 --- a/kt/moq/MIGRATE.md +++ b/kt/moq/MIGRATE.md @@ -4,6 +4,10 @@ How to update your code for each breaking change, newest first. ## Unreleased +### Catalog renditions carry their warmup ([#7864](https://github.com/moq-dev/moq/pull/7864)) + +`Video` and `Audio` gain `warmup`, the catalog's decoder warmup; it defaults to `null` as the last field, so existing code keeps working. `TrackConsumer` and the audio and video consumers read it from the rendition and start that much earlier. + ### Separate datagram tracks Replace stream-track `appendDatagram` / `insertDatagram` / `recvDatagram` with `BroadcastProducer.createDatagrams(name, info)` and `BroadcastConsumer.datagrams(name)`. The separate `DatagramProducer` has `append(timestamp, payload)` and `insert(Datagram)`; `DatagramConsumer` has `recv()` and the `datagrams()` flow. Use `priority()` / `priorityChanged(last)` for aggregate demand, and `BroadcastRequests.datagrams()` / `DatagramRequests.requested()` for on-demand tracks. `DatagramInfo` has only publisher priority and optional timescale. diff --git a/py/moq-rs/MIGRATE.md b/py/moq-rs/MIGRATE.md index 0167baab510..467e0d5bd88 100644 --- a/py/moq-rs/MIGRATE.md +++ b/py/moq-rs/MIGRATE.md @@ -4,6 +4,10 @@ How to update your code for each breaking change, newest first. ## Unreleased +### Catalog renditions carry their warmup ([#7864](https://github.com/moq-dev/moq/pull/7864)) + +`Video` and `Audio` gain `warmup`, the catalog's decoder warmup; it defaults to `None` as the last field, so existing code keeps working. `TrackConsumer` and the audio and video consumers read it from the rendition and start that much earlier. + ### Separate datagram tracks Replace stream-track `append_datagram` / `insert_datagram` / `recv_datagram` with `broadcast.create_datagrams(name, info)` and `broadcast.consume().datagrams(name)`. The returned `DatagramProducer` uses `append(payload, timestamp=...)` and `insert(Datagram(...))`; `DatagramConsumer` supports `recv()` and async iteration. `DatagramInfo` contains priority and optional timescale. Demand is `priority()` / `await priority_changed(last)`; on-demand handlers come from `broadcast.requests().datagrams()`. Stream tracks of the same name are independent. diff --git a/quest/m1/README.md b/quest/m1/README.md index e3bbdf1e474..434ce740c58 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -180,7 +180,6 @@ their end report, so an embedder's drain alert pages on real cuts. - [Ordered reads can skip owed holes, and snapshot latest() does](/quest/m1/ordered-owed-holes.md) - moq-net's `Ordered` gains a read that skips past an owed hole to the newest buffered group, while keeping the reader's bounds - [The relay reports native sessions it force-closes at the drain deadline](/quest/m1/relay-drain-cuts.md) - the drain's public outcome and live tally separate sessions cut while open from sessions only finishing their end report, so an embedder's drain alert pages on real cuts - [A Rust relay never replays from group 0 to a lite-03/04 subscriber](/quest/m1/relay-inflight-replay.md) - A Rust relay or publisher serving a lite-03 or lite-04 SUBSCRIBE from a track copy that is still resolving starts at the newest group once the copy resolves -- [Every moq-mux consumer gets the catalog warmup from one entry point](/quest/m1/mux-warmup.md) - one moq-mux call widens the subscription and configures the consumer, adopted by moq-audio, moq-c, moq-ffi's raw consumer, and moqsrc - [Rename moq-rtc to moq-webrtc](/quest/m1/webrtc-rename.md) - one published `moq-webrtc` crate holds the gateway, the ICE driver, and the data-channel transport - [Vendor and trim the WebRTC stack](/quest/m1/webrtc-vendor.md) - str0m, dimpl, `is`, and sctp-proto live in moq-webrtc with unused code removed, so moq-rtc can take rind - [MoQ on rind](/quest/m1/rind/README.md) - every MoQ crate takes its crypto from rind, a pure-Rust graviola fork, so no build compiles C crypto diff --git a/quest/m1/mux-warmup.md b/quest/m1/mux-warmup.md deleted file mode 100644 index 500c5b98a8e..00000000000 --- a/quest/m1/mux-warmup.md +++ /dev/null @@ -1,46 +0,0 @@ -# [M] Every moq-mux consumer gets the catalog warmup from one entry point - -## Goal - -A moq-mux container consumer is built from a subscription delay and the -catalog rendition's `warmup` in one call. That call widens the subscription -and configures the consumer, so a caller can't forget the warmup or apply it -in the wrong order. Every raw consumer uses it: moq-audio, moq-c, moq-ffi's -`MoqMediaTrackConsumer`, and moq-gst's `moqsrc`. moq-ffi's raw consumer and -`moqsrc` then start early enough for a warmed-up decoder, as moq-c's -`moq_consume_audio`/`video` already do. - -## Plan - -Decided 2026-11-03 (maintainer, /quest-plan of the follow-ups from -[moq-dev/moq#7723](https://github.com/moq-dev/moq/pull/7723)): milestone m1. -#7723's two follow-ups are one quest, since the simpler entry point is how -the two missing consumers get the warmup. - -- Today (`rs/moq-mux/src/container/consumer.rs:340-371`, and - `rs/moq-mux/MIGRATE.md:18`) the caller must add the warmup to the - subscription's `delay.max` before subscribing. It must then call - `with_warmup`, which carves the warmup back out, and call it before - `with_delay`. moq-audio (`decode/consumer.rs:245`) and moq-c - (`consume.rs:540-550`) do this by hand. -- Missing today: `rs/moq-ffi/src/media.rs:1717-1748` - (`MoqMediaTrackConsumer::subscribe`) and - `rs/moq-gst/src/source/imp.rs:1649-1673` subscribe with the caller's delay - and never call `with_warmup`. The deleted moq-c quest - (`quest/m1/c/raw-consume-warmup.md`, removed in `cd7f9a9a38`) wrongly - assumed moq-ffi already matched. -- Shape: one moq-mux constructor or builder takes the track or subscription - request, the `Delay`, and the warmup. It returns the widened subscription - and a consumer configured to match, or subscribes itself. Remove the - order-sensitive `with_warmup` + `with_delay` pair from the public API, - and record the break in `MIGRATE.md`. -- moq-video has its own two-decoder scheme (`decode/consumer.rs:176-185`). - Move it onto the new entry point if it fits. If not, record why here. -- Test each consumer: a test that starts a track with a nonzero catalog - warmup fails without that consumer's adoption. -- Builds on [moq-dev/moq#7697](https://github.com/moq-dev/moq/pull/7697) - (merged), which replaced the bindings' `max_delay` with - `Delay { min, max }` and edited `rs/moq-ffi/src/media.rs`. - -Public API: moq-mux's container consumer setup changes (breaking). The -bindings' subscribe surface is unchanged. Wire: none. diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index ab92b59dc07..c596a70683b 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -200,19 +200,15 @@ impl Consumer { let name = name.into(); let warmup = catalog.warmup.unwrap_or_default(); + let playout = moq_mux::container::Playout::new(options.delay, warmup); let track = broadcast.track(hang::catalog::Reference::track(catalog, &name))?; // The whole delay from the start, as the container keeps it: the transport enforces the // max too, so a narrower one while joining would drop the groups queued behind a reader // slow to take its first, and the publisher should never see a stated zero min first. let mut subscriber = track .subscribe( - moq_net::track::Subscription::default() - .with_priority(hang::catalog::PRIORITY.audio) - .with_delay( - options - .delay - .with_max(options.delay.max.map(|max| max.saturating_add(warmup))), - ), + playout + .subscription(moq_net::track::Subscription::default().with_priority(hang::catalog::PRIORITY.audio)), ) .await?; // Latest skips cached history; a warmup is primed by trimming the newest group's @@ -247,11 +243,9 @@ impl Consumer { // wire: `moq import fmp4` publishes CMAF. Reading a moof+mdat fragment as a // varint timestamp plus a payload decodes to garbage rather than failing. let container = moq_mux::catalog::hang::Container::try_from(catalog)?; - // The subscription's budget includes the warmup, so the start reaches the playout delay - // plus the warmup back, while the container skips at the max delay alone. - let track = moq_mux::container::Consumer::new(track, container) - .with_warmup(warmup) - .with_delay(options.delay); + // The start reaches the playout delay plus the warmup back, while the container skips at + // the max delay alone. + let track = playout.consumer(track, container); Ok(Self { target: Target::new(delay, options.estimate, catalog.jitter), diff --git a/rs/moq-c/src/consume.rs b/rs/moq-c/src/consume.rs index 71a47a26430..7aa54de22b9 100644 --- a/rs/moq-c/src/consume.rs +++ b/rs/moq-c/src/consume.rs @@ -533,7 +533,7 @@ impl Consume { // instead of assuming Legacy, otherwise CMAF/fMP4 sources (e.g. ffmpeg moqenc, // browser @moq/publish) are misread as raw frames. let container = moq_mux::catalog::hang::Container::try_from(config)?; - let warmup = config.warmup(); + let playout = moq_mux::container::Playout::new(delay, config.warmup()); // The rendition may live in a sibling broadcast, so resolve its reference rather than // assuming the catalog's own broadcast serves the track. let reference = config.broadcast().cloned(); @@ -548,10 +548,9 @@ impl Consume { |mut close| async move { driver .watch(async move { - let subscription = moq_net::track::Subscription::default() - .with_priority(C::PRIORITY) - // The publisher must retain the history the decoder needs too. - .with_delay(delay.with_max(delay.max.map(|max| max.saturating_add(warmup)))); + // The publisher must retain the history the decoder needs too. + let subscription = + playout.subscription(moq_net::track::Subscription::default().with_priority(C::PRIORITY)); // Resolving a sibling broadcast can wait too, so the close races both steps. let setup = async { let broadcast = resolve(broadcast, origin, reference).await?; @@ -560,9 +559,7 @@ impl Consume { let Some(track) = Self::until_closed(&mut close, setup).await? else { return Ok(()); }; - let track = moq_mux::container::Consumer::new(track, container) - .with_warmup(warmup) - .with_delay(delay); + let track = playout.consumer(track, container); Self::run_track(on_read, track, close).await }) .await diff --git a/rs/moq-ffi/MIGRATE.md b/rs/moq-ffi/MIGRATE.md index 83da290efe7..824cdce4bd1 100644 --- a/rs/moq-ffi/MIGRATE.md +++ b/rs/moq-ffi/MIGRATE.md @@ -5,6 +5,11 @@ These are the UniFFI names; each language wrapper has its own `MIGRATE.md` in it ## Unreleased +### Catalog renditions carry their warmup ([#7864](https://github.com/moq-dev/moq/pull/7864)) + +`MoqVideo` and `MoqAudio` gain `warmup`, the catalog's decoder warmup, null when absent. `MoqMediaTrackConsumer::subscribe` reads it from the rendition: it starts that much further back than the subscription's min delay and asks the publisher for its max delay plus the warmup, while it still skips at the max delay alone. `MoqAudioConsumer::subscribe` and `MoqVideoConsumer::subscribe` now honor it too. +The field defaults to null, so only code that builds these records exhaustively, such as a Rust struct literal or an unkeyed Go literal, changes. Pass the rendition from the catalog unchanged to keep its warmup. Regenerate bindings and rebuild the matching native library together. + ### Separate datagram tracks `MoqTrackProducer::{append_datagram, insert_datagram}` and `MoqTrackConsumer::recv_datagram` are removed. Use `MoqBroadcastProducer::create_datagrams(name, info)` and `MoqBroadcastConsumer::datagrams(name)` for separate `MoqDatagramProducer`/`MoqDatagramConsumer` handles. `append(timestamp, payload)` assigns a sequence; `insert(MoqDatagram)` preserves one; `recv()` reads it. `MoqDatagramInfo` has only priority and optional timescale. Watch aggregate subscriber priority with `priority()` / `priority_changed(last)`, and handle dynamic tracks with `MoqBroadcastRequests::datagrams()` / `MoqDatagramRequests::requested()`. Stream and datagram tracks of the same name have independent demand and lifetimes. diff --git a/rs/moq-ffi/src/audio.rs b/rs/moq-ffi/src/audio.rs index 8e2e56d1cd0..47da8b173a3 100644 --- a/rs/moq-ffi/src/audio.rs +++ b/rs/moq-ffi/src/audio.rs @@ -625,6 +625,7 @@ fn audio_config(catalog_audio: crate::media::MoqAudio) -> Result, /// The framing used to carry this rendition. pub container: MoqContainer, + /// How much media a decoder needs before its output is clean: after a join, decode from the + /// group start, present at the start plus this, and join that much earlier. Absent means none. + /// [`MoqMediaTrackConsumer::subscribe`] and the decoding consumers start this much earlier. + #[uniffi(default = None)] + pub warmup: Option, } /// An audio rendition and the configuration needed to decode it. @@ -198,6 +203,11 @@ pub struct MoqAudio { /// floors an estimated playout delay. Absent means frames flush at once. #[uniffi(default = None)] pub jitter: Option, + /// How much media a decoder needs before its output is clean: after a join, decode from the + /// group start, present at the start plus this, and join that much earlier. Absent means none. + /// [`MoqMediaTrackConsumer::subscribe`] and the decoding consumers start this much earlier. + #[uniffi(default = None)] + pub warmup: Option, } /// An exact media timestamp, `value` units at `scale` units per second, mirroring [`moq_net::Timestamp`]. @@ -621,6 +631,7 @@ pub(crate) fn convert_catalog(catalog: &moq_mux::catalog::hang::Catalog std::time::Duration { + match self { + Self::Video { config, .. } => config.warmup, + Self::Audio { config, .. } => config.warmup, + } + .unwrap_or_default() + } + /// Split into the broadcast reference and its instance, the track name, and the container. pub(crate) fn into_parts(self) -> (Option, Option, String, MoqContainer) { let (id, broadcast, instance, track, container) = match self { @@ -1529,6 +1550,97 @@ mod read_tests { use moq_mux::container::Container as _; use std::time::Duration; + /// A catalog rendition's warmup survives conversion, widens the raw subscription past the + /// caller's max delay, keeping the min delay as asked, and starts the reader that much further + /// back: with newest-first delivery it waits for a group covering the min delay plus the warmup. + #[tokio::test(start_paused = true)] + async fn raw_media_catalog_warmup_reaches_the_subscription() { + use crate::consumer::{MoqDelay, MoqSubscription, moq_subscription_defaults}; + + let warmup = Duration::from_millis(300); + let delay = Duration::from_millis(200); + for video in [false, true] { + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast.create_track("media", hang::container::track_info(0)).unwrap(); + let mut catalog = moq_mux::catalog::hang::Catalog::default(); + let mut audio = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); + audio.warmup = Some(warmup); + catalog.audio.renditions.insert("media".into(), audio); + let mut config = hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::VP8); + config.warmup = Some(warmup); + catalog.video.renditions.insert("media".into(), config); + let catalog = convert_catalog(&catalog); + let (rendition, kind) = match video { + true => ( + MoqRendition::Video { + id: "media".into(), + config: catalog.video["media"].clone(), + }, + moq_mux::container::Kind::Video, + ), + false => ( + MoqRendition::Audio { + id: "media".into(), + config: catalog.audio["media"].clone(), + }, + moq_mux::container::Kind::Audio, + ), + }; + let reader = MoqMediaTrackConsumer::subscribe( + &MoqBroadcastConsumer::new(broadcast.consume()), + MoqMediaTrackConfig { + rendition, + subscription: Some(MoqSubscription { + delay: MoqDelay { + min: Some(delay), + max: Some(delay), + }, + ..moq_subscription_defaults() + }), + }, + ) + .await + .unwrap(); + let subscription = track.subscription().unwrap(); + assert_eq!(subscription.delay.min, Some(delay), "video={video}"); + assert_eq!(subscription.delay.max, Some(delay + warmup), "video={video}"); + + let write = |sequence: u64, micros| { + let mut group = track.create_group(sequence.into()).unwrap(); + moq_mux::catalog::hang::Container::Legacy(kind) + .write( + &mut group, + &[moq_mux::container::Frame { + timestamp: moq_net::Timestamp::from_micros(micros).unwrap(), + payload: bytes::Bytes::from_static(b"media"), + keyframe: true, + duration: None, + }], + ) + .unwrap(); + group.finish().unwrap(); + }; + // Newest first: group 4 covers the min delay alone, but not the warmup too. + write(5, 5_000_000); + write(4, 4_700_000); + assert!( + kio::clock::timeout(Duration::from_millis(50), reader.next()) + .await + .is_err(), + "video={video}: the start waits for a group covering the warmup" + ); + write(3, 4_400_000); + track.finish().unwrap(); + let mut micros = Vec::new(); + while let Some(read) = reader.next().await.unwrap() { + if let MoqMediaRead::Frame { frame } = read { + micros.push(frame.timestamp.value); + } + } + assert_eq!(micros, [4_400_000, 4_700_000, 5_000_000], "video={video}"); + } + } + #[tokio::test(start_paused = true)] async fn encoded_media_preserves_breaks() { let broadcast = moq_net::broadcast::Info::new().produce(); @@ -1554,6 +1666,7 @@ mod read_tests { enabled: true, container: MoqContainer::Legacy, jitter: None, + warmup: None, }, }, subscription: None, @@ -1745,6 +1858,10 @@ impl MoqMediaTrackConsumer { /// Follows the rendition's broadcast reference and track name, so a rendition served by a /// sibling broadcast or under another track name subscribes to the right track, and fails /// one whose broadcast is proven to serve another stream than its instance. + /// + /// A rendition's [`warmup`](MoqVideo::warmup) starts the read that much further back than + /// the subscription's min delay, and widens the max delay the publisher serves by as much, + /// while the reader still skips at the max delay alone. #[uniffi::constructor] pub async fn subscribe( broadcast: &MoqBroadcastConsumer, @@ -1758,6 +1875,7 @@ impl MoqMediaTrackConsumer { MoqRendition::Audio { .. } => moq_mux::container::Kind::Audio, MoqRendition::Video { .. } => moq_mux::container::Kind::Video, }; + let warmup = rendition.warmup(); let (reference, instance, name, container) = rendition.into_parts(); // Parse the container before resolving or subscribing so we don't leave a dangling // subscription if init parsing fails. @@ -1769,11 +1887,13 @@ impl MoqMediaTrackConsumer { .map(moq_net::track::Subscription::try_from) .transpose()? .unwrap_or_default(); + let playout = moq_mux::container::Playout::new(subscription.delay, warmup); + let subscription = playout.subscription(subscription); let track = broadcast .driver .watch(async { Ok(resolved.track(&name)?.subscribe(subscription).await?) }) .await?; - let consumer = moq_mux::container::Consumer::new(track, media); + let consumer = playout.consumer(track, media); Ok(Arc::new(MoqMediaTrackConsumer { reads: Polled::new(consumer).supervised(broadcast.driver.clone()), })) diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index ee880532191..48a5bc1f66d 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -296,6 +296,7 @@ fn rendition(id: &str, container: MoqContainer) -> MoqRendition { enabled: true, framerate: None, container, + warmup: None, }, } } @@ -329,6 +330,7 @@ fn sibling_audio(reference: &str) -> crate::media::MoqAudio { enabled: true, container: MoqContainer::Legacy, jitter: None, + warmup: None, } } @@ -9284,6 +9286,7 @@ async fn video_subscribe_configuration_boundary() { display_aspect: None, bitrate: None, framerate: None, + warmup: None, }; let result = MoqVideoConsumer::subscribe( &consumer, diff --git a/rs/moq-ffi/src/video.rs b/rs/moq-ffi/src/video.rs index 98d482d2228..ba87c408eea 100644 --- a/rs/moq-ffi/src/video.rs +++ b/rs/moq-ffi/src/video.rs @@ -1260,6 +1260,7 @@ fn video_config(catalog_video: crate::media::MoqVideo) -> Result cmaf); the broadcast and track identify the media, apart from its catalog ID. -#[derive(Clone, PartialEq)] +/// +/// The catalog `warmup` rides along without being part of the shape: it matters only to a join, +/// so a changed one reaches the next reader without restarting the current one. +#[derive(Clone)] struct Shape { caps: gst::Caps, container: hang::catalog::Container, track: String, broadcast: Option, instance: Option, + warmup: Duration, +} + +impl PartialEq for Shape { + fn eq(&self, other: &Self) -> bool { + self.caps == other.caps + && self.container == other.container + && self.track == other.track + && self.broadcast == other.broadcast + && self.instance == other.instance + } } /// A rendition by kind and moq track name: what a pad is kept by. @@ -1060,6 +1074,7 @@ fn reconcile( kind, caps: Result, container: &hang::catalog::Container, + warmup: Option, reference: &dyn hang::catalog::Reference| match caps { Ok(caps) => { let shape = Shape { @@ -1068,16 +1083,31 @@ fn reconcile( track: reference.track(name).to_string(), broadcast: reference.broadcast().cloned(), instance: reference.instance().cloned(), + warmup: warmup.unwrap_or_default(), }; desired.insert((kind, name.clone()), shape); } Err(err) => gst::warning!(CAT, "ignoring {kind:?} rendition {name}: {err:?}"), }; for (name, config) in &playable.video.renditions { - insert(name, TrackKind::Video, video_caps(config), &config.container, config); + insert( + name, + TrackKind::Video, + video_caps(config), + &config.container, + config.warmup, + config, + ); } for (name, config) in &playable.audio.renditions { - insert(name, TrackKind::Audio, audio_caps(config), &config.container, config); + insert( + name, + TrackKind::Audio, + audio_caps(config), + &config.container, + config.warmup, + config, + ); } // A disabled rendition keeps its pad even when an enabled alternative displaced its reader, @@ -1396,9 +1426,11 @@ impl Follower { self.renditions .retain(|rendition, entry| match wanted.remove(rendition) { Some(shape) => { - // A new shape hands the pad to a new reader. - if shape != entry.shape { - entry.shape = shape; + // A new shape hands the pad to a new reader, and any shape keeps the latest + // warmup for the next one. + let reshaped = shape != entry.shape; + entry.shape = shape; + if reshaped { entry.reader = None; fresh.insert(rendition.clone()); } @@ -1641,12 +1673,15 @@ impl Reader { // ending a broadcast produces for a name nobody served). A publisher that answers neither // leaves this waiting, so racing `cancel` keeps such a reader reapable, and holding the // wait here keeps it off every other rendition. + // The catalog warmup reaches that much further back for a decoder to start clean. + let playout = + moq_mux::container::Playout::new(moq_net::track::Delay::default().with_max(max_delay), shape.warmup); let subscriber = tokio::select! { _ = cancel.changed() => return Exit::Done, subscriber = async { instance .track(&shape.track)? - .subscribe(moq_net::track::Subscription::default().with_delay(moq_net::track::Delay::default().with_max(max_delay))) + .subscribe(playout.subscription(moq_net::track::Subscription::default())) .await } => match subscriber { Ok(subscriber) => subscriber, @@ -1671,7 +1706,7 @@ impl Reader { return Exit::Done; } }; - let track = moq_mux::container::Consumer::new(subscriber, container); + let track = playout.consumer(subscriber, container); // Only a rendition that gets this far claims a pad id. let play = Play { @@ -1724,7 +1759,8 @@ impl Play { let (kind, name) = (rendition.0, &rendition.1); let mut sent_segment = false; // Each play starts a new stream on the pad, which opusdec decodes from a fresh state. - let mut pre_skip = PreSkip::new(&shape.caps); + let mut clip = OpusClip::new(&shape.caps, shape.warmup); + let mut first = true; loop { tokio::select! { // The follower stopped this reader (the instance changed, the rendition was @@ -1738,7 +1774,16 @@ impl Play { } frame = track.read() => match frame { Ok(Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. })) => {} - Ok(Some(moq_mux::container::Read::Frame { frame, .. })) => { + Ok(Some(moq_mux::container::Read::Frame { frame, group })) => { + // Nothing precedes group 0, and a group delivers from its frame 0 or not at + // all, so a play that reads it first decodes from the encoder's first packet, + // which has no history to warm up from. + if std::mem::take(&mut first) + && group == 0 + && let Some(clip) = &mut clip + { + clip.track_start(); + } let mut buffer = match build_buffer(frame, kind) { Ok(buffer) => buffer, Err(err) => { @@ -1748,8 +1793,8 @@ impl Play { return; } }; - if let Some(pre_skip) = &mut pre_skip - && let Err(err) = pre_skip.apply(buffer.get_mut().unwrap()) + if let Some(clip) = &mut clip + && let Err(err) = clip.apply(buffer.get_mut().unwrap()) { if let Some(obj) = element.upgrade() { gst::element_error!(obj, gst::StreamError::Format, ("track {name} has an invalid Opus packet"), ["{err:#}"]); @@ -1903,51 +1948,83 @@ fn build_buffer(frame: moq_mux::container::Frame, kind: TrackKind) -> Result Option { +impl OpusClip { + /// The clip of an `audio/x-opus` caps' OpusHead pre-skip and the rendition's `warmup`, or `None` + /// for other caps or nothing to clip. + fn new(caps: &gst::CapsRef, warmup: Duration) -> Option { let structure = caps.structure(0).filter(|s| s.name() == "audio/x-opus")?; - let headers = structure.get_optional::("streamheader").ok()??; - let head = headers.first()?.get::().ok()?; - let head = head.map_readable().ok()?; - let total = moq_mux::codec::opus::Config::parse(&mut head.as_slice()) - .ok()? - .pre_skip - .into(); - (total > 0).then_some(Self { + let total = structure + .get_optional::("streamheader") + .ok() + .flatten() + .and_then(|headers| { + let head = headers.first()?.get::().ok()?; + let head = head.map_readable().ok()?; + let config = moq_mux::codec::opus::Config::parse(&mut head.as_slice()).ok()?; + Some(u64::from(config.pre_skip)) + }) + .unwrap_or(0); + // Rounded up, so presentation never starts inside the warmup. + let warmup = u64::try_from((warmup.as_nanos() * 48_000).div_ceil(1_000_000_000)).unwrap_or(u64::MAX); + (total > 0 || warmup > 0).then_some(Self { total, remaining: total, + warmup, }) } - /// Clip this packet's share of the pre-skip, and stamp it where its first kept sample plays. - /// A packet's timestamp is where its decode would start without the pre-skip, so each one plays - /// the pre-skip earlier, less what it clips itself. + /// Decode from the track's first packet: the pre-skip is all the priming its encoder declared, + /// so no warmup is clipped. + fn track_start(&mut self) { + self.warmup = 0; + } + + /// Clip this packet's share of the pre-skip and then of the warmup, and stamp it where its + /// first kept sample plays. A packet's timestamp is where its decode would start without the + /// pre-skip, so each one plays the pre-skip earlier, less what it clips of it, and later by + /// what it clips of the warmup. fn apply(&mut self, buffer: &mut gst::BufferRef) -> Result<()> { - let mut clip = 0; - if self.remaining > 0 { - let samples = - moq_mux::codec::opus::packet_samples(&buffer.map_readable()?).context("malformed Opus packet")?; - clip = self.remaining.min(samples.into()); + let (mut clip, mut warm) = (0, 0); + if self.remaining > 0 || self.warmup > 0 { + let samples: u64 = moq_mux::codec::opus::packet_samples(&buffer.map_readable()?) + .context("malformed Opus packet")? + .into(); + clip = self.remaining.min(samples); self.remaining -= clip; - gst_audio::AudioClippingMeta::add(buffer, gst::format::Default::from_u64(clip), gst::format::Default::ZERO); + warm = self.warmup.min(samples - clip); + self.warmup -= warm; + gst_audio::AudioClippingMeta::add( + buffer, + gst::format::Default::from_u64(clip + warm), + gst::format::Default::ZERO, + ); } - let early = gst::ClockTime::from_nseconds( - (self.total - clip) + let time = |samples: u64| { + samples .mul_div_round(gst::ClockTime::SECOND.nseconds(), 48_000) - .context("pre-skip overflows the clock")?, - ); + .map(gst::ClockTime::from_nseconds) + .context("clip overflows the clock") + }; + let (early, late) = (time(self.total - clip)?, time(warm)?); // Only a packet clipped whole can reach before zero, and it plays nothing. - buffer.set_pts(buffer.pts().map(|pts| pts.saturating_sub(early))); + buffer.set_pts(buffer.pts().map(|pts| pts.saturating_add(late).saturating_sub(early))); Ok(()) } } @@ -2088,6 +2165,7 @@ fn opus_caps(config: &hang::catalog::AudioConfig, rate: i32, channels: i32) -> R mod tests { use super::{Timeline, ended}; use moq_net::Timestamp; + use std::time::Duration; /// A reader cancelled as its track ends records the end only when no media is left to read, /// passing over the markers a publisher closes with. @@ -2250,7 +2328,7 @@ mod tests { fn opus_pre_skip_is_clipped_across_packets() { gst::init().unwrap(); let mut config = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); - assert!(super::PreSkip::new(&super::audio_caps(&config).unwrap()).is_none()); + assert!(super::OpusClip::new(&super::audio_caps(&config).unwrap(), Duration::ZERO).is_none()); config.description = Some( moq_mux::codec::opus::Config::new(48_000, 2) .with_pre_skip(312) @@ -2264,7 +2342,7 @@ mod tests { (0xe0, 2_500, &[(120, 0), (120, 0), (72, 0), (0, 1_000), (0, 3_500)][..]), (0xf8, 20_000, &[(312, 0), (0, 13_500)][..]), ] { - let mut pre_skip = super::PreSkip::new(&caps).unwrap(); + let mut pre_skip = super::OpusClip::new(&caps, Duration::ZERO).unwrap(); for (index, &(clip, pts)) in expected.iter().enumerate() { let mut buffer = gst::Buffer::from_slice([toc]); let buffer = buffer.get_mut().unwrap(); @@ -2284,6 +2362,81 @@ mod tests { } } + /// After a join, the catalog warmup is clipped from the decoded output after the pre-skip, so + /// the first sample that plays is the first frame plus the warmup, while every packet still + /// reaches the decoder and keeps its place on the timeline. The track's first packet has no + /// warmup to clip. + #[test] + fn opus_warmup_is_clipped_after_the_pre_skip() { + gst::init().unwrap(); + let warmup = Duration::from_millis(80); + let mut config = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); + let bare = super::audio_caps(&config).unwrap(); + config.description = Some( + moq_mux::codec::opus::Config::new(48_000, 2) + .with_pre_skip(312) + .encode() + .unwrap(), + ); + let headed = super::audio_caps(&config).unwrap(); + assert!(super::OpusClip::new(&gst::Caps::new_empty_simple("video/x-h264"), warmup).is_none()); + + // 20 ms (960-sample) CELT packets: (clipped samples, pts in microseconds) for each, after a + // join, and from the track's first packet. + for (caps, track_start, expected) in [ + ( + &bare, + false, + &[ + (960, 20_000), + (960, 40_000), + (960, 60_000), + (960, 80_000), + (0, 80_000), + (0, 100_000), + ][..], + ), + ( + &headed, + false, + &[ + (960, 13_500), + (960, 33_500), + (960, 53_500), + (960, 73_500), + (312, 80_000), + (0, 93_500), + ][..], + ), + (&bare, true, &[(0, 0), (0, 20_000)][..]), + (&headed, true, &[(312, 0), (0, 13_500)][..]), + ] { + let mut clip = super::OpusClip::new(caps, warmup).unwrap(); + if track_start { + clip.track_start(); + } + for (index, &(clipped, pts)) in expected.iter().enumerate() { + let mut buffer = gst::Buffer::from_slice([0xf8u8]); + let buffer = buffer.get_mut().unwrap(); + buffer.set_pts(gst::ClockTime::from_useconds(index as u64 * 20_000)); + clip.apply(buffer).unwrap(); + let meta = buffer + .meta::() + .map(|meta| meta.start().value() as u64); + assert_eq!( + meta.unwrap_or(0), + clipped, + "packet {index} of {caps}, start {track_start}" + ); + assert_eq!( + buffer.pts(), + Some(gst::ClockTime::from_useconds(pts)), + "packet {index} of {caps}, start {track_start}" + ); + } + } + } + #[test] fn the_first_frame_fixes_the_segment_start_and_base() { gst::init().unwrap(); @@ -2822,6 +2975,56 @@ mod session_tests { super::RUNTIME.block_on(session).unwrap().unwrap(); } + /// A rendition's catalog warmup widens its subscription past the session max delay, so the + /// publisher keeps the media its decoder needs before the start. + #[test] + fn a_rendition_subscribes_with_its_catalog_warmup() { + let warmup = Duration::from_millis(300); + for kind in [TrackKind::Video, TrackKind::Audio] { + let element = element(); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let mut dynamic = broadcast.requests(); + let mut catalog = + moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + { + let mut guard = catalog.modify().unwrap(); + match kind { + TrackKind::Video => { + let mut video = video_rendition(); + video.warmup = Some(warmup); + guard.video.renditions = BTreeMap::from([("media".to_string(), video)]); + } + TrackKind::Audio => { + let mut audio = audio_rendition(); + audio.warmup = Some(warmup); + guard.audio.renditions = BTreeMap::from([("media".to_string(), audio)]); + } + } + } + + let max_delay = Duration::from_secs(5); + let (shutdown, mut cancelled) = watch::channel(false); + let weak = element.downgrade(); + let source = moq_mux::Source::new(origin().consume(), broadcast.consume()); + let session = super::RUNTIME.spawn(async move { + let stopping = cancelled.clone(); + Session::new(weak, stopping, max_delay) + .follow_catalog(source, &mut cancelled) + .await + }); + + let request = await_request(&mut dynamic); + assert_eq!( + request.subscription().unwrap().delay.max, + Some(max_delay + warmup), + "{kind:?}" + ); + + let _ = shutdown.send(true); + super::RUNTIME.block_on(session).unwrap().unwrap(); + } + } + /// A rendition delisted while its reader is still subscribing ends that reader, and answering /// the subscription afterwards must not resurrect it into a pad. Listing it again starts it /// anew, since the reader that left is no longer the rendition's. @@ -3504,6 +3707,8 @@ mod session_tests { broadcast: moq_net::broadcast::Producer, catalog: moq_mux::catalog::Producer, video: moq_mux::container::Producer, + /// The video track, for what its readers subscribe with. + track: moq_net::track::Producer, start: kio::clock::Instant, } @@ -3536,6 +3741,7 @@ mod session_tests { let mut guard = catalog.modify().unwrap(); guard.video.renditions = BTreeMap::from([("video".to_string(), config)]); } + let track = video.clone(); let video = moq_mux::container::Producer::new( video, moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), @@ -3545,6 +3751,7 @@ mod session_tests { broadcast, catalog, video, + track, start: kio::clock::now(), } } @@ -3699,6 +3906,7 @@ mod session_tests { track: "video".into(), broadcast: None, instance: None, + warmup: Duration::ZERO, }, instance, serial: 1, @@ -4050,6 +4258,67 @@ mod session_tests { recorder.assert_one_pad_held(); } + /// A warmup-only catalog update neither restarts the reader playing the rendition nor is lost: + /// the next instance's reader subscribes with the updated warmup. + #[test] + fn a_warmup_update_keeps_the_reader_and_reaches_the_next_one() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let mut config = video_rendition(); + config.broadcast = Some(moq_net::path::Relative::new("./cam").unwrap()); + config.warmup = Some(Duration::from_millis(300)); + let mut room = Publisher::new(&origin, Default::default(), config); + let session = follow(&element, &origin); + + let mut first = Publisher::sibling(&origin, Default::default()); + let pad = recorder.written(&mut first, 1); + assert_eq!( + first.track.subscription().unwrap().delay.max, + Some(DEFAULT_MAX_DELAY + Duration::from_millis(300)) + ); + + // Change only the warmup, then list an audio rendition in the same catalog. Its pad shows + // the run took the warmup too, since a catalog snapshot carries both, and the cam follower + // takes a plan before any instance event. + let audio = room + .broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let mut audio = moq_mux::container::Producer::new( + audio, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), + ); + { + let mut guard = room.catalog.modify().unwrap(); + guard.video.renditions.get_mut("video").unwrap().warmup = Some(Duration::from_millis(600)); + guard.audio.renditions = BTreeMap::from([("audio".to_string(), audio_rendition())]); + } + audio.write(room.frame(9)).unwrap(); + await_pad(&element, "audio_"); + assert_eq!(recorder.written(&mut first, 2), pad); + + let mut follow = watch(&origin, "cam"); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); + drop(first); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::End(_))); + let mut second = Publisher::sibling(&origin, Default::default()); + assert_eq!(recorder.written(&mut second, 3), pad, "the return moved to another pad"); + assert_eq!( + second.track.subscription().unwrap().delay.max, + Some(DEFAULT_MAX_DELAY + Duration::from_millis(600)) + ); + stop(session); + + let seen = recorder.seen(); + assert_eq!( + recorder.count(|seen| matches!(seen, Seen::Start(start) if *start == pad)), + 1, + "only the next instance starts a new stream on the pad: {seen:?}" + ); + } + /// A rendition that joins a sibling instance after another started on it shares that instance's /// timeline, so both map the instance's timestamps to one running time. #[test] diff --git a/rs/moq-mux/MIGRATE.md b/rs/moq-mux/MIGRATE.md index 090dd32c02e..d9d14bff8b7 100644 --- a/rs/moq-mux/MIGRATE.md +++ b/rs/moq-mux/MIGRATE.md @@ -4,6 +4,23 @@ How to adopt new APIs and update for breaking changes, newest first. ## Unreleased +### `container::Playout` replaces `with_warmup` ([#7864](https://github.com/moq-dev/moq/pull/7864)) + +`container::Consumer::with_warmup` is gone. Build a `container::Playout::new(delay, warmup)` from the `Delay` and the catalog rendition's `warmup`, subscribe with `playout.subscription(subscription)`, which widens the max delay by the warmup, and build the consumer with `playout.consumer(track, format)`, which skips at the max delay alone. The order no longer matters, and a track subscribed without the widening is widened when the consumer is built. + +```rust +// Before +let subscription = Subscription::default().with_delay(delay.with_max(delay.max.map(|max| max + warmup))); +let track = broadcast.track(name)?.subscribe(subscription).await?; +let consumer = Consumer::new(track, format).with_warmup(warmup).with_delay(delay); +// After +let playout = Playout::new(delay, warmup); +let track = broadcast.track(name)?.subscribe(playout.subscription(Subscription::default())).await?; +let consumer = playout.consumer(track, format); +``` + +A rendition without a warmup keeps `Consumer::new(track, format).with_delay(delay)`. + ### `catalog::Config::with_max_age(None)` is unlimited ([#7729](https://github.com/moq-dev/moq/pull/7729)) `Config::with_max_age` no longer treats `None` as "keep hang's default": an unset `Config` mints media tracks with `hang::container::MAX_AGE` (30 s), as before, `None` sets no publisher limit, and a value is that value. @@ -45,8 +62,8 @@ HLS replaces a refused importer immediately; transient fetch failures still keep `container::Consumer::set_max_delay(Duration)` is `set_delay(Delay)`, and `with_delay(Delay)` is its builder. `moq_net::track::Delay { min, max }` defaults to zero for both: `max` (`None` for no limit) is the max delay that `set_max_delay` took. `set_delay` puts the whole delay on the subscription, and `Consumer::new` inherits it from there. -`min` is the playout delay, and a consumer reads an unstated (`None`) one as zero. An unfloored consumer starts at the group covering the live edge less `min` plus any `with_warmup`, waiting for it when the newest group arrives first. Before the first read, a backlog that already landed starts at the first group whose first frame is within the max delay, plus the warmup, of the newest content, keeping the group covering the live edge less `min` and the warmup. -Replace `consumer.set_max_delay(d)` with `consumer.set_delay(Delay::default().with_max(d))`, and set `min` to your playout delay. Call `with_warmup` before `with_delay`. +`min` is the playout delay, and a consumer reads an unstated (`None`) one as zero. An unfloored consumer starts at the group covering the live edge less `min` plus any `Playout` warmup, waiting for it when the newest group arrives first. Before the first read, a backlog that already landed starts at the first group whose first frame is within the max delay, plus the warmup, of the newest content, keeping the group covering the live edge less `min` and the warmup. +Replace `consumer.set_max_delay(d)` with `consumer.set_delay(Delay::default().with_max(d))`, and set `min` to your playout delay. A rendition with a catalog warmup builds its consumer with `container::Playout` instead. ### Compression moves to the track diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index da4be0b5188..bcdc7893b04 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -75,7 +75,7 @@ pub(crate) enum Item { /// Delivery starts at the lowest absolute start among the /// [`Subscription::ranges`](moq_net::track::Subscription::ranges) when every range names one, waiting for that group under the budget; without one it /// starts at the lowest group that arrived, waiting for one covering the live edge less -/// [`Delay::min`] plus any [`with_warmup`](Self::with_warmup). Both default to zero, so it waits +/// [`Delay::min`] plus any [`Playout::warmup`]. Both default to zero, so it waits /// for nothing. Until the first item, the start also moves past each group whose first frame /// trails the newest content by more than the max delay plus the warmup, as `@moq/hang`'s /// consumer measures, unless that group covers the live edge less the min delay and the warmup: @@ -94,6 +94,7 @@ pub(crate) enum Item { /// Put the initial [`Delay`] on the [`moq_net::track::Subscription`] before /// subscribing. [`new`](Self::new) inherits it, reading an unstated [`Delay::min`] as zero, and /// [`with_delay`](Self::with_delay) or [`set_delay`](Self::set_delay) sets the whole [`Delay`]. +/// A rendition with a catalog `warmup` builds both with a [`Playout`] instead. /// /// ## Gaps and ends /// @@ -156,7 +157,7 @@ pub struct Consumer { // How far behind the live edge an unfloored start reaches, before the warmup: see `Delay::min`. min_delay: std::time::Duration, - // How much further than the min delay an unfloored start reaches: see `with_warmup`. + // How much further than the min delay an unfloored start reaches: see `Playout`. warmup: std::time::Duration, // When the newest group had landed as the first one arrived, once an unfloored start waits. @@ -273,6 +274,67 @@ fn gap(edge: std::time::Duration, resume: Timestamp) -> Item { Item::Gap { timestamp, duration } } +/// A [`Consumer`]'s [`Delay`] and the catalog rendition's warmup, applied together. +/// +/// The warmup is how much media a decoder needs before its output is clean, such as an Opus +/// decoder or the max-delay decoder of an intra-refresh rendition: pass the catalog rendition's +/// `warmup`. Subscribe with [`subscription`](Self::subscription), then build the consumer with +/// [`consumer`](Self::consumer). +/// +/// The subscription carries the max delay plus the warmup, since the publisher and the transport +/// skip what falls outside that budget, while the consumer skips at the max delay alone. +/// +/// Where delivery would start wherever the publisher does, it starts no later than the group +/// covering the live edge less [`Delay::min`] and the warmup: one that starts at or before that +/// point. Both default to zero, which the newest group covers, so the start waits for nothing. +/// Otherwise groups can arrive newest first, so when none that arrived covers that point the +/// start waits for an earlier one, until one covers it, the subscription's ranges hold none +/// earlier, the track finishes, the max buffer fills, or the newest group's arrival is that whole +/// reach ago. The wall clock bounds a quiet or paused track, which never ages out in media time. +/// The start is then the lowest group that arrived, and the max delay skips from there. Until the +/// reader is the warmup past both its first frame and that point, the skip also tolerates the +/// warmup, so the media leading up to the start is not dropped as stale. +/// +/// A subscription that names a start already waits for it, and a live start takes the newest +/// group. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Playout { + /// The delay the consumer plays at, and skips past. + pub delay: Delay, + /// How much further back than [`Delay::min`] the start reaches. + pub warmup: std::time::Duration, +} + +impl Playout { + /// Play at `delay`, starting `warmup` further back. + pub fn new(delay: Delay, warmup: std::time::Duration) -> Self { + Self { delay, warmup } + } + + /// `subscription` with this playout's delay, its max widened by the warmup. + pub fn subscription(&self, subscription: moq_net::track::Subscription) -> moq_net::track::Subscription { + subscription.with_delay(self.subscribed()) + } + + /// Decode `track` as `format` at this playout. + /// + /// Subscribe `track` with [`subscription`](Self::subscription): the publisher may already + /// have skipped what a narrower one left out. A track subscribed otherwise has its subscription + /// set to that delay here, widened or narrowed. + pub fn consumer(&self, track: moq_net::track::Subscriber, format: F) -> Consumer { + let mut consumer = Consumer::new(track, format); + consumer.warmup = self.warmup; + consumer.set_delay(self.delay); + consumer + } + + /// The delay the subscription carries. + fn subscribed(&self) -> Delay { + self.delay + .with_max(self.delay.max.map(|max| max.saturating_add(self.warmup))) + } +} + impl Consumer { /// Create a Consumer wrapping the given moq-lite consumer, decoding `format`. /// @@ -334,41 +396,6 @@ impl Consumer { self } - /// Reach `warmup` further back than the min delay for the first group, so a decoder that needs - /// that much media before its output is clean starts early enough, such as an Opus decoder or - /// the max-delay decoder of an intra-refresh rendition. Pass the catalog rendition's `warmup`. - /// - /// Put the max delay plus `warmup` on the [`moq_net::track::Subscription`], since the publisher - /// and the transport skip what falls outside that budget. This carves `warmup` back out of the - /// subscription's budget as it stands, so the consumer skips at the max delay alone. Call it - /// before [`with_delay`](Self::with_delay), which keeps the warmup on the subscription. - /// - /// Where delivery would start wherever the publisher does, it starts no later than the group - /// covering the live edge less [`Delay::min`] and the warmup: one that starts at or before that - /// point. Both default to zero, which the newest group covers, so the start waits for nothing. - /// Otherwise groups can arrive newest first, so when none that arrived covers that point the - /// start waits for an earlier one, until one covers it, the subscription's ranges hold none - /// earlier, the track finishes, the max buffer fills, or the newest group's arrival is that whole - /// reach ago. The wall clock bounds a quiet or paused track, which never ages out in media time. - /// The start is then the lowest group that arrived, and the max delay skips from there. Until the - /// reader is the warmup past both its first frame and that point, the skip also tolerates the - /// warmup, so the media leading up to the start is not dropped as stale. - /// - /// A subscription that names a start already waits for it, and a live start takes the newest - /// group. - pub fn with_warmup(mut self, warmup: std::time::Duration) -> Self { - // Carve the warmup out of what was asked for, then clamp to max age as `new` does, so a - // budget within the track's max age is not cut by the clamp too. - if let Some(track) = &self.track { - let requested = track.subscription().delay.max.unwrap_or(std::time::Duration::MAX); - self.max_delay = requested - .saturating_sub(warmup) - .min(track.info().max_age.unwrap_or(std::time::Duration::MAX)); - } - self.warmup = warmup; - self - } - /// Set the [`Delay`] as a builder; see [`set_delay`](Self::set_delay). pub fn with_delay(mut self, delay: Delay) -> Self { self.set_delay(delay); @@ -544,7 +571,7 @@ impl Consumer { // blocks, so the skip below would play it from its oldest group and stay that far behind // for good. The start moves past each group whose first frame trails the newest content // by more than the budget, as `@moq/hang`'s consumer measures, unless that group covers - // the live edge less the reach (see `with_warmup`), which the start promises. + // the live edge less the reach (see `Playout`), which the start promises. if self.edge.is_none() && self.start_skip(waiter)? { continue; } @@ -880,7 +907,7 @@ impl Consumer { } // Where an unfloored start begins, the lowest group that arrived, or `None` while it waits for - // one covering the reach (see `with_warmup`), registering the waiter on whatever would end the wait. + // one covering the reach (see `Playout`), registering the waiter on whatever would end the wait. fn reach_start(&mut self, waiter: &kio::Waiter, finished: bool) -> Option { let lowest = self.pending.front().expect("a group has a frame").sequence; // The publisher serves nothing older than the max delay, so a min delay past it reaches no @@ -1119,7 +1146,7 @@ impl Consumer { /// /// [`Delay::max`] is clamped to the track's max age like [`new`](Self::new). The /// subscription keeps the requested delay verbatim, its max plus any - /// [`with_warmup`](Self::with_warmup). An unstated [`Delay::min`] reads as zero here, and moves + /// [`Playout::warmup`]. An unstated [`Delay::min`] reads as zero here, and moves /// only a start still to come. A min past the max starts at the max, since the publisher serves /// nothing older. pub fn set_delay(&mut self, delay: Delay) { @@ -1135,7 +1162,7 @@ impl Consumer { // The transport enforces the same budget on the subscription itself, so a // tolerance set here has to reach it: otherwise moq-net skips the very groups // this consumer was told to wait for, before they ever get here. It keeps the - // warmup too, as `with_warmup` asks of the subscription. + // warmup too, as [`Playout::subscription`] does. let delay = delay.with_max(delay.max.map(|max| max.saturating_add(self.warmup))); if track.subscription().delay != delay { let subscription = track.subscription().with_delay(delay); @@ -3347,14 +3374,15 @@ mod tests { max: Duration, warmup: Duration, ) -> Consumer { - let consumer_track = track - .subscribe(moq_net::track::Subscription::default().with_delay(Delay::default().with_max(max + warmup))); - Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)) - .with_warmup(warmup) - .with_delay(Delay { + let playout = Playout::new( + Delay { min: Some(min), max: Some(max), - }) + }, + warmup, + ); + let consumer_track = track.subscribe(playout.subscription(moq_net::track::Subscription::default())); + playout.consumer(consumer_track, Container::Legacy(crate::container::Kind::Data)) } /// The live edge's stream wins the race, yet the start waits for a group covering the reach. @@ -3544,8 +3572,8 @@ mod tests { assert_eq!(frame.timestamp, ts(4_400_000)); } - /// The warmup is carved out of the requested budget before the max age clamp, so a max delay - /// within the track's max age stays whole. + /// The max age clamps the max delay alone, not the subscription's max plus the warmup, so a max + /// delay within the track's max age stays whole. #[tokio::test] async fn the_warmup_is_carved_before_the_max_age_clamp() { let mut info = hang::container::track_info(hang::catalog::PRIORITY.audio); @@ -3721,6 +3749,25 @@ mod tests { assert_eq!(consumer.max_delay, Duration::from_millis(200)); } + /// A track subscribed without the warmup is widened by the playout, and its consumer still + /// starts the warmup further back. + #[tokio::test] + async fn the_playout_widens_a_narrow_subscription() { + let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let delay = Delay { + min: Some(Duration::from_millis(100)), + max: Some(Duration::from_millis(200)), + }; + let consumer_track = track.subscribe(moq_net::track::Subscription::default().with_delay(delay)); + let consumer = Playout::new(delay, Duration::from_millis(300)) + .consumer(consumer_track, Container::Legacy(crate::container::Kind::Data)); + let subscription = consumer.track.as_ref().unwrap().subscription(); + assert_eq!(subscription.delay.min, Some(Duration::from_millis(100))); + assert_eq!(subscription.delay.max, Some(Duration::from_millis(500))); + assert_eq!(consumer.max_delay, Duration::from_millis(200)); + assert_eq!(consumer.warmup, Duration::from_millis(300)); + } + /// An unlimited max delay keeps the subscription unlimited, and the min delay still sets the /// start's reach. #[tokio::test] diff --git a/rs/moq-mux/src/container/mod.rs b/rs/moq-mux/src/container/mod.rs index 2de73f7a31f..a1110a298da 100644 --- a/rs/moq-mux/src/container/mod.rs +++ b/rs/moq-mux/src/container/mod.rs @@ -31,7 +31,7 @@ pub mod mkv; pub mod mp4; pub mod ts; -pub use consumer::{Consumer, DEFAULT_MAX_BUFFER}; +pub use consumer::{Consumer, DEFAULT_MAX_BUFFER, Playout}; pub use group::GroupConsumer; pub use producer::Producer; pub(crate) use source::{ExportSource, Resume}; diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index e0410c0f1c3..07e768aa03d 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -319,14 +319,17 @@ impl Reader { err => err, })?; + // The max-delay decoder tolerates its warmup as lateness, so its delay already holds the + // warmup on both bounds: neither lane passes the catalog warmup to the playout, which would + // count it twice, on the subscription and in the start's reach. + let playout = moq_mux::container::Playout::new(delay, std::time::Duration::ZERO); let track = broadcast.track(hang::catalog::Reference::track(catalog, name))?; let mut subscriber = track .subscribe( - moq_net::track::Subscription::default() - .with_priority(hang::catalog::PRIORITY.video) - // The whole delay from the first request, as the container keeps it, so the - // publisher never sees a stated zero min before the container's update. - .with_delay(delay), + // The whole delay from the first request, as the container keeps it, so the + // publisher never sees a stated zero min before the container's update. + playout + .subscription(moq_net::track::Subscription::default().with_priority(hang::catalog::PRIORITY.video)), ) .await?; // A decoder often opens on a track that is already cached: a replacement @@ -360,8 +363,8 @@ impl Reader { // On a cold network join the live edge's stream often lands first. The container starts // at the group covering the live edge less the min delay, waiting for it, so the // max-delay decoder, whose min delay includes the warmup, still gets the group its warmup - // needs. That decoder tolerates the warmup as lateness too, so it passes no further warmup. - let track = moq_mux::container::Consumer::new(track, container).with_delay(delay); + // needs. + let track = playout.consumer(track, container); Ok(Self { decoder, diff --git a/swift/MIGRATE.md b/swift/MIGRATE.md index 99088bb1411..2b02176b343 100644 --- a/swift/MIGRATE.md +++ b/swift/MIGRATE.md @@ -4,6 +4,10 @@ How to update your code for each breaking change, newest first. ## Unreleased +### Catalog renditions carry their warmup ([#7864](https://github.com/moq-dev/moq/pull/7864)) + +`Media.Video` and `Media.Audio` gain `warmup`, the catalog's decoder warmup; it defaults to `nil`, so existing initializers still compile. `Media.TrackConsumer` and the audio and video consumers read it from the rendition and start that much earlier. + ### Separate datagram tracks Replace stream-track `appendDatagram` / `insertDatagram` / `recvDatagram` with `BroadcastProducer.createDatagrams(name:info:)` and `BroadcastConsumer.datagrams(name:)`. `DatagramProducer.append(_:timestamp:)` assigns a sequence; `insert(_:)` takes a `Datagram`. `DatagramConsumer.recv()` and `AsyncSequence` read its independent cursor. Use `priority()` / `priorityChanged(last:)` for demand, and `BroadcastRequests.datagrams()` / `DatagramRequests.requested()` for on-demand tracks. `DatagramInfo` has priority and optional timescale, without stream ranges or cache settings.