diff --git a/cpp/moq/MIGRATE.md b/cpp/moq/MIGRATE.md index c3f53f138ba..43f753fb5b8 100644 --- a/cpp/moq/MIGRATE.md +++ b/cpp/moq/MIGRATE.md @@ -2,6 +2,13 @@ ## Unreleased +### Decoder outputs take a `moq::Delay` ([#7311](https://github.com/moq-dev/moq/pull/7311)) + +`moq::AudioDecoderOutput` replaces `max_delay` and the playout `delay` with `std::optional delay` and `bool estimate = true`. +`moq::VideoDecoderOutput::max_delay` is `std::optional delay`. +`moq::Delay{min, max}`: `max` (`std::nullopt` for no limit) is the old `max_delay`, and `min` is the playout delay, where a cold join starts. +A fixed audio playout delay `d` is `delay = moq::Delay{d, max}` with `estimate = false`. + ### Paths with an empty segment are refused ([#6648](https://github.com/moq-dev/moq/pull/6648)) Publishing, serving, or requesting a broadcast at a path with a leading, trailing, or doubled `/` fails instead of collapsing it. Pass the canonical path, such as `room/cam`. diff --git a/cpp/moq/test/probe.cpp b/cpp/moq/test/probe.cpp index e9934089a86..71a3812f765 100644 --- a/cpp/moq/test/probe.cpp +++ b/cpp/moq/test/probe.cpp @@ -121,7 +121,7 @@ void audio_roundtrip() { decoded.format = moq::AudioSampleFormat::kF32; // The track's whole retention: reading starts only after finish, and the budget ages // groups by wall clock, so a stalled host would otherwise skip the start. - decoded.max_delay = std::chrono::seconds(30); + decoded.delay = moq::Delay{std::chrono::seconds(0), std::chrono::seconds(30)}; auto consumer = ok(moq::AudioConsumer::subscribe(receiver, "tone", snapshot->audio.at("tone"), decoded).get(), "audio consumer"); for (size_t packet = 0; packet < 8; packet++) { std::vector data(960 * 2 * sizeof(float)); diff --git a/cpp/obs/test/moq-audio-encoder-test.cpp b/cpp/obs/test/moq-audio-encoder-test.cpp index 0ac27c34ca0..6c1f969702f 100644 --- a/cpp/obs/test/moq-audio-encoder-test.cpp +++ b/cpp/obs/test/moq-audio-encoder-test.cpp @@ -236,7 +236,7 @@ void CheckSubscriberDecodes(const char *preset, size_t channels) moq::AudioDecoderOutput output{moq::AudioSampleFormat::kF32}; output.channels = static_cast(channels); // The packets are written faster than real time, so let none be skipped as stale. - output.max_delay = std::chrono::seconds(10); + output.delay = moq::Delay{std::chrono::seconds(0), std::chrono::seconds(10)}; auto audio = TestOk(moq::AudioConsumer::subscribe(consumer, name, rendition, output).get(), "audio consumer"); const int count = 50; diff --git a/dart/moq/MIGRATE.md b/dart/moq/MIGRATE.md index 70ae6e9c3ab..10ec0483f9c 100644 --- a/dart/moq/MIGRATE.md +++ b/dart/moq/MIGRATE.md @@ -4,6 +4,13 @@ How to update your code for each breaking change, newest first. ## Unreleased +### Decoder outputs take a `Delay` ([#7311](https://github.com/moq-dev/moq/pull/7311)) + +`audio.DecoderOutput` replaces `maxDelay` and the playout `delay: Duration?` with `delay: media.Delay?` and `estimate: bool = true`. +`video.DecoderOutput.maxDelay` is `delay: media.Delay?`. +`Delay(min: ..., max: ...)`: `max` (null for no limit) is the old `maxDelay`, and `min` is the playout delay, where a cold join starts. +A fixed audio playout delay `d` is `delay: Delay(min: d, max: ...)` with `estimate: false`. + ### Compression moves to the track `package:moq/flate.dart` is removed: publish opaque bytes on a `TrackProducer`. diff --git a/dart/moq/lib/media.dart b/dart/moq/lib/media.dart index f683abe94fd..0238b8b70ef 100644 --- a/dart/moq/lib/media.dart +++ b/dart/moq/lib/media.dart @@ -75,6 +75,9 @@ typedef ContainerInit = MoqContainerInit; /// A width and height pair, in pixels. typedef Dimensions = MoqDimensions; +/// How far behind the live edge a decoder plays: `min` is the playout delay and where a cold join starts, and `max`, null for no limit, is how far a stalled group may fall behind before it is skipped. `min` stays on the consumer; a `min` past `max` starts at `max`. +typedef Delay = MoqDelay; + /// Where an audio or video decoder starts on a track that already holds groups: `latest` for a live player. typedef DecodeStart = MoqDecodeStart; diff --git a/dart/moq/lib/video.dart b/dart/moq/lib/video.dart index b7171c06451..0d220a93aaa 100644 --- a/dart/moq/lib/video.dart +++ b/dart/moq/lib/video.dart @@ -37,7 +37,7 @@ typedef EncoderInput = MoqVideoEncoderInput; /// The codec, bitrate, group mode, and backend preference. typedef EncoderOutput = MoqVideoEncoderOutput; -/// What a [Consumer] delivers: an optional resize, a max delay, and whether to keep the decoder's surface (macOS and iOS). +/// What a [Consumer] delivers: an optional resize, a delay, and whether to keep the decoder's surface (macOS and iOS). typedef DecoderOutput = MoqVideoDecoderOutput; /// One raw picture with its presentation timestamp. diff --git a/dart/moq/test/moq_test.dart b/dart/moq/test/moq_test.dart index a772ace5575..025d8538ae8 100644 --- a/dart/moq/test/moq_test.dart +++ b/dart/moq/test/moq_test.dart @@ -681,7 +681,11 @@ void main() { final fixedOutput = moq_audio.DecoderOutput( format: moq_audio.SampleFormat.f32, - delay: const Duration(milliseconds: 80), + delay: moq_media.Delay( + min: Duration(milliseconds: 80), + max: Duration.zero, + ), + estimate: false, ); final fixed = await moq_audio.Consumer.subscribe( broadcast: consumer, diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index 535337e1521..64ecbf455ab 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -16,15 +16,15 @@ class MoqAudioDecoderOutput { final MoqAudioSampleFormat format; final int? sampleRate; final int? channels; - final Duration? maxDelay; - final Duration? delay; + final MoqDelay? delay; + final bool estimate; final MoqDecodeStart? start; MoqAudioDecoderOutput({ required this.format, this.sampleRate = null, this.channels = null, - this.maxDelay = null, this.delay = null, + this.estimate = true, this.start = null, }); } @@ -51,16 +51,16 @@ class FfiConverterMoqAudioDecoderOutput { ); final channels = channels_lifted.value; new_offset += channels_lifted.bytesRead; - final maxDelay_lifted = FfiConverterOptionalDuration.read( - Uint8List.view(buf.buffer, new_offset), - ); - final maxDelay = maxDelay_lifted.value; - new_offset += maxDelay_lifted.bytesRead; - final delay_lifted = FfiConverterOptionalDuration.read( + final delay_lifted = FfiConverterOptionalMoqDelay.read( Uint8List.view(buf.buffer, new_offset), ); final delay = delay_lifted.value; new_offset += delay_lifted.bytesRead; + final estimate_lifted = FfiConverterBool.read( + Uint8List.view(buf.buffer, new_offset), + ); + final estimate = estimate_lifted.value; + new_offset += estimate_lifted.bytesRead; final start_lifted = FfiConverterOptionalMoqDecodeStart.read( Uint8List.view(buf.buffer, new_offset), ); @@ -71,8 +71,8 @@ class FfiConverterMoqAudioDecoderOutput { format: format, sampleRate: sampleRate, channels: channels, - maxDelay: maxDelay, delay: delay, + estimate: estimate, start: start, ), new_offset - buf.offsetInBytes, @@ -84,8 +84,8 @@ class FfiConverterMoqAudioDecoderOutput { FfiConverterMoqAudioSampleFormat.allocationSize(value.format) + FfiConverterOptionalUInt32.allocationSize(value.sampleRate) + FfiConverterOptionalUInt32.allocationSize(value.channels) + - FfiConverterOptionalDuration.allocationSize(value.maxDelay) + - FfiConverterOptionalDuration.allocationSize(value.delay) + + FfiConverterOptionalMoqDelay.allocationSize(value.delay) + + FfiConverterBool.allocationSize(value.estimate) + FfiConverterOptionalMoqDecodeStart.allocationSize(value.start) + 0; final buf = Uint8List(total_length); @@ -107,12 +107,12 @@ class FfiConverterMoqAudioDecoderOutput { value.channels, Uint8List.view(buf.buffer, new_offset), ); - new_offset += FfiConverterOptionalDuration.write( - value.maxDelay, + new_offset += FfiConverterOptionalMoqDelay.write( + value.delay, Uint8List.view(buf.buffer, new_offset), ); - new_offset += FfiConverterOptionalDuration.write( - value.delay, + new_offset += FfiConverterBool.write( + value.estimate, Uint8List.view(buf.buffer, new_offset), ); new_offset += FfiConverterOptionalMoqDecodeStart.write( @@ -126,8 +126,8 @@ class FfiConverterMoqAudioDecoderOutput { return FfiConverterMoqAudioSampleFormat.allocationSize(value.format) + FfiConverterOptionalUInt32.allocationSize(value.sampleRate) + FfiConverterOptionalUInt32.allocationSize(value.channels) + - FfiConverterOptionalDuration.allocationSize(value.maxDelay) + - FfiConverterOptionalDuration.allocationSize(value.delay) + + FfiConverterOptionalMoqDelay.allocationSize(value.delay) + + FfiConverterBool.allocationSize(value.estimate) + FfiConverterOptionalMoqDecodeStart.allocationSize(value.start) + 0; } @@ -1711,6 +1711,65 @@ class FfiConverterMoqDatagram { } } +class MoqDelay { + final Duration min; + final Duration? max; + MoqDelay({required this.min, this.max}); +} + +class FfiConverterMoqDelay { + static MoqDelay lift(RustBuffer buf) { + return FfiConverterMoqDelay.read(buf.asUint8List()).value; + } + + static LiftRetVal read(Uint8List buf) { + int new_offset = buf.offsetInBytes; + final min_lifted = FfiConverterDuration.read( + Uint8List.view(buf.buffer, new_offset), + ); + final min = min_lifted.value; + new_offset += min_lifted.bytesRead; + final max_lifted = FfiConverterOptionalDuration.read( + Uint8List.view(buf.buffer, new_offset), + ); + final max = max_lifted.value; + new_offset += max_lifted.bytesRead; + return LiftRetVal( + MoqDelay(min: min, max: max), + new_offset - buf.offsetInBytes, + ); + } + + static RustBuffer lower(MoqDelay value) { + final total_length = + FfiConverterDuration.allocationSize(value.min) + + FfiConverterOptionalDuration.allocationSize(value.max) + + 0; + final buf = Uint8List(total_length); + write(value, buf); + return toRustBuffer(buf); + } + + static int write(MoqDelay value, Uint8List buf) { + int new_offset = buf.offsetInBytes; + new_offset += FfiConverterDuration.write( + value.min, + Uint8List.view(buf.buffer, new_offset), + ); + new_offset += FfiConverterOptionalDuration.write( + value.max, + Uint8List.view(buf.buffer, new_offset), + ); + return new_offset - buf.offsetInBytes; + } + + static int allocationSize(MoqDelay value) { + return FfiConverterDuration.allocationSize(value.min) + + FfiConverterOptionalDuration.allocationSize(value.max) + + 0; + } +} + class MoqDimensions { final int width; final int height; @@ -4196,12 +4255,12 @@ class FfiConverterMoqWebSocketConfig { class MoqVideoDecoderOutput { final MoqDimensions? resize; - final Duration? maxDelay; + final MoqDelay? delay; final bool surface; final MoqDecodeStart? start; MoqVideoDecoderOutput({ this.resize = null, - this.maxDelay = null, + this.delay = null, this.surface = false, this.start = null, }); @@ -4219,11 +4278,11 @@ class FfiConverterMoqVideoDecoderOutput { ); final resize = resize_lifted.value; new_offset += resize_lifted.bytesRead; - final maxDelay_lifted = FfiConverterOptionalDuration.read( + final delay_lifted = FfiConverterOptionalMoqDelay.read( Uint8List.view(buf.buffer, new_offset), ); - final maxDelay = maxDelay_lifted.value; - new_offset += maxDelay_lifted.bytesRead; + final delay = delay_lifted.value; + new_offset += delay_lifted.bytesRead; final surface_lifted = FfiConverterBool.read( Uint8List.view(buf.buffer, new_offset), ); @@ -4237,7 +4296,7 @@ class FfiConverterMoqVideoDecoderOutput { return LiftRetVal( MoqVideoDecoderOutput( resize: resize, - maxDelay: maxDelay, + delay: delay, surface: surface, start: start, ), @@ -4248,7 +4307,7 @@ class FfiConverterMoqVideoDecoderOutput { static RustBuffer lower(MoqVideoDecoderOutput value) { final total_length = FfiConverterOptionalMoqDimensions.allocationSize(value.resize) + - FfiConverterOptionalDuration.allocationSize(value.maxDelay) + + FfiConverterOptionalMoqDelay.allocationSize(value.delay) + FfiConverterBool.allocationSize(value.surface) + FfiConverterOptionalMoqDecodeStart.allocationSize(value.start) + 0; @@ -4263,8 +4322,8 @@ class FfiConverterMoqVideoDecoderOutput { value.resize, Uint8List.view(buf.buffer, new_offset), ); - new_offset += FfiConverterOptionalDuration.write( - value.maxDelay, + new_offset += FfiConverterOptionalMoqDelay.write( + value.delay, Uint8List.view(buf.buffer, new_offset), ); new_offset += FfiConverterBool.write( @@ -4280,7 +4339,7 @@ class FfiConverterMoqVideoDecoderOutput { static int allocationSize(MoqVideoDecoderOutput value) { return FfiConverterOptionalMoqDimensions.allocationSize(value.resize) + - FfiConverterOptionalDuration.allocationSize(value.maxDelay) + + FfiConverterOptionalMoqDelay.allocationSize(value.delay) + FfiConverterBool.allocationSize(value.surface) + FfiConverterOptionalMoqDecodeStart.allocationSize(value.start) + 0; @@ -15459,6 +15518,51 @@ class FfiConverterOptionalMoqDecodeStart { } } +class FfiConverterOptionalMoqDelay { + static MoqDelay? lift(RustBuffer buf) { + return FfiConverterOptionalMoqDelay.read(buf.asUint8List()).value; + } + + static LiftRetVal read(Uint8List buf) { + if (ByteData.view(buf.buffer, buf.offsetInBytes).getInt8(0) == 0) { + return LiftRetVal(null, 1); + } + final result = FfiConverterMoqDelay.read( + Uint8List.view(buf.buffer, buf.offsetInBytes + 1), + ); + return LiftRetVal(result.value, result.bytesRead + 1); + } + + static int allocationSize([MoqDelay? value]) { + if (value == null) { + return 1; + } + return FfiConverterMoqDelay.allocationSize(value) + 1; + } + + static RustBuffer lower(MoqDelay? value) { + if (value == null) { + return toRustBuffer(Uint8List.fromList([0])); + } + final buf = Uint8List(FfiConverterOptionalMoqDelay.allocationSize(value)); + FfiConverterOptionalMoqDelay.write(value, buf); + return toRustBuffer(buf); + } + + static int write(MoqDelay? value, Uint8List buf) { + if (value == null) { + buf[0] = 0; + return 1; + } + buf[0] = 1; + return FfiConverterMoqDelay.write( + value, + Uint8List.view(buf.buffer, buf.offsetInBytes + 1), + ) + + 1; + } +} + class FfiConverterOptionalMoqDimensions { static MoqDimensions? lift(RustBuffer buf) { return FfiConverterOptionalMoqDimensions.read(buf.asUint8List()).value; @@ -20937,7 +21041,7 @@ void _checkApiChecksums() { if (uniffi_moq_ffi_checksum_method_moqaudioconsumer_cancel() != 62285) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqaudioconsumer_delay() != 48052) { + if (uniffi_moq_ffi_checksum_method_moqaudioconsumer_delay() != 18211) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqaudioconsumer_next() != 7650) { diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index 8a808a5cab7..242036b00e6 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -472,9 +472,11 @@ or one beyond the track's retention window, is an error at configuration time. Supported or refused, not warned about and continued. In the browser this is the numeric `delay` setting; natively it is -`decode::Options::delay`, where `None` selects the estimator. The two spell the -same thing the same way on purpose. It is distinct from `max_delay`, which is the -live-edge skip budget and not a buffer. +`decode::Options::delay.min` with `decode::Options::estimate` set to `false`. +`estimate` defaults to `true`, which runs the estimator instead and never goes +below that `min`. The two spell the same thing the same way on purpose. It is +distinct from `delay.max`, which is the live-edge skip budget and not a buffer. +The same `min` is where a cold join starts. ## Constants diff --git a/doc/lib/index.md b/doc/lib/index.md index 1dcafa6bba6..ddc34d5a9a0 100644 --- a/doc/lib/index.md +++ b/doc/lib/index.md @@ -77,7 +77,7 @@ how it looks in that language: - **Connection stats** and **errors** that tell an auth rejection (don't retry) from a shutdown or a transport failure. - **Authorization in band**: read the grant the relay sent back (the patterns this side may publish and subscribe to, and when it lapses), and present a fresh token before the current one expires, without reconnecting. It needs `moq-lite-07-wip`, which is opt-in, or moq-transport draft-18+ with the [MoQ Auth extension](/draft/moq-auth). -C's audio decoder neither takes nor reports a playout delay, and C has no in-band authorization. +C's audio decoder takes a playout delay only as the floor `moq_delay.min_us` under its estimate, and reports none; C has no in-band authorization. ## Upgrading diff --git a/doc/lib/rs/hang.md b/doc/lib/rs/hang.md index 4e669380196..d56435f89b7 100644 --- a/doc/lib/rs/hang.md +++ b/doc/lib/rs/hang.md @@ -22,7 +22,12 @@ cargo add hang hang is the format, not the pipeline. [`moq-mux`](/lib/rs/moq-mux) produces the catalog for you and decodes any container in group order -(`moq_mux::container::Consumer`), skipping groups that fall past your max delay. +(`moq_mux::container::Consumer`). Its `Delay { min, max }` says how far behind +the live edge to play: it skips groups that fall past `max`, and starts at the +group covering the live edge less `min` (your playout delay), plus a rendition's +catalog `warmup` passed to `with_warmup`, waiting up to that long for it when the +newest group arrives first. Both default to zero, which starts at the newest +group at once; `max` never moves the start. [`moq-video`](/lib/rs/moq-video) and [`moq-audio`](/lib/rs/moq-audio) publish from a device. Examples: [`video.rs`](https://github.com/moq-dev/moq/blob/main/rs/hang/examples/video.rs) diff --git a/go/wrapper/MIGRATE.md b/go/wrapper/MIGRATE.md index 3bbdd1df7d5..1ac64f9a9a2 100644 --- a/go/wrapper/MIGRATE.md +++ b/go/wrapper/MIGRATE.md @@ -4,6 +4,13 @@ How to update your code for each breaking change, newest first. ## Unreleased +### Decoder outputs take a `media.Delay` ([#7311](https://github.com/moq-dev/moq/pull/7311)) + +`audio.DecoderOutput` replaces `MaxDelay` and the playout `Delay *time.Duration` with `Delay *media.Delay` and `Estimate *bool` (nil estimates). +`video.DecoderOutput.MaxDelay` is `Delay *media.Delay`. +`media.Delay{Min, Max}`: `Max` (nil for no limit) is the old `MaxDelay`, and `Min` is the playout delay, where a cold join starts. +A fixed audio playout delay `d` is `Delay: &media.Delay{Min: d, Max: &max}` with `Estimate` set to false. + ### Compression moves to the track `moq.dev/moq/flate` is removed: publish opaque bytes on a `TrackProducer`. diff --git a/go/wrapper/audio/audio.go b/go/wrapper/audio/audio.go index cf76962345e..432ff028329 100644 --- a/go/wrapper/audio/audio.go +++ b/go/wrapper/audio/audio.go @@ -51,17 +51,20 @@ type DecoderOutput struct { Format SampleFormat SampleRate *uint32 Channels *uint32 - // MaxDelay bounds how stale a group may get before it is skipped; it adds no buffering. - MaxDelay *time.Duration - // Delay fixes the jitter buffer to hold ahead of the playhead; nil estimates it from arrival timing. - Delay *time.Duration + // Delay.Min is the jitter buffer to hold ahead of the playhead, and where a cold join starts; + // Delay.Max bounds how stale a group may get before it is skipped, and adds no buffering. nil + // is zero for both. + Delay *media.Delay + // Estimate measures the playout delay from arrival timing, never below Delay.Min; false holds + // exactly Delay.Min. nil estimates. + Estimate *bool // Start is where to begin on a track that already holds groups; zero is media.DecodeStartOldest. // A live player sets media.DecodeStartLatest so a reopened decoder skips the cached backlog. Start media.DecodeStart } func (o DecoderOutput) ffi() (ffi.MoqAudioDecoderOutput, error) { - if err := bridge.NonNegative(o.MaxDelay, "audio max delay"); err != nil { + if err := delay(o.Delay, "audio"); err != nil { return ffi.MoqAudioDecoderOutput{}, err } // Go's zero value is the native null: the oldest group. @@ -69,7 +72,19 @@ func (o DecoderOutput) ffi() (ffi.MoqAudioDecoderOutput, error) { if o.Start != 0 { start = &o.Start } - return ffi.MoqAudioDecoderOutput{Format: o.Format, SampleRate: o.SampleRate, Channels: o.Channels, MaxDelay: o.MaxDelay, Delay: o.Delay, Start: start}, bridge.NonNegative(o.Delay, "audio delay") + estimate := o.Estimate == nil || *o.Estimate + return ffi.MoqAudioDecoderOutput{Format: o.Format, SampleRate: o.SampleRate, Channels: o.Channels, Delay: o.Delay, Estimate: estimate, Start: start}, nil +} + +// delay refuses a negative bound, which the FFI cannot carry. +func delay(d *media.Delay, kind string) error { + if d == nil { + return nil + } + if err := bridge.NonNegative(&d.Min, kind+" min delay"); err != nil { + return err + } + return bridge.NonNegative(d.Max, kind+" max delay") } // Frame is interleaved PCM with its presentation timestamp. diff --git a/go/wrapper/media/media.go b/go/wrapper/media/media.go index 9acaab95917..2c769bc6860 100644 --- a/go/wrapper/media/media.go +++ b/go/wrapper/media/media.go @@ -23,6 +23,10 @@ type ( Audio = ffi.MoqAudio // Catalog is a broadcast's manifest: its video and audio renditions plus display metadata. Catalog = ffi.MoqCatalog + // Delay is how far behind the live edge a decoder plays: Min is the playout delay and where a + // cold join starts, and Max, nil for no limit, is how far a stalled group may fall behind before + // it is skipped. Min stays on the consumer; a Min past Max starts at Max. + Delay = ffi.MoqDelay // Dimensions is a width and height in pixels. Dimensions = ffi.MoqDimensions // DecodeStart is where an audio or video decoder starts on a track that already holds groups. diff --git a/go/wrapper/moq_test.go b/go/wrapper/moq_test.go index 22ee427df03..dd927abf536 100644 --- a/go/wrapper/moq_test.go +++ b/go/wrapper/moq_test.go @@ -288,7 +288,13 @@ func TestAudioConsumerDelay(t *testing.T) { } fixedDelay := 80 * time.Millisecond - fixed, err := moqaudio.Subscribe(ctx, bc, track, rendition, moqaudio.DecoderOutput{Format: moq.AudioSampleFormatF32, Delay: &fixedDelay}) + noMax := time.Duration(0) + estimate := false + fixed, err := moqaudio.Subscribe(ctx, bc, track, rendition, moqaudio.DecoderOutput{ + Format: moq.AudioSampleFormatF32, + Delay: &moqmedia.Delay{Min: fixedDelay, Max: &noMax}, + Estimate: &estimate, + }) if err != nil { t.Fatal(err) } diff --git a/go/wrapper/video/video.go b/go/wrapper/video/video.go index c7fb40318d7..17eb552ce7e 100644 --- a/go/wrapper/video/video.go +++ b/go/wrapper/video/video.go @@ -59,11 +59,13 @@ func packet(raw ffi.MoqVideoPacket) Packet { return Packet{Timestamp: time.Duration(raw.TimestampUs) * time.Microsecond, Data: raw.Data, Keyframe: raw.Keyframe} } -// DecoderOutput configures an optional resize, a max delay, whether frames keep the decoder's surface (macOS only), and where to start. +// DecoderOutput configures an optional resize, a delay, whether frames keep the decoder's surface (macOS only), and where to start. type DecoderOutput struct { - Resize *media.Dimensions - MaxDelay *time.Duration - Surface bool + Resize *media.Dimensions + // Delay.Min is the playout delay, where a cold join starts; Delay.Max is how far a stalled + // group may fall behind before it is skipped. nil is zero for both. + Delay *media.Delay + Surface bool // Start is where to begin on a track that already holds groups; zero is media.DecodeStartOldest. // A live player sets media.DecodeStartLatest so a reopened decoder skips the cached backlog. Start media.DecodeStart @@ -75,7 +77,14 @@ func (o DecoderOutput) ffi() (ffi.MoqVideoDecoderOutput, error) { if o.Start != 0 { start = &o.Start } - return ffi.MoqVideoDecoderOutput{Resize: o.Resize, MaxDelay: o.MaxDelay, Surface: o.Surface, Start: start}, bridge.NonNegative(o.MaxDelay, "video max delay") + out := ffi.MoqVideoDecoderOutput{Resize: o.Resize, Delay: o.Delay, Surface: o.Surface, Start: start} + if o.Delay == nil { + return out, nil + } + if err := bridge.NonNegative(&o.Delay.Min, "video min delay"); err != nil { + return out, err + } + return out, bridge.NonNegative(o.Delay.Max, "video max delay") } // Producer encodes raw pictures and publishes them as a track in a broadcast's catalog. diff --git a/js/hang/MIGRATE.md b/js/hang/MIGRATE.md index 08015b3fd40..ba441db6466 100644 --- a/js/hang/MIGRATE.md +++ b/js/hang/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `Container.Consumer` takes a `delay` ([#7311](https://github.com/moq-dev/moq/pull/7311)) + +`ConsumerProps.maxDelay` is `delay: GetterInit`, where `Delay` is `{ min?: Milli; max?: Milli | null }` and an omitted field is zero. +Replace `maxDelay: d` with `delay: { max: d }`, or derive it from a signal. `max: null` is no limit. +`min` is the playout delay: the consumer starts at the group covering the live edge less `min` plus any `warmup`, waiting for it when the newest group arrives first. `max` never moves the start. + ### The catalog rendition cap is removed `Catalog.MAX_RENDITIONS` and `Catalog.TooManyRenditions` are gone: `checkRenditions`, `encode`, and `watch` accept any number of renditions. diff --git a/js/hang/api/container.api.md b/js/hang/api/container.api.md index 54efd4ea80a..0db0d680e8d 100644 --- a/js/hang/api/container.api.md +++ b/js/hang/api/container.api.md @@ -25,14 +25,20 @@ export class Consumer { } export interface ConsumerProps { + delay?: Signals.GetterInit; format: Format; maxBuffer?: number; - maxDelay?: Signals.GetterInit; predecessor?: Consumer; + warmup?: Milli; } export const DEFAULT_MAX_BUFFER: number; +export interface Delay { + max?: Milli | null; + min?: Milli; +} + export interface Format { decode(payload: Uint8Array, timestamp: Net.Time.Timestamp | undefined): Frame[]; end?(frame: Frame): Micro | undefined; diff --git a/js/hang/bench/slow-group.ts b/js/hang/bench/slow-group.ts index a1622c64879..9fb8b7e6fbf 100644 --- a/js/hang/bench/slow-group.ts +++ b/js/hang/bench/slow-group.ts @@ -3,16 +3,16 @@ import { Milli } from "@moq/clock"; import { Time, Track, Varint } from "@moq/net"; import { Signal } from "@moq/signals"; -import { Consumer } from "../src/container/consumer.ts"; +import { Consumer, type Delay } from "../src/container/consumer.ts"; import { Format } from "../src/container/legacy.ts"; console.log("groups,warnings,bytes,catchup_ms"); for (const count of [1, 16, 256, 1024]) { const track = new Track.Producer("tone"); - const maxDelay = new Signal(Milli(30_000)); + const delay = new Signal({ max: Milli(30_000) }); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format: new Format("data"), - maxDelay, + delay, }); const write = (sequence: number) => { const group = track.createGroup(sequence); @@ -31,7 +31,7 @@ for (const count of [1, 16, 256, 1024]) { try { for (let i = 0; i < count; i++) write(i); await buffered(count - 1); - maxDelay.set(Milli.zero); + delay.set({ max: Milli.zero }); const started = performance.now(); write(count); await buffered(count); diff --git a/js/hang/src/container/consumer.backpressure.test.ts b/js/hang/src/container/consumer.backpressure.test.ts index 7930dd07f40..bfc5bfb7855 100644 --- a/js/hang/src/container/consumer.backpressure.test.ts +++ b/js/hang/src/container/consumer.backpressure.test.ts @@ -57,7 +57,7 @@ test("Consumer holds frames sharing a timestamp within the max buffer", async () const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format, - maxDelay: Milli(5_000), + delay: { max: Milli(5_000) }, maxBuffer, }); try { @@ -92,7 +92,7 @@ test("Consumer holds many groups within the max buffer and reaches the live edge const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format, - maxDelay: Milli(100), + delay: { max: Milli(100) }, maxBuffer, }); try { @@ -127,7 +127,7 @@ test("Consumer skips a stalled group once group overhead alone fills the max buf const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format, - maxDelay: Milli(5_000), + delay: { max: Milli(5_000) }, // Two groups' overhead, so nothing past the stalled head would decode without its successor leading. maxBuffer: 512, }); @@ -154,7 +154,7 @@ test("Consumer reaches a readable group past a stalled successor once group over const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format, - maxDelay: Milli.zero, + delay: { max: Milli.zero }, // Three groups' overhead: the readable group cannot decode until the stalled ones are moved past. maxBuffer: 768, }); @@ -183,7 +183,7 @@ test("Consumer admits reads completing together one at a time against the max bu const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format, - maxDelay: Milli(5_000), + delay: { max: Milli(5_000) }, maxBuffer: 4096, }); try { @@ -218,7 +218,7 @@ test("Consumer charges a small view of a large wire frame the wire frame", async const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format: new Header("data"), - maxDelay: Milli(5_000), + delay: { max: Milli(5_000) }, maxBuffer: 16 * 1024, }); try { @@ -254,7 +254,7 @@ test("Consumer stamps frames the max buffer held back with when they landed, not const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format: new Format("data"), - maxDelay: Milli(5_000), + delay: { max: Milli(5_000) }, // One group's overhead and one frame: the second frame waits in the transport until the first is read. maxBuffer: 300, }); @@ -283,7 +283,7 @@ test("Consumer expires a hole behind a full max buffer from when the group past const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format: new Format("data"), - maxDelay: Milli(500), + delay: { max: Milli(500) }, // Two groups' overhead: group 3 waits in the transport until groups 0 and 1 are read. maxBuffer: 512, }); @@ -316,7 +316,7 @@ for (const maxBuffer of [0, 256]) { const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format: new Format("data"), - maxDelay: Milli(5_000), + delay: { max: Milli(5_000) }, // No more than one group's overhead: the successor is pulled anyway, so the head can be skipped. maxBuffer, }); @@ -343,7 +343,7 @@ test("Consumer moves past a stalled backlog group below the cursor once the max const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format: new Format("data"), - maxDelay: Milli(5_000), + delay: { max: Milli(5_000) }, // Two groups' overhead: the backlog group and the active one leave no room for the newer group. maxBuffer: 512, }); @@ -371,7 +371,7 @@ test("Consumer reads what a promoted head already holds before skipping it under const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format: new Format("data"), - maxDelay: Milli(5_000), + delay: { max: Milli(5_000) }, maxBuffer: 1024, }); try { @@ -416,7 +416,7 @@ test("Consumer raises a malformed frame in a head promoted under pressure", asyn const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format: new Format("data"), - maxDelay: Milli(5_000), + delay: { max: Milli(5_000) }, maxBuffer: 1024, }); try { @@ -449,7 +449,7 @@ test("Consumer raises a malformed group an age skip passes after the track finis const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format: new Format("data"), - maxDelay: Milli(500), + delay: { max: Milli(500) }, }); try { const first = track.createGroup(0); @@ -479,7 +479,7 @@ test("Consumer reads a finished marker group's end before skipping under pressur const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format: new Format("data"), - maxDelay: Milli(5_000), + delay: { max: Milli(5_000) }, maxBuffer: 1024, }); try { @@ -515,7 +515,7 @@ test("Consumer reads a group whose frame fails to read as cut short while the re const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format: new Format("data"), - maxDelay: Milli(5_000), + delay: { max: Milli(5_000) }, }); try { const stalled = track.createGroup(0); diff --git a/js/hang/src/container/consumer.holes.test.ts b/js/hang/src/container/consumer.holes.test.ts index 7b0d7fecc7f..b1f9de2114f 100644 --- a/js/hang/src/container/consumer.holes.test.ts +++ b/js/hang/src/container/consumer.holes.test.ts @@ -12,7 +12,7 @@ test.each([0, 100])("a sequence hole before a paused data marker expires at %sms const track = new Track.Producer("data"); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format: new Format("data"), - maxDelay: Milli(budget), + delay: { max: Milli(budget) }, }); const results: Read[] = []; const reading = (async () => { @@ -57,7 +57,7 @@ test("a filled hole cancels its timer and the next hole gets a fresh budget", as const track = new Track.Producer("data"); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format: new Format("data"), - maxDelay: Milli(100), + delay: { max: Milli(100) }, }); const results: Read[] = []; const reading = (async () => { @@ -108,7 +108,7 @@ test("the holes a stalled source left behind share one budget", async () => { const track = new Track.Producer("data"); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format: new Format("data"), - maxDelay: Milli(100), + delay: { max: Milli(100) }, }); const results: Read[] = []; const reading = (async () => { @@ -149,7 +149,7 @@ test("a hole is late from the earliest arrival past it", async () => { const track = new Track.Producer("data"); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format: new Format("data"), - maxDelay: Milli(100), + delay: { max: Milli(100) }, }); const results: Read[] = []; const reading = (async () => { @@ -207,7 +207,7 @@ test("a hole in a drained backlog is late from the earliest arrival past it, not // Joined now, the backlog drains in sequence order: group 3 before group 5, which landed first. const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format: new Format("data"), - maxDelay: Milli(500), + delay: { max: Milli(500) }, }); const results: Read[] = []; const reading = (async () => { diff --git a/js/hang/src/container/consumer.nonsequential.test.ts b/js/hang/src/container/consumer.nonsequential.test.ts index 14bb3bda9ad..d9bc17d5467 100644 --- a/js/hang/src/container/consumer.nonsequential.test.ts +++ b/js/hang/src/container/consumer.nonsequential.test.ts @@ -37,7 +37,7 @@ test("Consumer waits on a non-sequential hang group because it has no per-frame const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe(), { format: new LegacyFormat("data"), - maxDelay: 5000 as Milli, + delay: { max: 5000 as Milli }, }); // Group A at a large sequence, completed so the cursor advances past it. diff --git a/js/hang/src/container/consumer.outoforder.test.ts b/js/hang/src/container/consumer.outoforder.test.ts index de150e86f65..34c6092fb57 100644 --- a/js/hang/src/container/consumer.outoforder.test.ts +++ b/js/hang/src/container/consumer.outoforder.test.ts @@ -57,7 +57,7 @@ test("out-of-order groups are delivered rather than dropped", async () => { const track = new Track.Producer("test").accept({ timescale: Time.Timescale.MILLI, maxAge: Milli(30_000) }); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(5000) }), { format: new LegacyFormat("data"), - maxDelay: 5000 as Milli, + delay: { max: 5000 as Milli }, }); // The live edge lands first, and delivery starts there rather than waiting. @@ -86,7 +86,7 @@ test("a below-cursor group still downloading is not truncated", async () => { const track = new Track.Producer("test").accept({ timescale: Time.Timescale.MILLI, maxAge: Milli(30_000) }); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(5000) }), { format: new LegacyFormat("data"), - maxDelay: 5000 as Milli, + delay: { max: 5000 as Milli }, }); // The live edge group arrives first and starts delivery (still open). @@ -125,7 +125,7 @@ test("delayed older groups survive a floor established by two newer groups", asy const track = new Track.Producer("test").accept({ timescale: Time.Timescale.MILLI, maxAge: Milli(30_000) }); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(5000) }), { format: new LegacyFormat("data"), - maxDelay: Milli(5000), + delay: { max: Milli(5000) }, }); try { publish(track, 3, 30); diff --git a/js/hang/src/container/consumer.rewind.test.ts b/js/hang/src/container/consumer.rewind.test.ts index 3c196cd5bb5..251048703dd 100644 --- a/js/hang/src/container/consumer.rewind.test.ts +++ b/js/hang/src/container/consumer.rewind.test.ts @@ -17,7 +17,7 @@ function setup() { const track = new Track.Producer("rewind"); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format: new Format("video"), - maxDelay: Milli(30_000), + delay: { max: Milli(30_000) }, }); return { track, consumer }; } diff --git a/js/hang/src/container/consumer.skips.test.ts b/js/hang/src/container/consumer.skips.test.ts index 3592cfbd6dd..33a29afae63 100644 --- a/js/hang/src/container/consumer.skips.test.ts +++ b/js/hang/src/container/consumer.skips.test.ts @@ -2,17 +2,17 @@ import { expect, spyOn, test } from "bun:test"; import { Milli } from "@moq/clock"; import { Time, Track, Varint } from "@moq/net"; import { Signal } from "@moq/signals"; -import { Consumer } from "./consumer.ts"; +import { Consumer, type Delay } from "./consumer.ts"; import { Format } from "./legacy.ts"; for (const count of [1, 16, 256]) { test(`Consumer summarizes ${count} skipped groups once per catch-up`, async () => { const warn = spyOn(console, "warn").mockImplementation(() => {}); const track = new Track.Producer("tone"); - const maxDelay = new Signal(Milli(30_000)); + const delay = new Signal({ max: Milli(30_000) }); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format: new Format("data"), - maxDelay, + delay, }); const write = (sequence: number, timestamp: number) => { const group = track.createGroup(sequence); @@ -27,7 +27,7 @@ for (const count of [1, 16, 256]) { for (let i = 0; i < count; i++) write(i * 2, i * 1000); await buffered(count - 1); expect(warn).not.toHaveBeenCalled(); - maxDelay.set(Milli.zero); + delay.set({ max: Milli.zero }); const newest = write(count * 2, count * 1000); await buffered(count); expect(warn).toHaveBeenCalledTimes(1); diff --git a/js/hang/src/container/consumer.startup.test.ts b/js/hang/src/container/consumer.startup.test.ts index d0100600206..a40396c2a77 100644 --- a/js/hang/src/container/consumer.startup.test.ts +++ b/js/hang/src/container/consumer.startup.test.ts @@ -12,7 +12,7 @@ function setup(maxDelay = 100) { const track = new Track.Producer("video"); const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { format: new Format("video"), - maxDelay: Milli(maxDelay), + delay: { max: Milli(maxDelay) }, }); const results: Read[] = []; const warn = spyOn(console, "warn").mockImplementation(() => {}); diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index ffac3e4bf0a..932bd1ec773 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -377,7 +377,7 @@ test("Legacy Producer discontinuity on a data track reads as an end at the edge" producer.encode(new Uint8Array([1]), 5_000_000 as Micro, true); producer.close(); - const consumer = new Consumer(subscriber, { format: new LegacyFormat("data"), maxDelay: 30_000 as Milli }); + const consumer = new Consumer(subscriber, { format: new LegacyFormat("data"), delay: { max: 30_000 as Milli } }); const items: string[] = []; for (let read = await consumer.next(); read; read = await consumer.next()) items.push(describe(read)); expect(items).toEqual(["0", "20000", "end 20000", "5000000"]); @@ -391,7 +391,10 @@ test("Legacy Producer discontinuity on a data track reads as an end at the edge" test("Consumer refuses a data marker that shares its group", async () => { for (const second of [new Uint8Array(), encodeLegacyFrame(10_000 as Micro, new Uint8Array([1]))]) { const track = new Track.Producer("data"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 500 as Milli }); + const consumer = new Consumer(replay(track), { + format: new LegacyFormat("data"), + delay: { max: 500 as Milli }, + }); try { const group = track.appendGroup(); group.writeFrame({ payload: new Uint8Array(), timestamp: Time.Timestamp.fromMicros(0) }); @@ -553,7 +556,7 @@ function frameOf(read: Read | string | undefined): Frame { test("Consumer delivers frames from a single group", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 500 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), delay: { max: 500 as Milli } }); writeGroupWithLegacyFrames(track, 0, [0 as Micro, 33_000 as Micro]); track.finish(); @@ -568,7 +571,7 @@ test("Consumer delivers frames from a single group", async () => { test("Consumer forces keyframe true at index 0", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 500 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), delay: { max: 500 as Milli } }); writeGroupWithLegacyFrames(track, 0, [0 as Micro, 33_000 as Micro]); track.finish(); @@ -593,7 +596,7 @@ test("Consumer index spans MoQ frames for keyframe detection", async () => { }; const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: multiFormat, maxDelay: 500 as Milli }); + const consumer = new Consumer(replay(track), { format: multiFormat, delay: { max: 500 as Milli } }); const group = track.createGroup(0); group.writeFrame({ payload: new Uint8Array([0x01]), timestamp: Time.Timestamp.now() }); // first MoQ frame → 3 samples @@ -626,7 +629,7 @@ for (const [where, payloads] of [ }; const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: failingFormat, maxDelay: 500 as Milli }); + const consumer = new Consumer(replay(track), { format: failingFormat, delay: { max: 500 as Milli } }); const g0 = track.createGroup(0); for (const byte of payloads) @@ -657,7 +660,7 @@ for (const [where, payloads] of [ // Errors come in delivery order, so a decode error waits behind the items ahead of it, a gap included. test("Consumer delivers everything ahead of a decode error", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 10_000 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), delay: { max: 10_000 as Milli } }); writeGroupWithLegacyFrames(track, 0, [0 as Micro]); writeGroupWithLegacyFrames(track, 2, [100_000 as Micro]); @@ -686,7 +689,7 @@ test("Consumer keeps the first of several decode errors", async () => { }; const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: failingFormat, maxDelay: 500 as Milli }); + const consumer = new Consumer(replay(track), { format: failingFormat, delay: { max: 500 as Milli } }); for (const sequence of [0, 1]) { const group = track.createGroup(sequence); @@ -717,7 +720,7 @@ test("Consumer releases its open groups when an error ends it", async () => { }; const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: failingFormat, maxDelay: 500 as Milli }); + const consumer = new Consumer(replay(track), { format: failingFormat, delay: { max: 500 as Milli } }); const malformed = track.createGroup(0); malformed.writeFrame({ payload: new Uint8Array([0xff]), timestamp: Time.Timestamp.now() }); @@ -749,7 +752,7 @@ test("Consumer keeps frames decoded before a group reset (truncated GoP)", async }; const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format, maxDelay: 500 as Milli }); + const consumer = new Consumer(replay(track), { format, delay: { max: 500 as Milli } }); const g0 = track.createGroup(0); for (const timestamp of [1, 2]) { @@ -773,7 +776,7 @@ test("Consumer keeps frames decoded before a group reset (truncated GoP)", async test("Consumer close returns undefined from next()", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 500 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), delay: { max: 500 as Milli } }); const promise = consumer.next(); consumer.close(); @@ -784,7 +787,7 @@ test("Consumer close returns undefined from next()", async () => { test("Consumer throws on concurrent next() calls", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 500 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), delay: { max: 500 as Milli } }); // First call blocks waiting for data void consumer.next(); @@ -797,7 +800,7 @@ test("Consumer throws on concurrent next() calls", async () => { test("Consumer skips groups via PTS-span when over the max delay", async () => { const track = new Track.Producer("test"); // Zero max delay = skip everything that's not the latest - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 0 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), delay: { max: 0 as Milli } }); // Write groups with increasing timestamps. With a 0 max delay, any PTS span > 0 triggers skip. writeGroupWithLegacyFrames(track, 0, [0 as Micro]); @@ -819,7 +822,7 @@ test("Consumer measures arrivals as they land", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe(), { format: new LegacyFormat("audio"), - maxDelay: 5000 as Milli, + delay: { max: 5000 as Milli }, }); const next = new Track.Producer("next"); @@ -838,7 +841,7 @@ test("Consumer measures arrivals as they land", async () => { // A replacement on the same path continues the estimate rather than restarting cold. const replacement = new Consumer(next.subscribe(), { format: new LegacyFormat("audio"), - maxDelay: 5000 as Milli, + delay: { max: 5000 as Milli }, predecessor: consumer, }); expect(replacement.spread.peek()).toBe(consumer.spread.peek()); @@ -857,7 +860,7 @@ test("Consumer measures arrivals as they land", async () => { test("Consumer delivers groups in sequence order regardless of arrival order", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 500 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), delay: { max: 500 as Milli } }); writeGroupWithLegacyFrames(track, 2, [60_000 as Micro]); writeGroupWithLegacyFrames(track, 0, [0 as Micro]); @@ -877,7 +880,7 @@ test("Consumer delivers groups in sequence order regardless of arrival order", a test("Consumer delivers a group that arrives below the cursor", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 500 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), delay: { max: 500 as Milli } }); // Group.Producer 5 arrives first (sets active = 5) writeGroupWithLegacyFrames(track, 5, [100_000 as Micro]); @@ -903,7 +906,7 @@ test("Consumer delivers a group that arrives below the cursor", async () => { test("Consumer next() yields no gap before the first frame", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 500 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), delay: { max: 500 as Milli } }); writeGroupWithLegacyFrames(track, 0, [0 as Micro, 33_000 as Micro]); writeGroupWithLegacyFrames(track, 1, [66_000 as Micro]); @@ -925,7 +928,7 @@ test("Consumer next() yields no gap before the first frame", async () => { test("Consumer reads nothing after close, even the frame a gap was returned ahead of", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 0 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), delay: { max: 0 as Milli } }); writeGroupWithLegacyFrames(track, 0, [0 as Micro]); await settle(); @@ -946,7 +949,7 @@ test("Consumer keeps a trailing duration marker internal", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe(), { format: new LegacyFormat("video"), - maxDelay: 500 as Milli, + delay: { max: 500 as Milli }, }); const group = track.createGroup(0); @@ -976,7 +979,7 @@ test("Consumer reads a leading marker as an end and keeps the first media keyfra const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe(), { format: new LegacyFormat("video"), - maxDelay: 500 as Milli, + delay: { max: 500 as Milli }, }); const group = track.createGroup(0); @@ -1010,7 +1013,7 @@ test("Consumer reads an empty LOC payload as the frame's duration", async () => ], ]); - const consumer = new Consumer(replay(track), { format: new LocFormat("video"), maxDelay: 500 as Milli }); + const consumer = new Consumer(replay(track), { format: new LocFormat("video"), delay: { max: 500 as Milli } }); await settle(); const media = frameOf(await consumer.next()); expect(media.payload).toEqual(new Uint8Array([0xde, 0xad])); @@ -1033,7 +1036,7 @@ test("Consumer passes over LOC duration markers", async () => { ], ]); - const consumer = new Consumer(replay(track), { format: new LocFormat("video"), maxDelay: 500 as Milli }); + const consumer = new Consumer(replay(track), { format: new LocFormat("video"), delay: { max: 500 as Milli } }); await settle(); const first = frameOf(await consumer.next()); expect(first.payload).toEqual(new Uint8Array([0xde, 0xad])); @@ -1057,7 +1060,7 @@ test("Consumer preserves empty Legacy and LOC data frames", async () => { writeLoc(track, [[[0 as Micro, []]]]); } const format = kind === "legacy" ? new LegacyFormat("data") : new LocFormat(); - const consumer = new Consumer(replay(track), { format, maxDelay: 500 as Milli }); + const consumer = new Consumer(replay(track), { format, delay: { max: 500 as Milli } }); const frame = frameOf(await consumer.next()); expect(frame.payload).toEqual(new Uint8Array()); expect(frame.keyframe).toBe(true); @@ -1081,7 +1084,7 @@ const wallClock = () => Time.Timestamp.now(); test("Consumer aborts a group below the previous group start", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 30_000 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), delay: { max: 30_000 as Milli } }); writeGroupWithLegacyFrames(track, 0, [10_000_000 as Micro], wallClock); writeGroupWithLegacyFrames(track, 1, [0 as Micro, 100_000 as Micro], wallClock); @@ -1096,7 +1099,7 @@ test("Consumer aborts a group below the previous group start", async () => { // A later group's rewind does not mask an earlier group's decode error. test("Consumer throws the first error in delivery order", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 30_000 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), delay: { max: 30_000 as Milli } }); writeGroupWithLegacyFrames(track, 0, [10_000_000 as Micro], wallClock); const malformed = track.createGroup(1); @@ -1138,7 +1141,7 @@ test("Consumer checks a whole decoded batch before a skip drops its group", asyn }; const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: batchFormat, maxDelay: 500 as Milli }); + const consumer = new Consumer(replay(track), { format: batchFormat, delay: { max: 500 as Milli } }); write(0, [1_000_000]).finish(); expect((await nextFrame(consumer))?.frame?.timestamp).toBe(1_000_000 as Micro); @@ -1164,7 +1167,7 @@ test("Consumer checks a whole decoded batch before a skip drops its group", asyn test("Consumer aborts a rewind carried by a later arrival", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 30_000 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), delay: { max: 30_000 as Milli } }); writeGroupWithLegacyFrames(track, 0, [10_000_000 as Micro], wallClock); await settle(); @@ -1183,7 +1186,7 @@ test("Consumer aborts a rewind carried by a later arrival", async () => { // rewind, so the detector compares against the previous group's start. test("Consumer treats B-frame reordering within a group as continuous", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 30_000 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), delay: { max: 30_000 as Milli } }); writeGroupWithLegacyFrames(track, 0, [0 as Micro, 66_000 as Micro, 33_000 as Micro]); await settle(); @@ -1204,7 +1207,7 @@ test("Consumer treats B-frame reordering within a group as continuous", async () test("Consumer accepts open-GOP leading pictures above the previous group", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 30_000 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), delay: { max: 30_000 as Milli } }); writeGroupWithLegacyFrames(track, 0, [0 as Micro, 33_000 as Micro]); writeGroupWithLegacyFrames(track, 1, [66_000 as Micro, 50_000 as Micro]); @@ -1220,7 +1223,7 @@ test("Consumer accepts open-GOP leading pictures above the previous group", asyn // the max delay loses nothing. test("Consumer reads no gap when retiring a fully delivered group", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 150 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), delay: { max: 150 as Milli } }); writeGroupWithLegacyFrames(track, 0, [0 as Micro]); await settle(); @@ -1239,7 +1242,7 @@ test("Consumer reads no gap when retiring a fully delivered group", async () => // An unread trailing endpoint carries no media, so retiring its group loses nothing. test("Consumer reads no gap when retiring a group holding only its endpoint", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("video"), maxDelay: 150 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("video"), delay: { max: 150 as Milli } }); const group = track.createGroup(0); group.writeFrame({ payload: encodeLegacy(0 as Micro), timestamp: Time.Timestamp.fromMicros(0) }); @@ -1264,7 +1267,7 @@ test("Consumer reads no gap when retiring a group holding only its endpoint", as // next group is adjacent. test("Consumer reads a gap when retiring a truncated group", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 150 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), delay: { max: 150 as Milli } }); const group = track.createGroup(0); group.writeFrame({ payload: encodeLegacy(0 as Micro), timestamp: Time.Timestamp.fromMicros(0) }); @@ -1286,7 +1289,7 @@ test("Consumer reads a gap when retiring a truncated group", async () => { // An open group may still have a tail in flight, so adjacency proves nothing when it is retired. test("Consumer reads a gap when retiring an open group", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 150 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), delay: { max: 150 as Milli } }); const group = track.createGroup(0); group.writeFrame({ payload: encodeLegacy(0 as Micro), timestamp: Time.Timestamp.fromMicros(0) }); @@ -1309,7 +1312,7 @@ test("Consumer reads a gap when retiring an open group", async () => { test("Consumer buffered signal updates as frames arrive", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 500 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), delay: { max: 500 as Milli } }); expect(consumer.buffered.peek()).toEqual([]); @@ -1332,7 +1335,7 @@ test("Consumer buffered signal updates as frames arrive", async () => { test("Consumer recovers from gap in group sequence numbers", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 100 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), delay: { max: 100 as Milli } }); writeGroupWithLegacyFrames(track, 0, [0 as Micro, 20_000 as Micro]); writeGroupWithLegacyFrames(track, 1, [40_000 as Micro, 60_000 as Micro]); @@ -1363,7 +1366,7 @@ test("Consumer handles empty decode result without deadlock", async () => { }; const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: emptyThenValid, maxDelay: 500 as Milli }); + const consumer = new Consumer(replay(track), { format: emptyThenValid, delay: { max: 500 as Milli } }); const group = track.createGroup(0); group.writeFrame({ payload: new Uint8Array([0x01]), timestamp: Time.Timestamp.now() }); // empty decode @@ -1389,7 +1392,7 @@ test("Consumer preserves empty media from formats without endpoint markers", asy }, }; const track = new Track.Producer("test"); - const consumer = new Consumer(track.subscribe(), { format, maxDelay: 500 as Milli }); + const consumer = new Consumer(track.subscribe(), { format, delay: { max: 500 as Milli } }); const group = track.createGroup(0); group.writeFrame({ payload: new Uint8Array([1]), timestamp: Time.Timestamp.now() }); group.finish(); @@ -1406,7 +1409,7 @@ test("Consumer with CmafFormat delivers correct timestamps", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new CmafFormat(TEST_INIT), - maxDelay: 500 as Milli, + delay: { max: 500 as Milli }, }); const group = track.createGroup(0); @@ -1435,7 +1438,7 @@ test("Consumer with CmafFormat times an untimed track from tfdt", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new CmafFormat(TEST_INIT), - maxDelay: 500 as Milli, + delay: { max: 500 as Milli }, }); const group = track.createGroup(0); @@ -1478,7 +1481,7 @@ const durationFormat: ContainerFormat = { test("Consumer duration-skips a stalled group once it is covered", async () => { const track = new Track.Producer("test"); // Latency dwarfs the gap, so only duration coverage can trigger the skip. - const consumer = new Consumer(replay(track), { format: durationFormat, maxDelay: 10_000 as Milli }); + const consumer = new Consumer(replay(track), { format: durationFormat, delay: { max: 10_000 as Milli } }); // Group.Producer 0: one frame at ts=0 lasting 33ms, never closed (stalled). const g0 = track.createGroup(0); @@ -1514,7 +1517,7 @@ test("Consumer does not duration-skip when the gap is not covered", async () => }; const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: shortFormat, maxDelay: 10_000 as Milli }); + const consumer = new Consumer(replay(track), { format: shortFormat, delay: { max: 10_000 as Milli } }); // Group.Producer 0 stays open and later receives a second frame; nothing covers the gap, // so that late frame must survive rather than being skipped. @@ -1547,7 +1550,7 @@ test("Consumer delivers a PTS-contiguous next group whose sequence jumped (Legac const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: timedLegacy, - maxDelay: 500 as Milli, + delay: { max: 500 as Milli }, }); // Group A (seq 1_000_000): one 3000-tick sample, so its content ends at 33_333µs (3000/90000 * 1e6). @@ -1587,7 +1590,7 @@ test("Consumer waits on a PTS gap instead of skipping to a later buffered group // Large max delay so the gap can't be age-skipped during the test window. const consumer = new Consumer(replay(track), { format: timedLegacy, - maxDelay: 10_000 as Milli, + delay: { max: 10_000 as Milli }, }); // Group A (seq 1_000_000): content ends at 33_333µs. @@ -1635,7 +1638,7 @@ test("Consumer delivers a contiguous group after one that completed out of order const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: timedLegacy, - maxDelay: 10_000 as Milli, + delay: { max: 10_000 as Milli }, }); // A (seq 1_000_000) stays open so it remains the active group. @@ -1678,7 +1681,7 @@ test("Consumer plays the head once a waited-out gap exceeds the budget (Legacy)" const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: timedLegacy, - maxDelay: 100 as Milli, + delay: { max: 100 as Milli }, }); // A (seq 1000): one frame, ends at 3000 ticks (33_333µs). @@ -1708,7 +1711,7 @@ test("Consumer waits on a gap within the budget, then plays the head once it exc const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: timedLegacy, - maxDelay: 100 as Milli, + delay: { max: 100 as Milli }, }); // A (seq 1000): one frame, ends at 3000 ticks (33_333µs). @@ -1745,7 +1748,7 @@ test("Consumer waits on a gap within the budget, then plays the head once it exc // adjacency doesn't rule out a group the max delay check dropped on the way past. test("Consumer yields no gap while nothing is dropped", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 100 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), delay: { max: 100 as Milli } }); writeGroupWithLegacyFrames(track, 0, [0 as Micro, 33_000 as Micro]); await settle(); @@ -1762,7 +1765,7 @@ test("Consumer yields no gap while nothing is dropped", async () => { test("Consumer reads a marker group as an end, not a gap", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 2_000 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), delay: { max: 2_000 as Milli } }); writeGroupWithLegacyFrames(track, 0, [0 as Micro]); writeMarkerGroup(track, 1, 0 as Micro); @@ -1779,7 +1782,7 @@ test("Consumer reads a marker group as an end, not a gap", async () => { test("Consumer empty groups mean nothing", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 2_000 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), delay: { max: 2_000 as Milli } }); writeGroupWithLegacyFrames(track, 0, [0 as Micro]); const empty = track.createGroup(1); @@ -1795,7 +1798,7 @@ test("Consumer empty groups mean nothing", async () => { test("Consumer reads a latency skip as one gap", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 0 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), delay: { max: 0 as Milli } }); writeGroupWithLegacyFrames(track, 0, [0 as Micro]); await settle(); @@ -1812,7 +1815,7 @@ test("Consumer reads a latency skip as one gap", async () => { test("Consumer reads a timestamp hole as a gap", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 0 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), delay: { max: 0 as Milli } }); writeGroupWithLegacyFrames(track, 0, [0 as Micro]); await settle(); @@ -1833,7 +1836,7 @@ test("Consumer reads a timestamp hole as a gap", async () => { test("Consumer reads a shed marker group as a gap", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 0 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), delay: { max: 0 as Milli } }); const group0 = track.createGroup(0); group0.writeFrame({ @@ -1860,7 +1863,7 @@ test("Consumer yields no gap across a PTS-contiguous group id jump (Legacy)", as const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: timedLegacy, - maxDelay: 10_000 as Milli, + delay: { max: 10_000 as Milli }, }); const a = track.createGroup(1_000_000); @@ -1882,7 +1885,7 @@ test("a live video frame is delivered before its duration marker arrives", async const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe(), { format: new LegacyFormat("video"), - maxDelay: 500 as Milli, + delay: { max: 500 as Milli }, }); const group = track.appendGroup(); group.writeFrame({ @@ -1926,7 +1929,7 @@ test("audio discontinuity writes no duration marker", async () => { for (const code of [StreamCode.Cancel, StreamCode.Internal, StreamCode.Old, StreamCode.Evicted, StreamCode(1234)]) { test(`Consumer preserves frames and continues after group reset ${code}`, async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 0 as Milli }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), delay: { max: 0 as Milli } }); const error = spyOn(console, "error").mockImplementation(() => {}); try { const group = track.appendGroup(); @@ -2018,7 +2021,7 @@ for (const end of [ const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), - maxDelay: 10_000 as Milli, + delay: { max: 10_000 as Milli }, }); try { writeGroupWithLegacyFrames(track, 0, [0 as Micro]); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index d38796a1f44..d89bfae392d 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -5,21 +5,36 @@ import type { Format } from "./format"; import { Jitter } from "./jitter"; import type { BufferedRanges, Frame, ReceivedFrame } from "./types"; -/** Options for constructing a {@link Consumer}. */ -export interface ConsumerProps { - /** The container format used to decode each MoQ frame. */ - format: Format; +/** How far behind the live edge a {@link Consumer} plays, in milliseconds. Mirrors Rust's `Delay`. */ +export interface Delay { /** - * How stale a group may get before it is skipped, in milliseconds (default: 0). + * The target latency: the playout or jitter-buffer delay a player holds behind the live edge + * (default: 0). + * + * The start is the group covering the live edge less `min`, plus any {@link ConsumerProps.warmup}, + * waiting for it when the newest group arrives first. Consumer-only: it never reaches the + * subscription. A `min` past `max` starts at `max`, since the publisher serves nothing older. + */ + min?: Milli; + /** + * How stale a group may get before it is skipped (default: 0), or `null` for no limit. * * Measured as the span from the oldest buffered frame to the newest, so it bounds how long a * late group is waited for. A missing sequence also expires this much monotonic time after * the first group past it arrived, even when the next group is a marker on a paused track. - * The local half of the subscription's - * `maxDelay`; both measure the same budget, one on the wire and one as frames are read. + * The local half of the subscription's `maxDelay`; both measure the same budget, one on the wire + * and one as frames are read. It bounds lateness only: it never moves the start. */ + max?: Milli | null; +} + +/** Options for constructing a {@link Consumer}. */ +export interface ConsumerProps { + /** The container format used to decode each MoQ frame. */ + format: Format; + /** How far behind the live edge to play (default: zero for both bounds). See {@link Delay}. */ // Read-only: a Getter (e.g. another component's output) is accepted directly. - maxDelay?: GetterInit; + delay?: GetterInit; /** * Cap the bytes of decoded frames held ahead of the reader, counting the whole wire payload they * share and each frame's and group's own overhead (default: {@link DEFAULT_MAX_BUFFER}). Mirrors @@ -32,6 +47,25 @@ export interface ConsumerProps { * more of what it waits behind can be held. */ maxBuffer?: number; + /** + * How much further than the min delay the start reaches, in milliseconds (default: 0), so a + * decoder that needs that much media before its output is clean starts early enough, such as an + * Opus decoder or the max-delay decoder of an intra-refresh rendition. Pass the catalog + * rendition's `warmup`. Mirrors Rust's `Consumer::with_warmup`. + * + * Put the max delay plus the warmup on the subscription, since the publisher skips what falls + * outside that budget. The start is no later than the group covering the live edge less + * {@link Delay.min} and the warmup: one that starts at or before that point. Both default to zero, + * which the newest group covers, so the start waits for nothing. Otherwise groups can arrive + * newest first, so when none that arrived covers that point the start waits for an earlier one, + * until one covers it, group 0 arrived, the track finishes, the max buffer fills, or the newest + * group's arrival is that whole reach ago. The wall clock bounds a quiet or paused track, which + * never ages out in media time. Delivery then starts at the lowest group that arrived. Until the + * reader is the warmup past both its first frame and that point, the age skip keeps a group that + * covers the reach and tolerates the warmup, so the media leading up to the start is not dropped + * as stale. After that, the max delay alone governs it. + */ + warmup?: Milli; /** * The consumer this one replaces on the same path, such as the previous rendition. * @@ -123,12 +157,28 @@ function continues(prev: Group, next: Group | undefined): next is Group { ); } -/** Reads frames from a MoQ track in order, buffering groups and skipping ones that fall `maxDelay` behind. */ +// The max delay in microseconds, infinite when unlimited. +function maxMicros(delay: Delay): number { + return delay.max === null ? Number.POSITIVE_INFINITY : Micro.fromMilli(delay.max ?? Milli.zero); +} + +/** Reads frames from a MoQ track in order, buffering groups and skipping ones that fall the max delay behind. */ export class Consumer { #track: Moq.Track.Subscriber; #format: Format; - #maxDelay: Getter; + #delay: Getter; #maxBuffer: number; + // How much further than the min delay the start reaches, and the age skip tolerates while warming. + #warmup: Milli; + // The start may still wait for a group covering the reach, until that wait ends for good. + #starting = true; + // When the newest group had landed as the first one arrived, once the start waits. + #anchor?: Instant; + // The point the start reached back to, the live edge less the reach, once its wait ended. + #target?: number; + // Where the warming ends: the warmup past the first frame read, or past the start's target when + // that is later. Undefined before the first frame. + #warmed?: Micro; // What the buffered groups hold, against the max buffer. #used = 0; // Group watchers and the track loop parked on a full max buffer. @@ -182,8 +232,9 @@ export class Consumer { constructor(track: Moq.Track.Subscriber, props: ConsumerProps) { this.#track = track; this.#format = props.format; - this.#maxDelay = getter(props.maxDelay ?? Milli.zero); + this.#delay = getter(props.delay ?? {}); this.#maxBuffer = props.maxBuffer ?? DEFAULT_MAX_BUFFER; + this.#warmup = props.warmup ?? Milli.zero; this.#jitter = props.predecessor ? props.predecessor.#jitter.fork() : new Jitter(); this.#spread = new Signal(this.#jitter.measured); this.spread = this.#spread; @@ -441,7 +492,7 @@ export class Consumer { // This also handles gaps in group sequence numbers: if #active points to a missing // group, the span proves the missing content is too old to wait for. while (this.#groups.length >= 2) { - const threshold = Micro.fromMilli(this.#maxDelay.peek()); + const threshold = this.#ageBudget(); const first = this.#groups[0]; // Check the difference between the earliest and latest known frames. @@ -460,6 +511,12 @@ export class Consumer { const age = max - min; if (age <= threshold) break; + // Until the reader warms, a group may cover the start's reach though it began before it: it is + // stale only once its successor starts a whole budget behind the newest content. + if (this.#warming()) { + const nextStart = this.#groups[1].start; + if (nextStart === undefined || max - nextStart < threshold) break; + } this.#groups.shift(); this.#active = this.#groups[0]?.consumer.sequence; @@ -482,6 +539,12 @@ export class Consumer { } if (skipped) { + // What the start would wait for is older than the reach, so it is past. The reader still + // warms until the start's target, as it would had the wait ended on its own. + if (this.#starting) { + this.#starting = false; + this.#target = this.#startTarget(this.#reach(this.#delay.peek())); + } console.warn( `skipping slow groups: track=${this.#track.name} ${start} -> ${this.#active} count=${skipped}`, ); @@ -668,6 +731,9 @@ export class Consumer { } } + // The start may wait for a group covering the reach, the min delay plus the warmup. + const starting = this.#reachStart(ended !== undefined); + // If #active points below all buffered groups -- e.g. a finished group's `+ 1` // fallback fired because no later group was buffered yet, and the real (large-gap, // non-sequential) next group has since arrived -- promote #active to the first real @@ -684,21 +750,21 @@ export class Consumer { // behind a hole can end a paused track without more arrivals. It is late since a group // past it arrived, not since the cursor got here: once the edge stops moving, a backlog // with every other group missing would otherwise drain one budget per hole. - if (this.#active !== undefined && this.#groups.length > 0) { + if (starting === undefined && this.#active !== undefined && this.#groups.length > 0) { const head = this.#groups[0]; if (head.consumer.sequence > this.#active) { this.#hole = head.late; const startup = this.#start === undefined; const contiguous = ptsContiguous(this.#presentedEnd, head.frames.at(0)?.timestamp); const empty = head.empty && head.consumer.done; - const maxDelay = Micro.fromMilli(this.#maxDelay.peek()); + const maxDelay = maxMicros(this.#delay.peek()); const skipHole = head.frames.length > 0 && (maxDelay === 0 || (this.#presentedEnd !== undefined && head.latest !== undefined && head.latest - this.#presentedEnd > maxDelay)); - const elapsed = Instant.durationSince(now(), this.#hole) >= Nano.fromMicro(maxDelay); + const elapsed = Instant.durationSince(now(), this.#hole) >= maxDelay * 1000; // The groups past the hole already fill the max buffer, so it can hold none of what would arrive into it. const full = this.#full(); if (startup || empty || contiguous || skipHole || elapsed || full || ended !== undefined) { @@ -717,7 +783,12 @@ export class Consumer { } const head = this.#groups.at(0); - if (head && this.#active !== undefined && head.consumer.sequence <= this.#active) { + if ( + starting === undefined && + head && + this.#active !== undefined && + head.consumer.sequence <= this.#active + ) { const seq = head.consumer.sequence; const frame = head.frames.shift(); if (frame) { @@ -734,6 +805,10 @@ export class Consumer { } this.#mediaGroup = seq; + // A group covering the target began before it, so the reader warms until the target too. + this.#warmed ??= Micro( + Math.max(frame.timestamp, this.#target ?? frame.timestamp) + Micro.fromMilli(this.#warmup), + ); if (this.#start === undefined || seq > this.#start.group) { this.#floor = this.#start?.timestamp; this.#start = { group: seq, timestamp: frame.timestamp }; @@ -801,13 +876,16 @@ export class Consumer { const abort = this.#signals.abort; if (abort.aborted) return undefined; let timer: Effect | undefined; - if (this.#hole !== undefined) { + const since = starting ?? this.#hole; + if (since !== undefined) { timer = new Effect(); - const since = this.#hole; timer.run((effect) => { - const budget = Nano.fromMilli(effect.get(this.#maxDelay)); + const delay = effect.get(this.#delay); + // The start waits out its reach, and a hole the max delay, which an unlimited one never does. + const budget = starting === undefined ? maxMicros(delay) : this.#reach(delay); + if (!Number.isFinite(budget)) return; const elapsed = Instant.durationSince(now(), since); - effect.timer(() => this.#notify?.(), Nano(Math.max(0, budget - elapsed))); + effect.timer(() => this.#notify?.(), Nano(Math.max(0, budget * 1000 - elapsed))); }); } @@ -826,6 +904,67 @@ export class Consumer { } } + // How far behind the live edge the start reaches, in microseconds: the min delay plus the warmup. + // The publisher serves nothing older than the max delay, so a min delay past it reaches no further. + #reach(delay: Delay): number { + return Math.min(Micro.fromMilli(delay.min ?? Milli.zero), maxMicros(delay)) + Micro.fromMilli(this.#warmup); + } + + // While the start still waits for a group covering the live edge less the min delay and the + // warmup, when the newest group had landed as the first one arrived: the wait ends a reach after + // it. Undefined once the start is free to go, for good. + #reachStart(ended: boolean): Instant | undefined { + if (!this.#starting) return undefined; + // Nothing has a wire frame yet, so nothing can start either. + if (!this.#groups.some((group) => !group.empty)) return undefined; + + const reach = this.#reach(this.#delay.peek()); + // The newest group covers a zero reach. + if (reach === 0) { + this.#starting = false; + return undefined; + } + this.#anchor ??= this.#groups[this.#groups.length - 1].consumer.arrived; + const target = this.#startTarget(reach); + const covered = + target !== undefined && this.#groups.some((group) => group.start !== undefined && group.start <= target); + // Nothing earlier can arrive or be held. + const lowest = this.#groups[0].consumer.sequence === 0; + // Wall time: the newest group landed a whole reach ago. + const elapsed = Instant.durationSince(now(), this.#anchor) >= reach * 1000; + if (ended || this.#full() || target === undefined || covered || lowest || elapsed) { + this.#starting = false; + this.#target = target; + return undefined; + } + return this.#anchor; + } + + // The point the start reaches back to: the live edge (the newest content) less `reach`. Undefined + // for a zero reach, which the newest group covers, and before any timed content, since untimed + // content such as a marker has no edge to reach back from. + #startTarget(reach: number): number | undefined { + if (reach === 0) return undefined; + let edge: number | undefined; + for (const group of this.#groups) { + if (group.latest !== undefined && (edge === undefined || group.latest > edge)) edge = group.latest; + } + // Media starts at zero, so a reach past it is covered there. + return edge === undefined ? undefined : Math.max(0, edge - reach); + } + + // How stale the buffered span may get before the age skip drops its oldest group, in microseconds: + // the max delay, plus the warmup while the reader warms. + #ageBudget(): number { + const maxDelay = maxMicros(this.#delay.peek()); + return this.#warming() ? maxDelay + Micro.fromMilli(this.#warmup) : maxDelay; + } + + // The reader has not yet read past the start's target and the warmup past its first frame. + #warming(): boolean { + return this.#warmed === undefined || (this.#edge ?? 0) < this.#warmed; + } + // Return `read`, or a gap ahead of it when media went missing after an earlier item. #emit(read: Exclude): Read { const resume = read.kind === "frame" ? read.frame.timestamp : read.end; diff --git a/js/hang/src/container/consumer.warmup.test.ts b/js/hang/src/container/consumer.warmup.test.ts new file mode 100644 index 00000000000..6270d9696c7 --- /dev/null +++ b/js/hang/src/container/consumer.warmup.test.ts @@ -0,0 +1,226 @@ +import { expect, onTestFinished, spyOn, test } from "bun:test"; +import { Manual, Micro, Milli, Nano } from "@moq/clock"; +import { type Group, Time, Track } from "@moq/net"; +import { Consumer, type Read } from "./consumer.ts"; +import { encodeFrame, Format } from "./legacy.ts"; + +const BUDGET = 500; + +// A consumer that starts `min` plus `warmup` behind the live edge and skips at `max`, `null` for no limit. +function setup(delay = BUDGET, warmup = 0, min = delay, max: number | null = delay) { + const clock = new Manual(); + const installed = clock.install(); + // Drain the in-memory readers and their continuations without moving time. + const flush = () => clock.advance(Nano.zero); + const track = new Track.Producer("video"); + // A null max delay sets no subscriber limit; an omitted one is zero. + const consumer = new Consumer(track.subscribe({ maxDelay: max === null ? null : Milli(max + warmup) }), { + format: new Format("video"), + delay: { min: Milli(min), max: max === null ? null : Milli(max) }, + warmup: Milli(warmup), + }); + const results: Read[] = []; + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const reading = (async () => { + for (;;) { + const result = await consumer.next(); + if (!result) return; + results.push(result); + } + })(); + onTestFinished(async () => { + consumer.close(); + track.finish(); + track.close(); + try { + await reading; + } finally { + warn.mockRestore(); + installed.close(); + } + }); + return { clock, track, results, flush }; +} + +// Each frame as "@<µs>". +function frames(results: Read[]): string[] { + return results.flatMap((read) => (read.kind === "frame" ? [`${read.group}@${read.frame.timestamp}`] : [])); +} + +function write(group: Group.Producer, timestamp: number) { + group.writeFrame({ + payload: encodeFrame(new Uint8Array([1]), Micro(timestamp)), + timestamp: Time.Timestamp.now(), + }); +} + +function group(track: Track.Producer, sequence: number, timestamps: number[]): Group.Producer { + const group = track.createGroup(sequence); + for (const timestamp of timestamps) write(group, timestamp); + return group; +} + +test("the start waits for an earlier group when the live edge arrives first", async () => { + const { track, results, flush } = setup(); + group(track, 5, [5_000_000]); + await flush(); + expect(frames(results)).toEqual([]); + + group(track, 4, [4_500_000]).finish(); + await flush(); + expect(frames(results)).toEqual(["4@4500000", "5@5000000"]); +}); + +test("a zero reach starts at a live edge that arrives first", async () => { + const { track, results, flush } = setup(0); + group(track, 5, [5_000_000]); + await flush(); + expect(frames(results)).toEqual(["5@5000000"]); +}); + +test("the warmup reaches further back for the start", async () => { + for (const [warmup, start] of [ + [0, "4@4800000"], + [300, "3@4500000"], + ] as const) { + const { track, results, flush } = setup(200, warmup); + // Newest first: group 4 covers the max delay alone, but not the warmup too. + group(track, 5, [5_000_000]).finish(); + group(track, 4, [4_800_000]).finish(); + await flush(); + group(track, 3, [4_500_000]).finish(); + await flush(); + expect(frames(results)[0]).toEqual(start); + results.length = 0; + } +}); + +test("a quiet track starts within one reach", async () => { + const { clock, track, results, flush } = setup(200, 300); + group(track, 5, [5_000_000]); + await flush(); + await clock.advance(Nano.fromMilli(Milli(BUDGET - 50))); + expect(frames(results)).toEqual([]); + + await clock.advance(Nano.fromMilli(Milli(50))); + expect(frames(results)).toEqual(["5@5000000"]); +}); + +test("the start needs no wait once a group covers the reach", async () => { + const { track, results, flush } = setup(); + group(track, 5, [5_000_000, 5_000_000 + BUDGET * 1000]); + await flush(); + expect(frames(results)).toEqual(["5@5000000", "5@5500000"]); +}); + +test("the start keeps a group that began before the reach but covers it", async () => { + const { track, results, flush } = setup(200, 300); + group(track, 5, [5_000_000]); + await flush(); + // Group 4 begins 600ms before the newest content, past the 500ms reach, yet covers its 4.5s target. + group(track, 4, [4_400_000, 4_600_000, 4_900_000]).finish(); + await flush(); + expect(frames(results)[0]).toEqual("4@4400000"); +}); + +test("the age skip still drops a group its successor covers", async () => { + const { track, results, flush } = setup(); + group(track, 5, [5_000_000]); + await flush(); + // Group 6 starts a whole budget behind group 7's content, so group 5 cannot cover the reach. + group(track, 6, [5_000_000 + (BUDGET + 100) * 1000]); + group(track, 7, [5_000_000 + (2 * BUDGET + 200) * 1000]); + await flush(); + expect(frames(results)[0]).toEqual("6@5600000"); +}); + +test("a large max delay alone starts at a live edge that arrives first", async () => { + const { track, results, flush } = setup(0, 0, 0, 10_000); + group(track, 5, [5_000_000]); + await flush(); + expect(frames(results)).toEqual(["5@5000000"]); +}); + +test("a min delay past the max delay reaches only the max delay", async () => { + const { clock, track, results, flush } = setup(200, 0, 2_000); + group(track, 5, [5_000_000]); + await flush(); + await clock.advance(Nano.fromMilli(Milli(150))); + expect(frames(results)).toEqual([]); + + await clock.advance(Nano.fromMilli(Milli(50))); + expect(frames(results)).toEqual(["5@5000000"]); +}); + +test("an unlimited max delay still reaches the min delay", async () => { + const { track, results, flush } = setup(BUDGET, 0, BUDGET, null); + group(track, 5, [5_000_000]); + await flush(); + expect(frames(results)).toEqual([]); + + group(track, 4, [4_500_000]).finish(); + await flush(); + expect(frames(results)).toEqual(["4@4500000", "5@5000000"]); +}); + +test("the covering group is kept while its warmup arrives after the first read", async () => { + const { track, results, flush } = setup(200, 300); + group(track, 5, [5_000_000]); + await flush(); + // Group 4 covers the 4.5s target, and its first frame is read before the rest of it lands. + const covering = group(track, 4, [4_400_000]); + await flush(); + expect(frames(results)).toEqual(["4@4400000"]); + + for (const timestamp of [4_420_000, 4_700_000, 4_900_000]) { + write(covering, timestamp); + await flush(); + } + covering.finish(); + await flush(); + expect(frames(results)).toEqual(["4@4400000", "4@4420000", "4@4700000", "4@4900000", "5@5000000"]); +}); + +test("the covering group is kept while it arrives after the first read, without a warmup", async () => { + const { track, results, flush } = setup(200); + group(track, 5, [5_000_000]); + await flush(); + // Group 4 covers the 4.8s target, so its media before that point is older than the max delay. + const covering = group(track, 4, [4_400_000]); + await flush(); + expect(frames(results)).toEqual(["4@4400000"]); + + for (const timestamp of [4_433_000, 4_700_000, 4_900_000]) { + write(covering, timestamp); + await flush(); + } + covering.finish(); + await flush(); + expect(frames(results)).toEqual(["4@4400000", "4@4433000", "4@4700000", "4@4900000", "5@5000000"]); +}); + +test("an age skip that ends the start still warms until the start's target", async () => { + const { track, results, flush } = setup(200, 300); + // Group 4 alone does not cover its 4.5s target, so the start waits. + group(track, 4, [5_000_000]); + await flush(); + // Groups 5 and 6 decode in one pass: group 6's 6.2s edge puts the 500ms reach's target at 5.7s, + // and the age skip drops group 4, whose successor starts past the reach, before the start sees it. + const covering = group(track, 5, []); + const newest = group(track, 6, []); + await flush(); + write(covering, 5_600_000); + write(covering, 5_900_000); + write(newest, 6_200_000); + await flush(); + expect(frames(results)).toEqual(["5@5600000", "5@5900000"]); + + // The reader warms until 6.0s (the target plus the warmup), not 5.9s (the first frame plus the + // warmup), so newer content does not yet drop the covering group. + write(newest, 6_210_000); + await flush(); + write(covering, 5_950_000); + covering.finish(); + await flush(); + expect(frames(results)).toEqual(["5@5600000", "5@5900000", "5@5950000", "6@6200000", "6@6210000"]); +}); diff --git a/js/hang/src/container/index.ts b/js/hang/src/container/index.ts index 14d2edf2cc7..8c4ce6fcb2d 100644 --- a/js/hang/src/container/index.ts +++ b/js/hang/src/container/index.ts @@ -7,7 +7,7 @@ /** Re-export of {@link https://jsr.io/@moq/loc | @moq/loc}, the LOC container. */ export * as Loc from "@moq/loc"; -export { Consumer, type ConsumerProps, DEFAULT_MAX_BUFFER, type Read } from "./consumer"; +export { Consumer, type ConsumerProps, DEFAULT_MAX_BUFFER, type Delay, type Read } from "./consumer"; export type { Format } from "./format"; /** The legacy varint container. */ export * as Legacy from "./legacy"; diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 2a4b3471a29..3373fc636f4 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -141,7 +141,9 @@ export class Decoder { // The subscription's max delay: the shared budget, raised to the estimator's ceiling in "auto". #subscribeMaxDelay = new Signal(Milli.zero); - #maxDelay = new Signal(Milli.zero); + // The container consumer's delay: the playout delay to start at and the shared budget. Its warmup + // reaches the start further back, to the media the decoder primes on. + #delay = new Signal({}); // The codec's frame duration: the catalog constant, refined by each frame's own duration. #frame = new Signal(undefined); @@ -217,13 +219,15 @@ export class Decoder { } // A group the relay expires is never observed, so a subscription cut to the target would cap the - // estimate at the target it already holds. The container consumer keeps the shared budget as its - // local skip, since it observes each frame before applying it. See doc/concept/audio-jitter.md. + // estimate at the target it already holds. The container consumer keeps the shared max delay as its + // local skip, since it observes each frame before applying it, and carves out the warmup the + // subscription adds once it has warmed, as Rust's does. See doc/concept/audio-jitter.md. #runSubscribeMaxDelay(effect: Effect): void { + const delay = effect.get(this.sync.out.delay); const maxDelay = effect.get(this.sync.out.maxDelay); const auto = effect.get(this.sync.in.delay) === "auto"; const warmup = Milli(effect.get(this.source.out.config)?.warmup ?? 0); - this.#maxDelay.set(Milli(maxDelay + warmup)); + this.#delay.set({ min: delay, max: maxDelay }); this.#subscribeMaxDelay.set(Milli((auto ? Math.max(maxDelay, AUTO_MAX_DELAY) : maxDelay) + warmup)); } @@ -706,7 +710,8 @@ export class Decoder { const continues = outgoing?.path === path; const consumer = new Container.Consumer(sub, { format, - maxDelay: this.#maxDelay, + delay: this.#delay, + warmup: Milli(config.warmup ?? 0), predecessor: continues ? outgoing.consumer : undefined, }); effect.cleanup(() => consumer.close()); diff --git a/js/watch/src/audio/playout.test.ts b/js/watch/src/audio/playout.test.ts index e22d2f823a4..1bba803197a 100644 --- a/js/watch/src/audio/playout.test.ts +++ b/js/watch/src/audio/playout.test.ts @@ -80,7 +80,7 @@ async function measure(t: Arrival[], budget: (target: Milli) => Milli): Promise< const subscriber = track.subscribe({ maxDelay: budget(Milli.zero) }); const consumer = new Container.Consumer(subscriber, { format: new Container.Legacy.Format("audio"), - maxDelay: AUTO_MAX_DELAY, + delay: { max: AUTO_MAX_DELAY }, }); const result: Measured = { targets: [], delivered: 0 }; diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index bc1123ad79a..c3449156b6b 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -20,7 +20,7 @@ import { Manual, type Micro, Milli, Nano, type Second } from "@moq/clock"; import type * as Catalog from "@moq/hang/catalog"; import * as Container from "@moq/hang/container"; import { type Group, Time, Track } from "@moq/net"; -import { Effect, Signal } from "@moq/signals"; +import { Derived, Effect, Signal } from "@moq/signals"; import { type Arrival as Diagnostic, gap } from "../arrivals"; import { type Delay, Sync } from "../sync"; import { Backpressure } from "./buffer"; @@ -134,7 +134,7 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato const track = new Track.Producer("audio"); const consumer = new Container.Consumer(track.subscribe({ maxDelay: WIRE_MAX_DELAY }), { format: new Container.Legacy.Format("audio"), - maxDelay: sync.out.maxDelay, + delay: new Derived([sync.out.delay, sync.out.maxDelay], (min, max) => ({ min, max })), }); // Match the decoder: a codec constant until a delivered packet states its own duration. diff --git a/js/watch/src/retention.test.ts b/js/watch/src/retention.test.ts index 912bd4fdcb7..b262f3fdde7 100644 --- a/js/watch/src/retention.test.ts +++ b/js/watch/src/retention.test.ts @@ -3,7 +3,7 @@ import { type Micro, Milli, Nano, now, yieldNow } from "@moq/clock"; import { Container } from "@moq/hang"; import * as Moq from "@moq/net"; import { Time } from "@moq/net"; -import { Effect, Signal } from "@moq/signals"; +import { Derived, Effect, Signal } from "@moq/signals"; import { nextMedia, subscribeMedia } from "./media"; import { Sync } from "./sync"; @@ -94,7 +94,10 @@ test("a long subscription through the player path leaves nothing behind", async maxDelay: sync.out.maxDelay, }); if (!sub) throw new Error("no subscription"); - const consumer = new Container.Consumer(sub, { format, maxDelay: sync.out.maxDelay }); + const consumer = new Container.Consumer(sub, { + format, + delay: new Derived([sync.out.maxDelay], (max) => ({ max })), + }); // Presentations overlap, as a decoder's outputs do, so the clock's sleeps are shared. const presenting = new Set>(); diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index eb7d8b0283c..0ca3ddda970 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -5,6 +5,7 @@ import * as Util from "@moq/hang/util"; import type * as Moq from "@moq/net"; import { type Computed, + Derived, Effect, type Getter, getter, @@ -491,7 +492,8 @@ class DecoderTrack { #consume(effect: Effect, sub: Moq.Track.Subscriber, format: Container.Format): Container.Consumer { const consumer = new Container.Consumer(sub, { format, - maxDelay: this.sync.out.maxDelay, + // The playout delay is where a cold join starts, and the shared max delay how late it may fall. + delay: new Derived([this.sync.out.delay, this.sync.out.maxDelay], (min, max) => ({ min, max })), predecessor: this.consumer ?? this.#predecessor, }); effect.cleanup(() => consumer.close()); diff --git a/kt/moq/MIGRATE.md b/kt/moq/MIGRATE.md index f5324b79443..c5e09fe8227 100644 --- a/kt/moq/MIGRATE.md +++ b/kt/moq/MIGRATE.md @@ -4,6 +4,13 @@ How to update your code for each breaking change, newest first. ## Unreleased +### Decoder outputs take a `Delay` ([#7311](https://github.com/moq-dev/moq/pull/7311)) + +`audio.DecoderOutput` replaces `maxDelay` and the playout `delay: Duration?` with `delay: media.Delay?` and `estimate: Boolean = true`. +`video.DecoderOutput.maxDelay` is `delay: media.Delay?`. +`Delay(min, max)`: `max` (null for no limit) is the old `maxDelay`, and `min` is the playout delay, where a cold join starts. +A fixed audio playout delay `d` is `delay = Delay(min = d, max = ...)` with `estimate = false`. + ### Compression moves to the track `dev.moq.flate` is removed: publish opaque bytes on a `TrackProducer`. diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/media/Media.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/media/Media.kt index 229cda09012..b85d2cc462a 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/media/Media.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/media/Media.kt @@ -57,6 +57,8 @@ typealias ContainerInit = uniffi.moq.MoqContainerInit typealias Audio = uniffi.moq.MoqAudio /** A width and height pair, in pixels. */ typealias Dimensions = uniffi.moq.MoqDimensions +/** How far behind the live edge a decoder plays: `min` is the playout delay and where a cold join starts, and `max`, null for no limit, is how far a stalled group may fall behind before it is skipped. `min` stays on the consumer; a `min` past `max` starts at `max`. */ +typealias Delay = uniffi.moq.MoqDelay /** Where an audio or video decoder starts on a track that already holds groups: `LATEST` for a live player. */ typealias DecodeStart = uniffi.moq.MoqDecodeStart diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/video/Video.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/video/Video.kt index fb99df34bbc..bac200ba0cd 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/video/Video.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/video/Video.kt @@ -10,7 +10,7 @@ typealias Consumer = uniffi.moq.MoqVideoConsumer typealias EncoderInput = uniffi.moq.MoqVideoEncoderInput /** The codec, bitrate, group mode, and backend preference. */ typealias EncoderOutput = uniffi.moq.MoqVideoEncoderOutput -/** What a [Consumer] delivers: an optional resize, a max delay, and the decoder's surface (macOS only). */ +/** What a [Consumer] delivers: an optional resize, a delay, and the decoder's surface (macOS only). */ typealias DecoderOutput = uniffi.moq.MoqVideoDecoderOutput /** One raw picture with its presentation timestamp. */ typealias Frame = uniffi.moq.MoqVideoFrame diff --git a/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt b/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt index f96a36d52db..95b38abada3 100644 --- a/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt +++ b/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt @@ -761,7 +761,11 @@ class SmokeTest { val catalog = CatalogConsumer.subscribe(consumer).next()!! val rendition = catalog.audio["mic"]!! - val fixedOutput = AudioDecoderOutput(format = AudioSampleFormat.F32, delay = java.time.Duration.ofMillis(80)) + val fixedOutput = AudioDecoderOutput( + format = AudioSampleFormat.F32, + delay = Delay(min = java.time.Duration.ofMillis(80), max = java.time.Duration.ZERO), + estimate = false, + ) val fixed = AudioConsumer.subscribe(consumer, "mic", rendition, fixedOutput) val estimated = AudioConsumer.subscribe(consumer, "mic", rendition, AudioDecoderOutput(format = AudioSampleFormat.F32)) assertEquals(java.time.Duration.ofMillis(80), fixed.delay()) diff --git a/py/moq-rs/MIGRATE.md b/py/moq-rs/MIGRATE.md index 4e0326e3bf5..4072adde08b 100644 --- a/py/moq-rs/MIGRATE.md +++ b/py/moq-rs/MIGRATE.md @@ -4,6 +4,13 @@ How to update your code for each breaking change, newest first. ## Unreleased +### Decoder outputs take a `Delay` ([#7311](https://github.com/moq-dev/moq/pull/7311)) + +`audio.DecoderOutput` replaces `max_delay` and the playout `delay: timedelta | None` with `delay: media.Delay | None` and `estimate: bool = True`. +`video.DecoderOutput.max_delay` is `delay: media.Delay | None`. +`media.Delay(min=..., max=...)`: `max` (`None` for no limit) is the old `max_delay`, and `min` is the playout delay, where a cold join starts. +A fixed audio playout delay `d` is `delay=Delay(min=d, max=...)` with `estimate=False`. + ### Compression moves to the track `moq.flate` is removed: publish opaque bytes on a `TrackProducer`. diff --git a/py/moq-rs/moq/audio.py b/py/moq-rs/moq/audio.py index e12bccfa6e6..25e607111f6 100644 --- a/py/moq-rs/moq/audio.py +++ b/py/moq-rs/moq/audio.py @@ -21,7 +21,7 @@ from ._read import End, Gap, _end, _gap from ._records import _from_us, _to_us from ._stream import _Stream -from .media import Audio, AudioInit, CatalogProducer, DecodeStart +from .media import Audio, AudioInit, CatalogProducer, DecodeStart, Delay from .publish import TrackDemand from .session import Allocator, Reservation from .subscribe import BroadcastConsumer @@ -83,19 +83,20 @@ def _ffi(self) -> MoqAudioEncoderOutput: class DecoderOutput: """The PCM layout a :class:`Consumer` delivers, how stale a group may get, and its playout delay. - ``delay`` fixes the jitter buffer to hold ahead of the playhead; ``None`` - estimates it from arrival timing. ``max_delay`` only bounds how stale a - group may get before it is skipped, and adds no buffering. ``start`` picks - where to begin on a track that already holds groups, ``None`` for the oldest: - a live player passes ``DecodeStart.LATEST`` so a reopened decoder skips the - cached backlog. + ``delay.min`` is the jitter buffer to hold ahead of the playhead, and where a + cold join starts; with ``estimate`` (the default) it floors the delay measured + from arrival timing instead. ``delay.max`` only bounds how stale a group may + get before it is skipped, and adds no buffering. ``None`` is zero for both. + ``start`` picks where to begin on a track that already holds groups, ``None`` + for the oldest: a live player passes ``DecodeStart.LATEST`` so a reopened + decoder skips the cached backlog. """ format: SampleFormat sample_rate: int | None = None channels: int | None = None - max_delay: timedelta | None = None - delay: timedelta | None = None + delay: Delay | None = None + estimate: bool = True start: DecodeStart | None = None def _ffi(self) -> MoqAudioDecoderOutput: @@ -103,8 +104,8 @@ def _ffi(self) -> MoqAudioDecoderOutput: format=self.format, sample_rate=self.sample_rate, channels=self.channels, - max_delay=self.max_delay, delay=self.delay, + estimate=self.estimate, start=self.start, ) diff --git a/py/moq-rs/moq/media.py b/py/moq-rs/moq/media.py index 2d60c6af289..bf06e6e8228 100644 --- a/py/moq-rs/moq/media.py +++ b/py/moq-rs/moq/media.py @@ -40,6 +40,7 @@ MoqRendition, ) from moq_ffi import MoqDecodeStart as DecodeStart +from moq_ffi import MoqDelay as Delay from moq_ffi import MoqDimensions as Dimensions from moq_ffi import MoqVideo as Video from moq_ffi import MoqVideoFormat as VideoFormat @@ -428,6 +429,7 @@ async def catalog(broadcast: BroadcastConsumer) -> Catalog: "Video", "Catalog", "Container", + "Delay", "Dimensions", "DecodeStart", "AudioFormat", diff --git a/py/moq-rs/moq/video.py b/py/moq-rs/moq/video.py index 36501a46fc4..d96f782461b 100644 --- a/py/moq-rs/moq/video.py +++ b/py/moq-rs/moq/video.py @@ -21,7 +21,7 @@ from ._read import End, Gap, _end, _gap from ._records import _from_us, _to_us from ._stream import _Stream -from .media import CatalogProducer, DecodeStart, Dimensions, Video +from .media import CatalogProducer, DecodeStart, Delay, Dimensions, Video from .publish import TrackDemand from .session import Allocator, Reservation from .subscribe import BroadcastConsumer @@ -64,22 +64,24 @@ def _from_ffi(value: MoqVideoPacket) -> Packet: @dataclass(frozen=True) class DecoderOutput: - """What a :class:`Consumer` delivers: an optional resize, a max delay, and the decoder's surface (macOS only). + """What a :class:`Consumer` delivers: an optional resize, a delay, and the decoder's surface (macOS only). - ``start`` picks where to begin on a track that already holds groups, ``None`` for the - oldest: a live player passes ``DecodeStart.LATEST`` so a reopened decoder skips the - cached backlog. + ``delay.min`` is the playout delay, where a cold join starts; ``delay.max`` is how + far behind the live edge a stalled group may fall before it is skipped. ``None`` is + zero for both. ``start`` picks where to begin on a track that already holds groups, + ``None`` for the oldest: a live player passes ``DecodeStart.LATEST`` so a reopened + decoder skips the cached backlog. """ resize: Dimensions | None = None - max_delay: timedelta | None = None + delay: Delay | None = None surface: bool = False start: DecodeStart | None = None def _ffi(self) -> MoqVideoDecoderOutput: return MoqVideoDecoderOutput( resize=self.resize, - max_delay=self.max_delay, + delay=self.delay, surface=self.surface, start=self.start, ) diff --git a/py/moq-rs/tests/test_local.py b/py/moq-rs/tests/test_local.py index 3efedb002bc..f208690971a 100644 --- a/py/moq-rs/tests/test_local.py +++ b/py/moq-rs/tests/test_local.py @@ -1513,7 +1513,10 @@ async def test_audio_consumer_reports_its_playout_delay(): rendition = catalog.audio["mic"] def output(delay: timedelta | None) -> moq.audio.DecoderOutput: - return moq.audio.DecoderOutput(format=moq.AudioSampleFormat.F32, delay=delay) + if delay is None: + return moq.audio.DecoderOutput(format=moq.AudioSampleFormat.F32) + fixed = moq.media.Delay(min=delay, max=timedelta(0)) + return moq.audio.DecoderOutput(format=moq.AudioSampleFormat.F32, delay=fixed, estimate=False) fixed = await moq.audio.Consumer.subscribe(broadcast_consumer, "mic", rendition, output(timedelta(milliseconds=80))) estimated = await moq.audio.Consumer.subscribe(broadcast_consumer, "mic", rendition, output(None)) diff --git a/quest/m2/intra-refresh/README.md b/quest/m2/intra-refresh/README.md index f94e4872335..4030eb12dda 100644 --- a/quest/m2/intra-refresh/README.md +++ b/quest/m2/intra-refresh/README.md @@ -49,7 +49,6 @@ make these presentation guarantees true even when sweep lengths change. ## Required -- [Join earlier group](/quest/m2/intra-refresh/join-earlier-group.md) - max-delay decoders, native and browser, start a group before the live edge even when the live edge arrives first - [Dual decoder](/quest/m2/intra-refresh/dual-decoder.md) - js/watch runs a min-delay and a max-delay decoder and shows the max-delay picture whenever it has caught up - [Plan: recovery boundaries](/quest/m2/intra-refresh/plan-boundary.md) - settle publication ordering for a later, longer sweep - [Consumer warmup](/quest/m2/intra-refresh/consumer-warmup.md) - the transcoder decodes the groups covering `warmup` as history before each group, except at a true IDR diff --git a/quest/m2/intra-refresh/dual-decoder.md b/quest/m2/intra-refresh/dual-decoder.md index 46e5db99a25..9ce0326e948 100644 --- a/quest/m2/intra-refresh/dual-decoder.md +++ b/quest/m2/intra-refresh/dual-decoder.md @@ -57,9 +57,17 @@ Decided in the 2026-10-14 planning interview: newest group in `Container.Consumer` first, and `DecoderTrack` then drops the earlier group as stale. That would leave the max-delay decoder starting mid-sweep. Its consumer must wait for the lower group within its budget - before emitting a newer one, through the opt-in in - [join earlier group](/quest/m2/intra-refresh/join-earlier-group.md), which - fixes it at the source for Rust and `js/hang` alike. + before emitting a newer one. ✅ Decided 2026-10-30 (maintainer): a + `Container.Consumer` takes `delay: { min, max }` (ms) and starts at the + group covering the live edge less `min` (the playout delay) plus its + `warmup` prop (the catalog's name and units), waiting for it under that + reach. `max` (`null` for no limit) only bounds lateness and never moves the + start; a `min` past `max` starts at `max`. All default to zero, which + starts at the newest group at once. This decoder passes the playout delay + plus the catalog `warmup` as `min`, and the shared max delay plus the + `warmup` as `max`, so its start reaches the earlier group without the + `warmup` prop, as Rust's max-delay decoder does + ([moq-dev/moq#7311](https://github.com/moq-dev/moq/pull/7311)). - H.264 recovery-point frames sent as `key` chunks pass Chromium's WebCodecs keyframe check (`ParseH264KeyFrame` accepts `is_sei_recovery_point`; WPT `videoDecoder-h264-sei`). Chromium rejects a non-IRAP H.265 frame sent as @@ -80,10 +88,6 @@ Decided in the 2026-10-14 planning interview: dropped frames for one versus two decoders on a 1080p rendition, in a browser, recorded in the PR. -## Required - -- [Join earlier group](/quest/m2/intra-refresh/join-earlier-group.md) - the `js/hang` opt-in the max-delay decoder joins with - ## Related - [H.264 import](/quest/m2/intra-refresh/h264-import.md) - publishes the real `warmup` for an end-to-end x264 check diff --git a/quest/m2/intra-refresh/join-earlier-group.md b/quest/m2/intra-refresh/join-earlier-group.md deleted file mode 100644 index 28cb3d1a247..00000000000 --- a/quest/m2/intra-refresh/join-earlier-group.md +++ /dev/null @@ -1,48 +0,0 @@ -# [M] Max-delay decoders join a group earlier even when the live edge arrives first - -## Goal - -On a cold network join, the max-delay decoder of a `warmup` rendition starts -at the group before the live edge, in both `moq_video::decode::Consumer` and -the browser [dual decoder](/quest/m2/intra-refresh/dual-decoder.md), even -when the live-edge group's stream arrives first. Every other container -consumer keeps starting where the publisher does, exactly as today. - -## Plan - -Decided while iterating on the native dual decoder (#5842), with the -maintainer away; the agent took the recommended option for each: - -- The problem: an unfloored `moq_mux::container::Consumer` adopts the lowest - group that has arrived when it first reads, and drops a lower group that - arrives later as old. The upstream request aggregates to - `Order::Descending`, so the live-edge group often wins the race. The - max-delay decoder then starts mid-sweep, like the min-delay one, and shows - stripes for a sweep. Cached joins are fine today: `decode::Consumer` moves - that decoder's local cursor one group below `track.latest()`. -- An opt-in on the container consumer, not a new default. Waiting for an - earlier group in every unfloored consumer would break - `startup_mid_stream_live_track_starts_at_the_served_group` and delay the - first frame of every budgeted consumer when no earlier group exists. -- Semantics: once the first group arrives, wait for the sequence below it as - long as `Subscription::max_delay` says it could still be fresh, measured on - both clocks the subscription documents. In media time, that lasts until - the newest content is a budget past where the first group starts. In wall - time, until the first group arrived a budget ago. The wall clock bounds a - quiet or paused track, which never ages out in media time. -- One quest for both sides so the semantics match: the Rust opt-in in - `moq_mux::container::Consumer`, used by `decode::Consumer`'s max-delay - decoder, and the same opt-in in `js/hang`'s `Container.Consumer`, used by - the browser dual decoder. Mirror the name across Rust and JS. `moq play`'s - max-delay decoder (`rs/moq-cli/src/play/media.rs`) subscribes the same way - and takes the opt-in too. -- Open, the maintainer's call: the opt-in's name and shape. Options are a - builder such as `with_lookback(groups)`, a count of earlier groups to wait - for; a bool such as `with_earlier_group()`; or a config field. Recommend - the builder with a count, since a `warmup` longer than one group would - want more than one. -- Tests, with mocked time: newest-first arrival delivers the earlier group - first in Rust and JS; a quiet track with no earlier group starts within - one budget of wall time; consumers without the opt-in are unchanged; a - `decode::Consumer` cold join whose live edge arrives first hands over to a - max-delay decoder that started a group earlier. diff --git a/rs/moq-audio/MIGRATE.md b/rs/moq-audio/MIGRATE.md index 51df70a4bf0..6d2eb4e3932 100644 --- a/rs/moq-audio/MIGRATE.md +++ b/rs/moq-audio/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `decode::Options` takes a `Delay` and `estimate` ([#7311](https://github.com/moq-dev/moq/pull/7311)) + +`decode::Options::max_delay: Duration` and the playout `delay: Option` are one `delay: decode::Delay` (`moq_mux::container::Delay`) plus `estimate: bool`. +`delay.max` is the old `max_delay`, and `delay.min` is the playout delay, where a cold join starts. +A fixed `delay: Some(d)` is `delay.min = d` with `estimate: false`. The default `None` (estimate from arrivals) is `estimate: true`, which now never goes below `delay.min`. + ### `Level` is at the crate root `encode::Level` is `moq_audio::Level`, since the program bus's meters report it too. diff --git a/rs/moq-audio/src/decode/conformance.rs b/rs/moq-audio/src/decode/conformance.rs index cf79e67c37b..efa28af0fde 100644 --- a/rs/moq-audio/src/decode/conformance.rs +++ b/rs/moq-audio/src/decode/conformance.rs @@ -211,7 +211,10 @@ async fn consumer_preserves_reference_samples_and_timeline() { for vector in VECTORS { let (_source, broadcast, name, config) = import(vector); let options = decode::Options { - max_delay: Duration::from_secs(1), + delay: decode::Delay { + max: Some(Duration::from_secs(1)), + ..Default::default() + }, ..Default::default() }; let mut decoder = decode::Consumer::new(&broadcast, &config, name, options).await.unwrap(); diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 9e071c7ee8c..d40957e9004 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -12,6 +12,7 @@ use super::decoder::{Config, Decoder}; use crate::jitter::{self, Target}; use crate::resample::{Remix, Resampler}; use crate::{Activity, Error, Format, Frame, Layout}; +pub use moq_mux::container::Delay; /// One item of a [`Consumer`] read, in delivery order: decoded PCM, or a break in it. /// @@ -69,24 +70,38 @@ pub struct Output { } /// Subscription, decoder, and output policy for [`Consumer`]. -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug)] #[non_exhaustive] pub struct Options { /// Low-level decoder configuration. pub decoder: Config, /// PCM output conversion. pub output: Output, - /// How far a group may fall behind the live edge before the subscription skips it. - pub max_delay: std::time::Duration, + /// How far behind the live edge playback runs. + /// + /// [`Delay::min`] is the playout delay: the jitter buffer a player holds ahead of the + /// playhead, reported by [`Consumer::delay`], and where a cold join starts. With + /// [`estimate`](Self::estimate) it is a floor under the measured delay instead. A `min` + /// beyond the track's retention is refused. [`Delay::max`] is how far a group may fall + /// behind the live edge before the subscription skips it; it adds no buffering. + pub delay: Delay, + /// Measure the playout delay from arrival timing, never below [`Delay::min`] (default: + /// `true`). `false` holds exactly `delay.min`. + pub estimate: bool, /// Initial cached-group policy. pub start: Start, - /// A fixed playout delay, or `None` to estimate it from arrival timing. - /// - /// The jitter buffer a player holds ahead of the playhead, reported by - /// [`Consumer::delay`]. Distinct from [`max_delay`](Self::max_delay), which is - /// the live-edge skip budget and adds no buffering. A fixed delay beyond the - /// track's retention is refused. - pub delay: Option, +} + +impl Default for Options { + fn default() -> Self { + Self { + decoder: Config::default(), + output: Output::default(), + delay: Delay::default(), + estimate: true, + start: Start::default(), + } + } } impl Options { @@ -181,7 +196,7 @@ impl Consumer { .subscribe( moq_net::track::Subscription::default() .with_priority(hang::catalog::PRIORITY.audio) - .with_max_delay(options.max_delay.saturating_add(warmup)), + .with_max_delay(options.delay.max.map(|max| max.saturating_add(warmup))), ) .await?; // Latest skips cached history; a warmup is primed by trimming the newest group's @@ -197,10 +212,15 @@ impl Consumer { } let track = subscriber; let retention = track.info().max_age; - let max_delay = options.max_delay.min(retention.unwrap_or(Duration::MAX)); + let max_delay = options + .delay + .max + .unwrap_or(Duration::MAX) + .min(retention.unwrap_or(Duration::MAX)); // Holding more than the publisher keeps would wait on media it has already // discarded, so the caller asked for something impossible. - if let (Some(delay), Some(retention)) = (options.delay, retention) + let delay = options.delay.min; + if let Some(retention) = retention && delay > retention { return Err(Error::Unsupported(format!( @@ -211,10 +231,14 @@ impl Consumer { // wire: `moq import fmp4` publishes CMAF. Reading a moof+mdat fragment as a // varint timestamp plus a payload decodes to garbage rather than failing. let container = moq_mux::catalog::hang::Container::try_from(catalog)?; - let track = moq_mux::container::Consumer::new(track, container); + // The subscription's budget includes the warmup, so the start reaches the playout delay + // plus the warmup back, while the container skips at the max delay alone. + let track = moq_mux::container::Consumer::new(track, container) + .with_warmup(warmup) + .with_delay(options.delay); Ok(Self { - target: Target::new(options.delay, catalog.jitter), + target: Target::new(options.delay.min, options.estimate, catalog.jitter), decoder, track, resampler: None, @@ -256,7 +280,7 @@ impl Consumer { } /// The playout delay to hold ahead of the playhead right now: the fixed - /// [`Options::delay`], or the estimate from every frame read so far. + /// [`Delay::min`], or with [`Options::estimate`] the estimate from every frame read so far. /// /// The estimate measures when [`read`](Self::read) hands each packet over, so /// it is only as honest as the caller is prompt: read as soon as the previous @@ -818,7 +842,10 @@ mod tests { &catalog, "audio", Options { - max_delay: Duration::from_secs(10), + delay: Delay { + max: Some(Duration::from_secs(10)), + ..Delay::default() + }, ..Options::new() }, ) @@ -935,7 +962,10 @@ mod tests { "audio", Options { start: Start::Latest, - max_delay: Duration::from_secs(10), + delay: Delay { + max: Some(Duration::from_secs(10)), + ..Delay::default() + }, ..Options::new() }, ) @@ -988,7 +1018,10 @@ mod tests { "audio", Options { start: Start::Latest, - max_delay: Duration::from_secs(1), + delay: Delay { + max: Some(Duration::from_secs(1)), + ..Delay::default() + }, ..Options::new() }, ) @@ -1069,7 +1102,10 @@ mod tests { "audio", Options { start: Start::Latest, - max_delay: Duration::from_secs(10), + delay: Delay { + max: Some(Duration::from_secs(10)), + ..Delay::default() + }, ..Options::new() }, ) @@ -1113,7 +1149,10 @@ mod tests { &catalog, "audio", Options { - max_delay: Duration::from_secs(1), + delay: Delay { + max: Some(Duration::from_secs(1)), + ..Delay::default() + }, output: Output { sample_rate: Some(44_100), ..Output::default() @@ -1166,7 +1205,10 @@ mod tests { &catalog, "audio", Options { - max_delay: Duration::from_secs(10), + delay: Delay { + max: Some(Duration::from_secs(10)), + ..Delay::default() + }, ..Options::new() }, ) @@ -1199,7 +1241,10 @@ mod tests { &catalog, "audio", Options { - max_delay: Duration::from_secs(10), + delay: Delay { + max: Some(Duration::from_secs(10)), + ..Delay::default() + }, output: Output { sample_rate: Some(48_000), ..Output::default() @@ -1341,7 +1386,10 @@ mod tests { ); let catalog = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Pcm, 48_000, 1); let options = Options { - max_delay: Duration::from_secs(1), + delay: Delay { + max: Some(Duration::from_secs(1)), + ..Delay::default() + }, ..Options::new() }; let mut consumer = Consumer::new(&broadcast.consume(), &catalog, "audio", options) @@ -1393,7 +1441,10 @@ mod tests { sample_rate: Some(48_000), ..Output::default() }, - max_delay: Duration::from_secs(1), + delay: Delay { + max: Some(Duration::from_secs(1)), + ..Delay::default() + }, ..Options::new() }, ) @@ -1467,7 +1518,10 @@ mod tests { sample_rate: Some(48_000), ..Output::default() }, - max_delay: std::time::Duration::from_secs(1), + delay: Delay { + max: Some(std::time::Duration::from_secs(1)), + ..Delay::default() + }, ..Options::new() }, ) @@ -1601,7 +1655,10 @@ mod tests { sample_rate: Some(44_100), ..Output::default() }, - max_delay: std::time::Duration::from_secs(1), + delay: Delay { + max: Some(std::time::Duration::from_secs(1)), + ..Delay::default() + }, ..Options::new() }, ) @@ -1684,7 +1741,10 @@ mod tests { sample_rate: Some(out_rate), ..Output::default() }, - max_delay: std::time::Duration::from_secs(1), + delay: Delay { + max: Some(std::time::Duration::from_secs(1)), + ..Delay::default() + }, ..Options::new() }, ) @@ -1931,7 +1991,10 @@ mod tests { &catalog, "audio", Options { - max_delay: std::time::Duration::from_secs(1), + delay: Delay { + max: Some(std::time::Duration::from_secs(1)), + ..Delay::default() + }, ..Options::new() }, ) @@ -2019,7 +2082,11 @@ mod tests { &catalog, "audio", Options { - delay: case.delay.map(duration), + delay: Delay { + min: case.delay.map_or(Duration::ZERO, duration), + ..Delay::default() + }, + estimate: case.delay.is_none(), ..Options::new() }, ) @@ -2133,7 +2200,11 @@ mod tests { let catalog = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Pcm, 48_000, 1); let options = |delay| Options { - delay: Some(std::time::Duration::from_millis(delay)), + delay: Delay { + min: std::time::Duration::from_millis(delay), + ..Delay::default() + }, + estimate: false, ..Options::new() }; let err = Consumer::new(&subscriber, &catalog, "audio", options(101)) @@ -2162,7 +2233,10 @@ mod tests { &catalog, "audio", Options { - max_delay: std::time::Duration::from_millis(500), + delay: Delay { + max: Some(std::time::Duration::from_millis(500)), + ..Delay::default() + }, ..Options::new() }, ) @@ -2201,7 +2275,10 @@ mod tests { &catalog, "audio", Options { - max_delay: std::time::Duration::from_secs(1), + delay: Delay { + max: Some(std::time::Duration::from_secs(1)), + ..Delay::default() + }, ..Options::new() }, ) @@ -2255,7 +2332,10 @@ mod tests { &catalog, "audio", Options { - max_delay: std::time::Duration::from_secs(1), + delay: Delay { + max: Some(std::time::Duration::from_secs(1)), + ..Delay::default() + }, ..Options::new() }, ) @@ -2314,7 +2394,10 @@ mod tests { &catalog, "audio", Options { - max_delay, + delay: Delay { + max: Some(max_delay), + ..Delay::default() + }, ..Options::new() }, ) diff --git a/rs/moq-audio/src/decode/mod.rs b/rs/moq-audio/src/decode/mod.rs index dc0850838ff..a0e71455cd0 100644 --- a/rs/moq-audio/src/decode/mod.rs +++ b/rs/moq-audio/src/decode/mod.rs @@ -19,6 +19,6 @@ mod consumer; mod decoded; mod decoder; -pub use consumer::{Consumer, Options, Output, Read, Start}; +pub use consumer::{Consumer, Delay, Options, Output, Read, Start}; pub use decoded::Decoded; pub use decoder::{Config, Decoder, Kind}; diff --git a/rs/moq-audio/src/encode/producer.rs b/rs/moq-audio/src/encode/producer.rs index b7de46d10a2..e3ce9ca3578 100644 --- a/rs/moq-audio/src/encode/producer.rs +++ b/rs/moq-audio/src/encode/producer.rs @@ -735,7 +735,10 @@ mod tests { &decoder_config, "audio", DecodeOptions { - max_delay: Duration::from_secs(1), + delay: crate::decode::Delay { + max: Some(Duration::from_secs(1)), + ..Default::default() + }, ..DecodeOptions::new() }, ) @@ -915,7 +918,10 @@ mod tests { &decoder_config, "audio", DecodeOptions { - max_delay: Duration::from_millis(500), + delay: crate::decode::Delay { + max: Some(Duration::from_millis(500)), + ..Default::default() + }, ..DecodeOptions::new() }, ) diff --git a/rs/moq-audio/src/jitter.rs b/rs/moq-audio/src/jitter.rs index 21330ae1986..62f295b8a81 100644 --- a/rs/moq-audio/src/jitter.rs +++ b/rs/moq-audio/src/jitter.rs @@ -53,11 +53,13 @@ pub(crate) enum Target { } impl Target { - /// A fixed target for `Some`, otherwise the estimator floored at `advertised`. - pub(crate) fn new(delay: Option, advertised: Option) -> Self { - match delay { - Some(delay) => Self::Fixed(millis(delay)), - None => Self::Auto(Box::new(Estimator::new(advertised.map_or(0.0, millis)))), + /// The estimator floored at `delay` and `advertised` when `estimate`, otherwise `delay` fixed. + pub(crate) fn new(delay: Duration, estimate: bool, advertised: Option) -> Self { + match estimate { + false => Self::Fixed(millis(delay)), + true => Self::Auto(Box::new(Estimator::new(millis( + advertised.unwrap_or_default().max(delay), + )))), } } @@ -430,7 +432,11 @@ pub(crate) mod tests { fn conforms_to_the_corpus() { for name in CORPUS { let case = case(name); - let mut target = Target::new(case.delay.map(duration), case.advertised.map(duration)); + let mut target = Target::new( + case.delay.map_or(Duration::ZERO, duration), + case.delay.is_none(), + case.advertised.map(duration), + ); target.set_frame(case.frame); let series = case.series(); @@ -450,7 +456,7 @@ pub(crate) mod tests { /// ahead of it must not carry the media-time gap into the target. #[test] fn a_timestamp_jump_is_not_a_delay() { - let mut target = Target::new(None, None); + let mut target = Target::new(Duration::ZERO, true, None); target.set_frame(20.0); target.observe(0.0, 0.0); for i in 0..500 { @@ -461,14 +467,14 @@ pub(crate) mod tests { #[test] fn an_advertised_span_floors_rather_than_adds() { - let target = Target::new(None, Some(Duration::from_millis(200))); + let target = Target::new(Duration::ZERO, true, Some(Duration::from_millis(200))); // The prior reads 100 ms, under the advertised 200 ms. assert_eq!(target.millis(), 200.0); } #[test] fn the_measured_term_is_bounded_by_the_histogram() { - let mut target = Target::new(None, None); + let mut target = Target::new(Duration::ZERO, true, None); target.set_frame(20.0); // A frame every 20 ms of media arriving a whole second apart: every interval // sees a delay far past the last bucket. diff --git a/rs/moq-audio/tests/integration/import.rs b/rs/moq-audio/tests/integration/import.rs index 9d2a52df539..413705a061c 100644 --- a/rs/moq-audio/tests/integration/import.rs +++ b/rs/moq-audio/tests/integration/import.rs @@ -25,7 +25,7 @@ async fn ts_surround_opus_decodes() { let mut options = decode::Options::default(); options.output.format = Format::F32; // The whole file is imported before anything decodes it. - options.max_delay = Duration::from_secs(30); + options.delay.max = Some(Duration::from_secs(30)); let mut consumer = decode::Consumer::new(&broadcast_consumer, config, name, options) .await .unwrap(); diff --git a/rs/moq-audio/tests/integration/roundtrip.rs b/rs/moq-audio/tests/integration/roundtrip.rs index 86de7c061df..ff4fcbbf9b3 100644 --- a/rs/moq-audio/tests/integration/roundtrip.rs +++ b/rs/moq-audio/tests/integration/roundtrip.rs @@ -90,7 +90,7 @@ async fn opus_roundtrip(case: Case) { options.output.sample_rate = Some(case.rate); options.output.layout = Some(case.layout); // The whole clip is queued before subscription; retain every group. - options.max_delay = Duration::from_secs(30); + options.delay.max = Some(Duration::from_secs(30)); let mut consumer = decode::Consumer::new(&source, rendition, "audio", options) .await .unwrap(); diff --git a/rs/moq-c/MIGRATE.md b/rs/moq-c/MIGRATE.md index c5970dc36b2..29f031fc42e 100644 --- a/rs/moq-c/MIGRATE.md +++ b/rs/moq-c/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### Media consumers take a `moq_delay` ([#7311](https://github.com/moq-dev/moq/pull/7311)) + +`moq_consume_video` and `moq_consume_audio` take `const moq_delay *delay` in place of `uint64_t max_delay_us`; NULL is the default. +`moq_video_decoder_output.max_delay_us` and `moq_audio_decoder_output.max_delay_us` are `moq_delay delay`, which changes both structs' layout. +Set `delay.max_us` to the old `max_delay_us`, `delay.max_unlimited` for no limit, and `delay.min_us` to your playout delay, where a cold join starts. A zeroed `moq_delay` is zero for both. + ### A subscription without `max_delay_present` has no limit A `moq_subscription` with `max_delay_present` false sets no subscriber limit, leaving the publisher's max age alone to bound it, instead of moq-net's default of zero. diff --git a/rs/moq-c/README.md b/rs/moq-c/README.md index af18306b768..b295f9c5855 100644 --- a/rs/moq-c/README.md +++ b/rs/moq-c/README.md @@ -126,11 +126,11 @@ int32_t moq_consume_video_enabled(uint32_t catalog, uint32_t index, bool *dst); int32_t moq_consume_audio_config(uint32_t catalog, uint32_t index, moq_audio_config *dst); // Consuming: Video -int32_t moq_consume_video(uint32_t catalog, uint32_t index, uint64_t max_delay_us, moq_status_callback on_frame, void *user_data); +int32_t moq_consume_video(uint32_t catalog, uint32_t index, const moq_delay *delay, moq_status_callback on_frame, void *user_data); int32_t moq_consume_video_cancel(uint32_t track); // Consuming: Audio -int32_t moq_consume_audio(uint32_t catalog, uint32_t index, uint64_t max_delay_us, moq_status_callback on_frame, void *user_data); +int32_t moq_consume_audio(uint32_t catalog, uint32_t index, const moq_delay *delay, moq_status_callback on_frame, void *user_data); int32_t moq_consume_audio_cancel(uint32_t track); // Consuming: Frames diff --git a/rs/moq-c/c-tests/audio-decode-output.c b/rs/moq-c/c-tests/audio-decode-output.c index 4262c61c1d6..377916abfaa 100644 --- a/rs/moq-c/c-tests/audio-decode-output.c +++ b/rs/moq-c/c-tests/audio-decode-output.c @@ -96,7 +96,7 @@ static void on_read(void *data, int32_t id) { int main(void) { Context c = {.mutex = PTHREAD_MUTEX_INITIALIZER, .changed = PTHREAD_COND_INITIALIZER}; - moq_audio_decoder_output invalid = {999, 0, 0, 0}; + moq_audio_decoder_output invalid = {999, 0, 0, {0}}; CHECK(moq_decode_audio(INT32_MAX, 0, &invalid, on_read, &c) == -15); CHECK(moq_decode_audio(INT32_MAX, 0, NULL, on_read, &c) == -6); @@ -134,7 +134,7 @@ int main(void) { for (size_t format = 0; format < sizeof(formats) / sizeof(formats[0]); format++) { for (uint32_t channels = 1; channels <= 2; channels++) { c.format = formats[format]; c.channels = channels; c.frames = 0; c.decode_done = 0; - moq_audio_decoder_output decoded = {c.format, 48000, channels, 30000000}; + moq_audio_decoder_output decoded = {c.format, 48000, channels, {.max_us = 30000000}}; int32_t decoder = moq_decode_audio((uint32_t)c.catalog, 0, &decoded, on_read, &c); CHECK(decoder > 0); pthread_mutex_lock(&c.mutex); diff --git a/rs/moq-c/c-tests/decode-output.c b/rs/moq-c/c-tests/decode-output.c index 82822d80864..0ac2272e112 100644 --- a/rs/moq-c/c-tests/decode-output.c +++ b/rs/moq-c/c-tests/decode-output.c @@ -31,12 +31,16 @@ _Static_assert(MOQ_VIDEO_PIXEL_FORMAT_I420 == 0, "I420 discriminant moved"); _Static_assert(MOQ_VIDEO_PIXEL_FORMAT_RGBA == 1, "RGBA discriminant moved"); -_Static_assert(offsetof(moq_video_decoder_output, max_delay_us) == 0, "max_delay_us moved"); -_Static_assert(offsetof(moq_video_decoder_output, format) == 8, "format moved"); -_Static_assert(offsetof(moq_video_decoder_output, width) == 12, "width moved"); -_Static_assert(offsetof(moq_video_decoder_output, height) == 16, "height moved"); -_Static_assert(offsetof(moq_video_decoder_output, start) == 20, "start moved"); -_Static_assert(sizeof(moq_video_decoder_output) == 24, "decoder output size changed"); +_Static_assert(offsetof(moq_delay, min_us) == 0, "min_us moved"); +_Static_assert(offsetof(moq_delay, max_us) == 8, "max_us moved"); +_Static_assert(offsetof(moq_delay, max_unlimited) == 16, "max_unlimited moved"); +_Static_assert(sizeof(moq_delay) == 24, "delay size changed"); +_Static_assert(offsetof(moq_video_decoder_output, delay) == 0, "delay moved"); +_Static_assert(offsetof(moq_video_decoder_output, format) == 24, "format moved"); +_Static_assert(offsetof(moq_video_decoder_output, width) == 28, "width moved"); +_Static_assert(offsetof(moq_video_decoder_output, height) == 32, "height moved"); +_Static_assert(offsetof(moq_video_decoder_output, start) == 36, "start moved"); +_Static_assert(sizeof(moq_video_decoder_output) == 40, "decoder output size changed"); _Static_assert(MOQ_DECODE_READ_FRAME == 0, "frame read discriminant moved"); _Static_assert(MOQ_DECODE_READ_GAP == 1, "gap read discriminant moved"); _Static_assert(MOQ_DECODE_READ_END == 2, "end read discriminant moved"); @@ -66,14 +70,14 @@ static void ignore_status(void *user_data, int32_t status) { static void check_refusals(void) { // Unknown pixel format. - moq_video_decoder_output bad_format = {0, 999, 0, 0}; + moq_video_decoder_output bad_format = {{0}, 999, 0, 0}; if (moq_decode_video(1, 0, &bad_format, NULL, NULL) != MOQ_ERR_INVALID_CODE) fail("error: unknown format was not refused with InvalidCode: %s\n", moq_error()); // Half-set, odd, and unrepresentable sizes. const uint32_t bad_sizes[][2] = {{320, 0}, {0, 240}, {321, 240}, {320, 241}, {UINT32_MAX - 1, UINT32_MAX - 1}}; for (size_t i = 0; i < sizeof(bad_sizes) / sizeof(bad_sizes[0]); i++) { - moq_video_decoder_output bad_size = {0, MOQ_VIDEO_PIXEL_FORMAT_I420, bad_sizes[i][0], bad_sizes[i][1]}; + moq_video_decoder_output bad_size = {{0}, MOQ_VIDEO_PIXEL_FORMAT_I420, bad_sizes[i][0], bad_sizes[i][1]}; if (moq_decode_video(1, 0, &bad_size, NULL, NULL) != MOQ_ERR_INVALID_CONFIG) fail("error: size %ux%u was not refused with InvalidConfig: %s\n", bad_sizes[i][0], bad_sizes[i][1], moq_error()); @@ -84,7 +88,7 @@ static void check_refusals(void) { fail("error: null output was not refused with InvalidPointer: %s\n", moq_error()); // A valid request gets past validation and fails on the bogus catalog. - moq_video_decoder_output valid = {0, MOQ_VIDEO_PIXEL_FORMAT_RGBA, 160, 120}; + moq_video_decoder_output valid = {{0}, MOQ_VIDEO_PIXEL_FORMAT_RGBA, 160, 120}; if (moq_decode_video(INT32_MAX, 0, &valid, ignore_status, NULL) != MOQ_ERR_CATALOG_NOT_FOUND) fail("error: valid request did not reach catalog lookup: %s\n", moq_error()); @@ -272,11 +276,11 @@ int main(void) { pthread_mutex_unlock(&c.mu); // The I420 native default, RGBA at native size, and a resized I420. - moq_video_decoder_output native_i420 = {10000000, MOQ_VIDEO_PIXEL_FORMAT_I420, 0, 0}; + moq_video_decoder_output native_i420 = {{.max_us = 10000000}, MOQ_VIDEO_PIXEL_FORMAT_I420, 0, 0}; decode_once(&c, catalog, &native_i420, WIDTH, HEIGHT, WIDTH * HEIGHT * 3 / 2, producer); - moq_video_decoder_output native_rgba = {10000000, MOQ_VIDEO_PIXEL_FORMAT_RGBA, 0, 0}; + moq_video_decoder_output native_rgba = {{.max_us = 10000000}, MOQ_VIDEO_PIXEL_FORMAT_RGBA, 0, 0}; decode_once(&c, catalog, &native_rgba, WIDTH, HEIGHT, WIDTH * HEIGHT * 4, producer); - moq_video_decoder_output small_i420 = {10000000, MOQ_VIDEO_PIXEL_FORMAT_I420, 160, 120}; + moq_video_decoder_output small_i420 = {{.max_us = 10000000}, MOQ_VIDEO_PIXEL_FORMAT_I420, 160, 120}; decode_once(&c, catalog, &small_i420, 160, 120, 160 * 120 * 3 / 2, producer); if (moq_consume_catalog_free((uint32_t)catalog) < 0) diff --git a/rs/moq-c/src/api.rs b/rs/moq-c/src/api.rs index b3322b0bcf8..16800ceca47 100644 --- a/rs/moq-c/src/api.rs +++ b/rs/moq-c/src/api.rs @@ -694,6 +694,43 @@ pub enum moq_demand { MOQ_DEMAND_UNWANTED = 2, } +/// How far behind the live edge a media consumer plays. +/// +/// A zeroed struct is the default, and so is a null pointer where one is taken: start at +/// the newest group and skip aggressively. +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub struct moq_delay { + /// The target latency in microseconds: the playout or jitter-buffer delay held behind + /// the live edge, and where a cold join starts. It stays on the consumer and never + /// reaches the subscription. A `min_us` past the max starts at the max, since the + /// publisher serves nothing older. + pub min_us: u64, + /// How far a group may fall behind the live edge before it is skipped, in + /// microseconds. It bounds lateness only, never the start. Ignored when + /// `max_unlimited` is set. + pub max_us: u64, + /// No max: wait on a stalled group as long as anything could. + pub max_unlimited: bool, +} + +impl From for moq_mux::container::Delay { + fn from(delay: moq_delay) -> Self { + Self { + min: std::time::Duration::from_micros(delay.min_us), + max: (!delay.max_unlimited).then(|| std::time::Duration::from_micros(delay.max_us)), + } + } +} + +/// A [moq_delay] from a pointer that may be null, for the defaults. +/// +/// # Safety +/// - `delay` must be null or point to a valid [moq_delay]. +pub(crate) unsafe fn parse_delay(delay: *const moq_delay) -> moq_mux::container::Delay { + unsafe { delay.as_ref() }.copied().unwrap_or_default().into() +} + /// Subscriber-side raw track delivery preferences. /// /// A null [moq_consume_track] or [moq_consume_track_update] `subscription` @@ -3916,7 +3953,8 @@ pub unsafe extern "C" fn moq_consume_catalog_section( /// Consume a video track from a broadcast, delivering encoded frames, gaps, and encoder ends in order. /// -/// - `max_delay_us` controls the maximum amount of buffering allowed before skipping a GoP. +/// - `delay` is how far behind the live edge to play: where a cold join starts and when a +/// stalled GoP is skipped. NULL uses the defaults (see [moq_delay]). /// - `on_read` is called with a positive read ID per item, then exactly once /// more with a terminal code: `0` (closed cleanly) or a negative error. After /// the terminal (`<= 0`) callback, `on_read` is never called again and @@ -3926,23 +3964,24 @@ pub unsafe extern "C" fn moq_consume_catalog_section( /// Returns a non-zero handle to the track on success, or a negative code on failure. /// /// # Safety +/// - `delay` must be NULL or point to a valid [moq_delay]. /// - The caller must keep `user_data` valid until the terminal (`<= 0`) `on_read` callback. #[unsafe(no_mangle)] pub unsafe extern "C" fn moq_consume_video( catalog: u32, index: u32, - max_delay_us: u64, + delay: *const moq_delay, on_read: ffi::moq_status_callback, user_data: *mut c_void, ) -> i32 { ffi::enter(move || { let catalog = ffi::parse_id(catalog)?; let index = index as usize; - let max_delay = std::time::Duration::from_micros(max_delay_us); + let delay = unsafe { parse_delay(delay) }; let on_read = unsafe { ffi::OnStatus::new(user_data, on_read)? }; State::lock() .consume - .media::(catalog, index, max_delay, on_read) + .media::(catalog, index, delay, on_read) }) } @@ -3967,28 +4006,30 @@ pub extern "C" fn moq_consume_video_cancel(track: u32) -> i32 { /// the terminal (`<= 0`) callback, `on_read` is never called again and /// `user_data` is never touched again, so release `user_data` there. The /// terminal callback fires even after [moq_consume_audio_cancel]. -/// The `max_delay_us` parameter controls how long to wait before skipping frames. +/// `delay` is how far behind the live edge to play: where a cold join starts and when +/// frames are skipped. NULL uses the defaults (see [moq_delay]). /// /// Returns a non-zero handle to the track on success, or a negative code on failure. /// /// # Safety +/// - `delay` must be NULL or point to a valid [moq_delay]. /// - The caller must keep `user_data` valid until the terminal (`<= 0`) `on_read` callback. #[unsafe(no_mangle)] pub unsafe extern "C" fn moq_consume_audio( catalog: u32, index: u32, - max_delay_us: u64, + delay: *const moq_delay, on_read: ffi::moq_status_callback, user_data: *mut c_void, ) -> i32 { ffi::enter(move || { let catalog = ffi::parse_id(catalog)?; let index = index as usize; - let max_delay = std::time::Duration::from_micros(max_delay_us); + let delay = unsafe { parse_delay(delay) }; let on_read = unsafe { ffi::OnStatus::new(user_data, on_read)? }; State::lock() .consume - .media::(catalog, index, max_delay, on_read) + .media::(catalog, index, delay, on_read) }) } diff --git a/rs/moq-c/src/audio.rs b/rs/moq-c/src/audio.rs index 0fd7227286e..8801776bb96 100644 --- a/rs/moq-c/src/audio.rs +++ b/rs/moq-c/src/audio.rs @@ -118,13 +118,12 @@ pub struct moq_audio_decoder_output { /// layout as `moq_audio_encoder_input.channels` describes, and the decoder /// remixes to it. pub channels: u32, - /// Upper bound on buffering before skipping a stalled group, in - /// microseconds. Same congestion-control knob as - /// `moq_consume_audio`'s `max_delay_us`. 0 = skip - /// aggressively (the moq-mux default); set to your playout - /// buffer (tens to a few hundred ms) for a softer skip. This bounds - /// group staleness rather than setting a minimum playout delay. - pub max_delay_us: u64, + /// How far behind the live edge to play. Same knob as `moq_consume_audio`'s + /// `delay`. `min_us` floors the playout delay, which is otherwise measured + /// from arrival timing, and sets where a cold join starts. `max_us` bounds + /// group staleness and adds no buffering. Zeroed: start at the newest group + /// and skip aggressively (the moq-mux default). + pub delay: crate::api::moq_delay, /// `moq_decode_start` discriminant: where to start on a track that already /// holds groups. 0 = the oldest, decoding everything cached; a live player /// sets `MOQ_DECODE_START_LATEST`. @@ -572,7 +571,7 @@ pub unsafe extern "C" fn moq_decode_audio( config.output.layout = zeroable(raw.channels) .map(moq_audio::Layout::from_channels) .transpose()?; - config.max_delay = Duration::from_micros(raw.max_delay_us); + config.delay = raw.delay.into(); let on_read = unsafe { OnStatus::new(user_data, on_read)? }; diff --git a/rs/moq-c/src/consume.rs b/rs/moq-c/src/consume.rs index 703a9020896..b187e78da9f 100644 --- a/rs/moq-c/src/consume.rs +++ b/rs/moq-c/src/consume.rs @@ -496,7 +496,7 @@ impl Consume { &mut self, catalog: Id, index: usize, - max_delay: std::time::Duration, + delay: moq_mux::container::Delay, on_read: OnStatus, ) -> Result where @@ -528,10 +528,10 @@ impl Consume { .subscribe( moq_net::track::Subscription::default() .with_priority(C::PRIORITY) - .with_max_delay(max_delay), + .with_max_delay(delay.max), ) .await?; - let track = moq_mux::container::Consumer::new(track, container); + let track = moq_mux::container::Consumer::new(track, container).with_delay(delay); Self::run_track(on_read, track, close).await }) .await diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index 2ed889ef1d9..2a89a0732df 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -7,6 +7,13 @@ use std::time::Duration; const TIMEOUT: Duration = Duration::from_secs(10); +/// A ten-second max delay, the media consumers' usual budget here. +const TEN_SECONDS: moq_delay = moq_delay { + min_us: 0, + max_us: 10_000_000, + max_unlimited: false, +}; + /// Convert a positive `i32` return value to `u32`, panicking on error. fn id(raw: i32) -> u32 { assert!(raw > 0, "expected positive id, got {raw}"); @@ -1293,7 +1300,7 @@ fn raw_loc_video_uses_the_declared_catalog_container() { assert!(video_cfg.container.init.is_null()); let frame_cb = Callback::new(); - let consumer = id(unsafe { moq_consume_video(catalog, 0, 10_000_000, Some(channel_callback), frame_cb.ptr) }); + let consumer = id(unsafe { moq_consume_video(catalog, 0, &TEN_SECONDS, Some(channel_callback), frame_cb.ptr) }); let timestamp_us = 42_000; let payload = b"codec frame"; @@ -2828,7 +2835,7 @@ fn double_close_all_resource_types() { let catalog_id = id(catalog_cb.recv()); let frame_cb = Callback::new(); - let track = id(unsafe { moq_consume_audio(catalog_id, 0, 10_000_000, Some(channel_callback), frame_cb.ptr) }); + let track = id(unsafe { moq_consume_audio(catalog_id, 0, &TEN_SECONDS, Some(channel_callback), frame_cb.ptr) }); let payload = b"test"; assert_eq!( @@ -4043,7 +4050,7 @@ fn local_publish_consume() { ); let frame_cb = Callback::new(); - let track = id(unsafe { moq_consume_audio(catalog_id, 0, 10_000_000, Some(channel_callback), frame_cb.ptr) }); + let track = id(unsafe { moq_consume_audio(catalog_id, 0, &TEN_SECONDS, Some(channel_callback), frame_cb.ptr) }); let payload = b"opus audio payload data"; let timestamp_us: u64 = 1_000_000; @@ -4208,7 +4215,7 @@ fn consume_audio_follows_a_sibling_broadcast_reference() { let catalog_id = id(catalog_cb.recv()); let frame_cb = Callback::new(); - let track = id(unsafe { moq_consume_audio(catalog_id, 0, 10_000_000, Some(channel_callback), frame_cb.ptr) }); + let track = id(unsafe { moq_consume_audio(catalog_id, 0, &TEN_SECONDS, Some(channel_callback), frame_cb.ptr) }); // Published on the sibling, so it only arrives if the reference was followed. let payload = b"opus audio payload data"; @@ -4669,7 +4676,7 @@ fn video_publish_consume() { ); let frame_cb = Callback::new(); - let track = id(unsafe { moq_consume_video(catalog_id, 0, 10_000_000, Some(channel_callback), frame_cb.ptr) }); + let track = id(unsafe { moq_consume_video(catalog_id, 0, &TEN_SECONDS, Some(channel_callback), frame_cb.ptr) }); let keyframe = [0x00, 0x00, 0x00, 0x01, 0x65, 0xAA, 0xBB, 0xCC]; assert_eq!( @@ -4862,7 +4869,10 @@ fn audio_raw_decode_reads_the_encoder_end() { format: moq_audio_sample_format::MOQ_AUDIO_SAMPLE_FORMAT_F32 as u32, sample_rate: 0, channels: 0, - max_delay_us: 30_000_000, + delay: moq_delay { + max_us: 30_000_000, + ..moq_delay::default() + }, start: 0, }; let read_cb = Callback::new(); @@ -4964,7 +4974,10 @@ fn video_raw_publish_consume() { let catalog_id = id(catalog_cb.recv()); let decoder = moq_video_decoder_output { - max_delay_us: 10_000_000, + delay: moq_delay { + max_us: 10_000_000, + ..moq_delay::default() + }, format: moq_video_pixel_format::MOQ_VIDEO_PIXEL_FORMAT_I420 as u32, width: 0, height: 0, @@ -5032,7 +5045,7 @@ fn video_raw_decode_output_rejected() { use crate::ffi::ReturnCode; let bad_format = moq_video_decoder_output { - max_delay_us: 0, + delay: moq_delay::default(), format: 999, width: 0, height: 0, @@ -5044,7 +5057,7 @@ fn video_raw_decode_output_rejected() { ); let bad_start = moq_video_decoder_output { - max_delay_us: 0, + delay: moq_delay::default(), format: moq_video_pixel_format::MOQ_VIDEO_PIXEL_FORMAT_I420 as u32, width: 0, height: 0, @@ -5057,7 +5070,7 @@ fn video_raw_decode_output_rejected() { for (width, height) in [(320, 0), (0, 240), (321, 240), (320, 241), (u32::MAX - 1, u32::MAX - 1)] { let bad_size = moq_video_decoder_output { - max_delay_us: 0, + delay: moq_delay::default(), format: moq_video_pixel_format::MOQ_VIDEO_PIXEL_FORMAT_I420 as u32, width, height, @@ -5076,7 +5089,7 @@ fn video_raw_decode_output_rejected() { ); let valid = moq_video_decoder_output { - max_delay_us: 0, + delay: moq_delay::default(), format: moq_video_pixel_format::MOQ_VIDEO_PIXEL_FORMAT_RGBA as u32, width: 160, height: 120, @@ -5176,7 +5189,10 @@ fn decode_first_frame(output: &moq_video_decoder_output) -> (u32, u32, usize) { #[test] fn video_raw_decode_rgba() { let output = moq_video_decoder_output { - max_delay_us: 10_000_000, + delay: moq_delay { + max_us: 10_000_000, + ..moq_delay::default() + }, format: moq_video_pixel_format::MOQ_VIDEO_PIXEL_FORMAT_RGBA as u32, width: 0, height: 0, @@ -5190,7 +5206,10 @@ fn video_raw_decode_rgba() { #[test] fn video_raw_decode_resize() { let output = moq_video_decoder_output { - max_delay_us: 10_000_000, + delay: moq_delay { + max_us: 10_000_000, + ..moq_delay::default() + }, format: moq_video_pixel_format::MOQ_VIDEO_PIXEL_FORMAT_I420 as u32, width: 160, height: 120, @@ -5597,7 +5616,10 @@ fn video_raw_decode() { // Subscribe + decode before publishing frames so the keyframe group is delivered. let output = moq_video_decoder_output { - max_delay_us: 10_000_000, + delay: moq_delay { + max_us: 10_000_000, + ..moq_delay::default() + }, format: moq_video_pixel_format::MOQ_VIDEO_PIXEL_FORMAT_I420 as u32, width: 0, height: 0, @@ -5669,7 +5691,7 @@ fn multiple_frames_ordering() { let catalog_id = id(catalog_cb.recv()); let frame_cb = Callback::new(); - let track = id(unsafe { moq_consume_audio(catalog_id, 0, 10_000_000, Some(channel_callback), frame_cb.ptr) }); + let track = id(unsafe { moq_consume_audio(catalog_id, 0, &TEN_SECONDS, Some(channel_callback), frame_cb.ptr) }); let timestamps: [u64; 5] = [0, 20_000, 40_000, 60_000, 80_000]; for (i, &ts) in timestamps.iter().enumerate() { diff --git a/rs/moq-c/src/video.rs b/rs/moq-c/src/video.rs index 83c13a75f2a..c8f67092025 100644 --- a/rs/moq-c/src/video.rs +++ b/rs/moq-c/src/video.rs @@ -14,7 +14,6 @@ use std::ffi::{c_char, c_void}; use std::sync::{Arc, OnceLock}; -use std::time::Duration; use tokio::sync::oneshot; @@ -167,11 +166,12 @@ pub struct moq_video_encoder_frame { /// with the `libmoq.a` they link. #[repr(C)] pub struct moq_video_decoder_output { - /// Upper bound on buffering before skipping a stalled group, in - /// microseconds. Same congestion-control knob as - /// `moq_consume_video`'s `max_delay_us`. 0 = skip aggressively - /// (the moq-mux default); set to your playout buffer for a softer skip. - pub max_delay_us: u64, + /// How far behind the live edge to play: where a cold join starts and when + /// a stalled group is skipped. Same knob as `moq_consume_video`'s `delay`. + /// Zeroed: start at the newest group and skip aggressively (the moq-mux + /// default); set `min_us` to your playout delay and `max_us` at or past it + /// for a softer skip. + pub delay: crate::api::moq_delay, /// `moq_video_pixel_format` discriminant. Unknown values fail /// [`moq_decode_video`] rather than decoding into an assumed layout. pub format: u32, @@ -1201,7 +1201,7 @@ pub unsafe extern "C" fn moq_decode_video( let mut options = moq_video::decode::Options::new(); options.start = start; - options.max_delay = Duration::from_micros(raw.max_delay_us); + options.delay = raw.delay.into(); // The C caller takes packed pixels, so let a backend that can decode // straight to the CPU do that rather than downloading afterwards. options.decoder.output = moq_video::Output::Cpu; diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 7d4eff35425..1cb0b53c3b2 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -267,7 +267,7 @@ impl Media { // doubles as the staleness budget on the wire. let max_delay = self.args.video_max_delay(&snapshot.audio); // A decoder and its subscription, starting `lookback` groups before the live edge. - let open = async |max_delay: Duration, lookback: u64| { + let open = async |delay: moq_mux::container::Delay, lookback: u64| { // The window's renderer refuses a GPU surface it cannot import, so // the decoder downloads those instead, all of them on a device // that imports none. @@ -279,7 +279,7 @@ impl Media { .subscribe( moq_net::track::Subscription::default() .with_priority(hang::catalog::PRIORITY.video) - .with_max_delay(max_delay), + .with_max_delay(delay.max), ) .await?; // Start at the live edge without asking the shared publisher @@ -290,10 +290,17 @@ impl Media { subscriber = subscriber.with_range(largest.group.saturating_sub(lookback)..); } let format = catalog::hang::Container::try_from(config)?; - Ok::<_, anyhow::Error>((moq_mux::container::Consumer::new(subscriber, format), decoder)) + // The playout delay is the target latency; the floor above bounds how far + // back the start can wait, but the consumer still learns both bounds. + let track = moq_mux::container::Consumer::new(subscriber, format).with_delay(delay); + Ok::<_, anyhow::Error>((track, decoder)) }; let opened = async { - let (track, decoder) = open(max_delay, 0).await?; + let delay = moq_mux::container::Delay { + min: self.args.video_delay(), + max: Some(max_delay), + }; + let (track, decoder) = open(delay, 0).await?; // An intra-refresh rendition also runs a decoder with one more sweep // of budget, joining a group earlier so its pictures stay clean, as // `moq_video::decode::Consumer` does. Both share one upstream request. @@ -301,7 +308,13 @@ impl Media { let refresh = if warmup.is_zero() { None } else { - let (track, decoder) = open(max_delay.saturating_add(warmup), 1).await?; + // The min-delay decoder's delay plus one sweep, as in + // `moq_video::decode::Consumer`. + let delay = moq_mux::container::Delay { + min: delay.min.saturating_add(warmup), + max: Some(max_delay.saturating_add(warmup)), + }; + let (track, decoder) = open(delay, 1).await?; let interval = config .framerate .filter(|rate| rate.is_finite() && *rate > 0.0) @@ -383,8 +396,11 @@ impl Media { // asked for, so a smaller budget would skip a group the playhead could // still have reached, and would size the hole fill below to a playhead // that does not exist. - decode.max_delay = self.args.max_delay().max(AUDIO_BUFFER_MIN); - decode.delay = self.args.fixed_delay(); + decode.delay = moq_audio::decode::Delay { + min: self.args.fixed_delay().unwrap_or_default(), + max: Some(self.args.max_delay().max(AUDIO_BUFFER_MIN)), + }; + decode.estimate = self.args.fixed_delay().is_none(); // The sink and the frame-duration math below both assume f32, // so ask for it rather than inheriting the decoder default. decode.output.format = moq_audio::Format::F32; @@ -890,8 +906,11 @@ mod tests { .unwrap(); let mut producer = moq_mux::container::Producer::new(track, Container::Legacy(moq_mux::container::Kind::Audio)); let mut options = moq_audio::decode::Options::new(); - options.delay = Some(delay); - options.max_delay = delay; + options.delay = moq_audio::decode::Delay { + min: delay, + max: Some(delay), + }; + options.estimate = false; options.output.format = moq_audio::Format::F32; let consumer = moq_audio::decode::Consumer::new( &broadcast.consume(), @@ -993,8 +1012,11 @@ mod tests { let mut audio = moq_mux::container::Producer::new(audio, Container::Legacy(moq_mux::container::Kind::Audio)); let mut video = moq_mux::container::Producer::new(video, Container::Legacy(moq_mux::container::Kind::Video)); let mut options = moq_audio::decode::Options::new(); - options.delay = Some(delay); - options.max_delay = delay; + options.delay = moq_audio::decode::Delay { + min: delay, + max: Some(delay), + }; + options.estimate = false; options.output.format = moq_audio::Format::F32; let consumer = moq_audio::decode::Consumer::new( &broadcast.consume(), @@ -1149,8 +1171,11 @@ mod tests { .unwrap(); let mut producer = moq_mux::container::Producer::new(track, Container::Legacy(moq_mux::container::Kind::Audio)); let mut options = moq_audio::decode::Options::new(); - options.delay = Some(delay); - options.max_delay = delay; + options.delay = moq_audio::decode::Delay { + min: delay, + max: Some(delay), + }; + options.estimate = false; options.output.format = moq_audio::Format::F32; options.output.sample_rate = Some(SAMPLE_RATE); let consumer = moq_audio::decode::Consumer::new( @@ -1232,8 +1257,11 @@ mod tests { producer.discontinuity().unwrap(); // Joins at the end marker: the stopped encoder's media is already behind it. let mut options = moq_audio::decode::Options::new(); - options.delay = Some(delay); - options.max_delay = delay; + options.delay = moq_audio::decode::Delay { + min: delay, + max: Some(delay), + }; + options.estimate = false; options.start = moq_audio::decode::Start::Latest; options.output.format = moq_audio::Format::F32; let consumer = moq_audio::decode::Consumer::new( @@ -1284,8 +1312,11 @@ mod tests { .unwrap(); let mut producer = moq_mux::container::Producer::new(track, Container::Legacy(moq_mux::container::Kind::Audio)); let mut options = moq_audio::decode::Options::new(); - options.delay = Some(delay); - options.max_delay = delay; + options.delay = moq_audio::decode::Delay { + min: delay, + max: Some(delay), + }; + options.estimate = false; options.output.format = moq_audio::Format::F32; let consumer = moq_audio::decode::Consumer::new( &broadcast.consume(), @@ -2133,7 +2164,7 @@ mod tests { }); let mut options = moq_video::decode::Options::new(); options.decoder.kind = moq_video::decode::Kind::Software; - options.max_delay = delay; + options.delay.max = Some(delay); let decoder = moq_video::decode::Sink::open(&catalog, &options.decoder).await.unwrap(); let subscriber = broadcast .consume() diff --git a/rs/moq-ffi/MIGRATE.md b/rs/moq-ffi/MIGRATE.md index 93ae354dd37..6ee04bb9f27 100644 --- a/rs/moq-ffi/MIGRATE.md +++ b/rs/moq-ffi/MIGRATE.md @@ -5,6 +5,13 @@ These are the UniFFI names; each language wrapper has its own `MIGRATE.md` in it ## Unreleased +### Decoder outputs take a `MoqDelay` ([#7311](https://github.com/moq-dev/moq/pull/7311)) + +`MoqAudioDecoderOutput` replaces `max_delay` and the playout `delay: Option` with `delay: Option` and `estimate: bool` (default `true`). +`MoqVideoDecoderOutput::max_delay` is `delay: Option`. +`MoqDelay { min, max }`: `max` (null for no limit) is the old `max_delay`, and `min` is the playout delay, where a cold join starts. A null `delay` is zero for both. +A fixed audio playout delay `d` is `delay: MoqDelay { min: d, max }` with `estimate: false`; with `estimate` it floors the estimate instead. + ### Compression moves to the track The `MoqFlate*` producers and `MoqFlateConfig` are deleted: publish opaque bytes on a `MoqTrackProducer`. diff --git a/rs/moq-ffi/src/audio.rs b/rs/moq-ffi/src/audio.rs index 90f1abd54b4..f0cd65ff311 100644 --- a/rs/moq-ffi/src/audio.rs +++ b/rs/moq-ffi/src/audio.rs @@ -164,21 +164,20 @@ pub struct MoqAudioDecoderOutput { /// the decoder remixes to it. #[uniffi(default = None)] pub channels: Option, - /// Upper bound on buffering before skipping a stalled group. Same - /// congestion-control knob as - /// [`MoqSubscription::max_delay`](crate::consumer::MoqSubscription::max_delay): - /// when a group stalls and a newer group is more than this far ahead, - /// the consumer skips. `None` keeps the moq-mux default of zero (skip - /// aggressively). Distinct from [`delay`](Self::delay): this one - /// bounds how stale a group may be and adds no buffering. + /// How far behind the live edge to play. `min` is the playout delay: the + /// jitter buffer to hold ahead of the playhead, reported by + /// [`MoqAudioConsumer::delay`], and where a cold join starts; with + /// [`estimate`](Self::estimate) it floors the measured delay instead. A `min` + /// beyond the track's retention is refused. `max` is the congestion-control + /// knob [`MoqSubscription::max_delay`](crate::consumer::MoqSubscription::max_delay) + /// carries: a stalled group more than this far behind a newer one is skipped, + /// with no buffering added. `None` keeps the moq-mux default of zero for both. #[uniffi(default = None)] - pub max_delay: Option, - /// A fixed playout delay, or `None` to estimate it from arrival timing. This - /// is the jitter buffer to hold ahead of the playhead, reported by - /// [`MoqAudioConsumer::delay`]. A fixed delay beyond the track's retention - /// is refused. - #[uniffi(default = None)] - pub delay: Option, + pub delay: Option, + /// Measure the playout delay from arrival timing, never below `delay.min`. + /// `false` holds exactly `delay.min`. + #[uniffi(default = true)] + pub estimate: bool, /// Where to start on a track that already holds groups, or `None` for the oldest. A live /// player picks [`Latest`](crate::media::MoqDecodeStart::Latest) so a reopened decoder /// skips the backlog. @@ -597,7 +596,8 @@ impl MoqAudioConsumer { } /// The playout delay to hold ahead of the playhead: the fixed - /// [`MoqAudioDecoderOutput::delay`], or the estimate from every frame read + /// [`MoqAudioDecoderOutput::delay`]'s min, or with + /// [`MoqAudioDecoderOutput::estimate`] the estimate from every frame read /// so far. /// /// The estimate measures when [`Self::next`] returns each frame, so call it @@ -670,8 +670,8 @@ impl MoqAudioConsumer { config.output.format = output.format.into(); config.output.sample_rate = output.sample_rate; config.output.layout = output.channels.map(moq_audio::Layout::from_channels).transpose()?; - config.max_delay = output.max_delay.unwrap_or_default(); - config.delay = output.delay; + config.delay = output.delay.map(Into::into).unwrap_or_default(); + config.estimate = output.estimate; config.start = match output.start.unwrap_or_default() { crate::media::MoqDecodeStart::Oldest => moq_audio::decode::Start::Oldest, crate::media::MoqDecodeStart::Latest => moq_audio::decode::Start::Latest, diff --git a/rs/moq-ffi/src/media.rs b/rs/moq-ffi/src/media.rs index ddac8c5fb0b..3da7421872b 100644 --- a/rs/moq-ffi/src/media.rs +++ b/rs/moq-ffi/src/media.rs @@ -23,6 +23,27 @@ pub struct MoqDimensions { pub height: u32, } +/// How far behind the live edge a decoder plays, the counterpart of `moq_mux::container::Delay`. +#[derive(Clone, Copy, Debug, uniffi::Record)] +pub struct MoqDelay { + /// The target latency: the playout or jitter-buffer delay held behind the live edge, and + /// where a cold join starts. It stays on the consumer and never reaches the subscription. A + /// `min` past `max` starts at `max`, since the publisher serves nothing older. + pub min: std::time::Duration, + /// How far a group may fall behind the live edge before it is skipped, or null for no limit. + /// It bounds lateness only, never the start. + pub max: Option, +} + +impl From for moq_mux::container::Delay { + fn from(delay: MoqDelay) -> Self { + Self { + min: delay.min, + max: delay.max, + } + } +} + /// Where an audio or video decoder starts on a track that already holds groups. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, uniffi::Enum)] pub enum MoqDecodeStart { diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index e1bbd7ce5b7..f68658f3143 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -315,8 +315,8 @@ fn audio_output() -> crate::audio::MoqAudioDecoderOutput { format: MoqAudioSampleFormat::F32, sample_rate: None, channels: None, - max_delay: None, delay: None, + estimate: true, start: None, } } @@ -710,7 +710,10 @@ async fn audio_subscribes_a_held_disabled_rendition() { "voice".into(), muted, MoqAudioDecoderOutput { - max_delay: Some(Duration::from_secs(30)), + delay: Some(crate::media::MoqDelay { + min: Duration::ZERO, + max: Some(Duration::from_secs(30)), + }), ..audio_output() }, ) @@ -774,7 +777,10 @@ async fn audio_raw_publish_from_many_threads() { MoqAudioDecoderOutput { // The track's whole retention: reading starts only after finish, and the budget // ages groups by wall clock, so a stalled host would otherwise skip the start. - max_delay: Some(Duration::from_secs(30)), + delay: Some(crate::media::MoqDelay { + min: Duration::ZERO, + max: Some(Duration::from_secs(30)), + }), ..audio_output() }, ) @@ -908,13 +914,17 @@ async fn raw_audio_decode_delay() { .expect("expected a raw audio catalog"); let rendition = catalog.audio.get("microphone").unwrap().clone(); - let subscribe = |delay| { + let subscribe = |delay: Option| { MoqAudioConsumer::subscribe( &consumer, "microphone".into(), rendition.clone(), MoqAudioDecoderOutput { - delay, + delay: delay.map(|min| crate::media::MoqDelay { + min, + max: Some(Duration::ZERO), + }), + estimate: delay.is_none(), ..audio_output() }, ) @@ -1009,7 +1019,10 @@ async fn raw_audio_decode_reports_encoder_ends() { snapshot.audio["microphone"].clone(), MoqAudioDecoderOutput { // Reading starts only after finish, so keep every group. - max_delay: Some(Duration::from_secs(30)), + delay: Some(crate::media::MoqDelay { + min: Duration::ZERO, + max: Some(Duration::from_secs(30)), + }), ..audio_output() }, ) @@ -4111,7 +4124,10 @@ async fn video_decode_start() { MoqVideoDecoderOutput { start, // The track's whole retention, so a slow host does not skip the first group. - max_delay: Some(Duration::from_secs(30)), + delay: Some(crate::media::MoqDelay { + min: Duration::ZERO, + max: Some(Duration::from_secs(30)), + }), ..Default::default() }, ) diff --git a/rs/moq-ffi/src/video.rs b/rs/moq-ffi/src/video.rs index 68d123df536..6dafb4fc784 100644 --- a/rs/moq-ffi/src/video.rs +++ b/rs/moq-ffi/src/video.rs @@ -1043,11 +1043,13 @@ pub struct MoqVideoDecoderOutput { /// dimensions rather than assuming this took. Both dimensions must be even. #[uniffi(default = None)] pub resize: Option, - /// Upper bound on buffering before skipping a stalled group. Same knob as - /// [`MoqAudioDecoderOutput::max_delay`](crate::audio::MoqAudioDecoderOutput::max_delay). - /// `None` keeps the moq-mux default of zero (skip aggressively). + /// How far behind the live edge to play: where a cold join starts and when a + /// stalled group is skipped. Same knob as + /// [`MoqAudioDecoderOutput::delay`](crate::audio::MoqAudioDecoderOutput::delay). + /// `None` keeps the moq-mux default of zero for both (start at the newest group, + /// skip aggressively). #[uniffi(default = None)] - pub max_delay: Option, + pub delay: Option, /// Keep each frame in the surface its decoder produced, for /// [`MoqVideoDecodedFrame::surface`]. `false`, the default, downloads every /// frame to CPU memory as it is decoded, so @@ -1309,7 +1311,7 @@ impl MoqVideoConsumer { let resolved = broadcast.resolve_inner(reference.as_deref(), epoch.as_deref()).await?; options.decoder.scale_hint = output.resize.map(|size| moq_video::Size::new(size.width, size.height)); - options.max_delay = output.max_delay.unwrap_or_default(); + options.delay = output.delay.map(Into::into).unwrap_or_default(); options.start = match output.start.unwrap_or_default() { crate::media::MoqDecodeStart::Oldest => moq_video::decode::Start::Oldest, crate::media::MoqDecodeStart::Latest => moq_video::decode::Start::Latest, diff --git a/rs/moq-mux/MIGRATE.md b/rs/moq-mux/MIGRATE.md index e684af7e784..9abb9fd4679 100644 --- a/rs/moq-mux/MIGRATE.md +++ b/rs/moq-mux/MIGRATE.md @@ -4,6 +4,13 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A `Delay` sets where a consumer starts ([#7311](https://github.com/moq-dev/moq/pull/7311)) + +`container::Consumer::set_max_delay(Duration)` is `set_delay(Delay)`, and `with_delay(Delay)` is its builder. +`container::Delay { min, max }` defaults to zero for both: `max` (`None` for no limit) is the max delay that `set_max_delay` took, and the subscription still carries it. +`min` is the playout delay. An unfloored consumer starts at the group covering the live edge less `min` plus any `with_warmup`, waiting for it when the newest group arrives first; the max delay never moves the start. +Replace `consumer.set_max_delay(d)` with `consumer.set_delay(Delay { max: Some(d), ..Delay::default() })`, and set `min` to your playout delay. Call `with_warmup` before `with_delay`. + ### Compression moves to the track `catalog.json.z` is gone: `catalog::Producer` publishes one `catalog.json` with `compress: true` (the MSF `catalog` stays uncompressed), and HLS record tracks are compressed too. diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 3f566a91a93..2481abd1063 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -73,7 +73,9 @@ pub(crate) enum Item { /// /// Delivery starts at the lowest absolute start among the /// [`Subscription::ranges`](moq_net::track::Subscription::ranges) when every range names one, waiting for that group under the budget; without one it -/// starts wherever the publisher does, adopted from the first served groups. From there +/// starts at the lowest group that arrived, waiting for one covering the live edge less +/// [`Delay::min`] plus any [`with_warmup`](Self::with_warmup). Both default to zero, so it waits +/// for nothing. The max delay never moves the start. From there /// the same budget is the one rule that catches the consumer up to the live edge, and /// history the publisher no longer serves expires like any other gap, since its reach /// sits a full budget behind the newest content. @@ -86,7 +88,7 @@ pub(crate) enum Item { /// /// Put the initial max delay on the [`moq_net::track::Subscription`] before /// subscribing. [`new`](Self::new) inherits that budget, and -/// [`set_max_delay`](Self::set_max_delay) changes it mid-stream. +/// [`with_delay`](Self::with_delay) or [`set_delay`](Self::set_delay) sets the whole [`Delay`]. /// /// ## Gaps and ends /// @@ -146,10 +148,26 @@ pub struct Consumer { // Latch onto the newest group instead, discarding what arrived below it. newest: bool, + // How far behind the live edge an unfloored start reaches, before the warmup: see `Delay::min`. + min_delay: std::time::Duration, + + // How much further than the min delay an unfloored start reaches: see `with_warmup`. + warmup: std::time::Duration, + + // When the newest group had landed as the first one arrived, once an unfloored start waits. + anchor: Option, + + // The point an unfloored start reached back to, the live edge less the reach, once it chose. + target: Option, + + // Where the warming ends: the warmup past the first frame delivered, or past the start's target + // when that is later. `None` before the first frame. + warmed: Option, + // How far we may drift from the live edge before skipping a group. max_delay: std::time::Duration, - // Wakes the reader once the hole at the cursor expires. + // Wakes the reader once the hole at the cursor, or the start's wait, expires. hole_timer: Option>>, // The furthest presentation point delivered so far: max(timestamp + duration), endpoints @@ -188,6 +206,34 @@ pub struct Consumer { /// about one group duration. const CONTIGUITY_TOLERANCE: std::time::Duration = std::time::Duration::from_millis(1); +/// How far behind the live edge a [`Consumer`] plays: where it starts, and how late it may fall. +/// +/// Both default to zero, which starts at the newest group and skips aggressively. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Delay { + /// The target latency: the playout or jitter-buffer delay a player holds behind the live edge. + /// + /// The start is the group covering the live edge less `min`, plus any + /// [`with_warmup`](Consumer::with_warmup), waiting for it when the newest group arrives first. + /// Consumer-only: it never reaches the subscription. A `min` past `max` starts at `max`, since + /// the publisher serves nothing older. + pub min: std::time::Duration, + /// How far a group may fall behind the live edge before it is skipped, or `None` for no limit. + /// + /// The subscription carries it as [`moq_net::track::Subscription::max_delay`]. It bounds + /// lateness only: it never moves the start. + pub max: Option, +} + +impl Default for Delay { + fn default() -> Self { + Self { + min: std::time::Duration::ZERO, + max: Some(std::time::Duration::ZERO), + } + } +} + /// The default [`Consumer::set_max_buffer`]: one group at the transport's size cap. pub const DEFAULT_MAX_BUFFER: usize = moq_net::group::MAX_CACHE_BYTES as usize; @@ -282,6 +328,11 @@ impl Consumer { gated: false, startup: start.is_none(), newest: false, + min_delay: std::time::Duration::ZERO, + warmup: std::time::Duration::ZERO, + anchor: None, + target: None, + warmed: None, max_delay, hole_timer: None, edge: None, @@ -305,6 +356,47 @@ impl Consumer { self } + /// Reach `warmup` further back than the min delay for the first group, so a decoder that needs + /// that much media before its output is clean starts early enough, such as an Opus decoder or + /// the max-delay decoder of an intra-refresh rendition. Pass the catalog rendition's `warmup`. + /// + /// Put the max delay plus `warmup` on the [`moq_net::track::Subscription`], since the publisher + /// and the transport skip what falls outside that budget. This carves `warmup` back out of the + /// subscription's budget as it stands, so the consumer skips at the max delay alone. Call it + /// before [`with_delay`](Self::with_delay), which keeps the warmup on the subscription. + /// + /// Where delivery would start wherever the publisher does, it starts no later than the group + /// covering the live edge less [`Delay::min`] and the warmup: one that starts at or before that + /// point. Both default to zero, which the newest group covers, so the start waits for nothing. + /// Otherwise groups can arrive newest first, so when none that arrived covers that point the + /// start waits for an earlier one, until one covers it, the subscription's ranges hold none + /// earlier, the track finishes, the max buffer fills, or the newest group's arrival is that whole + /// reach ago. The wall clock bounds a quiet or paused track, which never ages out in media time. + /// The start is then the lowest group that arrived, and the max delay skips from there. Until the + /// reader is the warmup past both its first frame and that point, the skip also tolerates the + /// warmup, so the media leading up to the start is not dropped as stale. + /// + /// A subscription that names a start already waits for it, and a live start takes the newest + /// group. + pub fn with_warmup(mut self, warmup: std::time::Duration) -> Self { + // Carve the warmup out of what was asked for, then clamp to retention as `new` does, so a + // budget the track can keep is not cut by the clamp too. + if let Some(track) = &self.track { + let requested = track.subscription().max_delay.unwrap_or(std::time::Duration::MAX); + self.max_delay = requested + .saturating_sub(warmup) + .min(track.info().max_age.unwrap_or(std::time::Duration::MAX)); + } + self.warmup = warmup; + self + } + + /// Set the [`Delay`] as a builder; see [`set_delay`](Self::set_delay). + pub fn with_delay(mut self, delay: Delay) -> Self { + self.set_delay(delay); + self + } + /// The group sequence of the last event read. pub(crate) fn group(&self) -> u64 { self.group @@ -452,7 +544,11 @@ impl Consumer { } start } - false => self.pending.front().expect("a group has a frame").sequence, + // The start may still wait for a group earlier than any that arrived. + false => match self.reach_start(waiter, finished) { + Some(start) => start, + None => return Poll::Pending, + }, }; // Nothing has been read, so discarding what lies below the start is no gap. self.discard(|group| group.sequence < start)?; @@ -483,6 +579,10 @@ impl Consumer { if self.start.is_none_or(|(start, _)| start != seq) { self.start = Some((seq, ts)); } + // A group covering the target began before it, so the reader warms until the target too. + let first = std::time::Duration::from(ts); + self.warmed + .get_or_insert_with(|| first.max(self.target.unwrap_or(first)).saturating_add(self.warmup)); let end = std::time::Duration::from(ts) + frame.duration.map(std::time::Duration::from).unwrap_or_default(); self.advance(end); @@ -667,9 +767,13 @@ impl Consumer { // while the newer groups race ahead on their own streams. Skip early too // once it has presented up to where the next group begins (duration // coverage), since nothing is left worth waiting for. + // While the reader warms, the start reached the warmup further back than the max delay. + let budget = match self.warming() { + true => self.max_delay.saturating_add(self.warmup), + false => self.max_delay, + }; let should_skip = next_group.is_some_and(|(_, next_start)| { - max_timestamp.saturating_sub(next_start) >= self.max_delay - || current_end.is_some_and(|end| end >= next_start) + max_timestamp.saturating_sub(next_start) >= budget || current_end.is_some_and(|end| end >= next_start) }); let target = match should_skip { true => next_group.map(|(i, start)| (i, Some(start))), @@ -721,6 +825,71 @@ impl Consumer { } } + // Where an unfloored start begins, the lowest group that arrived, or `None` while it waits for + // one covering the reach (see `with_warmup`), registering the waiter on whatever would end the wait. + fn reach_start(&mut self, waiter: &kio::Waiter, finished: bool) -> Option { + let lowest = self.pending.front().expect("a group has a frame").sequence; + // The publisher serves nothing older than the max delay, so a min delay past it reaches no + // further. + let reach = self.min_delay.min(self.max_delay).saturating_add(self.warmup); + // The newest group covers a zero reach. + if reach.is_zero() { + return Some(lowest); + } + + // The live edge, the newest content, read once looking ahead has decoded it. + let mut edge = None; + for group in self.pending.iter_mut().rev() { + if let Poll::Ready(Some(ts)) = group.poll_max_timestamp(waiter, &self.format) { + edge = Some(std::time::Duration::from(ts)); + break; + } + } + // Untimed content, such as a marker, has no edge to reach back from. + let Some(edge) = edge else { + return Some(lowest); + }; + let target = edge.saturating_sub(reach); + self.target = Some(target); + let covered = self.pending.iter().any(|group| { + group + .min_timestamp + .is_some_and(|start| std::time::Duration::from(start) <= target) + }); + if covered { + return Some(lowest); + } + + // Nothing earlier can arrive or be held. Looking ahead may have filled the max buffer, + // which then registers no waiter on more frames. + let floor = self.subscriber().next_in_range(0); + if finished || self.memory.full() || floor.is_none_or(|floor| lowest <= floor) { + return Some(lowest); + } + + // Wall time: the newest group landed a whole reach ago. + let arrived = *self + .anchor + .get_or_insert_with(|| self.pending.back().expect("a group has a frame").group.arrived()); + if arrived.elapsed() >= reach { + return Some(lowest); + } + if let Some(wake) = arrived.checked_add(reach) { + let timer = self + .hole_timer + .get_or_insert_with(|| Box::pin(kio::clock::sleep_until(wake))); + timer.as_mut().reset(wake); + let _ = waiter.poll_future(timer.as_mut()); + } + None + } + + // The reader has not yet read past the start's target and the warmup past its first frame. + fn warming(&self) -> bool { + self.warmed + .is_none_or(|warmed| self.edge.is_none_or(|edge| edge < warmed)) + } + // The subscription, which only an error ending the reader drops. fn subscriber(&mut self) -> &mut moq_net::track::Subscriber { self.track.as_mut().expect("read only until an error ends the reader") @@ -872,20 +1041,31 @@ impl Consumer { } } - /// Set the max delay mid-stream, clamped to the track's retention window like - /// [`new`](Self::new). The subscription keeps the requested value verbatim, - /// matching [`moq_net::track::Subscription::max_delay`]. - pub fn set_max_delay(&mut self, max_delay: std::time::Duration) { + /// Set the [`Delay`], mid-stream or before the first read. + /// + /// [`Delay::max`] is clamped to the track's retention window like [`new`](Self::new). The + /// subscription keeps the requested value verbatim, matching + /// [`moq_net::track::Subscription::max_delay`], plus any [`with_warmup`](Self::with_warmup). + /// [`Delay::min`] stays here: it moves only a start still to come. + pub fn set_delay(&mut self, delay: Delay) { // An ended reader has no subscription left to tune. let Some(track) = &mut self.track else { return; }; - self.max_delay = max_delay.min(track.info().max_age.unwrap_or(std::time::Duration::MAX)); + self.min_delay = delay.min; + self.max_delay = delay + .max + .unwrap_or(std::time::Duration::MAX) + .min(track.info().max_age.unwrap_or(std::time::Duration::MAX)); // The transport enforces the same budget on the subscription itself, so a // tolerance set here has to reach it: otherwise moq-net skips the very groups - // this consumer was told to wait for, before they ever get here. - let subscription = track.subscription().with_max_delay(max_delay); - let _ = track.update(subscription); + // this consumer was told to wait for, before they ever get here. It keeps the + // warmup too, as `with_warmup` asks of the subscription. + let max = delay.max.map(|max| max.saturating_add(self.warmup)); + if track.subscription().max_delay != max { + let subscription = track.subscription().with_max_delay(max); + let _ = track.update(subscription); + } } /// Cap the bytes of decoded frames held ahead of the reader, counting the wire payload they share @@ -1358,7 +1538,10 @@ mod tests { ) -> Consumer { let control = track.control(); let mut consumer = Consumer::new(track, Container::Legacy(crate::container::Kind::Data)); - consumer.set_max_delay(max_delay); + consumer.set_delay(Delay { + max: Some(max_delay), + ..Delay::default() + }); control .update(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(30))) .unwrap(); @@ -2912,6 +3095,401 @@ mod tests { assert_eq!(micros, vec![0, 100_000], "the slow stream's frames are delivered"); } + // ---- Start reach ---- + + /// An unfloored consumer that starts `min` plus `warmup` behind the live edge and skips at `max`. + fn reach_consumer( + track: &mut moq_net::track::Producer, + min: Duration, + max: Duration, + warmup: Duration, + ) -> Consumer { + let consumer_track = track.subscribe(moq_net::track::Subscription::default().with_max_delay(max + warmup)); + Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)) + .with_warmup(warmup) + .with_delay(Delay { min, max: Some(max) }) + } + + /// The live edge's stream wins the race, yet the start waits for a group covering the reach. + #[tokio::test] + async fn the_start_waits_for_an_earlier_group_when_the_live_edge_arrives_first() { + tokio::time::pause(); + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let mut consumer = reach_consumer( + &mut track, + Duration::from_millis(500), + Duration::from_millis(500), + Duration::ZERO, + ); + + write_group(&mut track, 5, &[ts(5_000_000)]); + assert!( + kio::clock::timeout(Duration::from_millis(50), consumer.read()) + .await + .is_err(), + "an earlier group within the budget must be waited for" + ); + + write_group(&mut track, 4, &[ts(4_800_000)]); + track.finish().unwrap(); + + let frames = read_all(&mut consumer).await.unwrap(); + let micros: Vec = frames.iter().map(|f| f.timestamp.as_micros()).collect(); + assert_eq!( + micros, + vec![4_800_000, 5_000_000], + "the earlier group is delivered first" + ); + } + + /// With no min delay and no warmup, the same race starts at the live edge and drops the earlier + /// group, as before. + #[tokio::test] + async fn a_zero_reach_starts_at_a_live_edge_that_arrives_first() { + tokio::time::pause(); + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let mut consumer = reach_consumer(&mut track, Duration::ZERO, Duration::ZERO, Duration::ZERO); + + write_group(&mut track, 5, &[ts(5_000_000)]); + let frame = kio::clock::timeout(Duration::from_millis(50), consumer.read_frame()) + .await + .expect("nothing to wait for") + .unwrap() + .unwrap(); + assert_eq!(frame.timestamp, ts(5_000_000)); + + write_group(&mut track, 4, &[ts(4_000_000)]); + track.finish().unwrap(); + assert!( + read_all(&mut consumer).await.unwrap().is_empty(), + "the earlier group is old" + ); + } + + /// The warmup reaches past the min delay for the start, waiting for the group covering the live + /// edge less both. + #[tokio::test] + async fn the_warmup_reaches_further_back_for_the_start() { + tokio::time::pause(); + for (warmup, expected) in [ + (Duration::ZERO, vec![4_700_000, 5_000_000]), + (Duration::from_millis(300), vec![4_400_000, 4_700_000, 5_000_000]), + ] { + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let mut consumer = reach_consumer( + &mut track, + Duration::from_millis(200), + Duration::from_millis(200), + warmup, + ); + + // Newest first: group 4 covers the min delay alone, but not the warmup too. + write_group(&mut track, 5, &[ts(5_000_000)]); + write_group(&mut track, 4, &[ts(4_700_000)]); + let first = kio::clock::timeout(Duration::from_millis(50), consumer.read_frame()).await; + write_group(&mut track, 3, &[ts(4_400_000)]); + track.finish().unwrap(); + + let frames = first.ok().map(|frame| frame.unwrap().unwrap()); + let frames = frames.into_iter().chain(read_all(&mut consumer).await.unwrap()); + let micros: Vec = frames.map(|f| f.timestamp.as_micros()).collect(); + assert_eq!(micros, expected, "warmup {warmup:?}"); + } + } + + /// A hole is skipped at the max delay alone: the warmup does not widen it. + #[tokio::test] + async fn the_warmup_does_not_widen_the_skip() { + tokio::time::pause(); + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let mut consumer = reach_consumer( + &mut track, + Duration::from_millis(200), + Duration::from_millis(200), + Duration::from_millis(300), + ); + + write_group(&mut track, 0, &[ts(0)]); + let frame = consumer.read_frame().await.unwrap().unwrap(); + assert_eq!(frame.timestamp, ts(0)); + + // Group 1 is missing, and group 2's content runs 250ms past where it starts: more than the + // max delay, though less than the max delay plus the warmup. + write_group(&mut track, 2, &[ts(100_000), ts(350_000)]); + let frame = kio::clock::timeout(Duration::from_millis(50), consumer.read_frame()) + .await + .expect("the hole is skipped at the max delay") + .unwrap() + .unwrap(); + assert_eq!(frame.timestamp, ts(100_000)); + } + + /// The group covering the reach stays while the rest of its warmup arrives after the first read, + /// though its successor starts past the max delay alone. + #[tokio::test] + async fn the_covering_group_is_kept_while_its_warmup_arrives_after_the_first_read() { + tokio::time::pause(); + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let mut consumer = reach_consumer(&mut track, Duration::ZERO, Duration::ZERO, Duration::from_millis(80)); + + write_group(&mut track, 5, &[ts(5_000_000)]); + // Group 4 covers the 4.92s target, and its first frame is read before the rest of it lands. + let mut covering = track.create_group(moq_net::group::Info { sequence: 4 }).unwrap(); + let write = |group: &mut moq_net::group::Producer, micros| { + let frame = Frame { + timestamp: ts(micros), + payload: Bytes::from_static(&[0xDE, 0xAD]), + keyframe: false, + duration: None, + }; + Container::Legacy(crate::container::Kind::Data) + .write(group, &[frame]) + .unwrap(); + }; + write(&mut covering, 4_900_000); + let frame = kio::clock::timeout(Duration::from_millis(50), consumer.read_frame()) + .await + .expect("group 4 covers the reach") + .unwrap() + .unwrap(); + assert_eq!(frame.timestamp, ts(4_900_000)); + assert!( + kio::clock::timeout(Duration::from_millis(50), consumer.read()) + .await + .is_err(), + "the covering group's next frame is waited for" + ); + + write(&mut covering, 4_950_000); + covering.finish().unwrap(); + track.finish().unwrap(); + let micros: Vec = read_all(&mut consumer) + .await + .unwrap() + .iter() + .map(|f| f.timestamp.as_micros()) + .collect(); + assert_eq!(micros, vec![4_950_000, 5_000_000], "the warmup is not skipped as stale"); + } + + /// The warmup is carved out of the requested budget before the retention clamp, so a max delay + /// the track can keep stays whole. + #[tokio::test] + async fn the_warmup_is_carved_before_the_retention_clamp() { + let mut info = hang::container::track_info(hang::catalog::PRIORITY.audio); + info.max_age = Some(Duration::from_secs(1)); + let mut track = track_producer("test", info); + let consumer = reach_consumer( + &mut track, + Duration::ZERO, + Duration::from_secs(1), + Duration::from_millis(300), + ); + assert_eq!(consumer.max_delay, Duration::from_secs(1)); + } + + /// A quiet track with no earlier group starts once its first group arrived a whole reach ago. + #[tokio::test] + async fn a_quiet_track_starts_within_one_reach() { + tokio::time::pause(); + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let mut consumer = reach_consumer( + &mut track, + Duration::from_millis(200), + Duration::from_millis(200), + Duration::from_millis(300), + ); + + write_group(&mut track, 5, &[ts(5_000_000)]); + assert!( + kio::clock::timeout(Duration::from_millis(450), consumer.read()) + .await + .is_err(), + "the wait lasts while an earlier group could still cover the reach" + ); + let frame = kio::clock::timeout(Duration::from_millis(100), consumer.read_frame()) + .await + .expect("the wait ends a reach after the first group arrived") + .unwrap() + .unwrap(); + assert_eq!(frame.timestamp, ts(5_000_000)); + } + + /// Media a reach past where the newest group starts ends the wait without the wall clock. + #[tokio::test] + async fn the_start_needs_no_wait_once_a_group_covers_the_reach() { + tokio::time::pause(); + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let mut consumer = reach_consumer( + &mut track, + Duration::from_millis(500), + Duration::from_millis(500), + Duration::ZERO, + ); + + write_group(&mut track, 5, &[ts(5_000_000), ts(5_500_000)]); + let frame = kio::clock::timeout(Duration::from_millis(50), consumer.read_frame()) + .await + .expect("the newest group covers the reach") + .unwrap() + .unwrap(); + assert_eq!(frame.timestamp, ts(5_000_000)); + } + + /// A full max buffer can hold none of what the start waits for, so the wait ends at once. + #[tokio::test] + async fn the_start_wait_ends_once_the_max_buffer_is_full() { + tokio::time::pause(); + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let mut consumer = reach_consumer( + &mut track, + Duration::from_millis(500), + Duration::from_millis(500), + Duration::ZERO, + ); + consumer.set_max_buffer(1); + + write_group(&mut track, 5, &[ts(5_000_000), ts(5_100_000)]); + let frame = kio::clock::timeout(Duration::from_millis(50), consumer.read_frame()) + .await + .expect("the max buffer cannot hold an earlier group") + .unwrap() + .unwrap(); + assert_eq!(frame.timestamp, ts(5_000_000)); + } + + /// A group below the subscriber's range never arrives, so the start does not wait for it. + #[tokio::test] + async fn the_start_stops_at_the_subscriber_range() { + tokio::time::pause(); + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_millis(500))) + .with_range(5..); + let mut consumer = + Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)).with_delay(Delay { + min: Duration::from_millis(500), + max: Some(Duration::from_millis(500)), + }); + assert!( + consumer.startup, + "a local range names no start, so the start still reaches back" + ); + + write_group(&mut track, 5, &[ts(5_000_000)]); + let frame = kio::clock::timeout(Duration::from_millis(50), consumer.read_frame()) + .await + .expect("nothing below the range can arrive") + .unwrap() + .unwrap(); + assert_eq!(frame.timestamp, ts(5_000_000)); + } + + /// A default min delay starts at a live edge that arrives first however large the max delay is: + /// the max delay bounds lateness, not the start. + #[tokio::test] + async fn a_large_max_delay_alone_starts_at_once() { + tokio::time::pause(); + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let mut consumer = reach_consumer(&mut track, Duration::ZERO, Duration::from_secs(10), Duration::ZERO); + + write_group(&mut track, 5, &[ts(5_000_000)]); + let frame = kio::clock::timeout(Duration::from_millis(50), consumer.read_frame()) + .await + .expect("nothing to wait for") + .unwrap() + .unwrap(); + assert_eq!(frame.timestamp, ts(5_000_000)); + } + + /// A min delay past the max delay reaches back only the max delay, since the publisher serves + /// nothing older: a quiet track starts once its first group arrived the max delay ago. + #[tokio::test] + async fn a_min_delay_past_the_max_delay_reaches_the_max_delay() { + tokio::time::pause(); + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let mut consumer = reach_consumer( + &mut track, + Duration::from_secs(2), + Duration::from_millis(200), + Duration::ZERO, + ); + + write_group(&mut track, 5, &[ts(5_000_000)]); + assert!( + kio::clock::timeout(Duration::from_millis(150), consumer.read()) + .await + .is_err(), + "the wait lasts the max delay" + ); + let frame = kio::clock::timeout(Duration::from_millis(100), consumer.read_frame()) + .await + .expect("the wait ends at the max delay, not the min delay") + .unwrap() + .unwrap(); + assert_eq!(frame.timestamp, ts(5_000_000)); + } + + /// The min delay stays on the consumer: the subscription carries the max delay plus the warmup. + #[tokio::test] + async fn the_min_delay_never_reaches_the_subscription() { + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let consumer = reach_consumer( + &mut track, + Duration::from_secs(2), + Duration::from_millis(200), + Duration::from_millis(300), + ); + let subscription = consumer.track.as_ref().unwrap().subscription(); + assert_eq!(subscription.max_delay, Some(Duration::from_millis(500))); + assert_eq!(consumer.max_delay, Duration::from_millis(200)); + } + + /// An unlimited max delay keeps the subscription unlimited, and the min delay still sets the + /// start's reach. + #[tokio::test] + async fn an_unlimited_max_delay_still_reaches_the_min_delay() { + tokio::time::pause(); + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let consumer_track = track.subscribe(moq_net::track::Subscription::default().with_max_delay(None)); + let mut consumer = + Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)).with_delay(Delay { + min: Duration::from_millis(500), + max: None, + }); + assert_eq!(consumer.track.as_ref().unwrap().subscription().max_delay, None); + + write_group(&mut track, 5, &[ts(5_000_000)]); + assert!( + kio::clock::timeout(Duration::from_millis(50), consumer.read()) + .await + .is_err(), + "an earlier group within the min delay must be waited for" + ); + write_group(&mut track, 4, &[ts(4_500_000)]); + track.finish().unwrap(); + let frames = read_all(&mut consumer).await.unwrap(); + let micros: Vec = frames.iter().map(|f| f.timestamp.as_micros()).collect(); + assert_eq!(micros, vec![4_500_000, 5_000_000]); + } + + /// A subscription without a max delay reaches back for history, so it starts at once. + #[tokio::test] + async fn an_unbounded_subscription_starts_at_once() { + tokio::time::pause(); + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let consumer_track = track.subscribe(moq_net::track::Subscription::default()); + let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); + + write_group(&mut track, 5, &[ts(5_000_000)]); + let frame = kio::clock::timeout(Duration::from_millis(50), consumer.read_frame()) + .await + .expect("an unbounded reach waits for nothing") + .unwrap() + .unwrap(); + assert_eq!(frame.timestamp, ts(5_000_000)); + } + // ---- Decode errors ---- /// A container that decodes each frame's payload as an 8-byte LE microsecond @@ -3376,7 +3954,7 @@ mod tests { } #[tokio::test] - async fn set_max_delay_changes_behavior() { + async fn set_delay_changes_behavior() { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); let consumer_track = track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))); @@ -3388,7 +3966,10 @@ mod tests { let frame = consumer.read_frame().await.unwrap().unwrap(); assert_eq!(frame.timestamp, ts(0)); - consumer.set_max_delay(Duration::from_millis(100)); + consumer.set_delay(Delay { + max: Some(Duration::from_millis(100)), + ..Delay::default() + }); assert!(consumer.read().await.unwrap().is_none()); } diff --git a/rs/moq-mux/src/container/mod.rs b/rs/moq-mux/src/container/mod.rs index 2c11f751514..0ec8e6a96e4 100644 --- a/rs/moq-mux/src/container/mod.rs +++ b/rs/moq-mux/src/container/mod.rs @@ -30,7 +30,7 @@ pub mod loc; pub mod mkv; pub mod ts; -pub use consumer::{Consumer, DEFAULT_MAX_BUFFER}; +pub use consumer::{Consumer, DEFAULT_MAX_BUFFER, Delay}; pub use group::GroupConsumer; pub use producer::Producer; pub(crate) use source::{ExportSource, Resume}; diff --git a/rs/moq-video/MIGRATE.md b/rs/moq-video/MIGRATE.md index 694d0e87a1c..409d76f4636 100644 --- a/rs/moq-video/MIGRATE.md +++ b/rs/moq-video/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `decode::Options::max_delay` is `delay` ([#7311](https://github.com/moq-dev/moq/pull/7311)) + +`decode::Options::max_delay: Duration` is `delay: decode::Delay` (`moq_mux::container::Delay`). +Replace `max_delay: d` with `delay: Delay { min: playout, max: Some(d) }`: `min` is your playout delay, where a cold join starts, and `max` (`None` for no limit) is the old max delay. + ### A capture reopened onto a new identity publishes a new track ([#7110](https://github.com/moq-dev/moq/pull/7110)) When a reopened camera or encoder emits a different codec string than the probed rendition, such as a higher level for a larger mode, `encode::Producer` publishes under the next free `/` and finishes the old track. diff --git a/rs/moq-video/README.md b/rs/moq-video/README.md index 3f16c3b9111..c38a4f02fb7 100644 --- a/rs/moq-video/README.md +++ b/rs/moq-video/README.md @@ -235,7 +235,9 @@ backend, or Linux without DMA-BUF import). `capture::Config::output` makes the s for captured frames. `decode::Config::scale_hint` asks a decoder with a hardware scaler (NVDEC) to emit that size; it is a hint, so check `Frame::size` and use `Frame::resize` for the exact size. `decode::Consumer` takes `decode::Options`, -which pairs that config with the subscription's `start` and `max_delay`. +which pairs that config with the subscription's `start` and `delay`: a +`decode::Delay` whose `min` is your playout delay, where a cold join starts, and +whose `max` is how far behind the live edge a stalled group may fall. Common feature sets: diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index b3b2ff6fddd..32deba77a08 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -13,6 +13,7 @@ use super::select::{Lane, Next, Selector}; use super::sink::Sink; use crate::Error; use crate::Frame; +pub use moq_mux::container::Delay; /// One item of a [`Consumer`] read, in delivery order: a decoded picture, or a break /// between them. @@ -88,13 +89,15 @@ pub struct Options { pub decoder: Config, /// Where to start on a track that already holds groups. pub start: Start, - /// How far playback may drift from the live edge before a stalled group is - /// skipped. Defaults to [`std::time::Duration::ZERO`](std::time::Duration::ZERO) - /// (skip aggressively); set it to your playout buffer for a softer skip. - /// Applied to the transport subscription and inherited by + /// How far behind the live edge playback runs. [`Delay::min`] is the + /// player's playout delay, where a cold join starts; [`Delay::max`] is how + /// far playback may drift before a stalled group is skipped. Both default to + /// zero: start at the newest group and skip aggressively. Set `min` to your + /// playout delay and `max` at or past it for a softer skip. The max is + /// applied to the transport subscription, and both reach /// [`moq_mux::container::Consumer`]. A rendition with a catalog `warmup` - /// runs a second decoder at this plus the warmup (see [`Consumer`]). - pub max_delay: std::time::Duration, + /// runs a second decoder at both plus the warmup (see [`Consumer`]). + pub delay: Delay, } impl Options { @@ -112,9 +115,9 @@ impl Options { /// `moq_audio::decode::Consumer`. /// /// A rendition with a catalog `warmup` (intra refresh, so no keyframes) runs -/// two decoders. One skips at [`Options::max_delay`] and may show refresh -/// stripes after a join or a skip. The other tolerates `warmup` more and joins -/// a group earlier, so its pictures stay clean, and they replace the first +/// two decoders. One skips at [`Options::delay`]'s max and may show refresh +/// stripes after a join or a skip. The other starts and tolerates `warmup` more +/// and joins a group earlier, so its pictures stay clean, and they replace the first /// decoder's once it catches up. Both subscriptions share one upstream request. pub struct Consumer { /// The min-delay and max-delay decoders, indexed by [`Lane`]. @@ -168,16 +171,20 @@ impl Consumer { options: Options, ) -> Result { let name = name.into(); - let min = Reader::open(broadcast, catalog, &name, &options, options.max_delay, 0).await?; + let min = Reader::open(broadcast, catalog, &name, &options, options.delay, 0).await?; let warmup = catalog.warmup.unwrap_or_default(); let max = if warmup.is_zero() { None } else { - // One more sweep of budget, joining a group earlier, so this decoder - // starts at a recovery point a whole sweep before the live edge. - let budget = options.max_delay.saturating_add(warmup); - Some(Reader::open(broadcast, catalog, &name, &options, budget, 1).await?) + // One more sweep both to start and to fall behind, joining a group earlier, + // so this decoder starts at a recovery point a whole sweep before the + // min-delay one. + let delay = Delay { + min: options.delay.min.saturating_add(warmup), + max: options.delay.max.map(|max| max.saturating_add(warmup)), + }; + Some(Reader::open(broadcast, catalog, &name, &options, delay, 1).await?) }; // The catalog's rate when it has one, else whatever the pictures show. @@ -295,14 +302,14 @@ impl Consumer { } impl Reader { - /// Subscribe with `max_delay`, starting `lookback` groups before the live + /// Subscribe with `delay`'s max, starting `lookback` groups before the live /// edge when the caller asked for [`Start::Latest`]. async fn open( broadcast: &moq_net::broadcast::Consumer, catalog: &VideoConfig, name: &str, options: &Options, - max_delay: std::time::Duration, + delay: Delay, lookback: u64, ) -> Result { let decoder = Sink::open(catalog, &options.decoder).await?; @@ -312,7 +319,7 @@ impl Reader { .subscribe( moq_net::track::Subscription::default() .with_priority(hang::catalog::PRIORITY.video) - .with_max_delay(max_delay), + .with_max_delay(delay.max), ) .await?; // A decoder often opens on a track that is already cached: a replacement @@ -343,7 +350,11 @@ impl Reader { // wire: `moq import fmp4` publishes CMAF. Reading a moof+mdat fragment as a // varint timestamp plus a payload decodes to garbage rather than failing. let container = moq_mux::catalog::hang::Container::try_from(catalog)?; - let track = moq_mux::container::Consumer::new(track, container); + // On a cold network join the live edge's stream often lands first. The container starts + // at the group covering the live edge less the min delay, waiting for it, so the + // max-delay decoder, whose min delay includes the warmup, still gets the group its warmup + // needs. That decoder tolerates the warmup as lateness too, so it passes no further warmup. + let track = moq_mux::container::Consumer::new(track, container).with_delay(delay); Ok(Self { decoder, @@ -574,7 +585,10 @@ mod tests { // Wide enough to keep every group fresh: the age budget on its own // delivers only the live edge, which would pass this test without // `Start::Latest` doing anything. - max_delay: std::time::Duration::from_secs(10), + delay: Delay { + max: Some(std::time::Duration::from_secs(10)), + ..Delay::default() + }, ..Options::new() }, ) @@ -655,7 +669,10 @@ mod tests { }, // A budget that keeps every group fresh, so the start policy is the // only thing deciding what is read. - max_delay: std::time::Duration::from_secs(10), + delay: Delay { + max: Some(std::time::Duration::from_secs(10)), + ..Delay::default() + }, ..Options::new() }, ) @@ -724,7 +741,10 @@ mod tests { "video", Options { decoder: decoder.clone(), - max_delay, + delay: Delay { + max: Some(max_delay), + ..Delay::default() + }, ..Options::new() }, )) @@ -772,7 +792,10 @@ mod tests { kind: Kind::Named(probe::BUFFERED_NAME.into()), ..Config::new() }, - max_delay: std::time::Duration::from_secs(10), + delay: Delay { + max: Some(std::time::Duration::from_secs(10)), + ..Delay::default() + }, ..Options::new() }, ) @@ -805,10 +828,7 @@ mod tests { [1000, 1100, 2000, 1800, 1900], "join trims, continuous delivery keeps every leading picture" ); - consumer - .reader(Lane::Min) - .track - .set_max_delay(std::time::Duration::ZERO); + consumer.reader(Lane::Min).track.set_delay(Delay::default()); write(12, &[3000]); write(13, &[4000, 3800, 3900, 4100]); track.finish().unwrap(); @@ -919,7 +939,10 @@ mod tests { kind: Kind::Named(probe::BUFFERED_NAME.into()), ..Config::new() }, - max_delay: std::time::Duration::from_secs(10), + delay: Delay { + max: Some(std::time::Duration::from_secs(10)), + ..Delay::default() + }, ..Options::new() }, ) @@ -1143,7 +1166,10 @@ mod tests { ..Config::new() }, start, - max_delay, + delay: Delay { + max: Some(max_delay), + ..Delay::default() + }, }, ) .await @@ -1211,6 +1237,50 @@ mod tests { } } + /// On a cold network join the live edge's stream often lands first. The + /// max-delay decoder still starts at the group before it, and takes over. + #[tokio::test] + async fn a_cold_join_whose_live_edge_arrives_first_starts_a_group_earlier() { + let (broadcast, mut track) = live_track(); + let mut live = group(&mut track, 1, &[4, 5]); + + let mut consumer = dual( + &broadcast, + Start::Latest, + std::time::Duration::ZERO, + probe::IMMEDIATE_NAME, + ) + .await; + + // Only the live edge has landed: the max-delay decoder waits for the group below it. + // Mocked time, so the timeout and the warmup's deadline never race; no decode runs + // while it waits, and the probe's thread would let mocked time skip ahead if one did. + let max = consumer.reader(Lane::Max); + tokio::time::pause(); + assert!( + kio::clock::timeout(std::time::Duration::from_millis(50), max.read()) + .await + .is_err(), + "the max-delay decoder started at the live edge" + ); + tokio::time::resume(); + group(&mut track, 0, &[0, 1, 2, 3]).finish().unwrap(); + let Ok(Some(Read::Frame(first))) = max.read().await else { + panic!("the max-delay decoder produced no picture"); + }; + assert_eq!( + first.timestamp, + Timestamp::from_micros(0).unwrap(), + "it starts a group earlier" + ); + + assert_eq!(read(&mut consumer).await, (4, Lane::Min), "the live edge shows first"); + assert_eq!(read(&mut consumer).await.0, 5); + catch_up(&mut consumer).await; + write(&mut live, &[6]); + assert_eq!(read(&mut consumer).await, (6, Lane::Max)); + } + /// A stall shorter than a sweep makes the min-delay decoder skip, while the /// max-delay one waits it out, and takes the picture back once it catches up, /// never alternating in between. @@ -1346,7 +1416,10 @@ mod tests { kind: Kind::Named(probe::IMMEDIATE_NAME.into()), ..Config::new() }, - max_delay, + delay: Delay { + max: Some(max_delay), + ..Delay::default() + }, ..Options::new() }, ) diff --git a/rs/moq-video/src/decode/mod.rs b/rs/moq-video/src/decode/mod.rs index 6b2e74d3939..410c66f9ba2 100644 --- a/rs/moq-video/src/decode/mod.rs +++ b/rs/moq-video/src/decode/mod.rs @@ -29,7 +29,7 @@ mod sink; mod vui; pub use backend::{Codec, NAMES}; -pub use consumer::{Consumer, Options, Read, Start}; +pub use consumer::{Consumer, Delay, Options, Read, Start}; pub use decoder::{Config, Decoder, Kind}; pub use select::{Lane, Next, Selector}; pub use sink::Sink; diff --git a/swift/MIGRATE.md b/swift/MIGRATE.md index 9f9fb9be298..39478780f17 100644 --- a/swift/MIGRATE.md +++ b/swift/MIGRATE.md @@ -4,6 +4,13 @@ How to update your code for each breaking change, newest first. ## Unreleased +### Decoder outputs take a `Media.Delay` ([#7311](https://github.com/moq-dev/moq/pull/7311)) + +`Audio.DecoderOutput` replaces `maxDelay` and the playout `delay: TimeInterval?` with `delay: Media.Delay?` and `estimate: Bool = true`. +`Video.DecoderOutput.maxDelay` is `delay: Media.Delay?`. +`Media.Delay(min:max:)`: `max` (nil for no limit) is the old `maxDelay`, and `min` is the playout delay, where a cold join starts. +A fixed audio playout delay `d` is `delay: Media.Delay(min: d, max: ...)` with `estimate: false`. + ### Compression moves to the track `Flate` is removed: publish opaque bytes on a `TrackProducer`. diff --git a/swift/Sources/Moq/Aliases.swift b/swift/Sources/Moq/Aliases.swift index 79cc6830900..64608e2b215 100644 --- a/swift/Sources/Moq/Aliases.swift +++ b/swift/Sources/Moq/Aliases.swift @@ -18,13 +18,14 @@ public typealias AudioEncoderInput = MoqFFI.MoqAudioEncoderInput /// The codec-side audio encoder settings: codec, rate, channels, /// bitrate, and frame duration. public typealias AudioEncoderOutput = MoqFFI.MoqAudioEncoderOutput -/// What a `VideoConsumer` decodes to: an optional resize, a max delay, and +/// What a `VideoConsumer` decodes to: an optional resize, a delay, and /// whether frames keep the decoder's native surface. public typealias VideoDecoderOutput = MoqVideoDecoderOutput /// One decoded video frame, owning the decoder's surface until released; /// `pixels(format:)` converts it to packed CPU pixels. public typealias VideoDecodedFrame = MoqVideoDecodedFrame -/// The decoded PCM layout, maximum group staleness, and playout delay. +/// The decoded PCM layout, its delay (playout and group staleness), and whether the +/// playout delay is estimated. public typealias AudioDecoderOutput = MoqFFI.MoqAudioDecoderOutput /// A raw PCM sample format, mirroring WebCodecs `AudioData.format`. public typealias AudioSampleFormat = MoqFFI.MoqAudioSampleFormat diff --git a/swift/Sources/Moq/Audio.swift b/swift/Sources/Moq/Audio.swift index 56abcdbbac7..73720184007 100644 --- a/swift/Sources/Moq/Audio.swift +++ b/swift/Sources/Moq/Audio.swift @@ -101,8 +101,8 @@ public enum Audio { try await ffi.next() } - /// The playout delay to hold ahead of the playhead: the fixed `DecoderOutput.delay`, - /// or the estimate from every frame read so far. + /// The playout delay to hold ahead of the playhead: the fixed `DecoderOutput.delay.min`, + /// or with `DecoderOutput.estimate` the estimate from every frame read so far. public var delay: TimeInterval { ffi.delay() } /// Cancel all current and future reads. diff --git a/swift/Sources/Moq/Media.swift b/swift/Sources/Moq/Media.swift index 54c9dc4e32b..c122400bc6e 100644 --- a/swift/Sources/Moq/Media.swift +++ b/swift/Sources/Moq/Media.swift @@ -29,6 +29,10 @@ public enum Media { public typealias Audio = MoqFFI.MoqAudio /// A width and height in pixels. public typealias Dimensions = MoqFFI.MoqDimensions + /// How far behind the live edge a decoder plays: `min` is the playout delay and where a + /// cold join starts, and `max`, nil for no limit, is how far a stalled group may fall + /// behind before it is skipped. `min` stays on the consumer; a `min` past `max` starts at `max`. + public typealias Delay = MoqFFI.MoqDelay /// Where an audio or video decoder starts on a track that already holds /// groups: `.latest` for a live player. public typealias DecodeStart = MoqFFI.MoqDecodeStart diff --git a/swift/Sources/Moq/Video.swift b/swift/Sources/Moq/Video.swift index 5c3140602ea..8ba81a30fec 100644 --- a/swift/Sources/Moq/Video.swift +++ b/swift/Sources/Moq/Video.swift @@ -9,7 +9,7 @@ public enum Video { public typealias EncoderInput = VideoEncoderInput /// The codec, bitrate, group mode, and backend preference. public typealias EncoderOutput = VideoEncoderOutput - /// What a `Consumer` delivers: an optional resize, a max delay, and the decoder's surface (macOS only). + /// What a `Consumer` delivers: an optional resize, a delay, and the decoder's surface (macOS only). public typealias DecoderOutput = VideoDecoderOutput /// One raw picture with its presentation timestamp. public typealias Frame = VideoFrame diff --git a/swift/Tests/MoqTests/SmokeTests.swift b/swift/Tests/MoqTests/SmokeTests.swift index 5a9c90c8117..20b163828b8 100644 --- a/swift/Tests/MoqTests/SmokeTests.swift +++ b/swift/Tests/MoqTests/SmokeTests.swift @@ -617,7 +617,7 @@ final class SmokeTests: XCTestCase { let fixed = try await Audio.Consumer.subscribe( broadcast: consumer, id: "mic", rendition: rendition, - output: AudioDecoderOutput(format: .f32, delay: 0.08) + output: AudioDecoderOutput(format: .f32, delay: Media.Delay(min: 0.08, max: 0), estimate: false) ) let estimated = try await Audio.Consumer.subscribe( broadcast: consumer, id: "mic", rendition: rendition, diff --git a/test/interop/clients/c/subscribe.c b/test/interop/clients/c/subscribe.c index f0c6ca8d617..cabba3f28d4 100644 --- a/test/interop/clients/c/subscribe.c +++ b/test/interop/clients/c/subscribe.c @@ -245,7 +245,7 @@ static void start_audio(ctx_t *c, int32_t catalog) { moq_audio_decoder_output out; memset(&out, 0, sizeof(out)); out.format = MOQ_AUDIO_SAMPLE_FORMAT_F32; - out.max_delay_us = MAX_DELAY_US; + out.delay.max_us = MAX_DELAY_US; int32_t track = moq_decode_audio((uint32_t)catalog, i, &out, on_audio, &c->audio[i]); pthread_mutex_lock(&c->mu); if (track > 0) c->audio[i].track = track; @@ -324,7 +324,8 @@ static void on_catalog(void *ud, int32_t catalog) { moq_video_config vcfg; memset(&vcfg, 0, sizeof(vcfg)); if (moq_consume_video_config((uint32_t)catalog, 0, &vcfg) == 0) { - int32_t track = moq_consume_video((uint32_t)catalog, 0, MAX_DELAY_US, on_frame, ud); + moq_delay delay = {.max_us = MAX_DELAY_US}; + int32_t track = moq_consume_video((uint32_t)catalog, 0, &delay, on_frame, ud); if (track > 0) { pthread_mutex_lock(&c->mu); c->video_track = track; diff --git a/test/interop/clients/cpp/main.cpp b/test/interop/clients/cpp/main.cpp index c91a89abc6e..4771c32061d 100644 --- a/test/interop/clients/cpp/main.cpp +++ b/test/interop/clients/cpp/main.cpp @@ -295,7 +295,7 @@ void decode_audio( } moq::AudioDecoderOutput output{}; output.format = moq::AudioSampleFormat::kF32; - output.max_delay = MAX_DELAY; + output.delay = moq::Delay{std::chrono::nanoseconds(0), MAX_DELAY}; auto decoder = until(moq::AudioConsumer::subscribe(consumer, id, audio, output), deadline, "audio consumer"); std::ofstream log(out + ".txt"); diff --git a/test/interop/clients/go/main.go b/test/interop/clients/go/main.go index 27bd64942a9..8082754527c 100644 --- a/test/interop/clients/go/main.go +++ b/test/interop/clients/go/main.go @@ -465,8 +465,8 @@ func decodeAudio(ctx context.Context, consumer *moq.BroadcastConsumer, id string codec, _, _ := strings.Cut(rendition.Codec, ".") out += "." + codec decoder, err := moqaudio.Subscribe(ctx, consumer, id, rendition, moqaudio.DecoderOutput{ - Format: moq.AudioSampleFormatF32, - MaxDelay: durationPtr(maxDelay), + Format: moq.AudioSampleFormatF32, + Delay: &moqmedia.Delay{Max: durationPtr(maxDelay)}, }) if err != nil { return err diff --git a/test/interop/clients/js-native/finish.ts b/test/interop/clients/js-native/finish.ts index 9449a496070..40804bf29e4 100644 --- a/test/interop/clients/js-native/finish.ts +++ b/test/interop/clients/js-native/finish.ts @@ -38,7 +38,7 @@ export async function readFinish( const subscription = broadcast .track(Catalog.track(name, config)) .subscribe({ maxDelay, groups: { start: { included: 0 } } }); - const media = new Container.Consumer(subscription, { format, maxDelay }); + const media = new Container.Consumer(subscription, { format, delay: { max: maxDelay } }); let count = 0; try { for (;;) { diff --git a/test/interop/clients/js-native/subscribe.ts b/test/interop/clients/js-native/subscribe.ts index 2247ce38c04..41d97c793d8 100644 --- a/test/interop/clients/js-native/subscribe.ts +++ b/test/interop/clients/js-native/subscribe.ts @@ -119,7 +119,10 @@ function containerFormat(config: Catalog.VideoConfig | Catalog.AudioConfig, kind async function video(bc: Moq.Broadcast.Consumer, name: string, config: Catalog.VideoConfig): Promise { const sub = bc.track(name).subscribe({ priority: 0, maxDelay: MAX_DELAY }); - const consumer = new Container.Consumer(sub, { format: containerFormat(config, "video"), maxDelay: MAX_DELAY }); + const consumer = new Container.Consumer(sub, { + format: containerFormat(config, "video"), + delay: { max: MAX_DELAY }, + }); try { for (;;) { const next = await consumer.next(); @@ -185,7 +188,7 @@ async function audio(bc: Moq.Broadcast.Consumer, name: string, config: Catalog.A const sub = bc.track(name).subscribe({ priority: Catalog.PRIORITY.audio, maxDelay: AUDIO_MAX_DELAY }); const consumer = new Container.Consumer(sub, { format: containerFormat(config, "audio"), - maxDelay: AUDIO_MAX_DELAY, + delay: { max: AUDIO_MAX_DELAY }, }); const feed = (async () => { // The consumer also hands back the backlog the relay served behind the live edge, older diff --git a/test/interop/clients/python/interop.py b/test/interop/clients/python/interop.py index 2ba8042d34b..6d519f6db12 100644 --- a/test/interop/clients/python/interop.py +++ b/test/interop/clients/python/interop.py @@ -256,7 +256,7 @@ async def _decode_audio( """ out = f"{out}.{audio.codec.split('.')[0]}" decoder = await moq.audio.Consumer.subscribe( - consumer, id, audio, moq.audio.DecoderOutput(format=moq.audio.SampleFormat.F32, max_delay=MAX_DELAY) + consumer, id, audio, moq.audio.DecoderOutput(format=moq.audio.SampleFormat.F32, delay=moq.media.Delay(min=timedelta(0), max=MAX_DELAY)) ) frame_bytes = 4 * audio.channel_count decoded = 0 diff --git a/test/interop/clients/rust/audio.rs b/test/interop/clients/rust/audio.rs index 8c5d5b0ba02..cff30cd8c1f 100644 --- a/test/interop/clients/rust/audio.rs +++ b/test/interop/clients/rust/audio.rs @@ -94,7 +94,7 @@ async fn decode( // runtime starved by the others can fall a second behind. That is this reader's speed, not the // delivery under test. A track's max-age caps it, so interop.sh's Rust publisher keeps its // media at least this long. - options.max_delay = Duration::from_secs(5); + options.delay.max = Some(Duration::from_secs(5)); let mut decoder = moq_audio::decode::Consumer::new(&rendition, audio, name, options).await?; let rate = decoder.sample_rate(); let channels = decoder.layout().channels();