diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index 9f83a722186..671fdfcbd49 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -432,6 +432,7 @@ class MoqAudio { final int? bitrate; final bool enabled; final MoqContainer container; + final int? jitterUs; MoqAudio({ this.label = null, this.broadcast = null, @@ -443,6 +444,7 @@ class MoqAudio { this.bitrate, this.enabled = true, required this.container, + this.jitterUs = null, }); } @@ -503,6 +505,11 @@ class FfiConverterMoqAudio { ); final container = container_lifted.value; new_offset += container_lifted.bytesRead; + final jitterUs_lifted = FfiConverterOptionalUInt64.read( + Uint8List.view(buf.buffer, new_offset), + ); + final jitterUs = jitterUs_lifted.value; + new_offset += jitterUs_lifted.bytesRead; return LiftRetVal( MoqAudio( label: label, @@ -515,6 +522,7 @@ class FfiConverterMoqAudio { bitrate: bitrate, enabled: enabled, container: container, + jitterUs: jitterUs, ), new_offset - buf.offsetInBytes, ); @@ -532,6 +540,7 @@ class FfiConverterMoqAudio { FfiConverterOptionalUInt64.allocationSize(value.bitrate) + FfiConverterBool.allocationSize(value.enabled) + FfiConverterMoqContainer.allocationSize(value.container) + + FfiConverterOptionalUInt64.allocationSize(value.jitterUs) + 0; final buf = Uint8List(total_length); write(value, buf); @@ -580,6 +589,10 @@ class FfiConverterMoqAudio { value.container, Uint8List.view(buf.buffer, new_offset), ); + new_offset += FfiConverterOptionalUInt64.write( + value.jitterUs, + Uint8List.view(buf.buffer, new_offset), + ); return new_offset - buf.offsetInBytes; } @@ -594,6 +607,7 @@ class FfiConverterMoqAudio { FfiConverterOptionalUInt64.allocationSize(value.bitrate) + FfiConverterBool.allocationSize(value.enabled) + FfiConverterMoqContainer.allocationSize(value.container) + + FfiConverterOptionalUInt64.allocationSize(value.jitterUs) + 0; } } diff --git a/doc/lib/index.md b/doc/lib/index.md index 425606d0699..da0c27fbc50 100644 --- a/doc/lib/index.md +++ b/doc/lib/index.md @@ -38,7 +38,7 @@ how it looks in that language: - **Connect** to a relay with TLS options and a JWT in the URL, or **serve** sessions yourself and accept or reject each request by path. - **Reconnect** automatically with backoff when the transport drops. - **Discover** broadcasts by prefix, wait for a specific one, or request one by path. Advertise an exact path (a created broadcast is invisible until announced), or claim a prefix and serve requests beneath it on demand. -- **Publish and subscribe to media** with the hang catalog filled in from the bitstream, or hand over raw pixels and PCM and let the binding encode and decode (VideoToolbox, Media Foundation, NVENC, openh264, Opus). The built-in video encoder can follow the connection's send estimate. +- **Publish and subscribe to media** with the hang catalog filled in from the bitstream, or hand over raw pixels and PCM and let the binding encode and decode (VideoToolbox, Media Foundation, NVENC, openh264, Opus). The built-in video encoder can follow the connection's send estimate. The audio decoder reports the playout delay to buffer ahead of the playhead, fixed or estimated from arrivals, so read each frame as soon as the last is handled rather than pacing reads to the speaker. - **Raw tracks** of arbitrary bytes, with per-subscriber priority and max delay, and best-effort datagrams. - **JSON tracks** as a latest value with merge-patch deltas, or as an append log. - **Fetch** a single cached group by sequence. @@ -47,4 +47,5 @@ how it looks in that language: - **Connection stats** and **errors** that tell an auth rejection (don't retry) from a shutdown or a transport failure. Dart is the exception on codecs: its published binaries carry no encoder or -decoder, so it moves already-encoded frames. +decoder, so it moves already-encoded frames. C's audio decoder neither takes +nor reports a playout delay. diff --git a/go/wrapper/audio/audio.go b/go/wrapper/audio/audio.go index 4882c81841a..2966e3ed39c 100644 --- a/go/wrapper/audio/audio.go +++ b/go/wrapper/audio/audio.go @@ -61,17 +61,24 @@ func (o EncoderOutput) ffi() (ffi.MoqAudioEncoderOutput, error) { return ffi.MoqAudioEncoderOutput{Codec: o.Codec, Track: o.Track, SampleRate: o.SampleRate, Channels: o.Channels, Bitrate: o.Bitrate, FrameDurationUs: uint32(frame), Preset: o.Preset}, err } -// DecoderOutput configures the PCM a Consumer delivers and how stale a group may get before it is skipped. +// DecoderOutput configures the PCM a Consumer delivers, how stale a group may get before it is skipped, and the playout delay. type DecoderOutput struct { Format SampleFormat SampleRate *uint32 Channels *uint32 - MaxDelay *time.Duration + // MaxDelay bounds how stale a group may get before it is skipped; it adds no buffering. + MaxDelay *time.Duration + // Delay fixes the jitter buffer to hold ahead of the playhead; nil estimates it from arrival timing. + Delay *time.Duration } func (o DecoderOutput) ffi() (ffi.MoqAudioDecoderOutput, error) { - delay, err := bridge.OptionalDuration(o.MaxDelay, "audio max delay") - return ffi.MoqAudioDecoderOutput{Format: o.Format, SampleRate: o.SampleRate, Channels: o.Channels, MaxDelayUs: delay}, err + maxDelay, err := bridge.OptionalDuration(o.MaxDelay, "audio max delay") + if err != nil { + return ffi.MoqAudioDecoderOutput{}, err + } + delay, err := bridge.OptionalDuration(o.Delay, "audio delay") + return ffi.MoqAudioDecoderOutput{Format: o.Format, SampleRate: o.SampleRate, Channels: o.Channels, MaxDelayUs: maxDelay, DelayUs: delay}, err } // Frame is interleaved PCM with its presentation timestamp. @@ -203,5 +210,11 @@ func (d *Consumer) Next(ctx context.Context) (*Frame, error) { // All ranges over decoded frames until the track ends or the loop breaks. func (d *Consumer) All(ctx context.Context) iter.Seq2[*Frame, error] { return bridge.Seq(ctx, d.Next) } +// Delay is the playout delay to hold ahead of the playhead: the fixed one, or the estimate from every frame read so far. +// The estimate measures when Next returns, so call it again as soon as the previous frame is handled. +func (d *Consumer) Delay() time.Duration { + return time.Duration(d.inner.DelayUs()) * time.Microsecond +} + // Cancel stops the stream. func (d *Consumer) Cancel() { d.inner.Cancel() } diff --git a/go/wrapper/moq_test.go b/go/wrapper/moq_test.go index 4c0c69501b0..db2d9c4db7e 100644 --- a/go/wrapper/moq_test.go +++ b/go/wrapper/moq_test.go @@ -213,6 +213,86 @@ func TestEncodeAudioWithOpusObject(t *testing.T) { } } +// A fixed playout delay is reported back as is, and an estimated one moves as frames are read. +func TestAudioConsumerDelay(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + + origin := newOrigin(t) + broadcast, err := origin.CreateBroadcast("audio-delay") + if err != nil { + t.Fatal(err) + } + track := "mic" + catalogProducer, err := moqmedia.NewCatalogProducer(broadcast) + if err != nil { + t.Fatal(err) + } + audio, err := moqaudio.NewProducer(catalogProducer, + moqaudio.EncoderInput{Format: moq.AudioSampleFormatF32, SampleRate: 48000, Channels: 1}, + moqaudio.EncoderOutput{Codec: moq.OpusAudioCodec(), Track: &track}, + nil, + ) + if err != nil { + t.Fatal(err) + } + if err := broadcast.Announce(moq.Route{}); err != nil { + t.Fatal(err) + } + + bc, err := origin.Consume().RequestBroadcast(ctx, "audio-delay") + if err != nil { + t.Fatal(err) + } + catalog, err := moqmedia.CatalogSnapshot(ctx, bc) + if err != nil { + t.Fatal(err) + } + rendition, ok := catalog.Audio[track] + if !ok { + t.Fatalf("catalog has no %q rendition: %v", track, catalog.Audio) + } + + fixedDelay := 80 * time.Millisecond + fixed, err := moqaudio.Subscribe(ctx, bc, track, rendition, moqaudio.DecoderOutput{Format: moq.AudioSampleFormatF32, Delay: &fixedDelay}) + if err != nil { + t.Fatal(err) + } + defer fixed.Cancel() + estimated, err := moqaudio.Subscribe(ctx, bc, track, rendition, moqaudio.DecoderOutput{Format: moq.AudioSampleFormatF32}) + if err != nil { + t.Fatal(err) + } + defer estimated.Cancel() + if got := fixed.Delay(); got != fixedDelay { + t.Fatalf("fixed delay = %v, want %v", got, fixedDelay) + } + initial := estimated.Delay() + + // Two 20 ms Opus frames of silence at 48 kHz mono. + for i := range 2 { + if err := audio.Write(moqaudio.Frame{Timestamp: time.Duration(i) * 20 * time.Millisecond, Data: make([]byte, 960*4)}); err != nil { + t.Fatal(err) + } + } + for _, consumer := range []*moqaudio.Consumer{fixed, estimated} { + if frame, err := consumer.Next(ctx); err != nil || frame == nil { + t.Fatalf("next = %v, %v", frame, err) + } + } + + if got := fixed.Delay(); got != fixedDelay { + t.Fatalf("fixed delay after a read = %v, want %v", got, fixedDelay) + } + if got := estimated.Delay(); got == initial { + t.Fatalf("estimated delay stayed at %v after a read", got) + } + + if err := audio.Finish(); err != nil { + t.Fatal(err) + } +} + // FrameDurationUs is microseconds so Opus' 2.5 ms frame is expressible at all, // and a duration outside the Opus set is refused rather than silently rounded. func TestEncodeAudioFrameDurations(t *testing.T) { diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Durations.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Durations.kt index 9618ba94aef..33c80ede56c 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Durations.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Durations.kt @@ -41,6 +41,14 @@ val TrackInfo.maxAge: Duration? val AudioDecoderOutput.maxDelay: Duration? get() = maxDelayUs?.toLong()?.microseconds +/** Fixed playout delay to hold ahead of the playhead, or null to estimate it from arrival timing. */ +val AudioDecoderOutput.delay: Duration? + get() = delayUs?.toLong()?.microseconds + +/** Playout delay to hold ahead of the playhead: the fixed one, or the estimate from every frame read so far. */ +val dev.moq.audio.Consumer.delay: Duration + get() = delayUs().toLong().microseconds + /** Upper bound on buffering before a stalled group is skipped, or null for the default. */ val VideoDecoderOutput.maxDelay: Duration? get() = maxDelayUs?.toLong()?.microseconds diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/audio/Audio.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/audio/Audio.kt index cd5f442d0df..ac9e6d37e16 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/audio/Audio.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/audio/Audio.kt @@ -10,7 +10,7 @@ typealias Consumer = uniffi.moq.MoqAudioConsumer typealias EncoderInput = uniffi.moq.MoqAudioEncoderInput /** The codec side: codec, track name, rate, channels, bitrate, frame duration (0 takes the preset's or codec's own frame), and preset. */ typealias EncoderOutput = uniffi.moq.MoqAudioEncoderOutput -/** The PCM layout a [Consumer] delivers and how stale a group may get. */ +/** The PCM layout a [Consumer] delivers, how stale a group may get, and its playout delay. */ typealias DecoderOutput = uniffi.moq.MoqAudioDecoderOutput /** Interleaved PCM with its presentation timestamp. */ typealias Frame = uniffi.moq.MoqAudioFrame diff --git a/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt b/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt index 1164de4a7e6..781e514d0f4 100644 --- a/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt +++ b/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt @@ -1,6 +1,7 @@ package dev.moq import dev.moq.media.* +import dev.moq.audio.Consumer as AudioConsumer import dev.moq.audio.Producer as AudioProducer import dev.moq.video.Consumer as VideoConsumer import dev.moq.video.Producer as VideoProducer @@ -585,6 +586,46 @@ class SmokeTest { } } + /** A fixed playout delay is reported back as is, and an estimated one moves as frames are read. */ + @Test + fun `audio consumer reports its playout delay`() = runTest { + OriginProducer(OriginConfig()).use { origin -> + origin.createBroadcast("audio-delay").use { broadcast -> + val audio = AudioProducer( + CatalogProducer(broadcast), + AudioEncoderInput(format = AudioSampleFormat.F32, sampleRate = 48_000u, channels = 1u), + AudioEncoderOutput(codec = AudioCodec.opus(), track = "mic"), + ) + broadcast.announce(Route()) + + val consumer = origin.consume().requestBroadcast("audio-delay") + val catalog = CatalogConsumer.subscribe(consumer).next()!! + val rendition = catalog.audio["mic"]!! + + val fixedOutput = AudioDecoderOutput(format = AudioSampleFormat.F32, delayUs = 80_000uL) + assertEquals(80.milliseconds, fixedOutput.delay) + val fixed = AudioConsumer.subscribe(consumer, "mic", rendition, fixedOutput) + val estimated = AudioConsumer.subscribe(consumer, "mic", rendition, AudioDecoderOutput(format = AudioSampleFormat.F32)) + assertEquals(80.milliseconds, fixed.delay) + val initial = estimated.delay + + // Two 20 ms Opus frames of silence at 48 kHz mono. + for (i in 0 until 2) { + audio.write(AudioFrame(timestampUs = i.toULong() * 20_000uL, data = ByteArray(960 * 4))) + } + assertNotNull(fixed.next()) + assertNotNull(estimated.next()) + + assertEquals(80.milliseconds, fixed.delay) + assertTrue(estimated.delay != initial, "the estimate should move as frames are read") + + fixed.cancel() + estimated.cancel() + audio.finish() + } + } + } + @Test fun `decoded video frame owns its picture`() = runTest { OriginProducer(OriginConfig()).use { origin -> diff --git a/py/moq-rs/moq/audio.py b/py/moq-rs/moq/audio.py index ff74bf4ed48..52ea28b4ed1 100644 --- a/py/moq-rs/moq/audio.py +++ b/py/moq-rs/moq/audio.py @@ -78,12 +78,18 @@ def _ffi(self) -> MoqAudioEncoderOutput: @dataclass(frozen=True) class DecoderOutput: - """The PCM layout a :class:`Consumer` delivers and how stale a group may get.""" + """The PCM layout a :class:`Consumer` delivers, how stale a group may get, and its playout delay. + + ``delay`` fixes the jitter buffer to hold ahead of the playhead; ``None`` + estimates it from arrival timing. ``max_delay`` only bounds how stale a + group may get before it is skipped, and adds no buffering. + """ format: SampleFormat sample_rate: int | None = None channels: int | None = None max_delay: timedelta | None = None + delay: timedelta | None = None def _ffi(self) -> MoqAudioDecoderOutput: return MoqAudioDecoderOutput( @@ -91,6 +97,7 @@ def _ffi(self) -> MoqAudioDecoderOutput: sample_rate=self.sample_rate, channels=self.channels, max_delay_us=_opt_us(self.max_delay, "max_delay"), + delay_us=_opt_us(self.delay, "delay"), ) @@ -171,6 +178,15 @@ async def __anext__(self) -> Frame: raise StopAsyncIteration return Frame._from_ffi(value) + @property + def delay(self) -> timedelta: + """The playout delay to hold ahead of the playhead: the fixed one, or the estimate so far. + + The estimate measures when each frame is read, so read the next frame as + soon as the previous one is handled rather than pacing reads to the speaker. + """ + return _from_us(self._inner.delay_us()) + def cancel(self) -> None: """Cancel the subscription and stop delivering audio frames.""" self._inner.cancel() diff --git a/py/moq-rs/tests/test_local.py b/py/moq-rs/tests/test_local.py index 5b0b832fb80..598d7f463c7 100644 --- a/py/moq-rs/tests/test_local.py +++ b/py/moq-rs/tests/test_local.py @@ -1275,6 +1275,7 @@ def test_optional_binding_records_use_none_defaults(): assert decoder.sample_rate is None assert decoder.channels is None assert decoder.max_delay is None + assert decoder.delay is None def test_audio_encoder_returns_namespaced_records(): @@ -1338,6 +1339,44 @@ def test_encode_audio_with_opus_object(): broadcast.close() +async def test_audio_consumer_reports_its_playout_delay(): + """A fixed playout delay is reported back as is; an estimated one moves as frames are read.""" + origin = moq.OriginProducer() + broadcast = create_announced(origin, "audio-delay") + audio = moq.audio.Producer( + moq.media.CatalogProducer(broadcast), + moq.AudioEncoderInput(format=moq.AudioSampleFormat.F32, sample_rate=48_000, channels=1), + moq.audio.EncoderOutput(codec=moq.AudioCodec.opus(), track="mic"), + ) + + consumer = origin.consume() + broadcast_consumer = await asyncio.wait_for(consumer.request_broadcast("audio-delay"), timeout=5.0) + catalog = await asyncio.wait_for(moq.media.catalog(broadcast_consumer), timeout=5.0) + rendition = catalog.audio["mic"] + + def output(delay: timedelta | None) -> moq.audio.DecoderOutput: + return moq.audio.DecoderOutput(format=moq.AudioSampleFormat.F32, delay=delay) + + fixed = await moq.audio.Consumer.subscribe(broadcast_consumer, "mic", rendition, output(timedelta(milliseconds=80))) + estimated = await moq.audio.Consumer.subscribe(broadcast_consumer, "mic", rendition, output(None)) + assert fixed.delay == timedelta(milliseconds=80) + initial = estimated.delay + + # Two 20 ms Opus frames of silence at 48 kHz mono. + for i in range(2): + audio.write(moq.audio.Frame(timestamp=timedelta(milliseconds=20 * i), data=bytes(960 * 4))) + await asyncio.wait_for(anext(fixed), timeout=5.0) + await asyncio.wait_for(anext(estimated), timeout=5.0) + + assert fixed.delay == timedelta(milliseconds=80) + assert estimated.delay != initial + + fixed.cancel() + estimated.cancel() + audio.finish() + broadcast.close() + + async def test_decode_video_frame(): """A decoded frame owns its picture and converts to either layout, even after its consumer is cancelled.""" origin = moq.OriginProducer() diff --git a/quest/m2/README.md b/quest/m2/README.md index 561fe933ca0..863051a6d3e 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -46,7 +46,6 @@ ranked just below deterministic TS legs, which it requires. - [Time test sweep](/quest/m2/time-test-sweep.md) - unrelated test conversion and the workspace lint ratchet -- [Binding audio delay](/quest/m2/binding-surface.md) - moq-ffi and every wrapper configure and observe audio playout delay - [Pipelined requests](/quest/m2/pipeline-requests/README.md) - SUBSCRIBE and the first FETCH go out with the track-info request at every hop, so first data arrives a round trip sooner per hop - [OBS native codecs](/quest/m2/obs-moq-video/README.md) - replace FFmpeg video and audio decoding with moq-video and moq-audio, deliver GPU frames, and use native audio/video encoders - [OBS multitrack](/quest/m2/obs-multitrack.md) - the obs-moq output publishes OBS's native multitrack video encoders as simulcast renditions, configured like OBS multitrack and over obs-websocket diff --git a/quest/m2/binding-surface.md b/quest/m2/binding-surface.md deleted file mode 100644 index 9df262aef2f..00000000000 --- a/quest/m2/binding-surface.md +++ /dev/null @@ -1,28 +0,0 @@ -# [M] Audio decode delay in every binding - -## Goal - -moq-ffi and every wrapper (Python, Go, Swift, Kotlin, Dart) expose -`decode::Options::delay` and `decode::Consumer::delay()` so applications can -configure and observe audio playout delay. - -## Plan - -- One PR, each wrapper touched once, in its own idiom: durations as the - language's duration type where the wrapper already uses one, handles over - flat methods where a surface has more than one call. -- moq-ffi only, not the hand-written moq-c: the [generated C](/quest/m1/c/README.md) and - C++ bindings inherit it from moq-ffi. -- Update `doc/lib/{py,swift,kt,go,dart}` in the same PR. -- Test configuration and observed delay in every wrapper that has tests. - -Decided in the 2026-10-06 audit: the native decode delay API is on `main` -(#4162, `rs/moq-audio/src/decode/consumer.rs`), so this no longer waits on -the jitter-target line, whose remaining Watch proof bindings do not use. -Each wrapper adds delay to the reshaped audio decoder rather than to the -removed `BroadcastConsumer.decode_audio`. - -## Related - -- [FFI shape](/quest/m1/ffi-shape/README.md) - reshapes the binding namespaces -- [Audio jitter target](/quest/m1/audio-jitter-target/README.md) - landed the native decode delay API this exposes diff --git a/quest/m2/dart-codecs.md b/quest/m2/dart-codecs.md index a6627360f7d..5ebb01ef675 100644 --- a/quest/m2/dart-codecs.md +++ b/quest/m2/dart-codecs.md @@ -28,8 +28,9 @@ against the Kotlin and Swift artifacts that already pay it, and update Deployment floors stay as they are (Android API 24, iOS 16); a codec addition that raises either is a separate decision. -This quest owns Dart integration of the already-landed `MoqVideoConsumer`: -expose it through the Dart wrapper, ship codec-enabled artifacts, and test +This quest owns Dart integration of the already-landed `MoqVideoConsumer` and +`MoqAudioConsumer` (including its playout `delay_us`): expose them through the +Dart wrapper, ship codec-enabled artifacts, and test subscribed video decoding from Dart in CI. Update the capability docs in the same change. Do not finish with codecs enabled but the consumer unexposed. diff --git a/rs/moq-ffi/CHANGELOG.md b/rs/moq-ffi/CHANGELOG.md index eaadcd5ad54..9fd0b08a70a 100644 --- a/rs/moq-ffi/CHANGELOG.md +++ b/rs/moq-ffi/CHANGELOG.md @@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `MoqAudioEncoderOutput::preset` takes a `MoqAudioPreset` (low latency, balanced, quality). - `MoqAudioEncoder::track_init` returns the init that publishes its packets, and `MoqAudioEncoder::set_bitrate` retunes it live. +- `MoqAudioDecoderOutput::delay_us` fixes the audio playout delay, and + `MoqAudioConsumer::delay_us()` reports it, fixed or estimated from arrivals. +- `MoqAudio::jitter_us` carries the catalog's advertised jitter, which floors + the estimated audio playout delay. ### Changed diff --git a/rs/moq-ffi/src/audio.rs b/rs/moq-ffi/src/audio.rs index fbab2beb50b..d91929fba5d 100644 --- a/rs/moq-ffi/src/audio.rs +++ b/rs/moq-ffi/src/audio.rs @@ -6,6 +6,7 @@ //! Opus and AAC encode/decode inside the FFI boundary. use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Duration; use crate::bandwidth::{MoqBandwidth, MoqReservation}; @@ -174,12 +175,16 @@ pub struct MoqAudioDecoderOutput { /// [`MoqSubscription::max_delay_us`](crate::consumer::MoqSubscription::max_delay_us): /// when a group stalls and a newer group is more than this far ahead, /// the consumer skips. `None` keeps the moq-mux default of zero (skip - /// aggressively). Named `_max` to leave room for a future - /// `min_buffer_us` (jitter-buffer floor), which is a distinct knob: this - /// one bounds how stale a group may be, that one how much to hold before - /// presenting. + /// aggressively). Distinct from [`delay_us`](Self::delay_us): this one + /// bounds how stale a group may be and adds no buffering. #[uniffi(default = None)] pub max_delay_us: Option, + /// A fixed playout delay in microseconds, or `None` to estimate it from + /// arrival timing. This is the jitter buffer to hold ahead of the playhead, + /// reported by [`MoqAudioConsumer::delay_us`]. A fixed delay beyond the + /// track's retention is refused. + #[uniffi(default = None)] + pub delay_us: Option, } /// One audio frame: payload bytes plus a presentation timestamp. @@ -472,12 +477,15 @@ impl MoqAudioProducer { struct ConsumerInner { consumer: moq_audio::decode::Consumer, + delay: Arc, } impl ConsumerInner { async fn next(&mut self) -> Result, MoqError> { loop { - match self.consumer.read().await? { + let read = self.consumer.read().await; + self.delay.store(micros(self.consumer.delay()), Ordering::Relaxed); + match read? { Some(moq_audio::decode::Read::Frame { frame, .. }) => return Ok(Some(frame.into())), Some(moq_audio::decode::Read::End { .. }) => {} None => return Ok(None), @@ -486,10 +494,17 @@ impl ConsumerInner { } } +fn micros(duration: Duration) -> u64 { + duration.as_micros().try_into().unwrap_or(u64::MAX) +} + /// Consumer for a raw-audio track. #[derive(uniffi::Object)] pub struct MoqAudioConsumer { task: Task, + /// Mirrors the decoder's delay after each read, so it stays readable while + /// [`Self::next`] holds the decoder. + delay: Arc, } #[uniffi::export] @@ -499,6 +514,17 @@ impl MoqAudioConsumer { self.task.run(|mut state| async move { state.next().await }).await } + /// The playout delay to hold ahead of the playhead, in microseconds: the + /// fixed [`MoqAudioDecoderOutput::delay_us`], or the estimate from every + /// frame read so far. + /// + /// The estimate measures when [`Self::next`] returns each frame, so call it + /// again as soon as the previous frame is handled rather than pacing reads + /// to the speaker. + pub fn delay_us(&self) -> u64 { + self.delay.load(Ordering::Relaxed) + } + /// Make current and future reads return `Cancelled`. /// /// Terminal: the decoder session is released here, not when the handle is. @@ -528,6 +554,7 @@ fn audio_config(catalog_audio: crate::media::MoqAudio) -> Result, } /// A payload and the time it should be presented. @@ -509,6 +513,9 @@ pub(crate) fn convert_catalog(catalog: &moq_mux::catalog::hang::Catalog crate::media::MoqAudio { bitrate: None, enabled: true, container: MoqContainer::Legacy, + jitter_us: None, } } @@ -297,6 +298,7 @@ fn audio_output() -> crate::audio::MoqAudioDecoderOutput { sample_rate: None, channels: None, max_delay_us: None, + delay_us: None, } } @@ -659,6 +661,94 @@ async fn raw_audio_activity() { broadcast.close().unwrap(); } +/// The decoder's playout delay is configurable and stays observable while a read waits: +/// the decoder sits behind the read's lock, so the getter must not need it. +#[cfg(feature = "audio")] +#[tokio::test] +async fn raw_audio_decode_delay() { + use crate::audio::*; + + // 20 ms of mono silence at 48 kHz. + const FRAME_SAMPLES: usize = 960; + const FRAME_US: u64 = 20_000; + const FIXED_US: u64 = 80_000; + + let broadcast = MoqBroadcastProducer::new().unwrap(); + let consumer = broadcast.consume().unwrap(); + let catalog_consumer = MoqMediaCatalogConsumer::subscribe(&consumer).await.unwrap(); + let audio = MoqAudioProducer::new( + &MoqMediaCatalogProducer::new(broadcast.clone()).unwrap(), + MoqAudioEncoderInput { + format: MoqAudioSampleFormat::F32, + sample_rate: 48_000, + channels: 1, + }, + MoqAudioEncoderOutput { + track: Some("microphone".into()), + codec: MoqAudioCodec::opus(), + sample_rate: None, + channels: None, + bitrate: None, + frame_duration_us: FRAME_US as u32, + preset: None, + }, + None, + ) + .unwrap(); + let catalog = tokio::time::timeout(TIMEOUT, catalog_consumer.next()) + .await + .expect("timed out waiting for raw audio catalog") + .unwrap() + .expect("expected a raw audio catalog"); + let rendition = catalog.audio.get("microphone").unwrap().clone(); + + let subscribe = |delay_us| { + MoqAudioConsumer::subscribe( + &consumer, + "microphone".into(), + rendition.clone(), + MoqAudioDecoderOutput { + delay_us, + ..audio_output() + }, + ) + }; + let fixed = subscribe(Some(FIXED_US)).await.unwrap(); + let estimated = subscribe(None).await.unwrap(); + assert_eq!(fixed.delay_us(), FIXED_US); + let initial = estimated.delay_us(); + + // Both reads park on the decoder before any audio exists. + let read = |decoded: Arc| tokio::spawn(async move { decoded.next().await }); + let (fixed_read, estimated_read) = (read(fixed.clone()), read(estimated.clone())); + tokio::task::yield_now().await; + assert_eq!(fixed.delay_us(), FIXED_US, "readable while a read holds the decoder"); + + for i in 0..2 { + audio + .write(MoqAudioFrame { + timestamp_us: i * FRAME_US, + data: vec![0; FRAME_SAMPLES * std::mem::size_of::()], + }) + .unwrap(); + } + for read in [fixed_read, estimated_read] { + tokio::time::timeout(TIMEOUT, read) + .await + .expect("timed out waiting for decoded audio") + .unwrap() + .unwrap() + .expect("expected a decoded frame"); + } + + assert_eq!(fixed.delay_us(), FIXED_US, "a fixed delay ignores arrivals"); + // The decoded frame's duration joins the estimate, so the mirror must have moved. + assert_ne!(estimated.delay_us(), initial, "the estimate updates as frames are read"); + + audio.finish().unwrap(); + broadcast.close().unwrap(); +} + /// `frame_duration_us` is microseconds so Opus' 2.5 ms frame survives the trip, where /// the old integer millisecond field truncated it to 2 ms and libopus refused to encode. #[cfg(feature = "audio")] diff --git a/swift/Sources/Moq/Audio.swift b/swift/Sources/Moq/Audio.swift index 72dc3f1348c..ab276250038 100644 --- a/swift/Sources/Moq/Audio.swift +++ b/swift/Sources/Moq/Audio.swift @@ -6,7 +6,7 @@ public enum Audio { public typealias EncoderInput = AudioEncoderInput /// The codec side: codec, track name, rate, channels, bitrate, frame duration, and preset. public typealias EncoderOutput = AudioEncoderOutput - /// The PCM layout a `Consumer` delivers and how stale a group may get. + /// The PCM layout a `Consumer` delivers, how stale a group may get, and its playout delay. public typealias DecoderOutput = AudioDecoderOutput /// Interleaved PCM with its presentation timestamp. public typealias Frame = AudioFrame @@ -94,6 +94,10 @@ public enum Audio { try await ffi.next() } + /// The playout delay to hold ahead of the playhead, in microseconds: the fixed + /// `DecoderOutput.delayUs`, or the estimate from every frame read so far. + public var delayUs: UInt64 { ffi.delayUs() } + /// Cancel all current and future reads. public func cancel() { ffi.cancel() diff --git a/swift/Tests/MoqTests/SmokeTests.swift b/swift/Tests/MoqTests/SmokeTests.swift index 343662555bf..5e2b845dbe9 100644 --- a/swift/Tests/MoqTests/SmokeTests.swift +++ b/swift/Tests/MoqTests/SmokeTests.swift @@ -454,6 +454,54 @@ final class SmokeTests: XCTestCase { try broadcast.close() } + /// A fixed playout delay is reported back as is, and an estimated one moves + /// as frames are read. + func testAudioConsumerReportsItsPlayoutDelay() async throws { + let origin = OriginProducer() + let broadcast = try origin.createBroadcast(path: "audio-delay") + let audio = try Audio.Producer( + catalog: Media.CatalogProducer(broadcast: broadcast), + input: AudioEncoderInput(format: .f32, sampleRate: 48_000, channels: 1), + output: AudioEncoderOutput(codec: AudioCodec.opus(), track: "mic") + ) + try broadcast.announce() + + let consumer = try await origin.consume().requestBroadcast(path: "audio-delay") + let catalogs = try await Media.CatalogConsumer.subscribe(broadcast: consumer) + // XCTUnwrap takes an autoclosure, which can't hold an await. + let nextCatalog = try await catalogs.next() + let catalog = try XCTUnwrap(nextCatalog) + let rendition = try XCTUnwrap(catalog.audio["mic"]) + + let fixed = try await Audio.Consumer.subscribe( + broadcast: consumer, id: "mic", rendition: rendition, + output: AudioDecoderOutput(format: .f32, delayUs: 80_000) + ) + let estimated = try await Audio.Consumer.subscribe( + broadcast: consumer, id: "mic", rendition: rendition, + output: AudioDecoderOutput(format: .f32) + ) + XCTAssertEqual(fixed.delayUs, 80_000) + let initial = estimated.delayUs + + // Two 20 ms Opus frames of silence at 48 kHz mono. + for i in 0..<2 { + try audio.write(AudioFrame(timestampUs: UInt64(i) * 20_000, data: Data(count: 960 * 4))) + } + let fixedFrame = try await fixed.next() + XCTAssertNotNil(fixedFrame) + let estimatedFrame = try await estimated.next() + XCTAssertNotNil(estimatedFrame) + + XCTAssertEqual(fixed.delayUs, 80_000) + XCTAssertNotEqual(estimated.delayUs, initial, "the estimate should move as frames are read") + + fixed.cancel() + estimated.cancel() + try audio.finish() + try broadcast.close() + } + /// A decoded frame owns its picture: it converts to either CPU layout on /// demand and stays readable after its consumer is cancelled. func testDecodeVideoFrame() async throws {