diff --git a/doc/lib/go/index.md b/doc/lib/go/index.md index c9532888663..48f37c04a97 100644 --- a/doc/lib/go/index.md +++ b/doc/lib/go/index.md @@ -107,6 +107,12 @@ one: `FetchGroup`/`FetchMediaGroup`, `Dynamic()` with `Requests(ctx)`, `AppendDatagram`/`Datagrams(ctx)`, `SetCatalogSection`, `Used`/`Unused`, `Session().Stats()`. `moq.IsAuthError` and `moq.IsShutdown` classify errors. `moq.ProtocolError(err)` is the structured protocol failure (scope, verbatim code, kind) when the peer sent one. +`DecodeVideo` picks the decoded CPU pixel layout: `VideoDecoderOutput.Format` +is I420 when nil, or `VideoPixelFormatRgba` for four bytes a pixel, and every +`VideoDecodedFrame` repeats the layout it was decoded to. `Resize` is best +effort: only NVDEC has a built-in scaler, so read each frame's own `Width` and +`Height` rather than assuming it took. + - API reference: [pkg.go.dev/github.com/moq-dev/moq-go](https://pkg.go.dev/github.com/moq-dev/moq-go) - Source: [`go/`](https://github.com/moq-dev/moq/tree/main/go); `just go check` builds and tests locally - Mirrors the vanity path resolves to: [moq-dev/moq-go](https://github.com/moq-dev/moq-go) (wrapper), [moq-dev/moq-go-ffi](https://github.com/moq-dev/moq-go-ffi) (raw bindings and static libraries) diff --git a/doc/lib/kt/index.md b/doc/lib/kt/index.md index 00414821282..949bcbb0ee1 100644 --- a/doc/lib/kt/index.md +++ b/doc/lib/kt/index.md @@ -82,6 +82,12 @@ connection's send estimate; pass it to `encodeVideo` / `encodeAudio` or (scope, verbatim code, kind) when the peer sent one. Cancelling the collecting coroutine cancels the native side. +`decodeVideo` picks the decoded CPU pixel layout: `VideoDecoderOutput.format` +is `VideoPixelFormat.I420` when null, or `VideoPixelFormat.RGBA` for four bytes +a pixel, and every frame repeats the layout it was decoded to. `resize` is best +effort: only NVDEC has a built-in scaler, and MediaCodec is not it, so read each +frame's own `width` and `height` rather than assuming it took. + - API reference: [javadoc.io/doc/dev.moq/moq](https://javadoc.io/doc/dev.moq/moq) - Source: [`kt/`](https://github.com/moq-dev/moq/tree/main/kt); `just kt check` builds and tests locally - Artifacts: [dev.moq:moq](https://central.sonatype.com/artifact/dev.moq/moq), [dev.moq:moq-ffi](https://central.sonatype.com/artifact/dev.moq/moq-ffi) diff --git a/doc/lib/py/index.md b/doc/lib/py/index.md index a8db4f48906..d2632d7c238 100644 --- a/doc/lib/py/index.md +++ b/doc/lib/py/index.md @@ -94,6 +94,12 @@ app-owned track. `moq.is_auth(err)` and `moq.is_shutdown(err)` classify errors. Each server request reports a `moq.Transport` enum, including QUIC, Iroh, WebSocket, TCP, and Unix sockets. +`decode_video` picks the decoded CPU pixel layout: `VideoDecoderOutput.format` +is `VideoPixelFormat.I420` when unset, or `VideoPixelFormat.RGBA` for four +bytes a pixel, and every frame repeats the layout it was decoded to. `resize` +is best effort: only NVDEC has a built-in scaler, so read each frame's own +`width` and `height` rather than assuming it took. + - API reference: [moq-rs.readthedocs.io](https://moq-rs.readthedocs.io) - Source and examples: [`py/moq-rs`](https://github.com/moq-dev/moq/tree/main/py/moq-rs) - Raw bindings: [`moq-ffi`](https://pypi.org/project/moq-ffi/) on PyPI, for the unwrapped API diff --git a/doc/lib/swift/index.md b/doc/lib/swift/index.md index 4f7f1e3bfc2..5be2d630d2b 100644 --- a/doc/lib/swift/index.md +++ b/doc/lib/swift/index.md @@ -85,6 +85,12 @@ divides the connection's send estimate; pass it to `encodeVideo` / `isShutdown` classify errors. `protocolError` is the structured protocol failure (scope, verbatim code, kind) when the peer sent one. +`decodeVideo` picks the decoded CPU pixel layout: `VideoDecoderOutput.format` +is `.i420` when unset, or `.rgba` for four bytes a pixel, and every frame +repeats the layout it was decoded to. `resize` is best effort: only NVDEC has a +built-in scaler, and VideoToolbox is not it, so read each frame's own `width` +and `height` rather than assuming it took. + - API reference: [Swift Package Index (DocC)](https://swiftpackageindex.com/moq-dev/moq-swift/documentation/moq) - Source: [`swift/`](https://github.com/moq-dev/moq/tree/main/swift); `just swift check` builds and tests on a Mac - Packages SPM resolves: [moq-dev/moq-swift](https://github.com/moq-dev/moq-swift), [moq-dev/moq-swift-ffi](https://github.com/moq-dev/moq-swift-ffi) diff --git a/go/wrapper/moq_test.go b/go/wrapper/moq_test.go index 10cfd122fc0..c80da6a9f4c 100644 --- a/go/wrapper/moq_test.go +++ b/go/wrapper/moq_test.go @@ -240,6 +240,121 @@ func TestVideoPropertiesUseDefaultedFields(t *testing.T) { } } +// TestDecodeVideoFormat pins the decode side picking its CPU layout: an unset +// Format is I420, and RGBA is four bytes a pixel, with each frame naming the +// layout it was decoded to. +func TestDecodeVideoFormat(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + + origin := moq.NewOriginProducer() + broadcast, err := origin.CreateBroadcast("video-decode-format") + if err != nil { + t.Fatal(err) + } + track := "camera" + video, err := broadcast.EncodeVideo( + moq.VideoEncoderInput{Format: moq.VideoPixelFormatRgba, Width: 320, Height: 240, Framerate: 30}, + // Software both ways so the test is deterministic everywhere. + moq.VideoEncoderOutput{Codec: moq.VideoCodecH264, Track: &track, Kind: moq.SoftwareEncoder()}, + nil, + ) + if err != nil { + t.Fatal(err) + } + if err := broadcast.Announce(moq.Route{}); err != nil { + t.Fatal(err) + } + + // Seed the track so a subscriber joining below lands on encoded media. + rgba := make([]byte, 320*240*4) + for i := range rgba { + rgba[i] = 0x80 + } + if err := video.Cut(); err != nil { + t.Fatal(err) + } + for i := range 10 { + if err := video.Write(moq.VideoFrame{TimestampUs: uint64(i) * 33333, Data: rgba}); err != nil { + t.Fatal(err) + } + } + + bc, err := origin.Consume().RequestBroadcast(ctx, "video-decode-format") + if err != nil { + t.Fatal(err) + } + catalog, err := bc.Catalog(ctx) + if err != nil { + t.Fatal(err) + } + rendition, ok := catalog.Video[track] + if !ok { + t.Fatalf("catalog has no %q rendition: %v", track, catalog.Video) + } + + // Two subscribers over one publication, so the same encoded frames are read + // twice and only the requested layout differs. + i420, err := bc.DecodeVideo(ctx, track, rendition, moq.VideoDecoderOutput{}) + if err != nil { + t.Fatal(err) + } + defer i420.Cancel() + format := moq.VideoPixelFormatRgba + packed, err := bc.DecodeVideo(ctx, track, rendition, moq.VideoDecoderOutput{Format: &format}) + if err != nil { + t.Fatal(err) + } + defer packed.Cancel() + + // Keep the encoder fed so both decoders see frames after they joined. + for i := 10; i < 40; i++ { + if err := video.Write(moq.VideoFrame{TimestampUs: uint64(i) * 33333, Data: rgba}); err != nil { + t.Fatal(err) + } + } + + frame, err := i420.Next(ctx) + if err != nil { + t.Fatal(err) + } + if frame == nil { + t.Fatal("expected an I420 frame") + } + if frame.Format != moq.VideoPixelFormatI420 { + t.Fatalf("format = %v, want I420", frame.Format) + } + if want := int(frame.Width) * int(frame.Height) * 3 / 2; len(frame.Data) != want { + t.Fatalf("I420 length = %d, want %d", len(frame.Data), want) + } + + frame, err = packed.Next(ctx) + if err != nil { + t.Fatal(err) + } + if frame == nil { + t.Fatal("expected an RGBA frame") + } + if frame.Format != moq.VideoPixelFormatRgba { + t.Fatalf("format = %v, want RGBA", frame.Format) + } + if want := int(frame.Width) * int(frame.Height) * 4; len(frame.Data) != want { + t.Fatalf("RGBA length = %d, want %d", len(frame.Data), want) + } + for i := 3; i < len(frame.Data); i += 4 { + if frame.Data[i] != 0xFF { + t.Fatalf("RGBA alpha at %d = %#x, want 0xff", i, frame.Data[i]) + } + } + + if err := video.Finish(); err != nil { + t.Fatal(err) + } + if err := broadcast.Finish(); err != nil { + t.Fatal(err) + } +} + func TestFetchGroupAndServeDynamicMiss(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), testTimeout) defer cancel() diff --git a/go/wrapper/subscribe.go b/go/wrapper/subscribe.go index 3ba56d0c401..d791ca491e6 100644 --- a/go/wrapper/subscribe.go +++ b/go/wrapper/subscribe.go @@ -116,9 +116,10 @@ func (b *BroadcastConsumer) DecodeAudio( return &AudioConsumer{inner: inner}, nil } -// DecodeVideo subscribes to a video track and decodes it inside the bindings, -// yielding packed I420. catalogVideo comes from the catalog. output.Resize is -// best effort, so read each frame's own dimensions. +// DecodeVideo subscribes to a video track and decodes it inside the bindings. +// catalogVideo comes from the catalog. output.Format picks the packed CPU layout +// every frame arrives in, defaulting to I420 when nil; each frame repeats it. +// output.Resize is best effort, so read each frame's own dimensions. func (b *BroadcastConsumer) DecodeVideo( ctx context.Context, name string, diff --git a/go/wrapper/types.go b/go/wrapper/types.go index 819ede075d9..40bf4f53eaa 100644 --- a/go/wrapper/types.go +++ b/go/wrapper/types.go @@ -52,11 +52,11 @@ type ( // Video describes one catalog rendition, including whether the publisher recommends temporarily avoiding it. Video = ffi.MoqVideo // VideoHint supplies catalog fields a video stream can't reveal itself, such as bitrate, filling only the gaps. - // VideoDecodedFrame is one decoded video frame: packed I420, its dimensions, and a timestamp in microseconds. + VideoHint = ffi.MoqVideoHint + // VideoDecodedFrame is one decoded video frame: packed pixels, the layout they are in, their dimensions, and a timestamp in microseconds. VideoDecodedFrame = ffi.MoqVideoDecodedFrame - // VideoDecoderOutput configures what DecodeVideo delivers: an optional resize plus a max age. + // VideoDecoderOutput configures what DecodeVideo delivers: an optional pixel format and resize, plus a max age. VideoDecoderOutput = ffi.MoqVideoDecoderOutput - VideoHint = ffi.MoqVideoHint // AudioFormat is a single audio codec an importer can parse. AudioFormat = ffi.MoqAudioFormat // VideoFormat is a single video codec an importer can parse. @@ -67,7 +67,7 @@ type ( VideoProperties = ffi.MoqVideoProperties // VideoCodec identifies a published video track's codec: H.264 or H.265. VideoCodec = ffi.MoqVideoCodec - // VideoPixelFormat is a raw pixel layout (I420 or RGBA) written to a VideoProducer. + // VideoPixelFormat is a CPU pixel layout (I420 or RGBA): written to a VideoProducer, or delivered by DecodeVideo. VideoPixelFormat = ffi.MoqVideoPixelFormat // VideoEncoderInput declares the pixel layout, resolution, and framerate of frames written to a video producer. VideoEncoderInput = ffi.MoqVideoEncoderInput @@ -184,7 +184,8 @@ func OpusAudioCodec() *AudioCodec { return ffi.MoqAudioCodecOpus() } -// VideoPixelFormat values: the raw pixel layout fed to the in-process encoder. +// VideoPixelFormat values: the raw pixel layout fed to the in-process encoder, +// and the one the in-process decoder delivers. const ( // VideoPixelFormatI420 is tightly-packed planar I420: Y, then U, then V. VideoPixelFormatI420 = ffi.MoqVideoPixelFormatI420 diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt index 2d362fc8114..4ebdbefd57e 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt @@ -76,7 +76,7 @@ typealias MediaGroupConsumer = uniffi.moq.MoqMediaGroupConsumer typealias AudioProducer = uniffi.moq.MoqAudioProducer /** The read side of a raw-audio track: yields decoded PCM frames. */ typealias AudioConsumer = uniffi.moq.MoqAudioConsumer -/** The read side of a video track decoded inside the bindings: yields packed I420 frames. */ +/** The read side of a video track decoded inside the bindings: yields packed frames in the layout asked for. */ typealias VideoConsumer = uniffi.moq.MoqVideoConsumer /** The write side of a raw-video track; pixels written here are encoded inside the FFI boundary. */ typealias VideoProducer = uniffi.moq.MoqVideoProducer @@ -138,9 +138,9 @@ typealias AudioCodec = uniffi.moq.MoqAudioCodec typealias AudioSampleFormat = uniffi.moq.MoqAudioSampleFormat /** The PCM layout an [AudioConsumer] should decode to. */ typealias AudioDecoderOutput = uniffi.moq.MoqAudioDecoderOutput -/** What a [VideoConsumer] decodes to: an optional resize plus a latency budget. */ +/** What a [VideoConsumer] decodes to: an optional pixel format and resize, plus a latency budget. */ typealias VideoDecoderOutput = uniffi.moq.MoqVideoDecoderOutput -/** One decoded video frame: packed I420, its dimensions, and a timestamp. */ +/** One decoded video frame: packed pixels, the layout they are in, its dimensions, and a timestamp. */ typealias VideoDecodedFrame = uniffi.moq.MoqVideoDecodedFrame /** The PCM layout the caller feeds an [AudioProducer]. */ typealias AudioEncoderInput = uniffi.moq.MoqAudioEncoderInput @@ -150,7 +150,7 @@ typealias AudioEncoderOutput = uniffi.moq.MoqAudioEncoderOutput typealias VideoFrame = uniffi.moq.MoqVideoFrame /** A video codec identifier (H.264 or H.265). */ typealias VideoCodec = uniffi.moq.MoqVideoCodec -/** A raw pixel layout (I420 or RGBA) fed to a [VideoProducer]. */ +/** A CPU pixel layout (I420 or RGBA): fed to a [VideoProducer], or delivered by `decodeVideo`. */ typealias VideoPixelFormat = uniffi.moq.MoqVideoPixelFormat /** The pixel layout, resolution, and framerate the caller feeds a [VideoProducer]. */ typealias VideoEncoderInput = uniffi.moq.MoqVideoEncoderInput diff --git a/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt b/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt index 85e56bbcfd6..f101109f23f 100644 --- a/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt +++ b/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt @@ -357,6 +357,59 @@ class SmokeTest { } } + @Test + fun `decode video picks its pixel format`() = runTest { + OriginProducer(OriginConfig()).use { origin -> + origin.createBroadcast("video-decode-format").use { broadcast -> + val video = broadcast.encodeVideo( + VideoEncoderInput(format = VideoPixelFormat.RGBA, width = 320u, height = 240u, framerate = 30u), + // Software both ways so the test is deterministic everywhere. + VideoEncoderOutput(codec = VideoCodec.H264, track = "camera", kind = softwareEncoder), + null, + ) + broadcast.announce(Route()) + + // Seed the track so a subscriber joining below lands on encoded media. + val rgba = ByteArray(320 * 240 * 4) { 0x80.toByte() } + video.cut() + for (i in 0 until 10) { + video.write(VideoFrame(timestampUs = i.toULong() * 33_333uL, data = rgba)) + } + + val consumer = origin.consume().requestBroadcast("video-decode-format") + val catalog = consumer.subscribeCatalog().next()!! + val rendition = catalog.video["camera"]!! + + // Two subscribers over one publication, so the same encoded frames + // are read twice and only the requested layout differs. + val i420 = consumer.decodeVideo("camera", rendition, VideoDecoderOutput()) + val packed = consumer.decodeVideo( + "camera", + rendition, + VideoDecoderOutput(format = VideoPixelFormat.RGBA), + ) + + // Keep the encoder fed so both decoders see frames after they joined. + for (i in 10 until 40) { + video.write(VideoFrame(timestampUs = i.toULong() * 33_333uL, data = rgba)) + } + + val planar = i420.next()!! + assertEquals(VideoPixelFormat.I420, planar.format) + assertEquals(planar.width.toInt() * planar.height.toInt() * 3 / 2, planar.data.size) + + val frame = packed.next()!! + assertEquals(VideoPixelFormat.RGBA, frame.format) + assertEquals(frame.width.toInt() * frame.height.toInt() * 4, frame.data.size) + assertTrue(frame.data.indices.filter { it % 4 == 3 }.all { frame.data[it] == 0xFF.toByte() }) + + i420.cancel() + packed.cancel() + video.finish() + } + } + } + @Test fun `encode audio with opus object`() { val input = AudioEncoderInput(format = AudioSampleFormat.F32, sampleRate = 48_000u, channels = 1u) diff --git a/py/moq-rs/moq/subscribe.py b/py/moq-rs/moq/subscribe.py index 25933bad4fc..7bb80427f97 100644 --- a/py/moq-rs/moq/subscribe.py +++ b/py/moq-rs/moq/subscribe.py @@ -271,9 +271,9 @@ class VideoConsumer: """Async iterator of decoded video frames. Built via :meth:`BroadcastConsumer.decode_video`. Each frame is - tightly-packed I420 and carries its own ``width`` and ``height``: - ``output.resize`` is best effort, so read the frame rather than - assuming it took. + tightly packed in its own ``format`` and carries its own ``width`` + and ``height``: ``output.resize`` is best effort, so read the frame + rather than assuming it took. """ def __init__(self, inner: MoqVideoConsumer) -> None: @@ -513,9 +513,12 @@ async def decode_video( ``catalog_video`` comes from the catalog (e.g. ``await broadcast.catalog()`` followed by ``catalog.video[name]``). - Frames arrive as tightly-packed I420. An unrecognized codec raises - here; a recognized one no native backend handles raises when the - decoder opens, both before the first frame. + An unrecognized codec raises here; a recognized one no native backend + handles raises when the decoder opens, both before the first frame. + + ``output.format`` picks the packed CPU layout frames arrive in and + defaults to :attr:`VideoPixelFormat.I420`, which is what a decoder + produces natively; each frame repeats the layout it was decoded to. ``output.resize`` asks the decoder for a different size and is best effort, so read each frame's own dimensions. diff --git a/py/moq-rs/tests/test_local.py b/py/moq-rs/tests/test_local.py index 14d53607f39..04185c19c5d 100644 --- a/py/moq-rs/tests/test_local.py +++ b/py/moq-rs/tests/test_local.py @@ -1037,6 +1037,60 @@ def test_encode_audio_with_opus_object(): broadcast.finish() +async def test_decode_video_format(): + """The decode side picks its CPU layout: unset is I420, RGBA is four bytes a pixel.""" + origin = moq.OriginProducer() + broadcast = create_announced(origin, "video-decode-format") + video = broadcast.encode_video( + moq.VideoEncoderInput(format=moq.VideoPixelFormat.RGBA, width=320, height=240, framerate=30), + # Software both ways so the test is deterministic everywhere. uniffi nests + # each variant inside the enum without declaring it a subclass, so the cast + # is what makes the variant typecheck. + moq.VideoEncoderOutput( + codec=moq.VideoCodec.H264, + track="camera", + kind=cast(moq.VideoEncoderKind, moq.VideoEncoderKind.SOFTWARE()), + ), + ) + + # Seed the track so a subscriber joining below lands on encoded media. + rgba = bytes([0x80]) * (320 * 240 * 4) + video.cut() + for i in range(10): + video.write(moq.VideoFrame(timestamp_us=i * 33_333, data=rgba)) + + consumer = origin.consume() + broadcast_consumer = await asyncio.wait_for(consumer.request_broadcast("video-decode-format"), timeout=5.0) + catalog = await asyncio.wait_for(broadcast_consumer.catalog(), timeout=5.0) + track_name = next(iter(catalog.video)) + rendition = catalog.video[track_name] + + # Two subscribers over one publication, so the same encoded frames are read + # twice and only the requested layout differs. + i420 = await broadcast_consumer.decode_video(track_name, rendition) + packed = await broadcast_consumer.decode_video( + track_name, rendition, moq.VideoDecoderOutput(format=moq.VideoPixelFormat.RGBA) + ) + + # Keep the encoder fed so both decoders see frames after they joined. + for i in range(10, 40): + video.write(moq.VideoFrame(timestamp_us=i * 33_333, data=rgba)) + + frame = await asyncio.wait_for(anext(i420), timeout=5.0) + assert frame.format == moq.VideoPixelFormat.I420 + assert len(frame.data) == frame.width * frame.height * 3 // 2 + + frame = await asyncio.wait_for(anext(packed), timeout=5.0) + assert frame.format == moq.VideoPixelFormat.RGBA + assert len(frame.data) == frame.width * frame.height * 4 + assert all(frame.data[i] == 0xFF for i in range(3, len(frame.data), 4)), "RGBA output should be opaque" + + i420.cancel() + packed.cancel() + video.finish() + broadcast.finish() + + async def test_announce_then_unannounce_is_visible(): origin = moq.OriginProducer() broadcast = origin.create_broadcast("live") diff --git a/quest/dev/release.md b/quest/dev/release.md index 5b2c714521c..04eecb84c42 100644 --- a/quest/dev/release.md +++ b/quest/dev/release.md @@ -55,8 +55,8 @@ with its PR and the replacement call: - bindings: the Go module is `moq.dev/moq` with `context.Context` on every blocking call (#2957); `MoqAudioCodec` is an `opus()` object (#3671); the configuration setters are fallible (#3642); durations are microseconds and - the rate estimates are `estimated_*` (#3744); the decode format knob from - [Decode format](/quest/next/ffi-decode-format.md). + the rate estimates are `estimated_*` (#3744); `MoqVideoDecoderOutput.format` + picks I420 or RGBA decode output, additively. Release-notes outline, the additions worth leading with, in order of value to a consumer: prefix routes and wildcard Pattern events (#3225, #3649); @@ -86,7 +86,6 @@ Public API: none beyond the required quests. Wire: none. ## Required -- [Decode format](/quest/next/ffi-decode-format.md) - the C-only decode knob reaches every uniffi binding - [Binding parity](/quest/next/binding-parity.md) - every wrapper reaches every moq-ffi method - [Binding docs](/quest/next/binding-docs.md) - the binding pages compile against the wrappers - The merged relay has soaked on moq.pro staging and the maintainer has signed it off diff --git a/quest/next/README.md b/quest/next/README.md index 3d7c8e1d4d9..6e2c0724e2d 100644 --- a/quest/next/README.md +++ b/quest/next/README.md @@ -31,7 +31,6 @@ transport, benchmark tooling); worktrees isolate commits, not semantics. - [Headless player](/quest/next/watch-player.md) - `Watch.Player` assembles the pipeline the element, the room, and moq.pro each rebuild - [@moq/net additive](/quest/next/js-net-additive.md) - a live-broadcasts getter, `Table.dynamic`, credential refresh before redial, inferred `share` - [Last frame duration](/quest/next/mux-last-frame-duration.md) - a group's final frame keeps its duration on 90 kHz and nanosecond imports -- [Decode format](/quest/next/ffi-decode-format.md) - the C-only decode pixel format knob reaches every uniffi binding - [Publisher clocks](/quest/next/publisher-clock.md) - wire the shared clock through native and browser publisher restarts - [Broadcast route](/quest/next/js-broadcast-route.md) - JS Announce.Broadcast goes live on any claim matching its path, like Rust routed() - [io_uring check](/quest/next/check-uring-feature.md) - a moq-relay diff compiles the io-uring feature in `just check`, not only nightly diff --git a/quest/next/ffi-decode-format.md b/quest/next/ffi-decode-format.md deleted file mode 100644 index 6ea4e46b182..00000000000 --- a/quest/next/ffi-decode-format.md +++ /dev/null @@ -1,23 +0,0 @@ -# [S] The decode pixel format reaches every uniffi binding - -## Goal - -A Python, Swift, Kotlin, Go, or Dart consumer chooses I420 or RGBA decode -output the way a C consumer does. `moq_video_decoder_output` in libmoq -carries `format`, `width`, and `height` (#3674), while -`MoqVideoDecoderOutput` in `rs/moq-ffi/src/video.rs` has only a best-effort -`resize` and documents that there is no format to choose. - -## Plan - -Add `format: MoqVideoPixelFormat` with an I420 default to -`MoqVideoDecoderOutput`, reusing the enum the encode side already exports, -and make the delivered frame carry the format it was decoded to. Keep -`resize` best-effort but state per backend which honor it. Follow the -Cross-Package Sync row for `rs/moq-ffi`: libmoq maps its struct onto the -shared record instead of a private conversion, the Go wrapper and the -hand-written Python and Dart wrappers gain the field, and each -`doc/lib/*/index.md` gets the "Raw decode output" bullet the C page has. -Test one RGBA decode per binding smoke suite. - -Public API: additive field on a uniffi record and its wrappers. Wire: none. diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index bf2a3320e78..0f9f734fe39 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -1830,6 +1830,114 @@ async fn video_raw_publish_consume() { broadcast.finish().unwrap(); } +/// The decode side picks its CPU pixel layout: an unset `format` delivers I420, +/// and RGBA delivers `width * height * 4` bytes, with each frame naming the +/// layout it was decoded to. +#[cfg(feature = "video")] +#[tokio::test] +async fn video_decode_format() { + use crate::video::*; + + let origin = MoqOriginProducer::new(MoqOriginConfig::default()); + let broadcast = create_announced(&origin, "video-decode-format"); + + let video = broadcast + .encode_video( + MoqVideoEncoderInput { + format: MoqVideoPixelFormat::Rgba, + width: 320, + height: 240, + framerate: 30, + }, + MoqVideoEncoderOutput { + codec: MoqVideoCodec::H264, + track: Some("camera".into()), + bitrate: None, + gop: None, + // Software both ways so the test is deterministic everywhere. + kind: MoqVideoEncoderKind::Software, + }, + None, + ) + .unwrap(); + + // Seed the track so a subscriber joining below lands on encoded media. + let rgba = vec![0x80u8; 320 * 240 * 4]; + video.cut().unwrap(); + for i in 0..10u64 { + video + .write(MoqVideoFrame { + timestamp_us: i * 33_333, + data: rgba.clone(), + }) + .unwrap(); + } + + let consumer = origin.consume(); + let broadcast_consumer = await_announced(&consumer, "video-decode-format").await; + let catalog_consumer = broadcast_consumer.subscribe_catalog().await.unwrap(); + let catalog = tokio::time::timeout(TIMEOUT, catalog_consumer.next()) + .await + .expect("timed out") + .unwrap() + .expect("expected catalog"); + let (track, rendition) = catalog.video.iter().next().unwrap(); + + // Two subscribers over one publication, so the same encoded frames are read + // twice and only the requested layout differs. + let i420 = broadcast_consumer + .decode_video(track.clone(), rendition.clone(), MoqVideoDecoderOutput::default()) + .await + .unwrap(); + let rgba_out = broadcast_consumer + .decode_video( + track.clone(), + rendition.clone(), + MoqVideoDecoderOutput { + format: Some(MoqVideoPixelFormat::Rgba), + ..Default::default() + }, + ) + .await + .unwrap(); + + // Keep the encoder fed so both decoders see frames after they joined. + for i in 10..40u64 { + video + .write(MoqVideoFrame { + timestamp_us: i * 33_333, + data: rgba.clone(), + }) + .unwrap(); + } + + let frame = tokio::time::timeout(TIMEOUT, i420.next()) + .await + .expect("timed out") + .unwrap() + .expect("expected an I420 frame"); + assert!(matches!(frame.format, MoqVideoPixelFormat::I420)); + assert_eq!(frame.data.len(), frame.width as usize * frame.height as usize * 3 / 2); + + let frame = tokio::time::timeout(TIMEOUT, rgba_out.next()) + .await + .expect("timed out") + .unwrap() + .expect("expected an RGBA frame"); + assert!(matches!(frame.format, MoqVideoPixelFormat::Rgba)); + assert_eq!(frame.data.len(), frame.width as usize * frame.height as usize * 4); + // Every fourth byte is alpha, so an opaque frame proves the conversion ran rather than handing back planes. + assert!( + frame.data.chunks_exact(4).all(|px| px[3] == 0xFF), + "RGBA output should be opaque" + ); + + i420.cancel(); + rgba_out.cancel(); + video.finish().unwrap(); + broadcast.finish().unwrap(); +} + /// Regression: a `MoqVideoProducer` is shared, so its calls land on whichever /// thread the caller is on, and none of them need be the thread that published. /// Holding a bare `Encoder` made that unsound on Windows, where the codec's COM diff --git a/rs/moq-ffi/src/video.rs b/rs/moq-ffi/src/video.rs index 6c6854f02a1..bf81a5961c8 100644 --- a/rs/moq-ffi/src/video.rs +++ b/rs/moq-ffi/src/video.rs @@ -21,7 +21,8 @@ use crate::consumer::MoqBroadcastConsumer; use crate::error::MoqError; use crate::producer::MoqBroadcastProducer; -/// Pixel layout of the raw frames passed to [`MoqVideoProducer::write`]. +/// A CPU pixel layout: what [`MoqVideoProducer::write`] is fed, and what +/// [`MoqBroadcastConsumer::decode_video`] hands back. #[derive(Clone, Copy, uniffi::Enum)] pub enum MoqVideoPixelFormat { /// Tightly-packed planar I420: Y, then U, then V, no row padding @@ -559,16 +560,16 @@ async fn follow_reservation( /// How a subscriber wants decoded video delivered. /// -/// Frames always arrive as tightly-packed I420 (Y, then U, then V, no row -/// padding), so there is no pixel format to choose: a decoder's native output is -/// flattened to CPU I420 at delivery, since the FFI boundary can't hand back a -/// GPU surface. -#[derive(Clone, uniffi::Record)] +/// A decoder's native output is flattened to CPU pixels at delivery, since the +/// FFI boundary can't hand back a GPU surface; `format` picks the layout it is +/// flattened to. +#[derive(Clone, Default, uniffi::Record)] pub struct MoqVideoDecoderOutput { /// Ask the decoder to emit frames at this size instead of the stream's - /// native one. Best effort: a backend with a built-in scaler honors it for - /// free, others ignore it, so read each frame's own dimensions rather than - /// assuming this took. Both dimensions must be even. + /// native one. Best effort: only NVDEC has a built-in scaler and honors it for + /// free; VideoToolbox, Media Foundation, MediaCodec, VAAPI, V4L2, and openh264 + /// ignore it and decode at the stream's native size. Read each frame's own + /// dimensions rather than assuming this took. Both dimensions must be even. #[uniffi(default = None)] pub resize: Option, /// Upper bound on buffering before skipping a stalled group, in @@ -577,9 +578,18 @@ pub struct MoqVideoDecoderOutput { /// `None` keeps the moq-mux default of zero (skip aggressively). #[uniffi(default = None)] pub max_age_us: Option, + /// CPU pixel layout every frame is delivered in. `None` delivers + /// [`MoqVideoPixelFormat::I420`], which is what a decoder produces natively, + /// so asking for RGBA costs a conversion per frame. + /// + /// Spelled as an option rather than an I420-valued field because uniffi has no + /// enum default, and a required field would break every existing caller. + #[uniffi(default = None)] + pub format: Option, } -/// One decoded video frame: packed I420 plus the size it actually decoded to. +/// One decoded video frame: packed pixels plus the layout and size they +/// actually decoded to. /// /// Unlike [`MoqVideoFrame`] on the publish side, this carries dimensions: there /// they are fixed by the encoder config, here they are whatever the stream @@ -592,12 +602,17 @@ pub struct MoqVideoDecodedFrame { pub width: u32, /// Frame height in pixels. pub height: u32, - /// Tightly-packed I420: Y, then U, then V. + /// The pixels, in `format`: I420 is Y, then U, then V (`width * height * 3 / + /// 2` bytes); RGBA is `width * height * 4` bytes. Neither has row padding. pub data: Vec, + /// The layout `data` is in, which is what + /// [`MoqVideoDecoderOutput::format`] asked for. + pub format: MoqVideoPixelFormat, } struct VideoConsumerInner { consumer: moq_video::decode::Consumer, + format: MoqVideoPixelFormat, } impl VideoConsumerInner { @@ -607,18 +622,23 @@ impl VideoConsumerInner { }; let size = frame.size(); - // CPU output was asked for, so this is a move rather than a download: - // uniffi has no handle type to hand back a texture with anyway. - let data = frame - .surface - .into_i420() - .map_err(|err| MoqError::Codec(err.to_string()))? - .into_data(); + // CPU output was asked for, so I420 is a move rather than a download: + // uniffi has no handle type to hand back a texture with anyway. RGBA is the + // one layout no decoder produces, so it costs a conversion here. + let data = match self.format { + MoqVideoPixelFormat::I420 => frame.surface.into_i420().map(|i420| i420.into_data()), + MoqVideoPixelFormat::Rgba => frame + .surface + .to_rgba(&moq_video::convert::Config::default()) + .map(|rgba| rgba.into_data()), + } + .map_err(|err| MoqError::Codec(err.to_string()))?; Ok(Some(MoqVideoDecodedFrame { timestamp_us: frame.timestamp.as_micros() as u64, width: size.width, height: size.height, + format: self.format, data, })) } @@ -700,8 +720,9 @@ impl MoqBroadcastConsumer { let broadcast = self.resolve_inner(reference.as_deref()).await?; let mut options = moq_video::decode::Options::default(); - // The bindings hand back packed I420, so let a backend that can decode - // straight to the CPU do that rather than downloading afterwards. + // The bindings hand back packed CPU pixels whatever the format, so let a + // backend that can decode straight to the CPU do that rather than + // downloading afterwards. options.decoder.output = moq_video::Output::Cpu; options.decoder.scale_hint = output.resize.map(|size| moq_video::Size::new(size.width, size.height)); options.max_age = output @@ -712,7 +733,12 @@ impl MoqBroadcastConsumer { let consumer = moq_video::decode::Consumer::new(&broadcast, &cfg, name, options).await?; Ok(Arc::new(MoqVideoConsumer { - task: crate::ffi::Task::new(VideoConsumerInner { consumer }), + task: crate::ffi::Task::new(VideoConsumerInner { + consumer, + // Resolved here rather than at the boundary: a Go caller gets no + // uniffi default, so an unset field has to mean I420 in Rust. + format: output.format.unwrap_or(MoqVideoPixelFormat::I420), + }), })) } } diff --git a/swift/Sources/Moq/Aliases.swift b/swift/Sources/Moq/Aliases.swift index 72ae8cb5bae..dcfef7b25ea 100644 --- a/swift/Sources/Moq/Aliases.swift +++ b/swift/Sources/Moq/Aliases.swift @@ -42,9 +42,11 @@ public typealias AudioEncoderInput = MoqFFI.MoqAudioEncoderInput /// The encoder-side config for a published audio track: codec, rate, channels, /// bitrate, and frame duration. public typealias AudioEncoderOutput = MoqFFI.MoqAudioEncoderOutput -/// What a `VideoConsumer` decodes to: an optional resize plus a max age. +/// What a `VideoConsumer` decodes to: an optional pixel format and resize, +/// plus a max age. public typealias VideoDecoderOutput = MoqVideoDecoderOutput -/// One decoded video frame: packed I420, its dimensions, and a timestamp. +/// One decoded video frame: packed pixels, the layout they are in, their +/// dimensions, and a timestamp. public typealias VideoDecodedFrame = MoqVideoDecodedFrame /// The PCM layout an `AudioConsumer` decodes to, plus its max age. public typealias AudioDecoderOutput = MoqFFI.MoqAudioDecoderOutput @@ -60,7 +62,8 @@ public typealias VideoEncoderInput = MoqFFI.MoqVideoEncoderInput /// The encoder-side config for a published video track: name, codec, bitrate, /// keyframe interval, and backend preference. public typealias VideoEncoderOutput = MoqFFI.MoqVideoEncoderOutput -/// A raw pixel layout (I420 or RGBA) fed to a `VideoProducer`. +/// A CPU pixel layout (I420 or RGBA): fed to a `VideoProducer`, or delivered +/// by `decodeVideo`. public typealias VideoPixelFormat = MoqFFI.MoqVideoPixelFormat /// A video codec identifier (H.264 or H.265). public typealias VideoCodec = MoqFFI.MoqVideoCodec diff --git a/swift/Sources/Moq/Broadcast.swift b/swift/Sources/Moq/Broadcast.swift index b78257fdc4e..3be12735d1f 100644 --- a/swift/Sources/Moq/Broadcast.swift +++ b/swift/Sources/Moq/Broadcast.swift @@ -82,9 +82,11 @@ public final class BroadcastConsumer: Sendable { AudioConsumer(try await ffi.decodeAudio(name: name, catalogAudio: catalogAudio, output: output)) } - /// Subscribe to a video track and decode it inside the bindings, yielding - /// packed I420. `catalogVideo` is the matching rendition from the catalog. + /// Subscribe to a video track and decode it inside the bindings. + /// `catalogVideo` is the matching rendition from the catalog. /// + /// `output.format` picks the packed CPU layout every frame arrives in and + /// defaults to I420; each frame repeats the layout it was decoded to. /// `output.resize` is best effort, so read each frame's own dimensions. public func decodeVideo( name: String, diff --git a/swift/Sources/Moq/Video.swift b/swift/Sources/Moq/Video.swift index 68aada97bc4..b5e1ac6a168 100644 --- a/swift/Sources/Moq/Video.swift +++ b/swift/Sources/Moq/Video.swift @@ -1,7 +1,8 @@ import MoqFFI /// Read side of a video track decoded inside the bindings. Iterating yields -/// tightly-packed I420 frames, each carrying the size it actually decoded to. +/// tightly-packed frames, each carrying the pixel layout and the size it +/// actually decoded to. public final class VideoConsumer: AsyncSequence, Sendable { /// The decoded video frame emitted by this sequence. public typealias Element = VideoDecodedFrame diff --git a/swift/Tests/MoqTests/SmokeTests.swift b/swift/Tests/MoqTests/SmokeTests.swift index f64d14d2555..537170b4d2d 100644 --- a/swift/Tests/MoqTests/SmokeTests.swift +++ b/swift/Tests/MoqTests/SmokeTests.swift @@ -279,6 +279,63 @@ final class SmokeTests: XCTestCase { try broadcast.finish() } + /// The decode side picks its CPU layout: an unset `format` is I420, and RGBA + /// is four bytes a pixel, with each frame naming the layout it decoded to. + func testDecodeVideoFormat() async throws { + let origin = OriginProducer() + let broadcast = try origin.createBroadcast(path: "video-decode-format") + let video = try broadcast.encodeVideo( + input: VideoEncoderInput(format: .rgba, width: 320, height: 240, framerate: 30), + // Software both ways so the test is deterministic everywhere. + output: VideoEncoderOutput(codec: .h264, track: "camera", kind: .software) + ) + try broadcast.announce() + + // Seed the track so a subscriber joining below lands on encoded media. + let rgba = Data(repeating: 0x80, count: 320 * 240 * 4) + try video.cut() + for i in 0..<10 { + try video.write(VideoFrame(timestampUs: UInt64(i) * 33_333, data: rgba)) + } + + let consumer = try await origin.consume().requestBroadcast(path: "video-decode-format") + let catalogs = try await consumer.subscribeCatalog() + // XCTUnwrap takes an autoclosure, which can't hold an await. + let nextCatalog = try await catalogs.next() + let catalog = try XCTUnwrap(nextCatalog) + let rendition = try XCTUnwrap(catalog.video["camera"]) + + // Two subscribers over one publication, so the same encoded frames are + // read twice and only the requested layout differs. + let i420 = try await consumer.decodeVideo(name: "camera", catalogVideo: rendition) + defer { i420.cancel() } + let packed = try await consumer.decodeVideo( + name: "camera", + catalogVideo: rendition, + output: VideoDecoderOutput(format: .rgba) + ) + defer { packed.cancel() } + + // Keep the encoder fed so both decoders see frames after they joined. + for i in 10..<40 { + try video.write(VideoFrame(timestampUs: UInt64(i) * 33_333, data: rgba)) + } + + let nextPlanar = try await i420.next() + let planar = try XCTUnwrap(nextPlanar) + XCTAssertEqual(planar.format, .i420) + XCTAssertEqual(planar.data.count, Int(planar.width) * Int(planar.height) * 3 / 2) + + let nextPacked = try await packed.next() + let frame = try XCTUnwrap(nextPacked) + XCTAssertEqual(frame.format, .rgba) + XCTAssertEqual(frame.data.count, Int(frame.width) * Int(frame.height) * 4) + XCTAssertTrue(stride(from: 3, to: frame.data.count, by: 4).allSatisfy { frame.data[$0] == 0xFF }) + + try video.finish() + try broadcast.finish() + } + func testEncodeAudioWithOpusObject() throws { // The config retains the codec, so releasing either first must still encode. let input = AudioEncoderInput(format: .f32, sampleRate: 48_000, channels: 1)