From 3bfa84f26f333a68c46753917f431c368a61b18d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 9 Nov 2026 05:35:32 -0800 Subject: [PATCH 1/5] chore(quest): claim quest/m2/video-codec-receive.md Quest-Claim: 7c1530dc566d7f32 From 71c4444f29013b0675cf405696637ab18dfe933e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 9 Nov 2026 06:06:44 -0800 Subject: [PATCH 2/5] feat(video)!: codecs split submission from receive Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + dart/moq_ffi/lib/src/moq.dart | 154 ++++----- demo/carla/bridge/src/stream.rs | 10 +- demo/carla/bridge/tests/gpu.rs | 6 +- go/wrapper/video/video.go | 17 +- py/moq-rs/moq/video.py | 21 +- py/moq-rs/tests/test_local.py | 29 ++ rs/moq-boy/src/video.rs | 27 +- rs/moq-c/README.md | 3 +- rs/moq-c/src/encoder.rs | 113 +++---- rs/moq-c/src/test.rs | 10 +- rs/moq-c/src/test/parity.rs | 36 ++- rs/moq-c/src/video.rs | 28 +- rs/moq-cli/src/play/media.rs | 56 ++-- rs/moq-cli/src/play/video.rs | 67 ++-- rs/moq-ffi/src/video.rs | 108 ++++--- rs/moq-transcode/benches/ladder.rs | 3 +- rs/moq-transcode/src/feed.rs | 8 +- rs/moq-transcode/src/lib.rs | 18 +- rs/moq-transcode/src/rung.rs | 178 +++++------ rs/moq-video/Cargo.toml | 6 + rs/moq-video/benches/codec.rs | 279 +++++++++++++++++ rs/moq-video/examples/d3d11-resize-probe.rs | 29 +- rs/moq-video/examples/encode-presets.rs | 28 +- rs/moq-video/src/capture/wgc.rs | 4 +- rs/moq-video/src/decode/backend/mediacodec.rs | 74 +++-- .../src/decode/backend/mediafoundation.rs | 35 ++- rs/moq-video/src/decode/backend/mod.rs | 62 +++- rs/moq-video/src/decode/backend/nvdec.rs | 57 ++-- rs/moq-video/src/decode/backend/openh264.rs | 92 +++--- rs/moq-video/src/decode/backend/probe.rs | 105 +++++-- rs/moq-video/src/decode/backend/v4l2.rs | 69 ++-- rs/moq-video/src/decode/backend/vaapi.rs | 73 +++-- .../src/decode/backend/videotoolbox.rs | 34 +- rs/moq-video/src/decode/backend/vpx.rs | 34 +- rs/moq-video/src/decode/conformance.rs | 2 +- rs/moq-video/src/decode/consumer.rs | 27 +- rs/moq-video/src/decode/decoder.rs | 191 +++++++----- rs/moq-video/src/decode/sink.rs | 115 ++++--- rs/moq-video/src/decode/vui.rs | 2 +- rs/moq-video/src/encode/backend/mediacodec.rs | 61 ++-- .../src/encode/backend/mediafoundation.rs | 66 ++-- rs/moq-video/src/encode/backend/mod.rs | 68 ++-- rs/moq-video/src/encode/backend/nvenc.rs | 42 +-- rs/moq-video/src/encode/backend/openh264.rs | 44 +-- rs/moq-video/src/encode/backend/probe.rs | 30 +- rs/moq-video/src/encode/backend/v4l2.rs | 50 +-- rs/moq-video/src/encode/backend/vaapi.rs | 66 ++-- .../src/encode/backend/videotoolbox.rs | 50 +-- rs/moq-video/src/encode/encoder.rs | 295 ++++++++++-------- rs/moq-video/src/encode/producer.rs | 62 ++-- rs/moq-video/src/encode/sink.rs | 200 ++++++------ rs/moq-video/src/frame/cuda_test.rs | 14 +- rs/moq-video/src/render/export.rs | 4 +- rs/moq-video/src/render/export/dx12.rs | 4 +- rs/moq-video/src/render/export/metal.rs | 6 +- rs/moq-video/src/render/export/vulkan.rs | 4 +- rs/moq-video/src/render/renderer.rs | 12 +- rs/moq-video/src/worker.rs | 35 ++- swift/Sources/Moq/Video.swift | 14 +- 60 files changed, 2044 insertions(+), 1294 deletions(-) create mode 100644 rs/moq-video/benches/codec.rs diff --git a/Cargo.lock b/Cargo.lock index 18b6155f02b..c2300c271c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5084,6 +5084,7 @@ dependencies = [ "anyhow", "ash", "ashpd", + "bench-harness", "block2 0.6.2", "bytemuck", "bytes", diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index a8dba5dcbb4..5adcc89cc54 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -8,7 +8,9 @@ import "dart:ffi"; import "dart:io" show Platform, File, Directory; import "dart:isolate"; import "dart:typed_data"; + import "package:ffi/ffi.dart"; + import "uniffi_runtime.dart"; export "uniffi_runtime.dart"; @@ -16542,9 +16544,9 @@ class FfiConverterMoqVideoDecodedFrame { abstract class MoqVideoEncoderInterface { void cut(); - List encode({required MoqVideoFrame frame}); - List finish(); - List flush(); + void encode({required MoqVideoFrame frame}); + void flush(); + MoqVideoPacket? receive(); void setBitrate({required int bitrate}); } @@ -16607,36 +16609,32 @@ class MoqVideoEncoder implements MoqVideoEncoderInterface { }, moqExceptionErrorHandler); } - List encode({required MoqVideoFrame frame}) { - return rustCallWithLifter( - (status) => uniffi_moq_ffi_fn_method_moqvideoencoder_encode( + void encode({required MoqVideoFrame frame}) { + return rustCall((status) { + uniffi_moq_ffi_fn_method_moqvideoencoder_encode( uniffiClonePointer(), FfiConverterMoqVideoFrame.lower(frame), status, - ), - FfiConverterSequenceMoqVideoPacket.lift, - moqExceptionErrorHandler, - ); + ); + }, moqExceptionErrorHandler); } - List finish() { - return rustCallWithLifter( - (status) => uniffi_moq_ffi_fn_method_moqvideoencoder_finish( + void flush() { + return rustCall((status) { + uniffi_moq_ffi_fn_method_moqvideoencoder_flush( uniffiClonePointer(), status, - ), - FfiConverterSequenceMoqVideoPacket.lift, - moqExceptionErrorHandler, - ); + ); + }, moqExceptionErrorHandler); } - List flush() { + MoqVideoPacket? receive() { return rustCallWithLifter( - (status) => uniffi_moq_ffi_fn_method_moqvideoencoder_flush( + (status) => uniffi_moq_ffi_fn_method_moqvideoencoder_receive( uniffiClonePointer(), status, ), - FfiConverterSequenceMoqVideoPacket.lift, + FfiConverterOptionalMoqVideoPacket.lift, moqExceptionErrorHandler, ); } @@ -19103,6 +19101,53 @@ class FfiConverterOptionalMoqVideoHint { } } +class FfiConverterOptionalMoqVideoPacket { + static MoqVideoPacket? lift(RustBuffer buf) { + return FfiConverterOptionalMoqVideoPacket.read(buf.asUint8List()).value; + } + + static LiftRetVal read(Uint8List buf) { + if (ByteData.view(buf.buffer, buf.offsetInBytes).getInt8(0) == 0) { + return LiftRetVal(null, 1); + } + final result = FfiConverterMoqVideoPacket.read( + Uint8List.view(buf.buffer, buf.offsetInBytes + 1), + ); + return LiftRetVal(result.value, result.bytesRead + 1); + } + + static int allocationSize([MoqVideoPacket? value]) { + if (value == null) { + return 1; + } + return FfiConverterMoqVideoPacket.allocationSize(value) + 1; + } + + static RustBuffer lower(MoqVideoPacket? value) { + if (value == null) { + return toRustBuffer(Uint8List.fromList([0])); + } + final buf = Uint8List( + FfiConverterOptionalMoqVideoPacket.allocationSize(value), + ); + FfiConverterOptionalMoqVideoPacket.write(value, buf); + return toRustBuffer(buf); + } + + static int write(MoqVideoPacket? value, Uint8List buf) { + if (value == null) { + buf[0] = 0; + return 1; + } + buf[0] = 1; + return FfiConverterMoqVideoPacket.write( + value, + Uint8List.view(buf.buffer, buf.offsetInBytes + 1), + ) + + 1; + } +} + class FfiConverterOptionalMoqVideoProducer { static MoqVideoProducer? lift(RustBuffer buf) { return FfiConverterOptionalMoqVideoProducer.read(buf.asUint8List()).value; @@ -19602,51 +19647,6 @@ class FfiConverterSequenceMoqRange { } } -class FfiConverterSequenceMoqVideoPacket { - static List lift(RustBuffer buf) { - return FfiConverterSequenceMoqVideoPacket.read(buf.asUint8List()).value; - } - - static LiftRetVal> read(Uint8List buf) { - List res = []; - final length = buf.buffer.asByteData(buf.offsetInBytes).getInt32(0); - int offset = buf.offsetInBytes + 4; - for (var i = 0; i < length; i++) { - final ret = FfiConverterMoqVideoPacket.read( - Uint8List.view(buf.buffer, offset), - ); - offset += ret.bytesRead; - res.add(ret.value); - } - return LiftRetVal(res, offset - buf.offsetInBytes); - } - - static int write(List value, Uint8List buf) { - buf.buffer.asByteData(buf.offsetInBytes).setInt32(0, value.length); - int offset = buf.offsetInBytes + 4; - for (var i = 0; i < value.length; i++) { - offset += FfiConverterMoqVideoPacket.write( - value[i], - Uint8List.view(buf.buffer, offset), - ); - } - return offset - buf.offsetInBytes; - } - - static int allocationSize(List value) { - return value - .map((l) => FfiConverterMoqVideoPacket.allocationSize(l)) - .fold(0, (a, b) => a + b) + - 4; - } - - static RustBuffer lower(List value) { - final buf = Uint8List(allocationSize(value)); - write(value, buf); - return toRustBuffer(buf); - } -} - class FfiConverterSequenceString { static List lift(RustBuffer buf) { return FfiConverterSequenceString.read(buf.asUint8List()).value; @@ -23030,19 +23030,19 @@ external void uniffi_moq_ffi_fn_method_moqvideoencoder_cut( Pointer uniffiStatus, ); -@Native< - RustBuffer Function(Pointer, RustBuffer, Pointer) ->(assetId: _uniffiAssetId) -external RustBuffer uniffi_moq_ffi_fn_method_moqvideoencoder_encode( +@Native, RustBuffer, Pointer)>( + assetId: _uniffiAssetId, +) +external void uniffi_moq_ffi_fn_method_moqvideoencoder_encode( Pointer ptr, RustBuffer frame, Pointer uniffiStatus, ); -@Native, Pointer)>( +@Native, Pointer)>( assetId: _uniffiAssetId, ) -external RustBuffer uniffi_moq_ffi_fn_method_moqvideoencoder_finish( +external void uniffi_moq_ffi_fn_method_moqvideoencoder_flush( Pointer ptr, Pointer uniffiStatus, ); @@ -23050,7 +23050,7 @@ external RustBuffer uniffi_moq_ffi_fn_method_moqvideoencoder_finish( @Native, Pointer)>( assetId: _uniffiAssetId, ) -external RustBuffer uniffi_moq_ffi_fn_method_moqvideoencoder_flush( +external RustBuffer uniffi_moq_ffi_fn_method_moqvideoencoder_receive( Pointer ptr, Pointer uniffiStatus, ); @@ -24256,10 +24256,10 @@ external int uniffi_moq_ffi_checksum_method_moqvideoencoder_cut(); external int uniffi_moq_ffi_checksum_method_moqvideoencoder_encode(); @Native(assetId: _uniffiAssetId) -external int uniffi_moq_ffi_checksum_method_moqvideoencoder_finish(); +external int uniffi_moq_ffi_checksum_method_moqvideoencoder_flush(); @Native(assetId: _uniffiAssetId) -external int uniffi_moq_ffi_checksum_method_moqvideoencoder_flush(); +external int uniffi_moq_ffi_checksum_method_moqvideoencoder_receive(); @Native(assetId: _uniffiAssetId) external int uniffi_moq_ffi_checksum_method_moqvideoencoder_set_bitrate(); @@ -25164,13 +25164,13 @@ void _checkApiChecksums() { if (uniffi_moq_ffi_checksum_method_moqvideoencoder_cut() != 28527) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqvideoencoder_encode() != 513) { + if (uniffi_moq_ffi_checksum_method_moqvideoencoder_encode() != 42494) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqvideoencoder_finish() != 5918) { + if (uniffi_moq_ffi_checksum_method_moqvideoencoder_flush() != 24167) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqvideoencoder_flush() != 4585) { + if (uniffi_moq_ffi_checksum_method_moqvideoencoder_receive() != 33059) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqvideoencoder_set_bitrate() != 34315) { diff --git a/demo/carla/bridge/src/stream.rs b/demo/carla/bridge/src/stream.rs index 59df1ba1262..8c35d2b43a5 100644 --- a/demo/carla/bridge/src/stream.rs +++ b/demo/carla/bridge/src/stream.rs @@ -531,8 +531,8 @@ impl Encoding { if planned.keyframe { encoder.cut().context("request an IDR")?; } - let packets = encoder.encode(&frame).context("encode")?; - for packet in packets { + encoder.encode(&frame).context("encode")?; + while let Some(packet) = encoder.receive().context("encode")? { let keyframe = packet.keyframe; if !self.gate.admit(planned.index, keyframe) { continue; @@ -578,10 +578,10 @@ fn probe(config: &moq_video::encode::Config) -> Result { let luma = size.width as usize * size.height as usize; let gray = I420::new(size, vec![0x80; luma + luma / 2])?; let mut encoder = Encoder::new(config)?; - let mut packets = encoder.encode(&Frame::new(Surface::I420(gray), moq_net::Timestamp::from_micros(0)?))?; - packets.extend(encoder.finish()?); + encoder.encode(&Frame::new(Surface::I420(gray), moq_net::Timestamp::from_micros(0)?))?; + encoder.flush()?; let mut sets = Vec::new(); - for packet in &packets { + while let Some(packet) = encoder.receive()? { for nal in nals(&packet.payload)? { if matches!(nal.first().map(|header| header & 0x1f), Some(7 | 8)) { sets.push(nal); diff --git a/demo/carla/bridge/tests/gpu.rs b/demo/carla/bridge/tests/gpu.rs index ec7c8cbf9c9..2cc771d4541 100644 --- a/demo/carla/bridge/tests/gpu.rs +++ b/demo/carla/bridge/tests/gpu.rs @@ -152,10 +152,10 @@ impl Worker { let timestamp = moq_net::Timestamp::from_micros(frame.timestamp_us).unwrap(); // The decoder may hold a picture back; match pictures to // packets in order rather than one for one. - for picture in decoder + decoder .decode(&frame.payload, timestamp, frame.keyframe) - .expect("decode the packet") - { + .expect("decode the packet"); + while let Some(picture) = decoder.receive().expect("decode the packet") { let (number, rgb) = expected.pop_front().expect("more pictures than packets"); let i420 = picture.surface.to_i420().expect("CPU picture"); let size = SIZES[usize::from(frame.rendition)]; diff --git a/go/wrapper/video/video.go b/go/wrapper/video/video.go index d9a5b3e7647..f1b39716fd0 100644 --- a/go/wrapper/video/video.go +++ b/go/wrapper/video/video.go @@ -140,10 +140,11 @@ func NewEncoder(input EncoderInput, output EncoderOutput) (*Encoder, error) { return &Encoder{inner}, nil } -// Encode encodes one picture, returning any access units now available. -func (e *Encoder) Encode(value Frame) ([]Packet, error) { - return e.inner.Encode(value) -} +// Encode submits one picture, whose access units Receive returns. +func (e *Encoder) Encode(value Frame) error { return e.inner.Encode(value) } + +// Receive pops the oldest ready access unit, or nil while the codec holds the rest; call it until nil. +func (e *Encoder) Receive() (*Packet, error) { return e.inner.Receive() } // Cut forces the next encoded picture to start a new group. func (e *Encoder) Cut() error { return e.inner.Cut() } @@ -151,11 +152,11 @@ func (e *Encoder) Cut() error { return e.inner.Cut() } // SetBitrate retunes the live encoder in bits per second. func (e *Encoder) SetBitrate(value uint64) error { return e.inner.SetBitrate(value) } -// Flush drains buffered access units and keeps the encoder open. -func (e *Encoder) Flush() ([]Packet, error) { return e.inner.Flush() } +// Flush marks the end of input, so Receive returns every access unit the codec still holds, without closing the encoder. +func (e *Encoder) Flush() error { return e.inner.Flush() } -// Finish drains and closes the encoder. -func (e *Encoder) Finish() ([]Packet, error) { return e.inner.Finish() } +// Close releases the encoder, dropping any access unit not yet received. +func (e *Encoder) Close() { e.inner.Destroy() } // Consumer subscribes to a video rendition and yields decoded frames. type Consumer struct{ inner *ffi.MoqVideoConsumer } diff --git a/py/moq-rs/moq/video.py b/py/moq-rs/moq/video.py index 7eb2ed6b61a..1dc19087fb8 100644 --- a/py/moq-rs/moq/video.py +++ b/py/moq-rs/moq/video.py @@ -196,9 +196,14 @@ class Encoder: def __init__(self, input: EncoderInput, output: EncoderOutput) -> None: self._inner = MoqVideoEncoder(input, output) - def encode(self, frame: Frame) -> list[Packet]: - """Encode one picture, returning zero or more access units.""" - return [Packet._from_ffi(value) for value in self._inner.encode(frame._ffi())] + def encode(self, frame: Frame) -> None: + """Submit one picture, whose access units ``receive`` returns.""" + self._inner.encode(frame._ffi()) + + def receive(self) -> Packet | None: + """Pop the oldest ready access unit, or ``None`` while the codec holds the rest; call until ``None``.""" + value = self._inner.receive() + return None if value is None else Packet._from_ffi(value) def cut(self) -> None: """Start a group at the next encoded picture.""" @@ -208,13 +213,9 @@ def set_bitrate(self, bitrate: int) -> None: """Retune the live encoder in bits per second.""" self._inner.set_bitrate(bitrate) - def flush(self) -> list[Packet]: - """Drain buffered pictures while keeping the encoder open.""" - return [Packet._from_ffi(value) for value in self._inner.flush()] - - def finish(self) -> list[Packet]: - """Drain and close the encoder.""" - return [Packet._from_ffi(value) for value in self._inner.finish()] + def flush(self) -> None: + """Mark the end of input, so ``receive`` returns every access unit the codec still holds.""" + self._inner.flush() __all__ = [ diff --git a/py/moq-rs/tests/test_local.py b/py/moq-rs/tests/test_local.py index 5a91aaf5b66..838ff528a6b 100644 --- a/py/moq-rs/tests/test_local.py +++ b/py/moq-rs/tests/test_local.py @@ -1559,6 +1559,35 @@ def test_audio_encoder_returns_namespaced_records(): assert isinstance(finish, moq.audio.EncoderFinish) +def test_video_encoder_submits_and_receives(): + """The codec-only video encoder takes pictures and hands back access units one at a time.""" + encoder = moq.video.Encoder( + moq.video.EncoderInput( + format=moq.video.PixelFormat.I420, + width=64, + height=48, + framerate=moq.video.FrameRate(numerator=30, denominator=1), + ), + moq.video.EncoderOutput( + codec=moq.video.Codec.H264, + kind=cast(moq.video.EncoderKind, moq.video.EncoderKind.SOFTWARE()), + ), + ) + packets: list[moq.video.Packet] = [] + picture = bytes([0x80]) * (64 * 48 * 3 // 2) + for i in range(3): + if i == 2: + encoder.cut() + encoder.encode(moq.video.Frame(timestamp=moq.Timestamp(i * 33_333, 1_000_000), data=picture)) + while (packet := encoder.receive()) is not None: + packets.append(packet) + encoder.flush() + while (packet := encoder.receive()) is not None: + packets.append(packet) + assert [packet.keyframe for packet in packets] == [True, False, True] + assert all(packet.data for packet in packets) + + def test_audio_encoder_frame_init_and_bitrate(): """A frame duration sets the frame size, the encoder reports its OpusHead, and the bitrate retunes live.""" output = moq.audio.EncoderOutput( diff --git a/rs/moq-boy/src/video.rs b/rs/moq-boy/src/video.rs index 87fa1b0efd7..9b164a7ec7e 100644 --- a/rs/moq-boy/src/video.rs +++ b/rs/moq-boy/src/video.rs @@ -169,17 +169,26 @@ fn encoder_thread( continue; } }; - match enc.encode(&moq_video::Frame::new(surface, ts)) { - Ok(encoded) => { - if let Err(e) = producer.publish(&encoded) { - // Publish only fails once the track/broadcast is gone, which - // is terminal -- stop rather than flooding logs every frame. - tracing::error!(error = %e, "video publish failed; stopping encoder"); - return; + // A single bad frame is tolerable; keep going. + if let Err(e) = enc.encode(&moq_video::Frame::new(surface, ts)) { + tracing::error!(error = %e, "H.264 encode error"); + } + loop { + match enc.receive() { + Ok(Some(encoded)) => { + if let Err(e) = producer.publish(&encoded) { + // Publish only fails once the track/broadcast is gone, which + // is terminal -- stop rather than flooding logs every frame. + tracing::error!(error = %e, "video publish failed; stopping encoder"); + return; + } + } + Ok(None) => break, + Err(e) => { + tracing::error!(error = %e, "H.264 encode error"); + break; } } - // A single bad frame is tolerable; keep going. - Err(e) => tracing::error!(error = %e, "H.264 encode error"), } encode_duration.store(start.elapsed().as_micros() as u64, Ordering::Relaxed); } diff --git a/rs/moq-c/README.md b/rs/moq-c/README.md index 2725f617627..1bff57a27e5 100644 --- a/rs/moq-c/README.md +++ b/rs/moq-c/README.md @@ -209,11 +209,10 @@ int32_t moq_audio_encoder_packet(uint32_t encoder, uint32_t index, moq_frame *ds int32_t moq_audio_encoder_free(uint32_t encoder); int32_t moq_video_encoder_create(const moq_video_encoder_input *input, const moq_video_encoder_output *output); int32_t moq_video_encoder_encode(uint32_t encoder, const moq_video_encoder_frame *frame); +int32_t moq_video_encoder_receive(uint32_t encoder, moq_frame *dst); int32_t moq_video_encoder_cut(uint32_t encoder); int32_t moq_video_encoder_bitrate(uint32_t encoder, uint64_t bitrate); int32_t moq_video_encoder_flush(uint32_t encoder); -int32_t moq_video_encoder_finish(uint32_t encoder); -int32_t moq_video_encoder_packet(uint32_t encoder, uint32_t index, moq_frame *dst); int32_t moq_video_encoder_free(uint32_t encoder); ``` diff --git a/rs/moq-c/src/encoder.rs b/rs/moq-c/src/encoder.rs index d8f43af0cf9..5fad0a8ea12 100644 --- a/rs/moq-c/src/encoder.rs +++ b/rs/moq-c/src/encoder.rs @@ -1,9 +1,10 @@ //! Codec-only encoders, independent of any broadcast. //! -//! Each call that produces packets keeps them on the encoder handle, readable by index with -//! [moq_audio_encoder_packet] or [moq_video_encoder_packet] until the next such call or the -//! handle is freed; [moq_audio_encoder_receive] keeps the one packet it writes the same way. Publish them with [crate::moq_publish_audio] or [crate::moq_publish_video] -//! and [crate::moq_publish_media_frame]. +//! Encoding submits a frame; [moq_audio_encoder_receive] and [moq_video_encoder_receive] pop +//! one packet at a time, which the encoder handle keeps until its next call or until it is +//! freed. [moq_audio_encoder_finish] keeps its packets the same way, readable by index with +//! [moq_audio_encoder_packet]. Publish them with [crate::moq_publish_audio] or +//! [crate::moq_publish_video] and [crate::moq_publish_media_frame]. use std::time::Duration; @@ -81,31 +82,11 @@ impl AudioEncoder { } struct VideoEncoder { - /// `None` once finished; the packets it drained stay readable. - encoder: Option, + encoder: moq_video::encode::Sink, format: moq_video_pixel_format, size: moq_video::Size, - packets: Vec, -} - -impl VideoEncoder { - fn encoder(&mut self) -> Result<&mut moq_video::encode::Sink, Error> { - self.encoder - .as_mut() - .ok_or_else(|| Error::InvalidConfig("the encoder finished".into())) - } - - fn keep(&mut self, packets: Vec) -> Result { - self.packets = packets - .into_iter() - .map(|packet| Packet { - payload: packet.payload, - timestamp: Some(Box::new(packet.timestamp.into())), - keyframe: packet.keyframe, - }) - .collect(); - count(&self.packets) - } + /// The last received access unit, owning what [moq_video_encoder_receive] lent out. + packet: Option, } /// Run `f` on an encoder resolved under the global lock, then locked on its own. @@ -359,19 +340,18 @@ pub unsafe extern "C" fn moq_video_encoder_create( // Opened before the global lock is taken: bringing up a hardware encoder is slow. let encoder = crate::video::block_on(moq_video::encode::Sink::open(&config))?; State::lock().encoders.video.insert(Shared::new(VideoEncoder { - encoder: Some(encoder), + encoder, format, size: config.size(), - packets: Vec::new(), + packet: None, })) }) } /// Encode one raw picture in the input's layout and resolution. /// -/// Returns the number of access units now available (zero or more, since a backend that -/// pipelines hands back an earlier picture's), read with [moq_video_encoder_packet], or a -/// negative code on failure. +/// Returns zero, or a negative code on failure. Read its access units with +/// [moq_video_encoder_receive], which a pipelining backend readies only after later pictures. /// /// # Safety /// - `frame` must point at a valid [moq_video_encoder_frame] whose `data` holds `data_size` bytes. @@ -389,8 +369,8 @@ pub unsafe extern "C" fn moq_video_encoder_encode(encoder: u32, frame: *const mo moq_video_pixel_format::MOQ_VIDEO_PIXEL_FORMAT_RGBA => moq_video::Surface::rgba(data, video.size)?, }; let frame = moq_video::Frame::new(surface, timestamp); - let packets = crate::video::block_on(video.encoder()?.encode(frame))?; - video.keep(packets) + crate::video::block_on(video.encoder.encode(frame))?; + Ok(0) }) }) } @@ -400,7 +380,7 @@ pub unsafe extern "C" fn moq_video_encoder_encode(encoder: u32, frame: *const mo pub extern "C" fn moq_video_encoder_cut(encoder: u32) -> i32 { ffi::enter(move || { with(encoder, video, |video| { - crate::video::block_on(video.encoder()?.cut())?; + crate::video::block_on(video.encoder.cut())?; Ok(()) }) }) @@ -411,63 +391,54 @@ pub extern "C" fn moq_video_encoder_cut(encoder: u32) -> i32 { pub extern "C" fn moq_video_encoder_bitrate(encoder: u32, bitrate: u64) -> i32 { ffi::enter(move || { with(encoder, video, |video| { - crate::video::block_on( - video - .encoder()? - .set_bitrate(moq_net::bandwidth::Rate::from_bps(bitrate)), - )?; + crate::video::block_on(video.encoder.set_bitrate(moq_net::bandwidth::Rate::from_bps(bitrate)))?; Ok(()) }) }) } -/// Drain the access units the codec still holds, keeping the encoder open. +/// Pop the oldest ready access unit into `dst`; call until it returns zero after each +/// [moq_video_encoder_encode] or [moq_video_encoder_flush]. +/// +/// Returns one when it wrote an access unit, zero while the codec holds the rest, or a negative +/// code. `payload` and `timestamp` borrow the encoder until its next call or +/// [moq_video_encoder_free]; `keyframe` says whether the unit starts from a keyframe. /// -/// Returns their count, read with [moq_video_encoder_packet], or a negative code. +/// # Safety +/// - `dst` must point at a writable [moq_frame]. #[unsafe(no_mangle)] -pub extern "C" fn moq_video_encoder_flush(encoder: u32) -> i32 { +pub unsafe extern "C" fn moq_video_encoder_receive(encoder: u32, dst: *mut moq_frame) -> i32 { ffi::enter(move || { + let dst = unsafe { dst.as_mut() }.ok_or(Error::InvalidPointer)?; with(encoder, video, |video| { - let packets = crate::video::block_on(video.encoder()?.flush())?; - video.keep(packets) + let Some(ready) = video.encoder.receive()? else { + return Ok(0); + }; + let packet = video.packet.insert(Packet { + payload: ready.payload, + timestamp: Some(Box::new(ready.timestamp.into())), + keyframe: ready.keyframe, + }); + *dst = packet.info(); + Ok(1) }) }) } -/// Drain and close the encoder. +/// Mark the end of input, so [moq_video_encoder_receive] returns every access unit the codec +/// still holds. The encoder stays open: the next picture starts new input. /// -/// Returns the number of access units drained, read with [moq_video_encoder_packet], or a -/// negative code. Nothing more can be encoded; free the handle once the packets are read. +/// Returns zero, or a negative code. #[unsafe(no_mangle)] -pub extern "C" fn moq_video_encoder_finish(encoder: u32) -> i32 { +pub extern "C" fn moq_video_encoder_flush(encoder: u32) -> i32 { ffi::enter(move || { with(encoder, video, |video| { - let encoder = video - .encoder - .take() - .ok_or_else(|| Error::InvalidConfig("the encoder finished".into()))?; - let packets = crate::video::block_on(encoder.finish())?; - video.keep(packets) + crate::video::block_on(video.encoder.flush())?; + Ok(0) }) }) } -/// Read access unit `index` from the last encode, flush, or finish. -/// -/// `payload` and `timestamp` borrow the encoder until its next call or -/// [moq_video_encoder_free]; `keyframe` says whether the unit starts from a keyframe. Returns -/// zero, or a negative code, `MOQ_ERROR_NO_INDEX` past the last unit. -/// -/// # Safety -/// - `dst` must point at a writable [moq_frame]. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn moq_video_encoder_packet(encoder: u32, index: u32, dst: *mut moq_frame) -> i32 { - ffi::enter(move || { - let dst = unsafe { dst.as_mut() }.ok_or(Error::InvalidPointer)?; - with(encoder, video, |video| packet(&video.packets, index, dst)) - }) -} - /// Close a codec-only video encoder and release its access units. #[unsafe(no_mangle)] pub extern "C" fn moq_video_encoder_free(encoder: u32) -> i32 { diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index b90e01e91ae..a720df524a6 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -6962,9 +6962,15 @@ fn video_raw_decode() { } let surface = moq_video::Surface::rgba(&gray, moq_video::Size::new(320, 240)).unwrap(); let frame = moq_video::Frame::new(surface, moq_net::Timestamp::from_micros(i * 33_333).unwrap()); - frames.extend(encoder.encode(&frame).unwrap()); + encoder.encode(&frame).unwrap(); + while let Some(encoded) = encoder.receive().unwrap() { + frames.push(encoded); + } + } + encoder.flush().unwrap(); + while let Some(encoded) = encoder.receive().unwrap() { + frames.push(encoded); } - frames.extend(encoder.finish().unwrap()); assert!(!frames.is_empty(), "encoder produced no frames"); let origin = id(moq_origin_create()); diff --git a/rs/moq-c/src/test/parity.rs b/rs/moq-c/src/test/parity.rs index 1fdafea9446..69f482acf66 100644 --- a/rs/moq-c/src/test/parity.rs +++ b/rs/moq-c/src/test/parity.rs @@ -648,28 +648,36 @@ fn video_encoder_without_a_broadcast() { }; let encoder = id(unsafe { moq_video_encoder_create(&input, &output) }); let rgba = gray_rgba(320, 240); - let mut packets = 0; + let mut packet: moq_frame = unsafe { std::mem::zeroed() }; + let mut keyframes = Vec::new(); + let mut receive = |keyframes: &mut Vec| loop { + match unsafe { moq_video_encoder_receive(encoder, &mut packet) } { + 0 => break, + 1 => { + assert!(packet.payload_size > 0); + assert!(!packet.timestamp.is_null(), "an access unit carries its timestamp"); + keyframes.push(packet.keyframe); + } + code => panic!("receive failed: {code}"), + } + }; for index in 0..5 { + if index == 3 { + assert_eq!(moq_video_encoder_cut(encoder), 0); + } let frame = moq_video_encoder_frame { timestamp: micros(index * 33_333), data: rgba.as_ptr(), data_size: rgba.len(), }; - let count = unsafe { moq_video_encoder_encode(encoder, &frame) }; - assert!(count >= 0, "encode failed: {count}"); - packets += count; - } - assert_eq!(moq_video_encoder_cut(encoder), 0); - let drained = moq_video_encoder_finish(encoder); - assert!(drained >= 0, "finish failed: {drained}"); - assert!(packets + drained > 0, "the encoder produced access units"); - if drained > 0 { - let mut packet: moq_frame = unsafe { std::mem::zeroed() }; - assert_eq!(unsafe { moq_video_encoder_packet(encoder, 0, &mut packet) }, 0); - assert!(packet.payload_size > 0); + assert_eq!(unsafe { moq_video_encoder_encode(encoder, &frame) }, 0); + receive(&mut keyframes); } - assert!(moq_video_encoder_finish(encoder) < 0, "finishing twice fails"); + assert_eq!(moq_video_encoder_flush(encoder), 0); + receive(&mut keyframes); + assert_eq!(keyframes, [true, false, false, true, false], "keyframes at the start and the cut"); assert_eq!(moq_video_encoder_free(encoder), 0); + assert!(moq_video_encoder_free(encoder) < 0); } /// Only Apple decoders hand out a native surface; elsewhere asking for one is refused up front. diff --git a/rs/moq-c/src/video.rs b/rs/moq-c/src/video.rs index 595444af404..950be0547ae 100644 --- a/rs/moq-c/src/video.rs +++ b/rs/moq-c/src/video.rs @@ -644,9 +644,9 @@ impl VideoEncoder { let frame = moq_video::Frame::new(surface, timestamp); // A backend that pipelines hands back an earlier frame's output, so this is // zero or more access units rather than one per call. - let encoded = block_on(encoder.encode(frame))?; + block_on(encoder.encode(frame))?; let before = self.producer.demand(); - let published = self.producer.publish(&encoded); + let published = publish(encoder, &mut self.producer); // A frame whose decoder identity changed, such as a resume at a ceiling past the declared // level, moved the rendition to a newly named track, whose demand the claim has to watch. // The claim moves even when publishing on the new track failed, so it never stays on the @@ -702,7 +702,7 @@ impl VideoEncoder { if let Some(encoder) = self.encoder.take() { self.target = moq_mux::rate::Target::Disabled; // The encoder is gone either way, so a lost tail only costs those frames. - let drained = block_on(encoder.finish()).and_then(|encoded| self.producer.publish(&encoded)); + let drained = drain(encoder, &mut self.producer); if let Err(err) = drained { tracing::warn!(error = %err, "failed to drain a disabled rendition's encoder"); } @@ -756,7 +756,7 @@ impl VideoEncoder { // Drain the codec into the track before ending it, so the last frames land // in it rather than being dropped with the encoder. let drained = match encoder { - Some(encoder) => block_on(encoder.finish()).and_then(|encoded| producer.publish(&encoded)), + Some(encoder) => drain(encoder, &mut producer), None => Ok(()), }; finalize(producer, drained, &children, crate::publish::Child::Video(id))?; @@ -766,6 +766,26 @@ impl VideoEncoder { } } +/// Publish every access unit `encoder` has ready. +fn publish( + encoder: &mut moq_video::encode::Sink, + producer: &mut moq_video::encode::Producer, +) -> Result<(), moq_video::Error> { + while let Some(encoded) = encoder.receive()? { + producer.publish(&encoded)?; + } + Ok(()) +} + +/// Flush `encoder` and publish its tail, closing it. +fn drain( + mut encoder: moq_video::encode::Sink, + producer: &mut moq_video::encode::Producer, +) -> Result<(), moq_video::Error> { + block_on(encoder.flush())?; + publish(&mut encoder, producer) +} + impl Video { /// Advertise a track for an already-opened encoder. /// diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 59235ddcb5a..9c6b71f1c0a 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -1139,16 +1139,20 @@ mod tests { /// a stalled speaker never holds the picture. #[tokio::test] async fn stalled_audio_no_longer_holds_video() { - struct Passthrough; + struct Passthrough(Option); impl super::super::video::Decoder for Passthrough { - async fn decode(&mut self, frame: moq_mux::container::Frame) -> anyhow::Result> { - Ok(vec![moq_video::Frame::new( + async fn decode(&mut self, frame: moq_mux::container::Frame) -> anyhow::Result<()> { + self.0 = Some(moq_video::Frame::new( moq_video::Surface::rgba(&[128; 16 * 16 * 4], moq_video::Size::new(16, 16))?, frame.timestamp, - )]) + )); + Ok(()) + } + async fn flush(&mut self) -> anyhow::Result<()> { + Ok(()) } - async fn flush(&mut self) -> anyhow::Result> { - Ok(Vec::new()) + fn receive(&mut self) -> anyhow::Result> { + Ok(self.0.take()) } } tokio::time::pause(); @@ -1208,7 +1212,7 @@ mod tests { }); let video_task = tokio::spawn(video_play.run( moq_mux::container::Consumer::new(subscriber, Container::Legacy(moq_mux::container::Kind::Video)), - Passthrough, + Passthrough(None), None, )); let mut shown = Vec::new(); @@ -1262,19 +1266,24 @@ mod tests { } async fn quick_resumes(audio_end: bool, video_end: bool, audio_latency: Duration) { - struct Buffered(Vec); + struct Buffered(Vec, std::collections::VecDeque); impl super::super::video::Decoder for Buffered { - async fn decode(&mut self, frame: moq_mux::container::Frame) -> anyhow::Result> { + async fn decode(&mut self, frame: moq_mux::container::Frame) -> anyhow::Result<()> { self.0.push(moq_video::Frame::new( moq_video::Surface::rgba(&[128; 16 * 16 * 4], moq_video::Size::new(16, 16))?, frame.timestamp, )); self.0.sort_by_key(|frame| frame.timestamp); let ready = self.0.len().saturating_sub(2); - Ok(self.0.drain(..ready).collect()) + self.1.extend(self.0.drain(..ready)); + Ok(()) + } + async fn flush(&mut self) -> anyhow::Result<()> { + self.1.extend(self.0.drain(..)); + Ok(()) } - async fn flush(&mut self) -> anyhow::Result> { - Ok(std::mem::take(&mut self.0)) + fn receive(&mut self) -> anyhow::Result> { + Ok(self.1.pop_front()) } } tokio::time::pause(); @@ -1338,7 +1347,7 @@ mod tests { }); let video_task = tokio::spawn(video_play.run( moq_mux::container::Consumer::new(subscriber, Container::Legacy(moq_mux::container::Kind::Video)), - Buffered(Vec::new()), + Buffered(Vec::new(), Default::default()), None, )); let picture = |index: u64, keyframe: bool| moq_mux::container::Frame { @@ -1656,16 +1665,20 @@ mod tests { #[tokio::test] async fn an_evicted_replacement_gop_gives_way_to_the_next_replacement() { - struct Passthrough; + struct Passthrough(Option); impl super::super::video::Decoder for Passthrough { - async fn decode(&mut self, frame: moq_mux::container::Frame) -> anyhow::Result> { - Ok(vec![moq_video::Frame::new( + async fn decode(&mut self, frame: moq_mux::container::Frame) -> anyhow::Result<()> { + self.0 = Some(moq_video::Frame::new( moq_video::Surface::rgba(&[128; 16 * 16 * 4], moq_video::Size::new(16, 16))?, frame.timestamp, - )]) + )); + Ok(()) + } + async fn flush(&mut self) -> anyhow::Result<()> { + Ok(()) } - async fn flush(&mut self) -> anyhow::Result> { - Ok(Vec::new()) + fn receive(&mut self) -> anyhow::Result> { + Ok(self.0.take()) } } tokio::time::pause(); @@ -1696,7 +1709,7 @@ mod tests { }; let video_task = tokio::spawn(video_play.run( moq_mux::container::Consumer::new(subscriber, Container::Legacy(moq_mux::container::Kind::Video)), - Passthrough, + Passthrough(None), None, )); let picture = |millis: u64, keyframe: bool| moq_mux::container::Frame { @@ -2388,7 +2401,8 @@ mod tests { for index in 0..=60 { let surface = moq_video::Surface::rgba(&vec![128; 64 * 64 * 4], moq_video::Size::new(64, 64)).unwrap(); let frame = moq_video::Frame::new(surface, moq_net::Timestamp::from_millis(index * 33).unwrap()); - for encoded in encoder.encode(&frame).unwrap() { + encoder.encode(&frame).unwrap(); + while let Some(encoded) = encoder.receive().unwrap() { producer .write(moq_mux::container::Frame { timestamp: encoded.timestamp, diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 4d990ea3c40..67008c908f5 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -70,18 +70,26 @@ pub(super) struct Refresh { /// The production sink and the deterministic buffered decoder used by tests. pub(super) trait Decoder { - async fn decode(&mut self, frame: Frame) -> anyhow::Result>; - async fn flush(&mut self) -> anyhow::Result>; + /// Submit one frame, whose pictures `receive` returns. + async fn decode(&mut self, frame: Frame) -> anyhow::Result<()>; + /// Mark the end of input, so `receive` returns every picture the codec holds. + async fn flush(&mut self) -> anyhow::Result<()>; + /// Pop the next ready picture, in display order. + fn receive(&mut self) -> anyhow::Result>; } impl Decoder for moq_video::decode::Sink { - async fn decode(&mut self, frame: Frame) -> anyhow::Result> { + async fn decode(&mut self, frame: Frame) -> anyhow::Result<()> { Ok(self.decode(frame.payload, frame.timestamp, frame.keyframe).await?) } - async fn flush(&mut self) -> anyhow::Result> { + async fn flush(&mut self) -> anyhow::Result<()> { Ok(self.flush().await?) } + + fn receive(&mut self) -> anyhow::Result> { + Ok(self.receive()?) + } } /// One encoder segment's media, and the transition it resumes when it @@ -404,7 +412,7 @@ impl Video { } } }; - let frames = match step { + match step { Step::Wait(Some(at)) => { tokio::select! { _ = kio::clock::sleep_until(at) => {}, @@ -434,7 +442,7 @@ impl Video { wake.notify_one(); // Never race a codec call: cancelling Sink::decode poisons it. The // joined receiver continues observing arrivals during this await. - decoder.decode(frame).await? + decoder.decode(frame).await?; } Step::Resume => { // Read under the same lock as the pop: a gap replaces the buffers and @@ -486,22 +494,20 @@ impl Video { Step::Flush => { let flushed = generation.take(); held.clear(); - let frames = decoder.flush().await?; + decoder.flush().await?; // A gap during the await already discarded these pictures' generation. if flushed != Some(buffers.lock().unwrap().front().unwrap().generation) { + while decoder.receive()?.is_some() {} continue; } - frames } - }; - // A codec returns display order, so a picture held before the newest - // one it returned was dropped rather than delayed. - if let Some(last) = frames.iter().map(|frame| frame.timestamp).max() { - held.retain(|held| *held > last); } let now = Instant::now(); let mut selector = selector.lock().unwrap(); - for frame in frames { + while let Some(frame) = decoder.receive()? { + // A codec returns display order, so a picture held before one it + // returned was dropped rather than delayed. + held.retain(|held| *held > frame.timestamp); selector.push(Lane::Min, Decoded::Frame(frame), now); } }; @@ -518,14 +524,15 @@ impl Video { else { return std::future::pending::>().await; }; - let push = |frames: Vec| { + let push = |codec: &mut D| -> anyhow::Result<()> { let now = Instant::now(); let mut selector = selector.lock().unwrap(); - for frame in frames { + while let Some(frame) = codec.receive()? { selector.push(Lane::Max, Decoded::Frame(frame), now); } drop(selector); self.changed.notify_one(); + Ok(()) }; // After a join or a loss, pictures presenting before the first recovery // point reference media this decoder never had. @@ -538,18 +545,21 @@ impl Video { let (group, mut frame) = match refresh.read().await? { Some(Read::Frame { group, frame }) => (group, frame), Some(Read::Gap { .. }) => { - push(codec.flush().await?); + codec.flush().await?; + push(&mut codec)?; recover = true; continue; } // A fresh encoder has no preceding GOP to reference, so no recovery. Some(Read::End { group, .. }) => { - push(codec.flush().await?); + codec.flush().await?; + push(&mut codec)?; fence = Some(group); continue; } None => { - push(codec.flush().await?); + codec.flush().await?; + push(&mut codec)?; selector.lock().unwrap().finish(Lane::Max); self.changed.notify_one(); return std::future::pending::>().await; @@ -619,7 +629,8 @@ impl Video { }; frame.timestamp = mapped; // Never race a codec call: cancelling Sink::decode poisons it. - push(codec.decode(frame).await?); + codec.decode(frame).await?; + push(&mut codec)?; } }; let decode = async { @@ -653,6 +664,7 @@ mod tests { /// How long each decode takes. cost: Duration, held: Vec, + ready: VecDeque, decoded: Arc>>, flushed: Arc>, } @@ -670,6 +682,7 @@ mod tests { size: MIN, cost: Duration::ZERO, held: Vec::new(), + ready: VecDeque::new(), decoded: Default::default(), flushed: Default::default(), } @@ -690,7 +703,7 @@ mod tests { } impl Decoder for Buffered { - async fn decode(&mut self, frame: Frame) -> anyhow::Result> { + async fn decode(&mut self, frame: Frame) -> anyhow::Result<()> { if !self.cost.is_zero() { kio::clock::sleep(self.cost).await; } @@ -701,12 +714,18 @@ mod tests { self.held.push(moq_video::Frame::new(surface, frame.timestamp)); self.held.sort_by_key(|frame| frame.timestamp); let bumped = self.held.len().saturating_sub(self.depth); - Ok(self.held.drain(..bumped).collect()) + self.ready.extend(self.held.drain(..bumped)); + Ok(()) } - async fn flush(&mut self) -> anyhow::Result> { + async fn flush(&mut self) -> anyhow::Result<()> { *self.flushed.lock().unwrap() += 1; - Ok(std::mem::take(&mut self.held)) + self.ready.extend(self.held.drain(..)); + Ok(()) + } + + fn receive(&mut self) -> anyhow::Result> { + Ok(self.ready.pop_front()) } } diff --git a/rs/moq-ffi/src/video.rs b/rs/moq-ffi/src/video.rs index 52db4e462f0..31304775dce 100644 --- a/rs/moq-ffi/src/video.rs +++ b/rs/moq-ffi/src/video.rs @@ -190,15 +190,14 @@ pub struct MoqVideoPacket { pub keyframe: bool, } -fn video_packets(packets: Vec) -> Vec { - packets - .into_iter() - .map(|packet| MoqVideoPacket { +impl From for MoqVideoPacket { + fn from(packet: moq_video::encode::Encoded) -> Self { + Self { timestamp: packet.timestamp.into(), data: packet.payload.to_vec(), keyframe: packet.keyframe, - }) - .collect() + } + } } fn video_encoder_config( @@ -217,9 +216,13 @@ fn video_encoder_config( } /// A codec-only video encoder, independent of any broadcast. +/// +/// Submit pictures with [`encode`](Self::encode) and pop access units with +/// [`receive`](Self::receive) until it returns none. At the end of input, +/// [`flush`](Self::flush) and receive the rest before releasing it. #[derive(uniffi::Object)] pub struct MoqVideoEncoder { - inner: std::sync::Mutex>, + inner: std::sync::Mutex, format: MoqVideoPixelFormat, size: moq_video::Size, } @@ -232,62 +235,71 @@ impl MoqVideoEncoder { let config = video_encoder_config(&input, &output)?; let encoder = block_on(moq_video::encode::Sink::open(&config))?; Ok(Arc::new(Self { - inner: std::sync::Mutex::new(Some(encoder)), + inner: std::sync::Mutex::new(encoder), format: input.format, size: config.size(), })) } - /// Encode one raw picture and return any access units now available. - pub fn encode(&self, frame: MoqVideoFrame) -> Result, MoqError> { + /// Submit one raw picture, whose access units `receive` returns. + pub fn encode(&self, frame: MoqVideoFrame) -> Result<(), MoqError> { let surface = match self.format { MoqVideoPixelFormat::I420 => moq_video::Surface::I420(moq_video::I420::new(self.size, frame.data)?), MoqVideoPixelFormat::Rgba => moq_video::Surface::rgba(&frame.data, self.size)?, }; let frame = moq_video::Frame::new(surface, frame.timestamp.try_into()?); let mut guard = self.inner.lock().map_err(|_| panic_error())?; - Ok(video_packets(block_on( - guard.as_mut().ok_or(MoqError::Closed)?.encode(frame), - )?)) + block_on(guard.encode(frame))?; + Ok(()) + } + + /// Pop the oldest ready access unit, or none while the codec holds the rest; call until none. + pub fn receive(&self) -> Result, MoqError> { + let mut guard = self.inner.lock().map_err(|_| panic_error())?; + Ok(guard.receive()?.map(MoqVideoPacket::from)) } /// Force the next encoded picture to start a new group. pub fn cut(&self) -> Result<(), MoqError> { let mut guard = self.inner.lock().map_err(|_| panic_error())?; - block_on(guard.as_mut().ok_or(MoqError::Closed)?.cut())?; + block_on(guard.cut())?; Ok(()) } /// Retune the live encoder in bits per second. pub fn set_bitrate(&self, bitrate: u64) -> Result<(), MoqError> { let mut guard = self.inner.lock().map_err(|_| panic_error())?; - block_on( - guard - .as_mut() - .ok_or(MoqError::Closed)? - .set_bitrate(moq_net::bandwidth::Rate::from_bps(bitrate)), - )?; + block_on(guard.set_bitrate(moq_net::bandwidth::Rate::from_bps(bitrate)))?; Ok(()) } - /// Drain buffered access units while keeping the encoder open. - pub fn flush(&self) -> Result, MoqError> { + /// Mark the end of input, so `receive` returns every access unit the codec still + /// holds. The encoder stays open: the next picture starts new input. + pub fn flush(&self) -> Result<(), MoqError> { let mut guard = self.inner.lock().map_err(|_| panic_error())?; - Ok(video_packets(block_on( - guard.as_mut().ok_or(MoqError::Closed)?.flush(), - )?)) + block_on(guard.flush())?; + Ok(()) } +} - /// Drain and close the encoder. - pub fn finish(&self) -> Result, MoqError> { - let encoder = self - .inner - .lock() - .map_err(|_| panic_error())? - .take() - .ok_or(MoqError::Closed)?; - Ok(video_packets(block_on(encoder.finish())?)) +/// Publish every access unit `encoder` has ready. +fn publish( + encoder: &mut moq_video::encode::Sink, + producer: &mut moq_video::encode::Producer, +) -> Result<(), moq_video::Error> { + while let Some(encoded) = encoder.receive()? { + producer.publish(&encoded)?; } + Ok(()) +} + +/// Flush `encoder` and publish its tail, closing it. +fn drain( + mut encoder: moq_video::encode::Sink, + producer: &mut moq_video::encode::Producer, +) -> Result<(), moq_video::Error> { + block_on(encoder.flush())?; + publish(&mut encoder, producer) } /// End a video track, given the result of draining its encoder into it. @@ -474,8 +486,8 @@ impl VideoProducer { let frame = moq_video::Frame::new(surface, frame.timestamp.try_into()?); // A backend that pipelines hands back an earlier frame's output, so this is // zero or more access units rather than one per call. - let encoded = block_on(encoder.encode(frame))?; - self.producer.publish(&encoded)?; + block_on(encoder.encode(frame))?; + publish(encoder, &mut self.producer)?; Ok(()) } @@ -516,7 +528,7 @@ impl VideoProducer { if let Some(encoder) = self.encoder.take() { self.target = moq_mux::rate::Target::Disabled; // The encoder is gone either way, so a lost tail only costs those frames. - let drained = block_on(encoder.finish()).and_then(|encoded| self.producer.publish(&encoded)); + let drained = drain(encoder, &mut self.producer); if let Err(err) = drained { tracing::warn!(error = %err, "failed to drain a disabled rendition's encoder"); } @@ -570,7 +582,7 @@ impl VideoProducer { // Drain the codec into the track before ending it, so the last frames land // in it rather than being dropped with the encoder. let drained = match encoder { - Some(encoder) => block_on(encoder.finish()).and_then(|encoded| producer.publish(&encoded)), + Some(encoder) => drain(encoder, &mut producer), None => Ok(()), }; (finalize(producer, drained, children), follower) @@ -1457,7 +1469,7 @@ mod finish_tests { /// The codec-only encoder every wrapper exposes (`moq.video.Encoder` and /// friends): a keyframe opens the stream, `cut` forces the next one, and - /// `finish` closes it. No test in any language had driven it. + /// `flush` readies the tail without closing it. #[test] fn encoder_emits_keyframes_where_asked() { let (width, height) = (64u32, 48u32); @@ -1490,20 +1502,28 @@ mod finish_tests { }; let mut packets = Vec::new(); + let receive = |packets: &mut Vec| { + while let Some(packet) = encoder.receive().unwrap() { + packets.push(packet); + } + }; for i in 0..3 { - packets.extend(encoder.encode(picture(i)).unwrap()); + encoder.encode(picture(i)).unwrap(); + receive(&mut packets); } - packets.extend(encoder.flush().unwrap()); + encoder.flush().unwrap(); + receive(&mut packets); encoder.set_bitrate(250_000).unwrap(); encoder.cut().unwrap(); - packets.extend(encoder.encode(picture(3)).unwrap()); - packets.extend(encoder.finish().unwrap()); + encoder.encode(picture(3)).unwrap(); + encoder.flush().unwrap(); + receive(&mut packets); let keyframes: Vec = packets.iter().filter(|p| p.keyframe).map(|p| p.timestamp).collect(); assert_eq!(packets.len(), 4, "one access unit per picture"); assert_eq!(keyframes, [ntsc(0), ntsc(3)], "keyframes at the start and the cut"); assert!(packets.iter().all(|p| !p.data.is_empty()), "an empty access unit"); - assert!(matches!(encoder.encode(picture(4)), Err(MoqError::Closed))); + assert!(encoder.receive().unwrap().is_none(), "flush readied everything"); } #[tokio::test] diff --git a/rs/moq-transcode/benches/ladder.rs b/rs/moq-transcode/benches/ladder.rs index 0075767710e..25bd340297b 100644 --- a/rs/moq-transcode/benches/ladder.rs +++ b/rs/moq-transcode/benches/ladder.rs @@ -90,7 +90,8 @@ async fn fanout(sinks: &mut [(moq_video::Size, moq_video::encode::Sink)], frame: Scale::Caller => Arc::new(frame.resize(*size, &resize).unwrap()), Scale::Encoder => frame.clone(), }; - sink.encode(frame).await.unwrap() + sink.encode(frame).await.unwrap(); + while sink.receive().unwrap().is_some() {} })) .await; } diff --git a/rs/moq-transcode/src/feed.rs b/rs/moq-transcode/src/feed.rs index 99ad22b5150..afa460b19bb 100644 --- a/rs/moq-transcode/src/feed.rs +++ b/rs/moq-transcode/src/feed.rs @@ -219,7 +219,8 @@ async fn decode(inner: &Inner, sender: &broadcast::Sender) -> Result<(), E let keyframe = frame.keyframe || first; first = false; - for decoded in decoder.decode(frame.payload, timestamp, keyframe).await? { + decoder.decode(frame.payload, timestamp, keyframe).await?; + while let Some(decoded) = decoder.receive()? { let _ = sender.send(Item::Frame(Arc::new(decoded))); } } @@ -227,15 +228,16 @@ async fn decode(inner: &Inner, sender: &broadcast::Sender) -> Result<(), E // A buffered decoder still owns this group's tail. Drain it before the // boundary so those frames stay in this group rather than appearing after // the next group's keyframe. - let tail = decoder.flush().await?; + decoder.flush().await?; // A lost group is never finished short, and every rung hears of it now // rather than at a next group that may never come. if complete { - for decoded in tail { + while let Some(decoded) = decoder.receive()? { let _ = sender.send(Item::Frame(Arc::new(decoded))); } let _ = sender.send(Item::End); } else { + while decoder.receive()?.is_some() {} let _ = sender.send(Item::Lost); } } diff --git a/rs/moq-transcode/src/lib.rs b/rs/moq-transcode/src/lib.rs index 857c79a2a29..144f1a2afc5 100644 --- a/rs/moq-transcode/src/lib.rs +++ b/rs/moq-transcode/src/lib.rs @@ -358,7 +358,8 @@ mod tests { .unwrap(); encoder.cut().unwrap(); let gray = vec![0x80u8; 320 * 240 * 4]; - for encoded in encoder.encode(&gray_frame(&gray, 0)).unwrap() { + encoder.encode(&gray_frame(&gray, 0)).unwrap(); + while let Some(encoded) = encoder.receive().unwrap() { hang::container::Frame { timestamp: encoded.timestamp, payload: encoded.payload, @@ -410,7 +411,8 @@ mod tests { if index == 0 { encoder.cut().unwrap(); } - for encoded in encoder.encode(&gray_frame(&gray, timestamp)).unwrap() { + encoder.encode(&gray_frame(&gray, timestamp)).unwrap(); + while let Some(encoded) = encoder.receive().unwrap() { let frame = hang::container::Frame { timestamp: encoded.timestamp, payload: encoded.payload, @@ -479,7 +481,8 @@ mod tests { encoder.cut().unwrap(); let timestamps: Vec<_> = (0..5).map(|index| (sequence * 5 + index) * 33_333).collect(); for ×tamp in ×tamps { - for encoded in encoder.encode(&gray_frame(&gray, timestamp)).unwrap() { + encoder.encode(&gray_frame(&gray, timestamp)).unwrap(); + while let Some(encoded) = encoder.receive().unwrap() { hang::container::Frame { timestamp: encoded.timestamp, payload: encoded.payload, @@ -1457,7 +1460,8 @@ mod tests { encoder.cut().unwrap(); let gray = vec![0x80u8; 320 * 240 * 4]; for timestamp in [0, 33_333] { - for encoded in encoder.encode(&gray_frame(&gray, timestamp)).unwrap() { + encoder.encode(&gray_frame(&gray, timestamp)).unwrap(); + while let Some(encoded) = encoder.receive().unwrap() { hang::container::Frame { timestamp: encoded.timestamp, payload: encoded.payload, @@ -1548,7 +1552,8 @@ mod tests { let mut write = |group: &mut moq_net::group::Producer, timestamps: &[u64]| { encoder.cut().unwrap(); for ×tamp in timestamps { - for encoded in encoder.encode(&gray_frame(&gray, timestamp)).unwrap() { + encoder.encode(&gray_frame(&gray, timestamp)).unwrap(); + while let Some(encoded) = encoder.receive().unwrap() { hang::container::Frame { timestamp: encoded.timestamp, payload: encoded.payload, @@ -1646,7 +1651,8 @@ mod tests { let mut write = |group: &mut moq_net::group::Producer, timestamps: &[u64]| { encoder.cut().unwrap(); for ×tamp in timestamps { - for encoded in encoder.encode(&gray_frame(&gray, timestamp)).unwrap() { + encoder.encode(&gray_frame(&gray, timestamp)).unwrap(); + while let Some(encoded) = encoder.receive().unwrap() { hang::container::Frame { timestamp: encoded.timestamp, payload: encoded.payload, diff --git a/rs/moq-transcode/src/rung.rs b/rs/moq-transcode/src/rung.rs index 68599a603c9..150a991bf02 100644 --- a/rs/moq-transcode/src/rung.rs +++ b/rs/moq-transcode/src/rung.rs @@ -18,6 +18,7 @@ //! output group N maps to source group N and a player switching renditions //! lands on the same content. +use std::collections::VecDeque; use std::future::Future; use std::sync::Arc; @@ -347,6 +348,7 @@ async fn live(rung: &Rung, producer: &mut moq_net::track::Producer) -> Result Result { if let Some(mut output) = current.take() { // The source group is complete, so this one has to be too: a // hardware encoder is still holding its last frames. - write(&mut output, &active, encoder.flush().await?)?; + encoder.flush().await?; + drain(&mut output, &active, &mut encoder)?; output.finish()?; } // Or during the flush. @@ -703,50 +707,45 @@ async fn transcode_group_inner( let keyframe = frame.keyframe || first; first = false; - write( - output, - active, - pipeline.process(frame.payload, timestamp, keyframe).await?, - )?; + pipeline.process(frame.payload, timestamp, keyframe).await?; + drain(output, active, &mut pipeline.encoder)?; } } // One-shot group: drain both codec stages. Each packet keeps the timestamp of // the frame it was encoded from, so the tail stays in step. - write(output, active, pipeline.finish().await?)?; + pipeline.flush().await?; + drain(output, active, &mut pipeline.encoder)?; Ok(()) } -/// Append encoded frames to the output group in the legacy hang framing, metering -/// what reached the track. +/// Write everything the encoder has ready to the output group. +fn drain(output: &mut moq_net::group::Producer, active: &crate::active::Guard, encoder: &mut Encoder) -> Result<(), Error> { + while let Some(encoded) = encoder.receive()? { + write(output, active, encoded)?; + } + Ok(()) +} + +/// Append one encoded frame to the output group in the legacy hang framing, +/// metering it once it reached the track. /// -/// The frames written before a failure are banked anyway: they are on the group -/// and a consumer may already have read them, so dropping them from the meters -/// would understate the bill exactly when something went wrong. +/// Metered per frame, so the frames written before a failure are banked anyway: +/// they are on the group and a consumer may already have read them, so dropping +/// them from the meters would understate the bill exactly when something went wrong. fn write( output: &mut moq_net::group::Producer, active: &crate::active::Guard, - encoded: Vec, + encoded: moq_video::encode::Encoded, ) -> Result<(), Error> { - let mut frames = 0; - let mut bytes = 0; - - let result: Result<(), Error> = (|| { - for encoded in encoded { - let size = encoded.payload.len() as u64; - let frame = hang::container::Frame { - timestamp: encoded.timestamp, - payload: encoded.payload, - }; - frame.write_to(output)?; - frames += 1; - bytes += size; - } - Ok(()) - })(); - - active.produced(frames, bytes); - result + let size = encoded.payload.len() as u64; + let frame = hang::container::Frame { + timestamp: encoded.timestamp, + payload: encoded.payload, + }; + frame.write_to(output)?; + active.produced(1, size); + Ok(()) } /// Decode -> resize -> encode for one fetched group of one rung. @@ -777,14 +776,9 @@ impl Pipeline { }) } - /// Transcode one container payload into zero or more encoded frames, each - /// carrying the presentation time of the picture it came from. - async fn process( - &mut self, - payload: Bytes, - timestamp: moq_net::Timestamp, - keyframe: bool, - ) -> Result, Error> { + /// Transcode one container payload, readying zero or more encoded frames on + /// the encoder, each carrying the presentation time of the picture it came from. + async fn process(&mut self, payload: Bytes, timestamp: moq_net::Timestamp, keyframe: bool) -> Result<(), Error> { // This group opens on an IDR. The encoder holds the request until a picture // actually arrives, which matters because a decoder that buffers returns // nothing for the access unit that asked for one. @@ -792,25 +786,24 @@ impl Pipeline { self.encoder.cut().await?; } - let mut encoded = Vec::new(); - for raw in self.decoder.decode(payload, timestamp, keyframe).await? { - encoded.extend(self.encoder.encode(&self.rung, raw).await?); - } - Ok(encoded) + self.decoder.decode(payload, timestamp, keyframe).await?; + self.transfer().await } - /// Drain the decoder and then the encoder, keeping every frame's timestamp. - /// - /// Consumes the pipeline, since finishing the encoder consumes it: a one-shot - /// group's pipeline is done once both codec stages are drained. - async fn finish(mut self) -> Result, Error> { - let mut encoded = Vec::new(); - for raw in self.decoder.flush().await? { - encoded.extend(self.encoder.encode(&self.rung, raw).await?); - } + /// Flush the decoder and then the encoder, keeping every frame's timestamp: + /// a one-shot group's pipeline is done once both codec stages are drained. + async fn flush(&mut self) -> Result<(), Error> { + self.decoder.flush().await?; + self.transfer().await?; + self.encoder.flush().await + } - encoded.extend(self.encoder.finish().await?); - Ok(encoded) + /// Hand every decoded frame to the encoder. + async fn transfer(&mut self) -> Result<(), Error> { + while let Some(raw) = self.decoder.receive()? { + self.encoder.encode(&self.rung, raw).await?; + } + Ok(()) } } @@ -824,6 +817,8 @@ impl Pipeline { struct Encoder { /// The open encoder and the space it signals. open: Option<(moq_video::encode::Sink, moq_video::Color)>, + /// Encoders closed by a switch, flushed and holding their tails, oldest first. + retired: VecDeque, /// A cut asked for while no encoder was open, applied when one opens. pending_cut: bool, } @@ -840,12 +835,8 @@ impl Encoder { /// Encode one frame, at any size, reopening first if its space changed. /// - /// The previous encoder's tail comes out ahead of the frame, in order. - async fn encode( - &mut self, - rung: &Rung, - frame: impl Into>, - ) -> Result, Error> { + /// The previous encoder's tail is received ahead of the frame, in order. + async fn encode(&mut self, rung: &Rung, frame: impl Into>) -> Result<(), Error> { let frame = frame.into(); // The space the encoder will see, after it scales the frame: an unlabelled // source means what its own size implies, and the resize labels it so. @@ -855,15 +846,15 @@ impl Encoder { .surface .color() .unwrap_or_else(|| moq_video::Color::infer(frame.size())); - let mut encoded = Vec::new(); if let Some((_, signalled)) = &self.open && *signalled != color { - let (sink, _) = self.open.take().expect("checked above"); + let (mut sink, _) = self.open.take().expect("checked above"); // The new parameter sets only reach a decoder on an IDR. Owed before the old encoder // drains, so a drain that fails still leaves the next encoder owing it. self.pending_cut = true; - encoded = sink.finish().await?; + sink.flush().await?; + self.retired.push_back(sink); } let sink = match &mut self.open { Some((sink, _)) => sink, @@ -878,24 +869,30 @@ impl Encoder { &mut self.open.insert((sink, color)).0 } }; - encoded.extend(sink.encode(frame).await?); - Ok(encoded) + sink.encode(frame).await?; + Ok(()) } - /// Drain the frames the encoder still holds. - async fn flush(&mut self) -> Result, Error> { + /// Pop the next encoded frame: a retired encoder's tail first, then the open one's. + fn receive(&mut self) -> Result, Error> { + while let Some(retired) = self.retired.front_mut() { + if let Some(encoded) = retired.receive()? { + return Ok(Some(encoded)); + } + self.retired.pop_front(); + } match &mut self.open { - Some((sink, _)) => Ok(sink.flush().await?), - None => Ok(Vec::new()), + Some((sink, _)) => Ok(sink.receive()?), + None => Ok(None), } } - /// Drain and close the encoder. - async fn finish(self) -> Result, Error> { - match self.open { - Some((sink, _)) => Ok(sink.finish().await?), - None => Ok(Vec::new()), + /// Ready the frames the encoder still holds. + async fn flush(&mut self) -> Result<(), Error> { + if let Some((sink, _)) = &mut self.open { + sink.flush().await?; } + Ok(()) } } @@ -1145,13 +1142,19 @@ mod tests { let mut encoder = Encoder::default(); encoder.cut().await.unwrap(); - let mut encoded = encoder.encode(&rung, frame(Color::BT709, 0)).await.unwrap(); - encoded.extend(encoder.encode(&rung, frame(Color::BT709, 33_333)).await.unwrap()); - let switched = encoder.encode(&rung, frame(pq, 66_666)).await.unwrap(); - encoded.extend(encoder.finish().await.unwrap()); + encoder.encode(&rung, frame(Color::BT709, 0)).await.unwrap(); + encoder.encode(&rung, frame(Color::BT709, 33_333)).await.unwrap(); + encoder.encode(&rung, frame(pq, 66_666)).await.unwrap(); + encoder.flush().await.unwrap(); + let mut encoded = Vec::new(); + while let Some(unit) = encoder.receive().unwrap() { + encoded.push(unit); + } - assert!(encoded.first().expect("the first frame").keyframe); - assert!(switched.last().expect("the switched frame").keyframe); + let timestamps: Vec = encoded.iter().map(|e| e.timestamp.as_micros() as u64).collect(); + assert_eq!(timestamps, [0, 33_333, 66_666], "the old tail comes out ahead of the switch"); + assert!(encoded[0].keyframe); + assert!(encoded[2].keyframe, "the switched frame"); } /// An unlabelled HD frame means BT.709, and scaling it to an SD rung keeps @@ -1171,9 +1174,9 @@ mod tests { let mut encoder = Encoder::default(); encoder.cut().await.unwrap(); assert_eq!(rung.rendition.applied(), None); - let mut encoded = encoder.encode(&rung, frame).await.unwrap(); - encoded.extend(encoder.finish().await.unwrap()); - assert!(encoded.first().expect("the frame").keyframe); + encoder.encode(&rung, frame).await.unwrap(); + encoder.flush().await.unwrap(); + assert!(encoder.receive().unwrap().expect("the frame").keyframe); // The rendition reports the opened encoder's preset, the one the caller requested by default. let applied = rung.rendition.applied().expect("the encoder opened"); assert_eq!(applied.preset, Some(moq_video::encode::Preset::LowLatency)); @@ -1227,7 +1230,8 @@ mod tests { false, ); - assert!(write(&mut group, &guard, vec![good, bad]).is_err()); + write(&mut group, &guard, good).unwrap(); + assert!(write(&mut group, &guard, bad).is_err()); assert_eq!(rendition.frames(), 1); assert_eq!(rendition.bytes(), 5); // One frame is still a start, so the rendition reads as encoding. diff --git a/rs/moq-video/Cargo.toml b/rs/moq-video/Cargo.toml index 7b8346ec723..4544d8a0360 100644 --- a/rs/moq-video/Cargo.toml +++ b/rs/moq-video/Cargo.toml @@ -273,6 +273,7 @@ windows = { version = "0.62", features = [ cfg_aliases = "0.2.2" [dev-dependencies] +criterion = { package = "bench-harness", path = "../bench-harness" } # Native Vulkan producer for the opt-in Vulkan/CUDA interop hardware exercise. ash = { workspace = true } # `poll!`, to cancel an `encode::Sink` request exactly where a `select!` would. @@ -289,5 +290,10 @@ tokio = { workspace = true, features = ["test-util"] } tracing-test = { workspace = true } workspace-hack = { path = "../workspace-hack" } +[[bench]] +name = "codec" +harness = false +required-features = ["openh264"] + [lints] workspace = true diff --git a/rs/moq-video/benches/codec.rs b/rs/moq-video/benches/codec.rs new file mode 100644 index 00000000000..d2d36252a3f --- /dev/null +++ b/rs/moq-video/benches/codec.rs @@ -0,0 +1,279 @@ +//! What the codec API costs around the codec: OpenH264 encode and decode of a short +//! clip, through the inline front ends ([`encode::Encoder`], [`decode::Decoder`]) and +//! the threaded [`encode::Sink`] and [`decode::Sink`], swept over a tiny picture, where +//! the codec is cheap and the API shows, and a modest one, where the codec dominates. +//! +//! Every case submits each frame and receives what it readied before the next, then +//! flushes and receives the tail, the way a live caller drives a codec. +//! +//! Run with `cargo bench -p moq-video --features openh264 --bench codec`. A table comes +//! first: allocations and allocated bytes per frame from an untimed pass, counting the +//! codec's own and every frame's output. Criterion then times each case. +use std::alloc::{GlobalAlloc, Layout as AllocLayout, System}; +use std::hint::black_box; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::Duration; + +use bytes::Bytes; +use criterion::{BenchmarkId, Criterion, Throughput}; +use moq_net::Timestamp; +use moq_video::{Frame, FrameRate, I420, Size, Surface, decode, encode}; + +/// Frames per case: one group, so every case encodes a single keyframe. +const FRAMES: u64 = 30; +const SIZES: [(u32, u32); 2] = [(64, 64), (640, 360)]; + +struct Counter; + +static COUNTING: AtomicBool = AtomicBool::new(false); +static ALLOCATIONS: AtomicU64 = AtomicU64::new(0); +static ALLOCATED: AtomicU64 = AtomicU64::new(0); + +#[global_allocator] +static ALLOCATOR: Counter = Counter; + +impl Counter { + fn count(size: usize) { + if COUNTING.load(Ordering::Relaxed) { + ALLOCATIONS.fetch_add(1, Ordering::Relaxed); + ALLOCATED.fetch_add(size as u64, Ordering::Relaxed); + } + } +} + +unsafe impl GlobalAlloc for Counter { + unsafe fn alloc(&self, layout: AllocLayout) -> *mut u8 { + Self::count(layout.size()); + unsafe { System.alloc(layout) } + } + + unsafe fn alloc_zeroed(&self, layout: AllocLayout) -> *mut u8 { + Self::count(layout.size()); + unsafe { System.alloc_zeroed(layout) } + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: AllocLayout, new_size: usize) -> *mut u8 { + Self::count(new_size); + unsafe { System.realloc(ptr, layout, new_size) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: AllocLayout) { + unsafe { System.dealloc(ptr, layout) } + } +} + +/// Which front end drives the codec. +#[derive(Clone, Copy)] +enum Path { + /// The caller's thread. + Inline, + /// The `Sink`'s own codec thread. + Sink, +} + +impl Path { + fn name(self) -> &'static str { + match self { + Self::Inline => "inline", + Self::Sink => "sink", + } + } +} + +/// One clip at one size: its raw pictures, and the access units they encode to. +struct Clip { + size: Size, + pictures: Vec>, + units: Vec<(Bytes, Timestamp, bool)>, + catalog: hang::catalog::VideoConfig, +} + +impl Clip { + fn new(size: Size) -> Self { + let pictures: Vec> = (0..FRAMES).map(|index| Arc::new(picture(size, index))).collect(); + let config = config(size); + let catalog = pollster::block_on(config.probe()).expect("probe the encoder"); + let mut encoder = encode::Encoder::new(&config).expect("OpenH264 encoder"); + let mut units = Vec::new(); + for picture in &pictures { + encoder.encode(picture).unwrap(); + while let Some(encoded) = encoder.receive().unwrap() { + units.push((encoded.payload, encoded.timestamp, encoded.keyframe)); + } + } + encoder.flush().unwrap(); + while let Some(encoded) = encoder.receive().unwrap() { + units.push((encoded.payload, encoded.timestamp, encoded.keyframe)); + } + assert_eq!(units.len() as u64, FRAMES, "one access unit per picture"); + Self { + size, + pictures, + units, + catalog, + } + } + + fn name(&self, stage: &str, path: Path) -> String { + format!("{stage}/{}/{}x{}", path.name(), self.size.width, self.size.height) + } +} + +fn config(size: Size) -> encode::Config { + let mut config = encode::Config::new(size.width, size.height, FrameRate::new(30, 1).unwrap()); + config.kind = encode::Kind::Software; + config +} + +/// A gradient that moves with `index`, so every picture has new content to encode. +fn picture(size: Size, index: u64) -> Frame { + let (width, height) = (size.width as usize, size.height as usize); + let mut data = vec![128u8; width * height * 3 / 2]; + for y in 0..height { + for x in 0..width { + data[y * width + x] = ((x + y + index as usize * 4) % 256) as u8; + } + } + let i420 = I420::new(size, data).unwrap(); + Frame::new(Surface::I420(i420), Timestamp::from_micros(index * 33_333).unwrap()) +} + +/// Encode the clip from a fresh encoder, receiving after every picture. +fn encode(clip: &Clip, path: Path) -> u64 { + let config = config(clip.size); + let mut received = 0; + match path { + Path::Inline => { + let mut encoder = encode::Encoder::new(&config).unwrap(); + for picture in &clip.pictures { + encoder.encode(picture).unwrap(); + while let Some(encoded) = encoder.receive().unwrap() { + black_box(encoded); + received += 1; + } + } + encoder.flush().unwrap(); + while let Some(encoded) = encoder.receive().unwrap() { + black_box(encoded); + received += 1; + } + } + Path::Sink => pollster::block_on(async { + let mut sink = encode::Sink::open(&config).await.unwrap(); + for picture in &clip.pictures { + sink.encode(picture.clone()).await.unwrap(); + while let Some(encoded) = sink.receive().unwrap() { + black_box(encoded); + received += 1; + } + } + sink.flush().await.unwrap(); + while let Some(encoded) = sink.receive().unwrap() { + black_box(encoded); + received += 1; + } + }), + } + received +} + +/// Decode the clip from a fresh decoder, receiving after every access unit. +fn decode(clip: &Clip, path: Path) -> u64 { + let mut config = decode::Config::new(); + config.kind = decode::Kind::Software; + let mut received = 0; + match path { + Path::Inline => { + let mut decoder = decode::Decoder::new(&clip.catalog, &config).unwrap(); + for (payload, timestamp, keyframe) in &clip.units { + decoder.decode(payload, *timestamp, *keyframe).unwrap(); + while let Some(frame) = decoder.receive().unwrap() { + black_box(frame); + received += 1; + } + } + decoder.flush().unwrap(); + while let Some(frame) = decoder.receive().unwrap() { + black_box(frame); + received += 1; + } + } + Path::Sink => pollster::block_on(async { + let mut sink = decode::Sink::open(&clip.catalog, &config).await.unwrap(); + for (payload, timestamp, keyframe) in &clip.units { + sink.decode(payload.clone(), *timestamp, *keyframe).await.unwrap(); + while let Some(frame) = sink.receive().unwrap() { + black_box(frame); + received += 1; + } + } + sink.flush().await.unwrap(); + while let Some(frame) = sink.receive().unwrap() { + black_box(frame); + received += 1; + } + }), + } + received +} + +type Stage = fn(&Clip, Path) -> u64; +const STAGES: [(&str, Stage); 2] = [("encode", encode), ("decode", decode)]; +const PATHS: [Path; 2] = [Path::Inline, Path::Sink]; + +/// Print allocations per frame for every case, from one untimed pass each. +fn counts(clips: &[Clip]) { + println!("{:<28} {:>14} {:>14}", "case", "allocs/frame", "bytes/frame"); + for clip in clips { + for (stage, run) in STAGES { + for path in PATHS { + ALLOCATIONS.store(0, Ordering::Relaxed); + ALLOCATED.store(0, Ordering::Relaxed); + COUNTING.store(true, Ordering::Relaxed); + let received = run(clip, path); + COUNTING.store(false, Ordering::Relaxed); + assert_eq!(received, FRAMES, "{}", clip.name(stage, path)); + println!( + "{:<28} {:>14.2} {:>14.0}", + clip.name(stage, path), + ALLOCATIONS.load(Ordering::Relaxed) as f64 / FRAMES as f64, + ALLOCATED.load(Ordering::Relaxed) as f64 / FRAMES as f64, + ); + } + } + } +} + +fn bench_codec(c: &mut Criterion, clips: &[Clip]) { + let mut group = c.benchmark_group("codec"); + group.sample_size(10); + group.warm_up_time(Duration::from_millis(500)); + group.measurement_time(Duration::from_secs(3)); + group.throughput(Throughput::Elements(FRAMES)); + for clip in clips { + for (stage, run) in STAGES { + for path in PATHS { + group.bench_function(BenchmarkId::from_parameter(clip.name(stage, path)), |b| { + b.iter(|| run(clip, path)); + }); + } + } + } + group.finish(); +} + +fn main() { + // Nextest lists all targets as potential test binaries. + if std::env::args().any(|arg| arg == "--list") { + return; + } + let clips: Vec = SIZES + .into_iter() + .map(|(width, height)| Clip::new(Size::new(width, height))) + .collect(); + counts(&clips); + let mut criterion = Criterion::default().configure_from_args(); + bench_codec(&mut criterion, &clips); + criterion.final_summary(); +} diff --git a/rs/moq-video/examples/d3d11-resize-probe.rs b/rs/moq-video/examples/d3d11-resize-probe.rs index 61e9a07a786..87db3b7a65e 100644 --- a/rs/moq-video/examples/d3d11-resize-probe.rs +++ b/rs/moq-video/examples/d3d11-resize-probe.rs @@ -852,10 +852,15 @@ mod probe { Surface::I420(gradient(index)), moq_net::Timestamp::from_micros(index * 33_333).unwrap(), ); - for encoded in encoder.encode(&frame).context("encode")? { + encoder.encode(&frame).context("encode")?; + while let Some(encoded) = encoder.receive().context("encode")? { out.push(encoded.payload.to_vec()); } } + encoder.flush().context("flush")?; + while let Some(encoded) = encoder.receive().context("flush")? { + out.push(encoded.payload.to_vec()); + } Ok(out) } @@ -1053,7 +1058,10 @@ mod probe { let mut frames = Vec::new(); for (index, unit) in stream()?.into_iter().enumerate() { let timestamp = moq_net::Timestamp::from_micros(index as u64 * 33_333).unwrap(); - frames.extend(decoder.decode(&unit.into(), timestamp, index == 0).context("decode")?); + decoder.decode(&unit.into(), timestamp, index == 0).context("decode")?; + while let Some(frame) = decoder.receive().context("decode")? { + frames.push(frame); + } } let Some(Surface::Texture(texture)) = frames.first().map(|f| &f.surface) else { bail!("the crate's decoder did not produce a GPU texture on this machine"); @@ -1115,11 +1123,12 @@ mod probe { let mut frames = Vec::new(); for (index, unit) in units.iter().enumerate() { let timestamp = moq_net::Timestamp::from_micros(index as u64 * 33_333).unwrap(); - frames.extend( - decoder - .decode(&unit.clone().into(), timestamp, index == 0) - .context("decode")?, - ); + decoder + .decode(&unit.clone().into(), timestamp, index == 0) + .context("decode")?; + while let Some(frame) = decoder.receive().context("decode")? { + frames.push(frame); + } } let Some(first) = frames.into_iter().next() else { bail!("the decoder did not produce a frame on this machine"); @@ -1164,6 +1173,11 @@ mod probe { encoder .encode(&resized) .with_context(|| format!("ladder rung {rung} encode"))?; + while encoder + .receive() + .with_context(|| format!("ladder rung {rung} encode"))? + .is_some() + {} } } Ok(()) @@ -1202,6 +1216,7 @@ mod probe { decoder .decode(&unit.clone().into(), timestamp, index == 0) .context("live ladder decode")?; + while decoder.receive().context("live ladder decode")?.is_some() {} timestamp_index += 1; } } diff --git a/rs/moq-video/examples/encode-presets.rs b/rs/moq-video/examples/encode-presets.rs index d6a06a73d62..20fbe04e56d 100644 --- a/rs/moq-video/examples/encode-presets.rs +++ b/rs/moq-video/examples/encode-presets.rs @@ -144,15 +144,18 @@ fn run(config: &Config, pictures: &[&[u8]], paced: bool) -> anyhow::Result let mut returned = Vec::with_capacity(pictures.len()); let mut stream = Vec::new(); - let mut collect = - |packets: Vec, submitted: &[Instant], emitted: &mut [Option]| { - let now = Instant::now(); - for packet in packets { - let index = (packet.timestamp.as_micros() / frame_time.as_micros()) as usize; - emitted[index] = Some(now - submitted[index]); - stream.extend_from_slice(&packet.payload); - } - }; + let mut collect = |encoder: &mut Encoder, + submitted: &[Instant], + emitted: &mut [Option]| + -> anyhow::Result<()> { + let now = Instant::now(); + while let Some(packet) = encoder.receive()? { + let index = (packet.timestamp.as_micros() / frame_time.as_micros()) as usize; + emitted[index] = Some(now - submitted[index]); + stream.extend_from_slice(&packet.payload); + } + Ok(()) + }; let start = Instant::now(); let cpu = cpu_time(); @@ -166,11 +169,12 @@ fn run(config: &Config, pictures: &[&[u8]], paced: bool) -> anyhow::Result let i420 = I420::new(config.size(), picture.to_vec())?; let timestamp = moq_net::Timestamp::from_micros((frame_time * index as u32).as_micros() as u64)?; submitted.push(Instant::now()); - let packets = encoder.encode(&Frame::new(Surface::I420(i420), timestamp))?; - collect(packets, &submitted, &mut emitted); + encoder.encode(&Frame::new(Surface::I420(i420), timestamp))?; + collect(&mut encoder, &submitted, &mut emitted)?; returned.push(emitted.iter().filter(|e| e.is_some()).count()); } - collect(encoder.finish()?, &submitted, &mut emitted); + encoder.flush()?; + collect(&mut encoder, &submitted, &mut emitted)?; let (wall, cpu) = (start.elapsed(), cpu_time() - cpu); // Only knowable at the end: a frame that never came back was skipped, not held. diff --git a/rs/moq-video/src/capture/wgc.rs b/rs/moq-video/src/capture/wgc.rs index aeb1c92bdfa..3dac1d28278 100644 --- a/rs/moq-video/src/capture/wgc.rs +++ b/rs/moq-video/src/capture/wgc.rs @@ -329,7 +329,9 @@ mod tests { assert_eq!(delivery.deadline(), Some(now)); let (surface, timestamp) = delivery.next(now).unwrap().unwrap(); let frame = crate::Frame::new(crate::Surface::I420(surface.clone()), timestamp); - producer.publish(&encoder.encode(&frame).unwrap()).unwrap(); + for encoded in encoder.encode_all(&frame).unwrap() { + producer.publish(&encoded).unwrap(); + } let received = loop { if let moq_mux::container::Read::Frame { frame, .. } = subscriber.read().await.unwrap().unwrap() { break frame; diff --git a/rs/moq-video/src/decode/backend/mediacodec.rs b/rs/moq-video/src/decode/backend/mediacodec.rs index aa182d25a1a..7645c5ce3e2 100644 --- a/rs/moq-video/src/decode/backend/mediacodec.rs +++ b/rs/moq-video/src/decode/backend/mediacodec.rs @@ -23,6 +23,7 @@ //! stalls decoding. That is the same trade the VideoToolbox and NVDEC backends //! make: draw and drop. +use std::collections::VecDeque; use std::sync::{Arc, Condvar, Mutex}; use std::time::Duration; @@ -120,6 +121,8 @@ pub(crate) struct MediaCodec { /// Whether the last acquire found every slot held, so the stall is reported /// once on the way in rather than on every poll. stalled: bool, + /// Pictures acquired from the reader, waiting for `receive`. + ready: VecDeque, } /// Where the codec's input stands between flushes. @@ -238,6 +241,7 @@ impl MediaCodec { pending_images: 0, input: Input::Idle, stalled: false, + ready: VecDeque::new(), })) } @@ -246,7 +250,7 @@ impl MediaCodec { /// Waits for a buffer rather than dropping the access unit when the codec has /// none free: a decoder that skips one produces a broken picture until the /// next keyframe, which is a far worse trade than the wait. - fn submit(&mut self, access_unit: &[u8], timestamp: Timestamp, out: &mut Vec) -> Result<(), Error> { + fn submit(&mut self, access_unit: &[u8], timestamp: Timestamp) -> Result<(), Error> { let time = timestamp.as_micros().min(u64::MAX as u128) as u64; for _ in 0..SUBMIT_ROUNDS { @@ -290,7 +294,7 @@ impl MediaCodec { // Every input buffer is still with the codec; collecting its output is // what frees one. - self.drain(OUTPUT_TIMEOUT, out)?; + self.drain(OUTPUT_TIMEOUT)?; } Err(Error::codec(anyhow::anyhow!( @@ -298,9 +302,9 @@ impl MediaCodec { ))) } - /// Render every decoded picture the codec has ready and collect the images + /// Render every decoded picture the codec has ready and queue the images /// they turn into. - fn drain(&mut self, timeout: Duration, out: &mut Vec) -> Result { + fn drain(&mut self, timeout: Duration) -> Result { loop { let ended = match self .codec @@ -323,7 +327,7 @@ impl MediaCodec { info.flags() & FLAG_END_OF_STREAM != 0 } DequeuedOutputBufferInfoResult::TryAgainLater => { - self.collect(out)?; + self.collect()?; return Ok(false); } // The stream's geometry is read off each image rather than off the @@ -336,7 +340,7 @@ impl MediaCodec { // Eagerly, because releasing for rendering queues the picture into the // reader before it returns, so the image is normally there already and // waiting for the next access unit would add a frame of latency. - self.collect(out)?; + self.collect()?; if ended { return Ok(true); } @@ -344,7 +348,7 @@ impl MediaCodec { } /// Acquire every rendered picture the reader has ready. - fn collect(&mut self, out: &mut Vec) -> Result<(), Error> { + fn collect(&mut self) -> Result<(), Error> { loop { match self .reader @@ -362,7 +366,7 @@ impl MediaCodec { let timestamp = Timestamp::from_nanos(nanos.max(0) as u64)?; let (left, top, width, height) = image_size(&image)?; let buffer = HardwareBuffer::new(self.reader.clone(), image, left, top, width, height); - out.push(Frame::new(Surface::HardwareBuffer(buffer), timestamp)); + self.ready.push_back(Frame::new(Surface::HardwareBuffer(buffer), timestamp)); } // The picture is on its way but not queued yet, so it comes out on the // next round rather than being lost. @@ -394,12 +398,11 @@ impl MediaCodec { } /// Signal end of input and wait until every delayed output buffer is released. - fn drain_tail(&mut self) -> Result, Error> { - let mut out = Vec::new(); - self.signal_end_of_input(&mut out)?; + fn drain_tail(&mut self) -> Result<(), Error> { + self.signal_end_of_input()?; let mut ended = false; for _ in 0..DRAIN_ROUNDS { - if self.drain(DRAIN_TIMEOUT, &mut out)? { + if self.drain(DRAIN_TIMEOUT)? { ended = true; break; } @@ -415,7 +418,7 @@ impl MediaCodec { // ImageReader callback so the final pictures are part of this stream rather // than appearing after the next keyframe. for _ in 0..DRAIN_ROUNDS { - self.collect(&mut out)?; + self.collect()?; if self.pending_images == 0 { break; } @@ -430,11 +433,11 @@ impl MediaCodec { self.pending_images = 0; } - Ok(out) + Ok(()) } /// Queue the empty input buffer that marks the end of a ByteBuffer stream. - fn signal_end_of_input(&mut self, out: &mut Vec) -> Result<(), Error> { + fn signal_end_of_input(&mut self) -> Result<(), Error> { for _ in 0..DRAIN_ROUNDS { let queued = match self .codec @@ -452,7 +455,7 @@ impl MediaCodec { if queued { return Ok(()); } - self.drain(OUTPUT_TIMEOUT, out)?; + self.drain(OUTPUT_TIMEOUT)?; } Err(Error::codec(anyhow::anyhow!( @@ -462,21 +465,24 @@ impl MediaCodec { } impl Backend for MediaCodec { - fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result, Error> { + fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result<(), Error> { self.input.check()?; - let mut out = Vec::new(); // Collect what the codec finished while the caller was elsewhere, which is // also what frees the input buffer the submission below needs. - self.drain(OUTPUT_TIMEOUT, &mut out)?; - self.submit(&access_unit, timestamp, &mut out)?; - self.drain(OUTPUT_TIMEOUT, &mut out)?; - Ok(out) + self.drain(OUTPUT_TIMEOUT)?; + self.submit(&access_unit, timestamp)?; + self.drain(OUTPUT_TIMEOUT)?; + Ok(()) + } + + fn receive(&mut self) -> Result, Error> { + Ok(self.ready.pop_front()) } - fn flush(&mut self) -> Result, Error> { + fn flush(&mut self) -> Result<(), Error> { self.input.check()?; if self.input == Input::Idle { - return Ok(Vec::new()); + return Ok(()); } // Flushed whatever the drain did: one that never saw end of stream still @@ -593,26 +599,26 @@ mod tests { let timestamp = Timestamp::from_micros(index * 33_333).unwrap(); let frame = Frame::new(Surface::I420(i420.clone()), timestamp); encoder.cut().unwrap(); - for encoded in encoder.encode(&frame).unwrap() { - frames.extend(decoder.decode(encoded.payload, encoded.timestamp, true).unwrap()); + for encoded in encoder.encode_all(&frame).unwrap() { + frames.extend(decoder.decode_all(encoded.payload, encoded.timestamp, true).unwrap()); } } - for encoded in encoder.flush().unwrap() { - frames.extend(decoder.decode(encoded.payload, encoded.timestamp, true).unwrap()); + for encoded in encoder.flush_all().unwrap() { + frames.extend(decoder.decode_all(encoded.payload, encoded.timestamp, true).unwrap()); } - frames.extend(decoder.flush().unwrap()); + frames.extend(decoder.flush_all().unwrap()); // A flush drains the previous stream and leaves the same decoder reusable. let timestamp = Timestamp::from_micros(1_000_000).unwrap(); let frame = Frame::new(Surface::I420(i420.clone()), timestamp); encoder.cut().unwrap(); - for encoded in encoder.encode(&frame).unwrap() { - frames.extend(decoder.decode(encoded.payload, encoded.timestamp, true).unwrap()); + for encoded in encoder.encode_all(&frame).unwrap() { + frames.extend(decoder.decode_all(encoded.payload, encoded.timestamp, true).unwrap()); } - for encoded in encoder.flush().unwrap() { - frames.extend(decoder.decode(encoded.payload, encoded.timestamp, true).unwrap()); + for encoded in encoder.flush_all().unwrap() { + frames.extend(decoder.decode_all(encoded.payload, encoded.timestamp, true).unwrap()); } - frames.extend(decoder.flush().unwrap()); + frames.extend(decoder.flush_all().unwrap()); let frame = frames.first().expect("at least one decoded frame"); assert_eq!(frame.size(), size); diff --git a/rs/moq-video/src/decode/backend/mediafoundation.rs b/rs/moq-video/src/decode/backend/mediafoundation.rs index 70c4ea0ad02..18e9b05e234 100644 --- a/rs/moq-video/src/decode/backend/mediafoundation.rs +++ b/rs/moq-video/src/decode/backend/mediafoundation.rs @@ -93,6 +93,8 @@ pub(crate) struct MediaFoundation { last_timestamp: Option, /// Mark the first accepted sample after a drain as a new stream epoch. discontinuity: bool, + /// Pictures the MFT handed back, waiting for `receive`. + ready: VecDeque, /// Kept alive for the MFT's lifetime; the MFT holds its own ref but we own the /// pairing. Drops before `_com`. _manager: IMFDXGIDeviceManager, @@ -146,6 +148,7 @@ impl MediaFoundation { pending: VecDeque::new(), last_timestamp: None, discontinuity: false, + ready: VecDeque::new(), _manager: manager, _com: com, }; @@ -308,12 +311,12 @@ impl MediaFoundation { /// Reports whether the decoder got anywhere, which is not the same as whether /// it handed back a picture: renegotiating the output type is progress too, and /// it produces no frame. - fn drain_output(&mut self, out: &mut Vec) -> Result { + fn drain_output(&mut self) -> Result { if !self.output_configured { return Ok(false); } let mut progressed = false; - while self.process_output(out)? { + while self.process_output()? { progressed = true; } Ok(progressed) @@ -322,7 +325,7 @@ impl MediaFoundation { /// One `ProcessOutput`. Returns `true` if it produced a frame or handled a /// stream change (so the caller keeps draining), `false` on /// `MF_E_TRANSFORM_NEED_MORE_INPUT` (the decoder wants the next access unit). - fn process_output(&mut self, out: &mut Vec) -> Result { + fn process_output(&mut self) -> Result { // In DXVA mode the MFT provides its own texture-backed sample, so we pass a // null slot; otherwise we hand it a system-memory buffer to fill. let provided = if self.provides_samples { @@ -351,7 +354,8 @@ impl MediaFoundation { if let Some(sample) = sample { let sample_time = unsafe { sample.GetSampleTime() }.ok(); let surface = self.sample_to_surface(&sample)?; - out.push(Frame::new(surface, self.take_timestamp(sample_time))); + let timestamp = self.take_timestamp(sample_time); + self.ready.push_back(Frame::new(surface, timestamp)); } Ok(true) } @@ -425,9 +429,7 @@ impl MediaFoundation { } impl Backend for MediaFoundation { - fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result, Error> { - let mut out = Vec::new(); - + fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result<(), Error> { let sample = self.build_sample(&access_unit)?; if self.discontinuity { unsafe { sample.SetUINT32(&MFSampleExtension_Discontinuity, 1) } @@ -447,7 +449,7 @@ impl Backend for MediaFoundation { // renegotiates the output type here and produces no picture doing // it, and that is exactly the case where feeding the same access // unit again is the right move. - if !self.drain_output(&mut out)? { + if !self.drain_output()? { return Err(Error::codec(anyhow::anyhow!( "decoder rejected the access unit with no output to drain" ))); @@ -471,22 +473,25 @@ impl Backend for MediaFoundation { self.configure_output()?; } - self.drain_output(&mut out)?; - Ok(out) + self.drain_output()?; + Ok(()) } - fn flush(&mut self) -> Result, Error> { + fn receive(&mut self) -> Result, Error> { + Ok(self.ready.pop_front()) + } + + fn flush(&mut self) -> Result<(), Error> { // Whatever the drain does, the next access unit opens a new stream. self.discontinuity = true; if !self.output_configured { self.pending.clear(); - return Ok(Vec::new()); + return Ok(()); } - let mut out = Vec::new(); let drained = unsafe { self.transform.ProcessMessage(MFT_MESSAGE_COMMAND_DRAIN, 0) } .map_err(|e| mf_err("drain", e)) - .and_then(|()| self.drain_output(&mut out)); + .and_then(|()| self.drain_output()); // Only after the drain, which pairs the delayed pictures with these. self.pending.clear(); if let Err(err) = drained { @@ -503,7 +508,7 @@ impl Backend for MediaFoundation { unsafe { let _ = self.transform.ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0); } - Ok(out) + Ok(()) } fn name(&self) -> &str { diff --git a/rs/moq-video/src/decode/backend/mod.rs b/rs/moq-video/src/decode/backend/mod.rs index 446e89fd502..f83f49276c9 100644 --- a/rs/moq-video/src/decode/backend/mod.rs +++ b/rs/moq-video/src/decode/backend/mod.rs @@ -130,24 +130,54 @@ impl From for Stream { } } -/// An opened decoder. Feed it prepared access units in decode order; get back -/// zero or more decoded frames (zero while the decoder is still buffering, e.g. -/// before the first keyframe's parameter sets). +/// An opened decoder. Submit prepared access units in decode order, then +/// receive zero or more decoded frames (zero while the decoder is still +/// buffering, e.g. before the first keyframe's parameter sets). +/// +/// A backend that readies pictures synchronously holds them in a reused queue, +/// so no call allocates to hand them back. pub(crate) trait Backend { - /// Decode one access unit stamped with its presentation `timestamp`. + /// Submit one access unit stamped with its presentation `timestamp`. /// `keyframe` marks a random-access frame. Takes an owned [`Bytes`] so a /// backend can split codec units without copying. /// Backends that decode one-in one-out echo the input timestamp; NVDEC and /// MediaCodec thread timestamps through the codec, so they survive decoder /// delay and frame reordering. - fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, keyframe: bool) -> Result, Error>; + fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, keyframe: bool) -> Result<(), Error>; + + /// Pop the oldest ready picture, or `None` while the codec holds the rest. + fn receive(&mut self) -> Result, Error>; - /// Return the pictures the codec still holds once the stream has ended. + /// Mark the end of input, so [`receive`](Self::receive) returns every picture + /// the codec still holds. The next access unit submitted starts new input. /// - /// Backends configured for zero delay return no frames. A backend that - /// reorders pictures overrides this so the end of a track does not drop its - /// buffered tail. - fn flush(&mut self) -> Result, Error>; + /// Backends configured for zero delay hold nothing. A backend that reorders + /// pictures queues its tail here so the end of a track does not drop it. + fn flush(&mut self) -> Result<(), Error>; + + /// Submit one access unit and collect everything it readied. + #[cfg(test)] + fn decode_all(&mut self, access_unit: Bytes, timestamp: Timestamp, keyframe: bool) -> Result, Error> { + self.decode(access_unit, timestamp, keyframe)?; + self.receive_all() + } + + /// Flush and collect the tail. + #[cfg(test)] + fn flush_all(&mut self) -> Result, Error> { + self.flush()?; + self.receive_all() + } + + /// Collect everything ready. + #[cfg(test)] + fn receive_all(&mut self) -> Result, Error> { + let mut frames = Vec::new(); + while let Some(frame) = self.receive()? { + frames.push(frame); + } + Ok(frames) + } /// The decoder name in use, e.g. `"videotoolbox"` (for logging). fn name(&self) -> &str; @@ -446,12 +476,16 @@ mod tests { } impl Backend for Stub { - fn decode(&mut self, _access_unit: Bytes, _timestamp: Timestamp, _keyframe: bool) -> Result, Error> { - Ok(Vec::new()) + fn decode(&mut self, _access_unit: Bytes, _timestamp: Timestamp, _keyframe: bool) -> Result<(), Error> { + Ok(()) + } + + fn receive(&mut self) -> Result, Error> { + Ok(None) } - fn flush(&mut self) -> Result, Error> { - Ok(Vec::new()) + fn flush(&mut self) -> Result<(), Error> { + Ok(()) } fn name(&self) -> &str { diff --git a/rs/moq-video/src/decode/backend/nvdec.rs b/rs/moq-video/src/decode/backend/nvdec.rs index ee7d7533a8c..9686f87d869 100644 --- a/rs/moq-video/src/decode/backend/nvdec.rs +++ b/rs/moq-video/src/decode/backend/nvdec.rs @@ -23,6 +23,7 @@ //! correct presentation times even across reordering. use core::ffi::{c_int, c_uint, c_ulong, c_ulonglong, c_void}; +use std::collections::VecDeque; use std::ptr; use std::sync::Arc; @@ -55,6 +56,8 @@ pub(crate) struct Nvdec { /// Boxed so its address is stable: the parser holds a raw pointer to it for /// the lifetime of the parser (callbacks dereference it during parse). state: Box, + /// Pictures mapped and copied out, waiting for `receive`. + ready: VecDeque, } /// State shared with the parser's C callbacks via the user-data pointer. @@ -152,12 +155,13 @@ impl Nvdec { parser: Some(parser), codec: cuda_codec, state, + ready: VecDeque::new(), })) } - /// Submit one parser packet and copy every picture its display callback made - /// ready before returning. - fn parse(&mut self, packet: &mut CUVIDSOURCEDATAPACKET) -> Result, Error> { + /// Submit one parser packet and queue a copy of every picture its display + /// callback made ready before returning. + fn parse(&mut self, packet: &mut CUVIDSOURCEDATAPACKET) -> Result<(), Error> { // Parser callbacks and the map/copy below all need the CUDA context current // on this thread. self.state @@ -180,13 +184,20 @@ impl Nvdec { return Err(codec_err(format!("cuvidParseVideoData: {result:?}"))); } - let ready = std::mem::take(&mut self.state.ready); - ready.iter().map(|disp| self.state.map_frame(disp)).collect() + // Taken out and put back so the callback's buffer keeps its capacity. + let mut displayed = std::mem::take(&mut self.state.ready); + let mapped = displayed.iter().try_for_each(|disp| { + self.ready.push_back(self.state.map_frame(disp)?); + Ok(()) + }); + displayed.clear(); + self.state.ready = displayed; + mapped } } impl Backend for Nvdec { - fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result, Error> { + fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result<(), Error> { let mut packet = CUVIDSOURCEDATAPACKET { // ENDOFPICTURE: each payload is one complete access unit, so the // parser sees the picture boundary without waiting for the next AU. @@ -201,7 +212,11 @@ impl Backend for Nvdec { self.parse(&mut packet) } - fn flush(&mut self) -> Result, Error> { + fn receive(&mut self) -> Result, Error> { + Ok(self.ready.pop_front()) + } + + fn flush(&mut self) -> Result<(), Error> { if self.parser.is_none() { // A failed replacement ended the session; never install another parser. return Err(Error::CodecGone("NVDEC parser replacement failed after a drain".into())); @@ -214,7 +229,7 @@ impl Backend for Nvdec { payload: ptr::null(), timestamp: 0, }; - let frames = self.parse(&mut packet); + let parsed = self.parse(&mut packet); // Past end-of-stream the parser accepts data but never decodes it again // (CUVID_PKT_DISCONTINUITY does not reopen it), so whatever the drain did, // the next access unit goes to a new parser. The cuvid decoder is kept: the @@ -223,7 +238,7 @@ impl Backend for Nvdec { let parser = create_parser(self.codec, &mut self.state) .map_err(|err| Error::CodecGone(format!("NVDEC parser replacement: {err}")))?; self.parser = Some(parser); - frames + parsed } fn name(&self) -> &str { @@ -660,14 +675,14 @@ mod tests { if i == 0 { encoder.cut().unwrap(); } - for encoded in encoder.encode(&gradient_frame(&rgba, w, h, i)).unwrap() { - for decoded in decoder.decode(encoded.payload, encoded.timestamp, i == 0).unwrap() { + for encoded in encoder.encode_all(&gradient_frame(&rgba, w, h, i)).unwrap() { + for decoded in decoder.decode_all(encoded.payload, encoded.timestamp, i == 0).unwrap() { let i420 = decoded.surface.to_i420().unwrap().into_owned(); out.push((decoded.timestamp.as_micros() as u64, i420)); } } } - for decoded in decoder.flush().unwrap() { + for decoded in decoder.flush_all().unwrap() { let i420 = decoded.surface.to_i420().unwrap().into_owned(); out.push((decoded.timestamp.as_micros() as u64, i420)); } @@ -696,16 +711,16 @@ mod tests { let mut timestamps = Vec::new(); encoder.cut().unwrap(); for i in 0..10u64 { - for encoded in encoder.encode(&gradient_frame(&rgba, w, h, group * 10 + i)).unwrap() { - let frames = decoder.decode(encoded.payload, encoded.timestamp, i == 0).unwrap(); + for encoded in encoder.encode_all(&gradient_frame(&rgba, w, h, group * 10 + i)).unwrap() { + let frames = decoder.decode_all(encoded.payload, encoded.timestamp, i == 0).unwrap(); timestamps.extend(frames.iter().map(|frame| frame.timestamp.as_micros() as u64)); } } - let frames = decoder.flush().unwrap(); + let frames = decoder.flush_all().unwrap(); timestamps.extend(frames.iter().map(|frame| frame.timestamp.as_micros() as u64)); // A second flush lands on a parser that has seen no packet yet. assert!( - decoder.flush().unwrap().is_empty(), + decoder.flush_all().unwrap().is_empty(), "a repeated flush hands back nothing" ); decoded.push(timestamps); @@ -845,8 +860,8 @@ mod tests { if i == 0 { source.cut().unwrap(); } - for encoded in source.encode(&gradient_frame(&rgba, w, h, i)).unwrap() { - frames.extend(decoder.decode(encoded.payload, encoded.timestamp, i == 0).unwrap()); + for encoded in source.encode_all(&gradient_frame(&rgba, w, h, i)).unwrap() { + frames.extend(decoder.decode_all(encoded.payload, encoded.timestamp, i == 0).unwrap()); } } frames @@ -862,9 +877,9 @@ mod tests { if i == 0 { nvenc.cut().unwrap(); } - packets.extend(nvenc.encode(&out).unwrap()); + packets.extend(nvenc.encode_all(&out).unwrap()); } - packets.extend(nvenc.finish().unwrap()); + packets.extend(nvenc.flush_all().unwrap()); assert!(!packets.is_empty(), "NVENC produced no packets from CUDA frames"); // Decode the re-encoded stream in software and compare to the source. @@ -888,7 +903,7 @@ mod tests { .unwrap(); let mut last = None; for (i, encoded) in packets.into_iter().enumerate() { - for out in check.decode(encoded.payload, encoded.timestamp, i == 0).unwrap() { + for out in check.decode_all(encoded.payload, encoded.timestamp, i == 0).unwrap() { last = Some(out.surface.to_i420().unwrap().into_owned()); } } diff --git a/rs/moq-video/src/decode/backend/openh264.rs b/rs/moq-video/src/decode/backend/openh264.rs index 2b321da6942..3f49fb09c17 100644 --- a/rs/moq-video/src/decode/backend/openh264.rs +++ b/rs/moq-video/src/decode/backend/openh264.rs @@ -5,7 +5,7 @@ //! I420. use std::cmp::Reverse; -use std::collections::BinaryHeap; +use std::collections::{BinaryHeap, VecDeque}; use bytes::Bytes; use moq_net::Timestamp; @@ -88,6 +88,7 @@ pub(crate) struct Openh264 { /// Access units the decoder has refused since the last picture came out, so /// a run of them logs once rather than once per picture. lost: u64, + ready: VecDeque, } /// Builds an openh264 decoder configured the way this backend needs one. @@ -141,19 +142,20 @@ impl Openh264 { decoder, pending: BinaryHeap::new(), lost: 0, + ready: VecDeque::new(), }) } /// Records an access unit the decoder refused. /// - /// Returns no frames for a state that describes the picture, so playback - /// carries on and recovers at the next keyframe, and the error itself for a - /// state that describes the decoder. + /// Succeeds with no picture for a state that describes the picture, so + /// playback carries on and recovers at the next keyframe, and returns the + /// error itself for a state that describes the decoder. /// /// Only a state in [`DECODER_RESET`] takes the outstanding timestamps with /// it. After the others the decoder still holds the pictures those /// timestamps belong to. - fn picture_lost(&mut self, timestamp: Timestamp, err: &openh264::Error) -> Result, Error> { + fn picture_lost(&mut self, timestamp: Timestamp, err: &openh264::Error) -> Result<(), Error> { let state = i32::try_from(err.native_code()).unwrap_or(dsErrorFree); if state == dsErrorFree || state & !PICTURE_LOST != 0 { return Err(Error::codec(anyhow::anyhow!("openh264 decode: {err}"))); @@ -185,7 +187,7 @@ impl Openh264 { } else { tracing::trace!(decoder = NAME, state = format_args!("{state:#06x}"), "picture lost"); } - Ok(Vec::new()) + Ok(()) } /// Takes the timestamp the next picture out belongs to, reporting a recovery @@ -213,9 +215,9 @@ impl Openh264 { } } - /// The pictures the decoder still holds, each stamped with its access unit's - /// timestamp. - fn drain_tail(&mut self) -> Result, Error> { + /// Queue the pictures the decoder still holds, each stamped with its access + /// unit's timestamp. + fn drain_tail(&mut self) -> Result<(), Error> { let tail = self .decoder .flush_remaining() @@ -223,31 +225,37 @@ impl Openh264 { // `tail` borrows the decoder, so copy the pictures before taking their // timestamps. The placeholder is replaced before any frame reaches a // caller; an unstamped picture is dropped below. - let decoded: Vec = tail - .iter() - .map(|yuv| picture(yuv, Timestamp::ZERO)) - .collect::>()?; - drop(tail); - - let mut frames = Vec::with_capacity(decoded.len()); - let mut unstamped = 0usize; - for mut frame in decoded { - match self.picture_out() { - Some(timestamp) => { - frame.timestamp = timestamp; - frames.push(frame); + let start = self.ready.len(); + for yuv in tail.iter() { + match picture(yuv, Timestamp::ZERO) { + Ok(frame) => self.ready.push_back(frame), + Err(err) => { + self.ready.truncate(start); + return Err(err); } - None => unstamped += 1, } } + drop(tail); + + // Once the timestamps run out they stay out, so the unstamped pictures are + // the trailing ones. + let mut stamped = start; + while stamped < self.ready.len() + && let Some(timestamp) = self.picture_out() + { + self.ready[stamped].timestamp = timestamp; + stamped += 1; + } + let unstamped = self.ready.len() - stamped; if unstamped > 0 { + self.ready.truncate(stamped); tracing::warn!( decoder = NAME, dropped = unstamped, "flushed pictures had no timestamp waiting for them" ); } - Ok(frames) + Ok(()) } } @@ -273,7 +281,7 @@ fn picture(yuv: &impl YUVSource, timestamp: Timestamp) -> Result { } impl Backend for Openh264 { - fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result, Error> { + fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result<(), Error> { self.picture_in(timestamp); // `None` means the decoder took the access unit but has no picture yet: @@ -285,19 +293,21 @@ impl Backend for Openh264 { Err(err) => return self.picture_lost(timestamp, &err), }; - match decoded { - Some(mut frame) => { - if let Some(timestamp) = self.picture_out() { - frame.timestamp = timestamp; - } - Ok(vec![frame]) + if let Some(mut frame) = decoded { + if let Some(timestamp) = self.picture_out() { + frame.timestamp = timestamp; } - None => Ok(Vec::new()), + self.ready.push_back(frame); } + Ok(()) + } + + fn receive(&mut self) -> Result, Error> { + Ok(self.ready.pop_front()) } - /// Return the reordered pictures still buffered at the end of the stream. - fn flush(&mut self) -> Result, Error> { + /// Queue the reordered pictures still buffered at the end of the stream. + fn flush(&mut self) -> Result<(), Error> { let tail = self.drain_tail(); self.pending.clear(); self.lost = 0; @@ -397,11 +407,11 @@ mod tests { }; frames.extend( decoder - .decode(payload, at(index), keyframe) + .decode_all(payload, at(index), keyframe) .unwrap_or_else(|e| panic!("access unit {index} ended the stream: {e}")), ); } - frames.extend(decoder.flush().expect("the drain works")); + frames.extend(decoder.flush_all().expect("the drain works")); frames } @@ -419,11 +429,11 @@ mod tests { let mut decoded = 0; for (index, (payload, keyframe)) in access_units().into_iter().enumerate() { decoded += decoder - .decode(payload, at(index), keyframe) + .decode_all(payload, at(index), keyframe) .unwrap_or_else(|e| panic!("round {round} access unit {index}: {e}")) .len(); } - decoded += decoder.flush().expect("the drain works").len(); + decoded += decoder.flush_all().expect("the drain works").len(); assert_eq!(decoded, PICTURES, "round {round} lost pictures"); } } @@ -437,11 +447,11 @@ mod tests { let mut decoder = open(); let mut timestamps = Vec::new(); for (index, (payload, keyframe)) in access_units().into_iter().enumerate() { - for frame in decoder.decode(payload, at(index), keyframe).expect("decodes") { + for frame in decoder.decode_all(payload, at(index), keyframe).expect("decodes") { timestamps.push(frame.timestamp.as_micros()); } } - for frame in decoder.flush().expect("the drain works") { + for frame in decoder.flush_all().expect("the drain works") { timestamps.push(frame.timestamp.as_micros()); } @@ -465,7 +475,7 @@ mod tests { payload }; let frames = decoder - .decode(payload, at(index), keyframe) + .decode_all(payload, at(index), keyframe) .unwrap_or_else(|e| panic!("access unit {index} ended the stream: {e}")) .len(); decoded += frames; diff --git a/rs/moq-video/src/decode/backend/probe.rs b/rs/moq-video/src/decode/backend/probe.rs index d9d5fb4d05e..2a48688f44f 100644 --- a/rs/moq-video/src/decode/backend/probe.rs +++ b/rs/moq-video/src/decode/backend/probe.rs @@ -8,6 +8,7 @@ //! Reachable only through [`Kind::Named`](super::Kind), never through `Auto` / //! `Hardware` / `Software`, so it can't be picked by accident. +use std::collections::VecDeque; use std::sync::Mutex; use std::sync::atomic::{AtomicBool, Ordering}; use std::thread::ThreadId; @@ -141,15 +142,18 @@ pub(crate) const SIZE: Size = Size { height: 240, }; -pub(crate) struct Probe; +pub(crate) struct Probe(VecDeque); -pub(crate) struct Buffered(Option); +pub(crate) struct Buffered { + held: Option, + ready: VecDeque, +} -pub(crate) struct Immediate; +pub(crate) struct Immediate(VecDeque); -pub(crate) struct Single; +pub(crate) struct Single(VecDeque); -pub(crate) struct Native; +pub(crate) struct Native(VecDeque); pub(crate) struct Refusing; @@ -159,32 +163,40 @@ pub(crate) struct BlockingFlush; impl Probe { pub(crate) fn open(_stream: &Stream, _config: &Config) -> Result, Error> { record("open"); - Ok(Box::new(Self)) + Ok(Box::new(Self(VecDeque::new()))) } } impl Buffered { pub(crate) fn open(_stream: &Stream, _config: &Config) -> Result, Error> { - Ok(Box::new(Self(None))) + Ok(Box::new(Self { + held: None, + ready: VecDeque::new(), + })) } } impl Immediate { pub(crate) fn open(_stream: &Stream, _config: &Config) -> Result, Error> { - Ok(Box::new(Self)) + Ok(Box::new(Self(VecDeque::new()))) } } impl Backend for Immediate { - fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result, Error> { + fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result<(), Error> { if access_unit == CORRUPT { return Err(anyhow::anyhow!("probe: corrupt access unit").into()); } - Ok(vec![frame(timestamp)?]) + self.0.push_back(frame(timestamp)?); + Ok(()) + } + + fn receive(&mut self) -> Result, Error> { + Ok(self.0.pop_front()) } - fn flush(&mut self) -> Result, Error> { - Ok(Vec::new()) + fn flush(&mut self) -> Result<(), Error> { + Ok(()) } fn name(&self) -> &str { @@ -197,17 +209,22 @@ impl Single { if SINGLE_OPEN.swap(true, Ordering::SeqCst) { return Err(anyhow::anyhow!("probe: out of decoder sessions").into()); } - Ok(Box::new(Self)) + Ok(Box::new(Self(VecDeque::new()))) } } impl Backend for Single { - fn decode(&mut self, _access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result, Error> { - Ok(vec![frame(timestamp)?]) + fn decode(&mut self, _access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result<(), Error> { + self.0.push_back(frame(timestamp)?); + Ok(()) + } + + fn receive(&mut self) -> Result, Error> { + Ok(self.0.pop_front()) } - fn flush(&mut self) -> Result, Error> { - Ok(Vec::new()) + fn flush(&mut self) -> Result<(), Error> { + Ok(()) } fn name(&self) -> &str { @@ -224,7 +241,7 @@ impl Drop for Single { impl Native { pub(crate) fn open(_stream: &Stream, config: &Config) -> Result, Error> { *NATIVE_OPENED.lock().unwrap() = Some(config.clone()); - Ok(Box::new(Self)) + Ok(Box::new(Self(VecDeque::new()))) } /// A mid-gray picture at [`SIZE`] in the most native representation this @@ -246,12 +263,17 @@ impl Native { } impl Backend for Native { - fn decode(&mut self, _access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result, Error> { - Ok(vec![Self::frame(timestamp)?]) + fn decode(&mut self, _access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result<(), Error> { + self.0.push_back(Self::frame(timestamp)?); + Ok(()) } - fn flush(&mut self) -> Result, Error> { - Ok(Vec::new()) + fn receive(&mut self) -> Result, Error> { + Ok(self.0.pop_front()) + } + + fn flush(&mut self) -> Result<(), Error> { + Ok(()) } fn name(&self) -> &str { @@ -283,13 +305,18 @@ fn frame(timestamp: Timestamp) -> Result { impl Backend for Probe { /// One mid-gray frame per access unit, carrying the timestamp it came in /// with, so a test can tell which payload a frame came from. - fn decode(&mut self, _access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result, Error> { + fn decode(&mut self, _access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result<(), Error> { record("decode"); - Ok(vec![frame(timestamp)?]) + self.0.push_back(frame(timestamp)?); + Ok(()) + } + + fn receive(&mut self) -> Result, Error> { + Ok(self.0.pop_front()) } - fn flush(&mut self) -> Result, Error> { - Ok(Vec::new()) + fn flush(&mut self) -> Result<(), Error> { + Ok(()) } fn name(&self) -> &str { @@ -299,17 +326,21 @@ impl Backend for Probe { #[cfg(not(apple))] impl Backend for BlockingFlush { - fn decode(&mut self, _access_unit: Bytes, _timestamp: Timestamp, _keyframe: bool) -> Result, Error> { - Ok(Vec::new()) + fn decode(&mut self, _access_unit: Bytes, _timestamp: Timestamp, _keyframe: bool) -> Result<(), Error> { + Ok(()) } - fn flush(&mut self) -> Result, Error> { + fn receive(&mut self) -> Result, Error> { + Ok(None) + } + + fn flush(&mut self) -> Result<(), Error> { FLUSH_ENTERED.store(true, Ordering::SeqCst); FLUSH_ENTERING.notify_waiters(); while !FLUSH_RELEASED.load(Ordering::SeqCst) { std::thread::yield_now(); } - Ok(Vec::new()) + Ok(()) } fn name(&self) -> &str { @@ -318,12 +349,18 @@ impl Backend for BlockingFlush { } impl Backend for Buffered { - fn decode(&mut self, _access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result, Error> { - Ok(self.0.replace(frame(timestamp)?).into_iter().collect()) + fn decode(&mut self, _access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result<(), Error> { + self.ready.extend(self.held.replace(frame(timestamp)?)); + Ok(()) + } + + fn receive(&mut self) -> Result, Error> { + Ok(self.ready.pop_front()) } - fn flush(&mut self) -> Result, Error> { - Ok(self.0.take().into_iter().collect()) + fn flush(&mut self) -> Result<(), Error> { + self.ready.extend(self.held.take()); + Ok(()) } fn name(&self) -> &str { diff --git a/rs/moq-video/src/decode/backend/v4l2.rs b/rs/moq-video/src/decode/backend/v4l2.rs index c2c74967f4c..6060443c147 100644 --- a/rs/moq-video/src/decode/backend/v4l2.rs +++ b/rs/moq-video/src/decode/backend/v4l2.rs @@ -111,6 +111,8 @@ pub(crate) struct V4l2 { /// microseconds, and a [`Timestamp`] is an instant at its own scale, which a /// round trip through microseconds would change. submitted: VecDeque<(Duration, Timestamp)>, + /// Pictures copied out of the driver, waiting for `receive`. + ready: VecDeque, } /// Where the decoder stands, with what each stage needs. @@ -184,6 +186,7 @@ impl V4l2 { coded, phase: Phase::Sizing { since: None }, submitted: VecDeque::new(), + ready: VecDeque::new(), })) } @@ -334,13 +337,13 @@ impl V4l2 { ) } - /// Collect every picture the driver has finished, re-queueing each buffer as + /// Queue every picture the driver has finished, re-queueing each buffer as /// it is copied out. /// /// Returns whether the sequence ended, which the driver says with /// `V4L2_BUF_FLAG_LAST` on its last picture and then with `EPIPE` on any /// dequeue past it. - fn drain(&mut self, frames: &mut Vec) -> Result { + fn drain(&mut self) -> Result { let Phase::Decoding(pictures) = &self.phase else { return Ok(false); }; @@ -375,7 +378,7 @@ impl V4l2 { if let Some(decoded) = decoded { let timestamp = restore(&mut self.submitted, buffer.timestamp)?; - frames.push(Frame::new(Surface::I420(decoded), timestamp)); + self.ready.push_back(Frame::new(Surface::I420(decoded), timestamp)); } if buffer.last() { return Ok(true); @@ -383,7 +386,7 @@ impl V4l2 { } } - /// Take the pictures still in the CAPTURE queue when a sequence ends. + /// Queue the pictures still in the CAPTURE queue when a sequence ends. /// /// A source change is an implicit drain: the driver decodes everything from /// before the change, marks the last of it with `V4L2_BUF_FLAG_LAST`, and only @@ -392,12 +395,12 @@ impl V4l2 { /// is a visible gap at each mid-stream resolution change. See /// `Documentation/userspace-api/media/v4l/dev-decoder.rst`, "Dynamic Resolution /// Change". - fn drain_tail(&mut self) -> Result, Error> { - let mut frames = Vec::new(); + fn drain_tail(&mut self) -> Result<(), Error> { + let start = self.ready.len(); let deadline = Instant::now() + DRAIN_TIMEOUT; loop { - if self.drain(&mut frames)? { - return Ok(frames); + if self.drain()? { + return Ok(()); } if Instant::now() >= deadline { // Renegotiated anyway: a tail the driver never finished is worth less @@ -405,10 +408,10 @@ impl V4l2 { // make it arrive. tracing::warn!( decoder = NAME, - frames = frames.len(), + frames = self.ready.len() - start, "V4L2 decoder did not end the sequence within {DRAIN_TIMEOUT:?}" ); - return Ok(frames); + return Ok(()); } self.device.wait(POLL_INTERVAL); } @@ -419,17 +422,16 @@ impl V4l2 { /// Returns whether a source change arrived during the drain. That event stops /// the decoder at the old format, so the caller must renegotiate before it can /// continue through any access units queued for the new format. - fn drain_sequence(&mut self) -> Result<(Vec, bool), Error> { - let mut frames = Vec::new(); + fn drain_sequence(&mut self) -> Result { let mut ended = false; let mut changed = false; let mut deadline = Instant::now() + DRAIN_TIMEOUT; loop { - let before = frames.len(); + let before = self.ready.len(); let reclaimed = self.reclaim_coded()?; changed |= self.device.take_source_change(); if !ended { - ended = self.drain(&mut frames)?; + ended = self.drain()?; } // A source change stops the old sequence at LAST while access units for @@ -437,10 +439,10 @@ impl V4l2 { // continue, so do not wait here for buffers the stopped decoder cannot // return yet. if ended && (changed || self.coded.outstanding() == 0) { - return Ok((frames, changed)); + return Ok(changed); } - if reclaimed > 0 || frames.len() > before { + if reclaimed > 0 || self.ready.len() > before { deadline = Instant::now() + DRAIN_TIMEOUT; } else if Instant::now() >= deadline { return Err(Error::codec(anyhow::anyhow!( @@ -484,18 +486,14 @@ impl V4l2 { Ok(()) } - /// End the stream and drain it, returning the decoder ready for the next + /// End the stream and drain it, leaving the decoder ready for the next /// stream's keyframe or [`Phase::Failed`]. - fn drain_stream(&mut self) -> Result, Error> { - let mut frames = Vec::new(); + fn drain_stream(&mut self) -> Result<(), Error> { loop { // Refused, no drain began, so the decoder carries on as it was. self.device.decoder_cmd(V4L2_DEC_CMD_STOP)?; let changed = match self.drain_sequence() { - Ok((tail, changed)) => { - frames.extend(tail); - changed - } + Ok(changed) => changed, Err(err) => { self.reset()?; return Err(err); @@ -514,7 +512,7 @@ impl V4l2 { // even after every OUTPUT buffer was returned. Resume it, then always run // a fresh drain so the flush still covers everything submitted before it. if !changed { - return Ok(frames); + return Ok(()); } } } @@ -550,7 +548,7 @@ fn restore(submitted: &mut VecDeque<(Duration, Timestamp)>, key: Duration) -> Re } impl Backend for V4l2 { - fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result, Error> { + fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result<(), Error> { if let Phase::Failed = self.phase { return Err(failed()); } @@ -560,7 +558,6 @@ impl Backend for V4l2 { // event is worth waiting for. After it, checking costs one ioctl and a // missed resolution change would decode the rest of the stream at the old // geometry. - let mut frames = Vec::new(); if let Phase::Sizing { since } = &mut self.phase { let since = *since.get_or_insert_with(Instant::now); let deadline = Instant::now() + SOURCE_CHANGE_TIMEOUT; @@ -577,7 +574,7 @@ impl Backend for V4l2 { ?waited, "V4L2 decoder has not reported the stream's size" ); - return Ok(Vec::new()); + return Ok(()); } self.device.wait(POLL_INTERVAL); } @@ -585,15 +582,19 @@ impl Backend for V4l2 { } else if self.device.take_source_change() { // The pictures decoded before the change are still in the CAPTURE queue, // and the kernel wants them taken before the queue is released. - frames = self.drain_tail()?; + self.drain_tail()?; self.negotiate()?; } - self.drain(&mut frames)?; - Ok(frames) + self.drain()?; + Ok(()) + } + + fn receive(&mut self) -> Result, Error> { + Ok(self.ready.pop_front()) } - fn flush(&mut self) -> Result, Error> { + fn flush(&mut self) -> Result<(), Error> { match self.phase { Phase::Failed => Err(failed()), // A drain only begins with both queues streaming. Before the driver has @@ -604,12 +605,12 @@ impl Backend for V4l2 { // them, so none can surface after the next stream's keyframe. self.reset()?; self.phase = Phase::Sizing { since: None }; - Ok(Vec::new()) + Ok(()) } Phase::Decoding(_) => { - let frames = self.drain_stream()?; + self.drain_stream()?; self.submitted.clear(); - Ok(frames) + Ok(()) } } } diff --git a/rs/moq-video/src/decode/backend/vaapi.rs b/rs/moq-video/src/decode/backend/vaapi.rs index ef15cfc370c..d7269581e35 100644 --- a/rs/moq-video/src/decode/backend/vaapi.rs +++ b/rs/moq-video/src/decode/backend/vaapi.rs @@ -56,6 +56,7 @@ //! `moq-vaapi` keeps the surface alongside the descriptor and reads it back //! through `vaDeriveImage` rather than trying to read a tiled buffer as rows. +use std::collections::VecDeque; use std::path::{Path, PathBuf}; use bytes::Bytes; @@ -72,6 +73,7 @@ pub(crate) const NAME: &str = "vaapi"; pub(crate) struct Vaapi { decoder: Decoder, export: Export, + ready: VecDeque, } /// Whether pictures are handed out as DMA-BUFs rather than downloaded. @@ -148,12 +150,16 @@ impl Vaapi { exporting, "opened decoder" ); - Ok(Box::new(Self { decoder, export })) + Ok(Box::new(Self { + decoder, + export, + ready: VecDeque::new(), + })) } /// Decodes one access unit into GPU-resident frames, or `None` once the /// driver has shown it will not export. - fn decode_shared(&mut self, access_unit: &Bytes, timestamp: u64) -> Option, Error>> { + fn decode_shared(&mut self, access_unit: &Bytes, timestamp: u64) -> Option> { if self.export == Export::Downloading { return None; } @@ -164,7 +170,7 @@ impl Vaapi { /// The same for the stream's tail, so a track that ends does not switch to /// downloading for its last few pictures. - fn flush_shared(&mut self) -> Option, Error>> { + fn flush_shared(&mut self) -> Option> { if self.export == Export::Downloading { return None; } @@ -173,8 +179,8 @@ impl Vaapi { Some(self.exported(exported)) } - /// Hands back the pictures a shared decode produced, and decides what a - /// failure to produce any means. + /// Queues the pictures a shared decode produced, and decides what a failure + /// to produce any means. /// /// A driver can decode without being able to share what it decoded, and /// native output permits CPU pictures, so until one picture @@ -187,7 +193,7 @@ impl Vaapi { /// is a decode error rather than a verdict on the driver. Reporting it as /// one keeps a corrupt access unit from silently costing the rest of the /// session its fast path. - fn exported(&mut self, exported: anyhow::Result>) -> Result, Error> { + fn exported(&mut self, exported: anyhow::Result>) -> Result<(), Error> { match exported { Ok(frames) => { // An access unit whose pictures are all still in the DPB proves @@ -195,7 +201,8 @@ impl Vaapi { if !frames.is_empty() { self.export = Export::Exporting; } - Ok(frames) + self.ready.extend(frames); + Ok(()) } Err(err) if self.export == Export::Exporting => { Err(Error::codec(err.context("VAAPI decode to a shared surface"))) @@ -203,14 +210,14 @@ impl Vaapi { Err(err) => { tracing::warn!(%err, "VAAPI cannot hand out decoded surfaces; downloading them instead"); self.export = Export::Downloading; - Ok(Vec::new()) + Ok(()) } } } } impl Backend for Vaapi { - fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result, Error> { + fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result<(), Error> { // The timestamp rides through the decoder with the picture, so it // survives the DPB reordering a stream with B-frames goes through. let timestamp = timestamp.as_micros() as u64; @@ -223,11 +230,15 @@ impl Backend for Vaapi { .decode(&access_unit, timestamp) .map_err(|e| Error::codec(anyhow::anyhow!("VAAPI decode: {e:?}")))?; - convert(decoded) + self.convert(decoded) } - /// Return the tail held by the DPB at the end of the stream. - fn flush(&mut self) -> Result, Error> { + fn receive(&mut self) -> Result, Error> { + Ok(self.ready.pop_front()) + } + + /// Queue the tail held by the DPB at the end of the stream. + fn flush(&mut self) -> Result<(), Error> { if let Some(frames) = self.flush_shared() { return frames; } @@ -237,7 +248,7 @@ impl Backend for Vaapi { .flush() .map_err(|e| Error::codec(anyhow::anyhow!("VAAPI flush: {e:?}")))?; - convert(decoded) + self.convert(decoded) } fn name(&self) -> &str { @@ -285,17 +296,17 @@ fn offered(codec: moq_vaapi::Codec, named: Option<&Path>, profile: VAProfile::Ty Some(false) } -/// Deinterleave each decoded picture's NV12 into the CPU I420 the rest of the -/// crate speaks, keeping the timestamp the picture was coded with. -fn convert(decoded: Vec) -> Result, Error> { - decoded - .into_iter() - .map(|frame| { +impl Vaapi { + /// Deinterleave each decoded picture's NV12 into the CPU I420 the rest of the + /// crate speaks, keeping the timestamp the picture was coded with. + fn convert(&mut self, decoded: Vec) -> Result<(), Error> { + for frame in decoded { let i420 = I420::from_nv12(&frame.data, crate::Size::new(frame.width, frame.height))?; let timestamp = Timestamp::from_micros(frame.timestamp).unwrap_or(Timestamp::ZERO); - Ok(Frame::new(Surface::I420(i420), timestamp)) - }) - .collect() + self.ready.push_back(Frame::new(Surface::I420(i420), timestamp)); + } + Ok(()) + } } /// Wrap each exported picture as the DMA-BUF surface a renderer imports, keeping @@ -415,8 +426,8 @@ mod tests { } let surface = Surface::rgba(&rgba, crate::Size::new(w, h)).unwrap(); let frame = Frame::new(surface, Timestamp::from_micros(i * 33_333).unwrap()); - for encoded in encoder.encode(&frame).unwrap() { - decoded.extend(decoder.decode(encoded.payload, encoded.timestamp, i == 0).unwrap()); + for encoded in encoder.encode_all(&frame).unwrap() { + decoded.extend(decoder.decode_all(encoded.payload, encoded.timestamp, i == 0).unwrap()); } } @@ -470,10 +481,10 @@ mod tests { } let surface = Surface::rgba(&rgba, crate::Size::new(w, h)).unwrap(); let frame = Frame::new(surface, Timestamp::from_micros(i * 33_333).unwrap()); - for encoded in encoder.encode(&frame).unwrap() { + for encoded in encoder.encode_all(&frame).unwrap() { let (payload, timestamp) = (encoded.payload, encoded.timestamp); - exported.extend(exporting.decode(payload.clone(), timestamp, i == 0).unwrap()); - downloaded.extend(downloading.decode(payload, timestamp, i == 0).unwrap()); + exported.extend(exporting.decode_all(payload.clone(), timestamp, i == 0).unwrap()); + downloaded.extend(downloading.decode_all(payload, timestamp, i == 0).unwrap()); } } assert!(!exported.is_empty(), "VAAPI produced no frames"); @@ -558,8 +569,8 @@ mod tests { } let surface = Surface::rgba(&rgba, crate::Size::new(w, h)).unwrap(); let frame = Frame::new(surface, Timestamp::from_micros(i * 33_333).unwrap()); - for encoded in encoder.encode(&frame).unwrap() { - streamed.extend(decoder.decode(encoded.payload, encoded.timestamp, i == 0).unwrap()); + for encoded in encoder.encode_all(&frame).unwrap() { + streamed.extend(decoder.decode_all(encoded.payload, encoded.timestamp, i == 0).unwrap()); } } assert!( @@ -567,7 +578,7 @@ mod tests { "the DPB held nothing back, so this test proves nothing" ); - let flushed = decoder.flush().unwrap(); + let flushed = decoder.flush_all().unwrap(); let timestamps: Vec = streamed .iter() .chain(&flushed) @@ -578,6 +589,6 @@ mod tests { // A second flush has nothing left to hand back, so the drain is idempotent // and a caller that flushes twice does not see a picture twice. - assert!(decoder.flush().unwrap().is_empty()); + assert!(decoder.flush_all().unwrap().is_empty()); } } diff --git a/rs/moq-video/src/decode/backend/videotoolbox.rs b/rs/moq-video/src/decode/backend/videotoolbox.rs index d1f24a1be6d..6a77ab04923 100644 --- a/rs/moq-video/src/decode/backend/videotoolbox.rs +++ b/rs/moq-video/src/decode/backend/videotoolbox.rs @@ -151,6 +151,8 @@ pub(crate) struct VideoToolbox { /// VideoToolbox returns through the callback. pending: VecDeque<(i64, Timestamp)>, sink: Box, + /// Pictures paired with their timestamps, waiting for `receive`. + ready: VecDeque, } impl VideoToolbox { @@ -180,6 +182,7 @@ impl VideoToolbox { sample_index: 0, pending: VecDeque::new(), sink: Box::new(Sink::default()), + ready: VecDeque::new(), }; if let Some(parameter_sets) = &stream.parameter_sets { let split = Split::new(codec, parameter_sets.clone())?; @@ -292,20 +295,23 @@ impl VideoToolbox { Ok(self.pending.remove(index).expect("index found above").1) } - /// Convert every callback result collected by the last decode or drain. - fn take_frames(&mut self) -> Result, Error> { - std::mem::take(&mut self.sink.frames) - .into_iter() - .map(|(surface, sample_time)| { - let timestamp = self.take_timestamp(sample_time)?; - Ok(Frame::new(Surface::PixelBuffer(surface), timestamp)) - }) - .collect() + /// Queue every callback result collected by the last decode or drain. + fn take_frames(&mut self) -> Result<(), Error> { + // Taken out and put back so the callback's buffer keeps its capacity. + let mut frames = std::mem::take(&mut self.sink.frames); + let queued = frames.drain(..).try_for_each(|(surface, sample_time)| { + let timestamp = self.take_timestamp(sample_time)?; + self.ready.push_back(Frame::new(Surface::PixelBuffer(surface), timestamp)); + Ok(()) + }); + frames.clear(); + self.sink.frames = frames; + queued } } impl Backend for VideoToolbox { - fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result, Error> { + fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result<(), Error> { // Split the Annex-B access unit, pull out any parameter sets, and gather // the slices into length-prefixed (4-byte) form. let Split { vps, sps, pps, avcc } = Split::new(self.codec, access_unit)?; @@ -317,7 +323,7 @@ impl Backend for VideoToolbox { self.ensure_session(vps, sps, pps)?; // No parameter sets yet (e.g. a delta frame before the first keyframe). let Some(live) = &self.session else { - return Ok(Vec::new()); + return Ok(()); }; // A parameter-set-only access unit has nothing to decode. @@ -350,7 +356,11 @@ impl Backend for VideoToolbox { self.take_frames() } - fn flush(&mut self) -> Result, Error> { + fn receive(&mut self) -> Result, Error> { + Ok(self.ready.pop_front()) + } + + fn flush(&mut self) -> Result<(), Error> { self.sink.frames.clear(); self.sink.error = None; diff --git a/rs/moq-video/src/decode/backend/vpx.rs b/rs/moq-video/src/decode/backend/vpx.rs index 1d3a71347af..db23729f34c 100644 --- a/rs/moq-video/src/decode/backend/vpx.rs +++ b/rs/moq-video/src/decode/backend/vpx.rs @@ -9,6 +9,7 @@ //! back a different picture than the stream coded, so a picture in any other //! format is refused rather than converted. +use std::collections::VecDeque; use std::ffi::CStr; use std::ptr; @@ -32,6 +33,7 @@ pub(crate) struct Vpx { ctx: vpx_codec_ctx_t, codec: Codec, chain: Chain, + ready: VecDeque, } /// Whether the decoder holds the references the next delta frame needs. @@ -90,6 +92,7 @@ impl Vpx { ctx, codec, chain: Chain::Intact, + ready: VecDeque::new(), }) } @@ -228,10 +231,10 @@ fn picture(codec: Codec, img: &vpx_image_t, timestamp: Timestamp) -> Result Result, Error> { + fn decode(&mut self, frame: Bytes, timestamp: Timestamp, keyframe: bool) -> Result<(), Error> { if matches!(self.chain, Chain::Broken { .. }) && !keyframe { self.picture_lost("reference chain broken"); - return Ok(Vec::new()); + return Ok(()); } let len = u32::try_from(frame.len()).map_err(|_| { @@ -249,11 +252,11 @@ impl Backend for Vpx { return Err(Error::codec(anyhow::anyhow!("{NAME} decode: {reason}"))); } self.picture_lost(&reason); - return Ok(Vec::new()); + return Ok(()); } if self.corrupted()? { self.picture_lost("corrupt frame"); - return Ok(Vec::new()); + return Ok(()); } // Only now the keyframe has decoded whole: one refused on the way back // keeps counting into the same run. @@ -265,18 +268,21 @@ impl Backend for Vpx { // Without frame threading a call hands back at most the one picture it // shows, so the input timestamp is that picture's. A frame that is not // shown (a VP8 alt-ref coded on its own) hands back none. - let mut frames = Vec::new(); let mut iter: vpx_codec_iter_t = ptr::null(); // SAFETY: each image stays valid until the next decode call. while let Some(img) = unsafe { vpx_codec_get_frame(&mut self.ctx, &mut iter).as_ref() } { - frames.push(picture(self.codec, img, timestamp)?); + self.ready.push_back(picture(self.codec, img, timestamp)?); } - Ok(frames) + Ok(()) + } + + fn receive(&mut self) -> Result, Error> { + Ok(self.ready.pop_front()) } /// libvpx decodes without delay here, so nothing is ever held back. - fn flush(&mut self) -> Result, Error> { - Ok(Vec::new()) + fn flush(&mut self) -> Result<(), Error> { + Ok(()) } fn name(&self) -> &str { @@ -427,13 +433,13 @@ mod tests { for (index, frame) in frames(vector.ivf).into_iter().enumerate() { let key = keyframe(vector.codec, &frame); let pictures = decoder - .decode(damage(index, frame), at(index), key) + .decode_all(damage(index, frame), at(index), key) .unwrap_or_else(|e| panic!("{} frame {index} ended the stream: {e}", vector.name)); out.extend(pictures.into_iter().map(|picture| (index, picture))); } out.extend( decoder - .flush() + .flush_all() .unwrap() .into_iter() .map(|picture| (usize::MAX, picture)), @@ -561,7 +567,7 @@ mod tests { // The two-bit frame marker (VP9 section 6.2). _ => frame[0] = 0, } - match open(vector.codec).decode(Bytes::from(frame), at(0), true) { + match open(vector.codec).decode_all(Bytes::from(frame), at(0), true) { Err(Error::Codec(_)) => {} Err(other) => panic!("{}: expected Codec, got {other:?}", vector.name), Ok(frames) => panic!("{}: decoded {} pictures", vector.name, frames.len()), @@ -584,7 +590,7 @@ mod tests { #[test] fn a_picture_that_is_not_i420_is_refused() { let frame = frames(VP9_PROFILE1).remove(0); - match open(Codec::Vp9).decode(frame, at(0), true) { + match open(Codec::Vp9).decode_all(frame, at(0), true) { Err(Error::UnsupportedCodec(reason)) => assert!(reason.contains("I444"), "{reason}"), Err(other) => panic!("expected UnsupportedCodec, got {other:?}"), Ok(frames) => panic!("expected UnsupportedCodec, decoded {} pictures", frames.len()), @@ -646,7 +652,7 @@ mod tests { let key = keyframe(Codec::Vp9, &frame); decoded.extend( decoder - .decode(&frame, at(index), key) + .decode_all(&frame, at(index), key) .unwrap() .into_iter() .map(|f| (index, f)), diff --git a/rs/moq-video/src/decode/conformance.rs b/rs/moq-video/src/decode/conformance.rs index 2a20607ffb6..fd197e20154 100644 --- a/rs/moq-video/src/decode/conformance.rs +++ b/rs/moq-video/src/decode/conformance.rs @@ -256,7 +256,7 @@ fn decode(name: &str, vector: &Vector) -> Vec { let mut decoded = Vec::new(); for unit in access_units(vector) { let frames = decoder - .decode(unit.payload, unit.timestamp, unit.keyframe) + .decode_all(unit.payload, unit.timestamp, unit.keyframe) .unwrap_or_else(|e| panic!("{name} failed to decode {}: {e}", vector.name)); decoded.extend(frames); } diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index c2357cf62e5..ea0c664c0f8 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -419,7 +419,8 @@ impl Reader { // nothing in between to wait on. A caller that treats a codec // error as one lost picture and carries on then spins. self.drained = true; - self.pending.extend(tail?.into_iter().map(Read::Frame)); + tail?; + self.receive()?; continue; }; @@ -431,19 +432,25 @@ impl Reader { continue; } - let pictures = self - .decoder + self.decoder .decode(mux_frame.payload, mux_frame.timestamp, mux_frame.keyframe) .await?; - self.pending.extend(pictures.into_iter().map(Read::Frame)); + self.receive()?; } } + /// Queue every picture the decoder has ready. + fn receive(&mut self) -> Result<(), Error> { + while let Some(picture) = self.decoder.receive()? { + self.pending.push_back(Read::Frame(picture)); + } + Ok(()) + } + /// Queue every picture the decoder still holds. async fn drain(&mut self) -> Result<(), Error> { - let pictures = self.decoder.flush().await?; - self.pending.extend(pictures.into_iter().map(Read::Frame)); - Ok(()) + self.decoder.flush().await?; + self.receive() } } @@ -498,7 +505,9 @@ mod tests { encoder.cut().unwrap(); let surface = crate::Surface::rgba(&rgba, crate::Size::new(320, 240)).unwrap(); let frame = crate::Frame::new(surface, moq_net::Timestamp::from_micros(index * 33_333).unwrap()); - producer.publish(&encoder.encode(&frame).unwrap()).unwrap(); + for encoded in encoder.encode_all(&frame).unwrap() { + producer.publish(&encoded).unwrap(); + } } let origin = produce_origin(); @@ -1075,7 +1084,7 @@ mod tests { } let surface = crate::Surface::rgba(&rgba, crate::Size::new(320, 240)).unwrap(); let frame = crate::Frame::new(surface, moq_net::Timestamp::from_micros(index * 33_333).unwrap()); - for encoded in encoder.encode(&frame).unwrap() { + for encoded in encoder.encode_all(&frame).unwrap() { producer .write(moq_mux::container::Frame { timestamp: encoded.timestamp, diff --git a/rs/moq-video/src/decode/decoder.rs b/rs/moq-video/src/decode/decoder.rs index b0c1f0e4bca..dc139f5beb9 100644 --- a/rs/moq-video/src/decode/decoder.rs +++ b/rs/moq-video/src/decode/decoder.rs @@ -174,6 +174,9 @@ pub struct Decoder { /// A reordering backend can still be holding pictures from before the /// switch, which keep the old space until output reaches the keyframe. switches: VecDeque<(Timestamp, Option)>, + /// Set by [`flush`](Self::flush) until the tail is received or more input + /// arrives, so a switch no picture reached still applies to the next stream. + flushed: bool, /// Keeps direct use bound to the constructing thread, regardless of backend. _thread_bound: PhantomData>, } @@ -258,6 +261,7 @@ impl Decoder { got_keyframe: false, color: declared(catalog), switches: VecDeque::new(), + flushed: false, _thread_bound: PhantomData, }) } @@ -267,29 +271,31 @@ impl Decoder { self.backend.name() } - /// Decode one container frame, returning zero or more raw frames. `timestamp` is - /// this frame's presentation time; it rides through the decoder and comes back on - /// each output frame, so a reordering decoder (B-frames) stamps every picture - /// with its own presentation time rather than this access unit's. With no - /// reordering the two coincide. - pub fn decode(&mut self, payload: &Bytes, timestamp: Timestamp, keyframe: bool) -> Result, Error> { + /// Submit one container frame, whose raw frames come out of + /// [`receive`](Self::receive). `timestamp` is this frame's presentation time; + /// it rides through the decoder and comes back on each output frame, so a + /// reordering decoder (B-frames) stamps every picture with its own + /// presentation time rather than this access unit's. With no reordering the + /// two coincide. + pub fn decode(&mut self, payload: &Bytes, timestamp: Timestamp, keyframe: bool) -> Result<(), Error> { // Wait for the first keyframe: a decoder started mid-GOP can't decode // delta frames, and the parameter sets ride along with the keyframe. if !self.got_keyframe && !keyframe { - return Ok(Vec::new()); + return Ok(()); } - let frames = self.submit(payload, timestamp, keyframe); + self.flushed = false; + let submitted = self.submit(payload, timestamp, keyframe); // Set by how the keyframe went rather than by its arrival: one that failed // leaves nothing for the deltas behind it to predict from. if keyframe { - self.got_keyframe = frames.is_ok(); + self.got_keyframe = submitted.is_ok(); } - self.deliver(frames?) + submitted } /// Convert one access unit and hand it to the backend. - fn submit(&mut self, payload: &Bytes, timestamp: Timestamp, keyframe: bool) -> Result, Error> { + fn submit(&mut self, payload: &Bytes, timestamp: Timestamp, keyframe: bool) -> Result<(), Error> { let access_unit = self.conversion.access_unit(payload, keyframe)?; if keyframe && let Some(color) = super::vui::signalled(self.codec, &access_unit) @@ -300,49 +306,80 @@ impl Decoder { self.backend.decode(access_unit, timestamp, keyframe) } - /// Return the frames the backend still holds once the stream has ended. + /// Pop the oldest ready raw frame, or `None` while the decoder holds the rest. /// - /// Call this after the last access unit and before dropping the decoder. The - /// decoder remains reusable and waits for a keyframe before accepting the - /// next stream. - pub fn flush(&mut self) -> Result, Error> { + /// Call until it returns `None` after each [`decode`](Self::decode) and + /// [`flush`](Self::flush). + pub fn receive(&mut self) -> Result, Error> { + let Some(frame) = self.backend.receive()? else { + // The tail is out, so a switch no picture reached still applies. + if std::mem::take(&mut self.flushed) + && let Some((_, color)) = self.switches.drain(..).next_back() + { + self.color = color; + } + return Ok(None); + }; + self.deliver(frame).map(Some) + } + + /// Mark the end of input, so [`receive`](Self::receive) returns the frames + /// the backend still holds once the stream has ended. + /// + /// Call this after the last access unit, and receive, before dropping the + /// decoder. The decoder remains reusable and waits for a keyframe before + /// accepting the next stream. + pub fn flush(&mut self) -> Result<(), Error> { self.got_keyframe = false; - let frames = self.backend.flush()?; - let frames = self.deliver(frames)?; - // Nothing is held any more, so a switch no picture reached still applies. - if let Some((_, color)) = self.switches.drain(..).next_back() { - self.color = color; - } - Ok(frames) + self.backend.flush()?; + self.flushed = true; + Ok(()) } - /// Put the backend's pictures in the representation [`Config::output`] - /// asked for. + /// Put a backend picture in the representation [`Config::output`] asked for. /// /// The one place the choice is enforced, so a backend only has to know /// about it when decoding to the CPU directly is cheaper than downloading /// afterwards (VAAPI). A surface with no CPU path fails here rather than /// arriving GPU-resident at a caller that said it could not take one. - fn deliver(&mut self, frames: Vec) -> Result, Error> { - frames - .into_iter() - .map(|frame| { - // Output is in presentation order, and nothing decoded after a - // keyframe is shown before it, so a picture's timestamp says which - // side of a switch it is on. - while let Some(&(at, color)) = self.switches.front() - && frame.timestamp >= at - { - self.color = color; - self.switches.pop_front(); - } - let mut surface = self.output.convert(frame.surface)?; - if let Some(color) = self.color { - surface.label(color); - } - Ok(Frame::new(surface, frame.timestamp)) - }) - .collect() + fn deliver(&mut self, frame: Frame) -> Result { + // Output is in presentation order, and nothing decoded after a keyframe is + // shown before it, so a picture's timestamp says which side of a switch it + // is on. + while let Some(&(at, color)) = self.switches.front() + && frame.timestamp >= at + { + self.color = color; + self.switches.pop_front(); + } + let mut surface = self.output.convert(frame.surface)?; + if let Some(color) = self.color { + surface.label(color); + } + Ok(Frame::new(surface, frame.timestamp)) + } + + /// Submit one container frame and collect everything it readied. + #[cfg(test)] + pub(crate) fn decode_all(&mut self, payload: &Bytes, timestamp: Timestamp, keyframe: bool) -> Result, Error> { + self.decode(payload, timestamp, keyframe)?; + self.receive_all() + } + + /// Flush and collect the tail. + #[cfg(test)] + pub(crate) fn flush_all(&mut self) -> Result, Error> { + self.flush()?; + self.receive_all() + } + + #[cfg(test)] + fn receive_all(&mut self) -> Result, Error> { + let mut frames = Vec::new(); + while let Some(frame) = self.receive()? { + frames.push(frame); + } + Ok(frames) } } @@ -492,11 +529,11 @@ mod tests { encoder.cut().unwrap(); } // Distinct, spread-apart timestamps so a round-tripped value is unambiguous. - for encoded in encoder.encode(&gray_frame(i)).unwrap() { - decoded.extend(decoder.decode(encoded.payload, encoded.timestamp, keyframe).unwrap()); + for encoded in encoder.encode_all(&gray_frame(i)).unwrap() { + decoded.extend(decoder.decode_all(encoded.payload, encoded.timestamp, keyframe).unwrap()); } } - decoded.extend(decoder.flush().unwrap()); + decoded.extend(decoder.flush_all().unwrap()); assert!(!decoded.is_empty(), "decoder produced no frames"); for out in &decoded { @@ -581,12 +618,12 @@ mod tests { if keyframe { encoder.cut().unwrap(); } - for encoded in encoder.encode(&gray_frame(i)).unwrap() { + for encoded in encoder.encode_all(&gray_frame(i)).unwrap() { assert!( encoded.payload.starts_with(&[0, 0, 0, 1]) || encoded.payload.starts_with(&[0, 0, 1]), "the test feeds Annex-B, not length-prefixed NALs" ); - decoded.extend(decoder.decode(&encoded.payload, encoded.timestamp, keyframe).unwrap()); + decoded.extend(decoder.decode_all(&encoded.payload, encoded.timestamp, keyframe).unwrap()); } } @@ -641,12 +678,12 @@ mod tests { if keyframe { encoder.cut().unwrap(); } - for encoded in encoder.encode(&gray_frame(i)).unwrap() { + for encoded in encoder.encode_all(&gray_frame(i)).unwrap() { let sample = length_prefixed(&encoded.payload); - decoded.extend(decoder.decode(&sample, encoded.timestamp, keyframe).unwrap()); + decoded.extend(decoder.decode_all(&sample, encoded.timestamp, keyframe).unwrap()); } } - decoded.extend(decoder.flush().unwrap()); + decoded.extend(decoder.flush_all().unwrap()); assert!(!decoded.is_empty(), "decoder produced no frames"); for out in &decoded { @@ -804,7 +841,7 @@ mod tests { Surface::I420(i420.with_color(color)), Timestamp::from_micros(0).unwrap(), ); - encoder.encode(&frame).unwrap().remove(0).payload + encoder.encode_all(&frame).unwrap().remove(0).payload }; let (first, second) = ( Timestamp::from_micros(0).unwrap(), @@ -813,12 +850,12 @@ mod tests { let kind = super::Kind::Named(probe::BUFFERED_NAME.into()); let mut decoder = super::Decoder::new(&probe_catalog(), &decode_config(kind)).unwrap(); - assert!(decoder.decode(&keyframe(Color::BT709), first, true).unwrap().is_empty()); - let held = decoder.decode(&keyframe(Color::BT601), second, true).unwrap(); + assert!(decoder.decode_all(&keyframe(Color::BT709), first, true).unwrap().is_empty()); + let held = decoder.decode_all(&keyframe(Color::BT601), second, true).unwrap(); assert_eq!(held.len(), 1); assert_eq!(held[0].timestamp, first); assert_eq!(held[0].surface.color(), Some(Color::BT709)); - let tail = decoder.flush().unwrap(); + let tail = decoder.flush_all().unwrap(); assert_eq!(tail[0].timestamp, second); assert_eq!(tail[0].surface.color(), Some(Color::BT601)); } @@ -840,9 +877,9 @@ mod tests { let mut decoder = super::Decoder::new(&catalog, &decode_config(kind)).unwrap(); let payload = bytes::Bytes::from_static(&[0]); decoder - .decode(&payload, Timestamp::from_micros(0).unwrap(), true) + .decode_all(&payload, Timestamp::from_micros(0).unwrap(), true) .unwrap(); - decoder.flush().unwrap().remove(0).surface.color() + decoder.flush_all().unwrap().remove(0).surface.color() }; let pq = hang::catalog::VP9 { @@ -870,17 +907,17 @@ mod tests { let at = |micros| Timestamp::from_micros(micros).unwrap(); let corrupt = bytes::Bytes::from_static(probe::CORRUPT); - assert!(decoder.decode(&corrupt, at(0), true).is_err()); + assert!(decoder.decode_all(&corrupt, at(0), true).is_err()); let delta = bytes::Bytes::from_static(&[0]); - assert!(decoder.decode(&delta, at(33_333), false).unwrap().is_empty()); + assert!(decoder.decode_all(&delta, at(33_333), false).unwrap().is_empty()); // The next keyframe that decodes opens it. - assert_eq!(decoder.decode(&delta, at(66_666), true).unwrap().len(), 1); - assert_eq!(decoder.decode(&delta, at(99_999), false).unwrap().len(), 1); + assert_eq!(decoder.decode_all(&delta, at(66_666), true).unwrap().len(), 1); + assert_eq!(decoder.decode_all(&delta, at(99_999), false).unwrap().len(), 1); // And one that fails mid-stream closes it again. - assert!(decoder.decode(&corrupt, at(133_332), true).is_err()); - assert!(decoder.decode(&delta, at(166_665), false).unwrap().is_empty()); + assert!(decoder.decode_all(&corrupt, at(133_332), true).is_err()); + assert!(decoder.decode_all(&delta, at(166_665), false).unwrap().is_empty()); } /// An inline-H.264 catalog the test probes accept. @@ -904,7 +941,7 @@ mod tests { }; let mut decoder = super::Decoder::new(&probe_catalog(), &config).expect("the native probe opens"); let mut frames = decoder - .decode( + .decode_all( &bytes::Bytes::from_static(b"access unit"), Timestamp::from_micros(0).unwrap(), true, @@ -1074,8 +1111,8 @@ mod tests { if keyframe { encoder.cut().unwrap(); } - for encoded in encoder.encode(&gray_frame(i)).unwrap() { - decoded.extend(decoder.decode(encoded.payload, encoded.timestamp, keyframe).unwrap()); + for encoded in encoder.encode_all(&gray_frame(i)).unwrap() { + decoded.extend(decoder.decode_all(encoded.payload, encoded.timestamp, keyframe).unwrap()); } } @@ -1135,9 +1172,9 @@ mod tests { if i == 0 { encoder.cut().unwrap(); } - packets += encoder.encode(out).unwrap().len(); + packets += encoder.encode_all(out).unwrap().len(); } - packets += encoder.finish().unwrap().len(); + packets += encoder.flush_all().unwrap().len(); assert!(packets > 0, "re-encoding decoded surfaces produced no packets"); } @@ -1240,8 +1277,8 @@ mod tests { if keyframe { encoder.cut().unwrap(); } - for encoded in encoder.encode(&flat_frame(i, level(i), size)).unwrap() { - decoded.extend(decoder.decode(encoded.payload, encoded.timestamp, keyframe).unwrap()); + for encoded in encoder.encode_all(&flat_frame(i, level(i), size)).unwrap() { + decoded.extend(decoder.decode_all(encoded.payload, encoded.timestamp, keyframe).unwrap()); } } @@ -1377,9 +1414,9 @@ mod tests { if i == 0 { encoder.cut().unwrap(); } - reencoded.extend(encoder.encode(out).unwrap()); + reencoded.extend(encoder.encode_all(out).unwrap()); } - reencoded.extend(encoder.finish().unwrap()); + reencoded.extend(encoder.flush_all().unwrap()); assert!( !reencoded.is_empty(), "re-encoding decoded textures produced no packets" @@ -1393,7 +1430,7 @@ mod tests { for (i, encoded) in reencoded.iter().enumerate() { out.extend( decoder - .decode(encoded.payload.clone(), encoded.timestamp, i == 0) + .decode_all(encoded.payload.clone(), encoded.timestamp, i == 0) .unwrap(), ); } @@ -1463,9 +1500,9 @@ mod tests { if i == 0 { encoder.cut().unwrap(); } - packets += encoder.encode(out).unwrap().len(); + packets += encoder.encode_all(out).unwrap().len(); } - packets += encoder.finish().unwrap().len(); + packets += encoder.flush_all().unwrap().len(); assert!(packets > 0, "re-encoding resized textures produced no packets"); } diff --git a/rs/moq-video/src/decode/sink.rs b/rs/moq-video/src/decode/sink.rs index 46a0a893175..2f62e2b7805 100644 --- a/rs/moq-video/src/decode/sink.rs +++ b/rs/moq-video/src/decode/sink.rs @@ -29,8 +29,10 @@ use threaded::Inner; /// A [`Decoder`](super::Decoder) confined to one thread, driven from anywhere. /// /// Same shape as [`Decoder`](super::Decoder), except that -/// [`decode`](Self::decode) is `async` and takes its payload by value (it may -/// cross a thread). Reach for this instead of a `Decoder` whenever the decoder +/// [`decode`](Self::decode) and [`flush`](Self::flush) are `async` and `decode` +/// takes its payload by value (it may cross a thread). [`receive`](Self::receive) +/// stays synchronous: each `decode` and `flush` has already moved what the codec +/// readied to this side. Reach for this instead of a `Decoder` whenever the decoder /// outlives a single thread's stack: a spawned task, an object shared across /// threads, an FFI handle. A `Decoder` you build, drive, and drop inside one /// function needs none of it. @@ -66,20 +68,30 @@ impl Sink { self.0.name() } - /// Decode one access unit, waiting for whatever pictures it yields. + /// Submit one access unit, waiting until the codec has taken it. /// - /// Otherwise [`Decoder::decode`](super::Decoder::decode): zero or more frames, - /// since a backend that reorders holds pictures back. `payload` is a [`Bytes`], - /// so handing it over is a refcount rather than a copy. - pub async fn decode(&mut self, payload: Bytes, timestamp: Timestamp, keyframe: bool) -> Result, Error> { + /// Otherwise [`Decoder::decode`](super::Decoder::decode): its pictures come out + /// of [`receive`](Self::receive), zero or more of them, since a backend that + /// reorders holds pictures back. `payload` is a [`Bytes`], so handing it over + /// is a refcount rather than a copy. + pub async fn decode(&mut self, payload: Bytes, timestamp: Timestamp, keyframe: bool) -> Result<(), Error> { self.0.decode(payload, timestamp, keyframe).await } - /// Drain the frames the decoder still holds once the stream has ended. + /// Pop the oldest ready frame, or `None` while the decoder holds the rest. + /// + /// Call until it returns `None` after each [`decode`](Self::decode) and + /// [`flush`](Self::flush). + pub fn receive(&mut self) -> Result, Error> { + self.0.receive() + } + + /// Mark the end of input, so [`receive`](Self::receive) returns the frames the + /// decoder still holds once the stream has ended. /// /// Otherwise [`Decoder::flush`](super::Decoder::flush), confined to the codec /// thread like [`decode`](Self::decode). - pub async fn flush(&mut self) -> Result, Error> { + pub async fn flush(&mut self) -> Result<(), Error> { self.0.flush().await } } @@ -92,27 +104,40 @@ mod threaded { use tokio::sync::{mpsc, oneshot}; use super::super::decoder::{Config, Decoder}; - use crate::worker::{Ready, Worker}; + use crate::worker::{Queue, Ready, Worker}; use crate::{Error, Frame}; - /// Work for the decode thread. One variant today; the enum is here so a - /// future request (a flush, a reset) lands in order with the frames around it - /// rather than racing them. + /// Work for the decode thread, in order with the frames around it. enum Request { Decode { payload: Bytes, timestamp: Timestamp, keyframe: bool, - resp: oneshot::Sender, Error>>, + resp: oneshot::Sender>, }, Flush { - resp: oneshot::Sender, Error>>, + resp: oneshot::Sender>, }, } + /// Move everything the decoder has ready into `queue`. + fn collect(decoder: &mut Decoder, queue: &Queue) -> Result<(), Error> { + let mut queue = queue.lock(); + while let Some(frame) = decoder.receive()? { + queue.push_back(frame); + } + Ok(()) + } + /// Build a decoder and serve requests until the channel closes. Runs entirely /// on the decode thread; see [`crate::worker`]. - fn run(catalog: VideoConfig, config: Config, ready: Ready, mut requests: mpsc::UnboundedReceiver) { + fn run( + catalog: VideoConfig, + config: Config, + queue: Queue, + ready: Ready, + mut requests: mpsc::UnboundedReceiver, + ) { let mut decoder = match Decoder::new(&catalog, &config) { Ok(decoder) => decoder, Err(err) => return ready.err(err), @@ -132,10 +157,14 @@ mod threaded { keyframe, resp, } => { - let _ = resp.send(decoder.decode(&payload, timestamp, keyframe)); + let decoded = decoder + .decode(&payload, timestamp, keyframe) + .and_then(|()| collect(&mut decoder, &queue)); + let _ = resp.send(decoded); } Request::Flush { resp } => { - let _ = resp.send(decoder.flush()); + let flushed = decoder.flush().and_then(|()| collect(&mut decoder, &queue)); + let _ = resp.send(flushed); } } } @@ -143,30 +172,30 @@ mod threaded { } /// A [`Decoder`] running on its own thread. See the module docs. - pub struct Inner(Worker); + pub struct Inner { + worker: Worker, + queue: Queue, + } impl Inner { pub async fn open(catalog: &VideoConfig, config: &Config) -> Result { let catalog = catalog.clone(); let config = config.clone(); + let queue = Queue::default(); + let filled = queue.clone(); let worker = Worker::open("moq-video-decode", move |ready, requests| { - run(catalog, config, ready, requests) + run(catalog, config, filled, ready, requests) }) .await?; - Ok(Self(worker)) + Ok(Self { worker, queue }) } pub fn name(&self) -> &str { - self.0.info() + self.worker.info() } - pub async fn decode( - &mut self, - payload: Bytes, - timestamp: Timestamp, - keyframe: bool, - ) -> Result, Error> { - self.0 + pub async fn decode(&mut self, payload: Bytes, timestamp: Timestamp, keyframe: bool) -> Result<(), Error> { + self.worker .request(|resp| Request::Decode { payload, timestamp, @@ -176,8 +205,12 @@ mod threaded { .await } - pub async fn flush(&mut self) -> Result, Error> { - self.0.request(|resp| Request::Flush { resp }).await + pub fn receive(&mut self) -> Result, Error> { + Ok(self.queue.pop()) + } + + pub async fn flush(&mut self) -> Result<(), Error> { + self.worker.request(|resp| Request::Flush { resp }).await } } } @@ -210,16 +243,15 @@ mod inline { /// Async only to match the threaded `Inner`; there's no thread to hand /// this to, so it decodes inline. - pub async fn decode( - &mut self, - payload: Bytes, - timestamp: Timestamp, - keyframe: bool, - ) -> Result, Error> { + pub async fn decode(&mut self, payload: Bytes, timestamp: Timestamp, keyframe: bool) -> Result<(), Error> { self.0.decode(&payload, timestamp, keyframe) } - pub async fn flush(&mut self) -> Result, Error> { + pub fn receive(&mut self) -> Result, Error> { + self.0.receive() + } + + pub async fn flush(&mut self) -> Result<(), Error> { self.0.flush() } } @@ -281,11 +313,12 @@ mod tests { let caller = std::thread::spawn(move || { let mut guard = sink.lock().unwrap(); let sink = guard.as_mut().unwrap(); - let frames = pollster::block_on(sink.decode(Bytes::from_static(b"au"), at(index), index == 0)).unwrap(); + pollster::block_on(sink.decode(Bytes::from_static(b"au"), at(index), index == 0)).unwrap(); // The frame really came back, so the assertions below are about a // decoder that ran rather than one that no-opped. - assert_eq!(frames.len(), 1); - assert_eq!(frames[0].timestamp, at(index)); + let frame = sink.receive().unwrap().expect("one frame per access unit"); + assert!(sink.receive().unwrap().is_none()); + assert_eq!(frame.timestamp, at(index)); std::thread::current().id() }); callers.push(caller.join().unwrap()); diff --git a/rs/moq-video/src/decode/vui.rs b/rs/moq-video/src/decode/vui.rs index 6fd3018f309..fb26cdeb53c 100644 --- a/rs/moq-video/src/decode/vui.rs +++ b/rs/moq-video/src/decode/vui.rs @@ -158,7 +158,7 @@ mod tests { Surface::I420(i420.with_color(color)), moq_net::Timestamp::from_micros(0).unwrap(), ); - encoder.encode(&frame).unwrap().remove(0).payload + encoder.encode_all(&frame).unwrap().remove(0).payload }; let both = |first: Color, second: Color| { let mut unit = keyframe(first).to_vec(); diff --git a/rs/moq-video/src/encode/backend/mediacodec.rs b/rs/moq-video/src/encode/backend/mediacodec.rs index df79d2a3ed5..a4f16c81c31 100644 --- a/rs/moq-video/src/encode/backend/mediacodec.rs +++ b/rs/moq-video/src/encode/backend/mediacodec.rs @@ -151,6 +151,8 @@ pub(crate) struct MediaCodec { keyframe_pending: bool, /// Whether the codec still takes input. What it owes a flush is `pending`. input: Input, + /// Access units the codec handed back and `receive` has not popped yet. + ready: VecDeque, } /// Whether the codec takes input. @@ -158,8 +160,8 @@ pub(crate) struct MediaCodec { enum Input { /// Taking frames, and back to it after every flush. Open, - /// End of input was signalled for the last time ([`Backend::finish`]), or a - /// flush could not take the codec back out of it, so it refuses input for good. + /// A flush could not take the codec back out of end of input, so it refuses + /// input for good. Ended, } @@ -211,6 +213,7 @@ impl MediaCodec { frame_index: 0, keyframe_pending: false, input: Input::Open, + ready: VecDeque::new(), })) } @@ -277,10 +280,10 @@ impl MediaCodec { Ok(()) } - /// Collect every access unit the codec has ready, waiting `timeout` on each + /// Queue every access unit the codec has ready, waiting `timeout` on each /// round. Returns whether it reported end of stream, which is the only thing /// that says the tail is out rather than still coming. - fn drain(&mut self, timeout: Duration, out: &mut Vec) -> Result { + fn drain(&mut self, timeout: Duration) -> Result { loop { let taken = match self .codec @@ -318,7 +321,7 @@ impl MediaCodec { } else { unit }; - out.push(Encoded::new(payload, timestamp, keyframe)); + self.ready.push_back(Encoded::new(payload, timestamp, keyframe)); } if flags & FLAG_END_OF_STREAM != 0 { @@ -327,7 +330,7 @@ impl MediaCodec { } } - /// Signal end of input and wait the codec's tail out, returning everything it + /// Signal end of input and wait the codec's tail out, queueing everything it /// was still holding. /// /// The wait is the point. An encoder that buffers won't part with the frames @@ -335,16 +338,15 @@ impl MediaCodec { /// happens to be ready already truncates the stream: that is one access unit /// lost per group, and on a live track the frame does not vanish but /// reappears ahead of the next group's keyframe. - fn drain_tail(&mut self) -> Result, Error> { - let mut out = Vec::new(); - self.signal_end_of_input(&mut out)?; + fn drain_tail(&mut self) -> Result<(), Error> { + self.signal_end_of_input()?; for _ in 0..DRAIN_ROUNDS { - if self.drain(DRAIN_TIMEOUT, &mut out)? { + if self.drain(DRAIN_TIMEOUT)? { // Anything still outstanding was discarded by the codec rather than // encoded, so it will never be paired with an output. self.pending.clear(); - return Ok(out); + return Ok(()); } } @@ -358,7 +360,7 @@ impl MediaCodec { /// /// ByteBuffer mode has no `AMediaCodec_signalEndOfInputStream`: that one is /// for a codec fed through a surface. - fn signal_end_of_input(&mut self, out: &mut Vec) -> Result<(), Error> { + fn signal_end_of_input(&mut self) -> Result<(), Error> { let sample_time = self.sample_time(); for _ in 0..DRAIN_ROUNDS { let queued = match self @@ -379,7 +381,7 @@ impl MediaCodec { } // Every input buffer is still with the codec; collecting its output is // what frees one. - self.drain(OUTPUT_TIMEOUT, out)?; + self.drain(OUTPUT_TIMEOUT)?; } Err(Error::codec(anyhow::anyhow!( @@ -389,25 +391,28 @@ impl MediaCodec { } impl Backend for MediaCodec { - fn encode(&mut self, frame: &Frame, cut: bool) -> Result, Error> { + fn encode(&mut self, frame: &Frame, cut: bool) -> Result<(), Error> { self.input.check()?; - let mut out = Vec::new(); // Collect whatever the codec finished while the caller was elsewhere, so // the input buffer asked for below isn't stuck behind an output nobody // picked up. - self.drain(OUTPUT_TIMEOUT, &mut out)?; + self.drain(OUTPUT_TIMEOUT)?; self.submit(frame, cut || self.keyframe_pending)?; - self.drain(OUTPUT_TIMEOUT, &mut out)?; - Ok(out) + self.drain(OUTPUT_TIMEOUT)?; + Ok(()) + } + + fn receive(&mut self) -> Result, Error> { + Ok(self.ready.pop_front()) } - fn flush(&mut self) -> Result, Error> { + fn flush(&mut self) -> Result<(), Error> { self.input.check()?; // Every frame fed has already come back out, so there is no tail to wait // for and no reason to cycle the codec through end of stream. This is the // common case: the format asks for a one-frame pipeline. if self.pending.is_empty() { - return Ok(Vec::new()); + return Ok(()); } // Flushed whatever the drain did: end of input leaves the codec refusing @@ -432,15 +437,6 @@ impl Backend for MediaCodec { tail } - fn finish(&mut self) -> Result, Error> { - self.input.check()?; - self.input = Input::Ended; - if self.pending.is_empty() { - return Ok(Vec::new()); - } - self.drain_tail() - } - fn set_bitrate(&mut self, bitrate: u64) -> Result<(), Error> { // Settable on a live codec and applied without an IDR, which is what the // rate control loop wants. The NDK does warn that a parameter change may @@ -729,11 +725,8 @@ mod tests { let i420 = I420::new(size, vec![0x80; I420::len(size).unwrap()]).unwrap(); let frame = Frame::new(crate::Surface::I420(i420), timestamp(0)); - let mut encoded = backend.encode(&frame, true).unwrap(); - if encoded.is_empty() { - encoded = backend.flush().unwrap(); - } - + let mut encoded = backend.encode_all(&frame, true).unwrap(); + encoded.extend(backend.flush_all().unwrap()); let annexb: Vec = encoded.iter().flat_map(|frame| frame.payload.iter().copied()).collect(); moq_mux::codec::h264::config(&annexb).expect("a keyframe carrying its own parameter sets"); } diff --git a/rs/moq-video/src/encode/backend/mediafoundation.rs b/rs/moq-video/src/encode/backend/mediafoundation.rs index b1329f32099..fa21835d034 100644 --- a/rs/moq-video/src/encode/backend/mediafoundation.rs +++ b/rs/moq-video/src/encode/backend/mediafoundation.rs @@ -84,6 +84,8 @@ pub(crate) struct MediaFoundation { /// more access units than we fed it frames. Repeating a time is far kinder to a /// consumer than the jump to zero the alternative would produce. last_timestamp: Option, + /// Access units the MFT handed back and `receive` has not popped yet. + ready: VecDeque, /// Kept alive for the MFT's lifetime once a texture frame binds a device. _manager: Option, _com: ComGuard, @@ -96,8 +98,7 @@ enum Phase { /// frame itself. Unstarted, Streaming(Stream), - /// The stream ended for the last time ([`Backend::finish`]), or the MFT - /// failed to start, drain, or restart, so it takes no more frames. + /// The MFT failed to start, drain, or restart, so it takes no more frames. Ended, } @@ -164,6 +165,7 @@ impl MediaFoundation { sample_index: 0, pending: VecDeque::new(), last_timestamp: None, + ready: VecDeque::new(), _manager: None, _com: com, })) @@ -449,27 +451,27 @@ impl MediaFoundation { Ok(buffer) } - /// Block on events until the MFT is ready for input, collecting any output + /// Block on events until the MFT is ready for input, queueing any output /// that arrives meanwhile. Runs on the dedicated encode thread (see /// `encode::sink`), so this blocking wait never parks a tokio worker. - fn wait_for_input(&mut self, out: &mut Vec) -> Result<(), Error> { + fn wait_for_input(&mut self) -> Result<(), Error> { while !self.stream()?.needs_input { let event = unsafe { self.events .GetEvent(MF_EVENT_FLAG_NONE) .map_err(|e| mf_err("GetEvent", e))? }; - self.handle_event(&event, out)?; + self.handle_event(&event)?; } Ok(()) } /// Drain events already queued without blocking (called after feeding input). - fn drain_ready(&mut self, out: &mut Vec) -> Result<(), Error> { + fn drain_ready(&mut self) -> Result<(), Error> { loop { match unsafe { self.events.GetEvent(MF_EVENT_FLAG_NO_WAIT) } { Ok(event) => { - self.handle_event(&event, out)?; + self.handle_event(&event)?; } Err(e) if e.code() == MF_E_NO_EVENTS_AVAILABLE => return Ok(()), Err(e) => return Err(mf_err("GetEvent (drain)", e)), @@ -480,7 +482,7 @@ impl MediaFoundation { /// Act on one event from the MFT, returning whether it reported a drain /// complete, which is how a drain knows the tail is out rather than still /// coming. - fn handle_event(&mut self, event: &IMFMediaEvent, out: &mut Vec) -> Result { + fn handle_event(&mut self, event: &IMFMediaEvent) -> Result { // Surface an async failure (e.g. an MEError event) instead of looping in // `wait_for_input` forever for an input request that will never come. unsafe { event.GetStatus() } @@ -496,7 +498,7 @@ impl MediaFoundation { NEED_INPUT => self.stream()?.needs_input = true, HAVE_OUTPUT => { if let Some(packet) = self.process_output()? { - out.push(packet); + self.ready.push_back(packet); } } DRAIN_COMPLETE => return Ok(true), @@ -555,7 +557,7 @@ impl MediaFoundation { Ok(Some(Encoded::new(payload, self.take_timestamp(sample_time), keyframe))) } - /// End the stream and wait the MFT's tail out, returning everything it was + /// End the stream and wait the MFT's tail out, queueing everything it was /// still holding. /// /// The blocking wait is the point. A hardware encoder holds a frame or two, and @@ -570,8 +572,7 @@ impl MediaFoundation { /// set that up are checked for the same reason: the wait is only bounded by an /// MFT that accepted the drain, so failing to reach that state has to be an /// error here rather than a block later. - fn drain(&mut self) -> Result, Error> { - let mut out = Vec::new(); + fn drain(&mut self) -> Result<(), Error> { unsafe { self.transform .ProcessMessage(MFT_MESSAGE_NOTIFY_END_OF_STREAM, 0) @@ -583,8 +584,8 @@ impl MediaFoundation { loop { let event = unsafe { self.events.GetEvent(MF_EVENT_FLAG_NONE) }.map_err(|e| mf_err("GetEvent (drain)", e))?; - if self.handle_event(&event, &mut out)? { - return Ok(out); + if self.handle_event(&event)? { + return Ok(()); } } } @@ -594,23 +595,22 @@ impl MediaFoundation { /// A drain cut short leaves the MFT somewhere between streams, with its /// completion perhaps still queued for the next drain to mistake for its own, /// so nothing after one can be trusted. - fn cycle(&mut self) -> Result, Error> { - let cycled = self.drain().and_then(|out| { + fn cycle(&mut self) -> Result<(), Error> { + let cycled = self.drain().and_then(|()| { // A drain leaves the MFT stopped: it stops asking for input until a new // stream starts. unsafe { self.transform .ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0) - .map_err(|e| mf_err("start of stream", e))?; + .map_err(|e| mf_err("start of stream", e)) } - Ok(out) }); match cycled { - Ok(out) => { + Ok(()) => { let stream = self.stream()?; stream.needs_input = false; stream.discontinuity = true; - Ok(out) + Ok(()) } Err(err) => { self.phase = Phase::Ended; @@ -659,13 +659,12 @@ impl MediaFoundation { } impl Backend for MediaFoundation { - fn encode(&mut self, frame: &Frame, cut: bool) -> Result, Error> { + fn encode(&mut self, frame: &Frame, cut: bool) -> Result<(), Error> { if let Phase::Unstarted = self.phase { self.start(&frame.surface)?; } - let mut out = Vec::new(); - self.wait_for_input(&mut out)?; + self.wait_for_input()?; if cut { self.set_codec(&CODECAPI_AVEncVideoForceKeyFrame, variant_u32(1))?; @@ -695,29 +694,20 @@ impl Backend for MediaFoundation { self.pending.push_back((sample_time, frame.timestamp)); self.sample_index += 1; - self.drain_ready(&mut out)?; - Ok(out) + self.drain_ready() } - fn flush(&mut self) -> Result, Error> { - if let Phase::Unstarted = self.phase { - return Ok(Vec::new()); - } - // An ended encoder fails here rather than being sent a drain. - self.stream()?; - self.cycle() + fn receive(&mut self) -> Result, Error> { + Ok(self.ready.pop_front()) } - fn finish(&mut self) -> Result, Error> { + fn flush(&mut self) -> Result<(), Error> { if let Phase::Unstarted = self.phase { - self.phase = Phase::Ended; - return Ok(Vec::new()); + return Ok(()); } // An ended encoder fails here rather than being sent a drain. self.stream()?; - let out = self.drain(); - self.phase = Phase::Ended; - out + self.cycle() } fn set_bitrate(&mut self, bitrate: u64) -> Result<(), Error> { diff --git a/rs/moq-video/src/encode/backend/mod.rs b/rs/moq-video/src/encode/backend/mod.rs index 2872c430db3..f5ab121bed8 100644 --- a/rs/moq-video/src/encode/backend/mod.rs +++ b/rs/moq-video/src/encode/backend/mod.rs @@ -8,9 +8,9 @@ //! the right importer. //! //! Output is stamped with the timestamp of the frame it was encoded from, not -//! whichever frame happened to be going in. A backend that flushes each frame -//! before returning just echoes the input timestamp; one that buffers (Media -//! Foundation, MediaCodec) correlates through the codec's own sample clock. +//! whichever frame happened to be going in. A backend that readies each frame's +//! output as it is submitted just echoes the input timestamp; one that buffers +//! (Media Foundation, MediaCodec) correlates through the codec's own sample clock. //! //! [`open`] picks the best backend for a [`Codec`](super::Codec) + //! [`Kind`](super::Kind): only candidates that support the requested codec are @@ -45,12 +45,15 @@ mod v4l2; #[cfg(all(target_os = "linux", feature = "vaapi"))] mod vaapi; -/// An opened video encoder. Feed it frames at the configured resolution; get -/// back zero or more access units in the codec's wire framing, each stamped with -/// the timestamp of the frame it came from and marked when it is a keyframe, -/// whether forced by `cut` or placed by the backend's own GOP cadence. +/// An opened video encoder. Submit frames at the configured resolution, then +/// receive zero or more access units in the codec's wire framing, each stamped +/// with the timestamp of the frame it came from and marked when it is a +/// keyframe, whether forced by `cut` or placed by the backend's own GOP cadence. +/// +/// A backend that readies output synchronously holds it in a reused queue, so +/// no call allocates to hand it back. pub(crate) trait Backend { - /// Encode one frame, opening a group at it (an IDR) when `cut` is set. + /// Submit one frame, opening a group at it (an IDR) when `cut` is set. /// Backends place group boundaries on their own every /// [`Config::keyframe_interval`] frames, so this is only the encoder's extra /// request: a caller's [`Encoder::cut`](super::Encoder::cut), or a @@ -58,10 +61,13 @@ pub(crate) trait Backend { /// arrives only on a backend whose [`can_cut`](Self::can_cut) said yes. A /// backend that drops the frame carries the cut to the next one it encodes, /// or the group waits a whole join for its keyframe. - fn encode(&mut self, frame: &Frame, cut: bool) -> Result, Error>; + fn encode(&mut self, frame: &Frame, cut: bool) -> Result<(), Error>; + + /// Pop the oldest ready access unit, or `None` while the codec holds the rest. + fn receive(&mut self) -> Result, Error>; - /// Return every access unit the codec is still holding, leaving the encoder - /// usable for the frames that follow. + /// Mark the end of input, so [`receive`](Self::receive) returns every access + /// unit the codec still holds. The next frame submitted starts new input. /// /// The caller reaches for this at a boundary the output has to respect, which /// on a live track is a group: a codec that pipelines would otherwise carry the @@ -71,10 +77,32 @@ pub(crate) trait Backend { /// No default, even though most backends have nothing to hold: a pipelined one /// that inherited an empty implementation would drop frames at every boundary /// and look like it worked. - fn flush(&mut self) -> Result, Error>; + fn flush(&mut self) -> Result<(), Error>; + + /// Submit one frame and collect everything it readied. + #[cfg(test)] + fn encode_all(&mut self, frame: &Frame, cut: bool) -> Result, Error> { + self.encode(frame, cut)?; + self.receive_all() + } + + /// Flush and collect the tail. + #[cfg(test)] + #[allow(dead_code, reason = "only the platform backends' tests use it")] + fn flush_all(&mut self) -> Result, Error> { + self.flush()?; + self.receive_all() + } - /// Flush the encoder for the last time, returning any buffered access units. - fn finish(&mut self) -> Result, Error>; + /// Collect everything ready. + #[cfg(test)] + fn receive_all(&mut self) -> Result, Error> { + let mut encoded = Vec::new(); + while let Some(unit) = self.receive()? { + encoded.push(unit); + } + Ok(encoded) + } /// Retune the live encoder to `bitrate` bits per second, taking effect from /// roughly the next frame. Called as the congestion controller's estimate @@ -577,16 +605,16 @@ mod tests { } impl Backend for Stub { - fn encode(&mut self, _frame: &Frame, _cut: bool) -> Result, Error> { - Ok(Vec::new()) + fn encode(&mut self, _frame: &Frame, _cut: bool) -> Result<(), Error> { + Ok(()) } - fn flush(&mut self) -> Result, Error> { - Ok(Vec::new()) + fn receive(&mut self) -> Result, Error> { + Ok(None) } - fn finish(&mut self) -> Result, Error> { - Ok(Vec::new()) + fn flush(&mut self) -> Result<(), Error> { + Ok(()) } fn set_bitrate(&mut self, _bitrate: u64) -> Result<(), Error> { diff --git a/rs/moq-video/src/encode/backend/nvenc.rs b/rs/moq-video/src/encode/backend/nvenc.rs index 618a59b1fbc..20ecd84c7e0 100644 --- a/rs/moq-video/src/encode/backend/nvenc.rs +++ b/rs/moq-video/src/encode/backend/nvenc.rs @@ -32,6 +32,7 @@ //! rung of a rendition ladder can be too small for NVENC even though the GPU //! conversion and resize handle it. +use std::collections::VecDeque; use std::num::NonZeroUsize; use std::sync::Arc; @@ -85,6 +86,7 @@ pub(crate) struct Nvenc { codec: Codec, size: crate::Size, converter: Option, + ready: VecDeque, } impl Nvenc { @@ -235,6 +237,7 @@ impl Nvenc { .map(|_| cuda::Converter::new(cuda.ordinal(), config.resolved_color(), NonZeroUsize::new(8).unwrap())) .transpose()?, size: config.size(), + ready: VecDeque::new(), _cuda: cuda, timestamp: 0, codec: config.codec, @@ -243,7 +246,7 @@ impl Nvenc { } impl Backend for Nvenc { - fn encode(&mut self, frame: &Frame, cut: bool) -> Result, Error> { + fn encode(&mut self, frame: &Frame, cut: bool) -> Result<(), Error> { // The published image shares its full-size conversion across renditions. // Smaller encoders resize that NV12 buffer, so capture converts only once // for each declared color space and never downloads its pixels. @@ -355,23 +358,22 @@ impl Backend for Nvenc { }; // The bitstream lock above is synchronous, so this is that frame's output. - Ok(if data.is_empty() { - Vec::new() - } else { + if !data.is_empty() { // moq-nvenc hands back only the bytes, not the picture type it locked them with. let keyframe = keyframe_annexb(self.codec, &data); - vec![Encoded::new(Bytes::from(data), frame.timestamp, keyframe)] - }) + self.ready + .push_back(Encoded::new(Bytes::from(data), frame.timestamp, keyframe)); + } + Ok(()) } - fn flush(&mut self) -> Result, Error> { - // Each encode locks its own output synchronously, so nothing is buffered. - Ok(Vec::new()) + fn receive(&mut self) -> Result, Error> { + Ok(self.ready.pop_front()) } - fn finish(&mut self) -> Result, Error> { + fn flush(&mut self) -> Result<(), Error> { // Each encode locks its own output synchronously, so nothing is buffered. - Ok(Vec::new()) + Ok(()) } fn set_bitrate(&mut self, bitrate: u64) -> Result<(), Error> { @@ -585,7 +587,7 @@ mod tests { if i == 0 || i == 5 { encoder.cut().unwrap(); } - let encoded = encoder.encode(&gray_frame(&frame, i.into())).unwrap(); + let encoded = encoder.encode_all(&gray_frame(&frame, i.into())).unwrap(); let joined: Vec = encoded.iter().flat_map(|f| f.payload.iter()).copied().collect(); if i == 0 { first = joined; @@ -634,7 +636,7 @@ mod tests { let frame = gray_rgba(320, 240); for i in 0..5 { - let encoded = encoder.encode(&gray_frame(&frame, i)).unwrap(); + let encoded = encoder.encode_all(&gray_frame(&frame, i)).unwrap(); assert_eq!(encoded.len(), 1, "{codec:?} {preset:?} held frame {i}"); } } @@ -657,7 +659,7 @@ mod tests { }; let frame = gray_rgba(320, 240); - assert!(!encoder.encode(&gray_frame(&frame, 0)).unwrap().is_empty()); + assert!(!encoder.encode_all(&gray_frame(&frame, 0)).unwrap().is_empty()); encoder .set_bitrate(moq_net::bandwidth::Rate::ZERO) .expect_err("a zero rate must be refused"); @@ -666,7 +668,7 @@ mod tests { .set_bitrate(moq_net::bandwidth::Rate::from_bps(500_000)) .unwrap(); for i in 1..5 { - assert!(!encoder.encode(&gray_frame(&frame, i)).unwrap().is_empty()); + assert!(!encoder.encode_all(&gray_frame(&frame, i)).unwrap().is_empty()); } } @@ -697,7 +699,7 @@ mod tests { if i == 0 || i == 5 { encoder.cut().unwrap(); } - let encoded = encoder.encode(&gray_frame(&frame, i.into())).unwrap(); + let encoded = encoder.encode_all(&gray_frame(&frame, i.into())).unwrap(); let joined: Vec = encoded.iter().flat_map(|f| f.payload.iter()).copied().collect(); if i == 0 { first = joined; @@ -741,7 +743,7 @@ mod tests { // Never force a keyframe (only frame 0 would be); the backend must insert // IDRs at frames 3 and 6 on its own. for i in 0..7u32 { - let encoded = encoder.encode(&gray_frame(&frame, i.into())).unwrap(); + let encoded = encoder.encode_all(&gray_frame(&frame, i.into())).unwrap(); let joined: Vec = encoded.iter().flat_map(|f| f.payload.iter()).copied().collect(); let types = h264_nal_types(&joined); if types.contains(&5) { @@ -809,8 +811,8 @@ mod tests { } let surface = crate::Surface::rgba(&rgba, crate::Size::new(w, h)).unwrap(); let frame = crate::Frame::new(surface, moq_net::Timestamp::from_micros(i * 33_333).unwrap()); - for encoded in encoder.encode(&frame).unwrap() { - for out in decoder.decode(encoded.payload, encoded.timestamp, i == 0).unwrap() { + for encoded in encoder.encode_all(&frame).unwrap() { + for out in decoder.decode_all(encoded.payload, encoded.timestamp, i == 0).unwrap() { decoded = Some(out.surface.to_i420().unwrap().into_owned()); } } @@ -864,7 +866,7 @@ mod tests { let rgba = textured_rgba(w, h, (i * 4) as usize); let surface = crate::Surface::rgba(&rgba, crate::Size::new(w, h)).unwrap(); let frame = crate::Frame::new(surface, moq_net::Timestamp::from_micros(i * 33_333).unwrap()); - for encoded in encoder.encode(&frame).unwrap() { + for encoded in encoder.encode_all(&frame).unwrap() { // Skip the first GOP: rate control is still converging there. if i < 30 { continue; diff --git a/rs/moq-video/src/encode/backend/openh264.rs b/rs/moq-video/src/encode/backend/openh264.rs index def568eb136..ca415ce08aa 100644 --- a/rs/moq-video/src/encode/backend/openh264.rs +++ b/rs/moq-video/src/encode/backend/openh264.rs @@ -3,6 +3,8 @@ //! The fallback when no hardware encoder is available. Emits Annex-B with //! in-band SPS/PPS, ready for `moq_mux::codec::h264::Import` in avc3 mode. +use std::collections::VecDeque; + use bytes::Bytes; use openh264::OpenH264API; use openh264::encoder::{ @@ -30,6 +32,7 @@ pub(crate) struct Openh264 { pending: Option, /// Whether a frame has gone through, i.e. whether the encoder exists yet. started: bool, + ready: VecDeque, } impl Openh264 { @@ -99,6 +102,7 @@ impl Openh264 { applied, pending: None, started: false, + ready: VecDeque::new(), }) } @@ -155,7 +159,7 @@ impl Openh264 { } impl Backend for Openh264 { - fn encode(&mut self, frame: &Frame, cut: bool) -> Result, Error> { + fn encode(&mut self, frame: &Frame, cut: bool) -> Result<(), Error> { // A rate deferred from before the encoder existed lands here, ahead of the // frame rather than after it, so a rejected rate can't cost us a frame's // packets on the way out. @@ -188,21 +192,20 @@ impl Backend for Openh264 { // be set from the next frame on. self.started = true; // One access unit out per frame in, so it carries that frame's timestamp. - Ok(if bytes.is_empty() { - Vec::new() - } else { - vec![Encoded::new(Bytes::from(bytes), frame.timestamp, keyframe)] - }) + if !bytes.is_empty() { + self.ready + .push_back(Encoded::new(Bytes::from(bytes), frame.timestamp, keyframe)); + } + Ok(()) } - fn flush(&mut self) -> Result, Error> { - // Low-delay: nothing is buffered, so there's nothing to flush. - Ok(Vec::new()) + fn receive(&mut self) -> Result, Error> { + Ok(self.ready.pop_front()) } - fn finish(&mut self) -> Result, Error> { + fn flush(&mut self) -> Result<(), Error> { // Low-delay: nothing is buffered, so there's nothing to flush. - Ok(Vec::new()) + Ok(()) } fn set_bitrate(&mut self, bitrate: u64) -> Result<(), Error> { @@ -263,7 +266,7 @@ mod tests { (Preset::Quality, MEDIUM_COMPLEXITY, Preset::Balanced), ] { let mut enc = Openh264::new(&Config { preset, ..config() }).unwrap(); - enc.encode(&gray(), true).unwrap(); + enc.encode_all(&gray(), true).unwrap(); assert_eq!(enc.read_complexity(), complexity, "{preset:?}"); assert_eq!(enc.applied().preset, Some(applied), "{preset:?}"); } @@ -274,7 +277,7 @@ mod tests { #[test] fn set_bitrate_reaches_the_encoder() { let mut enc = Openh264::new(&config()).unwrap(); - enc.encode(&gray(), true).unwrap(); + enc.encode_all(&gray(), true).unwrap(); let lower = config().resolved_bitrate().as_bps() / 2; enc.set_bitrate(lower).unwrap(); @@ -288,7 +291,7 @@ mod tests { #[test] fn set_bitrate_above_the_opening_rate_is_rejected() { let mut enc = Openh264::new(&config()).unwrap(); - enc.encode(&gray(), true).unwrap(); + enc.encode_all(&gray(), true).unwrap(); let higher = config().resolved_bitrate().as_bps() * 4; assert!(enc.set_bitrate(higher).is_err()); @@ -300,7 +303,7 @@ mod tests { #[test] fn set_bitrate_at_the_opening_rate_is_accepted() { let mut enc = Openh264::new(&config()).unwrap(); - enc.encode(&gray(), true).unwrap(); + enc.encode_all(&gray(), true).unwrap(); let opened = config().resolved_bitrate().as_bps(); enc.set_bitrate(opened / 2).unwrap(); @@ -320,11 +323,11 @@ mod tests { // Deferred: the encoder doesn't exist yet. enc.set_bitrate(opened / 2).unwrap(); - enc.encode(&gray(), true).unwrap(); + enc.encode_all(&gray(), true).unwrap(); // Live: this is the rate the caller last asked for. enc.set_bitrate(opened / 4).unwrap(); - enc.encode(&gray(), false).unwrap(); + enc.encode_all(&gray(), false).unwrap(); assert_eq!(enc.read_bitrate(), (opened / 4) as i64, "the deferred rate resurrected"); } @@ -353,8 +356,9 @@ mod tests { let surface = Surface::rgba(&rgba, size).unwrap(); let frame = Frame::new(surface, moq_net::Timestamp::from_micros(0).unwrap()); - let encoded = enc.encode(&frame, true).unwrap(); - let annexb = &encoded.first().expect("a keyframe").payload; + let encoded = enc.encode_all(&frame, true).unwrap(); + let encoded = encoded.first().expect("a keyframe"); + let annexb = &encoded.payload; assert_eq!(declared_color(annexb), Some(described), "{size} SPS color description"); @@ -380,7 +384,7 @@ mod tests { timestamp: moq_net::Timestamp::from_micros(index * 33_333).unwrap(), ..gray() }; - for unit in enc.encode(&frame, index == 45).unwrap() { + for unit in enc.encode_all(&frame, index == 45).unwrap() { assert_eq!( unit.keyframe, super::super::keyframe_annexb(crate::encode::Codec::H264, &unit.payload), diff --git a/rs/moq-video/src/encode/backend/probe.rs b/rs/moq-video/src/encode/backend/probe.rs index 3c75cab810a..5b233a145f6 100644 --- a/rs/moq-video/src/encode/backend/probe.rs +++ b/rs/moq-video/src/encode/backend/probe.rs @@ -7,6 +7,7 @@ //! It also holds each frame back by one, like the Media Foundation MFT, so a //! drain that loses the tail shows up as a missing frame rather than passing. +use std::collections::VecDeque; use std::sync::Mutex; use std::thread::ThreadId; @@ -82,7 +83,9 @@ pub(crate) fn refuse_once(bitrate: u64) { } pub(crate) struct Probe { - pending: Option, + /// The frame held back by one. + held: Option, + ready: VecDeque, cuts: bool, } @@ -90,7 +93,8 @@ impl Probe { pub(crate) fn open(_config: &Config) -> Result, Error> { record("open"); Ok(Box::new(Self { - pending: None, + held: None, + ready: VecDeque::new(), cuts: true, })) } @@ -98,14 +102,15 @@ impl Probe { pub(crate) fn open_no_cut(_config: &Config) -> Result, Error> { record("open"); Ok(Box::new(Self { - pending: None, + held: None, + ready: VecDeque::new(), cuts: false, })) } } impl Backend for Probe { - fn encode(&mut self, frame: &Frame, cut: bool) -> Result, Error> { + fn encode(&mut self, frame: &Frame, cut: bool) -> Result<(), Error> { // Uncontended unless a test is holding the codec here on purpose. drop(GATE.lock().unwrap_or_else(|err| err.into_inner())); record("encode"); @@ -117,18 +122,19 @@ impl Backend for Probe { // The payload is the frame's timestamp, so a test can tell which frame a // packet came from independently of what it's stamped with. let payload = bytes::Bytes::from(frame.timestamp.as_micros().to_string()); - let previous = self.pending.replace(Encoded::new(payload, frame.timestamp, cut)); - Ok(previous.into_iter().collect()) + let previous = self.held.replace(Encoded::new(payload, frame.timestamp, cut)); + self.ready.extend(previous); + Ok(()) } - fn flush(&mut self) -> Result, Error> { - record("flush"); - Ok(self.pending.take().into_iter().collect()) + fn receive(&mut self) -> Result, Error> { + Ok(self.ready.pop_front()) } - fn finish(&mut self) -> Result, Error> { - record("finish"); - Ok(self.pending.take().into_iter().collect()) + fn flush(&mut self) -> Result<(), Error> { + record("flush"); + self.ready.extend(self.held.take()); + Ok(()) } fn set_bitrate(&mut self, bitrate: u64) -> Result<(), Error> { diff --git a/rs/moq-video/src/encode/backend/v4l2.rs b/rs/moq-video/src/encode/backend/v4l2.rs index 14015cbc584..805686cfe19 100644 --- a/rs/moq-video/src/encode/backend/v4l2.rs +++ b/rs/moq-video/src/encode/backend/v4l2.rs @@ -119,6 +119,8 @@ pub(crate) struct V4l2 { size: Size, /// Frames the driver has taken and not yet answered with an access unit. pending: Pending, + /// Access units dequeued and not yet received. + ready: VecDeque, phase: Phase, /// Whether the driver takes `VIDIOC_ENCODER_CMD`. Cleared the first time it /// refuses one, after which a flush can only take what the codec has already @@ -275,6 +277,7 @@ impl V4l2 { planes, size, pending: Pending::default(), + ready: VecDeque::new(), phase: Phase::Stopped, drainable: true, cuts, @@ -365,12 +368,12 @@ impl V4l2 { Ok(()) } - /// Collect every access unit the driver has finished, re-queueing each coded + /// Queue every access unit the driver has finished, re-queueing each coded /// buffer as it is emptied. /// /// Returns whether the driver marked the end of a sequence, which is how a /// drain knows it is over. - fn drain(&mut self, out: &mut Vec) -> Result { + fn drain(&mut self) -> Result { self.reclaim()?; loop { @@ -406,7 +409,7 @@ impl V4l2 { // whatever went in last, and the timestamp is that frame's own. // Read from the bitstream: drivers set `V4L2_BUF_FLAG_KEYFRAME` unevenly. let keyframe = keyframe_annexb(Codec::H264, &payload); - out.push(Encoded::new(payload, timestamp, keyframe)); + self.ready.push_back(Encoded::new(payload, timestamp, keyframe)); } if buffer.last() { @@ -428,13 +431,13 @@ impl V4l2 { /// the stream stopped, and a flush falls on every group boundary, so the first /// boundary would time out and take the broadcast with it. bcm2835-codec /// happens to be one-in-one-out, so a Pi would not have shown this. - fn drain_tail(&mut self) -> Result, Error> { + fn drain_tail(&mut self) -> Result<(), Error> { match self.phase { Phase::Failed => return Err(failed()), // The sequence needs both queues streaming. `VIDIOC_ENCODER_CMD` // succeeds without starting one otherwise, and nothing is held // before a frame goes in anyway. - Phase::Stopped => return Ok(Vec::new()), + Phase::Stopped => return Ok(()), Phase::Streaming => {} } @@ -460,7 +463,7 @@ impl V4l2 { /// Run the rest of the drain sequence, up to and including the restart, or /// stop both queues when it fails. - fn drain_to_last(&mut self) -> Result, Error> { + fn drain_to_last(&mut self) -> Result<(), Error> { let drained = self.resume_after_last(); if drained.is_err() { self.stop()?; @@ -468,17 +471,16 @@ impl V4l2 { drained } - fn resume_after_last(&mut self) -> Result, Error> { - let mut out = Vec::new(); + fn resume_after_last(&mut self) -> Result<(), Error> { let mut deadline = Instant::now() + FLUSH_TIMEOUT; loop { - let before = out.len(); - if self.drain(&mut out)? { + let before = self.ready.len(); + if self.drain()? { break; } // Progress earns more time, so a slow encoder finishes a long tail while // a stalled one still gives up after `FLUSH_TIMEOUT` of silence. - if out.len() > before { + if self.ready.len() > before { deadline = Instant::now() + FLUSH_TIMEOUT; } else if Instant::now() >= deadline { return Err(Error::codec(anyhow::anyhow!( @@ -504,7 +506,7 @@ impl V4l2 { // A stopped encoder accepts OUTPUT buffers but does not process them, so // without this the frames after the boundary would sit in the driver. self.device.encoder_cmd(V4L2_ENC_CMD_START)?; - Ok(out) + Ok(()) } /// Take what the codec has already finished, for a driver that cannot be told @@ -512,16 +514,15 @@ impl V4l2 { /// /// Only a drain where the encoder holds nothing it has not been asked for, /// which is what an empty [`Pending`] says here. - fn wait_out(&mut self) -> Result, Error> { - let mut out = Vec::new(); + fn wait_out(&mut self) -> Result<(), Error> { let mut deadline = Instant::now() + FLUSH_TIMEOUT; loop { - let before = out.len(); - self.drain(&mut out)?; + let before = self.ready.len(); + self.drain()?; if self.pending.is_empty() { - return Ok(out); + return Ok(()); } - if out.len() > before { + if self.ready.len() > before { deadline = Instant::now() + FLUSH_TIMEOUT; } else if Instant::now() >= deadline { return Err(Error::codec(anyhow::anyhow!( @@ -535,7 +536,7 @@ impl V4l2 { } impl Backend for V4l2 { - fn encode(&mut self, frame: &Frame, cut: bool) -> Result, Error> { + fn encode(&mut self, frame: &Frame, cut: bool) -> Result<(), Error> { if self.phase == Phase::Failed { return Err(failed()); } @@ -578,16 +579,15 @@ impl Backend for V4l2 { self.start()?; } - let mut out = Vec::new(); - self.drain(&mut out)?; - Ok(out) + self.drain()?; + Ok(()) } - fn flush(&mut self) -> Result, Error> { - self.drain_tail() + fn receive(&mut self) -> Result, Error> { + Ok(self.ready.pop_front()) } - fn finish(&mut self) -> Result, Error> { + fn flush(&mut self) -> Result<(), Error> { self.drain_tail() } diff --git a/rs/moq-video/src/encode/backend/vaapi.rs b/rs/moq-video/src/encode/backend/vaapi.rs index a7f6263ccbb..84567aa0dc7 100644 --- a/rs/moq-video/src/encode/backend/vaapi.rs +++ b/rs/moq-video/src/encode/backend/vaapi.rs @@ -41,7 +41,7 @@ //! `vaapi_1080p_ladder_stays_on_the_gpu`, which requests hardware explicitly and //! fails without it. -use std::collections::HashSet; +use std::collections::{HashSet, VecDeque}; use bytes::Bytes; use moq_vaapi::encode::{Config as VaapiConfig, Encoder}; @@ -63,6 +63,7 @@ pub(crate) struct Vaapi { /// The CPU path's NV12 staging buffer, kept so a frame reuses the last one's /// allocation. nv12: Vec, + ready: VecDeque, /// Frames encoded from a DMA-BUF on the GPU, for the tests to tell the two /// paths apart. #[cfg(test)] @@ -118,6 +119,7 @@ impl Vaapi { codec: config.codec, refused: HashSet::new(), nv12: Vec::new(), + ready: VecDeque::new(), #[cfg(test)] gpu_frames: 0, }) @@ -172,7 +174,7 @@ impl Vaapi { } impl Backend for Vaapi { - fn encode(&mut self, frame: &Frame, cut: bool) -> Result, Error> { + fn encode(&mut self, frame: &Frame, cut: bool) -> Result<(), Error> { let gpu = match &frame.surface { Surface::DmaBuf(buffer) => self.encode_dmabuf(buffer, cut), _ => None, @@ -183,25 +185,23 @@ impl Backend for Vaapi { }; // Submitted and read back within the call, so this is that frame's output. - Ok(if annexb.is_empty() { - Vec::new() - } else { + if !annexb.is_empty() { // moq-vaapi places the IDRs but reports only the bytes. let keyframe = keyframe_annexb(self.codec, &annexb); - vec![Encoded::new(Bytes::from(annexb), frame.timestamp, keyframe)] - }) + self.ready + .push_back(Encoded::new(Bytes::from(annexb), frame.timestamp, keyframe)); + } + Ok(()) } - fn flush(&mut self) -> Result, Error> { - // The encoder submits and reads back synchronously per frame, so nothing - // is ever buffered. - Ok(Vec::new()) + fn receive(&mut self) -> Result, Error> { + Ok(self.ready.pop_front()) } - fn finish(&mut self) -> Result, Error> { + fn flush(&mut self) -> Result<(), Error> { // The encoder submits and reads back synchronously per frame, so nothing - // is buffered at shutdown. - Ok(Vec::new()) + // is ever buffered. + Ok(()) } fn set_bitrate(&mut self, bitrate: u64) -> Result<(), Error> { @@ -319,15 +319,15 @@ mod tests { let mut software = decoder("openh264", Output::Cpu); let mut decoded = Vec::new(); for (index, frame) in frames.iter().enumerate() { - for encoded in backend.encode(frame, index == 0).expect("encode") { + for encoded in backend.encode_all(frame, index == 0).expect("encode") { decoded.extend( software - .decode(encoded.payload, encoded.timestamp, index == 0) + .decode_all(encoded.payload, encoded.timestamp, index == 0) .expect("decode"), ); } } - decoded.extend(software.flush().expect("flush")); + decoded.extend(software.flush_all().expect("flush")); assert_eq!(decoded.len(), frames.len(), "pictures went missing"); decoded } @@ -414,11 +414,11 @@ mod tests { source.cut().unwrap(); } let frame = Frame::new(Surface::rgba(&rgba, size).unwrap(), at(i)); - for encoded in source.encode(&frame).unwrap() { - frames.extend(hardware.decode(encoded.payload, encoded.timestamp, i == 0).unwrap()); + for encoded in source.encode_all(&frame).unwrap() { + frames.extend(hardware.decode_all(encoded.payload, encoded.timestamp, i == 0).unwrap()); } } - frames.extend(hardware.flush().unwrap()); + frames.extend(hardware.flush_all().unwrap()); assert!( frames.iter().all(|frame| matches!(frame.surface, Surface::DmaBuf(_))), "the decoder handed out CPU frames, so nothing here tests the GPU path" @@ -527,16 +527,16 @@ mod tests { let mut decoded = Vec::new(); for (index, frame) in frames.iter().enumerate() { - for encoded in backend.encode(frame, index == 0).expect("encode") { + for encoded in backend.encode_all(frame, index == 0).expect("encode") { assert_eq!(encoded.keyframe, index == 0, "picture {index} keyframe flag"); decoded.extend( hardware - .decode(encoded.payload, encoded.timestamp, encoded.keyframe) + .decode_all(encoded.payload, encoded.timestamp, encoded.keyframe) .expect("decode"), ); } } - decoded.extend(hardware.flush().expect("flush")); + decoded.extend(hardware.flush_all().expect("flush")); assert_eq!(backend.gpu_frames, 3, "the encoder fell back to the CPU"); assert!(backend.refused.is_empty()); assert_eq!(decoded.len(), frames.len(), "pictures went missing"); @@ -558,7 +558,7 @@ mod tests { if index == 3 { backend.set_bitrate(250_000).expect("set the bitrate"); } - assert!(!backend.encode(frame, index == 0).expect("encode").is_empty()); + assert!(!backend.encode_all(frame, index == 0).expect("encode").is_empty()); } assert_eq!(backend.encoder.config().bitrate, 250_000); } @@ -628,7 +628,7 @@ mod tests { }; let rgba = [255u8, 0, 0, 255].repeat(size.pixels() as usize); let frame = Frame::new(Surface::rgba(&rgba, size).unwrap(), at(0)); - let encoded = backend.encode(&frame, true).unwrap(); + let encoded = backend.encode_all(&frame, true).unwrap(); let annexb = &encoded.first().expect("a keyframe").payload; assert_eq!(declared_color(annexb), Some(described), "{size} SPS color description"); } @@ -671,22 +671,22 @@ mod hardware_tests { for index in 0..FRAMES { let timestamp = Timestamp::from_micros(index as u64 * 33_333).unwrap(); let frame = Frame::new(Surface::I420(pixels.clone()), timestamp); - for packet in source.encode(&frame).expect("encode source") { + for packet in source.encode_all(&frame).expect("encode source") { frames.extend( decoder - .decode(&packet.payload, packet.timestamp, packet.keyframe) + .decode_all(&packet.payload, packet.timestamp, packet.keyframe) .expect("decode source"), ); } } - for packet in source.finish().expect("finish source") { + for packet in source.flush_all().expect("finish source") { frames.extend( decoder - .decode(&packet.payload, packet.timestamp, packet.keyframe) + .decode_all(&packet.payload, packet.timestamp, packet.keyframe) .expect("decode source tail"), ); } - frames.extend(decoder.flush().expect("flush source decoder")); + frames.extend(decoder.flush_all().expect("flush source decoder")); assert_eq!(frames.len(), FRAMES); assert!( frames.iter().all(|frame| matches!(frame.surface, Surface::DmaBuf(_))), @@ -717,15 +717,15 @@ mod hardware_tests { "{size}: resize downloaded pixels" ); assert_eq!(scaled.size(), size); - for packet in encoder.encode(&scaled, index == 0).expect("encode rung") { + for packet in encoder.encode_all(&scaled, index == 0).expect("encode rung") { decoded.extend( output - .decode(&packet.payload, packet.timestamp, packet.keyframe) + .decode_all(&packet.payload, packet.timestamp, packet.keyframe) .expect("decode rung"), ); } } - decoded.extend(output.flush().expect("flush rung decoder")); + decoded.extend(output.flush_all().expect("flush rung decoder")); assert_eq!(encoder.gpu_frames, FRAMES as u64, "{size}: encoder used CPU transfers"); assert!(encoder.refused.is_empty()); assert_eq!(decoded.len(), FRAMES, "{size}: lost output pictures"); diff --git a/rs/moq-video/src/encode/backend/videotoolbox.rs b/rs/moq-video/src/encode/backend/videotoolbox.rs index 338d6b27bdb..33cbb1b827c 100644 --- a/rs/moq-video/src/encode/backend/videotoolbox.rs +++ b/rs/moq-video/src/encode/backend/videotoolbox.rs @@ -12,11 +12,13 @@ //! thread-bound with it; only the Apple `Sink::Inner` keeps the serialized //! `Send` wrapper, safe because `Sink` serializes every call. +use std::collections::VecDeque; use std::ffi::{c_int, c_void}; use std::ptr::{self, NonNull}; use std::slice; use bytes::{BufMut, Bytes, BytesMut}; +use moq_net::Timestamp; use objc2_core_foundation::{ CFDictionary, CFNumber, CFNumberType, CFRetained, CFString, CFType, kCFBooleanFalse, kCFBooleanTrue, }; @@ -50,13 +52,15 @@ use crate::{Error, Frame}; pub(crate) const NAME: &str = "videotoolbox"; -/// Where the C output callback drops finished frames, read back after each -/// `encode_frame` + `complete_frames`. Lives behind a `Box` so its address is -/// stable for the lifetime of the session that holds it as a refcon. +/// Where the C output callback queues finished frames, popped by `receive`. +/// Lives behind a `Box` so its address is stable for the lifetime of the +/// session that holds it as a refcon. struct Sink { codec: Codec, - /// Each access unit, and whether it is a keyframe. - packets: Vec<(Bytes, bool)>, + /// The timestamp of the frame going in, which every access unit the callback + /// sees during its `complete_frames` came from. + timestamp: Timestamp, + ready: VecDeque, error: Option, } @@ -89,7 +93,8 @@ impl VideoToolbox { let mut sink = Box::new(Sink { codec: config.codec, - packets: Vec::new(), + timestamp: Timestamp::ZERO, + ready: VecDeque::new(), error: None, }); let refcon = (&mut *sink as *mut Sink).cast::(); @@ -216,9 +221,12 @@ impl VideoToolbox { } impl Backend for VideoToolbox { - fn encode(&mut self, frame: &Frame, cut: bool) -> Result, Error> { - self.sink.packets.clear(); + fn encode(&mut self, frame: &Frame, cut: bool) -> Result<(), Error> { self.sink.error = None; + // `complete_frames` below forces this frame out before returning, so every + // access unit the callback sees until then came from it. + self.sink.timestamp = frame.timestamp; + let before = self.sink.ready.len(); // Zero-copy when the capture handed us a surface; otherwise upload I420. let pixel_buffer = match &frame.surface { @@ -271,23 +279,17 @@ impl Backend for VideoToolbox { ))); } // A dropped frame comes back with no packet and no error. - self.keyframe_pending = cut && self.sink.packets.is_empty(); - // `complete_frames` above forces this frame out before returning, so every - // packet collected here came from it and carries its timestamp. - Ok(std::mem::take(&mut self.sink.packets) - .into_iter() - .map(|(payload, keyframe)| Encoded::new(payload, frame.timestamp, keyframe)) - .collect()) + self.keyframe_pending = cut && self.sink.ready.len() == before; + Ok(()) } - fn flush(&mut self) -> Result, Error> { - // complete_frames runs per-encode, so nothing is ever buffered. - Ok(Vec::new()) + fn receive(&mut self) -> Result, Error> { + Ok(self.sink.ready.pop_front()) } - fn finish(&mut self) -> Result, Error> { - // complete_frames runs per-encode, so nothing is buffered at shutdown. - Ok(Vec::new()) + fn flush(&mut self) -> Result<(), Error> { + // complete_frames runs per-encode, so nothing is ever buffered. + Ok(()) } fn set_bitrate(&mut self, bitrate: u64) -> Result<(), Error> { @@ -354,7 +356,7 @@ unsafe extern "C-unwind" fn output_callback( return; // dropped frame }; match annexb_from_sample(sample, sink.codec) { - Ok(Some(packet)) => sink.packets.push(packet), + Ok(Some((payload, keyframe))) => sink.ready.push_back(Encoded::new(payload, sink.timestamp, keyframe)), Ok(None) => {} Err(status) => sink.error = Some(status), } @@ -630,7 +632,7 @@ mod tests { Surface::I420(I420::new(size, vec![0x80; I420::len(size).unwrap()]).unwrap()), timestamp, ); - let packets = encoder.encode(&frame, index == 1).unwrap(); + let packets = encoder.encode_all(&frame, index == 1).unwrap(); assert_eq!(packets.len(), 1, "{codec:?} {preset:?}"); assert_eq!(packets[0].timestamp, timestamp); if index <= 1 { @@ -652,7 +654,7 @@ mod tests { expected == Preset::Balanced ); } - assert!(encoder.finish().unwrap().is_empty()); + assert!(encoder.flush_all().unwrap().is_empty()); } } } diff --git a/rs/moq-video/src/encode/encoder.rs b/rs/moq-video/src/encode/encoder.rs index a7e8d0205a4..68a57b9c9e4 100644 --- a/rs/moq-video/src/encode/encoder.rs +++ b/rs/moq-video/src/encode/encoder.rs @@ -298,13 +298,19 @@ impl Config { // No cut: the first picture out of a fresh encoder is a keyframe on every backend, which // is what carries the parameter sets, and a backend that cannot cut still probes. - let mut encoded = sink.encode(frame).await?; + sink.encode(frame).await?; + let mut annexb = Vec::new(); + while let Some(encoded) = sink.receive()? { + annexb.extend_from_slice(&encoded.payload); + } // A backend that pipelines holds the first frame, so drain it rather than reading nothing. - if encoded.is_empty() { - encoded = sink.flush().await?; + if annexb.is_empty() { + sink.flush().await?; + while let Some(encoded) = sink.receive()? { + annexb.extend_from_slice(&encoded.payload); + } } - let annexb: Vec = encoded.iter().flat_map(|frame| frame.payload.iter().copied()).collect(); let parsed = match self.codec { Codec::H264 => moq_mux::codec::h264::config(&annexb), Codec::H265 => moq_mux::codec::h265::config(&annexb), @@ -357,9 +363,10 @@ pub(crate) fn default_bitrate(size: Size, framerate: FrameRate) -> moq_net::band moq_net::bandwidth::Rate::from_bps((size.pixels() as f64 * framerate.as_f64() * 0.07) as u64) } -/// Video encoder. Build one with [`Encoder::new`], feed it raw [`Frame`]s via -/// [`encode`](Self::encode), and publish the resulting [`Encoded`] access units -/// through a [`Producer`](super::Producer) built for the same [`Codec`]. +/// Video encoder. Build one with [`Encoder::new`], submit raw [`Frame`]s via +/// [`encode`](Self::encode), [`receive`](Self::receive) the resulting +/// [`Encoded`] access units until it returns `None`, and publish them through a +/// [`Producer`](super::Producer) built for the same [`Codec`]. /// /// An encoder is bound to the thread that opens it. Use [`Sink`](super::Sink) /// when the owner can move between threads. @@ -513,13 +520,13 @@ impl Encoder { } } - /// Encode one raw [`Frame`], whether it came from capture, a decoder (the + /// Submit one raw [`Frame`], whether it came from capture, a decoder (the /// transcode input path), or your own pixels via /// [`Surface::rgba`](crate::Surface::rgba). /// - /// Returns zero or more encoded access units, each carrying the timestamp of the - /// raw frame it came from: a backend that buffers hands back an earlier frame's - /// output, so the two don't always line up. + /// Its access units come out of [`receive`](Self::receive), each carrying the + /// timestamp of the raw frame it came from: a backend that buffers readies an + /// earlier frame's output, so the two don't always line up. /// /// A GPU surface feeds a hardware encoder on the same device directly /// (NVDEC -> NVENC never leaves the GPU, a `CVPixelBuffer` goes straight to @@ -530,7 +537,7 @@ impl Encoder { /// [`Frame::resize`](crate::Frame::resize), which /// [`decode::Config::scale_hint`](crate::decode::Config::scale_hint) lets a /// hardware decoder make a no-op. - pub fn encode(&mut self, frame: &Frame) -> Result, Error> { + pub fn encode(&mut self, frame: &Frame) -> Result<(), Error> { // A transposed frame is why this compares the shape rather than a byte // count: 240x320 and 320x240 hold the same number of bytes. #[cfg(target_os = "linux")] @@ -578,20 +585,33 @@ impl Encoder { } let timestamp = std::time::Duration::from(frame.timestamp); let cut = self.pending_cut || (self.backend.can_cut() && self.join.due(timestamp)); - let encoded = self.backend.encode(frame, cut)?; + self.backend.encode(frame, cut)?; // Cleared only once the frame is through: a failed encode produced no // picture, so the request still belongs to whatever comes next rather than // being swallowed. The size check above returns early for the same reason. self.pending_cut = false; self.join.encoded(timestamp, cut); - self.join.observe(&encoded); + Ok(()) + } + + /// Pop the oldest ready access unit, or `None` while the codec holds the rest. + /// + /// Call until it returns `None` after each [`encode`](Self::encode) and + /// [`flush`](Self::flush). + pub fn receive(&mut self) -> Result, Error> { + let encoded = self.backend.receive()?; + if let Some(encoded) = &encoded { + self.join.observe(encoded); + } Ok(encoded) } - /// Return every access unit the codec is still holding, leaving the encoder - /// ready for the frames that follow. Each keeps the timestamp of the raw frame - /// it was encoded from, so a drained tail stays in step with what came before - /// it. + /// Mark the end of input, so [`receive`](Self::receive) returns every access + /// unit the codec is still holding. The encoder stays ready for the frames + /// that follow. Each keeps the timestamp of the raw frame it was encoded from, + /// so a drained tail stays in step with what came before it. + /// + /// Call it before dropping the encoder, or its last frames are lost. /// /// Reach for this at a boundary the output has to respect, which for a live /// broadcast is a group: a hardware codec that pipelines holds the last frames @@ -602,20 +622,31 @@ impl Encoder { /// Not free: emptying the pipeline gives up the overlap between one frame's /// encode and the next frame's submission, so flush at boundaries rather than /// per frame. - pub fn flush(&mut self) -> Result, Error> { - let encoded = self.backend.flush()?; - self.join.observe(&encoded); - Ok(encoded) + pub fn flush(&mut self) -> Result<(), Error> { + self.backend.flush() } - /// Flush the encoder, returning any buffered frames. Each keeps the timestamp - /// of the raw frame it was encoded from, so a drained tail stays in step with - /// what was published before it. - /// - /// Consumes the encoder: nothing can be encoded after a flush, so this is the - /// last call rather than one leaving a drained encoder in your hands. - pub fn finish(mut self) -> Result, Error> { - self.backend.finish() + /// Submit `frame` and collect everything it readied. + #[cfg(test)] + pub(crate) fn encode_all(&mut self, frame: &Frame) -> Result, Error> { + self.encode(frame)?; + self.receive_all() + } + + /// Flush and collect the tail. + #[cfg(test)] + pub(crate) fn flush_all(&mut self) -> Result, Error> { + self.flush()?; + self.receive_all() + } + + #[cfg(test)] + fn receive_all(&mut self) -> Result, Error> { + let mut encoded = Vec::new(); + while let Some(unit) = self.receive()? { + encoded.push(unit); + } + Ok(encoded) } } @@ -661,11 +692,11 @@ impl Join { } } - /// Count the keyframes among `encoded`, including those the backend placed on + /// Count `encoded` if it is a keyframe, including one the backend placed on /// its own count, so the next is measured from the latest. - fn observe(&mut self, encoded: &[Encoded]) { - for keyframe in encoded.iter().filter(|encoded| encoded.keyframe) { - self.last = self.last.max(Some(keyframe.timestamp.into())); + fn observe(&mut self, encoded: &Encoded) { + if encoded.keyframe { + self.last = self.last.max(Some(encoded.timestamp.into())); } } } @@ -677,6 +708,8 @@ mod tests { allow(dead_code, unused_imports, reason = "only the `openh264` feature uses it") )] + use std::collections::VecDeque; + use super::*; use crate::{I420, Surface}; @@ -745,9 +778,9 @@ mod tests { if i == 0 { encoder.cut().unwrap(); } - frames.extend(encoder.encode(&gray_frame(320, 240, i)).unwrap()); + frames.extend(encoder.encode_all(&gray_frame(320, 240, i)).unwrap()); } - frames.extend(encoder.finish().unwrap()); + frames.extend(encoder.flush_all().unwrap()); assert!(!frames.is_empty(), "encoder produced no packets"); @@ -789,8 +822,8 @@ mod tests { let size = Size::new(320, 240); let i420 = I420::new(size, vec![0x80u8; I420::len(size).unwrap()]).unwrap(); let frame = Frame::new(Surface::I420(i420), at(0)); - let mut frames = encoder.encode(&frame).unwrap(); - frames.extend(encoder.finish().unwrap()); + let mut frames = encoder.encode_all(&frame).unwrap(); + frames.extend(encoder.flush_all().unwrap()); assert!(!frames.is_empty()); let packets = payloads(&frames); assert!(packets[0].starts_with(&[0, 0, 0, 1]) || packets[0].starts_with(&[0, 0, 1])); @@ -803,7 +836,7 @@ mod tests { let Ok(mut encoder) = Encoder::new(&Config::new(320, 240, crate::FrameRate::new(30, 1).unwrap())) else { return; }; - assert!(matches!(encoder.encode(&gray_frame(640, 480, 0)), Err(Error::Codec(_)))); + assert!(matches!(encoder.encode_all(&gray_frame(640, 480, 0)), Err(Error::Codec(_)))); } /// An encoder told to resize scales a frame at another size itself, so a @@ -815,8 +848,8 @@ mod tests { let Ok(mut encoder) = Encoder::new(&config) else { return; }; - let mut frames = encoder.encode(&gray_frame(640, 480, 0)).unwrap(); - frames.extend(encoder.finish().unwrap()); + let mut frames = encoder.encode_all(&gray_frame(640, 480, 0)).unwrap(); + frames.extend(encoder.flush_all().unwrap()); assert!(!frames.is_empty()); } @@ -835,7 +868,7 @@ mod tests { gray_rgba(320, 240).len(), "the byte counts must collide" ); - assert!(matches!(encoder.encode(&transposed), Err(Error::Codec(_)))); + assert!(matches!(encoder.encode_all(&transposed), Err(Error::Codec(_)))); } #[test] @@ -872,9 +905,9 @@ mod tests { if i == 0 { encoder.cut().unwrap(); } - frames.extend(encoder.encode(&gray_frame(320, 240, i)).unwrap()); + frames.extend(encoder.encode_all(&gray_frame(320, 240, i)).unwrap()); } - frames.extend(encoder.finish().unwrap()); + frames.extend(encoder.flush_all().unwrap()); assert!(!frames.is_empty(), "encoder produced no packets"); // VideoToolbox completes each frame before returning, so the timestamps come @@ -920,9 +953,9 @@ mod tests { if i == 0 { encoder.cut().unwrap(); } - frames.extend(encoder.encode(&gray_frame(320, 240, i)).unwrap()); + frames.extend(encoder.encode_all(&gray_frame(320, 240, i)).unwrap()); } - frames.extend(encoder.finish().unwrap()); + frames.extend(encoder.flush_all().unwrap()); assert!(!frames.is_empty(), "encoder produced no packets"); let packets = payloads(&frames); @@ -977,9 +1010,9 @@ mod tests { encoder.cut().unwrap(); } let frame = Frame::new(Surface::PixelBuffer(nv12_surface(320, 240)), at(i)); - frames.extend(encoder.encode(&frame).unwrap()); + frames.extend(encoder.encode_all(&frame).unwrap()); } - frames.extend(encoder.finish().unwrap()); + frames.extend(encoder.flush_all().unwrap()); assert!(!frames.is_empty()); let packets = payloads(&frames); @@ -1003,8 +1036,8 @@ mod tests { encoder.cut().unwrap(); let frame = Frame::new(Surface::PixelBuffer(nv12_surface(320, 240)), at(0)); - let mut frames = encoder.encode(&frame).unwrap(); - frames.extend(encoder.finish().unwrap()); + let mut frames = encoder.encode_all(&frame).unwrap(); + frames.extend(encoder.flush_all().unwrap()); assert!(!frames.is_empty()); let packets = payloads(&frames); @@ -1085,9 +1118,9 @@ mod tests { if i == 0 { encoder.cut().unwrap(); } - frames.extend(encoder.encode(&gray_frame(640, 480, i)).unwrap()); + frames.extend(encoder.encode_all(&gray_frame(640, 480, i)).unwrap()); } - frames.extend(encoder.finish().unwrap()); + frames.extend(encoder.flush_all().unwrap()); assert!(!frames.is_empty(), "encoder produced no packets"); // The MFT buffers, so packets come back stamped with the frame they were @@ -1141,9 +1174,9 @@ mod tests { if i == 0 { encoder.cut().unwrap(); } - frames.extend(encoder.encode(&frame).unwrap()); + frames.extend(encoder.encode_all(&frame).unwrap()); } - frames.extend(encoder.finish().unwrap()); + frames.extend(encoder.flush_all().unwrap()); // On a GPU this exercises the zero-copy texture path; the assert guards // against silently testing only the CPU fallback. @@ -1173,14 +1206,14 @@ mod tests { assert_eq!(opened, config.resolved_bitrate()); // Encode first: this is the live-retune path, once the encoder exists. - encoder.encode(&gray_frame(320, 240, 0)).unwrap(); + encoder.encode_all(&gray_frame(320, 240, 0)).unwrap(); let halved = opened.scaled(0.5); encoder.set_bitrate(halved).unwrap(); assert_eq!(encoder.bitrate(), halved); // The retuned encoder must still emit a decodable keyframe, not wedge. - let frames = encoder.encode(&gray_frame(320, 240, 1)).unwrap(); + let frames = encoder.encode_all(&gray_frame(320, 240, 1)).unwrap(); assert!(!frames.is_empty(), "encoder produced nothing after a retune"); let packets = payloads(&frames); assert!(packets[0].starts_with(&[0, 0, 0, 1]) || packets[0].starts_with(&[0, 0, 1])); @@ -1203,12 +1236,12 @@ mod tests { assert_eq!(encoder.bitrate(), halved); // The deferred rate is applied during this encode, which must still work. - let frames = encoder.encode(&gray_frame(320, 240, 0)).unwrap(); + let frames = encoder.encode_all(&gray_frame(320, 240, 0)).unwrap(); assert!(!frames.is_empty()); // And the encoder is live now, so a further retune takes the direct path. encoder.set_bitrate(halved.scaled(0.5)).unwrap(); - assert!(encoder.encode(&gray_frame(320, 240, 1)).is_ok()); + assert!(encoder.encode_all(&gray_frame(320, 240, 1)).is_ok()); } #[test] @@ -1220,26 +1253,30 @@ mod tests { } /// A backend that holds each frame back by one, like the Media Foundation MFT: - /// `encode` returns the *previous* frame's access unit and `finish` drains the - /// last. The payload is the frame's timestamp in microseconds, so a test can - /// tell which frame a packet came from independently of what it's stamped with. + /// `encode` readies the *previous* frame's access unit and `flush` the last. + /// The payload is the frame's timestamp in microseconds, so a test can tell + /// which frame a packet came from independently of what it's stamped with. + #[derive(Default)] struct Delayed { - pending: Option, + held: Option, + ready: VecDeque, } impl Backend for Delayed { - fn encode(&mut self, frame: &Frame, cut: bool) -> Result, Error> { + fn encode(&mut self, frame: &Frame, cut: bool) -> Result<(), Error> { let payload = bytes::Bytes::from(frame.timestamp.as_micros().to_string()); - let previous = self.pending.replace(Encoded::new(payload, frame.timestamp, cut)); - Ok(previous.into_iter().collect()) + self.ready + .extend(self.held.replace(Encoded::new(payload, frame.timestamp, cut))); + Ok(()) } - fn flush(&mut self) -> Result, Error> { - Ok(self.pending.take().into_iter().collect()) + fn receive(&mut self) -> Result, Error> { + Ok(self.ready.pop_front()) } - fn finish(&mut self) -> Result, Error> { - self.flush() + fn flush(&mut self) -> Result<(), Error> { + self.ready.extend(self.held.take()); + Ok(()) } fn set_bitrate(&mut self, _bitrate: u64) -> Result<(), Error> { @@ -1283,17 +1320,17 @@ mod tests { } impl Backend for Recorder { - fn encode(&mut self, _frame: &Frame, cut: bool) -> Result, Error> { + fn encode(&mut self, _frame: &Frame, cut: bool) -> Result<(), Error> { self.log.lock().unwrap().push(cut); - Ok(Vec::new()) + Ok(()) } - fn flush(&mut self) -> Result, Error> { - Ok(Vec::new()) + fn receive(&mut self) -> Result, Error> { + Ok(None) } - fn finish(&mut self) -> Result, Error> { - Ok(Vec::new()) + fn flush(&mut self) -> Result<(), Error> { + Ok(()) } fn set_bitrate(&mut self, _bitrate: u64) -> Result<(), Error> { @@ -1324,15 +1361,15 @@ mod tests { let mut encoder = encoder_with(Box::new(backend), &config); // Nothing asked for: `Config::join` places the boundaries on its own. - encoder.encode(&gray_frame(320, 240, 0)).unwrap(); + encoder.encode_all(&gray_frame(320, 240, 0)).unwrap(); // Asked twice before a frame exists: one cut, on the next frame. encoder.cut().unwrap(); encoder.cut().unwrap(); - encoder.encode(&gray_frame(320, 240, 1)).unwrap(); + encoder.encode_all(&gray_frame(320, 240, 1)).unwrap(); // And it does not carry into the frame after. - encoder.encode(&gray_frame(320, 240, 2)).unwrap(); + encoder.encode_all(&gray_frame(320, 240, 2)).unwrap(); assert_eq!(*log.lock().unwrap(), vec![false, true, false]); } @@ -1354,7 +1391,7 @@ mod tests { if i % 3 == 0 { encoder.cut().unwrap(); } - encoder.encode(&gray_frame(320, 240, i)).unwrap(); + encoder.encode_all(&gray_frame(320, 240, i)).unwrap(); } assert_eq!(*log.lock().unwrap(), vec![true, false, false, true, false, false]); @@ -1387,7 +1424,7 @@ mod tests { // Encoding carries on at the backend's own boundaries; the refusal is the // caller's to act on, not a broken encoder. - encoder.encode(&gray_frame(320, 240, 0)).unwrap(); + encoder.encode_all(&gray_frame(320, 240, 0)).unwrap(); assert_eq!(*log.lock().unwrap(), vec![false]); } @@ -1424,7 +1461,7 @@ mod tests { let backend = Recorder { log: log.clone(), cuts }; let mut encoder = encoder_with(Box::new(backend), &config); for µs in × { - encoder.encode(&gray_at(micros)).unwrap(); + encoder.encode_all(&gray_at(micros)).unwrap(); } let log = log.lock().unwrap(); if !cuts { @@ -1472,7 +1509,7 @@ mod tests { let micros = (i * 33_333) .checked_add_signed(jitter[i as usize % jitter.len()]) .unwrap(); - encoder.encode(&gray_at(micros)).unwrap(); + encoder.encode_all(&gray_at(micros)).unwrap(); } let log = log.lock().unwrap(); @@ -1486,22 +1523,25 @@ mod tests { log: std::sync::Arc>>, interval: u32, count: u32, + ready: VecDeque, } impl Backend for Counting { - fn encode(&mut self, frame: &Frame, cut: bool) -> Result, Error> { + fn encode(&mut self, frame: &Frame, cut: bool) -> Result<(), Error> { self.log.lock().unwrap().push(cut); let keyframe = cut || self.count.is_multiple_of(self.interval); self.count = if keyframe { 1 } else { self.count + 1 }; - Ok(vec![Encoded::new(bytes::Bytes::new(), frame.timestamp, keyframe)]) + self.ready + .push_back(Encoded::new(bytes::Bytes::new(), frame.timestamp, keyframe)); + Ok(()) } - fn flush(&mut self) -> Result, Error> { - Ok(Vec::new()) + fn receive(&mut self) -> Result, Error> { + Ok(self.ready.pop_front()) } - fn finish(&mut self) -> Result, Error> { - Ok(Vec::new()) + fn flush(&mut self) -> Result<(), Error> { + Ok(()) } fn set_bitrate(&mut self, _bitrate: u64) -> Result<(), Error> { @@ -1531,12 +1571,13 @@ mod tests { log: log.clone(), interval: config.keyframe_interval(), count: 0, + ready: VecDeque::new(), }; let mut encoder = encoder_with(Box::new(backend), &config); // Three seconds at 60 fps: the backend keys every half second. for i in 0..180 { - encoder.encode(&gray_at(i * 16_667)).unwrap(); + encoder.encode_all(&gray_at(i * 16_667)).unwrap(); } assert!(log.lock().unwrap().iter().all(|cut| !cut)); } @@ -1558,10 +1599,10 @@ mod tests { let mut encoder = encoder_with(Box::new(backend), &config); for i in 0..20 { - encoder.encode(&gray_at(10_000_000 + i * 33_333)).unwrap(); + encoder.encode_all(&gray_at(10_000_000 + i * 33_333)).unwrap(); } for i in 0..40 { - encoder.encode(&gray_at(i * 33_333)).unwrap(); + encoder.encode_all(&gray_at(i * 33_333)).unwrap(); } let log = log.lock().unwrap(); @@ -1584,7 +1625,7 @@ mod tests { let mut keys = Vec::new(); for i in 0..70 { - for encoded in encoder.encode(&gray_at(i * 50_000)).unwrap() { + for encoded in encoder.encode_all(&gray_at(i * 50_000)).unwrap() { if encoded.keyframe { keys.push(encoded.timestamp.as_micros() / 50_000); } @@ -1655,7 +1696,7 @@ mod tests { if i == 3 { encoder.cut().unwrap(); } - let encoded = encoder.encode(&gray_frame(320, 240, i)).unwrap(); + let encoded = encoder.encode_all(&gray_frame(320, 240, i)).unwrap(); let joined: Vec = encoded.iter().flat_map(|f| f.payload.iter()).copied().collect(); per_frame.push(nal_types(&joined)); } @@ -1683,16 +1724,16 @@ mod tests { struct Failing; impl Backend for Failing { - fn encode(&mut self, _frame: &Frame, _cut: bool) -> Result, Error> { + fn encode(&mut self, _frame: &Frame, _cut: bool) -> Result<(), Error> { Err(Error::codec(anyhow::anyhow!("no"))) } - fn flush(&mut self) -> Result, Error> { - Ok(Vec::new()) + fn receive(&mut self) -> Result, Error> { + Ok(None) } - fn finish(&mut self) -> Result, Error> { - Ok(Vec::new()) + fn flush(&mut self) -> Result<(), Error> { + Ok(()) } fn set_bitrate(&mut self, _bitrate: u64) -> Result<(), Error> { @@ -1718,17 +1759,17 @@ mod tests { let mut encoder = encoder_with(Box::new(Failing), &config); encoder.cut().unwrap(); - assert!(encoder.encode(&gray_frame(320, 240, 0)).is_err()); + assert!(encoder.encode_all(&gray_frame(320, 240, 0)).is_err()); assert!(encoder.pending_cut, "the backend error swallowed the request"); // Rejected before the backend ever sees it, for the same reason. - let mut encoder = encoder_with(Box::new(Delayed { pending: None }), &config); + let mut encoder = encoder_with(Box::new(Delayed::default()), &config); encoder.cut().unwrap(); - assert!(encoder.encode(&gray_frame(640, 480, 0)).is_err()); + assert!(encoder.encode_all(&gray_frame(640, 480, 0)).is_err()); assert!(encoder.pending_cut, "the size check swallowed the request"); // ...and the next frame that does go through claims it. - encoder.encode(&gray_frame(320, 240, 1)).unwrap(); + encoder.encode_all(&gray_frame(320, 240, 1)).unwrap(); assert!(!encoder.pending_cut); } @@ -1740,13 +1781,13 @@ mod tests { #[test] fn a_buffering_backend_keeps_each_frames_timestamp() { let config = Config::new(320, 240, crate::FrameRate::new(30, 1).unwrap()); - let mut encoder = encoder_with(Box::new(Delayed { pending: None }), &config); + let mut encoder = encoder_with(Box::new(Delayed::default()), &config); // The packet handed back while frame `i` goes in belongs to frame `i - 1`, so // it has to be stamped one frame back. Stamping at the call site (the only // option when encode just returned bytes) would shift the whole stream. for i in 0..5 { - let encoded = encoder.encode(&gray_frame(320, 240, i)).unwrap(); + let encoded = encoder.encode_all(&gray_frame(320, 240, i)).unwrap(); if i == 0 { assert!(encoded.is_empty(), "the first frame is still buffered"); continue; @@ -1758,14 +1799,14 @@ mod tests { assert_eq!(&encoded[0].payload[..], at(i - 1).as_micros().to_string().as_bytes()); } - // The tail: frame 4 never came back from an encode call, so `finish` drains + // The tail: frame 4 never came back from an encode call, so `flush` drains // it, still carrying its own time rather than one the caller picks. - let tail = encoder.finish().unwrap(); + let tail = encoder.flush_all().unwrap(); assert_eq!(tail.len(), 1); assert_eq!(tail[0].timestamp, at(4)); assert!( - encoder_with(Box::new(Delayed { pending: None }), &config) - .finish() + encoder_with(Box::new(Delayed::default()), &config) + .flush_all() .unwrap() .is_empty() ); @@ -1776,18 +1817,18 @@ mod tests { /// without that they surface in the *next* group, ahead of its keyframe, where /// a subscriber joining there cannot decode them. /// - /// The encoder stays usable afterwards, which is what separates this from - /// `finish`: a live track flushes at every group and keeps going. + /// The encoder stays usable afterwards: a live track flushes at every group + /// and keeps going. #[test] fn a_flush_empties_a_pipelined_backend_and_leaves_it_running() { let config = Config::new(320, 240, crate::FrameRate::new(30, 1).unwrap()); - let mut encoder = encoder_with(Box::new(Delayed { pending: None }), &config); + let mut encoder = encoder_with(Box::new(Delayed::default()), &config); let mut group = Vec::new(); for i in 0..3 { - group.extend(encoder.encode(&gray_frame(320, 240, i)).unwrap()); + group.extend(encoder.encode_all(&gray_frame(320, 240, i)).unwrap()); } - group.extend(encoder.flush().unwrap()); + group.extend(encoder.flush_all().unwrap()); // All three frames land in the group they belong to, in order. let times: Vec<_> = group.iter().map(|packet| packet.timestamp).collect(); @@ -1795,9 +1836,9 @@ mod tests { // The next group starts clean: nothing carried over, and the encoder still // takes frames. - let mut next = encoder.encode(&gray_frame(320, 240, 3)).unwrap(); + let mut next = encoder.encode_all(&gray_frame(320, 240, 3)).unwrap(); assert!(next.is_empty(), "frame 3 is buffered, so nothing comes back yet"); - next.extend(encoder.finish().unwrap()); + next.extend(encoder.flush_all().unwrap()); let times: Vec<_> = next.iter().map(|packet| packet.timestamp).collect(); assert_eq!(times, vec![at(3)], "the flush left something behind"); } @@ -1825,7 +1866,7 @@ mod tests { let surface = crate::frame::Surface::rgba(&rgba, size).unwrap(); encoder.cut().unwrap(); let frames = encoder - .encode(&Frame::new(surface, moq_net::Timestamp::from_micros(0).unwrap())) + .encode_all(&Frame::new(surface, moq_net::Timestamp::from_micros(0).unwrap())) .unwrap(); let keyframe = frames.first().expect("a keyframe"); @@ -1863,7 +1904,7 @@ mod tests { }; let mut encoder = Encoder::new(&config).unwrap(); encoder.cut().unwrap(); - let err = encoder.encode(&scaled).expect_err("a mismatched frame is refused"); + let err = encoder.encode_all(&scaled).expect_err("a mismatched frame is refused"); assert!( matches!( err, @@ -1883,7 +1924,7 @@ mod tests { }; let mut encoder = Encoder::new(&config).unwrap(); encoder.cut().unwrap(); - let frames = encoder.encode(&scaled).expect("a declared space encodes"); + let frames = encoder.encode_all(&scaled).expect("a declared space encodes"); let keyframe = frames.first().expect("a keyframe"); assert_eq!(declared_color(&keyframe.payload), Some(BT709_DESCRIBED)); @@ -1953,8 +1994,8 @@ mod tests { }; let mut encoder = Encoder::new(&config).unwrap(); encoder.cut().unwrap(); - let mut encoded = encoder.encode(&frame).unwrap(); - encoded.extend(encoder.finish().unwrap()); + let mut encoded = encoder.encode_all(&frame).unwrap(); + encoded.extend(encoder.flush_all().unwrap()); assert_eq!( declared_color(&encoded[0].payload), Some(described()), @@ -1972,9 +2013,9 @@ mod tests { let mut decoder = crate::decode::Decoder::new(&catalog, &decode).unwrap(); let mut decoded = Vec::new(); for unit in &encoded { - decoded.extend(decoder.decode(&unit.payload, unit.timestamp, unit.keyframe).unwrap()); + decoded.extend(decoder.decode_all(&unit.payload, unit.timestamp, unit.keyframe).unwrap()); } - decoded.extend(decoder.flush().unwrap()); + decoded.extend(decoder.flush_all().unwrap()); let picture = decoded.first().expect("a decoded picture"); assert_eq!(picture.surface.color(), Some(color), "{source} decoded label"); @@ -1999,7 +2040,7 @@ mod tests { }) .unwrap(); assert!(matches!( - guessed.encode(&scaled), + guessed.encode_all(&scaled), Err(Error::ColorMismatch { frame, .. }) if frame == color )); @@ -2011,7 +2052,7 @@ mod tests { }) .unwrap(); encoder.cut().unwrap(); - let encoded = encoder.encode(&scaled).unwrap(); + let encoded = encoder.encode_all(&scaled).unwrap(); assert_eq!( declared_color(&encoded[0].payload), Some(described()), @@ -2044,8 +2085,8 @@ mod tests { }; let mut encoder = Encoder::new(&config).expect("requested encoder must be available"); assert_eq!(encoder.name(), backend); - let mut encoded = encoder.encode(&frame).expect("encode WGC texture"); - encoded.extend(encoder.finish().unwrap()); + let mut encoded = encoder.encode_all(&frame).expect("encode WGC texture"); + encoded.extend(encoder.flush_all().unwrap()); assert!(!encoded.is_empty(), "{backend} must produce a frame"); assert!( encoded @@ -2116,7 +2157,7 @@ mod external_tests { crate::Surface::Vulkan(image), moq_net::Timestamp::from_micros(0).unwrap(), ); - let error = encoder.encode(&frame).unwrap_err(); + let error = encoder.encode_all(&frame).unwrap_err(); assert!(error.to_string().contains(&device().to_string()), "{error}"); assert!(frame.surface.to_i420().is_err()); } diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index db9f759ca4d..a2dca930f2c 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -158,23 +158,21 @@ impl Producer { largest.is_none_or(|largest| wanted > largest).then_some(wanted) } - /// Publish already-encoded frames, each at its own timestamp. Each frame is one - /// whole access unit in the producer's codec framing. - pub fn publish(&mut self, encoded: &[Encoded]) -> Result<(), Error> { - for frame in encoded { - let timestamp = Some(frame.timestamp); - // The encoder emits one whole access unit per frame, so flush to emit it. - match &mut self.codecs { - Codecs::H264 { split, import } => { - let mut frames = split.decode(&frame.payload, timestamp)?; - frames.extend(split.flush(timestamp)?); - import.decode(frames)?; - } - Codecs::H265 { split, import } => { - let mut frames = split.decode(&frame.payload, timestamp)?; - frames.extend(split.flush(timestamp)?); - import.decode(frames)?; - } + /// Publish one already-encoded frame at its own timestamp: one whole access unit + /// in the producer's codec framing. + pub fn publish(&mut self, encoded: &Encoded) -> Result<(), Error> { + let timestamp = Some(encoded.timestamp); + // The encoder emits one whole access unit per frame, so flush to emit it. + match &mut self.codecs { + Codecs::H264 { split, import } => { + let mut frames = split.decode(&encoded.payload, timestamp)?; + frames.extend(split.flush(timestamp)?); + import.decode(frames)?; + } + Codecs::H265 { split, import } => { + let mut frames = split.decode(&encoded.payload, timestamp)?; + frames.extend(split.flush(timestamp)?); + import.decode(frames)?; } } Ok(()) @@ -1017,10 +1015,12 @@ async fn capture_loop( if forced.due(wanted) { encoder.cut().await?; } - let Some(encoded) = wait_capture(&demand, on_demand, encoder.encode(frame)).await? else { + if wait_capture(&demand, on_demand, encoder.encode(frame)).await?.is_none() { break; - }; - producer.publish(&encoded)?; + } + while let Some(encoded) = encoder.receive()? { + producer.publish(&encoded)?; + } // A frame whose decoder identity differs from the probed one (a reopen onto another // mode or backend) publishes under a new name and ends this track. Capture follows it // there with a claim against the new track, which nobody watches yet: its viewers find @@ -1116,9 +1116,13 @@ mod tests { for i in 0..10u64 { let surface = crate::Surface::rgba(&rgba, crate::Size::new(320, 240)).unwrap(); let frame = Frame::new(surface, Timestamp::from_micros(i * 33_333).unwrap()); - producer.publish(&encoder.encode(&frame).unwrap()).unwrap(); + for encoded in encoder.encode_all(&frame).unwrap() { + producer.publish(&encoded).unwrap(); + } + } + for encoded in encoder.flush_all().unwrap() { + producer.publish(&encoded).unwrap(); } - producer.publish(&encoder.finish().unwrap()).unwrap(); let (_, resolved) = rendition(&catalog).expect("the importer should have registered a video rendition"); // Jitter and delay aside, which are measured from the frames rather than declared by either. @@ -1190,7 +1194,9 @@ mod tests { encoder.cut().unwrap(); let surface = crate::Surface::rgba(&rgba, crate::Size::new(320, 240)).unwrap(); let frame = Frame::new(surface, Timestamp::from_micros(timestamp).unwrap()); - producer.publish(&encoder.encode(&frame).unwrap()).unwrap(); + for encoded in encoder.encode_all(&frame).unwrap() { + producer.publish(&encoded).unwrap(); + } } producer.finish().unwrap(); @@ -1224,7 +1230,9 @@ mod tests { let rgba = vec![0x80u8; usize::try_from(config.width * config.height * 4).unwrap()]; let surface = crate::Surface::rgba(&rgba, crate::Size::new(config.width, config.height)).unwrap(); let frame = Frame::new(surface, Timestamp::from_micros(timestamp).unwrap()); - producer.publish(&encoder.encode(&frame).unwrap()).unwrap(); + for encoded in encoder.encode_all(&frame).unwrap() { + producer.publish(&encoded).unwrap(); + } capture_stopped(&mut producer).unwrap(); } @@ -1339,7 +1347,9 @@ mod tests { let rgba = vec![0x80u8; 320 * 240 * 4]; let surface = crate::Surface::rgba(&rgba, crate::Size::new(320, 240)).unwrap(); let frame = Frame::new(surface, Timestamp::from_micros(0).unwrap()); - producer.publish(&encoder.encode(&frame).unwrap()).unwrap(); + for encoded in encoder.encode_all(&frame).unwrap() { + producer.publish(&encoded).unwrap(); + } producer.finish().unwrap(); assert!(subscriber.recv_group().await.unwrap().is_some()); @@ -2472,7 +2482,7 @@ mod finished { let mut producer = Producer::new(track, catalog.reserve(), rendition.clone()).unwrap(); for pts in [0, 33_333] { let pts = moq_net::Timestamp::from_micros(pts).unwrap(); - producer.publish(&[Encoded::new(keyframe(), pts, true)]).unwrap(); + producer.publish(&Encoded::new(keyframe(), pts, true)).unwrap(); } producer.finish().unwrap(); producer diff --git a/rs/moq-video/src/encode/sink.rs b/rs/moq-video/src/encode/sink.rs index 9d3dcdddf8a..4412f90d4dd 100644 --- a/rs/moq-video/src/encode/sink.rs +++ b/rs/moq-video/src/encode/sink.rs @@ -11,8 +11,8 @@ //! same reason: an FFI object shared between threads opens the apartment on //! whichever thread built it and closes it on whichever thread drops it. //! Confining the whole encoder lifetime to one thread fixes both; frames are -//! `Send` there (Windows D3D11 textures and CPU I420 both are) and packets come -//! back over a channel. +//! `Send` there (Windows D3D11 textures and CPU I420 both are), and the thread +//! moves the packets each call readies into a queue the caller pops. //! //! macOS and iOS keep encoding inline: VideoToolbox has no COM apartment to balance and //! doesn't block on an event loop, so a thread would only add a hop, and its @@ -32,8 +32,10 @@ use threaded::Inner; /// An [`Encoder`](super::Encoder) confined to one thread, driven from anywhere. /// /// Same shape as [`Encoder`](super::Encoder), one method at a time, except that -/// the calls are `async` and [`encode`](Self::encode) takes the frame by value -/// (it may cross a thread). Reach for this instead of an `Encoder` whenever the +/// the calls that reach the codec are `async` and [`encode`](Self::encode) takes +/// the frame by value (it may cross a thread). [`receive`](Self::receive) stays +/// synchronous: each `encode` and `flush` has already moved what the codec +/// readied to this side. Reach for this instead of an `Encoder` whenever the /// encoder outlives a single thread's stack: an object shared across threads, an /// FFI handle, a task that migrates between executor workers. An `Encoder` you /// build, drive, and drop inside one function needs none of it. @@ -49,7 +51,7 @@ use threaded::Inner; /// it slide. The codec runs on its own thread, so a request that has been queued /// runs whether or not anyone is still waiting: dropping the future (racing it in /// a `select!`, giving it a timeout) leaves the codec a step ahead of the stream, -/// holding output nobody received. Rather than let the next call carry on and +/// holding output nobody collected. Rather than let the next call carry on and /// publish a track quietly missing those frames, the sink refuses every call /// after a cancelled one. Drop it and open another. /// @@ -94,10 +96,10 @@ impl Sink { self.0.cut().await } - /// Encode one frame, waiting for its access units. + /// Submit one frame, waiting until the codec has taken it. /// - /// Otherwise [`Encoder::encode`](super::Encoder::encode): zero or more access - /// units, each stamped with the frame it came from. + /// Otherwise [`Encoder::encode`](super::Encoder::encode): its access units come + /// out of [`receive`](Self::receive), each stamped with the frame it came from. /// /// One call at a time, so the sink never queues raw frames of its own: the /// only pictures in flight are the one being encoded and whatever the codec @@ -110,10 +112,18 @@ impl Sink { /// frame out to several encoders (a transcode ladder) hands over a clone of /// the handle rather than a copy of the pixels. Pass a [`Frame`] and it is /// wrapped for you. - pub async fn encode(&mut self, frame: impl Into>) -> Result, Error> { + pub async fn encode(&mut self, frame: impl Into>) -> Result<(), Error> { self.0.encode(frame.into()).await } + /// Pop the oldest ready access unit, or `None` while the codec holds the rest. + /// + /// Call until it returns `None` after each [`encode`](Self::encode) and + /// [`flush`](Self::flush). + pub fn receive(&mut self) -> Result, Error> { + self.0.receive() + } + /// Retune the encoder, waiting for the backend's verdict. See /// [`Encoder::set_bitrate`](super::Encoder::set_bitrate) for what a failure /// means (not fatal: stop adapting, keep encoding). @@ -121,27 +131,20 @@ impl Sink { self.0.set_bitrate(bitrate).await } - /// Empty the codec at a boundary the output has to respect, leaving it ready - /// for the frames that follow. See [`Encoder::flush`](super::Encoder::flush). + /// Mark the end of input, so [`receive`](Self::receive) returns every access + /// unit the codec still holds, leaving it ready for the frames that follow. + /// See [`Encoder::flush`](super::Encoder::flush). /// /// A live track needs this at every group boundary: a backend that pipelines /// is still holding the last frames of a group when it ends, and they would /// otherwise surface in the next group ahead of its keyframe, where a /// subscriber joining there cannot decode them. Publishing frame by frame /// with no group structure needs none of it. - pub async fn flush(&mut self) -> Result, Error> { - self.0.flush().await - } - - /// Drain the codec, returning every access unit it was still holding, and - /// shut the encoder down. /// - /// Consumes the sink, like [`Encoder::finish`](super::Encoder::finish). - /// Dropping a sink without this is fine and tears down just as cleanly, it - /// just discards the tail: publish the returned frames before ending a track, - /// or its last pictures never reach a subscriber. - pub async fn finish(self) -> Result, Error> { - self.0.finish().await + /// Flush and receive before dropping the sink, or its last pictures never + /// reach a subscriber. + pub async fn flush(&mut self) -> Result<(), Error> { + self.0.flush().await } } @@ -153,18 +156,18 @@ mod threaded { use super::super::Encoded; use super::super::encoder::{Applied, Config, Encoder}; - use crate::worker::{Ready, Worker}; + use crate::worker::{Queue, Ready, Worker}; use crate::{Error, Frame}; /// Work for the encode thread. Every variant goes down the same channel so a /// cut or a bitrate change lands in order with the frames around it, rather /// than racing them. enum Request { - /// A frame to encode, plus a oneshot to return the resulting access units - /// (or an error) in order. + /// A frame to encode, plus a oneshot to report, in order, whether the codec + /// took it. Encode { frame: Arc, - resp: oneshot::Sender, Error>>, + resp: oneshot::Sender>, }, /// Cut a group at the next frame, reporting whether the backend can: a /// refusal has to reach the caller, since the alternative is a group @@ -178,18 +181,8 @@ mod threaded { bitrate: moq_net::bandwidth::Rate, resp: oneshot::Sender>, }, - /// Empty the codec at a group boundary, leaving it running. Unlike - /// `Finish` the encoder survives, so this is served like any other - /// request. - Flush { - resp: oneshot::Sender, Error>>, - }, - /// Drain the codec and shut down, returning the tail. Last request the - /// thread serves: `Encoder::finish` consumes the encoder, so the loop has - /// to break out rather than come back round for another frame. - Finish { - resp: oneshot::Sender, Error>>, - }, + /// Mark the end of input at a group boundary, leaving the codec running. + Flush { resp: oneshot::Sender> }, } /// What the encode thread reports once its encoder is open. @@ -198,9 +191,23 @@ mod threaded { applied: Applied, } + /// Move everything the encoder has ready into `queue`. + fn collect(encoder: &mut Encoder, queue: &Queue) -> Result<(), Error> { + let mut queue = queue.lock(); + while let Some(encoded) = encoder.receive()? { + queue.push_back(encoded); + } + Ok(()) + } + /// Build an encoder for `config` and serve requests until the channel closes. /// Runs entirely on the encode thread; see [`crate::worker`]. - fn run(config: Config, ready: Ready, mut requests: mpsc::UnboundedReceiver) { + fn run( + config: Config, + queue: Queue, + ready: Ready, + mut requests: mpsc::UnboundedReceiver, + ) { let mut encoder = match Encoder::new(&config) { Ok(encoder) => encoder, Err(err) => return ready.err(err), @@ -215,14 +222,13 @@ mod threaded { } // Serve each request in arrival order. The encoder and its COM / MFT - // handles are created, used, and dropped only on this thread. `finish` - // consumes the encoder, so it breaks out and drains below rather than - // serving another request. - let mut draining = None; + // handles are created, used, and dropped only on this thread, which the + // encoder drops on once the channel closes. while let Some(req) = requests.blocking_recv() { match req { Request::Encode { frame, resp } => { - let _ = resp.send(encoder.encode(&frame)); + let encoded = encoder.encode(&frame).and_then(|()| collect(&mut encoder, &queue)); + let _ = resp.send(encoded); } Request::Cut { resp } => { let _ = resp.send(encoder.cut()); @@ -231,60 +237,57 @@ mod threaded { let _ = resp.send(encoder.set_bitrate(bitrate)); } Request::Flush { resp } => { - let _ = resp.send(encoder.flush()); - } - Request::Finish { resp } => { - draining = Some(resp); - break; + let flushed = encoder.flush().and_then(|()| collect(&mut encoder, &queue)); + let _ = resp.send(flushed); } } } - // The drain runs here, on this thread, and consumes the encoder; otherwise - // `encoder` drops here. Either way the COM apartment it opened closes on - // the thread that opened it. - if let Some(resp) = draining { - let _ = resp.send(encoder.finish()); - } } /// An [`Encoder`] running on its own thread. See the module docs. - pub struct Inner(Worker); + pub struct Inner { + worker: Worker, + queue: Queue, + } impl Inner { pub async fn open(config: &Config) -> Result { let config = config.clone(); - let worker = Worker::open("moq-video-encode", move |ready, requests| run(config, ready, requests)).await?; - Ok(Self(worker)) + let queue = Queue::default(); + let filled = queue.clone(); + let worker = Worker::open("moq-video-encode", move |ready, requests| { + run(config, filled, ready, requests) + }) + .await?; + Ok(Self { worker, queue }) } pub fn name(&self) -> &str { - &self.0.info().name + &self.worker.info().name } pub fn applied(&self) -> Applied { - self.0.info().applied.clone() + self.worker.info().applied.clone() } pub async fn cut(&mut self) -> Result<(), Error> { - self.0.request(|resp| Request::Cut { resp }).await + self.worker.request(|resp| Request::Cut { resp }).await } - pub async fn encode(&mut self, frame: Arc) -> Result, Error> { - self.0.request(|resp| Request::Encode { frame, resp }).await + pub async fn encode(&mut self, frame: Arc) -> Result<(), Error> { + self.worker.request(|resp| Request::Encode { frame, resp }).await } - pub async fn set_bitrate(&mut self, bitrate: moq_net::bandwidth::Rate) -> Result<(), Error> { - self.0.request(|resp| Request::SetBitrate { bitrate, resp }).await + pub fn receive(&mut self) -> Result, Error> { + Ok(self.queue.pop()) } - pub async fn flush(&mut self) -> Result, Error> { - self.0.request(|resp| Request::Flush { resp }).await + pub async fn set_bitrate(&mut self, bitrate: moq_net::bandwidth::Rate) -> Result<(), Error> { + self.worker.request(|resp| Request::SetBitrate { bitrate, resp }).await } - pub async fn finish(mut self) -> Result, Error> { - // `self` drops on the way out, which drops the sender and joins the - // thread that just drained and released the encoder. - self.0.request(|resp| Request::Finish { resp }).await + pub async fn flush(&mut self) -> Result<(), Error> { + self.worker.request(|resp| Request::Flush { resp }).await } } } @@ -324,21 +327,21 @@ mod inline { self.0.cut() } - pub async fn encode(&mut self, frame: Arc) -> Result, Error> { + pub async fn encode(&mut self, frame: Arc) -> Result<(), Error> { self.0.encode(&frame) } + pub fn receive(&mut self) -> Result, Error> { + self.0.receive() + } + pub async fn set_bitrate(&mut self, bitrate: moq_net::bandwidth::Rate) -> Result<(), Error> { self.0.set_bitrate(bitrate) } - pub async fn flush(&mut self) -> Result, Error> { + pub async fn flush(&mut self) -> Result<(), Error> { self.0.flush() } - - pub async fn finish(self) -> Result, Error> { - self.0.finish() - } } } @@ -490,7 +493,7 @@ mod tests { // Drive it the way an FFI handle gets driven: a fresh caller thread every // time, none of them the thread that opened it. let mut callers = vec![std::thread::current().id()]; - let mut flushed = Vec::new(); + let mut received = Vec::new(); for index in 0..3u64 { let sink = sink.clone(); let caller = std::thread::spawn(move || { @@ -501,27 +504,35 @@ mod tests { pollster::block_on(sink.set_bitrate(moq_net::bandwidth::Rate::from_bps(500_000 + index))).unwrap(); // Only the first frame closes a group, so the two after it stay in // the codec and leave the drain below something to find. - let flushed = match index { - 0 => pollster::block_on(sink.flush()).unwrap(), - _ => Vec::new(), - }; - (std::thread::current().id(), flushed) + if index == 0 { + pollster::block_on(sink.flush()).unwrap(); + } + let mut received = Vec::new(); + while let Some(frame) = sink.receive().unwrap() { + received.push(frame.timestamp.as_micros()); + } + (std::thread::current().id(), received) }); - let (caller, drained) = caller.join().unwrap(); + let (caller, frames) = caller.join().unwrap(); callers.push(caller); - flushed.extend(drained); + received.extend(frames); } // The probe holds each frame back by one, so a flush that reached the codec - // hands back the frame the group ended on. An inherited no-op would return - // nothing here and silently drop it into the next group. - let flushed: Vec<_> = flushed.iter().map(|frame| frame.timestamp.as_micros()).collect(); - assert_eq!(flushed, vec![0], "the group boundary did not empty the codec"); + // hands back the frame the group ended on, ahead of frame 1, which frame 2 + // pushed out. An inherited no-op would leave frame 0 to surface after the + // boundary, in the next group. + assert_eq!(received, vec![0, 33_333], "the group boundary did not empty the codec"); - // ...and finished, so dropped, from yet another. + // ...and drained, then dropped, from yet another. let closer = std::thread::spawn(move || { - let sink = sink.lock().unwrap().take().unwrap(); - let tail = pollster::block_on(sink.finish()).unwrap(); + let mut sink = sink.lock().unwrap().take().unwrap(); + pollster::block_on(sink.flush()).unwrap(); + let mut tail = Vec::new(); + while let Some(frame) = sink.receive().unwrap() { + tail.push(frame.timestamp.as_micros()); + } + drop(sink); (std::thread::current().id(), tail) }); let (closer, tail) = closer.join().unwrap(); @@ -529,11 +540,10 @@ mod tests { // Frame 2 never came back from an encode call and no flush claimed it, so // the drain has to. Dropping the sink instead would lose it silently. - let tail: Vec<_> = tail.iter().map(|frame| frame.timestamp.as_micros()).collect(); assert_eq!(tail, vec![2 * 33_333], "the drain lost the codec's tail"); let log = probe::take(); - for what in ["open", "encode", "set_bitrate", "flush", "finish", "drop"] { + for what in ["open", "encode", "set_bitrate", "flush", "drop"] { assert!(log.iter().any(|(event, _)| *event == what), "no {what} in {log:?}"); } diff --git a/rs/moq-video/src/frame/cuda_test.rs b/rs/moq-video/src/frame/cuda_test.rs index ea037f71115..bdd0baf3b08 100644 --- a/rs/moq-video/src/frame/cuda_test.rs +++ b/rs/moq-video/src/frame/cuda_test.rs @@ -248,7 +248,7 @@ async fn vulkan_cuda_convert_resize_encode() { sd_encoder.cut().expect("NVENC cuts on request"); } let packets = hd - .encode(&VideoFrame::new(Surface::Cuda(converted), timestamp)) + .encode_all(&VideoFrame::new(Surface::Cuda(converted), timestamp)) .unwrap(); assert_eq!(packets.len(), 1, "one access unit per frame at {i}"); assert_eq!(packets[0].timestamp, timestamp, "timestamp preserved at {i}"); @@ -258,13 +258,13 @@ async fn vulkan_cuda_convert_resize_encode() { if idr { assert!(types.contains(&7) && types.contains(&8), "IDR at {i} lacks SPS/PPS"); } - for out in decoder.decode(packets[0].payload.clone(), timestamp, i == 0).unwrap() { + for out in decoder.decode_all(packets[0].payload.clone(), timestamp, i == 0).unwrap() { decoded = Some(out.surface.to_i420().unwrap().into_owned()); expected = Some(reference(&gradient(size, Format::Rgba8, i as usize * 4), size, color)); } let packets = sd_encoder - .encode(&VideoFrame::new(Surface::Cuda(scaled), timestamp)) + .encode_all(&VideoFrame::new(Surface::Cuda(scaled), timestamp)) .unwrap(); assert_eq!(packets.len(), 1); assert_eq!(packets[0].timestamp, timestamp); @@ -422,11 +422,11 @@ async fn vulkan_cuda_three_view_workload() { .expect("resize"); let packets = encode_hd - .measure(|| hd.encode(&VideoFrame::new(Surface::Cuda(converted), timestamp))) + .measure(|| hd.encode_all(&VideoFrame::new(Surface::Cuda(converted), timestamp))) .expect("encode HD"); bytes.0 += packets.iter().map(|p| p.payload.len()).sum::(); let packets = encode_sd - .measure(|| small.encode(&VideoFrame::new(Surface::Cuda(scaled), timestamp))) + .measure(|| small.encode_all(&VideoFrame::new(Surface::Cuda(scaled), timestamp))) .expect("encode SD"); bytes.1 += packets.iter().map(|p| p.payload.len()).sum::(); @@ -572,14 +572,14 @@ async fn vulkan_cuda_auto_external_encode() { let output = encoder.size(); encoder.cut().unwrap(); let encoded = encoder - .encode(&frame) + .encode_all(&frame) .expect("external image encoded entirely on the GPU"); assert_eq!(encoded.len(), 1, "one access unit for {output} capture {capture}"); let mut decoded = Vec::new(); for packet in encoded { decoded.extend( decoder - .decode(packet.payload, packet.timestamp, packet.keyframe) + .decode_all(packet.payload, packet.timestamp, packet.keyframe) .unwrap(), ); } diff --git a/rs/moq-video/src/render/export.rs b/rs/moq-video/src/render/export.rs index 8b6367bf54f..4f18b7cbca0 100644 --- a/rs/moq-video/src/render/export.rs +++ b/rs/moq-video/src/render/export.rs @@ -576,8 +576,8 @@ pub(crate) mod tests { let red = texture(&device, &queue, size, [255, 0, 0, 255]); let frame = exporter.copy(&red, Timestamp::ZERO).unwrap().unwrap().wait().unwrap(); - let mut encoded = encoder.encode(&frame).unwrap(); - encoded.extend(encoder.finish().unwrap()); + let mut encoded = encoder.encode_all(&frame).unwrap(); + encoded.extend(encoder.flush_all().unwrap()); assert!(encoded.first().is_some_and(|encoded| encoded.keyframe)); } diff --git a/rs/moq-video/src/render/export/dx12.rs b/rs/moq-video/src/render/export/dx12.rs index efc8507558d..1d47baa8003 100644 --- a/rs/moq-video/src/render/export/dx12.rs +++ b/rs/moq-video/src/render/export/dx12.rs @@ -365,9 +365,9 @@ mod tests { .expect("a free slot") .wait() .unwrap(); - encoded.extend(encoder.encode(&frame).unwrap()); + encoded.extend(encoder.encode_all(&frame).unwrap()); } - encoded.extend(encoder.finish().unwrap()); + encoded.extend(encoder.flush_all().unwrap()); assert!(encoded.first().is_some_and(|encoded| encoded.keyframe)); } } diff --git a/rs/moq-video/src/render/export/metal.rs b/rs/moq-video/src/render/export/metal.rs index 737885c3dfe..f0894268bb3 100644 --- a/rs/moq-video/src/render/export/metal.rs +++ b/rs/moq-video/src/render/export/metal.rs @@ -322,7 +322,7 @@ mod tests { let actual = center(&frame); assert!(near(actual, [255, 0, 0, 255]), "{actual:?}"); - let mut encoded = encoder.encode(&frame).unwrap(); + let mut encoded = encoder.encode_all(&frame).unwrap(); drop(frame); drop(held); for index in 1..12u64 { @@ -333,9 +333,9 @@ mod tests { .expect("a free buffer") .wait() .unwrap(); - encoded.extend(encoder.encode(&frame).unwrap()); + encoded.extend(encoder.encode_all(&frame).unwrap()); } - encoded.extend(encoder.finish().unwrap()); + encoded.extend(encoder.flush_all().unwrap()); assert!(encoded.first().is_some_and(|encoded| encoded.keyframe)); } } diff --git a/rs/moq-video/src/render/export/vulkan.rs b/rs/moq-video/src/render/export/vulkan.rs index ea1ee7dc816..53189abfa5f 100644 --- a/rs/moq-video/src/render/export/vulkan.rs +++ b/rs/moq-video/src/render/export/vulkan.rs @@ -676,7 +676,7 @@ mod tests { let pending = exporter.copy(&red, timestamp).unwrap().expect("a slot came back"); let frame = pending.wait().unwrap(); assert!(matches!(frame.surface, Surface::Vulkan(_))); - encoded.extend(encoder.encode(&frame).unwrap()); + encoded.extend(encoder.encode_all(&frame).unwrap()); drop(frame); let super::super::Slots::Vulkan(export) = &exporter.slots else { unreachable!("checked the route above"); @@ -687,7 +687,7 @@ mod tests { "a slot was replaced rather than reused" ); } - encoded.extend(encoder.finish().unwrap()); + encoded.extend(encoder.flush_all().unwrap()); assert_eq!(encoded.len(), Exporter::DEPTH * 4); assert!(encoded[0].keyframe); } diff --git a/rs/moq-video/src/render/renderer.rs b/rs/moq-video/src/render/renderer.rs index 1ed5a169538..08ecdd43b47 100644 --- a/rs/moq-video/src/render/renderer.rs +++ b/rs/moq-video/src/render/renderer.rs @@ -1180,16 +1180,16 @@ mod tests { } let surface = Surface::rgba(&rgba, size).expect("a valid RGBA frame"); let frame = Frame::new(surface, Timestamp::from_micros(index * 33_333).unwrap()); - for unit in encoder.encode(&frame).expect("encode a picture") { + for unit in encoder.encode_all(&frame).expect("encode a picture") { let timestamp = unit.timestamp; exported.extend( exporting - .decode(unit.payload.clone(), timestamp, index == 0) + .decode_all(unit.payload.clone(), timestamp, index == 0) .expect("decode to the GPU"), ); downloaded.extend( downloading - .decode(unit.payload, timestamp, index == 0) + .decode_all(unit.payload, timestamp, index == 0) .expect("decode to the CPU"), ); } @@ -1343,12 +1343,12 @@ mod tests { encoder.cut().unwrap(); } let frame = Frame::new(picture(index), Timestamp::from_micros(index * 33_333).unwrap()); - for unit in encoder.encode(&frame).expect("encode a picture") { + for unit in encoder.encode_all(&frame).expect("encode a picture") { let exported = exporting - .decode(unit.payload.clone(), unit.timestamp, index == 0) + .decode_all(unit.payload.clone(), unit.timestamp, index == 0) .expect("decode to the GPU"); let downloaded = downloading - .decode(unit.payload, unit.timestamp, index == 0) + .decode_all(unit.payload, unit.timestamp, index == 0) .expect("decode to the CPU"); assert_eq!(exported.len(), downloaded.len(), "the two decoders disagreed"); diff --git a/rs/moq-video/src/worker.rs b/rs/moq-video/src/worker.rs index aa56c394887..9f667898a0b 100644 --- a/rs/moq-video/src/worker.rs +++ b/rs/moq-video/src/worker.rs @@ -2,7 +2,7 @@ //! //! Shared by [`encode::Sink`](crate::encode::Sink) and //! [`decode::Sink`](crate::decode::Sink), which differ only in the requests they -//! serve. The reason both need it is the same: on Windows a Media Foundation +//! serve and the outputs they queue. The reason both need it is the same: on Windows a Media Foundation //! transform's COM apartment is per-thread, so the codec has to be created, //! driven, and dropped on one thread, and a `!Send`-in-practice handle cannot be //! left to migrate between executor workers or FFI callers. @@ -11,12 +11,45 @@ //! the cancellation poison, joining on drop). Each codec supplies its own //! request type and serve loop. +use std::collections::VecDeque; +use std::sync::{Arc, Mutex, MutexGuard}; use std::thread::JoinHandle; use tokio::sync::{mpsc, oneshot}; use crate::Error; +/// Outputs the codec thread readied and the caller has not popped yet. +/// +/// The thread fills it before replying to the call that readied them, so a +/// caller that awaited that call pops synchronously. Reused, so it stops +/// allocating once it has held one call's worth. +pub(crate) struct Queue(Arc>>); + +impl Queue { + /// Lock the queue. A thread that panicked holding it left only whole items. + pub(crate) fn lock(&self) -> MutexGuard<'_, VecDeque> { + self.0.lock().unwrap_or_else(|err| err.into_inner()) + } + + /// Pop the oldest output. + pub(crate) fn pop(&self) -> Option { + self.lock().pop_front() + } +} + +impl Default for Queue { + fn default() -> Self { + Self(Arc::default()) + } +} + +impl Clone for Queue { + fn clone(&self) -> Self { + Self(self.0.clone()) + } +} + /// The reply channel a worker thread uses to report that its codec is built. /// /// Sending what the codec resolved to (`I`, at least its backend name) hands the diff --git a/swift/Sources/Moq/Video.swift b/swift/Sources/Moq/Video.swift index bf2eb255890..f02502b31d5 100644 --- a/swift/Sources/Moq/Video.swift +++ b/swift/Sources/Moq/Video.swift @@ -211,8 +211,11 @@ public enum Video { ffi = try MoqVideoEncoder(input: input, output: output) } - /// Encode one picture, returning any access units now available. - public func encode(_ frame: Frame) throws -> [Packet] { try ffi.encode(frame: frame) } + /// Submit one picture, whose access units `receive` returns. + public func encode(_ frame: Frame) throws { try ffi.encode(frame: frame) } + + /// Pop the oldest ready access unit, or nil while the codec holds the rest; call until nil. + public func receive() throws -> Packet? { try ffi.receive() } /// Force the next encoded picture to start a new group. public func cut() throws { try ffi.cut() } @@ -220,10 +223,7 @@ public enum Video { /// Retune the live encoder in bits per second. public func setBitrate(_ bitrate: UInt64) throws { try ffi.setBitrate(bitrate: bitrate) } - /// Drain buffered access units and keep the encoder open. - public func flush() throws -> [Packet] { try ffi.flush() } - - /// Drain and close the encoder. - public func finish() throws -> [Packet] { try ffi.finish() } + /// Mark the end of input, so `receive` returns every access unit the codec still holds, without closing the encoder. + public func flush() throws { try ffi.flush() } } } From 6c0d45e9b5f68efdbdf467992286f6b8bece63dd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 9 Nov 2026 06:32:04 -0800 Subject: [PATCH 3/5] feat(video): openh264 codec bench, nightly coverage, and close the quest Co-Authored-By: Claude Opus 5.5 --- quest/m2/README.md | 1 - quest/m2/video-codec-receive.md | 17 --------- rs/moq-c/src/test/parity.rs | 6 ++- rs/moq-transcode/src/rung.rs | 12 +++++- rs/moq-video/Cargo.toml | 4 +- .../benches/{codec.rs => openh264.rs} | 2 +- rs/moq-video/examples/encode-presets.rs | 22 +++++------ rs/moq-video/src/capture/wgc.rs | 4 +- rs/moq-video/src/decode/backend/mediacodec.rs | 3 +- rs/moq-video/src/decode/backend/nvdec.rs | 5 ++- .../src/decode/backend/videotoolbox.rs | 3 +- rs/moq-video/src/decode/consumer.rs | 4 +- rs/moq-video/src/decode/decoder.rs | 38 ++++++++++++++++--- rs/moq-video/src/encode/encoded.rs | 2 +- rs/moq-video/src/encode/encoder.rs | 11 +++++- rs/moq-video/src/encode/producer.rs | 12 +++--- rs/moq-video/src/frame/cuda_test.rs | 5 ++- sh/gh/nightly.sh | 8 ++-- 18 files changed, 97 insertions(+), 62 deletions(-) delete mode 100644 quest/m2/video-codec-receive.md rename rs/moq-video/benches/{codec.rs => openh264.rs} (99%) diff --git a/quest/m2/README.md b/quest/m2/README.md index f52ea88f8bc..f1f9fd317b5 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -197,7 +197,6 @@ quests that require them. - [Stream credit by priority](/quest/m1/stream-credit/README.md) - every stream opens through one priority entry point, and a higher-priority group takes over or resets a lower one when credit runs out (held in m1 by an open claim) - [Finalize moq-lite-07](/quest/m2/lite07-finalize.md) - when the maintainer cuts it, lite-07 negotiates as `moq-lite-07` and the next release ships it - [Unbounded groups](/quest/m2/unbounded-groups/README.md) - groups and frames lose their size caps; a group keeps a 32 MB tail, and readers behind it replay through paced flights or refresh -- [moq-video codecs split submission from receive](/quest/m2/video-codec-receive.md) - video encoders and decoders submit, then `receive` one output, the shape moq-audio took, so no call allocates a `Vec` of outputs - [Connection state carries each phase's data](/quest/m2/connection-state-phases.md) - moq-tokio's `Observation`, moq-ffi's `MoqConnectionState`, and moq-c's `moq_connection_state` each hold one phase per value, so no field is readable in a phase where it means nothing - [JS time values are types, not numbers](/quest/m2/js-time-types.md) - every public JS duration and instant is a `Clock.Duration` or `Clock.Instant` object, so a unit mistake fails to compile - [Small API fixes before the cut](/quest/m2/api-shape/README.md) - one path and one spelling per item, names by role, and no dead or wrong-role public surface across net, transport, media, native, apps, JS, and bindings diff --git a/quest/m2/video-codec-receive.md b/quest/m2/video-codec-receive.md deleted file mode 100644 index 1f695e86daa..00000000000 --- a/quest/m2/video-codec-receive.md +++ /dev/null @@ -1,17 +0,0 @@ -# [L] moq-video codecs split submission from receive - -## Goal - -moq-video's encoders and decoders take the same send/receive shape moq-audio took in [moq-dev/moq#8270](https://github.com/moq-dev/moq/pull/8270): `encode`/`decode` only submit and return `Result<()>`, `receive` pops one ready output (`Ok(None)` when nothing is ready), and `flush` marks the end of input so `receive` drains what the codec held. -No call allocates a `Vec` just to hand back its outputs. - -## Plan - -- Decided 2026-11-08 (maintainer, on #8270): the FFmpeg/MediaCodec split, rejecting a caller-owned `&mut Vec` buffer and keeping `Vec`. moq-video still returns `Vec` from its backend traits, `encode::Sink`, and the decoder front end, so the two crates disagree until this lands. -- Apply it to every backend (VA-API, NVDEC, Media Foundation, VideoToolbox, MediaCodec, openh264, and the rest under `rs/moq-video/src/{encode,decode}/backend/`), the front ends, and their tests. A backend that readies output synchronously holds it in a reused `VecDeque`. -- Mirror it through moq-ffi (`MoqVideoEncoder.encode`/`flush`/`finish`), its Python, Swift, Go, Dart, and Kotlin wrappers, moq-c (`moq_video_encoder_encode`/`flush`, with a `moq_video_encoder_receive` like `moq_audio_encoder_receive`), and the OBS video encoder. -- Benchmark the encode and decode paths base vs branch; allocations should drop or match. - -## Related - -- [moq-dev/moq#8270](https://github.com/moq-dev/moq/pull/8270) - the moq-audio change whose shape this copies diff --git a/rs/moq-c/src/test/parity.rs b/rs/moq-c/src/test/parity.rs index 69f482acf66..3117b74420a 100644 --- a/rs/moq-c/src/test/parity.rs +++ b/rs/moq-c/src/test/parity.rs @@ -675,7 +675,11 @@ fn video_encoder_without_a_broadcast() { } assert_eq!(moq_video_encoder_flush(encoder), 0); receive(&mut keyframes); - assert_eq!(keyframes, [true, false, false, true, false], "keyframes at the start and the cut"); + assert_eq!( + keyframes, + [true, false, false, true, false], + "keyframes at the start and the cut" + ); assert_eq!(moq_video_encoder_free(encoder), 0); assert!(moq_video_encoder_free(encoder) < 0); } diff --git a/rs/moq-transcode/src/rung.rs b/rs/moq-transcode/src/rung.rs index 150a991bf02..d43d34b32d2 100644 --- a/rs/moq-transcode/src/rung.rs +++ b/rs/moq-transcode/src/rung.rs @@ -720,7 +720,11 @@ async fn transcode_group_inner( } /// Write everything the encoder has ready to the output group. -fn drain(output: &mut moq_net::group::Producer, active: &crate::active::Guard, encoder: &mut Encoder) -> Result<(), Error> { +fn drain( + output: &mut moq_net::group::Producer, + active: &crate::active::Guard, + encoder: &mut Encoder, +) -> Result<(), Error> { while let Some(encoded) = encoder.receive()? { write(output, active, encoded)?; } @@ -1152,7 +1156,11 @@ mod tests { } let timestamps: Vec = encoded.iter().map(|e| e.timestamp.as_micros() as u64).collect(); - assert_eq!(timestamps, [0, 33_333, 66_666], "the old tail comes out ahead of the switch"); + assert_eq!( + timestamps, + [0, 33_333, 66_666], + "the old tail comes out ahead of the switch" + ); assert!(encoded[0].keyframe); assert!(encoded[2].keyframe, "the switched frame"); } diff --git a/rs/moq-video/Cargo.toml b/rs/moq-video/Cargo.toml index 4544d8a0360..43476cba5df 100644 --- a/rs/moq-video/Cargo.toml +++ b/rs/moq-video/Cargo.toml @@ -273,9 +273,9 @@ windows = { version = "0.62", features = [ cfg_aliases = "0.2.2" [dev-dependencies] -criterion = { package = "bench-harness", path = "../bench-harness" } # Native Vulkan producer for the opt-in Vulkan/CUDA interop hardware exercise. ash = { workspace = true } +criterion = { package = "bench-harness", path = "../bench-harness" } # `poll!`, to cancel an `encode::Sink` request exactly where a `select!` would. futures = { workspace = true } # `Manual::install`, a clock the capture-loop tests move while the encoder thread works. @@ -291,7 +291,7 @@ tracing-test = { workspace = true } workspace-hack = { path = "../workspace-hack" } [[bench]] -name = "codec" +name = "openh264" harness = false required-features = ["openh264"] diff --git a/rs/moq-video/benches/codec.rs b/rs/moq-video/benches/openh264.rs similarity index 99% rename from rs/moq-video/benches/codec.rs rename to rs/moq-video/benches/openh264.rs index d2d36252a3f..cdf9b66dae8 100644 --- a/rs/moq-video/benches/codec.rs +++ b/rs/moq-video/benches/openh264.rs @@ -6,7 +6,7 @@ //! Every case submits each frame and receives what it readied before the next, then //! flushes and receives the tail, the way a live caller drives a codec. //! -//! Run with `cargo bench -p moq-video --features openh264 --bench codec`. A table comes +//! Run with `cargo bench -p moq-video --features openh264 --bench openh264`. A table comes //! first: allocations and allocated bytes per frame from an untimed pass, counting the //! codec's own and every frame's output. Criterion then times each case. use std::alloc::{GlobalAlloc, Layout as AllocLayout, System}; diff --git a/rs/moq-video/examples/encode-presets.rs b/rs/moq-video/examples/encode-presets.rs index 20fbe04e56d..6a9e3af61eb 100644 --- a/rs/moq-video/examples/encode-presets.rs +++ b/rs/moq-video/examples/encode-presets.rs @@ -144,18 +144,16 @@ fn run(config: &Config, pictures: &[&[u8]], paced: bool) -> anyhow::Result let mut returned = Vec::with_capacity(pictures.len()); let mut stream = Vec::new(); - let mut collect = |encoder: &mut Encoder, - submitted: &[Instant], - emitted: &mut [Option]| - -> anyhow::Result<()> { - let now = Instant::now(); - while let Some(packet) = encoder.receive()? { - let index = (packet.timestamp.as_micros() / frame_time.as_micros()) as usize; - emitted[index] = Some(now - submitted[index]); - stream.extend_from_slice(&packet.payload); - } - Ok(()) - }; + let mut collect = + |encoder: &mut Encoder, submitted: &[Instant], emitted: &mut [Option]| -> anyhow::Result<()> { + let now = Instant::now(); + while let Some(packet) = encoder.receive()? { + let index = (packet.timestamp.as_micros() / frame_time.as_micros()) as usize; + emitted[index] = Some(now - submitted[index]); + stream.extend_from_slice(&packet.payload); + } + Ok(()) + }; let start = Instant::now(); let cpu = cpu_time(); diff --git a/rs/moq-video/src/capture/wgc.rs b/rs/moq-video/src/capture/wgc.rs index 3dac1d28278..b6f54a51fc7 100644 --- a/rs/moq-video/src/capture/wgc.rs +++ b/rs/moq-video/src/capture/wgc.rs @@ -330,8 +330,8 @@ mod tests { let (surface, timestamp) = delivery.next(now).unwrap().unwrap(); let frame = crate::Frame::new(crate::Surface::I420(surface.clone()), timestamp); for encoded in encoder.encode_all(&frame).unwrap() { - producer.publish(&encoded).unwrap(); - } + producer.publish(&encoded).unwrap(); + } let received = loop { if let moq_mux::container::Read::Frame { frame, .. } = subscriber.read().await.unwrap().unwrap() { break frame; diff --git a/rs/moq-video/src/decode/backend/mediacodec.rs b/rs/moq-video/src/decode/backend/mediacodec.rs index 7645c5ce3e2..ded7d7c62de 100644 --- a/rs/moq-video/src/decode/backend/mediacodec.rs +++ b/rs/moq-video/src/decode/backend/mediacodec.rs @@ -366,7 +366,8 @@ impl MediaCodec { let timestamp = Timestamp::from_nanos(nanos.max(0) as u64)?; let (left, top, width, height) = image_size(&image)?; let buffer = HardwareBuffer::new(self.reader.clone(), image, left, top, width, height); - self.ready.push_back(Frame::new(Surface::HardwareBuffer(buffer), timestamp)); + self.ready + .push_back(Frame::new(Surface::HardwareBuffer(buffer), timestamp)); } // The picture is on its way but not queued yet, so it comes out on the // next round rather than being lost. diff --git a/rs/moq-video/src/decode/backend/nvdec.rs b/rs/moq-video/src/decode/backend/nvdec.rs index 9686f87d869..85876d865ba 100644 --- a/rs/moq-video/src/decode/backend/nvdec.rs +++ b/rs/moq-video/src/decode/backend/nvdec.rs @@ -711,7 +711,10 @@ mod tests { let mut timestamps = Vec::new(); encoder.cut().unwrap(); for i in 0..10u64 { - for encoded in encoder.encode_all(&gradient_frame(&rgba, w, h, group * 10 + i)).unwrap() { + for encoded in encoder + .encode_all(&gradient_frame(&rgba, w, h, group * 10 + i)) + .unwrap() + { let frames = decoder.decode_all(encoded.payload, encoded.timestamp, i == 0).unwrap(); timestamps.extend(frames.iter().map(|frame| frame.timestamp.as_micros() as u64)); } diff --git a/rs/moq-video/src/decode/backend/videotoolbox.rs b/rs/moq-video/src/decode/backend/videotoolbox.rs index 6a77ab04923..25f6d5909ad 100644 --- a/rs/moq-video/src/decode/backend/videotoolbox.rs +++ b/rs/moq-video/src/decode/backend/videotoolbox.rs @@ -301,7 +301,8 @@ impl VideoToolbox { let mut frames = std::mem::take(&mut self.sink.frames); let queued = frames.drain(..).try_for_each(|(surface, sample_time)| { let timestamp = self.take_timestamp(sample_time)?; - self.ready.push_back(Frame::new(Surface::PixelBuffer(surface), timestamp)); + self.ready + .push_back(Frame::new(Surface::PixelBuffer(surface), timestamp)); Ok(()) }); frames.clear(); diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index ea0c664c0f8..4e08acbd4d0 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -506,8 +506,8 @@ mod tests { let surface = crate::Surface::rgba(&rgba, crate::Size::new(320, 240)).unwrap(); let frame = crate::Frame::new(surface, moq_net::Timestamp::from_micros(index * 33_333).unwrap()); for encoded in encoder.encode_all(&frame).unwrap() { - producer.publish(&encoded).unwrap(); - } + producer.publish(&encoded).unwrap(); + } } let origin = produce_origin(); diff --git a/rs/moq-video/src/decode/decoder.rs b/rs/moq-video/src/decode/decoder.rs index dc139f5beb9..cb55ba97dbb 100644 --- a/rs/moq-video/src/decode/decoder.rs +++ b/rs/moq-video/src/decode/decoder.rs @@ -361,7 +361,12 @@ impl Decoder { /// Submit one container frame and collect everything it readied. #[cfg(test)] - pub(crate) fn decode_all(&mut self, payload: &Bytes, timestamp: Timestamp, keyframe: bool) -> Result, Error> { + pub(crate) fn decode_all( + &mut self, + payload: &Bytes, + timestamp: Timestamp, + keyframe: bool, + ) -> Result, Error> { self.decode(payload, timestamp, keyframe)?; self.receive_all() } @@ -530,7 +535,11 @@ mod tests { } // Distinct, spread-apart timestamps so a round-tripped value is unambiguous. for encoded in encoder.encode_all(&gray_frame(i)).unwrap() { - decoded.extend(decoder.decode_all(encoded.payload, encoded.timestamp, keyframe).unwrap()); + decoded.extend( + decoder + .decode_all(encoded.payload, encoded.timestamp, keyframe) + .unwrap(), + ); } } decoded.extend(decoder.flush_all().unwrap()); @@ -623,7 +632,11 @@ mod tests { encoded.payload.starts_with(&[0, 0, 0, 1]) || encoded.payload.starts_with(&[0, 0, 1]), "the test feeds Annex-B, not length-prefixed NALs" ); - decoded.extend(decoder.decode_all(&encoded.payload, encoded.timestamp, keyframe).unwrap()); + decoded.extend( + decoder + .decode_all(&encoded.payload, encoded.timestamp, keyframe) + .unwrap(), + ); } } @@ -850,7 +863,12 @@ mod tests { let kind = super::Kind::Named(probe::BUFFERED_NAME.into()); let mut decoder = super::Decoder::new(&probe_catalog(), &decode_config(kind)).unwrap(); - assert!(decoder.decode_all(&keyframe(Color::BT709), first, true).unwrap().is_empty()); + assert!( + decoder + .decode_all(&keyframe(Color::BT709), first, true) + .unwrap() + .is_empty() + ); let held = decoder.decode_all(&keyframe(Color::BT601), second, true).unwrap(); assert_eq!(held.len(), 1); assert_eq!(held[0].timestamp, first); @@ -1112,7 +1130,11 @@ mod tests { encoder.cut().unwrap(); } for encoded in encoder.encode_all(&gray_frame(i)).unwrap() { - decoded.extend(decoder.decode_all(encoded.payload, encoded.timestamp, keyframe).unwrap()); + decoded.extend( + decoder + .decode_all(encoded.payload, encoded.timestamp, keyframe) + .unwrap(), + ); } } @@ -1278,7 +1300,11 @@ mod tests { encoder.cut().unwrap(); } for encoded in encoder.encode_all(&flat_frame(i, level(i), size)).unwrap() { - decoded.extend(decoder.decode_all(encoded.payload, encoded.timestamp, keyframe).unwrap()); + decoded.extend( + decoder + .decode_all(encoded.payload, encoded.timestamp, keyframe) + .unwrap(), + ); } } diff --git a/rs/moq-video/src/encode/encoded.rs b/rs/moq-video/src/encode/encoded.rs index 68e46411c17..5fc94fe9cfb 100644 --- a/rs/moq-video/src/encode/encoded.rs +++ b/rs/moq-video/src/encode/encoded.rs @@ -12,7 +12,7 @@ use moq_net::Timestamp; /// /// The encoder carries the timestamp through the codec rather than leaving the /// caller to guess one at publish time, so a backend that buffers a frame or -/// drains a tail from [`Encoder::finish`](super::Encoder::finish) still stamps +/// drains a tail on [`Encoder::flush`](super::Encoder::flush) still stamps /// each access unit with the time of the picture it encoded. #[derive(Clone, Debug)] #[non_exhaustive] diff --git a/rs/moq-video/src/encode/encoder.rs b/rs/moq-video/src/encode/encoder.rs index 68a57b9c9e4..f1942ecd2f6 100644 --- a/rs/moq-video/src/encode/encoder.rs +++ b/rs/moq-video/src/encode/encoder.rs @@ -836,7 +836,10 @@ mod tests { let Ok(mut encoder) = Encoder::new(&Config::new(320, 240, crate::FrameRate::new(30, 1).unwrap())) else { return; }; - assert!(matches!(encoder.encode_all(&gray_frame(640, 480, 0)), Err(Error::Codec(_)))); + assert!(matches!( + encoder.encode_all(&gray_frame(640, 480, 0)), + Err(Error::Codec(_)) + )); } /// An encoder told to resize scales a frame at another size itself, so a @@ -2013,7 +2016,11 @@ mod tests { let mut decoder = crate::decode::Decoder::new(&catalog, &decode).unwrap(); let mut decoded = Vec::new(); for unit in &encoded { - decoded.extend(decoder.decode_all(&unit.payload, unit.timestamp, unit.keyframe).unwrap()); + decoded.extend( + decoder + .decode_all(&unit.payload, unit.timestamp, unit.keyframe) + .unwrap(), + ); } decoded.extend(decoder.flush_all().unwrap()); let picture = decoded.first().expect("a decoded picture"); diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index a2dca930f2c..77de90bbba3 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -1117,8 +1117,8 @@ mod tests { let surface = crate::Surface::rgba(&rgba, crate::Size::new(320, 240)).unwrap(); let frame = Frame::new(surface, Timestamp::from_micros(i * 33_333).unwrap()); for encoded in encoder.encode_all(&frame).unwrap() { - producer.publish(&encoded).unwrap(); - } + producer.publish(&encoded).unwrap(); + } } for encoded in encoder.flush_all().unwrap() { producer.publish(&encoded).unwrap(); @@ -1195,8 +1195,8 @@ mod tests { let surface = crate::Surface::rgba(&rgba, crate::Size::new(320, 240)).unwrap(); let frame = Frame::new(surface, Timestamp::from_micros(timestamp).unwrap()); for encoded in encoder.encode_all(&frame).unwrap() { - producer.publish(&encoded).unwrap(); - } + producer.publish(&encoded).unwrap(); + } } producer.finish().unwrap(); @@ -1231,8 +1231,8 @@ mod tests { let surface = crate::Surface::rgba(&rgba, crate::Size::new(config.width, config.height)).unwrap(); let frame = Frame::new(surface, Timestamp::from_micros(timestamp).unwrap()); for encoded in encoder.encode_all(&frame).unwrap() { - producer.publish(&encoded).unwrap(); - } + producer.publish(&encoded).unwrap(); + } capture_stopped(&mut producer).unwrap(); } diff --git a/rs/moq-video/src/frame/cuda_test.rs b/rs/moq-video/src/frame/cuda_test.rs index bdd0baf3b08..158869770f4 100644 --- a/rs/moq-video/src/frame/cuda_test.rs +++ b/rs/moq-video/src/frame/cuda_test.rs @@ -258,7 +258,10 @@ async fn vulkan_cuda_convert_resize_encode() { if idr { assert!(types.contains(&7) && types.contains(&8), "IDR at {i} lacks SPS/PPS"); } - for out in decoder.decode_all(packets[0].payload.clone(), timestamp, i == 0).unwrap() { + for out in decoder + .decode_all(packets[0].payload.clone(), timestamp, i == 0) + .unwrap() + { decoded = Some(out.surface.to_i420().unwrap().into_owned()); expected = Some(reference(&gradient(size, Format::Rgba8, i as usize * 4), size, color)); } diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index 30ca3654485..b79053c4e52 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -138,13 +138,15 @@ case "$1" in rust-extra) criterion qmux --manifest-path rs/qmux/Cargo.toml --bench receive criterion msf -p moq-msf --bench catalog - # The allocation/window/play-buffer/batch/receive-bursts tables keep their stdout in the run; + # The allocation/window/play-buffer/batch/receive-bursts/openh264 tables keep their stdout in the run; # only Criterion estimates enter the trend report. criterion extra --no-fail-fast -p moq-crypto -p moq-json -p moq-archive -p moq-cli -p moq-mux -p moq-audio \ - -p moq-quic -p moq-sock -p moq-tokio -p moq-transcode -p moq-hls --features moq-json/bench,moq-quic/fuzz \ + -p moq-quic -p moq-sock -p moq-tokio -p moq-transcode -p moq-hls -p moq-video \ + --features moq-json/bench,moq-quic/fuzz,moq-video/openh264 \ --bench protect --bench sign --bench codec --bench table --bench allocations --bench window \ --bench producer --bench play_buffer --bench jitter --bench sync --bench decode_clock --bench ts_programs --bench ts_video \ - --bench bus --bench batches --bench parameter_sets --bench sent_packets --bench demux --bench receive_bursts --bench ladder --bench fanout --bench include + --bench bus --bench batches --bench parameter_sets --bench sent_packets --bench demux --bench receive_bursts --bench ladder --bench fanout --bench include \ + --bench openh264 criterion audio-capture -p moq-audio --features capture --bench capture # moq-auth's published release is its own dev-dependency, so `-p moq-auth` is ambiguous. # The concurrent cases' latency percentiles stay in the run's stdout. From 168ba354918873d27f42a9606caffb6c24afa108 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 9 Nov 2026 06:37:45 -0800 Subject: [PATCH 4/5] fix(transcode): close a switched encoder before opening its replacement Co-Authored-By: Claude Opus 5.5 --- rs/moq-transcode/src/rung.rs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/rs/moq-transcode/src/rung.rs b/rs/moq-transcode/src/rung.rs index d43d34b32d2..8adb81a8d44 100644 --- a/rs/moq-transcode/src/rung.rs +++ b/rs/moq-transcode/src/rung.rs @@ -821,8 +821,8 @@ impl Pipeline { struct Encoder { /// The open encoder and the space it signals. open: Option<(moq_video::encode::Sink, moq_video::Color)>, - /// Encoders closed by a switch, flushed and holding their tails, oldest first. - retired: VecDeque, + /// The tail of encoders closed by a switch, received ahead of the open one's output. + tail: VecDeque, /// A cut asked for while no encoder was open, applied when one opens. pending_cut: bool, } @@ -858,7 +858,11 @@ impl Encoder { // drains, so a drain that fails still leaves the next encoder owing it. self.pending_cut = true; sink.flush().await?; - self.retired.push_back(sink); + while let Some(encoded) = sink.receive()? { + self.tail.push_back(encoded); + } + // Closed before its replacement opens, so a switch never needs a second hardware session. + drop(sink); } let sink = match &mut self.open { Some((sink, _)) => sink, @@ -877,13 +881,10 @@ impl Encoder { Ok(()) } - /// Pop the next encoded frame: a retired encoder's tail first, then the open one's. + /// Pop the next encoded frame: a closed encoder's tail first, then the open one's. fn receive(&mut self) -> Result, Error> { - while let Some(retired) = self.retired.front_mut() { - if let Some(encoded) = retired.receive()? { - return Ok(Some(encoded)); - } - self.retired.pop_front(); + if let Some(encoded) = self.tail.pop_front() { + return Ok(Some(encoded)); } match &mut self.open { Some((sink, _)) => Ok(sink.receive()?), From 25458c241068bf0d5681e6cf7b5cd5da14124872 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 10:13:24 -0800 Subject: [PATCH 5/5] fix(video): address review findings on submit/receive split Co-Authored-By: Claude Opus 5.5 --- rs/moq-c/src/video.rs | 4 ++-- rs/moq-ffi/src/video.rs | 4 ++-- rs/moq-video/benches/openh264.rs | 3 ++- rs/moq-video/src/decode/backend/mod.rs | 1 + rs/moq-video/src/decode/backend/nvdec.rs | 5 +++++ rs/moq-video/src/decode/consumer.rs | 5 ++++- 6 files changed, 16 insertions(+), 6 deletions(-) diff --git a/rs/moq-c/src/video.rs b/rs/moq-c/src/video.rs index 950be0547ae..22ac6c9c984 100644 --- a/rs/moq-c/src/video.rs +++ b/rs/moq-c/src/video.rs @@ -642,8 +642,8 @@ impl VideoEncoder { return Ok(()); }; let frame = moq_video::Frame::new(surface, timestamp); - // A backend that pipelines hands back an earlier frame's output, so this is - // zero or more access units rather than one per call. + // A backend that pipelines readies an earlier frame's output, so `publish` + // drains zero or more access units rather than one per call. block_on(encoder.encode(frame))?; let before = self.producer.demand(); let published = publish(encoder, &mut self.producer); diff --git a/rs/moq-ffi/src/video.rs b/rs/moq-ffi/src/video.rs index 31304775dce..51fea7f494f 100644 --- a/rs/moq-ffi/src/video.rs +++ b/rs/moq-ffi/src/video.rs @@ -484,8 +484,8 @@ impl VideoProducer { return Ok(()); }; let frame = moq_video::Frame::new(surface, frame.timestamp.try_into()?); - // A backend that pipelines hands back an earlier frame's output, so this is - // zero or more access units rather than one per call. + // A backend that pipelines readies an earlier frame's output, so `publish` + // drains zero or more access units rather than one per call. block_on(encoder.encode(frame))?; publish(encoder, &mut self.producer)?; Ok(()) diff --git a/rs/moq-video/benches/openh264.rs b/rs/moq-video/benches/openh264.rs index cdf9b66dae8..fa13f5c19ac 100644 --- a/rs/moq-video/benches/openh264.rs +++ b/rs/moq-video/benches/openh264.rs @@ -8,7 +8,8 @@ //! //! Run with `cargo bench -p moq-video --features openh264 --bench openh264`. A table comes //! first: allocations and allocated bytes per frame from an untimed pass, counting the -//! codec's own and every frame's output. Criterion then times each case. +//! Rust allocations of the codec wrapper and every frame's output, not OpenH264's +//! native ones. Criterion then times each case. use std::alloc::{GlobalAlloc, Layout as AllocLayout, System}; use std::hint::black_box; use std::sync::Arc; diff --git a/rs/moq-video/src/decode/backend/mod.rs b/rs/moq-video/src/decode/backend/mod.rs index f83f49276c9..1f9254ad0e9 100644 --- a/rs/moq-video/src/decode/backend/mod.rs +++ b/rs/moq-video/src/decode/backend/mod.rs @@ -164,6 +164,7 @@ pub(crate) trait Backend { /// Flush and collect the tail. #[cfg(test)] + #[allow(dead_code, reason = "only the platform backends' tests use it")] fn flush_all(&mut self) -> Result, Error> { self.flush()?; self.receive_all() diff --git a/rs/moq-video/src/decode/backend/nvdec.rs b/rs/moq-video/src/decode/backend/nvdec.rs index 85876d865ba..f7ea9c2704d 100644 --- a/rs/moq-video/src/decode/backend/nvdec.rs +++ b/rs/moq-video/src/decode/backend/nvdec.rs @@ -186,12 +186,17 @@ impl Nvdec { // Taken out and put back so the callback's buffer keeps its capacity. let mut displayed = std::mem::take(&mut self.state.ready); + // A failed map discards this parse's pictures so an error never precedes them. + let queued = self.ready.len(); let mapped = displayed.iter().try_for_each(|disp| { self.ready.push_back(self.state.map_frame(disp)?); Ok(()) }); displayed.clear(); self.state.ready = displayed; + if mapped.is_err() { + self.ready.truncate(queued); + } mapped } } diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index 4e08acbd4d0..7852616faa8 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -419,8 +419,11 @@ impl Reader { // nothing in between to wait on. A caller that treats a codec // error as one lost picture and carries on then spins. self.drained = true; + // Queue what the flush readied before reporting its error, so a + // later read still hands those pictures over. + let received = self.receive(); tail?; - self.receive()?; + received?; continue; };