From 0f27fb2eeb6103284c4b135466341a1e62e18db1 Mon Sep 17 00:00:00 2001 From: Frando Date: Wed, 2 Sep 2026 00:47:04 +0200 Subject: [PATCH 1/3] feat(moq-video): add the VAAPI H.264 decoder The decode half of the VAAPI backend, next to the encoder that has been behind the same feature since it landed. On Linux the hardware decode tier was NVIDIA-only; this gives Intel and AMD boxes a hardware H.264 path instead of falling through to openh264. moq-vaapi's new `decode` module does the work: one Annex-B access unit in, tightly-packed NV12 out, with the parameter sets read from the stream. This adapts it the way the encode backend does, deinterleaving NV12 to the CPU I420 the rest of the crate speaks, and registers it after NVDEC in the hardware candidate list so a machine with both keeps preferring NVDEC. Output is CPU I420 rather than a zero-copy DMA-BUF surface on purpose. VA-API decode targets on Intel come back Y-tiled (DRM modifier 0x100000000000002), which the Vulkan renderer cannot import, so exporting one here would only move the download somewhere less convenient. That changes once the re-tile lands. `I420::from_nv12` and `deinterleave_uv` were already written for the Media Foundation and PipeWire capture paths; their cfg now includes the vaapi feature rather than growing a second copy. Tested on Intel Meteor Lake (iHD 26.1.5): the added round trip encodes a gradient with openh264 and decodes it through the hardware, asserting the returned planes match the input and that each picture carries its own timestamp. It skips itself on a host with no usable VA stack, like the NVDEC tests do. --- Cargo.lock | 3 +- Cargo.toml | 4 +- rs/moq-video/Cargo.toml | 21 +-- rs/moq-video/src/decode/backend/mod.rs | 13 +- rs/moq-video/src/decode/backend/vaapi.rs | 192 +++++++++++++++++++++++ rs/moq-video/src/frame.rs | 12 +- 6 files changed, 227 insertions(+), 18 deletions(-) create mode 100644 rs/moq-video/src/decode/backend/vaapi.rs diff --git a/Cargo.lock b/Cargo.lock index 3c16d018276..36fce42c9ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5133,8 +5133,7 @@ dependencies = [ [[package]] name = "moq-vaapi" version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4209e5f9401f2ed76eb5a82c807966be235c483a6932777623cec30950ba7d4" +source = "git+https://github.com/Frando/vaapi?branch=pr%2Fdecode#b2550c8f63a7a982e607dc482a5131457f9dd5d2" dependencies = [ "anyhow", "bindgen 0.70.1", diff --git a/Cargo.toml b/Cargo.toml index 0e01abba28f..b3b1c0c7cbc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -131,7 +131,9 @@ moq-token-cli = { version = "0.5.46", path = "rs/moq-token-cli" } moq-transcode = { version = "0.0.15", path = "rs/moq-transcode", default-features = false } # Standalone crate (moq-dev/vaapi); vendored from cros-libva + cros-codecs. # dlopen's libva at runtime (no libva-dev at build, no NEEDED libva in the binary). -moq-vaapi = "0.0.3" +# The decoder this crate's VAAPI backend calls is not in 0.0.3 yet; see +# moq-dev/vaapi#2. Point this back at a release once one carries `decode`. +moq-vaapi = { git = "https://github.com/Frando/vaapi", branch = "pr/decode" } # `default-features = false` is here for `capture` (V4L2, libclang), which the # cross-compiled Python, Swift, Kotlin, and Go builds have no use for: adding # `features = ["capture"]` to a consumer that ships in those bindings pulls the diff --git a/rs/moq-video/Cargo.toml b/rs/moq-video/Cargo.toml index 48c8ef40619..eecbfc2c866 100644 --- a/rs/moq-video/Cargo.toml +++ b/rs/moq-video/Cargo.toml @@ -35,11 +35,12 @@ nvidia = ["dep:cudarc", "dep:moq-nvenc", "dep:libloading"] # feature tables name only `nvidia`. nvenc = ["nvidia"] nvdec = ["nvidia"] -# Intel/AMD VAAPI hardware encoder (Linux). Off by default because the backend has -# never been validated on real hardware, not because of anything it costs the -# build: moq-vaapi dlopen's libva, so an enabled-vaapi build needs no libva at -# build time and still starts on a libva-less host, where automatic backend -# selection falls back to the next encoder like the NVENC backend does. +# Intel/AMD VAAPI hardware codecs (Linux): the H.264 encoder and decoder. Off by +# default because the encoder has never been validated on real hardware (the +# decoder has), not because of anything it costs the build: moq-vaapi dlopen's +# libva, so an enabled-vaapi build needs no libva at build time and still starts +# on a libva-less host, where automatic backend selection falls back to the next +# candidate like the NVENC and NVDEC backends do. vaapi = ["dmabuf", "dep:moq-vaapi"] # Linux DMA-BUF surface vocabulary and CPU fallback support. Enabled by every # backend that can produce or consume one. It pulls no graphics API by itself. @@ -125,11 +126,11 @@ libloading = { workspace = true, optional = true } # Architecture-correct DMA_BUF_IOCTL_SYNC request values for CPU access. linux-raw-sys = { workspace = true, optional = true } moq-nvenc = { workspace = true, optional = true } -# Intel/AMD VAAPI hardware encoder, behind the opt-in `vaapi` feature. As of -# moq-vaapi 0.0.3 libva is dlopen'd at runtime, so the binary carries no NEEDED -# libva: the build needs no libva-dev, and a libva-less host still loads. Automatic -# backend selection then falls back to the next encoder instead of failing at load, -# matching the NVENC backend. +# Intel/AMD VAAPI hardware H.264 encoder and decoder, behind the opt-in `vaapi` +# feature. As of moq-vaapi 0.0.3 libva is dlopen'd at runtime, so the binary carries +# no NEEDED libva: the build needs no libva-dev, and a libva-less host still loads. +# Automatic backend selection then falls back to the next candidate instead of +# failing at load, matching the NVENC and NVDEC backends. moq-vaapi = { workspace = true, optional = true } # PipeWire video stream for the portal's node, behind the `pipewire` feature. # Links libpipewire-0.3 via pkg-config at build time; bindgen needs libclang. diff --git a/rs/moq-video/src/decode/backend/mod.rs b/rs/moq-video/src/decode/backend/mod.rs index 8f0fbb01e8f..0dabbcaff67 100644 --- a/rs/moq-video/src/decode/backend/mod.rs +++ b/rs/moq-video/src/decode/backend/mod.rs @@ -8,8 +8,8 @@ //! //! [`open`] picks the best backend for a [`Codec`] and [`Config`], trying //! hardware candidates (platform-gated: VideoToolbox on macOS, Media Foundation -//! / DXVA on Windows, NVDEC on Linux) before the openh264 software fallback, -//! exactly like the encode side. Only backends that support the requested codec +//! / DXVA on Windows, NVDEC then VAAPI on Linux) before the openh264 software +//! fallback, exactly like the encode side. Only backends that support the requested codec //! are considered: there is no software H.265 or AV1 decoder, so those tracks //! have no fallback below the hardware path. @@ -33,6 +33,9 @@ mod mediafoundation; #[cfg(all(target_os = "linux", feature = "nvidia"))] mod nvdec; +#[cfg(all(target_os = "linux", feature = "vaapi"))] +mod vaapi; + /// The video codec a decoder handles. Derived from the catalog, not chosen by the /// caller. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -99,6 +102,12 @@ const HARDWARE: &[Candidate] = &[ supports: |c| matches!(c, Codec::H264 | Codec::H265 | Codec::Av1), open: nvdec::Nvdec::open, }, + #[cfg(all(target_os = "linux", feature = "vaapi"))] + Candidate { + name: vaapi::NAME, + supports: |c| matches!(c, Codec::H264), + open: vaapi::Vaapi::open, + }, ]; const SOFTWARE: Candidate = Candidate { diff --git a/rs/moq-video/src/decode/backend/vaapi.rs b/rs/moq-video/src/decode/backend/vaapi.rs new file mode 100644 index 00000000000..7362aa0955d --- /dev/null +++ b/rs/moq-video/src/decode/backend/vaapi.rs @@ -0,0 +1,192 @@ +//! Intel/AMD VAAPI hardware H.264 decode via the opt-in `moq-vaapi` crate on Linux. +//! +//! The decode half of [`encode::backend::vaapi`](crate::encode). `moq-vaapi` is a +//! focused VA-API H.264 codec vendored and trimmed from cros-libva + +//! discord/cros-codecs. Its decoder takes one Annex-B access unit with the +//! parameter sets inline and hands back tightly-packed NV12, which this +//! deinterleaves to the CPU I420 the rest of the crate speaks. +//! +//! libva is `dlopen`'d at runtime, so a VAAPI-enabled build needs no libva at +//! build time and the binary carries no `NEEDED libva`. A libva-less host, a +//! missing render node, or a driver with no H.264 decode entrypoint makes +//! `Decoder::new` return an error; under automatic selection +//! [`backend::open`](super::open) then moves on to the next candidate, like the +//! NVDEC backend. +//! +//! Progressive 8-bit 4:2:0 only, which is everything a browser's `VideoEncoder`, +//! WebRTC, or this crate's own encoders emit. The decoder rejects an interlaced or +//! high-bit-depth sequence at its first SPS, which surfaces here as a decode +//! error rather than wrong pixels. +//! +//! Unlike NVDEC, which cuvid lets us pin to zero display delay, output trails the +//! input by one picture even without B-frames: H.264's DPB releases a picture only +//! once a later one needs its slot. The decoded frames still carry their own +//! timestamps, so nothing downstream has to know. +//! +//! The output is CPU I420 rather than a zero-copy [`Surface::DmaBuf`]: VA-API +//! decode targets on Intel come back Y-tiled (DRM modifier `0x100000000000002`), +//! a modifier the Vulkan renderer cannot import, so a DMA-BUF here would only move +//! the download somewhere less convenient. + +use bytes::Bytes; +use moq_net::Timestamp; +use moq_vaapi::decode::{Config as VaapiConfig, Decoder}; + +use super::{Backend, Codec, Config}; +use crate::frame::{I420, Surface}; +use crate::{Error, Frame}; + +pub(crate) const NAME: &str = "vaapi"; + +pub(crate) struct Vaapi { + decoder: Decoder, +} + +// The decoder is `!Send` (libva uses `Rc` internally) but is created, used, and +// dropped only on the dedicated decode thread (see `decode::sink`); the `Send` +// impl just lets the boxed trait object satisfy `Backend: Send`. +unsafe impl Send for Vaapi {} + +impl Vaapi { + /// VA-API H.265 and AV1 decode exist but are not wired up in `moq-vaapi`, so + /// this handles H.264 only. `config` carries no hardware scaler request we can + /// honor: VA-API's scaler is a separate VPP pipeline, so callers scale the + /// frames themselves. + pub(crate) fn open(codec: Codec, _config: &Config) -> Result, Error> { + if codec != Codec::H264 { + return Err(Error::Codec(anyhow::anyhow!("VAAPI cannot decode {}", codec.label()))); + } + + let decoder = + Decoder::new(VaapiConfig::new()).map_err(|e| Error::Codec(anyhow::anyhow!("VAAPI decoder init: {e:?}")))?; + + tracing::info!(decoder = NAME, "opened H.264 decoder"); + Ok(Box::new(Self { decoder })) + } +} + +impl Backend for Vaapi { + fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result, Error> { + // The timestamp rides through the decoder with the picture, so it + // survives the DPB reordering a stream with B-frames goes through. + let decoded = self + .decoder + .decode(&access_unit, timestamp.as_micros() as u64) + .map_err(|e| Error::Codec(anyhow::anyhow!("VAAPI decode: {e:?}")))?; + + decoded + .into_iter() + .map(|frame| { + let i420 = I420::from_nv12(&frame.data, frame.width, frame.height)?; + let timestamp = Timestamp::from_micros(frame.timestamp).unwrap_or(Timestamp::ZERO); + Ok(Frame::new(Surface::I420(i420), timestamp)) + }) + .collect() + } + + fn name(&self) -> &str { + NAME + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::decode::{Config as DecodeConfig, Kind as DecodeKind}; + use crate::encode::{Config as EncodeConfig, Encoder, Kind as EncodeKind}; + + /// Real hardware only: skip on a box with no libva or no VA-API H.264 decode + /// entrypoint, so these are no-ops on CI and validate on an Intel or AMD box. + fn hw_available() -> bool { + Decoder::new(VaapiConfig::new()).is_ok() + } + + fn decode_config() -> DecodeConfig { + DecodeConfig { + kind: DecodeKind::Named(NAME.into()), + ..DecodeConfig::new() + } + } + + /// On a host with no usable VA stack, opening must return an `Err` (so + /// `Kind::Auto` falls through to openh264) rather than panicking inside libva. + /// A no-op on a box that does have one. + #[test] + fn missing_driver_errors_instead_of_panicking() { + if hw_available() { + return; + } + assert!(Vaapi::open(Codec::H264, &decode_config()).is_err()); + } + + /// A static RGBA gradient that varies in both axes, so the chroma planes have + /// spatial structure and a pitch or plane-split bug corrupts the picture. + fn gradient_rgba(width: u32, height: u32) -> Vec { + let (w, h) = (width as usize, height as usize); + let mut buf = vec![0u8; w * h * 4]; + for y in 0..h { + for x in 0..w { + let i = (y * w + x) * 4; + buf[i] = (x * 255 / w) as u8; + buf[i + 1] = (y * 255 / h) as u8; + buf[i + 2] = ((x + y) * 255 / (w + h)) as u8; + buf[i + 3] = 255; + } + } + buf + } + + /// Mean absolute error between two equal-length planes. + fn mae(a: &[u8], b: &[u8]) -> u64 { + assert_eq!(a.len(), b.len()); + a.iter().zip(b).map(|(x, y)| x.abs_diff(*y) as u64).sum::() / a.len() as u64 + } + + /// H.264 through the real hardware: openh264 encodes a gradient (Annex-B with + /// inline SPS/PPS) and VA-API decodes it. Asserts the downloaded picture + /// matches the input, which a plane split or stride bug would shear, and that + /// the timestamp rides through with its picture. + #[test] + fn vaapi_h264_round_trip() { + if !hw_available() { + return; + } + let (w, h) = (320u32, 240u32); + let rgba = gradient_rgba(w, h); + let expected = I420::from_rgba(&rgba, w * 4, w, h).unwrap(); + + let mut encoder = Encoder::new(&EncodeConfig { + kind: EncodeKind::Software, + ..EncodeConfig::new(w, h, 30) + }) + .unwrap(); + let mut decoder = Vaapi::open(Codec::H264, &decode_config()).expect("VAAPI H.264 decoder"); + + let mut decoded = Vec::new(); + for i in 0..10u64 { + if i == 0 { + encoder.keyframe(); + } + let surface = Surface::rgba(&rgba, crate::Size::new(w, h)).unwrap(); + let frame = Frame::new(surface, Timestamp::from_micros(i * 33_333).unwrap()); + for encoded in encoder.encode(&frame).unwrap() { + decoded.extend(decoder.decode(encoded.payload, encoded.timestamp, i == 0).unwrap()); + } + } + + assert!(!decoded.is_empty(), "VAAPI produced no frames"); + for (i, frame) in decoded.iter().enumerate() { + // openh264 encodes IPPP, so the pictures come back in feed order. + assert_eq!( + frame.timestamp.as_micros(), + i as u128 * 33_333, + "timestamp did not ride the picture" + ); + let i420 = frame.surface.to_i420().unwrap(); + assert_eq!((i420.width(), i420.height()), (w, h)); + assert!(mae(i420.y(), expected.y()) < 8, "Y plane corrupt"); + assert!(mae(i420.u(), expected.u()) < 8, "U plane corrupt"); + assert!(mae(i420.v(), expected.v()) < 8, "V plane corrupt"); + } + } +} diff --git a/rs/moq-video/src/frame.rs b/rs/moq-video/src/frame.rs index f3df4581648..b03491fe8d7 100644 --- a/rs/moq-video/src/frame.rs +++ b/rs/moq-video/src/frame.rs @@ -776,8 +776,11 @@ impl I420 { /// Split tightly-packed NV12 (Y plane `width * height`, then interleaved UV /// `width/2 * height/2` pairs) into planar I420. A chroma deinterleave, no /// color-space conversion. Used by the Windows Media Foundation and Linux - /// PipeWire capture paths. - #[cfg(any(target_os = "windows", all(target_os = "linux", feature = "pipewire")))] + /// PipeWire capture paths, and by the VAAPI decoder. + #[cfg(any( + target_os = "windows", + all(target_os = "linux", any(feature = "pipewire", feature = "vaapi")) + ))] pub(crate) fn from_nv12(nv12: &[u8], width: u32, height: u32) -> Result { let (w, h) = (width as usize, height as usize); let luma = w * h; @@ -917,7 +920,10 @@ pub(crate) fn interleave_uv(u: &[u8], v: &[u8], uv: &mut [u8]) { /// Split a packed NV12 chroma plane into separate U and V planes, the inverse of /// [`interleave_uv`]. -#[cfg(any(target_os = "windows", all(target_os = "linux", feature = "pipewire")))] +#[cfg(any( + target_os = "windows", + all(target_os = "linux", any(feature = "pipewire", feature = "vaapi")) +))] pub(crate) fn deinterleave_uv(uv: &[u8], u: &mut [u8], v: &mut [u8]) { for (pair, (u, v)) in uv.chunks_exact(2).zip(u.iter_mut().zip(v)) { *u = pair[0]; From 44aafe68c0d4f3c87147a105939e3186ac523fb6 Mon Sep 17 00:00:00 2001 From: Frando Date: Wed, 2 Sep 2026 13:23:21 +0200 Subject: [PATCH 2/3] fix(moq-video): drain the decoder when the track ends `Consumer::read` returned `None` the moment the track did, which for a backend that buffers means the last pictures of every stream are dropped on the floor. VAAPI is that backend: H.264 releases a picture from the DPB only once a later one needs its slot, and the slot count follows the sequence's reference and reorder limits rather than the reorder depth the stream actually used, so the tail is several pictures rather than one. `Backend::flush` is the drain, with a default returning no frames so the backends that decode one-in one-out need nothing. That is all of them but VAAPI: openh264 is one-in one-out, Media Foundation runs with its output-reorder buffer disabled, NVDEC pins `ulMaxDisplayDelay` to zero, and VideoToolbox decodes without temporal processing. VAAPI overrides it onto `moq_vaapi::decode::Decoder::flush`, which runs the C.4.5 bumping process to completion. It is exposed up through `decode::Decoder` and `decode::Sink`, where it lands in order on the codec thread behind the access units it follows, and `Consumer::read` calls it once when the track ends and hands out what comes back before reporting the end. Measured on Intel Meteor Lake (iHD 26.1.5). Feeding five openh264-encoded pictures through the hardware returned four; the sixth picture of an x264 stream at its defaults costs two, and at ref=3 with no B-frames it costs three. Both new tests were watched failing against the old code with exactly the last picture missing, and both assert that `decode` alone came up short before asserting that the flush makes up the difference, so neither can quietly go vacuous. Not addressed here: moq-transcode decodes group by group without a flush at the boundaries, so a VAAPI decoder there carries the tail of one group into the next, past the keyframe that opens it. The encode side already flushes at every boundary for that reason. It needs the same treatment and its own test. --- rs/moq-video/src/decode/backend/mod.rs | 22 +++++- rs/moq-video/src/decode/backend/vaapi.rs | 99 +++++++++++++++++++++--- rs/moq-video/src/decode/consumer.rs | 87 ++++++++++++++++++++- rs/moq-video/src/decode/decoder.rs | 18 +++++ rs/moq-video/src/decode/sink.rs | 33 +++++++- 5 files changed, 243 insertions(+), 16 deletions(-) diff --git a/rs/moq-video/src/decode/backend/mod.rs b/rs/moq-video/src/decode/backend/mod.rs index 0dabbcaff67..7b5ba9f76b1 100644 --- a/rs/moq-video/src/decode/backend/mod.rs +++ b/rs/moq-video/src/decode/backend/mod.rs @@ -33,8 +33,10 @@ mod mediafoundation; #[cfg(all(target_os = "linux", feature = "nvidia"))] mod nvdec; +// Crate-visible so `decode::Consumer`'s end-of-track test can name the one +// backend that holds pictures back; nothing outside the tests reaches for it. #[cfg(all(target_os = "linux", feature = "vaapi"))] -mod vaapi; +pub(crate) mod vaapi; /// The video codec a decoder handles. Derived from the catalog, not chosen by the /// caller. @@ -67,6 +69,24 @@ pub(crate) trait Backend: Send { /// reordering. fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, keyframe: bool) -> Result, Error>; + /// Return the pictures the codec is still holding, in output order, once the + /// stream has ended. + /// + /// The last access unit of a stream is not the last picture out of a decoder + /// that buffers: H.264 releases a picture from the DPB only when a later one + /// needs its slot, so the tail sits there until something asks for it. That is + /// several pictures, not one, since the DPB bumps against the sequence's + /// reference and reorder limits rather than the reorder depth actually used. + /// + /// Defaults to no frames, which is right for every backend configured for zero + /// delay: openh264, Media Foundation with its reorder buffer disabled, NVDEC + /// with `ulMaxDisplayDelay` at zero, and VideoToolbox decoding without temporal + /// processing all hand each picture back within the call that fed it. Override + /// it in a backend that holds pictures across calls, or its stream ends short. + fn flush(&mut self) -> Result, Error> { + Ok(Vec::new()) + } + /// The decoder name in use, e.g. `"videotoolbox"` (for logging). fn name(&self) -> &str; } diff --git a/rs/moq-video/src/decode/backend/vaapi.rs b/rs/moq-video/src/decode/backend/vaapi.rs index 7362aa0955d..85e9011cc29 100644 --- a/rs/moq-video/src/decode/backend/vaapi.rs +++ b/rs/moq-video/src/decode/backend/vaapi.rs @@ -19,9 +19,13 @@ //! error rather than wrong pixels. //! //! Unlike NVDEC, which cuvid lets us pin to zero display delay, output trails the -//! input by one picture even without B-frames: H.264's DPB releases a picture only -//! once a later one needs its slot. The decoded frames still carry their own -//! timestamps, so nothing downstream has to know. +//! input even without B-frames: H.264's DPB releases a picture only once a later +//! one needs its slot, and the slot count comes from the sequence's reference and +//! reorder limits rather than the reorder depth actually used. A stream coded with +//! three reference frames therefore keeps three pictures in hand, which is why +//! this backend implements [`Backend::flush`] and the layers above call it when +//! the track ends. The decoded frames carry their own timestamps, so the delay +//! itself needs nothing downstream. //! //! The output is CPU I420 rather than a zero-copy [`Surface::DmaBuf`]: VA-API //! decode targets on Intel come back Y-tiled (DRM modifier `0x100000000000002`), @@ -74,14 +78,19 @@ impl Backend for Vaapi { .decode(&access_unit, timestamp.as_micros() as u64) .map_err(|e| Error::Codec(anyhow::anyhow!("VAAPI decode: {e:?}")))?; - decoded - .into_iter() - .map(|frame| { - let i420 = I420::from_nv12(&frame.data, frame.width, frame.height)?; - let timestamp = Timestamp::from_micros(frame.timestamp).unwrap_or(Timestamp::ZERO); - Ok(Frame::new(Surface::I420(i420), timestamp)) - }) - .collect() + convert(decoded) + } + + /// The tail of the stream, which is why this backend overrides the trait's + /// no-op: the DPB is holding as many pictures as the sequence's reference and + /// reorder limits allow, and nothing else will ever ask for them. + fn flush(&mut self) -> Result, Error> { + let decoded = self + .decoder + .flush() + .map_err(|e| Error::Codec(anyhow::anyhow!("VAAPI flush: {e:?}")))?; + + convert(decoded) } fn name(&self) -> &str { @@ -89,6 +98,19 @@ impl Backend for Vaapi { } } +/// Deinterleave each decoded picture's NV12 into the CPU I420 the rest of the +/// crate speaks, keeping the timestamp the picture was coded with. +fn convert(decoded: Vec) -> Result, Error> { + decoded + .into_iter() + .map(|frame| { + let i420 = I420::from_nv12(&frame.data, frame.width, frame.height)?; + let timestamp = Timestamp::from_micros(frame.timestamp).unwrap_or(Timestamp::ZERO); + Ok(Frame::new(Surface::I420(i420), timestamp)) + }) + .collect() +} + #[cfg(test)] mod tests { use super::*; @@ -189,4 +211,59 @@ mod tests { assert!(mae(i420.v(), expected.v()) < 8, "V plane corrupt"); } } + + /// Regression: every picture fed in comes back out, which needs the flush. + /// + /// H.264 releases a picture from the DPB only once a later one needs its slot, + /// so a stream simply stopping leaves its tail there. How many depends on the + /// sequence's reference and reorder limits, not on the reorder depth the + /// stream used: openh264 codes one reference frame and loses the last picture, + /// x264's default three loses three. The `decode` half of the assertion is + /// what keeps this honest, since a decoder that held nothing back would pass + /// the rest of it without a flush ever running. + #[test] + fn flushing_returns_the_tail_the_dpb_holds() { + if !hw_available() { + return; + } + const FRAMES: u64 = 5; + let (w, h) = (320u32, 240u32); + let rgba = gradient_rgba(w, h); + + let mut encoder = Encoder::new(&EncodeConfig { + kind: EncodeKind::Software, + ..EncodeConfig::new(w, h, 30) + }) + .unwrap(); + let mut decoder = Vaapi::open(Codec::H264, &decode_config()).expect("VAAPI H.264 decoder"); + + let mut streamed = Vec::new(); + for i in 0..FRAMES { + if i == 0 { + encoder.keyframe(); + } + let surface = Surface::rgba(&rgba, crate::Size::new(w, h)).unwrap(); + let frame = Frame::new(surface, Timestamp::from_micros(i * 33_333).unwrap()); + for encoded in encoder.encode(&frame).unwrap() { + streamed.extend(decoder.decode(encoded.payload, encoded.timestamp, i == 0).unwrap()); + } + } + assert!( + (streamed.len() as u64) < FRAMES, + "the DPB held nothing back, so this test proves nothing" + ); + + let flushed = decoder.flush().unwrap(); + let timestamps: Vec = streamed + .iter() + .chain(&flushed) + .map(|frame| frame.timestamp.as_micros()) + .collect(); + let expected: Vec = (0..FRAMES as u128).map(|i| i * 33_333).collect(); + assert_eq!(timestamps, expected, "the stream lost pictures at its end"); + + // A second flush has nothing left to hand back, so the drain is idempotent + // and a caller that flushes twice does not see a picture twice. + assert!(decoder.flush().unwrap().is_empty()); + } } diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index bd753754752..5f7d4e1caed 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -24,6 +24,10 @@ pub struct Consumer { /// One AU yields one frame in the low-delay path, but a backend may hand back /// more, so we buffer to keep `read` one-frame-per-call. pending: VecDeque, + /// Set once the track has ended and the decoder has been flushed, so the + /// drain runs exactly once and later reads report the end instead of asking + /// a track that is already done. + drained: bool, } impl Consumer { @@ -55,6 +59,7 @@ impl Consumer { decoder, track, pending: VecDeque::new(), + drained: false, }) } @@ -64,14 +69,26 @@ impl Consumer { } /// Read the next decoded I420 frame, or `None` when the track ends. + /// + /// The end of the track is not the end of the frames: a backend that buffers + /// is still holding the tail of the stream when the last access unit arrives, + /// so the track ending flushes the decoder and returns what comes out before + /// reporting the end. pub async fn read(&mut self) -> Result, Error> { loop { if let Some(frame) = self.pending.pop_front() { return Ok(Some(frame)); } + if self.drained { + return Ok(None); + } let Some(mux_frame) = self.track.read().await? else { - return Ok(None); + // The flush runs once, whether or not it yields anything, and the + // loop comes back around to hand out whatever it did. + self.drained = true; + self.pending.extend(self.decoder.flush().await?); + continue; }; self.pending.extend( @@ -150,4 +167,72 @@ mod tests { let frame = consumer.read().await.unwrap().expect("decoded frame"); assert_eq!(frame.size(), crate::Size::new(320, 240)); } + + /// Regression: the track ending is not the stream ending. A VAAPI decoder is + /// still holding the tail of the stream in its DPB when the last access unit + /// arrives, so `read` has to drain it before reporting the end or every track + /// stops short of the pictures it published. + /// + /// Real hardware only, because it is the only backend that holds anything + /// back: the rest decode one-in one-out and would pass this without a drain + /// ever running. + #[cfg(all(target_os = "linux", feature = "vaapi"))] + #[tokio::test] + async fn the_track_ending_drains_the_decoder() { + use crate::decode::backend::vaapi; + + const FRAMES: u64 = 5; + let config = EncodeConfig { + kind: EncodeKind::Software, + ..EncodeConfig::new(320, 240, 30) + }; + let catalog = config.probe().await.expect("probe the software encoder"); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast.create_track("video", hang::container::track_info()).unwrap(); + let subscriber = broadcast.consume(); + let mut producer = moq_mux::container::Producer::new(track, moq_mux::catalog::hang::Container::Legacy); + + let mut encoder = Encoder::new(&config).unwrap(); + let rgba = vec![0x80u8; 320 * 240 * 4]; + for index in 0..FRAMES { + if index == 0 { + encoder.keyframe(); + } + let surface = crate::Surface::rgba(&rgba, crate::Size::new(320, 240)).unwrap(); + let frame = crate::Frame::new(surface, moq_net::Timestamp::from_micros(index * 33_333).unwrap()); + for encoded in encoder.encode(&frame).unwrap() { + producer + .write(moq_mux::container::Frame { + timestamp: encoded.timestamp, + duration: None, + payload: encoded.payload, + keyframe: index == 0, + }) + .unwrap(); + } + } + producer.finish().unwrap(); + + let decode = Config { + kind: Kind::Named(vaapi::NAME.into()), + ..Config::new() + }; + // The hardware gate: no libva, no render node, or no H.264 decode + // entrypoint and the named backend refuses to open. + let Ok(mut consumer) = Consumer::new(&subscriber, &catalog, "video", decode).await else { + return; + }; + + let mut timestamps = Vec::new(); + while let Some(frame) = consumer.read().await.unwrap() { + timestamps.push(frame.timestamp.as_micros()); + } + let expected: Vec = (0..FRAMES as u128).map(|index| index * 33_333).collect(); + assert_eq!(timestamps, expected, "the track ended before the stream did"); + + // The end stays the end: the drain runs once, so a caller that keeps + // reading past it does not get the tail a second time. + assert!(consumer.read().await.unwrap().is_none()); + } } diff --git a/rs/moq-video/src/decode/decoder.rs b/rs/moq-video/src/decode/decoder.rs index 635741fd07d..15d4fbe6062 100644 --- a/rs/moq-video/src/decode/decoder.rs +++ b/rs/moq-video/src/decode/decoder.rs @@ -172,6 +172,24 @@ impl Decoder { self.backend.decode(access_unit, timestamp, keyframe) } + + /// Return the frames the backend is still holding, in output order, once the + /// stream has ended. + /// + /// Call this after the last access unit and before dropping the decoder, or a + /// backend that buffers ends its stream short: a VAAPI H.264 decoder holds the + /// tail of every stream in its DPB, as many pictures as the sequence's + /// reference and reorder limits allow. Backends that decode one-in one-out + /// return nothing here, so calling it is always safe and never necessary to + /// think about per backend. + /// + /// The decoder stays usable but goes back to waiting for a keyframe, since a + /// flushed codec has no reference pictures left to decode a delta frame + /// against. + pub fn flush(&mut self) -> Result, Error> { + self.got_keyframe = false; + self.backend.flush() + } } fn is_supported_av1(av1: &AV1) -> bool { diff --git a/rs/moq-video/src/decode/sink.rs b/rs/moq-video/src/decode/sink.rs index 84cd30f6717..be50ba729cd 100644 --- a/rs/moq-video/src/decode/sink.rs +++ b/rs/moq-video/src/decode/sink.rs @@ -73,6 +73,16 @@ impl Sink { pub async fn decode(&mut self, payload: Bytes, timestamp: Timestamp, keyframe: bool) -> Result, Error> { self.0.decode(payload, timestamp, keyframe).await } + + /// Drain the decoder once the stream has ended, returning the frames it is + /// still holding. See [`Decoder::flush`](super::Decoder::flush). + /// + /// A track that ends without this ends short wherever the backend buffers: + /// VAAPI hands back a picture only when a later one needs its DPB slot, so the + /// last few pictures of every stream never come out on their own. + pub async fn flush(&mut self) -> Result, Error> { + self.0.flush().await + } } #[cfg(not(target_os = "macos"))] @@ -86,9 +96,9 @@ mod threaded { use crate::worker::{Ready, Worker}; use crate::{Error, Frame}; - /// Work for the decode thread. One variant today; the enum is here so a - /// future request (a flush, a reset) lands in order with the frames around it - /// rather than racing them. + /// Work for the decode thread. Each variant lands in order with the frames + /// around it rather than racing them, which is what a flush needs: it has to + /// run after the last access unit, not alongside it. enum Request { Decode { payload: Bytes, @@ -96,6 +106,11 @@ mod threaded { keyframe: bool, resp: oneshot::Sender, Error>>, }, + /// Drain the decoder at the end of the stream, returning the frames it + /// still holds. + Flush { + resp: oneshot::Sender, Error>>, + }, } /// Build a decoder and serve requests until the channel closes. Runs entirely @@ -122,6 +137,9 @@ mod threaded { } => { let _ = resp.send(decoder.decode(&payload, timestamp, keyframe)); } + Request::Flush { resp } => { + let _ = resp.send(decoder.flush()); + } } } // `decoder` drops here, on this thread, balancing the COM apartment. @@ -160,6 +178,10 @@ mod threaded { }) .await } + + pub async fn flush(&mut self) -> Result, Error> { + self.0.request(|resp| Request::Flush { resp }).await + } } } @@ -194,6 +216,11 @@ mod inline { ) -> Result, Error> { self.0.decode(&payload, timestamp, keyframe) } + + /// Async only to match the threaded `Inner`, as with `decode`. + pub async fn flush(&mut self) -> Result, Error> { + self.0.flush() + } } } From 8517062f7813bbea4bdacccb411993a8e9202213 Mon Sep 17 00:00:00 2001 From: Frando Date: Wed, 2 Sep 2026 13:57:34 +0200 Subject: [PATCH 3/3] fix(moq-video): stop DmaBufExport being dead code without render `vaapi` enables `dmabuf` without `render`, so in that build nothing reads the producer lease or splits the export apart, and clippy fails the crate on two dead-code errors. The lease has to stay a field either way: dropping it would release the producer's buffer while the descriptor is still outstanding. So it keeps an allow with the reason written down, and the method is gated on the feature that calls it. --- rs/moq-video/src/frame.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/rs/moq-video/src/frame.rs b/rs/moq-video/src/frame.rs index b03491fe8d7..557a4126994 100644 --- a/rs/moq-video/src/frame.rs +++ b/rs/moq-video/src/frame.rs @@ -148,6 +148,11 @@ impl DmaBufPlane { #[cfg(all(target_os = "linux", feature = "dmabuf"))] pub struct DmaBufExport { fd: OwnedFd, + /// The producer lease the descriptor borrows from, held so the buffer + /// outlives the export. Only [`into_parts`](Self::into_parts) reads it, and + /// that is the render module's way in, so in a build without `render` the + /// field does its entire job by existing until `Drop`. + #[cfg_attr(not(feature = "render"), allow(dead_code))] inner: Arc, } @@ -211,6 +216,12 @@ impl DmaBufExport { std::os::fd::AsFd::as_fd(&self.fd) } + /// Splits the export into its descriptor and the lease that keeps it valid. + /// + /// Only the renderer needs the two apart, so this is gated with it: `vaapi` + /// and `pipewire` both enable `dmabuf` without `render`, and an ungated + /// method would be dead code in those builds. + #[cfg(feature = "render")] pub(crate) fn into_parts(self) -> (OwnedFd, Arc) { (self.fd, self.inner) }