diff --git a/Cargo.lock b/Cargo.lock index ab053a33f47..259d2934758 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4859,9 +4859,9 @@ dependencies = [ [[package]] name = "moq-vaapi" -version = "0.0.4" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "091df44de531fc6e5346da91f2dd7d1ebbb4a2e741ddadaf49aef017e659cb57" +checksum = "a697143830ed57c5ce0e96016e845e0b097f607c536847c12890e3bd23ff4db9" dependencies = [ "anyhow", "bindgen 0.70.1", diff --git a/Cargo.toml b/Cargo.toml index 5a529c9cd67..9a4d0496c32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -179,7 +179,7 @@ moq-uring = { version = "0.0.2", path = "rs/moq-uring", default-features = false moq-v4l = { version = "0.0.1", path = "rs/moq-v4l" } # Standalone crate (moq-dev/vaapi); vendored from cros-libva + cros-codecs. # dlopen's libva at runtime (no libva-dev at build, no NEEDED libva in the binary). -moq-vaapi = "0.0.4" +moq-vaapi = "0.1.0" # `default-features = false` is here for `capture` (V4L2, libclang), which the # cross-compiled Python, Swift, Kotlin, and Go builds have no use for: adding # `features = ["capture"]` to a consumer that ships in those bindings pulls the diff --git a/quest/m4/video-vaapi.md b/quest/m4/video-vaapi.md index 350f5c73d66..e2bc947fce5 100644 --- a/quest/m4/video-vaapi.md +++ b/quest/m4/video-vaapi.md @@ -9,21 +9,23 @@ release first. ## Plan -Four gaps, one external dependency. +Three gaps, one external dependency. **Decode.** The H.264 decoder landed (moq-vaapi 0.0.4, `decode/backend/vaapi.rs`), with the default `decode::Config::output` of `Output::Native` handing out DMA-BUF surfaces the renderer imports without a download. H.265 decode is still missing, so a Linux box without NVDEC has no hardware path for it. -**The encoder.** Ours is a 111-line CPU-only adapter whose own header says it -is unvalidated on hardware. iroh-live's imports a DMA-BUF directly and does -scale and convert through VPP, validated on Intel Meteor Lake. Take theirs and -reshape it to our surface rather than growing ours toward it. +**The encoder.** H.264 is done. `moq-vaapi` imports DMA-BUFs and scales and +converts them through VPP (`dmabuf`, `vpp`, `Encoder::encode_dmabuf`, +`Encoder::set_bitrate`); `encode/backend/vaapi.rs` encodes a `Surface::DmaBuf` +without a download, and `Surface::resize` scales one through VPP. Validated on +Intel Meteor Lake. What is left is H.265, below. **H.265.** The VAAPI backend advertises H.264 only. `moq-vaapi` 0.0.2 vendors the HEVC buffer types (`src/buffer/hevc.rs`) but its `Encoder` is hardcoded to -`VAProfileH264Main` / `VAEntrypointEncSlice`, so exposing an HEVC encoder is a +`VAProfileH264Main` (with `VAEntrypointEncSlice`, or the low-power entrypoint +where that is all a device has), so exposing an HEVC encoder is a change to that crate, not a flag here. **Build cost.** `moq-vaapi` 0.0.3 dlopens libva (no `DT_NEEDED`), so a @@ -43,6 +45,6 @@ already falls back cleanly, since `Encoder::new` returns `Err` and ## Required -- A `moq-dev/vaapi` release exposing an HEVC encoder, the decode half and a - VPP wrapper, and shipping pre-generated bindings instead of a bindgen build - script +- A `moq-dev/vaapi` release exposing an HEVC encoder (H.264 decode is in + 0.0.4; DMA-BUF encode and VPP shipped in 0.1.0) and pre-generated bindings + instead of a bindgen build script diff --git a/rs/moq-video/Cargo.toml b/rs/moq-video/Cargo.toml index 3e5e6508d8e..c7fe32b53e6 100644 --- a/rs/moq-video/Cargo.toml +++ b/rs/moq-video/Cargo.toml @@ -51,8 +51,8 @@ openh264 = ["dep:openh264", "dep:openh264-sys2"] # default because moq-vaapi's build script runs bindgen over its vendored libva # headers, so the build host needs libclang. At runtime libva is dlopen'd: the # binary carries no NEEDED libva and still starts on a libva-less host, where -# automatic backend selection falls back to the next candidate. The decoder is -# hardware-validated, while the encoder is not yet. +# automatic backend selection falls back to the next candidate. Both are +# validated on Intel Meteor Lake. vaapi = ["dmabuf", "dep:moq-vaapi"] # The V4L2 stateful memory-to-memory hardware codecs on Linux: the H.264 encoder # and decoder most ARM SoCs expose (a Raspberry Pi's VideoCore, and the diff --git a/rs/moq-video/DESIGN-native-codecs.md b/rs/moq-video/DESIGN-native-codecs.md index 3ee8a755c7a..33217db86f8 100644 --- a/rs/moq-video/DESIGN-native-codecs.md +++ b/rs/moq-video/DESIGN-native-codecs.md @@ -3,9 +3,9 @@ Status: **phases 2-6 implemented** (openh264, VideoToolbox, NVENC, VAAPI, capture swap + ffmpeg removal). Capture is now native on all three platforms (AVFoundation / ScreenCaptureKit on macOS, V4L2 on Linux, Media Foundation on Windows), so -**nokhwa is fully removed**. VAAPI is back via discord/cros-codecs with an NV12 -surface-upload input path; the zero-copy dmabuf capture is a follow-up. See "As -built" at the bottom for where the implementation diverged from this plan. +**nokhwa is fully removed**. VAAPI is back via moq-vaapi, which encodes DMA-BUFs +on the GPU and uploads CPU frames as NV12. See "As built" at the bottom for where +the implementation diverged from this plan. ## Goal @@ -412,25 +412,24 @@ software (see `backend::open`). > libva and vendors the libva headers its bindgen reads, so VAAPI costs the build > nothing: no libva-dev, no `NEEDED libva.so.2`, no entry in the nix devShell. -**Input is an NV12 surface upload, not zero-copy dmabuf.** The encoder wants an -NV12 VA surface, but UVC webcams deliver YUYV/MJPEG (decoded to CPU I420); they -rarely expose NV12 to import zero-copy. So `backend/vaapi.rs` drives -`new_native_vaapi` with a `VaSurfacePool`: each frame uploads I420 into a pooled -surface as NV12 (`libva::Image`, honoring plane pitches) and encodes the surface. -This works with the existing CPU V4L2 capture, no new capture code. - -Follow-up (not in this PR): the **zero-copy dmabuf path** for the rare NV12-capable -V4L2 source. Re-add `Frame::DmaBuf`, a V4L2 `VIDIOC_EXPBUF` capture (the `v4l` -crate exposes the raw ioctl but no dmabuf stream), and a `requires_dmabuf` capture -coupling, then import the dmabuf into a VA surface (`MemoryType::DrmPrime2`). - -**NOT YET VALIDATED ON HARDWARE.** Compiles on Linux with libva headers; written -against discord/cros-codecs `discord-0.0.5` with type/field names checked against -source. Needs a Linux + Intel/AMD GPU to confirm: (1) the `low_power` entrypoint -(recent Intel iHD often requires the low-power encode entrypoint, AMD the full -one; we request full and let `Kind::Auto` fall back); (2) the NV12 upload -pitch/offset handling round-trips; (3) `cargo deny` accepts the new transitive -licenses (drm, drm-fourcc, etc.) once the vaapi graph resolves. +**Input: DMA-BUFs stay on the GPU, CPU frames are uploaded.** A +`Surface::DmaBuf` is encoded without a download. An NV12 buffer at the encoder's +size (a VA-API decode, or one `Surface::resize` already scaled through VPP) is +imported and encoded in place when that size is a whole number of macroblocks; +anything else the driver imports, packed RGB from a PipeWire screen capture in +particular, goes through VPP into the encoder's own surface. A layout the driver +refuses falls back to the CPU path and is not tried again. Every other surface +is converted to I420, interleaved to NV12, and uploaded into the encoder's input +surface, which is the path a UVC webcam's YUYV or MJPEG frames take. A V4L2 capture that exports DMA-BUFs (`VIDIOC_EXPBUF`) would let an +NV12-capable camera skip the upload too; it is not built. + +**Validated on Intel Meteor Lake** (iHD 26.1.5) by the tests in +`encode/backend/vaapi.rs`: they encode CPU frames, VA-API decodes handed over as +DMA-BUFs, and packed RGB buffers at and above the encoder's size, decode each +stream with openh264, and compare pixels. They skip without a VA-API device. +moq-vaapi opens the full encode entrypoint and falls back to the low-power one +(`VAEntrypointEncSliceLP`) that some Intel parts expose alone; no part that +needs the fallback has run it yet. ### NVENC ships via dlopen (no driver dependency at build or load) diff --git a/rs/moq-video/src/decode/backend/vaapi.rs b/rs/moq-video/src/decode/backend/vaapi.rs index 632cd616fe8..7d92c6bca18 100644 --- a/rs/moq-video/src/decode/backend/vaapi.rs +++ b/rs/moq-video/src/decode/backend/vaapi.rs @@ -47,15 +47,12 @@ //! back through `vaDeriveImage` rather than trying to read a tiled buffer as //! rows. -use std::os::fd::{AsFd, OwnedFd}; -use std::sync::{Arc, Mutex}; - use bytes::Bytes; use moq_net::Timestamp; use moq_vaapi::decode::{Config as VaapiConfig, Decoder, ExportedFrame}; use super::{Backend, Codec, Config}; -use crate::frame::{DmaBuf, DmaBufFrame, DmaBufPlane, DrmFormat, I420, Surface}; +use crate::frame::{I420, Surface, vaapi}; use crate::{Error, Frame, Output}; pub(crate) const NAME: &str = "vaapi"; @@ -204,122 +201,13 @@ fn share(exported: Vec) -> anyhow::Result> { .into_iter() .map(|frame| { let timestamp = Timestamp::from_micros(frame.timestamp).unwrap_or(Timestamp::ZERO); - Ok(Frame::new(Surface::DmaBuf(adopt(frame)?), timestamp)) + // A decode target names no color space, so the renderer infers one + // from the frame size exactly as it does for a downloaded picture. + Ok(Frame::new(Surface::DmaBuf(vaapi::adopt(frame, None)?), timestamp)) }) .collect() } -/// Describe an exported picture as a [`DmaBuf`]: the driver's format modifier, -/// and the offset and pitch of each of its memory planes. -/// -/// The width and height are the visible frame rather than the exported extent, -/// which is the driver's padded allocation. Neither the pitches nor the offsets -/// follow from the visible size, which is exactly why they are read off the -/// export rather than computed from it. -/// -/// # Errors -/// -/// When the export is not the one shape a [`DmaBuf`] can describe: a single NV12 -/// layer whose planes all live in a single object. The Intel and AMD drivers -/// export exactly that, and the alternatives are refused rather than guessed at, -/// because every one of them draws as a plausible-looking picture made of the -/// wrong bytes. -fn adopt(frame: ExportedFrame) -> anyhow::Result { - let (width, height) = (frame.width, frame.height); - // One object, because a consumer imports every plane from the one descriptor - // `Exported::export` hands out, and one layer, because the planes are read - // off it as a group. Both are what `VA_EXPORT_SURFACE_COMPOSED_LAYERS` asks - // for; neither is what it guarantees. - let [object] = frame.descriptor.objects.as_slice() else { - anyhow::bail!( - "VA-API exported {} objects, expected one holding every plane", - frame.descriptor.objects.len() - ); - }; - let [layer] = frame.descriptor.layers.as_slice() else { - anyhow::bail!( - "VA-API exported {} layers, expected one composed layer", - frame.descriptor.layers.len() - ); - }; - if layer.drm_format != DrmFormat::NV12.as_raw() { - anyhow::bail!("VA-API exported DRM format {:#x}, expected NV12", layer.drm_format); - } - - // `num_planes` and the arrays it indexes both come from the driver, and only - // the arrays are bounded, so indexing on the count would panic rather than - // fail. - let count = layer.num_planes as usize; - anyhow::ensure!( - count <= layer.offset.len(), - "VA-API exported {count} planes, more than a PRIME descriptor holds" - ); - let planes = (0..count) - .map(|plane| DmaBufPlane::new(layer.offset[plane], layer.pitch[plane])) - .collect(); - let modifier = object.drm_format_modifier; - - // A decode target names no color space, so the renderer infers one from the - // frame size exactly as it does for a downloaded picture. - DmaBuf::new( - DrmFormat::NV12, - modifier, - width, - height, - planes, - None, - Arc::new(Exported::new(frame)), - ) - .map_err(|e| anyhow::anyhow!("{e}")) -} - -/// A decoded picture the consumer holds as a DMA-BUF. -/// -/// Both of the things a consumer can do with one: hand a descriptor to a -/// graphics API, or give up on drawing it and read the pixels back. Dropping the -/// last clone destroys the surface, which is what returns its allocation to the -/// driver. -struct Exported { - /// Locked because [`DmaBufFrame`] hands out `&self` while what is behind it - /// is a single libva surface: `download_i420` maps that surface, and two - /// threads doing so at once is more than libva promises to serialize. The - /// frame is [`Send`] on its own, so a lock is enough and no `unsafe impl` is - /// involved. - frame: Mutex, -} - -impl Exported { - fn new(frame: ExportedFrame) -> Self { - Self { - frame: Mutex::new(frame), - } - } -} - -impl DmaBufFrame for Exported { - /// Vulkan takes ownership of an imported descriptor on success and closes it - /// on failure, so every import needs one of its own and the original stays - /// with the picture. - fn export(&self) -> std::io::Result { - let frame = self.frame.lock().expect("poisoned"); - let object = frame.descriptor.objects.first().ok_or_else(|| { - std::io::Error::new(std::io::ErrorKind::InvalidData, "the VA-API export carries no object") - })?; - object.fd.as_fd().try_clone_to_owned() - } - - /// Read the picture back through the retained surface rather than the - /// descriptor: a decode target is tiled, so mapping the file descriptor as - /// rows would be wrong. - fn download_i420(&self) -> Result { - let frame = self.frame.lock().expect("poisoned"); - let nv12 = frame - .download() - .map_err(|e| Error::Codec(anyhow::anyhow!("read a VA-API decode surface back: {e:?}")))?; - I420::from_nv12(&nv12.data, crate::Size::new(nv12.width, nv12.height)) - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/rs/moq-video/src/encode/backend/mod.rs b/rs/moq-video/src/encode/backend/mod.rs index fdb637e8c61..96530f8cf85 100644 --- a/rs/moq-video/src/encode/backend/mod.rs +++ b/rs/moq-video/src/encode/backend/mod.rs @@ -357,7 +357,7 @@ pub(crate) mod test_util { /// BT.601 goes out as SMPTE 170M primaries and matrix (code point 6) with the /// BT.709 transfer curve (1). The two curves are defined identically, and /// CoreVideo's SMPTE 170M transfer constant is deprecated while Media - /// Foundation has none at all, so 1 is the only value all four backends can + /// Foundation has none at all, so 1 is the only value all five backends can /// actually emit. pub(crate) const BT601_DESCRIBED: Described = Described { primaries: 6, diff --git a/rs/moq-video/src/encode/backend/vaapi.rs b/rs/moq-video/src/encode/backend/vaapi.rs index ce4f3b8c6e4..be0fc8e6a55 100644 --- a/rs/moq-video/src/encode/backend/vaapi.rs +++ b/rs/moq-video/src/encode/backend/vaapi.rs @@ -1,8 +1,8 @@ //! Intel/AMD VAAPI hardware backend via the `moq-vaapi` crate on Linux. //! //! `moq-vaapi` is a focused VA-API H.264 encoder vendored and trimmed from -//! cros-libva + discord/cros-codecs. It takes tightly-packed NV12 and emits an -//! Annex-B elementary stream with in-band SPS/PPS, matching avc3 mode. +//! cros-libva + discord/cros-codecs. It emits an Annex-B elementary stream with +//! in-band SPS/PPS, matching avc3 mode, and labels the color space in the VUI. //! //! As of moq-vaapi 0.0.3 libva is `dlopen`'d at runtime, so a VAAPI-enabled build //! needs no libva at build time and the binary carries no `NEEDED libva`. A @@ -10,38 +10,68 @@ //! driver), makes `Encoder::new` return an error; under automatic selection //! [`backend::open`](super::open) then moves on to openh264, like the NVENC backend. //! -//! Our captures hand us CPU I420 (webcams deliver YUYV/MJPEG, decoded to I420), -//! so each frame is interleaved to NV12 before encoding. +//! A [`Surface::DmaBuf`] is encoded without touching the CPU. An NV12 buffer at +//! the encoder's size (a VA-API decode, or one [`Surface::resize`] already +//! scaled on the GPU) is imported and encoded in place when its size is a +//! whole number of macroblocks. Anything else the driver imports, packed RGB +//! from a PipeWire screen capture above all, goes through the video processor +//! into the encoder's own surface first. RGB and YUV labelled with another +//! space are converted into the stream's; unlabelled YUV is taken to be in it. //! -//! NOT YET VALIDATED ON HARDWARE: `moq-vaapi`'s encode path is compile-verified -//! only, so the emitted bitstream needs a Linux + Intel/AMD GPU to confirm at -//! playback. +//! When the driver refuses to encode a buffer, the backend says so once and +//! sends every later buffer of the same format and modifier through the CPU +//! path. A failure before the driver sees the buffer, such as a producer fence +//! that times out, affects that frame only. +//! +//! Every other surface takes the CPU path: [`Surface::to_i420`], interleaved to +//! NV12, and uploaded. +//! +//! Validated on Intel Meteor Lake with iHD 26.1.5: the tests below encode CPU +//! frames, VA-API decodes handed over as DMA-BUFs, and packed RGB buffers at +//! and above the encoder's size, decode each stream with openh264, and compare +//! pixels. They skip on a machine without a VA-API device. + +use std::collections::HashSet; use bytes::Bytes; use moq_vaapi::encode::{Config as VaapiConfig, Encoder}; use super::super::encoder::{Config, Gop}; use super::{Backend, Encoded}; -use crate::frame::I420; -use crate::{Error, Frame}; +use crate::frame::{DmaBuf, DrmFormat, I420, vaapi}; +use crate::{Error, Frame, Surface}; pub(crate) const NAME: &str = "vaapi"; pub(crate) struct Vaapi { encoder: Encoder, + /// Buffer layouts the driver refused to encode, by format and modifier, so + /// each costs one failed attempt rather than one per frame. + refused: HashSet<(DrmFormat, u64)>, + /// Frames encoded from a DMA-BUF on the GPU, for the tests to tell the two + /// paths apart. + #[cfg(test)] + gpu_frames: u64, } impl Vaapi { pub(crate) fn open(config: &Config) -> Result, Error> { + Ok(Box::new(Self::new(config)?)) + } + + fn new(config: &Config) -> Result { let bitrate = config.resolved_bitrate().as_bps().min(u32::MAX as u64) as u32; let Gop::Keyframe { interval } = config.gop; - let vaapi = VaapiConfig::new( - config.width, - config.height, - config.framerate.rounded(), - bitrate, - interval, - ); + let vaapi = VaapiConfig { + color: vaapi::color(config.resolved_color()), + ..VaapiConfig::new( + config.width, + config.height, + config.framerate.rounded(), + bitrate, + interval, + ) + }; let encoder = Encoder::new(vaapi).map_err(|e| Error::Codec(anyhow::anyhow!("VAAPI encoder init: {e:?}")))?; tracing::info!( @@ -50,18 +80,72 @@ impl Vaapi { height = config.height, "opened H.264 encoder" ); - Ok(Box::new(Self { encoder })) + Ok(Self { + encoder, + refused: HashSet::new(), + #[cfg(test)] + gpu_frames: 0, + }) } -} -impl Backend for Vaapi { - fn encode(&mut self, frame: &Frame, cut: bool) -> Result, Error> { + /// Encodes `buffer` on the GPU, or returns `None` when it has to go through the CPU instead. + /// + /// Holds the producer's lease until the encode has read the buffer. + fn encode_dmabuf(&mut self, buffer: &DmaBuf, cut: bool) -> Option> { + let key = (buffer.format(), buffer.modifier()); + if self.refused.contains(&key) { + return None; + } + let (descriptor, lease) = match vaapi::import(buffer) { + Ok(imported) => imported, + Err(err) => { + tracing::debug!(encoder = NAME, %err, "DMA-BUF export failed; encoding this frame through the CPU"); + return None; + } + }; + let annexb = self.encoder.encode_dmabuf(descriptor, cut); + drop(lease); + match annexb { + Ok(annexb) => { + #[cfg(test)] + { + self.gpu_frames += 1; + } + Some(annexb) + } + Err(err) => { + tracing::warn!( + encoder = NAME, + format = ?key.0, + modifier = format_args!("{:#x}", key.1), + err = format!("{err:#}"), + "VAAPI cannot encode this DMA-BUF layout on the GPU; encoding it through the CPU from now on" + ); + self.refused.insert(key); + None + } + } + } + + fn encode_cpu(&mut self, frame: &Frame, cut: bool) -> Result, Error> { let i420 = frame.surface.to_i420()?; let nv12 = i420_to_nv12(&i420); - let annexb = self - .encoder + self.encoder .encode_nv12(&nv12, cut) - .map_err(|e| Error::Codec(anyhow::anyhow!("VAAPI encode: {e:?}")))?; + .map_err(|e| Error::Codec(e.context("VAAPI encode"))) + } +} + +impl Backend for Vaapi { + fn encode(&mut self, frame: &Frame, cut: bool) -> Result, Error> { + let gpu = match &frame.surface { + Surface::DmaBuf(buffer) => self.encode_dmabuf(buffer, cut), + _ => None, + }; + let annexb = match gpu { + Some(annexb) => annexb, + None => self.encode_cpu(frame, cut)?, + }; // Submitted and read back within the call, so this is that frame's output. Ok(if annexb.is_empty() { @@ -83,14 +167,12 @@ impl Backend for Vaapi { Ok(Vec::new()) } - fn set_bitrate(&mut self, _bitrate: u64) -> Result<(), Error> { - // moq-vaapi rebuilds its rate control parameter from `Config::bitrate` on - // every frame, so this needs nothing more than a setter on that config to - // work. It doesn't have one as of 0.0.3 and the field is private, so the - // rate is fixed at open until moq-vaapi exposes one. Rebuilding the - // session per estimate isn't worth it: that forces an IDR on a link that - // just told us it's congested. - Err(Error::BitrateUnsupported(NAME)) + fn set_bitrate(&mut self, bitrate: u64) -> Result<(), Error> { + // The rate control parameters go out with every frame, so the new rate + // applies from the next one without an IDR. + self.encoder + .set_bitrate(bitrate.min(u32::MAX as u64) as u32) + .map_err(|e| Error::Codec(e.context("VAAPI bitrate change"))) } fn can_cut(&self) -> bool { @@ -119,3 +201,321 @@ fn i420_to_nv12(i420: &I420) -> Vec { } out } + +// The round trips decode with openh264, an independent decoder. +#[cfg(all(test, feature = "openh264"))] +mod tests { + use moq_net::Timestamp; + + use super::*; + use crate::decode::backend as decode_backend; + use crate::decode::{Codec as DecodeCodec, Config as DecodeConfig, Kind as DecodeKind}; + use crate::encode::{Encoder as CrateEncoder, Kind as EncodeKind}; + use crate::{Output, Rate, Size}; + + const WIDTH: u32 = 320; + const HEIGHT: u32 = 240; + + fn config() -> Config { + Config { + kind: EncodeKind::Named(NAME.into()), + ..Config::new(WIDTH, HEIGHT, Rate::new(30, 1).unwrap()) + } + } + + /// Real hardware only: the backend itself, so a test can see which path + /// it took, or `None` to skip on a box with no VA-API H.264 encoder. + fn backend() -> Option { + match Vaapi::new(&config()) { + Ok(backend) => Some(backend), + Err(err) => { + eprintln!("skipping: no VA-API H.264 encoder: {err}"); + None + } + } + } + + fn decoder(name: &str, output: Output) -> Box { + let config = DecodeConfig { + kind: DecodeKind::Named(name.into()), + output, + ..DecodeConfig::new() + }; + decode_backend::open(DecodeCodec::H264, &config).expect("open the decoder") + } + + /// A static RGBA gradient that varies in both axes, so the chroma planes have + /// spatial structure and a pitch or plane-split bug corrupts the picture. + fn gradient_rgba(size: Size) -> Vec { + let (w, h) = (size.width as usize, size.height as usize); + let mut rgba = vec![0u8; w * h * 4]; + for y in 0..h { + for x in 0..w { + let i = (y * w + x) * 4; + rgba[i] = (x * 255 / w) as u8; + rgba[i + 1] = (y * 255 / h) as u8; + rgba[i + 2] = ((x + y) * 255 / (w + h)) as u8; + rgba[i + 3] = 255; + } + } + rgba + } + + /// Mean absolute difference between two planes. + fn mae(a: &[u8], b: &[u8]) -> u64 { + assert_eq!(a.len(), b.len()); + a.iter().zip(b).map(|(&x, &y)| x.abs_diff(y) as u64).sum::() / a.len() as u64 + } + + fn at(index: u64) -> Timestamp { + Timestamp::from_micros(index * 33_333).unwrap() + } + + /// Encodes each frame with `backend` and decodes the stream with openh264, + /// an independent decoder. + fn round_trip(backend: &mut Vaapi, frames: &[Frame]) -> Vec { + let mut software = decoder("openh264", Output::Cpu); + let mut decoded = Vec::new(); + for (index, frame) in frames.iter().enumerate() { + for encoded in backend.encode(frame, index == 0).expect("encode") { + decoded.extend( + software + .decode(encoded.payload, encoded.timestamp, index == 0) + .expect("decode"), + ); + } + } + decoded.extend(software.flush().expect("flush")); + assert_eq!(decoded.len(), frames.len(), "pictures went missing"); + decoded + } + + /// Asserts every plane of `decoded` is within `tolerance` of `expected` on average. + /// + /// A lossy round trip of this smooth gradient lands within 2 to 4 code + /// values; 8 (10 across two generations of coding) leaves room for encoder + /// variation while a swapped chroma plane or a stride bug misses by 30 or + /// more. + fn assert_close(decoded: &Frame, expected: &I420, tolerance: u64) { + let i420 = decoded.surface.to_i420().unwrap(); + assert_eq!((i420.width(), i420.height()), (expected.width(), expected.height())); + for (plane, (a, b)) in [ + (i420.y(), expected.y()), + (i420.u(), expected.u()), + (i420.v(), expected.v()), + ] + .into_iter() + .enumerate() + { + let error = mae(a, b); + assert!(error < tolerance, "plane {plane} is off by {error} on average"); + } + } + + /// CPU frames come back from an independent decoder as the picture that + /// went in, which a stride or chroma-order bug in the upload would break. + #[test] + fn cpu_frames_round_trip_through_a_software_decoder() { + let Some(mut backend) = backend() else { return }; + let size = Size::new(WIDTH, HEIGHT); + let rgba = gradient_rgba(size); + let expected = I420::from_rgba(&rgba, WIDTH * 4, size).unwrap(); + let frames: Vec = (0..5) + .map(|i| Frame::new(Surface::rgba(&rgba, size).unwrap(), at(i))) + .collect(); + + for frame in round_trip(&mut backend, &frames) { + assert_close(&frame, &expected, 8); + } + } + + /// A VA-API decode handed out as DMA-BUFs is re-encoded without leaving the + /// GPU, the transcode path, and still carries the picture. + #[test] + fn decoded_dmabufs_are_encoded_on_the_gpu() { + let Some(mut backend) = backend() else { return }; + let size = Size::new(WIDTH, HEIGHT); + let rgba = gradient_rgba(size); + let expected = I420::from_rgba(&rgba, WIDTH * 4, size).unwrap(); + + // A software-encoded source stream, decoded by VA-API into DMA-BUFs. + let mut source = CrateEncoder::new(&Config { + kind: EncodeKind::Software, + ..Config::new(WIDTH, HEIGHT, Rate::new(30, 1).unwrap()) + }) + .unwrap(); + let mut hardware = decoder(NAME, Output::Native); + let mut frames = Vec::new(); + for i in 0..5 { + if i == 0 { + source.cut().unwrap(); + } + let frame = Frame::new(Surface::rgba(&rgba, size).unwrap(), at(i)); + for encoded in source.encode(&frame).unwrap() { + frames.extend(hardware.decode(encoded.payload, encoded.timestamp, i == 0).unwrap()); + } + } + frames.extend(hardware.flush().unwrap()); + assert!( + frames.iter().all(|frame| matches!(frame.surface, Surface::DmaBuf(_))), + "the decoder handed out CPU frames, so nothing here tests the GPU path" + ); + + let decoded = round_trip(&mut backend, &frames); + assert!( + backend.gpu_frames > 0 && backend.refused.is_empty(), + "the encoder fell back to the CPU" + ); + for frame in decoded { + // Two generations of lossy coding. + assert_close(&frame, &expected, 10); + } + } + + /// A packed RGB DMA-BUF, what a PipeWire screen capture delivers, is scaled + /// and converted on the GPU and encoded from there. + #[test] + fn an_rgb_dmabuf_is_scaled_and_encoded_on_the_gpu() { + let Some(mut backend) = backend() else { return }; + let source_size = Size::new(WIDTH * 2, HEIGHT * 2); + let rgba = gradient_rgba(source_size); + let Some(buffer) = vaapi::testing::bgrx_dmabuf(&rgba, source_size) else { + eprintln!("skipping: no VA-API device to allocate a BGRX surface on"); + return; + }; + let frame = Frame::new(Surface::DmaBuf(buffer), at(0)); + + let scaled = frame + .resize(Size::new(WIDTH, HEIGHT), &crate::resize::Config::default()) + .expect("resize"); + let Surface::DmaBuf(ref buffer) = scaled.surface else { + panic!("the resize left the GPU"); + }; + assert_eq!(buffer.format(), crate::DrmFormat::NV12); + assert_eq!((buffer.width(), buffer.height()), (WIDTH, HEIGHT)); + + let expected = I420::from_rgba(&rgba, source_size.width * 4, source_size) + .unwrap() + .resize(Size::new(WIDTH, HEIGHT)) + .unwrap(); + // The GPU read-back of the scaled buffer, before any coding. + let read_back = scaled.surface.to_i420().expect("read the scaled buffer back"); + assert_eq!(read_back.color(), Some(crate::Color::infer(source_size))); + + let decoded = round_trip(&mut backend, std::slice::from_ref(&scaled)); + assert!( + backend.gpu_frames > 0 && backend.refused.is_empty(), + "the encoder fell back to the CPU" + ); + assert_close(&decoded[0], &expected, 10); + } + + /// The encoder takes a packed RGB DMA-BUF at its own size directly. + #[test] + fn an_rgb_dmabuf_at_the_encoder_size_is_converted_on_the_gpu() { + let Some(mut backend) = backend() else { return }; + let size = Size::new(WIDTH, HEIGHT); + let rgba = gradient_rgba(size); + let Some(buffer) = vaapi::testing::bgrx_dmabuf(&rgba, size) else { + eprintln!("skipping: no VA-API device to allocate a BGRX surface on"); + return; + }; + let expected = I420::from_rgba(&rgba, WIDTH * 4, size).unwrap(); + + let decoded = round_trip(&mut backend, &[Frame::new(Surface::DmaBuf(buffer), at(0))]); + assert!( + backend.gpu_frames > 0 && backend.refused.is_empty(), + "the encoder fell back to the CPU" + ); + assert_close(&decoded[0], &expected, 10); + } + + /// A bitrate change reaches the encoder instead of being refused. + #[test] + fn the_bitrate_can_change_mid_stream() { + let Some(mut backend) = backend() else { return }; + let size = Size::new(WIDTH, HEIGHT); + let rgba = gradient_rgba(size); + let frames: Vec = (0..6) + .map(|i| Frame::new(Surface::rgba(&rgba, size).unwrap(), at(i))) + .collect(); + for (index, frame) in frames.iter().enumerate() { + if index == 3 { + backend.set_bitrate(250_000).expect("set the bitrate"); + } + assert!(!backend.encode(frame, index == 0).expect("encode").is_empty()); + } + assert_eq!(backend.encoder.config().bitrate, 250_000); + } + + /// A buffer the driver cannot import is encoded through the CPU, and its + /// layout is not tried on the GPU again. + #[test] + fn a_refused_dmabuf_is_encoded_through_the_cpu() { + let Some(mut backend) = backend() else { return }; + let size = Size::new(WIDTH, HEIGHT); + let rgba = gradient_rgba(size); + let expected = I420::from_rgba(&rgba, WIDTH * 4, size).unwrap(); + let frames: Vec = (0..3) + .map(|i| { + let buffer = vaapi::testing::unimportable_dmabuf(expected.clone()); + Frame::new(Surface::DmaBuf(buffer), at(i)) + }) + .collect(); + + let decoded = round_trip(&mut backend, &frames); + assert_eq!(backend.gpu_frames, 0); + assert_eq!(backend.refused.len(), 1, "the refused layout is remembered once"); + for frame in decoded { + assert_close(&frame, &expected, 8); + } + } + + /// A buffer the video processor cannot import is resized on the CPU, the + /// same as without VA-API. + #[test] + fn a_refused_dmabuf_is_resized_on_the_cpu() { + let size = Size::new(WIDTH, HEIGHT); + let expected = I420::from_rgba(&gradient_rgba(size), WIDTH * 4, size).unwrap(); + let buffer = vaapi::testing::unimportable_dmabuf(expected.clone()); + let half = Size::new(WIDTH / 2, HEIGHT / 2); + let scaled = Frame::new(Surface::DmaBuf(buffer), at(0)) + .resize(half, &crate::resize::Config::default()) + .expect("resize"); + let Surface::I420(ref pixels) = scaled.surface else { + panic!("a refused buffer came back on the GPU"); + }; + let reference = expected.resize(half).unwrap(); + assert_eq!(mae(pixels.y(), reference.y()), 0, "the CPU path is the CPU resize"); + } + + /// The SPS names the color space the stream was configured with, read back + /// out of the bitstream. + #[test] + fn the_sps_declares_the_color_space() { + use super::super::test_util::{BT601_DESCRIBED, BT709_DESCRIBED, declared_color}; + use crate::Color; + + // Opposite of the space `resolved_color` would infer from the size, so a + // backend that ignores `Config::color` cannot pass. + for (size, color, described) in [ + (Size::new(640, 480), Color::Bt709Limited, BT709_DESCRIBED), + (Size::new(1920, 1080), Color::Bt601Limited, BT601_DESCRIBED), + ] { + let config = Config { + kind: EncodeKind::Named(NAME.into()), + color: Some(color), + ..Config::new(size.width, size.height, Rate::new(30, 1).unwrap()) + }; + let Ok(mut backend) = Vaapi::new(&config) else { + eprintln!("skipping: no VA-API H.264 encoder"); + return; + }; + let rgba = [255u8, 0, 0, 255].repeat(size.pixels() as usize); + let frame = Frame::new(Surface::rgba(&rgba, size).unwrap(), at(0)); + let encoded = backend.encode(&frame, true).unwrap(); + let annexb = &encoded.first().expect("a keyframe").payload; + assert_eq!(declared_color(annexb), Some(described), "{size} SPS color description"); + } + } +} diff --git a/rs/moq-video/src/frame.rs b/rs/moq-video/src/frame.rs index 6c166c96e11..4a403ccc46b 100644 --- a/rs/moq-video/src/frame.rs +++ b/rs/moq-video/src/frame.rs @@ -543,6 +543,19 @@ impl Surface { Surface::I420(texture.download_i420()?.resize(size)?) } }, + // Scaled by the VA-API video processor, which also converts packed + // RGB to NV12 on the way, so the result is ready for the VAAPI + // encoder to import. If VA-API is missing or refuses the buffer, + // download and resize on the CPU. + #[cfg(all(target_os = "linux", feature = "vaapi"))] + Surface::DmaBuf(buffer) if config.output == crate::Output::Native => match vaapi::resize(buffer, size) { + Ok(scaled) => Surface::DmaBuf(scaled), + Err(err) => { + static WARN_ONCE: std::sync::Once = std::sync::Once::new(); + WARN_ONCE.call_once(|| tracing::warn!(%err, "GPU resize failed; falling back to the CPU")); + Surface::I420(buffer.inner.download_i420()?.resize(size)?) + } + }, #[allow(unreachable_patterns)] other => Surface::I420(other.to_i420()?.into_owned().resize(size)?), }) @@ -1916,6 +1929,10 @@ pub mod vulkan; #[path = "frame/cuda.rs"] pub mod cuda; +#[cfg(all(target_os = "linux", feature = "vaapi"))] +#[path = "frame/vaapi.rs"] +pub(crate) mod vaapi; + // Compiled for every test build so its policy tests run without a GPU. #[cfg(any(test, all(target_os = "linux", feature = "nvidia")))] #[path = "frame/pool.rs"] diff --git a/rs/moq-video/src/frame/vaapi.rs b/rs/moq-video/src/frame/vaapi.rs new file mode 100644 index 00000000000..84f30df9aa4 --- /dev/null +++ b/rs/moq-video/src/frame/vaapi.rs @@ -0,0 +1,388 @@ +//! VA-API's side of [`DmaBuf`](super::DmaBuf): handing one to `moq-vaapi`, +//! wrapping what `moq-vaapi` exports, and scaling one on the GPU. +//! +//! The decoder and the encoder backends and +//! [`Surface::resize`](super::Surface::resize) all meet +//! `moq-vaapi` here, so there is one place that knows how a crate `DmaBuf` maps +//! to a VA-API import descriptor and back. +//! +//! Scaling goes through the VA-API video processor on whichever device answers +//! first. It stays on the GPU: the input is imported, blitted into an NV12 +//! surface of the target size (converting packed RGB on the way), and that +//! surface is exported as a new DMA-BUF, which the VAAPI encoder then imports +//! in turn. A simulcast publisher resizing one captured frame into several +//! renditions pays one blit per rendition instead of a download and a CPU +//! scale each. + +use std::cell::RefCell; +use std::collections::HashSet; +use std::os::fd::{AsFd, OwnedFd}; +use std::sync::{Arc, Mutex}; + +use moq_vaapi::decode::ExportedFrame; +use moq_vaapi::dmabuf::{DmaBuf as VaapiDmaBuf, Plane}; +use moq_vaapi::vpp::Processor; +use moq_vaapi::{Matrix, VA_FOURCC_NV12}; + +use super::{DmaBuf, DmaBufExport, DmaBufFrame, DmaBufPlane, DrmFormat, I420}; +use crate::{Color, Error, Size}; + +/// Describes `buffer` for a VA-API import, returning the producer lease to hold until the import is done with. +/// +/// Waits on the producer's write fence first, since a VA-API import does not. +/// The descriptor handed to `moq-vaapi` is a duplicate, owned by the imported +/// surface; the returned lease keeps the producer from recycling the buffer +/// underneath it. +pub(crate) fn import(buffer: &DmaBuf) -> Result<(VaapiDmaBuf, DmaBufExport), Error> { + let export = buffer + .export() + .map_err(|e| Error::Codec(anyhow::anyhow!("export a DMA-BUF for VA-API: {e}")))?; + let fd: OwnedFd = export + .as_fd() + .try_clone_to_owned() + .map_err(|e| Error::Codec(anyhow::anyhow!("duplicate a DMA-BUF descriptor: {e}")))?; + let descriptor = VaapiDmaBuf { + drm_format: buffer.format().as_raw(), + modifier: buffer.modifier(), + width: buffer.width(), + height: buffer.height(), + planes: buffer + .planes() + .iter() + .map(|plane| Plane { + offset: plane.offset(), + pitch: plane.stride(), + }) + .collect(), + fd, + color: buffer.color().map(color), + }; + Ok((descriptor, export)) +} + +/// Returns the `moq-vaapi` color space matching `color`. +pub(crate) fn color(color: Color) -> moq_vaapi::Color { + match color { + Color::Bt601Limited => moq_vaapi::Color { + matrix: Matrix::Bt601, + full_range: false, + }, + Color::Bt601Full => moq_vaapi::Color { + matrix: Matrix::Bt601, + full_range: true, + }, + Color::Bt709Limited => moq_vaapi::Color { + matrix: Matrix::Bt709, + full_range: false, + }, + Color::Bt709Full => moq_vaapi::Color { + matrix: Matrix::Bt709, + full_range: true, + }, + } +} + +/// Returns whether `format` is packed RGB, which carries no YUV matrix of its own. +pub(crate) fn is_rgb(format: DrmFormat) -> bool { + matches!( + format, + DrmFormat::XRGB8888 | DrmFormat::ARGB8888 | DrmFormat::XBGR8888 | DrmFormat::ABGR8888 + ) +} + +thread_local! { + /// The video processor this thread scales with, opened on the first resize. + /// + /// Per thread because a processor keeps its contexts behind an `Rc`. Each + /// thread that resizes pays for one display and one context per size. A + /// failed open is kept too, so a machine without VA-API finds that out once + /// per thread rather than once per frame. + static PROCESSOR: RefCell>> = const { RefCell::new(None) }; + + /// Buffer layouts the video processor refused on this thread, by format and + /// modifier, so a buffer it cannot read goes straight to the CPU path. + static REFUSED: RefCell> = RefCell::new(HashSet::new()); +} + +/// Scales `buffer` to `size` on the GPU, returning an NV12 DMA-BUF. +/// +/// The color handling matches the CPU resize. A YUV input keeps whatever space +/// it was labelled with, including none. A packed RGB input is converted into +/// [`Color::infer`] for its own size, the space the CPU download would have +/// picked, and the result is labelled with it. +/// +/// # Errors +/// +/// Fails when this thread has no video processor, when the processor refused +/// this buffer's format and modifier before, and when the export, the import, +/// or the blit fails. The caller falls back to the CPU. +pub(crate) fn resize(buffer: &DmaBuf, size: Size) -> Result { + let key = (buffer.format(), buffer.modifier()); + if REFUSED.with(|refused| refused.borrow().contains(&key)) { + return Err(Error::Unsupported(format!( + "the VA-API video processor refused {:?} with modifier {:#x}", + key.0, key.1 + ))); + } + let (input_space, output_space, label) = match is_rgb(buffer.format()) { + true => { + let inferred = Color::infer(Size::new(buffer.width(), buffer.height())); + (None, Some(color(inferred)), Some(inferred)) + } + false => (buffer.color().map(color), buffer.color().map(color), buffer.color()), + }; + + let exported = PROCESSOR.with(|slot| -> Result { + let mut slot = slot.borrow_mut(); + // Checked before exporting, so a machine without VA-API does not pay a + // fence wait and a descriptor duplicate on every frame to learn it again. + let processor = slot + .get_or_insert_with(|| Processor::open().map_err(|e| format!("{e:#}"))) + .as_ref() + .map_err(|e| Error::Unsupported(format!("no VA-API video processor: {e}")))?; + let (descriptor, lease) = import(buffer)?; + let processed = processor.import(descriptor).and_then(|input| { + processor.process( + &input, + VA_FOURCC_NV12, + (size.width, size.height), + input_space, + output_space, + ) + }); + // The blit has synced, so the producer's buffer is free to go back. + drop(lease); + let output = processed.map_err(|e| { + REFUSED.with(|refused| refused.borrow_mut().insert(key)); + Error::Codec(e.context("scale a DMA-BUF with the VA-API video processor")) + })?; + ExportedFrame::from_surface(output, 0).map_err(Error::Codec) + })?; + adopt(exported, label).map_err(Error::Codec) +} + +/// Describes an exported picture as a [`DmaBuf`]: the driver's format modifier, +/// and the offset and pitch of each of its memory planes. +/// +/// The width and height are the visible frame rather than the exported extent, +/// which is the driver's padded allocation. Neither the pitches nor the offsets +/// follow from the visible size, which is exactly why they are read off the +/// export rather than computed from it. `color` is the space the pixels are in, +/// where the producer knows it. +/// +/// # Errors +/// +/// When the export is not the one shape a [`DmaBuf`] can describe: a single NV12 +/// layer whose planes all live in a single object. The Intel and AMD drivers +/// export exactly that, and the alternatives are refused rather than guessed at, +/// because every one of them draws as a plausible-looking picture made of the +/// wrong bytes. +pub(crate) fn adopt(frame: ExportedFrame, color: Option) -> anyhow::Result { + let (width, height) = (frame.width, frame.height); + // One object, because a consumer imports every plane from the one descriptor + // `Exported::export` hands out, and one layer, because the planes are read + // off it as a group. Both are what `VA_EXPORT_SURFACE_COMPOSED_LAYERS` asks + // for; neither is what it guarantees. + let [object] = frame.descriptor.objects.as_slice() else { + anyhow::bail!( + "VA-API exported {} objects, expected one holding every plane", + frame.descriptor.objects.len() + ); + }; + let [layer] = frame.descriptor.layers.as_slice() else { + anyhow::bail!( + "VA-API exported {} layers, expected one composed layer", + frame.descriptor.layers.len() + ); + }; + if layer.drm_format != DrmFormat::NV12.as_raw() { + anyhow::bail!("VA-API exported DRM format {:#x}, expected NV12", layer.drm_format); + } + + // `num_planes` and the arrays it indexes both come from the driver, and only + // the arrays are bounded, so indexing on the count would panic rather than + // fail. + let count = layer.num_planes as usize; + anyhow::ensure!( + count <= layer.offset.len(), + "VA-API exported {count} planes, more than a PRIME descriptor holds" + ); + let planes = (0..count) + .map(|plane| DmaBufPlane::new(layer.offset[plane], layer.pitch[plane])) + .collect(); + let modifier = object.drm_format_modifier; + + DmaBuf::new( + DrmFormat::NV12, + modifier, + width, + height, + planes, + color, + Arc::new(Exported { + frame: Mutex::new(frame), + color, + }), + ) + .map_err(|e| anyhow::anyhow!("{e}")) +} + +/// A picture on a VA-API surface the consumer holds as a DMA-BUF. +/// +/// Both of the things a consumer can do with one: hand a descriptor to a +/// graphics API, or give up on drawing it and read the pixels back. Dropping the +/// last clone destroys the surface, which is what returns its allocation to the +/// driver. +struct Exported { + /// Locked because [`DmaBufFrame`] hands out `&self` while what is behind it + /// is a single libva surface: `download_i420` maps that surface, and two + /// threads doing so at once is more than libva promises to serialize. The + /// frame is [`Send`] on its own, so a lock is enough and no `unsafe impl` is + /// involved. + frame: Mutex, + /// The space the pixels are in, for the read-back to carry. + color: Option, +} + +impl DmaBufFrame for Exported { + /// Vulkan takes ownership of an imported descriptor on success and closes it + /// on failure, so every import needs one of its own and the original stays + /// with the picture. + fn export(&self) -> std::io::Result { + let frame = self.frame.lock().expect("poisoned"); + let object = frame.descriptor.objects.first().ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::InvalidData, "the VA-API export carries no object") + })?; + object.fd.as_fd().try_clone_to_owned() + } + + /// Read the picture back through the retained surface rather than the + /// descriptor: a VA-API surface is tiled, so mapping the file descriptor as + /// rows would be wrong. + fn download_i420(&self) -> Result { + let frame = self.frame.lock().expect("poisoned"); + let nv12 = frame + .download() + .map_err(|e| Error::Codec(anyhow::anyhow!("read a VA-API surface back: {e:?}")))?; + let i420 = I420::from_nv12(&nv12.data, crate::Size::new(nv12.width, nv12.height))?; + Ok(match self.color { + Some(color) => i420.with_color(color), + None => i420, + }) + } +} + +// For the encoder backend's tests, which need openh264 to decode. +#[cfg(all(test, feature = "openh264"))] +pub(crate) mod testing { + use std::os::fd::OwnedFd; + use std::sync::Arc; + + use moq_vaapi::{Display, Image, Surface as VaSurface, UsageHint, VA_FOURCC_BGRX, VA_RT_FORMAT_RGB32}; + + use super::super::{DmaBuf, DmaBufFrame, DmaBufPlane, DrmFormat, I420}; + use crate::{Error, Size}; + + /// A driver surface exported as a DMA-BUF, standing in for a PipeWire buffer. + struct Allocated { + _surface: VaSurface<()>, + fd: OwnedFd, + } + + impl DmaBufFrame for Allocated { + fn export(&self) -> std::io::Result { + self.fd.try_clone() + } + + fn download_i420(&self) -> Result { + Err(Error::Unsupported("a test buffer has no CPU path".into())) + } + } + + /// A buffer no driver imports, whose CPU read-back still works. + struct Unimportable { + fd: OwnedFd, + pixels: I420, + } + + impl DmaBufFrame for Unimportable { + fn export(&self) -> std::io::Result { + self.fd.try_clone() + } + + fn download_i420(&self) -> Result { + Ok(self.pixels.clone()) + } + } + + /// Returns an NV12 DMA-BUF the driver refuses to import (its descriptor is + /// `/dev/null` and its modifier made up) whose CPU read-back yields `pixels`. + pub(crate) fn unimportable_dmabuf(pixels: I420) -> DmaBuf { + let fd = OwnedFd::from(std::fs::File::open("/dev/null").expect("open /dev/null")); + let (width, height) = (pixels.width(), pixels.height()); + DmaBuf::new( + DrmFormat::NV12, + 0x00ff_ffff_ffff_fffe, + width, + height, + vec![DmaBufPlane::new(0, width), DmaBufPlane::new(width * height, width)], + None, + Arc::new(Unimportable { fd, pixels }), + ) + .expect("a valid description") + } + + /// Returns a BGRX DMA-BUF holding `rgba`, allocated by VA-API, or `None` without a device. + pub(crate) fn bgrx_dmabuf(rgba: &[u8], size: Size) -> Option { + let display = Display::open()?; + let (width, height) = (size.width, size.height); + let surface = display + .create_surfaces( + VA_RT_FORMAT_RGB32, + Some(VA_FOURCC_BGRX), + width, + height, + Some(UsageHint::USAGE_HINT_EXPORT), + vec![()], + ) + .ok()? + .pop()?; + let format = display + .query_image_formats() + .ok()? + .into_iter() + .find(|format| format.fourcc == VA_FOURCC_BGRX)?; + { + let mut image = Image::create_from(&surface, format, (width, height), (width, height)).ok()?; + let va_image = *image.image(); + let data = image.as_mut(); + for y in 0..height as usize { + for x in 0..width as usize { + let from = (y * width as usize + x) * 4; + let to = va_image.offsets[0] as usize + y * va_image.pitches[0] as usize + x * 4; + let [r, g, b, _] = [rgba[from], rgba[from + 1], rgba[from + 2], rgba[from + 3]]; + data[to..to + 4].copy_from_slice(&[b, g, r, 255]); + } + } + } + surface.sync().ok()?; + let mut exported = surface.export_prime().ok()?; + let layer = &exported.layers[0]; + assert_eq!(layer.drm_format, DrmFormat::XRGB8888.as_raw()); + let plane = DmaBufPlane::new(layer.offset[0], layer.pitch[0]); + let object = exported.objects.remove(0); + DmaBuf::new( + DrmFormat::XRGB8888, + object.drm_format_modifier, + width, + height, + vec![plane], + None, + Arc::new(Allocated { + _surface: surface, + fd: object.fd, + }), + ) + .ok() + } +}