diff --git a/doc/lib/rs/moq-video.md b/doc/lib/rs/moq-video.md index 1c5ab813366..11249d2e36f 100644 --- a/doc/lib/rs/moq-video.md +++ b/doc/lib/rs/moq-video.md @@ -20,7 +20,7 @@ ffmpeg, no GStreamer, no system codec to install. Highlights: -- **Automatic backend selection**, hardware first. Linux GPU libraries are `dlopen`ed at runtime, so one binary starts anywhere and warns when it falls back to software. openh264 (the default-on `openh264` feature) is statically linked as the H.264 fallback; H.265 is hardware-only; AV1 decodes via NVDEC. The VAAPI encoder is compile-verified but not yet validated on hardware. +- **Automatic backend selection**, hardware first. Linux GPU libraries are `dlopen`ed at runtime, so one binary starts anywhere and warns when it falls back to software. openh264 (the default-on `openh264` feature) is statically linked as the H.264 fallback; H.265 is hardware-only; AV1 decodes via NVDEC. The VAAPI encoder, decoder, and GPU resize share one render node: the first whose driver does all three, or the one the `MOQ_VAAPI_DEVICE` environment variable names (for example `/dev/dri/renderD129`). - **Publish on demand.** `encode::publish_capture` advertises the track up front and opens the camera only while someone subscribes. - **GPU ownership where the platform allows.** Matching codec backends consume their native GPU surfaces directly. The renderer imports `CVPixelBuffer` and supported DMA-BUF formats. Linux/NVIDIA producers can import dedicated Vulkan RGBA8 slots into CUDA with timeline-semaphore ordering and completion-driven slot return. Vulkan/CUDA surfaces deliberately have no CPU pixel fallback; other surfaces use the typed `Surface::into_i420()` and configured `Surface::to_rgba(config)` when needed. - **Live bitrate control** where the selected backend supports it, without forcing a keyframe. An unsupported backend keeps its opening rate. diff --git a/rs/moq-video/src/decode/backend/vaapi.rs b/rs/moq-video/src/decode/backend/vaapi.rs index 7d92c6bca18..25b63e81a5c 100644 --- a/rs/moq-video/src/decode/backend/vaapi.rs +++ b/rs/moq-video/src/decode/backend/vaapi.rs @@ -11,7 +11,8 @@ //! missing render node, or a driver with no H.264 decode entrypoint makes //! `Decoder::new` return an error; under automatic selection //! [`backend::open`](super::open) then moves on to the next candidate, like the -//! NVDEC backend. +//! NVDEC backend. The render node is the one the encoder and the GPU resize +//! share, which `MOQ_VAAPI_DEVICE` can name; see `frame::vaapi::device`. //! //! Progressive 8-bit 4:2:0 only, which is everything a browser's `VideoEncoder`, //! WebRTC, or this crate's own encoders emit. The decoder rejects an interlaced or @@ -37,15 +38,17 @@ //! downloading, which native output permits. //! //! [`Output::Cpu`](crate::Output::Cpu) downloads inside the backend rather than -//! leaving it to the generic conversion, because exporting is not free: -//! handing a surface out retires it from the decoder's recycling pool, since a -//! later picture decoded over it would corrupt one the consumer still holds, so -//! it costs an allocation per picture on top of the download the CPU consumer -//! pays anyway. A native consumer that still wants pixels is not stranded: +//! leaving it to the generic conversion, because exporting is not free: each +//! picture costs a PRIME export, and its surface stays out of the decoder's +//! recycling pool until the frame drops, since a later picture decoded over it +//! would corrupt one the consumer still holds. A native consumer that draws a +//! picture and lets it go gets the surface decoded into again rather than a new +//! allocation per picture. One that still wants pixels is not stranded: //! [`Surface::into_i420`](crate::Surface::into_i420) answers, because -//! `moq-vaapi` keeps the retired surface alongside the descriptor and reads it -//! back through `vaDeriveImage` rather than trying to read a tiled buffer as -//! rows. +//! `moq-vaapi` keeps the surface alongside the descriptor and reads it back +//! through `vaDeriveImage` rather than trying to read a tiled buffer as rows. + +use std::path::Path; use bytes::Bytes; use moq_net::Timestamp; @@ -79,8 +82,10 @@ impl Vaapi { return Err(Error::Codec(anyhow::anyhow!("VAAPI cannot decode {}", codec.label()))); } - let decoder = - Decoder::new(VaapiConfig::new()).map_err(|e| Error::Codec(anyhow::anyhow!("VAAPI decoder init: {e:?}")))?; + let vaapi = VaapiConfig { + device: vaapi::device().map(Path::to_path_buf), + }; + let decoder = Decoder::new(vaapi).map_err(|e| Error::Codec(anyhow::anyhow!("VAAPI decoder init: {e:?}")))?; let exporting = config.output == Output::Native; tracing::info!(decoder = NAME, exporting, "opened H.264 decoder"); diff --git a/rs/moq-video/src/encode/backend/vaapi.rs b/rs/moq-video/src/encode/backend/vaapi.rs index be0fc8e6a55..ec9061ac22e 100644 --- a/rs/moq-video/src/encode/backend/vaapi.rs +++ b/rs/moq-video/src/encode/backend/vaapi.rs @@ -9,6 +9,8 @@ //! libva-less host, or a present-but-unusable VA stack (no render node, no usable //! driver), makes `Encoder::new` return an error; under automatic selection //! [`backend::open`](super::open) then moves on to openh264, like the NVENC backend. +//! The render node is the one the decoder and the GPU resize share, which +//! `MOQ_VAAPI_DEVICE` can name; see `frame::vaapi::device`. //! //! A [`Surface::DmaBuf`] is encoded without touching the CPU. An NV12 buffer at //! the encoder's size (a VA-API decode, or one [`Surface::resize`] already @@ -32,6 +34,7 @@ //! pixels. They skip on a machine without a VA-API device. use std::collections::HashSet; +use std::path::Path; use bytes::Bytes; use moq_vaapi::encode::{Config as VaapiConfig, Encoder}; @@ -48,6 +51,9 @@ pub(crate) struct Vaapi { /// Buffer layouts the driver refused to encode, by format and modifier, so /// each costs one failed attempt rather than one per frame. refused: HashSet<(DrmFormat, u64)>, + /// The CPU path's NV12 staging buffer, kept so a frame reuses the last one's + /// allocation. + nv12: Vec, /// Frames encoded from a DMA-BUF on the GPU, for the tests to tell the two /// paths apart. #[cfg(test)] @@ -63,6 +69,7 @@ impl Vaapi { let bitrate = config.resolved_bitrate().as_bps().min(u32::MAX as u64) as u32; let Gop::Keyframe { interval } = config.gop; let vaapi = VaapiConfig { + device: vaapi::device().map(Path::to_path_buf), color: vaapi::color(config.resolved_color()), ..VaapiConfig::new( config.width, @@ -76,6 +83,7 @@ impl Vaapi { tracing::info!( encoder = NAME, + device = ?encoder.config().device, width = config.width, height = config.height, "opened H.264 encoder" @@ -83,6 +91,7 @@ impl Vaapi { Ok(Self { encoder, refused: HashSet::new(), + nv12: Vec::new(), #[cfg(test)] gpu_frames: 0, }) @@ -129,9 +138,9 @@ impl Vaapi { fn encode_cpu(&mut self, frame: &Frame, cut: bool) -> Result, Error> { let i420 = frame.surface.to_i420()?; - let nv12 = i420_to_nv12(&i420); + i420_to_nv12(&i420, &mut self.nv12); self.encoder - .encode_nv12(&nv12, cut) + .encode_nv12(&self.nv12, cut) .map_err(|e| Error::Codec(e.context("VAAPI encode"))) } } @@ -184,22 +193,21 @@ impl Backend for Vaapi { } } -/// Interleave tightly-packed I420 into tightly-packed NV12: copy Y as-is, then -/// interleave U and V into the chroma plane. -fn i420_to_nv12(i420: &I420) -> Vec { +/// Interleave tightly-packed I420 into tightly-packed NV12 in `out`: copy Y +/// as-is, then interleave U and V into the chroma plane. +/// +/// `out` is resized to the frame and every byte of it is written, so it can be +/// reused from frame to frame without clearing. +fn i420_to_nv12(i420: &I420, out: &mut Vec) { let (w, h) = (i420.width as usize, i420.height as usize); let (cw, ch) = (w / 2, h / 2); - let mut out = vec![0u8; w * h + 2 * cw * ch]; - out[..w * h].copy_from_slice(i420.y()); - - let (u, v) = (i420.u(), i420.v()); - let uv = &mut out[w * h..]; - for i in 0..cw * ch { - uv[i * 2] = u[i]; - uv[i * 2 + 1] = v[i]; + out.resize(w * h + 2 * cw * ch, 0); + let (y, uv) = out.split_at_mut(w * h); + y.copy_from_slice(i420.y()); + for ((pair, &u), &v) in uv.as_chunks_mut::<2>().0.iter_mut().zip(i420.u()).zip(i420.v()) { + *pair = [u, v]; } - out } // The round trips decode with openh264, an independent decoder. @@ -312,6 +320,27 @@ mod tests { } } + /// The reused staging buffer takes each frame's size and carries nothing + /// over from a larger frame before it. Needs no hardware. + #[test] + fn the_nv12_staging_buffer_is_rewritten_per_frame() { + let mut nv12 = Vec::new(); + let large = Size::new(64, 64); + let large = I420::from_rgba(&gradient_rgba(large), large.width * 4, large).unwrap(); + i420_to_nv12(&large, &mut nv12); + + let small = Size::new(32, 16); + let small = I420::from_rgba(&gradient_rgba(small), small.width * 4, small).unwrap(); + i420_to_nv12(&small, &mut nv12); + + let luma = small.y().len(); + assert_eq!(nv12.len(), luma * 3 / 2); + assert_eq!(&nv12[..luma], small.y()); + for (index, pair) in nv12[luma..].as_chunks::<2>().0.iter().enumerate() { + assert_eq!(*pair, [small.u()[index], small.v()[index]], "chroma pair {index}"); + } + } + /// CPU frames come back from an independent decoder as the picture that /// went in, which a stride or chroma-order bug in the upload would break. #[test] diff --git a/rs/moq-video/src/frame/vaapi.rs b/rs/moq-video/src/frame/vaapi.rs index 84f30df9aa4..569421bcbe2 100644 --- a/rs/moq-video/src/frame/vaapi.rs +++ b/rs/moq-video/src/frame/vaapi.rs @@ -6,8 +6,9 @@ //! `moq-vaapi` here, so there is one place that knows how a crate `DmaBuf` maps //! to a VA-API import descriptor and back. //! -//! Scaling goes through the VA-API video processor on whichever device answers -//! first. It stays on the GPU: the input is imported, blitted into an NV12 +//! All three open the render node [`device`] names, so a picture moving between +//! them stays on one GPU. Scaling goes through the VA-API video processor on +//! that node. It stays on the GPU: the input is imported, blitted into an NV12 //! surface of the target size (converting packed RGB on the way), and that //! surface is exported as a new DMA-BUF, which the VAAPI encoder then imports //! in turn. A simulcast publisher resizing one captured frame into several @@ -17,7 +18,8 @@ use std::cell::RefCell; use std::collections::HashSet; use std::os::fd::{AsFd, OwnedFd}; -use std::sync::{Arc, Mutex}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, OnceLock}; use moq_vaapi::decode::ExportedFrame; use moq_vaapi::dmabuf::{DmaBuf as VaapiDmaBuf, Plane}; @@ -27,6 +29,45 @@ use moq_vaapi::{Matrix, VA_FOURCC_NV12}; use super::{DmaBuf, DmaBufExport, DmaBufFrame, DmaBufPlane, DrmFormat, I420}; use crate::{Color, Error, Size}; +/// Environment variable naming the render node, such as `/dev/dri/renderD129`, that [`device`] returns. +const DEVICE_ENV: &str = "MOQ_VAAPI_DEVICE"; + +/// Returns the render node the VA-API encoder, decoder, and resize open, or `None` for each to find its own. +/// +/// [`DEVICE_ENV`] names one outright, and a named node is used even when it +/// fails to open, so a typo is an error rather than a silent fallback. +/// Otherwise this is the first node whose driver encodes and decodes H.264 and +/// post-processes video. Sharing one node is what lets a DMA-BUF move from the +/// decoder through a resize into the encoder without a copy; on a machine with +/// two GPUs, the first node that decodes need not be one that encodes. Where no +/// node offers all three, each opens the first node that offers what it needs. +/// +/// Resolved once per process, since finding it opens every render node. +pub(crate) fn device() -> Option<&'static Path> { + static DEVICE: OnceLock> = OnceLock::new(); + DEVICE + .get_or_init(|| { + if let Some(node) = std::env::var_os(DEVICE_ENV) { + let node = PathBuf::from(node); + tracing::info!(device = %node.display(), "using the VA-API render node {DEVICE_ENV} names"); + return Some(node); + } + let node = moq_vaapi::DrmDeviceIterator::default().find(|node| { + moq_vaapi::Display::open_drm_display(node).is_ok_and(|display| { + moq_vaapi::encode::probe(&display).is_ok() + && moq_vaapi::decode::probe(&display).is_ok() + && moq_vaapi::vpp::probe(&display).is_ok() + }) + }); + match &node { + Some(node) => tracing::debug!(device = %node.display(), "VA-API render node"), + None => tracing::debug!("no VA-API render node encodes, decodes, and scales; each opens its own"), + } + node + }) + .as_deref() +} + /// Describes `buffer` for a VA-API import, returning the producer lease to hold until the import is done with. /// /// Waits on the producer's write fence first, since a VA-API import does not. @@ -137,7 +178,13 @@ pub(crate) fn resize(buffer: &DmaBuf, size: Size) -> Result { // Checked before exporting, so a machine without VA-API does not pay a // fence wait and a descriptor duplicate on every frame to learn it again. let processor = slot - .get_or_insert_with(|| Processor::open().map_err(|e| format!("{e:#}"))) + .get_or_insert_with(|| { + match device() { + Some(node) => Processor::new(node), + None => Processor::open(), + } + .map_err(|e| format!("{e:#}")) + }) .as_ref() .map_err(|e| Error::Unsupported(format!("no VA-API video processor: {e}")))?; let (descriptor, lease) = import(buffer)?; @@ -231,8 +278,8 @@ pub(crate) fn adopt(frame: ExportedFrame, color: Option) -> anyhow::Resul /// /// Both of the things a consumer can do with one: hand a descriptor to a /// graphics API, or give up on drawing it and read the pixels back. Dropping the -/// last clone destroys the surface, which is what returns its allocation to the -/// driver. +/// last clone hands a decoded picture's surface back to the decoder, which +/// decodes into it again, and destroys any other surface. struct Exported { /// Locked because [`DmaBufFrame`] hands out `&self` while what is behind it /// is a single libva surface: `download_i420` maps that surface, and two @@ -272,6 +319,29 @@ impl DmaBufFrame for Exported { } } +#[cfg(test)] +mod tests { + use super::*; + + /// Without an override, the node the codecs share is one that does + /// everything it is shared for. + #[test] + fn the_shared_node_encodes_decodes_and_scales() { + if std::env::var_os(DEVICE_ENV).is_some() { + eprintln!("skipping: {DEVICE_ENV} names the node"); + return; + } + let Some(node) = device() else { + eprintln!("skipping: no render node encodes, decodes, and scales"); + return; + }; + let display = moq_vaapi::Display::open_drm_display(node).expect("reopen the shared node"); + moq_vaapi::encode::probe(&display).expect("the shared node encodes H.264"); + moq_vaapi::decode::probe(&display).expect("the shared node decodes H.264"); + moq_vaapi::vpp::probe(&display).expect("the shared node scales"); + } +} + // For the encoder backend's tests, which need openh264 to decode. #[cfg(all(test, feature = "openh264"))] pub(crate) mod testing { @@ -334,7 +404,12 @@ pub(crate) mod testing { /// Returns a BGRX DMA-BUF holding `rgba`, allocated by VA-API, or `None` without a device. pub(crate) fn bgrx_dmabuf(rgba: &[u8], size: Size) -> Option { - let display = Display::open()?; + // The same node resize and the encoder open. On two GPUs `Display::open` + // can be a different device, and the processor then refuses the import. + let display = match super::device() { + Some(node) => Display::open_drm_display(node).ok()?, + None => Display::open()?, + }; let (width, height) = (size.width, size.height); let surface = display .create_surfaces( diff --git a/rs/moq-video/src/render/renderer.rs b/rs/moq-video/src/render/renderer.rs index fa2049201ae..1eb0575ef3b 100644 --- a/rs/moq-video/src/render/renderer.rs +++ b/rs/moq-video/src/render/renderer.rs @@ -1287,6 +1287,130 @@ mod tests { } } + /// A decoder that decodes into its surfaces again still has every picture + /// drawn as the one just decoded. + /// + /// Each picture is drawn and dropped before the next access unit goes in, + /// the way a player draws a picture and moves on, so the decoder gets its + /// surfaces back once the renderer's GPU work is done with them. A surface + /// handed back too early shows up as a picture with the block where a later + /// one has it. The same stream decoded to the CPU is the reference. + /// + /// Ignored: needs a Vulkan GPU and a VA-API device. Run with + /// `cargo test -p moq-video --features render,vaapi recycled -- --ignored --nocapture`. + #[cfg(all(target_os = "linux", feature = "vaapi"))] + #[tokio::test] + #[ignore = "needs a Vulkan GPU and a VA-API device"] + async fn recycled_decoder_surfaces_draw_the_latest_picture() { + use std::os::unix::fs::MetadataExt as _; + + use crate::decode::backend::{self, Codec, vaapi}; + + const PICTURES: u64 = 30; + let Some((device, queue)) = dmabuf_gpu().await else { + eprintln!("skipping: no Vulkan adapter with DMA-BUF external memory"); + return; + }; + let decode = |output| { + backend::open( + Codec::H264, + &crate::decode::Config { + kind: crate::decode::Kind::Named(vaapi::NAME.into()), + output, + ..crate::decode::Config::new() + }, + ) + }; + let Ok(mut exporting) = decode(crate::Output::Native) else { + eprintln!("skipping: no VA-API H.264 decoder"); + return; + }; + let mut downloading = decode(crate::Output::Cpu).expect("a second decoder"); + + // A block moving across a gradient, so a picture drawn from a stale + // import shows the block where an earlier picture had it. + let size = Size::new(320, 240); + let (width, height) = (size.width, size.height); + let picture = |step: u64| { + let mut rgba = vec![0u8; (width * height * 4) as usize]; + let x0 = (step as u32 * 8) % (width / 2); + for y in 0..height { + for x in 0..width { + let index = ((y * width + x) * 4) as usize; + let block = (height / 4..height / 2).contains(&y) && (x0..x0 + width / 4).contains(&x); + rgba[index] = if block { 240 } else { (x * 255 / width) as u8 }; + rgba[index + 1] = (y * 255 / height) as u8; + rgba[index + 2] = ((x + y) * 255 / (width + height)) as u8; + rgba[index + 3] = 255; + } + } + Surface::rgba(&rgba, size).expect("a valid RGBA frame") + }; + let mut encoder = crate::encode::Encoder::new(&crate::encode::Config { + kind: crate::encode::Kind::Software, + ..crate::encode::Config::new(width, height, crate::Rate::new(30, 1).unwrap()) + }) + .expect("a software H.264 encoder"); + + let config = Config { + usage: wgpu::TextureUsages::COPY_SRC, + ..Config::new() + }; + let mut importing = Renderer::new(&device, &queue, config.clone()).expect("a renderer"); + let mut uploading = Renderer::new(&device, &queue, config).expect("a renderer"); + + let mut buffers = std::collections::HashSet::new(); + let mut drawn = 0; + for index in 0..PICTURES { + if index == 0 { + encoder.cut().unwrap(); + } + let frame = Frame::new(picture(index), Timestamp::from_micros(index * 33_333).unwrap()); + for unit in encoder.encode(&frame).expect("encode a picture") { + let exported = exporting + .decode(unit.payload.clone(), unit.timestamp, index == 0) + .expect("decode to the GPU"); + let downloaded = downloading + .decode(unit.payload, unit.timestamp, index == 0) + .expect("decode to the CPU"); + assert_eq!(exported.len(), downloaded.len(), "the two decoders disagreed"); + + for (gpu, cpu) in exported.iter().zip(&downloaded) { + let Surface::DmaBuf(buffer) = &gpu.surface else { + panic!("picture {index} did not come back GPU-resident"); + }; + let export = buffer.export().expect("export the decoded surface"); + let file = std::fs::File::from(export.as_fd().try_clone_to_owned().expect("duplicate")); + buffers.insert(file.metadata().expect("stat the DMA-BUF").ino()); + drop((file, export)); + + let texture = importing.render(gpu).expect("draw the imported picture"); + assert_eq!(importing.strikes, 0, "picture {index} fell back to the CPU"); + let zero_copy = readback(&device, &queue, &texture).await; + let texture = uploading.render(cpu).expect("draw the downloaded picture"); + let reference = readback(&device, &queue, &texture).await; + + for (pixel, (&imported, &expected)) in zero_copy.iter().zip(&reference).enumerate() { + let drift = (0..4).map(|c| imported[c].abs_diff(expected[c])).max().unwrap_or(0); + let (x, y) = (pixel % width as usize, pixel / width as usize); + assert!( + drift <= 2, + "picture {index} at ({x}, {y}): imported {imported:?}, downloaded {expected:?}" + ); + } + drawn += 1; + } + } + } + + eprintln!("{drawn} pictures drawn from {} buffers", buffers.len()); + assert!(drawn > 0, "the decoder produced no pictures"); + assert!( + (buffers.len() as u64) < PICTURES, + "the decoder decoded every picture into a new buffer, so nothing here is reused" + ); + } + /// A pool-backed NV12 surface, shaped like a hardware decode's output. #[cfg(target_os = "macos")] fn pooled(size: Size, rgba: [u8; 4]) -> crate::Surface {