From 7d981eb1bc62f0d5bfd68165cdd77cbe303b627d Mon Sep 17 00:00:00 2001 From: Frando Date: Thu, 24 Sep 2026 09:57:00 +0200 Subject: [PATCH 1/4] feat(moq-video): capture cameras through PipeWire `Source::Camera` was V4L2 only on Linux. That leaves a Flatpak or Snap app without a camera, since the sandbox has no /dev/video access, and a Raspberry Pi CSI camera unusable, since libcamera presents it as a PipeWire node rather than a V4L2 node that yields processed frames. With the `pipewire` feature, `cameras()` now also lists PipeWire `Video/Source` nodes with the `Camera` role as `pipewire:`, and `pipewire` alone opens the session manager's default camera. The id picks the backend, so a `/dev/videoN` path still always means V4L2. The stream reuses the ScreenCast loop, which now also connects to the session socket, resolves a camera by name from the registry, and converts YUY2. Inside a sandbox the default camera and the listing go through the xdg-desktop-portal Camera interface instead. Co-Authored-By: Claude Opus 5.5 (1M context) --- doc/bin/cli.md | 4 +- doc/lib/rs/moq-video.md | 11 +- rs/moq-cli/Cargo.toml | 4 +- rs/moq-cli/src/publish.rs | 3 +- rs/moq-video/Cargo.toml | 23 +- rs/moq-video/src/capture/mod.rs | 120 +++++- rs/moq-video/src/capture/pipewire.rs | 419 +++++++++++++++----- rs/moq-video/src/capture/pipewire/camera.rs | 391 ++++++++++++++++++ rs/moq-video/src/frame.rs | 4 +- 9 files changed, 852 insertions(+), 127 deletions(-) create mode 100644 rs/moq-video/src/capture/pipewire/camera.rs diff --git a/doc/bin/cli.md b/doc/bin/cli.md index fcef336af15..54e9a13e8c0 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -126,8 +126,8 @@ audio is Opus. The camera is opened only while someone is watching, and `--bitrate` is the opening ceiling. Backends with live bitrate control lower it to fit the connection's bandwidth estimate. `moq devices` prints every source id. Requires the `capture` feature; on Linux that needs the ALSA headers for -the microphone, and `--display` also needs the `pipewire` feature (links -libpipewire). +the microphone, and `--display` and `pipewire:` cameras also need the +`pipewire` feature (links libpipewire). ## Transcode diff --git a/doc/lib/rs/moq-video.md b/doc/lib/rs/moq-video.md index 6d2d2c7fcf2..b986760acc5 100644 --- a/doc/lib/rs/moq-video.md +++ b/doc/lib/rs/moq-video.md @@ -39,6 +39,15 @@ and an empty rate list means no discrete intervals were reported. Device errors are returned rather than treated as an empty list. Other platforms return `Error::Unsupported`. +With `pipewire` enabled, `capture::cameras` also lists PipeWire camera nodes as +`pipewire:` after the V4L2 devices, and `pipewire` alone opens the +session manager's default camera. That reaches cameras V4L2 cannot: a Raspberry +Pi CSI camera behind libcamera, and any camera from inside a Flatpak or Snap +sandbox, where the default camera comes through the xdg-desktop-portal Camera +interface. These cameras stream their first raw YUY2 or NV12 mode: MJPEG-only +modes are not offered, `Config::width` and `height` are not applied yet, and +`camera_modes` returns `Error::Unsupported` for them. + `capture::Config::framerate` is an `Option` request in the same exact type; the stream reports the rate the device accepted, or `None` when the driver reported none. @@ -59,7 +68,7 @@ cargo add moq-video --features capture # camera + screen capture, no system bu cargo add moq-video --features render # wgpu rendering cargo add moq-video --features v4l2 # Linux V4L2 M2M codecs, no system build deps cargo add moq-video --features vaapi # Linux VAAPI codecs (bindgen needs libclang) -cargo add moq-video --features pipewire # Wayland screen capture (links libpipewire) +cargo add moq-video --features pipewire # Wayland screen + PipeWire cameras (links libpipewire) cargo add moq-video --no-default-features --features openh264 # software H.264 only cargo add moq-video --no-default-features --features nvidia # Linux NVIDIA only, no C++ or wgpu ``` diff --git a/rs/moq-cli/Cargo.toml b/rs/moq-cli/Cargo.toml index 9c496a7b7b4..a4c291b1bc3 100644 --- a/rs/moq-cli/Cargo.toml +++ b/rs/moq-cli/Cargo.toml @@ -74,8 +74,8 @@ vaapi = ["moq-video?/vaapi", "moq-transcode?/vaapi"] # checks its bindings in. Only applies when `capture`, `transcode`, or `play` # pulls in moq-video. v4l2 = ["moq-video?/v4l2", "moq-transcode?/v4l2"] -# Screen capture for `capture --display` on Linux (xdg-desktop-portal + PipeWire), -# opt-in like moq-video's own `pipewire` because it links libpipewire-0.3 at +# Screen capture for `capture --display` on Linux (xdg-desktop-portal + PipeWire) +# and PipeWire cameras for `capture --camera pipewire:`, opt-in like moq-video's own `pipewire` because it links libpipewire-0.3 at # build time. Only bites when `capture` pulls in moq-video and moq-audio. # Also reaches audio, so `devices`, microphone capture and playback go through # the sound server rather than its ALSA plugin on a machine running one. diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index b26616c98fc..6360d44f6d1 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -51,7 +51,8 @@ impl From for moq_video::encode::Codec { #[usage(group("audio-source"))] pub struct CaptureArgs { /// Capture a camera, by the id `moq devices` reports (an AVFoundation - /// `uniqueID`, `/dev/videoN` path, or Media Foundation symbolic link). + /// `uniqueID`, `/dev/videoN` path, `pipewire:` node, or Media Foundation + /// symbolic link). /// Bare `--camera`, or no source flag at all, opens the default camera. #[usage(long, group = "video-source")] pub camera: Option>, diff --git a/rs/moq-video/Cargo.toml b/rs/moq-video/Cargo.toml index 2828effa43d..3e5e6508d8e 100644 --- a/rs/moq-video/Cargo.toml +++ b/rs/moq-video/Cargo.toml @@ -86,10 +86,12 @@ render = [ "objc2-core-video/CVMetalTextureCache", "objc2-core-video/objc2-metal", ] -# Screen capture on Linux via xdg-desktop-portal + PipeWire (Wayland and X11). -# Off by default because the `pipewire` crate links libpipewire-0.3 via pkg-config -# at build time (and its bindgen needs libclang), so it only belongs in builds -# that actually capture displays. Camera capture (V4L2) needs only what +# Screen capture on Linux via xdg-desktop-portal + PipeWire (Wayland and X11), +# and camera capture from PipeWire camera nodes: libcamera sensors such as a +# Raspberry Pi CSI camera, and any camera from inside a Flatpak or Snap sandbox +# through the camera portal. Off by default because the `pipewire` crate links +# libpipewire-0.3 via pkg-config at build time (and its bindgen needs libclang), +# so it only belongs in builds that use it. V4L2 camera capture needs only what # `capture` already does. pipewire = ["capture", "dmabuf", "dep:pipewire", "dep:ashpd"] @@ -136,10 +138,12 @@ yuv = "0.8.14" ndk = { version = "0.9", optional = true, default-features = false, features = ["media", "api-level-26"] } [target.'cfg(target_os="linux")'.dependencies] -# xdg-desktop-portal ScreenCast negotiation for the `pipewire` feature (pure Rust, -# zbus over DBus). The portal owns source selection: the user picks a monitor in a -# compositor dialog, and we get back a PipeWire fd + node id to stream from. -ashpd = { version = "0.13", optional = true, default-features = false, features = ["tokio", "screencast"] } +# xdg-desktop-portal ScreenCast and Camera negotiation for the `pipewire` feature +# (pure Rust, zbus over DBus). The ScreenCast portal owns source selection: the +# user picks a monitor in a compositor dialog, and we get back a PipeWire fd + +# node id to stream from. The Camera portal grants a sandboxed app a PipeWire +# remote that exposes the cameras. +ashpd = { version = "0.13", optional = true, default-features = false, features = ["tokio", "screencast", "camera"] } # The NVIDIA codecs, behind the default-on `nvidia` feature. Everything is # dlopen'd at runtime: cudarc loads CUDA, and moq-nvenc (an in-tree fork of # nvidia-video-codec-sdk, see rs/moq-nvenc) loads libnvidia-encode. So the binary @@ -176,7 +180,8 @@ moq-v4l = { workspace = true, optional = true } # then falls back to the next candidate instead of failing at load, matching the # NVENC and NVDEC backends. moq-vaapi = { workspace = true, optional = true } -# PipeWire video stream for the portal's node, behind the `pipewire` feature. +# PipeWire video stream for a portal's node or a camera node, behind the +# `pipewire` feature. # Links libpipewire-0.3 via pkg-config at build time; bindgen needs libclang. pipewire = { version = "0.10", optional = true } # X11 display/window enumeration and capture. Pure Rust and used when no diff --git a/rs/moq-video/src/capture/mod.rs b/rs/moq-video/src/capture/mod.rs index 9e7bd5a9690..969745572ea 100644 --- a/rs/moq-video/src/capture/mod.rs +++ b/rs/moq-video/src/capture/mod.rs @@ -2,9 +2,9 @@ //! per-source: //! - macOS camera -> AVFoundation, screen -> ScreenCaptureKit, both yielding //! zero-copy `CVPixelBuffer` surfaces straight to VideoToolbox. -//! - Linux camera -> native V4L2 (YUYV / MJPEG -> CPU I420), X11 display and -//! window -> X11, Wayland display -> xdg-desktop-portal + PipeWire -//! (`pipewire` feature). +//! - Linux camera -> native V4L2 (YUYV / MJPEG -> CPU I420), or a PipeWire +//! camera node (`pipewire` feature), X11 display and window -> X11, Wayland +//! display -> xdg-desktop-portal + PipeWire (`pipewire` feature). //! - Windows camera -> native Media Foundation (`IMFSourceReader`), screen -> //! DXGI Desktop Duplication, window -> GDI (BGRA -> CPU I420). //! @@ -43,7 +43,7 @@ mod v4l2; #[cfg(target_os = "linux")] mod x11; -// Portal + PipeWire screen capture on Linux. +// Portal + PipeWire screen capture and PipeWire camera capture on Linux. #[cfg(all(target_os = "linux", feature = "pipewire"))] mod pipewire; @@ -82,6 +82,13 @@ pub enum Source { /// The identifiers from [`cameras`] are an AVFoundation `uniqueID` on macOS, /// a `/dev/videoN` path on Linux, and a Media Foundation symbolic link on /// Windows. Bare numeric indices remain accepted on Linux and Windows. + /// + /// With the `pipewire` feature, Linux also lists PipeWire camera nodes as + /// `pipewire:`, and `pipewire` alone opens the session manager's + /// default camera. The id picks the backend: a `/dev/videoN` path always opens + /// V4L2, and a `pipewire` id always opens PipeWire. `None` opens + /// `/dev/video0`, except inside a sandbox (Flatpak or Snap), where it opens + /// PipeWire's default camera through the camera portal. Camera(Option), /// A whole display. `None` opens the main display. @@ -364,7 +371,15 @@ pub async fn open(config: &Config) -> Result { } #[cfg(target_os = "linux")] { - v4l2::open(config, device.as_deref()).await + match LinuxCamera::select(device.as_deref(), sandboxed()) { + LinuxCamera::V4l2(device) => v4l2::open(config, device).await, + #[cfg(feature = "pipewire")] + LinuxCamera::PipeWire(node) => pipewire::camera::open(config, node).await, + #[cfg(not(feature = "pipewire"))] + LinuxCamera::PipeWire(_) => Err(Error::Unsupported( + "PipeWire camera capture without the `pipewire` feature".to_string(), + )), + } } #[cfg(target_os = "windows")] { @@ -436,6 +451,12 @@ pub async fn open(config: &Config) -> Result { } /// List the available cameras and the identifiers [`Source::Camera`] accepts. +/// +/// On Linux with the `pipewire` feature, V4L2 devices come first and PipeWire +/// camera nodes follow, so a webcam that PipeWire also serves appears once per +/// backend. A session without PipeWire lists V4L2 alone. Inside a sandbox the +/// PipeWire nodes come through the camera portal, which may ask the user for +/// camera access. pub async fn cameras() -> Result, Error> { #[cfg(target_os = "macos")] { @@ -443,7 +464,10 @@ pub async fn cameras() -> Result, Error> { } #[cfg(target_os = "linux")] { - blocking(v4l2::cameras).await + let cameras = blocking(v4l2::cameras).await?; + #[cfg(feature = "pipewire")] + let cameras = [cameras, pipewire::camera::cameras().await?].concat(); + Ok(cameras) } #[cfg(target_os = "windows")] { @@ -462,7 +486,8 @@ pub async fn cameras() -> Result, Error> { /// are ones [`open`] could negotiate rather than everything the driver /// advertises. /// -/// Linux only; other backends return [`Error::Unsupported`]. +/// Linux V4L2 only; PipeWire cameras and other platforms return +/// [`Error::Unsupported`]. /// Rates are exact device reports and [`Config::framerate`] accepts the same type. /// V4L2 prefers the closest geometry, then the accepted rate /// nearest that request, then the cheaper conversion format. @@ -473,7 +498,10 @@ pub async fn camera_modes(camera: Option<&str>) -> Result, Error> { let _ = camera; #[cfg(target_os = "linux")] { - let camera = camera.map(str::to_string); + let camera = match LinuxCamera::select(camera, sandboxed()) { + LinuxCamera::V4l2(camera) => camera.map(str::to_string), + LinuxCamera::PipeWire(_) => return Err(Error::Unsupported("listing PipeWire camera modes".to_string())), + }; blocking(move || v4l2::modes(camera.as_deref())).await } #[cfg(not(target_os = "linux"))] @@ -538,6 +566,51 @@ pub async fn apps() -> Result, Error> { } } +/// The id prefix of a PipeWire camera, and on its own the default one. +#[cfg(any(target_os = "linux", test))] +const PIPEWIRE: &str = "pipewire"; + +/// The Linux backend a [`Source::Camera`] selector names. +#[cfg(any(target_os = "linux", test))] +#[derive(Debug, PartialEq, Eq)] +enum LinuxCamera<'a> { + /// A V4L2 device, by path or index. `None` opens `/dev/video0`. + V4l2(Option<&'a str>), + /// A PipeWire camera, by `node.name`. `None` lets the session manager pick. + PipeWire(Option<&'a str>), +} + +#[cfg(any(target_os = "linux", test))] +impl<'a> LinuxCamera<'a> { + /// Parse a camera selector. A sandbox has no V4L2 devices to open by + /// default, so its default camera is PipeWire's, reached through the portal. + fn select(selector: Option<&'a str>, sandboxed: bool) -> Self { + match selector { + None if sandboxed => Self::PipeWire(None), + None => Self::V4l2(None), + Some(PIPEWIRE) => Self::PipeWire(None), + Some(selector) => match selector.strip_prefix(PIPEWIRE).and_then(|rest| rest.strip_prefix(':')) { + Some(name) => Self::PipeWire(Some(name)), + None => Self::V4l2(Some(selector)), + }, + } + } +} + +/// Whether this process runs in a Flatpak or Snap sandbox. Without the +/// `pipewire` feature there is no portal to use, so it never matters. +#[cfg(target_os = "linux")] +fn sandboxed() -> bool { + #[cfg(feature = "pipewire")] + { + ashpd::is_sandboxed() + } + #[cfg(not(feature = "pipewire"))] + { + false + } +} + /// Run synchronous platform enumeration off the async runtime's worker threads. #[cfg(any(target_os = "linux", target_os = "windows"))] async fn blocking(f: F) -> Result @@ -549,3 +622,34 @@ where .await .map_err(|err| Error::Codec(anyhow::anyhow!("capture enumeration thread failed: {err}")))? } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn camera_selectors_name_their_backend() { + assert_eq!(LinuxCamera::select(None, false), LinuxCamera::V4l2(None)); + assert_eq!(LinuxCamera::select(None, true), LinuxCamera::PipeWire(None)); + for sandboxed in [false, true] { + assert_eq!( + LinuxCamera::select(Some("/dev/video2"), sandboxed), + LinuxCamera::V4l2(Some("/dev/video2")) + ); + assert_eq!(LinuxCamera::select(Some("1"), sandboxed), LinuxCamera::V4l2(Some("1"))); + assert_eq!( + LinuxCamera::select(Some("pipewire"), sandboxed), + LinuxCamera::PipeWire(None) + ); + assert_eq!( + LinuxCamera::select(Some("pipewire:v4l2_input.pci-0000_00_14.0-usb-0_4_1.0"), sandboxed), + LinuxCamera::PipeWire(Some("v4l2_input.pci-0000_00_14.0-usb-0_4_1.0")) + ); + } + // Only the whole prefix selects PipeWire; anything else is a device path. + assert_eq!( + LinuxCamera::select(Some("pipewired"), false), + LinuxCamera::V4l2(Some("pipewired")) + ); + } +} diff --git a/rs/moq-video/src/capture/pipewire.rs b/rs/moq-video/src/capture/pipewire.rs index 0b0146aab79..2fcb52d9d8f 100644 --- a/rs/moq-video/src/capture/pipewire.rs +++ b/rs/moq-video/src/capture/pipewire.rs @@ -1,14 +1,15 @@ -//! Screen capture via xdg-desktop-portal + PipeWire (Linux, Wayland and X11). +//! Screen and camera capture via PipeWire (Linux, Wayland and X11). //! -//! The ScreenCast portal owns source selection: [`open`] pops the compositor's +//! The ScreenCast portal owns screen selection: [`open`] pops the compositor's //! picker dialog, the user chooses a monitor, and the portal hands us a PipeWire -//! fd + node id. A dedicated thread then runs the PipeWire main loop, forwarding -//! DMA-BUF frames without copying when the compositor offers them and converting -//! shared-memory frames to CPU [`I420`] otherwise. It pushes both into the shared -//! [`FrameChannel`] (callback-driven like the macOS delegate, not a pull-style -//! pump). +//! fd + node id. Cameras are PipeWire nodes too, reached as described in +//! [`camera`]. For either, a dedicated thread then runs the PipeWire main loop, +//! forwarding DMA-BUF frames without copying when the producer offers them and +//! converting shared-memory frames to CPU [`I420`] otherwise. It pushes both into +//! the shared [`FrameChannel`] (callback-driven like the macOS delegate, not a +//! pull-style pump). //! -//! Two quirks worth knowing: +//! Two screen quirks worth knowing: //! - `publish_capture` releases the capture while unwatched and reopens it on //! demand. A fresh portal session would re-prompt the picker every time, so the //! portal's restore token is kept in a process-wide slot and replayed on the @@ -19,6 +20,7 @@ //! - Compositors only deliver frames on damage, so a static screen would starve //! the encoder. A loop timer re-emits the last frame whenever a frame interval //! passes without a fresh one, mirroring the Windows Desktop Duplication pacing. +//! Cameras deliver every interval, so their streams have no such timer. use std::borrow::Cow; use std::cell::RefCell; @@ -44,13 +46,14 @@ use crate::{Color, Error, Size}; const DEFAULT_FRAMERATE: u32 = 30; // libspa 0.10 omits this flag from its safe wrapper, but exposes the raw bits. const CHUNK_FLAG_EMPTY: i32 = 1 << 1; -/// The compositor sends the negotiated format right after the stream connects; +/// The producer sends the negotiated format right after the stream connects; /// if nothing arrives the session is broken (or the grant was revoked mid-setup). const FORMAT_TIMEOUT: Duration = Duration::from_secs(10); /// ScreenCast compositors deliver the current content as a first frame right -/// after negotiation; none arriving means the session is broken, so fail `open` -/// rather than hand the encoder a stream that will never produce (same -/// first-frame wait as the macOS ScreenCaptureKit backend). +/// after negotiation, and a camera starts streaming once linked; none arriving +/// means the session is broken, so fail `open` rather than hand the encoder a +/// stream that will never produce (same first-frame wait as the macOS +/// ScreenCaptureKit backend). const FIRST_FRAME_TIMEOUT: Duration = Duration::from_secs(5); /// The portal restore token from the last grant, replayed on the next [`open`] @@ -62,6 +65,8 @@ fn err(ctx: &str, e: impl std::fmt::Display) -> Error { Error::Codec(anyhow::anyhow!("{ctx}: {e}")) } +pub(super) mod camera; + /// Open a portal screen capture and stream its frames from a PipeWire loop thread. pub(super) async fn open(config: &Config, device: Option<&str>) -> Result { if let Some(device) = device { @@ -69,7 +74,73 @@ pub(super) async fn open(config: &Config, device: Option<&str>) -> Result, + target: Target, + label: String, + /// The frame size to prefer, or `None` to leave the size out of the offers + /// and take the producer's first mode. Any size is accepted either way. + /// + /// Cameras use `None`: given a size range, spa-v4l2 answers with modes the + /// device does not have (a webcam with YUY2 only at 640x480 was offered YUY2 + /// at the range's 1280x720 default) and then fails the link. + size: Option, +} +/// Whether a loop captures a screen or a camera. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Kind { + Screen, + Camera, +} + +impl Kind { + /// The raw formats to offer, most preferred first. + fn formats(self) -> &'static [VideoFormat] { + match self { + Self::Screen => &SCREEN_FORMATS, + Self::Camera => &CAMERA_FORMATS, + } + } + + /// The producer, as error messages name it. + fn producer(self) -> &'static str { + match self { + Self::Screen => "the compositor", + Self::Camera => "the camera", + } + } +} + +/// The node a capture loop links to. +enum Target { + /// A node id granted by the ScreenCast portal. + Node(u32), + /// A camera by `node.name`, or `None` for the session manager's default. + Camera(Option), +} + +/// Spawn the PipeWire loop for `capture` and wait for its format and first frame. +async fn start(config: &Config, capture: Capture, session: Option) -> Result { + let kind = capture.kind; let chan = FrameChannel::new(); let framerate = config .framerate @@ -94,8 +165,7 @@ pub(super) async fn open(config: &Config, device: Option<&str>) -> Result) -> Result result?, Ok(Err(_)) => { return Err(Error::Codec(anyhow::anyhow!( - "screen capture thread exited before negotiating a format" + "{kind:?} capture thread exited before negotiating a format" ))); } Err(_) => { return Err(Error::Codec(anyhow::anyhow!( - "no video format from the compositor within {FORMAT_TIMEOUT:?}" + "no video format from {} within {FORMAT_TIMEOUT:?}", + kind.producer() ))); } }; @@ -144,16 +215,18 @@ pub(super) async fn open(config: &Config, device: Option<&str>) -> Result return Err(error), Ok(Ok(None)) | Err(_) => { return Err(Error::Codec(anyhow::anyhow!( - "no frames from the compositor within {FIRST_FRAME_TIMEOUT:?}" + "no frames from {} within {FIRST_FRAME_TIMEOUT:?}", + kind.producer() ))); } }; tracing::info!( - node = node_id, + ?kind, + source = %geo.label, width = geo.width, height = geo.height, - "opened screen capture (PipeWire)" + "opened PipeWire capture" ); Ok(Stream::new( @@ -247,8 +320,9 @@ struct LoopGuard { quit: pw::channel::Sender<()>, handle: Option>, /// Held so the portal session outlives the loop; dropping it closes the - /// session only after the loop thread has been joined above. - _session: SessionGuard, + /// session only after the loop thread has been joined above. Cameras have + /// no portal session to hold. + _session: Option, } impl Drop for LoopGuard { @@ -646,8 +720,7 @@ impl Drop for Dequeued<'_> { } struct CaptureLoop { - fd: OwnedFd, - node_id: u32, + capture: Capture, framerate: u32, chan: Arc, state: Rc>, @@ -693,12 +766,26 @@ fn fixate_modifier(param: &spa::pod::Pod, modifier: u64) -> Option> { .map(|serialized| serialized.0.into_inner()) } -/// Connect to the portal's PipeWire node and run until the stream ends, the +/// Connect to a portal's PipeWire remote, or to the session's own socket when +/// there is none. +fn connect(context: &pw::context::ContextRc, remote: Option) -> Result { + match remote { + Some(fd) => context.connect_fd_rc(fd, None), + None => context.connect_rc(None), + } +} + +/// Connect to the target PipeWire node and run until the stream ends, the /// consumer drops, or the format changes. fn run_loop(args: CaptureLoop) -> Result<(), Error> { let CaptureLoop { - fd, - node_id, + capture: Capture { + kind, + remote, + target, + label, + size, + }, framerate, chan, state, @@ -710,17 +797,23 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { let mainloop = pw::main_loop::MainLoopRc::new(None).map_err(|e| err("pipewire main loop", e))?; let context = pw::context::ContextRc::new(&mainloop, None).map_err(|e| err("pipewire context", e))?; - let core = context - .connect_fd_rc(fd, None) - .map_err(|e| err("pipewire connect", e))?; + let core = connect(&context, remote).map_err(|e| err("pipewire connect", e))?; + let node_id = match target { + Target::Node(id) => Some(id), + Target::Camera(name) => camera::resolve(&camera::scan(&mainloop, &core)?, name.as_deref())?, + }; + let (name, role) = match kind { + Kind::Screen => ("moq-screen", "Screen"), + Kind::Camera => ("moq-camera", "Camera"), + }; let stream = pw::stream::StreamRc::new( core, - "moq-screen", + name, pw::properties::properties! { *pw::keys::MEDIA_TYPE => "Video", *pw::keys::MEDIA_CATEGORY => "Capture", - *pw::keys::MEDIA_ROLE => "Screen", + *pw::keys::MEDIA_ROLE => role, }, ) .map_err(|e| err("pipewire stream", e))?; @@ -755,18 +848,20 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { pw::stream::StreamState::Error(_) | pw::stream::StreamState::Unconnected ); if done { - // The compositor ended the stream while our loop was live (the - // user hit "stop sharing", or the output went away). Forget the - // restore token so the demand-driven reopen asks again instead - // of silently resuming a grant the user just revoked. Our own - // teardown quits the loop before anything disconnects, so it - // never reaches this path. - *RESTORE_TOKEN.lock().unwrap() = None; + // The producer ended the stream while our loop was live (the + // user hit "stop sharing", or the output or camera went away). + // Forget the screen restore token so the demand-driven reopen + // asks again instead of silently resuming a grant the user just + // revoked. Our own teardown quits the loop before anything + // disconnects, so it never reaches this path. + if kind == Kind::Screen { + *RESTORE_TOKEN.lock().unwrap() = None; + } state.borrow_mut().terminal = Some(Error::SourceUnavailable(match &new { pw::stream::StreamState::Error(error) => format!("PipeWire stream failed: {error}"), - _ => "the selected screen is no longer available".to_string(), + _ => format!("the selected {} is no longer available", role.to_lowercase()), })); - tracing::debug!(state = ?new, "screen capture stream ended"); + tracing::debug!(?kind, state = ?new, "capture stream ended"); if let Some(mainloop) = mainloop.upgrade() { mainloop.quit(); } @@ -836,7 +931,7 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { tracing::warn!(width = size.width, height = size.height, "unusable capture size"); return; } - let color = match pipewire_color(state.format, width, height) { + let color = match pipewire_color(kind, state.format, width, height) { Ok(color) => color, Err(e) => { match state.geo_tx.take() { @@ -864,7 +959,7 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { width, height, framerate, - label: format!("pipewire:{node_id}"), + label: label.clone(), })); } else if format_requires_restart(state.geometry, state.color, width, height, color) { // The encoder's geometry and VUI are fixed when it opens. End the @@ -916,6 +1011,7 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { Ok(stride) if stride > 0 => stride, _ => match state.format.format() { VideoFormat::NV12 => state.format.size().width, + VideoFormat::YUY2 => state.format.size().width.saturating_mul(2), VideoFormat::BGRx | VideoFormat::BGRA | VideoFormat::RGBx | VideoFormat::RGBA => { state.format.size().width.saturating_mul(4) } @@ -1019,7 +1115,11 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { match DmaBuf::new(format, modifier, layout.width, layout.height, planes, color, inner) { Ok(frame) => { chan.push(Surface::DmaBuf(frame.clone())); - state.last = Some(Last::DmaBuf(frame)); + // Only the pacing timer reads `last`. A camera must not + // hold a leased buffer back from its producer's pool. + if kind == Kind::Screen { + state.last = Some(Last::DmaBuf(frame)); + } state.fresh = true; } Err(e) => tracing::warn!(error = %e, "invalid PipeWire DMA-BUF"), @@ -1060,12 +1160,14 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { match convert(state.format.format(), bytes.as_ref(), layout, color) { Ok(i420) => { chan.push(Surface::I420(i420.clone())); - state.last = Some(Last::I420(i420)); + if kind == Kind::Screen { + state.last = Some(Last::I420(i420)); + } state.fresh = true; } Err(e) => { // Persistent (bad format), not per-frame; stop rather than spam. - tracing::warn!(error = %e, "screen frame conversion failed; stopping capture"); + tracing::warn!(?kind, error = %e, "frame conversion failed; stopping capture"); state.terminal = Some(e); if let Some(mainloop) = mainloop.upgrade() { mainloop.quit(); @@ -1079,7 +1181,7 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { // DMA-BUF formats come first so PipeWire prefers them. Each has a matching // shared-memory offer without a modifier as the required fallback. - let offers = format_offers(framerate); + let offers = format_offers(kind.formats(), framerate, size); let mut params = offers .iter() .map(|offer| { @@ -1090,14 +1192,15 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { stream .connect( spa::utils::Direction::Input, - Some(node_id), + node_id, pw::stream::StreamFlags::AUTOCONNECT | pw::stream::StreamFlags::MAP_BUFFERS, &mut params, ) .map_err(|e| err("pipewire stream connect", e))?; // Re-emit the last frame when an interval passes without a fresh one, so a - // static screen still produces a steady stream for the encoder. + // static screen still produces a steady stream for the encoder. A camera + // never retains a `last` frame and never arms the timer. let timer = mainloop.loop_().add_timer({ let state = state.clone(); let chan = chan.clone(); @@ -1111,11 +1214,13 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { } } }); - let interval = Duration::from_micros(1_000_000 / framerate as u64); - timer - .update_timer(Some(interval), Some(interval)) - .into_result() - .map_err(|e| err("pipewire timer", e))?; + if kind == Kind::Screen { + let interval = Duration::from_micros(1_000_000 / framerate as u64); + timer + .update_timer(Some(interval), Some(interval)) + .into_result() + .map_err(|e| err("pipewire timer", e))?; + } // Quit when the Stream drops. let _quit = quit_rx.attach(mainloop.loop_(), { @@ -1181,6 +1286,7 @@ fn frame_data_size(format: VideoFormat, layout: FrameLayout) -> Option { let height = layout.height as usize; let row_size = match format { VideoFormat::NV12 => width, + VideoFormat::YUY2 => width.checked_mul(2)?, VideoFormat::BGRx | VideoFormat::BGRA | VideoFormat::RGBx | VideoFormat::RGBA => width.checked_mul(4)?, _ => return None, }; @@ -1194,6 +1300,9 @@ fn frame_data_size(format: VideoFormat, layout: FrameLayout) -> Option { .checked_sub(1)? .checked_mul(stride)? .checked_add(row_size), + // The packed 4:2:2 converter reads whole rows, including the last row's + // padding, so the frame has to span every stride it covers. + VideoFormat::YUY2 => height.checked_mul(stride), _ => height.checked_sub(1)?.checked_mul(stride)?.checked_add(row_size), } } @@ -1230,16 +1339,17 @@ fn replace_video_format( Ok(result) } -/// Preserve the color description that names NV12 samples. Unknown fields use +/// Preserve the color description that names YUV samples. Unknown fields use /// the same size-based, limited-range fallback as the encoder. Reject matrices /// the crate cannot represent rather than writing a false 601/709 VUI. -fn pipewire_color(format: VideoInfoRaw, width: u32, height: u32) -> Result, Error> { - if format.format() != VideoFormat::NV12 { +fn pipewire_color(kind: Kind, format: VideoInfoRaw, width: u32, height: u32) -> Result, Error> { + if !matches!(format.format(), VideoFormat::NV12 | VideoFormat::YUY2) { return Ok(None); } let size = Size::new(width, height); let color = color_from_pipewire(format.color_range(), format.color_matrix(), size)?; validate_pipewire_description( + kind, color.unwrap_or_else(|| Color::infer(size)), format.color_primaries(), format.transfer_function(), @@ -1257,7 +1367,7 @@ fn color_from_pipewire(range: u32, matrix: u32, size: Size) -> Result false, _ => { return Err(Error::Codec(anyhow::anyhow!( - "unsupported PipeWire NV12 color range {range}" + "unsupported PipeWire YUV color range {range}" ))); } }; @@ -1269,7 +1379,7 @@ fn color_from_pipewire(range: u32, matrix: u32, size: Size) -> Result false, _ => { return Err(Error::Codec(anyhow::anyhow!( - "unsupported PipeWire NV12 color matrix {matrix}" + "unsupported PipeWire YUV color matrix {matrix}" ))); } }; @@ -1293,14 +1403,25 @@ fn color_from_pipewire(range: u32, matrix: u32, size: Size) -> Result Result<(), Error> { +fn validate_pipewire_description(kind: Kind, color: Color, primaries: u32, transfer: u32) -> Result<(), Error> { let expected_primaries = match color { Color::Bt601Limited | Color::Bt601Full => spa::sys::SPA_VIDEO_COLOR_PRIMARIES_SMPTE170M, Color::Bt709Limited | Color::Bt709Full => spa::sys::SPA_VIDEO_COLOR_PRIMARIES_BT709, }; - if primaries != spa::sys::SPA_VIDEO_COLOR_PRIMARIES_UNKNOWN && primaries != expected_primaries { + // UVC webcams describe their YUY2 as a BT.601 matrix with BT.709 primaries + // (an integrated USB webcam does, through spa-v4l2), a pairing `Color` + // cannot name. The matrix and range decide how the samples decode, + // so a camera keeps them and accepts either SDR primaries. + let primaries_match = primaries == spa::sys::SPA_VIDEO_COLOR_PRIMARIES_UNKNOWN + || primaries == expected_primaries + || (kind == Kind::Camera + && matches!( + primaries, + spa::sys::SPA_VIDEO_COLOR_PRIMARIES_SMPTE170M | spa::sys::SPA_VIDEO_COLOR_PRIMARIES_BT709 + )); + if !primaries_match { return Err(Error::Codec(anyhow::anyhow!( - "PipeWire NV12 primaries {primaries} do not match the negotiated matrix" + "PipeWire YUV primaries {primaries} do not match the negotiated matrix" ))); } if !matches!( @@ -1311,7 +1432,7 @@ fn validate_pipewire_description(color: Color, primaries: u32, transfer: u32) -> | SPA_VIDEO_TRANSFER_BT2020_10 ) { return Err(Error::Codec(anyhow::anyhow!( - "unsupported PipeWire NV12 transfer function {transfer}" + "unsupported PipeWire YUV transfer function {transfer}" ))); } Ok(()) @@ -1327,11 +1448,14 @@ fn format_requires_restart( geometry.is_some_and(|geometry| geometry != (width, height) || color != next_color) } -/// Convert one strided screen frame to tightly-packed I420. +/// Convert one strided frame to tightly-packed I420. fn convert(format: VideoFormat, bytes: &[u8], layout: FrameLayout, color: Option) -> Result { match format { - VideoFormat::NV12 => { - let frame = nv12_to_i420(bytes, layout)?; + VideoFormat::NV12 | VideoFormat::YUY2 => { + let frame = match format { + VideoFormat::NV12 => nv12_to_i420(bytes, layout)?, + _ => I420::from_yuyv(bytes, layout.stride, crate::Size::new(layout.width, layout.height))?, + }; Ok(match color { Some(color) => frame.with_color(color), None => frame, @@ -1361,7 +1485,7 @@ fn drm_format(format: VideoFormat) -> Option { } } -const PIPEWIRE_FORMATS: [VideoFormat; 5] = [ +const SCREEN_FORMATS: [VideoFormat; 5] = [ VideoFormat::BGRx, VideoFormat::BGRA, VideoFormat::RGBx, @@ -1369,8 +1493,15 @@ const PIPEWIRE_FORMATS: [VideoFormat; 5] = [ VideoFormat::NV12, ]; -/// Serialize one `EnumFormat` pod for a concrete pixel format. -fn format_offer(framerate: u32, format: VideoFormat, dmabuf: bool) -> Vec { +/// Among the shared-memory offers YUY2 comes first: it is one plane on every +/// producer, and the buffer offer asks for one data block, while libcamera may +/// deliver NV12 as two. NV12 still leads as a DMA-BUF, which only matches a +/// producer that advertises a modifier. MJPEG is not offered yet. +const CAMERA_FORMATS: [VideoFormat; 2] = [VideoFormat::YUY2, VideoFormat::NV12]; + +/// Serialize one `EnumFormat` pod for a concrete pixel format, accepting any +/// size and preferring `size` when there is one. +fn format_offer(framerate: u32, format: VideoFormat, dmabuf: bool, size: Option) -> Vec { let mut obj = spa::pod::object!( spa::utils::SpaTypes::ObjectParamFormat, spa::param::ParamType::EnumFormat, @@ -1385,21 +1516,6 @@ fn format_offer(framerate: u32, format: VideoFormat, dmabuf: bool) -> Vec { spa::param::format::MediaSubtype::Raw ), spa::pod::property!(spa::param::format::FormatProperties::VideoFormat, Id, format), - spa::pod::property!( - spa::param::format::FormatProperties::VideoSize, - Choice, - Range, - Rectangle, - spa::utils::Rectangle { - width: 1920, - height: 1080 - }, - spa::utils::Rectangle { width: 1, height: 1 }, - spa::utils::Rectangle { - width: 8192, - height: 8192 - } - ), spa::pod::property!( spa::param::format::FormatProperties::VideoFramerate, Choice, @@ -1413,6 +1529,28 @@ fn format_offer(framerate: u32, format: VideoFormat, dmabuf: bool) -> Vec { spa::utils::Fraction { num: 1000, denom: 1 } ), ); + if let Some(size) = size { + obj.properties.insert( + 3, + spa::pod::Property::new( + spa::param::format::FormatProperties::VideoSize.as_raw(), + spa::pod::Value::Choice(spa::pod::ChoiceValue::Rectangle(spa::utils::Choice( + spa::utils::ChoiceFlags::empty(), + spa::utils::ChoiceEnum::Range { + default: spa::utils::Rectangle { + width: size.width, + height: size.height, + }, + min: spa::utils::Rectangle { width: 1, height: 1 }, + max: spa::utils::Rectangle { + width: 8192, + height: 8192, + }, + }, + ))), + ), + ); + } if dmabuf { obj.properties.push(spa::pod::Property { key: spa::param::format::FormatProperties::VideoModifier.as_raw(), @@ -1434,17 +1572,17 @@ fn format_offer(framerate: u32, format: VideoFormat, dmabuf: bool) -> Vec { .into_inner() } -/// Serialize DMA-BUF formats first and shared-memory fallbacks second. -fn format_offers(framerate: u32) -> Vec> { - PIPEWIRE_FORMATS - .into_iter() - .map(|format| format_offer(framerate, format, true)) - .chain( - PIPEWIRE_FORMATS - .into_iter() - .map(|format| format_offer(framerate, format, false)), - ) - .collect() +/// Serialize DMA-BUF formats first and shared-memory fallbacks second. Only a +/// format with a DRM fourcc to import as gets a DMA-BUF offer. +fn format_offers(formats: &[VideoFormat], framerate: u32, size: Option) -> Vec> { + let dmabuf = formats + .iter() + .filter(|format| drm_format(**format).is_some()) + .map(|&format| format_offer(framerate, format, true, size)); + let shared = formats + .iter() + .map(|&format| format_offer(framerate, format, false, size)); + dmabuf.chain(shared).collect() } /// Serialize `SPA_PARAM_Buffers` for the negotiated memory representation. @@ -1496,8 +1634,8 @@ mod tests { /// The serialized format offer must parse back as a valid pod. #[test] fn format_offer_is_valid_pod() { - let offers = format_offers(30); - assert_eq!(offers.len(), PIPEWIRE_FORMATS.len() * 2); + let offers = format_offers(&SCREEN_FORMATS, 30, Some(Size::new(1920, 1080))); + assert_eq!(offers.len(), SCREEN_FORMATS.len() * 2); for (index, bytes) in offers.iter().enumerate() { let (remaining, value) = spa::pod::deserialize::PodDeserializer::deserialize_any_from(bytes) .expect("format offer did not round-trip"); @@ -1506,13 +1644,13 @@ mod tests { panic!("format offer is not an object"); }; let property = |key| object.properties.iter().find(|property| property.key == key); - let format = PIPEWIRE_FORMATS[index % PIPEWIRE_FORMATS.len()]; + let format = SCREEN_FORMATS[index % SCREEN_FORMATS.len()]; assert_eq!( property(spa::param::format::FormatProperties::VideoFormat.as_raw()).map(|p| &p.value), Some(&spa::pod::Value::Id(spa::utils::Id(format.as_raw()))) ); let modifier = property(spa::param::format::FormatProperties::VideoModifier.as_raw()); - if index < PIPEWIRE_FORMATS.len() { + if index < SCREEN_FORMATS.len() { let modifier = modifier.expect("DMA-BUF offer has no modifier"); assert_eq!( modifier.flags.bits(), @@ -1536,13 +1674,14 @@ mod tests { #[test] fn negotiated_modifier_must_be_present_and_fixed() { - let shared = format_offer(30, VideoFormat::BGRx, false); + let size = Some(Size::new(1920, 1080)); + let shared = format_offer(30, VideoFormat::BGRx, false, size); let shared = spa::pod::Pod::from_bytes(&shared).unwrap(); let mut format = VideoInfoRaw::default(); format.parse(shared).unwrap(); assert_eq!(negotiated_memory(shared, format), Some(NegotiatedMemory::SharedMemory)); - let offered = format_offer(30, VideoFormat::BGRx, true); + let offered = format_offer(30, VideoFormat::BGRx, true, size); let offered = spa::pod::Pod::from_bytes(&offered).unwrap(); format.parse(offered).unwrap(); assert_eq!(negotiated_memory(offered, format), Some(NegotiatedMemory::Fixating)); @@ -1553,6 +1692,82 @@ mod tests { assert_eq!(negotiated_memory(fixed, format), Some(NegotiatedMemory::DmaBuf(0))); } + /// A camera offer leaves the size to the producer, and never offers YUY2 + /// as a DMA-BUF, which has no DRM fourcc here. + #[test] + fn camera_offers_leave_the_size_to_the_producer() { + let decoded: Vec<_> = format_offers(&CAMERA_FORMATS, 30, None) + .iter() + .map(|bytes| { + let (_, value) = spa::pod::deserialize::PodDeserializer::deserialize_any_from(bytes).unwrap(); + let spa::pod::Value::Object(object) = value else { + panic!("format offer is not an object"); + }; + let property = |key: spa::param::format::FormatProperties| { + object + .properties + .iter() + .find(|property| property.key == key.as_raw()) + .map(|property| property.value.clone()) + }; + let Some(spa::pod::Value::Id(format)) = property(spa::param::format::FormatProperties::VideoFormat) + else { + panic!("format is not an id"); + }; + assert_eq!(property(spa::param::format::FormatProperties::VideoSize), None); + let dmabuf = property(spa::param::format::FormatProperties::VideoModifier).is_some(); + (VideoFormat::from_raw(format.0), dmabuf) + }) + .collect(); + assert_eq!( + decoded, + [ + (VideoFormat::NV12, true), + (VideoFormat::YUY2, false), + (VideoFormat::NV12, false), + ] + ); + } + + #[test] + fn yuy2_converts_with_the_negotiated_color() { + let layout = FrameLayout { + stride: 10, + width: 4, + height: 2, + source_height: 2, + }; + assert_eq!(frame_data_size(VideoFormat::YUY2, layout), Some(20)); + // Y0 U Y1 V per pixel pair, then two bytes of row padding. + let data = [ + 1, 100, 2, 200, 3, 101, 4, 201, 99, 99, // row 0 + 5, 100, 6, 200, 7, 101, 8, 201, 99, 99, // row 1 + ]; + let frame = convert(VideoFormat::YUY2, &data, layout, Some(Color::Bt601Limited)).unwrap(); + assert_eq!(frame.y(), &[1, 2, 3, 4, 5, 6, 7, 8]); + assert_eq!(frame.color(), Some(Color::Bt601Limited)); + } + + /// An integrated USB webcam's description through spa-v4l2: a camera + /// accepts it, and a screen still rejects the same mismatch. + #[test] + fn camera_accepts_bt601_matrix_with_bt709_primaries() { + let mut format = VideoInfoRaw::default(); + format.set_format(VideoFormat::YUY2); + format.set_color_range(spa::sys::SPA_VIDEO_COLOR_RANGE_16_235); + format.set_color_matrix(spa::sys::SPA_VIDEO_COLOR_MATRIX_BT601); + format.set_color_primaries(spa::sys::SPA_VIDEO_COLOR_PRIMARIES_BT709); + format.set_transfer_function(spa::sys::SPA_VIDEO_TRANSFER_BT709); + assert_eq!( + pipewire_color(Kind::Camera, format, 640, 480).unwrap(), + Some(Color::Bt601Limited) + ); + assert!(pipewire_color(Kind::Screen, format, 640, 480).is_err()); + + format.set_color_primaries(spa::sys::SPA_VIDEO_COLOR_PRIMARIES_BT2020); + assert!(pipewire_color(Kind::Camera, format, 640, 480).is_err()); + } + #[test] fn buffer_offer_is_valid_pod() { let dma_buf = 1 << DataType::DmaBuf.as_raw(); @@ -1728,11 +1943,11 @@ mod tests { format.set_color_matrix(spa::sys::SPA_VIDEO_COLOR_MATRIX_BT709); format.set_color_primaries(spa::sys::SPA_VIDEO_COLOR_PRIMARIES_BT2020); format.set_transfer_function(spa::sys::SPA_VIDEO_TRANSFER_BT709); - assert!(pipewire_color(format, 1920, 1080).is_err()); + assert!(pipewire_color(Kind::Screen, format, 1920, 1080).is_err()); format.set_color_range(spa::sys::SPA_VIDEO_COLOR_RANGE_UNKNOWN); format.set_color_matrix(spa::sys::SPA_VIDEO_COLOR_MATRIX_UNKNOWN); format.set_transfer_function(spa::sys::SPA_VIDEO_TRANSFER_SMPTE2084); - assert!(pipewire_color(format, 1920, 1080).is_err()); + assert!(pipewire_color(Kind::Screen, format, 1920, 1080).is_err()); let layout = FrameLayout { stride: 4, diff --git a/rs/moq-video/src/capture/pipewire/camera.rs b/rs/moq-video/src/capture/pipewire/camera.rs new file mode 100644 index 00000000000..7e5b945e9f6 --- /dev/null +++ b/rs/moq-video/src/capture/pipewire/camera.rs @@ -0,0 +1,391 @@ +//! PipeWire camera capture: the `Video/Source` nodes with the `Camera` role, +//! which is how PipeWire presents V4L2 webcams (spa-v4l2) and libcamera sensors +//! such as a Raspberry Pi CSI camera (spa-libcamera). A CSI camera has no V4L2 +//! node that yields processed frames, so on a Pi this is the only way to open it +//! as a camera. +//! +//! Outside a sandbox this connects to the session's PipeWire socket, which also +//! works headless. A sandbox cannot reach that socket, so there the camera portal +//! (`org.freedesktop.portal.Camera`) grants access and hands back a PipeWire +//! remote that exposes the cameras. Streaming reuses the parent module's loop. + +use std::cell::RefCell; +use std::os::fd::OwnedFd; +use std::rc::Rc; +use std::time::Duration; + +use pipewire as pw; + +use super::{Capture, Kind, Target, err}; +use crate::Error; +use crate::capture::{Camera, Config, PIPEWIRE, Stream}; + +/// How long the PipeWire daemon may take to list its objects. A connected +/// daemon answers in milliseconds, so running out means it is stuck. +const SCAN_TIMEOUT: Duration = Duration::from_secs(5); + +/// List the PipeWire cameras as [`Camera`]s with `pipewire:` ids. +pub(in crate::capture) async fn cameras() -> Result, Error> { + let remote = remote().await?; + let sandboxed = remote.is_some(); + let nodes = crate::capture::blocking(move || { + pw::init(); + let mainloop = pw::main_loop::MainLoopRc::new(None).map_err(|e| err("pipewire main loop", e))?; + let context = pw::context::ContextRc::new(&mainloop, None).map_err(|e| err("pipewire context", e))?; + let core = match super::connect(&context, remote) { + Ok(core) => core, + // PipeWire is optional on the host, and a host without it has no + // PipeWire cameras. A portal remote that fails to connect is broken. + Err(error) if !sandboxed => { + tracing::debug!(%error, "no PipeWire session to list cameras from"); + return Ok(Vec::new()); + } + Err(error) => return Err(err("pipewire connect", error)), + }; + scan(&mainloop, &core) + }) + .await?; + + Ok(nodes + .into_iter() + .map(|node| Camera { + id: format!("{PIPEWIRE}:{}", node.name), + name: node.description, + }) + .collect()) +} + +/// Open the camera node named `node`, or the session manager's default camera. +/// +/// The camera streams its first raw mode that this backend converts, so +/// `config.width` and `config.height` are not applied yet. The parent module's +/// `Capture::size` says why the offers carry no size. +pub(in crate::capture) async fn open(config: &Config, node: Option<&str>) -> Result { + let remote = remote().await?; + let label = match node { + Some(node) => format!("{PIPEWIRE}:{node}"), + None => PIPEWIRE.to_string(), + }; + super::start( + config, + Capture { + kind: Kind::Camera, + remote, + target: Target::Camera(node.map(str::to_string)), + label, + size: None, + }, + None, + ) + .await +} + +/// The camera portal's PipeWire remote inside a sandbox, or `None` to use the +/// session's socket. +async fn remote() -> Result, Error> { + if !ashpd::is_sandboxed() { + return Ok(None); + } + let portal = ashpd::desktop::camera::Camera::new() + .await + .map_err(|e| err("camera portal", e))?; + if !portal.is_present().await.map_err(|e| err("camera portal", e))? { + return Err(Error::SourceUnavailable( + "the camera portal reports no camera".to_string(), + )); + } + // The portal asks the user unless the sandbox's permission store already + // holds a grant, so this blocks on the user the first time. + portal + .request_access(Default::default()) + .await + .map_err(|e| err("camera portal access", e))? + .response() + .map_err(|e| Error::PermissionDenied(format!("camera access request: {e}")))?; + let fd = portal + .open_pipe_wire_remote(Default::default()) + .await + .map_err(|e| err("camera portal pipewire remote", e))?; + Ok(Some(fd)) +} + +/// A PipeWire camera node. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct Node { + /// The registry id to link to. Only valid for this connection's lifetime. + id: u32, + /// `node.name`, which stays the same across restarts and reboots. + name: String, + /// `node.description`, or the best name the node has short of it. + description: String, +} + +impl Node { + /// Build a node from a registry global's properties, or `None` when it is + /// not a camera. The criterion is the one xdg-desktop-portal applies when it + /// decides which nodes a sandboxed app may see. + fn from_props<'a>(id: u32, prop: impl Fn(&str) -> Option<&'a str>) -> Option { + if prop(*pw::keys::MEDIA_CLASS)? != "Video/Source" || prop(*pw::keys::MEDIA_ROLE)? != "Camera" { + return None; + } + let name = prop(*pw::keys::NODE_NAME).filter(|name| !name.is_empty())?; + let description = prop(*pw::keys::NODE_DESCRIPTION) + .or_else(|| prop(*pw::keys::NODE_NICK)) + .filter(|description| !description.is_empty()) + .unwrap_or(name); + Some(Self { + id, + name: name.to_string(), + description: description.to_string(), + }) + } +} + +/// List the camera nodes on `core`, running `mainloop` until the daemon has +/// sent every registry global. +pub(super) fn scan(mainloop: &pw::main_loop::MainLoopRc, core: &pw::core::CoreRc) -> Result, Error> { + let registry = core.get_registry_rc().map_err(|e| err("pipewire registry", e))?; + let nodes = Rc::new(RefCell::new(Vec::new())); + let outcome: Rc>>> = Rc::new(RefCell::new(None)); + + // The daemon answers a sync only after every global it announced before it. + let pending = core.sync(0).map_err(|e| err("pipewire sync", e))?; + let _core = core + .add_listener_local() + .done({ + let outcome = outcome.clone(); + let mainloop = mainloop.downgrade(); + move |id, seq| { + if id == pw::core::PW_ID_CORE && seq == pending { + outcome.borrow_mut().get_or_insert(Ok(())); + if let Some(mainloop) = mainloop.upgrade() { + mainloop.quit(); + } + } + } + }) + .error({ + let outcome = outcome.clone(); + let mainloop = mainloop.downgrade(); + move |id, _, _, message| { + if id == pw::core::PW_ID_CORE { + outcome.borrow_mut().get_or_insert(Err(err("pipewire", message))); + if let Some(mainloop) = mainloop.upgrade() { + mainloop.quit(); + } + } + } + }) + .register(); + let _registry = registry + .add_listener_local() + .global({ + let nodes = nodes.clone(); + move |global| { + if global.type_ != pw::types::ObjectType::Node { + return; + } + let Some(props) = global.props else { return }; + if let Some(node) = Node::from_props(global.id, |key| props.get(key)) { + nodes.borrow_mut().push(node); + } + } + }) + .register(); + let timer = mainloop.loop_().add_timer({ + let mainloop = mainloop.downgrade(); + move |_| { + if let Some(mainloop) = mainloop.upgrade() { + mainloop.quit(); + } + } + }); + timer + .update_timer(Some(SCAN_TIMEOUT), None) + .into_result() + .map_err(|e| err("pipewire timer", e))?; + + mainloop.run(); + let outcome = outcome.borrow_mut().take(); + match outcome { + Some(Ok(())) => Ok(nodes.take()), + Some(Err(error)) => Err(error), + None => Err(Error::Codec(anyhow::anyhow!( + "PipeWire did not list its objects within {SCAN_TIMEOUT:?}" + ))), + } +} + +/// The node id to link to for `name`, or `None` to let the session manager +/// pick its default camera. Fails when the camera is not there, rather than +/// leaving the stream unlinked until the format wait runs out. +pub(super) fn resolve(nodes: &[Node], name: Option<&str>) -> Result, Error> { + let Some(name) = name else { + if nodes.is_empty() { + return Err(Error::SourceUnavailable("PipeWire has no camera".to_string())); + } + return Ok(None); + }; + match nodes.iter().find(|node| node.name == name) { + Some(node) => Ok(Some(node.id)), + None => { + let found: Vec<_> = nodes.iter().map(|node| node.name.as_str()).collect(); + Err(Error::SourceUnavailable(format!( + "no PipeWire camera named {name} (found: {})", + found.join(", ") + ))) + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + + fn node(props: &[(&str, &str)]) -> Option { + let props: HashMap<_, _> = props.iter().copied().collect(); + Node::from_props(62, |key| props.get(key).copied()) + } + + const WEBCAM: [(&str, &str); 4] = [ + ("media.class", "Video/Source"), + ("media.role", "Camera"), + ("node.name", "v4l2_input.pci-0000_00_14.0-usb-0_4_1.0"), + ("node.description", "Integrated Camera (V4L2)"), + ]; + + #[test] + fn camera_nodes_are_video_sources_with_the_camera_role() { + assert_eq!( + node(&WEBCAM), + Some(Node { + id: 62, + name: "v4l2_input.pci-0000_00_14.0-usb-0_4_1.0".to_string(), + description: "Integrated Camera (V4L2)".to_string(), + }) + ); + + let mut screen = WEBCAM; + screen[1] = ("media.role", "Screen"); + assert_eq!(node(&screen), None); + let mut sink = WEBCAM; + sink[0] = ("media.class", "Video/Sink"); + assert_eq!(node(&sink), None); + // The name is the id's only stable part, so a node without one is unusable. + assert_eq!(node(&WEBCAM[..2]), None); + } + + #[test] + fn a_camera_without_a_description_falls_back_to_its_nick_then_name() { + let libcamera = [ + ("media.class", "Video/Source"), + ("media.role", "Camera"), + ("node.name", "libcamera_input./base/soc/i2c0mux/i2c@1/imx708@1a"), + ("node.nick", "imx708"), + ]; + assert_eq!(node(&libcamera).unwrap().description, "imx708"); + assert_eq!( + node(&libcamera[..3]).unwrap().description, + "libcamera_input./base/soc/i2c0mux/i2c@1/imx708@1a" + ); + } + + #[test] + fn resolve_finds_the_named_camera_or_fails() { + let nodes = [node(&WEBCAM).unwrap()]; + assert_eq!( + resolve(&nodes, Some("v4l2_input.pci-0000_00_14.0-usb-0_4_1.0")).unwrap(), + Some(62) + ); + assert_eq!(resolve(&nodes, None).unwrap(), None); + let error = resolve(&nodes, Some("missing")).unwrap_err().to_string(); + assert!(error.contains("v4l2_input.pci-0000_00_14.0-usb-0_4_1.0"), "{error}"); + assert!(matches!(resolve(&[], None), Err(Error::SourceUnavailable(_)))); + } + + /// Lists the PipeWire cameras over the session socket. Needs no camera and + /// never turns one on, so a host without PipeWire just lists nothing. + #[tokio::test] + async fn lists_cameras_over_the_session_socket() { + if ashpd::is_sandboxed() { + eprintln!("skipping: listing inside a sandbox would ask the camera portal"); + return; + } + let cameras = cameras().await.expect("listing PipeWire cameras"); + for camera in &cameras { + assert!(camera.id.starts_with("pipewire:"), "{}", camera.id); + assert!(!camera.name.is_empty()); + } + eprintln!("PipeWire cameras: {cameras:?}"); + } + + /// Captures a few frames from each PipeWire camera over the session socket. + /// Ignored because it turns the cameras on: + /// `cargo test -p moq-video --features pipewire pipewire_camera -- --ignored`. + #[tokio::test] + #[ignore = "turns on every PipeWire camera on the host"] + async fn pipewire_camera_captures_frames() { + if ashpd::is_sandboxed() { + eprintln!("skipping: capturing inside a sandbox would ask the camera portal"); + return; + } + let cameras = cameras().await.expect("listing PipeWire cameras"); + if cameras.is_empty() { + eprintln!("skipping: no PipeWire camera"); + return; + } + + let mut captured = 0; + for camera in cameras { + let config = Config { + source: camera.source(), + ..Default::default() + }; + let mut stream = match crate::capture::open(&config).await { + Ok(stream) => stream, + // An IR camera offers only GRAY8, which this backend does not convert. + Err(error) => { + eprintln!("{}: not captured: {error}", camera.id); + continue; + } + }; + assert_eq!(stream.label(), camera.id); + assert!(stream.width() >= 2 && stream.width().is_multiple_of(2), "bad width"); + assert!(stream.height() >= 2 && stream.height().is_multiple_of(2), "bad height"); + for i in 0..5 { + let frame = stream + .read() + .await + .unwrap_or_else(|error| panic!("{}: read frame {i}: {error}", camera.id)) + .unwrap_or_else(|| panic!("{}: no frame {i}", camera.id)); + assert_eq!(frame.surface.width(), stream.width()); + assert_eq!(frame.surface.height(), stream.height()); + } + eprintln!( + "{}: captured 5 frames at {}x{}, {:?} fps, color {:?}", + camera.id, + stream.width(), + stream.height(), + stream.framerate(), + stream.color() + ); + captured += 1; + } + assert!(captured > 0, "no PipeWire camera could be captured"); + + // `pipewire` alone leaves the choice to the session manager. + let config = Config { + source: crate::capture::Source::Camera(Some(PIPEWIRE.to_string())), + ..Default::default() + }; + let mut stream = crate::capture::open(&config).await.expect("default PipeWire camera"); + assert_eq!(stream.label(), PIPEWIRE); + stream + .read() + .await + .expect("read") + .expect("a frame from the default camera"); + } +} diff --git a/rs/moq-video/src/frame.rs b/rs/moq-video/src/frame.rs index a2915c09f42..05cf9045ba6 100644 --- a/rs/moq-video/src/frame.rs +++ b/rs/moq-video/src/frame.rs @@ -866,7 +866,7 @@ impl I420 { /// Convert packed YUYV (YUV 4:2:2, `stride` bytes per row) to I420. A chroma /// resample (4:2:2 -> 4:2:0), no color-space conversion. Used for the raw - /// V4L2 capture path (Linux). + /// V4L2 and PipeWire camera capture paths (Linux). #[cfg(all(target_os = "linux", feature = "capture"))] pub(crate) fn from_yuyv(yuyv: &[u8], stride: u32, size: Size) -> Result { use yuv::{YuvPackedImage, yuyv422_to_yuv420}; @@ -883,7 +883,7 @@ impl I420 { yuyv422_to_yuv420(&mut planar, &packed) .map_err(|e| Error::Codec(anyhow::anyhow!("yuyv422_to_yuv420 failed for {width}x{height}: {e}")))?; // A chroma resample, not a color conversion: these samples are in - // whatever space the camera produced, which nothing here names. + // whatever space the camera produced, which the caller names if it can. Self::pack(&planar, size, None) } From 9058cfa61ff5bc8c61849a77775737e02f8fc6ca Mon Sep 17 00:00:00 2001 From: Frando Date: Thu, 24 Sep 2026 10:09:04 +0200 Subject: [PATCH 2/4] feat(moq-video): choose the PipeWire camera mode like V4L2 does PipeWire cameras streamed the first raw mode the producer listed, so `Config::width` and `height` had no effect, a webcam that offers YUY2 only at VGA sizes could not go higher, and `camera_modes` had nothing to say about them. Offering a size range to steer negotiation does not work: spa-v4l2 accepts a size its format lacks and then fails the link. The camera's own `EnumFormat` list is now read before the stream connects, the mode is picked by the selection the V4L2 backend uses (moved to `capture::mode` so both share it), and the stream offers exactly that mode. MJPEG modes are candidates too, decoded through the V4L2 backend's JPEG path, now `I420::from_mjpeg`. `camera_modes` lists the same modes for `pipewire:` ids, and `pipewire` alone resolves to the camera with the highest session priority so it has a list to read. Co-Authored-By: Claude Opus 5.5 (1M context) --- doc/lib/rs/moq-video.md | 17 +- rs/moq-video/src/capture/mod.rs | 40 +- rs/moq-video/src/capture/mode.rs | 166 ++++ rs/moq-video/src/capture/pipewire.rs | 220 +++--- rs/moq-video/src/capture/pipewire/camera.rs | 802 +++++++++++++++++--- rs/moq-video/src/capture/v4l2.rs | 118 +-- rs/moq-video/src/frame.rs | 29 +- 7 files changed, 1036 insertions(+), 356 deletions(-) create mode 100644 rs/moq-video/src/capture/mode.rs diff --git a/doc/lib/rs/moq-video.md b/doc/lib/rs/moq-video.md index b986760acc5..1c5ab813366 100644 --- a/doc/lib/rs/moq-video.md +++ b/doc/lib/rs/moq-video.md @@ -41,18 +41,19 @@ are returned rather than treated as an empty list. Other platforms return With `pipewire` enabled, `capture::cameras` also lists PipeWire camera nodes as `pipewire:` after the V4L2 devices, and `pipewire` alone opens the -session manager's default camera. That reaches cameras V4L2 cannot: a Raspberry -Pi CSI camera behind libcamera, and any camera from inside a Flatpak or Snap -sandbox, where the default camera comes through the xdg-desktop-portal Camera -interface. These cameras stream their first raw YUY2 or NV12 mode: MJPEG-only -modes are not offered, `Config::width` and `height` are not applied yet, and -`camera_modes` returns `Error::Unsupported` for them. +camera with the highest session priority, the session manager's default. That +reaches cameras V4L2 cannot: a Raspberry Pi CSI camera behind libcamera, and any +camera from inside a Flatpak or Snap sandbox, where the default camera comes +through the xdg-desktop-portal Camera interface. The mode is chosen from the +node's own format list by the same rules as V4L2 (below) and offered exactly, +across YUY2, NV12, RGB, and MJPEG, and `camera_modes` lists the same modes. `capture::Config::framerate` is an `Option` request in the same exact type; the stream reports the rate the device accepted, or `None` when the driver reported none. -V4L2 chooses the closest geometry, then the format whose accepted rate is -nearest the request, then the cheaper conversion when both match equally well. +V4L2 and PipeWire cameras choose the closest geometry, then the format whose +accepted rate is nearest the request, then the cheaper conversion when both +match equally well. ```rust let mut video = moq_video::decode::Consumer::new(&broadcast, &rendition, "video", Default::default()).await?; diff --git a/rs/moq-video/src/capture/mod.rs b/rs/moq-video/src/capture/mod.rs index 969745572ea..25fe9784c50 100644 --- a/rs/moq-video/src/capture/mod.rs +++ b/rs/moq-video/src/capture/mod.rs @@ -39,6 +39,9 @@ mod surface; // Native V4L2 camera capture on Linux. #[cfg(target_os = "linux")] mod v4l2; +// The mode choice V4L2 and PipeWire cameras share. +#[cfg(target_os = "linux")] +mod mode; // Native X11 display and window capture, including the portal fallback. #[cfg(target_os = "linux")] mod x11; @@ -84,11 +87,12 @@ pub enum Source { /// Windows. Bare numeric indices remain accepted on Linux and Windows. /// /// With the `pipewire` feature, Linux also lists PipeWire camera nodes as - /// `pipewire:`, and `pipewire` alone opens the session manager's - /// default camera. The id picks the backend: a `/dev/videoN` path always opens - /// V4L2, and a `pipewire` id always opens PipeWire. `None` opens - /// `/dev/video0`, except inside a sandbox (Flatpak or Snap), where it opens - /// PipeWire's default camera through the camera portal. + /// `pipewire:`, and `pipewire` alone opens the camera with the + /// highest session priority, which the session manager treats as default. + /// The id picks the backend: a `/dev/videoN` path always opens V4L2, and a + /// `pipewire` id always opens PipeWire. `None` opens `/dev/video0`, except + /// inside a sandbox (Flatpak or Snap), where it opens PipeWire's default + /// camera through the camera portal. Camera(Option), /// A whole display. `None` opens the main display. @@ -486,10 +490,10 @@ pub async fn cameras() -> Result, Error> { /// are ones [`open`] could negotiate rather than everything the driver /// advertises. /// -/// Linux V4L2 only; PipeWire cameras and other platforms return -/// [`Error::Unsupported`]. +/// Linux only (V4L2, and PipeWire cameras with the `pipewire` feature); other +/// platforms return [`Error::Unsupported`]. /// Rates are exact device reports and [`Config::framerate`] accepts the same type. -/// V4L2 prefers the closest geometry, then the accepted rate +/// Both Linux backends prefer the closest geometry, then the accepted rate /// nearest that request, then the cheaper conversion format. /// /// An empty list is not a failure: it means the driver enumerated nothing this @@ -498,11 +502,18 @@ pub async fn camera_modes(camera: Option<&str>) -> Result, Error> { let _ = camera; #[cfg(target_os = "linux")] { - let camera = match LinuxCamera::select(camera, sandboxed()) { - LinuxCamera::V4l2(camera) => camera.map(str::to_string), - LinuxCamera::PipeWire(_) => return Err(Error::Unsupported("listing PipeWire camera modes".to_string())), - }; - blocking(move || v4l2::modes(camera.as_deref())).await + match LinuxCamera::select(camera, sandboxed()) { + LinuxCamera::V4l2(camera) => { + let camera = camera.map(str::to_string); + blocking(move || v4l2::modes(camera.as_deref())).await + } + #[cfg(feature = "pipewire")] + LinuxCamera::PipeWire(node) => pipewire::camera::modes(node).await, + #[cfg(not(feature = "pipewire"))] + LinuxCamera::PipeWire(_) => Err(Error::Unsupported( + "PipeWire camera modes without the `pipewire` feature".to_string(), + )), + } } #[cfg(not(target_os = "linux"))] { @@ -576,7 +587,8 @@ const PIPEWIRE: &str = "pipewire"; enum LinuxCamera<'a> { /// A V4L2 device, by path or index. `None` opens `/dev/video0`. V4l2(Option<&'a str>), - /// A PipeWire camera, by `node.name`. `None` lets the session manager pick. + /// A PipeWire camera, by `node.name`. `None` picks the camera with the + /// highest session priority, the one the session manager links by default. PipeWire(Option<&'a str>), } diff --git a/rs/moq-video/src/capture/mode.rs b/rs/moq-video/src/capture/mode.rs new file mode 100644 index 00000000000..7c8af3d62bd --- /dev/null +++ b/rs/moq-video/src/capture/mode.rs @@ -0,0 +1,166 @@ +//! Mode choice and listing shared by the Linux camera backends (V4L2 and +//! PipeWire), so both land on the same mode for the same request. + +use std::collections::{BTreeMap, BTreeSet}; + +use super::{Mode, Rate}; +use crate::Size; + +/// What the caller asked the camera for. +#[derive(Clone, Copy, Debug)] +pub(super) struct Request { + pub size: Size, + pub framerate: Option, +} + +/// One mode a camera can stream, as the selection weighs it. +#[derive(Clone, Copy, Debug)] +pub(super) struct Candidate { + pub size: Size, + pub framerate: Option, + /// What converting the format to I420 costs: lower is cheaper. + pub cost: u8, +} + +/// Pick the candidate nearest `want`: geometry first, then the rate nearest the +/// request, then the cheaper conversion. A size that cannot feed I420 is never +/// picked, and among equals the earliest candidate wins. +pub(super) fn nearest( + candidates: impl IntoIterator, + want: Request, + candidate: impl Fn(&T) -> Candidate, +) -> Option { + candidates + .into_iter() + .filter(|item| candidate(item).size.validate("camera resolution").is_ok()) + .min_by(|left, right| { + let (left, right) = (candidate(left), candidate(right)); + distance(left.size, want.size) + .cmp(&distance(right.size, want.size)) + .then_with(|| rate_distance(left.framerate, right.framerate, want.framerate)) + .then_with(|| left.cost.cmp(&right.cost)) + }) +} + +fn rate_distance(left: Option, right: Option, want: Option) -> std::cmp::Ordering { + let Some(want) = want else { + return std::cmp::Ordering::Equal; + }; + match (left, right) { + (Some(left), Some(right)) => (left.as_f64() - want.as_f64()) + .abs() + .total_cmp(&(right.as_f64() - want.as_f64()).abs()), + (Some(_), None) => std::cmp::Ordering::Less, + (None, Some(_)) => std::cmp::Ordering::Greater, + (None, None) => std::cmp::Ordering::Equal, + } +} + +/// How far a mode lands from the requested geometry, summed over both +/// dimensions. Zero is an exact match. +fn distance(size: Size, want: Size) -> u64 { + u64::from(size.width.abs_diff(want.width)) + u64::from(size.height.abs_diff(want.height)) +} + +/// Merge the sizes and rates a camera reports into [`Mode`]s, largest first +/// with each mode's rates highest first. The encoder sees I420 whatever format +/// carried a mode, so the format is not something a caller can act on. +pub(super) fn modes(reported: impl IntoIterator)>) -> Vec { + let mut sizes: BTreeMap<(u32, u32), BTreeSet> = BTreeMap::new(); + for (size, rates) in reported { + sizes.entry((size.width, size.height)).or_default().extend(rates); + } + let mut modes: Vec = sizes + .into_iter() + .map(|((width, height), framerates)| Mode { + width, + height, + // Highest first, so the rate a caller most often wants is the one it + // reads without scanning. + framerates: framerates.into_iter().rev().collect(), + }) + .collect(); + modes.sort_by_key(|mode| std::cmp::Reverse(u64::from(mode.width) * u64::from(mode.height))); + modes +} + +#[cfg(test)] +mod tests { + use super::*; + + fn rate(fps: u32) -> Rate { + Rate::new(fps, 1).unwrap() + } + + #[test] + fn rate_scoring_preserves_fractional_precision_and_handles_unknown_rates() { + let ntsc = Some(Rate::new(60000, 1001).unwrap()); + let thirty = Some(rate(30)); + assert!(rate_distance(ntsc, thirty, Some(rate(60))).is_lt()); + assert!(rate_distance(thirty, ntsc, Some(rate(30))).is_lt()); + assert!(rate_distance(ntsc, None, Some(rate(60))).is_lt()); + assert!(rate_distance(ntsc, thirty, None).is_eq()); + } + + /// Distance is symmetric in the two dimensions and zero only on an exact hit, + /// so a mode that overshoots is no better than one that undershoots by as much. + #[test] + fn distance_is_zero_only_on_an_exact_match() { + let want = Size::new(1280, 720); + assert_eq!(distance(Size::new(1280, 720), want), 0); + assert_eq!(distance(Size::new(1280, 600), want), 120); + assert_eq!(distance(Size::new(1280, 840), want), 120); + } + + #[test] + fn nearest_prefers_geometry_then_rate_then_cost_then_order() { + let candidate = |width, height, fps: Option, cost| Candidate { + size: Size::new(width, height), + framerate: fps.map(rate), + cost, + }; + let want = |framerate| Request { + size: Size::new(1280, 720), + framerate, + }; + let candidates = [ + candidate(640, 480, Some(30), 0), + candidate(1280, 720, Some(30), 1), + candidate(1280, 720, Some(15), 0), + candidate(1280, 720, Some(30), 0), + candidate(1281, 720, Some(30), 0), + ]; + let pick = |framerate| nearest(candidates, want(framerate), |c| *c).unwrap(); + + let chosen = pick(Some(rate(30))); + assert_eq!((chosen.framerate, chosen.cost), (Some(rate(30)), 0)); + // With no rate requested, the first of the cheapest exact sizes wins. + assert_eq!(pick(None).framerate, Some(rate(15))); + // An odd size is never picked, even when it is the only exact one. + assert!(nearest([candidate(1281, 721, None, 0)], want(None), |c| *c).is_none()); + } + + #[test] + fn modes_merge_sizes_and_sort_largest_and_fastest_first() { + let modes = modes([ + (Size::new(640, 480), vec![rate(30)]), + (Size::new(1280, 720), vec![rate(15)]), + (Size::new(640, 480), vec![rate(60), rate(30)]), + ]); + assert_eq!( + modes, + [ + Mode { + width: 1280, + height: 720, + framerates: vec![rate(15)], + }, + Mode { + width: 640, + height: 480, + framerates: vec![rate(60), rate(30)], + }, + ] + ); + } +} diff --git a/rs/moq-video/src/capture/pipewire.rs b/rs/moq-video/src/capture/pipewire.rs index 2fcb52d9d8f..4ac1ea79209 100644 --- a/rs/moq-video/src/capture/pipewire.rs +++ b/rs/moq-video/src/capture/pipewire.rs @@ -38,6 +38,7 @@ use spa::buffer::DataType; use spa::param::video::{VideoFormat, VideoInfoRaw}; use super::channel::FrameChannel; +use super::mode::Request; use super::pump::Geometry; use super::{Config, Stream}; use crate::frame::{DmaBuf, DmaBufFrame, DmaBufPlane, DrmFormat, I420, Surface, wait_dma_buf_readable}; @@ -81,8 +82,6 @@ pub(super) async fn open(config: &Config, device: Option<&str>) -> Result, target: Target, label: String, - /// The frame size to prefer, or `None` to leave the size out of the offers - /// and take the producer's first mode. Any size is accepted either way. - /// - /// Cameras use `None`: given a size range, spa-v4l2 answers with modes the - /// device does not have (a webcam with YUY2 only at 640x480 was offered YUY2 - /// at the range's 1280x720 default) and then fails the link. - size: Option, } /// Whether a loop captures a screen or a camera. @@ -113,14 +105,6 @@ enum Kind { } impl Kind { - /// The raw formats to offer, most preferred first. - fn formats(self) -> &'static [VideoFormat] { - match self { - Self::Screen => &SCREEN_FORMATS, - Self::Camera => &CAMERA_FORMATS, - } - } - /// The producer, as error messages name it. fn producer(self) -> &'static str { match self { @@ -134,8 +118,9 @@ impl Kind { enum Target { /// A node id granted by the ScreenCast portal. Node(u32), - /// A camera by `node.name`, or `None` for the session manager's default. - Camera(Option), + /// A camera by `node.name`, or `None` for the default one, streaming the + /// mode nearest the request. + Camera(Option, Request), } /// Spawn the PipeWire loop for `capture` and wait for its format and first frame. @@ -784,7 +769,6 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { remote, target, label, - size, }, framerate, chan, @@ -798,9 +782,13 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { let mainloop = pw::main_loop::MainLoopRc::new(None).map_err(|e| err("pipewire main loop", e))?; let context = pw::context::ContextRc::new(&mainloop, None).map_err(|e| err("pipewire context", e))?; let core = connect(&context, remote).map_err(|e| err("pipewire connect", e))?; - let node_id = match target { - Target::Node(id) => Some(id), - Target::Camera(name) => camera::resolve(&camera::scan(&mainloop, &core)?, name.as_deref())?, + let (node_id, offers) = match target { + Target::Node(id) => (id, format_offers(framerate)), + Target::Camera(name, want) => { + let (id, mode) = camera::choose(&mainloop, &core, name.as_deref(), want)?; + tracing::debug!(node = id, ?mode, "chose PipeWire camera mode"); + (id, mode.offers()) + } }; let (name, role) = match kind { @@ -879,14 +867,22 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { let Ok((media_type, media_subtype)) = spa::param::format_utils::parse_format(param) else { return; }; - if media_type != spa::param::format::MediaType::Video - || media_subtype != spa::param::format::MediaSubtype::Raw - { + if media_type != spa::param::format::MediaType::Video { return; } let mut state = state.borrow_mut(); - if let Err(e) = replace_video_format(&mut state.format, |format| format.parse(param)) { + if media_subtype == spa::param::format::MediaSubtype::Mjpg { + // libspa parses only raw formats. MJPEG is recorded as an + // `Encoded` raw format carrying the negotiated size and rate. + let Some(format) = camera::mjpeg_format(param) else { + tracing::warn!("failed to parse the pipewire MJPEG format"); + return; + }; + state.format = format; + } else if media_subtype != spa::param::format::MediaSubtype::Raw { + return; + } else if let Err(e) = replace_video_format(&mut state.format, |format| format.parse(param)) { tracing::warn!(error = %e, "failed to parse pipewire video format"); return; } @@ -1031,9 +1027,16 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { ChunkKind::Empty => return, ChunkKind::Invalid => return, } - let Some(required) = frame_data_size(state.format.format(), layout) else { - tracing::warn!("pipewire frame layout overflows its buffer"); - return; + // A JPEG spans its chunk, whatever its layout would suggest. + let required = match state.format.format() { + VideoFormat::Encoded => size, + format => { + let Some(required) = frame_data_size(format, layout) else { + tracing::warn!("pipewire frame layout overflows its buffer"); + return; + }; + required + } }; if dmabuf { let Some(format) = drm_format(state.format.format()) else { @@ -1181,7 +1184,6 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { // DMA-BUF formats come first so PipeWire prefers them. Each has a matching // shared-memory offer without a modifier as the required fallback. - let offers = format_offers(kind.formats(), framerate, size); let mut params = offers .iter() .map(|offer| { @@ -1192,7 +1194,7 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { stream .connect( spa::utils::Direction::Input, - node_id, + Some(node_id), pw::stream::StreamFlags::AUTOCONNECT | pw::stream::StreamFlags::MAP_BUFFERS, &mut params, ) @@ -1467,6 +1469,8 @@ fn convert(format: VideoFormat, bytes: &[u8], layout: FrameLayout, color: Option VideoFormat::RGBx | VideoFormat::RGBA => { I420::from_rgba(bytes, layout.stride, crate::Size::new(layout.width, layout.height)) } + // Only a camera negotiates MJPEG, which the format records as `Encoded`. + VideoFormat::Encoded => I420::from_mjpeg(bytes, crate::Size::new(layout.width, layout.height)), other => Err(Error::Codec(anyhow::anyhow!( "pipewire negotiated an unsupported video format {other:?}" ))), @@ -1493,15 +1497,8 @@ const SCREEN_FORMATS: [VideoFormat; 5] = [ VideoFormat::NV12, ]; -/// Among the shared-memory offers YUY2 comes first: it is one plane on every -/// producer, and the buffer offer asks for one data block, while libcamera may -/// deliver NV12 as two. NV12 still leads as a DMA-BUF, which only matches a -/// producer that advertises a modifier. MJPEG is not offered yet. -const CAMERA_FORMATS: [VideoFormat; 2] = [VideoFormat::YUY2, VideoFormat::NV12]; - -/// Serialize one `EnumFormat` pod for a concrete pixel format, accepting any -/// size and preferring `size` when there is one. -fn format_offer(framerate: u32, format: VideoFormat, dmabuf: bool, size: Option) -> Vec { +/// Serialize one `EnumFormat` pod for a concrete pixel format. +fn format_offer(framerate: u32, format: VideoFormat, dmabuf: bool) -> Vec { let mut obj = spa::pod::object!( spa::utils::SpaTypes::ObjectParamFormat, spa::param::ParamType::EnumFormat, @@ -1516,6 +1513,21 @@ fn format_offer(framerate: u32, format: VideoFormat, dmabuf: bool, size: Option< spa::param::format::MediaSubtype::Raw ), spa::pod::property!(spa::param::format::FormatProperties::VideoFormat, Id, format), + spa::pod::property!( + spa::param::format::FormatProperties::VideoSize, + Choice, + Range, + Rectangle, + spa::utils::Rectangle { + width: 1920, + height: 1080 + }, + spa::utils::Rectangle { width: 1, height: 1 }, + spa::utils::Rectangle { + width: 8192, + height: 8192 + } + ), spa::pod::property!( spa::param::format::FormatProperties::VideoFramerate, Choice, @@ -1529,60 +1541,48 @@ fn format_offer(framerate: u32, format: VideoFormat, dmabuf: bool, size: Option< spa::utils::Fraction { num: 1000, denom: 1 } ), ); - if let Some(size) = size { - obj.properties.insert( - 3, - spa::pod::Property::new( - spa::param::format::FormatProperties::VideoSize.as_raw(), - spa::pod::Value::Choice(spa::pod::ChoiceValue::Rectangle(spa::utils::Choice( - spa::utils::ChoiceFlags::empty(), - spa::utils::ChoiceEnum::Range { - default: spa::utils::Rectangle { - width: size.width, - height: size.height, - }, - min: spa::utils::Rectangle { width: 1, height: 1 }, - max: spa::utils::Rectangle { - width: 8192, - height: 8192, - }, - }, - ))), - ), - ); - } if dmabuf { - obj.properties.push(spa::pod::Property { - key: spa::param::format::FormatProperties::VideoModifier.as_raw(), - flags: spa::pod::PropertyFlags::from_bits_retain( - spa::sys::SPA_POD_PROP_FLAG_MANDATORY | spa::sys::SPA_POD_PROP_FLAG_DONT_FIXATE, - ), - value: spa::pod::Value::Choice(spa::pod::ChoiceValue::Long(spa::utils::Choice( - spa::utils::ChoiceFlags::empty(), - spa::utils::ChoiceEnum::Enum { - default: 0, - alternatives: vec![0], - }, - ))), - }); + obj.properties.push(linear_modifier()); } - spa::pod::serialize::PodSerializer::serialize(std::io::Cursor::new(Vec::new()), &spa::pod::Value::Object(obj)) + serialize_format(obj) +} + +/// The `VideoModifier` property of a DMA-BUF offer: linear only, left for the +/// producer to fixate. +fn linear_modifier() -> spa::pod::Property { + spa::pod::Property { + key: spa::param::format::FormatProperties::VideoModifier.as_raw(), + flags: spa::pod::PropertyFlags::from_bits_retain( + spa::sys::SPA_POD_PROP_FLAG_MANDATORY | spa::sys::SPA_POD_PROP_FLAG_DONT_FIXATE, + ), + value: spa::pod::Value::Choice(spa::pod::ChoiceValue::Long(spa::utils::Choice( + spa::utils::ChoiceFlags::empty(), + spa::utils::ChoiceEnum::Enum { + default: 0, + alternatives: vec![0], + }, + ))), + } +} + +fn serialize_format(object: spa::pod::Object) -> Vec { + spa::pod::serialize::PodSerializer::serialize(std::io::Cursor::new(Vec::new()), &spa::pod::Value::Object(object)) .expect("serializing a static format pod cannot fail") .0 .into_inner() } -/// Serialize DMA-BUF formats first and shared-memory fallbacks second. Only a -/// format with a DRM fourcc to import as gets a DMA-BUF offer. -fn format_offers(formats: &[VideoFormat], framerate: u32, size: Option) -> Vec> { - let dmabuf = formats - .iter() - .filter(|format| drm_format(**format).is_some()) - .map(|&format| format_offer(framerate, format, true, size)); - let shared = formats - .iter() - .map(|&format| format_offer(framerate, format, false, size)); - dmabuf.chain(shared).collect() +/// Serialize DMA-BUF formats first and shared-memory fallbacks second. +fn format_offers(framerate: u32) -> Vec> { + SCREEN_FORMATS + .into_iter() + .map(|format| format_offer(framerate, format, true)) + .chain( + SCREEN_FORMATS + .into_iter() + .map(|format| format_offer(framerate, format, false)), + ) + .collect() } /// Serialize `SPA_PARAM_Buffers` for the negotiated memory representation. @@ -1634,7 +1634,7 @@ mod tests { /// The serialized format offer must parse back as a valid pod. #[test] fn format_offer_is_valid_pod() { - let offers = format_offers(&SCREEN_FORMATS, 30, Some(Size::new(1920, 1080))); + let offers = format_offers(30); assert_eq!(offers.len(), SCREEN_FORMATS.len() * 2); for (index, bytes) in offers.iter().enumerate() { let (remaining, value) = spa::pod::deserialize::PodDeserializer::deserialize_any_from(bytes) @@ -1674,14 +1674,13 @@ mod tests { #[test] fn negotiated_modifier_must_be_present_and_fixed() { - let size = Some(Size::new(1920, 1080)); - let shared = format_offer(30, VideoFormat::BGRx, false, size); + let shared = format_offer(30, VideoFormat::BGRx, false); let shared = spa::pod::Pod::from_bytes(&shared).unwrap(); let mut format = VideoInfoRaw::default(); format.parse(shared).unwrap(); assert_eq!(negotiated_memory(shared, format), Some(NegotiatedMemory::SharedMemory)); - let offered = format_offer(30, VideoFormat::BGRx, true, size); + let offered = format_offer(30, VideoFormat::BGRx, true); let offered = spa::pod::Pod::from_bytes(&offered).unwrap(); format.parse(offered).unwrap(); assert_eq!(negotiated_memory(offered, format), Some(NegotiatedMemory::Fixating)); @@ -1692,43 +1691,6 @@ mod tests { assert_eq!(negotiated_memory(fixed, format), Some(NegotiatedMemory::DmaBuf(0))); } - /// A camera offer leaves the size to the producer, and never offers YUY2 - /// as a DMA-BUF, which has no DRM fourcc here. - #[test] - fn camera_offers_leave_the_size_to_the_producer() { - let decoded: Vec<_> = format_offers(&CAMERA_FORMATS, 30, None) - .iter() - .map(|bytes| { - let (_, value) = spa::pod::deserialize::PodDeserializer::deserialize_any_from(bytes).unwrap(); - let spa::pod::Value::Object(object) = value else { - panic!("format offer is not an object"); - }; - let property = |key: spa::param::format::FormatProperties| { - object - .properties - .iter() - .find(|property| property.key == key.as_raw()) - .map(|property| property.value.clone()) - }; - let Some(spa::pod::Value::Id(format)) = property(spa::param::format::FormatProperties::VideoFormat) - else { - panic!("format is not an id"); - }; - assert_eq!(property(spa::param::format::FormatProperties::VideoSize), None); - let dmabuf = property(spa::param::format::FormatProperties::VideoModifier).is_some(); - (VideoFormat::from_raw(format.0), dmabuf) - }) - .collect(); - assert_eq!( - decoded, - [ - (VideoFormat::NV12, true), - (VideoFormat::YUY2, false), - (VideoFormat::NV12, false), - ] - ); - } - #[test] fn yuy2_converts_with_the_negotiated_color() { let layout = FrameLayout { diff --git a/rs/moq-video/src/capture/pipewire/camera.rs b/rs/moq-video/src/capture/pipewire/camera.rs index 7e5b945e9f6..2865c40a1d7 100644 --- a/rs/moq-video/src/capture/pipewire/camera.rs +++ b/rs/moq-video/src/capture/pipewire/camera.rs @@ -8,6 +8,12 @@ //! works headless. A sandbox cannot reach that socket, so there the camera portal //! (`org.freedesktop.portal.Camera`) grants access and hands back a PipeWire //! remote that exposes the cameras. Streaming reuses the parent module's loop. +//! +//! The mode is chosen here rather than negotiated: the node's `EnumFormat` list +//! goes through the same selection as a V4L2 device, and the stream offers +//! exactly the chosen mode. Offering a size range instead lets spa-v4l2 accept +//! a size its format does not have (a webcam with YUY2 only at 640x480 took +//! YUY2 at 1280x720) and then fail the link. use std::cell::RefCell; use std::os::fd::OwnedFd; @@ -15,14 +21,21 @@ use std::rc::Rc; use std::time::Duration; use pipewire as pw; +use pw::spa; +use spa::param::format::{FormatProperties, MediaSubtype, MediaType}; +use spa::param::video::{VideoFormat, VideoInfoRaw}; +use spa::pod::{ChoiceValue, Value}; +use spa::utils::ChoiceEnum; -use super::{Capture, Kind, Target, err}; -use crate::Error; -use crate::capture::{Camera, Config, PIPEWIRE, Stream}; +use super::{Capture, DEFAULT_FRAMERATE, Kind, Target, drm_format, err, linear_modifier, serialize_format}; +use crate::capture::mode::{self, Request}; +use crate::capture::v4l2::{DEFAULT_HEIGHT, DEFAULT_WIDTH}; +use crate::capture::{Camera, Config, Mode, PIPEWIRE, Stream}; +use crate::{Error, Rate, Size}; -/// How long the PipeWire daemon may take to list its objects. A connected -/// daemon answers in milliseconds, so running out means it is stuck. -const SCAN_TIMEOUT: Duration = Duration::from_secs(5); +/// How long the PipeWire daemon may take to answer a query. A connected daemon +/// answers in milliseconds, so running out means it is stuck. +const QUERY_TIMEOUT: Duration = Duration::from_secs(5); /// List the PipeWire cameras as [`Camera`]s with `pipewire:` ids. pub(in crate::capture) async fn cameras() -> Result, Error> { @@ -42,7 +55,8 @@ pub(in crate::capture) async fn cameras() -> Result, Error> { } Err(error) => return Err(err("pipewire connect", error)), }; - scan(&mainloop, &core) + let registry = core.get_registry_rc().map_err(|e| err("pipewire registry", e))?; + scan(&mainloop, &core, ®istry) }) .await?; @@ -55,25 +69,49 @@ pub(in crate::capture) async fn cameras() -> Result, Error> { .collect()) } -/// Open the camera node named `node`, or the session manager's default camera. -/// -/// The camera streams its first raw mode that this backend converts, so -/// `config.width` and `config.height` are not applied yet. The parent module's -/// `Capture::size` says why the offers carry no size. +/// List the modes the camera named `node` (or the default camera) reports, for +/// the formats this backend converts. +pub(in crate::capture) async fn modes(node: Option<&str>) -> Result, Error> { + let remote = remote().await?; + let node = node.map(str::to_string); + crate::capture::blocking(move || { + pw::init(); + let mainloop = pw::main_loop::MainLoopRc::new(None).map_err(|e| err("pipewire main loop", e))?; + let context = pw::context::ContextRc::new(&mainloop, None).map_err(|e| err("pipewire context", e))?; + let core = super::connect(&context, remote).map_err(|e| err("pipewire connect", e))?; + let registry = core.get_registry_rc().map_err(|e| err("pipewire registry", e))?; + let nodes = scan(&mainloop, &core, ®istry)?; + let node = resolve(&nodes, node.as_deref())?; + let formats = formats(&mainloop, &core, ®istry, node)?; + Ok(mode::modes(formats.into_iter().flat_map(|format| { + format.sizes.into_iter().map(move |size| (size, format.rates.clone())) + }))) + }) + .await +} + +/// Open the camera node named `node`, or the default camera, in the mode +/// nearest `config`'s size and rate. pub(in crate::capture) async fn open(config: &Config, node: Option<&str>) -> Result { let remote = remote().await?; let label = match node { Some(node) => format!("{PIPEWIRE}:{node}"), None => PIPEWIRE.to_string(), }; + let want = Request { + size: Size::new( + config.width.unwrap_or(DEFAULT_WIDTH), + config.height.unwrap_or(DEFAULT_HEIGHT), + ), + framerate: config.framerate, + }; super::start( config, Capture { kind: Kind::Camera, remote, - target: Target::Camera(node.map(str::to_string)), + target: Target::Camera(node.map(str::to_string), want), label, - size: None, }, None, ) @@ -109,6 +147,27 @@ async fn remote() -> Result, Error> { Ok(Some(fd)) } +/// Pick the camera and its mode for a stream on `core`: the node id to link +/// to, and the mode to offer. +pub(super) fn choose( + mainloop: &pw::main_loop::MainLoopRc, + core: &pw::core::CoreRc, + name: Option<&str>, + want: Request, +) -> Result<(u32, Selected), Error> { + let registry = core.get_registry_rc().map_err(|e| err("pipewire registry", e))?; + let nodes = scan(mainloop, core, ®istry)?; + let node = resolve(&nodes, name)?; + let formats = formats(mainloop, core, ®istry, node)?; + let selected = select(&formats, want).ok_or_else(|| { + Error::Codec(anyhow::anyhow!( + "PipeWire camera {} offers no YUY2, NV12, RGB, or MJPEG mode", + node.name + )) + })?; + Ok((node.id, selected)) +} + /// A PipeWire camera node. #[derive(Clone, Debug, PartialEq, Eq)] pub(super) struct Node { @@ -118,6 +177,8 @@ pub(super) struct Node { name: String, /// `node.description`, or the best name the node has short of it. description: String, + /// `priority.session`, which the session manager ranks default nodes by. + priority: i32, } impl Node { @@ -133,22 +194,22 @@ impl Node { .or_else(|| prop(*pw::keys::NODE_NICK)) .filter(|description| !description.is_empty()) .unwrap_or(name); + let priority = prop(*pw::keys::PRIORITY_SESSION) + .and_then(|priority| priority.parse().ok()) + .unwrap_or(0); Some(Self { id, name: name.to_string(), description: description.to_string(), + priority, }) } } -/// List the camera nodes on `core`, running `mainloop` until the daemon has -/// sent every registry global. -pub(super) fn scan(mainloop: &pw::main_loop::MainLoopRc, core: &pw::core::CoreRc) -> Result, Error> { - let registry = core.get_registry_rc().map_err(|e| err("pipewire registry", e))?; - let nodes = Rc::new(RefCell::new(Vec::new())); +/// Run `mainloop` until the daemon has answered every request sent before +/// this one on `core`. +fn roundtrip(mainloop: &pw::main_loop::MainLoopRc, core: &pw::core::CoreRc) -> Result<(), Error> { let outcome: Rc>>> = Rc::new(RefCell::new(None)); - - // The daemon answers a sync only after every global it announced before it. let pending = core.sync(0).map_err(|e| err("pipewire sync", e))?; let _core = core .add_listener_local() @@ -177,6 +238,35 @@ pub(super) fn scan(mainloop: &pw::main_loop::MainLoopRc, core: &pw::core::CoreRc } }) .register(); + let timer = mainloop.loop_().add_timer({ + let mainloop = mainloop.downgrade(); + move |_| { + if let Some(mainloop) = mainloop.upgrade() { + mainloop.quit(); + } + } + }); + timer + .update_timer(Some(QUERY_TIMEOUT), None) + .into_result() + .map_err(|e| err("pipewire timer", e))?; + + mainloop.run(); + let outcome = outcome.borrow_mut().take(); + outcome.unwrap_or_else(|| { + Err(Error::Codec(anyhow::anyhow!( + "PipeWire did not answer within {QUERY_TIMEOUT:?}" + ))) + }) +} + +/// List the camera nodes in `registry`. +fn scan( + mainloop: &pw::main_loop::MainLoopRc, + core: &pw::core::CoreRc, + registry: &pw::registry::RegistryRc, +) -> Result, Error> { + let nodes = Rc::new(RefCell::new(Vec::new())); let _registry = registry .add_listener_local() .global({ @@ -192,50 +282,335 @@ pub(super) fn scan(mainloop: &pw::main_loop::MainLoopRc, core: &pw::core::CoreRc } }) .register(); - let timer = mainloop.loop_().add_timer({ - let mainloop = mainloop.downgrade(); - move |_| { - if let Some(mainloop) = mainloop.upgrade() { - mainloop.quit(); + // The registry announces every existing global before the sync reply. + roundtrip(mainloop, core)?; + Ok(nodes.take()) +} + +/// The camera named `name`, or the one with the highest session priority, +/// which is the camera the session manager links by default. Fails when the +/// camera is not there, rather than leaving the stream unlinked until the +/// format wait runs out. +fn resolve<'a>(nodes: &'a [Node], name: Option<&str>) -> Result<&'a Node, Error> { + let Some(name) = name else { + // `max_by_key` keeps the last of equals; the registry's first is wanted. + return nodes + .iter() + .rev() + .max_by_key(|node| node.priority) + .ok_or_else(|| Error::SourceUnavailable("PipeWire has no camera".to_string())); + }; + nodes.iter().find(|node| node.name == name).ok_or_else(|| { + let found: Vec<_> = nodes.iter().map(|node| node.name.as_str()).collect(); + Error::SourceUnavailable(format!("no PipeWire camera named {name} (found: {})", found.join(", "))) + }) +} + +/// The raw formats `convert` turns into I420. +const RAW_FORMATS: [VideoFormat; 6] = [ + VideoFormat::YUY2, + VideoFormat::NV12, + VideoFormat::BGRx, + VideoFormat::BGRA, + VideoFormat::RGBx, + VideoFormat::RGBA, +]; + +/// How a camera format carries its pixels. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Encoding { + /// A raw format [`super::convert`] handles. + Raw(VideoFormat), + /// Motion-JPEG, decoded per frame. + Mjpeg, +} + +impl Encoding { + /// What converting to I420 costs, in the V4L2 backend's terms: resampling + /// YUV is cheapest, then a color conversion, then a JPEG decode. + fn cost(self) -> u8 { + match self { + Self::Raw(VideoFormat::YUY2 | VideoFormat::NV12) => 0, + Self::Raw(_) => 1, + Self::Mjpeg => 2, + } + } +} + +/// One `EnumFormat` entry of a camera, for a format this backend converts. +#[derive(Clone, Debug, PartialEq)] +pub(super) struct Format { + encoding: Encoding, + /// The sizes on offer. A size range contributes its corners. + sizes: Vec, + /// The exact rates on offer, highest first. Empty for a rate range. + rates: Vec, + /// Whether the size is a range that also takes the requested size. + ranged: Option<(Size, Size)>, +} + +/// The mode a camera stream offers. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Selected { + encoding: Encoding, + size: Size, + /// `None` when the camera reports a rate range rather than exact rates. + framerate: Option, +} + +impl Selected { + /// Serialize the `EnumFormat` offers for exactly this mode, a DMA-BUF offer + /// first when the format has a DRM fourcc to import as. + pub(super) fn offers(&self) -> Vec> { + let dmabuf = match self.encoding { + Encoding::Raw(format) => drm_format(format).is_some(), + Encoding::Mjpeg => false, + }; + let mut offers = Vec::new(); + if dmabuf { + offers.push(self.offer(true)); + } + offers.push(self.offer(false)); + offers + } + + fn offer(&self, dmabuf: bool) -> Vec { + let property = |key: FormatProperties, value| spa::pod::Property::new(key.as_raw(), value); + let mut properties = vec![property( + FormatProperties::MediaType, + Value::Id(spa::utils::Id(MediaType::Video.as_raw())), + )]; + match self.encoding { + Encoding::Raw(format) => { + properties.push(property( + FormatProperties::MediaSubtype, + Value::Id(spa::utils::Id(MediaSubtype::Raw.as_raw())), + )); + properties.push(property( + FormatProperties::VideoFormat, + Value::Id(spa::utils::Id(format.as_raw())), + )); } + Encoding::Mjpeg => properties.push(property( + FormatProperties::MediaSubtype, + Value::Id(spa::utils::Id(MediaSubtype::Mjpg.as_raw())), + )), } - }); - timer - .update_timer(Some(SCAN_TIMEOUT), None) - .into_result() - .map_err(|e| err("pipewire timer", e))?; + properties.push(property( + FormatProperties::VideoSize, + Value::Rectangle(spa::utils::Rectangle { + width: self.size.width, + height: self.size.height, + }), + )); + let framerate = match self.framerate { + Some(rate) => Value::Fraction(fraction(rate)), + None => Value::Choice(ChoiceValue::Fraction(spa::utils::Choice( + spa::utils::ChoiceFlags::empty(), + ChoiceEnum::Range { + default: spa::utils::Fraction { + num: DEFAULT_FRAMERATE, + denom: 1, + }, + min: spa::utils::Fraction { num: 0, denom: 1 }, + max: spa::utils::Fraction { num: 1000, denom: 1 }, + }, + ))), + }; + properties.push(property(FormatProperties::VideoFramerate, framerate)); + if dmabuf { + properties.push(linear_modifier()); + } + serialize_format(spa::pod::Object { + type_: spa::utils::SpaTypes::ObjectParamFormat.as_raw(), + id: spa::param::ParamType::EnumFormat.as_raw(), + properties, + }) + } +} - mainloop.run(); - let outcome = outcome.borrow_mut().take(); - match outcome { - Some(Ok(())) => Ok(nodes.take()), - Some(Err(error)) => Err(error), - None => Err(Error::Codec(anyhow::anyhow!( - "PipeWire did not list its objects within {SCAN_TIMEOUT:?}" - ))), +fn fraction(rate: Rate) -> spa::utils::Fraction { + spa::utils::Fraction { + num: rate.numerator(), + denom: rate.denominator(), } } -/// The node id to link to for `name`, or `None` to let the session manager -/// pick its default camera. Fails when the camera is not there, rather than -/// leaving the stream unlinked until the format wait runs out. -pub(super) fn resolve(nodes: &[Node], name: Option<&str>) -> Result, Error> { - let Some(name) = name else { - if nodes.is_empty() { - return Err(Error::SourceUnavailable("PipeWire has no camera".to_string())); +/// List the formats a camera node offers that this backend converts. +fn formats( + mainloop: &pw::main_loop::MainLoopRc, + core: &pw::core::CoreRc, + registry: &pw::registry::RegistryRc, + node: &Node, +) -> Result, Error> { + let proxy: pw::node::Node = registry + .bind(&pw::registry::GlobalObject { + id: node.id, + permissions: pw::permissions::PermissionFlags::empty(), + type_: pw::types::ObjectType::Node, + version: 0, + props: None::<&spa::utils::dict::DictRef>, + }) + .map_err(|e| err("pipewire node", e))?; + let formats = Rc::new(RefCell::new(Vec::new())); + let _listener = proxy + .add_listener_local() + .param({ + let formats = formats.clone(); + move |_, id, _, _, param| { + if id != spa::param::ParamType::EnumFormat { + return; + } + let Some(param) = param else { return }; + match spa::pod::deserialize::PodDeserializer::deserialize_any_from(param.as_bytes()) { + Ok((_, value)) => formats.borrow_mut().extend(parse_format(&value)), + Err(e) => tracing::debug!(error = ?e, "unreadable PipeWire EnumFormat"), + } + } + }) + .register(); + proxy.enum_params(0, Some(spa::param::ParamType::EnumFormat), 0, u32::MAX); + roundtrip(mainloop, core)?; + Ok(formats.take()) +} + +/// Read one `EnumFormat` or `Format` pod into the formats it offers that this +/// backend converts. A pod may list several pixel formats in one choice. +fn parse_format(value: &Value) -> Vec { + let Value::Object(object) = value else { + return Vec::new(); + }; + let property = |key: FormatProperties| { + object + .properties + .iter() + .find(|property| property.key == key.as_raw()) + .map(|property| &property.value) + }; + if property(FormatProperties::MediaType) != Some(&Value::Id(spa::utils::Id(MediaType::Video.as_raw()))) { + return Vec::new(); + } + let encodings = match property(FormatProperties::MediaSubtype) { + Some(Value::Id(id)) if *id == spa::utils::Id(MediaSubtype::Mjpg.as_raw()) => vec![Encoding::Mjpeg], + Some(Value::Id(id)) if *id == spa::utils::Id(MediaSubtype::Raw.as_raw()) => { + let formats = match property(FormatProperties::VideoFormat) { + Some(Value::Id(id)) => vec![*id], + Some(Value::Choice(ChoiceValue::Id(choice))) => listed(&choice.1), + _ => Vec::new(), + }; + formats + .into_iter() + .map(|id| VideoFormat::from_raw(id.0)) + .filter(|format| RAW_FORMATS.contains(format)) + .map(Encoding::Raw) + .collect() } - return Ok(None); + _ => Vec::new(), + }; + + let size = |rectangle: &spa::utils::Rectangle| Size::new(rectangle.width, rectangle.height); + let (sizes, ranged) = match property(FormatProperties::VideoSize) { + Some(Value::Rectangle(rectangle)) => (vec![size(rectangle)], None), + Some(Value::Choice(ChoiceValue::Rectangle(choice))) => match &choice.1 { + ChoiceEnum::Range { min, max, .. } | ChoiceEnum::Step { min, max, .. } => { + (vec![size(max), size(min)], Some((size(min), size(max)))) + } + other => (listed(other).iter().map(size).collect(), None), + }, + _ => (Vec::new(), None), }; - match nodes.iter().find(|node| node.name == name) { - Some(node) => Ok(Some(node.id)), - None => { - let found: Vec<_> = nodes.iter().map(|node| node.name.as_str()).collect(); - Err(Error::SourceUnavailable(format!( - "no PipeWire camera named {name} (found: {})", - found.join(", ") - ))) + let rate = |fraction: &spa::utils::Fraction| Rate::new(fraction.num, fraction.denom).ok(); + let mut rates: Vec = match property(FormatProperties::VideoFramerate) { + Some(Value::Fraction(fraction)) => rate(fraction).into_iter().collect(), + Some(Value::Choice(ChoiceValue::Fraction(choice))) => listed(&choice.1).iter().filter_map(rate).collect(), + _ => Vec::new(), + }; + rates.sort_by(|left, right| right.cmp(left)); + rates.dedup(); + + encodings + .into_iter() + .map(|encoding| Format { + encoding, + sizes: sizes.clone(), + rates: rates.clone(), + ranged, + }) + .collect() +} + +/// The values a choice lists. A range lists none: it describes a continuum. +fn listed(choice: &ChoiceEnum) -> Vec { + let values = match choice { + ChoiceEnum::None(value) => vec![*value], + // SPA usually repeats the default among the alternatives. + ChoiceEnum::Enum { default, alternatives } => { + std::iter::once(*default).chain(alternatives.iter().copied()).collect() + } + ChoiceEnum::Range { .. } | ChoiceEnum::Step { .. } | ChoiceEnum::Flags { .. } => Vec::new(), + }; + let mut unique = Vec::with_capacity(values.len()); + for value in values { + if !unique.contains(&value) { + unique.push(value); } } + unique +} + +/// Pick the mode nearest `want` with the V4L2 backend's rules. A size range +/// that holds the requested size offers it exactly. +fn select(formats: &[Format], want: Request) -> Option { + let mut candidates = Vec::new(); + for format in formats { + let within = format.ranged.is_some_and(|(min, max)| { + (min.width..=max.width).contains(&want.size.width) && (min.height..=max.height).contains(&want.size.height) + }); + let sizes = within + .then_some(want.size) + .into_iter() + .chain(format.sizes.iter().copied()); + let rates: Vec> = if format.rates.is_empty() { + vec![None] + } else { + format.rates.iter().copied().map(Some).collect() + }; + for size in sizes { + for &framerate in &rates { + candidates.push(Selected { + encoding: format.encoding, + size, + framerate, + }); + } + } + } + mode::nearest(candidates, want, |selected| mode::Candidate { + size: selected.size, + framerate: selected.framerate, + cost: selected.encoding.cost(), + }) +} + +/// The negotiated MJPEG `Format` as the loop's `VideoInfoRaw`, with the +/// `Encoded` format standing for MJPEG and the negotiated size and rate. +pub(super) fn mjpeg_format(param: &spa::pod::Pod) -> Option { + let (_, value) = spa::pod::deserialize::PodDeserializer::deserialize_any_from(param.as_bytes()).ok()?; + let format = parse_format(&value).into_iter().next()?; + if format.encoding != Encoding::Mjpeg { + return None; + } + let size = *format.sizes.first()?; + let mut info = VideoInfoRaw::default(); + info.set_format(VideoFormat::Encoded); + info.set_size(spa::utils::Rectangle { + width: size.width, + height: size.height, + }); + if let Some(rate) = format.rates.first() { + info.set_framerate(fraction(*rate)); + } + Some(info) } #[cfg(test)] @@ -249,11 +624,12 @@ mod tests { Node::from_props(62, |key| props.get(key).copied()) } - const WEBCAM: [(&str, &str); 4] = [ + const WEBCAM: [(&str, &str); 5] = [ ("media.class", "Video/Source"), ("media.role", "Camera"), ("node.name", "v4l2_input.pci-0000_00_14.0-usb-0_4_1.0"), ("node.description", "Integrated Camera (V4L2)"), + ("priority.session", "1000"), ]; #[test] @@ -264,6 +640,7 @@ mod tests { id: 62, name: "v4l2_input.pci-0000_00_14.0-usb-0_4_1.0".to_string(), description: "Integrated Camera (V4L2)".to_string(), + priority: 1000, }) ); @@ -293,18 +670,228 @@ mod tests { } #[test] - fn resolve_finds_the_named_camera_or_fails() { - let nodes = [node(&WEBCAM).unwrap()]; - assert_eq!( - resolve(&nodes, Some("v4l2_input.pci-0000_00_14.0-usb-0_4_1.0")).unwrap(), - Some(62) - ); - assert_eq!(resolve(&nodes, None).unwrap(), None); + fn resolve_finds_the_named_camera_or_the_highest_priority_one() { + let webcam = node(&WEBCAM).unwrap(); + let infrared = Node { + id: 79, + name: "v4l2_input.pci-0000_00_14.0-usb-0_4_1.2".to_string(), + priority: 980, + ..webcam.clone() + }; + let nodes = [infrared.clone(), webcam.clone()]; + assert_eq!(resolve(&nodes, Some(&infrared.name)).unwrap().id, 79); + assert_eq!(resolve(&nodes, None).unwrap().id, 62); + // Equal priorities keep the registry's order. + let tied = [ + Node { + priority: 1000, + ..infrared + }, + webcam, + ]; + assert_eq!(resolve(&tied, None).unwrap().id, 79); + let error = resolve(&nodes, Some("missing")).unwrap_err().to_string(); assert!(error.contains("v4l2_input.pci-0000_00_14.0-usb-0_4_1.0"), "{error}"); assert!(matches!(resolve(&[], None), Err(Error::SourceUnavailable(_)))); } + fn rectangle(width: u32, height: u32) -> spa::utils::Rectangle { + spa::utils::Rectangle { width, height } + } + + /// One `EnumFormat` entry the way spa-v4l2 reports it for a UVC webcam. + fn enum_format(subtype: MediaSubtype, format: Option, size: spa::utils::Rectangle) -> Value { + let mut properties = vec![ + spa::pod::Property::new( + FormatProperties::MediaType.as_raw(), + Value::Id(spa::utils::Id(MediaType::Video.as_raw())), + ), + spa::pod::Property::new( + FormatProperties::MediaSubtype.as_raw(), + Value::Id(spa::utils::Id(subtype.as_raw())), + ), + spa::pod::Property::new(FormatProperties::VideoSize.as_raw(), Value::Rectangle(size)), + spa::pod::Property::new( + FormatProperties::VideoFramerate.as_raw(), + Value::Choice(ChoiceValue::Fraction(spa::utils::Choice( + spa::utils::ChoiceFlags::empty(), + ChoiceEnum::None(spa::utils::Fraction { num: 30, denom: 1 }), + ))), + ), + ]; + if let Some(format) = format { + properties.push(spa::pod::Property::new( + FormatProperties::VideoFormat.as_raw(), + Value::Id(spa::utils::Id(format.as_raw())), + )); + } + Value::Object(spa::pod::Object { + type_: spa::utils::SpaTypes::ObjectParamFormat.as_raw(), + id: spa::param::ParamType::EnumFormat.as_raw(), + properties, + }) + } + + /// The integrated webcam's list: MJPEG up to 1080p, YUY2 only at VGA sizes, + /// which is why the first mode is no answer to a size request. + fn webcam_formats() -> Vec { + [ + enum_format(MediaSubtype::Mjpg, None, rectangle(1280, 720)), + enum_format(MediaSubtype::Mjpg, None, rectangle(640, 480)), + enum_format(MediaSubtype::Mjpg, None, rectangle(1920, 1080)), + enum_format(MediaSubtype::Raw, Some(VideoFormat::YUY2), rectangle(640, 480)), + enum_format(MediaSubtype::Raw, Some(VideoFormat::YUY2), rectangle(640, 360)), + enum_format(MediaSubtype::Raw, Some(VideoFormat::GRAY8), rectangle(640, 360)), + ] + .iter() + .flat_map(parse_format) + .collect() + } + + #[test] + fn enum_formats_parse_to_the_convertible_formats() { + let formats = webcam_formats(); + // GRAY8 is not converted, so it is not a candidate. + assert_eq!(formats.len(), 5); + assert_eq!(formats[0].encoding, Encoding::Mjpeg); + assert_eq!(formats[0].sizes, [Size::new(1280, 720)]); + assert_eq!(formats[0].rates, [Rate::new(30, 1).unwrap()]); + assert_eq!(formats[3].encoding, Encoding::Raw(VideoFormat::YUY2)); + } + + #[test] + fn selection_matches_the_v4l2_rules() { + let formats = webcam_formats(); + let select = |width, height| { + let selected = select( + &formats, + Request { + size: Size::new(width, height), + framerate: None, + }, + ) + .unwrap(); + (selected.encoding, selected.size) + }; + // An exact size wins, and the cheaper YUY2 wins a tie with MJPEG. + assert_eq!( + select(640, 480), + (Encoding::Raw(VideoFormat::YUY2), Size::new(640, 480)) + ); + assert_eq!( + select(640, 360), + (Encoding::Raw(VideoFormat::YUY2), Size::new(640, 360)) + ); + assert_eq!(select(1920, 1080), (Encoding::Mjpeg, Size::new(1920, 1080))); + // No exact size: the nearest one, whatever format carries it. + assert_eq!(select(1280, 800), (Encoding::Mjpeg, Size::new(1280, 720))); + } + + #[test] + fn a_size_range_offers_the_requested_size() { + let ranged = Value::Object(spa::pod::Object { + type_: spa::utils::SpaTypes::ObjectParamFormat.as_raw(), + id: spa::param::ParamType::EnumFormat.as_raw(), + properties: vec![ + spa::pod::Property::new( + FormatProperties::MediaType.as_raw(), + Value::Id(spa::utils::Id(MediaType::Video.as_raw())), + ), + spa::pod::Property::new( + FormatProperties::MediaSubtype.as_raw(), + Value::Id(spa::utils::Id(MediaSubtype::Raw.as_raw())), + ), + spa::pod::Property::new( + FormatProperties::VideoFormat.as_raw(), + Value::Id(spa::utils::Id(VideoFormat::NV12.as_raw())), + ), + spa::pod::Property::new( + FormatProperties::VideoSize.as_raw(), + Value::Choice(ChoiceValue::Rectangle(spa::utils::Choice( + spa::utils::ChoiceFlags::empty(), + ChoiceEnum::Range { + default: rectangle(1920, 1080), + min: rectangle(64, 64), + max: rectangle(4608, 2592), + }, + ))), + ), + ], + }); + let formats = parse_format(&ranged); + assert_eq!(formats[0].sizes, [Size::new(4608, 2592), Size::new(64, 64)]); + // A rate range lists no exact rate. + assert!(formats[0].rates.is_empty()); + let selected = select( + &formats, + Request { + size: Size::new(1280, 720), + framerate: None, + }, + ) + .unwrap(); + assert_eq!((selected.size, selected.framerate), (Size::new(1280, 720), None)); + } + + /// The offer pins the chosen mode, and only NV12 gets a DMA-BUF offer. + #[test] + fn offers_pin_the_selected_mode() { + let decode = |bytes: &Vec| { + let (_, value) = spa::pod::deserialize::PodDeserializer::deserialize_any_from(bytes).unwrap(); + let Value::Object(object) = value else { + panic!("offer is not an object"); + }; + object + }; + let find = |object: &spa::pod::Object, key: FormatProperties| { + object + .properties + .iter() + .find(|property| property.key == key.as_raw()) + .map(|property| property.value.clone()) + }; + + let mjpeg = Selected { + encoding: Encoding::Mjpeg, + size: Size::new(1920, 1080), + framerate: Some(Rate::new(30, 1).unwrap()), + }; + let offers = mjpeg.offers(); + assert_eq!(offers.len(), 1); + let offer = decode(&offers[0]); + assert_eq!( + find(&offer, FormatProperties::MediaSubtype), + Some(Value::Id(spa::utils::Id(MediaSubtype::Mjpg.as_raw()))) + ); + assert_eq!( + find(&offer, FormatProperties::VideoSize), + Some(Value::Rectangle(rectangle(1920, 1080))) + ); + assert_eq!( + find(&offer, FormatProperties::VideoFramerate), + Some(Value::Fraction(spa::utils::Fraction { num: 30, denom: 1 })) + ); + // The negotiated format parses back into the loop's MJPEG description. + let info = mjpeg_format(spa::pod::Pod::from_bytes(&offers[0]).unwrap()).unwrap(); + assert_eq!(info.format(), VideoFormat::Encoded); + assert_eq!(info.size(), rectangle(1920, 1080)); + + let nv12 = Selected { + encoding: Encoding::Raw(VideoFormat::NV12), + ..mjpeg + }; + let offers = nv12.offers(); + assert_eq!(offers.len(), 2); + assert!(find(&decode(&offers[0]), FormatProperties::VideoModifier).is_some()); + assert!(find(&decode(&offers[1]), FormatProperties::VideoModifier).is_none()); + let yuy2 = Selected { + encoding: Encoding::Raw(VideoFormat::YUY2), + ..mjpeg + }; + assert_eq!(yuy2.offers().len(), 1); + } + /// Lists the PipeWire cameras over the session socket. Needs no camera and /// never turns one on, so a host without PipeWire just lists nothing. #[tokio::test] @@ -321,8 +908,30 @@ mod tests { eprintln!("PipeWire cameras: {cameras:?}"); } - /// Captures a few frames from each PipeWire camera over the session socket. - /// Ignored because it turns the cameras on: + /// Open `config` and check that five frames arrive at the reported size. + async fn capture(config: &Config) -> Result { + let mut stream = crate::capture::open(config).await?; + assert!(stream.width() >= 2 && stream.width().is_multiple_of(2), "bad width"); + assert!(stream.height() >= 2 && stream.height().is_multiple_of(2), "bad height"); + for i in 0..5 { + let frame = stream.read().await?.unwrap_or_else(|| panic!("no frame {i}")); + assert_eq!(frame.surface.width(), stream.width()); + assert_eq!(frame.surface.height(), stream.height()); + } + eprintln!( + "{}: captured 5 frames at {}x{}, {:?} fps, color {:?}", + stream.label(), + stream.width(), + stream.height(), + stream.framerate(), + stream.color() + ); + Ok(stream) + } + + /// Captures from each PipeWire camera over the session socket, at its + /// smallest mode and at its largest mode up to 1080p, and checks the stream + /// lands on the requested size. Ignored because it turns the cameras on: /// `cargo test -p moq-video --features pipewire pipewire_camera -- --ignored`. #[tokio::test] #[ignore = "turns on every PipeWire camera on the host"] @@ -339,53 +948,44 @@ mod tests { let mut captured = 0; for camera in cameras { - let config = Config { - source: camera.source(), - ..Default::default() - }; - let mut stream = match crate::capture::open(&config).await { - Ok(stream) => stream, - // An IR camera offers only GRAY8, which this backend does not convert. - Err(error) => { - eprintln!("{}: not captured: {error}", camera.id); - continue; - } + let modes = crate::capture::camera_modes(Some(&camera.id)) + .await + .expect("listing PipeWire camera modes"); + eprintln!("{}: modes {modes:?}", camera.id); + // An IR camera offers only GRAY8, which this backend does not convert. + let Some(smallest) = modes.last() else { + eprintln!("{}: no convertible mode", camera.id); + continue; }; - assert_eq!(stream.label(), camera.id); - assert!(stream.width() >= 2 && stream.width().is_multiple_of(2), "bad width"); - assert!(stream.height() >= 2 && stream.height().is_multiple_of(2), "bad height"); - for i in 0..5 { - let frame = stream - .read() + let largest = modes + .iter() + .find(|mode| mode.width <= 1920 && mode.height <= 1080) + .unwrap_or(smallest); + for mode in [smallest, largest] { + let config = Config { + source: camera.source(), + width: Some(mode.width), + height: Some(mode.height), + framerate: mode.max_framerate(), + ..Default::default() + }; + let stream = capture(&config) .await - .unwrap_or_else(|error| panic!("{}: read frame {i}: {error}", camera.id)) - .unwrap_or_else(|| panic!("{}: no frame {i}", camera.id)); - assert_eq!(frame.surface.width(), stream.width()); - assert_eq!(frame.surface.height(), stream.height()); + .unwrap_or_else(|error| panic!("{} at {}x{}: {error}", camera.id, mode.width, mode.height)); + assert_eq!(stream.label(), camera.id); + assert_eq!((stream.width(), stream.height()), (mode.width, mode.height)); + assert_eq!(stream.framerate(), mode.max_framerate()); } - eprintln!( - "{}: captured 5 frames at {}x{}, {:?} fps, color {:?}", - camera.id, - stream.width(), - stream.height(), - stream.framerate(), - stream.color() - ); captured += 1; } assert!(captured > 0, "no PipeWire camera could be captured"); - // `pipewire` alone leaves the choice to the session manager. + // `pipewire` alone picks the highest-priority camera, at the default size. let config = Config { source: crate::capture::Source::Camera(Some(PIPEWIRE.to_string())), ..Default::default() }; - let mut stream = crate::capture::open(&config).await.expect("default PipeWire camera"); + let stream = capture(&config).await.expect("default PipeWire camera"); assert_eq!(stream.label(), PIPEWIRE); - stream - .read() - .await - .expect("read") - .expect("a frame from the default camera"); } } diff --git a/rs/moq-video/src/capture/v4l2.rs b/rs/moq-video/src/capture/v4l2.rs index 128c6016b41..d3451e233b7 100644 --- a/rs/moq-video/src/capture/v4l2.rs +++ b/rs/moq-video/src/capture/v4l2.rs @@ -6,8 +6,6 @@ //! the pure-Rust [`zune_jpeg`], then converted). This is the CPU path feeding //! NVENC / VAAPI / openh264; there's no GPU surface here. -use std::collections::{BTreeMap, BTreeSet}; - use moq_v4l::buffer::Type as BufType; use moq_v4l::capability::Flags; use moq_v4l::frameinterval::FrameIntervalEnum; @@ -17,9 +15,9 @@ use moq_v4l::io::traits::CaptureStream; use moq_v4l::video::Capture; use moq_v4l::video::capture::Parameters; use moq_v4l::{Device, Format, FourCC}; -use zune_jpeg::zune_core::bytestream::ZCursor; use super::channel::FrameChannel; +use super::mode::{self, Request}; use super::pump::{self, Geometry}; use super::{Config, Mode, Rate, Stream}; use crate::frame::{I420, Surface}; @@ -75,34 +73,19 @@ pub(super) fn cameras() -> Result, Error> { /// on. pub(super) fn modes(selector: Option<&str>) -> Result, Error> { let (device, _) = open_device(selector)?; - let mut sizes: BTreeMap<(u32, u32), BTreeSet> = BTreeMap::new(); + let mut reported = Vec::new(); for candidate in Source::ALL { let fourcc = candidate.fourcc(); let enumerated = frame_sizes(&device, fourcc)?; for size in enumerated { for (width, height) in reported_sizes(size) { - sizes.entry((width, height)).or_default().extend(framerates( - &device, - fourcc, - Size::new(width, height), - )?); + let size = Size::new(width, height); + reported.push((size, framerates(&device, fourcc, size)?)); } } } - - let mut modes: Vec = sizes - .into_iter() - .map(|((width, height), framerates)| Mode { - width, - height, - // Highest first, so the rate a caller most often wants is the one it - // reads without scanning. - framerates: framerates.into_iter().rev().collect(), - }) - .collect(); - modes.sort_by_key(|mode| std::cmp::Reverse(u64::from(mode.width) * u64::from(mode.height))); - Ok(modes) + Ok(mode::modes(reported)) } /// The sizes worth reporting from one `VIDIOC_ENUM_FRAMESIZES` entry. @@ -260,9 +243,9 @@ pub(super) async fn open(config: &Config, device: Option<&str>) -> Result { // Only `bytesused` of the buffer holds the JPEG; the rest is stale. let jpeg = buf.get(..meta.bytesused as usize).unwrap_or(buf); - // zune-jpeg 0.5 reads through a seekable cursor, not a bare slice. - let mut decoder = zune_jpeg::JpegDecoder::new(ZCursor::new(jpeg)); - let rgb = decoder - .decode() - .map_err(|e| Error::Codec(anyhow::anyhow!("MJPEG decode: {e:?}")))?; - let (w, h) = decoder - .dimensions() - .ok_or_else(|| Error::Codec(anyhow::anyhow!("MJPEG frame had no dimensions")))?; - // The stream reports the negotiated size and the encoder is built - // from it, so a frame that decodes to another one can't be published - // as this stream's. - if w as u32 != self.width || h as u32 != self.height { - return Err(Error::Codec(anyhow::anyhow!( - "MJPEG frame is {w}x{h}, not the negotiated {}x{}", - self.width, - self.height - ))); - } - I420::from_rgb(&rgb, crate::Size::new(self.width, self.height))? + I420::from_mjpeg(jpeg, crate::Size::new(self.width, self.height))? } }; let timestamp = u64::try_from(meta.timestamp.sec) @@ -425,11 +390,6 @@ fn open_error(device: &str, error: std::io::Error) -> Error { } } -struct Request { - size: Size, - framerate: Option, -} - /// Negotiate the format we can convert to I420 that lands closest to `want`. /// /// V4L2's non-mutating `VIDIOC_TRY_FMT` is optional, so use the required @@ -524,39 +484,11 @@ fn closest( replies: impl IntoIterator)>, want: Request, ) -> Option<(Format, Source, Option)> { - replies - .into_iter() - .filter(|(format, _, _)| { - Size::new(format.width, format.height) - .validate("camera resolution") - .is_ok() - }) - .min_by(|(left, left_source, left_rate), (right, right_source, right_rate)| { - distance(*left, want.size) - .cmp(&distance(*right, want.size)) - .then_with(|| rate_distance(*left_rate, *right_rate, want.framerate)) - .then_with(|| left_source.cost().cmp(&right_source.cost())) - }) -} - -fn rate_distance(left: Option, right: Option, want: Option) -> std::cmp::Ordering { - let Some(want) = want else { - return std::cmp::Ordering::Equal; - }; - match (left, right) { - (Some(left), Some(right)) => (left.as_f64() - want.as_f64()) - .abs() - .total_cmp(&(right.as_f64() - want.as_f64()).abs()), - (Some(_), None) => std::cmp::Ordering::Less, - (None, Some(_)) => std::cmp::Ordering::Greater, - (None, None) => std::cmp::Ordering::Equal, - } -} - -/// How far a negotiated mode lands from the requested geometry, summed over both -/// dimensions. Zero is an exact match. -fn distance(format: Format, want: Size) -> u64 { - u64::from(format.width.abs_diff(want.width)) + u64::from(format.height.abs_diff(want.height)) + mode::nearest(replies, want, |(format, source, rate)| mode::Candidate { + size: Size::new(format.width, format.height), + framerate: *rate, + cost: source.cost(), + }) } fn set_format(device: &Device, format: Format) -> Result { @@ -625,6 +557,8 @@ mod tests { #[test] fn equivalent_rates_deduplicate_and_sort_numerically() { + use std::collections::BTreeSet; + let rates: BTreeSet<_> = [(1001, 30000), (1, 30), (2, 60), (2, 1)] .into_iter() .map(|(n, d)| rate(moq_v4l::Fraction::new(n, d)).unwrap()) @@ -668,16 +602,6 @@ mod tests { ); } - #[test] - fn rate_scoring_preserves_fractional_precision_and_handles_unknown_rates() { - let ntsc = Some(rate(moq_v4l::Fraction::new(1001, 60000)).unwrap()); - let thirty = Some(rate(moq_v4l::Fraction::new(1, 30)).unwrap()); - assert!(rate_distance(ntsc, thirty, Some(Rate::new(60, 1).unwrap())).is_lt()); - assert!(rate_distance(thirty, ntsc, Some(Rate::new(30, 1).unwrap())).is_lt()); - assert!(rate_distance(ntsc, None, Some(Rate::new(60, 1).unwrap())).is_lt()); - assert!(rate_distance(ntsc, thirty, None).is_eq()); - } - /// A one-pixel step over a 4K range is 8 million modes, and reporting them /// would say nothing the two corners do not. #[test] @@ -797,16 +721,6 @@ mod tests { assert!(closest(replies, request(1280, 720)).is_none()); } - /// Distance is symmetric in the two dimensions and zero only on an exact hit, - /// so a mode that overshoots is no better than one that undershoots by as much. - #[test] - fn distance_is_zero_only_on_an_exact_match() { - let want = Size::new(1280, 720); - assert_eq!(distance(Format::new(1280, 720, Source::Yuyv.fourcc()), want), 0); - assert_eq!(distance(Format::new(1280, 600, Source::Yuyv.fourcc()), want), 120); - assert_eq!(distance(Format::new(1280, 840, Source::Yuyv.fourcc()), want), 120); - } - /// Every format we advertise round-trips through its fourcc, which is what /// lets `negotiate` recognize the driver's substitution. #[test] diff --git a/rs/moq-video/src/frame.rs b/rs/moq-video/src/frame.rs index 05cf9045ba6..6c166c96e11 100644 --- a/rs/moq-video/src/frame.rs +++ b/rs/moq-video/src/frame.rs @@ -843,8 +843,7 @@ impl I420 { } /// Convert tightly-packed RGB (`width * height * 3` bytes) to I420 in - /// [`Color::infer`]'s color space for this size. Used for MJPEG capture - /// (Linux V4L2), which decodes to RGB. + /// [`Color::infer`]'s color space for this size. Used by [`Self::from_mjpeg`]. #[cfg(all(target_os = "linux", feature = "capture"))] pub(crate) fn from_rgb(rgb: &[u8], size: Size) -> Result { use yuv::rgb_to_yuv420; @@ -864,6 +863,32 @@ impl I420 { Self::pack(&planar, size, Some(color)) } + /// Decode one Motion-JPEG frame to I420 in [`Color::infer`]'s color space. + /// Used by the Linux V4L2 and PipeWire camera paths. + /// + /// The stream reports the negotiated size and the encoder is built from it, + /// so a frame that decodes to another size is an error rather than a frame + /// published as this stream's. + #[cfg(all(target_os = "linux", feature = "capture"))] + pub(crate) fn from_mjpeg(jpeg: &[u8], size: Size) -> Result { + use zune_jpeg::zune_core::bytestream::ZCursor; + + // zune-jpeg 0.5 reads through a seekable cursor, not a bare slice. + let mut decoder = zune_jpeg::JpegDecoder::new(ZCursor::new(jpeg)); + let rgb = decoder + .decode() + .map_err(|e| Error::Codec(anyhow::anyhow!("MJPEG decode: {e:?}")))?; + let (w, h) = decoder + .dimensions() + .ok_or_else(|| Error::Codec(anyhow::anyhow!("MJPEG frame had no dimensions")))?; + if w as u32 != size.width || h as u32 != size.height { + return Err(Error::Codec(anyhow::anyhow!( + "MJPEG frame is {w}x{h}, not the negotiated {size}" + ))); + } + Self::from_rgb(&rgb, size) + } + /// Convert packed YUYV (YUV 4:2:2, `stride` bytes per row) to I420. A chroma /// resample (4:2:2 -> 4:2:0), no color-space conversion. Used for the raw /// V4L2 and PipeWire camera capture paths (Linux). From e829fd502e35a661f14e72bd2f4d56639d62c896 Mon Sep 17 00:00:00 2001 From: Frando Date: Thu, 24 Sep 2026 14:41:36 +0200 Subject: [PATCH 3/4] fix(moq-video): list no PipeWire camera when the portal has none Inside a sandbox, `cameras()` asked the camera portal for a remote and failed with `SourceUnavailable` when the portal reported no camera. The Linux `capture::cameras` joins the V4L2 and PipeWire lists with `?`, so a Flatpak or Snap without a camera lost the V4L2 list as well. Listing now returns no PipeWire cameras in that case; opening a camera and reading its modes still fail with the error. Co-Authored-By: Claude Opus 5.5 (1M context) --- rs/moq-video/src/capture/pipewire/camera.rs | 27 +++++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/rs/moq-video/src/capture/pipewire/camera.rs b/rs/moq-video/src/capture/pipewire/camera.rs index 2865c40a1d7..76218672ec7 100644 --- a/rs/moq-video/src/capture/pipewire/camera.rs +++ b/rs/moq-video/src/capture/pipewire/camera.rs @@ -39,8 +39,16 @@ const QUERY_TIMEOUT: Duration = Duration::from_secs(5); /// List the PipeWire cameras as [`Camera`]s with `pipewire:` ids. pub(in crate::capture) async fn cameras() -> Result, Error> { - let remote = remote().await?; - let sandboxed = remote.is_some(); + let sandboxed = ashpd::is_sandboxed(); + let remote = if sandboxed { + match portal().await? { + Some(fd) => Some(fd), + // No camera lists as nothing, so the V4L2 cameras still show. + None => return Ok(Vec::new()), + } + } else { + None + }; let nodes = crate::capture::blocking(move || { pw::init(); let mainloop = pw::main_loop::MainLoopRc::new(None).map_err(|e| err("pipewire main loop", e))?; @@ -124,13 +132,22 @@ async fn remote() -> Result, Error> { if !ashpd::is_sandboxed() { return Ok(None); } + match portal().await? { + Some(fd) => Ok(Some(fd)), + None => Err(Error::SourceUnavailable( + "the camera portal reports no camera".to_string(), + )), + } +} + +/// Ask the camera portal for a PipeWire remote that exposes the cameras, or +/// `None` when the portal reports no camera. +async fn portal() -> Result, Error> { let portal = ashpd::desktop::camera::Camera::new() .await .map_err(|e| err("camera portal", e))?; if !portal.is_present().await.map_err(|e| err("camera portal", e))? { - return Err(Error::SourceUnavailable( - "the camera portal reports no camera".to_string(), - )); + return Ok(None); } // The portal asks the user unless the sandbox's permission store already // holds a grant, so this blocks on the user the first time. From fad5ea5e8ba6eb77779c5acaa12746f299436142 Mon Sep 17 00:00:00 2001 From: Frando Date: Thu, 24 Sep 2026 14:41:37 +0200 Subject: [PATCH 4/4] fix(moq-video): keep PipeWire size steps and rate ranges in selection A `Step` size choice was treated like a `Range`: the step was dropped, so a request between the corners was offered even when it fell off the camera's grid, and odd corners were kept raw, so the selection dropped them instead of moving them to an even size. Both are now read as a grid. Its corners go through `v4l2::bounds`, as a stepwise V4L2 size does, and the grid offers its even size nearest the request, the way a V4L2 driver rounds a stepwise request. A continuous rate range was ranked as "no rate", which loses to any exact rate another format lists for the same size, so a 60 fps request could pin an MJPEG mode at 30 fps while an NV12 range could deliver 60. A range that holds the requested rate now offers exactly that rate. Co-Authored-By: Claude Opus 5.5 (1M context) --- rs/moq-video/src/capture/pipewire/camera.rs | 308 ++++++++++++++++---- rs/moq-video/src/capture/v4l2.rs | 2 +- 2 files changed, 253 insertions(+), 57 deletions(-) diff --git a/rs/moq-video/src/capture/pipewire/camera.rs b/rs/moq-video/src/capture/pipewire/camera.rs index 76218672ec7..f19b0116e74 100644 --- a/rs/moq-video/src/capture/pipewire/camera.rs +++ b/rs/moq-video/src/capture/pipewire/camera.rs @@ -29,7 +29,7 @@ use spa::utils::ChoiceEnum; use super::{Capture, DEFAULT_FRAMERATE, Kind, Target, drm_format, err, linear_modifier, serialize_format}; use crate::capture::mode::{self, Request}; -use crate::capture::v4l2::{DEFAULT_HEIGHT, DEFAULT_WIDTH}; +use crate::capture::v4l2::{DEFAULT_HEIGHT, DEFAULT_WIDTH, bounds}; use crate::capture::{Camera, Config, Mode, PIPEWIRE, Stream}; use crate::{Error, Rate, Size}; @@ -362,8 +362,67 @@ pub(super) struct Format { sizes: Vec, /// The exact rates on offer, highest first. Empty for a rate range. rates: Vec, - /// Whether the size is a range that also takes the requested size. - ranged: Option<(Size, Size)>, + /// The size range, when the size is one, which also offers the grid size + /// nearest the request. + ranged: Option, + /// The lowest and highest rate, when the rate is a continuous range. + rate_range: Option<(spa::utils::Fraction, spa::utils::Fraction)>, +} + +/// A size range: every size from `min` to `max` in `step` increments. +#[derive(Clone, Copy, Debug, PartialEq)] +struct Grid { + min: Size, + max: Size, + step: Size, +} + +impl Grid { + /// The size on the grid nearest `want` with even dimensions, the way a V4L2 + /// driver rounds a request for a stepwise size. + fn nearest(&self, want: Size) -> Option { + let snap = |value: u32, min: u32, max: u32, step: u32| { + let (first, last) = bounds(min, max, step)?; + let value = value.clamp(first, last); + if first == last { + return Some(first); + } + // `bounds` spans more than one value only on a nonzero step. + let below = min + (value - min) / step * step; + [below.checked_sub(step), Some(below), below.checked_add(step)] + .into_iter() + .flatten() + .filter(|size| size.is_multiple_of(2) && (first..=last).contains(size)) + .min_by_key(|size| size.abs_diff(value)) + }; + Some(Size::new( + snap(want.width, self.min.width, self.max.width, self.step.width)?, + snap(want.height, self.min.height, self.max.height, self.step.height)?, + )) + } + + /// The largest and smallest sizes on the grid with even dimensions, the way + /// the V4L2 backend reports a stepwise size. + fn corners(&self) -> Vec { + let width = bounds(self.min.width, self.max.width, self.step.width); + let height = bounds(self.min.height, self.max.height, self.step.height); + let (Some((min_width, max_width)), Some((min_height, max_height))) = (width, height) else { + return Vec::new(); + }; + let (largest, smallest) = (Size::new(max_width, max_height), Size::new(min_width, min_height)); + if largest == smallest { + vec![largest] + } else { + vec![largest, smallest] + } + } +} + +/// Whether the rate range `min..=max` holds `rate`, compared as cross products +/// because a range may start at 0/1, which is no valid [`Rate`]. +fn holds(min: spa::utils::Fraction, max: spa::utils::Fraction, rate: Rate) -> bool { + let (num, denom) = (u64::from(rate.numerator()), u64::from(rate.denominator())); + u64::from(min.num) * denom <= num * u64::from(min.denom) && num * u64::from(max.denom) <= u64::from(max.num) * denom } /// The mode a camera stream offers. @@ -371,7 +430,7 @@ pub(super) struct Format { pub(super) struct Selected { encoding: Encoding, size: Size, - /// `None` when the camera reports a rate range rather than exact rates. + /// `None` when the camera reports a rate range and no rate in it was requested. framerate: Option, } @@ -528,22 +587,44 @@ fn parse_format(value: &Value) -> Vec { let size = |rectangle: &spa::utils::Rectangle| Size::new(rectangle.width, rectangle.height); let (sizes, ranged) = match property(FormatProperties::VideoSize) { Some(Value::Rectangle(rectangle)) => (vec![size(rectangle)], None), - Some(Value::Choice(ChoiceValue::Rectangle(choice))) => match &choice.1 { - ChoiceEnum::Range { min, max, .. } | ChoiceEnum::Step { min, max, .. } => { - (vec![size(max), size(min)], Some((size(min), size(max)))) + Some(Value::Choice(ChoiceValue::Rectangle(choice))) => { + let grid = match &choice.1 { + ChoiceEnum::Range { min, max, .. } => Some(Grid { + min: size(min), + max: size(max), + step: Size::new(1, 1), + }), + ChoiceEnum::Step { min, max, step, .. } => Some(Grid { + min: size(min), + max: size(max), + step: size(step), + }), + _ => None, + }; + match grid { + Some(grid) => (grid.corners(), Some(grid)), + None => (listed(&choice.1).iter().map(size).collect(), None), } - other => (listed(other).iter().map(size).collect(), None), - }, + } _ => (Vec::new(), None), }; let rate = |fraction: &spa::utils::Fraction| Rate::new(fraction.num, fraction.denom).ok(); - let mut rates: Vec = match property(FormatProperties::VideoFramerate) { + let framerate = property(FormatProperties::VideoFramerate); + let mut rates: Vec = match framerate { Some(Value::Fraction(fraction)) => rate(fraction).into_iter().collect(), Some(Value::Choice(ChoiceValue::Fraction(choice))) => listed(&choice.1).iter().filter_map(rate).collect(), _ => Vec::new(), }; rates.sort_by(|left, right| right.cmp(left)); rates.dedup(); + // Only a continuous range: whether a rate lies on a stepwise range's grid + // is the driver's call, so that offer stays open. + let rate_range = match framerate { + Some(Value::Choice(ChoiceValue::Fraction(spa::utils::Choice(_, ChoiceEnum::Range { min, max, .. })))) => { + Some((*min, *max)) + } + _ => None, + }; encodings .into_iter() @@ -552,6 +633,7 @@ fn parse_format(value: &Value) -> Vec { sizes: sizes.clone(), rates: rates.clone(), ranged, + rate_range, }) .collect() } @@ -576,19 +658,21 @@ fn listed(choice: &Choic } /// Pick the mode nearest `want` with the V4L2 backend's rules. A size range -/// that holds the requested size offers it exactly. +/// offers its size nearest the request, and a rate range that holds the +/// requested rate offers that rate. fn select(formats: &[Format], want: Request) -> Option { let mut candidates = Vec::new(); for format in formats { - let within = format.ranged.is_some_and(|(min, max)| { - (min.width..=max.width).contains(&want.size.width) && (min.height..=max.height).contains(&want.size.height) - }); - let sizes = within - .then_some(want.size) + let sizes = format + .ranged + .and_then(|grid| grid.nearest(want.size)) .into_iter() .chain(format.sizes.iter().copied()); let rates: Vec> = if format.rates.is_empty() { - vec![None] + let held = want + .framerate + .filter(|&rate| format.rate_range.is_some_and(|(min, max)| holds(min, max, rate))); + vec![held] } else { format.rates.iter().copied().map(Some).collect() }; @@ -805,50 +889,162 @@ mod tests { assert_eq!(select(1280, 800), (Encoding::Mjpeg, Size::new(1280, 720))); } - #[test] - fn a_size_range_offers_the_requested_size() { - let ranged = Value::Object(spa::pod::Object { + /// An NV12 `EnumFormat` entry with the given size and rate properties, the way + /// spa-libcamera reports a sensor. + fn nv12_format(size: Value, framerate: Option) -> Vec { + let property = |key: FormatProperties, value| spa::pod::Property::new(key.as_raw(), value); + let mut properties = vec![ + property( + FormatProperties::MediaType, + Value::Id(spa::utils::Id(MediaType::Video.as_raw())), + ), + property( + FormatProperties::MediaSubtype, + Value::Id(spa::utils::Id(MediaSubtype::Raw.as_raw())), + ), + property( + FormatProperties::VideoFormat, + Value::Id(spa::utils::Id(VideoFormat::NV12.as_raw())), + ), + property(FormatProperties::VideoSize, size), + ]; + properties.extend(framerate.map(|framerate| property(FormatProperties::VideoFramerate, framerate))); + parse_format(&Value::Object(spa::pod::Object { type_: spa::utils::SpaTypes::ObjectParamFormat.as_raw(), id: spa::param::ParamType::EnumFormat.as_raw(), - properties: vec![ - spa::pod::Property::new( - FormatProperties::MediaType.as_raw(), - Value::Id(spa::utils::Id(MediaType::Video.as_raw())), - ), - spa::pod::Property::new( - FormatProperties::MediaSubtype.as_raw(), - Value::Id(spa::utils::Id(MediaSubtype::Raw.as_raw())), - ), - spa::pod::Property::new( - FormatProperties::VideoFormat.as_raw(), - Value::Id(spa::utils::Id(VideoFormat::NV12.as_raw())), - ), - spa::pod::Property::new( - FormatProperties::VideoSize.as_raw(), - Value::Choice(ChoiceValue::Rectangle(spa::utils::Choice( - spa::utils::ChoiceFlags::empty(), - ChoiceEnum::Range { - default: rectangle(1920, 1080), - min: rectangle(64, 64), - max: rectangle(4608, 2592), - }, - ))), - ), - ], - }); - let formats = parse_format(&ranged); + properties, + })) + } + + fn size_choice(choice: ChoiceEnum) -> Value { + Value::Choice(ChoiceValue::Rectangle(spa::utils::Choice( + spa::utils::ChoiceFlags::empty(), + choice, + ))) + } + + fn ratio(num: u32, denom: u32) -> spa::utils::Fraction { + spa::utils::Fraction { num, denom } + } + + fn request(width: u32, height: u32, fps: Option) -> Request { + Request { + size: Size::new(width, height), + framerate: fps.map(|fps| Rate::new(fps, 1).unwrap()), + } + } + + #[test] + fn a_size_range_offers_the_requested_size() { + let formats = nv12_format( + size_choice(ChoiceEnum::Range { + default: rectangle(1920, 1080), + min: rectangle(64, 64), + max: rectangle(4608, 2592), + }), + None, + ); assert_eq!(formats[0].sizes, [Size::new(4608, 2592), Size::new(64, 64)]); + let selected = select(&formats, request(1280, 720, None)).unwrap(); + assert_eq!((selected.size, selected.framerate), (Size::new(1280, 720), None)); + // An odd request lands on the nearest even size rather than a corner. + let selected = select(&formats, request(1281, 721, None)).unwrap(); + assert_eq!(selected.size, Size::new(1280, 720)); + } + + /// A stepwise range offers only sizes on its grid, and its corners move to + /// even sizes the way `v4l2::bounds` moves a stepwise V4L2 size. + #[test] + fn a_stepwise_size_range_offers_the_nearest_even_size_on_its_grid() { + let eights = nv12_format( + size_choice(ChoiceEnum::Step { + default: rectangle(640, 480), + min: rectangle(32, 32), + max: rectangle(1921, 1081), + step: rectangle(8, 8), + }), + None, + ); + assert_eq!(eights[0].sizes, [Size::new(1920, 1080), Size::new(32, 32)]); + assert_eq!( + select(&eights, request(1280, 720, None)).unwrap().size, + Size::new(1280, 720) + ); + assert_eq!( + select(&eights, request(1270, 716, None)).unwrap().size, + Size::new(1272, 712) + ); + assert_eq!( + select(&eights, request(4000, 3000, None)).unwrap().size, + Size::new(1920, 1080) + ); + + // An odd step alternates parity, so the nearest grid size may be odd. + let threes = nv12_format( + size_choice(ChoiceEnum::Step { + default: rectangle(640, 480), + min: rectangle(1, 1), + max: rectangle(1919, 1079), + step: rectangle(3, 3), + }), + None, + ); + assert_eq!(threes[0].sizes, [Size::new(1918, 1078), Size::new(4, 4)]); + assert_eq!( + select(&threes, request(1280, 720, None)).unwrap().size, + Size::new(1282, 718) + ); + + // An even step from an odd minimum has no even size at all. + let odd = nv12_format( + size_choice(ChoiceEnum::Step { + default: rectangle(641, 481), + min: rectangle(1, 1), + max: rectangle(1919, 1079), + step: rectangle(2, 2), + }), + None, + ); + assert!(odd[0].sizes.is_empty()); + assert_eq!(select(&odd, request(1280, 720, None)), None); + } + + /// A rate range that holds the requested rate ranks as that rate, so an exact + /// rate elsewhere does not beat it. + #[test] + fn a_rate_range_offers_the_requested_rate() { + let range = Value::Choice(ChoiceValue::Fraction(spa::utils::Choice( + spa::utils::ChoiceFlags::empty(), + ChoiceEnum::Range { + default: ratio(30, 1), + min: ratio(0, 1), + max: ratio(60, 1), + }, + ))); + let mut formats = nv12_format(Value::Rectangle(rectangle(1280, 720)), Some(range)); // A rate range lists no exact rate. assert!(formats[0].rates.is_empty()); - let selected = select( - &formats, - Request { - size: Size::new(1280, 720), - framerate: None, - }, - ) - .unwrap(); - assert_eq!((selected.size, selected.framerate), (Size::new(1280, 720), None)); + formats.extend(parse_format(&enum_format( + MediaSubtype::Mjpg, + None, + rectangle(1280, 720), + ))); + + let pick = |fps| { + let selected = select(&formats, request(1280, 720, fps)).unwrap(); + (selected.encoding, selected.framerate) + }; + let nv12 = Encoding::Raw(VideoFormat::NV12); + let rate = |fps| Some(Rate::new(fps, 1).unwrap()); + assert_eq!(pick(Some(60)), (nv12, rate(60))); + assert_eq!(pick(Some(30)), (nv12, rate(30))); + // Out of the range, the exact rate elsewhere is the better answer. + assert_eq!(pick(Some(120)), (Encoding::Mjpeg, rate(30))); + // Nothing requested: the range stays open and the cheaper format wins. + assert_eq!(pick(None), (nv12, None)); + + assert!(holds(ratio(0, 1), ratio(30, 1), Rate::new(30000, 1001).unwrap())); + assert!(!holds(ratio(0, 1), ratio(30, 1), Rate::new(30001, 1000).unwrap())); } /// The offer pins the chosen mode, and only NV12 gets a DMA-BUF offer. diff --git a/rs/moq-video/src/capture/v4l2.rs b/rs/moq-video/src/capture/v4l2.rs index d3451e233b7..c1e8099db52 100644 --- a/rs/moq-video/src/capture/v4l2.rs +++ b/rs/moq-video/src/capture/v4l2.rs @@ -121,7 +121,7 @@ fn reported_sizes(size: FrameSizeEnum) -> Vec<(u32, u32)> { } /// First and last nonzero even values on the driver's step grid. -fn bounds(min: u32, max: u32, step: u32) -> Option<(u32, u32)> { +pub(super) fn bounds(min: u32, max: u32, step: u32) -> Option<(u32, u32)> { if min == max { return (min != 0 && min.is_multiple_of(2)).then_some((min, min)); }