diff --git a/doc/bin/cli.md b/doc/bin/cli.md index fcef336af15..54e9a13e8c0 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -126,8 +126,8 @@ audio is Opus. The camera is opened only while someone is watching, and `--bitrate` is the opening ceiling. Backends with live bitrate control lower it to fit the connection's bandwidth estimate. `moq devices` prints every source id. Requires the `capture` feature; on Linux that needs the ALSA headers for -the microphone, and `--display` also needs the `pipewire` feature (links -libpipewire). +the microphone, and `--display` and `pipewire:` cameras also need the +`pipewire` feature (links libpipewire). ## Transcode diff --git a/doc/lib/rs/moq-video.md b/doc/lib/rs/moq-video.md index 6d2d2c7fcf2..1c5ab813366 100644 --- a/doc/lib/rs/moq-video.md +++ b/doc/lib/rs/moq-video.md @@ -39,11 +39,21 @@ and an empty rate list means no discrete intervals were reported. Device errors are returned rather than treated as an empty list. Other platforms return `Error::Unsupported`. +With `pipewire` enabled, `capture::cameras` also lists PipeWire camera nodes as +`pipewire:` after the V4L2 devices, and `pipewire` alone opens the +camera with the highest session priority, the session manager's default. That +reaches cameras V4L2 cannot: a Raspberry Pi CSI camera behind libcamera, and any +camera from inside a Flatpak or Snap sandbox, where the default camera comes +through the xdg-desktop-portal Camera interface. The mode is chosen from the +node's own format list by the same rules as V4L2 (below) and offered exactly, +across YUY2, NV12, RGB, and MJPEG, and `camera_modes` lists the same modes. + `capture::Config::framerate` is an `Option` request in the same exact type; the stream reports the rate the device accepted, or `None` when the driver reported none. -V4L2 chooses the closest geometry, then the format whose accepted rate is -nearest the request, then the cheaper conversion when both match equally well. +V4L2 and PipeWire cameras choose the closest geometry, then the format whose +accepted rate is nearest the request, then the cheaper conversion when both +match equally well. ```rust let mut video = moq_video::decode::Consumer::new(&broadcast, &rendition, "video", Default::default()).await?; @@ -59,7 +69,7 @@ cargo add moq-video --features capture # camera + screen capture, no system bu cargo add moq-video --features render # wgpu rendering cargo add moq-video --features v4l2 # Linux V4L2 M2M codecs, no system build deps cargo add moq-video --features vaapi # Linux VAAPI codecs (bindgen needs libclang) -cargo add moq-video --features pipewire # Wayland screen capture (links libpipewire) +cargo add moq-video --features pipewire # Wayland screen + PipeWire cameras (links libpipewire) cargo add moq-video --no-default-features --features openh264 # software H.264 only cargo add moq-video --no-default-features --features nvidia # Linux NVIDIA only, no C++ or wgpu ``` diff --git a/rs/moq-cli/Cargo.toml b/rs/moq-cli/Cargo.toml index 9c496a7b7b4..a4c291b1bc3 100644 --- a/rs/moq-cli/Cargo.toml +++ b/rs/moq-cli/Cargo.toml @@ -74,8 +74,8 @@ vaapi = ["moq-video?/vaapi", "moq-transcode?/vaapi"] # checks its bindings in. Only applies when `capture`, `transcode`, or `play` # pulls in moq-video. v4l2 = ["moq-video?/v4l2", "moq-transcode?/v4l2"] -# Screen capture for `capture --display` on Linux (xdg-desktop-portal + PipeWire), -# opt-in like moq-video's own `pipewire` because it links libpipewire-0.3 at +# Screen capture for `capture --display` on Linux (xdg-desktop-portal + PipeWire) +# and PipeWire cameras for `capture --camera pipewire:`, opt-in like moq-video's own `pipewire` because it links libpipewire-0.3 at # build time. Only bites when `capture` pulls in moq-video and moq-audio. # Also reaches audio, so `devices`, microphone capture and playback go through # the sound server rather than its ALSA plugin on a machine running one. diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index b26616c98fc..6360d44f6d1 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -51,7 +51,8 @@ impl From for moq_video::encode::Codec { #[usage(group("audio-source"))] pub struct CaptureArgs { /// Capture a camera, by the id `moq devices` reports (an AVFoundation - /// `uniqueID`, `/dev/videoN` path, or Media Foundation symbolic link). + /// `uniqueID`, `/dev/videoN` path, `pipewire:` node, or Media Foundation + /// symbolic link). /// Bare `--camera`, or no source flag at all, opens the default camera. #[usage(long, group = "video-source")] pub camera: Option>, diff --git a/rs/moq-video/Cargo.toml b/rs/moq-video/Cargo.toml index 2828effa43d..3e5e6508d8e 100644 --- a/rs/moq-video/Cargo.toml +++ b/rs/moq-video/Cargo.toml @@ -86,10 +86,12 @@ render = [ "objc2-core-video/CVMetalTextureCache", "objc2-core-video/objc2-metal", ] -# Screen capture on Linux via xdg-desktop-portal + PipeWire (Wayland and X11). -# Off by default because the `pipewire` crate links libpipewire-0.3 via pkg-config -# at build time (and its bindgen needs libclang), so it only belongs in builds -# that actually capture displays. Camera capture (V4L2) needs only what +# Screen capture on Linux via xdg-desktop-portal + PipeWire (Wayland and X11), +# and camera capture from PipeWire camera nodes: libcamera sensors such as a +# Raspberry Pi CSI camera, and any camera from inside a Flatpak or Snap sandbox +# through the camera portal. Off by default because the `pipewire` crate links +# libpipewire-0.3 via pkg-config at build time (and its bindgen needs libclang), +# so it only belongs in builds that use it. V4L2 camera capture needs only what # `capture` already does. pipewire = ["capture", "dmabuf", "dep:pipewire", "dep:ashpd"] @@ -136,10 +138,12 @@ yuv = "0.8.14" ndk = { version = "0.9", optional = true, default-features = false, features = ["media", "api-level-26"] } [target.'cfg(target_os="linux")'.dependencies] -# xdg-desktop-portal ScreenCast negotiation for the `pipewire` feature (pure Rust, -# zbus over DBus). The portal owns source selection: the user picks a monitor in a -# compositor dialog, and we get back a PipeWire fd + node id to stream from. -ashpd = { version = "0.13", optional = true, default-features = false, features = ["tokio", "screencast"] } +# xdg-desktop-portal ScreenCast and Camera negotiation for the `pipewire` feature +# (pure Rust, zbus over DBus). The ScreenCast portal owns source selection: the +# user picks a monitor in a compositor dialog, and we get back a PipeWire fd + +# node id to stream from. The Camera portal grants a sandboxed app a PipeWire +# remote that exposes the cameras. +ashpd = { version = "0.13", optional = true, default-features = false, features = ["tokio", "screencast", "camera"] } # The NVIDIA codecs, behind the default-on `nvidia` feature. Everything is # dlopen'd at runtime: cudarc loads CUDA, and moq-nvenc (an in-tree fork of # nvidia-video-codec-sdk, see rs/moq-nvenc) loads libnvidia-encode. So the binary @@ -176,7 +180,8 @@ moq-v4l = { workspace = true, optional = true } # then falls back to the next candidate instead of failing at load, matching the # NVENC and NVDEC backends. moq-vaapi = { workspace = true, optional = true } -# PipeWire video stream for the portal's node, behind the `pipewire` feature. +# PipeWire video stream for a portal's node or a camera node, behind the +# `pipewire` feature. # Links libpipewire-0.3 via pkg-config at build time; bindgen needs libclang. pipewire = { version = "0.10", optional = true } # X11 display/window enumeration and capture. Pure Rust and used when no diff --git a/rs/moq-video/src/capture/mod.rs b/rs/moq-video/src/capture/mod.rs index 9e7bd5a9690..25fe9784c50 100644 --- a/rs/moq-video/src/capture/mod.rs +++ b/rs/moq-video/src/capture/mod.rs @@ -2,9 +2,9 @@ //! per-source: //! - macOS camera -> AVFoundation, screen -> ScreenCaptureKit, both yielding //! zero-copy `CVPixelBuffer` surfaces straight to VideoToolbox. -//! - Linux camera -> native V4L2 (YUYV / MJPEG -> CPU I420), X11 display and -//! window -> X11, Wayland display -> xdg-desktop-portal + PipeWire -//! (`pipewire` feature). +//! - Linux camera -> native V4L2 (YUYV / MJPEG -> CPU I420), or a PipeWire +//! camera node (`pipewire` feature), X11 display and window -> X11, Wayland +//! display -> xdg-desktop-portal + PipeWire (`pipewire` feature). //! - Windows camera -> native Media Foundation (`IMFSourceReader`), screen -> //! DXGI Desktop Duplication, window -> GDI (BGRA -> CPU I420). //! @@ -39,11 +39,14 @@ mod surface; // Native V4L2 camera capture on Linux. #[cfg(target_os = "linux")] mod v4l2; +// The mode choice V4L2 and PipeWire cameras share. +#[cfg(target_os = "linux")] +mod mode; // Native X11 display and window capture, including the portal fallback. #[cfg(target_os = "linux")] mod x11; -// Portal + PipeWire screen capture on Linux. +// Portal + PipeWire screen capture and PipeWire camera capture on Linux. #[cfg(all(target_os = "linux", feature = "pipewire"))] mod pipewire; @@ -82,6 +85,14 @@ pub enum Source { /// The identifiers from [`cameras`] are an AVFoundation `uniqueID` on macOS, /// a `/dev/videoN` path on Linux, and a Media Foundation symbolic link on /// Windows. Bare numeric indices remain accepted on Linux and Windows. + /// + /// With the `pipewire` feature, Linux also lists PipeWire camera nodes as + /// `pipewire:`, and `pipewire` alone opens the camera with the + /// highest session priority, which the session manager treats as default. + /// The id picks the backend: a `/dev/videoN` path always opens V4L2, and a + /// `pipewire` id always opens PipeWire. `None` opens `/dev/video0`, except + /// inside a sandbox (Flatpak or Snap), where it opens PipeWire's default + /// camera through the camera portal. Camera(Option), /// A whole display. `None` opens the main display. @@ -364,7 +375,15 @@ pub async fn open(config: &Config) -> Result { } #[cfg(target_os = "linux")] { - v4l2::open(config, device.as_deref()).await + match LinuxCamera::select(device.as_deref(), sandboxed()) { + LinuxCamera::V4l2(device) => v4l2::open(config, device).await, + #[cfg(feature = "pipewire")] + LinuxCamera::PipeWire(node) => pipewire::camera::open(config, node).await, + #[cfg(not(feature = "pipewire"))] + LinuxCamera::PipeWire(_) => Err(Error::Unsupported( + "PipeWire camera capture without the `pipewire` feature".to_string(), + )), + } } #[cfg(target_os = "windows")] { @@ -436,6 +455,12 @@ pub async fn open(config: &Config) -> Result { } /// List the available cameras and the identifiers [`Source::Camera`] accepts. +/// +/// On Linux with the `pipewire` feature, V4L2 devices come first and PipeWire +/// camera nodes follow, so a webcam that PipeWire also serves appears once per +/// backend. A session without PipeWire lists V4L2 alone. Inside a sandbox the +/// PipeWire nodes come through the camera portal, which may ask the user for +/// camera access. pub async fn cameras() -> Result, Error> { #[cfg(target_os = "macos")] { @@ -443,7 +468,10 @@ pub async fn cameras() -> Result, Error> { } #[cfg(target_os = "linux")] { - blocking(v4l2::cameras).await + let cameras = blocking(v4l2::cameras).await?; + #[cfg(feature = "pipewire")] + let cameras = [cameras, pipewire::camera::cameras().await?].concat(); + Ok(cameras) } #[cfg(target_os = "windows")] { @@ -462,9 +490,10 @@ pub async fn cameras() -> Result, Error> { /// are ones [`open`] could negotiate rather than everything the driver /// advertises. /// -/// Linux only; other backends return [`Error::Unsupported`]. +/// Linux only (V4L2, and PipeWire cameras with the `pipewire` feature); other +/// platforms return [`Error::Unsupported`]. /// Rates are exact device reports and [`Config::framerate`] accepts the same type. -/// V4L2 prefers the closest geometry, then the accepted rate +/// Both Linux backends prefer the closest geometry, then the accepted rate /// nearest that request, then the cheaper conversion format. /// /// An empty list is not a failure: it means the driver enumerated nothing this @@ -473,8 +502,18 @@ pub async fn camera_modes(camera: Option<&str>) -> Result, Error> { let _ = camera; #[cfg(target_os = "linux")] { - let camera = camera.map(str::to_string); - blocking(move || v4l2::modes(camera.as_deref())).await + match LinuxCamera::select(camera, sandboxed()) { + LinuxCamera::V4l2(camera) => { + let camera = camera.map(str::to_string); + blocking(move || v4l2::modes(camera.as_deref())).await + } + #[cfg(feature = "pipewire")] + LinuxCamera::PipeWire(node) => pipewire::camera::modes(node).await, + #[cfg(not(feature = "pipewire"))] + LinuxCamera::PipeWire(_) => Err(Error::Unsupported( + "PipeWire camera modes without the `pipewire` feature".to_string(), + )), + } } #[cfg(not(target_os = "linux"))] { @@ -538,6 +577,52 @@ pub async fn apps() -> Result, Error> { } } +/// The id prefix of a PipeWire camera, and on its own the default one. +#[cfg(any(target_os = "linux", test))] +const PIPEWIRE: &str = "pipewire"; + +/// The Linux backend a [`Source::Camera`] selector names. +#[cfg(any(target_os = "linux", test))] +#[derive(Debug, PartialEq, Eq)] +enum LinuxCamera<'a> { + /// A V4L2 device, by path or index. `None` opens `/dev/video0`. + V4l2(Option<&'a str>), + /// A PipeWire camera, by `node.name`. `None` picks the camera with the + /// highest session priority, the one the session manager links by default. + PipeWire(Option<&'a str>), +} + +#[cfg(any(target_os = "linux", test))] +impl<'a> LinuxCamera<'a> { + /// Parse a camera selector. A sandbox has no V4L2 devices to open by + /// default, so its default camera is PipeWire's, reached through the portal. + fn select(selector: Option<&'a str>, sandboxed: bool) -> Self { + match selector { + None if sandboxed => Self::PipeWire(None), + None => Self::V4l2(None), + Some(PIPEWIRE) => Self::PipeWire(None), + Some(selector) => match selector.strip_prefix(PIPEWIRE).and_then(|rest| rest.strip_prefix(':')) { + Some(name) => Self::PipeWire(Some(name)), + None => Self::V4l2(Some(selector)), + }, + } + } +} + +/// Whether this process runs in a Flatpak or Snap sandbox. Without the +/// `pipewire` feature there is no portal to use, so it never matters. +#[cfg(target_os = "linux")] +fn sandboxed() -> bool { + #[cfg(feature = "pipewire")] + { + ashpd::is_sandboxed() + } + #[cfg(not(feature = "pipewire"))] + { + false + } +} + /// Run synchronous platform enumeration off the async runtime's worker threads. #[cfg(any(target_os = "linux", target_os = "windows"))] async fn blocking(f: F) -> Result @@ -549,3 +634,34 @@ where .await .map_err(|err| Error::Codec(anyhow::anyhow!("capture enumeration thread failed: {err}")))? } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn camera_selectors_name_their_backend() { + assert_eq!(LinuxCamera::select(None, false), LinuxCamera::V4l2(None)); + assert_eq!(LinuxCamera::select(None, true), LinuxCamera::PipeWire(None)); + for sandboxed in [false, true] { + assert_eq!( + LinuxCamera::select(Some("/dev/video2"), sandboxed), + LinuxCamera::V4l2(Some("/dev/video2")) + ); + assert_eq!(LinuxCamera::select(Some("1"), sandboxed), LinuxCamera::V4l2(Some("1"))); + assert_eq!( + LinuxCamera::select(Some("pipewire"), sandboxed), + LinuxCamera::PipeWire(None) + ); + assert_eq!( + LinuxCamera::select(Some("pipewire:v4l2_input.pci-0000_00_14.0-usb-0_4_1.0"), sandboxed), + LinuxCamera::PipeWire(Some("v4l2_input.pci-0000_00_14.0-usb-0_4_1.0")) + ); + } + // Only the whole prefix selects PipeWire; anything else is a device path. + assert_eq!( + LinuxCamera::select(Some("pipewired"), false), + LinuxCamera::V4l2(Some("pipewired")) + ); + } +} diff --git a/rs/moq-video/src/capture/mode.rs b/rs/moq-video/src/capture/mode.rs new file mode 100644 index 00000000000..7c8af3d62bd --- /dev/null +++ b/rs/moq-video/src/capture/mode.rs @@ -0,0 +1,166 @@ +//! Mode choice and listing shared by the Linux camera backends (V4L2 and +//! PipeWire), so both land on the same mode for the same request. + +use std::collections::{BTreeMap, BTreeSet}; + +use super::{Mode, Rate}; +use crate::Size; + +/// What the caller asked the camera for. +#[derive(Clone, Copy, Debug)] +pub(super) struct Request { + pub size: Size, + pub framerate: Option, +} + +/// One mode a camera can stream, as the selection weighs it. +#[derive(Clone, Copy, Debug)] +pub(super) struct Candidate { + pub size: Size, + pub framerate: Option, + /// What converting the format to I420 costs: lower is cheaper. + pub cost: u8, +} + +/// Pick the candidate nearest `want`: geometry first, then the rate nearest the +/// request, then the cheaper conversion. A size that cannot feed I420 is never +/// picked, and among equals the earliest candidate wins. +pub(super) fn nearest( + candidates: impl IntoIterator, + want: Request, + candidate: impl Fn(&T) -> Candidate, +) -> Option { + candidates + .into_iter() + .filter(|item| candidate(item).size.validate("camera resolution").is_ok()) + .min_by(|left, right| { + let (left, right) = (candidate(left), candidate(right)); + distance(left.size, want.size) + .cmp(&distance(right.size, want.size)) + .then_with(|| rate_distance(left.framerate, right.framerate, want.framerate)) + .then_with(|| left.cost.cmp(&right.cost)) + }) +} + +fn rate_distance(left: Option, right: Option, want: Option) -> std::cmp::Ordering { + let Some(want) = want else { + return std::cmp::Ordering::Equal; + }; + match (left, right) { + (Some(left), Some(right)) => (left.as_f64() - want.as_f64()) + .abs() + .total_cmp(&(right.as_f64() - want.as_f64()).abs()), + (Some(_), None) => std::cmp::Ordering::Less, + (None, Some(_)) => std::cmp::Ordering::Greater, + (None, None) => std::cmp::Ordering::Equal, + } +} + +/// How far a mode lands from the requested geometry, summed over both +/// dimensions. Zero is an exact match. +fn distance(size: Size, want: Size) -> u64 { + u64::from(size.width.abs_diff(want.width)) + u64::from(size.height.abs_diff(want.height)) +} + +/// Merge the sizes and rates a camera reports into [`Mode`]s, largest first +/// with each mode's rates highest first. The encoder sees I420 whatever format +/// carried a mode, so the format is not something a caller can act on. +pub(super) fn modes(reported: impl IntoIterator)>) -> Vec { + let mut sizes: BTreeMap<(u32, u32), BTreeSet> = BTreeMap::new(); + for (size, rates) in reported { + sizes.entry((size.width, size.height)).or_default().extend(rates); + } + let mut modes: Vec = sizes + .into_iter() + .map(|((width, height), framerates)| Mode { + width, + height, + // Highest first, so the rate a caller most often wants is the one it + // reads without scanning. + framerates: framerates.into_iter().rev().collect(), + }) + .collect(); + modes.sort_by_key(|mode| std::cmp::Reverse(u64::from(mode.width) * u64::from(mode.height))); + modes +} + +#[cfg(test)] +mod tests { + use super::*; + + fn rate(fps: u32) -> Rate { + Rate::new(fps, 1).unwrap() + } + + #[test] + fn rate_scoring_preserves_fractional_precision_and_handles_unknown_rates() { + let ntsc = Some(Rate::new(60000, 1001).unwrap()); + let thirty = Some(rate(30)); + assert!(rate_distance(ntsc, thirty, Some(rate(60))).is_lt()); + assert!(rate_distance(thirty, ntsc, Some(rate(30))).is_lt()); + assert!(rate_distance(ntsc, None, Some(rate(60))).is_lt()); + assert!(rate_distance(ntsc, thirty, None).is_eq()); + } + + /// Distance is symmetric in the two dimensions and zero only on an exact hit, + /// so a mode that overshoots is no better than one that undershoots by as much. + #[test] + fn distance_is_zero_only_on_an_exact_match() { + let want = Size::new(1280, 720); + assert_eq!(distance(Size::new(1280, 720), want), 0); + assert_eq!(distance(Size::new(1280, 600), want), 120); + assert_eq!(distance(Size::new(1280, 840), want), 120); + } + + #[test] + fn nearest_prefers_geometry_then_rate_then_cost_then_order() { + let candidate = |width, height, fps: Option, cost| Candidate { + size: Size::new(width, height), + framerate: fps.map(rate), + cost, + }; + let want = |framerate| Request { + size: Size::new(1280, 720), + framerate, + }; + let candidates = [ + candidate(640, 480, Some(30), 0), + candidate(1280, 720, Some(30), 1), + candidate(1280, 720, Some(15), 0), + candidate(1280, 720, Some(30), 0), + candidate(1281, 720, Some(30), 0), + ]; + let pick = |framerate| nearest(candidates, want(framerate), |c| *c).unwrap(); + + let chosen = pick(Some(rate(30))); + assert_eq!((chosen.framerate, chosen.cost), (Some(rate(30)), 0)); + // With no rate requested, the first of the cheapest exact sizes wins. + assert_eq!(pick(None).framerate, Some(rate(15))); + // An odd size is never picked, even when it is the only exact one. + assert!(nearest([candidate(1281, 721, None, 0)], want(None), |c| *c).is_none()); + } + + #[test] + fn modes_merge_sizes_and_sort_largest_and_fastest_first() { + let modes = modes([ + (Size::new(640, 480), vec![rate(30)]), + (Size::new(1280, 720), vec![rate(15)]), + (Size::new(640, 480), vec![rate(60), rate(30)]), + ]); + assert_eq!( + modes, + [ + Mode { + width: 1280, + height: 720, + framerates: vec![rate(15)], + }, + Mode { + width: 640, + height: 480, + framerates: vec![rate(60), rate(30)], + }, + ] + ); + } +} diff --git a/rs/moq-video/src/capture/pipewire.rs b/rs/moq-video/src/capture/pipewire.rs index 0b0146aab79..4ac1ea79209 100644 --- a/rs/moq-video/src/capture/pipewire.rs +++ b/rs/moq-video/src/capture/pipewire.rs @@ -1,14 +1,15 @@ -//! Screen capture via xdg-desktop-portal + PipeWire (Linux, Wayland and X11). +//! Screen and camera capture via PipeWire (Linux, Wayland and X11). //! -//! The ScreenCast portal owns source selection: [`open`] pops the compositor's +//! The ScreenCast portal owns screen selection: [`open`] pops the compositor's //! picker dialog, the user chooses a monitor, and the portal hands us a PipeWire -//! fd + node id. A dedicated thread then runs the PipeWire main loop, forwarding -//! DMA-BUF frames without copying when the compositor offers them and converting -//! shared-memory frames to CPU [`I420`] otherwise. It pushes both into the shared -//! [`FrameChannel`] (callback-driven like the macOS delegate, not a pull-style -//! pump). +//! fd + node id. Cameras are PipeWire nodes too, reached as described in +//! [`camera`]. For either, a dedicated thread then runs the PipeWire main loop, +//! forwarding DMA-BUF frames without copying when the producer offers them and +//! converting shared-memory frames to CPU [`I420`] otherwise. It pushes both into +//! the shared [`FrameChannel`] (callback-driven like the macOS delegate, not a +//! pull-style pump). //! -//! Two quirks worth knowing: +//! Two screen quirks worth knowing: //! - `publish_capture` releases the capture while unwatched and reopens it on //! demand. A fresh portal session would re-prompt the picker every time, so the //! portal's restore token is kept in a process-wide slot and replayed on the @@ -19,6 +20,7 @@ //! - Compositors only deliver frames on damage, so a static screen would starve //! the encoder. A loop timer re-emits the last frame whenever a frame interval //! passes without a fresh one, mirroring the Windows Desktop Duplication pacing. +//! Cameras deliver every interval, so their streams have no such timer. use std::borrow::Cow; use std::cell::RefCell; @@ -36,6 +38,7 @@ use spa::buffer::DataType; use spa::param::video::{VideoFormat, VideoInfoRaw}; use super::channel::FrameChannel; +use super::mode::Request; use super::pump::Geometry; use super::{Config, Stream}; use crate::frame::{DmaBuf, DmaBufFrame, DmaBufPlane, DrmFormat, I420, Surface, wait_dma_buf_readable}; @@ -44,13 +47,14 @@ use crate::{Color, Error, Size}; const DEFAULT_FRAMERATE: u32 = 30; // libspa 0.10 omits this flag from its safe wrapper, but exposes the raw bits. const CHUNK_FLAG_EMPTY: i32 = 1 << 1; -/// The compositor sends the negotiated format right after the stream connects; +/// The producer sends the negotiated format right after the stream connects; /// if nothing arrives the session is broken (or the grant was revoked mid-setup). const FORMAT_TIMEOUT: Duration = Duration::from_secs(10); /// ScreenCast compositors deliver the current content as a first frame right -/// after negotiation; none arriving means the session is broken, so fail `open` -/// rather than hand the encoder a stream that will never produce (same -/// first-frame wait as the macOS ScreenCaptureKit backend). +/// after negotiation, and a camera starts streaming once linked; none arriving +/// means the session is broken, so fail `open` rather than hand the encoder a +/// stream that will never produce (same first-frame wait as the macOS +/// ScreenCaptureKit backend). const FIRST_FRAME_TIMEOUT: Duration = Duration::from_secs(5); /// The portal restore token from the last grant, replayed on the next [`open`] @@ -62,6 +66,8 @@ fn err(ctx: &str, e: impl std::fmt::Display) -> Error { Error::Codec(anyhow::anyhow!("{ctx}: {e}")) } +pub(super) mod camera; + /// Open a portal screen capture and stream its frames from a PipeWire loop thread. pub(super) async fn open(config: &Config, device: Option<&str>) -> Result { if let Some(device) = device { @@ -69,7 +75,57 @@ pub(super) async fn open(config: &Config, device: Option<&str>) -> Result, + target: Target, + label: String, +} + +/// Whether a loop captures a screen or a camera. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Kind { + Screen, + Camera, +} +impl Kind { + /// The producer, as error messages name it. + fn producer(self) -> &'static str { + match self { + Self::Screen => "the compositor", + Self::Camera => "the camera", + } + } +} + +/// The node a capture loop links to. +enum Target { + /// A node id granted by the ScreenCast portal. + Node(u32), + /// A camera by `node.name`, or `None` for the default one, streaming the + /// mode nearest the request. + Camera(Option, Request), +} + +/// Spawn the PipeWire loop for `capture` and wait for its format and first frame. +async fn start(config: &Config, capture: Capture, session: Option) -> Result { + let kind = capture.kind; let chan = FrameChannel::new(); let framerate = config .framerate @@ -94,8 +150,7 @@ pub(super) async fn open(config: &Config, device: Option<&str>) -> Result) -> Result result?, Ok(Err(_)) => { return Err(Error::Codec(anyhow::anyhow!( - "screen capture thread exited before negotiating a format" + "{kind:?} capture thread exited before negotiating a format" ))); } Err(_) => { return Err(Error::Codec(anyhow::anyhow!( - "no video format from the compositor within {FORMAT_TIMEOUT:?}" + "no video format from {} within {FORMAT_TIMEOUT:?}", + kind.producer() ))); } }; @@ -144,16 +200,18 @@ pub(super) async fn open(config: &Config, device: Option<&str>) -> Result return Err(error), Ok(Ok(None)) | Err(_) => { return Err(Error::Codec(anyhow::anyhow!( - "no frames from the compositor within {FIRST_FRAME_TIMEOUT:?}" + "no frames from {} within {FIRST_FRAME_TIMEOUT:?}", + kind.producer() ))); } }; tracing::info!( - node = node_id, + ?kind, + source = %geo.label, width = geo.width, height = geo.height, - "opened screen capture (PipeWire)" + "opened PipeWire capture" ); Ok(Stream::new( @@ -247,8 +305,9 @@ struct LoopGuard { quit: pw::channel::Sender<()>, handle: Option>, /// Held so the portal session outlives the loop; dropping it closes the - /// session only after the loop thread has been joined above. - _session: SessionGuard, + /// session only after the loop thread has been joined above. Cameras have + /// no portal session to hold. + _session: Option, } impl Drop for LoopGuard { @@ -646,8 +705,7 @@ impl Drop for Dequeued<'_> { } struct CaptureLoop { - fd: OwnedFd, - node_id: u32, + capture: Capture, framerate: u32, chan: Arc, state: Rc>, @@ -693,12 +751,25 @@ fn fixate_modifier(param: &spa::pod::Pod, modifier: u64) -> Option> { .map(|serialized| serialized.0.into_inner()) } -/// Connect to the portal's PipeWire node and run until the stream ends, the +/// Connect to a portal's PipeWire remote, or to the session's own socket when +/// there is none. +fn connect(context: &pw::context::ContextRc, remote: Option) -> Result { + match remote { + Some(fd) => context.connect_fd_rc(fd, None), + None => context.connect_rc(None), + } +} + +/// Connect to the target PipeWire node and run until the stream ends, the /// consumer drops, or the format changes. fn run_loop(args: CaptureLoop) -> Result<(), Error> { let CaptureLoop { - fd, - node_id, + capture: Capture { + kind, + remote, + target, + label, + }, framerate, chan, state, @@ -710,17 +781,27 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { let mainloop = pw::main_loop::MainLoopRc::new(None).map_err(|e| err("pipewire main loop", e))?; let context = pw::context::ContextRc::new(&mainloop, None).map_err(|e| err("pipewire context", e))?; - let core = context - .connect_fd_rc(fd, None) - .map_err(|e| err("pipewire connect", e))?; + let core = connect(&context, remote).map_err(|e| err("pipewire connect", e))?; + let (node_id, offers) = match target { + Target::Node(id) => (id, format_offers(framerate)), + Target::Camera(name, want) => { + let (id, mode) = camera::choose(&mainloop, &core, name.as_deref(), want)?; + tracing::debug!(node = id, ?mode, "chose PipeWire camera mode"); + (id, mode.offers()) + } + }; + let (name, role) = match kind { + Kind::Screen => ("moq-screen", "Screen"), + Kind::Camera => ("moq-camera", "Camera"), + }; let stream = pw::stream::StreamRc::new( core, - "moq-screen", + name, pw::properties::properties! { *pw::keys::MEDIA_TYPE => "Video", *pw::keys::MEDIA_CATEGORY => "Capture", - *pw::keys::MEDIA_ROLE => "Screen", + *pw::keys::MEDIA_ROLE => role, }, ) .map_err(|e| err("pipewire stream", e))?; @@ -755,18 +836,20 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { pw::stream::StreamState::Error(_) | pw::stream::StreamState::Unconnected ); if done { - // The compositor ended the stream while our loop was live (the - // user hit "stop sharing", or the output went away). Forget the - // restore token so the demand-driven reopen asks again instead - // of silently resuming a grant the user just revoked. Our own - // teardown quits the loop before anything disconnects, so it - // never reaches this path. - *RESTORE_TOKEN.lock().unwrap() = None; + // The producer ended the stream while our loop was live (the + // user hit "stop sharing", or the output or camera went away). + // Forget the screen restore token so the demand-driven reopen + // asks again instead of silently resuming a grant the user just + // revoked. Our own teardown quits the loop before anything + // disconnects, so it never reaches this path. + if kind == Kind::Screen { + *RESTORE_TOKEN.lock().unwrap() = None; + } state.borrow_mut().terminal = Some(Error::SourceUnavailable(match &new { pw::stream::StreamState::Error(error) => format!("PipeWire stream failed: {error}"), - _ => "the selected screen is no longer available".to_string(), + _ => format!("the selected {} is no longer available", role.to_lowercase()), })); - tracing::debug!(state = ?new, "screen capture stream ended"); + tracing::debug!(?kind, state = ?new, "capture stream ended"); if let Some(mainloop) = mainloop.upgrade() { mainloop.quit(); } @@ -784,14 +867,22 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { let Ok((media_type, media_subtype)) = spa::param::format_utils::parse_format(param) else { return; }; - if media_type != spa::param::format::MediaType::Video - || media_subtype != spa::param::format::MediaSubtype::Raw - { + if media_type != spa::param::format::MediaType::Video { return; } let mut state = state.borrow_mut(); - if let Err(e) = replace_video_format(&mut state.format, |format| format.parse(param)) { + if media_subtype == spa::param::format::MediaSubtype::Mjpg { + // libspa parses only raw formats. MJPEG is recorded as an + // `Encoded` raw format carrying the negotiated size and rate. + let Some(format) = camera::mjpeg_format(param) else { + tracing::warn!("failed to parse the pipewire MJPEG format"); + return; + }; + state.format = format; + } else if media_subtype != spa::param::format::MediaSubtype::Raw { + return; + } else if let Err(e) = replace_video_format(&mut state.format, |format| format.parse(param)) { tracing::warn!(error = %e, "failed to parse pipewire video format"); return; } @@ -836,7 +927,7 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { tracing::warn!(width = size.width, height = size.height, "unusable capture size"); return; } - let color = match pipewire_color(state.format, width, height) { + let color = match pipewire_color(kind, state.format, width, height) { Ok(color) => color, Err(e) => { match state.geo_tx.take() { @@ -864,7 +955,7 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { width, height, framerate, - label: format!("pipewire:{node_id}"), + label: label.clone(), })); } else if format_requires_restart(state.geometry, state.color, width, height, color) { // The encoder's geometry and VUI are fixed when it opens. End the @@ -916,6 +1007,7 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { Ok(stride) if stride > 0 => stride, _ => match state.format.format() { VideoFormat::NV12 => state.format.size().width, + VideoFormat::YUY2 => state.format.size().width.saturating_mul(2), VideoFormat::BGRx | VideoFormat::BGRA | VideoFormat::RGBx | VideoFormat::RGBA => { state.format.size().width.saturating_mul(4) } @@ -935,9 +1027,16 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { ChunkKind::Empty => return, ChunkKind::Invalid => return, } - let Some(required) = frame_data_size(state.format.format(), layout) else { - tracing::warn!("pipewire frame layout overflows its buffer"); - return; + // A JPEG spans its chunk, whatever its layout would suggest. + let required = match state.format.format() { + VideoFormat::Encoded => size, + format => { + let Some(required) = frame_data_size(format, layout) else { + tracing::warn!("pipewire frame layout overflows its buffer"); + return; + }; + required + } }; if dmabuf { let Some(format) = drm_format(state.format.format()) else { @@ -1019,7 +1118,11 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { match DmaBuf::new(format, modifier, layout.width, layout.height, planes, color, inner) { Ok(frame) => { chan.push(Surface::DmaBuf(frame.clone())); - state.last = Some(Last::DmaBuf(frame)); + // Only the pacing timer reads `last`. A camera must not + // hold a leased buffer back from its producer's pool. + if kind == Kind::Screen { + state.last = Some(Last::DmaBuf(frame)); + } state.fresh = true; } Err(e) => tracing::warn!(error = %e, "invalid PipeWire DMA-BUF"), @@ -1060,12 +1163,14 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { match convert(state.format.format(), bytes.as_ref(), layout, color) { Ok(i420) => { chan.push(Surface::I420(i420.clone())); - state.last = Some(Last::I420(i420)); + if kind == Kind::Screen { + state.last = Some(Last::I420(i420)); + } state.fresh = true; } Err(e) => { // Persistent (bad format), not per-frame; stop rather than spam. - tracing::warn!(error = %e, "screen frame conversion failed; stopping capture"); + tracing::warn!(?kind, error = %e, "frame conversion failed; stopping capture"); state.terminal = Some(e); if let Some(mainloop) = mainloop.upgrade() { mainloop.quit(); @@ -1079,7 +1184,6 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { // DMA-BUF formats come first so PipeWire prefers them. Each has a matching // shared-memory offer without a modifier as the required fallback. - let offers = format_offers(framerate); let mut params = offers .iter() .map(|offer| { @@ -1097,7 +1201,8 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { .map_err(|e| err("pipewire stream connect", e))?; // Re-emit the last frame when an interval passes without a fresh one, so a - // static screen still produces a steady stream for the encoder. + // static screen still produces a steady stream for the encoder. A camera + // never retains a `last` frame and never arms the timer. let timer = mainloop.loop_().add_timer({ let state = state.clone(); let chan = chan.clone(); @@ -1111,11 +1216,13 @@ fn run_loop(args: CaptureLoop) -> Result<(), Error> { } } }); - let interval = Duration::from_micros(1_000_000 / framerate as u64); - timer - .update_timer(Some(interval), Some(interval)) - .into_result() - .map_err(|e| err("pipewire timer", e))?; + if kind == Kind::Screen { + let interval = Duration::from_micros(1_000_000 / framerate as u64); + timer + .update_timer(Some(interval), Some(interval)) + .into_result() + .map_err(|e| err("pipewire timer", e))?; + } // Quit when the Stream drops. let _quit = quit_rx.attach(mainloop.loop_(), { @@ -1181,6 +1288,7 @@ fn frame_data_size(format: VideoFormat, layout: FrameLayout) -> Option { let height = layout.height as usize; let row_size = match format { VideoFormat::NV12 => width, + VideoFormat::YUY2 => width.checked_mul(2)?, VideoFormat::BGRx | VideoFormat::BGRA | VideoFormat::RGBx | VideoFormat::RGBA => width.checked_mul(4)?, _ => return None, }; @@ -1194,6 +1302,9 @@ fn frame_data_size(format: VideoFormat, layout: FrameLayout) -> Option { .checked_sub(1)? .checked_mul(stride)? .checked_add(row_size), + // The packed 4:2:2 converter reads whole rows, including the last row's + // padding, so the frame has to span every stride it covers. + VideoFormat::YUY2 => height.checked_mul(stride), _ => height.checked_sub(1)?.checked_mul(stride)?.checked_add(row_size), } } @@ -1230,16 +1341,17 @@ fn replace_video_format( Ok(result) } -/// Preserve the color description that names NV12 samples. Unknown fields use +/// Preserve the color description that names YUV samples. Unknown fields use /// the same size-based, limited-range fallback as the encoder. Reject matrices /// the crate cannot represent rather than writing a false 601/709 VUI. -fn pipewire_color(format: VideoInfoRaw, width: u32, height: u32) -> Result, Error> { - if format.format() != VideoFormat::NV12 { +fn pipewire_color(kind: Kind, format: VideoInfoRaw, width: u32, height: u32) -> Result, Error> { + if !matches!(format.format(), VideoFormat::NV12 | VideoFormat::YUY2) { return Ok(None); } let size = Size::new(width, height); let color = color_from_pipewire(format.color_range(), format.color_matrix(), size)?; validate_pipewire_description( + kind, color.unwrap_or_else(|| Color::infer(size)), format.color_primaries(), format.transfer_function(), @@ -1257,7 +1369,7 @@ fn color_from_pipewire(range: u32, matrix: u32, size: Size) -> Result false, _ => { return Err(Error::Codec(anyhow::anyhow!( - "unsupported PipeWire NV12 color range {range}" + "unsupported PipeWire YUV color range {range}" ))); } }; @@ -1269,7 +1381,7 @@ fn color_from_pipewire(range: u32, matrix: u32, size: Size) -> Result false, _ => { return Err(Error::Codec(anyhow::anyhow!( - "unsupported PipeWire NV12 color matrix {matrix}" + "unsupported PipeWire YUV color matrix {matrix}" ))); } }; @@ -1293,14 +1405,25 @@ fn color_from_pipewire(range: u32, matrix: u32, size: Size) -> Result Result<(), Error> { +fn validate_pipewire_description(kind: Kind, color: Color, primaries: u32, transfer: u32) -> Result<(), Error> { let expected_primaries = match color { Color::Bt601Limited | Color::Bt601Full => spa::sys::SPA_VIDEO_COLOR_PRIMARIES_SMPTE170M, Color::Bt709Limited | Color::Bt709Full => spa::sys::SPA_VIDEO_COLOR_PRIMARIES_BT709, }; - if primaries != spa::sys::SPA_VIDEO_COLOR_PRIMARIES_UNKNOWN && primaries != expected_primaries { + // UVC webcams describe their YUY2 as a BT.601 matrix with BT.709 primaries + // (an integrated USB webcam does, through spa-v4l2), a pairing `Color` + // cannot name. The matrix and range decide how the samples decode, + // so a camera keeps them and accepts either SDR primaries. + let primaries_match = primaries == spa::sys::SPA_VIDEO_COLOR_PRIMARIES_UNKNOWN + || primaries == expected_primaries + || (kind == Kind::Camera + && matches!( + primaries, + spa::sys::SPA_VIDEO_COLOR_PRIMARIES_SMPTE170M | spa::sys::SPA_VIDEO_COLOR_PRIMARIES_BT709 + )); + if !primaries_match { return Err(Error::Codec(anyhow::anyhow!( - "PipeWire NV12 primaries {primaries} do not match the negotiated matrix" + "PipeWire YUV primaries {primaries} do not match the negotiated matrix" ))); } if !matches!( @@ -1311,7 +1434,7 @@ fn validate_pipewire_description(color: Color, primaries: u32, transfer: u32) -> | SPA_VIDEO_TRANSFER_BT2020_10 ) { return Err(Error::Codec(anyhow::anyhow!( - "unsupported PipeWire NV12 transfer function {transfer}" + "unsupported PipeWire YUV transfer function {transfer}" ))); } Ok(()) @@ -1327,11 +1450,14 @@ fn format_requires_restart( geometry.is_some_and(|geometry| geometry != (width, height) || color != next_color) } -/// Convert one strided screen frame to tightly-packed I420. +/// Convert one strided frame to tightly-packed I420. fn convert(format: VideoFormat, bytes: &[u8], layout: FrameLayout, color: Option) -> Result { match format { - VideoFormat::NV12 => { - let frame = nv12_to_i420(bytes, layout)?; + VideoFormat::NV12 | VideoFormat::YUY2 => { + let frame = match format { + VideoFormat::NV12 => nv12_to_i420(bytes, layout)?, + _ => I420::from_yuyv(bytes, layout.stride, crate::Size::new(layout.width, layout.height))?, + }; Ok(match color { Some(color) => frame.with_color(color), None => frame, @@ -1343,6 +1469,8 @@ fn convert(format: VideoFormat, bytes: &[u8], layout: FrameLayout, color: Option VideoFormat::RGBx | VideoFormat::RGBA => { I420::from_rgba(bytes, layout.stride, crate::Size::new(layout.width, layout.height)) } + // Only a camera negotiates MJPEG, which the format records as `Encoded`. + VideoFormat::Encoded => I420::from_mjpeg(bytes, crate::Size::new(layout.width, layout.height)), other => Err(Error::Codec(anyhow::anyhow!( "pipewire negotiated an unsupported video format {other:?}" ))), @@ -1361,7 +1489,7 @@ fn drm_format(format: VideoFormat) -> Option { } } -const PIPEWIRE_FORMATS: [VideoFormat; 5] = [ +const SCREEN_FORMATS: [VideoFormat; 5] = [ VideoFormat::BGRx, VideoFormat::BGRA, VideoFormat::RGBx, @@ -1414,21 +1542,31 @@ fn format_offer(framerate: u32, format: VideoFormat, dmabuf: bool) -> Vec { ), ); if dmabuf { - obj.properties.push(spa::pod::Property { - key: spa::param::format::FormatProperties::VideoModifier.as_raw(), - flags: spa::pod::PropertyFlags::from_bits_retain( - spa::sys::SPA_POD_PROP_FLAG_MANDATORY | spa::sys::SPA_POD_PROP_FLAG_DONT_FIXATE, - ), - value: spa::pod::Value::Choice(spa::pod::ChoiceValue::Long(spa::utils::Choice( - spa::utils::ChoiceFlags::empty(), - spa::utils::ChoiceEnum::Enum { - default: 0, - alternatives: vec![0], - }, - ))), - }); + obj.properties.push(linear_modifier()); } - spa::pod::serialize::PodSerializer::serialize(std::io::Cursor::new(Vec::new()), &spa::pod::Value::Object(obj)) + serialize_format(obj) +} + +/// The `VideoModifier` property of a DMA-BUF offer: linear only, left for the +/// producer to fixate. +fn linear_modifier() -> spa::pod::Property { + spa::pod::Property { + key: spa::param::format::FormatProperties::VideoModifier.as_raw(), + flags: spa::pod::PropertyFlags::from_bits_retain( + spa::sys::SPA_POD_PROP_FLAG_MANDATORY | spa::sys::SPA_POD_PROP_FLAG_DONT_FIXATE, + ), + value: spa::pod::Value::Choice(spa::pod::ChoiceValue::Long(spa::utils::Choice( + spa::utils::ChoiceFlags::empty(), + spa::utils::ChoiceEnum::Enum { + default: 0, + alternatives: vec![0], + }, + ))), + } +} + +fn serialize_format(object: spa::pod::Object) -> Vec { + spa::pod::serialize::PodSerializer::serialize(std::io::Cursor::new(Vec::new()), &spa::pod::Value::Object(object)) .expect("serializing a static format pod cannot fail") .0 .into_inner() @@ -1436,11 +1574,11 @@ fn format_offer(framerate: u32, format: VideoFormat, dmabuf: bool) -> Vec { /// Serialize DMA-BUF formats first and shared-memory fallbacks second. fn format_offers(framerate: u32) -> Vec> { - PIPEWIRE_FORMATS + SCREEN_FORMATS .into_iter() .map(|format| format_offer(framerate, format, true)) .chain( - PIPEWIRE_FORMATS + SCREEN_FORMATS .into_iter() .map(|format| format_offer(framerate, format, false)), ) @@ -1497,7 +1635,7 @@ mod tests { #[test] fn format_offer_is_valid_pod() { let offers = format_offers(30); - assert_eq!(offers.len(), PIPEWIRE_FORMATS.len() * 2); + assert_eq!(offers.len(), SCREEN_FORMATS.len() * 2); for (index, bytes) in offers.iter().enumerate() { let (remaining, value) = spa::pod::deserialize::PodDeserializer::deserialize_any_from(bytes) .expect("format offer did not round-trip"); @@ -1506,13 +1644,13 @@ mod tests { panic!("format offer is not an object"); }; let property = |key| object.properties.iter().find(|property| property.key == key); - let format = PIPEWIRE_FORMATS[index % PIPEWIRE_FORMATS.len()]; + let format = SCREEN_FORMATS[index % SCREEN_FORMATS.len()]; assert_eq!( property(spa::param::format::FormatProperties::VideoFormat.as_raw()).map(|p| &p.value), Some(&spa::pod::Value::Id(spa::utils::Id(format.as_raw()))) ); let modifier = property(spa::param::format::FormatProperties::VideoModifier.as_raw()); - if index < PIPEWIRE_FORMATS.len() { + if index < SCREEN_FORMATS.len() { let modifier = modifier.expect("DMA-BUF offer has no modifier"); assert_eq!( modifier.flags.bits(), @@ -1553,6 +1691,45 @@ mod tests { assert_eq!(negotiated_memory(fixed, format), Some(NegotiatedMemory::DmaBuf(0))); } + #[test] + fn yuy2_converts_with_the_negotiated_color() { + let layout = FrameLayout { + stride: 10, + width: 4, + height: 2, + source_height: 2, + }; + assert_eq!(frame_data_size(VideoFormat::YUY2, layout), Some(20)); + // Y0 U Y1 V per pixel pair, then two bytes of row padding. + let data = [ + 1, 100, 2, 200, 3, 101, 4, 201, 99, 99, // row 0 + 5, 100, 6, 200, 7, 101, 8, 201, 99, 99, // row 1 + ]; + let frame = convert(VideoFormat::YUY2, &data, layout, Some(Color::Bt601Limited)).unwrap(); + assert_eq!(frame.y(), &[1, 2, 3, 4, 5, 6, 7, 8]); + assert_eq!(frame.color(), Some(Color::Bt601Limited)); + } + + /// An integrated USB webcam's description through spa-v4l2: a camera + /// accepts it, and a screen still rejects the same mismatch. + #[test] + fn camera_accepts_bt601_matrix_with_bt709_primaries() { + let mut format = VideoInfoRaw::default(); + format.set_format(VideoFormat::YUY2); + format.set_color_range(spa::sys::SPA_VIDEO_COLOR_RANGE_16_235); + format.set_color_matrix(spa::sys::SPA_VIDEO_COLOR_MATRIX_BT601); + format.set_color_primaries(spa::sys::SPA_VIDEO_COLOR_PRIMARIES_BT709); + format.set_transfer_function(spa::sys::SPA_VIDEO_TRANSFER_BT709); + assert_eq!( + pipewire_color(Kind::Camera, format, 640, 480).unwrap(), + Some(Color::Bt601Limited) + ); + assert!(pipewire_color(Kind::Screen, format, 640, 480).is_err()); + + format.set_color_primaries(spa::sys::SPA_VIDEO_COLOR_PRIMARIES_BT2020); + assert!(pipewire_color(Kind::Camera, format, 640, 480).is_err()); + } + #[test] fn buffer_offer_is_valid_pod() { let dma_buf = 1 << DataType::DmaBuf.as_raw(); @@ -1728,11 +1905,11 @@ mod tests { format.set_color_matrix(spa::sys::SPA_VIDEO_COLOR_MATRIX_BT709); format.set_color_primaries(spa::sys::SPA_VIDEO_COLOR_PRIMARIES_BT2020); format.set_transfer_function(spa::sys::SPA_VIDEO_TRANSFER_BT709); - assert!(pipewire_color(format, 1920, 1080).is_err()); + assert!(pipewire_color(Kind::Screen, format, 1920, 1080).is_err()); format.set_color_range(spa::sys::SPA_VIDEO_COLOR_RANGE_UNKNOWN); format.set_color_matrix(spa::sys::SPA_VIDEO_COLOR_MATRIX_UNKNOWN); format.set_transfer_function(spa::sys::SPA_VIDEO_TRANSFER_SMPTE2084); - assert!(pipewire_color(format, 1920, 1080).is_err()); + assert!(pipewire_color(Kind::Screen, format, 1920, 1080).is_err()); let layout = FrameLayout { stride: 4, diff --git a/rs/moq-video/src/capture/pipewire/camera.rs b/rs/moq-video/src/capture/pipewire/camera.rs new file mode 100644 index 00000000000..f19b0116e74 --- /dev/null +++ b/rs/moq-video/src/capture/pipewire/camera.rs @@ -0,0 +1,1204 @@ +//! PipeWire camera capture: the `Video/Source` nodes with the `Camera` role, +//! which is how PipeWire presents V4L2 webcams (spa-v4l2) and libcamera sensors +//! such as a Raspberry Pi CSI camera (spa-libcamera). A CSI camera has no V4L2 +//! node that yields processed frames, so on a Pi this is the only way to open it +//! as a camera. +//! +//! Outside a sandbox this connects to the session's PipeWire socket, which also +//! works headless. A sandbox cannot reach that socket, so there the camera portal +//! (`org.freedesktop.portal.Camera`) grants access and hands back a PipeWire +//! remote that exposes the cameras. Streaming reuses the parent module's loop. +//! +//! The mode is chosen here rather than negotiated: the node's `EnumFormat` list +//! goes through the same selection as a V4L2 device, and the stream offers +//! exactly the chosen mode. Offering a size range instead lets spa-v4l2 accept +//! a size its format does not have (a webcam with YUY2 only at 640x480 took +//! YUY2 at 1280x720) and then fail the link. + +use std::cell::RefCell; +use std::os::fd::OwnedFd; +use std::rc::Rc; +use std::time::Duration; + +use pipewire as pw; +use pw::spa; +use spa::param::format::{FormatProperties, MediaSubtype, MediaType}; +use spa::param::video::{VideoFormat, VideoInfoRaw}; +use spa::pod::{ChoiceValue, Value}; +use spa::utils::ChoiceEnum; + +use super::{Capture, DEFAULT_FRAMERATE, Kind, Target, drm_format, err, linear_modifier, serialize_format}; +use crate::capture::mode::{self, Request}; +use crate::capture::v4l2::{DEFAULT_HEIGHT, DEFAULT_WIDTH, bounds}; +use crate::capture::{Camera, Config, Mode, PIPEWIRE, Stream}; +use crate::{Error, Rate, Size}; + +/// How long the PipeWire daemon may take to answer a query. A connected daemon +/// answers in milliseconds, so running out means it is stuck. +const QUERY_TIMEOUT: Duration = Duration::from_secs(5); + +/// List the PipeWire cameras as [`Camera`]s with `pipewire:` ids. +pub(in crate::capture) async fn cameras() -> Result, Error> { + let sandboxed = ashpd::is_sandboxed(); + let remote = if sandboxed { + match portal().await? { + Some(fd) => Some(fd), + // No camera lists as nothing, so the V4L2 cameras still show. + None => return Ok(Vec::new()), + } + } else { + None + }; + let nodes = crate::capture::blocking(move || { + pw::init(); + let mainloop = pw::main_loop::MainLoopRc::new(None).map_err(|e| err("pipewire main loop", e))?; + let context = pw::context::ContextRc::new(&mainloop, None).map_err(|e| err("pipewire context", e))?; + let core = match super::connect(&context, remote) { + Ok(core) => core, + // PipeWire is optional on the host, and a host without it has no + // PipeWire cameras. A portal remote that fails to connect is broken. + Err(error) if !sandboxed => { + tracing::debug!(%error, "no PipeWire session to list cameras from"); + return Ok(Vec::new()); + } + Err(error) => return Err(err("pipewire connect", error)), + }; + let registry = core.get_registry_rc().map_err(|e| err("pipewire registry", e))?; + scan(&mainloop, &core, ®istry) + }) + .await?; + + Ok(nodes + .into_iter() + .map(|node| Camera { + id: format!("{PIPEWIRE}:{}", node.name), + name: node.description, + }) + .collect()) +} + +/// List the modes the camera named `node` (or the default camera) reports, for +/// the formats this backend converts. +pub(in crate::capture) async fn modes(node: Option<&str>) -> Result, Error> { + let remote = remote().await?; + let node = node.map(str::to_string); + crate::capture::blocking(move || { + pw::init(); + let mainloop = pw::main_loop::MainLoopRc::new(None).map_err(|e| err("pipewire main loop", e))?; + let context = pw::context::ContextRc::new(&mainloop, None).map_err(|e| err("pipewire context", e))?; + let core = super::connect(&context, remote).map_err(|e| err("pipewire connect", e))?; + let registry = core.get_registry_rc().map_err(|e| err("pipewire registry", e))?; + let nodes = scan(&mainloop, &core, ®istry)?; + let node = resolve(&nodes, node.as_deref())?; + let formats = formats(&mainloop, &core, ®istry, node)?; + Ok(mode::modes(formats.into_iter().flat_map(|format| { + format.sizes.into_iter().map(move |size| (size, format.rates.clone())) + }))) + }) + .await +} + +/// Open the camera node named `node`, or the default camera, in the mode +/// nearest `config`'s size and rate. +pub(in crate::capture) async fn open(config: &Config, node: Option<&str>) -> Result { + let remote = remote().await?; + let label = match node { + Some(node) => format!("{PIPEWIRE}:{node}"), + None => PIPEWIRE.to_string(), + }; + let want = Request { + size: Size::new( + config.width.unwrap_or(DEFAULT_WIDTH), + config.height.unwrap_or(DEFAULT_HEIGHT), + ), + framerate: config.framerate, + }; + super::start( + config, + Capture { + kind: Kind::Camera, + remote, + target: Target::Camera(node.map(str::to_string), want), + label, + }, + None, + ) + .await +} + +/// The camera portal's PipeWire remote inside a sandbox, or `None` to use the +/// session's socket. +async fn remote() -> Result, Error> { + if !ashpd::is_sandboxed() { + return Ok(None); + } + match portal().await? { + Some(fd) => Ok(Some(fd)), + None => Err(Error::SourceUnavailable( + "the camera portal reports no camera".to_string(), + )), + } +} + +/// Ask the camera portal for a PipeWire remote that exposes the cameras, or +/// `None` when the portal reports no camera. +async fn portal() -> Result, Error> { + let portal = ashpd::desktop::camera::Camera::new() + .await + .map_err(|e| err("camera portal", e))?; + if !portal.is_present().await.map_err(|e| err("camera portal", e))? { + return Ok(None); + } + // The portal asks the user unless the sandbox's permission store already + // holds a grant, so this blocks on the user the first time. + portal + .request_access(Default::default()) + .await + .map_err(|e| err("camera portal access", e))? + .response() + .map_err(|e| Error::PermissionDenied(format!("camera access request: {e}")))?; + let fd = portal + .open_pipe_wire_remote(Default::default()) + .await + .map_err(|e| err("camera portal pipewire remote", e))?; + Ok(Some(fd)) +} + +/// Pick the camera and its mode for a stream on `core`: the node id to link +/// to, and the mode to offer. +pub(super) fn choose( + mainloop: &pw::main_loop::MainLoopRc, + core: &pw::core::CoreRc, + name: Option<&str>, + want: Request, +) -> Result<(u32, Selected), Error> { + let registry = core.get_registry_rc().map_err(|e| err("pipewire registry", e))?; + let nodes = scan(mainloop, core, ®istry)?; + let node = resolve(&nodes, name)?; + let formats = formats(mainloop, core, ®istry, node)?; + let selected = select(&formats, want).ok_or_else(|| { + Error::Codec(anyhow::anyhow!( + "PipeWire camera {} offers no YUY2, NV12, RGB, or MJPEG mode", + node.name + )) + })?; + Ok((node.id, selected)) +} + +/// A PipeWire camera node. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct Node { + /// The registry id to link to. Only valid for this connection's lifetime. + id: u32, + /// `node.name`, which stays the same across restarts and reboots. + name: String, + /// `node.description`, or the best name the node has short of it. + description: String, + /// `priority.session`, which the session manager ranks default nodes by. + priority: i32, +} + +impl Node { + /// Build a node from a registry global's properties, or `None` when it is + /// not a camera. The criterion is the one xdg-desktop-portal applies when it + /// decides which nodes a sandboxed app may see. + fn from_props<'a>(id: u32, prop: impl Fn(&str) -> Option<&'a str>) -> Option { + if prop(*pw::keys::MEDIA_CLASS)? != "Video/Source" || prop(*pw::keys::MEDIA_ROLE)? != "Camera" { + return None; + } + let name = prop(*pw::keys::NODE_NAME).filter(|name| !name.is_empty())?; + let description = prop(*pw::keys::NODE_DESCRIPTION) + .or_else(|| prop(*pw::keys::NODE_NICK)) + .filter(|description| !description.is_empty()) + .unwrap_or(name); + let priority = prop(*pw::keys::PRIORITY_SESSION) + .and_then(|priority| priority.parse().ok()) + .unwrap_or(0); + Some(Self { + id, + name: name.to_string(), + description: description.to_string(), + priority, + }) + } +} + +/// Run `mainloop` until the daemon has answered every request sent before +/// this one on `core`. +fn roundtrip(mainloop: &pw::main_loop::MainLoopRc, core: &pw::core::CoreRc) -> Result<(), Error> { + let outcome: Rc>>> = Rc::new(RefCell::new(None)); + let pending = core.sync(0).map_err(|e| err("pipewire sync", e))?; + let _core = core + .add_listener_local() + .done({ + let outcome = outcome.clone(); + let mainloop = mainloop.downgrade(); + move |id, seq| { + if id == pw::core::PW_ID_CORE && seq == pending { + outcome.borrow_mut().get_or_insert(Ok(())); + if let Some(mainloop) = mainloop.upgrade() { + mainloop.quit(); + } + } + } + }) + .error({ + let outcome = outcome.clone(); + let mainloop = mainloop.downgrade(); + move |id, _, _, message| { + if id == pw::core::PW_ID_CORE { + outcome.borrow_mut().get_or_insert(Err(err("pipewire", message))); + if let Some(mainloop) = mainloop.upgrade() { + mainloop.quit(); + } + } + } + }) + .register(); + let timer = mainloop.loop_().add_timer({ + let mainloop = mainloop.downgrade(); + move |_| { + if let Some(mainloop) = mainloop.upgrade() { + mainloop.quit(); + } + } + }); + timer + .update_timer(Some(QUERY_TIMEOUT), None) + .into_result() + .map_err(|e| err("pipewire timer", e))?; + + mainloop.run(); + let outcome = outcome.borrow_mut().take(); + outcome.unwrap_or_else(|| { + Err(Error::Codec(anyhow::anyhow!( + "PipeWire did not answer within {QUERY_TIMEOUT:?}" + ))) + }) +} + +/// List the camera nodes in `registry`. +fn scan( + mainloop: &pw::main_loop::MainLoopRc, + core: &pw::core::CoreRc, + registry: &pw::registry::RegistryRc, +) -> Result, Error> { + let nodes = Rc::new(RefCell::new(Vec::new())); + let _registry = registry + .add_listener_local() + .global({ + let nodes = nodes.clone(); + move |global| { + if global.type_ != pw::types::ObjectType::Node { + return; + } + let Some(props) = global.props else { return }; + if let Some(node) = Node::from_props(global.id, |key| props.get(key)) { + nodes.borrow_mut().push(node); + } + } + }) + .register(); + // The registry announces every existing global before the sync reply. + roundtrip(mainloop, core)?; + Ok(nodes.take()) +} + +/// The camera named `name`, or the one with the highest session priority, +/// which is the camera the session manager links by default. Fails when the +/// camera is not there, rather than leaving the stream unlinked until the +/// format wait runs out. +fn resolve<'a>(nodes: &'a [Node], name: Option<&str>) -> Result<&'a Node, Error> { + let Some(name) = name else { + // `max_by_key` keeps the last of equals; the registry's first is wanted. + return nodes + .iter() + .rev() + .max_by_key(|node| node.priority) + .ok_or_else(|| Error::SourceUnavailable("PipeWire has no camera".to_string())); + }; + nodes.iter().find(|node| node.name == name).ok_or_else(|| { + let found: Vec<_> = nodes.iter().map(|node| node.name.as_str()).collect(); + Error::SourceUnavailable(format!("no PipeWire camera named {name} (found: {})", found.join(", "))) + }) +} + +/// The raw formats `convert` turns into I420. +const RAW_FORMATS: [VideoFormat; 6] = [ + VideoFormat::YUY2, + VideoFormat::NV12, + VideoFormat::BGRx, + VideoFormat::BGRA, + VideoFormat::RGBx, + VideoFormat::RGBA, +]; + +/// How a camera format carries its pixels. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Encoding { + /// A raw format [`super::convert`] handles. + Raw(VideoFormat), + /// Motion-JPEG, decoded per frame. + Mjpeg, +} + +impl Encoding { + /// What converting to I420 costs, in the V4L2 backend's terms: resampling + /// YUV is cheapest, then a color conversion, then a JPEG decode. + fn cost(self) -> u8 { + match self { + Self::Raw(VideoFormat::YUY2 | VideoFormat::NV12) => 0, + Self::Raw(_) => 1, + Self::Mjpeg => 2, + } + } +} + +/// One `EnumFormat` entry of a camera, for a format this backend converts. +#[derive(Clone, Debug, PartialEq)] +pub(super) struct Format { + encoding: Encoding, + /// The sizes on offer. A size range contributes its corners. + sizes: Vec, + /// The exact rates on offer, highest first. Empty for a rate range. + rates: Vec, + /// The size range, when the size is one, which also offers the grid size + /// nearest the request. + ranged: Option, + /// The lowest and highest rate, when the rate is a continuous range. + rate_range: Option<(spa::utils::Fraction, spa::utils::Fraction)>, +} + +/// A size range: every size from `min` to `max` in `step` increments. +#[derive(Clone, Copy, Debug, PartialEq)] +struct Grid { + min: Size, + max: Size, + step: Size, +} + +impl Grid { + /// The size on the grid nearest `want` with even dimensions, the way a V4L2 + /// driver rounds a request for a stepwise size. + fn nearest(&self, want: Size) -> Option { + let snap = |value: u32, min: u32, max: u32, step: u32| { + let (first, last) = bounds(min, max, step)?; + let value = value.clamp(first, last); + if first == last { + return Some(first); + } + // `bounds` spans more than one value only on a nonzero step. + let below = min + (value - min) / step * step; + [below.checked_sub(step), Some(below), below.checked_add(step)] + .into_iter() + .flatten() + .filter(|size| size.is_multiple_of(2) && (first..=last).contains(size)) + .min_by_key(|size| size.abs_diff(value)) + }; + Some(Size::new( + snap(want.width, self.min.width, self.max.width, self.step.width)?, + snap(want.height, self.min.height, self.max.height, self.step.height)?, + )) + } + + /// The largest and smallest sizes on the grid with even dimensions, the way + /// the V4L2 backend reports a stepwise size. + fn corners(&self) -> Vec { + let width = bounds(self.min.width, self.max.width, self.step.width); + let height = bounds(self.min.height, self.max.height, self.step.height); + let (Some((min_width, max_width)), Some((min_height, max_height))) = (width, height) else { + return Vec::new(); + }; + let (largest, smallest) = (Size::new(max_width, max_height), Size::new(min_width, min_height)); + if largest == smallest { + vec![largest] + } else { + vec![largest, smallest] + } + } +} + +/// Whether the rate range `min..=max` holds `rate`, compared as cross products +/// because a range may start at 0/1, which is no valid [`Rate`]. +fn holds(min: spa::utils::Fraction, max: spa::utils::Fraction, rate: Rate) -> bool { + let (num, denom) = (u64::from(rate.numerator()), u64::from(rate.denominator())); + u64::from(min.num) * denom <= num * u64::from(min.denom) && num * u64::from(max.denom) <= u64::from(max.num) * denom +} + +/// The mode a camera stream offers. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Selected { + encoding: Encoding, + size: Size, + /// `None` when the camera reports a rate range and no rate in it was requested. + framerate: Option, +} + +impl Selected { + /// Serialize the `EnumFormat` offers for exactly this mode, a DMA-BUF offer + /// first when the format has a DRM fourcc to import as. + pub(super) fn offers(&self) -> Vec> { + let dmabuf = match self.encoding { + Encoding::Raw(format) => drm_format(format).is_some(), + Encoding::Mjpeg => false, + }; + let mut offers = Vec::new(); + if dmabuf { + offers.push(self.offer(true)); + } + offers.push(self.offer(false)); + offers + } + + fn offer(&self, dmabuf: bool) -> Vec { + let property = |key: FormatProperties, value| spa::pod::Property::new(key.as_raw(), value); + let mut properties = vec![property( + FormatProperties::MediaType, + Value::Id(spa::utils::Id(MediaType::Video.as_raw())), + )]; + match self.encoding { + Encoding::Raw(format) => { + properties.push(property( + FormatProperties::MediaSubtype, + Value::Id(spa::utils::Id(MediaSubtype::Raw.as_raw())), + )); + properties.push(property( + FormatProperties::VideoFormat, + Value::Id(spa::utils::Id(format.as_raw())), + )); + } + Encoding::Mjpeg => properties.push(property( + FormatProperties::MediaSubtype, + Value::Id(spa::utils::Id(MediaSubtype::Mjpg.as_raw())), + )), + } + properties.push(property( + FormatProperties::VideoSize, + Value::Rectangle(spa::utils::Rectangle { + width: self.size.width, + height: self.size.height, + }), + )); + let framerate = match self.framerate { + Some(rate) => Value::Fraction(fraction(rate)), + None => Value::Choice(ChoiceValue::Fraction(spa::utils::Choice( + spa::utils::ChoiceFlags::empty(), + ChoiceEnum::Range { + default: spa::utils::Fraction { + num: DEFAULT_FRAMERATE, + denom: 1, + }, + min: spa::utils::Fraction { num: 0, denom: 1 }, + max: spa::utils::Fraction { num: 1000, denom: 1 }, + }, + ))), + }; + properties.push(property(FormatProperties::VideoFramerate, framerate)); + if dmabuf { + properties.push(linear_modifier()); + } + serialize_format(spa::pod::Object { + type_: spa::utils::SpaTypes::ObjectParamFormat.as_raw(), + id: spa::param::ParamType::EnumFormat.as_raw(), + properties, + }) + } +} + +fn fraction(rate: Rate) -> spa::utils::Fraction { + spa::utils::Fraction { + num: rate.numerator(), + denom: rate.denominator(), + } +} + +/// List the formats a camera node offers that this backend converts. +fn formats( + mainloop: &pw::main_loop::MainLoopRc, + core: &pw::core::CoreRc, + registry: &pw::registry::RegistryRc, + node: &Node, +) -> Result, Error> { + let proxy: pw::node::Node = registry + .bind(&pw::registry::GlobalObject { + id: node.id, + permissions: pw::permissions::PermissionFlags::empty(), + type_: pw::types::ObjectType::Node, + version: 0, + props: None::<&spa::utils::dict::DictRef>, + }) + .map_err(|e| err("pipewire node", e))?; + let formats = Rc::new(RefCell::new(Vec::new())); + let _listener = proxy + .add_listener_local() + .param({ + let formats = formats.clone(); + move |_, id, _, _, param| { + if id != spa::param::ParamType::EnumFormat { + return; + } + let Some(param) = param else { return }; + match spa::pod::deserialize::PodDeserializer::deserialize_any_from(param.as_bytes()) { + Ok((_, value)) => formats.borrow_mut().extend(parse_format(&value)), + Err(e) => tracing::debug!(error = ?e, "unreadable PipeWire EnumFormat"), + } + } + }) + .register(); + proxy.enum_params(0, Some(spa::param::ParamType::EnumFormat), 0, u32::MAX); + roundtrip(mainloop, core)?; + Ok(formats.take()) +} + +/// Read one `EnumFormat` or `Format` pod into the formats it offers that this +/// backend converts. A pod may list several pixel formats in one choice. +fn parse_format(value: &Value) -> Vec { + let Value::Object(object) = value else { + return Vec::new(); + }; + let property = |key: FormatProperties| { + object + .properties + .iter() + .find(|property| property.key == key.as_raw()) + .map(|property| &property.value) + }; + if property(FormatProperties::MediaType) != Some(&Value::Id(spa::utils::Id(MediaType::Video.as_raw()))) { + return Vec::new(); + } + let encodings = match property(FormatProperties::MediaSubtype) { + Some(Value::Id(id)) if *id == spa::utils::Id(MediaSubtype::Mjpg.as_raw()) => vec![Encoding::Mjpeg], + Some(Value::Id(id)) if *id == spa::utils::Id(MediaSubtype::Raw.as_raw()) => { + let formats = match property(FormatProperties::VideoFormat) { + Some(Value::Id(id)) => vec![*id], + Some(Value::Choice(ChoiceValue::Id(choice))) => listed(&choice.1), + _ => Vec::new(), + }; + formats + .into_iter() + .map(|id| VideoFormat::from_raw(id.0)) + .filter(|format| RAW_FORMATS.contains(format)) + .map(Encoding::Raw) + .collect() + } + _ => Vec::new(), + }; + + let size = |rectangle: &spa::utils::Rectangle| Size::new(rectangle.width, rectangle.height); + let (sizes, ranged) = match property(FormatProperties::VideoSize) { + Some(Value::Rectangle(rectangle)) => (vec![size(rectangle)], None), + Some(Value::Choice(ChoiceValue::Rectangle(choice))) => { + let grid = match &choice.1 { + ChoiceEnum::Range { min, max, .. } => Some(Grid { + min: size(min), + max: size(max), + step: Size::new(1, 1), + }), + ChoiceEnum::Step { min, max, step, .. } => Some(Grid { + min: size(min), + max: size(max), + step: size(step), + }), + _ => None, + }; + match grid { + Some(grid) => (grid.corners(), Some(grid)), + None => (listed(&choice.1).iter().map(size).collect(), None), + } + } + _ => (Vec::new(), None), + }; + let rate = |fraction: &spa::utils::Fraction| Rate::new(fraction.num, fraction.denom).ok(); + let framerate = property(FormatProperties::VideoFramerate); + let mut rates: Vec = match framerate { + Some(Value::Fraction(fraction)) => rate(fraction).into_iter().collect(), + Some(Value::Choice(ChoiceValue::Fraction(choice))) => listed(&choice.1).iter().filter_map(rate).collect(), + _ => Vec::new(), + }; + rates.sort_by(|left, right| right.cmp(left)); + rates.dedup(); + // Only a continuous range: whether a rate lies on a stepwise range's grid + // is the driver's call, so that offer stays open. + let rate_range = match framerate { + Some(Value::Choice(ChoiceValue::Fraction(spa::utils::Choice(_, ChoiceEnum::Range { min, max, .. })))) => { + Some((*min, *max)) + } + _ => None, + }; + + encodings + .into_iter() + .map(|encoding| Format { + encoding, + sizes: sizes.clone(), + rates: rates.clone(), + ranged, + rate_range, + }) + .collect() +} + +/// The values a choice lists. A range lists none: it describes a continuum. +fn listed(choice: &ChoiceEnum) -> Vec { + let values = match choice { + ChoiceEnum::None(value) => vec![*value], + // SPA usually repeats the default among the alternatives. + ChoiceEnum::Enum { default, alternatives } => { + std::iter::once(*default).chain(alternatives.iter().copied()).collect() + } + ChoiceEnum::Range { .. } | ChoiceEnum::Step { .. } | ChoiceEnum::Flags { .. } => Vec::new(), + }; + let mut unique = Vec::with_capacity(values.len()); + for value in values { + if !unique.contains(&value) { + unique.push(value); + } + } + unique +} + +/// Pick the mode nearest `want` with the V4L2 backend's rules. A size range +/// offers its size nearest the request, and a rate range that holds the +/// requested rate offers that rate. +fn select(formats: &[Format], want: Request) -> Option { + let mut candidates = Vec::new(); + for format in formats { + let sizes = format + .ranged + .and_then(|grid| grid.nearest(want.size)) + .into_iter() + .chain(format.sizes.iter().copied()); + let rates: Vec> = if format.rates.is_empty() { + let held = want + .framerate + .filter(|&rate| format.rate_range.is_some_and(|(min, max)| holds(min, max, rate))); + vec![held] + } else { + format.rates.iter().copied().map(Some).collect() + }; + for size in sizes { + for &framerate in &rates { + candidates.push(Selected { + encoding: format.encoding, + size, + framerate, + }); + } + } + } + mode::nearest(candidates, want, |selected| mode::Candidate { + size: selected.size, + framerate: selected.framerate, + cost: selected.encoding.cost(), + }) +} + +/// The negotiated MJPEG `Format` as the loop's `VideoInfoRaw`, with the +/// `Encoded` format standing for MJPEG and the negotiated size and rate. +pub(super) fn mjpeg_format(param: &spa::pod::Pod) -> Option { + let (_, value) = spa::pod::deserialize::PodDeserializer::deserialize_any_from(param.as_bytes()).ok()?; + let format = parse_format(&value).into_iter().next()?; + if format.encoding != Encoding::Mjpeg { + return None; + } + let size = *format.sizes.first()?; + let mut info = VideoInfoRaw::default(); + info.set_format(VideoFormat::Encoded); + info.set_size(spa::utils::Rectangle { + width: size.width, + height: size.height, + }); + if let Some(rate) = format.rates.first() { + info.set_framerate(fraction(*rate)); + } + Some(info) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + + fn node(props: &[(&str, &str)]) -> Option { + let props: HashMap<_, _> = props.iter().copied().collect(); + Node::from_props(62, |key| props.get(key).copied()) + } + + const WEBCAM: [(&str, &str); 5] = [ + ("media.class", "Video/Source"), + ("media.role", "Camera"), + ("node.name", "v4l2_input.pci-0000_00_14.0-usb-0_4_1.0"), + ("node.description", "Integrated Camera (V4L2)"), + ("priority.session", "1000"), + ]; + + #[test] + fn camera_nodes_are_video_sources_with_the_camera_role() { + assert_eq!( + node(&WEBCAM), + Some(Node { + id: 62, + name: "v4l2_input.pci-0000_00_14.0-usb-0_4_1.0".to_string(), + description: "Integrated Camera (V4L2)".to_string(), + priority: 1000, + }) + ); + + let mut screen = WEBCAM; + screen[1] = ("media.role", "Screen"); + assert_eq!(node(&screen), None); + let mut sink = WEBCAM; + sink[0] = ("media.class", "Video/Sink"); + assert_eq!(node(&sink), None); + // The name is the id's only stable part, so a node without one is unusable. + assert_eq!(node(&WEBCAM[..2]), None); + } + + #[test] + fn a_camera_without_a_description_falls_back_to_its_nick_then_name() { + let libcamera = [ + ("media.class", "Video/Source"), + ("media.role", "Camera"), + ("node.name", "libcamera_input./base/soc/i2c0mux/i2c@1/imx708@1a"), + ("node.nick", "imx708"), + ]; + assert_eq!(node(&libcamera).unwrap().description, "imx708"); + assert_eq!( + node(&libcamera[..3]).unwrap().description, + "libcamera_input./base/soc/i2c0mux/i2c@1/imx708@1a" + ); + } + + #[test] + fn resolve_finds_the_named_camera_or_the_highest_priority_one() { + let webcam = node(&WEBCAM).unwrap(); + let infrared = Node { + id: 79, + name: "v4l2_input.pci-0000_00_14.0-usb-0_4_1.2".to_string(), + priority: 980, + ..webcam.clone() + }; + let nodes = [infrared.clone(), webcam.clone()]; + assert_eq!(resolve(&nodes, Some(&infrared.name)).unwrap().id, 79); + assert_eq!(resolve(&nodes, None).unwrap().id, 62); + // Equal priorities keep the registry's order. + let tied = [ + Node { + priority: 1000, + ..infrared + }, + webcam, + ]; + assert_eq!(resolve(&tied, None).unwrap().id, 79); + + let error = resolve(&nodes, Some("missing")).unwrap_err().to_string(); + assert!(error.contains("v4l2_input.pci-0000_00_14.0-usb-0_4_1.0"), "{error}"); + assert!(matches!(resolve(&[], None), Err(Error::SourceUnavailable(_)))); + } + + fn rectangle(width: u32, height: u32) -> spa::utils::Rectangle { + spa::utils::Rectangle { width, height } + } + + /// One `EnumFormat` entry the way spa-v4l2 reports it for a UVC webcam. + fn enum_format(subtype: MediaSubtype, format: Option, size: spa::utils::Rectangle) -> Value { + let mut properties = vec![ + spa::pod::Property::new( + FormatProperties::MediaType.as_raw(), + Value::Id(spa::utils::Id(MediaType::Video.as_raw())), + ), + spa::pod::Property::new( + FormatProperties::MediaSubtype.as_raw(), + Value::Id(spa::utils::Id(subtype.as_raw())), + ), + spa::pod::Property::new(FormatProperties::VideoSize.as_raw(), Value::Rectangle(size)), + spa::pod::Property::new( + FormatProperties::VideoFramerate.as_raw(), + Value::Choice(ChoiceValue::Fraction(spa::utils::Choice( + spa::utils::ChoiceFlags::empty(), + ChoiceEnum::None(spa::utils::Fraction { num: 30, denom: 1 }), + ))), + ), + ]; + if let Some(format) = format { + properties.push(spa::pod::Property::new( + FormatProperties::VideoFormat.as_raw(), + Value::Id(spa::utils::Id(format.as_raw())), + )); + } + Value::Object(spa::pod::Object { + type_: spa::utils::SpaTypes::ObjectParamFormat.as_raw(), + id: spa::param::ParamType::EnumFormat.as_raw(), + properties, + }) + } + + /// The integrated webcam's list: MJPEG up to 1080p, YUY2 only at VGA sizes, + /// which is why the first mode is no answer to a size request. + fn webcam_formats() -> Vec { + [ + enum_format(MediaSubtype::Mjpg, None, rectangle(1280, 720)), + enum_format(MediaSubtype::Mjpg, None, rectangle(640, 480)), + enum_format(MediaSubtype::Mjpg, None, rectangle(1920, 1080)), + enum_format(MediaSubtype::Raw, Some(VideoFormat::YUY2), rectangle(640, 480)), + enum_format(MediaSubtype::Raw, Some(VideoFormat::YUY2), rectangle(640, 360)), + enum_format(MediaSubtype::Raw, Some(VideoFormat::GRAY8), rectangle(640, 360)), + ] + .iter() + .flat_map(parse_format) + .collect() + } + + #[test] + fn enum_formats_parse_to_the_convertible_formats() { + let formats = webcam_formats(); + // GRAY8 is not converted, so it is not a candidate. + assert_eq!(formats.len(), 5); + assert_eq!(formats[0].encoding, Encoding::Mjpeg); + assert_eq!(formats[0].sizes, [Size::new(1280, 720)]); + assert_eq!(formats[0].rates, [Rate::new(30, 1).unwrap()]); + assert_eq!(formats[3].encoding, Encoding::Raw(VideoFormat::YUY2)); + } + + #[test] + fn selection_matches_the_v4l2_rules() { + let formats = webcam_formats(); + let select = |width, height| { + let selected = select( + &formats, + Request { + size: Size::new(width, height), + framerate: None, + }, + ) + .unwrap(); + (selected.encoding, selected.size) + }; + // An exact size wins, and the cheaper YUY2 wins a tie with MJPEG. + assert_eq!( + select(640, 480), + (Encoding::Raw(VideoFormat::YUY2), Size::new(640, 480)) + ); + assert_eq!( + select(640, 360), + (Encoding::Raw(VideoFormat::YUY2), Size::new(640, 360)) + ); + assert_eq!(select(1920, 1080), (Encoding::Mjpeg, Size::new(1920, 1080))); + // No exact size: the nearest one, whatever format carries it. + assert_eq!(select(1280, 800), (Encoding::Mjpeg, Size::new(1280, 720))); + } + + /// An NV12 `EnumFormat` entry with the given size and rate properties, the way + /// spa-libcamera reports a sensor. + fn nv12_format(size: Value, framerate: Option) -> Vec { + let property = |key: FormatProperties, value| spa::pod::Property::new(key.as_raw(), value); + let mut properties = vec![ + property( + FormatProperties::MediaType, + Value::Id(spa::utils::Id(MediaType::Video.as_raw())), + ), + property( + FormatProperties::MediaSubtype, + Value::Id(spa::utils::Id(MediaSubtype::Raw.as_raw())), + ), + property( + FormatProperties::VideoFormat, + Value::Id(spa::utils::Id(VideoFormat::NV12.as_raw())), + ), + property(FormatProperties::VideoSize, size), + ]; + properties.extend(framerate.map(|framerate| property(FormatProperties::VideoFramerate, framerate))); + parse_format(&Value::Object(spa::pod::Object { + type_: spa::utils::SpaTypes::ObjectParamFormat.as_raw(), + id: spa::param::ParamType::EnumFormat.as_raw(), + properties, + })) + } + + fn size_choice(choice: ChoiceEnum) -> Value { + Value::Choice(ChoiceValue::Rectangle(spa::utils::Choice( + spa::utils::ChoiceFlags::empty(), + choice, + ))) + } + + fn ratio(num: u32, denom: u32) -> spa::utils::Fraction { + spa::utils::Fraction { num, denom } + } + + fn request(width: u32, height: u32, fps: Option) -> Request { + Request { + size: Size::new(width, height), + framerate: fps.map(|fps| Rate::new(fps, 1).unwrap()), + } + } + + #[test] + fn a_size_range_offers_the_requested_size() { + let formats = nv12_format( + size_choice(ChoiceEnum::Range { + default: rectangle(1920, 1080), + min: rectangle(64, 64), + max: rectangle(4608, 2592), + }), + None, + ); + assert_eq!(formats[0].sizes, [Size::new(4608, 2592), Size::new(64, 64)]); + let selected = select(&formats, request(1280, 720, None)).unwrap(); + assert_eq!((selected.size, selected.framerate), (Size::new(1280, 720), None)); + // An odd request lands on the nearest even size rather than a corner. + let selected = select(&formats, request(1281, 721, None)).unwrap(); + assert_eq!(selected.size, Size::new(1280, 720)); + } + + /// A stepwise range offers only sizes on its grid, and its corners move to + /// even sizes the way `v4l2::bounds` moves a stepwise V4L2 size. + #[test] + fn a_stepwise_size_range_offers_the_nearest_even_size_on_its_grid() { + let eights = nv12_format( + size_choice(ChoiceEnum::Step { + default: rectangle(640, 480), + min: rectangle(32, 32), + max: rectangle(1921, 1081), + step: rectangle(8, 8), + }), + None, + ); + assert_eq!(eights[0].sizes, [Size::new(1920, 1080), Size::new(32, 32)]); + assert_eq!( + select(&eights, request(1280, 720, None)).unwrap().size, + Size::new(1280, 720) + ); + assert_eq!( + select(&eights, request(1270, 716, None)).unwrap().size, + Size::new(1272, 712) + ); + assert_eq!( + select(&eights, request(4000, 3000, None)).unwrap().size, + Size::new(1920, 1080) + ); + + // An odd step alternates parity, so the nearest grid size may be odd. + let threes = nv12_format( + size_choice(ChoiceEnum::Step { + default: rectangle(640, 480), + min: rectangle(1, 1), + max: rectangle(1919, 1079), + step: rectangle(3, 3), + }), + None, + ); + assert_eq!(threes[0].sizes, [Size::new(1918, 1078), Size::new(4, 4)]); + assert_eq!( + select(&threes, request(1280, 720, None)).unwrap().size, + Size::new(1282, 718) + ); + + // An even step from an odd minimum has no even size at all. + let odd = nv12_format( + size_choice(ChoiceEnum::Step { + default: rectangle(641, 481), + min: rectangle(1, 1), + max: rectangle(1919, 1079), + step: rectangle(2, 2), + }), + None, + ); + assert!(odd[0].sizes.is_empty()); + assert_eq!(select(&odd, request(1280, 720, None)), None); + } + + /// A rate range that holds the requested rate ranks as that rate, so an exact + /// rate elsewhere does not beat it. + #[test] + fn a_rate_range_offers_the_requested_rate() { + let range = Value::Choice(ChoiceValue::Fraction(spa::utils::Choice( + spa::utils::ChoiceFlags::empty(), + ChoiceEnum::Range { + default: ratio(30, 1), + min: ratio(0, 1), + max: ratio(60, 1), + }, + ))); + let mut formats = nv12_format(Value::Rectangle(rectangle(1280, 720)), Some(range)); + // A rate range lists no exact rate. + assert!(formats[0].rates.is_empty()); + formats.extend(parse_format(&enum_format( + MediaSubtype::Mjpg, + None, + rectangle(1280, 720), + ))); + + let pick = |fps| { + let selected = select(&formats, request(1280, 720, fps)).unwrap(); + (selected.encoding, selected.framerate) + }; + let nv12 = Encoding::Raw(VideoFormat::NV12); + let rate = |fps| Some(Rate::new(fps, 1).unwrap()); + assert_eq!(pick(Some(60)), (nv12, rate(60))); + assert_eq!(pick(Some(30)), (nv12, rate(30))); + // Out of the range, the exact rate elsewhere is the better answer. + assert_eq!(pick(Some(120)), (Encoding::Mjpeg, rate(30))); + // Nothing requested: the range stays open and the cheaper format wins. + assert_eq!(pick(None), (nv12, None)); + + assert!(holds(ratio(0, 1), ratio(30, 1), Rate::new(30000, 1001).unwrap())); + assert!(!holds(ratio(0, 1), ratio(30, 1), Rate::new(30001, 1000).unwrap())); + } + + /// The offer pins the chosen mode, and only NV12 gets a DMA-BUF offer. + #[test] + fn offers_pin_the_selected_mode() { + let decode = |bytes: &Vec| { + let (_, value) = spa::pod::deserialize::PodDeserializer::deserialize_any_from(bytes).unwrap(); + let Value::Object(object) = value else { + panic!("offer is not an object"); + }; + object + }; + let find = |object: &spa::pod::Object, key: FormatProperties| { + object + .properties + .iter() + .find(|property| property.key == key.as_raw()) + .map(|property| property.value.clone()) + }; + + let mjpeg = Selected { + encoding: Encoding::Mjpeg, + size: Size::new(1920, 1080), + framerate: Some(Rate::new(30, 1).unwrap()), + }; + let offers = mjpeg.offers(); + assert_eq!(offers.len(), 1); + let offer = decode(&offers[0]); + assert_eq!( + find(&offer, FormatProperties::MediaSubtype), + Some(Value::Id(spa::utils::Id(MediaSubtype::Mjpg.as_raw()))) + ); + assert_eq!( + find(&offer, FormatProperties::VideoSize), + Some(Value::Rectangle(rectangle(1920, 1080))) + ); + assert_eq!( + find(&offer, FormatProperties::VideoFramerate), + Some(Value::Fraction(spa::utils::Fraction { num: 30, denom: 1 })) + ); + // The negotiated format parses back into the loop's MJPEG description. + let info = mjpeg_format(spa::pod::Pod::from_bytes(&offers[0]).unwrap()).unwrap(); + assert_eq!(info.format(), VideoFormat::Encoded); + assert_eq!(info.size(), rectangle(1920, 1080)); + + let nv12 = Selected { + encoding: Encoding::Raw(VideoFormat::NV12), + ..mjpeg + }; + let offers = nv12.offers(); + assert_eq!(offers.len(), 2); + assert!(find(&decode(&offers[0]), FormatProperties::VideoModifier).is_some()); + assert!(find(&decode(&offers[1]), FormatProperties::VideoModifier).is_none()); + let yuy2 = Selected { + encoding: Encoding::Raw(VideoFormat::YUY2), + ..mjpeg + }; + assert_eq!(yuy2.offers().len(), 1); + } + + /// Lists the PipeWire cameras over the session socket. Needs no camera and + /// never turns one on, so a host without PipeWire just lists nothing. + #[tokio::test] + async fn lists_cameras_over_the_session_socket() { + if ashpd::is_sandboxed() { + eprintln!("skipping: listing inside a sandbox would ask the camera portal"); + return; + } + let cameras = cameras().await.expect("listing PipeWire cameras"); + for camera in &cameras { + assert!(camera.id.starts_with("pipewire:"), "{}", camera.id); + assert!(!camera.name.is_empty()); + } + eprintln!("PipeWire cameras: {cameras:?}"); + } + + /// Open `config` and check that five frames arrive at the reported size. + async fn capture(config: &Config) -> Result { + let mut stream = crate::capture::open(config).await?; + assert!(stream.width() >= 2 && stream.width().is_multiple_of(2), "bad width"); + assert!(stream.height() >= 2 && stream.height().is_multiple_of(2), "bad height"); + for i in 0..5 { + let frame = stream.read().await?.unwrap_or_else(|| panic!("no frame {i}")); + assert_eq!(frame.surface.width(), stream.width()); + assert_eq!(frame.surface.height(), stream.height()); + } + eprintln!( + "{}: captured 5 frames at {}x{}, {:?} fps, color {:?}", + stream.label(), + stream.width(), + stream.height(), + stream.framerate(), + stream.color() + ); + Ok(stream) + } + + /// Captures from each PipeWire camera over the session socket, at its + /// smallest mode and at its largest mode up to 1080p, and checks the stream + /// lands on the requested size. Ignored because it turns the cameras on: + /// `cargo test -p moq-video --features pipewire pipewire_camera -- --ignored`. + #[tokio::test] + #[ignore = "turns on every PipeWire camera on the host"] + async fn pipewire_camera_captures_frames() { + if ashpd::is_sandboxed() { + eprintln!("skipping: capturing inside a sandbox would ask the camera portal"); + return; + } + let cameras = cameras().await.expect("listing PipeWire cameras"); + if cameras.is_empty() { + eprintln!("skipping: no PipeWire camera"); + return; + } + + let mut captured = 0; + for camera in cameras { + let modes = crate::capture::camera_modes(Some(&camera.id)) + .await + .expect("listing PipeWire camera modes"); + eprintln!("{}: modes {modes:?}", camera.id); + // An IR camera offers only GRAY8, which this backend does not convert. + let Some(smallest) = modes.last() else { + eprintln!("{}: no convertible mode", camera.id); + continue; + }; + let largest = modes + .iter() + .find(|mode| mode.width <= 1920 && mode.height <= 1080) + .unwrap_or(smallest); + for mode in [smallest, largest] { + let config = Config { + source: camera.source(), + width: Some(mode.width), + height: Some(mode.height), + framerate: mode.max_framerate(), + ..Default::default() + }; + let stream = capture(&config) + .await + .unwrap_or_else(|error| panic!("{} at {}x{}: {error}", camera.id, mode.width, mode.height)); + assert_eq!(stream.label(), camera.id); + assert_eq!((stream.width(), stream.height()), (mode.width, mode.height)); + assert_eq!(stream.framerate(), mode.max_framerate()); + } + captured += 1; + } + assert!(captured > 0, "no PipeWire camera could be captured"); + + // `pipewire` alone picks the highest-priority camera, at the default size. + let config = Config { + source: crate::capture::Source::Camera(Some(PIPEWIRE.to_string())), + ..Default::default() + }; + let stream = capture(&config).await.expect("default PipeWire camera"); + assert_eq!(stream.label(), PIPEWIRE); + } +} diff --git a/rs/moq-video/src/capture/v4l2.rs b/rs/moq-video/src/capture/v4l2.rs index 128c6016b41..c1e8099db52 100644 --- a/rs/moq-video/src/capture/v4l2.rs +++ b/rs/moq-video/src/capture/v4l2.rs @@ -6,8 +6,6 @@ //! the pure-Rust [`zune_jpeg`], then converted). This is the CPU path feeding //! NVENC / VAAPI / openh264; there's no GPU surface here. -use std::collections::{BTreeMap, BTreeSet}; - use moq_v4l::buffer::Type as BufType; use moq_v4l::capability::Flags; use moq_v4l::frameinterval::FrameIntervalEnum; @@ -17,9 +15,9 @@ use moq_v4l::io::traits::CaptureStream; use moq_v4l::video::Capture; use moq_v4l::video::capture::Parameters; use moq_v4l::{Device, Format, FourCC}; -use zune_jpeg::zune_core::bytestream::ZCursor; use super::channel::FrameChannel; +use super::mode::{self, Request}; use super::pump::{self, Geometry}; use super::{Config, Mode, Rate, Stream}; use crate::frame::{I420, Surface}; @@ -75,34 +73,19 @@ pub(super) fn cameras() -> Result, Error> { /// on. pub(super) fn modes(selector: Option<&str>) -> Result, Error> { let (device, _) = open_device(selector)?; - let mut sizes: BTreeMap<(u32, u32), BTreeSet> = BTreeMap::new(); + let mut reported = Vec::new(); for candidate in Source::ALL { let fourcc = candidate.fourcc(); let enumerated = frame_sizes(&device, fourcc)?; for size in enumerated { for (width, height) in reported_sizes(size) { - sizes.entry((width, height)).or_default().extend(framerates( - &device, - fourcc, - Size::new(width, height), - )?); + let size = Size::new(width, height); + reported.push((size, framerates(&device, fourcc, size)?)); } } } - - let mut modes: Vec = sizes - .into_iter() - .map(|((width, height), framerates)| Mode { - width, - height, - // Highest first, so the rate a caller most often wants is the one it - // reads without scanning. - framerates: framerates.into_iter().rev().collect(), - }) - .collect(); - modes.sort_by_key(|mode| std::cmp::Reverse(u64::from(mode.width) * u64::from(mode.height))); - Ok(modes) + Ok(mode::modes(reported)) } /// The sizes worth reporting from one `VIDIOC_ENUM_FRAMESIZES` entry. @@ -138,7 +121,7 @@ fn reported_sizes(size: FrameSizeEnum) -> Vec<(u32, u32)> { } /// First and last nonzero even values on the driver's step grid. -fn bounds(min: u32, max: u32, step: u32) -> Option<(u32, u32)> { +pub(super) fn bounds(min: u32, max: u32, step: u32) -> Option<(u32, u32)> { if min == max { return (min != 0 && min.is_multiple_of(2)).then_some((min, min)); } @@ -260,9 +243,9 @@ pub(super) async fn open(config: &Config, device: Option<&str>) -> Result { // Only `bytesused` of the buffer holds the JPEG; the rest is stale. let jpeg = buf.get(..meta.bytesused as usize).unwrap_or(buf); - // zune-jpeg 0.5 reads through a seekable cursor, not a bare slice. - let mut decoder = zune_jpeg::JpegDecoder::new(ZCursor::new(jpeg)); - let rgb = decoder - .decode() - .map_err(|e| Error::Codec(anyhow::anyhow!("MJPEG decode: {e:?}")))?; - let (w, h) = decoder - .dimensions() - .ok_or_else(|| Error::Codec(anyhow::anyhow!("MJPEG frame had no dimensions")))?; - // The stream reports the negotiated size and the encoder is built - // from it, so a frame that decodes to another one can't be published - // as this stream's. - if w as u32 != self.width || h as u32 != self.height { - return Err(Error::Codec(anyhow::anyhow!( - "MJPEG frame is {w}x{h}, not the negotiated {}x{}", - self.width, - self.height - ))); - } - I420::from_rgb(&rgb, crate::Size::new(self.width, self.height))? + I420::from_mjpeg(jpeg, crate::Size::new(self.width, self.height))? } }; let timestamp = u64::try_from(meta.timestamp.sec) @@ -425,11 +390,6 @@ fn open_error(device: &str, error: std::io::Error) -> Error { } } -struct Request { - size: Size, - framerate: Option, -} - /// Negotiate the format we can convert to I420 that lands closest to `want`. /// /// V4L2's non-mutating `VIDIOC_TRY_FMT` is optional, so use the required @@ -524,39 +484,11 @@ fn closest( replies: impl IntoIterator)>, want: Request, ) -> Option<(Format, Source, Option)> { - replies - .into_iter() - .filter(|(format, _, _)| { - Size::new(format.width, format.height) - .validate("camera resolution") - .is_ok() - }) - .min_by(|(left, left_source, left_rate), (right, right_source, right_rate)| { - distance(*left, want.size) - .cmp(&distance(*right, want.size)) - .then_with(|| rate_distance(*left_rate, *right_rate, want.framerate)) - .then_with(|| left_source.cost().cmp(&right_source.cost())) - }) -} - -fn rate_distance(left: Option, right: Option, want: Option) -> std::cmp::Ordering { - let Some(want) = want else { - return std::cmp::Ordering::Equal; - }; - match (left, right) { - (Some(left), Some(right)) => (left.as_f64() - want.as_f64()) - .abs() - .total_cmp(&(right.as_f64() - want.as_f64()).abs()), - (Some(_), None) => std::cmp::Ordering::Less, - (None, Some(_)) => std::cmp::Ordering::Greater, - (None, None) => std::cmp::Ordering::Equal, - } -} - -/// How far a negotiated mode lands from the requested geometry, summed over both -/// dimensions. Zero is an exact match. -fn distance(format: Format, want: Size) -> u64 { - u64::from(format.width.abs_diff(want.width)) + u64::from(format.height.abs_diff(want.height)) + mode::nearest(replies, want, |(format, source, rate)| mode::Candidate { + size: Size::new(format.width, format.height), + framerate: *rate, + cost: source.cost(), + }) } fn set_format(device: &Device, format: Format) -> Result { @@ -625,6 +557,8 @@ mod tests { #[test] fn equivalent_rates_deduplicate_and_sort_numerically() { + use std::collections::BTreeSet; + let rates: BTreeSet<_> = [(1001, 30000), (1, 30), (2, 60), (2, 1)] .into_iter() .map(|(n, d)| rate(moq_v4l::Fraction::new(n, d)).unwrap()) @@ -668,16 +602,6 @@ mod tests { ); } - #[test] - fn rate_scoring_preserves_fractional_precision_and_handles_unknown_rates() { - let ntsc = Some(rate(moq_v4l::Fraction::new(1001, 60000)).unwrap()); - let thirty = Some(rate(moq_v4l::Fraction::new(1, 30)).unwrap()); - assert!(rate_distance(ntsc, thirty, Some(Rate::new(60, 1).unwrap())).is_lt()); - assert!(rate_distance(thirty, ntsc, Some(Rate::new(30, 1).unwrap())).is_lt()); - assert!(rate_distance(ntsc, None, Some(Rate::new(60, 1).unwrap())).is_lt()); - assert!(rate_distance(ntsc, thirty, None).is_eq()); - } - /// A one-pixel step over a 4K range is 8 million modes, and reporting them /// would say nothing the two corners do not. #[test] @@ -797,16 +721,6 @@ mod tests { assert!(closest(replies, request(1280, 720)).is_none()); } - /// Distance is symmetric in the two dimensions and zero only on an exact hit, - /// so a mode that overshoots is no better than one that undershoots by as much. - #[test] - fn distance_is_zero_only_on_an_exact_match() { - let want = Size::new(1280, 720); - assert_eq!(distance(Format::new(1280, 720, Source::Yuyv.fourcc()), want), 0); - assert_eq!(distance(Format::new(1280, 600, Source::Yuyv.fourcc()), want), 120); - assert_eq!(distance(Format::new(1280, 840, Source::Yuyv.fourcc()), want), 120); - } - /// Every format we advertise round-trips through its fourcc, which is what /// lets `negotiate` recognize the driver's substitution. #[test] diff --git a/rs/moq-video/src/frame.rs b/rs/moq-video/src/frame.rs index a2915c09f42..6c166c96e11 100644 --- a/rs/moq-video/src/frame.rs +++ b/rs/moq-video/src/frame.rs @@ -843,8 +843,7 @@ impl I420 { } /// Convert tightly-packed RGB (`width * height * 3` bytes) to I420 in - /// [`Color::infer`]'s color space for this size. Used for MJPEG capture - /// (Linux V4L2), which decodes to RGB. + /// [`Color::infer`]'s color space for this size. Used by [`Self::from_mjpeg`]. #[cfg(all(target_os = "linux", feature = "capture"))] pub(crate) fn from_rgb(rgb: &[u8], size: Size) -> Result { use yuv::rgb_to_yuv420; @@ -864,9 +863,35 @@ impl I420 { Self::pack(&planar, size, Some(color)) } + /// Decode one Motion-JPEG frame to I420 in [`Color::infer`]'s color space. + /// Used by the Linux V4L2 and PipeWire camera paths. + /// + /// The stream reports the negotiated size and the encoder is built from it, + /// so a frame that decodes to another size is an error rather than a frame + /// published as this stream's. + #[cfg(all(target_os = "linux", feature = "capture"))] + pub(crate) fn from_mjpeg(jpeg: &[u8], size: Size) -> Result { + use zune_jpeg::zune_core::bytestream::ZCursor; + + // zune-jpeg 0.5 reads through a seekable cursor, not a bare slice. + let mut decoder = zune_jpeg::JpegDecoder::new(ZCursor::new(jpeg)); + let rgb = decoder + .decode() + .map_err(|e| Error::Codec(anyhow::anyhow!("MJPEG decode: {e:?}")))?; + let (w, h) = decoder + .dimensions() + .ok_or_else(|| Error::Codec(anyhow::anyhow!("MJPEG frame had no dimensions")))?; + if w as u32 != size.width || h as u32 != size.height { + return Err(Error::Codec(anyhow::anyhow!( + "MJPEG frame is {w}x{h}, not the negotiated {size}" + ))); + } + Self::from_rgb(&rgb, size) + } + /// Convert packed YUYV (YUV 4:2:2, `stride` bytes per row) to I420. A chroma /// resample (4:2:2 -> 4:2:0), no color-space conversion. Used for the raw - /// V4L2 capture path (Linux). + /// V4L2 and PipeWire camera capture paths (Linux). #[cfg(all(target_os = "linux", feature = "capture"))] pub(crate) fn from_yuyv(yuyv: &[u8], stride: u32, size: Size) -> Result { use yuv::{YuvPackedImage, yuyv422_to_yuv420}; @@ -883,7 +908,7 @@ impl I420 { yuyv422_to_yuv420(&mut planar, &packed) .map_err(|e| Error::Codec(anyhow::anyhow!("yuyv422_to_yuv420 failed for {width}x{height}: {e}")))?; // A chroma resample, not a color conversion: these samples are in - // whatever space the camera produced, which nothing here names. + // whatever space the camera produced, which the caller names if it can. Self::pack(&planar, size, None) }