diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 2bba16a682a..0d114c5107d 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -370,10 +370,12 @@ moq --client-connect https://relay.example.com/anon --broadcast screen.hang \ import capture --display --system-audio ``` -On Linux the NVENC (NVIDIA) and VAAPI (Intel/AMD) encoders and the PipeWire +On Linux the NVENC (NVIDIA), VAAPI (Intel/AMD), and V4L2 M2M (an ARM SoC's +codec block, such as a Raspberry Pi's VideoCore) encoders and the PipeWire screen capture are compiled in by default. To build `capture` without any of -them (software openh264 + V4L2 camera capture only), drop the default features. `capture` itself still needs libclang and the V4L2 headers for -the camera, and ALSA for the microphone: +them (software openh264 + V4L2 camera capture only), drop the default features. +`capture` itself still needs libclang and the V4L2 headers for the camera, and +ALSA for the microphone: ```bash cargo build --release -p moq-cli --no-default-features \ @@ -398,9 +400,9 @@ The Linux screen backend links libpipewire and is behind the default-on `pipewire` feature; drop it (like the codecs above) for a build without the dependency. The codec is chosen with `--codec` (`h264` default, or `h265`). For H.264 it picks a hardware encoder (VideoToolbox on macOS, NVENC on Linux NVIDIA, -or VAAPI on Linux Intel/AMD when built with the `vaapi` feature) when one is -present, falling back to the built-in software encoder (openh264); force either -with `--hardware` / `--software`. A hardware encoder that was compiled in but +VAAPI on Linux Intel/AMD, or an ARM SoC's V4L2 M2M encoder) when one is present, +falling back to the built-in software encoder (openh264); force either with +`--hardware` / `--software`. A hardware encoder that was compiled in but can't open, typically because its driver libraries aren't on the loader path, warns rather than falling back quietly. H.265 is hardware-only (VideoToolbox on macOS, Media Foundation on Windows). `--camera` takes a bare integer as a device index, diff --git a/doc/lib/rs/crate/moq-video.md b/doc/lib/rs/crate/moq-video.md index e040afda0dd..455f36e8cce 100644 --- a/doc/lib/rs/crate/moq-video.md +++ b/doc/lib/rs/crate/moq-video.md @@ -67,6 +67,7 @@ cargo add moq-video --features pipewire | `capture` | yes | Native device capture (`v4l` and `zune-jpeg` on Linux) | | `nvidia` | yes | NVENC encode and NVDEC decode on Linux (`cudarc`, `moq-nvenc`), dlopen'd at runtime | | `vaapi` | yes | Intel/AMD encode on Linux (`moq-vaapi`), dlopen'd at runtime; not yet validated on hardware | +| `v4l2` | yes | V4L2 M2M encode and decode on ARM SoCs (Raspberry Pi and friends), validated on a Raspberry Pi 4 | | `render` | yes | `wgpu`, the GPU renderer, and Linux DMA-BUF support | | `pipewire` | no | Wayland screen capture via xdg-desktop-portal and DMA-BUF | diff --git a/quest/m3/README.md b/quest/m3/README.md index 506bee98d5f..4f62fa49e64 100644 --- a/quest/m3/README.md +++ b/quest/m3/README.md @@ -12,7 +12,8 @@ its verdict lands and the follow-on work becomes concrete. ## Quests -- [Embedded video](/quest/m3/video-embedded.md) - V4L2 M2M codecs and EGL import, so moq-video works on a Pi +- [Embedded video](/quest/m3/video-embedded.md) - EGL import in the renderer, so moq-video presents on a Pi +- [Decoder drain](/quest/m3/decode-drain.md) - flush and finish for pipelined decoders, so no picture is lost at a track end or crosses a group boundary - [#2819](/quest/m3/2819-moq-video-carry-pipewire-dma-bufs-safely-into-the-vulkan.md) - moq-video: carry PipeWire DMA-BUFs safely into the Vulkan renderer - [#2893](/quest/m3/2893-video-validate-pipewire-dma-buf-capture-on-kde-hardware.md) - video: validate PipeWire DMA-BUF capture on KDE hardware - [Video hardware validation](/quest/m3/video-hardware.md) - run the encode, capture, and zero-copy paths that were written but never run on real machines diff --git a/quest/m3/decode-drain.md b/quest/m3/decode-drain.md new file mode 100644 index 00000000000..1a4717e4647 --- /dev/null +++ b/quest/m3/decode-drain.md @@ -0,0 +1,33 @@ +# [M] Drain pipelined decoders at group and track boundaries + +## Goal + +A decoder that holds pictures back (NVDEC, the V4L2 stateful decoder) hands +every one of them over before a track ends, and a transcode's output groups +begin with the picture that began the input group rather than a straggler +from the one before. + +## Plan + +`decode::Backend` has `decode` and `name` and nothing else: there is no way to +tell a backend the stream is pausing or over. A one-in-one-out decoder +(openh264, VideoToolbox) never needs one, but NVDEC and V4L2 both pipeline, so +the last few pictures of a finite stream stay inside the driver, and at a +group boundary in `moq-transcode` the encoder is flushed while the decoder is +not, so pictures from the closing group surface after the next group's first +access unit went in and are encoded ahead of its keyframe. + +Add `flush` and `finish` to `decode::Backend`, mirroring the encode trait and +with the same rule: no default implementation, so a pipelined backend cannot +inherit a no-op and look like it worked. The V4L2 implementation is the +kernel's drain sequence (`V4L2_DEC_CMD_STOP`, dequeue CAPTURE through +`V4L2_BUF_FLAG_LAST`, `V4L2_DEC_CMD_START`), with a source change that lands +mid-drain handled the way `drain_tail` already handles one. NVDEC's is an +end-of-stream packet through the parser. Call `flush` where `moq-transcode` +flushes the encoder and `finish` before a consumer reports the track over, and +cover it with a delayed-backend test that proves no picture is lost or crosses +a group boundary. + +## Related + +- [Embedded video](/quest/m3/video-embedded.md) - the V4L2 decoder this drains diff --git a/quest/m3/teleop/README.md b/quest/m3/teleop/README.md index 4fcb3275ccf..858ed6cb6f1 100644 --- a/quest/m3/teleop/README.md +++ b/quest/m3/teleop/README.md @@ -26,8 +26,9 @@ operator arbitration, and the latency instrumentation from scratch. Two things are genuinely missing rather than merely undocumented: the hang catalog is video plus audio only (location tracks arrived in moq#401 and were -dropped when the catalog became generic), and `moq-video` has no V4L2-M2M -encoder backend, so the boards that fly have no native hardware-encode path. +dropped when the catalog became generic), and `moq-video`'s V4L2-M2M encoder +backend is compiled into no released `moq-cli`, so the boards that fly have no +native hardware-encode path anyone can install. ### Two delivery classes, one session @@ -94,9 +95,9 @@ stating plainly because it is what a builder is comparing against. - [SITL proof and browser ground station](/quest/m3/teleop/proof.md) - ArduPilot SITL and a synthetic camera flown from a browser ground station, reproducible in five minutes -- [V4L2-M2M encoding](/quest/m3/teleop/v4l2-encode.md) - `moq-video` encodes in - hardware on the boards that fly, so `moq-cli` needs no GStreamer detour on a - Raspberry Pi +- [V4L2-M2M encoding](/quest/m3/teleop/v4l2-encode.md) - a released `moq-cli` + reaches `moq-video`'s hardware encoder on the boards that fly, and the boards + worth buying are written down - [Teleoperation use-case docs](/quest/m3/teleop/docs.md) - `doc/concept/use-case/other.md` becomes the teleoperation page, with a runnable non-media example beside it - [ROS 2 bridge](/quest/m3/teleop/ros2.md) - a ROS 2 bridge sibling to the diff --git a/quest/m3/teleop/v4l2-encode.md b/quest/m3/teleop/v4l2-encode.md index c7a3b8bb6ae..ad2cda784c1 100644 --- a/quest/m3/teleop/v4l2-encode.md +++ b/quest/m3/teleop/v4l2-encode.md @@ -1,30 +1,41 @@ -# [L] V4L2-M2M encoding +# [S] V4L2-M2M encoding ## Goal -`moq-video` encodes in hardware on the boards that fly, so `moq-cli` needs no -GStreamer detour on a Raspberry Pi, and the hardware worth buying is written -down. +A released `moq-cli` encodes in hardware on the boards that fly, so a Raspberry +Pi needs no GStreamer detour, and the hardware worth buying is written down. ## Plan -Add a V4L2-M2M stateful encoder backend next to openh264, NVENC, VAAPI, -VideoToolbox and MediaFoundation, pairing with the V4L2 capture path that -already exists. Mainline kernel API, no vendor SDK. It covers Pi 4, CM4 and -Zero 2 W via `bcm2835-codec`, which is what the drone world actually flies. +The backend exists: `moq-video`'s default-on `v4l2` feature drives the stateful +V4L2 M2M encoder and decoder through `rs/moq-video/src/v4l2.rs`, and both have +run on a Pi 4 (`bcm2835-codec`). What is left is getting it into people's +hands. -Rockchip is deliberately excluded: RK3588 encoding goes through rkmpp in a -vendor kernel rather than V4L2, and the existing `moq-gst` route already ships +Released `moq-cli` binaries are built with default features, so `capture` is +not compiled in and the backend reaches nobody who installs a release. A +hardware encoder nobody can install is not a fix; the feature flip is +[CLI packaging](/quest/m2/cli-packaging.md)'s, and this quest finishes once a +released binary on a Pi 4 publishes from `moq import capture` through the +hardware encoder. + +Write the hardware note down, in `doc/bin/cli.md` next to the capture build +instructions. Two landmines make it part of the deliverable: Raspberry Pi 5 has +no video encoder at all, and Jetson Orin Nano ships without NVENC. The boards +that still encode are Pi 4/CM4/Zero 2 W, Orin NX and above, and RK3588. +Rockchip stays excluded from the backend: RK3588 encoding goes through rkmpp in +a vendor kernel rather than V4L2, and the existing `moq-gst` route already ships aarch64 packages for it. Adding an rkmpp backend is a separate decision. -Two landmines make this worth owning, and make the hardware note part of the -deliverable rather than an afterthought: Raspberry Pi 5 has no video encoder at -all, and Jetson Orin Nano ships without NVENC. The boards that still encode are -Pi 4/CM4/Zero 2 W, Orin NX and above, and RK3588. +Validate what the Pi 4 run did not reach: `set_bitrate` on a running encoder +(congestion control retunes through it), resolutions past 640x360 where 1080p +codes as 1088 rows and the compose rectangle crops it back, and the other +`bcm2835-codec` boards (Zero 2 W, CM4). -Released `moq-cli` binaries are built with default features, so `capture` is -not compiled in. A hardware encoder nobody can install is not a fix; settle how -the capture-enabled build ships as part of this quest. +## Required + +- [CLI packaging](/quest/m2/cli-packaging.md) - the released binary has to + compile `capture` before a hardware encoder in it reaches anyone ## Related diff --git a/quest/m3/video-embedded.md b/quest/m3/video-embedded.md index e5e47878e25..15fd63162d1 100644 --- a/quest/m3/video-embedded.md +++ b/quest/m3/video-embedded.md @@ -2,26 +2,20 @@ ## Goal -`moq-video` encodes and decodes on Raspberry Pi and similar embedded hardware, -and can present there. Neither is possible today: there is no stateful V4L2 -codec backend, and the renderer's only zero-copy import is Vulkan. +`moq-video` presents on Raspberry Pi and similar embedded hardware. Encoding +and decoding there is the V4L2 M2M backends' job already; presenting is not +possible today, because the renderer's only zero-copy import is Vulkan and +those devices have no usable Vulkan driver. ## Plan -**V4L2 M2M encode and decode.** The stateful hardware codec path embedded -devices expose, and the one gap with no alternative: openh264 software encode -on a Pi is not real-time. The seam these need already exists, since timestamps -ride on `Frame` through encode, so a queued device draining frame N-k while -frame N is submitted stamps its output correctly rather than with the -submission clock. +**EGL/GLES import** in the renderer. The devices with a V4L2 codec are the ones +without a usable Vulkan driver, so the existing DMA-BUF import cannot reach +them. Same shape as the Vulkan path: alias the buffer, keep the per-path +fallback and three-strike disable, fall back to I420 when import fails. -**EGL/GLES import** in the renderer. The same devices are the ones without a -usable Vulkan driver, so the existing DMA-BUF import cannot reach them. Same -shape as the Vulkan path: alias the buffer, keep the per-path fallback and -three-strike disable, fall back to I420 when import fails. - -Hardware-gated end to end. Both halves need a real device to validate, and the -usual dlopen-and-degrade rule applies: a build without the device present must +Hardware-gated end to end. It needs a real device to validate, and the usual +dlopen-and-degrade rule applies: a build without the device present must degrade rather than fail to start. Two items from the same wave are deliberately not here. X11 MIT-SHM capture is diff --git a/rs/moq-cli/Cargo.toml b/rs/moq-cli/Cargo.toml index eb66f9cffd7..2ccf5a70259 100644 --- a/rs/moq-cli/Cargo.toml +++ b/rs/moq-cli/Cargo.toml @@ -23,7 +23,7 @@ path = "src/main.rs" doc = false [features] -default = ["iroh", "quinn", "websocket", "nvidia", "vaapi", "pipewire"] +default = ["iroh", "quinn", "websocket", "nvidia", "vaapi", "v4l2", "pipewire"] iroh = ["moq-native/iroh"] jemalloc = ["moq-native/jemalloc"] noq = ["moq-native/noq"] @@ -64,6 +64,12 @@ nvdec = ["nvidia"] # trimmable the same way: libva is dlopen'd at runtime, so it costs the build # nothing. Only bites when `capture` or `transcode` pulls in moq-video. vaapi = ["moq-video?/vaapi", "moq-transcode?/vaapi"] +# The V4L2 M2M codecs of an ARM SoC (a Raspberry Pi's VideoCore and the like), +# on by default like `nvidia` and trimmable the same way. The `v4l` crate it +# needs is the one `capture` already builds for the camera, so it costs nothing +# a capture build does not pay already. Only bites when `capture`, `transcode`, +# or `play` pulls in moq-video. +v4l2 = ["moq-video?/v4l2", "moq-transcode?/v4l2"] # Screen capture for `capture --display` on Linux (xdg-desktop-portal + PipeWire), # on by default like the hardware codecs above and trimmable the same way. Only # bites when `capture` pulls in moq-video; links libpipewire-0.3 at build time. diff --git a/rs/moq-transcode/Cargo.toml b/rs/moq-transcode/Cargo.toml index daf0063cb9e..6cceeb0ff9d 100644 --- a/rs/moq-transcode/Cargo.toml +++ b/rs/moq-transcode/Cargo.toml @@ -17,7 +17,7 @@ categories = ["multimedia", "multimedia::video", "multimedia::encoding"] # the whole point of a GPU transcode fleet, and it is the zero-copy GPU pipeline # (NVDEC decodes and scales in hardware, NVENC encodes the CUDA frame in place). # Both dlopen their driver at runtime, so disabling them only slims the compile. -default = ["nvidia", "vaapi"] +default = ["nvidia", "vaapi", "v4l2"] # NVENC + NVDEC (Linux, dlopen'd at runtime; a no-op elsewhere). nvidia = ["moq-video/nvidia"] # Compatibility aliases for the pre-consolidation names, kept so a manifest that @@ -27,6 +27,9 @@ nvenc = ["nvidia"] nvdec = ["nvidia"] # Intel/AMD VAAPI hardware encoder (Linux; libva dlopen'd at runtime). vaapi = ["moq-video/vaapi"] +# The V4L2 M2M codecs of an ARM SoC (Linux; the `v4l` crate's bindgen needs +# libclang at build time). +v4l2 = ["moq-video/v4l2"] [dependencies] bytes = { workspace = true } diff --git a/rs/moq-video/Cargo.toml b/rs/moq-video/Cargo.toml index 578b4424fb9..51cf67a2988 100644 --- a/rs/moq-video/Cargo.toml +++ b/rs/moq-video/Cargo.toml @@ -20,7 +20,7 @@ categories = ["multimedia", "multimedia::video", "multimedia::encoding"] # it links libpipewire at build time. Workspace consumers opt out at the root # manifest (`default-features = false`) and pick per crate, so the relay and the # language bindings never build the device or render stacks. -default = ["capture", "nvidia", "vaapi", "render"] +default = ["capture", "nvidia", "vaapi", "v4l2", "render"] # Native camera and screen capture. Enabled by default for direct consumers, # but workspace consumers opt in so codec-only bindings avoid device build # dependencies such as V4L2 and libclang. @@ -44,6 +44,21 @@ nvdec = ["nvidia"] # The backend has not been validated on real hardware yet; `Kind::Named` # ("openh264") sidesteps it if it misbehaves. vaapi = ["dmabuf", "dep:moq-vaapi"] +# The V4L2 stateful memory-to-memory hardware codecs on Linux: the H.264 encoder +# and decoder most ARM SoCs expose (a Raspberry Pi's VideoCore, and the +# equivalent block on Rockchip / Amlogic / Allwinner / Exynos). One feature +# because they are one dependency and one code path: both drive the same device +# node through `src/v4l2.rs`, so splitting them would save no build. Both have +# run on a Raspberry Pi 4 (bcm2835-codec, Raspberry Pi OS Bookworm): the encoder +# on `/dev/video11` produced Constrained Baseline 640x360 that plays back +# correctly, and the decoder played a 640x360 stream for a minute without a +# failure and is what automatic selection picks on that board. On by default +# because it costs the build nothing `capture` does not already: the `v4l` crate +# and its bindgen are the same ones camera capture pulls in. A host with no M2M +# codec node fails at open, so automatic backend selection falls through to the +# next encoder or decoder; `Kind::Named("openh264")` sidesteps it on a board +# whose driver misbehaves. +v4l2 = ["dep:v4l", "dep:libc"] # Linux DMA-BUF surface vocabulary and CPU fallback support. Enabled by every # backend that can produce or consume one. It pulls no graphics API by itself. dmabuf = ["dep:libc", "dep:linux-raw-sys"] @@ -116,9 +131,9 @@ ashpd = { version = "0.13", optional = true, default-features = false, features # extra dependencies); libnvrtc itself is never loaded since we ship pre-built PTX # (see frame/nv12_resize.ptx) that the driver JIT-compiles. cudarc = { workspace = true, features = ["nvrtc"], optional = true } -# Mapping a linear DMA-BUF for the universal CPU I420 fallback. Non-linear -# modifiers stay GPU-only and return an honest error instead of treating tiled -# memory as rows. +# Mapping a linear DMA-BUF for the universal CPU I420 fallback, and the V4L2 M2M +# codec node's poll / mmap flags. Non-linear modifiers stay GPU-only and return +# an honest error instead of treating tiled memory as rows. libc = { workspace = true, optional = true } # Probe for the NVIDIA driver libraries before calling cudarc / the NVENC SDK, # which panic (process-abort under release `panic = "abort"`) if their library is @@ -136,8 +151,12 @@ moq-vaapi = { workspace = true, optional = true } # PipeWire video stream for the portal's node, behind the `pipewire` feature. # Links libpipewire-0.3 via pkg-config at build time; bindgen needs libclang. pipewire = { version = "0.10", optional = true } -# Native V4L2 webcam capture (replaces nokhwa). `v4l` streams MMAP buffers; -# MJPEG cameras are decoded with the pure-Rust `zune-jpeg` (no libjpeg/IJG). +# Native V4L2 webcam capture (replaces nokhwa) and, behind the `v4l2` feature, +# the M2M hardware codecs. `v4l` streams MMAP buffers; MJPEG cameras are decoded +# with the pure-Rust `zune-jpeg` (no libjpeg/IJG). The codec backends use its raw +# layer (`v4l::v4l_sys` for the videodev2.h structs, `v4l::v4l2::vidioc` for the +# request codes) rather than a second V4L2 crate, so no ioctl struct is +# hand-rolled and the dependency is one the tree already carries. v4l = { version = "0.14", optional = true } # X11 display/window enumeration and capture. Pure Rust and used when no # Wayland session is active, so Linux screen capture still works without a diff --git a/rs/moq-video/src/decode/backend/mod.rs b/rs/moq-video/src/decode/backend/mod.rs index 8f0fbb01e8f..ec6e65ecee8 100644 --- a/rs/moq-video/src/decode/backend/mod.rs +++ b/rs/moq-video/src/decode/backend/mod.rs @@ -8,8 +8,8 @@ //! //! [`open`] picks the best backend for a [`Codec`] and [`Config`], trying //! hardware candidates (platform-gated: VideoToolbox on macOS, Media Foundation -//! / DXVA on Windows, NVDEC on Linux) before the openh264 software fallback, -//! exactly like the encode side. Only backends that support the requested codec +//! / DXVA on Windows, NVDEC then V4L2 on Linux) before the openh264 software +//! fallback, exactly like the encode side. Only backends that support the requested codec //! are considered: there is no software H.265 or AV1 decoder, so those tracks //! have no fallback below the hardware path. @@ -33,6 +33,9 @@ mod mediafoundation; #[cfg(all(target_os = "linux", feature = "nvidia"))] mod nvdec; +#[cfg(all(target_os = "linux", feature = "v4l2"))] +mod v4l2; + /// The video codec a decoder handles. Derived from the catalog, not chosen by the /// caller. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -99,6 +102,15 @@ const HARDWARE: &[Candidate] = &[ supports: |c| matches!(c, Codec::H264 | Codec::H265 | Codec::Av1), open: nvdec::Nvdec::open, }, + // Last of the Linux hardware decoders, for the same reason as its encode + // counterpart: the SoC blocks it drives are the only hardware on a board that + // has no NVIDIA GPU. + #[cfg(all(target_os = "linux", feature = "v4l2"))] + Candidate { + name: v4l2::NAME, + supports: |c| matches!(c, Codec::H264), + open: v4l2::V4l2::open, + }, ]; const SOFTWARE: Candidate = Candidate { diff --git a/rs/moq-video/src/decode/backend/v4l2.rs b/rs/moq-video/src/decode/backend/v4l2.rs new file mode 100644 index 00000000000..d6c01b48a82 --- /dev/null +++ b/rs/moq-video/src/decode/backend/v4l2.rs @@ -0,0 +1,519 @@ +//! Hardware H.264 decode via the V4L2 stateful M2M decoder (Linux). +//! +//! The mirror of the encode backend, on the same device abstraction and usually +//! on a sibling node of the same driver: a Raspberry Pi decodes on +//! `/dev/video10` and encodes on `/dev/video11`, both `bcm2835-codec`. Behind +//! the same default-on `v4l2` feature. +//! +//! Access units go in on the OUTPUT queue in decode order and pictures come back +//! on CAPTURE as CPU [`Surface::I420`](crate::Surface). The driver copies each +//! input buffer's timestamp onto the picture it produced, so presentation times +//! survive decoder delay without any bookkeeping here. +//! +//! Stateful is the operative word. The driver parses the stream itself and +//! announces the picture size with a `V4L2_EVENT_SOURCE_CHANGE`, which is why +//! the CAPTURE queue does not exist until the first parameter sets have been +//! fed: [`Backend::decode`] returns no frames until it arrives, which is the +//! buffering the trait already allows for. The same event later means the stream +//! changed size, and the CAPTURE queue is torn down and renegotiated. +//! +//! Only H.264, and only stateful. A Raspberry Pi 4's separate HEVC block and +//! Rockchip's `rkvdec` are *stateless* V4L2 decoders: they take per-slice +//! parameters through the media request API and expect userspace to have parsed +//! the bitstream, which is a different interface rather than another format +//! here. [`Config::resize`] is ignored, since these drivers scale on a separate +//! ISP node rather than on the decoder. +//! +//! Run on a Raspberry Pi 4 (`bcm2835-codec`, `/dev/video10`): a 720p H.264 +//! stream decodes and plays at a steady 30fps. The decoder holds several +//! hundred milliseconds of pictures in its own queue, measured at about 690ms +//! end to end against a software decoder on the same Pi, so a caller that +//! cares about delay more than CPU may prefer openh264 there. The ioctl +//! sequence follows the kernel's stateful decoder documentation and an +//! implementation that also ran on a Pi Zero 2 W and a Pi 3. + +use std::collections::VecDeque; +use std::time::{Duration, Instant}; + +use bytes::Bytes; +use moq_net::Timestamp; +use v4l::v4l_sys::V4L2_CID_MIN_BUFFERS_FOR_CAPTURE; + +use super::{Backend, Codec, Config}; +use crate::v4l2::{self, Dequeue, Device, Dir, Format, Planes, Queue, Rect, Request, Role}; +use crate::{Error, Frame, Size, Surface}; + +pub(crate) const NAME: &str = "v4l2"; + +/// Which node decodes: one that takes H.264 in and gives raw 4:2:0 back. +const ROLE: Role = Role { + env: "MOQ_V4L2_DECODER", + input: &[v4l2::H264], + output: v4l2::RAW, +}; + +/// Access units the driver may hold at once. +const CODED_BUFFERS: u32 = 4; + +/// Bytes to reserve per access unit. The catalog does not reach a backend, so +/// there is no resolution to size this from, and one megabyte clears a 1080p +/// keyframe at any bitrate worth sending over a network. +const CODED_SIZE: u32 = 1024 * 1024; + +/// Pictures to allocate beyond the driver's stated minimum, so a caller holding +/// a frame or two does not stall decoding. +const SPARE_PICTURES: u32 = 3; + +/// Pictures to allocate when the driver will not say what it needs. +const DEFAULT_PICTURES: u32 = 8; + +/// How long [`Backend::decode`] waits for the driver to take an access unit +/// before giving up on it. +const BUFFER_TIMEOUT: Duration = Duration::from_millis(500); + +/// How long a call waits for the source change that follows the first parameter +/// sets. Missing it is not an error on its own: a subscriber can join anywhere +/// in a group, so the first access units of a stream may well be undecodable +/// and the next one gets another look. +const SOURCE_CHANGE_TIMEOUT: Duration = Duration::from_millis(250); + +/// How long the decoder spends waiting for that source change in total before +/// giving up on the stream. +/// +/// Without a ceiling, a stream the driver can never size costs every call the +/// whole of [`SOURCE_CHANGE_TIMEOUT`] and returns nothing, which is a permanent +/// four frames a second with no error and no log. Twenty access units is far +/// more than a driver needs, since [`Decoder`](crate::decode::Decoder) holds +/// everything back until the first keyframe and the parameter sets that ride +/// with it. +const SOURCE_CHANGE_LIMIT: Duration = Duration::from_secs(5); + +/// How long a resolution change waits for the driver to hand back the pictures +/// it decoded before it. Missing the tail costs those pictures but not the +/// stream, so it is bounded rather than waited out. +const DRAIN_TIMEOUT: Duration = Duration::from_millis(500); + +/// How long each wait inside those loops parks for. +const POLL_INTERVAL: Duration = Duration::from_millis(5); + +/// Access units remembered for the pictures they will become. Well past what +/// any driver holds, so a picture the driver never produces costs a slot and not +/// the stream. +const REMEMBERED: usize = 64; + +pub(crate) struct V4l2 { + device: Device, + /// The OUTPUT queue: access units going in. + coded: Queue, + /// The CAPTURE queue, which exists only once the driver has reported the + /// stream's size. + pictures: Option, + /// When the first access unit went in, which is what + /// [`SOURCE_CHANGE_LIMIT`] is measured from. + since: Option, + /// Access units the driver has taken, by the key their picture will carry, + /// with the timestamp to give that picture. The buffer carries whole + /// microseconds, and a [`Timestamp`] is an instant at its own scale, which a + /// round trip through microseconds would change. + submitted: VecDeque<(Duration, Timestamp)>, +} + +/// The CAPTURE queue plus where a picture sits in one of its buffers. +struct Pictures { + queue: Queue, + planes: Planes, +} + +impl V4l2 { + pub(crate) fn open(codec: Codec, _config: &Config) -> Result, Error> { + if codec != Codec::H264 { + return Err(Error::UnsupportedCodec(format!("{NAME} decodes H.264 only"))); + } + + let device = v4l2::open(&ROLE)?; + // Before the first access unit, so the announcement of the stream's size + // cannot be missed. + device.subscribe_source_change()?; + + let coded = device.set_format( + Dir::Output, + &Request { + pixelformat: v4l2::H264, + // The driver reads the real dimensions out of the bitstream. This is + // only a hint for how large a coded buffer to allocate, which the + // explicit `sizeimage` overrides anyway. + size: Size::new(1920, 1088), + sizeimage: Some(CODED_SIZE), + color: None, + }, + )?; + if coded.pixelformat != v4l2::H264 { + return Err(Error::Codec(anyhow::anyhow!( + "V4L2 decoder answered an H264 request with {}", + v4l2::name(coded.pixelformat) + ))); + } + + let mut coded = Queue::alloc(&device, Dir::Output, coded, CODED_BUFFERS)?; + // Started immediately, unlike the encoder's: a stateful decoder has to be + // consuming before it can parse a stream and report its size. + coded.stream_on(&device)?; + + tracing::info!( + decoder = NAME, + device = %device.path().display(), + "opened H.264 decoder" + ); + + Ok(Box::new(Self { + device, + coded, + pictures: None, + since: None, + submitted: VecDeque::new(), + })) + } + + /// Hand one access unit to the driver, waiting for a free buffer. + fn submit(&mut self, access_unit: &Bytes, timestamp: Timestamp) -> Result<(), Error> { + let deadline = Instant::now() + BUFFER_TIMEOUT; + let index = loop { + while let Some(buffer) = self.coded.dequeue(&self.device)?.buffer() { + if buffer.failed() { + tracing::warn!( + decoder = NAME, + buffer = buffer.index, + "V4L2 decoder could not decode an access unit" + ); + } + self.coded.reclaim(buffer.index); + } + if let Some(index) = self.coded.take_free() { + break index; + } + if Instant::now() >= deadline { + return Err(Error::Codec(anyhow::anyhow!( + "V4L2 decoder held every input buffer for {BUFFER_TIMEOUT:?}" + ))); + } + self.device.wait(POLL_INTERVAL); + }; + + let capacity = self.coded.plane(index, 0).len(); + if access_unit.len() > capacity { + // The buffer goes back on the free list: losing one to a single oversized + // access unit would shrink the pool for the rest of the stream. + self.coded.reclaim(index); + return Err(Error::Codec(anyhow::anyhow!( + "access unit of {} bytes exceeds the V4L2 decoder's {capacity} byte buffer", + access_unit.len() + ))); + } + self.coded.plane_mut(index, 0)[..access_unit.len()].copy_from_slice(access_unit); + + let bytesused = [access_unit.len() as u32]; + let key = key(timestamp); + self.coded.queue(&self.device, index, &bytesused, key)?; + remember(&mut self.submitted, key, timestamp); + Ok(()) + } + + /// Negotiate the CAPTURE queue against the size the driver has just reported. + fn negotiate(&mut self) -> Result<(), Error> { + if let Some(pictures) = self.pictures.take() { + pictures.queue.release(&self.device)?; + } + + let format = self.capture_format()?; + // Read after the format is settled, since `VIDIOC_S_FMT` on CAPTURE resets + // the compose rectangle to the default for the format it just took. + // + // The coded size is rounded up to whole macroblocks, so the picture is the + // compose rectangle inside it. A driver that reports none codes exactly the + // picture. + let visible = self + .device + .visible(Dir::Capture) + .unwrap_or_else(|| Rect::whole(format.size)); + let planes = Planes::new(&format, visible)?; + + // The driver needs a minimum of its own to hold reference frames; anything + // below it decodes wrong or not at all. + let minimum = self + .device + .control(V4L2_CID_MIN_BUFFERS_FOR_CAPTURE) + .map_or(DEFAULT_PICTURES, |minimum| minimum.max(1) as u32 + SPARE_PICTURES); + + tracing::info!( + decoder = NAME, + format = v4l2::name(format.pixelformat), + coded = %format.size, + visible = %visible.size, + left = visible.left, + top = visible.top, + buffers = minimum, + "V4L2 decoder negotiated its output" + ); + + let mut queue = Queue::alloc(&self.device, Dir::Capture, format, minimum)?; + while let Some(index) = queue.take_free() { + queue.queue(&self.device, index, &[], Duration::ZERO)?; + } + queue.stream_on(&self.device)?; + + self.pictures = Some(Pictures { queue, planes }); + Ok(()) + } + + /// Choose the raw format the CAPTURE queue produces. + /// + /// `VIDIOC_G_FMT` reports the driver's own default, which need not be one this + /// code can lay out: amphion defaults to the tiled `NV12_8L128` and mtk-vcodec + /// to `MM21`. That the node offered NV12 at open does not settle it either, + /// since the set narrows to what the driver supports for the stream it has now + /// parsed. `VIDIOC_ENUM_FMT` re-reads that set and `VIDIOC_S_FMT` is the step + /// where userspace picks from it. See + /// `Documentation/userspace-api/media/v4l/dev-decoder.rst`, "Capture Setup", + /// steps 3 and 4. + fn capture_format(&self) -> Result { + let format = self.device.format(Dir::Capture)?; + if v4l2::RAW.contains(&format.pixelformat) { + return Ok(format); + } + + let offered = self.device.formats(Dir::Capture)?; + let Some(&pixelformat) = v4l2::RAW.iter().find(|code| offered.contains(code)) else { + return Err(Error::Codec(anyhow::anyhow!( + "V4L2 decoder defaults to {} and offers no 8-bit 4:2:0 format for this stream", + v4l2::name(format.pixelformat) + ))); + }; + + tracing::debug!( + decoder = NAME, + default = v4l2::name(format.pixelformat), + selected = v4l2::name(pixelformat), + "V4L2 decoder defaulted to a format this cannot read" + ); + self.device.set_format( + Dir::Capture, + &Request { + pixelformat, + // Unchanged from what the driver parsed out of the stream: this has no + // compose or scaling to ask for. + size: format.size, + sizeimage: None, + color: None, + }, + ) + } + + /// Collect every picture the driver has finished, re-queueing each buffer as + /// it is copied out. + /// + /// Returns whether the sequence ended, which the driver says with + /// `V4L2_BUF_FLAG_LAST` on its last picture and then with `EPIPE` on any + /// dequeue past it. + fn drain(&mut self, frames: &mut Vec) -> Result { + let Some(pictures) = &self.pictures else { + return Ok(false); + }; + + loop { + let buffer = match pictures.queue.dequeue(&self.device)? { + Dequeue::Buffer(buffer) => buffer, + Dequeue::Empty => return Ok(false), + Dequeue::Ended => return Ok(true), + }; + + // A zero-length picture is the driver marking the end of a sequence + // rather than a frame, which is what arrives just before a source change. + let decoded = match buffer.written(0) { + 0 => None, + // A buffer flagged `V4L2_BUF_FLAG_ERROR` dequeues successfully and holds + // a picture the driver could not decode, so reading it out would publish + // garbage under a valid timestamp. + _ if buffer.failed() => { + tracing::warn!( + decoder = NAME, + buffer = buffer.index, + "V4L2 decoder flagged a picture bad" + ); + None + } + _ => Some(pictures.planes.read(&pictures.queue, &buffer)?), + }; + // Back to the driver before anything can fail: a picture buffer left out + // of the pool is one the decoder never gets to write again. + pictures.queue.queue(&self.device, buffer.index, &[], Duration::ZERO)?; + + if let Some(decoded) = decoded { + let timestamp = restore(&mut self.submitted, buffer.timestamp)?; + frames.push(Frame::new(Surface::I420(decoded), timestamp)); + } + if buffer.last() { + return Ok(true); + } + } + } + + /// Take the pictures still in the CAPTURE queue when a sequence ends. + /// + /// A source change is an implicit drain: the driver decodes everything from + /// before the change, marks the last of it with `V4L2_BUF_FLAG_LAST`, and only + /// then is the CAPTURE queue free to be torn down. Releasing it as soon as the + /// event arrives discards every picture the driver had already decoded, which + /// is a visible gap at each mid-stream resolution change. See + /// `Documentation/userspace-api/media/v4l/dev-decoder.rst`, "Dynamic Resolution + /// Change". + fn drain_tail(&mut self) -> Result, Error> { + let mut frames = Vec::new(); + let deadline = Instant::now() + DRAIN_TIMEOUT; + loop { + if self.drain(&mut frames)? { + return Ok(frames); + } + if Instant::now() >= deadline { + // Renegotiated anyway: a tail the driver never finished is worth less + // than the rest of the stream, and holding the old queue open does not + // make it arrive. + tracing::warn!( + decoder = NAME, + frames = frames.len(), + "V4L2 decoder did not end the sequence within {DRAIN_TIMEOUT:?}" + ); + return Ok(frames); + } + self.device.wait(POLL_INTERVAL); + } + } +} + +/// The `struct timeval` an access unit rides the driver under, which the +/// driver copies onto the picture it becomes. +fn key(timestamp: Timestamp) -> Duration { + Duration::from_micros(timestamp.as_micros() as u64) +} + +/// Record an access unit handed to the driver, forgetting the oldest once +/// [`REMEMBERED`] are outstanding. +fn remember(submitted: &mut VecDeque<(Duration, Timestamp)>, key: Duration, timestamp: Timestamp) { + if submitted.len() == REMEMBERED { + submitted.pop_front(); + } + submitted.push_back((key, timestamp)); +} + +/// The timestamp the access unit behind a picture went in with. +/// +/// Falls back to the buffer's own when the driver stamped a picture with +/// nothing that was submitted, which is the best that can be said about it. +fn restore(submitted: &mut VecDeque<(Duration, Timestamp)>, key: Duration) -> Result { + // Searched rather than popped: a decoder is free to hand pictures back in + // presentation order, which is not the order they went in. + let found = submitted.iter().position(|(at, _)| *at == key); + match found.and_then(|at| submitted.remove(at)) { + Some((_, timestamp)) => Ok(timestamp), + None => Ok(Timestamp::from_micros(key.as_micros() as u64)?), + } +} + +impl Backend for V4l2 { + fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result, Error> { + self.submit(&access_unit, timestamp)?; + + // Before the first negotiation there is nowhere for a picture to go, so the + // event is worth waiting for. After it, checking costs one ioctl and a + // missed resolution change would decode the rest of the stream at the old + // geometry. + let mut frames = Vec::new(); + if self.pictures.is_none() { + let since = *self.since.get_or_insert_with(Instant::now); + let deadline = Instant::now() + SOURCE_CHANGE_TIMEOUT; + while !self.device.take_source_change() { + if Instant::now() >= deadline { + let waited = since.elapsed(); + if waited >= SOURCE_CHANGE_LIMIT { + return Err(Error::Codec(anyhow::anyhow!( + "V4L2 decoder did not report the stream's size within {waited:?}" + ))); + } + tracing::debug!( + decoder = NAME, + ?waited, + "V4L2 decoder has not reported the stream's size" + ); + return Ok(Vec::new()); + } + self.device.wait(POLL_INTERVAL); + } + self.negotiate()?; + } else if self.device.take_source_change() { + // The pictures decoded before the change are still in the CAPTURE queue, + // and the kernel wants them taken before the queue is released. + frames = self.drain_tail()?; + self.negotiate()?; + } + + self.drain(&mut frames)?; + Ok(frames) + } + + fn name(&self) -> &str { + NAME + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn micros(micros: u64) -> Timestamp { + Timestamp::from_micros(micros).unwrap() + } + + /// A picture comes back with the timestamp its access unit went in with, at + /// its own scale: the driver carries whole microseconds, and a 90 kHz tick is + /// not one, so anything rebuilt from the buffer would be a different instant. + /// Pictures may also come back in presentation rather than submission order. + #[test] + fn a_picture_carries_its_access_unit_timestamp_unchanged() { + let ninety_khz = moq_net::Timescale::new(90_000).unwrap(); + let first = Timestamp::new(3003, ninety_khz).unwrap(); + let second = Timestamp::new(6006, ninety_khz).unwrap(); + assert_ne!(Timestamp::from_micros(first.as_micros() as u64).unwrap(), first); + + let mut submitted = VecDeque::new(); + remember(&mut submitted, key(first), first); + remember(&mut submitted, key(second), second); + + assert_eq!(restore(&mut submitted, key(second)).unwrap(), second); + assert_eq!(restore(&mut submitted, key(first)).unwrap(), first); + assert!(submitted.is_empty()); + } + + /// A picture stamped with nothing that was submitted still gets a timestamp, + /// the buffer's own, rather than failing the stream. + #[test] + fn an_unknown_picture_keeps_the_buffer_timestamp() { + let mut submitted = VecDeque::new(); + remember(&mut submitted, key(micros(10)), micros(10)); + assert_eq!(restore(&mut submitted, Duration::from_micros(7)).unwrap(), micros(7)); + // The one that was submitted is still waiting for its picture. + assert_eq!(submitted.len(), 1); + } + + /// What is remembered is bounded, and it is the oldest that is forgotten. + #[test] + fn remembered_access_units_are_bounded() { + let mut submitted = VecDeque::new(); + for at in 0..=REMEMBERED as u64 { + remember(&mut submitted, key(micros(at)), micros(at)); + } + assert_eq!(submitted.len(), REMEMBERED); + assert_eq!(restore(&mut submitted, key(micros(0))).unwrap(), micros(0)); + assert_eq!(submitted.len(), REMEMBERED); + assert_eq!(restore(&mut submitted, key(micros(1))).unwrap(), micros(1)); + assert_eq!(submitted.len(), REMEMBERED - 1); + } +} diff --git a/rs/moq-video/src/decode/decoder.rs b/rs/moq-video/src/decode/decoder.rs index 635741fd07d..3bf02093338 100644 --- a/rs/moq-video/src/decode/decoder.rs +++ b/rs/moq-video/src/decode/decoder.rs @@ -30,8 +30,8 @@ pub enum Kind { Hardware, /// Software (openh264) only. Software, - /// A specific backend by name, e.g. `"videotoolbox"`, `"nvdec"`, - /// `"openh264"`. + /// A specific backend by name, e.g. `"videotoolbox"`, `"nvdec"`, `"v4l2"`, + /// or `"openh264"`. Named(String), } diff --git a/rs/moq-video/src/encode/backend/mod.rs b/rs/moq-video/src/encode/backend/mod.rs index 786517b9e60..ebcc6324215 100644 --- a/rs/moq-video/src/encode/backend/mod.rs +++ b/rs/moq-video/src/encode/backend/mod.rs @@ -35,6 +35,9 @@ mod mediafoundation; #[cfg(all(target_os = "linux", feature = "nvidia"))] mod nvenc; +#[cfg(all(target_os = "linux", feature = "v4l2"))] +mod v4l2; + #[cfg(all(target_os = "linux", feature = "vaapi"))] mod vaapi; @@ -111,6 +114,15 @@ const HARDWARE: &[Candidate] = &[ codecs: &[Codec::H264], open: vaapi::Vaapi::open, }, + // Last of the Linux hardware encoders: the SoC blocks it drives are the only + // hardware on a board that has neither an NVIDIA GPU nor a VAAPI stack, so it + // is never the one being chosen over a faster peer. + #[cfg(all(target_os = "linux", feature = "v4l2"))] + Candidate { + name: v4l2::NAME, + codecs: &[Codec::H264], + open: v4l2::V4l2::open, + }, ]; /// Software fallbacks, all platforms, always available so a box with no usable diff --git a/rs/moq-video/src/encode/backend/v4l2.rs b/rs/moq-video/src/encode/backend/v4l2.rs new file mode 100644 index 00000000000..13326068d8c --- /dev/null +++ b/rs/moq-video/src/encode/backend/v4l2.rs @@ -0,0 +1,1125 @@ +//! Hardware H.264 backend via the V4L2 stateful M2M encoder (Linux). +//! +//! The encoder most ARM SoCs ship: a Raspberry Pi's VideoCore (`bcm2835-codec`), +//! and the equivalent block on Rockchip, Amlogic, Allwinner, and Samsung parts. +//! Without it a Pi either republishes what `rpicam-vid` already encoded or +//! spends its CPU on openh264. +//! +//! Behind the default-on `v4l2` feature. It costs no runtime dependency (the +//! interface is ioctls on a device node), only the `v4l` crate's build-time +//! bindgen, and a host with no M2M node fails at open so automatic selection +//! falls through to the next encoder. +//! +//! Feeds the driver 8-bit 4:2:0 and takes back Annex-B with in-band SPS/PPS +//! ahead of every IDR, which is the avc3 shape the H.264 importer expects. +//! [`Planes`](crate::v4l2::Planes) owns the layout: the driver picks the raw +//! fourcc, the stride, and the padded row count, and none of the three has to be +//! what was asked for. +//! +//! Two things the driver will not do by itself, both learned on a Pi: +//! 1. `bcm2835-codec` defaults to H.264 level 1.0, which is 176x144. The level +//! has to be set from the resolution, framerate, and bitrate before +//! `VIDIOC_S_FMT` or the encoder refuses anything larger, and no ioctl +//! reports the default, so there is nothing to detect and the level is +//! always set. +//! 2. Repeated parameter sets have two controls and drivers implement one +//! each: `bcm2835-codec` has `REPEAT_SEQ_HEADER` and not +//! `PREPEND_SPSPPS_TO_IDR`. Both are asked for and whichever lands wins, +//! because a subscriber joining at a later keyframe can only start if that +//! keyframe carries SPS/PPS. +//! +//! Run on a Raspberry Pi 4 (`bcm2835-codec`, `/dev/video11`): 640x360 at 30fps +//! comes out as Constrained Baseline with SPS and PPS ahead of every keyframe, +//! and plays back with a correct picture. The ioctl sequence is carried over +//! from an implementation that also ran on a Pi Zero 2 W and a Pi 3; those two +//! have not been re-run with this port. + +use std::collections::VecDeque; +use std::time::{Duration, Instant}; + +use bytes::{Bytes, BytesMut}; +use moq_net::Timestamp; +use v4l::v4l_sys::{ + V4L2_CID_MPEG_VIDEO_BITRATE, V4L2_CID_MPEG_VIDEO_BITRATE_MODE, V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME, + V4L2_CID_MPEG_VIDEO_GOP_SIZE, V4L2_CID_MPEG_VIDEO_H264_LEVEL, V4L2_CID_MPEG_VIDEO_H264_PROFILE, + V4L2_CID_MPEG_VIDEO_HEADER_MODE, V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR, V4L2_CID_MPEG_VIDEO_REPEAT_SEQ_HEADER, + V4L2_ENC_CMD_START, V4L2_ENC_CMD_STOP, v4l2_mpeg_video_bitrate_mode_V4L2_MPEG_VIDEO_BITRATE_MODE_CBR, + v4l2_mpeg_video_h264_level, v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_1_0, + v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_1_1, + v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_1_2, + v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_1_3, + v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_2_0, + v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_2_1, + v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_2_2, + v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_3_0, + v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_3_1, + v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_3_2, + v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_4_0, + v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_4_1, + v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_4_2, + v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_5_0, + v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_5_1, + v4l2_mpeg_video_h264_profile_V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE, + v4l2_mpeg_video_header_mode_V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME, +}; + +use super::super::encoder::Config; +use super::{Backend, Encoded}; +use crate::v4l2::{self, Dequeue, Device, Dir, Planes, Queue, Rect, Request, Role}; +use crate::{Error, Frame, Size}; + +pub(crate) const NAME: &str = "v4l2"; + +/// Which node encodes: one that takes raw 4:2:0 in and gives H.264 back. +const ROLE: Role = Role { + env: "MOQ_V4L2_ENCODER", + input: v4l2::RAW, + output: &[v4l2::H264], +}; + +/// Raw frames the driver may hold at once. Enough to keep the hardware fed +/// while the caller converts the next frame, and no more: every buffer is a +/// full picture of memory on a board that has little. +const RAW_BUFFERS: u32 = 4; + +/// Coded buffers. More than [`RAW_BUFFERS`] so a run of large access units +/// never leaves the encoder with nowhere to write. +const CODED_BUFFERS: u32 = 8; + +/// How long [`Backend::encode`] waits for the driver to hand a raw buffer back +/// before giving up on the frame. Reached only if the hardware has stalled: +/// with [`RAW_BUFFERS`] outstanding, an encoder keeping up returns one within a +/// frame interval. +const BUFFER_TIMEOUT: Duration = Duration::from_millis(500); + +/// How long [`Backend::flush`] waits for the codec to hand back an access unit +/// for a frame it has already taken. +const FLUSH_TIMEOUT: Duration = Duration::from_millis(500); + +/// How long each wait inside those loops parks for. Short enough that the +/// timeouts above are honored closely, long enough not to spin. +const POLL_INTERVAL: Duration = Duration::from_millis(5); + +/// How much [`Pending`] will carry from coded buffers that hold no picture. +/// +/// Parameter sets are a few hundred bytes, so nothing under this is reached by +/// a driver behaving as documented. Past it the driver is emitting something +/// that is neither a picture nor a header worth keeping, and holding more of it +/// would only put more of it in front of the next access unit. +const CARRY_LIMIT: usize = 64 * 1024; + +pub(crate) struct V4l2 { + device: Device, + /// The OUTPUT queue: raw frames going in. + raw: Queue, + /// The CAPTURE queue: coded access units coming back. + coded: Queue, + /// Where a raw frame's samples go in a `raw` buffer. + planes: Planes, + size: Size, + /// Frames the driver has taken and not yet answered with an access unit. + pending: Pending, + /// Whether the driver takes `VIDIOC_ENCODER_CMD`. Cleared the first time it + /// refuses one, after which a flush can only take what the codec has already + /// finished. + drainable: bool, + /// Whether the driver takes `V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME`. Cleared + /// the first time it refuses one, which is worth saying once and not per + /// keyframe. + keyframes: bool, +} + +impl V4l2 { + pub(crate) fn open(config: &Config) -> Result, Error> { + let size = config.size(); + size.validate("V4L2 encode of")?; + + let device = v4l2::open(&ROLE)?; + + // Before `VIDIOC_S_FMT`: bcm2835-codec picks its encoder configuration + // from the profile and level at format time, and a level set afterwards + // does not take. + device.set_control( + V4L2_CID_MPEG_VIDEO_H264_PROFILE, + v4l2_mpeg_video_h264_profile_V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE as i32, + )?; + device.set_control(V4L2_CID_MPEG_VIDEO_H264_LEVEL, h264_level(config)?)?; + device.set_control(V4L2_CID_MPEG_VIDEO_GOP_SIZE, config.gop as i32)?; + set_bitrate(&device, config.resolved_bitrate())?; + // Constant rate is what a live uplink wants: the congestion controller + // already owns the rate, and a variable-rate encoder would spend it on the + // wrong frames. + device.try_control( + V4L2_CID_MPEG_VIDEO_BITRATE_MODE, + v4l2_mpeg_video_bitrate_mode_V4L2_MPEG_VIDEO_BITRATE_MODE_CBR as i32, + ); + + // The parameter sets belong in the first access unit rather than on a coded + // buffer of their own. A driver left in `HEADER_MODE_SEPARATE`, which is + // s5p-mfc's default, emits them matched to no source frame and with no + // timestamp worth publishing them under. Optional because not every driver + // has the control, and [`Pending`] carries them into the next access unit + // where it does not land. + device.try_control( + V4L2_CID_MPEG_VIDEO_HEADER_MODE, + v4l2_mpeg_video_header_mode_V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME as i32, + ); + + // The two spellings of "repeat the parameter sets", neither of which every + // driver has. A subscriber joining at any keyframe needs SPS/PPS in band + // ahead of it, so if neither lands the track is only joinable at its first + // keyframe and that is worth a warning. + let repeat = device.try_control(V4L2_CID_MPEG_VIDEO_REPEAT_SEQ_HEADER, 1) + | device.try_control(V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR, 1); + if !repeat { + tracing::warn!( + encoder = NAME, + device = %device.path().display(), + "driver repeats no parameter sets; subscribers can only join at the first keyframe" + ); + } + + // Coded first, raw second, which is the order + // `Documentation/userspace-api/media/v4l/dev-encoder.rst` gives under + // "Initialization" and not just a preference: an encoder derives a new + // OUTPUT format from the CAPTURE format it is given, so a raw format + // negotiated first is one the driver is free to have replaced by the time + // its stride is read. + let coded = device.set_format( + Dir::Capture, + &Request { + pixelformat: v4l2::H264, + size, + sizeimage: Some(coded_size(size)), + color: None, + }, + )?; + if coded.pixelformat != v4l2::H264 { + return Err(Error::Codec(anyhow::anyhow!( + "V4L2 encoder answered an H264 request with {}", + v4l2::name(coded.pixelformat) + ))); + } + + let raw = device.set_format( + Dir::Output, + &Request { + pixelformat: v4l2::NV12, + size, + sizeimage: None, + color: Some(config.resolved_color()), + }, + )?; + + // Rate control spends the bitrate per unit of time, so it needs to know how + // much time a frame is. A driver without the ioctl assumes 30. + if let Err(err) = device.set_framerate(Dir::Output, config.framerate) { + tracing::debug!(encoder = NAME, %err, "driver does not take a framerate"); + } + + let planes = Planes::new(&raw, Rect::whole(size))?; + + tracing::info!( + encoder = NAME, + device = %device.path().display(), + format = v4l2::name(raw.pixelformat), + stride = raw.planes[0].stride, + width = size.width, + height = size.height, + "opened H.264 encoder" + ); + + let raw = Queue::alloc(&device, Dir::Output, raw, RAW_BUFFERS)?; + let coded = Queue::alloc(&device, Dir::Capture, coded, CODED_BUFFERS)?; + + Ok(Box::new(Self { + device, + raw, + coded, + planes, + size, + pending: Pending::default(), + drainable: true, + keyframes: true, + })) + } + + /// Start both queues, which has to happen with a raw frame already queued. + /// + /// bcm2835-codec wants the first OUTPUT buffer queued before `STREAMON` and + /// the CAPTURE queue started last. The ordering is legal on every driver, so + /// it is unconditional rather than a quirk to detect. + fn start(&mut self) -> Result<(), Error> { + self.raw.stream_on(&self.device)?; + while let Some(index) = self.coded.take_free() { + self.coded.queue(&self.device, index, &[], Duration::ZERO)?; + } + self.coded.stream_on(&self.device) + } + + /// Take a raw buffer, waiting for the driver to release one if it holds them + /// all. + fn free_buffer(&mut self) -> Result { + let deadline = Instant::now() + BUFFER_TIMEOUT; + loop { + self.reclaim()?; + if let Some(index) = self.raw.take_free() { + return Ok(index); + } + if Instant::now() >= deadline { + return Err(Error::Codec(anyhow::anyhow!( + "V4L2 encoder held every input buffer for {BUFFER_TIMEOUT:?}" + ))); + } + self.device.wait(POLL_INTERVAL); + } + } + + /// Take back every raw buffer the driver has finished reading. + /// + /// A buffer flagged `V4L2_BUF_FLAG_ERROR` is one the driver gave up on, so no + /// access unit will ever answer that frame. + fn reclaim(&mut self) -> Result<(), Error> { + while let Some(buffer) = self.raw.dequeue(&self.device)?.buffer() { + if buffer.failed() { + tracing::warn!(encoder = NAME, buffer = buffer.index, "V4L2 encoder dropped a frame"); + self.pending.dropped(buffer.timestamp); + } + self.raw.reclaim(buffer.index); + } + Ok(()) + } + + /// Collect every access unit the driver has finished, re-queueing each coded + /// buffer as it is emptied. + /// + /// Returns whether the driver marked the end of a sequence, which is how a + /// drain knows it is over. + fn drain(&mut self, out: &mut Vec) -> Result { + self.reclaim()?; + + loop { + let buffer = match self.coded.dequeue(&self.device)? { + Dequeue::Buffer(buffer) => buffer, + Dequeue::Empty => return Ok(false), + // Past the buffer flagged `V4L2_BUF_FLAG_LAST`, which is the end of a + // sequence just as much as the flag itself is. + Dequeue::Ended => return Ok(true), + }; + + let payload = access_unit(self.coded.payload(&buffer, 0), buffer.written(0)); + // Back to the driver before anything can fail: a coded buffer left out + // of the pool is one the encoder never gets to write again. + self.coded.queue(&self.device, buffer.index, &[], Duration::ZERO)?; + + if buffer.failed() { + // Whatever is in the buffer is not a whole access unit, which is what a + // `coded_size` too small for a keyframe produces. Publishing it would + // put a truncated NAL in the middle of the track. + tracing::warn!( + encoder = NAME, + buffer = buffer.index, + bytes = buffer.bytesused[0], + "V4L2 encoder flagged an access unit bad" + ); + self.pending.dropped(buffer.timestamp); + } else if !payload.is_empty() + && let Some((payload, timestamp)) = self.pending.matched(buffer.timestamp, payload) + { + // The driver copies the raw buffer's timestamp onto the coded buffer its + // work came out on, so this is the picture that was encoded rather than + // whatever went in last, and the timestamp is that frame's own. + out.push(Encoded::new(payload, timestamp)); + } + + if buffer.last() { + return Ok(true); + } + } + } + + /// Empty the codec's pipeline, leaving it ready for the frames that follow. + /// + /// The kernel's drain sequence, which is the only thing that makes an encoder + /// release a frame it is still holding: `V4L2_ENC_CMD_STOP`, dequeue CAPTURE + /// until the buffer flagged `V4L2_BUF_FLAG_LAST`, then `V4L2_ENC_CMD_START` to + /// resume with all the state from before the drain. See + /// `Documentation/userspace-api/media/v4l/dev-encoder.rst`, "Drain". + /// + /// Waiting is not a substitute for it. An encoder deeper than one-in-one-out + /// (a lookahead, two-pass rate control) holds those frames until it is told + /// the stream stopped, and a flush falls on every group boundary, so the first + /// boundary would time out and take the broadcast with it. bcm2835-codec + /// happens to be one-in-one-out, so a Pi would not have shown this. + fn drain_tail(&mut self) -> Result, Error> { + // The sequence needs both queues streaming. `VIDIOC_ENCODER_CMD` succeeds + // without starting one otherwise, and there is nothing to drain before the + // first frame anyway. + if !self.raw.streaming() || !self.coded.streaming() { + return Ok(Vec::new()); + } + + if self.drainable + && let Err(err) = self.device.encoder_cmd(V4L2_ENC_CMD_STOP) + { + // A driver with no encoder command cannot be asked to stop, so the most + // that can be done is to take what it has already finished: complete on a + // one-in-one-out encoder, short of the tail on anything deeper. + tracing::warn!( + encoder = NAME, + %err, + "driver takes no encoder command; a group boundary can only drain what it has finished" + ); + self.drainable = false; + } + + match self.drainable { + true => self.drain_to_last(), + false => self.wait_out(), + } + } + + /// Run the rest of the drain sequence, up to and including the restart. + fn drain_to_last(&mut self) -> Result, Error> { + let mut out = Vec::new(); + let mut deadline = Instant::now() + FLUSH_TIMEOUT; + loop { + let before = out.len(); + if self.drain(&mut out)? { + break; + } + // Progress earns more time, so a slow encoder finishes a long tail while + // a stalled one still gives up after `FLUSH_TIMEOUT` of silence. + if out.len() > before { + deadline = Instant::now() + FLUSH_TIMEOUT; + } else if Instant::now() >= deadline { + return Err(Error::Codec(anyhow::anyhow!( + "V4L2 encoder did not finish its drain within {FLUSH_TIMEOUT:?}, holding {} frame(s)", + self.pending.len() + ))); + } + self.device.wait(POLL_INTERVAL); + } + + if !self.pending.is_empty() { + // The drain is over, so these are frames the driver took and never + // answered. Left standing they would make the next drain, or a fallback to + // `wait_out`, believe the codec still owes something. + tracing::debug!( + encoder = NAME, + frames = self.pending.len(), + "V4L2 encoder ended its drain still owing access units" + ); + self.pending.forget(); + } + + // A stopped encoder accepts OUTPUT buffers but does not process them, so + // without this the frames after the boundary would sit in the driver. + self.device.encoder_cmd(V4L2_ENC_CMD_START)?; + Ok(out) + } + + /// Take what the codec has already finished, for a driver that cannot be told + /// to stop. + /// + /// Only a drain where the encoder holds nothing it has not been asked for, + /// which is what an empty [`Pending`] says here. + fn wait_out(&mut self) -> Result, Error> { + let mut out = Vec::new(); + let mut deadline = Instant::now() + FLUSH_TIMEOUT; + loop { + let before = out.len(); + self.drain(&mut out)?; + if self.pending.is_empty() { + return Ok(out); + } + if out.len() > before { + deadline = Instant::now() + FLUSH_TIMEOUT; + } else if Instant::now() >= deadline { + return Err(Error::Codec(anyhow::anyhow!( + "V4L2 encoder held {} frame(s) for {FLUSH_TIMEOUT:?} without encoding them", + self.pending.len() + ))); + } + self.device.wait(POLL_INTERVAL); + } + } +} + +impl Backend for V4l2 { + fn encode(&mut self, frame: &Frame, keyframe: bool) -> Result, Error> { + if frame.size() != self.size { + return Err(Error::Codec(anyhow::anyhow!( + "V4L2 encoder opened for {} was given a {} frame", + self.size, + frame.size() + ))); + } + + let i420 = frame.surface.to_i420()?; + // Also reclaims finished input buffers, so the codec is drained even when + // the caller never asks for output. + let index = self.free_buffer()?; + self.planes.write(&mut self.raw, index, &i420)?; + + if keyframe && self.keyframes { + // A button control: the value is ignored, the write is the request. It + // applies to the next frame queued, so it goes in immediately before. + // + // Best-effort: a driver that answers `EINVAL` still keeps to + // `V4L2_CID_MPEG_VIDEO_GOP_SIZE`, so keyframes land on its own boundary + // rather than the caller's, which is a worse group layout and not a + // broken stream. + self.keyframes = self.device.try_control(V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME, 0); + if !self.keyframes { + tracing::warn!( + encoder = NAME, + device = %self.device.path().display(), + "driver takes no keyframe request; groups fall on the encoder's own GOP boundary" + ); + } + } + + let key = key(frame.timestamp); + let bytesused: Vec = self.raw.format().planes.iter().map(|plane| plane.sizeimage).collect(); + self.raw.queue(&self.device, index, &bytesused, key)?; + self.pending.queued(key, frame.timestamp); + + if !self.raw.streaming() { + self.start()?; + } + + let mut out = Vec::new(); + self.drain(&mut out)?; + Ok(out) + } + + fn flush(&mut self) -> Result, Error> { + self.drain_tail() + } + + fn finish(&mut self) -> Result, Error> { + self.drain_tail() + } + + fn set_bitrate(&mut self, bitrate: u64) -> Result<(), Error> { + // Settable on a streaming encoder and applied without an IDR, which is + // exactly what the congestion controller wants. A driver that refuses gets + // `BitrateUnsupported`, which tells the control loop to stop adapting + // rather than to stop encoding. + set_bitrate(&self.device, bitrate).map_err(|err| { + tracing::debug!(encoder = NAME, %err, "driver refused a bitrate change"); + Error::BitrateUnsupported(NAME) + }) + } + + fn name(&self) -> &str { + NAME + } +} + +/// The `struct timeval` a frame rides the driver under. +/// +/// A key rather than the timestamp itself: the buffer carries whole +/// microseconds, and a [`Timestamp`] is an instant at its own scale, which a +/// round trip through microseconds would change. So the frame is looked up by +/// this and answered with the timestamp it arrived with. +fn key(timestamp: Timestamp) -> Duration { + Duration::from_micros(timestamp.as_micros() as u64) +} + +/// Which frame each coded buffer answers, and what to do with one that answers +/// none. +/// +/// The driver copies an OUTPUT buffer's timestamp onto the CAPTURE buffer its +/// work came out on, so the timestamp is the only thing tying an access unit +/// back to a frame. Counting the two against each other does not work: under +/// `V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE` the first coded buffer is SPS/PPS +/// alone, answering no frame, and counting it would leave the encoder reporting +/// itself drained one access unit early at every group boundary while the +/// straggler surfaced in the next group ahead of its keyframe. +#[derive(Debug, Default)] +struct Pending { + /// Frames queued and not yet answered, in the order they were queued: the + /// key the driver will hand back, and the timestamp to answer it with. + frames: VecDeque<(Duration, Timestamp)>, + /// Bytes from a coded buffer that held no picture, waiting for the access + /// unit to go in front of. + header: Option, + /// Set once a picture has answered no frame, so a driver that stamps + /// nothing says so once rather than per access unit. + unstamped: bool, +} + +impl Pending { + /// Record a frame handed to the driver under `key`. + fn queued(&mut self, key: Duration, timestamp: Timestamp) { + self.frames.push_back((key, timestamp)); + } + + /// How many frames the driver has taken and not answered. + fn len(&self) -> usize { + self.frames.len() + } + + fn is_empty(&self) -> bool { + self.frames.is_empty() + } + + /// The access unit a coded buffer holds and the timestamp of the frame it + /// answers, or `None` when it answers no frame. + /// + /// Parameter sets that arrived on their own go in front of the next access + /// unit instead of being published as a frame of their own, which is both what + /// `HEADER_MODE_JOINED_WITH_1ST_FRAME` would have produced and what a + /// subscriber joining at that keyframe needs. + fn matched(&mut self, key: Duration, payload: Bytes) -> Option<(Bytes, Timestamp)> { + // Decided on the bytes before the timestamp is consulted: parameter sets on + // their own carry whatever timestamp the driver left behind, usually zero, + // and zero is also a timestamp a real frame can have (the first one of a + // capture, and the one [`Config::probe`] encodes). Matched by timestamp + // alone, the header would be published as that frame and the picture that + // answers it carried into the next access unit instead. + if !has_picture(&payload) { + self.carry(payload); + return None; + } + let Some(at) = self.frames.iter().position(|(frame, _)| *frame == key) else { + // A picture that answers no frame has no timestamp to be published + // under, and putting it in front of a later access unit would corrupt + // the track rather than repair it. It means the driver is not copying + // timestamps, which is worth saying once. + if !self.unstamped { + tracing::warn!( + encoder = NAME, + "V4L2 encoder answered a picture that matches no frame; the driver is not copying timestamps" + ); + self.unstamped = true; + } + return None; + }; + + // Frames ahead of the match were taken and never answered, and an encoder + // does not go back: nothing will answer them now. + let (_, timestamp) = self.frames[at]; + self.frames.drain(..=at); + let payload = match self.header.take() { + Some(header) => join(&header, &payload), + None => payload, + }; + Some((payload, timestamp)) + } + + /// Hold a coded buffer that holds no picture, to go in front of the next one + /// that does. + /// + /// Discards everything held past [`CARRY_LIMIT`], including what arrived + /// last: parameter sets are a few hundred bytes, so anything that large is + /// not a header a subscriber needs. + fn carry(&mut self, payload: Bytes) { + let held = self.header.as_ref().map_or(0, Bytes::len); + if held + payload.len() > CARRY_LIMIT { + tracing::debug!( + encoder = NAME, + held, + "V4L2 encoder discarded coded bytes that hold no picture" + ); + self.header = None; + return; + } + self.header = Some(match self.header.take() { + Some(header) => join(&header, &payload), + None => payload, + }); + } + + /// Forget the frame a buffer answered without a usable access unit. + /// + /// Only that frame: a raw buffer the driver gave up on says nothing about + /// the frames queued before it, whose access units may still be on the way. + fn dropped(&mut self, key: Duration) { + if let Some(at) = self.frames.iter().position(|(frame, _)| *frame == key) { + self.frames.remove(at); + } + } + + /// Forget every outstanding frame, for a drain the driver has declared over. + fn forget(&mut self) { + self.frames.clear(); + } +} + +/// Concatenate two pieces of one access unit, which are already Annex-B and so +/// need nothing between them. +fn join(header: &[u8], payload: &[u8]) -> Bytes { + let mut joined = BytesMut::with_capacity(header.len() + payload.len()); + joined.extend_from_slice(header); + joined.extend_from_slice(payload); + joined.freeze() +} + +fn set_bitrate(device: &Device, bitrate: u64) -> Result<(), Error> { + device.set_control(V4L2_CID_MPEG_VIDEO_BITRATE, bitrate.min(i32::MAX as u64) as i32) +} + +/// Whether an Annex-B buffer holds a coded picture. +/// +/// A VCL NAL (types 1 through 5) is what makes a buffer an access unit; a buffer +/// of parameter sets, SEI, or delimiters alone answers no frame. Start codes +/// cannot occur inside a NAL, so scanning for them finds every header. +fn has_picture(annexb: &[u8]) -> bool { + annexb + .windows(4) + .any(|bytes| matches!(bytes, [0, 0, 1, header] if (1..=5).contains(&(header & 0x1f)))) +} + +/// One row of H.264 Table A-1 (level limits), in the units the table uses. +struct Level { + code: v4l2_mpeg_video_h264_level, + /// `MaxMBPS`: macroblocks per second. + per_second: u32, + /// `MaxFS`: macroblocks per frame. + per_frame: u32, + /// `MaxBR` for the Baseline profile, in kbit/s. + kbps: u32, +} + +/// The four menu entries added in Linux 5.7, which `videodev2.h` on an older +/// build host does not have. The values are fixed by the UAPI, so they are +/// spelled here rather than taken from the bindings. +const LEVEL_5_2: v4l2_mpeg_video_h264_level = 16; +const LEVEL_6_0: v4l2_mpeg_video_h264_level = 17; +const LEVEL_6_1: v4l2_mpeg_video_h264_level = 18; +const LEVEL_6_2: v4l2_mpeg_video_h264_level = 19; + +/// Table A-1, as far as `V4L2_CID_MPEG_VIDEO_H264_LEVEL` spells it. Level 1b +/// is left out: it fits nothing 1.1 does not. +const LEVELS: &[Level] = &[ + Level { + code: v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_1_0, + per_second: 1_485, + per_frame: 99, + kbps: 64, + }, + Level { + code: v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_1_1, + per_second: 3_000, + per_frame: 396, + kbps: 192, + }, + Level { + code: v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_1_2, + per_second: 6_000, + per_frame: 396, + kbps: 384, + }, + Level { + code: v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_1_3, + per_second: 11_880, + per_frame: 396, + kbps: 768, + }, + Level { + code: v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_2_0, + per_second: 11_880, + per_frame: 396, + kbps: 2_000, + }, + Level { + code: v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_2_1, + per_second: 19_800, + per_frame: 792, + kbps: 4_000, + }, + Level { + code: v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_2_2, + per_second: 20_250, + per_frame: 1_620, + kbps: 4_000, + }, + Level { + code: v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_3_0, + per_second: 40_500, + per_frame: 1_620, + kbps: 10_000, + }, + Level { + code: v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_3_1, + per_second: 108_000, + per_frame: 3_600, + kbps: 14_000, + }, + Level { + code: v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_3_2, + per_second: 216_000, + per_frame: 5_120, + kbps: 20_000, + }, + Level { + code: v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_4_0, + per_second: 245_760, + per_frame: 8_192, + kbps: 20_000, + }, + Level { + code: v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_4_1, + per_second: 245_760, + per_frame: 8_192, + kbps: 50_000, + }, + Level { + code: v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_4_2, + per_second: 522_240, + per_frame: 8_704, + kbps: 50_000, + }, + Level { + code: v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_5_0, + per_second: 589_824, + per_frame: 22_080, + kbps: 135_000, + }, + Level { + code: v4l2_mpeg_video_h264_level_V4L2_MPEG_VIDEO_H264_LEVEL_5_1, + per_second: 983_040, + per_frame: 36_864, + kbps: 240_000, + }, + Level { + code: LEVEL_5_2, + per_second: 2_073_600, + per_frame: 36_864, + kbps: 240_000, + }, + Level { + code: LEVEL_6_0, + per_second: 4_177_920, + per_frame: 139_264, + kbps: 240_000, + }, + Level { + code: LEVEL_6_1, + per_second: 8_355_840, + per_frame: 139_264, + kbps: 480_000, + }, + Level { + code: LEVEL_6_2, + per_second: 16_711_680, + per_frame: 139_264, + kbps: 800_000, + }, +]; + +/// The lowest H.264 level the config fits in, as the +/// `V4L2_CID_MPEG_VIDEO_H264_LEVEL` menu spells it. +/// +/// All three limits count, not just the frame size: 1080p fits level 4.0 at +/// 30fps and needs 4.2 at 60, and a level that understates the stream is one a +/// driver may clamp the rate to and a decoder may refuse. bcm2835-codec defaults +/// to level 1.0, whose 99-macroblock limit is 176x144, so the level is always +/// set and always set before `VIDIOC_S_FMT`. +/// +/// # Errors +/// +/// A config past level 6.2, which no H.264 level fits and so nothing should be +/// asked to encode. A driver whose menu stops short of the level chosen refuses +/// the control instead, which fails the open the same way. +fn h264_level(config: &Config) -> Result { + let size = config.size(); + let per_frame = size.width.div_ceil(16) * size.height.div_ceil(16); + let per_second = per_frame as u64 * config.framerate as u64; + let kbps = config.resolved_bitrate().div_ceil(1_000); + LEVELS + .iter() + .find(|level| { + per_frame <= level.per_frame && per_second <= level.per_second as u64 && kbps <= level.kbps as u64 + }) + .map(|level| level.code as i32) + .ok_or_else(|| { + Error::Codec(anyhow::anyhow!( + "{size} at {}fps and {kbps} kbit/s exceeds H.264 level 6.2", + config.framerate + )) + }) +} + +/// How large a coded buffer to ask for, since the driver cannot size an access +/// unit from the picture dimensions. +/// +/// Half a raw frame is well clear of what a keyframe costs at any sane quality, +/// and the floor keeps a small picture's buffer larger than its own first +/// keyframe. +fn coded_size(size: Size) -> u32 { + const MIN: u32 = 256 * 1024; + const MAX: u32 = 4 * 1024 * 1024; + (size.pixels().min(u32::MAX as u64) as u32 / 2).clamp(MIN, MAX) +} + +/// Copy an access unit out of a coded buffer, without the driver's padding. +/// +/// Trailing zeroes past the last NAL are `trailing_zero_8bits`, which a decoder +/// discards, and some drivers pad a coded buffer with them rather than reporting +/// the exact length. Dropping them is only safe because the profile is +/// constrained baseline, hence CAVLC: a CABAC stream can carry meaningful +/// `cabac_zero_words` in the same position. +fn access_unit(buffer: &[u8], bytesused: u32) -> Bytes { + let used = (bytesused as usize).min(buffer.len()); + let end = buffer[..used] + .iter() + .rposition(|byte| *byte != 0) + .map_or(0, |at| at + 1); + Bytes::copy_from_slice(&buffer[..end]) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn config(width: u32, height: u32, framerate: u32) -> Config { + Config::new(width, height, framerate) + } + + fn micros(micros: u64) -> Timestamp { + Timestamp::from_micros(micros).unwrap() + } + + /// Queue a frame the way [`Backend::encode`] does, keyed on its timestamp. + fn queue(pending: &mut Pending, timestamp: Timestamp) { + pending.queued(key(timestamp), timestamp); + } + + /// The level has to clear the resolution, or bcm2835-codec refuses the format + /// outright. Spot-checked against Table A-1 rather than the driver's menu, so + /// a driver with a different menu ordering still gets a correct level. + #[test] + fn the_level_clears_the_resolution() { + assert_eq!(h264_level(&config(320, 240, 30)).unwrap(), 4); // 1.3 + assert_eq!(h264_level(&config(640, 480, 30)).unwrap(), 8); // 3.0 + assert_eq!(h264_level(&config(1280, 720, 30)).unwrap(), 9); // 3.1 + assert_eq!(h264_level(&config(1920, 1080, 30)).unwrap(), 11); // 4.0 + assert_eq!(h264_level(&config(3840, 2160, 30)).unwrap(), 15); // 5.1 + } + + /// The same picture at twice the rate is twice the macroblocks per second, + /// which is the column a frame-size lookup misses: 1080p60 is level 4.2, not + /// 4.0, and 720p60 is 3.2. + #[test] + fn the_level_clears_the_framerate() { + assert_eq!(h264_level(&config(1920, 1080, 60)).unwrap(), 13); // 4.2 + assert_eq!(h264_level(&config(1280, 720, 60)).unwrap(), 10); // 3.2 + assert_eq!(h264_level(&config(320, 240, 60)).unwrap(), 6); // 2.1 + } + + /// The bitrate is the third column. A 720p30 stream is level 3.1 until it is + /// asked for more than 3.1's 14 Mbit/s. + #[test] + fn the_level_clears_the_bitrate() { + let mut config = config(1280, 720, 30); + config.bitrate = Some(14_000_000); + assert_eq!(h264_level(&config).unwrap(), 9); // 3.1 + config.bitrate = Some(14_000_001); + assert_eq!(h264_level(&config).unwrap(), 10); // 3.2 + } + + /// The menu runs to 6.2, and 4K60 is already past 5.1: nothing is quietly + /// labelled with a level it does not fit, and past the table there is no + /// level to ask for at all. + #[test] + fn the_level_runs_to_the_end_of_the_menu() { + assert_eq!(h264_level(&config(3840, 2160, 60)).unwrap(), 16); // 5.2 + assert_eq!(h264_level(&config(7680, 4320, 60)).unwrap(), 18); // 6.1 + assert_eq!(h264_level(&config(7680, 4320, 120)).unwrap(), 19); // 6.2 + assert!(h264_level(&config(7680, 4320, 240)).is_err()); + } + + /// A VCL NAL is what makes a buffer a picture, behind either start code. + #[test] + fn a_picture_is_recognized_by_its_vcl_nal() { + assert!(has_picture(&[0, 0, 0, 1, 0x65])); + assert!(has_picture(&[0, 0, 1, 0x41])); + // SPS, PPS, SEI, and an access unit delimiter are not pictures. + assert!(!has_picture(&[0, 0, 0, 1, 0x67, 0x42, 0, 0, 0, 1, 0x68, 0xce])); + assert!(!has_picture(&[0, 0, 0, 1, 0x06, 0, 0, 0, 1, 0x09])); + assert!(!has_picture(&[])); + assert!(!has_picture(&[0x65])); + } + + #[test] + fn the_coded_buffer_is_bounded() { + assert_eq!(coded_size(Size::new(160, 120)), 256 * 1024); + assert_eq!(coded_size(Size::new(1920, 1080)), 1920 * 1080 / 2); + assert_eq!(coded_size(Size::new(7680, 4320)), 4 * 1024 * 1024); + } + + /// A driver that pipelines answers a frame several buffers later, and only + /// the timestamp says which frame that was. + #[test] + fn an_access_unit_is_matched_to_the_frame_it_came_from() { + let mut pending = Pending::default(); + for at in [0, 33_000, 66_000] { + queue(&mut pending, micros(at)); + } + assert_eq!(pending.len(), 3); + + let payload = Bytes::from_static(&[0, 0, 0, 1, 0x65]); + assert_eq!( + pending.matched(Duration::from_micros(0), payload.clone()), + Some((payload.clone(), micros(0))) + ); + assert_eq!(pending.len(), 2); + + // A frame the driver skipped goes with the one that overtook it, since an + // encoder never comes back to it. + assert!(pending.matched(Duration::from_micros(66_000), payload).is_some()); + assert!(pending.is_empty()); + } + + /// The case a bare count gets wrong: a coded buffer that answers no frame at + /// all, which is what `HEADER_MODE_SEPARATE` produces. + #[test] + fn separate_parameter_sets_join_the_next_access_unit() { + let mut pending = Pending::default(); + queue(&mut pending, micros(500)); + + // SPS/PPS on their own, under whatever timestamp the driver left behind. + assert_eq!( + pending.matched(Duration::from_micros(0), Bytes::from_static(&[0, 0, 0, 1, 0x67])), + None + ); + // Still owed the frame, which is what stops a flush finishing early. + assert_eq!(pending.len(), 1); + + let (joined, _) = pending + .matched(Duration::from_micros(500), Bytes::from_static(&[0, 0, 0, 1, 0x65])) + .unwrap(); + assert_eq!(&joined[..], &[0, 0, 0, 1, 0x67, 0, 0, 0, 1, 0x65]); + assert!(pending.is_empty()); + // Carried once, not onto every access unit after it. + let next = Bytes::from_static(&[0, 0, 0, 1, 0x41]); + queue(&mut pending, micros(533)); + assert_eq!( + pending.matched(Duration::from_micros(533), next.clone()), + Some((next, micros(533))) + ); + } + + /// The frame comes back with the timestamp it went in with, at its own + /// scale. The driver carries whole microseconds, and a 90 kHz tick is not + /// one, so anything rebuilt from the buffer would be a different instant. + #[test] + fn the_answer_carries_the_frame_timestamp_unchanged() { + let ninety_khz = moq_net::Timescale::new(90_000).unwrap(); + let timestamp = Timestamp::new(3003, ninety_khz).unwrap(); + assert_ne!(Timestamp::from_micros(timestamp.as_micros() as u64).unwrap(), timestamp); + + let mut pending = Pending::default(); + queue(&mut pending, timestamp); + let (_, answered) = pending + .matched(key(timestamp), Bytes::from_static(&[0, 0, 0, 1, 0x65])) + .unwrap(); + assert_eq!(answered, timestamp); + } + + /// The case the timestamp alone cannot settle: parameter sets stamped zero + /// while the frame they precede is stamped zero too, which the first frame of + /// a capture and the one [`Config::probe`] encodes both are. The header must + /// not be published as that frame. + #[test] + fn parameter_sets_stamped_like_the_first_frame_still_join_it() { + let mut pending = Pending::default(); + queue(&mut pending, micros(0)); + + assert_eq!( + pending.matched( + Duration::ZERO, + Bytes::from_static(&[0, 0, 0, 1, 0x67, 0, 0, 0, 1, 0x68]) + ), + None + ); + assert_eq!(pending.len(), 1); + + let (joined, _) = pending + .matched(Duration::ZERO, Bytes::from_static(&[0, 0, 0, 1, 0x65])) + .unwrap(); + assert_eq!(&joined[..], &[0, 0, 0, 1, 0x67, 0, 0, 0, 1, 0x68, 0, 0, 0, 1, 0x65]); + assert!(pending.is_empty()); + } + + /// What is carried in the hope of a picture to go in front of has to stop + /// growing. + #[test] + fn unmatched_coded_buffers_stop_accumulating() { + let mut pending = Pending::default(); + queue(&mut pending, micros(1)); + + let payload = Bytes::from(vec![0u8; 8 * 1024]); + for _ in 0..64 { + assert_eq!(pending.matched(Duration::from_micros(0), payload.clone()), None); + assert!(pending.header.as_ref().is_none_or(|held| held.len() <= CARRY_LIMIT)); + } + // The frame is still owed, so a flush does not report the codec drained. + assert_eq!(pending.len(), 1); + } + + /// A picture that answers no frame is dropped, not carried: it has no + /// timestamp to be published under, and in front of a later access unit it + /// would be a second picture in one frame. + #[test] + fn an_unmatched_picture_is_not_carried() { + let mut pending = Pending::default(); + queue(&mut pending, micros(500)); + + let stray = Bytes::from_static(&[0, 0, 0, 1, 0x65, 0xaa]); + assert_eq!(pending.matched(Duration::from_micros(999), stray), None); + assert!(pending.unstamped); + assert!(pending.header.is_none()); + assert_eq!(pending.len(), 1); + + let payload = Bytes::from_static(&[0, 0, 0, 1, 0x65]); + assert_eq!( + pending.matched(Duration::from_micros(500), payload.clone()), + Some((payload, micros(500))) + ); + } + + /// A frame the driver flagged bad is owed by nobody, and forgetting it must + /// not shift the frames queued around it: the raw buffers come back in their + /// own order, so a later frame can fail while an earlier one still has its + /// access unit on the way. + #[test] + fn a_dropped_frame_leaves_the_rest_matchable() { + let mut pending = Pending::default(); + for at in [10, 20, 30] { + queue(&mut pending, micros(at)); + } + + pending.dropped(Duration::from_micros(20)); + assert_eq!(pending.len(), 2); + // A timestamp that was never queued changes nothing. + pending.dropped(Duration::from_micros(999)); + assert_eq!(pending.len(), 2); + + let payload = Bytes::from_static(&[0, 0, 0, 1, 0x65]); + assert_eq!( + pending.matched(Duration::from_micros(10), payload.clone()), + Some((payload.clone(), micros(10))) + ); + assert_eq!( + pending.matched(Duration::from_micros(30), payload.clone()), + Some((payload, micros(30))) + ); + assert!(pending.is_empty()); + } + + /// The payload stops at the last non-zero byte, and a buffer the driver wrote + /// nothing into is not an access unit at all. + #[test] + fn padding_is_not_part_of_the_access_unit() { + let buffer = [0, 0, 0, 1, 0x65, 0x88, 0, 0, 0, 0]; + assert_eq!(&access_unit(&buffer, 10)[..], &[0, 0, 0, 1, 0x65, 0x88]); + assert!(access_unit(&[0; 8], 8).is_empty()); + // `bytesused` past the mapping is clamped rather than trusted. + assert_eq!(access_unit(&buffer, 64).len(), 6); + } +} diff --git a/rs/moq-video/src/encode/encoder.rs b/rs/moq-video/src/encode/encoder.rs index c9f2cd6957b..b58e031ba57 100644 --- a/rs/moq-video/src/encode/encoder.rs +++ b/rs/moq-video/src/encode/encoder.rs @@ -41,7 +41,7 @@ pub enum Kind { /// Software only (openh264 for H.264). Software, /// A specific backend by name, e.g. `"videotoolbox"`, `"nvenc"`, `"vaapi"`, - /// or `"openh264"`. + /// `"v4l2"`, or `"openh264"`. Named(String), } diff --git a/rs/moq-video/src/lib.rs b/rs/moq-video/src/lib.rs index 11672199122..bb714bfd835 100644 --- a/rs/moq-video/src/lib.rs +++ b/rs/moq-video/src/lib.rs @@ -23,8 +23,8 @@ //! - [`encode`] encodes frames with a native backend and publishes them through //! the matching `moq_mux::codec` importer, which handles catalog registration //! and framing. The codec is chosen via [`encode::Codec`]: H.264 (openh264 / -//! VideoToolbox / Media Foundation / NVENC / VAAPI) or H.265 (VideoToolbox / -//! Media Foundation / NVENC). Two entry points: +//! VideoToolbox / Media Foundation / NVENC / VAAPI / V4L2) or H.265 +//! (VideoToolbox / Media Foundation / NVENC). Two entry points: //! - `encode::publish_capture` captures a webcam and publishes it (turnkey). //! It encodes strictly on demand: the track and catalog are advertised up //! front (the camera opens once at startup so they can be exact), and the @@ -35,7 +35,8 @@ //! [`encode::Producer`] publishes the results. //! - [`decode`] subscribes to an H.264, H.265, or AV1 track and decodes it to //! raw frames with a native backend (VideoToolbox on macOS, Media Foundation / -//! DXVA on Windows, NVDEC on Linux, openh264 software fallback for H.264). +//! DXVA on Windows, NVDEC or an ARM SoC's V4L2 M2M decoder on Linux, openh264 +//! software fallback for H.264). //! [`decode::Consumer`] is the mirror of `moq_audio::decode::Consumer`. An //! NVDEC frame stays in CUDA memory and feeds [`encode::Encoder::encode`] //! zero-copy (the transcode path), scaled in hardware via @@ -51,8 +52,8 @@ //! ## API stability //! //! The public API is codec-agnostic: no public type, signature, or error -//! variant names a backend (openh264 / VideoToolbox / NVENC / NVDEC) or a codec -//! implementation. [`encode::Encoder`] takes a [`Frame`], +//! variant names a backend (openh264 / VideoToolbox / NVENC / NVDEC / V4L2) or a +//! codec implementation. [`encode::Encoder`] takes a [`Frame`], //! [`decode::Consumer`] returns one (CPU I420 on demand, GPU-resident when //! hardware decoded), and [`capture::Stream`] returns a [`Surface`]. So swapping //! or bumping any backend crate is not a breaking change for consumers. Config @@ -89,6 +90,9 @@ mod worker; #[cfg(target_os = "windows")] mod mf; +#[cfg(all(target_os = "linux", feature = "v4l2"))] +mod v4l2; + pub use color::Color; pub use error::Error; #[cfg(all(target_os = "linux", feature = "dmabuf"))] diff --git a/rs/moq-video/src/v4l2.rs b/rs/moq-video/src/v4l2.rs new file mode 100644 index 00000000000..dbf0bf3f29f --- /dev/null +++ b/rs/moq-video/src/v4l2.rs @@ -0,0 +1,1542 @@ +//! Shared V4L2 memory-to-memory device plumbing for the Linux hardware codec +//! backends. +//! +//! M2M is the kernel interface an ARM SoC exposes its video codec through: one +//! device node with two queues, OUTPUT for what userspace feeds in and CAPTURE +//! for what the hardware hands back. An encoder takes raw frames on OUTPUT and +//! returns an elementary stream on CAPTURE; a decoder runs the same node the +//! other way around. Both backends drive it through the [`Device`] and [`Queue`] +//! here, so the ioctl sequence, the buffer pool, and the plane arithmetic are +//! written once. +//! +//! Built on the raw layer of the `v4l` crate that the camera capture path +//! already uses: `v4l::v4l_sys` supplies the `videodev2.h` structs and +//! `v4l::v4l2::vidioc` the request codes, so nothing here hand-rolls a struct +//! offset. +//! +//! The driver, not the caller, decides the raw layout. `VIDIOC_S_FMT` is a +//! negotiation, and the [`Format`] that comes back can carry a different fourcc, +//! a wider stride, and more rows than were asked for. [`Planes`] reads that +//! answer and is the only thing either backend consults about where chroma +//! lives. + +use std::collections::VecDeque; +use std::fs::File; +use std::os::fd::AsRawFd; +use std::os::unix::fs::OpenOptionsExt; +use std::path::{Path, PathBuf}; +use std::ptr::NonNull; +use std::time::Duration; + +use v4l::v4l_sys::{ + V4L2_BUF_FLAG_ERROR, V4L2_BUF_FLAG_LAST, V4L2_CAP_DEVICE_CAPS, V4L2_CAP_STREAMING, V4L2_CAP_VIDEO_M2M, + V4L2_CAP_VIDEO_M2M_MPLANE, V4L2_EVENT_SOURCE_CHANGE, V4L2_SEL_TGT_COMPOSE, timeval, + v4l2_buf_type_V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE, v4l2_buf_type_V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, v4l2_buffer, + v4l2_capability, v4l2_colorspace_V4L2_COLORSPACE_REC709, v4l2_colorspace_V4L2_COLORSPACE_SMPTE170M, v4l2_control, + v4l2_encoder_cmd, v4l2_event, v4l2_event_subscription, v4l2_field_V4L2_FIELD_NONE, v4l2_fmtdesc, v4l2_format, + v4l2_memory_V4L2_MEMORY_MMAP, v4l2_plane, v4l2_quantization_V4L2_QUANTIZATION_FULL_RANGE, + v4l2_quantization_V4L2_QUANTIZATION_LIM_RANGE, v4l2_requestbuffers, v4l2_selection, v4l2_streamparm, + v4l2_xfer_func_V4L2_XFER_FUNC_709, v4l2_ycbcr_encoding_V4L2_YCBCR_ENC_601, v4l2_ycbcr_encoding_V4L2_YCBCR_ENC_709, +}; +use v4l::v4l2::vidioc; + +use crate::frame::I420; +use crate::{Color, Error, Size}; + +/// A V4L2 fourcc, which is its four ASCII bytes little-endian. +/// +/// `videodev2.h` spells these with a macro, so bindgen emits none of them. +const fn fourcc(code: [u8; 4]) -> u32 { + u32::from_le_bytes(code) +} + +/// Semi-planar 4:2:0, luma then interleaved chroma, in one buffer. +pub(crate) const NV12: u32 = fourcc(*b"NV12"); +/// The same layout with each plane in its own buffer. +pub(crate) const NV12M: u32 = fourcc(*b"NM12"); +/// Planar 4:2:0, luma then U then V, in one buffer. +pub(crate) const YUV420: u32 = fourcc(*b"YU12"); +/// The same layout with each plane in its own buffer. +pub(crate) const YUV420M: u32 = fourcc(*b"YM12"); +/// An H.264 elementary stream, one access unit per buffer. +pub(crate) const H264: u32 = fourcc(*b"H264"); + +/// The raw formats either backend can read and write, in preference order. +/// +/// All four are 8-bit 4:2:0, which is what `Surface::I420` converts to and from +/// without resampling. Which one a driver picks is its business: the Raspberry +/// Pi's `bcm2835-codec` answers an NV12 request with `YU12` on some firmware +/// revisions, and [`Planes`] handles whichever comes back. +pub(crate) const RAW: &[u32] = &[NV12, NV12M, YUV420, YUV420M]; + +/// Planes a format may use here: Y, U, and V. +const MAX_PLANES: usize = 3; + +/// The three requests the `v4l` crate does not export, built the way +/// `linux/ioctl.h` builds them. The shifts are `asm-generic`'s, which is what +/// `v4l`'s own table already assumes. +mod request { + use v4l::v4l_sys::{v4l2_event, v4l2_event_subscription, v4l2_selection}; + use v4l::v4l2::vidioc::_IOC_TYPE; + + const READ: u32 = 2; + const WRITE: u32 = 1; + + const fn code(dir: u32, nr: u32, size: usize) -> _IOC_TYPE { + ((dir as _IOC_TYPE) << 30) | ((size as _IOC_TYPE) << 16) | ((b'V' as _IOC_TYPE) << 8) | nr as _IOC_TYPE + } + + pub(super) const DQEVENT: _IOC_TYPE = code(READ, 89, size_of::()); + pub(super) const SUBSCRIBE_EVENT: _IOC_TYPE = code(WRITE, 90, size_of::()); + pub(super) const G_SELECTION: _IOC_TYPE = code(READ | WRITE, 94, size_of::()); + + #[cfg(test)] + mod tests { + use v4l::v4l_sys::{v4l2_capability, v4l2_format}; + use v4l::v4l2::vidioc; + + use super::*; + + /// The three codes above have to come out the way `linux/ioctl.h` builds + /// the rest, and nothing else here would notice if they did not: a wrong + /// code is `ENOTTY` at runtime on whichever board reaches it first. + /// + /// Checked against requests the `v4l` crate does export, one per direction, + /// so the shifts, the direction bits, and the struct size all have to agree + /// on whatever target this compiles for. + #[test] + fn the_codes_are_built_the_way_the_crate_builds_its_own() { + assert_eq!(code(READ, 0, size_of::()), vidioc::VIDIOC_QUERYCAP); + assert_eq!(code(WRITE, 18, size_of::()), vidioc::VIDIOC_STREAMON); + assert_eq!(code(READ | WRITE, 5, size_of::()), vidioc::VIDIOC_S_FMT); + } + } +} + +/// A `videodev2.h` struct an ioctl reads or fills. +/// +/// # Safety +/// +/// The type must be plain data with no niche, so that an all-zero value is a +/// valid one. Every struct in `videodev2.h` is, which is what lets each request +/// start from zero: the kernel wants the fields it reserves left that way. +unsafe trait Arg: Sized { + /// A zeroed value, the starting point for every request here. + fn zeroed() -> Self { + // SAFETY: the implementer promises all-zero is a valid value. + unsafe { std::mem::zeroed() } + } +} + +// SAFETY: each is a `videodev2.h` struct: integers, arrays, and unions of the +// same, with no reference, no enum, and no niche. +unsafe impl Arg for v4l2_buffer {} +unsafe impl Arg for v4l2_capability {} +unsafe impl Arg for v4l2_encoder_cmd {} +unsafe impl Arg for v4l2_event {} +unsafe impl Arg for v4l2_event_subscription {} +unsafe impl Arg for v4l2_fmtdesc {} +unsafe impl Arg for v4l2_format {} +unsafe impl Arg for v4l2_requestbuffers {} +unsafe impl Arg for v4l2_selection {} +unsafe impl Arg for v4l2_streamparm {} +unsafe impl Arg for [v4l2_plane; MAX_PLANES] {} + +/// Which queue of an M2M device, named the way V4L2 names them: from the +/// driver's point of view, not ours. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Dir { + /// What userspace feeds the device: raw frames to an encoder, coded bytes to + /// a decoder. + Output, + /// What the device hands back: coded bytes from an encoder, raw frames from a + /// decoder. + Capture, +} + +impl Dir { + fn buf_type(self) -> u32 { + match self { + Dir::Output => v4l2_buf_type_V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, + Dir::Capture => v4l2_buf_type_V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE, + } + } +} + +/// What to ask a queue for. The driver answers with a [`Format`] that may differ +/// in every field. +pub(crate) struct Request { + /// The fourcc to ask for. + pub pixelformat: u32, + /// The picture size to ask for. + pub size: Size, + /// Buffer size to ask for, which a compressed queue needs because the driver + /// cannot size an access unit from the picture dimensions. `None` leaves the + /// driver's own estimate, which is what a raw queue wants. + pub sizeimage: Option, + /// The color space of the samples. An encoder driver that fills the VUI reads + /// it from here, so the bitstream says what the pixels actually are. + pub color: Option, +} + +/// One plane of a negotiated [`Format`]. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Plane { + /// Bytes between the starts of adjacent rows, at least the row's width and + /// usually more. + pub stride: u32, + /// Bytes the driver wants this plane's buffer to be, alignment padding + /// included. + pub sizeimage: u32, +} + +/// The format a queue negotiated, read back from the driver's `VIDIOC_S_FMT` +/// answer rather than assumed from the request. +#[derive(Clone, Debug)] +pub(crate) struct Format { + /// The fourcc the driver chose, which need not be the one asked for. + pub pixelformat: u32, + /// The coded size, at least the requested one and rounded up to whatever the + /// hardware aligns to. + pub size: Size, + /// One entry per plane, in plane order. + pub planes: Vec, +} + +/// How to recognize the M2M node for one codec, since node numbering is per SoC. +pub(crate) struct Role { + /// Environment variable naming a node directly, for a driver whose format + /// enumeration doesn't match what it can actually do. + pub env: &'static str, + /// Fourccs the OUTPUT queue has to accept, any one of them. + pub input: &'static [u32], + /// Fourccs the CAPTURE queue has to produce, any one of them. + pub output: &'static [u32], +} + +/// Open the M2M node for `role`, searching every V4L2 node for one that converts +/// what the role names. +/// +/// The path is per SoC (`/dev/video11` encodes on a Raspberry Pi, and other SoCs +/// number theirs differently), so the node is identified by what it converts +/// rather than by a table of guesses. `role.env` overrides the search. +pub(crate) fn open(role: &Role) -> Result { + if let Ok(path) = std::env::var(role.env) { + let device = Device::open(Path::new(&path))?; + device.check(role)?; + return Ok(device); + } + + let mut nodes = v4l::context::enum_devices(); + nodes.sort_by_key(v4l::context::Node::index); + + let mut refused = Vec::new(); + for node in nodes { + let path = node.path(); + match Device::open(path).and_then(|device| device.check(role).map(|()| device)) { + Ok(device) => return Ok(device), + // Most nodes on a box are cameras or metadata nodes, so a miss is the + // common case and only interesting when the search finds nothing. + Err(err) => refused.push(format!("{}: {err}", path.display())), + } + } + + Err(Error::Codec(anyhow::anyhow!( + "no V4L2 M2M node converts {} to {} (set {} to name one; tried {})", + join_fourcc(role.input), + join_fourcc(role.output), + role.env, + match refused.is_empty() { + true => "no nodes".to_owned(), + false => refused.join(", "), + } + ))) +} + +/// Render fourccs as their ASCII spelling for an error message. +fn join_fourcc(codes: &[u32]) -> String { + codes.iter().copied().map(name).collect::>().join("/") +} + +/// The ASCII spelling of a fourcc, for logs and errors. +pub(crate) fn name(code: u32) -> String { + String::from_utf8_lossy(&code.to_le_bytes()).into_owned() +} + +/// An open V4L2 M2M device node. +pub(crate) struct Device { + file: File, + path: PathBuf, +} + +impl Device { + /// Open `path`, failing unless it reports a multi-planar M2M capability. + fn open(path: &Path) -> Result { + // Non-blocking, so a `VIDIOC_DQBUF` with nothing ready returns `EAGAIN` + // instead of parking the caller's thread. Waiting is explicit, through + // `wait`, and always bounded. + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .custom_flags(libc::O_NONBLOCK) + .open(path) + .map_err(|err| Error::Codec(anyhow::anyhow!("{}: open: {err}", path.display())))?; + + let device = Self { + file, + path: path.to_path_buf(), + }; + + let caps = device.capabilities()?; + if caps & V4L2_CAP_VIDEO_M2M_MPLANE == 0 { + // Single-planar M2M exists, but no codec driver ships only that, and + // supporting it would double every queue path below for no hardware. + let what = match caps & V4L2_CAP_VIDEO_M2M { + 0 => "not an M2M device", + _ => "single-planar M2M only", + }; + return Err(Error::Codec(anyhow::anyhow!("{}: {what}", path.display()))); + } + // Everything below streams MMAP buffers, so a node without streaming I/O is + // the wrong node however its formats enumerate. + if caps & V4L2_CAP_STREAMING == 0 { + return Err(Error::Codec(anyhow::anyhow!("{}: no streaming I/O", path.display()))); + } + Ok(device) + } + + /// Whether this node converts what `role` names. + fn check(&self, role: &Role) -> Result<(), Error> { + for (dir, wanted) in [(Dir::Output, role.input), (Dir::Capture, role.output)] { + let offered = self.formats(dir)?; + if !wanted.iter().any(|code| offered.contains(code)) { + return Err(Error::Codec(anyhow::anyhow!( + "offers {} where {} is needed", + join_fourcc(&offered), + join_fourcc(wanted) + ))); + } + } + Ok(()) + } + + /// The node path, for logs. + pub(crate) fn path(&self) -> &Path { + &self.path + } + + /// Issue `request` against this node. + /// + /// # Safety + /// + /// `arg` must be the struct type `request` was defined for. + unsafe fn ioctl(&self, request: vidioc::_IOC_TYPE, arg: &mut T) -> std::io::Result<()> { + // SAFETY: the caller guarantees `arg` matches `request`, and the pointer + // stays valid for the call, which is the only thing the kernel needs. + unsafe { v4l::v4l2::ioctl(self.file.as_raw_fd(), request, (arg as *mut T).cast()) } + } + + /// Wrap an ioctl failure with the node and the operation that failed. + fn err(&self, what: impl std::fmt::Display, err: std::io::Error) -> Error { + Error::Codec(anyhow::anyhow!("{}: {what}: {err}", self.path.display())) + } + + /// The capabilities of this node specifically, rather than of the driver as a + /// whole. A codec driver registers several nodes off one `v4l2_capability`, + /// so only `device_caps` says what *this* one does. + fn capabilities(&self) -> Result { + let mut caps = v4l2_capability::zeroed(); + // SAFETY: `VIDIOC_QUERYCAP` takes a `v4l2_capability`. + unsafe { self.ioctl(vidioc::VIDIOC_QUERYCAP, &mut caps) }.map_err(|err| self.err("QUERYCAP", err))?; + + Ok(match caps.capabilities & V4L2_CAP_DEVICE_CAPS { + 0 => caps.capabilities, + _ => caps.device_caps, + }) + } + + /// Every fourcc a queue offers, from `VIDIOC_ENUM_FMT`. + /// + /// A decoder's CAPTURE answer narrows once the stream has been parsed: what + /// comes back then is what the driver supports for that stream, which is a + /// subset of what it offered at open. + pub(crate) fn formats(&self, dir: Dir) -> Result, Error> { + let mut formats = Vec::new(); + for index in 0.. { + // The enumeration ends with `EINVAL`, which is not an error here. Any + // other failure is, but reporting it as an empty list would only turn a + // broken node into a confusing "offers nothing", so stop either way. + let mut desc = v4l2_fmtdesc::zeroed(); + desc.index = index; + desc.type_ = dir.buf_type(); + // SAFETY: `VIDIOC_ENUM_FMT` takes a `v4l2_fmtdesc`. + if unsafe { self.ioctl(vidioc::VIDIOC_ENUM_FMT, &mut desc) }.is_err() { + break; + } + formats.push(desc.pixelformat); + } + Ok(formats) + } + + /// Negotiate a queue's format, returning what the driver settled on. + /// + /// `VIDIOC_G_FMT` runs first because the struct carries fields this does not + /// set, colorimetry defaults among them, and `bcm2835-codec` rejects an + /// `S_FMT` built from zeroes. + pub(crate) fn set_format(&self, dir: Dir, request: &Request) -> Result { + let mut format = v4l2_format::zeroed(); + format.type_ = dir.buf_type(); + // SAFETY: `VIDIOC_G_FMT` takes a `v4l2_format`. + unsafe { self.ioctl(vidioc::VIDIOC_G_FMT, &mut format) }.map_err(|err| self.err("G_FMT", err))?; + + { + // SAFETY: the queue type is one of the `_MPLANE` pair, so the union + // holds `pix_mp`. + let pix = unsafe { &mut format.fmt.pix_mp }; + pix.width = request.size.width; + pix.height = request.size.height; + pix.pixelformat = request.pixelformat; + pix.field = v4l2_field_V4L2_FIELD_NONE; + // The driver raises this for a format whose planes live in separate + // buffers; asking for one plane is right for every contiguous format and + // harmless for the rest. + pix.num_planes = 1; + // Zero is how V4L2 asks the driver to size a plane itself. Left as + // `G_FMT` reported them, these would be the stride and buffer size of + // whatever format the queue held before this call, which is the wrong + // answer for any request that changes the dimensions. + for plane in &mut pix.plane_fmt { + plane.bytesperline = 0; + plane.sizeimage = 0; + } + if let Some(sizeimage) = request.sizeimage { + pix.plane_fmt[0].sizeimage = sizeimage; + } + if let Some(color) = request.color { + let (colorspace, ycbcr) = match color { + // SMPTE 170M primaries and matrix with the BT.709 transfer curve, + // the same triple the other backends emit: the two curves are + // defined identically and only 709 has a name everywhere. + Color::Bt601Limited | Color::Bt601Full => ( + v4l2_colorspace_V4L2_COLORSPACE_SMPTE170M, + v4l2_ycbcr_encoding_V4L2_YCBCR_ENC_601, + ), + Color::Bt709Limited | Color::Bt709Full => ( + v4l2_colorspace_V4L2_COLORSPACE_REC709, + v4l2_ycbcr_encoding_V4L2_YCBCR_ENC_709, + ), + }; + pix.colorspace = colorspace; + pix.__bindgen_anon_1.ycbcr_enc = ycbcr as u8; + pix.xfer_func = v4l2_xfer_func_V4L2_XFER_FUNC_709 as u8; + pix.quantization = match color.limited() { + true => v4l2_quantization_V4L2_QUANTIZATION_LIM_RANGE as u8, + false => v4l2_quantization_V4L2_QUANTIZATION_FULL_RANGE as u8, + }; + } + } + + // SAFETY: `VIDIOC_S_FMT` takes a `v4l2_format`. + unsafe { self.ioctl(vidioc::VIDIOC_S_FMT, &mut format) }.map_err(|err| self.err("S_FMT", err))?; + self.read_format(&format) + } + + fn read_format(&self, format: &v4l2_format) -> Result { + // SAFETY: only ever called on a struct whose type is one of the `_MPLANE` + // pair, so the union holds `pix_mp`. + let pix = unsafe { &format.fmt.pix_mp }; + let count = (pix.num_planes as usize).clamp(1, MAX_PLANES); + Ok(Format { + pixelformat: pix.pixelformat, + size: Size::new(pix.width, pix.height), + planes: pix.plane_fmt[..count] + .iter() + .map(|plane| Plane { + stride: plane.bytesperline, + sizeimage: plane.sizeimage, + }) + .collect(), + }) + } + + /// Read a queue's current format without changing it, which is how a decoder + /// learns the picture size the stream turned out to have. + pub(crate) fn format(&self, dir: Dir) -> Result { + let mut format = v4l2_format::zeroed(); + format.type_ = dir.buf_type(); + // SAFETY: `VIDIOC_G_FMT` takes a `v4l2_format`. + unsafe { self.ioctl(vidioc::VIDIOC_G_FMT, &mut format) }.map_err(|err| self.err("G_FMT", err))?; + self.read_format(&format) + } + + /// The visible rectangle of a decoded picture, or `None` from a driver that + /// does not report one. + /// + /// Coding happens in macroblocks, so a stream's coded size is rounded up and + /// the picture people are meant to see is the compose rectangle inside it: + /// 1080p codes as 1088 rows, of which the last 8 are not part of the picture. + /// The rectangle has an origin as well as a size, since H.264 can crop any + /// edge, so the offset is kept rather than assumed to be the corner. + pub(crate) fn visible(&self, dir: Dir) -> Option { + let mut selection = v4l2_selection::zeroed(); + selection.type_ = dir.buf_type(); + selection.target = V4L2_SEL_TGT_COMPOSE; + // SAFETY: `VIDIOC_G_SELECTION` takes a `v4l2_selection`. + unsafe { self.ioctl(request::G_SELECTION, &mut selection) }.ok()?; + + let rect = Rect { + left: selection.r.left.max(0) as u32, + top: selection.r.top.max(0) as u32, + size: Size::new(selection.r.width, selection.r.height), + }; + match rect.size.width == 0 || rect.size.height == 0 { + true => None, + false => Some(rect), + } + } + + /// Ask the driver to report resolution changes, which is the only way a + /// stateful decoder announces the picture size. + pub(crate) fn subscribe_source_change(&self) -> Result<(), Error> { + let mut subscription = v4l2_event_subscription::zeroed(); + subscription.type_ = V4L2_EVENT_SOURCE_CHANGE; + // SAFETY: `VIDIOC_SUBSCRIBE_EVENT` takes a `v4l2_event_subscription`. + unsafe { self.ioctl(request::SUBSCRIBE_EVENT, &mut subscription) } + .map_err(|err| self.err("SUBSCRIBE_EVENT", err)) + } + + /// Drain the event queue, reporting whether a source change was among them. + /// + /// Every pending event is taken rather than one, since a caller that read + /// them one per call would fall behind a driver that queues several. + pub(crate) fn take_source_change(&self) -> bool { + let mut changed = false; + loop { + let mut event = v4l2_event::zeroed(); + // SAFETY: `VIDIOC_DQEVENT` takes a `v4l2_event`. + if unsafe { self.ioctl(request::DQEVENT, &mut event) }.is_err() { + return changed; + } + changed |= event.type_ == V4L2_EVENT_SOURCE_CHANGE; + } + } + + /// Read a control the driver computes rather than one we set. + pub(crate) fn control(&self, id: u32) -> Result { + let mut control = v4l2_control { id, value: 0 }; + // SAFETY: `VIDIOC_G_CTRL` takes a `v4l2_control`. + unsafe { self.ioctl(vidioc::VIDIOC_G_CTRL, &mut control) } + .map_err(|err| self.err(format_args!("G_CTRL {id:#x}"), err))?; + Ok(control.value) + } + + /// Declare the input framerate, which rate control uses to spend the bitrate. + pub(crate) fn set_framerate(&self, dir: Dir, framerate: u32) -> Result<(), Error> { + let mut parm = v4l2_streamparm::zeroed(); + parm.type_ = dir.buf_type(); + // SAFETY: the buffer type just written picks the arm the driver reads, so + // the two have to agree: a CAPTURE queue's parameters are `capture` and an + // OUTPUT queue's are `output`. + let time_per_frame = unsafe { + match dir { + Dir::Output => &mut parm.parm.output.timeperframe, + Dir::Capture => &mut parm.parm.capture.timeperframe, + } + }; + time_per_frame.numerator = 1; + time_per_frame.denominator = framerate; + + // SAFETY: `VIDIOC_S_PARM` takes a `v4l2_streamparm`. + unsafe { self.ioctl(vidioc::VIDIOC_S_PARM, &mut parm) }.map_err(|err| self.err("S_PARM", err)) + } + + /// Issue a `VIDIOC_ENCODER_CMD`, which is how an encoder is told to drain and + /// then to resume. + /// + /// # Errors + /// + /// `EINVAL` or `ENOTTY` from a driver that implements no encoder commands, + /// which the caller has to be able to carry on without. + pub(crate) fn encoder_cmd(&self, cmd: u32) -> Result<(), Error> { + // The drain sequence wants `flags` and `pts` zero, which is where every + // request starts. + let mut command = v4l2_encoder_cmd::zeroed(); + command.cmd = cmd; + // SAFETY: `VIDIOC_ENCODER_CMD` takes a `v4l2_encoder_cmd`. + unsafe { self.ioctl(vidioc::VIDIOC_ENCODER_CMD, &mut command) } + .map_err(|err| self.err(format_args!("ENCODER_CMD {cmd}"), err)) + } + + /// Set a codec control, failing if the driver rejects it. + pub(crate) fn set_control(&self, id: u32, value: i32) -> Result<(), Error> { + let mut control = v4l2_control { id, value }; + // SAFETY: `VIDIOC_S_CTRL` takes a `v4l2_control`. + unsafe { self.ioctl(vidioc::VIDIOC_S_CTRL, &mut control) } + .map_err(|err| self.err(format_args!("S_CTRL {id:#x} = {value}"), err)) + } + + /// Set a control the driver is allowed not to have. + /// + /// Optional by design rather than by accident: the two ways to ask for + /// repeated parameter sets are each supported by only some drivers, so both + /// are offered and whichever lands wins. + pub(crate) fn try_control(&self, id: u32, value: i32) -> bool { + match self.set_control(id, value) { + Ok(()) => true, + Err(err) => { + tracing::debug!(control = format!("{id:#x}"), value, %err, "V4L2 control not supported"); + false + } + } + } + + /// Allocate `count` mmap buffers for a queue, returning how many the driver + /// actually gave. + fn request_buffers(&self, dir: Dir, count: u32) -> Result { + let mut request = v4l2_requestbuffers::zeroed(); + request.count = count; + request.type_ = dir.buf_type(); + request.memory = v4l2_memory_V4L2_MEMORY_MMAP; + + // SAFETY: `VIDIOC_REQBUFS` takes a `v4l2_requestbuffers`. + unsafe { self.ioctl(vidioc::VIDIOC_REQBUFS, &mut request) } + .map_err(|err| self.err(format_args!("REQBUFS {count}"), err))?; + Ok(request.count) + } + + fn stream(&self, dir: Dir, on: bool) -> Result<(), Error> { + let mut buf_type = dir.buf_type(); + let request = match on { + true => vidioc::VIDIOC_STREAMON, + false => vidioc::VIDIOC_STREAMOFF, + }; + // SAFETY: both take an `int` holding the buffer type. + unsafe { self.ioctl(request, &mut buf_type) }.map_err(|err| { + let what = match on { + true => "STREAMON", + false => "STREAMOFF", + }; + self.err(what, err) + }) + } + + /// Park until the device has something to hand over, or `timeout` elapses. + /// + /// Blocking, which is what the caller is after: every codec backend here runs + /// on its own OS thread, so nothing on an executor is parked. + /// + /// Buffers on either queue, or an event on a decoder that subscribed to one. + /// A caller that finds nothing ready afterwards just goes round again, so a + /// spurious wakeup costs a loop rather than a wrong answer. + /// + /// `revents` has to be read, not just the return value. `v4l2_m2m_poll` + /// answers `POLLERR` whenever neither queue has a buffer queued, which is an + /// ordinary transient here rather than a failure, and it is level triggered: + /// a caller that took the immediate return for readiness would spin at full + /// speed until its own deadline instead of parking. So the wait is finished + /// by sleeping, which is what the caller asked for either way. + pub(crate) fn wait(&self, timeout: Duration) { + let mut event = libc::pollfd { + fd: self.file.as_raw_fd(), + events: libc::POLLIN | libc::POLLOUT | libc::POLLPRI, + revents: 0, + }; + // SAFETY: one initialized `pollfd` for the length passed. + let ready = unsafe { libc::poll(&mut event, 1, timeout.as_millis().min(i32::MAX as u128) as libc::c_int) }; + // A negative return is `EINTR` or `ENOMEM` and leaves `revents` untouched, + // so there is nothing to read and the caller's own deadline still bounds it. + if ready > 0 && event.revents & (libc::POLLIN | libc::POLLOUT | libc::POLLPRI) == 0 { + std::thread::sleep(timeout); + } + } +} + +/// One mmap'd plane of one buffer. +struct Mapping { + ptr: NonNull, + len: usize, +} + +// SAFETY: a mapping is only reached through `&Queue` / `&mut Queue`, and a queue +// travels with the backend that owns it, which the codec threads move as one +// value. Nothing here is shared between threads. +unsafe impl Send for Mapping {} + +impl Drop for Mapping { + fn drop(&mut self) { + // SAFETY: the pointer and length are what `mmap` returned and this is the + // only owner, so nothing can still be reading the region. + let _ = unsafe { v4l::v4l2::munmap(self.ptr.as_ptr().cast(), self.len) }; + } +} + +/// A queue's pool of mmap'd buffers, plus which of them are ours to fill. +pub(crate) struct Queue { + dir: Dir, + format: Format, + buffers: Vec>, + /// Buffers the driver is done with, so userspace may write them again. + free: VecDeque, + streaming: bool, +} + +impl Queue { + /// Allocate and map `count` buffers for `format` on `dir`. + /// + /// Every buffer is zeroed once here rather than before each frame. Only the + /// visible region is written afterwards, so the alignment padding a driver + /// reads past the picture keeps the zeroes it started with, and a Raspberry + /// Pi is not spending 3 MB of writes a frame to put them back. + pub(crate) fn alloc(device: &Device, dir: Dir, format: Format, count: u32) -> Result { + let count = device.request_buffers(dir, count)?; + if count == 0 { + return Err(device.err("REQBUFS", std::io::Error::from(std::io::ErrorKind::OutOfMemory))); + } + + let mut buffers = Vec::with_capacity(count as usize); + for index in 0..count { + let mut planes = zeroed_planes(); + let mut buffer = new_buffer(dir, format.planes.len()); + buffer.index = index; + buffer.m.planes = planes.as_mut_ptr(); + + // SAFETY: `VIDIOC_QUERYBUF` takes a `v4l2_buffer`, whose plane array is + // the one above, valid for the call and long enough for `buffer.length`. + unsafe { device.ioctl(vidioc::VIDIOC_QUERYBUF, &mut buffer) } + .map_err(|err| device.err(format_args!("QUERYBUF {index}"), err))?; + + let mut mappings = Vec::with_capacity(format.planes.len()); + for plane in &planes[..format.planes.len()] { + let len = plane.length as usize; + // SAFETY: the offset is the cookie `QUERYBUF` just handed back for + // this plane, on an mmap queue, so `m` holds `mem_offset`. The length + // is the one it reported alongside. + let ptr = unsafe { + v4l::v4l2::mmap( + std::ptr::null_mut(), + len, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED, + device.file.as_raw_fd(), + plane.m.mem_offset as libc::off_t, + ) + } + .map_err(|err| device.err(format_args!("mmap buffer {index}"), err))?; + + let ptr = NonNull::new(ptr.cast::()) + .ok_or_else(|| device.err("mmap", std::io::Error::from(std::io::ErrorKind::InvalidData)))?; + // SAFETY: `mmap` just returned this region for `len` bytes and + // nothing else has a pointer into it yet. + unsafe { std::ptr::write_bytes(ptr.as_ptr(), 0, len) }; + mappings.push(Mapping { ptr, len }); + } + buffers.push(mappings); + } + + Ok(Self { + dir, + format, + buffers, + free: (0..count).collect(), + streaming: false, + }) + } + + /// The format this queue was allocated for. + pub(crate) fn format(&self) -> &Format { + &self.format + } + + /// Take a buffer userspace may write, or `None` while the driver holds them + /// all. + pub(crate) fn take_free(&mut self) -> Option { + self.free.pop_front() + } + + /// Hand a dequeued buffer back to the free list. + pub(crate) fn reclaim(&mut self, index: u32) { + self.free.push_back(index); + } + + /// One plane of one buffer, for the whole mapped length. + pub(crate) fn plane(&self, index: u32, plane: usize) -> &[u8] { + let mapping = &self.buffers[index as usize][plane]; + // SAFETY: the mapping is live for as long as this queue, and `&self` + // excludes a concurrent write through `plane_mut`. + unsafe { std::slice::from_raw_parts(mapping.ptr.as_ptr(), mapping.len) } + } + + /// One plane of a dequeued buffer from where its payload starts, which a + /// driver that puts a header on the plane reports through `data_offset`. + pub(crate) fn payload(&self, buffer: &Dequeued, plane: usize) -> &[u8] { + let mapping = self.plane(buffer.index, plane); + &mapping[(buffer.data_offset[plane] as usize).min(mapping.len())..] + } + + /// One plane of one buffer, writable. + pub(crate) fn plane_mut(&mut self, index: u32, plane: usize) -> &mut [u8] { + let mapping = &mut self.buffers[index as usize][plane]; + // SAFETY: as `plane`, and `&mut self` excludes any other view of it. The + // driver does not touch a buffer that is not queued, and a buffer reaches + // here only off the free list. + unsafe { std::slice::from_raw_parts_mut(mapping.ptr.as_ptr(), mapping.len) } + } + + /// Hand a buffer to the driver, `bytesused` bytes per plane, stamped with + /// `timestamp`. + /// + /// An M2M driver copies the OUTPUT timestamp onto the CAPTURE buffer its work + /// comes out on, so this is how a frame's presentation time survives a codec + /// that pipelines. + pub(crate) fn queue( + &self, + device: &Device, + index: u32, + bytesused: &[u32], + timestamp: Duration, + ) -> Result<(), Error> { + let mut planes = zeroed_planes(); + // Driven by the buffer's own plane count, not by `bytesused`: every plane + // the driver expects has to carry its mapped length, and a caller passing + // one payload size for a several-plane picture (a decoder queueing an empty + // buffer back) must not leave the rest reading as zero-length. + for (index, mapping) in self.buffers[index as usize].iter().enumerate() { + planes[index].bytesused = bytesused.get(index).copied().unwrap_or(0); + planes[index].length = mapping.len as u32; + } + + let mut buffer = new_buffer(self.dir, self.buffers[index as usize].len()); + buffer.index = index; + buffer.m.planes = planes.as_mut_ptr(); + buffer.timestamp.tv_sec = timestamp.as_secs() as _; + buffer.timestamp.tv_usec = timestamp.subsec_micros() as _; + + // SAFETY: `VIDIOC_QBUF` takes a `v4l2_buffer`, whose plane array is the one + // above, valid for the call and long enough for `buffer.length`. + unsafe { device.ioctl(vidioc::VIDIOC_QBUF, &mut buffer) } + .map_err(|err| device.err(format_args!("QBUF {index}"), err)) + } + + /// Take back a buffer the driver has finished with. + /// + /// The three answers are distinct because a drain needs them to be: "nothing + /// ready yet" is a reason to wait and "the sequence has ended" is a reason to + /// stop, and folding both into `None` is why a resolution change used to + /// discard the pictures decoded before it. + pub(crate) fn dequeue(&self, device: &Device) -> Result { + // A queue that has not been started holds nothing, and asking anyway is an + // error rather than the `EAGAIN` an empty one answers: `vb2` checks + // `q->streaming` before it looks for a buffer. A caller that collects + // finished buffers before it queues its first one would fail on that. + if !self.streaming { + return Ok(Dequeue::Empty); + } + + let mut planes = zeroed_planes(); + let mut buffer = new_buffer(self.dir, self.format.planes.len()); + buffer.m.planes = planes.as_mut_ptr(); + + // SAFETY: as `queue`, with `VIDIOC_DQBUF`. + if let Err(err) = unsafe { device.ioctl(vidioc::VIDIOC_DQBUF, &mut buffer) } { + return match err.kind() { + // Nothing ready, which on a non-blocking fd is the answer rather than + // a failure. + std::io::ErrorKind::WouldBlock => Ok(Dequeue::Empty), + // `EPIPE`, which is what the kernel answers a dequeue past the buffer + // flagged `V4L2_BUF_FLAG_LAST` with. Not an error: it says the drain + // this caller is driving has already finished. + std::io::ErrorKind::BrokenPipe => Ok(Dequeue::Ended), + _ => Err(device.err("DQBUF", err)), + }; + } + + let mut bytesused = [0; MAX_PLANES]; + let mut data_offset = [0; MAX_PLANES]; + for (index, plane) in planes.iter().enumerate() { + bytesused[index] = plane.bytesused; + data_offset[index] = plane.data_offset; + } + + Ok(Dequeue::Buffer(Dequeued { + index: buffer.index, + bytesused, + data_offset, + timestamp: timestamp(buffer.timestamp), + flags: buffer.flags, + })) + } + + /// Start the queue, which the driver only accepts once its buffers are + /// allocated. + pub(crate) fn stream_on(&mut self, device: &Device) -> Result<(), Error> { + device.stream(self.dir, true)?; + self.streaming = true; + Ok(()) + } + + /// Whether the queue has been started. + pub(crate) fn streaming(&self) -> bool { + self.streaming + } + + /// Stop the queue and hand its buffers back to the driver, which is how a + /// decoder re-negotiates CAPTURE when the stream changes size. + /// + /// Consumes the queue: the buffers it addressed no longer exist afterwards, + /// so there is nothing left that could be queued by mistake. + pub(crate) fn release(mut self, device: &Device) -> Result<(), Error> { + if self.streaming { + device.stream(self.dir, false)?; + self.streaming = false; + } + // Unmapped before the driver is asked to free them: `REQBUFS(0)` returns + // EBUSY while userspace still holds a mapping. + self.buffers.clear(); + self.free.clear(); + device.request_buffers(self.dir, 0).map(|_| ()) + } +} + +/// What a `VIDIOC_DQBUF` found. +#[derive(Clone, Copy, Debug)] +pub(crate) enum Dequeue { + /// A buffer the driver has finished with. + Buffer(Dequeued), + /// Nothing is ready yet, so a caller with time left should wait. + Empty, + /// The sequence has already ended, so there is nothing left to wait for. + Ended, +} + +impl Dequeue { + /// The buffer, if there was one. + /// + /// Folds `Ended` back into "nothing more", which is what a caller that is not + /// driving a drain wants: it collects what is ready and comes back later. + pub(crate) fn buffer(self) -> Option { + match self { + Dequeue::Buffer(buffer) => Some(buffer), + Dequeue::Empty | Dequeue::Ended => None, + } + } +} + +/// A buffer the driver has handed back. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Dequeued { + /// Which buffer of the pool it is. + pub index: u32, + /// Bytes the driver wrote, per plane, counted from the start of the plane + /// and so including any header ahead of the payload. Zero on an OUTPUT + /// buffer, which the driver only read. + pub bytesused: [u32; MAX_PLANES], + /// Where the payload starts in each plane. Zero from every codec driver + /// seen so far, but the UAPI lets a driver put a header ahead of the data + /// and say so here. + data_offset: [u32; MAX_PLANES], + /// The timestamp the matching OUTPUT buffer carried, copied through by the + /// driver. + pub timestamp: Duration, + /// The `V4L2_BUF_FLAG_*` bits the driver set, read through [`Dequeued::failed`] + /// and [`Dequeued::last`]. + flags: u32, +} + +impl Dequeued { + /// Bytes of payload in `plane`, past whatever header the driver put ahead + /// of it. + pub(crate) fn written(&self, plane: usize) -> u32 { + self.bytesused[plane].saturating_sub(self.data_offset[plane]) + } + + /// Whether the driver marked the buffer's contents unusable. + /// + /// `VIDIOC_DQBUF` succeeds for a buffer flagged `V4L2_BUF_FLAG_ERROR`, so the + /// flag is the only thing that separates a good buffer from a bad one: an + /// encoder that overran its coded buffer returns a truncated access unit this + /// way, and a decoder that failed a picture returns garbage. + pub(crate) fn failed(&self) -> bool { + self.flags & V4L2_BUF_FLAG_ERROR != 0 + } + + /// Whether this is the last buffer of a sequence. + /// + /// The kernel ends every drain with it, whether the drain was asked for with + /// an encoder or decoder command or implied by a resolution change. The buffer + /// may still be empty, in which case it marks the end and is not a frame. + pub(crate) fn last(&self) -> bool { + self.flags & V4L2_BUF_FLAG_LAST != 0 + } +} + +/// The timestamp a dequeued buffer carries. +/// +/// Both fields of a `struct timeval` are signed and 64 bits wide, and neither +/// is worth trusting into anything narrower: a driver that leaves a buffer's +/// timestamp alone hands back whatever the struct held, and +/// `tv_usec as u32 * 1_000` overflows for any `tv_usec` above 4_294_967, which +/// panics in a debug build. Negative values are not a time either, so they +/// clamp to zero rather than wrapping into the far future. +fn timestamp(time: timeval) -> Duration { + Duration::from_secs(time.tv_sec.max(0) as u64) + Duration::from_micros(time.tv_usec.max(0) as u64) +} + +/// A zeroed plane array for a `v4l2_buffer` to point at. +fn zeroed_planes() -> [v4l2_plane; MAX_PLANES] { + <[v4l2_plane; MAX_PLANES]>::zeroed() +} + +/// A zeroed `v4l2_buffer` addressing `planes` planes of an mmap queue. +fn new_buffer(dir: Dir, planes: usize) -> v4l2_buffer { + let mut buffer = v4l2_buffer::zeroed(); + buffer.type_ = dir.buf_type(); + buffer.memory = v4l2_memory_V4L2_MEMORY_MMAP; + // On a multi-planar queue this field is the length of the plane array, not a + // byte count. + buffer.length = planes as u32; + buffer +} + +/// A rectangle inside a coded picture: the part of it that is the picture. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Rect { + /// Pixels from the left edge of the coded picture. + pub left: u32, + /// Rows from the top of the coded picture. + pub top: u32, + pub size: Size, +} + +impl Rect { + /// The whole of a `size` picture, from its top-left corner. + pub(crate) fn whole(size: Size) -> Self { + Self { left: 0, top: 0, size } + } +} + +/// Where a raw 4:2:0 frame's samples sit inside a queue's buffers. +/// +/// The whole point of the type: a driver answers `VIDIOC_S_FMT` with its own +/// fourcc, its own stride, and its own buffer size, and none of the three has to +/// be what was asked for. `bcm2835-codec` aligns stride to 32 bytes and height +/// to 16 rows, and reports the row padding through `bytesperline` but the row +/// *count* only through `sizeimage`. Chroma therefore starts at +/// `stride * padded_rows`, not at `stride * height`: place it at the visible +/// height and the driver reads zeroes for chroma, which comes out as a green +/// picture. +pub(crate) struct Planes { + y: Component, + /// Interleaved chroma for a semi-planar format, or the U plane of a planar + /// one. + u: Component, + /// The V plane, absent when chroma is interleaved. + v: Option, + /// The visible size, which is what gets copied. It fits inside the format's + /// coded size, at the origin the components' offsets already account for. + size: Size, +} + +/// One component's position within the queue's buffers. +#[derive(Clone, Copy)] +struct Component { + plane: usize, + offset: usize, + stride: usize, +} + +impl Planes { + /// Work out where the `rect` part of a coded picture sits in `format`'s + /// buffers. + pub(crate) fn new(format: &Format, rect: Rect) -> Result { + let size = rect.size; + size.validate("V4L2 4:2:0 frame")?; + if rect.left + size.width > format.size.width || rect.top + size.height > format.size.height { + return Err(Error::Codec(anyhow::anyhow!( + "V4L2 negotiated {} for a {size} picture at {},{}", + format.size, + rect.left, + rect.top + ))); + } + // Chroma is subsampled by two in each direction, and H.264 crops in units + // of two for exactly that reason, so halving loses nothing. + let (left, top) = (rect.left as usize, rect.top as usize); + let (chroma_left, chroma_top) = (left / 2, top / 2); + + let interleaved = match format.pixelformat { + NV12 | NV12M => true, + YUV420 | YUV420M => false, + other => { + return Err(Error::Codec(anyhow::anyhow!( + "V4L2 chose the unsupported raw format {}", + name(other) + ))); + } + }; + + let luma = *format + .planes + .first() + .ok_or_else(|| Error::Codec(anyhow::anyhow!("V4L2 reported a format with no planes")))?; + let stride = luma.stride.max(format.size.width) as usize; + let y = Component { + plane: 0, + offset: top * stride + left, + stride, + }; + + // Keyed on the plane count rather than the fourcc: the contiguous and + // per-plane spellings of a layout differ only in where the planes land, and + // a driver is free to answer either request with the other. + let separate = format.planes.len() > 1; + let rows = padded_rows(luma, format.size.height); + + let (u, v) = match (interleaved, separate) { + (true, true) => { + let stride = format.planes[1].stride.max(format.size.width) as usize; + ( + Component { + plane: 1, + // Interleaved chroma is two bytes per sample pair, so a pixel + // column is a byte column. + offset: chroma_top * stride + left, + stride, + }, + None, + ) + } + (true, false) => ( + Component { + plane: 0, + offset: stride * rows + chroma_top * stride + left, + stride, + }, + None, + ), + (false, true) if format.planes.len() >= 3 => { + let u_stride = format.planes[1].stride.max(format.size.width / 2) as usize; + let v_stride = format.planes[2].stride.max(format.size.width / 2) as usize; + ( + Component { + plane: 1, + offset: chroma_top * u_stride + chroma_left, + stride: u_stride, + }, + Some(Component { + plane: 2, + offset: chroma_top * v_stride + chroma_left, + stride: v_stride, + }), + ) + } + (false, true) => { + return Err(Error::Codec(anyhow::anyhow!( + "V4L2 chose planar {} with {} planes", + name(format.pixelformat), + format.planes.len() + ))); + } + (false, false) => { + let chroma_stride = stride / 2; + let chroma_origin = chroma_top * chroma_stride + chroma_left; + ( + Component { + plane: 0, + offset: stride * rows + chroma_origin, + stride: chroma_stride, + }, + Some(Component { + plane: 0, + offset: stride * rows + chroma_stride * rows.div_ceil(2) + chroma_origin, + stride: chroma_stride, + }), + ) + } + }; + + Ok(Self { y, u, v, size }) + } + + /// Copy a frame into buffer `index`, laying each plane out the way the driver + /// asked for. + pub(crate) fn write(&self, queue: &mut Queue, index: u32, frame: &I420) -> Result<(), Error> { + let (width, height) = (self.size.width as usize, self.size.height as usize); + let (chroma_width, chroma_rows) = (width / 2, height / 2); + + scatter(queue.plane_mut(index, self.y.plane), self.y, frame.y(), width, height)?; + match self.v { + Some(v) => { + scatter( + queue.plane_mut(index, self.u.plane), + self.u, + frame.u(), + chroma_width, + chroma_rows, + )?; + scatter(queue.plane_mut(index, v.plane), v, frame.v(), chroma_width, chroma_rows)?; + } + None => interleave( + queue.plane_mut(index, self.u.plane), + self.u, + frame.u(), + frame.v(), + chroma_width, + chroma_rows, + )?, + } + Ok(()) + } + + /// Copy a picture out of a dequeued buffer, undoing the driver's stride and + /// chroma layout. + pub(crate) fn read(&self, queue: &Queue, buffer: &Dequeued) -> Result { + let (width, height) = (self.size.width as usize, self.size.height as usize); + let (chroma_width, chroma_rows) = (width / 2, height / 2); + + let mut data = vec![0u8; I420::len(self.size.width, self.size.height)]; + let (luma, chroma) = data.split_at_mut(width * height); + let (u, v) = chroma.split_at_mut(chroma_width * chroma_rows); + + gather(luma, queue.payload(buffer, self.y.plane), self.y, width, height)?; + match self.v { + Some(at) => { + gather( + u, + queue.payload(buffer, self.u.plane), + self.u, + chroma_width, + chroma_rows, + )?; + gather(v, queue.payload(buffer, at.plane), at, chroma_width, chroma_rows)?; + } + None => deinterleave( + u, + v, + queue.payload(buffer, self.u.plane), + self.u, + chroma_width, + chroma_rows, + )?, + } + + I420::new(self.size.width, self.size.height, data) + } +} + +/// Luma rows the driver reserves before chroma starts. +/// +/// A driver that pads the height reports the padding only through `sizeimage`, +/// so a contiguous 4:2:0 buffer's chroma offset has to be recovered from it. The +/// floor at the visible height keeps a driver whose `sizeimage` carries an +/// unrelated tail from placing chroma inside the picture. +fn padded_rows(luma: Plane, height: u32) -> usize { + let height = height as usize; + match luma.stride as usize { + // A driver that reports no stride is reporting no padding either. + 0 => height, + // One buffer holding 4:2:0: sizeimage = stride * rows * 3 / 2. + stride => (luma.sizeimage as usize * 2 / (stride * 3)).max(height), + } +} + +/// Copy `rows` rows of `width` tightly packed bytes into a strided plane. +fn scatter(dst: &mut [u8], at: Component, src: &[u8], width: usize, rows: usize) -> Result<(), Error> { + let len = dst.len(); + for row in 0..rows { + let start = at.offset + row * at.stride; + dst.get_mut(start..start + width) + .ok_or_else(|| short(len, start + width))? + .copy_from_slice(&src[row * width..][..width]); + } + Ok(()) +} + +/// Interleave tightly packed U and V rows into a strided semi-planar chroma +/// plane, `width` sample pairs per row. +fn interleave(dst: &mut [u8], at: Component, u: &[u8], v: &[u8], width: usize, rows: usize) -> Result<(), Error> { + let len = dst.len(); + for row in 0..rows { + let start = at.offset + row * at.stride; + let out = dst + .get_mut(start..start + width * 2) + .ok_or_else(|| short(len, start + width * 2))?; + let (u, v) = (&u[row * width..][..width], &v[row * width..][..width]); + for (pair, (u, v)) in out.chunks_exact_mut(2).zip(u.iter().zip(v)) { + pair[0] = *u; + pair[1] = *v; + } + } + Ok(()) +} + +/// Copy `rows` rows of `width` bytes out of a strided plane, tightly packed. +fn gather(dst: &mut [u8], src: &[u8], at: Component, width: usize, rows: usize) -> Result<(), Error> { + for row in 0..rows { + let start = at.offset + row * at.stride; + let line = src + .get(start..start + width) + .ok_or_else(|| short(src.len(), start + width))?; + dst[row * width..][..width].copy_from_slice(line); + } + Ok(()) +} + +/// Split a strided semi-planar chroma plane into tightly packed U and V, +/// `width` sample pairs per row. +fn deinterleave(u: &mut [u8], v: &mut [u8], src: &[u8], at: Component, width: usize, rows: usize) -> Result<(), Error> { + for row in 0..rows { + let start = at.offset + row * at.stride; + let line = src + .get(start..start + width * 2) + .ok_or_else(|| short(src.len(), start + width * 2))?; + let (u, v) = (&mut u[row * width..][..width], &mut v[row * width..][..width]); + for (pair, (u, v)) in line.chunks_exact(2).zip(u.iter_mut().zip(v)) { + *u = pair[0]; + *v = pair[1]; + } + } + Ok(()) +} + +fn short(len: usize, needed: usize) -> Error { + Error::Codec(anyhow::anyhow!( + "V4L2 buffer of {len} bytes is too small for the {needed} its format implies" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A single-buffer format the way a driver reports one. + fn format(pixelformat: u32, size: Size, stride: u32, rows: u32) -> Format { + Format { + pixelformat, + size, + planes: vec![Plane { + stride, + sizeimage: stride * rows * 3 / 2, + }], + } + } + + /// A `tv_usec` past a `u32` of nanoseconds used to panic in debug rather than + /// convert, and a driver that never wrote the field is free to hand back + /// anything at all. + #[test] + fn a_buffer_timestamp_survives_any_timeval() { + assert_eq!( + timestamp(timeval { + tv_sec: 12, + tv_usec: 345_678, + }), + Duration::from_micros(12_345_678) + ); + // Above `u32::MAX / 1_000` microseconds, which is what overflowed. + assert_eq!( + timestamp(timeval { + tv_sec: 0, + tv_usec: 5_000_000, + }), + Duration::from_secs(5) + ); + assert_eq!( + timestamp(timeval { + tv_sec: -1, + tv_usec: -1, + }), + Duration::ZERO + ); + } + + /// `bytesused` counts from the start of the plane, so the payload is what is + /// left past the header the driver reported, and a header past the count is + /// no payload at all. + #[test] + fn the_payload_is_past_the_data_offset() { + let buffer = Dequeued { + index: 0, + bytesused: [10, 0, 0], + data_offset: [4, 0, 0], + timestamp: Duration::ZERO, + flags: 0, + }; + assert_eq!(buffer.written(0), 6); + let buffer = Dequeued { + data_offset: [16, 0, 0], + ..buffer + }; + assert_eq!(buffer.written(0), 0); + } + + /// The bug the alignment work was about: chroma goes where the padded row + /// count puts it, not where the visible height would. + #[test] + fn chroma_follows_the_padded_height() { + // 640x360 padded to a 640-byte stride and 368 rows, which is what a + // 16-row-aligned driver reports. + let planes = Planes::new( + &format(NV12, Size::new(640, 368), 640, 368), + Rect::whole(Size::new(640, 360)), + ) + .unwrap(); + assert_eq!(planes.u.offset, 640 * 368); + assert!(planes.v.is_none()); + } + + /// A driver that pads the stride but not the height still has to be believed + /// about the stride. + #[test] + fn chroma_follows_the_padded_stride() { + let planes = Planes::new( + &format(YUV420, Size::new(360, 240), 384, 240), + Rect::whole(Size::new(360, 240)), + ) + .unwrap(); + assert_eq!(planes.y.stride, 384); + assert_eq!(planes.u.offset, 384 * 240); + assert_eq!(planes.u.stride, 192); + let v = planes.v.unwrap(); + assert_eq!(v.offset, 384 * 240 + 192 * 120); + assert_eq!(v.stride, 192); + } + + /// A compose rectangle that does not start at the corner moves every + /// component's origin, chroma by half as much in each direction. + #[test] + fn the_picture_starts_at_the_compose_origin() { + let rect = Rect { + left: 16, + top: 8, + size: Size::new(1888, 1072), + }; + let planes = Planes::new(&format(NV12, Size::new(1920, 1088), 1920, 1088), rect).unwrap(); + assert_eq!(planes.y.offset, 8 * 1920 + 16); + assert_eq!(planes.u.offset, 1920 * 1088 + 4 * 1920 + 16); + + let planes = Planes::new(&format(YUV420, Size::new(1920, 1088), 1920, 1088), rect).unwrap(); + assert_eq!(planes.y.offset, 8 * 1920 + 16); + assert_eq!(planes.u.offset, 1920 * 1088 + 4 * 960 + 8); + assert_eq!(planes.v.unwrap().offset, 1920 * 1088 + 960 * 544 + 4 * 960 + 8); + } + + /// Per-plane formats put each component in its own buffer at offset zero. + #[test] + fn separate_planes_start_at_zero() { + let format = Format { + pixelformat: NV12M, + size: Size::new(320, 240), + planes: vec![ + Plane { + stride: 320, + sizeimage: 320 * 240, + }, + Plane { + stride: 320, + sizeimage: 320 * 120, + }, + ], + }; + let planes = Planes::new(&format, Rect::whole(Size::new(320, 240))).unwrap(); + assert_eq!(planes.u.plane, 1); + assert_eq!(planes.u.offset, 0); + } + + /// A picture larger than what the driver negotiated would be written past the + /// end of the buffer, so it is refused rather than truncated. + #[test] + fn a_picture_larger_than_the_format_is_refused() { + let format = format(NV12, Size::new(320, 240), 320, 240); + assert!(Planes::new(&format, Rect::whole(Size::new(640, 480))).is_err()); + // So is a picture that fits only if its origin is ignored. + assert!( + Planes::new( + &format, + Rect { + left: 16, + top: 0, + size: Size::new(320, 240) + } + ) + .is_err() + ); + } + + /// A driver offering something we cannot lay out says so at open, not on the + /// first frame. + #[test] + fn an_unsupported_raw_format_is_refused() { + let format = format(fourcc(*b"RGB3"), Size::new(320, 240), 960, 240); + assert!(Planes::new(&format, Rect::whole(Size::new(320, 240))).is_err()); + } + + /// The written picture round-trips: every row lands at the driver's stride + /// and chroma is interleaved in the driver's order. + #[test] + fn writing_respects_the_stride() { + let width = 4; + let height = 4; + let stride = 8; + let at = Component { + plane: 0, + offset: 0, + stride, + }; + + let mut dst = vec![0u8; stride * height]; + let src: Vec = (0..(width * height) as u8).collect(); + scatter(&mut dst, at, &src, width, height).unwrap(); + assert_eq!(&dst[..width], &src[..width]); + assert_eq!(&dst[stride..stride + width], &src[width..width * 2]); + // The padding between rows is left alone. + assert_eq!(&dst[width..stride], &[0; 4]); + + let mut chroma = vec![0u8; stride * height]; + interleave(&mut chroma, at, &[1, 2], &[3, 4], 2, 1).unwrap(); + assert_eq!(&chroma[..4], &[1, 3, 2, 4]); + } + + /// Reading undoes writing, for both chroma layouts. The two directions are + /// separate code, so a stride or an interleave order that disagreed between + /// them would decode every frame with its colors swapped. + #[test] + fn reading_undoes_writing() { + let (width, rows, stride) = (4, 4, 8); + let at = Component { + plane: 0, + offset: 16, + stride, + }; + + let luma: Vec = (0..(width * rows) as u8).collect(); + let mut device = vec![0u8; at.offset + stride * rows]; + scatter(&mut device, at, &luma, width, rows).unwrap(); + let mut back = vec![0u8; width * rows]; + gather(&mut back, &device, at, width, rows).unwrap(); + assert_eq!(back, luma); + + let (u, v) = (vec![1, 2, 3, 4], vec![5, 6, 7, 8]); + let mut device = vec![0u8; at.offset + stride * rows]; + interleave(&mut device, at, &u, &v, 2, 2).unwrap(); + let (mut back_u, mut back_v) = (vec![0u8; 4], vec![0u8; 4]); + deinterleave(&mut back_u, &mut back_v, &device, at, 2, 2).unwrap(); + assert_eq!(back_u, u); + assert_eq!(back_v, v); + } + + /// A buffer smaller than the format implies is an error, not a panic in the + /// middle of a frame. + #[test] + fn a_short_buffer_errors() { + let at = Component { + plane: 0, + offset: 0, + stride: 8, + }; + let mut dst = vec![0u8; 8]; + assert!(scatter(&mut dst, at, &[0; 16], 4, 4).is_err()); + + let mut back = vec![0u8; 16]; + assert!(gather(&mut back, &[0; 8], at, 4, 4).is_err()); + } +}