From f978a74806057ecc62e154ff0aa09da1e89e4169 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 22:02:06 -0800 Subject: [PATCH 1/4] chore(quest): claim quest/m1/camera-nv12-macos.md Quest-Claim: 7fc048a766e63f4b From fc61c56fd0561a8b9cb9bbc4bb5300f952e4f4c6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 22:20:09 -0800 Subject: [PATCH 2/4] fix(video): request NV12 from the macOS camera AVCaptureVideoDataOutput without videoSettings passes the device's native format through, so a MacBook Air camera delivered 2vuy and Output::Render failed at the first frame. Request kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, as the display source already does. Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 1 - quest/m1/camera-nv12-macos.md | 30 --------------- rs/moq-video/src/capture/avfoundation.rs | 48 ++++++++++++++++++++---- 3 files changed, 41 insertions(+), 38 deletions(-) delete mode 100644 quest/m1/camera-nv12-macos.md diff --git a/quest/m1/README.md b/quest/m1/README.md index 8698a16b258..d2b6c83dc19 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -58,7 +58,6 @@ moves what remains of them once their claims merge. - [Session auth grants and replies stay consistent and bounded](/quest/m1/auth-stream-bounds.md) - an AUTH_ERROR revokes its grant, queued grants and retiring tokens are bounded, and dropping queued requests cannot deadlock - [Relay settings parse once into a validated configuration](/quest/m1/relay-config-checks.md) - settings that cannot work together are refused at load instead of accepted and ignored - [Track drop runs are forgotten below the floor](/quest/m1/drop-runs-prune.md) - a track's dropped-group record stays proportional to its window in Rust and JS -- [A macOS camera delivers frames the renderer imports](/quest/m1/camera-nv12-macos.md) - AVFoundation capture asks for NV12, so a camera whose native format is `2vuy` reaches the renderer - [Encoders keyframe by time, not frame count](/quest/m1/keyframe-by-time.md) - groups stay within the declared join interval when a source's frame rate drifts from the configured one ## Related diff --git a/quest/m1/camera-nv12-macos.md b/quest/m1/camera-nv12-macos.md deleted file mode 100644 index 55ed9179836..00000000000 --- a/quest/m1/camera-nv12-macos.md +++ /dev/null @@ -1,30 +0,0 @@ -# [S] A macOS camera delivers frames the renderer imports - -## Goal - -A camera opened through `moq_video::capture` on macOS delivers NV12 pixel -buffers, which the Metal import and the CPU download both read, instead of -the device's native format. Today a MacBook Air camera delivers `2vuy`, and -`Output::Render` fails at the first frame with `cannot download pixel format -0x32767579; expected NV12`. - -## Plan - -Found by moq-dev/studio's first Mac capture run -([moq-dev/studio#70](https://github.com/moq-dev/studio/pull/70)). Decided -2026-11-07 (maintainer, moq-dev/studio's /quest-unblock): ✅ m1 (rejected: m3), -since it is an additive fix to shipped code and blocks the studio's macOS -camera. - -`open` in `rs/moq-video/src/capture/avfoundation.rs` builds its -`AVCaptureVideoDataOutput` without `videoSettings`, so AVFoundation passes the -device's format through. Request `kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange` -through `kCVPixelBufferPixelFormatTypeKey`, as the display source already does -(`rs/moq-video/src/capture/screencapture.rs`). A local patch doing this -brought the camera to the studio's program and broadcast (1280x720 and -640x360 H.264 at 30 fps, correct colors). - -CI has no camera: test that the output requests NV12 by reading its -`videoSettings` back, and record a run on a Mac with a camera in the PR. - -Once it lands, moq-dev/studio bumps its `moq` pin. diff --git a/rs/moq-video/src/capture/avfoundation.rs b/rs/moq-video/src/capture/avfoundation.rs index cf7a1415250..8bc859ce8c2 100644 --- a/rs/moq-video/src/capture/avfoundation.rs +++ b/rs/moq-video/src/capture/avfoundation.rs @@ -2,8 +2,11 @@ //! //! `AVCaptureVideoDataOutput` delivers IOSurface-backed `CVPixelBuffer`s on a //! dispatch queue; the delegate wraps each as a [`Surface::PixelBuffer`] and pushes it -//! into the shared [`FrameChannel`], which the encode loop awaits. Frames reach -//! VideoToolbox with no copy and no color conversion. +//! into the shared [`FrameChannel`], which the encode loop awaits. The output +//! requests NV12, the one layout the encoder, the Metal import, and the CPU +//! download all read, so a camera whose native format differs (a MacBook Air's +//! `2vuy`) is converted by AVFoundation before the frame reaches us. Frames then +//! reach VideoToolbox with no copy. //! //! Sample delivery is the only callback AVFoundation makes on the happy path, so //! the same delegate also observes the notifications that say the session died: @@ -27,7 +30,10 @@ use objc2_av_foundation::{ AVError, AVFoundationErrorDomain, AVMediaType, AVMediaTypeVideo, }; use objc2_core_media::CMSampleBuffer; -use objc2_foundation::{NSError, NSNotification, NSNotificationCenter, NSObject, NSObjectProtocol, NSString}; +use objc2_core_video::{kCVPixelBufferPixelFormatTypeKey, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange}; +use objc2_foundation::{ + NSDictionary, NSError, NSNotification, NSNotificationCenter, NSNumber, NSObject, NSObjectProtocol, NSString, +}; use super::surface::surface_frame; use super::{Camera, Config, FrameChannel, Stream}; @@ -93,10 +99,8 @@ pub(super) async fn open(config: &Config, device: Option<&str>) -> Result) -> Result Retained { + let output = unsafe { AVCaptureVideoDataOutput::new() }; + + // Without a pixel format AVFoundation passes the device's native one through. + let key: &NSString = unsafe { kCVPixelBufferPixelFormatTypeKey }.as_ref(); + let format = NSNumber::new_u32(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange); + let format: &AnyObject = &format; + let settings = NSDictionary::from_slices(&[key], &[format]); + + unsafe { + output.setVideoSettings(Some(&settings)); + // Drop late frames instead of queuing them; we want the newest. + output.setAlwaysDiscardsLateVideoFrames(true); + } + output +} + /// Ensure the process is authorized to use the camera, prompting once if the /// decision hasn't been made yet. /// @@ -436,7 +458,7 @@ impl Delegate { /// grant, or a test-only hook in the capture path. #[cfg(test)] mod tests { - use objc2_foundation::{NSDictionary, NSInteger}; + use objc2_foundation::NSInteger; use super::*; @@ -451,6 +473,18 @@ mod tests { (chan, session, guard) } + /// CI has no camera, so check the request itself: the output `open` uses + /// asks AVFoundation for NV12. + #[test] + fn the_output_requests_nv12() { + let output = video_output(); + let settings = unsafe { output.videoSettings() }; + let key: &NSString = unsafe { kCVPixelBufferPixelFormatTypeKey }.as_ref(); + let format = settings.objectForKey(key).expect("pixel format requested"); + let format = format.downcast::().expect("pixel format is a number"); + assert_eq!(format.as_u32(), kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange); + } + /// Teardown must never call `stopRunning` while configuration is open. This /// is the setup-failure path when `canAddInput` or `canAddOutput` returns false. #[test] From d5a06b5696a30305313caaae732caace2e696d5e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 22:57:27 -0800 Subject: [PATCH 3/4] ci(video): run the AVFoundation capture tests on macOS Co-Authored-By: Claude Opus 5.5 --- rs/justfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rs/justfile b/rs/justfile index 23a8ab90add..91a2fde2d4e 100644 --- a/rs/justfile +++ b/rs/justfile @@ -155,6 +155,8 @@ macos-test: run just rs test -p kio -p hang -p moq-tokio -p moq-sock run just rs test -p moq-sock --features fast-apple-datapath run just rs test -p moq-video --no-default-features -E 'test(encode::backend::videotoolbox::tests)' + # Runner has no camera; these tests need none. + run just rs test -p moq-video --no-default-features --features capture -E 'test(/^capture::avfoundation::tests::/)' run just rs test -p moq-video --no-default-features --features render -E 'test(/^render::export::/)' # The runner's paravirtual Metal device proves the IOSurface import, ignored elsewhere. run just rs test -p moq-video --no-default-features --features render --lib --run-ignored only \ From 7fc216d345b698c46aad9e8212520bd47ff1fc50 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 23:42:29 -0800 Subject: [PATCH 4/4] test(video): check the NV12 settings, not the unconnected output videoSettings reads back nil on an output with no connected device, which the macOS runner showed by panicking in objc2's non-null getter. Co-Authored-By: Claude Opus 5.5 --- rs/moq-video/src/capture/avfoundation.rs | 29 +++++++++--------------- 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/rs/moq-video/src/capture/avfoundation.rs b/rs/moq-video/src/capture/avfoundation.rs index 8bc859ce8c2..db8c3012c03 100644 --- a/rs/moq-video/src/capture/avfoundation.rs +++ b/rs/moq-video/src/capture/avfoundation.rs @@ -99,8 +99,11 @@ pub(super) async fn open(config: &Config, device: Option<&str>) -> Result) -> Result Retained { - let output = unsafe { AVCaptureVideoDataOutput::new() }; - - // Without a pixel format AVFoundation passes the device's native one through. +/// The `videoSettings` that make the output deliver NV12. Without a pixel +/// format AVFoundation passes the device's native one through. +fn nv12_settings() -> Retained> { let key: &NSString = unsafe { kCVPixelBufferPixelFormatTypeKey }.as_ref(); let format = NSNumber::new_u32(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange); let format: &AnyObject = &format; - let settings = NSDictionary::from_slices(&[key], &[format]); - - unsafe { - output.setVideoSettings(Some(&settings)); - // Drop late frames instead of queuing them; we want the newest. - output.setAlwaysDiscardsLateVideoFrames(true); - } - output + NSDictionary::from_slices(&[key], &[format]) } /// Ensure the process is authorized to use the camera, prompting once if the @@ -473,12 +467,11 @@ mod tests { (chan, session, guard) } - /// CI has no camera, so check the request itself: the output `open` uses - /// asks AVFoundation for NV12. + /// CI has no camera, and `videoSettings` reads back nil until a device is + /// connected, so check the settings `open` hands the output. #[test] fn the_output_requests_nv12() { - let output = video_output(); - let settings = unsafe { output.videoSettings() }; + let settings = nv12_settings(); let key: &NSString = unsafe { kCVPixelBufferPixelFormatTypeKey }.as_ref(); let format = settings.objectForKey(key).expect("pixel format requested"); let format = format.downcast::().expect("pixel format is a number");