diff --git a/quest/m1/README.md b/quest/m1/README.md index 9ed7f402118..2e8082945d5 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -43,7 +43,6 @@ moves what remains of them once their claims merge. - [Push-to-talk's press-to-audible latency fits a budget on the defaults](/quest/m1/push-to-talk-latency.md) - press to audible on the browser defaults, excluding device output, stays under a kixtop-measured budget that `just test push-to-talk` enforces - [Lifecycles are explicit state](/quest/m1/explicit-state/README.md) - every lifecycle the 2026-11-07 audit found implicit is one enum or union, fixing the bugs in the 11 lifecycles it redesigns, with review and agent rules against new ones - [Track drop runs are forgotten below the floor](/quest/m1/drop-runs-prune.md) - a track's dropped-group record stays proportional to its window in Rust and JS -- [A macOS camera delivers frames the renderer imports](/quest/m1/camera-nv12-macos.md) - AVFoundation capture asks for NV12, so a camera whose native format is `2vuy` reaches the renderer ## Related diff --git a/quest/m1/camera-nv12-macos.md b/quest/m1/camera-nv12-macos.md deleted file mode 100644 index 55ed9179836..00000000000 --- a/quest/m1/camera-nv12-macos.md +++ /dev/null @@ -1,30 +0,0 @@ -# [S] A macOS camera delivers frames the renderer imports - -## Goal - -A camera opened through `moq_video::capture` on macOS delivers NV12 pixel -buffers, which the Metal import and the CPU download both read, instead of -the device's native format. Today a MacBook Air camera delivers `2vuy`, and -`Output::Render` fails at the first frame with `cannot download pixel format -0x32767579; expected NV12`. - -## Plan - -Found by moq-dev/studio's first Mac capture run -([moq-dev/studio#70](https://github.com/moq-dev/studio/pull/70)). Decided -2026-11-07 (maintainer, moq-dev/studio's /quest-unblock): ✅ m1 (rejected: m3), -since it is an additive fix to shipped code and blocks the studio's macOS -camera. - -`open` in `rs/moq-video/src/capture/avfoundation.rs` builds its -`AVCaptureVideoDataOutput` without `videoSettings`, so AVFoundation passes the -device's format through. Request `kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange` -through `kCVPixelBufferPixelFormatTypeKey`, as the display source already does -(`rs/moq-video/src/capture/screencapture.rs`). A local patch doing this -brought the camera to the studio's program and broadcast (1280x720 and -640x360 H.264 at 30 fps, correct colors). - -CI has no camera: test that the output requests NV12 by reading its -`videoSettings` back, and record a run on a Mac with a camera in the PR. - -Once it lands, moq-dev/studio bumps its `moq` pin. diff --git a/rs/justfile b/rs/justfile index 50b4e3c002e..29a5222c385 100644 --- a/rs/justfile +++ b/rs/justfile @@ -156,6 +156,8 @@ macos-test: run just rs test -p kio -p hang -p moq-tokio -p moq-sock run just rs test -p moq-sock --features fast-apple-datapath run just rs test -p moq-video --no-default-features -E 'test(encode::backend::videotoolbox::tests)' + # Runner has no camera; these tests need none. + run just rs test -p moq-video --no-default-features --features capture -E 'test(/^capture::avfoundation::tests::/)' run just rs test -p moq-video --no-default-features --features render -E 'test(/^render::export::/)' # The runner's paravirtual Metal device proves the IOSurface import, ignored elsewhere. run just rs test -p moq-video --no-default-features --features render --lib --run-ignored only \ diff --git a/rs/moq-video/src/capture/avfoundation.rs b/rs/moq-video/src/capture/avfoundation.rs index b8ff00f4522..8769a028d07 100644 --- a/rs/moq-video/src/capture/avfoundation.rs +++ b/rs/moq-video/src/capture/avfoundation.rs @@ -2,8 +2,11 @@ //! //! `AVCaptureVideoDataOutput` delivers IOSurface-backed `CVPixelBuffer`s on a //! dispatch queue; the delegate wraps each as a [`Surface::PixelBuffer`] and pushes it -//! into the shared [`FrameChannel`], which the encode loop awaits. Frames reach -//! VideoToolbox with no copy and no color conversion. +//! into the shared [`FrameChannel`], which the encode loop awaits. The output +//! requests NV12, the one layout the encoder, the Metal import, and the CPU +//! download all read, so a camera whose native format differs (a MacBook Air's +//! `2vuy`) is converted by AVFoundation before the frame reaches us. Frames then +//! reach VideoToolbox with no copy. //! //! Sample delivery is the only callback AVFoundation makes on the happy path, so //! the same delegate also observes the notifications that say the session died: @@ -27,7 +30,10 @@ use objc2_av_foundation::{ AVError, AVFoundationErrorDomain, AVMediaType, AVMediaTypeVideo, }; use objc2_core_media::CMSampleBuffer; -use objc2_foundation::{NSError, NSNotification, NSNotificationCenter, NSObject, NSObjectProtocol, NSString}; +use objc2_core_video::{kCVPixelBufferPixelFormatTypeKey, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange}; +use objc2_foundation::{ + NSDictionary, NSError, NSNotification, NSNotificationCenter, NSNumber, NSObject, NSObjectProtocol, NSString, +}; use super::surface::surface_frame; use super::{Camera, Config, FrameChannel, Stream}; @@ -95,6 +101,7 @@ pub(super) async fn open(config: &Config, device: Option<&str>) -> Result) -> Result Retained> { + let key: &NSString = unsafe { kCVPixelBufferPixelFormatTypeKey }.as_ref(); + let format = NSNumber::new_u32(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange); + let format: &AnyObject = &format; + NSDictionary::from_slices(&[key], &[format]) +} + /// Ensure the process is authorized to use the camera, prompting once if the /// decision hasn't been made yet. /// @@ -436,7 +452,7 @@ impl Delegate { /// grant, or a test-only hook in the capture path. #[cfg(test)] mod tests { - use objc2_foundation::{NSDictionary, NSInteger}; + use objc2_foundation::NSInteger; use super::*; @@ -451,6 +467,17 @@ mod tests { (chan, session, guard) } + /// CI has no camera, and `videoSettings` reads back nil until a device is + /// connected, so check the settings `open` hands the output. + #[test] + fn the_output_requests_nv12() { + let settings = nv12_settings(); + let key: &NSString = unsafe { kCVPixelBufferPixelFormatTypeKey }.as_ref(); + let format = settings.objectForKey(key).expect("pixel format requested"); + let format = format.downcast::().expect("pixel format is a number"); + assert_eq!(format.as_u32(), kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange); + } + /// Teardown must never call `stopRunning` while configuration is open. This /// is the setup-failure path when `canAddInput` or `canAddOutput` returns false. #[test]