diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 9be9c456..5166e07a 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -1,8 +1,9 @@ name: macOS # Builds the macOS app bundle for each architecture, with CEF for browser -# sources, signs it ad hoc with the release's hardened runtime and -# entitlements, runs the browser-source tests against it on Apple silicon, +# sources and the virtual camera's extension, signs it ad hoc with the +# release's hardened runtime and entitlements, less those only the release's +# provisioning profile grants, runs the browser-source tests against it on Apple silicon, # packs it into a disk image, and checks the app in the image launches and # lists devices. Pull requests that change the packaging run it; nightly runs # it against main. @@ -120,7 +121,10 @@ jobs: # Application .p12 (base64, legacy encryption so `security import` reads # it) and APPLE_CERTIFICATE_PASSWORD; APPLE_API_KEY, APPLE_API_ISSUER, and # APPLE_API_PRIVATE_KEY, the App Store Connect key ID, issuer ID, and .p8 - # text. quest/m0/signing-renew.md renews them. Nothing built runs until + # text; APPLE_PROVISIONING_PROFILE, the Developer ID profile for + # live.kixel.studio with the System Extension capability (base64), which + # lets the app install its camera extension. quest/m0/signing-renew.md + # renews them. Nothing built runs until # they are gone from the runner. environment: release @@ -192,6 +196,13 @@ jobs: echo "KEYCHAIN=$keychain" >>"$GITHUB_ENV" echo "IDENTITY=$identity" >>"$GITHUB_ENV" + # Where sign.sh reads the entitlements it grants from, and where the + # system looks for it when the app installs its camera extension. + - name: Embed the provisioning profile + env: + APPLE_PROVISIONING_PROFILE: ${{ secrets.APPLE_PROVISIONING_PROFILE }} + run: base64 --decode <<<"$APPLE_PROVISIONING_PROFILE" >"kixel.live Studio.app/Contents/embedded.provisionprofile" + - name: Sign run: packaging/macos/sign.sh "kixel.live Studio.app" "$IDENTITY" "$KEYCHAIN" @@ -236,6 +247,8 @@ jobs: assessment=$(spctl --assess --type execute --verbose=2 "kixel.live Studio.app" 2>&1) echo "$assessment" [[ $assessment == *"source=Notarized Developer ID"* ]] + # Allowed to install the camera extension. + codesign -d --entitlements - --xml "kixel.live Studio.app" | grep com.apple.developer.system-extension.install >/dev/null # The app as the image holds it, which is what users run. - name: Smoke test diff --git a/Cargo.lock b/Cargo.lock index 9c8be92d..6d515883 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -371,7 +371,7 @@ dependencies = [ "objc2-foundation 0.3.2", "parking_lot", "percent-encoding", - "windows-sys 0.59.0", + "windows-sys 0.61.2", "x11rb 0.13.2", ] @@ -1743,7 +1743,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d045de693cb712d0b22c6a64be5b953f67b3ce00ab5ad3dd5d8b441886ab8e1a" dependencies = [ "quote", - "syn 2.0.119", + "syn 3.0.6", ] [[package]] @@ -2488,7 +2488,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4818,7 +4818,9 @@ dependencies = [ "apple-native-keyring-store", "arboard", "ashpd", + "block2 0.6.2", "bytes", + "dispatch2", "eframe", "egui", "egui_extras", @@ -4840,11 +4842,19 @@ dependencies = [ "moq-relay", "moq-scene", "moq-stats", + "moq-studio-camera", "moq-tokio", "moq-v4l", "moq-video", "mp4-atom", + "objc2 0.6.5", "objc2-app-kit 0.3.2", + "objc2-core-foundation", + "objc2-core-media", + "objc2-core-media-io", + "objc2-core-video", + "objc2-foundation 0.3.2", + "objc2-system-extensions", "pipewire", "png 0.18.1", "pollster 1.0.1", @@ -4870,6 +4880,20 @@ dependencies = [ "zbus-secret-service-keyring-store", ] +[[package]] +name = "moq-studio-camera" +version = "0.1.0" +dependencies = [ + "block2 0.6.2", + "dispatch2", + "objc2 0.6.5", + "objc2-core-foundation", + "objc2-core-media", + "objc2-core-media-io", + "objc2-core-video", + "objc2-foundation 0.3.2", +] + [[package]] name = "moq-tokio" version = "0.19.23" @@ -5253,7 +5277,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5620,6 +5644,23 @@ dependencies = [ "objc2-core-video", ] +[[package]] +name = "objc2-core-media-io" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "705c4d968c0c4ca4827d6af083648809370c2276c286fe1ec468d339a0089fc0" +dependencies = [ + "bitflags 2.13.2", + "block2 0.6.2", + "dispatch2", + "libc", + "objc2 0.6.5", + "objc2-core-foundation", + "objc2-core-media", + "objc2-core-video", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-core-services" version = "0.3.2" @@ -5818,6 +5859,17 @@ dependencies = [ "objc2-foundation 0.2.2", ] +[[package]] +name = "objc2-system-extensions" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00ca10e1fd778a1dd13d579afb37168422ef4bbb36b6723179c2228f6b8372c" +dependencies = [ + "dispatch2", + "objc2 0.6.5", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-ui-kit" version = "0.2.2" @@ -7188,7 +7240,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7245,7 +7297,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -8426,10 +8478,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix 1.1.6", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -9969,7 +10021,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index da0d35f7..6851fc64 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["rs/moq-scene", "rs/moq-studio"] +members = ["rs/moq-scene", "rs/moq-studio", "rs/moq-studio-camera"] # The moq submodule is its own cargo workspace; exclude it so this one does not # try to adopt its members. exclude = ["moq"] @@ -32,6 +32,7 @@ moq-mux = { path = "moq/rs/moq-mux" } moq-net = { path = "moq/rs/moq-net" } moq-relay = { path = "moq/rs/moq-relay", default-features = false } moq-scene = { path = "rs/moq-scene" } +moq-studio-camera = { path = "rs/moq-studio-camera" } moq-stats = { path = "moq/rs/moq-stats" } moq-tokio = { path = "moq/rs/moq-tokio", default-features = false } moq-v4l = { path = "moq/rs/moq-v4l" } diff --git a/README.md b/README.md index c6bb74c1..abca9afa 100644 --- a/README.md +++ b/README.md @@ -2179,7 +2179,17 @@ cannot be added as a source of the picture it shows. open `/dev/video*` directly. The studio never loads that module, since it needs root. Load a device for each camera you turn on, such as two: `sudo modprobe v4l2loopback devices=2 exclusive_caps=1,1 card_label="kixel.live Studio","kixel.live Studio 2"`. -- **macOS and Windows:** not available yet. +- **macOS:** a camera extension the app carries. The first time you turn a + camera on, macOS asks you to allow it in System Settings > General > Login + Items & Extensions > Camera Extensions, which needs the app in + Applications; then turn the camera off and on again. From then on apps list + "kixel.live Studio", the program's camera, even while the studio is closed, + when it shows nothing, and "kixel.live Studio: " while a scene's + camera is on. A newer app replaces the extension, without a restart, the + first time you turn a camera on after updating. Only the app feeds the + extension's cameras, so the studio run from a build outside the app has no + virtual camera. +- **Windows:** not available yet. ## Development @@ -2202,7 +2212,8 @@ it. They draw on Mesa's software Vulkan driver, so need no display or GPU, and run on Linux only, where that driver builds. `SHOTS_DIR` picks another output directory; a full run clears old PNGs only from the default one. -`rs/moq-scene` is the UI-free compositor, `rs/moq-studio` the app, and +`rs/moq-scene` is the UI-free compositor, `rs/moq-studio` the app, +`rs/moq-studio-camera` the macOS virtual camera's extension, and `js/join` the join page, built on `@moq/room`, `@moq/publish`, and `@moq/watch`. Like the Rust crates, those come from the `moq` submodule rather than npm, so diff --git a/packaging/macos/Camera.plist b/packaging/macos/Camera.plist new file mode 100644 index 00000000..2ab0cd58 --- /dev/null +++ b/packaging/macos/Camera.plist @@ -0,0 +1,38 @@ + + + + + + CFBundleDevelopmentRegion + en + CFBundleDisplayName + kixel.live Studio Camera + CFBundleExecutable + moq-studio-camera + CFBundleIdentifier + live.kixel.studio.camera + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + kixel.live Studio Camera + CFBundlePackageType + SYSX + CFBundleShortVersionString + 0.0.0 + CFBundleVersion + 0.0.0 + CMIOExtension + + + CMIOExtensionMachServiceName + D7D5SDDB5Z.live.kixel.studio.camera + + LSMinimumSystemVersion + 13.0 + NSSystemExtensionUsageDescription + Shows your program and published scenes as cameras that video-call apps can pick. + + diff --git a/packaging/macos/bundle.sh b/packaging/macos/bundle.sh index 912b9be0..f77d4ab9 100755 --- a/packaging/macos/bundle.sh +++ b/packaging/macos/bundle.sh @@ -12,8 +12,10 @@ cd "$packaging/../.." # Info.plist's LSMinimumSystemVersion, so the linker and OpenH264's C build # target the oldest macOS the bundle claims. export MACOSX_DEPLOYMENT_TARGET=13.0 -# The studio, and the binary CEF's subprocesses run from (browser sources). -json=$(cargo build --locked --release -p moq-studio -p moq-scene --bin moq-studio --bin moq-browser-helper \ +# The studio, the binary CEF's subprocesses run from (browser sources), and +# the virtual camera's extension. +json=$(cargo build --locked --release -p moq-studio -p moq-scene -p moq-studio-camera \ + --bin moq-studio --bin moq-browser-helper --bin moq-studio-camera \ --features moq-studio/release,moq-studio/cef,moq-scene/cef --message-format=json-render-diagnostics) # The binaries, the build script's output, which holds the icon it drew, and # cef-dll-sys's, where it downloads CEF unless CEF_PATH names a copy. @@ -22,10 +24,11 @@ artifact() { } executable=$(artifact moq-studio) helper=$(artifact moq-browser-helper) +camera=$(artifact moq-studio-camera) out=$(jq -r 'select(.reason == "build-script-executed" and (.package_id | test("/rs/moq-studio#"))) | .out_dir' <<<"$json") cef=$(jq -r 'select(.reason == "build-script-executed" and (.package_id | test("cef-dll-sys"))) | .out_dir' <<<"$json") version=$(cargo metadata --locked --no-deps --format-version 1 | jq -r '.packages[] | select(.name == "moq-studio") | .version') -[[ -x $executable && -x $helper && -f $out/live.kixel.studio.icns && -n $version ]] +[[ -x $executable && -x $helper && -x $camera && -f $out/live.kixel.studio.icns && -n $version ]] frameworks=() while IFS= read -r framework; do frameworks+=("$framework") @@ -67,3 +70,14 @@ for variant in "" " (GPU)" " (Renderer)" " (Plugin)" " (Alerts)"; do plutil -lint "$contents/Info.plist" cp "$helper" "$contents/MacOS/$name" done + +# The virtual camera's Camera Extension, which the studio asks the system to +# activate from here. Named for its identifier, as the system requires. +contents="$app/Contents/Library/SystemExtensions/live.kixel.studio.camera.systemextension/Contents" +mkdir -p "$contents/MacOS" +cp "$packaging/Camera.plist" "$contents/Info.plist" +chmod 644 "$contents/Info.plist" +plutil -replace CFBundleShortVersionString -string "$version" "$contents/Info.plist" +plutil -replace CFBundleVersion -string "$version" "$contents/Info.plist" +plutil -lint "$contents/Info.plist" +cp "$camera" "$contents/MacOS/moq-studio-camera" diff --git a/packaging/macos/camera.entitlements.plist b/packaging/macos/camera.entitlements.plist new file mode 100644 index 00000000..53c45f0f --- /dev/null +++ b/packaging/macos/camera.entitlements.plist @@ -0,0 +1,15 @@ + + + + + + com.apple.security.app-sandbox + + com.apple.security.application-groups + + D7D5SDDB5Z.live.kixel.studio.camera + + + diff --git a/packaging/macos/sign.sh b/packaging/macos/sign.sh index 43644c24..a4fea6d8 100755 --- a/packaging/macos/sign.sh +++ b/packaging/macos/sign.sh @@ -4,12 +4,20 @@ # Application certificate's SHA-1 for a release, or `-` to sign ad hoc, which # runs the same runtime and entitlements without the release's secrets. # Optional: the keychain that holds the identity. +# A release bundle holds its Developer ID provisioning profile as +# Contents/embedded.provisionprofile, which grants the app the restricted +# entitlements installing the camera extension takes. An ad hoc signature +# cannot carry them, since macOS kills an app claiming them without a profile, +# so an ad hoc bundle runs but cannot install the extension. set -euo pipefail +shopt -s nullglob app=$1 identity=$2 keychain=${3:-} packaging=$(cd "$(dirname "$0")" && pwd) +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT flags=(--force --options runtime --sign "$identity") if [[ $identity == - ]]; then @@ -22,15 +30,55 @@ if [[ -n $keychain ]]; then flags+=(--keychain "$keychain") fi -# The entitlements each helper app's process needs, by its name: none but -# those that compile code at run time: the GPU and renderer helpers, and the -# plain one too, since CEF 156 starts pages' renderers and utility processes -# from it, and under the hardened runtime V8 cannot reserve its code range -# without `allow-jit`. +fail() { + echo "$1" >&2 + exit 1 +} + +# The app's entitlements: entitlements.plist, and for a release those its +# profile grants to install the camera extension, checked against the bundle. +app_entitlements=$packaging/entitlements.plist +if [[ $identity != - ]]; then + profile=$app/Contents/embedded.provisionprofile + [[ -f $profile ]] || fail "a release bundle needs its provisioning profile at $profile" + security cms -D -i "$profile" >"$work/profile.plist" + profile() { + /usr/libexec/PlistBuddy -c "Print :$1" "$work/profile.plist" + } + team=$(profile TeamIdentifier:0) + application=$(profile Entitlements:com.apple.application-identifier) + [[ $application == "$team.live.kixel.studio" ]] || + fail "the profile is for $application, not $team.live.kixel.studio" + [[ $(profile Entitlements:com.apple.developer.system-extension.install) == true ]] || + fail "the profile does not grant installing system extensions" + # The extension's Mach service, and the app group that serves it, are + # named for the team that signs it. + extensions=("$app"/Contents/Library/SystemExtensions/*.systemextension) + ((${#extensions[@]} == 1)) || fail "the bundle holds no single system extension" + service=$(/usr/libexec/PlistBuddy -c "Print :CMIOExtension:CMIOExtensionMachServiceName" \ + "${extensions[0]}/Contents/Info.plist") + [[ $service == "$team".* ]] || fail "the camera extension's service $service is not team $team's" + + app_entitlements=$work/entitlements.plist + cp "$packaging/entitlements.plist" "$app_entitlements" + /usr/libexec/PlistBuddy \ + -c "Add :com.apple.application-identifier string $application" \ + -c "Add :com.apple.developer.team-identifier string $team" \ + -c "Add :com.apple.developer.system-extension.install bool true" \ + "$app_entitlements" +fi + +# The entitlements each nested bundle's process needs, by its name: for the +# helper apps, none but those that compile code at run time: the GPU and +# renderer helpers, and the plain one too, since CEF 156 starts pages' +# renderers and utility processes from it, and under the hardened runtime V8 +# cannot reserve its code range without `allow-jit`. The camera extension's +# sandbox and app group. entitlements() { case $1 in *" Helper.app" | *" Helper (Renderer).app" | *" Helper (GPU).app") echo "$packaging/jit.entitlements.plist" ;; *.app) ;; + *.systemextension) echo "$packaging/camera.entitlements.plist" ;; *) echo "no entitlements known for $1" >&2 return 1 @@ -41,16 +89,15 @@ entitlements() { # Nested code is signed before the bundle that holds it, innermost first, each # with its own entitlements; the bundle's signature then seals theirs. A # framework's libraries go before the framework. -shopt -s nullglob for framework in "$app"/Contents/Frameworks/*.framework; do for library in "$framework"/Libraries/*.dylib; do codesign "${flags[@]}" "$library" done codesign "${flags[@]}" "$framework" done -for helper in "$app"/Contents/Frameworks/*.app; do - plist=$(entitlements "$helper") - codesign "${flags[@]}" ${plist:+--entitlements "$plist"} "$helper" +for nested in "$app"/Contents/Frameworks/*.app "$app"/Contents/Library/SystemExtensions/*.systemextension; do + plist=$(entitlements "$nested") + codesign "${flags[@]}" ${plist:+--entitlements "$plist"} "$nested" done -codesign "${flags[@]}" --entitlements "$packaging/entitlements.plist" "$app" +codesign "${flags[@]}" --entitlements "$app_entitlements" "$app" codesign --verify --strict --deep --verbose=2 "$app" diff --git a/packaging/macos/smoke.sh b/packaging/macos/smoke.sh index f25ca443..f6e55600 100755 --- a/packaging/macos/smoke.sh +++ b/packaging/macos/smoke.sh @@ -46,6 +46,15 @@ else fi executable=$app/Contents/MacOS/moq-studio +# The camera extension, where the system looks for it, signed with the +# sandbox every Camera Extension needs. Installing it takes a user's approval, +# which no runner gives. +extension=$app/Contents/Library/SystemExtensions/live.kixel.studio.camera.systemextension +[[ -x $extension/Contents/MacOS/moq-studio-camera ]] || fail "the app carries no camera extension" +codesign --verify --strict "$extension" || fail "the camera extension is not signed" +codesign -d --entitlements - --xml "$extension" | grep com.apple.security.app-sandbox >/dev/null || + fail "the camera extension is not sandboxed" + # Whether the studio has a window on screen. Listing windows by owner needs # no permission, unlike reading their titles. window() { diff --git a/quest/m0/vcam/README.md b/quest/m0/vcam/README.md index 753b67dd..cce6c47a 100644 --- a/quest/m0/vcam/README.md +++ b/quest/m0/vcam/README.md @@ -23,3 +23,8 @@ one for the program and one for each published scene, not saved in the scene file. Each OS backend is a `vcam::backend` in `rs/moq-studio`, fed tightly packed RGBA frames of what its camera shows on a `vcam` thread only while an app reads the camera, and names its camera after the scene it shows. + +The macOS backend is a CoreMediaIO Camera Extension the app bundle carries +(`rs/moq-studio-camera`). It always lists the program's camera, adds a +scene's camera when the studio asks through a custom property, and takes +pictures on each camera's sink stream only from the signed app. diff --git a/quest/m0/vcam/macos-try.md b/quest/m0/vcam/macos-try.md new file mode 100644 index 00000000..b3e9f36b --- /dev/null +++ b/quest/m0/vcam/macos-try.md @@ -0,0 +1,36 @@ +# [S] The macOS virtual camera reaches apps on a Mac + +## Goal + +On a Mac running a signed release, turning a virtual camera on installs the +camera extension after one approval in System Settings, and FaceTime, Zoom, +and Chrome list "kixel.live Studio" and "kixel.live Studio: " and show +their pictures. Installing the next release replaces the extension without a +reboot. Whatever breaks on the way is fixed here. + +## Plan + +moq-dev/studio#217 added the extension without running it: installing a +system extension takes the release's Developer ID profile and a person's +approval, which no runner has. Check first what it assumed: + +- The app reaches the extension's custom properties (`kcfg`, `krdg`) through + `CMIOObjectGetPropertyData` and `CMIOObjectSetPropertyData`, and hears + `krdg` change through a listener block. +- A camera's sink stream is the one whose `kCMIOStreamPropertyDirection` is 0. +- `consumeSampleBufferFromClient` waits for a picture; when it returns none, + the extension looks again a frame later. +- The extension's team-prefixed app group needs no provisioning profile of + its own under a Developer ID signature. +- Apps take 32-bit BGRA. + +`log stream --predicate 'subsystem == "com.apple.cmio"'` and +`systemextensionsctl list` show what the system made of it. + +## Hosts + +- macos - approving the extension needs a Mac's GUI session + +## Waiting + +- Maintainer: after the next release, on a Mac, install it to Applications, turn on Settings > Advanced > Virtual camera, allow the extension in System Settings > General > Login Items & Extensions > Camera Extensions, turn the camera on again, and pick "kixel.live Studio" in FaceTime or Photo Booth; do the same for a published scene's camera, and report what each showed. Only a person can approve a system extension (System Settings asks for the user's password or Touch ID), and no CI runner can install one. Check: `systemextensionsctl list` shows `live.kixel.studio.camera` as `[activated enabled]`. diff --git a/quest/m0/vcam/macos.md b/quest/m0/vcam/macos.md deleted file mode 100644 index f3b01cff..00000000 --- a/quest/m0/vcam/macos.md +++ /dev/null @@ -1,25 +0,0 @@ -# [M] macOS virtual camera - -## Goal - -The macOS app bundle carries a CoreMediaIO Camera Extension. Once the user -approves it, apps list a camera that carries the program or a published scene. An app -update installs the new extension without a reboot. - -## Plan - -Decided 2026-11-09 (maintainer, /quest-audit): ✅ the extension joins the -studio's app bundle and goes through its signing (rejected: a Related link -only; leaving the quests unlinked). - -`packaging/macos/bundle.sh` lays the bundle out; the extension goes in -`Contents/Library/SystemExtensions`. `packaging/macos/sign.sh` signs it, with -its own entitlements, before the app. The app then needs the -`com.apple.developer.system-extension.install` entitlement, which only a -provisioning profile grants: embed the `release` environment's -`APPLE_PROVISIONING_PROFILE` (the Developer ID profile for -`live.kixel.studio` with the System Extension capability, base64) as -`Contents/embedded.provisionprofile`, and sign with the -`com.apple.application-identifier` and `com.apple.developer.team-identifier` -entitlements it names. A pull request's ad hoc signature cannot carry these, -so its launch check needs them left out. diff --git a/rs/moq-studio-camera/Cargo.toml b/rs/moq-studio-camera/Cargo.toml new file mode 100644 index 00000000..1a59a4bf --- /dev/null +++ b/rs/moq-studio-camera/Cargo.toml @@ -0,0 +1,32 @@ +[package] +publish = false +name = "moq-studio-camera" +description = "The studio's macOS virtual camera: a CoreMediaIO Camera Extension, and what the studio says to it" +authors.workspace = true +repository.workspace = true +license.workspace = true + +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true + +# The extension packaging/macos/bundle.sh puts in the app bundle. Only the +# system's camera service starts it. +[[bin]] +name = "moq-studio-camera" +path = "src/main.rs" + +# The objc2 releases moq-video's camera capture builds on, already in the graph +# on macOS. +[target.'cfg(target_os = "macos")'.dependencies] +block2 = "0.6" +dispatch2 = "0.3" +objc2 = "0.6" +objc2-core-foundation = "0.3" +objc2-core-media = "0.3" +objc2-core-media-io = "0.3" +objc2-core-video = "0.3" +objc2-foundation = "0.3" + +[lints] +workspace = true diff --git a/rs/moq-studio-camera/src/extension.rs b/rs/moq-studio-camera/src/extension.rs new file mode 100644 index 00000000..60d5c52d --- /dev/null +++ b/rs/moq-studio-camera/src/extension.rs @@ -0,0 +1,776 @@ +//! The CoreMediaIO provider: the program's camera, a camera per scene the +//! studio asks for, and each camera's two streams. +//! +//! CoreMediaIO holds a source only weakly, so each object here keeps the +//! sources below it alive: the [`Provider`] its cameras, and each [`Camera`] +//! its device, its streams, and their [`Stream`] sources. + +use std::ffi::c_void; +use std::ptr::NonNull; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Mutex, OnceLock}; + +use block2::RcBlock; +use dispatch2::{DispatchQueue, DispatchQueueGlobalPriority, DispatchTime, GlobalQueueIdentifier}; +use moq_studio_camera::{APP, CONFIG, Config, READING, TEAM}; +use objc2::rc::{Retained, Weak}; +use objc2::runtime::{AnyObject, Bool, ProtocolObject}; +use objc2::{AnyThread, DefinedClass, Message, define_class, msg_send}; +use objc2_core_foundation::{CFDictionary, CFNumber, CFRetained, CFRunLoop, CFString, CFType}; +use objc2_core_media::{CMClock, CMFormatDescription, CMSampleBuffer, CMTime, CMVideoFormatDescriptionCreate}; +use objc2_core_media_io::{ + CMIOExtensionClient, CMIOExtensionDevice, CMIOExtensionDeviceProperties, CMIOExtensionDeviceSource, + CMIOExtensionPropertyDeviceModel, CMIOExtensionPropertyDeviceTransportType, + CMIOExtensionPropertyProviderManufacturer, CMIOExtensionPropertyProviderName, CMIOExtensionPropertyState, + CMIOExtensionPropertyStreamActiveFormatIndex, CMIOExtensionPropertyStreamFrameDuration, + CMIOExtensionPropertyStreamSinkBufferQueueSize, CMIOExtensionPropertyStreamSinkBufferUnderrunCount, + CMIOExtensionPropertyStreamSinkBuffersRequiredForStartup, CMIOExtensionPropertyStreamSinkEndOfData, + CMIOExtensionProvider, CMIOExtensionProviderProperties, CMIOExtensionProviderSource, CMIOExtensionScheduledOutput, + CMIOExtensionStream, CMIOExtensionStreamClockType, CMIOExtensionStreamDirection, + CMIOExtensionStreamDiscontinuityFlags, CMIOExtensionStreamFormat, CMIOExtensionStreamProperties, + CMIOExtensionStreamSource, +}; +use objc2_core_video::kCVPixelFormatType_32BGRA; +use objc2_foundation::{ + NSArray, NSDictionary, NSError, NSLocalizedDescriptionKey, NSNumber, NSObject, NSObjectProtocol, NSSet, NSString, + NSUUID, ns_string, +}; + +/// The pictures a sink holds before the extension takes them. Two lets the +/// studio queue the next picture while the extension forwards one. +const SINK_QUEUE: u32 = 2; + +/// `kIOAudioDeviceTransportTypeVirtual`, what apps show a software camera as. +const VIRTUAL: u32 = u32::from_be_bytes(*b"virt"); + +/// Serve the cameras until the system stops the extension. +pub fn run() -> ! { + let source = Provider::new(); + let provider = + unsafe { CMIOExtensionProvider::providerWithSource_clientQueue(ProtocolObject::from_ref(&*source), None) }; + source.start(&provider); + unsafe { CMIOExtensionProvider::startServiceWithProvider(&provider) }; + CFRunLoop::run(); + // The run loop returns only once every source is gone. + std::process::exit(0) +} + +/// An object handed to a dispatch queue. Each one sent touches its own state +/// only through mutexes, atomics, and CoreMediaIO objects, which CoreMediaIO +/// itself uses from any queue. +struct Queued(T); + +// SAFETY: as the type says. +unsafe impl Send for Queued {} + +impl Queued { + /// Unwrapped through a method, so a closure captures the whole wrapper + /// rather than the field it would destructure. + fn take(self) -> T { + self.0 + } +} + +/// The current host time, the clock every camera runs on, in nanoseconds. +fn now() -> u64 { + let time = unsafe { CMClock::host_time_clock().time() }; + let nanos = i128::from(time.value) * 1_000_000_000 / i128::from(time.timescale.max(1)); + u64::try_from(nanos).unwrap_or(0) +} + +unsafe extern "C" { + fn NSLog(format: &NSString, ...); +} + +/// Log to the unified log, where Console and `log stream` find it: the +/// system starts the extension with nowhere for stderr to go. +fn log(message: &str) { + unsafe { NSLog(ns_string!("%@"), &*NSString::from_str(message)) }; +} + +#[link(name = "Security", kind = "framework")] +unsafe extern "C" { + static kSecGuestAttributePid: &'static CFString; + fn SecCodeCopyGuestWithAttributes( + host: *const c_void, + attributes: &CFDictionary, + flags: u32, + guest: *mut *const c_void, + ) -> i32; + fn SecRequirementCreateWithString(text: &CFString, flags: u32, requirement: *mut *const c_void) -> i32; + fn SecCodeCheckValidity(code: *const c_void, flags: u32, requirement: *const c_void) -> i32; +} + +/// Whether the process `pid` is the studio: the app, signed by its team +/// with a certificate Apple issued. Its signing identifier alone proves +/// nothing, since anyone can sign anything with it. +fn studio(pid: i32) -> bool { + /// Take over a reference a Security create or copy function handed out. + fn owned(object: *const c_void) -> Option> { + NonNull::new(object.cast_mut().cast::()).map(|object| unsafe { CFRetained::from_raw(object) }) + } + + let requirement = CFString::from_str(&format!( + "identifier \"{APP}\" and anchor apple generic and certificate leaf[subject.OU] = \"{TEAM}\"" + )); + let pid = CFNumber::new_i32(pid); + let attributes = CFDictionary::::from_slices(&[unsafe { kSecGuestAttributePid }], &[&pid]); + let mut code = std::ptr::null(); + let mut compiled = std::ptr::null(); + let status = unsafe { SecCodeCopyGuestWithAttributes(std::ptr::null(), attributes.as_opaque(), 0, &mut code) }; + let code = owned(code).filter(|_| status == 0); + let status = unsafe { SecRequirementCreateWithString(&requirement, 0, &mut compiled) }; + let compiled = owned(compiled).filter(|_| status == 0); + let (Some(code), Some(compiled)) = (code, compiled) else { + return false; + }; + let code: *const CFType = &*code; + let compiled: *const CFType = &*compiled; + unsafe { SecCodeCheckValidity(code.cast(), 0, compiled.cast()) == 0 } +} + +/// An error for a client, saying `message`. +fn error(message: &str) -> Retained { + log(message); + let message = NSString::from_str(message); + let info = NSDictionary::from_slices( + &[unsafe { NSLocalizedDescriptionKey }], + &[message.as_ref() as &AnyObject], + ); + unsafe { NSError::errorWithDomain_code_userInfo(ns_string!("live.kixel.studio.camera"), 1, Some(&info)) } +} + +/// Write `err` to an Objective-C error out parameter, if the caller gave one. +fn fail(out: *mut *mut NSError, err: Retained) { + if let Some(out) = NonNull::new(out) { + unsafe { out.write(Retained::autorelease_return(err)) }; + } +} + +fn set_of(properties: &[&NSString]) -> Retained> { + NSSet::from_slice(properties) +} + +fn state(value: &AnyObject) -> Retained { + unsafe { CMIOExtensionPropertyState::propertyStateWithValue(Some(value)) } +} + +struct ProviderIvars { + provider: OnceLock>, + /// Every camera, the program's first. + cameras: Mutex>>, + /// The device id of the scene camera added last that no client feeds yet. + /// Any app can write [`CONFIG`], so a new ask removes it: one never holds + /// more than one camera the studio does not feed. + pending: Mutex>, +} + +define_class!( + #[unsafe(super(NSObject))] + #[name = "MoqStudioCameraProvider"] + #[ivars = ProviderIvars] + struct Provider; + + unsafe impl NSObjectProtocol for Provider {} + + unsafe impl CMIOExtensionProviderSource for Provider { + #[unsafe(method(connectClient:error:))] + fn connect_client(&self, _client: &CMIOExtensionClient, _error: *mut *mut NSError) -> Bool { + Bool::YES + } + + #[unsafe(method(disconnectClient:))] + fn disconnect_client(&self, _client: &CMIOExtensionClient) {} + + #[unsafe(method_id(availableProperties))] + fn available_properties(&self) -> Retained> { + set_of(&[unsafe { CMIOExtensionPropertyProviderName }, unsafe { + CMIOExtensionPropertyProviderManufacturer + }]) + } + + #[unsafe(method_id(providerPropertiesForProperties:error:))] + fn provider_properties( + &self, + properties: &NSSet, + _error: *mut *mut NSError, + ) -> Retained { + let result = + unsafe { CMIOExtensionProviderProperties::providerPropertiesWithDictionary(&NSDictionary::new()) }; + let name = NSString::from_str(moq_studio_camera::NAME); + unsafe { + if properties.containsObject(CMIOExtensionPropertyProviderName) { + result.setName(Some(&name)); + } + if properties.containsObject(CMIOExtensionPropertyProviderManufacturer) { + result.setManufacturer(Some(&name)); + } + } + result + } + + #[unsafe(method(setProviderProperties:error:))] + fn set_provider_properties( + &self, + _properties: &CMIOExtensionProviderProperties, + _error: *mut *mut NSError, + ) -> Bool { + // Both are read only. + Bool::YES + } + } +); + +impl Provider { + fn new() -> Retained { + let this = Self::alloc().set_ivars(ProviderIvars { + provider: OnceLock::new(), + cameras: Mutex::new(Vec::new()), + pending: Mutex::new(None), + }); + unsafe { msg_send![super(this), init] } + } + + /// Remember `provider` and add the program's camera to it. + fn start(&self, provider: &CMIOExtensionProvider) { + let _ = self.ivars().provider.set(Weak::from(provider)); + if let Err(err) = self.add(Config::program()) { + // Without it the studio has nothing to ask through, so the + // extension has nothing to serve. + log(&format!("cannot add the program's camera: {err}")); + std::process::exit(1); + } + } + + fn provider(&self) -> Option> { + self.ivars().provider.get().and_then(Weak::load) + } + + /// Show `config` on a camera of its own, replacing the camera with its + /// device id if that shows something else and the studio does not feed + /// it. Called on the provider's queue, never from inside a property + /// setter of the camera it replaces. + fn ask(&self, config: Config) { + let id = config.device(); + let replaced: Vec<_> = { + let cameras = self.ivars().cameras.lock().unwrap(); + if cameras.iter().any(|camera| camera.ivars().config == config) { + return; + } + let pending = self.ivars().pending.lock().unwrap().clone(); + let replaced: Vec<_> = cameras + .iter() + .filter(|camera| { + let other = camera.ivars().config.device(); + other == id || (config.scene.is_some() && pending.as_ref() == Some(&other)) + }) + .cloned() + .collect(); + // Any app may write the config, so none may pull a camera out from + // under the studio's feed. The studio's own reopen stops its sink + // before it asks, so this refuses only someone else's ask. + if replaced.iter().any(|camera| camera.fed()) { + log(&format!("refused to replace a fed camera with {config}")); + return; + } + replaced + }; + for camera in replaced { + self.remove(&camera); + } + match self.add(config.clone()) { + Ok(()) if config.scene.is_some() => *self.ivars().pending.lock().unwrap() = Some(id), + Ok(()) => {} + Err(err) => log(&format!("cannot add a camera for {config}: {err}")), + } + } + + fn add(&self, config: Config) -> Result<(), String> { + let provider = self.provider().ok_or("the provider is gone")?; + let camera = Camera::new(config, self)?; + let device = camera.ivars().device.get().expect("set by Camera::new"); + unsafe { provider.addDevice_error(device) }.map_err(|err| err.to_string())?; + self.ivars().cameras.lock().unwrap().push(camera); + Ok(()) + } + + fn remove(&self, camera: &Camera) { + let id = camera.ivars().config.device(); + if let Some(provider) = self.provider() + && let Some(device) = camera.ivars().device.get() + && let Err(err) = unsafe { provider.removeDevice_error(device) } + { + log(&format!("cannot remove the camera {id}: {err}")); + } + self.ivars() + .cameras + .lock() + .unwrap() + .retain(|other| !std::ptr::eq(&**other, camera)); + let mut pending = self.ivars().pending.lock().unwrap(); + if pending.as_ref() == Some(&id) { + *pending = None; + } + } + + /// The studio started feeding `camera`. + fn fed(&self, camera: &Camera) { + let mut pending = self.ivars().pending.lock().unwrap(); + if pending.as_ref() == Some(&camera.ivars().config.device()) { + *pending = None; + } + } + + /// The studio stopped feeding `camera`, having turned it off or quit: a + /// scene's camera goes with it. The program's stays, so apps keep it. + fn starved(&self, camera: &Camera) { + if camera.ivars().config.scene.is_some() { + self.remove(camera); + } + } + + /// Run `work` on the provider's queue, where CoreMediaIO calls every + /// source, after the call in progress returns. + fn later(&self, work: impl FnOnce(&Provider) + 'static) { + let Some(provider) = self.provider() else { return }; + let queue = unsafe { provider.clientQueue() }; + let this = Queued((self.retain(), work)); + queue.exec_async(move || { + let (this, work) = this.take(); + work(&this); + }); + } +} + +struct CameraIvars { + config: Config, + provider: Weak, + device: OnceLock>, + /// What apps read. + source: OnceLock<(Retained, Retained)>, + /// What the studio feeds. + sink: OnceLock<(Retained, Retained)>, + /// How many apps read the source stream. + readers: AtomicUsize, +} + +define_class!( + #[unsafe(super(NSObject))] + #[name = "MoqStudioCameraDevice"] + #[ivars = CameraIvars] + struct Camera; + + unsafe impl NSObjectProtocol for Camera {} + + unsafe impl CMIOExtensionDeviceSource for Camera { + #[unsafe(method_id(availableProperties))] + fn available_properties(&self) -> Retained> { + let config = NSString::from_str(&CONFIG.name()); + let reading = NSString::from_str(&READING.name()); + set_of(&[ + unsafe { CMIOExtensionPropertyDeviceTransportType }, + unsafe { CMIOExtensionPropertyDeviceModel }, + &config, + &reading, + ]) + } + + #[unsafe(method_id(devicePropertiesForProperties:error:))] + fn device_properties( + &self, + properties: &NSSet, + _error: *mut *mut NSError, + ) -> Retained { + let result = unsafe { CMIOExtensionDeviceProperties::devicePropertiesWithDictionary(&NSDictionary::new()) }; + unsafe { + if properties.containsObject(CMIOExtensionPropertyDeviceTransportType) { + result.setTransportType(Some(&NSNumber::new_u32(VIRTUAL))); + } + if properties.containsObject(CMIOExtensionPropertyDeviceModel) { + result.setModel(Some(ns_string!("kixel.live Studio"))); + } + } + let config = NSString::from_str(&CONFIG.name()); + if properties.containsObject(&config) { + let value = NSString::from_str(&self.ivars().config.to_string()); + unsafe { result.setPropertyState_forProperty(Some(&state(&value)), &config) }; + } + let reading = NSString::from_str(&READING.name()); + if properties.containsObject(&reading) { + unsafe { result.setPropertyState_forProperty(Some(&state(&self.reading_value())), &reading) }; + } + result + } + + #[unsafe(method(setDeviceProperties:error:))] + fn set_device_properties(&self, properties: &CMIOExtensionDeviceProperties, error: *mut *mut NSError) -> Bool { + let key = NSString::from_str(&CONFIG.name()); + let Some(asked) = unsafe { properties.propertiesDictionary() }.objectForKey(&key) else { + // The rest are read only. + return Bool::YES; + }; + let config = unsafe { asked.value() } + .and_then(|value| value.downcast::().ok()) + .ok_or_else(|| "a camera config is text".to_string()) + .and_then(|text| text.to_string().parse::().map_err(|err| err.to_string())) + .and_then(|config| match self.ivars().config.scene { + // Only the program's camera, which is always there, takes asks. + None => Ok(config), + Some(_) => Err("only the program's camera takes a camera config".to_string()), + }); + match (config, self.ivars().provider.load()) { + (Ok(config), Some(provider)) => { + // Replacing this very camera from inside its own setter + // would pull the device out from under the call. + provider.later(move |provider| provider.ask(config)); + Bool::YES + } + (Err(err), _) => { + fail(error, self::error(&err)); + Bool::NO + } + (_, None) => { + fail(error, self::error("the provider is gone")); + Bool::NO + } + } + } + } +); + +impl Camera { + /// A camera showing `config`, with its streams, not yet added. + fn new(config: Config, provider: &Provider) -> Result, String> { + let format = format(&config)?; + let id = NSUUID::from_string(&NSString::from_str(&config.device())).ok_or("a bad device id")?; + let name = NSString::from_str(&config.name()); + let framerate = config.framerate; + let this = Self::alloc().set_ivars(CameraIvars { + config, + provider: Weak::from(provider), + device: OnceLock::new(), + source: OnceLock::new(), + sink: OnceLock::new(), + readers: AtomicUsize::new(0), + }); + let this: Retained = unsafe { msg_send![super(this), init] }; + let device = unsafe { + CMIOExtensionDevice::deviceWithLocalizedName_deviceID_source(&name, &id, ProtocolObject::from_ref(&*this)) + }; + for direction in [CMIOExtensionStreamDirection::Source, CMIOExtensionStreamDirection::Sink] { + let sink = direction == CMIOExtensionStreamDirection::Sink; + let source = Stream::new(&this, sink, format.clone(), framerate); + let label = if sink { + ns_string!("kixel.live Studio sink") + } else { + &name + }; + let stream = unsafe { + CMIOExtensionStream::streamWithLocalizedName_streamID_direction_clockType_source( + label, + &NSUUID::UUID(), + direction, + CMIOExtensionStreamClockType::HostTime, + ProtocolObject::from_ref(&*source), + ) + }; + unsafe { device.addStream_error(&stream) }.map_err(|err| err.to_string())?; + let slot = if sink { &this.ivars().sink } else { &this.ivars().source }; + let _ = slot.set((stream, source)); + } + let _ = this.ivars().device.set(device); + Ok(this) + } + + /// Whether the studio feeds the camera. + fn fed(&self) -> bool { + self.ivars() + .sink + .get() + .is_some_and(|(_, stream)| stream.ivars().running.load(Ordering::Relaxed)) + } + + /// Whether an app reads the camera. + fn reading(&self) -> bool { + self.ivars().readers.load(Ordering::Relaxed) > 0 + } + + fn reading_value(&self) -> Retained { + NSString::from_str(if self.reading() { "1" } else { "0" }) + } + + /// Count an app starting to read the camera, or stopping, and say so + /// when the first starts or the last stops, so the studio copies + /// pictures only while any app reads. + fn read(&self, starts: bool) { + let readers = &self.ivars().readers; + let changed = match starts { + true => readers.fetch_add(1, Ordering::Relaxed) == 0, + false => readers + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |count| count.checked_sub(1)) + .is_ok_and(|count| count == 1), + }; + let Some(device) = self.ivars().device.get().filter(|_| changed) else { + return; + }; + let key = NSString::from_str(&READING.name()); + let changed = NSDictionary::from_slices(&[&*key], &[&*state(&self.reading_value())]); + unsafe { device.notifyPropertiesChanged(&changed) }; + } +} + +/// What apps read a camera as: 32-bit BGRA at its size and frame rate, the +/// format every app takes and the studio's RGBA becomes with a swizzle. +fn format(config: &Config) -> Result, String> { + let mut description: *const CMFormatDescription = std::ptr::null(); + let status = unsafe { + CMVideoFormatDescriptionCreate( + None, + kCVPixelFormatType_32BGRA, + config.width as i32, + config.height as i32, + None, + NonNull::from(&mut description), + ) + }; + let description = NonNull::new(description.cast_mut()) + .filter(|_| status == 0) + .ok_or_else(|| format!("cannot describe {config}: {status}"))?; + // SAFETY: a create function hands over its reference. + let description = unsafe { objc2_core_foundation::CFRetained::from_raw(description) }; + let duration = frame(config.framerate); + Ok(unsafe { + CMIOExtensionStreamFormat::streamFormatWithFormatDescription_maxFrameDuration_minFrameDuration_validFrameDurations( + &description, + duration, + duration, + None, + ) + }) +} + +fn frame(framerate: u32) -> CMTime { + unsafe { CMTime::new(1, framerate as i32) } +} + +struct StreamIvars { + camera: Weak, + /// Whether the studio feeds it, rather than apps reading it. + sink: bool, + format: Retained, + framerate: u32, + /// The studio, once it is allowed to feed the sink. + client: Mutex>>, + /// Whether the sink takes pictures. + running: AtomicBool, +} + +define_class!( + #[unsafe(super(NSObject))] + #[name = "MoqStudioCameraStream"] + #[ivars = StreamIvars] + struct Stream; + + unsafe impl NSObjectProtocol for Stream {} + + unsafe impl CMIOExtensionStreamSource for Stream { + #[unsafe(method_id(formats))] + fn formats(&self) -> Retained> { + NSArray::from_retained_slice(&[self.ivars().format.clone()]) + } + + #[unsafe(method_id(availableProperties))] + fn available_properties(&self) -> Retained> { + unsafe { + let mut properties = vec![ + CMIOExtensionPropertyStreamActiveFormatIndex, + CMIOExtensionPropertyStreamFrameDuration, + ]; + if self.ivars().sink { + properties.extend([ + CMIOExtensionPropertyStreamSinkBufferQueueSize, + CMIOExtensionPropertyStreamSinkBuffersRequiredForStartup, + CMIOExtensionPropertyStreamSinkBufferUnderrunCount, + CMIOExtensionPropertyStreamSinkEndOfData, + ]); + } + set_of(&properties) + } + } + + #[unsafe(method_id(streamPropertiesForProperties:error:))] + fn stream_properties( + &self, + properties: &NSSet, + _error: *mut *mut NSError, + ) -> Retained { + let result = unsafe { CMIOExtensionStreamProperties::streamPropertiesWithDictionary(&NSDictionary::new()) }; + unsafe { + if properties.containsObject(CMIOExtensionPropertyStreamActiveFormatIndex) { + result.setActiveFormatIndex(Some(&NSNumber::new_u32(0))); + } + if properties.containsObject(CMIOExtensionPropertyStreamFrameDuration) + && let Some(duration) = frame(self.ivars().framerate).as_dictionary(None) + { + // Toll-free bridged. + let duration: &NSDictionary = &*(&*duration as *const _ as *const NSDictionary); + result.setFrameDuration(Some(duration)); + } + if properties.containsObject(CMIOExtensionPropertyStreamSinkBufferQueueSize) { + result.setSinkBufferQueueSize(Some(&NSNumber::new_u32(SINK_QUEUE))); + } + if properties.containsObject(CMIOExtensionPropertyStreamSinkBuffersRequiredForStartup) { + result.setSinkBuffersRequiredForStartup(Some(&NSNumber::new_u32(1))); + } + if properties.containsObject(CMIOExtensionPropertyStreamSinkBufferUnderrunCount) { + result.setSinkBufferUnderrunCount(Some(&NSNumber::new_u32(0))); + } + if properties.containsObject(CMIOExtensionPropertyStreamSinkEndOfData) { + result.setSinkEndOfData(Some(&NSNumber::new_u32(0))); + } + } + result + } + + #[unsafe(method(setStreamProperties:error:))] + fn set_stream_properties( + &self, + _properties: &CMIOExtensionStreamProperties, + _error: *mut *mut NSError, + ) -> Bool { + // One format at one frame rate: there is nothing else to pick. + Bool::YES + } + + #[unsafe(method(authorizedToStartStreamForClient:))] + fn authorized(&self, client: &CMIOExtensionClient) -> Bool { + if !self.ivars().sink { + return Bool::YES; + } + // Only the studio feeds a camera, so no other app shows its + // pictures under the studio's name. + let studio = studio(unsafe { client.pid() }); + if studio { + *self.ivars().client.lock().unwrap() = Some(client.retain()); + } else { + log(&format!("refused to let pid {} feed a camera", unsafe { client.pid() })); + } + Bool::new(studio) + } + + #[unsafe(method(startStreamAndReturnError:))] + fn start_stream(&self, _error: *mut *mut NSError) -> Bool { + let Some(camera) = self.ivars().camera.load() else { + return Bool::YES; + }; + if self.ivars().sink { + self.ivars().running.store(true, Ordering::Relaxed); + if let Some(provider) = camera.ivars().provider.load() { + provider.fed(&camera); + } + self.consume(); + } else { + camera.read(true); + } + Bool::YES + } + + #[unsafe(method(stopStreamAndReturnError:))] + fn stop_stream(&self, _error: *mut *mut NSError) -> Bool { + let Some(camera) = self.ivars().camera.load() else { + return Bool::YES; + }; + if self.ivars().sink { + self.ivars().running.store(false, Ordering::Relaxed); + self.ivars().client.lock().unwrap().take(); + if let Some(provider) = camera.ivars().provider.load() { + // Removing the camera stops this stream, so not from inside + // its own stop. + provider.later(move |provider| provider.starved(&camera)); + } + } else { + camera.read(false); + } + Bool::YES + } + } +); + +impl Stream { + fn new(camera: &Camera, sink: bool, format: Retained, framerate: u32) -> Retained { + let this = Self::alloc().set_ivars(StreamIvars { + camera: Weak::from(camera), + sink, + format, + framerate, + client: Mutex::new(None), + running: AtomicBool::new(false), + }); + unsafe { msg_send![super(this), init] } + } + + /// Take the studio's next picture off the sink and show it on the + /// source, then take the one after, until the sink stops. + fn consume(&self) { + if !self.ivars().running.load(Ordering::Relaxed) { + return; + } + let Some(client) = self.ivars().client.lock().unwrap().clone() else { + return; + }; + let Some(camera) = self.ivars().camera.load() else { + return; + }; + let Some((sink, _)) = camera.ivars().sink.get() else { + return; + }; + let this = Weak::from(self); + let framerate = self.ivars().framerate; + let done = RcBlock::new( + move |picture: *mut CMSampleBuffer, + sequence: u64, + discontinuity: CMIOExtensionStreamDiscontinuityFlags, + _more: Bool, + _error: *mut NSError| { + let Some(this) = this.load() else { return }; + match unsafe { picture.as_ref() } { + Some(picture) => { + this.forward(picture, sequence, discontinuity); + this.consume(); + } + None => { + // Nothing to take yet: look again a frame later + // rather than spin. + let this = Queued(this); + let when = DispatchTime::NOW.time(1_000_000_000 / i64::from(framerate)); + let _ = DispatchQueue::global_queue(GlobalQueueIdentifier::Priority( + DispatchQueueGlobalPriority::Default, + )) + .after(when, move || { + this.take().consume(); + }); + } + } + }, + ); + unsafe { sink.consumeSampleBufferFromClient_completionHandler(&client, &done) }; + } + + /// Show `picture`, sink sample `sequence`, to the apps reading the camera. + fn forward(&self, picture: &CMSampleBuffer, sequence: u64, discontinuity: CMIOExtensionStreamDiscontinuityFlags) { + let Some(camera) = self.ivars().camera.load() else { + return; + }; + let (Some((source, _)), Some((sink, _))) = (camera.ivars().source.get(), camera.ivars().sink.get()) else { + return; + }; + let now = now(); + if camera.reading() { + unsafe { source.sendSampleBuffer_discontinuity_hostTimeInNanoseconds(picture, discontinuity, now) }; + } + let output = unsafe { + CMIOExtensionScheduledOutput::scheduledOutputWithSequenceNumber_hostTimeInNanoseconds(sequence, now) + }; + unsafe { sink.notifyScheduledOutputChanged(&output) }; + } +} diff --git a/rs/moq-studio-camera/src/lib.rs b/rs/moq-studio-camera/src/lib.rs new file mode 100644 index 00000000..e969e64c --- /dev/null +++ b/rs/moq-studio-camera/src/lib.rs @@ -0,0 +1,277 @@ +//! What the studio and its macOS camera extension agree on. +//! +//! The extension (this crate's binary) is a CoreMediaIO Camera Extension the +//! app bundle carries. It always lists one camera, the program's, and adds a +//! camera for a published scene when the studio asks. Each camera has a source +//! stream, which apps read, and a sink stream, which only the studio may feed: +//! the extension passes each picture the studio puts on the sink to the source. +//! +//! The studio talks to the extension through two custom properties on a +//! camera, the only channel CoreMediaIO offers a sandboxed extension: +//! [`CONFIG`], which reads as what the camera shows and, written on the +//! program's camera, asks for a camera; and [`READING`], whether an app reads +//! the camera, so the studio copies pictures only then. + +use std::fmt; +use std::str::FromStr; + +/// The app's bundle identifier, the only client allowed to feed a camera. +pub const APP: &str = "live.kixel.studio"; + +/// The team that signs the app and the extension, which names the extension's +/// Mach service and app group (`packaging/macos/Camera.plist` and +/// `camera.entitlements.plist`). +pub const TEAM: &str = "D7D5SDDB5Z"; + +/// The extension's bundle identifier, which the app asks the system to +/// activate. +pub const EXTENSION: &str = "live.kixel.studio.camera"; + +/// What apps list the program's camera as. A scene's camera adds `: `. +pub const NAME: &str = "kixel.live Studio"; + +/// The program camera's device id, fixed so apps remember it across launches +/// and updates. +pub const PROGRAM: &str = "6B1D0F4E-3C52-4E8B-9A57-2D0C1F6E8A10"; + +/// The custom property holding a camera's [`Config`] as text. +pub const CONFIG: Property = Property(*b"kcfg"); + +/// The custom property saying whether an app reads the camera: `1` or `0`. +pub const READING: Property = Property(*b"krdg"); + +/// A custom CoreMediaIO property, named by its four-character selector. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Property(pub [u8; 4]); + +impl Property { + /// The selector the studio addresses it by through the CoreMediaIO + /// hardware API. + pub fn selector(self) -> u32 { + u32::from_be_bytes(self.0) + } + + /// The name the extension declares it by: a global property of the + /// object it is on, as CoreMediaIO spells custom properties. + pub fn name(self) -> String { + format!("4cc_{}_glob_0000", String::from_utf8_lossy(&self.0)) + } +} + +/// The largest picture a camera shows on either side: 8K. +const MAX_SIZE: u32 = 8192; + +/// The highest frame rate a camera shows. +const MAX_FRAMERATE: u32 = 240; + +/// The longest scene name a camera is named after, in bytes. +const MAX_SCENE: usize = 256; + +/// What a camera shows: the program or a scene, at a size and frame rate. +/// +/// As text, `1920x1080@30` for the program and `1920x1080@30 Scene` for the +/// scene called `Scene`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Config { + /// The published scene, or `None` for the program. + pub scene: Option, + pub width: u32, + pub height: u32, + pub framerate: u32, +} + +impl Config { + /// The program at 1080p30, what its camera shows before the studio asks + /// for anything else. + pub fn program() -> Self { + Self { + scene: None, + width: 1920, + height: 1080, + framerate: 30, + } + } + + /// The device id of the camera showing it: [`PROGRAM`], or one derived + /// from the scene's name, so an app remembers a scene's camera too. + pub fn device(&self) -> String { + match &self.scene { + None => PROGRAM.to_string(), + Some(scene) => scene_device(scene), + } + } + + /// What apps list the camera as. + pub fn name(&self) -> String { + match &self.scene { + None => NAME.to_string(), + Some(scene) => format!("{NAME}: {scene}"), + } + } +} + +impl fmt::Display for Config { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}x{}@{}", self.width, self.height, self.framerate)?; + if let Some(scene) = &self.scene { + write!(f, " {scene}")?; + } + Ok(()) + } +} + +/// Why a [`Config`]'s text was refused. +#[derive(Debug, PartialEq, Eq)] +pub struct Invalid(String); + +impl fmt::Display for Invalid { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "invalid camera config {:?}", self.0) + } +} + +impl std::error::Error for Invalid {} + +impl FromStr for Config { + type Err = Invalid; + + /// Parses what [`Display`](fmt::Display) writes. Any app can write the + /// property, so this refuses anything the studio would not ask for. + fn from_str(text: &str) -> Result { + let invalid = || Invalid(text.chars().take(64).collect()); + let (format, scene) = match text.split_once(' ') { + Some((format, scene)) => (format, Some(scene)), + None => (text, None), + }; + let (size, framerate) = format.split_once('@').ok_or_else(invalid)?; + let (width, height) = size.split_once('x').ok_or_else(invalid)?; + let number = |text: &str, max: u32| { + // `parse` takes a leading `+`, which `Display` never writes. + text.bytes() + .all(|byte| byte.is_ascii_digit()) + .then(|| text.parse::().ok()) + .flatten() + .filter(|&value| (1..=max).contains(&value)) + .ok_or_else(invalid) + }; + let scene = match scene { + Some(scene) if scene.is_empty() || scene.len() > MAX_SCENE || scene.chars().any(char::is_control) => { + return Err(invalid()); + } + scene => scene.map(str::to_string), + }; + Ok(Self { + scene, + width: number(width, MAX_SIZE)?, + height: number(height, MAX_SIZE)?, + framerate: number(framerate, MAX_FRAMERATE)?, + }) + } +} + +/// A scene camera's device id: a UUID (version 8, custom) from the 128-bit +/// FNV-1a hash of its name. Never change it: apps remember cameras by it. +fn scene_device(scene: &str) -> String { + const OFFSET: u128 = 0x6c62272e07bb014262b821756295c58d; + const PRIME: u128 = 0x0000000001000000000000000000013b; + let mut hash = OFFSET; + for byte in scene.bytes() { + hash ^= u128::from(byte); + hash = hash.wrapping_mul(PRIME); + } + // The version and variant bits RFC 9562 puts in every UUID. + hash = (hash & !(0xf << 76)) | (0x8 << 76); + hash = (hash & !(0x3 << 62)) | (0x2 << 62); + let hex = format!("{hash:032X}"); + format!( + "{}-{}-{}-{}-{}", + &hex[..8], + &hex[8..12], + &hex[12..16], + &hex[16..20], + &hex[20..] + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn configs_round_trip() { + for text in ["1920x1080@30", "1280x720@60 Main", "1x1@1 a scene: with spaces"] { + let config: Config = text.parse().unwrap(); + assert_eq!(config.to_string(), text); + } + let config: Config = "640x480@25 Interview".parse().unwrap(); + assert_eq!( + config, + Config { + scene: Some("Interview".to_string()), + width: 640, + height: 480, + framerate: 25, + } + ); + } + + #[test] + fn refuses_what_the_studio_never_asks_for() { + for text in [ + "", + "1920x1080", + "1920@30", + "0x1080@30", + "1920x0@30", + "1920x1080@0", + "8193x1080@30", + "1920x1080@241", + "+1920x1080@30", + "1920x1080@30 ", + "1920x1080@30 a\nb", + "99999999999999999999x1080@30", + ] { + assert!(text.parse::().is_err(), "{text:?}"); + } + let long = format!("1920x1080@30 {}", "a".repeat(MAX_SCENE + 1)); + assert!(long.parse::().is_err()); + } + + #[test] + fn scene_cameras_keep_their_ids() { + let scene = |name: &str| Config { + scene: Some(name.to_string()), + ..Config::program() + }; + // Pinned: apps remember a camera by its id, so it must never change. + assert_eq!(scene("Main").device(), "6828B994-E175-8277-B806-E911DE48E2B8"); + assert_eq!(scene("Main").device(), scene("Main").device()); + assert_ne!(scene("Main").device(), scene("main").device()); + assert_eq!(Config::program().device(), PROGRAM); + for id in [scene("Main").device(), scene("").device(), PROGRAM.to_string()] { + let groups: Vec<_> = id.split('-').map(str::len).collect(); + assert_eq!(groups, [8, 4, 4, 4, 12], "{id}"); + assert!( + id.chars() + .all(|c| c == '-' || c.is_ascii_hexdigit() && !c.is_ascii_lowercase()) + ); + } + } + + #[test] + fn the_bundle_names_the_same_team() { + let service = format!("{TEAM}.{EXTENSION}"); + for file in ["Camera.plist", "camera.entitlements.plist"] { + let path = format!("{}/../../packaging/macos/{file}", env!("CARGO_MANIFEST_DIR")); + let plist = std::fs::read_to_string(&path).unwrap(); + assert!(plist.contains(&service), "{file} does not name {service}"); + } + } + + #[test] + fn names_properties_as_coremediaio_does() { + assert_eq!(CONFIG.name(), "4cc_kcfg_glob_0000"); + assert_eq!(CONFIG.selector(), 0x6b636667); + assert_eq!(READING.name(), "4cc_krdg_glob_0000"); + } +} diff --git a/rs/moq-studio-camera/src/main.rs b/rs/moq-studio-camera/src/main.rs new file mode 100644 index 00000000..c7826556 --- /dev/null +++ b/rs/moq-studio-camera/src/main.rs @@ -0,0 +1,16 @@ +//! The camera extension in the macOS app bundle. The system's camera service +//! starts it once the user allows it, and apps list the cameras it adds. + +#[cfg(target_os = "macos")] +mod extension; + +#[cfg(target_os = "macos")] +fn main() { + extension::run() +} + +#[cfg(not(target_os = "macos"))] +fn main() -> std::process::ExitCode { + eprintln!("moq-studio-camera is a macOS camera extension"); + std::process::ExitCode::FAILURE +} diff --git a/rs/moq-studio/Cargo.toml b/rs/moq-studio/Cargo.toml index 3306f6a8..a71632ba 100644 --- a/rs/moq-studio/Cargo.toml +++ b/rs/moq-studio/Cargo.toml @@ -157,6 +157,19 @@ objc2-app-kit = { version = "0.3", default-features = false, features = [ ] } # Relay tokens in the login keychain. apple-native-keyring-store = { version = "1.0", features = ["keychain"] } +# The virtual camera: activating the camera extension the app bundle carries, +# then feeding its cameras through CoreMediaIO. The releases moq-video's +# capture builds on. +block2 = "0.6" +dispatch2 = "0.3" +moq-studio-camera = { workspace = true } +objc2 = "0.6" +objc2-core-foundation = "0.3" +objc2-core-media = "0.3" +objc2-core-media-io = "0.3" +objc2-core-video = "0.3" +objc2-foundation = { version = "0.3", features = ["NSBundle"] } +objc2-system-extensions = "0.3" [target.'cfg(windows)'.dependencies] # Which virtual key the foreground window's layout puts on each key, so a diff --git a/rs/moq-studio/src/devices.rs b/rs/moq-studio/src/devices.rs index fae24c54..69d4d392 100644 --- a/rs/moq-studio/src/devices.rs +++ b/rs/moq-studio/src/devices.rs @@ -55,7 +55,19 @@ async fn find() -> Listing { .map_err(|err| err.to_string()) } - let cameras = picks(moq_video::capture::cameras().await, |camera| Pick { + let cameras = moq_video::capture::cameras().await; + // Never the program's virtual camera, which the camera extension lists + // even while it is off, and which setup or Add would feed what it shows + // back into. Every other one of the studio's is listed only while on, + // when Add leaves it out. + #[cfg(target_os = "macos")] + let cameras = cameras.map(|cameras| { + cameras + .into_iter() + .filter(|camera| camera.id != moq_studio_camera::PROGRAM) + .collect::>() + }); + let cameras = picks(cameras, |camera| Pick { name: camera.name, source: Source::Camera { id: Some(camera.id) }, }); diff --git a/rs/moq-studio/src/engine.rs b/rs/moq-studio/src/engine.rs index 40ab9093..ef07ab1c 100644 --- a/rs/moq-studio/src/engine.rs +++ b/rs/moq-studio/src/engine.rs @@ -1760,8 +1760,9 @@ impl Engine { /// Turn the program's virtual camera, or the published scene `scene`'s, /// on or off. Turning it on fails, saying why, when the scene is not - /// published or this machine has no way to publish a camera. A scene's - /// camera turns off once the scene is no longer published under its name. + /// published; the camera's status says why when this machine has no way + /// to publish one. A scene's camera turns off once the scene is no longer + /// published under its name. pub fn set_camera(&self, scene: Option<&str>, on: bool) -> anyhow::Result<()> { let canvas = { let state = self.state.lock().unwrap(); @@ -6807,7 +6808,19 @@ mod tests { (None, "pipewire:moq-studio"), (Some("tall"), "pipewire:moq-studio-tall"), ] { - let status = engine.state.lock().unwrap().camera(scene).expect("a camera"); + // The `vcam` thread opens the backend. + let status = kio::clock::timeout(TIMEOUT, async { + loop { + let status = engine.state.lock().unwrap().camera(scene).expect("a camera"); + if !status.backend.is_empty() || status.error.is_some() { + return status; + } + kio::clock::sleep(Duration::from_millis(100)).await; + } + }) + .await + .expect("the camera opens"); + assert_eq!(status.error, None); assert_eq!(status.backend, "PipeWire"); assert_eq!(status.id, id); assert!(!status.reading, "nothing reads the camera yet"); diff --git a/rs/moq-studio/src/ui/settings.rs b/rs/moq-studio/src/ui/settings.rs index 2d9eda42..00e8b2ae 100644 --- a/rs/moq-studio/src/ui/settings.rs +++ b/rs/moq-studio/src/ui/settings.rs @@ -1440,6 +1440,10 @@ impl Panels { ), }; let help = match &camera { + Some(camera) if camera.backend.is_empty() => match camera.error { + Some(_) => "It could not open.".to_string(), + None => "Opening the camera...".to_string(), + }, Some(camera) => format!( "{} at {}x{}, {} fps. {}; {} frames sent, {} dropped.", camera.backend, diff --git a/rs/moq-studio/src/vcam.rs b/rs/moq-studio/src/vcam.rs index 0b712373..1d920293 100644 --- a/rs/moq-studio/src/vcam.rs +++ b/rs/moq-studio/src/vcam.rs @@ -4,7 +4,8 @@ //! On Linux the camera is a PipeWire video node (with the `pipewire` feature), //! which PipeWire-aware apps such as browsers list as a camera. Without //! PipeWire it is an existing v4l2loopback device, since that kernel module -//! needs root to load and so is never the first choice. +//! needs root to load and so is never the first choice. On macOS it is a +//! camera of the CoreMediaIO camera extension the app bundle carries. //! //! The program clock offers every frame it draws of what a camera shows; //! [`Camera::offer`] has moq-video read it back only while an app reads the @@ -12,12 +13,14 @@ //! the backend. use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{Arc, Mutex, mpsc}; +use std::sync::{Arc, Condvar, Mutex, mpsc}; use anyhow::Context; use moq_scene::Canvas; use moq_video::render::readback::{Pending, Readback}; +#[cfg(target_os = "macos")] +mod macos; #[cfg(all(target_os = "linux", feature = "pipewire"))] mod pipewire; #[cfg(target_os = "linux")] @@ -25,7 +28,7 @@ mod v4l2; /// The program, or a published scene, as a camera other apps can open. /// -/// Dropping it removes the camera. +/// Dropping it removes the camera, on its own thread. pub struct Camera { device: wgpu::Device, queue: wgpu::Queue, @@ -34,7 +37,8 @@ pub struct Camera { readback: Readback, /// `None` once dropping. work: Option>, - worker: Option>, + /// Set once the `vcam` thread, and the backend with it, has ended. + ended: Arc, shared: Arc, } @@ -86,7 +90,8 @@ impl Shared { /// Where an open camera lives. #[derive(Clone, Default)] struct Opened { - /// For the Output panel: `PipeWire`, or the v4l2loopback device's path. + /// For the Output panel: `PipeWire`, the v4l2loopback device's path, or + /// `Camera extension`. backend: String, /// The id moq-video's camera capture lists it by. id: String, @@ -96,7 +101,8 @@ struct Opened { /// How the camera is doing, for the Output panel. pub struct Status { - /// Where the camera lives, such as a PipeWire node or a `/dev/videoN` path. + /// Where the camera lives, such as a PipeWire node or a `/dev/videoN` + /// path; empty until the backend opens. pub backend: String, /// The id the Devices panel would list the camera by, so it can leave it out. pub id: String, @@ -120,8 +126,11 @@ impl Camera { /// Publish the program, or the published scene called `scene`, drawn on /// `device` at `canvas`, as a camera named after it. /// - /// Fails, naming why each backend refused, when this machine has no way to - /// publish one. + /// The `vcam` thread opens the backend, since one may wait seconds on the + /// system, as macOS installing the camera extension does, and this runs + /// on the UI's thread. Until it has, [`Status::backend`] is empty; a + /// machine with no way to publish a camera reports why each backend + /// refused in [`Status::error`]. pub fn open( device: &wgpu::Device, queue: &wgpu::Queue, @@ -131,23 +140,41 @@ impl Camera { let readback = Readback::new(device, queue, size(canvas))?; let shared = Arc::new(Shared::default()); let scene = scene.map(str::to_string); - let sink = open(scene.as_deref(), canvas, &shared)?; let (work, pending) = mpsc::channel(); - let worker = { + let ended = Arc::new(Ended::default()); + // Taken here, before this camera can be dropped itself, so it waits + // only for cameras turned off before it turned on. + let closing = { + let mut closing = CLOSING.lock().unwrap(); + closing.retain(|ended| !ended.done()); + closing.clone() + }; + { let shared = shared.clone(); + let ended = ended.clone(); std::thread::Builder::new() .name("vcam".into()) - .spawn(move || run(pending, sink, scene.as_deref(), &shared)) - .context("cannot start the virtual camera thread")? - }; + .spawn(move || { + // Set however the thread ends, a panic included. + let _ended = EndOnDrop(ended); + for camera in closing { + camera.wait(); + } + match open(scene.as_deref(), canvas, &shared) { + Ok(sink) => run(pending, sink, scene.as_deref(), &shared), + Err(err) => shared.fail(format!("{err:#}")), + } + }) + .context("cannot start the virtual camera thread")?; + } Ok(Self { device: device.clone(), queue: queue.clone(), canvas, readback, work: Some(work), - worker: Some(worker), + ended, shared, }) } @@ -207,15 +234,52 @@ impl Camera { impl Drop for Camera { fn drop(&mut self) { - // Ends the thread, which removes the camera before this returns, so - // turning it straight back on finds the device free. + // Ends the thread, which removes the camera once its backend returns. + // That may wait on the system for seconds, as macOS finding the camera + // extension's does, and this runs on the UI's thread, so the next + // camera to open waits for it instead. self.work.take(); - if let Some(worker) = self.worker.take() { - let _ = worker.join(); + CLOSING.lock().unwrap().push(self.ended.clone()); + } +} + +/// The cameras turned off whose threads may still be closing their backends. +/// A camera waits for each before it opens, on its own thread, so it finds +/// every device they held free, such as a v4l2loopback device or a scene's +/// camera of the macOS extension. A camera waits only for those turned off +/// before it turned on, so no two wait for each other. +static CLOSING: Mutex>> = Mutex::new(Vec::new()); + +/// Whether a camera's thread has ended. +#[derive(Default)] +struct Ended { + done: Mutex, + changed: Condvar, +} + +impl Ended { + fn done(&self) -> bool { + *self.done.lock().unwrap() + } + + fn wait(&self) { + let mut done = self.done.lock().unwrap(); + while !*done { + done = self.changed.wait(done).unwrap(); } } } +/// Sets [`Ended`] when dropped. +struct EndOnDrop(Arc); + +impl Drop for EndOnDrop { + fn drop(&mut self) { + *self.0.done.lock().unwrap() = true; + self.0.changed.notify_all(); + } +} + fn size(canvas: Canvas) -> moq_video::Size { moq_video::Size::new(canvas.width, canvas.height) } @@ -267,7 +331,18 @@ fn backend( } } -#[cfg(not(target_os = "linux"))] +#[cfg(target_os = "macos")] +fn backend( + scene: Option<&str>, + size: moq_video::Size, + framerate: u32, + shared: &Arc, +) -> anyhow::Result<(Box, String, String)> { + let (extension, id) = macos::Extension::open(scene, size, framerate, shared.clone())?; + Ok((Box::new(extension), "Camera extension".to_string(), id)) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] fn backend( scene: Option<&str>, size: moq_video::Size, diff --git a/rs/moq-studio/src/vcam/macos.rs b/rs/moq-studio/src/vcam/macos.rs new file mode 100644 index 00000000..c72151b8 --- /dev/null +++ b/rs/moq-studio/src/vcam/macos.rs @@ -0,0 +1,596 @@ +//! The camera through the camera extension the macOS app bundle carries. +//! +//! The first camera turned on in a run of the app asks the system to activate +//! the extension: that installs it, or the newer one an app update carries, +//! without a reboot. The user allows it once, in System Settings. The studio +//! then asks the program's camera, which the extension always lists, for the +//! camera it wants, waits for that camera to show it, and feeds the camera's +//! sink stream BGRA pictures while an app reads it. `moq_studio_camera` says +//! how the two talk. + +use std::ffi::c_void; +use std::ptr::NonNull; +use std::sync::atomic::Ordering; +use std::sync::{Arc, Mutex, mpsc}; +use std::time::Duration; + +use anyhow::Context; +use block2::RcBlock; +use dispatch2::{DispatchQueue, DispatchQueueGlobalPriority, DispatchRetained, GlobalQueueIdentifier}; +use moq_studio_camera::{CONFIG, Config, EXTENSION, PROGRAM, Property, READING}; +use moq_video::Size; +use objc2::rc::Retained; +use objc2::runtime::ProtocolObject; +use objc2::{AnyThread, DefinedClass, define_class, msg_send}; +use objc2_core_foundation::{CFDictionary, CFNumber, CFRetained, CFString, CFType}; +use objc2_core_media::{ + CMClock, CMSampleBuffer, CMSampleTimingInfo, CMSimpleQueue, CMTime, CMVideoFormatDescription, + CMVideoFormatDescriptionCreateForImageBuffer, kCMTimeInvalid, +}; +use objc2_core_media_io::{ + CMIODeviceStartStream, CMIODeviceStopStream, CMIOObjectAddPropertyListenerBlock, CMIOObjectGetPropertyData, + CMIOObjectGetPropertyDataSize, CMIOObjectID, CMIOObjectPropertyAddress, CMIOObjectRemovePropertyListenerBlock, + CMIOObjectSetPropertyData, CMIOStreamCopyBufferQueue, kCMIODevicePropertyDeviceUID, kCMIODevicePropertyStreams, + kCMIOHardwarePropertyDevices, kCMIOObjectPropertyElementMain, kCMIOObjectPropertyScopeGlobal, + kCMIOObjectSystemObject, kCMIOStreamPropertyDirection, +}; +use objc2_core_video::{ + CVPixelBuffer, CVPixelBufferGetBaseAddress, CVPixelBufferGetBytesPerRow, CVPixelBufferLockBaseAddress, + CVPixelBufferLockFlags, CVPixelBufferPool, CVPixelBufferUnlockBaseAddress, kCVPixelBufferHeightKey, + kCVPixelBufferIOSurfacePropertiesKey, kCVPixelBufferPixelFormatTypeKey, kCVPixelBufferWidthKey, + kCVPixelFormatType_32BGRA, +}; +use objc2_foundation::{NSBundle, NSError, NSObject, NSObjectProtocol, NSString}; +use objc2_system_extensions::{ + OSSystemExtensionManager, OSSystemExtensionProperties, OSSystemExtensionReplacementAction, + OSSystemExtensionRequest, OSSystemExtensionRequestDelegate, OSSystemExtensionRequestResult, +}; + +use super::{Shared, Sink}; + +/// How long activating an extension already allowed may take, which +/// replacing it with an update's includes. +const ACTIVATE: Duration = Duration::from_secs(30); + +/// How long the extension may take to list the camera asked for. +const APPEAR: Duration = Duration::from_secs(10); + +/// Where the user allows the extension. +const ALLOW: &str = "System Settings > General > Login Items & Extensions > Camera Extensions"; + +/// Whether this run of the app has activated the extension. Held while a +/// camera opens, from asking for it until feeding it: the extension keeps +/// only the camera asked for last until the studio feeds it, so two cameras +/// opening at once, as after a frame rate change, would remove each other's. +static OPENING: Mutex = Mutex::new(false); + +/// A camera of the extension, fed from the `vcam` thread. +pub struct Extension { + /// The device id apps list the camera by. + id: String, + /// Whether it feeds a scene's camera, which goes away with its sink. + scene: bool, + device: CMIOObjectID, + stream: CMIOObjectID, + /// The sink's queue: the studio enqueues, the extension dequeues. + queue: CFRetained, + pool: CFRetained, + /// Made from the first picture, as every later one matches it. + format: Option>, + size: Size, + framerate: u32, + /// Says whether an app reads the camera, on `listening`. + reading: RcBlock, + listening: DispatchRetained, + shared: Arc, +} + +// SAFETY: CoreMediaIO's, CoreMedia's, and CoreVideo's objects are reference +// counted with atomics and safe to use from any thread; the `vcam` thread is +// the only one that uses them through this. The listener block captures only +// a `Shared` and a device id, both `Send`. +unsafe impl Send for Extension {} + +impl Extension { + /// Open the extension's camera of the program, or of the scene called + /// `scene`, at `size` and `framerate`, and start feeding it, with the id + /// apps list it by. + pub fn open( + scene: Option<&str>, + size: Size, + framerate: u32, + shared: Arc, + ) -> anyhow::Result<(Self, String)> { + let mut activated = OPENING.lock().unwrap(); + if !*activated { + activate()?; + *activated = true; + } + let config = Config { + scene: scene.map(str::to_string), + width: size.width, + height: size.height, + framerate, + }; + let wanted = config.to_string(); + let id = config.device(); + + let program = appear(PROGRAM, None) + .with_context(|| format!("the kixel.live Studio camera extension is not running: allow it in {ALLOW}"))?; + set_string(program, CONFIG, &wanted).context("asking the camera extension for a camera")?; + let device = appear(&id, Some(&wanted)).context("the camera extension never listed the camera")?; + let stream = sink(device)?; + + let listening = DispatchQueue::new("live.kixel.studio.vcam", None); + let reading = { + let shared = shared.clone(); + RcBlock::new(move |_count: u32, _addresses: *mut CMIOObjectPropertyAddress| { + read(device, &shared); + }) + }; + let address = address(READING.selector()); + let status = unsafe { + CMIOObjectAddPropertyListenerBlock(device, &address, Some(&listening), RcBlock::as_ptr(&reading)) + }; + anyhow::ensure!(status == 0, "cannot listen to the camera: {status}"); + read(device, &shared); + + let mut queue: *mut CMSimpleQueue = std::ptr::null_mut(); + let status = unsafe { CMIOStreamCopyBufferQueue(stream, Some(altered), std::ptr::null_mut(), &mut queue) }; + let queue = NonNull::new(queue) + .filter(|_| status == 0) + .map(|queue| unsafe { CFRetained::from_raw(queue) }) + .with_context(|| format!("cannot reach the camera's sink: {status}"))?; + let pool = pool(size)?; + let mut this = Self { + id: id.clone(), + // Not yet: only a camera the studio fed goes away when it stops. + scene: false, + device, + stream, + queue, + pool, + format: None, + size, + framerate, + reading, + listening, + shared, + }; + let status = unsafe { CMIODeviceStartStream(device, stream) }; + anyhow::ensure!(status == 0, "cannot start feeding the camera: {status}"); + this.scene = scene.is_some(); + Ok((this, id)) + } +} + +impl Sink for Extension { + fn write(&mut self, rgba: &[u8]) -> anyhow::Result<()> { + // The extension has not taken the last pictures yet: this one would + // only wait behind them. + if unsafe { self.queue.count() >= self.queue.capacity() } { + self.shared.dropped.fetch_add(1, Ordering::Relaxed); + return Ok(()); + } + let mut buffer: *mut CVPixelBuffer = std::ptr::null_mut(); + let status = unsafe { CVPixelBufferPool::create_pixel_buffer(None, &self.pool, NonNull::from(&mut buffer)) }; + let buffer = NonNull::new(buffer) + .filter(|_| status == 0) + .map(|buffer| unsafe { CFRetained::from_raw(buffer) }) + .with_context(|| format!("cannot get a picture buffer: {status}"))?; + swizzle(&buffer, rgba, self.size)?; + + let format = match &self.format { + Some(format) => format.clone(), + None => { + let mut format: *const CMVideoFormatDescription = std::ptr::null(); + let status = + unsafe { CMVideoFormatDescriptionCreateForImageBuffer(None, &buffer, NonNull::from(&mut format)) }; + let format = NonNull::new(format.cast_mut()) + .filter(|_| status == 0) + .map(|format| unsafe { CFRetained::from_raw(format) }) + .with_context(|| format!("cannot describe the picture: {status}"))?; + self.format.insert(format).clone() + } + }; + let timing = CMSampleTimingInfo { + duration: unsafe { CMTime::new(1, self.framerate as i32) }, + presentationTimeStamp: unsafe { CMClock::host_time_clock().time() }, + decodeTimeStamp: unsafe { kCMTimeInvalid }, + }; + let mut sample: *mut CMSampleBuffer = std::ptr::null_mut(); + let status = unsafe { + CMSampleBuffer::create_for_image_buffer( + None, + &buffer, + true, + None, + std::ptr::null_mut(), + &format, + NonNull::from(&timing), + NonNull::from(&mut sample), + ) + }; + let sample = NonNull::new(sample) + .filter(|_| status == 0) + .with_context(|| format!("cannot wrap the picture: {status}"))?; + // The queue takes the reference; the extension's side releases it. + let status = unsafe { self.queue.enqueue(sample.cast::()) }; + if status != 0 { + drop(unsafe { CFRetained::from_raw(sample) }); + anyhow::bail!("cannot hand the camera a picture: {status}"); + } + Ok(()) + } +} + +impl Drop for Extension { + fn drop(&mut self) { + let address = address(READING.selector()); + unsafe { + CMIOObjectRemovePropertyListenerBlock( + self.device, + &address, + Some(&self.listening), + RcBlock::as_ptr(&self.reading), + ) + }; + unsafe { CMIODeviceStopStream(self.device, self.stream) }; + // Stopping the sink of a scene's camera removes the camera. + if self.scene + && let Err(err) = vanish(&self.id) + { + tracing::warn!(err = %format!("{err:#}"), "virtual camera"); + } + } +} + +/// Called when the extension takes a picture off the sink's queue. Nothing +/// waits on it: the next `write` looks at the queue itself. +unsafe extern "C-unwind" fn altered(_stream: CMIOObjectID, _token: *mut c_void, _context: *mut c_void) {} + +/// Copy `rgba`, tightly packed `size` rows, into `buffer` as BGRA. +fn swizzle(buffer: &CVPixelBuffer, rgba: &[u8], size: Size) -> anyhow::Result<()> { + let width = size.width as usize * 4; + anyhow::ensure!( + rgba.len() == width * size.height as usize, + "a picture of {} bytes is not {}x{} RGBA", + rgba.len(), + size.width, + size.height + ); + let status = unsafe { CVPixelBufferLockBaseAddress(buffer, CVPixelBufferLockFlags(0)) }; + anyhow::ensure!(status == 0, "cannot lock the picture buffer: {status}"); + let stride = CVPixelBufferGetBytesPerRow(buffer); + let base = CVPixelBufferGetBaseAddress(buffer).cast::(); + let fits = !base.is_null() && stride >= width; + if fits { + // SAFETY: locked, and the pool made it `size` BGRA with `stride` bytes + // a row. + let target = unsafe { std::slice::from_raw_parts_mut(base, stride * size.height as usize) }; + bgra(target, stride, rgba, width); + } + unsafe { CVPixelBufferUnlockBaseAddress(buffer, CVPixelBufferLockFlags(0)) }; + anyhow::ensure!(fits, "the picture buffer has no room"); + Ok(()) +} + +/// Copy `rgba` rows of `width` bytes into `target` rows of `stride` bytes, +/// swapping red and blue. +fn bgra(target: &mut [u8], stride: usize, rgba: &[u8], width: usize) { + for (to, from) in target.chunks_exact_mut(stride).zip(rgba.chunks_exact(width)) { + let (to, _) = to[..width].as_chunks_mut::<4>(); + let (from, _) = from.as_chunks::<4>(); + for (to, [r, g, b, a]) in to.iter_mut().zip(from) { + *to = [*b, *g, *r, *a]; + } + } +} + +/// Reusable IOSurface-backed BGRA buffers at `size`, which reach the +/// extension without a copy. +fn pool(size: Size) -> anyhow::Result> { + let format = CFNumber::new_i32(kCVPixelFormatType_32BGRA as i32); + let width = CFNumber::new_i32(size.width as i32); + let height = CFNumber::new_i32(size.height as i32); + let surface = CFDictionary::::empty(); + let attributes = unsafe { + CFDictionary::::from_slices( + &[ + kCVPixelBufferPixelFormatTypeKey, + kCVPixelBufferWidthKey, + kCVPixelBufferHeightKey, + kCVPixelBufferIOSurfacePropertiesKey, + ], + &[&format, &width, &height, &surface], + ) + }; + let mut pool: *mut CVPixelBufferPool = std::ptr::null_mut(); + let status = + unsafe { CVPixelBufferPool::create(None, None, Some(attributes.as_opaque()), NonNull::from(&mut pool)) }; + NonNull::new(pool) + .filter(|_| status == 0) + .map(|pool| unsafe { CFRetained::from_raw(pool) }) + .with_context(|| format!("cannot make picture buffers: {status}")) +} + +/// Store whether an app reads the camera `device` in `shared`. +fn read(device: CMIOObjectID, shared: &Shared) { + match get_string(device, READING) { + Ok(reading) => shared.reading.store(reading == "1", Ordering::Relaxed), + Err(err) => shared.fail(format!("{err:#}")), + } +} + +/// Ask the system to activate the extension in the app bundle, which installs +/// it or the update's, replacing the old one. The extension +/// takes pictures only from the app, so a build run from outside it has no +/// camera. +fn activate() -> anyhow::Result<()> { + let bundle = NSBundle::mainBundle().bundleIdentifier(); + anyhow::ensure!( + bundle.is_some_and(|bundle| bundle.to_string() == moq_studio_camera::APP), + "the virtual camera runs only from the kixel.live Studio app" + ); + + let (tx, rx) = mpsc::channel(); + let delegate = Activation::new(tx); + let queue = DispatchQueue::global_queue(GlobalQueueIdentifier::Priority(DispatchQueueGlobalPriority::Default)); + unsafe { + let request = + OSSystemExtensionRequest::activationRequestForExtension_queue(&NSString::from_str(EXTENSION), &queue); + request.setDelegate(Some(ProtocolObject::from_ref(&*delegate))); + OSSystemExtensionManager::sharedManager().submitRequest(&request); + } + // The request holds its delegate weakly, and still finishes once the user + // allows the extension after this run stopped waiting on it. + std::mem::forget(delegate); + + match rx.recv_timeout(ACTIVATE) { + Ok(Activated::Done) => Ok(()), + Ok(Activated::Approval) => { + anyhow::bail!("allow the kixel.live Studio camera in {ALLOW}, then turn this camera off and on again") + } + Ok(Activated::Reboot) => anyhow::bail!("restart the Mac to finish installing the kixel.live Studio camera"), + Ok(Activated::Failed(err)) => anyhow::bail!("cannot install the kixel.live Studio camera: {err}"), + Err(_) => anyhow::bail!("the system never installed the kixel.live Studio camera"), + } +} + +/// How an activation request ended, or that it waits on the user. +enum Activated { + Done, + Approval, + Reboot, + Failed(String), +} + +define_class!( + #[unsafe(super(NSObject))] + #[name = "MoqStudioCameraActivation"] + #[ivars = Mutex>] + struct Activation; + + unsafe impl NSObjectProtocol for Activation {} + + unsafe impl OSSystemExtensionRequestDelegate for Activation { + #[unsafe(method(request:actionForReplacingExtension:withExtension:))] + fn replace( + &self, + _request: &OSSystemExtensionRequest, + _existing: &OSSystemExtensionProperties, + _update: &OSSystemExtensionProperties, + ) -> OSSystemExtensionReplacementAction { + // The bundle's is the one this app speaks to, newer or older. + OSSystemExtensionReplacementAction::Replace + } + + #[unsafe(method(requestNeedsUserApproval:))] + fn needs_approval(&self, _request: &OSSystemExtensionRequest) { + self.send(Activated::Approval); + } + + #[unsafe(method(request:didFinishWithResult:))] + fn finished(&self, _request: &OSSystemExtensionRequest, result: OSSystemExtensionRequestResult) { + self.send(match result { + OSSystemExtensionRequestResult::Completed => Activated::Done, + _ => Activated::Reboot, + }); + } + + #[unsafe(method(request:didFailWithError:))] + fn failed(&self, _request: &OSSystemExtensionRequest, error: &NSError) { + self.send(Activated::Failed(error.localizedDescription().to_string())); + } + } +); + +impl Activation { + fn new(tx: mpsc::Sender) -> Retained { + let this = Self::alloc().set_ivars(Mutex::new(tx)); + unsafe { msg_send![super(this), init] } + } + + fn send(&self, activated: Activated) { + // Nobody waits once the toggle reported the first answer. + let _ = self.ivars().lock().unwrap().send(activated); + } +} + +fn address(selector: u32) -> CMIOObjectPropertyAddress { + CMIOObjectPropertyAddress { + mSelector: selector, + mScope: kCMIOObjectPropertyScopeGlobal, + mElement: kCMIOObjectPropertyElementMain, + } +} + +/// Wait until a camera with the device id `id` is listed, showing `config` +/// if given, and return its object. +fn appear(id: &str, config: Option<&str>) -> anyhow::Result { + watch(|| find(id, config))?.with_context(|| format!("no camera {id}")) +} + +/// Wait until no camera with the device id `id` is listed, so turning it +/// straight back on finds the new one rather than the one going away. +fn vanish(id: &str) -> anyhow::Result<()> { + let gone = || Ok(find(id, None)?.is_none().then_some(())); + watch(gone)?.with_context(|| format!("the camera {id} never went away")) +} + +/// Check the cameras with `check` until it finds something, again each time +/// the list changes, for at most [`APPEAR`]; `None` once that passes. +fn watch(mut check: impl FnMut() -> anyhow::Result>) -> anyhow::Result> { + let (tx, rx) = mpsc::channel(); + let changed = RcBlock::new(move |_count: u32, _addresses: *mut CMIOObjectPropertyAddress| { + let _ = tx.send(()); + }); + let queue = DispatchQueue::new("live.kixel.studio.vcam.devices", None); + let address = address(kCMIOHardwarePropertyDevices); + let status = unsafe { + CMIOObjectAddPropertyListenerBlock( + kCMIOObjectSystemObject, + &address, + Some(&queue), + RcBlock::as_ptr(&changed), + ) + }; + anyhow::ensure!(status == 0, "cannot watch the cameras: {status}"); + let start = kio::clock::now(); + let found = loop { + match check() { + Ok(None) => {} + found => break found, + } + let left = APPEAR.saturating_sub(start.elapsed()); + if left.is_zero() || rx.recv_timeout(left).is_err() { + break Ok(None); + } + }; + unsafe { + CMIOObjectRemovePropertyListenerBlock( + kCMIOObjectSystemObject, + &address, + Some(&queue), + RcBlock::as_ptr(&changed), + ) + }; + found +} + +/// The listed camera with the device id `id`, showing `config` if given. +fn find(id: &str, config: Option<&str>) -> anyhow::Result> { + for device in get_ids(kCMIOObjectSystemObject, kCMIOHardwarePropertyDevices)? { + // A camera that went away as it was listed has no id left to read. + let Ok(uid) = get_cf_string(device, kCMIODevicePropertyDeviceUID) else { + continue; + }; + if uid != id { + continue; + } + match config { + None => return Ok(Some(device)), + Some(config) if get_string(device, CONFIG).is_ok_and(|shown| shown == config) => return Ok(Some(device)), + Some(_) => {} + } + } + Ok(None) +} + +/// The sink stream of the camera `device`: its output stream, in CoreMediaIO's +/// terms. +fn sink(device: CMIOObjectID) -> anyhow::Result { + for stream in get_ids(device, kCMIODevicePropertyStreams)? { + let mut direction = 0u32; + get( + stream, + kCMIOStreamPropertyDirection, + std::slice::from_mut(&mut direction), + )?; + if direction == 0 { + return Ok(stream); + } + } + anyhow::bail!("the camera has no sink") +} + +/// Read the property `selector` of `object` into `data`, returning how many +/// bytes it filled. +fn get(object: CMIOObjectID, selector: u32, data: &mut [T]) -> anyhow::Result { + let address = address(selector); + let mut used = 0u32; + let status = unsafe { + CMIOObjectGetPropertyData( + object, + &address, + 0, + std::ptr::null(), + std::mem::size_of_val(data) as u32, + &mut used, + data.as_mut_ptr().cast(), + ) + }; + anyhow::ensure!(status == 0, "cannot read property {selector:#x}: {status}"); + Ok(used as usize) +} + +/// The object ids the property `selector` of `object` lists. +fn get_ids(object: CMIOObjectID, selector: u32) -> anyhow::Result> { + let address = address(selector); + let mut size = 0u32; + let status = unsafe { CMIOObjectGetPropertyDataSize(object, &address, 0, std::ptr::null(), &mut size) }; + anyhow::ensure!(status == 0, "cannot size property {selector:#x}: {status}"); + let mut ids = vec![0; size as usize / std::mem::size_of::()]; + let used = get(object, selector, &mut ids)?; + ids.truncate(used / std::mem::size_of::()); + Ok(ids) +} + +/// A string property, which CoreMediaIO hands over as a retained CFString. +fn get_cf_string(object: CMIOObjectID, selector: u32) -> anyhow::Result { + let mut string: *const CFString = std::ptr::null(); + get(object, selector, std::slice::from_mut(&mut string))?; + let string = NonNull::new(string.cast_mut()).context("an empty string property")?; + Ok(unsafe { CFRetained::from_raw(string) }.to_string()) +} + +fn get_string(object: CMIOObjectID, property: Property) -> anyhow::Result { + get_cf_string(object, property.selector()) +} + +fn set_string(object: CMIOObjectID, property: Property, value: &str) -> anyhow::Result<()> { + let value = CFString::from_str(value); + let pointer: *const CFString = &*value; + let address = address(property.selector()); + let status = unsafe { + CMIOObjectSetPropertyData( + object, + &address, + 0, + std::ptr::null(), + std::mem::size_of_val(&pointer) as u32, + (&raw const pointer).cast(), + ) + }; + anyhow::ensure!(status == 0, "cannot set property {}: {status}", property.name()); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn swaps_red_and_blue_into_padded_rows() { + // Two pixels a row, two rows, into rows padded to 12 bytes. + let rgba = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let mut target = [0xff; 24]; + bgra(&mut target, 12, &rgba, 8); + assert_eq!( + target, + [ + 3, 2, 1, 4, 7, 6, 5, 8, 0xff, 0xff, 0xff, 0xff, 11, 10, 9, 12, 15, 14, 13, 16, 0xff, 0xff, 0xff, 0xff + ] + ); + } +}