VideoInjection: emulate AVCaptureSession.connections and the preview layer's coordinate conversions

CameraFixture/CameraFixture.swift

@@ -21,6 +21,7 @@
  *   CameraFixture nv12 <frames>
  *   CameraFixture rotate <angle> <frames>        -- clockwise, on the data output's connection
  *   CameraFixture preview
+ *   CameraFixture roi                            -- session connections, and preview-layer rects and points at each gravity and angle
  *   CameraFixture photo <out.jpg>
  *   CameraFixture movie <seconds> <out.mov>
  *
@@ -270,6 +271,40 @@ func recordMovie(_ session: AVCaptureSession, _ seconds: TimeInterval, _ path: S
   return duration >= max(0.1, seconds - 0.75) ? 0 : 7
 }
 
+func convertRegions(_ session: AVCaptureSession) -> Int32 {
+  report("roi connections no-output=\(session.connections.count)")
+  let output = AVCapturePhotoOutput()
+  if session.canAddOutput(output) { session.addOutput(output) }
+  report("roi connections photo-output=\(session.connections.count)")
+  let layer = AVCaptureVideoPreviewLayer(session: session)
+  layer.frame = CGRect(x: 0, y: 0, width: 300, height: 600)
+  report("roi mirrored=\(layer.connection?.isVideoMirrored == true ? 1 : 0)")
+  let rect = CGRect(x: 30, y: 150, width: 120, height: 200)
+  let point = CGPoint(x: 75, y: 450)
+  var drift: CGFloat = 0
+  for gravity in [AVLayerVideoGravity.resizeAspect, .resizeAspectFill, .resize] {
+    layer.videoGravity = gravity
+    for angle: CGFloat in [0, 90, 180, 270] {
+      if layer.connection?.isVideoRotationAngleSupported(angle) == true { layer.connection?.videoRotationAngle = angle }
+      let metadata = layer.metadataOutputRectConverted(fromLayerRect: rect)
+      let device = layer.captureDevicePointConverted(fromLayerPoint: point)
+      let rectBack = layer.layerRectConverted(fromMetadataOutputRect: metadata)
+      let pointBack = layer.layerPointConverted(fromCaptureDevicePoint: device)
+      drift = max(
+        drift, abs(rectBack.minX - rect.minX), abs(rectBack.minY - rect.minY), abs(rectBack.width - rect.width),
+        abs(rectBack.height - rect.height), abs(pointBack.x - point.x), abs(pointBack.y - point.y))
+      report(
+        String(
+          format: "roi gravity=%@ angle=%.0f rect=%.4f,%.4f,%.4f,%.4f point=%.4f,%.4f", gravity.rawValue, layer.connection?.videoRotationAngle ?? -1,
+          metadata.minX, metadata.minY, metadata.width, metadata.height, device.x, device.y))
+    }
+  }
+  session.inputs.forEach(session.removeInput)
+  report("roi connections no-input=\(session.connections.count)")
+  report(String(format: "roi round-trip drift=%.6f", drift))
+  return drift < 0.001 ? 0 : 11
+}
+
 func run(_ arguments: [String]) -> Int32 {
   let mode = arguments.count > 1 ? arguments[1] : "stream"
   let argument = arguments.count > 2 ? arguments[2] : nil
@@ -323,6 +358,8 @@ func run(_ arguments: [String]) -> Int32 {
     }
     session.stopRunning()
     return layer.contents == nil ? 9 : 0
+  case "roi":
+    return convertRegions(session)
   default:
     break
   }

Shims/Shimulator/VideoInjection/CaptureSwizzles.swift

@@ -38,6 +38,7 @@ func installSwizzles() {
     (session, "addOutput:", #selector(AVCaptureSession.shimulator_addOutput(_:))),
     (session, "removeOutput:", #selector(AVCaptureSession.shimulator_removeOutput(_:))),
     (session, "outputs", #selector(getter: AVCaptureSession.shimulator_outputs)),
+    (session, "connections", #selector(getter: AVCaptureSession.shimulator_connections)),
     (session, "canSetSessionPreset:", #selector(AVCaptureSession.shimulator_canSetSessionPreset(_:))),
     (session, "setSessionPreset:", #selector(setter: AVCaptureSession.shimulator_sessionPreset)),
     (session, "sessionPreset", #selector(getter: AVCaptureSession.shimulator_sessionPreset)),
@@ -55,6 +56,22 @@ func installSwizzles() {
     (AVCapturePhotoOutput.self, "capturePhotoWithSettings:delegate:", #selector(AVCapturePhotoOutput.shimulator_capturePhoto(with:delegate:))),
     (AVCaptureVideoPreviewLayer.self, "session", #selector(getter: AVCaptureVideoPreviewLayer.shimulator_session)),
     (AVCaptureVideoPreviewLayer.self, "connection", #selector(getter: AVCaptureVideoPreviewLayer.shimulator_connection)),
+    (
+      AVCaptureVideoPreviewLayer.self, "metadataOutputRectOfInterestForRect:",
+      #selector(AVCaptureVideoPreviewLayer.shimulator_metadataOutputRectOfInterest(for:))
+    ),
+    (
+      AVCaptureVideoPreviewLayer.self, "rectForMetadataOutputRectOfInterest:",
+      #selector(AVCaptureVideoPreviewLayer.shimulator_rectForMetadataOutputRectOfInterest(_:))
+    ),
+    (
+      AVCaptureVideoPreviewLayer.self, "captureDevicePointOfInterestForPoint:",
+      #selector(AVCaptureVideoPreviewLayer.shimulator_captureDevicePointOfInterest(for:))
+    ),
+    (
+      AVCaptureVideoPreviewLayer.self, "pointForCaptureDevicePointOfInterest:",
+      #selector(AVCaptureVideoPreviewLayer.shimulator_pointForCaptureDevicePointOfInterest(_:))
+    ),
     (AVCaptureVideoPreviewLayer.self, "setSession:", #selector(AVCaptureVideoPreviewLayer.shimulator_setSession(_:))),
     (AVCaptureVideoPreviewLayer.self, "initWithSession:", NSSelectorFromString("initShimulatorWithSession:")),
   ]
@@ -328,6 +345,12 @@ extension AVCaptureSession {
     return withNativeView(self) { self.shimulator_outputs } + outputs
   }
 
+  @objc dynamic var shimulator_connections: [AVCaptureConnection] {
+    guard let state = sessionState(self) else { return self.shimulator_connections }
+    let connections: [AVCaptureConnection] = synchronized(state) { state.outputs }.compactMap { virtualConnection($0) }.filter { !$0.virtualPorts.isEmpty }
+    return withNativeView(self) { self.shimulator_connections } + connections
+  }
+
   @objc dynamic func shimulator_canSetSessionPreset(_ preset: Preset) -> Bool {
     holdsConfiguration || withNativeView(self) { shimulator_canSetSessionPreset(preset) }
   }
@@ -467,6 +490,52 @@ extension AVCaptureVideoPreviewLayer {
     }
   }
 
+  /// Maps the normalised native frame onto the layer as the preview shows it: turned by the angle
+  /// the frames are turned by, mirrored when the connection is, and placed by `videoGravity`.
+  private var virtualFrameTransform: CGAffineTransform? {
+    guard let connection = connection as? ShimulatorCaptureConnection else { return nil }
+    let size = frameSize(connection)
+    let bounds = self.bounds
+    guard size.width > 0, size.height > 0, bounds.width > 0, bounds.height > 0 else { return nil }
+    var video = bounds
+    if videoGravity != .resize {
+      let widthScale = bounds.width / size.width
+      let heightScale = bounds.height / size.height
+      let scale = videoGravity == .resizeAspectFill ? max(widthScale, heightScale) : min(widthScale, heightScale)
+      video.size = CGSize(width: size.width * scale, height: size.height * scale)
+      video.origin = CGPoint(x: bounds.midX - video.width / 2, y: bounds.midY - video.height / 2)
+    }
+    let rotations: [CGFloat: CGAffineTransform] = [
+      90: CGAffineTransform(a: 0, b: 1, c: -1, d: 0, tx: 0, ty: 0),
+      180: CGAffineTransform(a: -1, b: 0, c: 0, d: -1, tx: 0, ty: 0),
+      270: CGAffineTransform(a: 0, b: -1, c: 1, d: 0, tx: 0, ty: 0),
+    ]
+    return CGAffineTransform(translationX: -0.5, y: -0.5)
+      .concatenating(rotations[connection.frameRotationAngle] ?? .identity)
+      .concatenating(CGAffineTransform(a: connection.isVideoMirrored ? -1 : 1, b: 0, c: 0, d: 1, tx: 0.5, ty: 0.5))
+      .concatenating(CGAffineTransform(a: video.width, b: 0, c: 0, d: video.height, tx: video.minX, ty: video.minY))
+  }
+
+  @objc dynamic func shimulator_metadataOutputRectOfInterest(for rect: CGRect) -> CGRect {
+    guard let transform = virtualFrameTransform else { return shimulator_metadataOutputRectOfInterest(for: rect) }
+    return rect.applying(transform.inverted())
+  }
+
+  @objc dynamic func shimulator_rectForMetadataOutputRectOfInterest(_ rect: CGRect) -> CGRect {
+    guard let transform = virtualFrameTransform else { return shimulator_rectForMetadataOutputRectOfInterest(rect) }
+    return rect.applying(transform)
+  }
+
+  @objc dynamic func shimulator_captureDevicePointOfInterest(for point: CGPoint) -> CGPoint {
+    guard let transform = virtualFrameTransform else { return shimulator_captureDevicePointOfInterest(for: point) }
+    return point.applying(transform.inverted())
+  }
+
+  @objc dynamic func shimulator_pointForCaptureDevicePointOfInterest(_ point: CGPoint) -> CGPoint {
+    guard let transform = virtualFrameTransform else { return shimulator_pointForCaptureDevicePointOfInterest(point) }
+    return point.applying(transform)
+  }
+
   @objc dynamic func shimulator_setSession(_ session: AVCaptureSession?) {
     VirtualCamera.shared.track(self, session: session)
     if sessionState(session) == nil { shimulator_setSession(session) }

Shims/Shimulator/VideoInjection/README.md

@@ -92,14 +92,18 @@ Each of these has already been a crash or a hang:
 | `AVCapturePhotoOutput` | JPEG, pixel buffer and `CGImageRepresentation` of the frame; overlapping captures queue |
 | `AVCaptureMovieFileOutput` | video-only H.264 `.mov` written to the destination, deleted on failure |
 | `AVCaptureVideoPreviewLayer` | live `contents`, `videoGravity` mapped onto `contentsGravity` |
+| `metadataOutputRectOfInterestForRect:`, `rectForMetadataOutputRectOfInterest:`, `captureDevicePointOfInterestForPoint:`, `pointForCaptureDevicePointOfInterest:` | between the layer and the normalised native frame, after `videoGravity`, the connection's rotation and mirroring, as on a device |
+| `AVCaptureSession.connections` | the virtual outputs' connections once the camera input is added, after any native ones |
 | `systemPreferredCamera`, `userPreferredCamera` | the back camera, as `defaultDeviceWithMediaType:` answers |
 | `videoMirrored` | a front camera's preview is mirrored and its outputs are not while `automaticallyAdjustsVideoMirroring` is on, as on a device; setting `videoMirrored` then raises, as it does there |
 
 Preview layers are tracked from `setSession:`, since apps bind them before adding the input.
 
 Not yet covered: `AVCaptureMultiCamSession`, depth and metadata outputs,
-`AVCaptureDeviceRotationCoordinator`, audio in recorded movies, an `activeFormat` that follows the preset (it is always 1920x1080), a session's
-`connections`, and `recordedDuration`.
+`transformedMetadataObjectForMetadataObject:`, `AVCaptureDeviceRotationCoordinator`,
+audio in recorded movies, an `activeFormat` that follows the preset (it is always
+1920x1080), a preview layer's connection in a session's `connections`, and
+`recordedDuration`.
 
 ## Presets and orientation
 

Shims/Shimulator/VideoInjection/VirtualCamera.swift

@@ -347,6 +347,16 @@ func makeConnection(_ ports: [AVCaptureInput.Port], _ session: AVCaptureSession?
   return connection
 }
 
+/// The shape of the frames a connection delivers: the preset's native size turned by the
+/// connection's rotation, as a video data output delivers physically rotated buffers. The file's
+/// own size only decides the crop.
+func frameSize(_ connection: ShimulatorCaptureConnection) -> CGSize {
+  let preset = sessionState(connection.virtualSession).map { state in synchronized(state) { state.preset } }
+  let size = preset.flatMap { presetSizes[$0] } ?? nativeSize
+  let angle = connection.rotationAngle
+  return angle == 90 || angle == 270 ? CGSize(width: size.height, height: size.width) : size
+}
+
 final class MovieRecording {
   let writer: AVAssetWriter, adaptor: AVAssetWriterInputPixelBufferAdaptor
   let delegate: AVCaptureFileOutputRecordingDelegate, url: URL, frameSize: CGSize, rotationAngle: CGFloat, mirrored: Bool
@@ -428,16 +438,6 @@ final class VirtualCamera: NSObject {
     return format
   }
 
-  /// The shape of the frames a connection delivers: the preset's native size turned by the
-  /// connection's rotation, as a video data output delivers physically rotated buffers. The file's
-  /// own size only decides the crop.
-  private func frameSize(_ connection: ShimulatorCaptureConnection) -> CGSize {
-    let preset = sessionState(connection.virtualSession).map { state in synchronized(state) { state.preset } }
-    let size = preset.flatMap { presetSizes[$0] } ?? nativeSize
-    let angle = connection.rotationAngle
-    return angle == 90 || angle == 270 ? CGSize(width: size.height, height: size.width) : size
-  }
-
   // MARK: - Injection
 
   /// Returns the file that holds no playable video, if either does, and leaves the camera unchanged.