diff --git a/CameraFixture/CameraFixture.swift b/CameraFixture/CameraFixture.swift index 32590f645..964d5923e 100644 --- a/CameraFixture/CameraFixture.swift +++ b/CameraFixture/CameraFixture.swift @@ -21,6 +21,7 @@ * CameraFixture nv12 * CameraFixture rotate -- clockwise, on the data output's connection * CameraFixture preview + * CameraFixture roi -- session connections, and preview-layer rects and points at each gravity and angle * CameraFixture photo * CameraFixture movie * @@ -270,6 +271,40 @@ func recordMovie(_ session: AVCaptureSession, _ seconds: TimeInterval, _ path: S return duration >= max(0.1, seconds - 0.75) ? 0 : 7 } +func convertRegions(_ session: AVCaptureSession) -> Int32 { + report("roi connections no-output=\(session.connections.count)") + let output = AVCapturePhotoOutput() + if session.canAddOutput(output) { session.addOutput(output) } + report("roi connections photo-output=\(session.connections.count)") + let layer = AVCaptureVideoPreviewLayer(session: session) + layer.frame = CGRect(x: 0, y: 0, width: 300, height: 600) + report("roi mirrored=\(layer.connection?.isVideoMirrored == true ? 1 : 0)") + let rect = CGRect(x: 30, y: 150, width: 120, height: 200) + let point = CGPoint(x: 75, y: 450) + var drift: CGFloat = 0 + for gravity in [AVLayerVideoGravity.resizeAspect, .resizeAspectFill, .resize] { + layer.videoGravity = gravity + for angle: CGFloat in [0, 90, 180, 270] { + if layer.connection?.isVideoRotationAngleSupported(angle) == true { layer.connection?.videoRotationAngle = angle } + let metadata = layer.metadataOutputRectConverted(fromLayerRect: rect) + let device = layer.captureDevicePointConverted(fromLayerPoint: point) + let rectBack = layer.layerRectConverted(fromMetadataOutputRect: metadata) + let pointBack = layer.layerPointConverted(fromCaptureDevicePoint: device) + drift = max( + drift, abs(rectBack.minX - rect.minX), abs(rectBack.minY - rect.minY), abs(rectBack.width - rect.width), + abs(rectBack.height - rect.height), abs(pointBack.x - point.x), abs(pointBack.y - point.y)) + report( + String( + format: "roi gravity=%@ angle=%.0f rect=%.4f,%.4f,%.4f,%.4f point=%.4f,%.4f", gravity.rawValue, layer.connection?.videoRotationAngle ?? -1, + metadata.minX, metadata.minY, metadata.width, metadata.height, device.x, device.y)) + } + } + session.inputs.forEach(session.removeInput) + report("roi connections no-input=\(session.connections.count)") + report(String(format: "roi round-trip drift=%.6f", drift)) + return drift < 0.001 ? 0 : 11 +} + func run(_ arguments: [String]) -> Int32 { let mode = arguments.count > 1 ? arguments[1] : "stream" let argument = arguments.count > 2 ? arguments[2] : nil @@ -323,6 +358,8 @@ func run(_ arguments: [String]) -> Int32 { } session.stopRunning() return layer.contents == nil ? 9 : 0 + case "roi": + return convertRegions(session) default: break } diff --git a/Shims/Shimulator/VideoInjection/CaptureSwizzles.swift b/Shims/Shimulator/VideoInjection/CaptureSwizzles.swift index 289cf77a8..fa62909b1 100644 --- a/Shims/Shimulator/VideoInjection/CaptureSwizzles.swift +++ b/Shims/Shimulator/VideoInjection/CaptureSwizzles.swift @@ -38,6 +38,7 @@ func installSwizzles() { (session, "addOutput:", #selector(AVCaptureSession.shimulator_addOutput(_:))), (session, "removeOutput:", #selector(AVCaptureSession.shimulator_removeOutput(_:))), (session, "outputs", #selector(getter: AVCaptureSession.shimulator_outputs)), + (session, "connections", #selector(getter: AVCaptureSession.shimulator_connections)), (session, "canSetSessionPreset:", #selector(AVCaptureSession.shimulator_canSetSessionPreset(_:))), (session, "setSessionPreset:", #selector(setter: AVCaptureSession.shimulator_sessionPreset)), (session, "sessionPreset", #selector(getter: AVCaptureSession.shimulator_sessionPreset)), @@ -55,6 +56,22 @@ func installSwizzles() { (AVCapturePhotoOutput.self, "capturePhotoWithSettings:delegate:", #selector(AVCapturePhotoOutput.shimulator_capturePhoto(with:delegate:))), (AVCaptureVideoPreviewLayer.self, "session", #selector(getter: AVCaptureVideoPreviewLayer.shimulator_session)), (AVCaptureVideoPreviewLayer.self, "connection", #selector(getter: AVCaptureVideoPreviewLayer.shimulator_connection)), + ( + AVCaptureVideoPreviewLayer.self, "metadataOutputRectOfInterestForRect:", + #selector(AVCaptureVideoPreviewLayer.shimulator_metadataOutputRectOfInterest(for:)) + ), + ( + AVCaptureVideoPreviewLayer.self, "rectForMetadataOutputRectOfInterest:", + #selector(AVCaptureVideoPreviewLayer.shimulator_rectForMetadataOutputRectOfInterest(_:)) + ), + ( + AVCaptureVideoPreviewLayer.self, "captureDevicePointOfInterestForPoint:", + #selector(AVCaptureVideoPreviewLayer.shimulator_captureDevicePointOfInterest(for:)) + ), + ( + AVCaptureVideoPreviewLayer.self, "pointForCaptureDevicePointOfInterest:", + #selector(AVCaptureVideoPreviewLayer.shimulator_pointForCaptureDevicePointOfInterest(_:)) + ), (AVCaptureVideoPreviewLayer.self, "setSession:", #selector(AVCaptureVideoPreviewLayer.shimulator_setSession(_:))), (AVCaptureVideoPreviewLayer.self, "initWithSession:", NSSelectorFromString("initShimulatorWithSession:")), ] @@ -328,6 +345,12 @@ extension AVCaptureSession { return withNativeView(self) { self.shimulator_outputs } + outputs } + @objc dynamic var shimulator_connections: [AVCaptureConnection] { + guard let state = sessionState(self) else { return self.shimulator_connections } + let connections: [AVCaptureConnection] = synchronized(state) { state.outputs }.compactMap { virtualConnection($0) }.filter { !$0.virtualPorts.isEmpty } + return withNativeView(self) { self.shimulator_connections } + connections + } + @objc dynamic func shimulator_canSetSessionPreset(_ preset: Preset) -> Bool { holdsConfiguration || withNativeView(self) { shimulator_canSetSessionPreset(preset) } } @@ -467,6 +490,52 @@ extension AVCaptureVideoPreviewLayer { } } + /// Maps the normalised native frame onto the layer as the preview shows it: turned by the angle + /// the frames are turned by, mirrored when the connection is, and placed by `videoGravity`. + private var virtualFrameTransform: CGAffineTransform? { + guard let connection = connection as? ShimulatorCaptureConnection else { return nil } + let size = frameSize(connection) + let bounds = self.bounds + guard size.width > 0, size.height > 0, bounds.width > 0, bounds.height > 0 else { return nil } + var video = bounds + if videoGravity != .resize { + let widthScale = bounds.width / size.width + let heightScale = bounds.height / size.height + let scale = videoGravity == .resizeAspectFill ? max(widthScale, heightScale) : min(widthScale, heightScale) + video.size = CGSize(width: size.width * scale, height: size.height * scale) + video.origin = CGPoint(x: bounds.midX - video.width / 2, y: bounds.midY - video.height / 2) + } + let rotations: [CGFloat: CGAffineTransform] = [ + 90: CGAffineTransform(a: 0, b: 1, c: -1, d: 0, tx: 0, ty: 0), + 180: CGAffineTransform(a: -1, b: 0, c: 0, d: -1, tx: 0, ty: 0), + 270: CGAffineTransform(a: 0, b: -1, c: 1, d: 0, tx: 0, ty: 0), + ] + return CGAffineTransform(translationX: -0.5, y: -0.5) + .concatenating(rotations[connection.frameRotationAngle] ?? .identity) + .concatenating(CGAffineTransform(a: connection.isVideoMirrored ? -1 : 1, b: 0, c: 0, d: 1, tx: 0.5, ty: 0.5)) + .concatenating(CGAffineTransform(a: video.width, b: 0, c: 0, d: video.height, tx: video.minX, ty: video.minY)) + } + + @objc dynamic func shimulator_metadataOutputRectOfInterest(for rect: CGRect) -> CGRect { + guard let transform = virtualFrameTransform else { return shimulator_metadataOutputRectOfInterest(for: rect) } + return rect.applying(transform.inverted()) + } + + @objc dynamic func shimulator_rectForMetadataOutputRectOfInterest(_ rect: CGRect) -> CGRect { + guard let transform = virtualFrameTransform else { return shimulator_rectForMetadataOutputRectOfInterest(rect) } + return rect.applying(transform) + } + + @objc dynamic func shimulator_captureDevicePointOfInterest(for point: CGPoint) -> CGPoint { + guard let transform = virtualFrameTransform else { return shimulator_captureDevicePointOfInterest(for: point) } + return point.applying(transform.inverted()) + } + + @objc dynamic func shimulator_pointForCaptureDevicePointOfInterest(_ point: CGPoint) -> CGPoint { + guard let transform = virtualFrameTransform else { return shimulator_pointForCaptureDevicePointOfInterest(point) } + return point.applying(transform) + } + @objc dynamic func shimulator_setSession(_ session: AVCaptureSession?) { VirtualCamera.shared.track(self, session: session) if sessionState(session) == nil { shimulator_setSession(session) } diff --git a/Shims/Shimulator/VideoInjection/README.md b/Shims/Shimulator/VideoInjection/README.md index 5ef9b09fa..76d23dd4a 100644 --- a/Shims/Shimulator/VideoInjection/README.md +++ b/Shims/Shimulator/VideoInjection/README.md @@ -92,14 +92,18 @@ Each of these has already been a crash or a hang: | `AVCapturePhotoOutput` | JPEG, pixel buffer and `CGImageRepresentation` of the frame; overlapping captures queue | | `AVCaptureMovieFileOutput` | video-only H.264 `.mov` written to the destination, deleted on failure | | `AVCaptureVideoPreviewLayer` | live `contents`, `videoGravity` mapped onto `contentsGravity` | +| `metadataOutputRectOfInterestForRect:`, `rectForMetadataOutputRectOfInterest:`, `captureDevicePointOfInterestForPoint:`, `pointForCaptureDevicePointOfInterest:` | between the layer and the normalised native frame, after `videoGravity`, the connection's rotation and mirroring, as on a device | +| `AVCaptureSession.connections` | the virtual outputs' connections once the camera input is added, after any native ones | | `systemPreferredCamera`, `userPreferredCamera` | the back camera, as `defaultDeviceWithMediaType:` answers | | `videoMirrored` | a front camera's preview is mirrored and its outputs are not while `automaticallyAdjustsVideoMirroring` is on, as on a device; setting `videoMirrored` then raises, as it does there | Preview layers are tracked from `setSession:`, since apps bind them before adding the input. Not yet covered: `AVCaptureMultiCamSession`, depth and metadata outputs, -`AVCaptureDeviceRotationCoordinator`, audio in recorded movies, an `activeFormat` that follows the preset (it is always 1920x1080), a session's -`connections`, and `recordedDuration`. +`transformedMetadataObjectForMetadataObject:`, `AVCaptureDeviceRotationCoordinator`, +audio in recorded movies, an `activeFormat` that follows the preset (it is always +1920x1080), a preview layer's connection in a session's `connections`, and +`recordedDuration`. ## Presets and orientation diff --git a/Shims/Shimulator/VideoInjection/VirtualCamera.swift b/Shims/Shimulator/VideoInjection/VirtualCamera.swift index cfce849e3..d04f621c5 100644 --- a/Shims/Shimulator/VideoInjection/VirtualCamera.swift +++ b/Shims/Shimulator/VideoInjection/VirtualCamera.swift @@ -347,6 +347,16 @@ func makeConnection(_ ports: [AVCaptureInput.Port], _ session: AVCaptureSession? return connection } +/// The shape of the frames a connection delivers: the preset's native size turned by the +/// connection's rotation, as a video data output delivers physically rotated buffers. The file's +/// own size only decides the crop. +func frameSize(_ connection: ShimulatorCaptureConnection) -> CGSize { + let preset = sessionState(connection.virtualSession).map { state in synchronized(state) { state.preset } } + let size = preset.flatMap { presetSizes[$0] } ?? nativeSize + let angle = connection.rotationAngle + return angle == 90 || angle == 270 ? CGSize(width: size.height, height: size.width) : size +} + final class MovieRecording { let writer: AVAssetWriter, adaptor: AVAssetWriterInputPixelBufferAdaptor let delegate: AVCaptureFileOutputRecordingDelegate, url: URL, frameSize: CGSize, rotationAngle: CGFloat, mirrored: Bool @@ -428,16 +438,6 @@ final class VirtualCamera: NSObject { return format } - /// The shape of the frames a connection delivers: the preset's native size turned by the - /// connection's rotation, as a video data output delivers physically rotated buffers. The file's - /// own size only decides the crop. - private func frameSize(_ connection: ShimulatorCaptureConnection) -> CGSize { - let preset = sessionState(connection.virtualSession).map { state in synchronized(state) { state.preset } } - let size = preset.flatMap { presetSizes[$0] } ?? nativeSize - let angle = connection.rotationAngle - return angle == 90 || angle == 270 ? CGSize(width: size.height, height: size.width) : size - } - // MARK: - Injection /// Returns the file that holds no playable video, if either does, and leaves the camera unchanged.