[Experiment] Portrait orientation

RealityMixer/Calibration/Builders/CalibrationBuilder.swift

@@ -10,12 +10,10 @@ import ARKit
 
 struct CalibrationBuilder {
 
-    static func fov(from frame: ARFrame) -> (Float, Float) {
-        let projection = frame.camera.projectionMatrix
+    static func fov(projectionMatrix projection: simd_float4x4, imageResolution: CGSize) -> (Float, Float) {
         let yScale = projection[1,1]
         let yFov = 2 * atan(1/yScale)
 
-        let imageResolution = frame.camera.imageResolution
         let xFov = yFov * Float(imageResolution.width / imageResolution.height)
         return (xFov, yFov)
     }
@@ -61,10 +59,10 @@ struct CalibrationBuilder {
         rightControllerPosition: Vector3,
         rightControllerScreenCoordinates: CGPoint,
         centerPose: Pose,
-        frame: ARFrame
+        imageResolution: CGSize,
+        projectionMatrix: simd_float4x4
     ) -> (SCNMatrix4, CalibrationResult) {
-        let imageResolution = frame.camera.imageResolution
-        let (xFov, yFov) = fov(from: frame)
+        let (xFov, yFov) = fov(projectionMatrix: projectionMatrix, imageResolution: imageResolution)
 
         let anglePerVerticalPixel = yFov/Float(imageResolution.height)
         let anglePerHorizontalPixel = xFov/Float(imageResolution.width)

RealityMixer/Calibration/ViewControllers/CalibrationViewController.swift

@@ -165,7 +165,21 @@ final class CalibrationViewController: UIViewController {
             saveButtonContainer.isHidden = true
             updateInfo("Step 3 of 4")
 
+            let viewPortSize = sceneView.bounds.size
+            let interfaceOrientation = UIApplication.shared.windows.first?.windowScene?.interfaceOrientation ?? UIInterfaceOrientation.unknown
+            let transform = frame.displayTransform(for: interfaceOrientation, viewportSize: viewPortSize).inverted()
+            let ciImage = CIImage(cvImageBuffer: frame.capturedImage).transformed(by: transform)
+            let projectionMatrix = frame.camera.projectionMatrix(for: interfaceOrientation, viewportSize: viewPortSize, zNear: 0.001, zFar: 1000.0)
+
+            let context = CIContext(options: nil)
+            guard let cgImage = context.createCGImage(ciImage, from: ciImage.extent) else {
+                return
+            }
+            let uiImage = UIImage(cgImage: cgImage)
+
             let viewController = ProjectionViewController(
+                uiImage: uiImage,
+                projectionMatrix: projectionMatrix,
                 scaleFactor: scaleFactor,
                 cameraOrigin: cameraOrigin,
                 rightControllerPosition: rightControllerPosition,

RealityMixer/Calibration/ViewControllers/ProjectionPickerViewController.swift

@@ -20,16 +20,17 @@ protocol ProjectionPickerViewControllerDelegate: AnyObject {
 
 final class ProjectionPickerViewController: UIViewController {
     weak var delegate: ProjectionPickerViewControllerDelegate?
+
+    private let image: UIImage
+    private let projectionMatrix: simd_float4x4
+
     private let scaleFactor: Double
     private let cameraOrigin: Vector3
     private let rightControllerPosition: Vector3
     private let frame: ARFrame
     private let lastPoseUpdate: PoseUpdate
 
-    private var image: UIImage {
-        UIImage(ciImage: CIImage(cvImageBuffer: frame.capturedImage))
-    }
-
+
     @IBOutlet private weak var imageView: UIImageView!
     @IBOutlet private weak var sceneOverlay: SCNView!
     @IBOutlet private weak var blueView: UIView!
@@ -55,12 +56,16 @@ final class ProjectionPickerViewController: UIViewController {
     private var first = true
 
     init(
+        uiImage: UIImage,
+        projectionMatrix: simd_float4x4,
         scaleFactor: Double,
         cameraOrigin: Vector3,
         rightControllerPosition: Vector3,
         frame: ARFrame,
         lastPoseUpdate: PoseUpdate
     ) {
+        self.image = uiImage
+        self.projectionMatrix = projectionMatrix
         self.scaleFactor = scaleFactor
         self.cameraOrigin = cameraOrigin
         self.rightControllerPosition = rightControllerPosition
@@ -100,20 +105,20 @@ final class ProjectionPickerViewController: UIViewController {
         let camera = SCNCamera()
         camera.zNear = 0.1
         camera.zFar = 100.0
-        let (xFov, yFov) = CalibrationBuilder.fov(from: frame)
+        let (xFov, yFov) = CalibrationBuilder.fov(projectionMatrix: projectionMatrix, imageResolution: image.size)
 
         let imageViewRatio = imageView.frame.size.width/imageView.frame.size.height
-        let imageRatio = frame.camera.imageResolution.width/frame.camera.imageResolution.height
+        let imageRatio = image.size.width/image.size.height
 
         if imageViewRatio > imageRatio {
-            let imageHeightInImageViewCoordinates = frame.camera.imageResolution.height * (imageView.frame.size.width/frame.camera.imageResolution.width)
+            let imageHeightInImageViewCoordinates = image.size.height * (imageView.frame.size.width/image.size.width)
             let distanceInImageViewCoordinates = (imageHeightInImageViewCoordinates * 0.5)/CGFloat(tan(yFov/2.0))
             let adjustedYFov = CGFloat(2.0 * atan2((imageView.frame.size.height * 0.5), distanceInImageViewCoordinates))
 
             camera.projectionDirection = .vertical
             camera.fieldOfView = (adjustedYFov * (180.0/CGFloat.pi))
         } else {
-            let imageWidthInImageViewCoordinates = frame.camera.imageResolution.width * (imageView.frame.size.height/frame.camera.imageResolution.height)
+            let imageWidthInImageViewCoordinates = image.size.width * (imageView.frame.size.height/image.size.height)
             let distanceInImageViewCoordinates = (imageWidthInImageViewCoordinates * 0.5)/CGFloat(tan(xFov/2.0))
             let adjustedXFov = CGFloat(2.0 * atan2((imageView.frame.size.width * 0.5), distanceInImageViewCoordinates))
 
@@ -189,7 +194,8 @@ final class ProjectionPickerViewController: UIViewController {
             rightControllerPosition: rightControllerPosition,
             rightControllerScreenCoordinates: pixelCoordinate(from: blueViewCenter),
             centerPose: lastPoseUpdate.trackingTransformRaw,
-            frame: frame
+            imageResolution: image.size,
+            projectionMatrix: projectionMatrix
         )
 
         mainNode?.transform = calibration.0

RealityMixer/Calibration/ViewControllers/ProjectionViewController.swift

@@ -26,6 +26,8 @@ final class ProjectionViewController: UIViewController {
     private var projectionPickerViewController: ProjectionPickerViewController
 
     init(
+        uiImage: UIImage,
+        projectionMatrix: simd_float4x4,
         scaleFactor: Double,
         cameraOrigin: Vector3,
         rightControllerPosition: Vector3,
@@ -34,6 +36,8 @@ final class ProjectionViewController: UIViewController {
         delegate: ProjectionViewControllerDelegate
     ) {
         self.projectionPickerViewController = ProjectionPickerViewController(
+            uiImage: uiImage,
+            projectionMatrix: projectionMatrix,
             scaleFactor: scaleFactor,
             cameraOrigin: cameraOrigin,
             rightControllerPosition: rightControllerPosition,

RealityMixer/Capture/Misc/ARKitHelpers.swift

@@ -10,10 +10,8 @@ import ARKit
 struct ARKitHelpers {
 
     // FIXME: Check this.
-    static func planeSizeForDistance(_ distance: Float, frame: ARFrame) -> CGSize {
-        let projection = frame.camera.projectionMatrix
+    static func planeSizeForDistance(_ distance: Float, imageResolution: CGSize, projection: simd_float4x4) -> CGSize {
         let yScale = projection[1,1]
-        let imageResolution = frame.camera.imageResolution
         let width = (2.0 * distance) * tan(atan(1/yScale) * Float(imageResolution.width / imageResolution.height))
         let height = width * Float(imageResolution.height / imageResolution.width)
         return CGSize(width: CGFloat(width), height: CGFloat(height))
@@ -30,8 +28,8 @@ struct ARKitHelpers {
         return planeNode
     }
 
-    static func makePlaneNodeForDistance(_ distance: Float, frame: ARFrame) -> SCNNode {
-        makePlane(size: planeSizeForDistance(distance, frame: frame), distance: distance)
+    static func makePlaneNodeForDistance(_ distance: Float, viewPortSize: CGSize, projectionMatrix: simd_float4x4) -> SCNNode {
+        makePlane(size: planeSizeForDistance(distance, imageResolution: viewPortSize, projection: projectionMatrix), distance: distance)
     }
 
     @discardableResult

RealityMixer/Capture/ViewControllers/ChromaKeyConfigurationViewController.swift

@@ -109,9 +109,9 @@ final class ChromaKeyConfigurationViewController: UIViewController {
         updateValueLabels()
     }
 
-    private func configureBackgroundPlane(with frame: ARFrame) {
+    private func configureBackgroundPlane(viewPortSize: CGSize, projectionMatrix: simd_float4x4) {
         let distance: Float = 100
-        let backgroundPlaneSize = ARKitHelpers.planeSizeForDistance(distance, frame: frame)
+        let backgroundPlaneSize = ARKitHelpers.planeSizeForDistance(distance, imageResolution: viewPortSize, projection: projectionMatrix)
         let backgroundPlaneNode = ARKitHelpers.makePlane(size: backgroundPlaneSize, distance: distance)
 
         let planeMaterial = backgroundPlaneNode.geometry?.firstMaterial
@@ -129,8 +129,8 @@ final class ChromaKeyConfigurationViewController: UIViewController {
         self.backgroundPlaneNode = backgroundPlaneNode
     }
 
-    private func configurePlane(with frame: ARFrame) {
-        let planeNode = ARKitHelpers.makePlaneNodeForDistance(0.1, frame: frame)
+    private func configurePlane(viewPortSize: CGSize, projectionMatrix: simd_float4x4) {
+        let planeNode = ARKitHelpers.makePlaneNodeForDistance(0.1, viewPortSize: viewPortSize, projectionMatrix: projectionMatrix)
         planeNode.geometry?.firstMaterial?.lightingModel = .constant
         planeNode.geometry?.firstMaterial?.transparencyMode = .rgbZero
         planeNode.geometry?.firstMaterial?.shaderModifiers = [.surface: Shaders.surfaceChromaKeyConfiguration()]
@@ -248,6 +248,7 @@ final class ChromaKeyConfigurationViewController: UIViewController {
                 return
             }
 
+            // FIXME: Image is rotated!
             self.maskImage = ChromaKeyMaskBuilder.buildMask(for: currentFrame, chromaConfiguration: currentConfiguration())
         } else {
             self.maskImage = nil
@@ -284,8 +285,12 @@ extension ChromaKeyConfigurationViewController: ARSessionDelegate {
 
     func session(_ session: ARSession, didUpdate frame: ARFrame) {
         if first {
-            configureBackgroundPlane(with: frame)
-            configurePlane(with: frame)
+            let viewPortSize = sceneView.bounds.size
+            let interfaceOrientation = UIApplication.shared.windows.first?.windowScene?.interfaceOrientation ?? UIInterfaceOrientation.unknown
+            let projectionMatrix = frame.camera.projectionMatrix(for: interfaceOrientation, viewportSize: viewPortSize, zNear: 0.001, zFar: 1000.0)
+
+            configureBackgroundPlane(viewPortSize: viewPortSize, projectionMatrix: projectionMatrix)
+            configurePlane(viewPortSize: viewPortSize, projectionMatrix: projectionMatrix)
             didUpdateValues()
             first = false
         }

RealityMixer/Capture/ViewControllers/MixedRealityViewController.swift

@@ -129,9 +129,9 @@ final class MixedRealityViewController: UIViewController {
         NotificationCenter.default.addObserver(self, selector: #selector(willResignActive), name: UIApplication.willResignActiveNotification, object: nil)
     }
 
-    private func configureBackground(with frame: ARFrame) {
+    private func configureBackground(viewPortSize: CGSize, projectionMatrix: simd_float4x4) {
         if case .hidden = configuration.backgroundLayerOptions.visibility { return }
-        let backgroundPlaneNode = ARKitHelpers.makePlaneNodeForDistance(100.0, frame: frame)
+        let backgroundPlaneNode = ARKitHelpers.makePlaneNodeForDistance(100.0, viewPortSize: viewPortSize, projectionMatrix: projectionMatrix)
 
         // Flipping image
         if configuration.shouldFlipOutput {
@@ -162,11 +162,11 @@ final class MixedRealityViewController: UIViewController {
         self.backgroundNode = backgroundPlaneNode
     }
 
-    private func configureMiddle(with frame: ARFrame) {
+    private func configureMiddle(viewPortSize: CGSize, projectionMatrix: simd_float4x4) {
         guard case .greenScreen = configuration.captureMode,
             let chromaConfiguration = chromaConfiguration
         else { return }
-        let middlePlaneNode = ARKitHelpers.makePlaneNodeForDistance(0.02, frame: frame)
+        let middlePlaneNode = ARKitHelpers.makePlaneNodeForDistance(0.02, viewPortSize: viewPortSize, projectionMatrix: projectionMatrix)
 
         middlePlaneNode.geometry?.firstMaterial?.transparencyMode = .rgbZero
 
@@ -203,9 +203,9 @@ final class MixedRealityViewController: UIViewController {
         self.middlePlaneNode = middlePlaneNode
     }
 
-    private func configureForeground(with frame: ARFrame) {
+    private func configureForeground(viewPortSize: CGSize, projectionMatrix: simd_float4x4) {
         guard case .visible(let useMagentaAsTransparency) = configuration.foregroundLayerOptions.visibility else { return }
-        let foregroundPlaneNode = ARKitHelpers.makePlaneNodeForDistance(0.01, frame: frame)
+        let foregroundPlaneNode = ARKitHelpers.makePlaneNodeForDistance(0.01, viewPortSize: viewPortSize, projectionMatrix: projectionMatrix)
 
         // Flipping image
         if configuration.shouldFlipOutput {
@@ -344,11 +344,16 @@ extension MixedRealityViewController: ARSessionDelegate {
     func session(_ session: ARSession, didUpdate frame: ARFrame) {
 
         if first {
-            configureBackground(with: frame)
-            configureMiddle(with: frame)
-            configureForeground(with: frame)
+            let viewPortSize = sceneView.bounds.size
+            let interfaceOrientation = UIApplication.shared.windows.first?.windowScene?.interfaceOrientation ?? UIInterfaceOrientation.unknown
+            let projectionMatrix = frame.camera.projectionMatrix(for: interfaceOrientation, viewportSize: viewPortSize, zNear: 0.001, zFar: 1000.0)
+
+            configureBackground(viewPortSize: viewPortSize, projectionMatrix: projectionMatrix)
+            configureMiddle(viewPortSize: viewPortSize, projectionMatrix: projectionMatrix)
+            configureForeground(viewPortSize: viewPortSize, projectionMatrix: projectionMatrix)
             first = false
         } else {
+            // TODO: Check this (this might need to be updated)
             cameraPoseSender?.didUpdate(frame: frame)
         }
 

RealityMixer/Info.plist

@@ -47,11 +47,8 @@
 	<string>UIStatusBarStyleLightContent</string>
 	<key>UISupportedInterfaceOrientations</key>
 	<array>
-		<string>UIInterfaceOrientationLandscapeRight</string>
-	</array>
-	<key>UISupportedInterfaceOrientations~ipad</key>
-	<array>
-		<string>UIInterfaceOrientationLandscapeRight</string>
+		<string>UIInterfaceOrientationPortrait</string>
+		<string>UIInterfaceOrientationPortraitUpsideDown</string>
 	</array>
 	<key>UIUserInterfaceStyle</key>
 	<string>Light</string>