Merge pull request #32 from fabio914/background-layer-chroma-key

Experiment: Transparency on the background layer

Instructions.md

@@ -113,8 +113,8 @@ Follow the instructions below if you don't have the record option inside your "C
 
 ## How to improve performance?
 
- - Make sure that your device is using a 5 GHz WiFi connection (e.g. 802.11ac), and that you don't have too many devices using the network at the same time while you're using the app.
+ - Make sure that your device is using a 5 GHz WiFi connection (e.g. 802.11ac), that you don't have too many devices using the network at the same time while you're using the app, and that you have a strong signal.
 
  - Make sure that your device is not on Low Power mode. It is recommended that you keep your device connected to a power outlet while using the app.
 
- - Select a small resoltion scale factor (`0.5x` or `0.25x`) and then calibrate again.
+ - Select a small resolution scale factor (`0.5x` or `0.25x`) and then calibrate again.

RealityMixer.xcodeproj/project.pbxproj

@@ -59,6 +59,7 @@
 		A328D980253CE63B00B90E62 /* MixedRealityViewController.xib in Resources */ = {isa = PBXBuildFile; fileRef = A328D97E253CE63B00B90E62 /* MixedRealityViewController.xib */; };
 		A328D98E253CF2EE00B90E62 /* ARKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = A328D98D253CF2EE00B90E62 /* ARKit.framework */; };
 		A328D990253CF2F400B90E62 /* SceneKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = A328D98F253CF2F400B90E62 /* SceneKit.framework */; };
+		A36C606825AE58FE0026DDE1 /* MixedRealityConfiguration.swift in Sources */ = {isa = PBXBuildFile; fileRef = A36C606725AE58FE0026DDE1 /* MixedRealityConfiguration.swift */; };
 		A39033ED256859A8004CB638 /* Shaders.swift in Sources */ = {isa = PBXBuildFile; fileRef = A39033EC256859A8004CB638 /* Shaders.swift */; };
 /* End PBXBuildFile section */
 
@@ -236,6 +237,7 @@
 		A328D97E253CE63B00B90E62 /* MixedRealityViewController.xib */ = {isa = PBXFileReference; lastKnownFileType = file.xib; path = MixedRealityViewController.xib; sourceTree = "<group>"; };
 		A328D98D253CF2EE00B90E62 /* ARKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = ARKit.framework; path = System/Library/Frameworks/ARKit.framework; sourceTree = SDKROOT; };
 		A328D98F253CF2F400B90E62 /* SceneKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = SceneKit.framework; path = System/Library/Frameworks/SceneKit.framework; sourceTree = SDKROOT; };
+		A36C606725AE58FE0026DDE1 /* MixedRealityConfiguration.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MixedRealityConfiguration.swift; sourceTree = "<group>"; };
 		A37850BC256C433C00C046FA /* Compatibility.md */ = {isa = PBXFileReference; lastKnownFileType = net.daringfireball.markdown; path = Compatibility.md; sourceTree = "<group>"; };
 		A39033EC256859A8004CB638 /* Shaders.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Shaders.swift; sourceTree = "<group>"; };
 /* End PBXFileReference section */
@@ -347,6 +349,7 @@
 			isa = PBXGroup;
 			children = (
 				A32416172554DA24008D4B90 /* Preference.swift */,
+				A36C606725AE58FE0026DDE1 /* MixedRealityConfiguration.swift */,
 			);
 			path = Model;
 			sourceTree = "<group>";
@@ -790,6 +793,7 @@
 				A32416312554E340008D4B90 /* CalibrationSceneNodeBuilder.swift in Sources */,
 				A324163A2554E4F5008D4B90 /* Vector3+SceneKit.swift in Sources */,
 				A32416082554D326008D4B90 /* CalibrationPayload.swift in Sources */,
+				A36C606825AE58FE0026DDE1 /* MixedRealityConfiguration.swift in Sources */,
 				A32416182554DA24008D4B90 /* Preference.swift in Sources */,
 				A32416352554E41E008D4B90 /* ProjectionViewController.swift in Sources */,
 			);
@@ -932,7 +936,7 @@
 				ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
 				CLANG_ENABLE_MODULES = YES;
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 8;
+				CURRENT_PROJECT_VERSION = 9;
 				DEVELOPMENT_TEAM = ZTJ55TM37B;
 				FRAMEWORK_SEARCH_PATHS = (
 					"$(inherited)",
@@ -948,7 +952,7 @@
 					"$(inherited)",
 					"$(PROJECT_DIR)/RealityMixer/FFmpeg-iOS/lib",
 				);
-				MARKETING_VERSION = 0.1.6;
+				MARKETING_VERSION = 0.1.7;
 				PRODUCT_BUNDLE_IDENTIFIER = com.bluenose.reality.mixer;
 				PRODUCT_NAME = "$(TARGET_NAME)";
 				SWIFT_OBJC_BRIDGING_HEADER = "RealityMixer/RealityMixer-Bridging-Header.h";
@@ -965,7 +969,7 @@
 				ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
 				CLANG_ENABLE_MODULES = YES;
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 8;
+				CURRENT_PROJECT_VERSION = 9;
 				DEVELOPMENT_TEAM = ZTJ55TM37B;
 				FRAMEWORK_SEARCH_PATHS = (
 					"$(inherited)",
@@ -981,7 +985,7 @@
 					"$(inherited)",
 					"$(PROJECT_DIR)/RealityMixer/FFmpeg-iOS/lib",
 				);
-				MARKETING_VERSION = 0.1.6;
+				MARKETING_VERSION = 0.1.7;
 				PRODUCT_BUNDLE_IDENTIFIER = com.bluenose.reality.mixer;
 				PRODUCT_NAME = "$(TARGET_NAME)";
 				SWIFT_OBJC_BRIDGING_HEADER = "RealityMixer/RealityMixer-Bridging-Header.h";

RealityMixer/Capture/Model/MixedRealityConfiguration.swift

@@ -0,0 +1,24 @@
+import Foundation
+
+struct MixedRealityConfiguration {
+
+    enum BackgroundVisibility {
+        enum ChromaKey {
+            case black
+            case green
+            case magenta
+        }
+
+        case visible
+        case chromaKey(ChromaKey)
+        case hidden
+    }
+
+    // Use magenta as the transparency color for the foreground plane
+    let shouldUseMagentaAsTransparency: Bool
+
+    let enableAudio: Bool
+    let enableAutoFocus: Bool
+    let shouldFlipOutput: Bool
+    let backgroundVisibility: BackgroundVisibility
+}

RealityMixer/Capture/Shaders/Shaders.swift

@@ -44,6 +44,31 @@ struct Shaders {
         b = BT709_nonLinearNormToLinear(b);
         return vec4(r, g, b, 1.0);
     }
+
+    """
+
+    static let chromaKey = """
+    float chromaKey(vec3 c, vec3 maskColor) {
+        float maskY = 0.2989 * maskColor.r + 0.5866 * maskColor.g + 0.1145 * maskColor.b;
+        float maskCr = 0.7132 * (maskColor.r - maskY);
+        float maskCb = 0.5647 * (maskColor.b - maskY);
+
+        float Y = 0.2989 * c.r + 0.5866 * c.g + 0.1145 * c.b;
+        float Cr = 0.7132 * (c.r - Y);
+        float Cb = 0.5647 * (c.b - Y);
+
+        if (distance(vec2(Cr, Cb), vec2(maskCr, maskCb)) < 0.18) {
+            return 1.0;
+        } else {
+            return 0.0;
+        }
+
+        //float sensitivity = 0.18; // 0 ... 1.0
+        //float smooth = 0.1; // 0 ... 1.0
+
+        //return 1.0 - smoothstep(sensitivity, sensitivity + smooth, distance(vec2(Cr, Cb), vec2(maskCr, maskCb)));
+    }
+
     """
 
     static let backgroundSurface = """
@@ -53,13 +78,56 @@ struct Shaders {
 
     vec2 backgroundCoords = vec2((_surface.diffuseTexcoord.x * 0.5), _surface.diffuseTexcoord.y);
 
-    float luma = texture2D(u_ambientTexture, backgroundCoords).r;
+    float luma = texture2D(u_transparentTexture, backgroundCoords).r;
+    vec2 chroma = texture2D(u_diffuseTexture, backgroundCoords).rg;
+
+    _surface.diffuse = yCbCrToRGB(luma, chroma);
+    _surface.transparent = vec4(0.0, 0.0, 0.0, 1.0);
+    """
+
+    static func backgroundSurfaceChromaKey(red: Float, green: Float, blue: Float) -> String {
+        """
+        \(yCrCbToRGB)
+        \(chromaKey)
+
+        #pragma body
+
+        vec2 backgroundCoords = vec2((_surface.diffuseTexcoord.x * 0.5), _surface.diffuseTexcoord.y);
+
+        float luma = texture2D(u_transparentTexture, backgroundCoords).r;
+        vec2 chroma = texture2D(u_diffuseTexture, backgroundCoords).rg;
+
+        vec4 textureColor = yCbCrToRGB(luma, chroma);
+        _surface.diffuse = textureColor;
+
+        float blendValue = chromaKey(textureColor.rgb, vec3(\(red), \(green), \(blue)));
+        _surface.transparent = vec4(blendValue, blendValue, blendValue, 1.0);
+        """
+    }
+
+    static let backgroundSurfaceWithBlackChromaKey = """
+    \(yCrCbToRGB)
+
+    #pragma body
+
+    vec2 backgroundCoords = vec2((_surface.diffuseTexcoord.x * 0.5), _surface.diffuseTexcoord.y);
+
+    float luma = texture2D(u_transparentTexture, backgroundCoords).r;
     vec2 chroma = texture2D(u_diffuseTexture, backgroundCoords).rg;
 
     _surface.diffuse = yCbCrToRGB(luma, chroma);
-    _surface.ambient = vec4(0.0, 0.0, 0.0, 1.0);
+
+    if (luma < 0.13) {
+        _surface.transparent = vec4(1.0, 1.0, 1.0, 1.0);
+    } else {
+        _surface.transparent = vec4(0.0, 0.0, 0.0, 1.0);
+    }
+
     """
 
+    static let backgroundSurfaceWithGreenChromaKey = backgroundSurfaceChromaKey(red: 0, green: 1, blue: 0)
+    static let backgroundSurfaceWithMagentaChromaKey = backgroundSurfaceChromaKey(red: 1, green: 0, blue: 1)
+
     static let foregroundSurfaceShared = """
     \(yCrCbToRGB)
 
@@ -92,6 +160,7 @@ struct Shaders {
     """
 
     static let magentaForegroundSurface = """
+    \(chromaKey)
     \(foregroundSurfaceShared)
 
     vec2 alphaCoords = vec2((_surface.transparentTexcoord.x * 0.25) + 0.5, _surface.transparentTexcoord.y);
@@ -101,19 +170,7 @@ struct Shaders {
 
     vec4 alphaColor = yCbCrToRGB(luma2, chroma2);
 
-    vec3 magenta = vec3(1.0, 0.0, 1.0);
-    float threshold = 0.10;
-
-    // FIXME: Consider using HSV to compare these colors
-    bool checkRed = (alphaColor.r >= (magenta.r - threshold));
-    bool checkGreen = (alphaColor.g >= (magenta.g - threshold) && alphaColor.g <= (magenta.g + threshold));
-    bool checkBlue = (alphaColor.b >= (magenta.b - threshold));
-
-    if (checkRed && checkGreen && checkBlue) {
-        // FIXME: This is not ideal, this is ignoring semi-transparent pixels
-        _surface.transparent = vec4(1.0, 1.0, 1.0, 1.0);
-    } else {
-        _surface.transparent = vec4(0.0, 0.0, 0.0, 1.0);
-    }
+    float blendValue = chromaKey(alphaColor.rgb, vec3(1.0, 0.0, 1.0));
+    _surface.transparent = vec4(blendValue, blendValue, blendValue, 1.0);
     """
 }

RealityMixer/Capture/ViewControllers/MixedRealityConnectionViewController.swift

@@ -17,7 +17,9 @@ final class MixedRealityConnectionViewController: UIViewController {
     @IBOutlet private weak var autoFocusSwitch: UISwitch!
     @IBOutlet private weak var magentaSwitch: UISwitch!
     @IBOutlet private weak var unflipSwitch: UISwitch!
-    @IBOutlet private weak var hideBackgroundSwitch: UISwitch!
+    @IBOutlet private weak var backgroundVisibilitySegmentedControl: UISegmentedControl!
+    @IBOutlet private weak var backgroundChromaKeySection: UIStackView!
+    @IBOutlet private weak var backgroundChromaKeySegmentedControl: UISegmentedControl!
     @IBOutlet private weak var infoLabel: UILabel!
     @IBOutlet private weak var secondInfoLabel: UILabel!
     @IBOutlet private weak var thirdInfoLabel: UILabel!
@@ -60,7 +62,7 @@ final class MixedRealityConnectionViewController: UIViewController {
         """
 
         secondInfoLabel.text = """
-         • Make sure that the Quest and this device are both connected to the same WiFi network. A 5 Ghz WiFi is recommended.
+         • Make sure that the Quest and this device are both connected to the same WiFi network, and that both have a strong signal. A 5 Ghz WiFi is recommended.
 
          • Make sure that the Reality Mixer app is allowed to access your camera and your local network. It'll ask for permission the first time you launch the calibration or mixed reality, however, you'll need to navigate to the system settings to be able to re-enable these permissions if you haven't given permissions during the first launch.
         """
@@ -125,7 +127,7 @@ final class MixedRealityConnectionViewController: UIViewController {
                     enableAudio: self.audioSwitch.isOn,
                     enableAutoFocus: self.autoFocusSwitch.isOn,
                     shouldFlipOutput: !self.unflipSwitch.isOn,
-                    shouldHideBackground: self.hideBackgroundSwitch.isOn
+                    backgroundVisibility: self.backgroundVisibility()
                 )
 
                 connectionAlert.dismiss(animated: false, completion: { [weak self] in
@@ -142,6 +144,30 @@ final class MixedRealityConnectionViewController: UIViewController {
         })
     }
 
+    private func backgroundVisibility() -> MixedRealityConfiguration.BackgroundVisibility {
+        switch backgroundVisibilitySegmentedControl.selectedSegmentIndex {
+        case 0:
+            return .visible
+        case 1:
+            return .chromaKey({
+                switch backgroundChromaKeySegmentedControl.selectedSegmentIndex {
+                case 0:
+                    return .black
+                case 1:
+                    return .green
+                default:
+                    return .magenta
+                }
+            }())
+        default:
+            return .hidden
+        }
+    }
+
+    @IBAction func backgroundVisibilityDidChange(_ sender: UISegmentedControl) {
+        backgroundChromaKeySection.isHidden = sender.selectedSegmentIndex != 1
+    }
+
     @IBAction private func startCalibrationAction(_ sender: Any) {
         let otherNavigationController = UINavigationController(rootViewController: CalibrationConnectionViewController())
         otherNavigationController.modalPresentationStyle = .overFullScreen