Distortion

unity/Assets/Scripts/AgentManager.cs

@@ -51,6 +51,7 @@ public class AgentManager : MonoBehaviour, ActionInvokable {
     private bool initializedInstanceSeg;
     private bool renderNormalsImage;
     private bool renderFlowImage;
+    private bool renderDistortionImage;
     private Socket sock = null;
 
     [SerializeField]
@@ -88,7 +89,8 @@ private enum serverTypes {
         "UpdateThirdPartyCamera",
         "ChangeResolution",
         "CoordinateFromRaycastThirdPartyCamera",
-        "ChangeQuality"
+        "ChangeQuality",
+        "SetDistortionShaderParams"
     };
     public HashSet<string> errorAllowedActions = new HashSet<string> { "Reset" };
 
@@ -334,6 +336,7 @@ public void Initialize(ServerAction action) {
         this.renderInstanceSegmentation = this.initializedInstanceSeg =
             action.renderInstanceSegmentation;
         this.renderFlowImage = action.renderFlowImage;
+        this.renderDistortionImage = action.renderDistortionImage;
         this.fastActionEmit = action.fastActionEmit;
 
         PhysicsSceneManager.SetDefaultSimulationParams(action.defaultPhysicsSimulationParams);
@@ -774,6 +777,7 @@ public void AddThirdPartyCamera(
             || renderInstanceSegmentation
             || renderNormalsImage
             || renderFlowImage
+            || renderDistortionImage
         ) {
             gameObject.AddComponent(typeof(ImageSynthesis));
         }
@@ -1837,6 +1841,22 @@ protected bool LoadBoolVariable(bool variable, string name) {
     public void SetCriticalErrorState() {
         this.agentManagerState = AgentState.Error;
     }
+
+    public ActionFinished SetDistortionShaderParams(float zoomPercent, float k1, float k2, float k3, float k4, float strength = 1.0f) {
+
+        if (this.primaryAgent.imageSynthesis == null) {
+            return new ActionFinished(success: false, errorMessage: "No imageSynthesis, make sure you pass 'renderDistortionImage = true' to the agent constructor.");
+        }
+        var material = this.primaryAgent.imageSynthesis.distortionMaterial;
+        material.SetFloat("_ZoomPercent", zoomPercent);
+        material.SetFloat("_k1", k1);
+        material.SetFloat("_k2", k2);
+        material.SetFloat("_k3", k3);
+        material.SetFloat("_k4", k4);
+        material.SetFloat("_LensDistortionStrength", strength);
+        return ActionFinished.Success;
+
+    }
 }
 
 [Serializable]
@@ -2589,6 +2609,7 @@ public class ServerAction {
     public bool renderInstanceSegmentation;
     public bool renderNormalsImage;
     public bool renderFlowImage;
+    public bool renderDistortionImage;
     public float cameraY = 0.675f;
     public bool placeStationary = true; // when placing/spawning an object, do we spawn it stationary (kinematic true) or spawn and let physics resolve final position
 

unity/Assets/Scripts/DebugInputField.cs

@@ -2677,6 +2677,20 @@ IEnumerator executeBatch(JArray jActions) {
                         break;
                     }
 
+                case "sdp": {
+                    Dictionary<string, object> action = new Dictionary<string, object>() {
+                            ["action"] = "SetDistortionShaderParams",
+                            ["zoomPercent"] = 0.84f,
+                            ["k1"] = 0.1f,
+                            ["k2"] = 0.01f,
+                            ["k3"] = -0.2f,
+                            ["k4"] = 0.0,
+                            ["strength"] = 1.0
+                        };
+                    this.AManager.ProcessControlCommand(new DynamicServerAction(action));
+                    break;
+                }
+
                 case "gopro1": {
                         Dictionary<string, object> action = new Dictionary<string, object>() {
                             ["action"] = "UpdateMainCamera",

unity/Assets/Scripts/ImageSynthesis/ImageSynthesis.cs

@@ -204,6 +204,8 @@ public class ImageSynthesis : MonoBehaviour {
             needsRescale = true
         }, // (see issue with Motion Vectors in @KNOWN ISSUES)
 
+        new CapturePass() { name = "_distortion" }
+
         // new CapturePass() { name = "_position" },
     };
 
@@ -245,7 +247,7 @@ public void updateCameraStatuses(bool enabled) {
     private Shader opticalFlowShader;
     private Shader depthShader;
 
-    private Shader screenCopyShader;
+    private Shader distortionShader;
 
     // public Shader positionShader;
 
@@ -259,7 +261,7 @@ public void updateCameraStatuses(bool enabled) {
     private Material opticalFlowMaterial;
     public Material depthMaterial;
 
-    private Material screenCopyMaterial;
+    public Material distortionMaterial;
 
     private RenderTexture renderTexture;
     System.Security.Cryptography.MD5 md5;
@@ -299,14 +301,14 @@ public void OnEnable() {
                 depthShader = Shader.Find("Hidden/Depth");
 
 #endif
-            if (!screenCopyShader) {
-                screenCopyShader = Shader.Find("Hidden/ScreenCopy");
+            if (!distortionShader) {
+                distortionShader = Shader.Find("Hidden/BarrelDistortion");
             }
 
             // if (!positionShader)
             //	positionShader = Shader.Find("Hidden/World");
 
-            opticalFlowSensitivity = 40.0f;
+            opticalFlowSensitivity = 50.0f;
 
             // use real camera to capture final image
             capturePasses[0].camera = GetComponent<Camera>();
@@ -484,8 +486,8 @@ public void OnCameraChange() {
 
         // screenCopyMaterial = new Material(screenCopyShader);
 
-        if (!screenCopyMaterial || screenCopyMaterial.shader != screenCopyShader) {
-            screenCopyMaterial = new Material(screenCopyShader);
+        if (!distortionMaterial || distortionMaterial.shader != distortionShader) {
+            distortionMaterial = new Material(distortionShader);
         }
 
         // capturePasses [1].camera.farClipPlane = 100;
@@ -525,6 +527,10 @@ public void OnCameraChange() {
             DepthTextureMode.Depth | DepthTextureMode.MotionVectors
         );
 
+         SetupCameraWithPostShader2(renderTexture, capturePasses[6].camera, distortionMaterial,
+        //  screenCopyMaterial: screenCopyMaterial,
+          DepthTextureMode.Depth);
+
 #if UNITY_EDITOR
         for (int i = 0; i < capturePasses.Length; i++) {
             // Debug.Log("Setting camera " + capturePasses[i].camera.gameObject.name + " to display " + i);

unity/Assets/Scripts/ImageSynthesis/Shaders/DepthBW.shader

@@ -92,7 +92,7 @@
                  // depth01 = pow(LinearEyeDepth(depth01), _DepthLevel);
                  float depth01 = (Linear01Depth(UNITY_SAMPLE_DEPTH(tex2D(_CameraDepthTexture, o.uv))));
                 //  (LinearEyeDepth(UNITY_SAMPLE_DEPTH(tex2D(_CameraDepthTexture, o.uv)))) / (_ProjectionParams.z - _ProjectionParams.y);
-                //  return fixed4(depth01, depth01, depth01, depth01);
+                 return fixed4(depth01, depth01, depth01, depth01);
                 // return fixed4(o.uv.x, 0, o.uv.y, 1.0);
                 // _ScreenParams.x