From 33248689a116730eb6c4e21d6bfbb48f20529a2f Mon Sep 17 00:00:00 2001 From: brimson Date: Thu, 21 Jul 2022 00:36:37 -0700 Subject: [PATCH] Invert the LK equation --- shaders/cMotionBlur.fx | 58 ++++++++++++++++++++------------------- shaders/cOpticalFlowLK.fx | 56 +++++++++++++++++++------------------ shaders/kDatamosh.fx | 52 ++++++++++++++++++----------------- 3 files changed, 86 insertions(+), 80 deletions(-) diff --git a/shaders/cMotionBlur.fx b/shaders/cMotionBlur.fx index e4f7270..ec882a3 100644 --- a/shaders/cMotionBlur.fx +++ b/shaders/cMotionBlur.fx @@ -251,7 +251,7 @@ namespace MotionBlur OutputColor0 = tex2D(Sample_Common_0, TexCoord); } - static const float BlurWeights[10] = + static const float BlurWeights[10] = { 0.06299088, 0.122137636, 0.10790718, 0.08633988, @@ -312,9 +312,14 @@ namespace MotionBlur void Derivatives_Temporal_PS(in float4 Position : SV_POSITION, in float2 TexCoord : TEXCOORD0, out float2 OutputColor0 : SV_TARGET0) { - float Current = tex2D(Sample_Common_1_A, TexCoord).x; - float Previous = tex2D(Sample_Common_1_C, TexCoord).x; - OutputColor0 = Current - Previous; + float I0 = tex2D(Sample_Common_1_C, TexCoord).x; + float I1 = tex2D(Sample_Common_1_A, TexCoord).x; + OutputColor0 = I0 - I1; + } + + void Copy_PS(in float4 Position : SV_POSITION, in float2 TexCoord : TEXCOORD0, out float4 OutputColor0 : SV_TARGET0) + { + OutputColor0 = tex2D(Sample_Common_1_A, TexCoord); } void Derivatives_Spatial_PS(in float4 Position : SV_POSITION, in float4 TexCoords[2] : TEXCOORD0, out float2 OutputColor0 : SV_TARGET0) @@ -324,14 +329,14 @@ namespace MotionBlur // A0 A1 // A2 B0 // C0 C1 - float A0 = tex2D(Sample_Common_1_A, TexCoords[0].xw).x * 4.0; // <-1.5, +0.5> - float A1 = tex2D(Sample_Common_1_A, TexCoords[0].yw).x * 4.0; // <+1.5, +0.5> - float A2 = tex2D(Sample_Common_1_A, TexCoords[0].xz).x * 4.0; // <-1.5, -0.5> - float B0 = tex2D(Sample_Common_1_A, TexCoords[0].yz).x * 4.0; // <+1.5, -0.5> - float B1 = tex2D(Sample_Common_1_A, TexCoords[1].xw).x * 4.0; // <-0.5, +1.5> - float B2 = tex2D(Sample_Common_1_A, TexCoords[1].yw).x * 4.0; // <+0.5, +1.5> - float C0 = tex2D(Sample_Common_1_A, TexCoords[1].xz).x * 4.0; // <-0.5, -1.5> - float C1 = tex2D(Sample_Common_1_A, TexCoords[1].yz).x * 4.0; // <+0.5, -1.5> + float A0 = tex2D(Sample_Common_1_C, TexCoords[0].xw).x * 4.0; // <-1.5, +0.5> + float A1 = tex2D(Sample_Common_1_C, TexCoords[0].yw).x * 4.0; // <+1.5, +0.5> + float A2 = tex2D(Sample_Common_1_C, TexCoords[0].xz).x * 4.0; // <-1.5, -0.5> + float B0 = tex2D(Sample_Common_1_C, TexCoords[0].yz).x * 4.0; // <+1.5, -0.5> + float B1 = tex2D(Sample_Common_1_C, TexCoords[1].xw).x * 4.0; // <-0.5, +1.5> + float B2 = tex2D(Sample_Common_1_C, TexCoords[1].yw).x * 4.0; // <+0.5, +1.5> + float C0 = tex2D(Sample_Common_1_C, TexCoords[1].xz).x * 4.0; // <-0.5, -1.5> + float C1 = tex2D(Sample_Common_1_C, TexCoords[1].yz).x * 4.0; // <+0.5, -1.5> OutputColor0 = 0.0; OutputColor0.x = ((B2 + A1 + B0 + C1) - (B1 + A0 + A2 + C0)) / 12.0; @@ -344,10 +349,10 @@ namespace MotionBlur // [TexCoord.xw TexCoord.zw] // [TexCoord.xy TexCoord.zy] float2 S[4]; - S[0] = tex2D(Sample_Common_1_C, TexCoord.xw).xy; - S[1] = tex2D(Sample_Common_1_C, TexCoord.zw).xy; - S[2] = tex2D(Sample_Common_1_C, TexCoord.xy).xy; - S[3] = tex2D(Sample_Common_1_C, TexCoord.zy).xy; + S[0] = tex2D(Sample_Common_1_A, TexCoord.xw).xy; + S[1] = tex2D(Sample_Common_1_A, TexCoord.zw).xy; + S[2] = tex2D(Sample_Common_1_A, TexCoord.xy).xy; + S[3] = tex2D(Sample_Common_1_A, TexCoord.zy).xy; // A.x = Ix^2 (A11); A.y = Iy^2 (A22); A.z = IxIy (A12) float3 A = 0.0; @@ -409,11 +414,6 @@ namespace MotionBlur Color = float4(Lucas_Kanade(0, Average2D(Sample_Common_2, Tex).xy, Tex), 0.0, _BlendFactor); } - void Copy_PS(in float4 Position : SV_POSITION, in float2 TexCoord : TEXCOORD0, out float4 OutputColor0 : SV_TARGET0) - { - OutputColor0 = tex2D(Sample_Common_1_A, TexCoord); - } - void Post_Blur_0_PS(in float4 Position : SV_POSITION, in float4 TexCoords[7] : TEXCOORD0, out float4 OutputColor0 : SV_TARGET0) { Gaussian_Blur(Sample_Optical_Flow, TexCoords, false, OutputColor0); @@ -469,13 +469,18 @@ namespace MotionBlur // Scale frame CREATE_PASS(Basic_VS, Blit_Frame_PS, Render_Common_1_A) - // Gaussian blur + // Pre-process Gaussian blur CREATE_PASS(Pre_Blur_0_VS, Pre_Blur_0_PS, Render_Common_1_B) CREATE_PASS(Pre_Blur_1_VS, Pre_Blur_1_PS, Render_Common_1_A) // Save this to store later - // Calculate derivative pyramids + // Calculate temporal derivative pyramids CREATE_PASS(Basic_VS, Derivatives_Temporal_PS, Render_Common_1_B) - CREATE_PASS(Derivatives_Spatial_VS, Derivatives_Spatial_PS, Render_Common_1_C) + + // Copy current convolved frame for next frame + CREATE_PASS(Basic_VS, Copy_PS, Render_Common_1_C) + + // Calculate spatial derivative pyramids + CREATE_PASS(Derivatives_Spatial_VS, Derivatives_Spatial_PS, Render_Common_1_A) // Bilinear Lucas-Kanade Optical Flow CREATE_PASS(LK_Level_6_VS, LK_Level_6_PS, Render_Common_6) @@ -496,7 +501,7 @@ namespace MotionBlur DestBlend = SRCALPHA; } - // Gaussian blur + // Post-process Gaussian blur CREATE_PASS(Post_Blur_0_VS, Post_Blur_0_PS, Render_Common_3_B) CREATE_PASS(Post_Blur_1_VS, Post_Blur_1_PS, Render_Common_3_A) @@ -509,8 +514,5 @@ namespace MotionBlur SRGBWriteEnable = TRUE; #endif } - - // Copy current convolved frame for next frame - CREATE_PASS(Basic_VS, Copy_PS, Render_Common_1_C) } } \ No newline at end of file diff --git a/shaders/cOpticalFlowLK.fx b/shaders/cOpticalFlowLK.fx index 751d8f1..d894ee3 100644 --- a/shaders/cOpticalFlowLK.fx +++ b/shaders/cOpticalFlowLK.fx @@ -305,9 +305,14 @@ namespace OpticalFlowLK void Derivatives_Temporal_PS(in float4 Position : SV_POSITION, in float2 TexCoord : TEXCOORD0, out float2 OutputColor0 : SV_TARGET0) { - float Current = tex2D(Sample_Common_1_A, TexCoord).x; - float Previous = tex2D(Sample_Common_1_C, TexCoord).x; - OutputColor0 = Current - Previous; + float I0 = tex2D(Sample_Common_1_C, TexCoord).x; + float I1 = tex2D(Sample_Common_1_A, TexCoord).x; + OutputColor0 = I0 - I1; + } + + void Copy_PS(in float4 Position : SV_POSITION, in float2 TexCoord : TEXCOORD0, out float4 OutputColor0 : SV_TARGET0) + { + OutputColor0 = tex2D(Sample_Common_1_A, TexCoord); } void Derivatives_Spatial_PS(in float4 Position : SV_POSITION, in float4 TexCoords[2] : TEXCOORD0, out float2 OutputColor0 : SV_TARGET0) @@ -317,14 +322,14 @@ namespace OpticalFlowLK // A0 A1 // A2 B0 // C0 C1 - float A0 = tex2D(Sample_Common_1_A, TexCoords[0].xw).x * 4.0; // <-1.5, +0.5> - float A1 = tex2D(Sample_Common_1_A, TexCoords[0].yw).x * 4.0; // <+1.5, +0.5> - float A2 = tex2D(Sample_Common_1_A, TexCoords[0].xz).x * 4.0; // <-1.5, -0.5> - float B0 = tex2D(Sample_Common_1_A, TexCoords[0].yz).x * 4.0; // <+1.5, -0.5> - float B1 = tex2D(Sample_Common_1_A, TexCoords[1].xw).x * 4.0; // <-0.5, +1.5> - float B2 = tex2D(Sample_Common_1_A, TexCoords[1].yw).x * 4.0; // <+0.5, +1.5> - float C0 = tex2D(Sample_Common_1_A, TexCoords[1].xz).x * 4.0; // <-0.5, -1.5> - float C1 = tex2D(Sample_Common_1_A, TexCoords[1].yz).x * 4.0; // <+0.5, -1.5> + float A0 = tex2D(Sample_Common_1_C, TexCoords[0].xw).x * 4.0; // <-1.5, +0.5> + float A1 = tex2D(Sample_Common_1_C, TexCoords[0].yw).x * 4.0; // <+1.5, +0.5> + float A2 = tex2D(Sample_Common_1_C, TexCoords[0].xz).x * 4.0; // <-1.5, -0.5> + float B0 = tex2D(Sample_Common_1_C, TexCoords[0].yz).x * 4.0; // <+1.5, -0.5> + float B1 = tex2D(Sample_Common_1_C, TexCoords[1].xw).x * 4.0; // <-0.5, +1.5> + float B2 = tex2D(Sample_Common_1_C, TexCoords[1].yw).x * 4.0; // <+0.5, +1.5> + float C0 = tex2D(Sample_Common_1_C, TexCoords[1].xz).x * 4.0; // <-0.5, -1.5> + float C1 = tex2D(Sample_Common_1_C, TexCoords[1].yz).x * 4.0; // <+0.5, -1.5> OutputColor0 = 0.0; OutputColor0.x = ((B2 + A1 + B0 + C1) - (B1 + A0 + A2 + C0)) / 12.0; @@ -337,10 +342,10 @@ namespace OpticalFlowLK // [TexCoord.xw TexCoord.zw] // [TexCoord.xy TexCoord.zy] float2 S[4]; - S[0] = tex2D(Sample_Common_1_C, TexCoord.xw).xy; - S[1] = tex2D(Sample_Common_1_C, TexCoord.zw).xy; - S[2] = tex2D(Sample_Common_1_C, TexCoord.xy).xy; - S[3] = tex2D(Sample_Common_1_C, TexCoord.zy).xy; + S[0] = tex2D(Sample_Common_1_A, TexCoord.xw).xy; + S[1] = tex2D(Sample_Common_1_A, TexCoord.zw).xy; + S[2] = tex2D(Sample_Common_1_A, TexCoord.xy).xy; + S[3] = tex2D(Sample_Common_1_A, TexCoord.zy).xy; // A.x = Ix^2 (A11); A.y = Iy^2 (A22); A.z = IxIy (A12) float3 A = 0.0; @@ -402,11 +407,6 @@ namespace OpticalFlowLK Color = float4(Lucas_Kanade(0, Average2D(Sample_Common_2, Tex).xy, Tex), 0.0, _BlendFactor); } - void Copy_PS(in float4 Position : SV_POSITION, in float2 TexCoord : TEXCOORD0, out float4 OutputColor0 : SV_TARGET0) - { - OutputColor0 = tex2D(Sample_Common_1_A, TexCoord); - } - void Post_Blur_0_PS(in float4 Position : SV_POSITION, in float4 TexCoords[7] : TEXCOORD0, out float4 OutputColor0 : SV_TARGET0) { Gaussian_Blur(Sample_Optical_Flow, TexCoords, false, OutputColor0); @@ -445,13 +445,18 @@ namespace OpticalFlowLK // Scale frame CREATE_PASS(Basic_VS, Blit_Frame_PS, Render_Common_1_A) - // Gaussian blur + // Pre-process Gaussian blur CREATE_PASS(Pre_Blur_0_VS, Pre_Blur_0_PS, Render_Common_1_B) CREATE_PASS(Pre_Blur_1_VS, Pre_Blur_1_PS, Render_Common_1_A) // Save this to store later - // Calculate derivative pyramids + // Calculate temporal derivative pyramids CREATE_PASS(Basic_VS, Derivatives_Temporal_PS, Render_Common_1_B) - CREATE_PASS(Derivatives_Spatial_VS, Derivatives_Spatial_PS, Render_Common_1_C) + + // Copy current convolved frame for next frame + CREATE_PASS(Basic_VS, Copy_PS, Render_Common_1_C) + + // Calculate spatial derivative pyramids + CREATE_PASS(Derivatives_Spatial_VS, Derivatives_Spatial_PS, Render_Common_1_A) // Bilinear Lucas-Kanade Optical Flow CREATE_PASS(LK_Level_6_VS, LK_Level_6_PS, Render_Common_6) @@ -472,7 +477,7 @@ namespace OpticalFlowLK DestBlend = SRCALPHA; } - // Gaussian blur + // Post-process Gaussian blur CREATE_PASS(Post_Blur_0_VS, Post_Blur_0_PS, Render_Common_3_B) CREATE_PASS(Post_Blur_1_VS, Post_Blur_1_PS, Render_Common_3_A) @@ -482,8 +487,5 @@ namespace OpticalFlowLK VertexShader = Basic_VS; PixelShader = Display_PS; } - - // Copy current convolved frame for next frame - CREATE_PASS(Basic_VS, Copy_PS, Render_Common_1_C) } } diff --git a/shaders/kDatamosh.fx b/shaders/kDatamosh.fx index c835a94..b4139f4 100644 --- a/shaders/kDatamosh.fx +++ b/shaders/kDatamosh.fx @@ -377,9 +377,14 @@ namespace Datamosh void Derivatives_Temporal_PS(in float4 Position : SV_POSITION, in float2 TexCoord : TEXCOORD0, out float2 OutputColor0 : SV_TARGET0) { - float Current = tex2D(Sample_Common_1_A, TexCoord).x; - float Previous = tex2D(Sample_Common_1_C, TexCoord).x; - OutputColor0 = Current - Previous; + float I0 = tex2D(Sample_Common_1_C, TexCoord).x; + float I1 = tex2D(Sample_Common_1_A, TexCoord).x; + OutputColor0 = I0 - I1; + } + + void Copy_PS(in float4 Position : SV_POSITION, in float2 TexCoord : TEXCOORD0, out float4 OutputColor0 : SV_TARGET0) + { + OutputColor0 = tex2D(Sample_Common_1_A, TexCoord); } void Derivatives_Spatial_PS(in float4 Position : SV_POSITION, in float4 TexCoords[2] : TEXCOORD0, out float2 OutputColor0 : SV_TARGET0) @@ -389,14 +394,14 @@ namespace Datamosh // A0 A1 // A2 B0 // C0 C1 - float A0 = tex2D(Sample_Common_1_A, TexCoords[0].xw).x * 4.0; // <-1.5, +0.5> - float A1 = tex2D(Sample_Common_1_A, TexCoords[0].yw).x * 4.0; // <+1.5, +0.5> - float A2 = tex2D(Sample_Common_1_A, TexCoords[0].xz).x * 4.0; // <-1.5, -0.5> - float B0 = tex2D(Sample_Common_1_A, TexCoords[0].yz).x * 4.0; // <+1.5, -0.5> - float B1 = tex2D(Sample_Common_1_A, TexCoords[1].xw).x * 4.0; // <-0.5, +1.5> - float B2 = tex2D(Sample_Common_1_A, TexCoords[1].yw).x * 4.0; // <+0.5, +1.5> - float C0 = tex2D(Sample_Common_1_A, TexCoords[1].xz).x * 4.0; // <-0.5, -1.5> - float C1 = tex2D(Sample_Common_1_A, TexCoords[1].yz).x * 4.0; // <+0.5, -1.5> + float A0 = tex2D(Sample_Common_1_C, TexCoords[0].xw).x * 4.0; // <-1.5, +0.5> + float A1 = tex2D(Sample_Common_1_C, TexCoords[0].yw).x * 4.0; // <+1.5, +0.5> + float A2 = tex2D(Sample_Common_1_C, TexCoords[0].xz).x * 4.0; // <-1.5, -0.5> + float B0 = tex2D(Sample_Common_1_C, TexCoords[0].yz).x * 4.0; // <+1.5, -0.5> + float B1 = tex2D(Sample_Common_1_C, TexCoords[1].xw).x * 4.0; // <-0.5, +1.5> + float B2 = tex2D(Sample_Common_1_C, TexCoords[1].yw).x * 4.0; // <+0.5, +1.5> + float C0 = tex2D(Sample_Common_1_C, TexCoords[1].xz).x * 4.0; // <-0.5, -1.5> + float C1 = tex2D(Sample_Common_1_C, TexCoords[1].yz).x * 4.0; // <+0.5, -1.5> OutputColor0 = 0.0; OutputColor0.x = ((B2 + A1 + B0 + C1) - (B1 + A0 + A2 + C0)) / 12.0; @@ -409,10 +414,10 @@ namespace Datamosh // [TexCoord.xw TexCoord.zw] // [TexCoord.xy TexCoord.zy] float2 S[4]; - S[0] = tex2D(Sample_Common_1_C, TexCoord.xw).xy; - S[1] = tex2D(Sample_Common_1_C, TexCoord.zw).xy; - S[2] = tex2D(Sample_Common_1_C, TexCoord.xy).xy; - S[3] = tex2D(Sample_Common_1_C, TexCoord.zy).xy; + S[0] = tex2D(Sample_Common_1_A, TexCoord.xw).xy; + S[1] = tex2D(Sample_Common_1_A, TexCoord.zw).xy; + S[2] = tex2D(Sample_Common_1_A, TexCoord.xy).xy; + S[3] = tex2D(Sample_Common_1_A, TexCoord.zy).xy; // A.x = Ix^2 (A11); A.y = Iy^2 (A22); A.z = IxIy (A12) float3 A = 0.0; @@ -474,11 +479,6 @@ namespace Datamosh Color = float4(Lucas_Kanade(0, Average2D(Sample_Common_2_A, Tex).xy, Tex), 0.0, _BlendFactor); } - void Copy_PS(in float4 Position : SV_POSITION, in float2 TexCoord : TEXCOORD0, out float4 OutputColor0 : SV_TARGET0) - { - OutputColor0 = tex2D(Sample_Common_1_A, TexCoord); - } - void Post_Blur_0_PS(in float4 Position : SV_POSITION, in float4 TexCoords[7] : TEXCOORD0, out float4 OutputColor0 : SV_TARGET0) { Gaussian_Blur(Sample_Optical_Flow, TexCoords, false, OutputColor0); @@ -602,9 +602,14 @@ namespace Datamosh CREATE_PASS(Pre_Blur_0_VS, Pre_Blur_0_PS, Render_Common_1_B) CREATE_PASS(Pre_Blur_1_VS, Pre_Blur_1_PS, Render_Common_1_A) // Save this to store later - // Calculate derivative pyramids + // Calculate temporal derivative pyramids CREATE_PASS(Basic_VS, Derivatives_Temporal_PS, Render_Common_1_B) - CREATE_PASS(Derivatives_Spatial_VS, Derivatives_Spatial_PS, Render_Common_1_C) + + // Copy current convolved frame for next frame + CREATE_PASS(Basic_VS, Copy_PS, Render_Common_1_C) + + // Calculate spatial derivative pyramids + CREATE_PASS(Derivatives_Spatial_VS, Derivatives_Spatial_PS, Render_Common_1_A) // Bilinear Optical Flow CREATE_PASS(LK_Level_6_VS, LK_Level_6_PS, Render_Common_6) @@ -625,9 +630,6 @@ namespace Datamosh DestBlend = SRCALPHA; } - // Copy current convolved frame for next frame - CREATE_PASS(Basic_VS, Copy_PS, Render_Common_1_C) - // Gaussian blur CREATE_PASS(Post_Blur_0_VS, Post_Blur_0_PS, Render_Common_2_B) CREATE_PASS(Post_Blur_1_VS, Post_Blur_1_PS, Render_Common_2_A)