From eb4626209221247b80ce59ac773f7248556ecd47 Mon Sep 17 00:00:00 2001 From: loganrooks Date: Fri, 6 Apr 2018 16:31:10 -0400 Subject: [PATCH 1/8] added documentation to model.py that better reflects content in FlowNet2 paper --- models.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/models.py b/models.py index 6ec7f32..7ac8a58 100755 --- a/models.py +++ b/models.py @@ -107,23 +107,27 @@ def init_deconv_bilinear(self, weight): return def forward(self, inputs): + # input format: 5d tensor, (batch, rgb, img #, x, y) + # normalizing imgs rgb_mean = inputs.contiguous().view(inputs.size()[:2]+(-1,)).mean(dim=-1).view(inputs.size()[:2] + (1,1,1,)) - x = (inputs - rgb_mean) / self.rgb_max + + # squish the img # dimension by concatenating the rgb channels x1 = x[:,:,0,:,:] x2 = x[:,:,1,:,:] x = torch.cat((x1,x2), dim = 1) # flownetc - flownetc_flow2 = self.flownetc(x)[0] + flownetc_flow2 = self.flownetc(x)[0] # only return flow2, since the network returns flow6 to flow2 flownetc_flow = self.upsample1(flownetc_flow2*self.div_flow) - # warp img1 to img0; magnitude of diff between img0 and and warped_img1, + # warp img1 to img0; magnitude of diff between img0 and and warped_img1, i.e. brightness error + # combine the flow with img1 to 'reconstruct' img0 resampled_img1 = self.resample1(x[:,3:,:,:], flownetc_flow) - diff_img0 = x[:,:3,:,:] - resampled_img1 + diff_img0 = x[:,:3,:,:] - resampled_img1 # brightness error norm_diff_img0 = self.channelnorm(diff_img0) - # concat img0, img1, img1->img0, flow, diff-mag ; + # concat img0, img1, warped img1, flow, normalized brightness error concat1 = torch.cat((x, resampled_img1, flownetc_flow/self.div_flow, norm_diff_img0), dim=1) # flownets1 From 534c4f4691cda4eb40c64246a224bf8b17d7b392 Mon Sep 17 00:00:00 2001 From: Fitsum Reda Date: Fri, 6 Apr 2018 14:37:36 -0700 Subject: [PATCH 2/8] fix a bug in channelnorm.py ctx wasn't collecting nrom_gram in forward for re-use in backward --- networks/channelnorm_package/functions/channelnorm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/networks/channelnorm_package/functions/channelnorm.py b/networks/channelnorm_package/functions/channelnorm.py index f84bc31..a4c19a6 100755 --- a/networks/channelnorm_package/functions/channelnorm.py +++ b/networks/channelnorm_package/functions/channelnorm.py @@ -12,7 +12,8 @@ def forward(ctx, input1, norm_deg=2): channelnorm.ChannelNorm_cuda_forward(input1, output, norm_deg) ctx.save_for_backward(input1, output) - + ctx.norm_deg = norm_deg + return output @staticmethod From 4c0a3b1fc3754e39124bd7694f073ffdfdb334f5 Mon Sep 17 00:00:00 2001 From: Fitsum Reda Date: Fri, 6 Apr 2018 15:21:11 -0700 Subject: [PATCH 3/8] Update README.md --- README.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1e0db2e..e78c565 100755 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ We've included caffe pre-trained models. Should you use these pre-trained weight [![Predicted flows on MPI-Sintel](./image.png)](https://www.youtube.com/watch?v=HtBmabY8aeU "Predicted flows on MPI-Sintel") ## Reference -If you find this implementation useful in your work, please acknowledge it appropriately and cite the paper using: +If you find this implementation useful in your work, please acknowledge it appropriately and cite the paper: ```` @InProceedings{IMKDB17, author = "E. Ilg and N. Mayer and T. Saikia and M. Keuper and A. Dosovitskiy and T. Brox", @@ -97,6 +97,15 @@ If you find this implementation useful in your work, please acknowledge it appro url = "http://lmb.informatik.uni-freiburg.de//Publications/2017/IMKDB17" } ```` - +``` +@misc{flownet2-pytorch, + author = {Fitsum Reda and Jon Barker and Bryan Catanzaro}, + title = {flownet2-pytorch: Pytorch implementation of FlowNet 2.0: FlowNet 2.0: Evolution of Optical Flow Estimation with Deep Networks}, + year = {2017}, + publisher = {GitHub}, + journal = {GitHub repository}, + howpublished = {\url{https://github.com/NVIDIA/flownet2-pytorch}} +} +``` ## Acknowledgments Parts of this code were derived, as noted in the code, from [ClementPinard/FlowNetPytorch](https://github.com/ClementPinard/FlowNetPytorch). From ee480bacc31beefdaacbe4f79562a4bb74476ed0 Mon Sep 17 00:00:00 2001 From: Fitsum Reda Date: Fri, 6 Apr 2018 15:21:50 -0700 Subject: [PATCH 4/8] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e78c565..f489e54 100755 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ If you find this implementation useful in your work, please acknowledge it appro ``` @misc{flownet2-pytorch, author = {Fitsum Reda and Jon Barker and Bryan Catanzaro}, - title = {flownet2-pytorch: Pytorch implementation of FlowNet 2.0: FlowNet 2.0: Evolution of Optical Flow Estimation with Deep Networks}, + title = {flownet2-pytorch: Pytorch implementation of FlowNet 2.0: Evolution of Optical Flow Estimation with Deep Networks}, year = {2017}, publisher = {GitHub}, journal = {GitHub repository}, From 74aedd173fc96ae0d45966091e8898b10de020ce Mon Sep 17 00:00:00 2001 From: Fitsum Reda Date: Fri, 6 Apr 2018 15:53:34 -0700 Subject: [PATCH 5/8] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f489e54..035bdca 100755 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ If you find this implementation useful in your work, please acknowledge it appro ```` ``` @misc{flownet2-pytorch, - author = {Fitsum Reda and Jon Barker and Bryan Catanzaro}, + author = {Fitsum Reda and Robert Pottorff and Jon Barker and Bryan Catanzaro}, title = {flownet2-pytorch: Pytorch implementation of FlowNet 2.0: Evolution of Optical Flow Estimation with Deep Networks}, year = {2017}, publisher = {GitHub}, From 6827b4c59d1b5d58bcce492e2d5ec2d6cd1e42de Mon Sep 17 00:00:00 2001 From: Fitsum Reda Date: Wed, 11 Jul 2018 15:12:03 -0700 Subject: [PATCH 6/8] add reference to a related optical flow work from NVIDIA link to paper and codes for PWC-Net, NVIDIA's optical flow model. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 035bdca..317eb49 100755 --- a/README.md +++ b/README.md @@ -107,5 +107,9 @@ If you find this implementation useful in your work, please acknowledge it appro howpublished = {\url{https://github.com/NVIDIA/flownet2-pytorch}} } ``` +## Related Optical Flow Work from Nvidia +Code (in Caffe and Pytorch): [PWC-Net](https://github.com/NVlabs/PWC-Net)
+Paper : [PWC-Net: CNNs for Optical Flow Using Pyramid, Warping, and Cost Volume](https://arxiv.org/abs/1709.02371). + ## Acknowledgments Parts of this code were derived, as noted in the code, from [ClementPinard/FlowNetPytorch](https://github.com/ClementPinard/FlowNetPytorch). From 88df51af95faf8251642e736c63bb18ee88c49a3 Mon Sep 17 00:00:00 2001 From: "xuezhi.zhang" Date: Fri, 13 Jul 2018 05:50:37 -0400 Subject: [PATCH 7/8] add requirements.txt --- requirements.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..825c162 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +setproctitle==1.1.10 +tensorboardX==1.2 +colorama==0.3.9 From 5de68bd2fbb6e40feddb99914b60d3350eb66a16 Mon Sep 17 00:00:00 2001 From: "xuezhi.zhang" Date: Fri, 13 Jul 2018 05:53:34 -0400 Subject: [PATCH 8/8] edit build.py, and install successfully. --- networks/channelnorm_package/build.py | 5 +++-- networks/correlation_package/build.py | 3 ++- networks/resample2d_package/build.py | 5 +++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/networks/channelnorm_package/build.py b/networks/channelnorm_package/build.py index 2066819..a9be1cd 100755 --- a/networks/channelnorm_package/build.py +++ b/networks/channelnorm_package/build.py @@ -24,8 +24,9 @@ package=False, relative_to=this_folder, define_macros=Defines, - extra_objects=[os.path.join(this_folder, Object) for Object in Objects] + extra_objects=[os.path.join(this_folder, Object) for Object in Objects], + extra_compile_args=['-std=c99'] ) if __name__ == '__main__': - ffi.build() \ No newline at end of file + ffi.build() diff --git a/networks/correlation_package/build.py b/networks/correlation_package/build.py index 9974e2a..c86dd76 100755 --- a/networks/correlation_package/build.py +++ b/networks/correlation_package/build.py @@ -24,7 +24,8 @@ package=False, relative_to=this_folder, define_macros=Defines, - extra_objects=[os.path.join(this_folder, Object) for Object in Objects] + extra_objects=[os.path.join(this_folder, Object) for Object in Objects], + extra_compile_args=['-std=c99'] ) if __name__ == '__main__': diff --git a/networks/resample2d_package/build.py b/networks/resample2d_package/build.py index f1de130..7ac66c7 100755 --- a/networks/resample2d_package/build.py +++ b/networks/resample2d_package/build.py @@ -24,8 +24,9 @@ package=False, relative_to=this_folder, define_macros=Defines, - extra_objects=[os.path.join(this_folder, Object) for Object in Objects] + extra_objects=[os.path.join(this_folder, Object) for Object in Objects], + extra_compile_args=['-std=c99'] ) if __name__ == '__main__': - ffi.build() \ No newline at end of file + ffi.build()