Add model neuflow2

hmorimitsu · Aug 20, 2024 · a61def9 · a61def9
1 parent a0399f1
commit a61def9
Show file tree

Hide file tree

Showing 27 changed files with 1,335 additions and 173 deletions.
diff --git a/README.md b/README.md
@@ -99,7 +99,8 @@ This is still under development, so some things may not work as intended. I plan
 - MatchFlow [https://arxiv.org/abs/2303.08384](https://arxiv.org/abs/2303.08384)
 - MemFlow [https://arxiv.org/abs/2404.04808](https://arxiv.org/abs/2404.04808)
 - MS-RAFT+ [https://arxiv.org/abs/2210.16900](https://arxiv.org/abs/2210.16900)
-- NeuFlow [https://arxiv.org/abs/2403.10425](https://arxiv.org/abs/2403.10425)
+- NeuFlow v1 [https://arxiv.org/abs/2403.10425](https://arxiv.org/abs/2403.10425)
+- NeuFlow v2 [https://arxiv.org/abs/2408.10161](https://arxiv.org/abs/2408.10161)
 - PWCNet [https://arxiv.org/abs/1709.02371](https://arxiv.org/abs/1709.02371)
 - RAFT [https://arxiv.org/abs/2003.12039](https://arxiv.org/abs/2003.12039)
 - RAPIDFlow [https://hmorimitsu.com/publication/2024-icra-rapidflow/](https://hmorimitsu.com/publication/2024-icra-rapidflow/)

diff --git a/docs/source/_static/plot-memory(gb)-fp32-sintel-final-occ-val_epe.html b/docs/source/_static/plot-memory(gb)-fp32-sintel-final-occ-val_epe.html
diff --git a/docs/source/_static/plot-memory(gb)-fp32-sintel-final-occ-val_outlier.html b/docs/source/_static/plot-memory(gb)-fp32-sintel-final-occ-val_outlier.html
diff --git a/docs/source/_static/plot-params-flops.html b/docs/source/_static/plot-params-flops.html
diff --git a/docs/source/_static/plot-sintel-final-occ-val_epe-kitti-2015-val_outlier.html b/docs/source/_static/plot-sintel-final-occ-val_epe-kitti-2015-val_outlier.html
diff --git a/docs/source/_static/plot-time(ms)-fp32-sintel-final-occ-val_epe.html b/docs/source/_static/plot-time(ms)-fp32-sintel-final-occ-val_epe.html
diff --git a/docs/source/_static/plot-time(ms)-fp32-sintel-final-occ-val_outlier.html b/docs/source/_static/plot-time(ms)-fp32-sintel-final-occ-val_outlier.html
diff --git a/docs/source/models/checkpoint_list.csv b/docs/source/models/checkpoint_list.csv
@@ -110,6 +110,9 @@ memflow_t-kitti-9eeabb65.ckpt,MemFlowNet_T_kitti.pth,
 ms_raft_plus-mixed-2bb01f62.ckpt,mixed.pth,https://github.com/cv-stuttgart/MS_RAFT_plus
 neuflow-things-c402aa7a.ckpt,neuflow_things.pth,https://github.com/neufieldrobotics/neuflow
 neuflow-sintel-0d969ea2.ckpt,neuflow_sintel.pth,
+neuflow2-mixed-acac1a70.ckpt,neuflow_mixed.pth,https://github.com/neufieldrobotics/NeuFlow_v2
+neuflow2-sintel-15c625f8.ckpt,neuflow_sintel.pth,
+neuflow2-things-6ed47437.ckpt,neuflow_things.pth,
 pwcdcnet-sintel-c7d08a46.ckpt,pwc_net.pth.tar,https://github.com/NVlabs/PWC-Net/tree/master/PyTorch
 pwcdcnet-things-cc223701.ckpt,pwc_net_chairs.pth.tar,
 pwcnet-sintel-533815e5.ckpt,pwc_net.pth.tar,

diff --git a/docs/source/models/models_list.rst b/docs/source/models/models_list.rst
@@ -288,8 +288,8 @@ MS-RAFT+
 
 - Model names: ``ms_raft+``
 
-NeuFlow
--------
+NeuFlow v1
+----------
 
 `[source code] <https://github.com/hmorimitsu/ptlflow/tree/main/ptlflow/models/neuflow>`__
 
@@ -299,6 +299,17 @@ NeuFlow
 
 - Model names: ``neuflow``
 
+NeuFlow v2
+----------
+
+`[source code] <https://github.com/hmorimitsu/ptlflow/tree/main/ptlflow/models/neuflow2>`__
+
+- Paper: **NeuFlow v2: Push High-Efficiency Optical Flow To the Limit** - `https://arxiv.org/abs/2408.10161 <hhttps://arxiv.org/abs/2408.10161>`_
+
+- Reference code: `https://github.com/neufieldrobotics/NeuFlow_v2 <https://github.com/neufieldrobotics/NeuFlow_v2>`_
+
+- Model names: ``neuflow2``
+
 PWCNet
 ------
 

diff --git a/docs/source/results/metrics_all.csv b/docs/source/results/metrics_all.csv
@@ -119,6 +119,9 @@ memflow_t,kitti,2.237,0.855,0.927,0.945,7.004,6.481,1.54,0.486,0.886,0.71,0.95,0
 ms_raft+,mixed,0.648,0.961,0.979,0.984,1.939,2.664,0.243,0.775,0.985,0.881,0.993,0.911,0.995,10.58,0.627,1.107,0.913,0.954,0.967,4.172,3.525,0.654,0.696,0.936,0.836,0.97,0.878,0.98,15.02,2.64,1.015,0.87,0.956,0.972,3.054,0.679,0.877,0.964,0.981,1.724,,,,,,
 neuflow,things,1.66,0.837,0.935,0.956,6.148,5.449,0.985,0.472,0.87,0.734,0.959,0.816,0.975,25.104,3.818,3.126,0.787,0.892,0.921,10.424,7.954,2.244,0.415,0.817,0.664,0.917,0.751,0.942,32.279,7.933,5.056,0.525,0.789,0.848,20.281,11.111,0.45,0.682,0.747,30.918,,,,,,
 neuflow,sintel,0.988,0.871,0.952,0.97,4.411,3.414,0.609,0.552,0.901,0.787,0.973,0.856,0.985,19.69,2.49,1.294,0.838,0.93,0.955,6.463,3.933,0.889,0.517,0.865,0.756,0.95,0.831,0.971,22.786,4.53,2.297,0.673,0.887,0.931,9.489,2.06,0.645,0.868,0.922,9.539,,,,,,
+neuflow2,mixed,0.999,0.906,0.96,0.972,3.733,3.821,0.544,0.601,0.935,0.803,0.979,0.861,0.987,18.277,1.973,1.661,0.861,0.931,0.951,6.513,5.012,1.112,0.546,0.888,0.753,0.951,0.819,0.967,23.268,4.593,1.293,0.802,0.938,0.963,4.691,1.666,0.756,0.912,0.946,6.106,,,,,,
+neuflow2,sintel,0.801,0.917,0.965,0.977,3.175,3.182,0.416,0.629,0.946,0.82,0.983,0.876,0.99,16.462,1.531,1.321,0.874,0.94,0.96,5.539,4.093,0.901,0.57,0.901,0.775,0.959,0.841,0.975,20.968,3.73,1.474,0.799,0.932,0.957,5.281,1.897,0.762,0.909,0.94,6.594,,,,,,
+neuflow2,things,1.265,0.898,0.956,0.969,4.176,4.54,0.684,0.585,0.928,0.791,0.976,0.85,0.985,19.584,2.26,2.767,0.845,0.916,0.937,8.054,7.032,1.962,0.518,0.874,0.726,0.937,0.793,0.955,26.094,5.958,1.725,0.725,0.911,0.948,7.089,4.065,0.63,0.83,0.88,14.515,,,,,,
 pwcnet,things,2.643,0.801,0.909,0.934,8.811,7.733,1.697,0.369,0.835,0.65,0.937,0.749,0.958,33.673,6.064,4.059,0.749,0.871,0.904,12.592,9.855,2.943,0.337,0.78,0.599,0.898,0.7,0.927,38.921,9.872,4.042,0.545,0.788,0.854,20.243,10.288,0.452,0.662,0.731,33.014,,,,,,
 pwcnet,sintel,1.849,0.872,0.936,0.952,6.164,6.205,1.12,0.502,0.906,0.724,0.96,0.795,0.972,26.298,3.858,2.338,0.836,0.912,0.935,8.387,6.74,1.597,0.47,0.866,0.694,0.936,0.772,0.955,29.229,6.096,1.93,0.76,0.915,0.946,6.921,3.122,0.696,0.869,0.911,10.184,,,,,,
 pwcnet_nodc,things,3.329,0.724,0.88,0.917,11.639,9.577,2.164,0.271,0.758,0.549,0.915,0.678,0.947,43.948,8.154,4.769,0.675,0.843,0.885,15.416,11.502,3.454,0.257,0.704,0.516,0.874,0.636,0.913,47.481,12.212,5.29,0.503,0.743,0.811,25.053,12.274,0.414,0.633,0.704,36.11,,,,,,

diff --git a/docs/source/results/model_benchmark-all.csv b/docs/source/results/model_benchmark-all.csv
@@ -1,71 +1,73 @@
 Model,Params,FLOPs,InputH,InputW,InputPx,Time(ms)-fp16,Memory(GB)-fp16,Time(ms)-fp32,Memory(GB)-fp32
-ccmr,10.781,4134.022,500,1000,500000,552.557,0.601,835.839,1.036
-ccmr+,11.524,11716.13,500,1000,500000,1376.328,1.549,2218.884,2.749
-craft,6.307,4204.715,500,1000,500000,470.66,4.305,,
-csflow,5.605,1826.398,500,1000,500000,171.228,2.465,275.076,1.75
-dicl,11.226,453.293,500,1000,500000,50.061,1.652,75.532,2.393
-dip,5.372,5780.579,500,1000,500000,506.814,0.904,871.789,1.366
-fastflownet,1.366,23.604,500,1000,500000,10.939,0.36,10.505,0.397
-flow1d,5.734,1523.984,500,1000,500000,134.127,0.628,204.744,0.64
-flowformer,16.168,3207.113,500,1000,500000,388.474,3.056,649.667,6.018
-flowformer++,16.152,3048.146,500,1000,500000,373.262,2.758,618.602,4.318
-flownet2,162.519,357.598,500,1000,500000,53.807,1.24,71.46,1.743
-flownetc,39.175,82.022,500,1000,500000,14.215,1.072,15.058,0.71
-flownetcs,77.871,139.24,500,1000,500000,21.664,0.747,27.372,0.854
-flownetcss,116.566,196.458,500,1000,500000,29.676,0.887,39.812,0.99
-flownets,38.677,53.172,500,1000,500000,7.065,0.772,8.599,0.597
-flownetsd,45.372,95.366,500,1000,500000,10.292,0.677,14.381,0.788
-gma,5.88,2540.319,500,1000,500000,191.244,0.981,335.372,1.344
-gmflow,4.68,428.6,500,1000,500000,53.696,0.782,91.601,1.293
-gmflow_refine,4.717,1063.111,500,1000,500000,125.947,1.555,229.172,2.873
-gmflow+,4.68,428.6,500,1000,500000,55.987,0.784,97.923,1.305
-gmflow+_sc2,4.717,1063.111,500,1000,500000,138.132,1.58,246.93,2.968
-gmflow+_sc2_refine6,7.361,2110.617,500,1000,500000,253.094,1.589,445.431,2.906
-gmflownet,9.343,2059.582,500,1000,500000,264.816,1.451,442.92,2.423
-gmflownet_mix,8.688,1871.501,500,1000,500000,210.362,1.274,352.418,1.98
-hd3,39.562,289.666,500,1000,500000,36.14,0.883,49.356,1.152
-hd3_ctxt,39.943,310.117,500,1000,500000,36.863,0.915,50.068,1.167
-irr_pwc,6.362,907.539,500,1000,500000,134.896,0.849,184.049,1.243
-irr_pwcnet,8.639,178.249,500,1000,500000,24.475,0.627,29.814,0.888
-irr_pwcnet_irr,3.354,170.251,500,1000,500000,23.737,0.638,29.693,0.869
-lcv_raft,5.323,1781.566,500,1000,500000,,,269.202,1.206
-lcv_raft_small,1.007,459.632,500,1000,500000,,,107.918,1.148
-liteflownet,5.38,309.909,500,1000,500000,48.376,0.702,62.952,0.85
-liteflownet2,6.429,140.781,500,1000,500000,30.627,0.658,29.953,0.734
-liteflownet2_pseudoreg,6.493,148.154,500,1000,500000,27.512,0.662,36.61,0.765
-liteflownet3,7.524,178.905,500,1000,500000,43.32,0.694,41.898,0.771
-liteflownet3_pseudoreg,7.588,186.278,500,1000,500000,41.062,0.695,53.816,0.774
-liteflownet3s,8.006,180.694,500,1000,500000,47.175,0.724,46.416,0.799
-liteflownet3s_pseudoreg,8.07,188.067,500,1000,500000,41.564,0.726,48.378,0.802
-llaflow,6.06,3313.444,500,1000,500000,219.433,1.426,440.06,2.49
-llaflow_raft,5.52,2553.575,500,1000,500000,198.813,1.427,365.654,2.627
-maskflownet,20.656,314.989,500,1000,500000,43.45,0.694,67.794,0.76
-maskflownet_s,10.514,162.837,500,1000,500000,26.849,0.67,43.748,0.998
-matchflow,15.446,2994.565,500,1000,500000,,,496.148,1.783
-matchflow_raft,14.824,2284.576,500,1000,500000,,,415.316,1.649
-memflow,6.273,1628.784,500,1000,500000,261.383,0.958,436.268,1.652
-memflow_t,12.705,1581.625,500,1000,500000,265.718,0.999,436.476,1.97
-ms_raft+,16.177,9658.707,500,1000,500000,926.363,1.505,1334.574,2.424
-neuflow,3.847,76.212,500,1000,500000,10.901,0.517,14.077,0.514
-pwcnet,9.374,157.381,500,1000,500000,27.771,0.564,31.734,0.98
-pwcnet_nodc,8.243,96.596,500,1000,500000,22.287,0.536,23.439,0.847
-raft,5.258,1780.45,500,1000,500000,169.757,0.864,271.958,1.185
-raft_small,0.99,459.365,500,1000,500000,61.693,0.844,108.428,1.121
-rapidflow,1.646,99.618,500,1000,500000,46.743,0.569,51.972,0.724
-rapidflow_it1,1.646,12.099,500,1000,500000,11.821,0.545,12.213,0.58
-rapidflow_it2,1.646,16.507,500,1000,500000,15.25,0.548,15.457,0.585
-rapidflow_it3,1.646,36.336,500,1000,500000,20.889,0.55,23.366,0.71
-rapidflow_it6,1.646,57.43,500,1000,500000,29.925,0.55,32.361,0.71
-rpknet,2.847,249.432,500,1000,500000,184.237,0.619,205.162,0.822
-scopeflow,6.362,907.292,500,1000,500000,135.956,0.625,182.263,0.931
-sea_raft_s,8.883,538.417,500,1000,500000,52.879,0.588,88.948,0.977
-sea_raft_m,19.664,922.464,500,1000,500000,77.735,0.647,136.776,1.039
-sea_raft_l,19.664,1242.004,500,1000,500000,125.98,0.702,201.806,1.05
-separableflow,8.346,1658.256,500,1000,500000,,,677.244,1.206
-skflow,6.273,2731.596,500,1000,500000,442.6,0.953,642.179,1.418
-splatflow,9.0,2540.18,500,1000,500000,177.233,0.779,321.923,1.268
-starflow,4.772,786.01,500,1000,500000,124.916,0.905,162.477,1.177
-vcn,10.311,86.289,500,1000,500000,193.327,1.229,273.436,2.011
-vcn_small,8.371,29.129,500,1000,500000,78.737,0.691,103.627,0.943
-videoflow_bof,12.659,3159.379,500,1000,500000,531.243,1.361,825.437,2.362
-videoflow_mof,13.453,3460.935,500,1000,500000,591.13,1.399,889.864,2.398
+ccmr,10.781,4134.022,500,1000,500000,382.907,0.708,572.08,1.188
+ccmr+,11.524,11716.13,500,1000,500000,855.701,1.64,1386.917,3.026
+craft,6.307,4204.715,500,1000,500000,315.168,4.437,475.565,8.178
+csflow,5.605,1826.398,500,1000,500000,102.378,3.461,155.09,1.89
+dicl,11.226,453.293,500,1000,500000,49.855,1.796,57.079,2.556
+dip,5.372,5780.579,500,1000,500000,341.781,1.056,513.528,1.631
+fastflownet,1.366,23.72,500,1000,500000,35.841,0.481,29.944,0.514
+flow1d,5.734,1523.984,500,1000,500000,86.135,0.745,122.418,0.764
+flowformer,16.168,3207.109,500,1000,500000,220.047,3.186,371.532,6.111
+flowformer++,16.152,3048.144,500,1000,500000,212.087,2.895,356.207,4.428
+flownet2,162.519,358.356,500,1000,500000,58.25,1.338,69.212,1.847
+flownetc,39.175,82.78,500,1000,500000,27.886,1.157,30.459,0.801
+flownetcs,77.871,139.998,500,1000,500000,35.427,0.827,39.28,0.953
+flownetcss,116.566,197.216,500,1000,500000,42.829,0.991,48.338,1.096
+flownets,38.677,53.172,500,1000,500000,4.851,0.876,5.652,0.69
+flownetsd,45.372,95.366,500,1000,500000,7.126,0.77,8.049,0.889
+gma,5.88,2540.319,500,1000,500000,106.156,1.086,189.866,1.438
+gmflow,4.68,428.6,500,1000,500000,28.921,0.894,50.696,1.402
+gmflow_refine,4.717,1063.111,500,1000,500000,72.332,1.663,132.252,2.987
+gmflow+,4.68,428.6,500,1000,500000,28.98,0.887,52.061,1.401
+gmflow+_sc2,4.717,1063.111,500,1000,500000,74.425,1.663,135.823,2.977
+gmflow+_sc2_refine6,7.361,2110.617,500,1000,500000,145.88,1.669,244.301,2.989
+gmflownet,9.343,2059.582,500,1000,500000,151.625,1.512,246.5,2.52
+gmflownet_mix,8.688,1871.501,500,1000,500000,127.058,1.237,198.365,2.038
+hd3,39.562,289.931,500,1000,500000,53.431,0.935,60.143,1.213
+hd3_ctxt,39.943,310.383,500,1000,500000,53.431,0.98,56.934,1.223
+irr_pwc,6.362,907.539,500,1000,500000,130.299,0.919,157.042,1.303
+irr_pwcnet,8.639,178.394,500,1000,500000,42.655,0.725,43.146,0.989
+irr_pwcnet_irr,3.354,170.375,500,1000,500000,41.807,0.725,49.523,0.946
+lcv_raft,5.323,1781.566,500,1000,500000,,,162.396,1.252
+lcv_raft_small,1.007,459.632,500,1000,500000,,,88.596,1.219
+liteflownet,5.38,310.077,500,1000,500000,65.545,0.729,74.459,0.914
+liteflownet2,6.429,140.855,500,1000,500000,46.831,0.733,39.062,0.799
+liteflownet2_pseudoreg,6.493,148.229,500,1000,500000,37.606,0.735,41.947,0.818
+liteflownet3,7.524,179.333,500,1000,500000,62.753,0.749,61.946,0.812
+liteflownet3_pseudoreg,7.588,186.706,500,1000,500000,61.368,0.748,65.64,0.812
+liteflownet3s,8.006,181.128,500,1000,500000,69.932,0.745,68.469,0.816
+liteflownet3s_pseudoreg,8.07,188.501,500,1000,500000,74.508,0.747,69.647,0.816
+llaflow,6.06,3313.444,500,1000,500000,125.561,1.461,244.039,2.512
+llaflow_raft,5.52,2553.575,500,1000,500000,115.749,1.49,198.493,2.672
+maskflownet,20.656,315.189,500,1000,500000,76.374,0.745,92.676,0.759
+maskflownet_s,10.514,162.96,500,1000,500000,47.269,0.733,62.912,1.057
+memflow,6.273,1628.784,500,1000,500000,157.404,1.156,244.581,1.861
+memflow_t,12.705,1581.625,500,1000,500000,159.091,1.313,259.675,2.267
+ms_raft+,16.177,9658.707,500,1000,500000,525.469,1.477,756.039,2.775
+neuflow,3.847,76.246,500,1000,500000,14.884,0.668,15.324,0.673
+neuflow2,9.03,255.868,500,1000,500000,22.123,0.903,20.515,1.125
+pwcnet,9.374,157.505,500,1000,500000,48.76,0.713,46.366,1.072
+pwcnet_nodc,8.243,96.719,500,1000,500000,44.44,0.666,43.733,0.986
+raft,5.258,1780.45,500,1000,500000,124.247,0.916,145.654,1.254
+raft_small,0.99,459.365,500,1000,500000,88.98,0.922,88.807,1.213
+rapidflow,1.646,114.735,500,1000,500000,22.244,0.622,30.773,1.007
+rapidflow_it1,1.646,12.155,500,1000,500000,6.7,0.512,8.722,0.58
+rapidflow_it2,1.646,17.449,500,1000,500000,9.363,0.513,11.398,0.585
+rapidflow_it3,1.646,51.45,500,1000,500000,11.617,0.628,16.119,1.009
+rapidflow_it6,1.646,72.545,500,1000,500000,15.331,0.616,20.98,1.005
+rpknet,2.847,264.55,500,1000,500000,112.008,0.686,148.554,1.118
+sea_raft,8.883,538.417,500,1000,500000,29.095,0.854,45.788,1.203
+sea_raft_s,8.883,538.417,500,1000,500000,29.475,0.856,46.55,1.203
+sea_raft_m,19.664,922.464,500,1000,500000,40.657,0.893,67.91,1.234
+sea_raft_l,19.664,1242.004,500,1000,500000,66.027,0.878,105.991,1.22
+scopeflow,6.362,907.539,500,1000,500000,180.012,0.726,198.476,1.202
+separableflow,8.346,1658.256,500,1000,500000,,,602.443,1.286
+skflow,6.273,2731.596,500,1000,500000,290.073,1.013,362.867,1.467
+starflow,4.772,786.3,500,1000,500000,177.326,0.93,189.626,1.314
+unimatch,4.68,428.6,500,1000,500000,27.738,0.92,52.008,1.421
+unimatch_sc2,4.717,1063.111,500,1000,500000,75.261,1.684,136.203,3.006
+unimatch_sc2_refine6,7.361,2110.617,500,1000,500000,145.863,1.689,243.728,3.02
+vcn,10.311,86.289,500,1000,500000,278.101,1.291,424.59,2.107
+vcn_small,8.371,29.129,500,1000,500000,118.486,0.782,161.606,1.001
+videoflow_bof,12.659,3159.379,500,1000,500000,420.908,1.453,481.482,2.456
+videoflow_mof,13.453,3460.933,500,1000,500000,471.891,1.439,545.409,2.464
diff --git a/docs/source/results/model_benchmark.rst b/docs/source/results/model_benchmark.rst
@@ -5,10 +5,10 @@ Model benchmark
 Test environment
 ================
 
-- PyTorch 2.1.0
+- PyTorch 2.2.2
 - Lightning 1.9.5
-- NVIDIA RTX 3070
-- CUDA 11.7
+- NVIDIA RTX 3090
+- CUDA 12.1
 
 .. csv-table::
     :file: model_benchmark-all.csv