[BUILD] Error in step 27 and 52

[mk2510]

Environment

Operating System: ubuntu cluster, using conda enviroment.
Version / Commit SHA: 7496bfa
CMake Version: 3.29.4
Compiler: gcc 11.2.0, nvcc V12.1.105

Describe the problem

      [52/93] /scratch/work/krahnm1/.conda_envs/xcube/bin/nvcc --generate-dependencies-with-compile --dependency-output /scratch/work/krahnm1/XCube/openvdb/fvdb/build/temp.linux-x86_64-cpython-310/src/detail/ops/MarchingCubes.o.d -I/scratch/work/krahnm1/XCube/openvdb/fvdb/src -I/scratch/work/krahnm1/XCube/openvdb/fvdb/../nanovdb -I/scratch/work/krahnm1/XCube/openvdb/fvdb/external/cutlass/include -I/scratch/work/krahnm1/XCube/openvdb/fvdb/external/c-blosc/install/include -I/scratch/work/krahnm1/XCube/openvdb/fvdb/external/cudnn_fe/include -I/scratch/work/krahnm1/XCube/openvdb/fvdb/external/cudnn/cudnn-linux-x86_64-9.1.0.70_cuda12-archive/include -I/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/torch/include -I/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/torch/include/torch/csrc/api/include -I/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/torch/include/TH -I/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/torch/include/THC -I/scratch/work/krahnm1/.conda_envs/xcube/include -I/scratch/work/krahnm1/.conda_envs/xcube/include/python3.10 -c -c /scratch/work/krahnm1/XCube/openvdb/fvdb/src/detail/ops/MarchingCubes.cu -o /scratch/work/krahnm1/XCube/openvdb/fvdb/build/temp.linux-x86_64-cpython-310/src/detail/ops/MarchingCubes.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options ''"'"'-fPIC'"'"'' -std=c++20 --extended-lambda --diag-suppress=186 -diag-suppress=3189 -DTORCH_API_INCLUDE_EXTENSION_H '-DPYBIND11_COMPILER_TYPE="_gcc"' '-DPYBIND11_STDLIB="_libstdcpp"' '-DPYBIND11_BUILD_ABI="_cxxabi1011"' -DTORCH_EXTENSION_NAME=fvdblib -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_80,code=sm_80 -ccbin /scratch/work/krahnm1/.conda_envs/xcube/bin/x86_64-conda-linux-gnu-cc
      ninja: build stopped: subcommand failed.
      Traceback (most recent call last):
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/torch/utils/cpp_extension.py", line 2104, in _run_ninja_build
          subprocess.run(
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/subprocess.py", line 526, in run
          raise CalledProcessError(retcode, process.args,
      subprocess.CalledProcessError: Command '['ninja', '-v']' returned non-zero exit status 1.
      
      The above exception was the direct cause of the following exception:
      
      Traceback (most recent call last):
        File "<string>", line 2, in <module>
        File "<pip-setuptools-caller>", line 34, in <module>
        File "/scratch/work/krahnm1/XCube/openvdb/fvdb/setup.py", line 297, in <module>
          setup(name='fvdb',
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/setuptools/__init__.py", line 117, in setup
          return distutils.core.setup(**attrs)
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/setuptools/_distutils/core.py", line 183, in setup
          return run_commands(dist)
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/setuptools/_distutils/core.py", line 199, in run_commands
          dist.run_commands()
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/setuptools/_distutils/dist.py", line 954, in run_commands
          self.run_command(cmd)
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/setuptools/dist.py", line 950, in run_command
          super().run_command(command)
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/setuptools/_distutils/dist.py", line 973, in run_command
          cmd_obj.run()
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/setuptools/command/bdist_wheel.py", line 398, in run
          self.run_command("build")
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/setuptools/_distutils/cmd.py", line 316, in run_command
          self.distribution.run_command(command)
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/setuptools/dist.py", line 950, in run_command
          super().run_command(command)
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/setuptools/_distutils/dist.py", line 973, in run_command
          cmd_obj.run()
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/setuptools/_distutils/command/build.py", line 135, in run
          self.run_command(cmd_name)
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/setuptools/_distutils/cmd.py", line 316, in run_command
          self.distribution.run_command(command)
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/setuptools/dist.py", line 950, in run_command
          super().run_command(command)
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/setuptools/_distutils/dist.py", line 973, in run_command
          cmd_obj.run()
        File "/scratch/work/krahnm1/XCube/openvdb/fvdb/setup.py", line 140, in run
          super().run()
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/setuptools/command/build_ext.py", line 98, in run
          _build_ext.run(self)
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/setuptools/_distutils/command/build_ext.py", line 359, in run
          self.build_extensions()
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/torch/utils/cpp_extension.py", line 868, in build_extensions
          build_ext.build_extensions(self)
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/setuptools/_distutils/command/build_ext.py", line 476, in build_extensions
          self._build_extensions_serial()
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/setuptools/_distutils/command/build_ext.py", line 502, in _build_extensions_serial
          self.build_extension(ext)
        File "/scratch/work/krahnm1/XCube/openvdb/fvdb/setup.py", line 73, in build_extension
          super().build_extension(_ext)
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/setuptools/command/build_ext.py", line 263, in build_extension
          _build_ext.build_extension(self, ext)
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/setuptools/_distutils/command/build_ext.py", line 557, in build_extension
          objects = self.compiler.compile(
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/torch/utils/cpp_extension.py", line 681, in unix_wrap_ninja_compile
          _write_ninja_file_and_compile_objects(
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/torch/utils/cpp_extension.py", line 1784, in _write_ninja_file_and_compile_objects
          _run_ninja_build(
        File "/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/torch/utils/cpp_extension.py", line 2120, in _run_ninja_build
          raise RuntimeError(message) from e
      RuntimeError: Error compiling objects for extension
      [end of output]
  
  note: This error originates from a subprocess, and is likely not a problem with pip.
  ERROR: Failed building wheel for fvdb
  Running setup.py clean for fvdb
Failed to build fvdb
WARNING: There was an error checking the latest version of pip.
ERROR: ERROR: Failed to build installable wheels for some pyproject.toml based projects (fvdb)

Furthermore I noticed:

[27/93] /scratch/work/krahnm1/.conda_envs/xcube/bin/nvcc --generate-dependencies-with-compile --dependency-output /scratch/work/krahnm1/XCube/openvdb/fvdb/build/temp.linux-x86_64-cpython-310/src/detail/ops/FillToGrid.o.d -I/scratch/work/krahnm1/XCube/openvdb/fvdb/src -I/scratch/work/krahnm1/XCube/openvdb/fvdb/../nanovdb -I/scratch/work/krahnm1/XCube/openvdb/fvdb/external/cutlass/include -I/scratch/work/krahnm1/XCube/openvdb/fvdb/external/c-blosc/install/include -I/scratch/work/krahnm1/XCube/openvdb/fvdb/external/cudnn_fe/include -I/scratch/work/krahnm1/XCube/openvdb/fvdb/external/cudnn/cudnn-linux-x86_64-9.1.0.70_cuda12-archive/include -I/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/torch/include -I/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/torch/include/torch/csrc/api/include -I/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/torch/include/TH -I/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/torch/include/THC -I/scratch/work/krahnm1/.conda_envs/xcube/include -I/scratch/work/krahnm1/.conda_envs/xcube/include/python3.10 -c -c /scratch/work/krahnm1/XCube/openvdb/fvdb/src/detail/ops/FillToGrid.cu -o /scratch/work/krahnm1/XCube/openvdb/fvdb/build/temp.linux-x86_64-cpython-310/src/detail/ops/FillToGrid.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options ''"'"'-fPIC'"'"'' -std=c++20 --extended-lambda --diag-suppress=186 -diag-suppress=3189 -DTORCH_API_INCLUDE_EXTENSION_H '-DPYBIND11_COMPILER_TYPE="_gcc"' '-DPYBIND11_STDLIB="_libstdcpp"' '-DPYBIND11_BUILD_ABI="_cxxabi1011"' -DTORCH_EXTENSION_NAME=fvdblib -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_80,code=sm_80 -ccbin /scratch/work/krahnm1/.conda_envs/xcube/bin/x86_64-conda-linux-gnu-cc
      FAILED: /scratch/work/krahnm1/XCube/openvdb/fvdb/build/temp.linux-x86_64-cpython-310/src/detail/ops/FillToGrid.o
      /scratch/work/krahnm1/.conda_envs/xcube/bin/nvcc --generate-dependencies-with-compile --dependency-output /scratch/work/krahnm1/XCube/openvdb/fvdb/build/temp.linux-x86_64-cpython-310/src/detail/ops/FillToGrid.o.d -I/scratch/work/krahnm1/XCube/openvdb/fvdb/src -I/scratch/work/krahnm1/XCube/openvdb/fvdb/../nanovdb -I/scratch/work/krahnm1/XCube/openvdb/fvdb/external/cutlass/include -I/scratch/work/krahnm1/XCube/openvdb/fvdb/external/c-blosc/install/include -I/scratch/work/krahnm1/XCube/openvdb/fvdb/external/cudnn_fe/include -I/scratch/work/krahnm1/XCube/openvdb/fvdb/external/cudnn/cudnn-linux-x86_64-9.1.0.70_cuda12-archive/include -I/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/torch/include -I/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/torch/include/torch/csrc/api/include -I/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/torch/include/TH -I/scratch/work/krahnm1/.conda_envs/xcube/lib/python3.10/site-packages/torch/include/THC -I/scratch/work/krahnm1/.conda_envs/xcube/include -I/scratch/work/krahnm1/.conda_envs/xcube/include/python3.10 -c -c /scratch/work/krahnm1/XCube/openvdb/fvdb/src/detail/ops/FillToGrid.cu -o /scratch/work/krahnm1/XCube/openvdb/fvdb/build/temp.linux-x86_64-cpython-310/src/detail/ops/FillToGrid.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options ''"'"'-fPIC'"'"'' -std=c++20 --extended-lambda --diag-suppress=186 -diag-suppress=3189 -DTORCH_API_INCLUDE_EXTENSION_H '-DPYBIND11_COMPILER_TYPE="_gcc"' '-DPYBIND11_STDLIB="_libstdcpp"' '-DPYBIND11_BUILD_ABI="_cxxabi1011"' -DTORCH_EXTENSION_NAME=fvdblib -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_80,code=sm_80 -ccbin /scratch/work/krahnm1/.conda_envs/xcube/bin/x86_64-conda-linux-gnu-cc
      /scratch/work/krahnm1/XCube/openvdb/fvdb/src/detail/ops/FillToGrid.cu(82): error: dispatchFillToGrid is not a template
        dispatchFillToGrid<torch::kCUDA>(const GridBatchImpl &fromGrid, const GridBatchImpl &toGrid,
        ^
      
      /scratch/work/krahnm1/XCube/openvdb/fvdb/src/detail/ops/FillToGrid.cu(103): error: dispatchFillToGrid is not a template
        dispatchFillToGrid<torch::kCPU>(const GridBatchImpl &fromGrid, const GridBatchImpl &toGrid,
        ^
      
      2 errors detected in the compilation of "/scratch/work/krahnm1/XCube/openvdb/fvdb/src/detail/ops/FillToGrid.cu".
``
### To Reproduce
1) I did run all of the make commands
2) When I run TORCH_CUDA_ARCH_LIST="8.0" pip install . I get the error message after 1h of building

How could I fix those building errors?