HEFFTE support in compressible FHD + Advanced diagnostics for compressible turbulence

exec/compressible_stag/GNUmakefile

@@ -14,7 +14,20 @@ MAX_SPEC      = 8
 MAX_REAC      = 5
 
 USE_PARTICLES = FALSE
-DO_TURB 	  = FALSE
+DO_TURB       = FALSE
+
+USE_HEFFTE_FFTW   = FALSE
+USE_HEFFTE_CUFFT  = FALSE
+USE_HEFFTE_ROCFFT = FALSE
+USE_DISTRIBUTED_FFT = TRUE
+
+ifeq ($(USE_HEFFTE_FFTW),TRUE)
+  HEFFTE_HOME ?= ../../../heffte/
+else ifeq ($(USE_HEFFTE_CUFFT),TRUE)
+  HEFFTE_HOME ?= ../../../heffte-org/build_aware/
+else ifeq ($(USE_HEFFTE_ROCFFT),TRUE)
+  HEFFTE_HOME ?= ../../../heffte-org/build_noaware/
+endif
 
 include $(AMREX_HOME)/Tools/GNUMake/Make.defs
 
@@ -41,23 +54,35 @@ include ../../src_rng/Make.package
 VPATH_LOCATIONS   += ../../src_rng/
 INCLUDE_LOCATIONS += ../../src_rng/
 
-
 include ../../src_common/Make.package
 VPATH_LOCATIONS   += ../../src_common/
 INCLUDE_LOCATIONS += ../../src_common/
 
+#ifeq ($(USE_HEFFTE_FFTW),TRUE)
+#  include $(HEFFTE_HOME)/src/Make.package
+#else ifeq ($(USE_HEFFTE_CUFFT),TRUE)
+#  include $(HEFFTE_HOME)/src/Make.package
+#else ifeq ($(USE_HEFFTE_ROCFFT),TRUE)
+#  include $(HEFFTE_HOME)/src/Make.package
+#endif
+
+ifeq ($(USE_HEFFTE_FFTW),TRUE)
+  DEFINES += -DHEFFTE_FFTW
+  LIBRARIES += -L$(FFTW_DIR) -lfftw3_mpi -lfftw3 -lfftw3f
+else ifeq ($(USE_HEFFTE_CUFFT),TRUE)
+  DEFINES += -DHEFFTE_CUFFT
+else ifeq ($(USE_HEFFTE_ROCFFT),TRUE)
+  DEFINES += -DHEFFTE_ROCFFT
+endif
+
 include $(AMREX_HOME)/Src/Base/Make.package
 
 include ../../src_analysis/Make.package
 VPATH_LOCATIONS   += ../../src_analysis/
 INCLUDE_LOCATIONS += ../../src_analysis/
-
+ 
 include $(AMREX_HOME)/Tools/GNUMake/Make.rules
 
-ifeq ($(findstring cgpu, $(HOST)), cgpu)
-  CXXFLAGS += $(FFTW)
-endif
-
 ifeq ($(USE_CUDA),TRUE)
   LIBRARIES += -lcufft
 else ifeq ($(USE_HIP),TRUE)
@@ -66,7 +91,7 @@ else ifeq ($(USE_HIP),TRUE)
   LIBRARY_LOCATIONS += $(ROC_PATH)/rocfft/lib
   LIBRARIES += -L$(ROC_PATH)/rocfft/lib -lrocfft
 else
-  LIBRARIES += -L$(FFTW_DIR) -lfftw3_mpi -lfftw3
+  LIBRARIES += -L$(FFTW_DIR) -lfftw3_mpi -lfftw3 -lfftw3f
 endif
 
 ifeq ($(DO_TURB), TRUE)

exec/compressible_stag/SPECTRAL_FILTER/GNUmakefile

@@ -0,0 +1,69 @@
+AMREX_HOME ?= ../../../../amrex/
+
+DEBUG         = FALSE
+USE_MPI       = TRUE
+USE_OMP       = FALSE
+USE_CUDA      = FALSE
+USE_HIP       = FALSE
+COMP          = gnu
+DIM           = 3
+TINY_PROFILE  = FALSE
+
+USE_HEFFTE_FFTW   = FALSE
+USE_HEFFTE_CUFFT  = FALSE
+USE_HEFFTE_ROCFFT = FALSE
+
+ifeq ($(USE_HEFFTE_FFTW),TRUE)
+  HEFFTE_HOME ?= ../../../../heffte/
+else ifeq ($(USE_HEFFTE_CUFFT),TRUE)
+  HEFFTE_HOME ?= ../../../../heffte-org/build_aware/
+else ifeq ($(USE_HEFFTE_ROCFFT),TRUE)
+  HEFFTE_HOME ?= ../../../../heffte/
+endif
+
+include $(AMREX_HOME)/Tools/GNUMake/Make.defs
+
+VPATH_LOCATIONS   += .
+INCLUDE_LOCATIONS += .
+
+#ifeq ($(USE_HEFFTE_FFTW),TRUE)
+#  include $(HEFFTE_HOME)/src/Make.package
+#else ifeq ($(USE_HEFFTE_CUFFT),TRUE)
+#  include $(HEFFTE_HOME)/src/Make.package
+#else ifeq ($(USE_HEFFTE_ROCFFT),TRUE)
+#  include $(HEFFTE_HOME)/src/Make.package
+#endif
+
+include ./Make.package
+ifeq ($(USE_HEFFTE_FFTW),TRUE)
+  DEFINES += -DHEFFTE_FFTW
+  LIBRARIES += -L$(FFTW_DIR) -lfftw3_mpi -lfftw3 -lfftw3f
+else ifeq ($(USE_HEFFTE_CUFFT),TRUE)
+  DEFINES += -DHEFFTE_CUFFT
+  VPATH_LOCATIONS += $(HEFFTE_HOME)/include
+  INCLUDE_LOCATIONS += $(HEFFTE_HOME)/include
+  LIBRARY_LOCATIONS += $(HEFFTE_HOME)/lib
+  LIBRARIES += -lheffte
+else ifeq ($(USE_HEFFTE_ROCFFT),TRUE)
+  DEFINES += -DHEFFTE_ROCFFT
+endif
+
+include $(AMREX_HOME)/Src/Base/Make.package
+
+include $(AMREX_HOME)/Tools/GNUMake/Make.rules
+
+ifeq ($(USE_CUDA),TRUE)
+  LIBRARIES += -lcufft
+else ifeq ($(USE_HIP),TRUE)
+  # Use rocFFT.  ROC_PATH is defined in amrex
+  INCLUDE_LOCATIONS += $(ROC_PATH)/rocfft/include
+  LIBRARY_LOCATIONS += $(ROC_PATH)/rocfft/lib
+  LIBRARIES += -L$(ROC_PATH)/rocfft/lib -lrocfft
+else
+  LIBRARIES += -L$(FFTW_DIR) -lfftw3_mpi -lfftw3 -lfftw3f
+endif
+
+ifeq ($(DO_TURB), TRUE)
+  DEFINES += -DTURB
+endif
+

exec/compressible_stag/SPECTRAL_FILTER/Make.package

@@ -0,0 +1,5 @@
+CEXE_sources += main.cpp
+CEXE_sources += main_driver.cpp
+CEXE_sources += spectral_functions.cpp
+
+CEXE_headers += spectral_functions.H

exec/compressible_stag/SPECTRAL_FILTER/build_frontier.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/bash
+
+## load necessary modules 
+module load craype-accel-amd-gfx90a
+module load amd-mixed
+#module load rocm/5.2.0  # waiting for 5.6 for next bump
+module load cray-mpich/8.1.23
+module load cce/15.0.0  # must be loaded after rocm
+
+# GPU-aware MPI
+export MPICH_GPU_SUPPORT_ENABLED=1
+
+# optimize CUDA compilation for MI250X
+export AMREX_AMD_ARCH=gfx90a
+
+# compiler environment hints
+##export CC=$(which hipcc)
+##export CXX=$(which hipcc)
+##export FC=$(which ftn)
+##export CFLAGS="-I${ROCM_PATH}/include"
+##export CXXFLAGS="-I${ROCM_PATH}/include -Wno-pass-failed"
+##export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64 ${PE_MPICH_GTL_DIR_amd_gfx90a} -lmpi_gtl_hsa"
+export LDFLAGS="-L${MPICH_DIR}/lib -lmpi ${CRAY_XPMEM_POST_LINK_OPTS} -lxpmem ${PE_MPICH_GTL_DIR_amd_gfx90a} ${PE_MPICH_GTL_LIBS_amd_gfx90a}"
+export CXXFLAGS="-I${MPICH_DIR}/include"
+export HIPFLAGS="--amdgpu-target=gfx90a"
+
+make -j10 USE_HIP=TRUE USE_HEFFTE_ROCFFT=TRUE USE_ASSERTION=TRUE

exec/compressible_stag/SPECTRAL_FILTER/build_perlmutter.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/bash
+
+# required dependencies
+module load cray-fftw
+module load cmake
+module load cudatoolkit
+
+module list
+
+# necessary to use CUDA-Aware MPI and run a job
+export CRAY_ACCEL_TARGET=nvidia80
+
+export MPICH_GPU_SUPPORT_ENABLED=1
+
+# optimize CUDA compilation for A100
+export AMREX_CUDA_ARCH=8.0
+
+# optimize CPU microarchitecture for AMD EPYC 3rd Gen (Milan/Zen3)
+# note: the cc/CC/ftn wrappers below add those
+export CXXFLAGS="-march=znver3"
+export CFLAGS="-march=znver3"
+
+# compiler environment hints
+export CC=cc
+export CXX=CC
+export FC=ftn
+export CUDACXX=$(which nvcc)
+export CUDAHOSTCXX=CC
+
+make -j10 USE_CUDA=TRUE USE_HEFFTE_CUFFT=TRUE USE_ASSERTION=TRUE MAX_SPEC=2

exec/compressible_stag/SPECTRAL_FILTER/main.cpp

@@ -0,0 +1,28 @@
+#include <AMReX.H>
+//#include <AMReX_ParmParse.H>
+
+// function declaration
+void main_driver (const char* argv);
+
+int main (int argc, char* argv[])
+{
+    amrex::Initialize(argc,argv);
+
+// this specific part has been moved to Flagellum/main_driver.cpp
+//    {
+//        amrex::ParmParse pp("particles");
+//#ifdef AMREX_USE_GPU
+//        bool particles_do_tiling = true;
+//#else
+//        bool particles_do_tiling = false;
+//#endif
+//        pp.queryAdd("do_tiling", particles_do_tiling);
+//    }
+
+    // argv[1] contains the name of the inputs file entered at the command line
+    main_driver(argv[1]);
+
+    amrex::Finalize();
+
+    return 0;
+}