Preliminary makefile for hile gpu

MAKE/Makefile.hile_gpu

@@ -0,0 +1,78 @@
+# compile on compute node
+# srun -p gpu -w x3000c0s8b1n0 -c 16 --mem=20G -t1-0 --pty bash
+#
+# export VLASIATOR_ARCH=hile_gpu
+# module unload perftools-base
+# module load papi
+# module load cray-pmi
+# module load cray-mpich
+# module load craype-accel-amd-gfx90a
+# module load rocm/6.2.0
+#
+# export MPICH_GPU_SUPPORT_ENABLED=1
+
+CMP = OMPI_CXX='hipcc' CC
+LNK = OMPI_CXX='hipcc' CC
+
+#======== Vectorization ==========
+#Set vector backend type for vlasov solvers, sets precision and length. 
+#Options: 
+# AVX:	    VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER
+# AVX512:   VEC8D_AGNER, VEC16F_AGNER
+# Fallback: VECTORCLASS = VEC_FALLBACK_GENERIC (Defaults to VECL8)
+VECTORCLASS = VEC_FALLBACK_GENERIC
+
+#===== Vector Lengths ====
+# Default for VEC_FALLBACK_GENERIC is WID=4, VECL=8 
+#WID=8
+#VECL=64
+WID=4
+VECL=16
+
+# Compile with GPU support (USE_HIP or USE_CUDA)
+USE_HIP=1
+
+#======= Compiler and compilation flags =========
+# NOTES on compiler flags:
+# CXXFLAGS is for compiler flags, they are always used
+# MATHFLAGS are for special math etc. flags, these are only applied on solver functions
+# LDFLAGS flags for linker
+# Important note: Do not edit COMPFLAGS in this file!
+
+CXXFLAGS += -g -ggdb -O3 -x hip --offload-arch=gfx90a:xnack- -march=znver3 -std=c++20 -funroll-loops -fopenmp -I. -Ihip -Iomp -I${CRAY_MPICH_DIR}/include -W -Wall -Wno-unused-parameter -Wno-unused-result -Wno-unused-function -Wno-unused-variable -Wno-unknown-pragmas -Wno-deprecated-register -Wno-unused-but-set-variable -Wno-ignored-attributes -Wno-unused-lambda-capture
+
+testpackage: CXXFLAGS = -g -ggdb -O2 -x hip --offload-arch=gfx90a:xnack- -march=znver3 -std=c++20 -fopenmp -I. -Ihip -Iomp -I${CRAY_MPICH_DIR}/include -W -Wall -Wno-unused-parameter -Wno-unused-result -Wno-unused-function -Wno-unused-variable -Wno-unknown-pragmas -Wno-deprecated-register -Wno-unused-but-set-variable -Wno-ignored-attributes -Wno-unused-lambda-capture
+
+LDFLAGS = -fopenmp -lrt -lpthread -L${CRAY_MPICH_DIR}/lib ${PE_MPICH_GTL_DIR_amd_gfx90a} -L${ROCM_PATH}/lib -L/usr/lib -lamdhip64
+LIB_MPI = -lmpi ${PE_MPICH_GTL_LIBS_amd_gfx90a}
+
+# -fgpu-rdc # relocatable device code, needed for the velocity mesh
+# -fgpu-sanitize -fno-omit-frame-pointer # memory debugging
+
+#======== PAPI ==========
+#Add PAPI_MEM define to use papi to report memory consumption?
+CXXFLAGS +=  -DPAPI_MEM
+testpackage: CXXFLAGS +=  -DPAPI_MEM
+
+#======== Allocator =========
+#Note: jemalloc not supported with GPUs
+
+#======== Libraries ===========
+# Select the base directory based on which project you are using:
+# TODO: Make group directory in new lustre
+LIBRARY_PREFIX = /wrk-kappa/users/markusb/vlasiator/libraries-hile_gpu
+
+# System boost
+LIB_BOOST = -lboost_program_options -L/usr/lib64 -Wl,-rpath=/usr/lib64
+
+# Compiled libraries
+INC_ZOLTAN = -isystem $(LIBRARY_PREFIX)/include
+LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/lib -lzoltan -Wl,-rpath=$(LIBRARY_PREFIX)/lib
+
+LIB_PAPI = -lpapi
+
+INC_VLSV = -isystem $(LIBRARY_PREFIX)/include
+LIB_VLSV = -L$(LIBRARY_PREFIX)/lib -lvlsv -Wl,-rpath=$(LIBRARY_PREFIX)/lib
+
+INC_PROFILE = -isystem $(LIBRARY_PREFIX)/include -D_ROCTX -I${ROCM_PATH}/include
+LIB_PROFILE = -L$(LIBRARY_PREFIX)/lib -lphiprof -Wl,-rpath=$(LIBRARY_PREFIX)/lib -Wl,-rpath=${ROCM_PATH}/lib -lroctx64 -lroctracer64

build_libraries.sh

@@ -33,6 +33,8 @@ elif [[ $PLATFORM == "-appleM1" ]]; then
    make -j 4 CCC=mpic++ CC=appleLLVM CCFLAGS="-fpic -O2 -std=c++17 -DCLOCK_ID=CLOCK_MONOTONIC -fopenmp" LDFLAGS="-fopenmp"
 elif [[ $PLATFORM == "-leonardo_dcgp_intel" ]]; then
    make -j 4 CCC="mpiicpc -cxx=icpx" CC="mpiicc -cc=icx" CCFLAGS="-fpic -O2 -std=c++17 -DCLOCK_ID=CLOCK_MONOTONIC -qopenmp" LDFLAGS="-qopenmp"
+elif [[ $PLATFORM == "-hile" || $PLATFORM == "-hile_gpu" ]]; then
+   make -j 4 CCC=CC CC=cc CCFLAGS="-fpic -O2 -std=c++17 -DCLOCK_ID=CLOCK_MONOTONIC -fopenmp" LDFLAGS="-fopenmp"
 else
    make -j 4 CCC=mpic++
 fi
@@ -49,6 +51,8 @@ fi
 cd vlsv
 if [[ $PLATFORM == "-leonardo_dcgp_intel" ]]; then
    make ARCH=arch CMP="mpiicpc -cxx=icpx"
+elif [[ $PLATFORM == "-hile" || $PLATFORM == "-hile_gpu" ]]; then
+   make ARCH=arch CMP=CC
 else
    make ARCH=arch
 fi
@@ -57,9 +61,9 @@ cp *.h $WORKSPACE/libraries${PLATFORM}/include
 cd ..
 
 # Build papi
-if [[ $PLATFORM != "-arriesgado" && $PLATFORM != "-appleM1"  && $PLATFORM != "-ukkogpu" ]]; then
+if [[ $PLATFORM != "-arriesgado" && $PLATFORM != "-appleM1" && $PLATFORM != "-ukkogpu" && $PLATFORM != "-hile" && $PLATFORM != "-hile_gpu" ]]; then
     # This fails on RISCV and MacOS
-    # UkkoGPU uses system module
+    # UkkoGPU and HILE use system module
     git clone https://github.com/icl-utk-edu/papi
     cd papi/src
     if [[ $PLATFORM == "-leonardo_dcgp_intel" ]]; then
@@ -75,14 +79,16 @@ if [[ $PLATFORM != "-arriesgado" && $PLATFORM != "-appleM1"  && $PLATFORM != "-u
 fi
 
 # Build jemalloc (not for GPU versions)
-if [[ $PLATFORM != "-leonardo_booster" && $PLATFORM != "-karolina_cuda"  && $PLATFORM != "-ukkogpu" ]]; then    
+if [[ $PLATFORM != "-leonardo_booster" && $PLATFORM != "-karolina_cuda" && $PLATFORM != "-ukkogpu" && $PLATFORM != "-hile_gpu" ]]; then    
     curl -O -L https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2
     tar xjf jemalloc-5.3.0.tar.bz2
     cd jemalloc-5.3.0
     if [[ $PLATFORM == "-arriesgado" ]]; then
         ./configure --prefix=$WORKSPACE/libraries${PLATFORM} --with-jemalloc-prefix=je_
     elif [[ $PLATFORM == "-leonardo_dcgp_intel" ]]; then
         ./configure --prefix=$WORKSPACE/libraries${PLATFORM} --with-jemalloc-prefix=je_ CC="mpiicc -cc=icx" CXX="mpiicpc -cxx=icpx"
+    if [[ $PLATFORM == "-hile" ]]; then
+        ./configure --prefix=$WORKSPACE/libraries${PLATFORM} --with-jemalloc-prefix=je_ CC="cc" CXX="CC"
     else
         ./configure --prefix=$WORKSPACE/libraries${PLATFORM} --with-jemalloc-prefix=je_ CC=mpicc CXX=mpic++
     fi
@@ -92,6 +98,7 @@ fi
 
 # Build Zoltan
 git clone https://github.com/sandialabs/Zoltan.git
+#rm -rf zoltan-build
 mkdir zoltan-build
 cd zoltan-build
 if [[ $PLATFORM == "-arriesgado" ]]; then
@@ -107,14 +114,16 @@ elif [[ $PLATFORM == "-leonardo_dcgp_intel" ]]; then
     sed -i 's/mpiicpc/mpiicpc -cxx=icpx/g' Makefile
     sed -i 's/mpiicpc/mpiicpc -cxx=icpx/g' src/Makefile
     sed -i 's/mpiicpc/mpiicpc -cxx=icpx/g' src/driver/Makefile
+elif [[ $PLATFORM == "-hile" ||  $PLATFORM == "-hile_gpu" ]]; then
+   ../Zoltan/configure --prefix=$WORKSPACE/libraries${PLATFORM} --enable-mpi --with-mpi-compilers --with-gnumake --with-id-type=ullong CC=cc CXX=CC
 else
     ../Zoltan/configure --prefix=$WORKSPACE/libraries${PLATFORM} --enable-mpi --with-mpi-compilers --with-gnumake --with-id-type=ullong CC=mpicc CXX=mpic++
 fi
 make -j 4 && make install
 cd ..
 
 # Build boost
-if [[ $PLATFORM == "-hile" || $PLATFORM == "-leonardo_booster" || $PLATFORM == "-leonardo_dcgp" || $PLATFORM == "-karolina_cuda" || $PLATFORM == "-karolina_gcc" || $PLATFORM == "-ukkogpu" ]]; then
+if [[ $PLATFORM == "-leonardo_booster" || $PLATFORM == "-leonardo_dcgp" || $PLATFORM == "-karolina_cuda" || $PLATFORM == "-karolina_gcc" || $PLATFORM == "-ukkogpu" ]]; then
     echo "### Downloading boost. ###"
     wget -q https://archives.boost.io/release/1.86.0/source/boost_1_86_0.tar.gz
     echo "### Extracting boost. ###"