Docs: Add compile-notes for frontera

Docs/compile-notes/frontera/README.md

@@ -0,0 +1,67 @@
+# Install CarpetX with Spack (Frontera)
+
+Use interactive session
+
+* Compile CPU version: `idev -m 120`
+
+* Compile GPU version: `idev -p rtx-dev -m 120`
+
+Download spack
+
+* `git clone -c feature.manyFiles=true https://github.com/spack/spack.git`
+
+* `git checkout relesases/v0.20`
+
+## Download CarpetX and SpacetimeX
+
+```
+curl -kLO https://raw.githubusercontent.com/gridaphobe/CRL/master/GetComponents
+chmod a+x GetComponents
+./GetComponents --root Cactus --parallel --no-shallow https://raw.githubusercontent.com/lwJi/SpacetimeX/main/scripts/spacetimex.th
+```
+
+```
+cd Cactus/repos
+git clone https://github.com/lwJi/SpacetimeX.git
+cd ../arrangements
+ln -s ../repos/SpacetimeX
+```
+
+## Intel-Oneapi version (`[email protected]`)
+
+Load `intel/23.1.0`
+
+* `module load intel/23.1.0`
+
+Setup spack
+
+* `. share/spack/setup-env.sh`
+
+* `spack compiler find`
+
+Create a dir where you want put `view` in (say `/work2/.../username/frontera/SpackView/oneapi`)
+
+* replace the last line of `oneapi-23.1.0/spack.yaml` with your dir (say `/work2/.../username/frontera/SpackView/oneapi/view`)
+
+* replace the dir `/work2/08708/liwei/frontera/SpackView/oneapi/view` (with say `/work2/.../username/frontera/SpackView/oneapi/view`)
+in `config_frontera_oneapi.cfg`
+
+Install other required packages
+
+* `env TMPDIR=$WORK/tmp spack --env-dir ./oneapi-23.1.0 compiler find`
+
+* `env TMPDIR=$WORK/tmp spack --env-dir ./oneapi-23.1.0 concretize --force`
+
+* `env TMPDIR=$WORK/tmp spack --env-dir ./oneapi-23.1.0 install --fail-fast`
+
+Install CarpetX
+
+* `cd Cactus`
+
+* `gmake CarpetX-oneapi options=config_frontera_oneapi.cfg`
+
+* `cp repos/AsterX/scripts/asterx.th configs/CarpetX-oneapi/ThornList`
+
+* `gmake -j16 CarpetX-oneapi`
+
+

Docs/compile-notes/frontera/configs/config-frontera-cuda.cfg

@@ -0,0 +1,96 @@
+# Configuration for an frontera installation: cuda
+
+# Whenever this version string changes, the application is configured 
+# and rebuilt from scratch 
+VERSION = frontera-2023-09-30
+
+CPP = cpp
+CC  = gcc
+CXX = nvcc -x cu
+FPP = cpp
+FC = gfortran
+F90 = gfortran
+LD = nvcc
+
+CPPFLAGS = -DSIMD_CPU
+CFLAGS   = -pipe -g -std=gnu11
+CXXFLAGS = -pipe -g0 -std=c++17 --compiler-options -std=gnu++17 --expt-relaxed-constexpr --extended-lambda --gpu-architecture sm_75 --forward-unknown-to-host-compiler --Werror ext-lambda-captures-this --relocatable-device-code=true --objdir-as-tempdir
+FPPFLAGS = -traditional
+F90FLAGS = -pipe -g -fcray-pointer -ffixed-line-length-none
+LIBS = nvToolsExt gfortran
+
+#CUCCFLAGS = -std=c++17
+#CUCC = nvcc
+#CUCCFLAGS = -std=c++14 -I/opt/apps/cuda/11.3/include #-L/opt/apps/cuda/11.3/lib64
+#LDFLAGS  = -rdynamic
+#LIBDIRS = /opt/apps/cuda/11.3/lib64
+
+C_LINE_DIRECTIVES = yes
+F_LINE_DIRECTIVES = yes
+
+INTEGER_PRECISION = 4
+REAL_PRECISION = 8
+
+# CUDA does not support these
+DISABLE_INT16 = yes
+DISABLE_REAL16 = yes
+
+DEBUG           = no
+CPP_DEBUG_FLAGS = -DCARPET_DEBUG
+C_DEBUG_FLAGS   = -fbounds-check -fsanitize=undefined -fstack-protector-all -ftrapv
+CXX_DEBUG_FLAGS = -fbounds-check -fsanitize=undefined -fstack-protector-all -ftrapv -lineinfo
+FPP_DEBUG_FLAGS = -DCARPET_DEBUG
+F90_DEBUG_FLAGS = -fcheck=bounds,do,mem,pointer,recursion -finit-character=65 -finit-integer=42424242 -finit-real=nan -fsanitize=undefined -fstack-protector-all -ftrapv
+
+OPTIMISE           = yes
+C_OPTIMISE_FLAGS   = -O3 -fcx-limited-range -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math -fno-signaling-nans -funsafe-math-optimizations
+CXX_OPTIMISE_FLAGS = -O3 -fcx-limited-range -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math -fno-signaling-nans -funsafe-math-optimizations
+F90_OPTIMISE_FLAGS = -O3 -fcx-limited-range -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math -fno-signaling-nans -funsafe-math-optimizations
+
+OPENMP           = yes
+CPP_OPENMP_FLAGS = -fopenmp
+FPP_OPENMP_FLAGS = -D_OPENMP
+
+WARN           = yes
+CPP_WARN_FLAGS = -Wall
+C_WARN_FLAGS =
+CXX_WARN_FLAGS =
+FPP_WARN_FLAGS = -Wall
+F90_WARN_FLAGS = -Wall -Wshadow -Wsurprising
+
+VECTORISE = no
+
+MPI_DIR = /opt/intel/compilers_and_libraries_2020.4.304/linux/mpi
+MPI_LIB_DIRS = /opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/lib/release
+MPI_INC_DIRS = /opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/include
+MPI_LIBS = mpi
+
+GSL_DIR = /opt/apps/intel19/gsl/2.6
+FFTW3_DIR = /opt/apps/intel19/impi19_0/fftw3/3.3.10
+HWLOC_DIR = /opt/apps/hwloc/1.11.12
+
+BLAS_DIR = NO_BUILD
+BLAS_LIBS = # /opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/lib/intel64
+LAPACK_DIR = NO_BUILD
+LAPACK_LIBS = # /opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/lib/intel64
+OPENBLAS_DIR = NO_BUILD
+OPENBLAS_LIBS = # /opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/lib/intel64
+
+PTHREADS_DIR = NO_BUILD
+
+HDF5_DIR = /work2/08708/liwei/frontera/SpackView/gcc/view
+ADIOS2_DIR = /work2/08708/liwei/frontera/SpackView/cuda/view
+ADIOS2_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/cuda/view/lib64
+ADIOS2_LIBS = adios2_cxx11_mpi adios2_cxx11
+AMREX_DIR = /work2/08708/liwei/frontera/SpackView/cuda/view
+NSIMD_DIR = /work2/08708/liwei/frontera/SpackView/cuda/view
+NSIMD_INC_DIRS = /work2/08708/liwei/frontera/SpackView/cuda/view/include
+NSIMD_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/cuda/view/lib
+NSIMD_ARCH = AVX2
+NSIMD_OPTIONS =  #AVX2 FMA
+OPENPMD_API_DIR = /work2/08708/liwei/frontera/SpackView/cuda/view
+OPENPMD_DIR = /work2/08708/liwei/frontera/SpackView/cuda/view
+SSHT_DIR = /work2/08708/liwei/frontera/SpackView/cuda/view
+SILO_DIR = /work2/08708/liwei/frontera/SpackView/cuda/view
+YAML_CPP_DIR = /work2/08708/liwei/frontera/SpackView/cuda/view
+ZLIB_DIR = /work2/08708/liwei/frontera/SpackView/cuda/view

Docs/compile-notes/frontera/configs/config-frontera-gcc.cfg

@@ -0,0 +1,84 @@
+# Configuration for an frontera installation: gcc
+
+# Whenever this version string changes, the application is configured 
+# and rebuilt from scratch 
+VERSION = frontera-2023-09-30
+
+CPP = cpp
+CC  = gcc
+CXX = g++
+FPP = cpp
+F77 = gfortran
+F90 = gfortran
+
+CFLAGS   = -pipe -g -std=gnu11
+CXXFLAGS = -pipe -g -std=gnu++17
+FPPFLAGS = -traditional
+F90FLAGS = -pipe -g -fcray-pointer -ffixed-line-length-none
+#LDFLAGS  = -Wl,-rpath=/work2/08708/liwei/frontera/SpackView/gcc/view/lib
+#CPPFLAGS = -DMPICH_IGNORE_CXX_SEEK
+
+C_LINE_DIRECTIVES = yes
+F_LINE_DIRECTIVES = yes
+
+INTEGER_PRECISION = 4
+REAL_PRECISION = 8
+
+DEBUG           = no
+CPP_DEBUG_FLAGS = -DCARPET_DEBUG
+C_DEBUG_FLAGS   = -fbounds-check -fsanitize=undefined -fstack-protector-all -ftrapv
+CXX_DEBUG_FLAGS = -fbounds-check -fsanitize=undefined -fstack-protector-all -ftrapv
+FPP_DEBUG_FLAGS = -DCARPET_DEBUG
+F90_DEBUG_FLAGS = -fcheck=bounds,do,mem,pointer,recursion -finit-character=65 -finit-integer=42424242 -finit-real=nan -fsanitize=undefined -fstack-protector-all -ftrapv
+
+OPTIMISE           = yes
+C_OPTIMISE_FLAGS   = -O3 -fcx-limited-range -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math -fno-signaling-nans -funsafe-math-optimizations
+CXX_OPTIMISE_FLAGS = -O3 -fcx-limited-range -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math -fno-signaling-nans -funsafe-math-optimizations
+F90_OPTIMISE_FLAGS = -O3 -fcx-limited-range -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math -fno-signaling-nans -funsafe-math-optimizations
+
+OPENMP           = yes
+CPP_OPENMP_FLAGS = -fopenmp
+FPP_OPENMP_FLAGS = -D_OPENMP
+
+WARN           = yes
+CPP_WARN_FLAGS = -Wall
+C_WARN_FLAGS =
+CXX_WARN_FLAGS =
+FPP_WARN_FLAGS = -Wall
+F90_WARN_FLAGS = -Wall -Wshadow -Wsurprising
+
+MPI_DIR = /opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64
+MPI_LIB_DIRS = /opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/lib/release
+MPI_INC_DIRS = /opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/include
+MPI_LIBS = mpi
+
+GSL_DIR = /opt/apps/intel19/gsl/2.6
+FFTW3_DIR = /opt/apps/intel19/impi19_0/fftw3/3.3.10
+HWLOC_DIR = /opt/apps/hwloc/1.11.12
+
+BLAS_DIR  = /opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/lib/intel64
+BLAS_LIBS = mkl_intel_lp64 mkl_core mkl_gnu_thread
+LAPACK_DIR  = /opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/lib/intel64
+LAPACK_LIBS = mkl_intel_lp64 mkl_core mkl_gnu_thread
+OPENBLAS_DIR  = /opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/lib/intel64
+OPENBLAS_LIBS = mkl_intel_lp64 mkl_core mkl_gnu_thread
+
+PTHREADS_DIR = NO_BUILD
+
+HDF5_DIR = /work2/08708/liwei/frontera/SpackView/gcc/view
+HDF5_LIBS = hdf5_hl_cpp hdf5_cpp hdf5hl_fortran hdf5_fortran hdf5_hl hdf5
+ADIOS2_DIR = /work2/08708/liwei/frontera/SpackView/gcc/view
+ADIOS2_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/gcc/view/lib64
+ADIOS2_LIBS = adios2_cxx11_mpi adios2_cxx11
+AMREX_DIR = /work2/08708/liwei/frontera/SpackView/gcc/view
+NSIMD_DIR = /work2/08708/liwei/frontera/SpackView/gcc/view
+NSIMD_INC_DIRS = /work2/08708/liwei/frontera/SpackView/gcc/view/include
+NSIMD_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/gcc/view/lib
+NSIMD_ARCH = AVX512_SKYLAKE
+NSIMD_OPTIONS =  #AVX2 FMA
+OPENPMD_API_DIR = /work2/08708/liwei/frontera/SpackView/gcc/view
+OPENPMD_DIR = /work2/08708/liwei/frontera/SpackView/gcc/view
+SSHT_DIR = /work2/08708/liwei/frontera/SpackView/gcc/view
+SILO_DIR = /work2/08708/liwei/frontera/SpackView/gcc/view
+YAML_CPP_DIR = /work2/08708/liwei/frontera/SpackView/gcc/view
+ZLIB_DIR = /work2/08708/liwei/frontera/SpackView/gcc/view

Docs/compile-notes/frontera/configs/config-frontera-oneapi.cfg

@@ -0,0 +1,93 @@
+# Configuration for an frontera installation: oneapi
+
+# Whenever this version string changes, the application is configured 
+# and rebuilt from scratch 
+VERSION = frontera-2023-09-30
+
+CPP = cpp
+CC  = icx
+CXX = icpx
+FPP = cpp
+F77 = ifx
+F90 = ifx
+
+CPPFLAGS = -g -D_XOPEN_SOURCE -D_XOPEN_SOURCE_EXTENDED -DSIMD_CPU
+FPPFLAGS = -g -traditional
+CFLAGS   = -g -pipe -std=c11
+CXXFLAGS = -g -pipe -std=c++17
+F77FLAGS = -g -traceback -debug all -xCORE-AVX512 -align -pad -safe-cray-ptr
+F90FLAGS = -g -traceback -debug all -xCORE-AVX512 -align -pad -safe-cray-ptr
+
+#LDFLAGS  = -Wl,-rpath=/work2/08708/liwei/frontera/SpackView/oneapi/view/lib
+#CPPFLAGS = -DMPICH_IGNORE_CXX_SEEK
+
+C_LINE_DIRECTIVES = yes
+F_LINE_DIRECTIVES = yes
+
+INTEGER_PRECISION = 4
+REAL_PRECISION = 8
+
+DEBUG           = no
+CPP_DEBUG_FLAGS = -DCARPET_DEBUG
+C_DEBUG_FLAGS   = -fbounds-check -fsanitize=undefined -fstack-protector-all -ftrapv
+CXX_DEBUG_FLAGS = -fbounds-check -fsanitize=undefined -fstack-protector-all -ftrapv
+FPP_DEBUG_FLAGS = -DCARPET_DEBUG
+F90_DEBUG_FLAGS = -fcheck=bounds,do,mem,pointer,recursion -finit-character=65 -finit-integer=42424242 -finit-real=nan -fsanitize=undefined -fstack-protector-all -ftrapv
+
+OPTIMISE           = yes
+C_OPTIMISE_FLAGS   = -Ofast
+CXX_OPTIMISE_FLAGS = -Ofast
+F90_OPTIMISE_FLAGS = -Ofast
+
+OPENMP           = yes
+CPP_OPENMP_FLAGS = -fopenmp
+FPP_OPENMP_FLAGS = -fopenmp
+C_OPENMP_FLAGS   = -fopenmp
+CXX_OPENMP_FLAGS = -fopenmp
+F77_OPENMP_FLAGS = -fopenmp
+F90_OPENMP_FLAGS = -fopenmp
+
+WARN           = yes
+CPP_WARN_FLAGS = -Wall
+C_WARN_FLAGS = -Wno-implicit-function-declaration
+CXX_WARN_FLAGS =
+FPP_WARN_FLAGS =
+F90_WARN_FLAGS =
+
+MPI_DIR = /opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64
+MPI_LIB_DIRS = /opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/lib/release
+MPI_INC_DIRS = /opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/include
+MPI_LIBS = mpi
+
+#FFTW3_DIR = /opt/apps/intel19/impi19_0/fftw3/3.3.10
+HWLOC_DIR = /opt/apps/hwloc/1.11.12
+
+BLAS_DIR  = NO_BUILD
+BLAS_LIBS = -mkl
+LAPACK_DIR  = NO_BUILD
+LAPACK_LIBS = -mkl
+OPENBLAS_DIR  = NO_BUILD
+OPENBLAS_LIBS = -mkl
+
+PTHREADS_DIR = NO_BUILD
+
+GSL_DIR = /work2/08708/liwei/frontera/SpackView/oneapi/view
+ADIOS2_DIR = /work2/08708/liwei/frontera/SpackView/oneapi/view
+ADIOS2_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/oneapi/view/lib64
+ADIOS2_LIBS = adios2_cxx11_mpi adios2_cxx11
+OPENPMD_API_DIR = /work2/08708/liwei/frontera/SpackView/oneapi/view
+OPENPMD_DIR = /work2/08708/liwei/frontera/SpackView/oneapi/view
+FFTW3_DIR = /work2/08708/liwei/frontera/SpackView/oneapi/view
+HDF5_DIR = /work2/08708/liwei/frontera/SpackView/oneapi/view
+HDF5_LIBS = hdf5_hl_cpp hdf5_cpp hdf5hl_fortran hdf5_fortran hdf5_hl hdf5
+SILO_DIR = /work2/08708/liwei/frontera/SpackView/oneapi/view
+BOOST_DIR = /work2/08708/liwei/frontera/SpackView/oneapi/view
+AMREX_DIR = /work2/08708/liwei/frontera/SpackView/oneapi/view
+NSIMD_DIR = /work2/08708/liwei/frontera/SpackView/oneapi/view
+NSIMD_INC_DIRS = /work2/08708/liwei/frontera/SpackView/oneapi/view/include
+NSIMD_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/oneapi/view/lib
+NSIMD_ARCH = AVX512_SKYLAKE
+NSIMD_OPTIONS =  #AVX2 FMA
+SSHT_DIR = /work2/08708/liwei/frontera/SpackView/oneapi/view
+YAML_CPP_DIR = /work2/08708/liwei/frontera/SpackView/oneapi/view
+ZLIB_DIR = /work2/08708/liwei/frontera/SpackView/oneapi/view