diff --git a/.github/workflows/ice.yml b/.github/workflows/ice.yml index 1879559..79fe7cb 100644 --- a/.github/workflows/ice.yml +++ b/.github/workflows/ice.yml @@ -1,6 +1,7 @@ name: 'Test on ICE' on: + push: pull_request: workflow_dispatch: @@ -18,13 +19,13 @@ jobs: # use mkl option - name: Build Packages run: | - ml gcc/12.3.0 mvapich2/2.3.7-1 intel-oneapi-mkl/2023.1.0 netcdf-fortran/4.6.0-mva2-hdf5-1.14 fftw/3.3.10-mva cmake - ./rbc.sh install + module load gcc/12.3.0 mvapich2/2.3.7-1 netcdf-c hdf5/1.14.1-2-mva2 intel-oneapi-mkl/2023.1.0 python/3.10.10 fftw/3.3.10-mva2 cmake + ./rbc.sh install-ice # will stop on any errors - name: Compile Cases run: | - ml gcc/12.3.0 mvapich2/2.3.7-1 intel-oneapi-mkl/2023.1.0 netcdf-fortran/4.6.0-mva2-hdf5-1.14 fftw/3.3.10-mva cmake + module load gcc/12.3.0 mvapich2/2.3.7-1 netcdf-c hdf5/1.14.1-2-mva2 intel-oneapi-mkl/2023.1.0 python/3.10.10 fftw/3.3.10-mva2 cmake set -e -x export PETSC_DIR=`pwd`/packages/petsc-3.19.6 export PETSC_ARCH=arch-linux-c-opt diff --git a/.github/workflows/phoenix.yml b/.github/workflows/phoenix.yml index c22d0a7..46e503f 100644 --- a/.github/workflows/phoenix.yml +++ b/.github/workflows/phoenix.yml @@ -1,6 +1,7 @@ name: 'Test on Phoenix' on: + push: pull_request: workflow_dispatch: @@ -9,6 +10,9 @@ jobs: name: Phoenix Runner runs-on: labels: gt + env: + ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true if: github.repository == 'Comp-Physics/RBC3D' continue-on-error: true steps: @@ -17,12 +21,12 @@ jobs: - name: Build Packages run: | - ml gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv + module load gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv ./rbc.sh install-phoenix - name: Make Case with Makefiles run: | - ml gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv netcdf-fortran cmake + module load gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv netcdf-fortran cmake set -e -x export FFTWROOT=`pwd`/packages/fftw-3.3.10/build export PETSC_DIR=`pwd`/packages/petsc-3.19.6 @@ -37,7 +41,7 @@ jobs: - name: Compile Cases with CMake run: | - ml gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv netcdf-fortran cmake + module load gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv netcdf-fortran cmake set -e -x export PETSC_DIR=`pwd`/packages/petsc-3.19.6 export PETSC_ARCH=arch-linux-c-opt diff --git a/.gitignore b/.gitignore index 8d33366..0569867 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ field packages* traction build +*.sbatch diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ad777c..b5b3955 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,7 +21,7 @@ set(USE_MKL OFF) if (DEFINED ENV{MKLROOT}) set(USE_MKL ON) - message("Using MKL lapack because MKLROOT=$ENV{MKLROOT}") + message(STATUS "MKLROOT: $ENV{MKLROOT}") # use mkl implementation of lapack set(BLA_VENDOR Intel10_64lp) endif() @@ -102,7 +102,10 @@ elseif (DEFINED ENV{NETCDF_FORTRAN_ROOT}) set(NETCDF_INCLUDE "$ENV{NETCDF_FORTRAN_ROOT}/include") set(NETCDF_LIB "-L$ENV{NETCDF_FORTRAN_ROOT}/lib -lnetcdff") else () - message(FATAL_ERROR "NETCDF_FORTRANROOT or NETCDF_FORTRAN_ROOT environment variables not found. Please module load netcdf-fortran.") + message(WARNING "NETCDF_FORTRANROOT and NETCDF_FORTRAN_ROOT environment variables from module load fftw not found. Assuming you installed NETCDF-FORTRAN in packages.") + set(NETCDF_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/packages/NETCDF_INST/include") + # -rpath linker option to avoid having to add netcdf library to LD_LIBRARY_PATH + set(NETCDF_LIB "-Wl,-rpath -Wl,${CMAKE_CURRENT_SOURCE_DIR}/packages/NETCDF_INST/lib -L${CMAKE_CURRENT_SOURCE_DIR}/packages/NETCDF_INST/lib -lnetcdff") endif() # spherepack library @@ -129,8 +132,6 @@ foreach (example ${examples_list}) endif() endforeach() -cmake_print_variables(LAPACK_LIBRARIES) - foreach (casedir ${dirlist}) # make binary dir inside /build for case executables diff --git a/Makefile.in b/Makefile.in index 776e859..6844716 100755 --- a/Makefile.in +++ b/Makefile.in @@ -8,6 +8,9 @@ PETSC_ARCH = $(PETSC_DIR)/arch-linux-c-opt SPHEREPACK_DIR = $(WORK_DIR)/packages/spherepack3.2 LAPACK_DIR = $(WORK_DIR)/packages/lapack-3.11 +# if netcdf-fortran is manually installed then this will need to be set instead +# NETCDF_DIR = $(WORK_DIR)/packages/NETCDF_INST +NETCDF_DIR = $(NETCDF_FORTRANROOT) # Makedependf90 binary MAKEDEPEND_BIN = $(WORK_DIR)/packages/makedepf90/makedepf90 @@ -17,7 +20,7 @@ vpath $(WORK_DIR)/common # Includes PETSC_INCLUDE = $(PETSC_DIR)/include PETSC_ARCH_INCLUDE = $(PETSC_ARCH)/include -NETCDF_INCLUDE = $(NETCDF_FORTRANROOT)/include +NETCDF_INCLUDE = $(NETCDF_DIR)/include # If you installed FFTW in packages through install/install-phoenix.sh # run: export FFTWROOT=`pwd`/packages/fftw-3.3.10/build from RBC3D root directory in terminal # or make a variable in this file @@ -31,7 +34,7 @@ INCLUDE = $(COMMON_INCLUDE) -I$(PETSC_INCLUDE) -I$(PETSC_ARCH_INCLUDE) -I$(NETCD COMMON_LIB = $(WORK_DIR)/common/libcommon.a SPHPK_LIB = -L$(SPHEREPACK_DIR)/lib -lspherepack FFTW_LIB = -L$(FFTWROOT)/lib -lfftw3 -NETCDF_LIB = -L$(NETCDF_FORTRANROOT)/lib -lnetcdff +NETCDF_LIB = -L$(NETCDF_DIR)/lib -lnetcdff PETSC_LIB = -Wl,-rpath,$(PETSC_ARCH)/lib -L$(PETSC_ARCH)/lib -lpetsc -lstdc++ MKL_LIB = -L$(MKL_ROOT)lib/intel64/ -lmkl_gf_lp64 -lmkl_core -lmkl_sequential -lpthread -lm -ldl BLAS_LIB = -L$(BLAS_DIR) -lrefblas diff --git a/README.md b/README.md index 4c03925..86e2c59 100644 --- a/README.md +++ b/README.md @@ -23,26 +23,33 @@ This codebase solves the boundary integral form of the Stokes equations via an a ### Installation -[![Test on ICE](https://github.com/comp-physics/RBC3D/actions/workflows/ice.yml/badge.svg)](https://github.com/comp-physics/RBC3D/actions/workflows/ice.yml) +
-To install on PACE Phoenix, you need to salloc a node to make sure srun is available and then run this in the RBC3D root directory: +To install on PACE Phoenix, you need to salloc a node to make sure `srun` is available and then run this in the RBC3D root directory: ```shell -ml gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv python/3.9.12-rkxvr6 netcdf-fortran cmake +module load gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv python/3.9.12-rkxvr6 netcdf-fortran cmake ./rbc.sh install-phoenix ``` Note that if the `gcc`, `mvapich2`, `mkl`, and `fftw` modules work on your Phoenix account, you should use this installer script for a faster build. ```shell -ml gcc mvapich2 mkl python/3.9.12-rkxvr6 netcdf-fortran fftw cmake +module load gcc mvapich2 mkl python/3.9.12-rkxvr6 netcdf-fortran fftw cmake ./rbc.sh install ``` -Or if you're on COC-ICE, you just need to load different modules to run the installer script. +Or if you're on the ICE cluster, you just need to load different modules to run the installer script. ```shell -ml gcc/12.3.0 mvapich2/2.3.7-1 intel-oneapi-mkl/2023.1.0 python/3.10.10 netcdf-fortran/4.6.0-mva2-hdf5-1.14 fftw/3.3.10-mva2 cmake -./rbc.sh install +module load gcc/12.3.0 mvapich2/2.3.7-1 netcdf-c hdf5/1.14.1-2-mva2 intel-oneapi-mkl/2023.1.0 python/3.10.10 fftw/3.3.10-mva2 cmake +./rbc.sh install-ice ``` Before you can run cmake, you must set these environment variables. You can place them in your `~/.bashrc`. If you didn't place `RBC3D` in your `$HOME` directory, then replace it with where you placed `RBC3D`. @@ -58,7 +65,7 @@ cd build cmake .. make case # or just `make` to make common and all the cases cd case -srun -n 1 ./initcond +srun -n 1 ./initcond # or mpiexec srun ./tube ``` diff --git a/common/ModPME.F90 b/common/ModPME.F90 index 70324c0..64be4f8 100644 --- a/common/ModPME.F90 +++ b/common/ModPME.F90 @@ -269,7 +269,7 @@ subroutine PME_Init allocate (tt(ixBgn(1):ixEnd(1), ixBgn(2):ixEnd(2), ixBgn(3):ixEnd(3), 3, 3)) ! Note: ! We need to extend the domain of velocity mesh in z-direction - ! since velocity interplation needs data in the buffer zon + ! since velocity interpolation needs data in the buffer zon allocate (vv(ixBgn(1):ixEnd(1), ixBgn(2):ixEnd(2), ixBgn(3) - PBspln:ixEnd(3), 3)) allocate (ffC(iqBgn(3):iqEnd(3), iqBgn(2):iqEnd(2), iqBgn(1):iqEnd(1), 3)) diff --git a/install/install-ice.sh b/install/install-ice.sh new file mode 100644 index 0000000..462d5fc --- /dev/null +++ b/install/install-ice.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +module load gcc/12.3.0 mvapich2/2.3.7-1 netcdf-c hdf5/1.14.1-2-mva2 intel-oneapi-mkl/2023.1.0 python/3.10.10 fftw/3.3.10-mva2 cmake + +# build and install netcdf-c in packages/NETCDF_INST +rm -fr packages +mkdir packages +cd packages + +export ROOTDIR=$(pwd) +export SRCNCDF=${ROOTDIR}/srcNETCDF +export INSNCDF=${ROOTDIR}/NETCDF_INST + +mkdir $SRCNCDF +mkdir $INSNCDF + +cd $SRCNCDF + +wget downloads.unidata.ucar.edu/netcdf-fortran/4.6.1/netcdf-fortran-4.6.1.tar.gz + +tar -xf netcdf-fortran-4.6.1.tar.gz + +# build and install netcdf-fortran in packages/NETCDF_INST +cd netcdf-fortran-4.6.1 +# instructions for build with shared libraries +# https://docs.unidata.ucar.edu/netcdf-c/current/building_netcdf_fortran.html +export NCDIR=${NETCDF_CROOT} +export NFDIR=${INSNCDF} +export CPPFLAGS=$CPPFLAGS" -I${NCDIR}/include" +export LDFLAGS=$LDFLAGS" -L${NCDIR}/lib" + +./configure --prefix=${INSNCDF} +if (($?)); then + echo "[install-ice.sh] Error: NETCDF-Fortran configure failed." + exit 1 +fi +make check +if (($?)); then + echo "[install-ice.sh] Error: NETCDF-Fortran tests failed." + exit 1 +fi +make install + +cd ../.. + +# building and installing petsc 3.19.6 in packages directory +wget https://ftp.mcs.anl.gov/pub/petsc/petsc-3.19.tar.gz +tar -xf petsc-3.19.tar.gz + +cd petsc-3.19.6 + +# using mpiexec here instead of srun but srun works too --with-mpiexec=mpiexec \ +./configure --with-cc=mpicc \ + --with-cxx=mpicxx \ + --with-fc=mpif90 \ + --with-fortran-datatypes \ + --with-debugging=0 \ + --COPTFLAGS=-g -O3 -march=native -mtune=native \ + --CXXOPTFLAGS=-g -O3 -march=native -mtune=native \ + --FOPTFLAGS=-g -O3 -march=native -mtune=native \ + --with-blaslapack-dir=$MKLROOT \ + --with-mpiexec=srun \ + --with-x11=0 --with-x=0 --with-windows-graphics=0 + +if (($?)); then + echo "[install-ice.sh] Error: PETSc configure failed. See configure.log for more details" + cat configure.log + exit 1 +fi + +make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt all +make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt check + +# build and install spherepack +cd .. +git clone https://github.com/comp-physics/spherepack3.2.git +cd spherepack3.2 +make -j 8 + +echo "Done installing RBC3D!" diff --git a/install/install-phoenix.sh b/install/install-phoenix.sh index 3bfcac8..400bc6d 100644 --- a/install/install-phoenix.sh +++ b/install/install-phoenix.sh @@ -1,7 +1,6 @@ #!/bin/bash -# salloc a node just in case before you run this because petsc tests use srun -ml gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv mkl python/3.9.12-rkxvr6 netcdf-fortran cmake +module load gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv mkl python/3.9.12-rkxvr6 netcdf-fortran cmake # create packages directory mkdir packages @@ -68,4 +67,4 @@ make -j 8 cd ../.. ./rbc.sh install-makedepf90 -echo "Done installing RBC3D!" \ No newline at end of file +echo "Done installing RBC3D!" diff --git a/install/install.sh b/install/install.sh index 5320129..c1cdca8 100644 --- a/install/install.sh +++ b/install/install.sh @@ -11,7 +11,6 @@ tar -xf petsc-3.19.tar.gz cd petsc-3.19.6 -# if these configure options don't work, it's probably a path issue ./configure --with-cc=mpicc \ --with-cxx=mpicxx \ --with-fc=mpif90 \ @@ -39,4 +38,4 @@ git clone https://github.com/comp-physics/spherepack3.2.git cd spherepack3.2 make -j 8 -echo "Done installing RBC3D!" \ No newline at end of file +echo "Done installing RBC3D!" diff --git a/install/readme.md b/install/readme.md index b475668..c9190c5 100644 --- a/install/readme.md +++ b/install/readme.md @@ -21,7 +21,7 @@ You will need `gcc`, `gfortran`, and a suitable MPI wrapper like `mvapich` (or t * To check for gfortran and gcc, see if `which gfortran` and `which gcc` return a path * Similarly, to see if you can run MPI commands for later, see if `which mpicc` or `which mpif90` return a path -You will also need python3 for the PETSc install and pip modules. On Phoenix, you can module load it via `ml python/3.9.12-rkxvr6`, and ICE has a similar module. You may also need to add the `~/.local/bin` directory to your PATH by adding this line to your `~/.bashrc`: +You will also need python3 for the PETSc install and pip modules. On Phoenix, you can module load it via `module load python/3.9.12-rkxvr6`, and ICE has a similar module. You may also need to add the `~/.local/bin` directory to your PATH by adding this line to your `~/.bashrc`: ```shell export PATH="$PATH:$HOME/.local/bin" @@ -118,6 +118,7 @@ make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt check * Later you will need information about where `netcdf-fortran` is installed for `Makefile.in`. * Get this via `module show netcdf-fortran` and looking at the `NETCDF_FORTRANROOT` * In my case, this is `/usr/local/pace-apps/spack/packages/linux-rhel7-x86_64/gcc-10.3.0/netcdf-fortran-4.5.4-yx5osuxluenmuvr3xnahmosfr3abeu2p/` +* An example of how to install `netcdf-fortran` manually is available in `install/install-ice`. ### Spherepack diff --git a/rbc.sh b/rbc.sh index 73c3b6d..16ce16f 100755 --- a/rbc.sh +++ b/rbc.sh @@ -35,6 +35,10 @@ if [ "$1" == 'install-makedepf90' ]; then . "$(pwd)/install/install-makedepf90.sh" $@; exit fi +if [ "$1" == 'install-ice' ]; then + . "$(pwd)/install/install-ice.sh" $@; exit +fi + if [ "$1" == 'cmake' ]; then . "$(pwd)/install/cmake.sh" $@; exit fi