ICE Runner

.github/workflows/ice.yml

@@ -1,6 +1,7 @@
 name: 'Test on ICE'
 
 on:
+    push:
     pull_request:
     workflow_dispatch:
 
@@ -18,13 +19,13 @@ jobs:
       # use mkl option
       - name: Build Packages
         run: |
-            ml gcc/12.3.0 mvapich2/2.3.7-1 intel-oneapi-mkl/2023.1.0 netcdf-fortran/4.6.0-mva2-hdf5-1.14 fftw/3.3.10-mva cmake
-            ./rbc.sh install
+            module load gcc/12.3.0 mvapich2/2.3.7-1 netcdf-c hdf5/1.14.1-2-mva2 intel-oneapi-mkl/2023.1.0 python/3.10.10 fftw/3.3.10-mva2 cmake
+            ./rbc.sh install-ice
 
       # will stop on any errors
       - name: Compile Cases
         run: |
-            ml gcc/12.3.0 mvapich2/2.3.7-1 intel-oneapi-mkl/2023.1.0 netcdf-fortran/4.6.0-mva2-hdf5-1.14 fftw/3.3.10-mva cmake
+            module load gcc/12.3.0 mvapich2/2.3.7-1 netcdf-c hdf5/1.14.1-2-mva2 intel-oneapi-mkl/2023.1.0 python/3.10.10 fftw/3.3.10-mva2 cmake
             set -e -x
             export PETSC_DIR=`pwd`/packages/petsc-3.19.6
             export PETSC_ARCH=arch-linux-c-opt

.github/workflows/phoenix.yml

@@ -1,6 +1,7 @@
 name: 'Test on Phoenix'
 
 on:
+    push:
     pull_request:
     workflow_dispatch:
 
@@ -9,6 +10,9 @@ jobs:
     name: Phoenix Runner
     runs-on:
         labels: gt
+    env:
+        ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
+        ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     if: github.repository == 'Comp-Physics/RBC3D'
     continue-on-error: true
     steps:
@@ -17,12 +21,12 @@ jobs:
 
       - name: Build Packages
         run: |
-            ml gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv
+            module load gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv
             ./rbc.sh install-phoenix
 
       - name: Make Case with Makefiles
         run: |
-            ml gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv netcdf-fortran cmake
+            module load gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv netcdf-fortran cmake
             set -e -x
             export FFTWROOT=`pwd`/packages/fftw-3.3.10/build
             export PETSC_DIR=`pwd`/packages/petsc-3.19.6
@@ -37,7 +41,7 @@ jobs:
 
       - name: Compile Cases with CMake
         run: |
-            ml gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv netcdf-fortran cmake
+            module load gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv netcdf-fortran cmake
             set -e -x
             export PETSC_DIR=`pwd`/packages/petsc-3.19.6
             export PETSC_ARCH=arch-linux-c-opt

.gitignore

@@ -15,3 +15,4 @@ field
 packages*
 traction
 build
+*.sbatch

CMakeLists.txt

@@ -21,7 +21,7 @@ set(USE_MKL OFF)
 
 if (DEFINED ENV{MKLROOT})
     set(USE_MKL ON)
-    message("Using MKL lapack because MKLROOT=$ENV{MKLROOT}")
+    message(STATUS "MKLROOT: $ENV{MKLROOT}")   
     # use mkl implementation of lapack
     set(BLA_VENDOR Intel10_64lp)
 endif()
@@ -102,7 +102,10 @@ elseif (DEFINED ENV{NETCDF_FORTRAN_ROOT})
     set(NETCDF_INCLUDE "$ENV{NETCDF_FORTRAN_ROOT}/include")
     set(NETCDF_LIB "-L$ENV{NETCDF_FORTRAN_ROOT}/lib -lnetcdff")
 else ()
-    message(FATAL_ERROR "NETCDF_FORTRANROOT or NETCDF_FORTRAN_ROOT environment variables not found. Please module load netcdf-fortran.")
+    message(WARNING "NETCDF_FORTRANROOT and NETCDF_FORTRAN_ROOT environment variables from module load fftw not found. Assuming you installed NETCDF-FORTRAN in packages.")
+    set(NETCDF_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/packages/NETCDF_INST/include")
+    # -rpath linker option to avoid having to add netcdf library to LD_LIBRARY_PATH
+    set(NETCDF_LIB "-Wl,-rpath -Wl,${CMAKE_CURRENT_SOURCE_DIR}/packages/NETCDF_INST/lib -L${CMAKE_CURRENT_SOURCE_DIR}/packages/NETCDF_INST/lib -lnetcdff")
 endif()
 
 # spherepack library
@@ -129,8 +132,6 @@ foreach (example ${examples_list})
     endif()
 endforeach()
 
-cmake_print_variables(LAPACK_LIBRARIES)
-
 foreach (casedir ${dirlist})
 
     # make binary dir inside /build for case executables

Makefile.in

@@ -8,6 +8,9 @@ PETSC_ARCH = $(PETSC_DIR)/arch-linux-c-opt
 SPHEREPACK_DIR = $(WORK_DIR)/packages/spherepack3.2
 
 LAPACK_DIR = $(WORK_DIR)/packages/lapack-3.11
+# if netcdf-fortran is manually installed then this will need to be set instead
+# NETCDF_DIR = $(WORK_DIR)/packages/NETCDF_INST
+NETCDF_DIR = $(NETCDF_FORTRANROOT)
 
 # Makedependf90 binary
 MAKEDEPEND_BIN = $(WORK_DIR)/packages/makedepf90/makedepf90
@@ -17,7 +20,7 @@ vpath $(WORK_DIR)/common
 # Includes
 PETSC_INCLUDE = $(PETSC_DIR)/include
 PETSC_ARCH_INCLUDE = $(PETSC_ARCH)/include
-NETCDF_INCLUDE = $(NETCDF_FORTRANROOT)/include
+NETCDF_INCLUDE = $(NETCDF_DIR)/include
 # If you installed FFTW in packages through install/install-phoenix.sh
 # run: export FFTWROOT=`pwd`/packages/fftw-3.3.10/build from RBC3D root directory in terminal
 # or make a variable in this file
@@ -31,7 +34,7 @@ INCLUDE = $(COMMON_INCLUDE) -I$(PETSC_INCLUDE) -I$(PETSC_ARCH_INCLUDE) -I$(NETCD
 COMMON_LIB = $(WORK_DIR)/common/libcommon.a
 SPHPK_LIB = -L$(SPHEREPACK_DIR)/lib -lspherepack
 FFTW_LIB = -L$(FFTWROOT)/lib -lfftw3
-NETCDF_LIB = -L$(NETCDF_FORTRANROOT)/lib -lnetcdff
+NETCDF_LIB = -L$(NETCDF_DIR)/lib -lnetcdff
 PETSC_LIB = -Wl,-rpath,$(PETSC_ARCH)/lib -L$(PETSC_ARCH)/lib -lpetsc -lstdc++
 MKL_LIB = -L$(MKL_ROOT)lib/intel64/ -lmkl_gf_lp64 -lmkl_core -lmkl_sequential -lpthread -lm -ldl
 BLAS_LIB = -L$(BLAS_DIR) -lrefblas

README.md

@@ -23,26 +23,33 @@ This codebase solves the boundary integral form of the Stokes equations via an a
 
 ### Installation
 
-[![Test on ICE](https://github.com/comp-physics/RBC3D/actions/workflows/ice.yml/badge.svg)](https://github.com/comp-physics/RBC3D/actions/workflows/ice.yml)
+<p align="left">
+  <a href="https://github.com/comp-physics/RBC3D/actions/workflows/phoenix.yml">
+    <img src="https://github.com/comp-physics/RBC3D/actions/workflows/phoenix.yml/badge.svg" />
+  </a>
+  <a href="https://github.com/comp-physics/RBC3D/actions/workflows/ice.yml">
+    <img src="https://github.com/comp-physics/RBC3D/actions/workflows/ice.yml/badge.svg" />
+  </a>
+</p>
 
-To install on PACE Phoenix, you need to salloc a node to make sure srun is available and then run this in the RBC3D root directory: 
+To install on PACE Phoenix, you need to salloc a node to make sure `srun` is available and then run this in the RBC3D root directory: 
 
 ```shell
-ml gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv python/3.9.12-rkxvr6 netcdf-fortran cmake
+module load gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv python/3.9.12-rkxvr6 netcdf-fortran cmake
 ./rbc.sh install-phoenix
 ```
 
 Note that if the `gcc`, `mvapich2`, `mkl`, and `fftw` modules work on your Phoenix account, you should use this installer script for a faster build.
 ```shell
-ml gcc mvapich2 mkl python/3.9.12-rkxvr6 netcdf-fortran fftw cmake
+module load gcc mvapich2 mkl python/3.9.12-rkxvr6 netcdf-fortran fftw cmake
 ./rbc.sh install
 ```
 
-Or if you're on COC-ICE, you just need to load different modules to run the installer script.
+Or if you're on the ICE cluster, you just need to load different modules to run the installer script.
 
 ```shell
-ml gcc/12.3.0 mvapich2/2.3.7-1 intel-oneapi-mkl/2023.1.0 python/3.10.10 netcdf-fortran/4.6.0-mva2-hdf5-1.14 fftw/3.3.10-mva2 cmake
-./rbc.sh install
+module load gcc/12.3.0 mvapich2/2.3.7-1 netcdf-c hdf5/1.14.1-2-mva2 intel-oneapi-mkl/2023.1.0 python/3.10.10 fftw/3.3.10-mva2 cmake
+./rbc.sh install-ice
 ```
 
 Before you can run cmake, you must set these environment variables. You can place them in your `~/.bashrc`. If you didn't place `RBC3D` in your `$HOME` directory, then replace it with where you placed `RBC3D`.
@@ -58,7 +65,7 @@ cd build
 cmake ..
 make case # or just `make` to make common and all the cases
 cd case
-srun -n 1 ./initcond
+srun -n 1 ./initcond # or mpiexec
 srun ./tube
 ```
 

common/ModPME.F90

@@ -269,7 +269,7 @@ subroutine PME_Init
     allocate (tt(ixBgn(1):ixEnd(1), ixBgn(2):ixEnd(2), ixBgn(3):ixEnd(3), 3, 3))
     ! Note:
     !  We need to extend the domain of velocity mesh in z-direction
-    !  since velocity interplation needs data in the buffer zon
+    !  since velocity interpolation needs data in the buffer zon
     allocate (vv(ixBgn(1):ixEnd(1), ixBgn(2):ixEnd(2), ixBgn(3) - PBspln:ixEnd(3), 3))
 
     allocate (ffC(iqBgn(3):iqEnd(3), iqBgn(2):iqEnd(2), iqBgn(1):iqEnd(1), 3))

install/install-ice.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+module load gcc/12.3.0 mvapich2/2.3.7-1 netcdf-c hdf5/1.14.1-2-mva2 intel-oneapi-mkl/2023.1.0 python/3.10.10 fftw/3.3.10-mva2 cmake
+
+# build and install netcdf-c in packages/NETCDF_INST
+rm -fr packages
+mkdir packages
+cd packages
+
+export ROOTDIR=$(pwd)
+export SRCNCDF=${ROOTDIR}/srcNETCDF
+export INSNCDF=${ROOTDIR}/NETCDF_INST
+
+mkdir $SRCNCDF
+mkdir $INSNCDF
+
+cd $SRCNCDF
+
+wget downloads.unidata.ucar.edu/netcdf-fortran/4.6.1/netcdf-fortran-4.6.1.tar.gz
+
+tar -xf netcdf-fortran-4.6.1.tar.gz
+
+# build and install netcdf-fortran in packages/NETCDF_INST
+cd netcdf-fortran-4.6.1
+# instructions for build with shared libraries
+# https://docs.unidata.ucar.edu/netcdf-c/current/building_netcdf_fortran.html
+export NCDIR=${NETCDF_CROOT}
+export NFDIR=${INSNCDF}
+export CPPFLAGS=$CPPFLAGS" -I${NCDIR}/include"
+export LDFLAGS=$LDFLAGS" -L${NCDIR}/lib"
+
+./configure --prefix=${INSNCDF}
+if (($?)); then
+    echo "[install-ice.sh] Error: NETCDF-Fortran configure failed."
+    exit 1
+fi
+make check
+if (($?)); then
+    echo "[install-ice.sh] Error: NETCDF-Fortran tests failed."
+    exit 1
+fi
+make install
+
+cd ../..
+
+# building and installing petsc 3.19.6 in packages directory
+wget https://ftp.mcs.anl.gov/pub/petsc/petsc-3.19.tar.gz
+tar -xf petsc-3.19.tar.gz
+
+cd petsc-3.19.6
+
+# using mpiexec here instead of srun but srun works too --with-mpiexec=mpiexec \
+./configure --with-cc=mpicc \
+    --with-cxx=mpicxx \
+    --with-fc=mpif90 \
+    --with-fortran-datatypes \
+    --with-debugging=0 \
+    --COPTFLAGS=-g -O3 -march=native -mtune=native \
+    --CXXOPTFLAGS=-g -O3 -march=native -mtune=native \
+    --FOPTFLAGS=-g -O3 -march=native -mtune=native \
+    --with-blaslapack-dir=$MKLROOT \
+    --with-mpiexec=srun \
+    --with-x11=0 --with-x=0 --with-windows-graphics=0
+
+if (($?)); then
+    echo "[install-ice.sh] Error: PETSc configure failed. See configure.log for more details"
+    cat configure.log
+    exit 1
+fi
+
+make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt all
+make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt check
+
+# build and install spherepack
+cd ..
+git clone https://github.com/comp-physics/spherepack3.2.git
+cd spherepack3.2
+make -j 8
+
+echo "Done installing RBC3D!"

install/install-phoenix.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 
-# salloc a node just in case before you run this because petsc tests use srun
-ml gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv mkl python/3.9.12-rkxvr6 netcdf-fortran cmake
+module load gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv mkl python/3.9.12-rkxvr6 netcdf-fortran cmake
 
 # create packages directory
 mkdir packages
@@ -68,4 +67,4 @@ make -j 8
 cd ../..
 ./rbc.sh install-makedepf90
 
-echo "Done installing RBC3D!"
+echo "Done installing RBC3D!"

install/install.sh

@@ -11,7 +11,6 @@ tar -xf petsc-3.19.tar.gz
 
 cd petsc-3.19.6
 
-# if these configure options don't work, it's probably a path issue
 ./configure --with-cc=mpicc \
     --with-cxx=mpicxx \
     --with-fc=mpif90 \
@@ -39,4 +38,4 @@ git clone https://github.com/comp-physics/spherepack3.2.git
 cd spherepack3.2
 make -j 8
 
-echo "Done installing RBC3D!"
+echo "Done installing RBC3D!"

install/readme.md

@@ -21,7 +21,7 @@ You will need `gcc`, `gfortran`, and a suitable MPI wrapper like `mvapich` (or t
 * To check for gfortran and gcc, see if `which gfortran` and `which gcc` return a path
 * Similarly, to see if you can run MPI commands for later, see if `which mpicc` or `which mpif90` return a path
 
-You will also need python3 for the PETSc install and pip modules. On Phoenix, you can module load it via `ml python/3.9.12-rkxvr6`, and ICE has a similar module. You may also need to add the `~/.local/bin` directory to your PATH by adding this line to your `~/.bashrc`: 
+You will also need python3 for the PETSc install and pip modules. On Phoenix, you can module load it via `module load python/3.9.12-rkxvr6`, and ICE has a similar module. You may also need to add the `~/.local/bin` directory to your PATH by adding this line to your `~/.bashrc`: 
 
 ```shell
 export PATH="$PATH:$HOME/.local/bin"
@@ -118,6 +118,7 @@ make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt check
 * Later you will need information about where `netcdf-fortran` is installed for `Makefile.in`. 
 * Get this via `module show netcdf-fortran` and looking at the `NETCDF_FORTRANROOT`
 * In my case, this is `/usr/local/pace-apps/spack/packages/linux-rhel7-x86_64/gcc-10.3.0/netcdf-fortran-4.5.4-yx5osuxluenmuvr3xnahmosfr3abeu2p/`
+* An example of how to install `netcdf-fortran` manually is available in `install/install-ice`.
 
 ### Spherepack
 

rbc.sh

@@ -35,6 +35,10 @@ if [ "$1" == 'install-makedepf90' ]; then
     . "$(pwd)/install/install-makedepf90.sh" $@; exit
 fi
 
+if [ "$1" == 'install-ice' ]; then
+    . "$(pwd)/install/install-ice.sh" $@; exit
+fi
+
 if [ "$1" == 'cmake' ]; then
     . "$(pwd)/install/cmake.sh" $@; exit
 fi