Mac build (#46)

comp-physics · Jul 18, 2024 · 9f04c0f · 9f04c0f
1 parent 6ca0575
commit 9f04c0f
Show file tree

Hide file tree

Showing 26 changed files with 763 additions and 83 deletions.
diff --git a/.github/scripts/_typos.toml b/.github/scripts/_typos.toml
@@ -16,4 +16,5 @@ thi = "thi"
 doub = "doub"
 Doub = "Doub"
 Numer = "Numer"
-thr = "thr"
+thr = "thr"
+nd = "nd"
diff --git a/.github/workflows/ice.yml b/.github/workflows/ice.yml
@@ -27,7 +27,7 @@ jobs:
         run: |
             module load gcc/12.3.0 mvapich2/2.3.7-1 netcdf-c hdf5/1.14.1-2-mva2 intel-oneapi-mkl/2023.1.0 python/3.10.10 fftw/3.3.10-mva2 cmake
             set -e -x
-            export PETSC_DIR=`pwd`/packages/petsc-3.19.6
+            export PETSC_DIR=`pwd`/packages/petsc-3.21.3
             export PETSC_ARCH=arch-linux-c-opt
             mkdir build
             cd build

diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml
@@ -0,0 +1,34 @@
+name: 'Test on MacOS'
+
+on:
+    push:
+    pull_request:
+    workflow_dispatch:
+
+jobs:
+  self:
+    name: Mac Runner
+    runs-on: macos-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Build Packages
+        run: |
+            set -x
+            brew install gcc mpich gfortran pkg-config
+            ln -s /opt/homebrew/bin/gfortran-14 /usr/local/bin/gfortran
+            ./rbc.sh install-mac
+
+      - name: Compile Cases
+        run: |
+            set -e -x
+            export PETSC_DIR=`pwd`/packages/petsc-3.21.3
+            export PETSC_ARCH=arch-darwin-c-opt
+            mkdir build
+            cd build
+            cmake ..
+            make
+            echo "/common and all cases in /examples compiled successfully!"
+
diff --git a/.github/workflows/phoenix.yml b/.github/workflows/phoenix.yml
@@ -29,7 +29,7 @@ jobs:
             module load gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv netcdf-fortran cmake
             set -e -x
             export FFTWROOT=`pwd`/packages/fftw-3.3.10/build
-            export PETSC_DIR=`pwd`/packages/petsc-3.19.6
+            export PETSC_DIR=`pwd`/packages/petsc-3.21.3
             export PETSC_ARCH=arch-linux-c-opt
             cd common
             make .depend
@@ -43,7 +43,7 @@ jobs:
         run: |
             module load gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv netcdf-fortran cmake
             set -e -x
-            export PETSC_DIR=`pwd`/packages/petsc-3.19.6
+            export PETSC_DIR=`pwd`/packages/petsc-3.21.3
             export PETSC_ARCH=arch-linux-c-opt
             mkdir build
             cd build

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -37,6 +37,14 @@ if (NOT DEFINED ENV{PETSC_ARCH})
 else ()
     # full path if PETSC_DIR and PETSC_ARCH env variables are set
     set(PETSC $ENV{PETSC_DIR}/$ENV{PETSC_ARCH})
+    if ("$ENV{PETSC_ARCH}" STREQUAL "arch-darwin-c-opt")
+        set(MAC on)
+    elseif("$ENV{PETSC_ARCH}" STREQUAL "arch-linux-c-opt")
+        set(LINUX on)
+    else ()
+        message(FATAL_ERROR "PETSC_ARCH $ENV{PETSC_ARCH} not recognized. Please add support in CMakeLists.txt")
+    endif()
+
 endif()
 
 set(ENV{PKG_CONFIG_PATH} ${PETSC}/lib/pkgconfig)
@@ -51,7 +59,14 @@ if (FORTRAN_COMPILER)
     set(CMAKE_Fortran_COMPILER ${FORTRAN_COMPILER})
     # same compile options as Makefile.in
     add_compile_options(-fallow-argument-mismatch -freal-4-real-8 -O3)
-    add_link_options(-ldl -lstdc++ -Wl,--copy-dt-needed-entries)
+    add_link_options(-ldl -lstdc++)
+    if (LINUX)
+        # ice cluster install needs this extra link option
+        add_link_options(-Wl,--copy-dt-needed-entries)
+    elseif(MAC)
+        # might be able to remove this if homebrew fixes bug
+        add_link_options(-Wl,-no_warn_duplicate_libraries)
+    endif()
 else ()
     message(FATAL_ERROR "PETSc Fortran compiler not found. Please ensure PETSc was configured with a Fortran compiler.")
 endif (FORTRAN_COMPILER)
@@ -85,11 +100,11 @@ endif(LAPACK_FOUND)
 if (DEFINED ENV{FFTWROOT})
     set(FFTW_INCLUDE "$ENV{FFTWROOT}/include")
     set(FFTW_LIB "-L$ENV{FFTWROOT}/lib -lfftw3")
-elseif(DEFINED ENV{FFTWROOT})
+elseif(DEFINED ENV{FFTW_ROOT})
     set(FFTW_INCLUDE "$ENV{FFTW_ROOT}/include")
     set(FFTW_LIB "-L$ENV{FFTW_ROOT}/lib -lfftw3")
 else ()
-    message(WARNING "FFTWROOT and FFTW_ROOT environment variables from module load fftw not found. Assuming you installed FFTW in packages.")
+    message(STATUS "FFTWROOT and FFTW_ROOT environment variables from module load fftw not found. Assuming you installed FFTW in packages.")
     set(FFTW_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/packages/fftw-3.3.10/build/include")
     set(FFTW_LIB "-L${CMAKE_CURRENT_SOURCE_DIR}/packages/fftw-3.3.10/build/lib -lfftw3")
 endif()
@@ -102,7 +117,7 @@ elseif (DEFINED ENV{NETCDF_FORTRAN_ROOT})
     set(NETCDF_INCLUDE "$ENV{NETCDF_FORTRAN_ROOT}/include")
     set(NETCDF_LIB "-L$ENV{NETCDF_FORTRAN_ROOT}/lib -lnetcdff")
 else ()
-    message(WARNING "NETCDF_FORTRANROOT and NETCDF_FORTRAN_ROOT environment variables from module load fftw not found. Assuming you installed NETCDF-FORTRAN in packages.")
+    message(STATUS "NETCDF_FORTRANROOT and NETCDF_FORTRAN_ROOT environment variables from module load fftw not found. Assuming you installed NETCDF-FORTRAN in packages.")
     set(NETCDF_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/packages/NETCDF_INST/include")
     # -rpath linker option to avoid having to add netcdf library to LD_LIBRARY_PATH
     set(NETCDF_LIB "-Wl,-rpath -Wl,${CMAKE_CURRENT_SOURCE_DIR}/packages/NETCDF_INST/lib -L${CMAKE_CURRENT_SOURCE_DIR}/packages/NETCDF_INST/lib -lnetcdff")

diff --git a/Makefile.in b/Makefile.in
@@ -2,15 +2,18 @@
 WORK_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 
 # Package directories (shouldn't need to change these)
-PETSC_DIR = $(WORK_DIR)/packages/petsc-3.19.6
+PETSC_DIR = $(WORK_DIR)/packages/petsc-3.21.3
 PETSC_ARCH = $(PETSC_DIR)/arch-linux-c-opt
 
 SPHEREPACK_DIR = $(WORK_DIR)/packages/spherepack3.2
 
 LAPACK_DIR = $(WORK_DIR)/packages/lapack-3.11
-# if netcdf-fortran is manually installed then this will need to be set instead
+# if netcdf-fortran is manually installed, uncomment line below and comment the other one
 # NETCDF_DIR = $(WORK_DIR)/packages/NETCDF_INST
 NETCDF_DIR = $(NETCDF_FORTRANROOT)
+# if fftw is manually installed, use this instead
+# FFTW_DIR = $(WORK_DIR)/packages/fftw-3.3.10/build
+FFTW_DIR = $(FFTWROOT)
 
 # Makedependf90 binary
 MAKEDEPEND_BIN = $(WORK_DIR)/packages/makedepf90/makedepf90
@@ -24,16 +27,16 @@ NETCDF_INCLUDE = $(NETCDF_DIR)/include
 # If you installed FFTW in packages through install/install-phoenix.sh
 # run: export FFTWROOT=`pwd`/packages/fftw-3.3.10/build from RBC3D root directory in terminal
 # or make a variable in this file
-FFTW_INCLUDE = $(FFTWROOT)/include
+FFTW_INCLUDE = $(FFTW_DIR)/include
 
 COMMON_INCLUDE = -I$(WORK_DIR)/common
-# you can remove -I$(FFTW_INCLUDE) if you module loaded fftw
+# you can remove -I$(FFTW_DIR) if you module loaded fftw
 INCLUDE = $(COMMON_INCLUDE) -I$(PETSC_INCLUDE) -I$(PETSC_ARCH_INCLUDE) -I$(NETCDF_INCLUDE) -I$(FFTW_INCLUDE)
 
 # Libraries
 COMMON_LIB = $(WORK_DIR)/common/libcommon.a
 SPHPK_LIB = -L$(SPHEREPACK_DIR)/lib -lspherepack
-FFTW_LIB = -L$(FFTWROOT)/lib -lfftw3
+FFTW_LIB = -L$(FFTW_DIR)/lib -lfftw3
 NETCDF_LIB = -L$(NETCDF_DIR)/lib -lnetcdff
 PETSC_LIB = -Wl,-rpath,$(PETSC_ARCH)/lib -L$(PETSC_ARCH)/lib -lpetsc -lstdc++
 MKL_LIB = -L$(MKL_ROOT)lib/intel64/ -lmkl_gf_lp64 -lmkl_core -lmkl_sequential -lpthread -lm -ldl

diff --git a/README.md b/README.md
@@ -23,55 +23,41 @@ This codebase solves the boundary integral form of the Stokes equations via an a
 
 ### Installation
 
-<p align="left">
-  <a href="https://github.com/comp-physics/RBC3D/actions/workflows/phoenix.yml">
-    <img src="https://github.com/comp-physics/RBC3D/actions/workflows/phoenix.yml/badge.svg" />
-  </a>
-  <a href="https://github.com/comp-physics/RBC3D/actions/workflows/ice.yml">
-    <img src="https://github.com/comp-physics/RBC3D/actions/workflows/ice.yml/badge.svg" />
-  </a>
-</p>
-
-To install on PACE Phoenix, you need to salloc a node to make sure `srun` is available and then run this in the RBC3D root directory: 
-
-```shell
-module load gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv python/3.9.12-rkxvr6 netcdf-fortran cmake
-./rbc.sh install-phoenix
-```
+To install on a mac from the cloned repository, you can 
 
-Note that if the `gcc`, `mvapich2`, `mkl`, and `fftw` modules work on your Phoenix account, you should use this installer script for a faster build.
 ```shell
-module load gcc mvapich2 mkl python/3.9.12-rkxvr6 netcdf-fortran fftw cmake
-./rbc.sh install
+brew install gcc mpich gfortran pkg-config wget cmake
+./rbc.sh install-mac
 ```
 
-Or if you're on the ICE cluster, you just need to load different modules to run the installer script.
+and then set these environment variables in your `~/.zshrc` or `~/.bashrc`. Note that `$HOME` will need to be replaced with the folder you cloned RBC3D in.
 
-```shell
-module load gcc/12.3.0 mvapich2/2.3.7-1 netcdf-c hdf5/1.14.1-2-mva2 intel-oneapi-mkl/2023.1.0 python/3.10.10 fftw/3.3.10-mva2 cmake
-./rbc.sh install-ice
-```
-
-Before you can run cmake, you must set these environment variables. You can place them in your `~/.bashrc`. If you didn't place `RBC3D` in your `$HOME` directory, then replace it with where you placed `RBC3D`.
 ```shell
 export PETSC_DIR=$HOME/RBC3D/packages/petsc-3.19.6
-export PETSC_ARCH=arch-linux-c-opt
+export PETSC_ARCH=arch-darwin-c-opt
 ```
 
 Then to execute and run a case, you can:
 ```shell
 mkdir build
 cd build
 cmake ..
-make case # or just `make` to make common and all the cases
-cd case
-srun -n 1 ./initcond # or mpiexec
-srun ./tube
+make -j 8 minicase # or just `make` to make common and all the cases
+cd minicase
+mpiexec -n 1 ./minit
+mpiexec -n 2 ./mtube # number of nodes can be changed
+```
+
+This will generate output files in `build/minicase/D`. To keep output files in `examples/minicase/D` and use input files in `examples/minicase/Input`, you can do this instead once files are built in the `build` directory:
+
+```shell
+cd examples/case
+mpiexec -n 1 ../../build/case/minit
+mpiexec -n 2 ../../build/case/mtube
 ```
 
-This will generate output files in `build/case/D`. To keep output files in `examples/case/D`, you can `cd examples/case` and `srun ../../build/case/initcond` and same for tube.
+To run a case with more cells and nodes, you should use a supercomputing cluster. Instructions on how to build RBC3D on a cluster are [available here](https://github.com/comp-physics/RBC3D/blob/master/install/readme.md).
 
-On other supercomputing clusters, it should be easy to replace the module loads with the modules available on your system. If one of these isn't available, you can follow the manual build instructions [available here](https://github.com/comp-physics/RBC3D/blob/master/install/readme.md).
 
 ### Papers that use RBC3D
 

diff --git a/common/ModConf.F90 b/common/ModConf.F90
@@ -139,8 +139,8 @@ subroutine InitMPI(split_comm)
         print *, 'Node ', i, ' of ', numNodes, ' running on ', trim(machinename)
       end do ! i
     else
-      call MPI_Send(lenname, 1, MPI_Integer, 0, 1, MPI_Comm_World, stat, ierr)
-      call MPI_Send(machinename, lenname, MPI_Character, 0, 1, MPI_Comm_World, stat, ierr)
+      call MPI_Send(lenname, 1, MPI_Integer, 0, 1, MPI_Comm_World, ierr)
+      call MPI_Send(machinename, lenname, MPI_Character, 0, 1, MPI_Comm_World, ierr)
     end if
     call MPI_Barrier(MPI_COMM_WORLD, ierr)
 

diff --git a/common/ModIO.F90 b/common/ModIO.F90
@@ -45,17 +45,17 @@ subroutine IO_Init
 
     if (rootWorld) then
       if (pgrad_out > 0) then
-        write (fn, FMT=fn_FMT), 'D/', 'pgrad', Nt0, '.dat'
+        write (fn, FMT=fn_FMT) 'D/', 'pgrad', Nt0, '.dat'
         open (pGrad_unit, file=trim(fn), action='write')
       end if
 
       if (flow_out > 0) then
-        write (fn, FMT=fn_FMT), 'D/', 'flow', Nt0, '.dat'
+        write (fn, FMT=fn_FMT) 'D/', 'flow', Nt0, '.dat'
         open (flow_unit, file=trim(fn), action='write')
       end if
 
       if (ftot_out > 0) then
-        write (fn, FMT=fn_FMT), 'D/', 'ftot', Nt0, '.dat'
+        write (fn, FMT=fn_FMT) 'D/', 'ftot', Nt0, '.dat'
         open (ftot_unit, file=trim(fn), action='write')
       end if
 
@@ -553,6 +553,8 @@ subroutine ReadWallMesh(fn, wall)
     integer, allocatable :: connect(:, :)
     character(*), parameter :: func_name = "ReadWallMesh"
 
+    print *, "fn: ", trim(fn)
+
     ! Check whether the file exists
     ierr = NF90_OPEN(trim(fn), NF90_NOWRITE, ncid)
     if (ierr .ne. 0) then
@@ -824,22 +826,21 @@ subroutine ReadRestart(fn)
       write (*, *) 'vBkg = ', vBkg
     end if
 
-    print *, '1'
     call MPI_Bcast(Lb, 3, MPI_WP, 0, MPI_Comm_World, ierr)
     call MPI_Bcast(iLb, 3, MPI_WP, 0, MPI_Comm_World, ierr)
 
     call MPI_Bcast(Nt0, 1, MPI_INTEGER, 0, MPI_Comm_World, ierr)
     call MPI_Bcast(time0, 1, MPI_WP, 0, MPI_Comm_World, ierr)
     call MPI_Bcast(vBkg, 3, MPI_WP, 0, MPI_Comm_World, ierr)
-    print *, '2'
+
     ! cells
     if (rootWorld) then
       read (restart_unit) nrbc
 
       write (*, *) 'nrbc = ', nrbc
     end if
     call MPI_Bcast(nrbc, 1, MPI_Integer, 0, MPI_Comm_World, ierr)
-    print *, '3'
+
     allocate (rbcs(nrbc))
 
     do irbc = 1, nrbc
@@ -848,7 +849,6 @@ subroutine ReadRestart(fn)
       if (rootWorld) then
         read (restart_unit) nlat0, nlon0
         read (restart_unit) nlat, nlon
-        ! celltype = 1; print *,"NO READ CELL TYPE"
         read (restart_unit) celltype
         read (restart_unit) starting_area
         write (*, *) 'irbc : ', irbc, ' nlat0 = ', nlat0, 'type = ', celltype, 'starting_area = ', starting_area
@@ -888,7 +888,7 @@ subroutine ReadRestart(fn)
       end if
       call MPI_Bcast(rbc%x, size(rbc%x), MPI_WP, 0, MPI_Comm_World, ierr)
     end do ! irbc
-    print *, '4'
+
     ! Walls
     if (rootWorld) then
       read (restart_unit) nwall

diff --git a/common/ModRbc.F90 b/common/ModRbc.F90
@@ -229,14 +229,14 @@ subroutine RBC_MakeSickle(cell, rad, xc)
     type(t_RBC) :: cell
     real(WP), optional :: xc(3)
 
-    integer :: ilat, ilon, ii
+    integer :: ilat, ilon, ii, ierr
     real(WP) :: th, phi, r_u, r_l, r, p, rad
 
     real(WP), dimension(0:5) :: a_l, a_u
     real(WP), dimension(2) :: b
 
     print *, "The RBC_MakeSickle Cell Configuration is Not Confirmed Working Yet - Aborting"
-    call MPI_Abort(MPI_COMM_WORLD, 1, 1)
+    call MPI_Abort(MPI_COMM_WORLD, 1, ierr)
 
     a_u = (/1.36, -0.0403, 0.306, -0.00169, -0.0360, -0.0277/)
     a_l = (/-0.806, -0.1141, -0.00678, 0.00212, 0.0201, 0.0284/)

diff --git a/common/ModTimeInt.F90 b/common/ModTimeInt.F90
@@ -64,7 +64,7 @@ subroutine TimeInt_Init
       ! call RBC_ComputeGeometry(rbcRef)  JBF:  not needed???
       rbcRefs(1)%patch => rbcPatch
       rbcRefs(2)%patch => rbcPatch
-      ! rbcRefs(3)%patch => rbcPatch
+      rbcRefs(3)%patch => rbcPatch
 
       if (PhysEwald) then
       do irbc = 1, nrbc
@@ -122,11 +122,9 @@ subroutine TimeInt_Euler
       clockBgn = MPI_WTime() ! Start timing
 
       ! Evolve cells
-      ! print *,"NO VEL"
       call Compute_Rbc_Vel
 
       ! Enforce no-slip condition on the wall
-      ! print *,"NO NO SLIP"
       call NoSlipWall
 
       ! Evolve RBC
@@ -140,8 +138,6 @@ subroutine TimeInt_Euler
       ! call FilterRbcs
       call ReboxRbcs
 
-      ! print *,"MULTIVOL"
-
       ! call AddR0Motion
 
       ! call LeukWallRepulsion
@@ -954,7 +950,7 @@ subroutine VolConstrainRbcs
       fac = (rbcRefs(rbc%celltype)%vol - rbc%vol)/rbc%area
       fac = SIGN(MIN(ABS(fac), epsDist/20.), fac)
       if (rootWorld .and. ABS(fac) .gt. epsDist/40.) then
-        print *, "VOL: ", rbcRefs(rbc%celltype)%vol, rbc%vol, fac
+        print *, "VOL: ", rbc%celltype, rbcRefs(rbc%celltype)%vol, rbc%vol, fac
       end if
       do ilat = 1, rbc%nlat
         do ilon = 1, rbc%nlon

diff --git a/examples/case/Input/tube.in b/examples/case/Input/tube.in
@@ -14,7 +14,7 @@
 10000                   ! Nt
 0.0014                  ! Ts
 
-100                      ! cell_out
+50                       ! cell_out
 -10000                   ! wall_out
 -10                      ! pGrad_out
 -10                      ! flow_out

diff --git a/examples/minicase/D/.gittouch b/examples/minicase/D/.gittouch
diff --git a/examples/minicase/Input/new_cyl_D6_L13_33.e b/examples/minicase/Input/new_cyl_D6_L13_33.e