Merge pull request #39 from suzanmanasreh/cmake

Cmake

.github/scripts/submit.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+#SBATCH --account=gts-sbryngelson3
+#SBATCH -N4 --ntasks-per-node=24
+#SBATCH --mem-per-cpu=2G
+#SBATCH -t1:00:00
+#SBATCH -q embers
+#SBATCH [email protected]
+#SBATCH --mail-type=BEGIN,END,FAIL
+
+cd $SLURM_SUBMIT_DIR
+echo "Running in $(pwd):"
+
+ml gcc mvapich2 mkl petsc netcdf-c netcdf-cxx netcdf-fortran fftw
+
+mkdir packages
+cd packages
+
+# build and install spherepack
+cd ..
+git clone https://github.com/comp-physics/spherepack3.2.git
+cd spherepack3.2
+make
+
+# build and install makedepf90
+cd ..
+git clone https://github.com/comp-physics/makedepf90.git
+cd makedepf90
+# it works without setting prefix
+make
+make install
+
+cd ../../common
+make .depend
+make
+
+cd ..examples/case
+make .depend
+make
+
+srun -n 1 ./initcond
+srun ./tube

.github/workflows/ice.yml

@@ -17,32 +17,23 @@ jobs:
         uses: actions/checkout@v4
 
       # use mkl option
-      - name: Build
+      - name: Build Packages
         run: |
             ml gcc/12.3.0 mvapich2/2.3.7-1 intel-oneapi-mkl/2023.1.0 python/3.10.10 netcdf-c/4.9.2-mva2-hdf5-1.14 netcdf-cxx/4.2-mva2-hdf5-1.14 netcdf-fortran/4.6.0-mva2-hdf5-1.14 fftw/3.3.10-mva
-            bash rbc.sh install-with-mkl
+            bash rbc.sh install
 
       # will stop on any errors
-      - name: Compile Common
+      - name: Compile Cases
         run: |
             ml gcc/12.3.0 mvapich2/2.3.7-1 intel-oneapi-mkl/2023.1.0 python/3.10.10 netcdf-c/4.9.2-mva2-hdf5-1.14 netcdf-cxx/4.2-mva2-hdf5-1.14 netcdf-fortran/4.6.0-mva2-hdf5-1.14 fftw/3.3.10-mva
             set -e -x
-            cd common
-            make .depend
+            export PETSC_DIR=`pwd`/RBC3D/packages/petsc-3.19.6
+            export PETSC_ARCH=arch-linux-c-opt
+            echo $PETSC_DIR
+            echo $PETSC_ARCH
+            mkdir build
+            cd build
+            cmake ..
             make
-            echo "/common compiled successfully!"
+            echo "/common and all cases in /examples compiled successfully!"
 
-      - name: Compile Example Cases
-        run: |
-            ml gcc/12.3.0 mvapich2/2.3.7-1 intel-oneapi-mkl/2023.1.0 python/3.10.10 netcdf-c/4.9.2-mva2-hdf5-1.14 netcdf-cxx/4.2-mva2-hdf5-1.14 netcdf-fortran/4.6.0-mva2-hdf5-1.14 fftw/3.3.10-mva
-            set -e -x
-            base_dir="examples"
-
-            cases=$(find "$base_dir" -maxdepth 1 -type d ! -path "$base_dir")
-
-            for case in $cases; do
-                make -C "$case" .depend
-                make -C "$case"
-            done
-            
-            echo "All cases in /examples compiled successfully!"

.gitignore

@@ -14,3 +14,12 @@ libcommon.a
 field
 packages
 traction
+test
+tes17
+pinv
+qf1d
+packages*
+testcases
+mycases
+sample_files/sample_walls/my_meshes
+build

CMakeLists.txt

@@ -0,0 +1,181 @@
+# To generate build files for common and all example cases:
+#   mkdir build
+#   cd build
+#   cmake ..
+#   make
+# Regenerate case files after changes:
+#   cmake .
+#   make case
+# Show each build command line as it is launched
+#   cmake -DCMAKE_VERBOSE_MAKEFILE=ON ..
+# Thank you Boost, Sundials
+
+include(CMakePrintHelpers)
+cmake_minimum_required(VERSION 3.20)
+
+project(RBC3D LANGUAGES Fortran C CXX)
+
+set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+# use mkl implementation of lapack
+set(BLA_VENDOR Intel10_64lp)
+
+if (NOT DEFINED ENV{PETSC_DIR} OR NOT DEFINED ENV{PETSC_ARCH})
+    message(FATAL_ERROR "PETSc environment variables are not set. Please export PETSC_DIR and PETSC_ARCH.")
+endif()
+
+# PETSC_DIR and PETSC_ARCH env variable must be set
+set(PETSC $ENV{PETSC_DIR}/$ENV{PETSC_ARCH})
+set(ENV{PKG_CONFIG_PATH} ${PETSC}/lib/pkgconfig)
+
+cmake_print_variables(PETSC)
+
+# use the same fortran compiler petsc was configured with
+execute_process (COMMAND pkg-config PETSc --variable=fcompiler COMMAND tr -d '\n' OUTPUT_VARIABLE FORTRAN_COMPILER)
+cmake_print_variables(FORTRAN_COMPILER)
+
+if (FORTRAN_COMPILER)
+    message(${FORTRAN_COMPILER})
+    set(CMAKE_Fortran_COMPILER ${FORTRAN_COMPILER})
+    add_compile_options(-fallow-argument-mismatch -freal-4-real-8 -O3)
+    add_link_options(-ldl -lstdc++ -Wl,--copy-dt-needed-entries)
+else ()
+    message(FATAL_ERROR "PETSc Fortran compiler not found. Please export PETSC_DIR and PETSC_ARCH.")
+endif (FORTRAN_COMPILER)
+
+find_package(PkgConfig REQUIRED)
+# find_package(LAPACK REQUIRED)
+
+if (PKG_CONFIG_FOUND)
+    pkg_check_modules(PETSC PETSc)
+else()
+    message(FATAL_ERROR "PkgConfig was not found.")
+endif()
+
+if (NOT PETSC_FOUND)
+    message(FATAL_ERROR "PETSc was not found through PkgConfig.")
+else()
+    message("PETSc was not found")
+    cmake_print_variables(PETSC_INCLUDE_DIRS)
+    cmake_print_variables(PETSC_LINK_LIBRARIES)
+endif()
+
+if (LAPACK_FOUND)
+  message(STATUS "LAPACK libraries: ${LAPACK_LIBRARIES}")
+endif(LAPACK_FOUND)
+
+# look for FFTWROOT or FFTW_ROOT
+if (DEFINED ENV{FFTWROOT})
+    # set(FFTW_INCLUDE "${FFTW_INCLUDE};$ENV{FFTWROOT}/include")
+    set(FFTW_LIB "-L$ENV{FFTWROOT}/lib -lfftw3")
+    cmake_print_variables(FFTW_LIB)
+elseif(DEFINED ENV{FFTWROOT})
+    set(FFTW_LIB "-L$ENV{FFTW_ROOT}/lib -lfftw3")
+    cmake_print_variables(FFTW_LIB)
+else ()
+    message(FATAL_ERROR "FFTWROOT and FFTW_ROOT environment variables not found. Please module load fftw.")
+endif()
+
+# look for NETCDF_FORTRANROOT or NETCDF_FORTRAN_ROOT 
+if (DEFINED ENV{NETCDF_FORTRANROOT})
+    set(NETCDF_INCLUDE "$ENV{NETCDF_FORTRANROOT}/include")
+    set(NETCDF_LIB "-L$ENV{NETCDF_FORTRANROOT}/lib -lnetcdff")
+    cmake_print_variables(NETCDF_INCLUDE)
+    cmake_print_variables(NETCDF_LIB)
+elseif (DEFINED ENV{NETCDF_FORTRAN_ROOT})
+    set(NETCDF_INCLUDE "$ENV{NETCDF_FORTRAN_ROOT}/include")
+    set(NETCDF_LIB "-L$ENV{NETCDF_FORTRAN_ROOT}/lib -lnetcdff")
+else ()
+    message(FATAL_ERROR "NETCDF_FORTRANROOT or NETCDF_FORTRAN_ROOT environment variables not found. Please module load netcdf-fortran.")
+endif()
+
+# spherepack library
+set(SPHPK_LIB "-L${CMAKE_CURRENT_SOURCE_DIR}/packages/spherepack3.2/lib -lspherepack")
+
+# build common
+add_subdirectory(common)
+install(TARGETS common LIBRARY DESTINATION "bin")
+
+include_directories("${CMAKE_CURRENT_BINARY_DIR}/common")
+
+set(examples_dir "examples")
+
+# glob for the directories in /examples
+file(GLOB examples_list RELATIVE ${CMAKE_SOURCE_DIR}/${examples_dir} ${CMAKE_SOURCE_DIR}/${examples_dir}/*)
+set(dirlist "")
+foreach (example ${examples_list})
+    if (IS_DIRECTORY ${CMAKE_SOURCE_DIR}/${examples_dir}/${example})
+        list(APPEND dirlist ${example})
+    endif()
+endforeach()
+
+cmake_print_variables(dirlist)
+
+set(ALLOW_DUPLICATE_CUSTOM_TARGETS TRUE)
+
+foreach (casedir ${dirlist})
+    cmake_print_variables(${casedir})
+
+    # make binary dir inside /build for case files
+    set(case_binary_dir "${CMAKE_CURRENT_BINARY_DIR}/${casedir}")
+    file(MAKE_DIRECTORY ${case_binary_dir})
+    make_directory("${case_binary_dir}/D")
+    file(GLOB inputs "${CMAKE_CURRENT_SOURCE_DIR}/${examples_dir}/${casedir}/Input/*")
+    cmake_print_variables(inputs)
+    file(COPY ${inputs} DESTINATION "${case_binary_dir}/Input")
+    cmake_print_variables(case_binary_dir)
+
+    # find all F90 files in the examples/case directory
+    file(GLOB case_F90s "${CMAKE_CURRENT_SOURCE_DIR}/${examples_dir}/${casedir}/*.F90")
+    cmake_print_variables(case_F90s)
+
+    # create a custom target with the name of the current case directory
+    add_custom_target(${casedir})
+
+    # loop through each F90 file
+    foreach(F90_filepath ${case_F90s})
+        # get the file name without extension
+        get_filename_component(file_target ${F90_filepath} NAME_WE)
+        add_executable(${file_target} ${F90_filepath})
+
+        # add target to its own case build directory
+        set_target_properties(${file_target} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${case_binary_dir})
+
+        # set include directories and link libraries
+        # petsc might not be necessary for all files (i.e. initcond) but it doesn't hurt to add it
+        target_include_directories(${file_target} PRIVATE ${PETSC_INCLUDE_DIRS})
+        target_link_libraries(${file_target} PRIVATE common ${PETSC_LINK_LIBRARIES})
+
+        # add dependency to the custom target
+        add_dependencies(${casedir} ${file_target})
+    endforeach()
+endforeach()
+
+# # make binary dir inside /build for case files
+# set(CASE_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/case")
+# file(MAKE_DIRECTORY ${CASE_BINARY_DIR})
+
+# add_subdirectory(examples/case case)
+# add_subdirectory(examples/case_sickles case_sickles)
+# # Find all Fortran files in the examples/case directory
+# file(GLOB case_F90s "${CMAKE_CURRENT_SOURCE_DIR}/examples/case/*.F90")
+# cmake_print_variables(case_F90s)
+
+# # Create a custom target with the name of the current directory
+# add_custom_target(case)
+
+# # Loop through each Fortran file
+# foreach(F90_filepath ${case_F90s})
+#     # Get the file name without extension
+#     get_filename_component(file_target ${F90_filepath} NAME_WE)
+#     add_executable(${file_target} ${F90_filepath})
+#     cmake_print_variables(file_target)
+
+#     set_target_properties(${file_target} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CASE_BINARY_DIR})
+
+#     # Set include directories and link libraries
+#     target_include_directories(${file_target} PRIVATE ${PETSC_INCLUDE_DIRS})
+#     target_link_libraries(${file_target} PRIVATE common ${PETSC_LINK_LIBRARIES})
+
+#     # Add dependency to the custom target
+#     add_dependencies(case ${file_target})
+# endforeach()

README.md

@@ -28,30 +28,34 @@ This codebase solves the boundary integral form of the Stokes equations via an a
 To install on PACE Phoenix, you need to salloc a node to make sure srun is available and then run this in the RBC3D root directory: 
 
 ```shell
-ml gcc mvapich2 mkl python/3.9.12-rkxvr6 netcdf-c netcdf-cxx netcdf-fortran fftw
-bash rbc.sh install-with-mkl
+ml gcc mvapich2 mkl python/3.9.12-rkxvr6 netcdf-fortran fftw cmake
+bash rbc.sh install
 ```
 
 Or if you're on COC-ICE, you just need to load different modules to run the installer script.
 
 ```shell
-ml gcc/12.3.0 mvapich2/2.3.7-1 intel-oneapi-mkl/2023.1.0 python/3.10.10 netcdf-c/4.9.2-mva2-hdf5-1.14 netcdf-cxx/4.2-mva2-hdf5-1.14 netcdf-fortran/4.6.0-mva2-hdf5-1.14 fftw/3.3.10-mva2
-bash rbc.sh install-with-mkl
+ml gcc/12.3.0 mvapich2/2.3.7-1 intel-oneapi-mkl/2023.1.0 python/3.10.10 netcdf-fortran/4.6.0-mva2-hdf5-1.14 fftw/3.3.10-mva2
+bash rbc.sh install
+```
+
+Before you can run cmake, you must add these to your `~/.bashrc`. If you didn't place `RBC3D` in your `$HOME` directory. Then replace, it with where you placed `RBC3D`.
+```shell
+export PETSC_DIR=$HOME/RBC3D/packages/petsc-3.19.6
+export PETSC_ARCH=arch-linux-c-opt
 ```
 
 Then to execute and run a case, you can:
 ```shell
-cd common
-make .depend
-make
-cd ..examples/case
-make .depend
-make
+mkdir build
+cd build
+cmake ..
+make case
 srun -n 1 ./initcond
 srun ./tube
 ```
 
-On other supercomputing clusters, it should be easy to replace the module loads with the modules available on your system and change the directories in `Makefile.in` to point to those modules. If one of these isn't available, you can follow the manual build instructions [available here](https://github.com/comp-physics/RBC3D/blob/master/install/readme.md).
+On other supercomputing clusters, it should be easy to replace the module loads with the modules available on your system and make sure modules are setting appropriate environment variables via the `module show` command. If one of these isn't available, you can follow the manual build instructions [available here](https://github.com/comp-physics/RBC3D/blob/master/install/readme.md).
 
 ### Papers that use RBC3D
 

commands.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+rm -fr build
+mkdir build
+cd build
+# cmake .. -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
+cmake ..

common/CMakeLists.txt

@@ -0,0 +1,6 @@
+file(GLOB common_F90s "*.F90")
+add_library(common STATIC "${common_F90s}")
+
+target_include_directories(common PRIVATE ${PETSC_INCLUDE_DIRS} ${NETCDF_INCLUDE})
+target_link_libraries(common ${PETSC_LINK_LIBRARIES} ${SPHPK_LIB} ${FFTW_LIB} ${NETCDF_LIB})
+

examples/carotid_web/Makefile

@@ -1 +1,22 @@
-include ../Makefile.mkl
+include ../../Makefile.in
+
+LIB = $(COMMON_LIB) $(SPHPK_LIB) $(FFTW_LIB) $(PETSC_LIB) $(NETCDF_LIB) $(MKL_LIB) $(STATIC)
+EXECUTABLES = carotid_tube carotid_initcond
+
+all : $(EXECUTABLES)
+
+lib : $(wildcard $(WORK_DIR)/common/*.F90)
+	make -C $(WORK_DIR)/common
+
+$(EXECUTABLES) : % : %.o $(wildcard $(COMMON_DIR)/*.F90)
+	make lib
+	$(FC) $(LDFLAGS) -o $@ $< $(LIB)
+
+clean :
+	-rm -f $(EXECUTABLES) *.o *.mod *.a core
+
+# Dependency
+.depend : $(wildcard *.F90)
+	$(MAKEDEPEND_BIN) $^ > .depend
+
+include .depend

examples/case/Input/tube.in

@@ -11,7 +11,7 @@
 0.0     !
 0.0     !
 
-10000                   ! Nt
+1000                    ! Nt
 0.0014                  ! Ts
 
 100                      ! cell_out