Skip to content

Commit

Permalink
Add self-build PETSc, improve CMake find using logic from preCICE
Browse files Browse the repository at this point in the history
  • Loading branch information
@kopperp
kopperp committed Jul 12, 2024
1 parent 23b2eb3 commit c73c5b1
Show file tree
Hide file tree
Showing 9 changed files with 205 additions and 81 deletions.
26 changes: 13 additions & 13 deletions .gitlab-ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@
# - if: '$CI_PIPELINE_CREATED_AT =~ /^\d\d\d\d-\d\d-\d[1,3,5,7,9]T.*/ || $CI_COMMIT_REF_NAME !~ /^gitlab.custom.*/'
# variables:
# PIPELINE_NAME: '🔵 MPICH: on ${CI_PIPELINE_CREATED_AT} 123 ${CI_COMMIT_REF_NAME} 123'
#
#
# - if: '$CI_PIPELINE_CREATED_AT =~ /^\d\d\d\d-\d\d-\d[0,2,4,6,8]T.*/|| $CI_COMMIT_REF_NAME =~ /^gitlab.custom.*/'
# variables:
# PIPELINE_NAME: '🟠 OpenMPI: $(date +%d) ${CI_COMMIT_REF_NAME}'
#
#
# - when: never

# ----------------------------------------------------------------------------------------------------------------------------------------------------
Expand Down Expand Up @@ -84,24 +84,24 @@ before_script:
- echo "Perform arithmetic with 0-prefixed numbers by telling bash to use base-10 by specifying 10#"
- if [ "$((10#$day % 2))" == "0" ]; then
echo "Even day number ($day). Loading OpenMPI";
module purge;
module load cmake/3.26.4 gcc/13.2.0 openmpi/4.1.5/gcc/13.2.0 hdf5/1.14.0/gcc/13.2.0/openmpi/4.1.5 hopr/master/gcc/13.2.0/openmpi/4.1.5/hdf5/1.14.0 petsc/3.19.3/gcc/13.2.0/openmpi/4.1.5;
module purge || true;
module load cmake/3.26.4 gcc/13.2.0 openmpi/4.1.5/gcc/13.2.0 hdf5/1.14.0/gcc/13.2.0/openmpi/4.1.5 hopr/master/gcc/13.2.0/openmpi/4.1.5/hdf5/1.14.0 petsc/3.19.3/gcc/13.2.0/openmpi/4.1.5 || true;
else
echo "Odd day number ($day). Loading MPICH";
module purge;
module load cmake/3.26.4 gcc/13.2.0 mpich/4.1.2/gcc/13.2.0 hdf5/1.14.0/gcc/13.2.0/mpich/4.1.2 hopr/master/gcc/13.2.0/mpich/4.1.2/hdf5/1.14.0 petsc/3.19.3/gcc/13.2.0/mpich/4.1.2;
module purge || true;
module load cmake/3.26.4 gcc/13.2.0 mpich/4.1.2/gcc/13.2.0 hdf5/1.14.0/gcc/13.2.0/mpich/4.1.2 hopr/master/gcc/13.2.0/mpich/4.1.2/hdf5/1.14.0 petsc/3.19.3/gcc/13.2.0/mpich/4.1.2 || true;
fi
- if [ -n "${DO_MPICH}" ]; then
echo "DO_MPICH activated. Loading MPICH.";
module purge;
module load cmake/3.26.4 gcc/13.2.0 mpich/4.1.2/gcc/13.2.0 hdf5/1.14.0/gcc/13.2.0/mpich/4.1.2 hopr/master/gcc/13.2.0/mpich/4.1.2/hdf5/1.14.0 petsc/3.19.3/gcc/13.2.0/mpich/4.1.2;
module purge || true;
module load cmake/3.26.4 gcc/13.2.0 mpich/4.1.2/gcc/13.2.0 hdf5/1.14.0/gcc/13.2.0/mpich/4.1.2 hopr/master/gcc/13.2.0/mpich/4.1.2/hdf5/1.14.0 petsc/3.19.3/gcc/13.2.0/mpich/4.1.2 || true;
fi
- if [ -n "${DO_MPICH_DEBUG}" ]; then
echo "DO_MPICH_DEBUG activated. Loading MPICH with --with-device=ch3:sock which allows over-subscription without performance loss but cannot be used for shared memory arrays safely!";
module purge;
module load cmake/3.26.4 gcc/13.2.0 mpich-debug/4.1.2/gcc/13.2.0 hdf5/1.14.0/gcc/13.2.0/mpich-debug/4.1.2 hopr/master/gcc/13.2.0/mpich-debug/4.1.2/hdf5/1.14.0 petsc/3.19.3/gcc/13.2.0/mpich-debug/4.1.2;
module purge || true;
module load cmake/3.26.4 gcc/13.2.0 mpich-debug/4.1.2/gcc/13.2.0 hdf5/1.14.0/gcc/13.2.0/mpich-debug/4.1.2 hopr/master/gcc/13.2.0/mpich-debug/4.1.2/hdf5/1.14.0 petsc/3.19.3/gcc/13.2.0/mpich-debug/4.1.2 || true;
fi
- module list
- module list || true

# ----------------------------------------------------------------------------------------------------------------------------------------------------
# Stages
Expand Down Expand Up @@ -461,7 +461,7 @@ build_DSMC:
- cmake .. -DCMAKE_BUILD_TYPE=Release -DPICLAS_TIMEDISCMETHOD=DSMC -DLIBS_BUILD_HDF5=OFF -DPICLAS_POLYNOMIAL_DEGREE=1 ; $GENERATOR -j $NCORES all
- if [ -n "${DO_WEEKLY}" ]; then
cd .. ;
mkdir -p build_radiation ; cd build_radiation ;
mkdir -p build_radiation ; cd build_radiation ;
cmake .. -DLIBS_BUILD_HDF5=OFF -DPICLAS_BUILD_POSTI=ON -DPOSTI_BUILD_SUPERB=ON -DPICLAS_READIN_CONSTANTS=ON -DPICLAS_TIMEDISCMETHOD=Radiation -DPICLAS_INTKIND8=ON ; $GENERATOR -j $NCORES all ;
fi

Expand Down Expand Up @@ -1071,4 +1071,4 @@ github:
- git clone -b master --single-branch [email protected]:piclas/piclas.git piclas_github ;
- cd piclas_github ;
- git remote add piclas-framework [email protected]:piclas-framework/piclas.git ;
- git push --force --follow-tags piclas-framework master ;
- git push --force --follow-tags piclas-framework master ;
102 changes: 59 additions & 43 deletions CMakeListsLib.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,9 +125,9 @@ IF (NOT HDF5_FOUND)
ENDIF()
# Hide all the HDF5 libs paths
MARK_AS_ADVANCED(FORCE HDF5_DIR)
# CMake version found
IF (HDF5_FOUND)
IF(${CMAKE_VERSION} VERSION_LESS "3.10.0")
# CMake version found
IF(LIBS_HDF5_CMAKE AND ${CMAKE_VERSION} VERSION_LESS "3.10.0")
MESSAGE (WARNING "HDF5 built with CMake was found! This feature is only supported for CMake 3.10+ and HDF5 1.10.2+!")
ENDIF()
MESSAGE (STATUS "HDF5 C libs:${HDF5_FOUND} static:${HDF5_static_C_FOUND} and shared:${HDF5_shared_C_FOUND}")
Expand Down Expand Up @@ -633,63 +633,79 @@ ENDIF()
#=========================================================================
# PETSc
#=========================================================================
OPTION(PICLAS_PETSC "Use the PETSc library" OFF)
IF(PICLAS_PETSC)
ADD_DEFINITIONS(-DUSE_PETSC=1)
IF(NOT DEFINED PETSC_DIR)
SET(PETSC_DIR $ENV{PETSC_DIR})
ENDIF()
IF(NOT DEFINED PETSC_ARCH)
SET(PETSC_ARCH $ENV{PETSC_ARCH})
OPTION(LIBS_USE_PETSC "Use the PETSc library" OFF)
IF(LIBS_USE_PETSC)
# pkg-config will look at /usr by default, add additional search paths here
IF(DEFINED PETSC_DIR)
# Add our PETSc to pkg-config search path
SET(PKG_CONFIG_PATH PETSC_DIR)
ENDIF()

IF(EXISTS "${PETSC_DIR}/include" AND EXISTS "${PETSC_DIR}/${PETSC_ARCH}/include")
SET(PETSC_INCLUDE_DIR "${PETSC_DIR}/include")
SET(PETSC_INC "${PETSC_DIR}/include" "${PETSC_DIR}/${PETSC_ARCH}/include")
ELSE()
MESSAGE(SEND_ERROR "PETSc includes not found")
ENDIF()
FIND_PACKAGE(PETSc QUIET)

IF(EXISTS "${PETSC_DIR}/${PETSC_ARCH}/lib/libpetsc.so")
SET(PETSC_LIB "${PETSC_DIR}/${PETSC_ARCH}/lib/libpetsc.so")
ELSEIf(EXISTS "${PETSC_DIR}/${PETSC_ARCH}/lib/libpetsc.a")
SET(PETSC_LIB "${PETSC_DIR}/${PETSC_ARCH}/lib/libpetsc.a")
IF (PETSc_FOUND)
MESSAGE (STATUS "[PETSc] found in system libraries")
SET(LIBS_BUILD_PETSC OFF CACHE BOOL "Compile and build preCICE library")
ELSE()
MESSAGE(SEND_ERROR "PETSc library not found")
MESSAGE (STATUS "[PETSc] not found in system libraries")
SET(LIBS_BUILD_PETSC ON CACHE BOOL "Compile and build preCICE library")
ENDIF()

IF(EXISTS ${PETSC_DIR}/${PETSC_ARCH}/conf/petscvariables)
FILE(STRINGS ${PETSC_DIR}/${PETSC_ARCH}/conf/petscvariables
PETSC_VARIABLES NEWLINE_CONSUME)
ELSEIf(EXISTS ${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/petscvariables)
FILE(STRINGS ${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/petscvariables
PETSC_VARIABLES NEWLINE_CONSUME)
ELSE()
MESSAGE(SEND_ERROR "PETSc variables not found")
ENDIF()
IF (LIBS_BUILD_PETSC)
# Set PETSc build dir
SET(LIBS_PETSc_DIR ${LIBS_EXTERNAL_LIB_DIR}/PETSc/build)
SET(PETSc_BUILD_DIR ${LIBS_EXTERNAL_LIB_DIR}/PETSc/linux-gnu-c)
SET(PETSc_LIB_LIBRARIES ${LIBS_EXTERNAL_LIB_DIR}/PETSc/linux-gnu-c/lib)
SET(PETSc_INCLUDE_DIRS ${LIBS_EXTERNAL_LIB_DIR}/PETSc/linux-gnu-c/include)
SET(PETSc_VERSION 3.21.2)

# Add PETSc project
EXTERNALPROJECT_ADD("PETSc"
GIT_REPOSITORY "https://gitlab.com/petsc/petsc.git"
GIT_TAG "v${PETSc_VERSION}"
GIT_PROGRESS TRUE
# USES_TERMINAL_DOWNLOAD true
# USES_TERMINAL_UPDATE true
# USES_TERMINAL_BUILD true
# USES_TERMINAL_INSTALL true
# USES_TERMINAL_TEST true
${${GITSHALLOW}}
PREFIX ${LIBS_PETSc_DIR}
INSTALL_DIR ${LIBS_EXTERNAL_LIB_DIR}/PETSc
BUILD_IN_SOURCE TRUE
UPDATE_COMMAND ""
CONFIGURE_COMMAND PETSC_DIR=${LIBS_PETSc_DIR}/src/PETSc ${LIBS_PETSc_DIR}/src/PETSc/configure --prefix=${PETSc_BUILD_DIR} --with-shared-libraries=1 --with-mpi-f90module-visibility=0 --with-bison=0 --with-cmake=0 --with-hdf5=1 --with-hdf5-fortran-bindings=1
# BUILD_COMMAND ${CMAKE_MAKE_PROGRAM} -j4
BUILD_BYPRODUCTS ${PETSc_LIB_LIBRARIES}/libpetsc.so
)

# Find "^#define PETSC_VERSION_MAJOR" and get only the numbers and remove trailing line breaks
SET(PETSC_VERSION_FILE "${PETSC_INCLUDE_DIR}/petscversion.h")
IF(EXISTS "${PETSC_VERSION_FILE}")
EXECUTE_PROCESS(COMMAND cat "${PETSC_VERSION_FILE}" COMMAND grep "^#define PETSC_VERSION_MAJOR" COMMAND grep -o "[[:digit:]]*" COMMAND tr -d '\n' OUTPUT_VARIABLE PICLAS_PETSC_VERSION_MAJOR)
EXECUTE_PROCESS(COMMAND cat "${PETSC_VERSION_FILE}" COMMAND grep "^#define PETSC_VERSION_MINOR" COMMAND grep -o "[[:digit:]]*" COMMAND tr -d '\n' OUTPUT_VARIABLE PICLAS_PETSC_VERSION_MINOR)
MESSAGE(STATUS "Found PETSC version [${PICLAS_PETSC_VERSION_MAJOR}.${PICLAS_PETSC_VERSION_MINOR}] in ${PETSC_VERSION_FILE}")
IF(${PICLAS_PETSC_VERSION_MAJOR}.${PICLAS_PETSC_VERSION_MINOR} VERSION_LESS 3.18)
# Actually add the preCICE lib paths to the linking paths
INCLUDE_DIRECTORIES(${PETSc_INCLUDE_DIRS})
INCLUDE_DIRECTORIES (${PETSc_LIB_LIBRARIES})
LIST(APPEND linkedlibs ${PETSc_LIB_LIBRARIES}/libpetsc.so)
LIST(APPEND SELFBUILTEXTERNALS PETSc)

ADD_DEFINITIONS(-DUSE_PETSC=1)
MESSAGE(STATUS "Compiling with self-built [PETSc] (v${PETSc_VERSION})")
ELSE()
# Check if PETSc version needs FIX317
IF (${PETSc_VERSION} VERSION_LESS 3.18)
ADD_DEFINITIONS(-DUSE_PETSC_FIX317=1)
ELSE()
ADD_DEFINITIONS(-DUSE_PETSC_FIX317=0)
ENDIF()
ELSE()
ADD_DEFINITIONS(-DUSE_PETSC_FIX317=1)
ENDIF()

INCLUDE_DIRECTORIES(${PETSC_INC})
LIST(APPEND linkedlibs ${PETSC_LIB})
INCLUDE_DIRECTORIES(${PETSc_INCLUDE_DIRS})
LIST(APPEND linkedlibs ${PETSc_LINK_LIBRARIES})

ADD_DEFINITIONS(-DUSE_PETSC=1)
MESSAGE(STATUS "Compiling with system [PETSc] (v${PETSc_VERSION}) [${PETSc_LINK_LIBRARIES}]")
ENDIF()

# Do not allow PETSc + INT8 as this is not implemented yet
IF(PICLAS_INTKIND8)
MESSAGE(FATAL_ERROR "PETSc + INT8 is not implemented yet!.")
ENDIF()
ELSE()
ADD_DEFINITIONS(-DUSE_PETSC=0)
ENDIF(PICLAS_PETSC)
ENDIF(LIBS_USE_PETSC)
3 changes: 2 additions & 1 deletion CMakeListsMachine.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ IF("${CMAKE_GENERATOR}" MATCHES "Ninja")
SET(NINJA_COLOR_DIAGNOSTICS "-fdiagnostics-color=always" CACHE INTERNAL "Flag if Ninja should attempt to color output")
ENDIF()
ENDIF()
MESSAGE(STATUS "Generating for [${CMAKE_GENERATOR}] build system")

# =========================================================================
# Some clusters requires setting the compilers by hand and invoking
Expand Down Expand Up @@ -340,4 +341,4 @@ SET(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" CACHE STRIN
SET(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" CACHE STRING "Release compiler flags" FORCE)
SET(CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE}" CACHE STRING "Profile compiler flags" FORCE)
SET(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" CACHE STRING "Debug compiler flags" FORCE)
SET(CMAKE_Fortran_FLAGS_SANITIZE "${CMAKE_Fortran_FLAGS_SANITIZE}" CACHE STRING "Sanitize compiler flags" FORCE)
SET(CMAKE_Fortran_FLAGS_SANITIZE "${CMAKE_Fortran_FLAGS_SANITIZE}" CACHE STRING "Sanitize compiler flags" FORCE)
Loading

0 comments on commit c73c5b1

Please sign in to comment.