diff --git a/cmake/thirdparty/SetupSeracThirdParty.cmake b/cmake/thirdparty/SetupSeracThirdParty.cmake index d9fba1287..8592d1ce6 100644 --- a/cmake/thirdparty/SetupSeracThirdParty.cmake +++ b/cmake/thirdparty/SetupSeracThirdParty.cmake @@ -195,6 +195,29 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND) endif() message(STATUS "ARPACK support is ${ARPACK_FOUND}") + #------------------------------------------------------------------------------ + # Enzyme + #------------------------------------------------------------------------------ + if (ENZYME_DIR) + serac_assert_is_directory(DIR_VARIABLE ENZYME_DIR) + set(Enzyme_ROOT ${ENZYME_DIR} CACHE PATH "") + find_dependency(Enzyme REQUIRED) + + serac_assert_find_succeeded(PROJECT_NAME Enzyme + TARGET LLDEnzymeFlags + DIR_VARIABLE ENZYME_DIR) + + serac_assert_find_succeeded(PROJECT_NAME Enzyme + TARGET ClangEnzymeFlags + DIR_VARIABLE ENZYME_DIR) + + message(STATUS "Enzyme support is ON") + set(ENZYME_FOUND TRUE) + else() + message(STATUS "Enzyme support is OFF") + set(ENZYME_FOUND FALSE) + endif() + #------------------------------------------------------------------------------ # MFEM #------------------------------------------------------------------------------ @@ -214,7 +237,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND) message(STATUS "Using MFEM submodule") #### Store Data that MFEM clears - set(tpls_to_save ADIAK AMGX AXOM CALIPER CAMP CONDUIT HDF5 + set(tpls_to_save ADIAK AMGX AXOM CALIPER CAMP CONDUIT ENZYME HDF5 HYPRE LUA METIS MFEM NETCDF PARMETIS PETSC RAJA SLEPC SUPERLU_DIST STRUMPACK SUNDIALS TRIBOL UMPIRE) @@ -292,6 +315,10 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND) endif() set(MFEM_USE_UMPIRE OFF CACHE BOOL "") set(MFEM_USE_ZLIB ON CACHE BOOL "") + if(ENZYME_DIR) + serac_assert_is_directory(DIR_VARIABLE ENZYME_DIR) + set(MFEM_USE_ENZYME ON CACHE BOOL "") + endif() #### MFEM Configuration Options @@ -469,78 +496,61 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND) set(ENABLE_FORTRAN ON CACHE BOOL "" FORCE) endif() - #------------------------------------------------------------------------------ - # Enzyme (used by Tribol) - #------------------------------------------------------------------------------ - if (ENZYME_DIR) - serac_assert_is_directory(DIR_VARIABLE ENZYME_DIR) - set(Enzyme_ROOT ${ENZYME_DIR} CACHE PATH "") - find_dependency(Enzyme REQUIRED) - - serac_assert_find_succeeded(PROJECT_NAME Enzyme - TARGET LLDEnzymeFlags - DIR_VARIABLE ENZYME_DIR) - - message(STATUS "Enzyme support is ON") - set(ENZYME_FOUND TRUE) - else() - message(STATUS "Enzyme support is OFF") - set(ENZYME_FOUND FALSE) - endif() - #------------------------------------------------------------------------------ # Tribol #------------------------------------------------------------------------------ - if (NOT SERAC_ENABLE_CODEVELOP) - if(TRIBOL_DIR) - serac_assert_is_directory(DIR_VARIABLE TRIBOL_DIR) + if (NOT SERAC_DISABLE_TRIBOL) + if (NOT SERAC_ENABLE_CODEVELOP) + if(TRIBOL_DIR) + serac_assert_is_directory(DIR_VARIABLE TRIBOL_DIR) - find_dependency(tribol REQUIRED PATHS "${TRIBOL_DIR}/lib/cmake") + find_dependency(tribol REQUIRED PATHS "${TRIBOL_DIR}/lib/cmake") - serac_assert_find_succeeded(PROJECT_NAME Tribol - TARGET tribol - DIR_VARIABLE TRIBOL_DIR) - blt_convert_to_system_includes(TARGET tribol) - set(TRIBOL_FOUND ON) + serac_assert_find_succeeded(PROJECT_NAME Tribol + TARGET tribol + DIR_VARIABLE TRIBOL_DIR) + blt_convert_to_system_includes(TARGET tribol) + set(TRIBOL_FOUND ON) + else() + set(TRIBOL_FOUND OFF) + endif() + + message(STATUS "Tribol support is " ${TRIBOL_FOUND}) else() - set(TRIBOL_FOUND OFF) - endif() - - message(STATUS "Tribol support is " ${TRIBOL_FOUND}) - else() - set(ENABLE_FORTRAN OFF CACHE BOOL "" FORCE) - # Otherwise we use the submodule - message(STATUS "Using Tribol submodule") - set(BUILD_REDECOMP ${SERAC_ENABLE_MPI} CACHE BOOL "") - set(TRIBOL_USE_MPI ${SERAC_ENABLE_MPI} CACHE BOOL "") - set(TRIBOL_ENABLE_TESTS OFF CACHE BOOL "") - set(TRIBOL_ENABLE_EXAMPLES OFF CACHE BOOL "") - set(TRIBOL_ENABLE_DOCS OFF CACHE BOOL "") + set(ENABLE_FORTRAN OFF CACHE BOOL "" FORCE) + # Otherwise we use the submodule + message(STATUS "Using Tribol submodule") + set(BUILD_REDECOMP ${SERAC_ENABLE_MPI} CACHE BOOL "") + set(TRIBOL_USE_MPI ${SERAC_ENABLE_MPI} CACHE BOOL "") + set(TRIBOL_ENABLE_TESTS OFF CACHE BOOL "") + set(TRIBOL_ENABLE_EXAMPLES OFF CACHE BOOL "") + set(TRIBOL_ENABLE_DOCS OFF CACHE BOOL "") + + if(${PROJECT_NAME} STREQUAL "smith") + set(tribol_repo_dir "${PROJECT_SOURCE_DIR}/serac/tribol") + else() + set(tribol_repo_dir "${PROJECT_SOURCE_DIR}/tribol") + endif() - if(${PROJECT_NAME} STREQUAL "smith") - set(tribol_repo_dir "${PROJECT_SOURCE_DIR}/serac/tribol") - else() - set(tribol_repo_dir "${PROJECT_SOURCE_DIR}/tribol") + add_subdirectory(${tribol_repo_dir} ${CMAKE_BINARY_DIR}/tribol) + + target_include_directories(redecomp PUBLIC + $ + ) + target_include_directories(tribol PUBLIC + $ + $ + $ + ) + target_include_directories(tribol_shared PUBLIC + $ + $ + $ + ) + + set(TRIBOL_FOUND TRUE CACHE BOOL "" FORCE) + set(ENABLE_FORTRAN ON CACHE BOOL "" FORCE) endif() - - add_subdirectory(${tribol_repo_dir} ${CMAKE_BINARY_DIR}/tribol) - - target_include_directories(redecomp PUBLIC - $ - ) - target_include_directories(tribol PUBLIC - $ - $ - $ - ) - target_include_directories(tribol_shared PUBLIC - $ - $ - $ - ) - - set(TRIBOL_FOUND TRUE CACHE BOOL "" FORCE) - set(ENABLE_FORTRAN ON CACHE BOOL "" FORCE) endif() #--------------------------------------------------------------------------- diff --git a/host-configs/dfem/ruby-toss_4_x86_64_ib-clang@19.1.7.cmake b/host-configs/dfem/ruby-toss_4_x86_64_ib-clang@19.1.7.cmake new file mode 100644 index 000000000..fa0cd4152 --- /dev/null +++ b/host-configs/dfem/ruby-toss_4_x86_64_ib-clang@19.1.7.cmake @@ -0,0 +1,138 @@ +#------------------------------------------------------------------------------ +# !!!! This is a generated file, edit at own risk !!!! +#------------------------------------------------------------------------------ +# CMake executable path: /usr/tce/bin/cmake +#------------------------------------------------------------------------------ + +set(CMAKE_PREFIX_PATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/caliper-2.12.1-5o3zfbeosigjwrp5ffizjs6sf7pyoqjv;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/tribol-0.1.0.18-my4bcnwnewxcnewg3xgbzvym64dfizuj;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/adiak-0.4.1-5wjrv4xwbm3y7yh7745ijc45sbcwqwqi;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/libunwind-1.8.1-tgmphz65mn62bautt2ksl45ckecw5e2c;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/axom-0.10.1.1-tkmjd26d72eqk4tts7hwlwognvwuyj5s;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/conduit-0.9.2-t2zobt76hkwap7d2ppddp3traorxpffl;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/mfem-4.8.0.1-gbv5tbeosslcqh2zhkcrgkwtqxob257j;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/raja-2024.07.0-t5ay2skmkw3ybywrq33tcyl5xbn5g3gv;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/umpire-2024.07.0-47yl3s6ob3ru4g6qnma4tiv6ob5fr6p6;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/netcdf-c-4.7.4-62ri63jaeyajwrrvwk75dbwvrnj6fony;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/slepc-3.21.2-rji2oyfq6x4lyovnrkfqnleoc3yrqrm6;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/sundials-6.7.0-wwas67vc5xfgayl2uimersm3xszlqv62;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/camp-2024.07.0-afpyaxeo4ulr2lkljpy2zr7rjeq64cox;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/fmt-11.0.2-xpfumigjok66664dhruf6tfaa7gwcn3n;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/hdf5-1.8.23-pu3kif4ltqdw2kcwvlswfze6pmmsjs6q;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/arpack-ng-3.9.0-ahgmmge3jagsx3n72wbxtcznbubcclca;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/petsc-3.21.6-elp7iw6xe64daspgy5ctubfo3jj4nd43;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/hypre-2.26.0-ta6n6j6ziyuw2tfejt27nkdulvjq27hu;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/strumpack-8.0.0-zmfwdjw54im5o6m3gqssnkvuagdzljwe;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/superlu-dist-8.1.2-k53l7hvjwbqnepeb7etlujfgzrzfliua;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/netlib-scalapack-2.2.2-fuk7tt7il4e6tekxxctc625g4xk4hqeu;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/parmetis-4.0.3-morhecieft72ri6gi2r6hsatzrgdhe5n;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/metis-5.1.0-ebbueyypx5bn5mekybxavligkykhm5fy;/usr/workspace/smithdev/devtools/toss_4_x86_64_ib/latest/python-3.11.7;/usr/workspace/smithdev/devtools/toss_4_x86_64_ib/latest/python-3.11.7;/usr/tce;/usr/workspace/smithdev/devtools/toss_4_x86_64_ib/latest/cppcheck-2.9;/usr/workspace/smithdev/devtools/toss_4_x86_64_ib/latest/doxygen-1.9.8;/usr/tce/packages/clang/clang-14.0.6;/usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-10.3.1;/usr/tce/packages/python/python-3.9.12" CACHE STRING "") + +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH "ON" CACHE STRING "") + +set(CMAKE_BUILD_RPATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/serac-develop-rxm4riztuqljjlnrhyomn4cqusqy5g6z/lib;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/serac-develop-rxm4riztuqljjlnrhyomn4cqusqy5g6z/lib64;;/usr/tce/packages/gcc/gcc-10.3.1/lib/gcc/x86_64-redhat-linux/10" CACHE STRING "") + +set(CMAKE_INSTALL_RPATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/serac-develop-rxm4riztuqljjlnrhyomn4cqusqy5g6z/lib;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/serac-develop-rxm4riztuqljjlnrhyomn4cqusqy5g6z/lib64;;/usr/tce/packages/gcc/gcc-10.3.1/lib/gcc/x86_64-redhat-linux/10" CACHE STRING "") + +set(CMAKE_BUILD_TYPE "Release" CACHE STRING "") + +#------------------------------------------------------------------------------ +# Compilers +#------------------------------------------------------------------------------ +# Compiler Spec: clang@=19.1.7 +#------------------------------------------------------------------------------ +if(DEFINED ENV{SPACK_CC}) + + set(CMAKE_C_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/spack/lib/spack/env/clang/clang" CACHE PATH "") + + set(CMAKE_CXX_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/spack/lib/spack/env/clang/clang++" CACHE PATH "") + + set(CMAKE_Fortran_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/spack/lib/spack/env/clang/gfortran" CACHE PATH "") + +else() + + set(CMAKE_C_COMPILER "/usr/WS2/smithdev/toss_4_x86_64_ib/llvm/19.1.7/bin/clang" CACHE PATH "") + + set(CMAKE_CXX_COMPILER "/usr/WS2/smithdev/toss_4_x86_64_ib/llvm/19.1.7/bin/clang++" CACHE PATH "") + + set(CMAKE_Fortran_COMPILER "/usr/tce/packages/gcc/gcc-10.3.1/bin/gfortran" CACHE PATH "") + +endif() + +set(CMAKE_C_FLAGS "--gcc-toolchain=/usr/tce/packages/gcc/gcc-10.3.1" CACHE STRING "") + +set(CMAKE_CXX_FLAGS "--gcc-toolchain=/usr/tce/packages/gcc/gcc-10.3.1" CACHE STRING "") + +set(CMAKE_Fortran_FLAGS "-fPIE -fPIC" CACHE STRING "") + +#------------------------------------------------------------------------------ +# MPI +#------------------------------------------------------------------------------ + +set(MPI_C_COMPILER "/usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-10.3.1/bin/mpicc" CACHE PATH "") + +set(MPI_CXX_COMPILER "/usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-10.3.1/bin/mpicxx" CACHE PATH "") + +set(MPI_Fortran_COMPILER "/usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-10.3.1/bin/mpif90" CACHE PATH "") + +set(MPIEXEC_NUMPROC_FLAG "-n" CACHE STRING "") + +set(ENABLE_MPI ON CACHE BOOL "") + +set(MPIEXEC_EXECUTABLE "/usr/bin/srun" CACHE PATH "") + +#------------------------------------------------------------------------------ +# Hardware +#------------------------------------------------------------------------------ + +set(ENABLE_OPENMP OFF CACHE BOOL "") + +#------------------------------------------------------------------------------ +# TPLs +#------------------------------------------------------------------------------ + +set(TPL_ROOT "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7" CACHE PATH "") + +set(AXOM_DIR "${TPL_ROOT}/axom-0.10.1.1-tkmjd26d72eqk4tts7hwlwognvwuyj5s" CACHE PATH "") + +set(CAMP_DIR "${TPL_ROOT}/camp-2024.07.0-afpyaxeo4ulr2lkljpy2zr7rjeq64cox" CACHE PATH "") + +set(CONDUIT_DIR "${TPL_ROOT}/conduit-0.9.2-t2zobt76hkwap7d2ppddp3traorxpffl" CACHE PATH "") + +set(LUA_DIR "/usr" CACHE PATH "") + +set(MFEM_DIR "${TPL_ROOT}/mfem-4.8.0.1-gbv5tbeosslcqh2zhkcrgkwtqxob257j" CACHE PATH "") + +set(HDF5_DIR "${TPL_ROOT}/hdf5-1.8.23-pu3kif4ltqdw2kcwvlswfze6pmmsjs6q" CACHE PATH "") + +set(HYPRE_DIR "${TPL_ROOT}/hypre-2.26.0-ta6n6j6ziyuw2tfejt27nkdulvjq27hu" CACHE PATH "") + +set(METIS_DIR "${TPL_ROOT}/metis-5.1.0-ebbueyypx5bn5mekybxavligkykhm5fy" CACHE PATH "") + +set(PARMETIS_DIR "${TPL_ROOT}/parmetis-4.0.3-morhecieft72ri6gi2r6hsatzrgdhe5n" CACHE PATH "") + +set(NETCDF_DIR "${TPL_ROOT}/netcdf-c-4.7.4-62ri63jaeyajwrrvwk75dbwvrnj6fony" CACHE PATH "") + +set(SUPERLUDIST_DIR "${TPL_ROOT}/superlu-dist-8.1.2-k53l7hvjwbqnepeb7etlujfgzrzfliua" CACHE PATH "") + +set(ARPACK_DIR "${TPL_ROOT}/arpack-ng-3.9.0-ahgmmge3jagsx3n72wbxtcznbubcclca" CACHE PATH "") + +set(ADIAK_DIR "${TPL_ROOT}/adiak-0.4.1-5wjrv4xwbm3y7yh7745ijc45sbcwqwqi" CACHE PATH "") + +# AMGX not built + +set(CALIPER_DIR "${TPL_ROOT}/caliper-2.12.1-5o3zfbeosigjwrp5ffizjs6sf7pyoqjv" CACHE PATH "") + +set(PETSC_DIR "${TPL_ROOT}/petsc-3.21.6-elp7iw6xe64daspgy5ctubfo3jj4nd43" CACHE PATH "") + +set(RAJA_DIR "${TPL_ROOT}/raja-2024.07.0-t5ay2skmkw3ybywrq33tcyl5xbn5g3gv" CACHE PATH "") + +set(SLEPC_DIR "${TPL_ROOT}/slepc-3.21.2-rji2oyfq6x4lyovnrkfqnleoc3yrqrm6" CACHE PATH "") + +set(STRUMPACK_DIR "${TPL_ROOT}/strumpack-8.0.0-zmfwdjw54im5o6m3gqssnkvuagdzljwe" CACHE PATH "") + +set(SUNDIALS_DIR "${TPL_ROOT}/sundials-6.7.0-wwas67vc5xfgayl2uimersm3xszlqv62" CACHE PATH "") + +set(UMPIRE_DIR "${TPL_ROOT}/umpire-2024.07.0-47yl3s6ob3ru4g6qnma4tiv6ob5fr6p6" CACHE PATH "") + +set(TRIBOL_DIR "${TPL_ROOT}/tribol-0.1.0.18-my4bcnwnewxcnewg3xgbzvym64dfizuj" CACHE PATH "") + +#------------------------------------------------------------------------------ +# Devtools +#------------------------------------------------------------------------------ + +set(DEVTOOLS_ROOT "/usr/WS2/smithdev/devtools/toss_4_x86_64_ib/2024_05_30_15_02_20/._view/psk2dcrijss6s4i6qmxplzthrzm3y7nh" CACHE PATH "") + +set(ATS_EXECUTABLE "${DEVTOOLS_ROOT}/python-3.11.7/bin/ats" CACHE PATH "") + +set(CLANGFORMAT_EXECUTABLE "/usr/tce/packages/clang/clang-14.0.6/bin/clang-format" CACHE PATH "") + +set(CLANGTIDY_EXECUTABLE "/usr/tce/packages/clang/clang-14.0.6/bin/clang-tidy" CACHE PATH "") + +set(ENABLE_DOCS ON CACHE BOOL "") + +set(SPHINX_EXECUTABLE "${DEVTOOLS_ROOT}/python-3.11.7/bin/sphinx-build" CACHE PATH "") + +set(CPPCHECK_EXECUTABLE "${DEVTOOLS_ROOT}/cppcheck-2.9/bin/cppcheck" CACHE PATH "") + +set(DOXYGEN_EXECUTABLE "${DEVTOOLS_ROOT}/doxygen-1.9.8/bin/doxygen" CACHE PATH "") + + diff --git a/host-configs/dfem/rzvernal-toss_4_x86_64_ib_cray-rocmcc@6.2.1_hip_dfem.cmake b/host-configs/dfem/rzvernal-toss_4_x86_64_ib_cray-rocmcc@6.2.1_hip_dfem.cmake new file mode 100644 index 000000000..f7132a3f6 --- /dev/null +++ b/host-configs/dfem/rzvernal-toss_4_x86_64_ib_cray-rocmcc@6.2.1_hip_dfem.cmake @@ -0,0 +1,149 @@ +#------------------------------------------------------------------------------ +# !!!! This is a generated file, edit at own risk !!!! +#------------------------------------------------------------------------------ +# CMake executable path: /usr/tce/bin/cmake +#------------------------------------------------------------------------------ + +set(CMAKE_PREFIX_PATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/caliper-2.12.1-h2elmogt2mtorvunf67jrcxqdcrzjjem;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/tribol-0.1.0.18-neize4eyd3wxmeuui44az2x5ecqfe2pd;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/adiak-0.4.1-djvwhzv7uucbw342y2lxn6mjv4onnzeh;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/libunwind-1.8.1-667lulihqw77lixyhoyezlqadqmw3ufq;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/axom-0.10.1.1-4udyvzqnaoic74uv3zcumyguv5mbqqcg;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/conduit-0.9.2-wo23sn3yekayvxjphe2eu4vvmvkqrovj;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/mfem-4.8.0.1-s6vr55m7kq5gylxswlaqy5usltbrslzd;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/raja-2024.07.0-m322mdffprdfso4vg7iox3n5goelhtyp;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/umpire-2024.07.0-h4aizqln4js5uo6o5dnkkjadcq5apw37;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/netcdf-c-4.7.4-m3moxxpcdu3id35a6ib662pmcpe6hlxt;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/strumpack-8.0.0-w5kwnpidy7gzm4q2kplzw7ip2hebkojg;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/sundials-6.7.0-ihjcn5jozjo6hqxvueljdahsvpg4fpyr;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/superlu-dist-8.1.2-ulxns2zfpqyuezhrgxjh7do45ezer2m5;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/camp-2024.07.0-223y52kngyoyilxr7gz3i4pe5btnm5uq;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/fmt-11.0.2-haismsyfch472jneh7tj5hoz5faug2wp;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/hdf5-1.8.23-zhqxrmbx5hqezzazejozk5l7wuz3ijx6;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/netlib-scalapack-2.2.2-zinikcerv4cwfrzt4kbhby6xm76fjmvq;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/hypre-2.26.0-sffmaz74r5zmnzir6l73deslw36yypsh;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/parmetis-4.0.3-7o2cbbpbawncpn24yftcj24gbopz4aev;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/metis-5.1.0-q4s4zr5oj2vsyc4eqjy22wrwndgb4fnh;/usr/tce;/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1/llvm;/opt/rocm-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1" CACHE STRING "") + +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH "ON" CACHE STRING "") + +set(CMAKE_BUILD_RPATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/serac-develop-og4fmn5dkubewiybduaji237suiuzssb/lib;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/serac-develop-og4fmn5dkubewiybduaji237suiuzssb/lib64;;/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12" CACHE STRING "") + +set(CMAKE_INSTALL_RPATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/serac-develop-og4fmn5dkubewiybduaji237suiuzssb/lib;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/serac-develop-og4fmn5dkubewiybduaji237suiuzssb/lib64;;/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12" CACHE STRING "") + +set(CMAKE_BUILD_TYPE "Release" CACHE STRING "") + +#------------------------------------------------------------------------------ +# Compilers +#------------------------------------------------------------------------------ +# Compiler Spec: rocmcc@=6.2.1 +#------------------------------------------------------------------------------ +if(DEFINED ENV{SPACK_CC}) + + set(CMAKE_C_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/spack/lib/spack/env/rocmcc/amdclang" CACHE PATH "") + + set(CMAKE_CXX_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/spack/lib/spack/env/rocmcc/amdclang++" CACHE PATH "") + + set(CMAKE_Fortran_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/spack/lib/spack/env/rocmcc/amdflang" CACHE PATH "") + +else() + + set(CMAKE_C_COMPILER "/opt/rocm-6.2.1/llvm/bin/amdclang" CACHE PATH "") + + set(CMAKE_CXX_COMPILER "/opt/rocm-6.2.1/llvm/bin/amdclang++" CACHE PATH "") + + set(CMAKE_Fortran_COMPILER "/opt/rocm-6.2.1/llvm/bin/amdflang" CACHE PATH "") + +endif() + +#------------------------------------------------------------------------------ +# MPI +#------------------------------------------------------------------------------ + +set(MPI_C_COMPILER "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.2.1/bin/mpicc" CACHE PATH "") + +set(MPI_CXX_COMPILER "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.2.1/bin/mpicxx" CACHE PATH "") + +set(MPI_Fortran_COMPILER "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.2.1/bin/mpif90" CACHE PATH "") + +set(MPIEXEC_NUMPROC_FLAG "-n" CACHE STRING "") + +set(ENABLE_MPI ON CACHE BOOL "") + +set(MPIEXEC_EXECUTABLE "/usr/global/tools/flux_wrappers/bin/srun" CACHE PATH "") + +#------------------------------------------------------------------------------ +# Hardware +#------------------------------------------------------------------------------ + +#------------------------------------------------ +# ROCm +#------------------------------------------------ + +set(HIP_ROOT_DIR "/opt/rocm-6.2.1" CACHE PATH "") + +set(CMAKE_HIP_COMPILER "/opt/rocm-6.2.1/llvm/bin/clang++" CACHE FILEPATH "") + +set(CMAKE_HIP_ARCHITECTURES "gfx90a" CACHE STRING "") + +set(AMDGPU_TARGETS "gfx90a" CACHE STRING "") + +set(GPU_TARGETS "gfx90a" CACHE STRING "") + +set(ENABLE_OPENMP OFF CACHE BOOL "") + +set(ENABLE_HIP ON CACHE BOOL "") + +set(ROCM_PATH "/opt/rocm-6.2.1" CACHE PATH "") + +set(CMAKE_EXE_LINKER_FLAGS "-L/opt/rocm-6.2.1/lib -Wl,-rpath,/opt/rocm-6.2.1/lib -L/opt/rocm-6.2.1/llvm/lib -Wl,-rpath,/opt/rocm-6.2.1/llvm/lib -lxpmem -L/opt/cray/pe/mpich/8.1.29/gtl/lib -Wl,-rpath,/opt/cray/pe/mpich/8.1.29/gtl/lib -lmpi_gtl_hsa -Wl,--disable-new-dtags -lflang -lflangrti -lamdhip64 -lhsakmt -lhsa-runtime64 -lamd_comgr -lpgmath -lhipblas" CACHE STRING "") + +#------------------------------------------------------------------------------ +# TPLs +#------------------------------------------------------------------------------ + +set(TPL_ROOT "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1" CACHE PATH "") + +set(AXOM_DIR "${TPL_ROOT}/axom-0.10.1.1-4udyvzqnaoic74uv3zcumyguv5mbqqcg" CACHE PATH "") + +set(CAMP_DIR "${TPL_ROOT}/camp-2024.07.0-223y52kngyoyilxr7gz3i4pe5btnm5uq" CACHE PATH "") + +set(CONDUIT_DIR "${TPL_ROOT}/conduit-0.9.2-wo23sn3yekayvxjphe2eu4vvmvkqrovj" CACHE PATH "") + +set(LUA_DIR "/usr" CACHE PATH "") + +set(MFEM_DIR "${TPL_ROOT}/mfem-4.8.0.1-s6vr55m7kq5gylxswlaqy5usltbrslzd" CACHE PATH "") + +set(HDF5_DIR "${TPL_ROOT}/hdf5-1.8.23-zhqxrmbx5hqezzazejozk5l7wuz3ijx6" CACHE PATH "") + +set(HYPRE_DIR "${TPL_ROOT}/hypre-2.26.0-sffmaz74r5zmnzir6l73deslw36yypsh" CACHE PATH "") + +set(METIS_DIR "${TPL_ROOT}/metis-5.1.0-q4s4zr5oj2vsyc4eqjy22wrwndgb4fnh" CACHE PATH "") + +set(PARMETIS_DIR "${TPL_ROOT}/parmetis-4.0.3-7o2cbbpbawncpn24yftcj24gbopz4aev" CACHE PATH "") + +set(NETCDF_DIR "${TPL_ROOT}/netcdf-c-4.7.4-m3moxxpcdu3id35a6ib662pmcpe6hlxt" CACHE PATH "") + +set(SUPERLUDIST_DIR "${TPL_ROOT}/superlu-dist-8.1.2-ulxns2zfpqyuezhrgxjh7do45ezer2m5" CACHE PATH "") + +set(ADIAK_DIR "${TPL_ROOT}/adiak-0.4.1-djvwhzv7uucbw342y2lxn6mjv4onnzeh" CACHE PATH "") + +# AMGX not built + +set(CALIPER_DIR "${TPL_ROOT}/caliper-2.12.1-h2elmogt2mtorvunf67jrcxqdcrzjjem" CACHE PATH "") + +# PETSC not built + +set(RAJA_DIR "${TPL_ROOT}/raja-2024.07.0-m322mdffprdfso4vg7iox3n5goelhtyp" CACHE PATH "") + +# SLEPC not built + +set(STRUMPACK_DIR "${TPL_ROOT}/strumpack-8.0.0-w5kwnpidy7gzm4q2kplzw7ip2hebkojg" CACHE PATH "") + +set(SUNDIALS_DIR "${TPL_ROOT}/sundials-6.7.0-ihjcn5jozjo6hqxvueljdahsvpg4fpyr" CACHE PATH "") + +set(UMPIRE_DIR "${TPL_ROOT}/umpire-2024.07.0-h4aizqln4js5uo6o5dnkkjadcq5apw37" CACHE PATH "") + +set(TRIBOL_DIR "${TPL_ROOT}/tribol-0.1.0.18-neize4eyd3wxmeuui44az2x5ecqfe2pd" CACHE PATH "") + +set(ENZYME_DIR "/usr/WS2/smithdev/toss_4_x86_64_ib_cray/enzyme/rocm-6.2.1/0.0.180" CACHE PATH "") + +#------------------------------------------------------------------------------ +# Devtools +#------------------------------------------------------------------------------ + +# Code checks disabled due to disabled devtools + +set(SERAC_ENABLE_CODE_CHECKS OFF CACHE BOOL "") + +set(ENABLE_CLANGFORMAT OFF CACHE BOOL "") + +set(ENABLE_CLANGTIDY OFF CACHE BOOL "") + +set(ENABLE_DOCS OFF CACHE BOOL "") + +set(SERAC_ENABLE_CODEVELOP ON CACHE BOOL "") + +set(SERAC_DISABLE_TRIBOL ON CACHE BOOL "") + diff --git a/host-configs/dfem/rzwhippet-toss_4_x86_64_ib-clang@19.1.7.cmake b/host-configs/dfem/rzwhippet-toss_4_x86_64_ib-clang@19.1.7.cmake new file mode 100644 index 000000000..747b9ce1a --- /dev/null +++ b/host-configs/dfem/rzwhippet-toss_4_x86_64_ib-clang@19.1.7.cmake @@ -0,0 +1,143 @@ +#------------------------------------------------------------------------------ +# !!!! This is a generated file, edit at own risk !!!! +#------------------------------------------------------------------------------ +# CMake executable path: /usr/tce/bin/cmake +#------------------------------------------------------------------------------ + +set(CMAKE_PREFIX_PATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/caliper-2.12.1-a4chnmcpoktn7vlyrjxbw2qskkkytkj3;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/tribol-0.1.0.18-my4bcnwnewxcnewg3xgbzvym64dfizuj;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/adiak-0.4.1-5wjrv4xwbm3y7yh7745ijc45sbcwqwqi;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/libunwind-1.8.1-tgmphz65mn62bautt2ksl45ckecw5e2c;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/axom-0.10.1.1-tkmjd26d72eqk4tts7hwlwognvwuyj5s;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/conduit-0.9.2-t2zobt76hkwap7d2ppddp3traorxpffl;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/mfem-4.8.0.1-gbv5tbeosslcqh2zhkcrgkwtqxob257j;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/raja-2024.07.0-t5ay2skmkw3ybywrq33tcyl5xbn5g3gv;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/umpire-2024.07.0-47yl3s6ob3ru4g6qnma4tiv6ob5fr6p6;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/netcdf-c-4.7.4-62ri63jaeyajwrrvwk75dbwvrnj6fony;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/slepc-3.21.2-rji2oyfq6x4lyovnrkfqnleoc3yrqrm6;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/sundials-6.7.0-wwas67vc5xfgayl2uimersm3xszlqv62;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/camp-2024.07.0-afpyaxeo4ulr2lkljpy2zr7rjeq64cox;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/fmt-11.0.2-xpfumigjok66664dhruf6tfaa7gwcn3n;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/hdf5-1.8.23-pu3kif4ltqdw2kcwvlswfze6pmmsjs6q;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/arpack-ng-3.9.0-ahgmmge3jagsx3n72wbxtcznbubcclca;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/petsc-3.21.6-elp7iw6xe64daspgy5ctubfo3jj4nd43;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/hypre-2.26.0-ta6n6j6ziyuw2tfejt27nkdulvjq27hu;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/strumpack-8.0.0-zmfwdjw54im5o6m3gqssnkvuagdzljwe;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/superlu-dist-8.1.2-k53l7hvjwbqnepeb7etlujfgzrzfliua;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/netlib-scalapack-2.2.2-fuk7tt7il4e6tekxxctc625g4xk4hqeu;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/parmetis-4.0.3-morhecieft72ri6gi2r6hsatzrgdhe5n;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/metis-5.1.0-ebbueyypx5bn5mekybxavligkykhm5fy;/usr/workspace/smithdev/devtools/toss_4_x86_64_ib/latest/python-3.11.7;/usr/workspace/smithdev/devtools/toss_4_x86_64_ib/latest/python-3.11.7;/usr/tce;/usr/workspace/smithdev/devtools/toss_4_x86_64_ib/latest/cppcheck-2.9;/usr/workspace/smithdev/devtools/toss_4_x86_64_ib/latest/doxygen-1.9.8;/usr/tce/packages/clang/clang-14.0.6;/usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-10.3.1;/usr/tce/packages/python/python-3.9.12" CACHE STRING "") + +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH "ON" CACHE STRING "") + +set(CMAKE_BUILD_RPATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/serac-develop-t2mywg4ueanbqllbapg5wvrivg3umjoj/lib;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/serac-develop-t2mywg4ueanbqllbapg5wvrivg3umjoj/lib64;;/usr/tce/packages/gcc/gcc-10.3.1/lib/gcc/x86_64-redhat-linux/10" CACHE STRING "") + +set(CMAKE_INSTALL_RPATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/serac-develop-t2mywg4ueanbqllbapg5wvrivg3umjoj/lib;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/serac-develop-t2mywg4ueanbqllbapg5wvrivg3umjoj/lib64;;/usr/tce/packages/gcc/gcc-10.3.1/lib/gcc/x86_64-redhat-linux/10" CACHE STRING "") + +set(CMAKE_BUILD_TYPE "Release" CACHE STRING "") + +#------------------------------------------------------------------------------ +# Compilers +#------------------------------------------------------------------------------ +# Compiler Spec: clang@=19.1.7 +#------------------------------------------------------------------------------ +if(DEFINED ENV{SPACK_CC}) + + set(CMAKE_C_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/spack/lib/spack/env/clang/clang" CACHE PATH "") + + set(CMAKE_CXX_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/spack/lib/spack/env/clang/clang++" CACHE PATH "") + + set(CMAKE_Fortran_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/spack/lib/spack/env/clang/gfortran" CACHE PATH "") + +else() + + set(CMAKE_C_COMPILER "/usr/WS2/smithdev/toss_4_x86_64_ib/llvm/19.1.7/bin/clang" CACHE PATH "") + + set(CMAKE_CXX_COMPILER "/usr/WS2/smithdev/toss_4_x86_64_ib/llvm/19.1.7/bin/clang++" CACHE PATH "") + + set(CMAKE_Fortran_COMPILER "/usr/tce/packages/gcc/gcc-10.3.1/bin/gfortran" CACHE PATH "") + +endif() + +set(CMAKE_C_FLAGS "--gcc-toolchain=/usr/tce/packages/gcc/gcc-10.3.1" CACHE STRING "") + +set(CMAKE_CXX_FLAGS "--gcc-toolchain=/usr/tce/packages/gcc/gcc-10.3.1" CACHE STRING "") + +set(CMAKE_Fortran_FLAGS "-fPIE -fPIC" CACHE STRING "") + +#------------------------------------------------------------------------------ +# MPI +#------------------------------------------------------------------------------ + +set(MPI_C_COMPILER "/usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-10.3.1/bin/mpicc" CACHE PATH "") + +set(MPI_CXX_COMPILER "/usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-10.3.1/bin/mpicxx" CACHE PATH "") + +set(MPI_Fortran_COMPILER "/usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-10.3.1/bin/mpif90" CACHE PATH "") + +set(MPIEXEC_NUMPROC_FLAG "-n" CACHE STRING "") + +set(ENABLE_MPI ON CACHE BOOL "") + +set(MPIEXEC_EXECUTABLE "/usr/bin/srun" CACHE PATH "") + +#------------------------------------------------------------------------------ +# Hardware +#------------------------------------------------------------------------------ + +set(ENABLE_OPENMP OFF CACHE BOOL "") + +#------------------------------------------------------------------------------ +# TPLs +#------------------------------------------------------------------------------ + +set(TPL_ROOT "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7" CACHE PATH "") + +set(AXOM_DIR "${TPL_ROOT}/axom-0.10.1.1-tkmjd26d72eqk4tts7hwlwognvwuyj5s" CACHE PATH "") + +set(CAMP_DIR "${TPL_ROOT}/camp-2024.07.0-afpyaxeo4ulr2lkljpy2zr7rjeq64cox" CACHE PATH "") + +set(CONDUIT_DIR "${TPL_ROOT}/conduit-0.9.2-t2zobt76hkwap7d2ppddp3traorxpffl" CACHE PATH "") + +set(LUA_DIR "/usr" CACHE PATH "") + +set(MFEM_DIR "${TPL_ROOT}/mfem-4.8.0.1-gbv5tbeosslcqh2zhkcrgkwtqxob257j" CACHE PATH "") + +set(HDF5_DIR "${TPL_ROOT}/hdf5-1.8.23-pu3kif4ltqdw2kcwvlswfze6pmmsjs6q" CACHE PATH "") + +set(HYPRE_DIR "${TPL_ROOT}/hypre-2.26.0-ta6n6j6ziyuw2tfejt27nkdulvjq27hu" CACHE PATH "") + +set(METIS_DIR "${TPL_ROOT}/metis-5.1.0-ebbueyypx5bn5mekybxavligkykhm5fy" CACHE PATH "") + +set(PARMETIS_DIR "${TPL_ROOT}/parmetis-4.0.3-morhecieft72ri6gi2r6hsatzrgdhe5n" CACHE PATH "") + +set(NETCDF_DIR "${TPL_ROOT}/netcdf-c-4.7.4-62ri63jaeyajwrrvwk75dbwvrnj6fony" CACHE PATH "") + +set(SUPERLUDIST_DIR "${TPL_ROOT}/superlu-dist-8.1.2-k53l7hvjwbqnepeb7etlujfgzrzfliua" CACHE PATH "") + +set(ARPACK_DIR "${TPL_ROOT}/arpack-ng-3.9.0-ahgmmge3jagsx3n72wbxtcznbubcclca" CACHE PATH "") + +set(ADIAK_DIR "${TPL_ROOT}/adiak-0.4.1-5wjrv4xwbm3y7yh7745ijc45sbcwqwqi" CACHE PATH "") + +# AMGX not built + +set(CALIPER_DIR "${TPL_ROOT}/caliper-2.12.1-a4chnmcpoktn7vlyrjxbw2qskkkytkj3" CACHE PATH "") + +set(PETSC_DIR "${TPL_ROOT}/petsc-3.21.6-elp7iw6xe64daspgy5ctubfo3jj4nd43" CACHE PATH "") + +set(RAJA_DIR "${TPL_ROOT}/raja-2024.07.0-t5ay2skmkw3ybywrq33tcyl5xbn5g3gv" CACHE PATH "") + +set(SLEPC_DIR "${TPL_ROOT}/slepc-3.21.2-rji2oyfq6x4lyovnrkfqnleoc3yrqrm6" CACHE PATH "") + +set(STRUMPACK_DIR "${TPL_ROOT}/strumpack-8.0.0-zmfwdjw54im5o6m3gqssnkvuagdzljwe" CACHE PATH "") + +set(SUNDIALS_DIR "${TPL_ROOT}/sundials-6.7.0-wwas67vc5xfgayl2uimersm3xszlqv62" CACHE PATH "") + +set(UMPIRE_DIR "${TPL_ROOT}/umpire-2024.07.0-47yl3s6ob3ru4g6qnma4tiv6ob5fr6p6" CACHE PATH "") + +set(TRIBOL_DIR "${TPL_ROOT}/tribol-0.1.0.18-my4bcnwnewxcnewg3xgbzvym64dfizuj" CACHE PATH "") + +set(ENZYME_DIR "/usr/WS2/smithdev/toss_4_x86_64_ib/enzyme/llvm_19.1.7/0.0.180" CACHE PATH "") + +#------------------------------------------------------------------------------ +# Devtools +#------------------------------------------------------------------------------ + +set(DEVTOOLS_ROOT "/usr/WS2/smithdev/devtools/toss_4_x86_64_ib/2024_05_30_15_09_55/._view/rkqkran3ydsuprr2wip5pdnz5wh7xwnr" CACHE PATH "") + +set(ATS_EXECUTABLE "${DEVTOOLS_ROOT}/python-3.11.7/bin/ats" CACHE PATH "") + +set(CLANGFORMAT_EXECUTABLE "/usr/tce/packages/clang/clang-14.0.6/bin/clang-format" CACHE PATH "") + +set(CLANGTIDY_EXECUTABLE "/usr/tce/packages/clang/clang-14.0.6/bin/clang-tidy" CACHE PATH "") + +set(ENABLE_DOCS ON CACHE BOOL "") + +set(SPHINX_EXECUTABLE "${DEVTOOLS_ROOT}/python-3.11.7/bin/sphinx-build" CACHE PATH "") + +set(CPPCHECK_EXECUTABLE "${DEVTOOLS_ROOT}/cppcheck-2.9/bin/cppcheck" CACHE PATH "") + +set(DOXYGEN_EXECUTABLE "${DEVTOOLS_ROOT}/doxygen-1.9.8/bin/doxygen" CACHE PATH "") + +set(SERAC_ENABLE_CODEVELOP ON CACHE BOOL "") + +set(SERAC_DISABLE_TRIBOL ON CACHE BOOL "") + diff --git a/host-configs/dfem/tioga-toss_4_x86_64_ib_cray-rocmcc@6.2.1_hip_dfem.cmake b/host-configs/dfem/tioga-toss_4_x86_64_ib_cray-rocmcc@6.2.1_hip_dfem.cmake new file mode 100644 index 000000000..4a75768ea --- /dev/null +++ b/host-configs/dfem/tioga-toss_4_x86_64_ib_cray-rocmcc@6.2.1_hip_dfem.cmake @@ -0,0 +1,149 @@ +#------------------------------------------------------------------------------ +# !!!! This is a generated file, edit at own risk !!!! +#------------------------------------------------------------------------------ +# CMake executable path: /usr/tce/bin/cmake +#------------------------------------------------------------------------------ + +set(CMAKE_PREFIX_PATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/caliper-2.12.1-h2elmogt2mtorvunf67jrcxqdcrzjjem;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/tribol-0.1.0.18-neize4eyd3wxmeuui44az2x5ecqfe2pd;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/adiak-0.4.1-djvwhzv7uucbw342y2lxn6mjv4onnzeh;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/libunwind-1.8.1-667lulihqw77lixyhoyezlqadqmw3ufq;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/axom-0.10.1.1-4udyvzqnaoic74uv3zcumyguv5mbqqcg;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/conduit-0.9.2-wo23sn3yekayvxjphe2eu4vvmvkqrovj;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/mfem-4.8.0.1-s6vr55m7kq5gylxswlaqy5usltbrslzd;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/raja-2024.07.0-m322mdffprdfso4vg7iox3n5goelhtyp;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/umpire-2024.07.0-h4aizqln4js5uo6o5dnkkjadcq5apw37;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/netcdf-c-4.7.4-m3moxxpcdu3id35a6ib662pmcpe6hlxt;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/strumpack-8.0.0-w5kwnpidy7gzm4q2kplzw7ip2hebkojg;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/sundials-6.7.0-ihjcn5jozjo6hqxvueljdahsvpg4fpyr;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/superlu-dist-8.1.2-ulxns2zfpqyuezhrgxjh7do45ezer2m5;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/camp-2024.07.0-223y52kngyoyilxr7gz3i4pe5btnm5uq;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/fmt-11.0.2-haismsyfch472jneh7tj5hoz5faug2wp;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/hdf5-1.8.23-zhqxrmbx5hqezzazejozk5l7wuz3ijx6;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/netlib-scalapack-2.2.2-zinikcerv4cwfrzt4kbhby6xm76fjmvq;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/hypre-2.26.0-sffmaz74r5zmnzir6l73deslw36yypsh;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/parmetis-4.0.3-7o2cbbpbawncpn24yftcj24gbopz4aev;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/metis-5.1.0-q4s4zr5oj2vsyc4eqjy22wrwndgb4fnh;/usr/tce;/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1/llvm;/opt/rocm-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1" CACHE STRING "") + +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH "ON" CACHE STRING "") + +set(CMAKE_BUILD_RPATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/serac-develop-og4fmn5dkubewiybduaji237suiuzssb/lib;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/serac-develop-og4fmn5dkubewiybduaji237suiuzssb/lib64;;/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12" CACHE STRING "") + +set(CMAKE_INSTALL_RPATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/serac-develop-og4fmn5dkubewiybduaji237suiuzssb/lib;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/serac-develop-og4fmn5dkubewiybduaji237suiuzssb/lib64;;/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12" CACHE STRING "") + +set(CMAKE_BUILD_TYPE "Release" CACHE STRING "") + +#------------------------------------------------------------------------------ +# Compilers +#------------------------------------------------------------------------------ +# Compiler Spec: rocmcc@=6.2.1 +#------------------------------------------------------------------------------ +if(DEFINED ENV{SPACK_CC}) + + set(CMAKE_C_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/spack/lib/spack/env/rocmcc/amdclang" CACHE PATH "") + + set(CMAKE_CXX_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/spack/lib/spack/env/rocmcc/amdclang++" CACHE PATH "") + + set(CMAKE_Fortran_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/spack/lib/spack/env/rocmcc/amdflang" CACHE PATH "") + +else() + + set(CMAKE_C_COMPILER "/opt/rocm-6.2.1/llvm/bin/amdclang" CACHE PATH "") + + set(CMAKE_CXX_COMPILER "/opt/rocm-6.2.1/llvm/bin/amdclang++" CACHE PATH "") + + set(CMAKE_Fortran_COMPILER "/opt/rocm-6.2.1/llvm/bin/amdflang" CACHE PATH "") + +endif() + +#------------------------------------------------------------------------------ +# MPI +#------------------------------------------------------------------------------ + +set(MPI_C_COMPILER "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.2.1/bin/mpicc" CACHE PATH "") + +set(MPI_CXX_COMPILER "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.2.1/bin/mpicxx" CACHE PATH "") + +set(MPI_Fortran_COMPILER "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.2.1/bin/mpif90" CACHE PATH "") + +set(MPIEXEC_NUMPROC_FLAG "-n" CACHE STRING "") + +set(ENABLE_MPI ON CACHE BOOL "") + +set(MPIEXEC_EXECUTABLE "/usr/global/tools/flux_wrappers/bin/srun" CACHE PATH "") + +#------------------------------------------------------------------------------ +# Hardware +#------------------------------------------------------------------------------ + +#------------------------------------------------ +# ROCm +#------------------------------------------------ + +set(HIP_ROOT_DIR "/opt/rocm-6.2.1" CACHE PATH "") + +set(CMAKE_HIP_COMPILER "/opt/rocm-6.2.1/llvm/bin/clang++" CACHE FILEPATH "") + +set(CMAKE_HIP_ARCHITECTURES "gfx90a" CACHE STRING "") + +set(AMDGPU_TARGETS "gfx90a" CACHE STRING "") + +set(GPU_TARGETS "gfx90a" CACHE STRING "") + +set(ENABLE_OPENMP OFF CACHE BOOL "") + +set(ENABLE_HIP ON CACHE BOOL "") + +set(ROCM_PATH "/opt/rocm-6.2.1" CACHE PATH "") + +set(CMAKE_EXE_LINKER_FLAGS "-L/opt/rocm-6.2.1/lib -Wl,-rpath,/opt/rocm-6.2.1/lib -L/opt/rocm-6.2.1/llvm/lib -Wl,-rpath,/opt/rocm-6.2.1/llvm/lib -lxpmem -L/opt/cray/pe/mpich/8.1.29/gtl/lib -Wl,-rpath,/opt/cray/pe/mpich/8.1.29/gtl/lib -lmpi_gtl_hsa -Wl,--disable-new-dtags -lflang -lflangrti -lamdhip64 -lhsakmt -lhsa-runtime64 -lamd_comgr -lpgmath -lhipblas" CACHE STRING "") + +#------------------------------------------------------------------------------ +# TPLs +#------------------------------------------------------------------------------ + +set(TPL_ROOT "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1" CACHE PATH "") + +set(AXOM_DIR "${TPL_ROOT}/axom-0.10.1.1-4udyvzqnaoic74uv3zcumyguv5mbqqcg" CACHE PATH "") + +set(CAMP_DIR "${TPL_ROOT}/camp-2024.07.0-223y52kngyoyilxr7gz3i4pe5btnm5uq" CACHE PATH "") + +set(CONDUIT_DIR "${TPL_ROOT}/conduit-0.9.2-wo23sn3yekayvxjphe2eu4vvmvkqrovj" CACHE PATH "") + +set(LUA_DIR "/usr" CACHE PATH "") + +set(MFEM_DIR "${TPL_ROOT}/mfem-4.8.0.1-s6vr55m7kq5gylxswlaqy5usltbrslzd" CACHE PATH "") + +set(HDF5_DIR "${TPL_ROOT}/hdf5-1.8.23-zhqxrmbx5hqezzazejozk5l7wuz3ijx6" CACHE PATH "") + +set(HYPRE_DIR "${TPL_ROOT}/hypre-2.26.0-sffmaz74r5zmnzir6l73deslw36yypsh" CACHE PATH "") + +set(METIS_DIR "${TPL_ROOT}/metis-5.1.0-q4s4zr5oj2vsyc4eqjy22wrwndgb4fnh" CACHE PATH "") + +set(PARMETIS_DIR "${TPL_ROOT}/parmetis-4.0.3-7o2cbbpbawncpn24yftcj24gbopz4aev" CACHE PATH "") + +set(NETCDF_DIR "${TPL_ROOT}/netcdf-c-4.7.4-m3moxxpcdu3id35a6ib662pmcpe6hlxt" CACHE PATH "") + +set(SUPERLUDIST_DIR "${TPL_ROOT}/superlu-dist-8.1.2-ulxns2zfpqyuezhrgxjh7do45ezer2m5" CACHE PATH "") + +set(ADIAK_DIR "${TPL_ROOT}/adiak-0.4.1-djvwhzv7uucbw342y2lxn6mjv4onnzeh" CACHE PATH "") + +# AMGX not built + +set(CALIPER_DIR "${TPL_ROOT}/caliper-2.12.1-h2elmogt2mtorvunf67jrcxqdcrzjjem" CACHE PATH "") + +# PETSC not built + +set(RAJA_DIR "${TPL_ROOT}/raja-2024.07.0-m322mdffprdfso4vg7iox3n5goelhtyp" CACHE PATH "") + +# SLEPC not built + +set(STRUMPACK_DIR "${TPL_ROOT}/strumpack-8.0.0-w5kwnpidy7gzm4q2kplzw7ip2hebkojg" CACHE PATH "") + +set(SUNDIALS_DIR "${TPL_ROOT}/sundials-6.7.0-ihjcn5jozjo6hqxvueljdahsvpg4fpyr" CACHE PATH "") + +set(UMPIRE_DIR "${TPL_ROOT}/umpire-2024.07.0-h4aizqln4js5uo6o5dnkkjadcq5apw37" CACHE PATH "") + +set(TRIBOL_DIR "${TPL_ROOT}/tribol-0.1.0.18-neize4eyd3wxmeuui44az2x5ecqfe2pd" CACHE PATH "") + +set(ENZYME_DIR "/usr/WS2/smithdev/toss_4_x86_64_ib_cray/enzyme/rocm-6.2.1/0.0.180" CACHE PATH "") + +#------------------------------------------------------------------------------ +# Devtools +#------------------------------------------------------------------------------ + +# Code checks disabled due to disabled devtools + +set(SERAC_ENABLE_CODE_CHECKS OFF CACHE BOOL "") + +set(ENABLE_CLANGFORMAT OFF CACHE BOOL "") + +set(ENABLE_CLANGTIDY OFF CACHE BOOL "") + +set(ENABLE_DOCS OFF CACHE BOOL "") + +set(SERAC_ENABLE_CODEVELOP ON CACHE BOOL "") + +set(SERAC_DISABLE_TRIBOL ON CACHE BOOL "") + diff --git a/mfem b/mfem index d9c1c34fd..3e3a5e5d1 160000 --- a/mfem +++ b/mfem @@ -1 +1 @@ -Subproject commit d9c1c34fdfaf3f7a9f56dfc82f7c083082a36fca +Subproject commit 3e3a5e5d14003d7856f779292fe032067c10fc1b diff --git a/scripts/spack/configs/toss_4_x86_64_ib/spack.yaml b/scripts/spack/configs/toss_4_x86_64_ib/spack.yaml index a0e1abaed..1ca277e7c 100644 --- a/scripts/spack/configs/toss_4_x86_64_ib/spack.yaml +++ b/scripts/spack/configs/toss_4_x86_64_ib/spack.yaml @@ -62,6 +62,23 @@ spack: fc: /usr/tce/packages/gcc/gcc-10.3.1/bin/gfortran spec: clang@16.0.6 target: x86_64 + - compiler: + environment: {} + extra_rpaths: [] + flags: + cflags: --gcc-toolchain=/usr/tce/packages/gcc/gcc-10.3.1 + cxxflags: --gcc-toolchain=/usr/tce/packages/gcc/gcc-10.3.1 + fflags: -fPIE -fPIC + modules: + - clang/19.1.7 + operating_system: rhel8 + paths: + cc: /usr/WS2/smithdev/toss_4_x86_64_ib/llvm/19.1.7/bin/clang + cxx: /usr/WS2/smithdev/toss_4_x86_64_ib/llvm/19.1.7/bin/clang++ + f77: /usr/tce/packages/gcc/gcc-10.3.1/bin/gfortran + fc: /usr/tce/packages/gcc/gcc-10.3.1/bin/gfortran + spec: clang@19.1.7 + target: x86_64 packages: all: diff --git a/scripts/spack/specs.json b/scripts/spack/specs.json index c128493d4..c5a1fe6a0 100644 --- a/scripts/spack/specs.json +++ b/scripts/spack/specs.json @@ -16,7 +16,8 @@ "toss_4_x86_64_ib": [ "gcc@10.3.1+devtools+profiling", "clang@14.0.6+devtools+profiling", - "clang@16.0.6+devtools+profiling~openmp" ], + "clang@16.0.6+devtools+profiling~openmp", + "clang@19.1.7+devtools+profiling~openmp" ], "toss_4_x86_64_ib_cray": [ "rocmcc@6.2.1~openmp+profiling+rocm+raja+umpire+strumpack~petsc amdgpu_target=gfx90a ^hip@6.2.1 ^rocprim@6.2.1 ^hsa-rocr-dev@6.2.1 ^llvm-amdgpu@6.2.1 ^hdf5 cflags=-Wno-int-conversion"], diff --git a/src/serac/physics/contact/contact_data.hpp b/src/serac/physics/contact/contact_data.hpp index 01335cc92..2bd963d1d 100644 --- a/src/serac/physics/contact/contact_data.hpp +++ b/src/serac/physics/contact/contact_data.hpp @@ -247,6 +247,7 @@ class ContactData { */ int num_pressure_dofs_; +#ifdef SERAC_USE_TRIBOL /** * @brief Tracks whether the Jacobian and pressure offsets need to be updated * @@ -254,6 +255,7 @@ class ContactData { * */ mutable bool offsets_up_to_date_; +#endif /** * @brief Offsets giving size of each block Jacobian contribution @@ -284,9 +286,11 @@ class ContactData { */ mutable mfem::Array global_pressure_dof_offsets_; +#ifdef SERAC_USE_TRIBOL int cycle_{0}; double time_{0.0}; double dt_{1.0}; +#endif }; } // namespace serac