Merge branch 'develop' into patch-2

.gitlab-ci.yml

@@ -50,6 +50,10 @@
 # ON_QUARTZ:
 # Should the Quartz pipeline run? Set to  "ON" or "OFF" to enable/disable.
 #
+# SPACK_PREFIX: prefix used for shared spack installation.
+# Usually this a spack version number that matches the version set in the uberenv_config.json file. 
+# Spack installs go in /usr/workspace/sundials/spack_installs/${SPACK_PREFIX}/$(hostname).
+#
 # SHARED_SPACK:
 # If "ON", then a shared spack install that has been pre-configured is utilized.
 # If "OFF", then a new spack instance is created for every build (meaning all TPLs have to be installed).
@@ -75,6 +79,8 @@ variables:
   VERBOSE_TEST: "OFF"
   ON_LASSEN: "ON"
   ON_QUARTZ: "ON"
+  ON_CORONA: "ON"
+  SPACK_PREFIX: "v0.19.1"
   SHARED_SPACK: "UPSTREAM"
   BENCHMARK: "OFF"
   BENCHMARK_NNODES: 4
@@ -89,7 +95,7 @@ stages:
   - q_build_and_test
   - l_build_and_test
   - l_build_and_bench
-  # - c_build_and_test
+  - c_build_and_test
 
 # These are also templates (.name) that define project specific build commands.
 # If an allocation exist with the name defined in this pipeline, the job will
@@ -101,9 +107,10 @@ stages:
       --job-name=${ALLOC_NAME} .gitlab/build_and_test.sh
 
 # Corona
-.build_toss_3_x86_64_ib_corona_script:
+.build_toss_4_x86_64_ib_corona_script:
   script:
-    - srun -p mi60 --interactive -t ${DEFAULT_TIME} -N 1 .gitlab/build_and_test.sh
+    - echo ${ALLOC_NAME}
+    - flux alloc -N 1 -t ${DEFAULT_TIME}  .gitlab/build_and_test.sh
 
 # CORAL systems use spectrum LSF instead of SLURM
 .build_blueos_3_ppc64le_ib_script:
@@ -123,3 +130,5 @@ include:
   - local: .gitlab/quartz-jobs.yml
   - local: .gitlab/lassen-templates.yml
   - local: .gitlab/lassen-jobs.yml
+  - local: .gitlab/corona-templates.yml
+  - local: .gitlab/corona-jobs.yml

.gitlab/build_and_test.sh

@@ -20,6 +20,7 @@ job_unique_id=${CI_JOB_ID:-""}
 sys_type=${SYS_TYPE:-""}
 py_env_path=${PYTHON_ENVIRONMENT_PATH:-""}
 
+spack_prefix=${SHARED_SPACK_PREFIX:-"v0.19.1"}
 shared_spack=${SHARED_SPACK:-"UPSTREAM"}
 
 # Dependencies
@@ -46,8 +47,16 @@ hostname=${hostname%%[0-9]*}
 BUILD_JOBS=${BUILD_JOBS:-"1"}
 
 # load newer python to try the clingo concretizer
-echo "module load python/3.8.2"
-module load python/3.8.2
+# machine specific loads
+if [[ "${hostname}" == "corona" ]]; then
+    echo "module load python/3.9.12"
+    module load python/3.9.12
+    echo "module load rocm/5.4.1"
+    module load rocm/5.4.1
+else
+    echo "module load python/3.8.2"
+    module load python/3.8.2
+fi
 
 if [[ "${option}" != "--build-only" && "${option}" != "--test-only" ]]
 then
@@ -89,7 +98,7 @@ then
 
     if [[ -d /usr/workspace/sundials ]]
     then
-        upstream="/usr/workspace/sundials/spack_installs/${hostname}"
+        upstream="/usr/workspace/sundials/spack_installs/${spack_prefix}/${hostname}"
         mkdir -p "${upstream}"
         upstream_opt="--upstream=${upstream}"
     fi
@@ -174,7 +183,7 @@ then
     mkdir -p "${build_dir}" && cd "${build_dir}"
 
     date
-    
+
     $cmake_exe --version
 
     # configure

.gitlab/corona-jobs.yml

@@ -0,0 +1,37 @@
+# ------------------------------------------------------------------------------
+# SUNDIALS Copyright Start
+# Copyright (c) 2002-2021, Lawrence Livermore National Security
+# and Southern Methodist University.
+# All rights reserved.
+#
+# See the top-level LICENSE and NOTICE files for details.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+# SUNDIALS Copyright End
+# ------------------------------------------------------------------------------
+
+# ------------------------------------------------------------------------------
+# HIP
+# ------------------------------------------------------------------------------
+
+# Builds with Hip
+corona_clang_hip:
+  parallel:
+    matrix:
+      - COMPILER_SPEC: [email protected]
+        AMDGPU_TARGET: [gfx906]
+  variables:
+    SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double amdgpu_target=${AMDGPU_TARGET} scheduler=flux +rocm+mpi"
+  extends: .corona_build_and_test
+
+# ------------------------------------------------------------------------------
+# HIP + TPLs
+# ------------------------------------------------------------------------------
+corona_clang_hip_tpls:
+  parallel:
+    matrix:
+      - COMPILER_SPEC: [email protected]
+        AMDGPU_TARGET: [gfx906]
+  variables:
+    SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 amdgpu_target=${AMDGPU_TARGET} scheduler=flux +rocm+mpi+magma+raja ^magma+rocm amdgpu_target=${AMDGPU_TARGET} ^raja+rocm~openmp~examples~exercises amdgpu_target=${AMDGPU_TARGET}"
+  extends: .corona_build_and_test

.gitlab/corona-templates.yml

@@ -0,0 +1,35 @@
+# ------------------------------------------------------------------------------
+# SUNDIALS Copyright Start
+# Copyright (c) 2002-2021, Lawrence Livermore National Security
+# and Southern Methodist University.
+# All rights reserved.
+#
+# See the top-level LICENSE and NOTICE files for details.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+# SUNDIALS Copyright End
+# ------------------------------------------------------------------------------
+
+# ------------------------------------------------------------------------------
+# Tags and rules to run tests on Corona
+# ------------------------------------------------------------------------------
+
+# Generic Corona build job, extending build script for Toss 4 x86_64 Systems
+.corona_build_and_test:
+  tags:
+    - shell
+    - corona
+  extends: [.build_toss_4_x86_64_ib_corona_script]
+  stage: c_build_and_test
+  needs: []
+  artifacts:
+    paths:
+      - spack-*.txt
+      - build_*/*
+    when: always
+  rules:
+    # Don't run if...
+    - if: '$CI_COMMIT_BRANCH =~ /_cnone/ || $ON_CORONA == "OFF" || $BENCHMARK == "ON"'
+      when: never
+    # Default is to run if previous stage succeeded
+    - when: on_success

.gitlab/lassen-jobs.yml

@@ -78,7 +78,7 @@ lassen_gcc_cuda_tpls:
       - COMPILER_SPEC: [email protected]
         CUDA_SPEC: [[email protected]]
   variables:
-    SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda+raja+magma+superlu-dist cuda_arch=70 ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^${CUDA_SPEC}"
+    SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda+raja+magma+superlu-dist+petsc+hypre+ginkgo cuda_arch=70 ^ginkgo+cuda cuda_arch=70 ^hypre ^petsc+cuda cuda_arch=70 ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^${CUDA_SPEC}"
   extends: .lassen_build_and_test
 
 # ------------------------------------------------------------------------------

.gitlab/quartz-jobs.yml

@@ -55,11 +55,11 @@
 quartz_clang_tpls:
   parallel:
     matrix:
-      - COMPILER_SPEC: [email protected].0
+      - COMPILER_SPEC: [email protected].1
         INDEX_SPEC: [~int64]
         PRECISION_SPEC: [double]
   variables:
-    SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu"
+    SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu +petsc ^[email protected] ^openblas"
   extends: .quartz_build_and_test
 
 quartz_gcc_tpls:
@@ -69,7 +69,7 @@ quartz_gcc_tpls:
         INDEX_SPEC: [~int64]
         PRECISION_SPEC: [double]
   variables:
-    SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu"
+    SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu +petsc ^[email protected]"
   extends: .quartz_build_and_test
 
 quartz_intel_tpls:
@@ -79,5 +79,5 @@ quartz_intel_tpls:
         INDEX_SPEC: [~int64]
         PRECISION_SPEC: [double]
   variables:
-    SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre ~superlu-dist +lapack +klu"
+    SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre ~superlu-dist +lapack +klu ^[email protected]"
   extends: .quartz_build_and_test

.gitlab/spack_packages/netlib-lapack/ibm-xl-3.9.1.patch

@@ -0,0 +1,108 @@
+Fixes for IBM XL and Cray CCE builds:
+
+* Correct path to the fallback configuration used to handle mangling for
+  C++/Fortran compatibility (CCE, XL)
+
+* Change logic for detecting recursive fortran flags to (a) Include XL
+(qrecur), and (b) Be explicit, since not every compiler will correctly reject
+an incorrect option (ALL)
+
+NOTE: This patch has been accepted upstream
+(see https://github.com/Reference-LAPACK/lapack/pull/621)
+
+##############################################################################
+
+diff -Naur a/CBLAS/CMakeLists.txt b/CBLAS/CMakeLists.txt
+--- a/CBLAS/CMakeLists.txt	2021-03-25 12:25:15.000000000 -0600
++++ b/CBLAS/CMakeLists.txt	2021-09-01 16:27:23.561355382 -0600
+@@ -11,9 +11,7 @@
+                          MACRO_NAMESPACE "F77_"
+                          SYMBOL_NAMESPACE "F77_")
+ if(NOT FortranCInterface_GLOBAL_FOUND OR NOT FortranCInterface_MODULE_FOUND)
+-  message(WARNING "Reverting to pre-defined include/lapacke_mangling.h")
+-    configure_file(include/lapacke_mangling_with_flags.h.in
+-                  ${LAPACK_BINARY_DIR}/include/lapacke_mangling.h)
++  message(WARNING "Reverting to pre-defined include/cblas_mangling.h")
+     configure_file(include/cblas_mangling_with_flags.h.in
+                  ${LAPACK_BINARY_DIR}/include/cblas_mangling.h)
+ endif()
+diff -Naur a/CMakeLists.txt b/CMakeLists.txt
+--- a/CMakeLists.txt	2021-03-25 12:25:15.000000000 -0600
++++ b/CMakeLists.txt	2021-09-02 09:49:18.070436958 -0600
+@@ -94,16 +94,22 @@
+
+ # Check if recursive flag exists
+ include(CheckFortranCompilerFlag)
+-check_fortran_compiler_flag("-recursive" _recursiveFlag)
+-check_fortran_compiler_flag("-frecursive" _frecursiveFlag)
+-check_fortran_compiler_flag("-Mrecursive" _MrecursiveFlag)
++if(CMAKE_Fortran_COMPILER_ID STREQUAL Flang)
++  check_fortran_compiler_flag("-Mrecursive" _MrecursiveFlag)
++elseif(CMAKE_Fortran_COMPILER_ID STREQUAL GNU)
++  check_fortran_compiler_flag("-frecursive" _frecursiveFlag)
++elseif(CMAKE_Fortran_COMPILER_ID STREQUAL Intel)
++  check_fortran_compiler_flag("-recursive" _recursiveFlag)
++elseif(CMAKE_Fortran_COMPILER_ID STREQUAL XL)
++  check_fortran_compiler_flag("-qrecur" _qrecurFlag)
++endif()
+
+ # Add recursive flag
+-if(_recursiveFlag)
+-  string(REGEX MATCH "-recursive" output_test <string> "${CMAKE_Fortran_FLAGS}")
++if(_MrecursiveFlag)
++  string(REGEX MATCH "-Mrecursive" output_test <string> "${CMAKE_Fortran_FLAGS}")
+   if(NOT output_test)
+-    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -recursive"
+-      CACHE STRING "Recursive flag must be set" FORCE)
++    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -Mrecursive"
++    CACHE STRING "Recursive flag must be set" FORCE)
+   endif()
+ elseif(_frecursiveFlag)
+   string(REGEX MATCH "-frecursive" output_test <string> "${CMAKE_Fortran_FLAGS}")
+@@ -111,11 +117,17 @@
+     set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -frecursive"
+     CACHE STRING "Recursive flag must be set" FORCE)
+   endif()
+-elseif(_MrecursiveFlag)
+-  string(REGEX MATCH "-Mrecursive" output_test <string> "${CMAKE_Fortran_FLAGS}")
++elseif(_recursiveFlag)
++  string(REGEX MATCH "-recursive" output_test <string> "${CMAKE_Fortran_FLAGS}")
+   if(NOT output_test)
+-    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -Mrecursive"
+-    CACHE STRING "Recursive flag must be set" FORCE)
++    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -recursive"
++      CACHE STRING "Recursive flag must be set" FORCE)
++  endif()
++elseif(_qrecurFlag)
++  string(REGEX MATCH "-qrecur" output_test <string> "${CMAKE_Fortran_FLAGS}")
++  if(NOT output_test)
++    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qrecur"
++      CACHE STRING "Recursive flag must be set" FORCE)
+   endif()
+ endif()
+
+@@ -124,7 +136,7 @@
+     set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fp-model strict")
+   endif()
+   if(CMAKE_Fortran_COMPILER_ID STREQUAL XL)
+-    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qnosave -qstrict=none")
++    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qnosave -qstrict")
+   endif()
+ # Delete libmtsk in linking sequence for Sun/Oracle Fortran Compiler.
+ # This library is not present in the Sun package SolarisStudio12.3-linux-x86-bin
+diff -Naur a/INSTALL/make.inc.XLF b/INSTALL/make.inc.XLF
+--- a/INSTALL/make.inc.XLF	2021-03-25 12:25:15.000000000 -0600
++++ b/INSTALL/make.inc.XLF	2021-09-02 09:50:02.664646455 -0600
+@@ -14,10 +14,10 @@
+ #  the compiler options desired when NO OPTIMIZATION is selected.
+ #
+ FC = xlf
+-FFLAGS = -O3 -qfixed -qnosave
++FFLAGS = -O3 -qfixed -qnosave -qrecur
+ # For -O2, add -qstrict=none
+ FFLAGS_DRV = $(FFLAGS)
+-FFLAGS_NOOPT = -O0 -qfixed -qnosave
++FFLAGS_NOOPT = -O0 -qfixed -qnosave -qrecur
+
+ #  Define LDFLAGS to the desired linker options for your machine.
+ #