Merge remote-tracking branch 'origin/master' into docstring-fixes

eth-cscs · Sep 7, 2023 · ba566e3 · ba566e3
2 parents 1a6cede + 52704e9
commit ba566e3
Show file tree

Hide file tree

Showing 133 changed files with 4,286 additions and 1,135 deletions.
diff --git a/.github/workflows/check_license_header.yaml b/.github/workflows/check_license_header.yaml
@@ -32,6 +32,7 @@ jobs:
                    -o  -name '*.h.in'                                             \
                    -o  -name '*.tpp'                                              \
                    -o  -name '*.cu'                                               \
+                   -o  -name '*.c'                                                \
                    -o  -name 'CMakeLists.txt'                                     \
                    -o  -name '.*'                                                 \
                ')'                                                                \
@@ -54,6 +55,7 @@ jobs:
                    -o  -name '*.h.in'                                             \
                    -o  -name '*.tpp'                                              \
                    -o  -name '*.cu'                                               \
+                   -o  -name '*.c'                                                \
                ')'                                                                \
             | xargs -I{} sh -c                                                    \
                 ".github/check_license.sh misc/HEADER {} > /dev/null || echo {}"  \

diff --git a/.gitignore b/.gitignore
@@ -45,7 +45,6 @@
 *.out
 *.app
 
-
 # ------------------- CMake
 
 CMakeLists.txt.user

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -46,6 +46,7 @@ option(DLAF_BUILD_TESTING "Build tests" ON)
 option(DLAF_BUILD_TESTING_HEADER "Build header tests" OFF)
 option(DLAF_BUILD_DOC "Build documentation" OFF)
 option(DLAF_WITH_PRECOMPILED_HEADERS "Use precompiled headers." OFF)
+option(DLAF_WITH_SCALAPACK "Build ScaLAPACK-like C API (requires ScaLAPACK)" OFF)
 
 if(DLAF_WITH_MKL)
   # When using MKL there is no need to set the number of threads with
@@ -148,8 +149,25 @@ if(DLAF_WITH_MKL)
   else()
     message(FATAL_ERROR "${MKL_LAPACK_TARGET} as LAPACK target has not been found")
   endif()
+
+  if(DLAF_WITH_SCALAPACK)
+    set(MKL_SCALAPACK_TARGET "mkl::scalapack_intel_32bit_seq_dyn"
+        CACHE STRING "MKL ScaLAPACK target (see FindMKL for details)"
+    )
+
+    if(TARGET ${MKL_SCALAPACK_TARGET})
+      set(SCALAPACK_FOUND TRUE)
+      add_library(DLAF::SCALAPACK INTERFACE IMPORTED GLOBAL)
+      target_link_libraries(DLAF::SCALAPACK INTERFACE ${MKL_SCALAPACK_TARGET})
+    else()
+      message(FATAL_ERROR "${MKL_SCALAPACK_TARGET} as ScaLAPACK target has not been found")
+    endif()
+  endif()
 else()
   find_package(LAPACK REQUIRED)
+  if(DLAF_WITH_SCALAPACK)
+    find_package(SCALAPACK REQUIRED)
+  endif()
 endif()
 
 # ----- pika

diff --git a/README.md b/README.md
@@ -101,6 +101,7 @@ target_link_libraries(<your_target> PRIVATE DLAF::DLAF)
 ### Documentation
 
 - [Documentation of `v0.1.0`](https://eth-cscs.github.io/DLA-Future/v0.1.0/)
+- [Documentation of `v0.2.0`](https://eth-cscs.github.io/DLA-Future/v0.2.0/)
 - [Documentation of `master` branch](https://eth-cscs.github.io/DLA-Future/master/)
 
 #### How to generate the documentation

diff --git a/ci/.gitlab-ci.yml b/ci/.gitlab-ci.yml
@@ -223,6 +223,28 @@ cuda release build gcc11:
     THREADS_PER_NODE: 24
     USE_CODECOV: "false"
 
+cuda release build gcc11 scalapack:
+  extends: .build_spack_common
+  variables:
+    BUILD_DOCKER_FILE: ci/docker/build.Dockerfile
+    DEPLOY_DOCKER_FILE: ci/docker/deploy.Dockerfile
+    BASE_IMAGE: docker.io/nvidia/cuda:11.7.1-devel-ubuntu22.04
+    DEPLOY_BASE_IMAGE: docker.io/ubuntu:22.04
+    EXTRA_APTGET: ""
+    EXTRA_APTGET_DEPLOY: "glibc-tools"
+    # glibc-tools is needed for libSegFault on ubuntu:22.04
+    COMPILER: [email protected]
+    CXXSTD: 17
+    USE_MKL: "ON"
+    USE_ROCBLAS: "OFF"
+    SPACK_ENVIRONMENT: ci/docker/release-cuda-scalapack.yaml
+    COMMON_SPACK_ENVIRONMENT: ci/docker/common.yaml
+    BUILD_IMAGE: $CSCS_REGISTRY_PATH/release-cuda-gcc11-scalapack/build
+    DEPLOY_IMAGE: $CSCS_REGISTRY_PATH/release-cuda-gcc11-scalapack/deploy:$CI_COMMIT_SHA
+    SLURM_CONSTRAINT: gpu
+    THREADS_PER_NODE: 24
+    USE_CODECOV: "false"
+
 cuda codecov build gcc11:
   extends: .build_spack_common
   variables:
@@ -245,6 +267,28 @@ cuda codecov build gcc11:
     THREADS_PER_NODE: 24
     USE_CODECOV: "true"
 
+cuda codecov build gcc11 scalapack:
+  extends: .build_spack_common
+  variables:
+    BUILD_DOCKER_FILE: ci/docker/build.Dockerfile
+    DEPLOY_DOCKER_FILE: ci/docker/codecov.Dockerfile
+    BASE_IMAGE: docker.io/nvidia/cuda:11.7.1-devel-ubuntu22.04
+    DEPLOY_BASE_IMAGE: docker.io/ubuntu:22.04
+    EXTRA_APTGET: ""
+    EXTRA_APTGET_DEPLOY: "glibc-tools"
+    # glibc-tools is needed for libSegFault on ubuntu:22.04
+    COMPILER: [email protected]
+    CXXSTD: 17
+    USE_MKL: "OFF"
+    USE_ROCBLAS: "OFF"
+    SPACK_ENVIRONMENT: ci/docker/debug-cuda-scalapack.yaml
+    COMMON_SPACK_ENVIRONMENT: ci/docker/common.yaml
+    BUILD_IMAGE: $CSCS_REGISTRY_PATH/codecov-cuda-gcc11/build
+    DEPLOY_IMAGE: $CSCS_REGISTRY_PATH/codecov-cuda-gcc11/deploy:$CI_COMMIT_SHA
+    SLURM_CONSTRAINT: gpu
+    THREADS_PER_NODE: 24
+    USE_CODECOV: "true"
+
 rocm release build clang14+rocm-5.3.3:
   extends: .build_spack_common
   variables:
@@ -332,6 +376,15 @@ cuda release test gcc11:
       - artifact: pipeline.yml
         job: cuda release build gcc11
 
+cuda release test gcc11 scalapack:
+  extends: .run_common
+  needs:
+    - cuda release build gcc11 scalapack
+  trigger:
+    include:
+      - artifact: pipeline.yml
+        job: cuda release build gcc11 scalapack
+
 cuda codecov test gcc11:
   extends: .run_common
   needs:
@@ -341,3 +394,13 @@ cuda codecov test gcc11:
     include:
       - artifact: pipeline.yml
         job: cuda codecov build gcc11
+
+cuda codecov test gcc11 scalapack:
+  extends: .run_common
+  needs:
+    - cuda codecov build gcc11 scalapack
+  trigger:
+    strategy: depend
+    include:
+      - artifact: pipeline.yml
+        job: cuda codecov build gcc11 scalapack
diff --git a/ci/docker/debug-cuda-scalapack.yaml b/ci/docker/debug-cuda-scalapack.yaml
@@ -0,0 +1,30 @@
+#
+# Distributed Linear Algebra with Future (DLAF)
+#
+# Copyright (c) 2018-2023, ETH Zurich
+# All rights reserved.
+#
+# Please, refer to the LICENSE file in the root directory.
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+spack:
+  include:
+  - common.yaml
+
+  view: false
+  concretizer:
+    unify:
+      true
+
+  specs:
+    - dla-future@master build_type=Debug +cuda +miniapps +scalapack +ci-test ^openblas threads=openmp ^mpich+fortran
+
+  packages:
+    all:
+      variants:
+        - 'build_type=Release'
+    pika:
+      variants:
+        - 'build_type=Debug'
+        - 'malloc=system'
diff --git a/ci/docker/release-cpu-stdexec.yaml b/ci/docker/release-cpu-stdexec.yaml
@@ -18,7 +18,7 @@ spack:
       true
 
   specs:
-    - dla-future@master +miniapps +ci-test +ci-check-threads ^intel-mkl threads=openmp ^mpich ^pika+stdexec ^stdexec@git.7a47a4aa411c1ca9adfcb152c28cc3dd7b156b4d=main
+    - dla-future@master +miniapps +ci-test +ci-check-threads ^intel-mkl threads=openmp ^mpich ^pika+stdexec ^stdexec@git.48c52df0f81c6151eecf4f39fa5eed2dc0216204=main
 
   packages:
     all:

diff --git a/ci/docker/release-cuda-scalapack.yaml b/ci/docker/release-cuda-scalapack.yaml
@@ -0,0 +1,26 @@
+#
+# Distributed Linear Algebra with Future (DLAF)
+#
+# Copyright (c) 2018-2023, ETH Zurich
+# All rights reserved.
+#
+# Please, refer to the LICENSE file in the root directory.
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+spack:
+  include:
+  - common.yaml
+
+  view: false
+  concretizer:
+    unify:
+      true
+
+  specs:
+    - dla-future@master +cuda +miniapps +scalapack +ci-test ^intel-mkl threads=openmp ^mpich
+
+  packages:
+    all:
+      variants:
+        - 'build_type=Release'
diff --git a/cmake/DLAF_AddTargetWarnings.cmake b/cmake/DLAF_AddTargetWarnings.cmake
@@ -21,10 +21,10 @@ macro(target_add_warnings target_name)
     ${target_name}
     PRIVATE -Wall
             -Wextra
-            -Wnon-virtual-dtor
+            $<$<COMPILE_LANGUAGE:CXX>:-Wnon-virtual-dtor>
             -Wunused
             -Wunused-local-typedefs
-            -Woverloaded-virtual
+            $<$<COMPILE_LANGUAGE:CXX>:-Woverloaded-virtual>
             -Wdangling-else
             -Wswitch-enum
             # Conversions

diff --git a/cmake/DLAF_AddTest.cmake b/cmake/DLAF_AddTest.cmake
@@ -88,6 +88,9 @@ function(DLAF_addTest test_target_name)
     set(_gtest_tgt DLAF_gtest_mpipika_main)
     set(IS_AN_MPI_TEST TRUE)
     set(IS_AN_PIKA_TEST TRUE)
+  elseif(DLAF_AT_USE_MAIN STREQUAL CAPI)
+    set(_gtest_tgt DLAF_gtest_mpi_main)
+    set(IS_AN_MPI_TEST TRUE)
   else()
     message(FATAL_ERROR "USE_MAIN=${DLAF_AT_USE_MAIN} is not a supported option")
   endif()
@@ -181,6 +184,36 @@ function(DLAF_addTest test_target_name)
     list(APPEND _TEST_ARGUMENTS ${_PIKA_EXTRA_ARGS_LIST})
   endif()
 
+  # Special treatment for C API tests
+  # C API tests require pika arguments to be hard-coded in the test file
+  if(DLAF_AT_USE_MAIN STREQUAL CAPI)
+    separate_arguments(_PIKA_EXTRA_ARGS_LIST_CAPI UNIX_COMMAND ${DLAF_PIKATEST_EXTRA_ARGS})
+
+    # --pika:bind=none is useful just in case more ranks are going to be allocated on the same node.
+    if((DLAF_AT_MPIRANKS GREATER 1) AND (NOT DLAF_TEST_THREAD_BINDING_ENABLED))
+      _set_element_to_fallback_value(_PIKA_EXTRA_ARGS_LIST_CAPI "--pika:bind" "--pika:bind=none")
+    endif()
+
+    if(IS_AN_MPI_TEST AND DLAF_MPI_PRESET STREQUAL "plain-mpi")
+      math(EXPR _DLAF_PIKA_THREADS "${MPIEXEC_MAX_NUMPROCS}/${DLAF_AT_MPIRANKS}")
+
+      if(_DLAF_PIKA_THREADS LESS 2)
+        set(_DLAF_PIKA_THREADS 2)
+      endif()
+
+      _set_element_to_fallback_value(
+        _PIKA_EXTRA_ARGS_LIST_CAPI "--pika:threads" "--pika:threads=${_DLAF_PIKA_THREADS}"
+      )
+    endif()
+
+    string(REPLACE ";" "\", \"" PIKA_EXTRA_ARGS_LIST_CAPI "${_PIKA_EXTRA_ARGS_LIST_CAPI}")
+
+    configure_file(
+      ${PROJECT_SOURCE_DIR}/test/include/dlaf_c_test/config.h.in ${CMAKE_CURRENT_BINARY_DIR}/config.h
+    )
+
+  endif()
+
   ### Test executable target
   add_executable(${test_target_name} ${DLAF_AT_SOURCES})
   target_link_libraries(
@@ -190,7 +223,9 @@ function(DLAF_addTest test_target_name)
     ${test_target_name} PRIVATE ${DLAF_AT_COMPILE_DEFINITIONS} $<$<BOOL:${IS_AN_MPI_TEST}>:
                                 NUM_MPI_RANKS=${DLAF_AT_MPIRANKS}>
   )
-  target_include_directories(${test_target_name} PRIVATE ${DLAF_AT_INCLUDE_DIRS})
+  target_include_directories(
+    ${test_target_name} PRIVATE ${DLAF_AT_INCLUDE_DIRS} ${CMAKE_CURRENT_BINARY_DIR}
+  )
   target_add_warnings(${test_target_name})
   DLAF_addPrecompiledHeaders(${test_target_name})
   add_test(NAME ${test_target_name} COMMAND ${_TEST_COMMAND} ${_TEST_ARGUMENTS})

diff --git a/cmake/FindSCALAPACK.cmake b/cmake/FindSCALAPACK.cmake
@@ -0,0 +1,39 @@
+#
+# Distributed Linear Algebra with Future (DLAF)
+#
+# Copyright (c) 2018-2023, ETH Zurich
+# All rights reserved.
+#
+# Please, refer to the LICENSE file in the root directory.
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+include(FindPackageHandleStandardArgs)
+find_package(PkgConfig REQUIRED)
+
+pkg_search_module(_SCALAPACK scalapack)
+
+find_library(
+  SCALAPACK_LIBRARY NAME scalapack
+  HINTS ${_SCALAPACK_LIBRARY_DIRS}
+        ENV
+        SCALAPACKROOT
+        SCALAPACK_ROOT
+        SCALAPACK_PREFIX
+        SCALAPACK_DIR
+        SCALAPACKDIR
+        /usr
+  PATH_SUFFIXES lib
+)
+
+find_package_handle_standard_args(SCALAPACK DEFAULT_MSG SCALAPACK_LIBRARY)
+
+mark_as_advanced(SCALAPACK_LIBRARY)
+
+if(SCALAPACK_FOUND)
+  if(NOT TARGET DLAF::SCALAPACK)
+    add_library(DLAF::SCALAPACK INTERFACE IMPORTED GLOBAL)
+  endif()
+
+  target_link_libraries(DLAF::SCALAPACK INTERFACE "${SCALAPACK_LIBRARY}")
+endif()
diff --git a/cmake/template/DLAFConfig.cmake.in b/cmake/template/DLAFConfig.cmake.in
@@ -23,6 +23,7 @@ set(DLAF_WITH_MKL @DLAF_WITH_MKL@)
 set(DLAF_WITH_CUDA @DLAF_WITH_CUDA@)
 set(DLAF_WITH_HIP @DLAF_WITH_HIP@)
 set(DLAF_WITH_GPU @DLAF_WITH_GPU@)
+set(DLAF_WITH_SCALAPACK @DLAF_WITH_SCALAPACK@)
 
 # ===== DEPENDENCIES
 include(CMakeFindDependencyMacro)
@@ -61,10 +62,20 @@ if(DLAF_WITH_MKL)
   set(MKL_LAPACK_TARGET "@MKL_LAPACK_TARGET@")
   add_library(DLAF::LAPACK INTERFACE IMPORTED GLOBAL)
   target_link_libraries(DLAF::LAPACK INTERFACE ${MKL_LAPACK_TARGET})
+
+  if(DLAF_WITH_SCALAPACK)
+    set(MKL_SCALAPACK_TARGET "@MKL_SCALAPACK_TARGET@")
+    add_library(DLAF::SCALAPACK INTERFACE IMPORTED GLOBAL)
+    target_link_libraries(DLAF::SCALAPACK INTERFACE ${MKL_SCALAPACK_TARGET})
+  endif()
 else()
   set(LAPACK_LIBRARY "@LAPACK_LIBRARY@")
-
   find_dependency(LAPACK)
+
+  if(DLAF_WITH_SCALAPACK)
+    set(SCALAPACK_LIBRARY "@SCALAPACK_LIBRARY@")
+    find_dependency(SCALAPACK)
+  endif()
 endif()
 
 # ----- pika

diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in
@@ -2149,6 +2149,7 @@ PREDEFINED             += DLAF_ASSERT_ENABLE
 PREDEFINED             += DLAF_ASSERT_MODERATE_ENABLE
 PREDEFINED             += DLAF_ASSERT_HEAVY_ENABLE
 PREDEFINED             += DLAF_WITH_GPU
+PREDEFINED             += DLAF_WITH_SCALAPACK
 
 # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
 # tag can be used to specify a list of macro names that should be expanded. The
-Original file line number
+Diff line change
@@ Expand Up / @@ -45,7 +45,6 @@ @@
     *.out
     *.app
     # ------------------- CMake
     CMakeLists.txt.user
@@ Expand Down @@