Update papi_sde to current status. Use externals.

CMakeLists.txt

@@ -89,6 +89,7 @@ if(MSVC OR WIN32 OR CYGWIN OR APPLE)
 else()
     option(GINKGO_BUILD_HWLOC "Build Ginkgo with HWLOC. Default is ON. If a system HWLOC is not found, then we try to build it ourselves. Switch this OFF to disable HWLOC." ON)
 endif()
+option(GINKGO_BUILD_PAPI_SDE "Build Ginkgo with PAPI SDE. Default is OFF." OFF)
 option(GINKGO_DPCPP_SINGLE_MODE "Do not compile double kernels for the DPC++ backend." OFF)
 option(GINKGO_INSTALL_RPATH "Set the RPATH when installing its libraries." ON)
 option(GINKGO_INSTALL_RPATH_ORIGIN "Add $ORIGIN (Linux) or @loader_path (MacOS) to the installation RPATH." ON)
@@ -199,13 +200,6 @@ endif()
 include(CheckIncludeFileCXX)
 check_include_file_cxx(cxxabi.h GKO_HAVE_CXXABI_H)
 
-# Automatically find PAPI and search for the required 'sde' component
-set(GINKGO_HAVE_PAPI_SDE 0)
-find_package(PAPI OPTIONAL_COMPONENTS sde)
-if(PAPI_sde_FOUND)
-    set(GINKGO_HAVE_PAPI_SDE 1)
-endif()
-
 # Automatically find TAU
 set(GINKGO_HAVE_TAU 0)
 find_package(PerfStubs QUIET)
@@ -241,6 +235,12 @@ else()
     set(GINKGO_HAVE_HWLOC 0)
     message(STATUS "HWLOC is being forcibly switched off")
 endif()
+if(GINKGO_BUILD_PAPI_SDE)
+    set(GINKGO_HAVE_PAPI_SDE 1)
+else()
+    set(GINKGO_HAVE_PAPI_SDE 0)
+    message(STATUS "PAPI SDE is being forcibly switched off")
+endif()
 
 set(GINKGO_HAVE_GPU_AWARE_MPI OFF)
 set(GINKGO_FORCE_SPMV_BLOCKING_COMM OFF)
@@ -281,6 +281,10 @@ endif()
 if(GINKGO_BUILD_HWLOC)
     find_package(HWLOC 2.1) # No need for QUIET as we ship FindHWLOC
 endif()
+if(GINKGO_BUILD_PAPI_SDE)
+    # No need for QUIET as we ship FindPAPI
+    find_package(PAPI OPTIONAL_COMPONENTS sde)
+endif()
 add_subdirectory(third_party)    # Third-party tools and libraries
 
 if(MSVC)

cmake/DownloadNonCMakeCMakeLists.txt.in

@@ -3,12 +3,25 @@ project(${package_name})
 
 include(ExternalProject)
 ExternalProject_Add(${package_name}
-    URL "${package_url}"
-    URL_HASH "${package_hash}"
+    URL               "${package_url}"
+    URL_HASH          "${package_hash}"
     DOWNLOAD_NO_PROGRESS TRUE
     SOURCE_DIR        "${CMAKE_CURRENT_BINARY_DIR}/src"
-    BINARY_DIR        "${CMAKE_CURRENT_BINARY_DIR}/build"
-    CONFIGURE_COMMAND "${config_command}" "${ARGN}"
-    INSTALL_COMMAND ""
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND     ""
+    INSTALL_COMMAND   ""
     UPDATE_DISCONNECTED ${GINKGO_SKIP_DEPENDENCY_UPDATE}
     )
+
+ExternalProject_Add_Step(${package_name} custom_configure
+    COMMAND "${config_command}" "${ARGN}"
+    WORKING_DIRECTORY "${working_dir}"
+    DEPENDEES download)
+ExternalProject_Add_Step(${package_name} custom_build
+    COMMAND make
+    WORKING_DIRECTORY "${working_dir}"
+    DEPENDEES custom_configure)
+ExternalProject_Add_Step(${package_name} custom_install
+    COMMAND make all install
+    WORKING_DIRECTORY "${working_dir}"
+    DEPENDEES custom_build)

cmake/get_info.cmake

@@ -190,16 +190,21 @@ ginkgo_print_module_footer(${detailed_log} "")
 
 ginkgo_print_generic_header(${minimal_log} "  Components:")
 ginkgo_print_generic_header(${detailed_log} "  Components:")
-if(PAPI_sde_FOUND)
+ginkgo_print_variable(${minimal_log} "GINKGO_BUILD_PAPI_SDE")
+ginkgo_print_variable(${detailed_log} "GINKGO_BUILD_PAPI_SDE")
+if(TARGET PAPI::PAPI)
     ginkgo_print_variable(${detailed_log} "PAPI_VERSION")
     ginkgo_print_variable(${detailed_log} "PAPI_INCLUDE_DIR")
     ginkgo_print_flags(${detailed_log} "PAPI_LIBRARY")
 endif()
+
 ginkgo_print_variable(${minimal_log} "GINKGO_BUILD_HWLOC")
 ginkgo_print_variable(${detailed_log} "GINKGO_BUILD_HWLOC")
-ginkgo_print_variable(${detailed_log} "HWLOC_VERSION")
-ginkgo_print_variable(${detailed_log} "HWLOC_LIBRARIES")
-ginkgo_print_variable(${detailed_log} "HWLOC_INCLUDE_DIRS")
+if(TARGET hwloc)
+    ginkgo_print_variable(${detailed_log} "HWLOC_VERSION")
+    ginkgo_print_variable(${detailed_log} "HWLOC_LIBRARIES")
+    ginkgo_print_variable(${detailed_log} "HWLOC_INCLUDE_DIRS")
+endif()
 
 _minimal(
     "

cmake/information_helpers.cmake

@@ -78,8 +78,9 @@ macro(ginkgo_interface_libraries_recursively INTERFACE_LIBS)
             list(TRANSFORM GINKGO_LIBS_INTERFACE_LIBS REPLACE "\\$<LINK_ONLY:(.*)>" "\\1")
             ginkgo_interface_libraries_recursively("${GINKGO_LIBS_INTERFACE_LIBS}")
         elseif(EXISTS "${_libs}")
-            if ("${_libs}" MATCHES "${PROJECT_BINARY_DIR}.*hwloc.so")
-                list(APPEND GINKGO_INTERFACE_LIBS_FOUND "${CMAKE_INSTALL_FULL_LIBDIR}/libhwloc.so")
+            if ("${_libs}" MATCHES "${PROJECT_BINARY_DIR}.*(papi|sde|pfm|hwloc).so")
+                get_filename_component(_lib_name "${_libs}" NAME)
+                list(APPEND GINKGO_INTERFACE_LIBS_FOUND "${CMAKE_INSTALL_FULL_LIBDIR}/${_lib_name}")
             else()
                 list(APPEND GINKGO_INTERFACE_LIBS_FOUND "${_libs}")
             endif()

cmake/install_helpers.cmake

@@ -80,11 +80,6 @@ function(ginkgo_install)
     install(FILES "${Ginkgo_BINARY_DIR}/include/ginkgo/config.hpp"
         DESTINATION "${CMAKE_INSTALL_FULL_INCLUDEDIR}/ginkgo"
         )
-    if (GINKGO_HAVE_PAPI_SDE)
-        install(FILES "${Ginkgo_SOURCE_DIR}/third_party/papi_sde/papi_sde_interface.h"
-            DESTINATION "${CMAKE_INSTALL_FULL_INCLUDEDIR}/third_party/papi_sde"
-            )
-    endif()
 
     if  (GINKGO_HAVE_HWLOC AND NOT HWLOC_FOUND)
         get_filename_component(HWLOC_LIB_PATH ${HWLOC_LIBRARIES} DIRECTORY)

cmake/package_helpers.cmake

@@ -7,9 +7,10 @@ set(NON_CMAKE_PACKAGE_DOWNLOADER_SCRIPT
 #   \param package_name     Name of the package
 #   \param package_url      Url of the package
 #   \param package_tag      Tag or version of the package to be downloaded.
+#   \param working_dir      The directory where the configure/build should happen.
 #   \param config_command   The command for the configuration step.
 #
-function(ginkgo_load_and_configure_package package_name package_url package_hash config_command)
+function(ginkgo_load_and_configure_package package_name package_url package_hash working_dir config_command)
     set(GINKGO_THIRD_PARTY_BUILD_TYPE "Debug")
     if (CMAKE_BUILD_TYPE MATCHES "[Rr][Ee][Ll][Ee][Aa][Ss][Ee]")
         set(GINKGO_THIRD_PARTY_BUILD_TYPE "Release")

core/CMakeLists.txt

@@ -113,9 +113,8 @@ target_link_libraries(ginkgo
 set(GKO_RPATH_ADDITIONS "")
 
 if(GINKGO_HAVE_PAPI_SDE)
-    target_link_libraries(ginkgo PUBLIC PAPI::PAPI)
-    list(GET PAPI_LIBRARIES 0 PAPI_FIRST_LIB)
-    get_filename_component(GKO_PAPI_LIBDIR "${PAPI_FIRST_LIB}" DIRECTORY)
+    target_link_libraries(ginkgo PUBLIC PAPI::PAPI_SDE)
+    get_filename_component(GKO_PAPI_LIBDIR "${PAPI_SDE_LIBRARIES}" DIRECTORY)
     list(APPEND GKO_RPATH_ADDITIONS "${GKO_PAPI_LIBDIR}")
 endif()
 

core/test/log/CMakeLists.txt

@@ -1,7 +1,7 @@
 ginkgo_create_test(convergence)
 ginkgo_create_test(logger)
 if (GINKGO_HAVE_PAPI_SDE)
-    ginkgo_create_test(papi PAPI::PAPI)
+    ginkgo_create_test(papi ADDITIONAL_LIBRARIES PAPI::PAPI)
 endif()
 ginkgo_create_test(performance_hint)
 ginkgo_create_test(profiler_hook)

core/test/log/papi.cpp

@@ -71,7 +71,11 @@ class Papi : public ::testing::Test {
         }
     }
 
-    void TearDown() { eventset = PAPI_NULL; }
+    void TearDown() {
+        logger = nullptr;
+        PAPI_destroy_eventset(&eventset);
+        PAPI_shutdown();
+    }
 
     template <typename U>
     const std::string init(const gko::log::Logger::mask_type& event,

include/ginkgo/core/log/papi.hpp

@@ -46,16 +46,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <mutex>
 
 
-#include <papi.h>
+#include <sde_lib.h>
 
 
 #include <ginkgo/core/base/polymorphic_object.hpp>
 #include <ginkgo/core/log/logger.hpp>
 
 
-#include "third_party/papi_sde/papi_sde_interface.h"
-
-
 namespace gko {
 namespace log {
 
@@ -213,7 +210,10 @@ class Papi : public Logger {
     create(std::shared_ptr<const gko::Executor>,
            const Logger::mask_type& enabled_events = Logger::all_events_mask)
     {
-        return std::shared_ptr<Papi>(new Papi(enabled_events));
+        return std::shared_ptr<Papi>(new Papi(enabled_events), [](auto logger){
+            papi_sde_shutdown(logger->get_handle());
+            delete logger;
+        });
     }
 
     /**
@@ -224,7 +224,11 @@ class Papi : public Logger {
     static std::shared_ptr<Papi> create(
         const Logger::mask_type& enabled_events = Logger::all_events_mask)
     {
-        return std::shared_ptr<Papi>(new Papi(enabled_events));
+        return std::shared_ptr<Papi>(new Papi(enabled_events), [](auto logger){
+            papi_sde_shutdown(logger->get_handle());
+            delete logger;
+        }
+);
     }
 
     /**
@@ -235,6 +239,13 @@ class Papi : public Logger {
      */
     const std::string get_handle_name() const { return name; }
 
+    /**
+     * Returns the corresponding papi_handle_t for this logger
+     *
+     * @return the corresponding papi_handle_t for this logger
+     */
+    const papi_handle_t get_handle() const { return papi_handle; }
+
 protected:
     [[deprecated("use single-parameter constructor")]] explicit Papi(
         std::shared_ptr<const gko::Executor> exec,
@@ -265,12 +276,10 @@ class Papi : public Logger {
 
         ~papi_queue()
         {
-            if (PAPI_is_initialized()) {
-                for (auto e : data) {
-                    std::ostringstream oss;
-                    oss << counter_name << "::" << e.first;
-                    papi_sde_unregister_counter(*handle, oss.str().c_str());
-                }
+            for (auto e : data) {
+                std::ostringstream oss;
+                oss << counter_name << "::" << e.first;
+                papi_sde_unregister_counter(*handle, oss.str().c_str());
             }
             data.clear();
         }

reference/test/log/CMakeLists.txt

@@ -1,4 +1,4 @@
 ginkgo_create_test(convergence)
 if (GINKGO_HAVE_PAPI_SDE)
-    ginkgo_create_test(papi PAPI::PAPI)
+    ginkgo_create_test(papi ADDITIONAL_LIBRARIES PAPI::PAPI)
 endif()

third_party/CMakeLists.txt

@@ -7,6 +7,10 @@ if(GINKGO_BUILD_HWLOC AND (NOT HWLOC_FOUND))
     add_subdirectory(hwloc)
 endif()
 
+if(GINKGO_BUILD_PAPI_SDE AND (NOT PAPI_FOUND))
+    add_subdirectory(papi_sde)
+endif()
+
 if(GINKGO_DEVEL_TOOLS)
     set(GCF_IGNORE_LIST "third_party" CACHE STRING "Ignore directories for GCF")
     add_subdirectory(git-cmake-format)

third_party/hwloc/CMakeLists.txt

@@ -2,6 +2,7 @@ message(STATUS "Configuring and building HWLOC")
 set(TPL_HWLOC_PATH "${PROJECT_BINARY_DIR}/third_party/hwloc")
 ginkgo_load_and_configure_package(hwloc_external "https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz"
     "SHA1=b94950e8958e1125ca75ecac0bc0259ee3d108c4"
+    ""
     "${TPL_HWLOC_PATH}/src/configure" "--disable-nvml" "--disable-cuda" "--disable-rsmi"
     )
 

third_party/papi_sde/CMakeLists.txt

@@ -0,0 +1,37 @@
+message(STATUS "Configuring and building PAPI-SDE")
+set(TPL_PAPI_PATH "${PROJECT_BINARY_DIR}/third_party/papi_sde/src")
+ginkgo_load_and_configure_package(papi_external
+    "https://bitbucket.org/terry_cojean/papi/get/77cdd0ba8db98d86c1459dd5f55013aba242d5d5.tar.gz"
+    "SHA1=540c18a14eeafb83cd60cbbf0a96706111dbff3b"
+    "${TPL_PAPI_PATH}/src"
+    "./configure" "--prefix=${TPL_PAPI_PATH}/install"
+    "--with-components=sde" "--with-libsde=yes" "--with-tests=no"
+    "--with-static-lib=no" "--with-shared-lib=yes"
+    )
+
+add_library(PAPI SHARED IMPORTED GLOBAL)
+add_library(PAPI::PAPI ALIAS PAPI)
+add_dependencies(PAPI papi_external)
+# NOTE: if changing this (e.g. to `.a`), please update the special case in
+# `cmake/information_helpers.cmake`
+set(PAPI_LIBRARIES "${TPL_PAPI_PATH}/install/lib/libpapi.so"
+    CACHE FILEPATH "The path to PAPI libraries" FORCE)
+set(PAPI_INCLUDE_DIRS "${TPL_PAPI_PATH}/install/include" CACHE PATH
+    "The directory containing the PAPI header, papi.h" FORCE)
+set_target_properties(PAPI PROPERTIES IMPORTED_LOCATION "${PAPI_LIBRARIES}")
+set_target_properties(PAPI PROPERTIES INTERFACE_LINK_LIBRARIES "${PAPI_LIBRARIES}")
+set_target_properties(PAPI PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${PAPI_INCLUDE_DIRS}")
+
+
+add_library(PAPI_SDE SHARED IMPORTED GLOBAL)
+add_library(PAPI::PAPI_SDE ALIAS PAPI_SDE)
+add_dependencies(PAPI_SDE papi_external)
+# NOTE: if changing this (e.g. to `.a`), please update the special case in
+# `cmake/information_helpers.cmake`
+set(PAPI_SDE_LIBRARIES "${TPL_PAPI_PATH}/install/lib/libsde.so"
+    CACHE FILEPATH "The path to PAPI SDE libraries" FORCE)
+set(PAPI_SDE_INCLUDE_DIRS "${TPL_PAPI_PATH}/install/include" CACHE PATH
+    "The directory containing the PAPI SDE header, sde_lib.h" FORCE)
+set_target_properties(PAPI_SDE PROPERTIES IMPORTED_LOCATION "${PAPI_SDE_LIBRARIES}")
+set_target_properties(PAPI_SDE PROPERTIES INTERFACE_LINK_LIBRARIES "${PAPI_SDE_LIBRARIES}")
+set_target_properties(PAPI_SDE PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${PAPI_SDE_INCLUDE_DIRS}")