Add craypat support (LLNL#431)

* Initial CrayPAT support

* Init/finalize CrayPAT

* Cleanup

* Add documentation for CrayPAT forwarding

CMakeLists.txt

@@ -78,6 +78,7 @@ add_caliper_option(WITH_KOKKOS    "Enable Kokkos profiling support" TRUE)
 add_caliper_option(WITH_PCP       "Enable performance co-pilot support" FALSE)
 add_caliper_option(WITH_VARIORUM  "Enable Variorum support" FALSE)
 add_caliper_option(WITH_UMPIRE    "Enable Umpire statistics support" FALSE)
+add_caliper_option(WITH_CRAYPAT   "Enable CrayPAT region forwarding support" FALSE)
 
 add_caliper_option(USE_EXTERNAL_GOTCHA "Use pre-installed gotcha instead of building our own" FALSE)
 
@@ -382,6 +383,18 @@ if (WITH_ROCTRACER OR WITH_ROCM)
   endif()
 endif()
 
+if (WITH_CRAYPAT)
+  include(FindCrayPAT)
+  if (CRAYPAT_FOUND)
+    set(CALIPER_HAVE_CRAYPAT TRUE)
+    set(CALIPER_CrayPAT_CMAKE_MSG "Yes, using ${CRAYPAT_LIBRARIES}")
+    list(APPEND CALIPER_EXTERNAL_LIBS ${CRAYPAT_LIBRARIES})
+  else()
+    message(WARNING "CrayPAT support requested but CrayPAT was not found!\n"
+      "Set CRAYPAT_ROOT to installation path and re-run cmake.")
+  endif()
+endif()
+
 if(WITH_TAU)
   if (CALIPER_HAVE_MPI)
     find_package(TAU QUIET)
@@ -570,7 +583,8 @@ set(CALIPER_MODULES
   PCP
   histogram
   variorum
-  umpire)
+  umpire
+  CrayPAT)
 
 foreach(_caliper_module ${CALIPER_MODULES})
   string(LENGTH "${_caliper_module}" _strlen)

caliper-config.h.in

@@ -23,6 +23,7 @@
 #cmakedefine CALIPER_HAVE_ROCTX
 #cmakedefine CALIPER_HAVE_ROCTRACER
 #cmakedefine CALIPER_HAVE_UMPIRE
+#cmakedefine CALIPER_HAVE_CRAYPAT
 
 #cmakedefine CALIPER_REDUCED_CONSTEXPR_USAGE
 

cmake/FindCrayPAT.cmake

@@ -0,0 +1,28 @@
+# Find CrayPAT libraries/headers
+
+find_path(CRAYPAT_ROOT
+  NAMES include/pat_api.h
+)
+
+find_library(CRAYPAT_LIBRARIES
+  NAMES _pat_base
+  HINTS ${CRAYPAT_ROOT}/lib
+)
+
+find_path(CRAYPAT_INCLUDE_DIRS
+  NAMES pat_api.h
+  HINTS ${CRAYPAT_ROOT}/include
+)
+
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(CRAYPAT
+  DEFAULT_MSG
+  CRAYPAT_LIBRARIES
+  CRAYPAT_INCLUDE_DIRS
+)
+
+mark_as_advanced(
+  CRAYPAT_INCLUDE_DIRS
+  CRAYPAT_LIBRARIES
+)

doc/sphinx/ThirdPartyTools.rst

@@ -59,3 +59,37 @@ By default, Caliper exports all regions with the NESTED attribute
 property. Alternatively, the region attributes to export can be
 specified explicitly in the `CALI_VTUNE_TRIGGER_ATTRIBUTES`
 configuration variable.
+
+CrayPAT
+--------------------------------
+
+The `craypat` service translates Caliper annotations into CrayPAT regions.
+This API is only available in recent (2022+) CrayPAT versions. It also
+only works with the "trace" call stack mode in CrayPAT.
+
+By default, the `craypat` service will attempt to initialize CrayPAT internally
+and record a CrayPAT region profile. Alternatively, you can use the `craypat`
+service in conjunction with `pat_run` to set additional recording options. In
+this case, make sure to set ``PAT_RT_CALLSTACK_MODE=trace``::
+
+  $ PAT_RT_CALLSTACK_MODE=trace CALI_SERVICES_ENABLE=craypat pat_run -w -E PAPI_TOT_CYC ./examples/apps/cxx-example
+  CrayPat/X:  Version 22.06.0 Revision 4b5ab6256  05/21/22 02:04:43
+  Experiment data directory written:
+  /<path>/cxx-example+91423-10t
+  $ pat_report ./cxx-example+91423-10t
+  [...]
+  Table 1:  Profile by Function Group and Function
+
+      Time% |     Time | Imb. |  Imb. | Calls | Group
+            |          | Time | Time% |       |  Function
+
+     100.0% | 0.000716 |   -- |    -- |   7.0 | Total
+  |------------------------------------------------------
+  |  100.0% | 0.000716 |   -- |    -- |   7.0 | USER
+  ||-----------------------------------------------------
+  ||  89.2% | 0.000639 |   -- |    -- |   4.0 | [R]foo
+  ||   4.7% | 0.000034 |   -- |    -- |   1.0 | [R]main
+  ||   3.9% | 0.000028 |   -- |    -- |   1.0 | [R]mainloop
+  ||   2.1% | 0.000015 |   -- |    -- |   1.0 | [R]init
+  |======================================================
+  [...]

src/services/CMakeLists.txt

@@ -114,6 +114,9 @@ endif()
 if (CALIPER_HAVE_UMPIRE)
   add_subdirectory(umpire)
 endif()
+if (CALIPER_HAVE_CRAYPAT)
+  add_subdirectory(craypat)
+endif()
 
 if (CALIPER_BUILD_TESTING)
   add_subdirectory(templates)

src/services/craypat/CMakeLists.txt

@@ -0,0 +1,8 @@
+set(CALIPER_CRAYPAT_SOURCES
+  CrayPAT.cpp)
+
+add_library(caliper-craypat OBJECT ${CALIPER_CRAYPAT_SOURCES})
+target_include_directories(caliper-craypat PRIVATE ${CRAYPAT_INCLUDE_DIRS})
+
+add_service_objlib("caliper-craypat")
+add_caliper_service("craypat CALIPER_HAVE_CRAYPAT")

src/services/craypat/CrayPAT.cpp

@@ -0,0 +1,115 @@
+// Copyright (c) 2015-2022, Lawrence Livermore National Security, LLC.
+// See top-level LICENSE file for details.
+
+// Caliper RocTX bindings
+
+#include "caliper/AnnotationBinding.h"
+
+#include "caliper/common/Attribute.h"
+#include "caliper/common/Log.h"
+
+#include <pat_api.h>
+
+#include <cstdlib>
+
+#include <fstream>
+#include <vector>
+
+extern char** environ;
+
+using namespace cali;
+
+extern "C"
+{
+
+void *sbrk(__intptr_t increment);
+void __pat_init (int, const char *[ ], const char *[ ], const void *, const void *);
+void __pat_exit (void);
+
+}
+
+namespace
+{
+
+std::vector<std::string> read_cmdline()
+{
+    std::vector<std::string> ret;
+
+    std::ifstream f("/proc/self/cmdline");
+    std::string s;
+
+    while (f) {
+        char c = f.get();
+
+        if (f.eof() || c == '\0') {
+            if (s.size() > 0) {
+                ret.emplace_back(std::move(s));
+                s.clear();
+            }
+        } else {
+            s.push_back(c);
+        }
+    }
+
+    return ret;
+}
+
+class CrayPATBinding : public AnnotationBinding
+{
+    std::vector<std::string> m_args;
+    std::vector<const char*> m_argv;
+
+public:
+
+    void on_begin(Caliper*, Channel*, const Attribute &attr, const Variant& value) {
+        if (attr.is_nested() && attr.type() == CALI_TYPE_STRING)
+            PAT_region_push(static_cast<const char*>(value.data()));
+    }
+
+    void on_end(Caliper*, Channel*, const Attribute& attr, const Variant& value) {
+        if (attr.is_nested() && attr.type() == CALI_TYPE_STRING)
+            PAT_region_pop(static_cast<const char*>(value.data()));
+    }
+
+    void initialize(Caliper*, Channel* channel) {
+        m_args = read_cmdline();
+
+        if (m_args.empty()) {
+            Log(0).stream()
+                << channel->name()
+                << ": craypat: Unable to initialize CrayPAT: cannot read command line"
+                << std::endl;
+
+            return;
+        }
+
+        m_argv.resize(m_args.size());
+        for (size_t i = 0; i < m_args.size(); ++i)
+            m_argv[i] = m_args[i].data();
+
+        m_argv.push_back(nullptr); // somehow craypat segfaults without this
+
+        putenv(const_cast<char*>("PAT_RT_CALLSTACK_MODE=trace"));
+        putenv(const_cast<char*>("PAT_RT_EXPERIMENT=trace"));
+
+        __pat_init(static_cast<int>(m_argv.size()-1), m_argv.data(), const_cast<const char**>(environ), __builtin_frame_address (0), sbrk (0L));
+
+        Log(1).stream() << channel->name() << ": craypat: CrayPAT initialized\n";
+    }
+
+    void finalize(Caliper*, Channel* channel) {
+        Log(1).stream() << channel->name() << ": craypat: Closing CrayPAT\n";
+        __pat_exit();
+    }
+
+    const char* service_tag() const { return "craypat"; }
+};
+
+} // namespace [anonymous]
+
+namespace cali
+{
+
+CaliperService craypat_service { "craypat", AnnotationBinding::make_binding<CrayPATBinding> };
+
+}