ROX-7380 cpu/heap profiling endpoint (#425)

builder/Dockerfile

@@ -23,6 +23,7 @@ RUN apt-get update \
         libcap-ng-dev \
         libcurl4-openssl-dev \
         libelf-dev \
+        libgoogle-perftools-dev \
         libssl-dev \
         libtool \
         libz-dev \

collector/CMakeLists.txt

@@ -7,6 +7,10 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall --std=c++11 -pthread -Wno-de
 set(CMAKE_CXX_FLAGS_DEBUG "-g -ggdb")
 set(CMAKE_CXX_FLAGS_RELEASE "-O3 -fno-strict-aliasing -DNDEBUG")
 
+if(NOT DEFINED ENV{DISABLE_PROFILING})
+	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DCOLLECTOR_PROFILING")
+endif()
+
 if(ADDRESS_SANITIZER)
 	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer -fsanitize=address")
 endif()
@@ -47,6 +51,11 @@ target_link_libraries(collector_lib z ssl crypto)
 
 add_executable(collector collector.cpp)
 target_link_libraries(collector collector_lib)
+
+if(NOT DEFINED ENV{DISABLE_PROFILING})
+	target_link_libraries(collector profiler tcmalloc)
+endif()
+
 target_link_libraries(collector libprometheus-cpp-pull.a)
 target_link_libraries(collector libprometheus-cpp-core.a)
 
@@ -59,6 +68,10 @@ enable_testing()
 # Unit Tests
 file(GLOB TEST_SRC_FILES ${PROJECT_SOURCE_DIR}/test/*.cpp)
 add_executable(runUnitTests ${TEST_SRC_FILES})
+
+if(NOT DEFINED ENV{DISABLE_PROFILING})
+	target_link_libraries(runUnitTests profiler tcmalloc)
+endif()
 target_link_libraries(runUnitTests collector_lib)
 if(DEFINED ENV{WITH_RHEL8_RPMS})
 target_link_libraries(runUnitTests gtest gtest_main gmock gmock_main)

collector/Makefile

@@ -40,6 +40,7 @@ container/bin/collector-rhel: $(HDRS) $(SRCS) txt-files generated-srcs-rhel $(sh
 		-v "$(CURDIR):/src:ro" \
 		-v "$(CURDIR)/../sysdig/:/sysdig:ro" \
 		-v "$(CURDIR)/cmake-build-rhel:/tmp/cmake-build" \
+		-e DISABLE_PROFILING="true" \
 		stackrox/collector-builder:rhel-$(COLLECTOR_BUILDER_TAG) /build-collector.sh
 	cp cmake-build-rhel/collector container/bin/collector.rhel
 	cp -r cmake-build-rhel/THIRD_PARTY_NOTICES/* container/THIRD_PARTY_NOTICES.rhel/

collector/container/Dockerfile

@@ -32,7 +32,7 @@ RUN apt-get update \
         libuuid1 \
         libcap-ng0 \
         libelf1 \
-        google-perftools \
+        libgoogle-perftools4 \
  && dpkg -r --force-all e2fslibs e2fsprogs apt debconf dpkg \
  && rm -rf /var/lib/apt \
  ;

collector/lib/CollectorService.cpp

@@ -36,6 +36,7 @@ extern "C" {
 #include "GetStatus.h"
 #include "LogLevel.h"
 #include "NetworkStatusNotifier.h"
+#include "ProfilerHandler.h"
 #include "SysdigService.h"
 #include "Utility.h"
 #include "civetweb/CivetServer.h"
@@ -67,6 +68,9 @@ void CollectorService::RunForever() {
   LogLevel setLogLevel;
   server.addHandler("/loglevel", setLogLevel);
 
+  ProfilerHandler profiler_handler;
+  server.addHandler(ProfilerHandler::kBaseRoute, profiler_handler);
+
   prometheus::Exposer exposer("9090");
   exposer.RegisterCollectable(registry);
 

collector/lib/NetworkStatusNotifier.cpp

@@ -28,6 +28,7 @@ You should have received a copy of the GNU General Public License along with thi
 #include "CollectorStats.h"
 #include "DuplexGRPC.h"
 #include "GRPCUtil.h"
+#include "Profiler.h"
 #include "ProtoUtil.h"
 #include "TimeUtil.h"
 #include "Utility.h"
@@ -141,6 +142,7 @@ void NetworkStatusNotifier::ReceiveIPNetworks(const sensor::IPNetworkList& netwo
 }
 
 void NetworkStatusNotifier::Run() {
+  Profiler::RegisterCPUThread();
   auto next_attempt = std::chrono::system_clock::now();
 
   while (thread_.PauseUntil(next_attempt)) {

collector/lib/Profiler.h

@@ -0,0 +1,66 @@
+#ifndef COLLECTOR_PROFILER_H
+#define COLLECTOR_PROFILER_H
+
+#include <memory>
+#include <string>
+
+struct Free {
+  void operator()(const void* p) const { free((void*)p); }
+};
+using MallocUniquePtr = std::unique_ptr<char, Free>;
+
+#ifdef COLLECTOR_PROFILING
+#  include "gperftools/heap-profiler.h"
+#  include "gperftools/profiler.h"
+
+namespace collector {
+class Profiler {
+ public:
+  static inline void RegisterCPUThread() { ProfilerRegisterThread(); }
+  static inline MallocUniquePtr AllocAndGetHeapProfile() {
+    if (!IsHeapProfilerEnabled()) {
+      return nullptr;
+    }
+    return MallocUniquePtr(GetHeapProfile());
+  }
+  static inline bool StartCPUProfiler(const std::string& output_dir) {
+    return ProfilerStart(output_dir.c_str());
+  }
+  static inline void StopCPUProfiler() {
+    ProfilerFlush();
+    ProfilerStop();
+  }
+  static inline void StartHeapProfiler() { HeapProfilerStart(""); }
+  static inline void StopHeapProfiler() {
+    if (!IsHeapProfilerEnabled()) {
+      return;
+    }
+    HeapProfilerStop();
+  }
+  static inline bool IsCPUProfilerEnabled() { return ProfilingIsEnabledForAllThreads(); }
+  static inline bool IsCPUProfilerSupported() { return true; }
+  static inline bool IsHeapProfilerEnabled() { return IsHeapProfilerRunning(); }
+  static inline bool IsHeapProfilerSupported() { return true; }
+};
+
+}  // namespace collector
+#else
+
+namespace collector {
+class Profiler {
+ public:
+  static inline void RegisterCPUThread() {}
+  static inline MallocUniquePtr AllocAndGetHeapProfile() { return MallocUniquePtr(nullptr); }
+  static inline bool StartCPUProfiler(const std::string& output_dir) { return false; }
+  static inline void StopCPUProfiler() {}
+  static inline void StartHeapProfiler() {}
+  static inline void StopHeapProfiler() {}
+  static inline bool IsCPUProfilerSupported() { return false; }
+  static inline bool IsCPUProfilerEnabled() { return false; }
+  static inline bool IsHeapProfilerSupported() { return false; }
+  static inline bool IsHeapProfilerEnabled() { return false; }
+};
+}  // namespace collector
+#endif
+
+#endif  //COLLECTOR_PROFILER_H

collector/lib/ProfilerHandler.cpp

@@ -0,0 +1,167 @@
+#include "ProfilerHandler.h"
+
+#include <algorithm>
+#include <cstdio>
+#include <cstring>
+
+#include <json/json.h>
+#include <sys/stat.h>
+
+#include "Logging.h"
+#include "Profiler.h"
+#include "Utility.h"
+
+namespace collector {
+
+const std::string ProfilerHandler::kCPUProfileFilename = "/module/cpu_profile";
+const std::string ProfilerHandler::kBaseRoute = "/profile";
+const std::string ProfilerHandler::kCPURoute = kBaseRoute + "/cpu";
+const std::string ProfilerHandler::kHeapRoute = kBaseRoute + "/heap";
+
+bool ProfilerHandler::ServerError(struct mg_connection* conn, const char* err) {
+  return mg_send_http_error(conn, 500, err) >= 0;
+}
+
+bool ProfilerHandler::ClientError(struct mg_connection* conn, const char* err) {
+  return mg_send_http_error(conn, 400, err) >= 0;
+}
+
+bool ProfilerHandler::SendCPUProfile(struct mg_connection* conn) {
+  WITH_LOCK(mutex_) {
+    if (cpu_profile_length_ == 0) {
+      return mg_send_http_ok(conn, "text/plain", 0) >= 0;
+    }
+    mg_send_mime_file(conn, kCPUProfileFilename.c_str(), "application/octet-stream");
+  }
+  return true;
+}
+
+bool ProfilerHandler::SendHeapProfile(struct mg_connection* conn) {
+  WITH_LOCK(mutex_) {
+    if (heap_profile_length_ == 0) {
+      return mg_send_http_ok(conn, "text/plain", 0) >= 0;
+    }
+    if (mg_send_http_ok(conn, "application/octet-stream", heap_profile_length_) < 0) {
+      return false;
+    }
+    if (mg_write(conn, heap_profile_.get(), heap_profile_length_) != heap_profile_length_) {
+      return false;
+    }
+  }
+  return true;
+}
+
+bool ProfilerHandler::SendStatus(struct mg_connection* conn) {
+  Json::Value resp(Json::objectValue);
+  WITH_LOCK(mutex_) {
+    resp["supports_cpu"] = Profiler::IsCPUProfilerSupported();
+    resp["supports_heap"] = Profiler::IsHeapProfilerSupported();
+    resp["cpu"] = Profiler::IsCPUProfilerEnabled() ? "on" : (cpu_profile_length_ > 0 ? "off" : "empty");
+    resp["heap"] = Profiler::IsHeapProfilerEnabled() ? "on" : (heap_profile_length_ > 0 ? "off" : "empty");
+  }
+  std::string json_body = resp.toStyledString();
+  if (mg_send_http_ok(conn, "application/json", json_body.length()) < 0) {
+    return false;
+  }
+  return mg_write(conn, json_body.c_str(), json_body.length()) >= 0;
+}
+
+bool ProfilerHandler::HandleCPURoute(struct mg_connection* conn, const std::string& post_data) {
+  WITH_LOCK(mutex_) {
+    if (post_data == "on") {
+      if (!Profiler::IsCPUProfilerEnabled()) {
+        if (!Profiler::StartCPUProfiler(kCPUProfileFilename)) {
+          return ServerError(conn, "failed starting cpu profiler");
+        }
+        CLOG(INFO) << "started cpu profiler";
+      }
+    } else if (post_data == "off") {
+      if (Profiler::IsCPUProfilerEnabled()) {
+        Profiler::StopCPUProfiler();
+        struct stat sdata;
+        if (stat(kCPUProfileFilename.c_str(), &sdata) == 0) {
+          cpu_profile_length_ = sdata.st_size;
+        }
+        CLOG(INFO) << "stopped cpu profiler, bytes=" << cpu_profile_length_;
+      }
+    } else if (post_data == "empty") {
+      if (cpu_profile_length_ != 0) {
+        cpu_profile_length_ = 0;
+        if (std::remove(kCPUProfileFilename.c_str()) != 0) {
+          CLOG(INFO) << "remove failed: " << StrError();
+          return ServerError(conn, "failure while deleting cpu profile");
+        }
+        CLOG(INFO) << "cleared cpu profile";
+      }
+    } else {
+      return ClientError(conn, "invalid post data");
+    }
+  }
+  return mg_send_http_ok(conn, "text/plain", 0) >= 0;
+}
+
+bool ProfilerHandler::HandleHeapRoute(struct mg_connection* conn, const std::string& post_data) {
+  WITH_LOCK(mutex_) {
+    if (post_data == "on") {
+      if (!Profiler::IsHeapProfilerEnabled()) {
+        Profiler::StartHeapProfiler();
+        CLOG(INFO) << "started heap profiler";
+      }
+    } else if (post_data == "off") {
+      if (Profiler::IsHeapProfilerEnabled()) {
+        heap_profile_ = Profiler::AllocAndGetHeapProfile();
+        if (heap_profile_ == nullptr) {
+          return ServerError(conn, "failed get heap profile");
+        }
+        heap_profile_length_ = strlen(heap_profile_.get());
+        Profiler::StopHeapProfiler();
+        CLOG(INFO) << "stopped heap profiler, bytes=" << heap_profile_length_;
+      }
+    } else if (post_data == "empty") {
+      if (heap_profile_length_ != 0) {
+        heap_profile_length_ = 0;
+        heap_profile_.reset();
+        CLOG(INFO) << "cleared heap profile";
+      }
+    } else {
+      return ClientError(conn, "invalid post data");
+    }
+  }
+  return mg_send_http_ok(conn, "text/plain", 0) >= 0;
+}
+
+bool ProfilerHandler::handlePost(CivetServer* server, struct mg_connection* conn) {
+  if (!Profiler::IsCPUProfilerSupported()) {
+    return ServerError(conn, "not supported");
+  }
+  const mg_request_info* req_info = mg_get_request_info(conn);
+  if (req_info == nullptr) {
+    return ServerError(conn, "unable to read request");
+  }
+  std::string uri(req_info->local_uri);
+  std::string post_data(server->getPostData(conn));
+  if (uri == kCPURoute) {
+    return HandleCPURoute(conn, post_data);
+  } else if (uri == kHeapRoute) {
+    return HandleHeapRoute(conn, post_data);
+  }
+  return ClientError(conn, "unknown route");
+}
+
+bool ProfilerHandler::handleGet(CivetServer* server, struct mg_connection* conn) {
+  const mg_request_info* req_info = mg_get_request_info(conn);
+  if (req_info == nullptr) {
+    return ServerError(conn, "unable to read request");
+  }
+  std::string uri = req_info->local_uri;
+  if (uri == kBaseRoute) {
+    return SendStatus(conn);
+  } else if (uri == kHeapRoute) {
+    return SendHeapProfile((conn));
+  } else if (uri == kCPURoute) {
+    return SendCPUProfile(conn);
+  }
+  return ClientError(conn, "unknown route");
+}
+
+}  // namespace collector

collector/lib/ProfilerHandler.h

@@ -0,0 +1,48 @@
+#ifndef COLLECTOR_PROFILERHANDLER_H
+#define COLLECTOR_PROFILERHANDLER_H
+
+#include <memory>
+#include <mutex>
+
+#include "Profiler.h"
+#include "civetweb/CivetServer.h"
+
+namespace collector {
+
+// Profiling Endpoints:
+// POST /profile/cpu
+//   - accepts post data of on|off|empty to enable, disable or clear/delete the latest cpu profile
+// POST /profile/heap
+//   - accepts post data of on|off|empty to enable, disable or clear/delete the latest heap profile
+// GET /profile
+//   - get profiling support and status
+// GET /profile/cpu
+//   - get latest cpu profile
+// GET /profile/heap
+//   - get latest heap profile
+class ProfilerHandler : public CivetHandler {
+ public:
+  static const std::string kCPUProfileFilename;
+  static const std::string kBaseRoute;
+  static const std::string kCPURoute;
+  static const std::string kHeapRoute;
+
+  bool handleGet(CivetServer* server, struct mg_connection* conn);
+  bool handlePost(CivetServer* server, struct mg_connection* conn);
+
+ private:
+  bool ServerError(struct mg_connection* conn, const char* err);
+  bool ClientError(struct mg_connection* conn, const char* err);
+  bool SendStatus(struct mg_connection* conn);
+  bool SendHeapProfile(struct mg_connection* conn);
+  bool SendCPUProfile(struct mg_connection* conn);
+  bool HandleCPURoute(struct mg_connection* conn, const std::string& post_data);
+  bool HandleHeapRoute(struct mg_connection* conn, const std::string& post_data);
+  MallocUniquePtr heap_profile_;
+  size_t heap_profile_length_;
+  size_t cpu_profile_length_;
+  std::mutex mutex_;
+};
+}  // namespace collector
+
+#endif  //COLLECTOR_PROFILERHANDLER_H