diff --git a/builder/Dockerfile b/builder/Dockerfile index 8e2e4f83eb..4c1cd0dd02 100644 --- a/builder/Dockerfile +++ b/builder/Dockerfile @@ -23,6 +23,7 @@ RUN apt-get update \ libcap-ng-dev \ libcurl4-openssl-dev \ libelf-dev \ + libgoogle-perftools-dev \ libssl-dev \ libtool \ libz-dev \ diff --git a/collector/CMakeLists.txt b/collector/CMakeLists.txt index 862df76515..6ff53fa240 100644 --- a/collector/CMakeLists.txt +++ b/collector/CMakeLists.txt @@ -7,6 +7,10 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall --std=c++11 -pthread -Wno-de set(CMAKE_CXX_FLAGS_DEBUG "-g -ggdb") set(CMAKE_CXX_FLAGS_RELEASE "-O3 -fno-strict-aliasing -DNDEBUG") +if(NOT DEFINED ENV{DISABLE_PROFILING}) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DCOLLECTOR_PROFILING") +endif() + if(ADDRESS_SANITIZER) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer -fsanitize=address") endif() @@ -47,6 +51,11 @@ target_link_libraries(collector_lib z ssl crypto) add_executable(collector collector.cpp) target_link_libraries(collector collector_lib) + +if(NOT DEFINED ENV{DISABLE_PROFILING}) + target_link_libraries(collector profiler tcmalloc) +endif() + target_link_libraries(collector libprometheus-cpp-pull.a) target_link_libraries(collector libprometheus-cpp-core.a) @@ -59,6 +68,10 @@ enable_testing() # Unit Tests file(GLOB TEST_SRC_FILES ${PROJECT_SOURCE_DIR}/test/*.cpp) add_executable(runUnitTests ${TEST_SRC_FILES}) + +if(NOT DEFINED ENV{DISABLE_PROFILING}) + target_link_libraries(runUnitTests profiler tcmalloc) +endif() target_link_libraries(runUnitTests collector_lib) if(DEFINED ENV{WITH_RHEL8_RPMS}) target_link_libraries(runUnitTests gtest gtest_main gmock gmock_main) diff --git a/collector/Makefile b/collector/Makefile index 321f96c6e7..ec89044336 100644 --- a/collector/Makefile +++ b/collector/Makefile @@ -40,6 +40,7 @@ container/bin/collector-rhel: $(HDRS) $(SRCS) txt-files generated-srcs-rhel $(sh -v "$(CURDIR):/src:ro" \ -v "$(CURDIR)/../sysdig/:/sysdig:ro" \ -v "$(CURDIR)/cmake-build-rhel:/tmp/cmake-build" \ + -e DISABLE_PROFILING="true" \ stackrox/collector-builder:rhel-$(COLLECTOR_BUILDER_TAG) /build-collector.sh cp cmake-build-rhel/collector container/bin/collector.rhel cp -r cmake-build-rhel/THIRD_PARTY_NOTICES/* container/THIRD_PARTY_NOTICES.rhel/ diff --git a/collector/container/Dockerfile b/collector/container/Dockerfile index d57c76747f..a1c45b8721 100644 --- a/collector/container/Dockerfile +++ b/collector/container/Dockerfile @@ -32,7 +32,7 @@ RUN apt-get update \ libuuid1 \ libcap-ng0 \ libelf1 \ - google-perftools \ + libgoogle-perftools4 \ && dpkg -r --force-all e2fslibs e2fsprogs apt debconf dpkg \ && rm -rf /var/lib/apt \ ; diff --git a/collector/lib/CollectorService.cpp b/collector/lib/CollectorService.cpp index 7de4359789..519c6b40a0 100644 --- a/collector/lib/CollectorService.cpp +++ b/collector/lib/CollectorService.cpp @@ -36,6 +36,7 @@ extern "C" { #include "GetStatus.h" #include "LogLevel.h" #include "NetworkStatusNotifier.h" +#include "ProfilerHandler.h" #include "SysdigService.h" #include "Utility.h" #include "civetweb/CivetServer.h" @@ -67,6 +68,9 @@ void CollectorService::RunForever() { LogLevel setLogLevel; server.addHandler("/loglevel", setLogLevel); + ProfilerHandler profiler_handler; + server.addHandler(ProfilerHandler::kBaseRoute, profiler_handler); + prometheus::Exposer exposer("9090"); exposer.RegisterCollectable(registry); diff --git a/collector/lib/NetworkStatusNotifier.cpp b/collector/lib/NetworkStatusNotifier.cpp index e4f3b54a8f..37591c4c80 100644 --- a/collector/lib/NetworkStatusNotifier.cpp +++ b/collector/lib/NetworkStatusNotifier.cpp @@ -28,6 +28,7 @@ You should have received a copy of the GNU General Public License along with thi #include "CollectorStats.h" #include "DuplexGRPC.h" #include "GRPCUtil.h" +#include "Profiler.h" #include "ProtoUtil.h" #include "TimeUtil.h" #include "Utility.h" @@ -141,6 +142,7 @@ void NetworkStatusNotifier::ReceiveIPNetworks(const sensor::IPNetworkList& netwo } void NetworkStatusNotifier::Run() { + Profiler::RegisterCPUThread(); auto next_attempt = std::chrono::system_clock::now(); while (thread_.PauseUntil(next_attempt)) { diff --git a/collector/lib/Profiler.h b/collector/lib/Profiler.h new file mode 100644 index 0000000000..b85dfdc050 --- /dev/null +++ b/collector/lib/Profiler.h @@ -0,0 +1,66 @@ +#ifndef COLLECTOR_PROFILER_H +#define COLLECTOR_PROFILER_H + +#include +#include + +struct Free { + void operator()(const void* p) const { free((void*)p); } +}; +using MallocUniquePtr = std::unique_ptr; + +#ifdef COLLECTOR_PROFILING +# include "gperftools/heap-profiler.h" +# include "gperftools/profiler.h" + +namespace collector { +class Profiler { + public: + static inline void RegisterCPUThread() { ProfilerRegisterThread(); } + static inline MallocUniquePtr AllocAndGetHeapProfile() { + if (!IsHeapProfilerEnabled()) { + return nullptr; + } + return MallocUniquePtr(GetHeapProfile()); + } + static inline bool StartCPUProfiler(const std::string& output_dir) { + return ProfilerStart(output_dir.c_str()); + } + static inline void StopCPUProfiler() { + ProfilerFlush(); + ProfilerStop(); + } + static inline void StartHeapProfiler() { HeapProfilerStart(""); } + static inline void StopHeapProfiler() { + if (!IsHeapProfilerEnabled()) { + return; + } + HeapProfilerStop(); + } + static inline bool IsCPUProfilerEnabled() { return ProfilingIsEnabledForAllThreads(); } + static inline bool IsCPUProfilerSupported() { return true; } + static inline bool IsHeapProfilerEnabled() { return IsHeapProfilerRunning(); } + static inline bool IsHeapProfilerSupported() { return true; } +}; + +} // namespace collector +#else + +namespace collector { +class Profiler { + public: + static inline void RegisterCPUThread() {} + static inline MallocUniquePtr AllocAndGetHeapProfile() { return MallocUniquePtr(nullptr); } + static inline bool StartCPUProfiler(const std::string& output_dir) { return false; } + static inline void StopCPUProfiler() {} + static inline void StartHeapProfiler() {} + static inline void StopHeapProfiler() {} + static inline bool IsCPUProfilerSupported() { return false; } + static inline bool IsCPUProfilerEnabled() { return false; } + static inline bool IsHeapProfilerSupported() { return false; } + static inline bool IsHeapProfilerEnabled() { return false; } +}; +} // namespace collector +#endif + +#endif //COLLECTOR_PROFILER_H diff --git a/collector/lib/ProfilerHandler.cpp b/collector/lib/ProfilerHandler.cpp new file mode 100644 index 0000000000..5855620ee9 --- /dev/null +++ b/collector/lib/ProfilerHandler.cpp @@ -0,0 +1,167 @@ +#include "ProfilerHandler.h" + +#include +#include +#include + +#include +#include + +#include "Logging.h" +#include "Profiler.h" +#include "Utility.h" + +namespace collector { + +const std::string ProfilerHandler::kCPUProfileFilename = "/module/cpu_profile"; +const std::string ProfilerHandler::kBaseRoute = "/profile"; +const std::string ProfilerHandler::kCPURoute = kBaseRoute + "/cpu"; +const std::string ProfilerHandler::kHeapRoute = kBaseRoute + "/heap"; + +bool ProfilerHandler::ServerError(struct mg_connection* conn, const char* err) { + return mg_send_http_error(conn, 500, err) >= 0; +} + +bool ProfilerHandler::ClientError(struct mg_connection* conn, const char* err) { + return mg_send_http_error(conn, 400, err) >= 0; +} + +bool ProfilerHandler::SendCPUProfile(struct mg_connection* conn) { + WITH_LOCK(mutex_) { + if (cpu_profile_length_ == 0) { + return mg_send_http_ok(conn, "text/plain", 0) >= 0; + } + mg_send_mime_file(conn, kCPUProfileFilename.c_str(), "application/octet-stream"); + } + return true; +} + +bool ProfilerHandler::SendHeapProfile(struct mg_connection* conn) { + WITH_LOCK(mutex_) { + if (heap_profile_length_ == 0) { + return mg_send_http_ok(conn, "text/plain", 0) >= 0; + } + if (mg_send_http_ok(conn, "application/octet-stream", heap_profile_length_) < 0) { + return false; + } + if (mg_write(conn, heap_profile_.get(), heap_profile_length_) != heap_profile_length_) { + return false; + } + } + return true; +} + +bool ProfilerHandler::SendStatus(struct mg_connection* conn) { + Json::Value resp(Json::objectValue); + WITH_LOCK(mutex_) { + resp["supports_cpu"] = Profiler::IsCPUProfilerSupported(); + resp["supports_heap"] = Profiler::IsHeapProfilerSupported(); + resp["cpu"] = Profiler::IsCPUProfilerEnabled() ? "on" : (cpu_profile_length_ > 0 ? "off" : "empty"); + resp["heap"] = Profiler::IsHeapProfilerEnabled() ? "on" : (heap_profile_length_ > 0 ? "off" : "empty"); + } + std::string json_body = resp.toStyledString(); + if (mg_send_http_ok(conn, "application/json", json_body.length()) < 0) { + return false; + } + return mg_write(conn, json_body.c_str(), json_body.length()) >= 0; +} + +bool ProfilerHandler::HandleCPURoute(struct mg_connection* conn, const std::string& post_data) { + WITH_LOCK(mutex_) { + if (post_data == "on") { + if (!Profiler::IsCPUProfilerEnabled()) { + if (!Profiler::StartCPUProfiler(kCPUProfileFilename)) { + return ServerError(conn, "failed starting cpu profiler"); + } + CLOG(INFO) << "started cpu profiler"; + } + } else if (post_data == "off") { + if (Profiler::IsCPUProfilerEnabled()) { + Profiler::StopCPUProfiler(); + struct stat sdata; + if (stat(kCPUProfileFilename.c_str(), &sdata) == 0) { + cpu_profile_length_ = sdata.st_size; + } + CLOG(INFO) << "stopped cpu profiler, bytes=" << cpu_profile_length_; + } + } else if (post_data == "empty") { + if (cpu_profile_length_ != 0) { + cpu_profile_length_ = 0; + if (std::remove(kCPUProfileFilename.c_str()) != 0) { + CLOG(INFO) << "remove failed: " << StrError(); + return ServerError(conn, "failure while deleting cpu profile"); + } + CLOG(INFO) << "cleared cpu profile"; + } + } else { + return ClientError(conn, "invalid post data"); + } + } + return mg_send_http_ok(conn, "text/plain", 0) >= 0; +} + +bool ProfilerHandler::HandleHeapRoute(struct mg_connection* conn, const std::string& post_data) { + WITH_LOCK(mutex_) { + if (post_data == "on") { + if (!Profiler::IsHeapProfilerEnabled()) { + Profiler::StartHeapProfiler(); + CLOG(INFO) << "started heap profiler"; + } + } else if (post_data == "off") { + if (Profiler::IsHeapProfilerEnabled()) { + heap_profile_ = Profiler::AllocAndGetHeapProfile(); + if (heap_profile_ == nullptr) { + return ServerError(conn, "failed get heap profile"); + } + heap_profile_length_ = strlen(heap_profile_.get()); + Profiler::StopHeapProfiler(); + CLOG(INFO) << "stopped heap profiler, bytes=" << heap_profile_length_; + } + } else if (post_data == "empty") { + if (heap_profile_length_ != 0) { + heap_profile_length_ = 0; + heap_profile_.reset(); + CLOG(INFO) << "cleared heap profile"; + } + } else { + return ClientError(conn, "invalid post data"); + } + } + return mg_send_http_ok(conn, "text/plain", 0) >= 0; +} + +bool ProfilerHandler::handlePost(CivetServer* server, struct mg_connection* conn) { + if (!Profiler::IsCPUProfilerSupported()) { + return ServerError(conn, "not supported"); + } + const mg_request_info* req_info = mg_get_request_info(conn); + if (req_info == nullptr) { + return ServerError(conn, "unable to read request"); + } + std::string uri(req_info->local_uri); + std::string post_data(server->getPostData(conn)); + if (uri == kCPURoute) { + return HandleCPURoute(conn, post_data); + } else if (uri == kHeapRoute) { + return HandleHeapRoute(conn, post_data); + } + return ClientError(conn, "unknown route"); +} + +bool ProfilerHandler::handleGet(CivetServer* server, struct mg_connection* conn) { + const mg_request_info* req_info = mg_get_request_info(conn); + if (req_info == nullptr) { + return ServerError(conn, "unable to read request"); + } + std::string uri = req_info->local_uri; + if (uri == kBaseRoute) { + return SendStatus(conn); + } else if (uri == kHeapRoute) { + return SendHeapProfile((conn)); + } else if (uri == kCPURoute) { + return SendCPUProfile(conn); + } + return ClientError(conn, "unknown route"); +} + +} // namespace collector diff --git a/collector/lib/ProfilerHandler.h b/collector/lib/ProfilerHandler.h new file mode 100644 index 0000000000..6c6e368f9c --- /dev/null +++ b/collector/lib/ProfilerHandler.h @@ -0,0 +1,48 @@ +#ifndef COLLECTOR_PROFILERHANDLER_H +#define COLLECTOR_PROFILERHANDLER_H + +#include +#include + +#include "Profiler.h" +#include "civetweb/CivetServer.h" + +namespace collector { + +// Profiling Endpoints: +// POST /profile/cpu +// - accepts post data of on|off|empty to enable, disable or clear/delete the latest cpu profile +// POST /profile/heap +// - accepts post data of on|off|empty to enable, disable or clear/delete the latest heap profile +// GET /profile +// - get profiling support and status +// GET /profile/cpu +// - get latest cpu profile +// GET /profile/heap +// - get latest heap profile +class ProfilerHandler : public CivetHandler { + public: + static const std::string kCPUProfileFilename; + static const std::string kBaseRoute; + static const std::string kCPURoute; + static const std::string kHeapRoute; + + bool handleGet(CivetServer* server, struct mg_connection* conn); + bool handlePost(CivetServer* server, struct mg_connection* conn); + + private: + bool ServerError(struct mg_connection* conn, const char* err); + bool ClientError(struct mg_connection* conn, const char* err); + bool SendStatus(struct mg_connection* conn); + bool SendHeapProfile(struct mg_connection* conn); + bool SendCPUProfile(struct mg_connection* conn); + bool HandleCPURoute(struct mg_connection* conn, const std::string& post_data); + bool HandleHeapRoute(struct mg_connection* conn, const std::string& post_data); + MallocUniquePtr heap_profile_; + size_t heap_profile_length_; + size_t cpu_profile_length_; + std::mutex mutex_; +}; +} // namespace collector + +#endif //COLLECTOR_PROFILERHANDLER_H