From b19466ba00eed4646ac7b98f06432bda1e297225 Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Fri, 11 Apr 2025 14:25:21 +0000 Subject: [PATCH 1/6] workaround --- src/datadog/datadog_agent.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/datadog/datadog_agent.cpp b/src/datadog/datadog_agent.cpp index bd0f5c0d..ac2d27d4 100644 --- a/src/datadog/datadog_agent.cpp +++ b/src/datadog/datadog_agent.cpp @@ -185,8 +185,11 @@ DatadogAgent::~DatadogAgent() { Expected DatadogAgent::send( std::vector>&& spans, const std::shared_ptr& response_handler) { - std::lock_guard lock(mutex_); - trace_chunks_.push_back(TraceChunk{std::move(spans), response_handler}); + { + std::lock_guard lock(mutex_); + trace_chunks_.push_back(TraceChunk{std::move(spans), response_handler}); + } + flush(); return nullopt; } From 0ca941ee261757b49d5dc4a17c50e6e62805c64c Mon Sep 17 00:00:00 2001 From: elsa Date: Mon, 28 Apr 2025 14:08:44 +0000 Subject: [PATCH 2/6] feat: write process storage * CMakeLists.txt * include/datadog/environment.h * include/datadog/tracer.h * include/datadog/tracer_config.h * include/datadog/tracer_signature.h * src/datadog/tracer.cpp * src/datadog/tracer_config.cpp --- CMakeLists.txt | 1 + include/datadog/environment.h | 1 + include/datadog/tracer.h | 1 + include/datadog/tracer_config.h | 9 +++++++ include/datadog/tracer_signature.h | 43 ++++++++++++++++++++++++++++++ src/datadog/tracer.cpp | 9 ++++++- src/datadog/tracer_config.cpp | 9 +++++++ 7 files changed, 72 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ecbf5612..8b9126f4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -203,6 +203,7 @@ if (BUILD_SHARED_LIBS) PUBLIC dd_trace::obj CURL::libcurl_shared + customlabels PRIVATE dd_trace::specs ) diff --git a/include/datadog/environment.h b/include/datadog/environment.h index 266737dc..efeabe1f 100644 --- a/include/datadog/environment.h +++ b/include/datadog/environment.h @@ -50,6 +50,7 @@ namespace environment { MACRO(DD_TRACE_TAGS_PROPAGATION_MAX_LENGTH) \ MACRO(DD_VERSION) \ MACRO(DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED) \ + MACRO(DD_TRACE_CORRELATE_FULL_HOST_PROFILES) \ MACRO(DD_TELEMETRY_HEARTBEAT_INTERVAL) \ MACRO(DD_TELEMETRY_METRICS_ENABLED) \ MACRO(DD_TELEMETRY_METRICS_INTERVAL_SECONDS) \ diff --git a/include/datadog/tracer.h b/include/datadog/tracer.h index 5c183795..d9bb43e0 100644 --- a/include/datadog/tracer.h +++ b/include/datadog/tracer.h @@ -54,6 +54,7 @@ class Tracer { Baggage::Options baggage_opts_; bool baggage_injection_enabled_; bool baggage_extraction_enabled_; + bool correlate_full_host_profiles_; public: // Create a tracer configured using the specified `config`, and optionally: diff --git a/include/datadog/tracer_config.h b/include/datadog/tracer_config.h index ab8608d0..af4be8ca 100644 --- a/include/datadog/tracer_config.h +++ b/include/datadog/tracer_config.h @@ -133,6 +133,14 @@ struct TracerConfig { // the `DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED` environment variable. Optional generate_128bit_trace_ids; + // `correlate_full_host_profiles` indicates whether we want to correlate + // traces and spans with profiles generated by the eBPF full host profiler. + // This correlation only works on linux, due to the eBPF-based nature of + // the profiler. It implies writing some process-level and thread-level + // data in variables which the profiler will then read from the process's + // memory. + Optional correlate_full_host_profiles; + // `runtime_id` denotes the current run of the application in which the tracer // is embedded. If `runtime_id` is not specified, then it defaults to a // pseudo-randomly generated value. A server that contains multiple tracers, @@ -197,6 +205,7 @@ class FinalizedTracerConfig final { std::shared_ptr logger; bool log_on_startup; bool generate_128bit_trace_ids; + bool correlate_full_host_profiles; Optional runtime_id; Clock clock; std::string integration_name; diff --git a/include/datadog/tracer_signature.h b/include/datadog/tracer_signature.h index 042b4d37..597d50c4 100644 --- a/include/datadog/tracer_signature.h +++ b/include/datadog/tracer_signature.h @@ -19,6 +19,14 @@ // polling the Datadog Agent. See // `RemoteConfigurationManager::process_response` in `remote_config.h`. +#ifdef __linux__ +#include +#include +extern "C" { +#include +} +#endif + #include #include "runtime_id.h" @@ -31,6 +39,17 @@ namespace datadog { namespace tracing { +#ifdef __linux__ +namespace { +void write_utf8_string(std::vector& buffer, const std::string& str) { + uint32_t length = str.length(); + buffer.insert(buffer.end(), reinterpret_cast(&length), + reinterpret_cast(&length) + sizeof(length)); + buffer.insert(buffer.end(), str.begin(), str.end()); +} +} // namespace +#endif + struct TracerSignature { RuntimeID runtime_id; std::string default_service; @@ -47,6 +66,30 @@ struct TracerSignature { library_version(tracer_version), library_language("cpp"), library_language_version(DD_TRACE_STRINGIFY(__cplusplus), 6) {} + +#ifdef __linux__ + // The process correlation storage contains information needed to + // correlate traces to profiles generated by dd-otel-host-profiler. + void generate_process_correlation_storage() { + std::vector buffer; + + // Currently, layout minor version is 2 to differ from Elastic's + // version which includes a socket path. + // Layout: + // https://github.com/elastic/apm/blob/149cd3e39a77a58002344270ed2ad35357bdd02d/specs/agents/universal-profiling-integration.md#process-storage-layout + uint16_t layout_minor_version = 2; + buffer.insert(buffer.end(), + reinterpret_cast(&layout_minor_version), + reinterpret_cast(&layout_minor_version) + + sizeof(layout_minor_version)); + + write_utf8_string(buffer, default_service); + write_utf8_string(buffer, default_environment); + write_utf8_string(buffer, runtime_id.string()); + + proc_storage_set(buffer.data(), buffer.size()); + } +#endif }; } // namespace tracing diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index e0f34a1e..cb8fd965 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -58,7 +58,8 @@ Tracer::Tracer(const FinalizedTracerConfig& config, tags_header_max_size_(config.tags_header_size), baggage_opts_(config.baggage_opts), baggage_injection_enabled_(false), - baggage_extraction_enabled_(false) { + baggage_extraction_enabled_(false), + correlate_full_host_profiles_(config.correlate_full_host_profiles) { telemetry::init(config.telemetry, logger_, config.http_client, config.event_scheduler, config.agent_url); if (config.report_hostname) { @@ -92,6 +93,12 @@ Tracer::Tracer(const FinalizedTracerConfig& config, } } +#ifdef __linux__ + // TODO: change the way this is done to handle programs that fork. + if (correlate_full_host_profiles_) + signature_.generate_process_correlation_storage(); +#endif + if (config.log_on_startup) { logger_->log_startup([configuration = this->config()](std::ostream& log) { log << "DATADOG TRACER CONFIGURATION - " << configuration; diff --git a/src/datadog/tracer_config.cpp b/src/datadog/tracer_config.cpp index 629cd130..bf685ae9 100644 --- a/src/datadog/tracer_config.cpp +++ b/src/datadog/tracer_config.cpp @@ -127,6 +127,10 @@ Expected load_tracer_env_config(Logger &logger) { lookup(environment::DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED)) { env_cfg.generate_128bit_trace_ids = !falsy(*enabled_env); } + if (auto enabled_env = + lookup(environment::DD_TRACE_CORRELATE_FULL_HOST_PROFILES)) { + env_cfg.correlate_full_host_profiles = !falsy(*enabled_env); + } // Baggage if (auto baggage_items_env = @@ -361,6 +365,11 @@ Expected finalize_config(const TracerConfig &user_config, ConfigMetadata(ConfigName::GENEREATE_128BIT_TRACE_IDS, to_string(final_config.generate_128bit_trace_ids), origin); + // Correlate with Full Host Profiles + final_config.correlate_full_host_profiles = + value_or(env_config->correlate_full_host_profiles, + user_config.correlate_full_host_profiles, false); + // Integration name & version final_config.integration_name = value_or( env_config->integration_name, user_config.integration_name, "datadog"); From 9dca1839b91e393a8c917dbf449c6a0d1021d859 Mon Sep 17 00:00:00 2001 From: elsa Date: Mon, 28 Apr 2025 14:14:42 +0000 Subject: [PATCH 3/6] feat: use custom labels to share trace & span ids * include/datadog/tracer.h * src/datadog/otel_identifiers.h * src/datadog/span.cpp * src/datadog/trace_segment.cpp * src/datadog/tracer.cpp --- include/datadog/tracer.h | 3 +++ src/datadog/otel_identifiers.h | 12 ++++++++++++ src/datadog/span.cpp | 15 +++++++++++++++ src/datadog/trace_segment.cpp | 15 +++++++++++++++ src/datadog/tracer.cpp | 32 ++++++++++++++++++++++++++++++++ 5 files changed, 77 insertions(+) create mode 100644 src/datadog/otel_identifiers.h diff --git a/include/datadog/tracer.h b/include/datadog/tracer.h index d9bb43e0..d88694db 100644 --- a/include/datadog/tracer.h +++ b/include/datadog/tracer.h @@ -106,6 +106,9 @@ class Tracer { std::string config() const; private: +#ifdef __linux__ + void correlate(const Span& span); +#endif void store_config(); }; diff --git a/src/datadog/otel_identifiers.h b/src/datadog/otel_identifiers.h new file mode 100644 index 00000000..dfcc2c2f --- /dev/null +++ b/src/datadog/otel_identifiers.h @@ -0,0 +1,12 @@ +#pragma once + +namespace datadog { +namespace tracing { + +// Based on the OTel Trace API +// https://opentelemetry.io/docs/specs/otel/trace/api +#define OTEL_TRACE_ID_IDENTIFIER "TraceId" +#define OTEL_SPAN_ID_IDENTIFIER "SpanId" + +} // namespace tracing +} // namespace datadog diff --git a/src/datadog/span.cpp b/src/datadog/span.cpp index f7419841..93c0af55 100644 --- a/src/datadog/span.cpp +++ b/src/datadog/span.cpp @@ -4,10 +4,12 @@ #include #include #include +#include #include #include +#include "otel_identifiers.h" #include "span_data.h" #include "tags.h" @@ -40,6 +42,19 @@ Span::~Span() { data_->duration = now - data_->start; } +#ifdef __linux__ + // When a span is finished, we must update the span_id to its parent's. + if (process_storage != nullptr && parent_id().has_value()) { + auto span_id = std::to_string(parent_id().value()); + custom_labels_labelset_set( + custom_labels_current_set, + {sizeof(OTEL_SPAN_ID_IDENTIFIER), + reinterpret_cast(OTEL_SPAN_ID_IDENTIFIER)}, + {span_id.size(), + reinterpret_cast(span_id.c_str())}); + } +#endif + trace_segment_->span_finished(); } diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index b7f74c0f..0c0edd77 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -1,3 +1,9 @@ +#ifdef __linux__ +extern "C" { +#include +} +#endif + #include #include #include @@ -254,6 +260,15 @@ void TraceSegment::span_finished() { } } +#ifdef __linux__ + // When all spans are finished, so is the current trace. + if (process_storage != nullptr) { + auto saved_set = custom_labels_current_set; + custom_labels_current_set = nullptr; + custom_labels_labelset_free(saved_set); + } +#endif + telemetry::counter::increment(metrics::tracer::trace_segments_closed); } diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index cb8fd965..628ab4a0 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -21,6 +21,7 @@ #include "hex.h" #include "json.hpp" #include "msgpack.h" +#include "otel_identifiers.h" #include "platform_util.h" #include "random.h" #include "span_data.h" @@ -108,6 +109,27 @@ Tracer::Tracer(const FinalizedTracerConfig& config, store_config(); } +#ifdef __linux__ +void Tracer::correlate(const Span& span) { + custom_labels_current_set = custom_labels_labelset_new(2); + auto trace_id = span.trace_id().hex_padded(); + custom_labels_labelset_set( + custom_labels_current_set, + {sizeof(OTEL_TRACE_ID_IDENTIFIER), + reinterpret_cast(OTEL_TRACE_ID_IDENTIFIER)}, + {trace_id.size(), + reinterpret_cast(trace_id.c_str())}); + + auto span_id = std::to_string(span.id()); + custom_labels_labelset_set( + custom_labels_current_set, + {sizeof(OTEL_SPAN_ID_IDENTIFIER), + reinterpret_cast(OTEL_SPAN_ID_IDENTIFIER)}, + {span_id.size(), + reinterpret_cast(span_id.c_str())}); +} +#endif + std::string Tracer::config() const { // clang-format off auto config = nlohmann::json::object({ @@ -199,6 +221,11 @@ Span Tracer::create_span(const SpanConfig& config) { Span span{span_data_ptr, segment, [generator = generator_]() { return generator->span_id(); }, clock_}; + +#ifdef __linux__ + if (correlate_full_host_profiles_) correlate(span); +#endif + return span; } @@ -410,6 +437,11 @@ Expected Tracer::extract_span(const DictReader& reader, Span span{span_data_ptr, segment, [generator = generator_]() { return generator->span_id(); }, clock_}; + +#ifdef __linux__ + if (correlate_full_host_profiles_) correlate(span); +#endif + return span; } From 83920d229ab59a63e4df27c76d04c73a953f6868 Mon Sep 17 00:00:00 2001 From: elsa Date: Tue, 29 Apr 2025 15:34:58 +0000 Subject: [PATCH 4/6] fix: include customlabels in cmake deps * CMakeLists.txt * cmake/deps/customlabels.cmake * include/datadog/tracer_signature.h * src/datadog/trace_segment.cpp --- CMakeLists.txt | 27 ++++++++++++++++++++- cmake/deps/customlabels.cmake | 38 ++++++++++++++++++++++++++++++ include/datadog/tracer_signature.h | 2 +- src/datadog/trace_segment.cpp | 2 +- 4 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 cmake/deps/customlabels.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b9126f4..6b2d9b7f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,11 @@ endif () set(DD_TRACE_TRANSPORT "curl" CACHE STRING "HTTP transport that dd-trace-cpp uses to communicate with the Datadog Agent, can be either 'none' or 'curl'") +if (CMAKE_SYSTEM_NAME STREQUAL "Linux") + include(cmake/deps/customlabels.cmake) + message(STATUS "customlabels Library is available") +endif() + if(DD_TRACE_TRANSPORT STREQUAL "curl") include(cmake/deps/curl.cmake) message(STATUS "DD_TRACE_TRANSPORT is set to 'curl', including curl") @@ -203,11 +208,17 @@ if (BUILD_SHARED_LIBS) PUBLIC dd_trace::obj CURL::libcurl_shared - customlabels PRIVATE dd_trace::specs ) + if (CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_link_libraries(dd_trace_cpp-shared + PUBLIC + customlabels + ) + endif () + install(TARGETS dd_trace_cpp-objects dd_trace_cpp-shared EXPORT dd_trace_cpp-export FILE_SET public_headers DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} @@ -250,6 +261,13 @@ if (BUILD_STATIC_LIBS) dd_trace::specs ) + if (CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_link_libraries(dd_trace_cpp-static + PUBLIC + customlabels + ) + endif () + install(TARGETS dd_trace_cpp-objects dd_trace_cpp-static EXPORT dd_trace_cpp-export FILE_SET public_headers DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} @@ -258,3 +276,10 @@ if (BUILD_STATIC_LIBS) RUNTIME DESTINATION ${CMAKE_INSTALL_LIBDIR} ) endif () + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_include_directories(dd_trace_cpp-objects + PRIVATE + ${customlabels_SOURCE_DIR}/src + ) +endif() diff --git a/cmake/deps/customlabels.cmake b/cmake/deps/customlabels.cmake new file mode 100644 index 00000000..cd24924b --- /dev/null +++ b/cmake/deps/customlabels.cmake @@ -0,0 +1,38 @@ +include(FetchContent) + +FetchContent_Declare(customlabels + GIT_REPOSITORY https://github.com/DataDog/custom-labels.git + GIT_TAG elsa/add-process-storage +) + +FetchContent_MakeAvailable(customlabels) + +add_library(customlabels SHARED + ${customlabels_SOURCE_DIR}/src/customlabels.c +) + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux") + if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") + set(TLS_DIALECT desc) + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64") + set(TLS_DIALECT gnu2) + else() + message(FATAL_ERROR "Only aarch64 and x86-64 are supported (found: ${CMAKE_SYSTEM_PROCESSOR})") + endif() + + include(CheckCompilerFlag) + check_compiler_flag(CXX "-ftls-dialect=${TLS_DIALECT}" TLS_DIALECT_OK) + if (TLS_DIALECT_OK) + target_compile_options(customlabels PRIVATE + -g + -fPIC + -ftls-model=global-dynamic + -mtls-dialect=${TLS_DIALECT} + ) + endif() +endif() + +target_include_directories(customlabels +PUBLIC + ${customlabels_SOURCE_DIR}/src +) diff --git a/include/datadog/tracer_signature.h b/include/datadog/tracer_signature.h index 597d50c4..00ade8d4 100644 --- a/include/datadog/tracer_signature.h +++ b/include/datadog/tracer_signature.h @@ -23,7 +23,7 @@ #include #include extern "C" { -#include +#include } #endif diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index 0c0edd77..f442c027 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -1,6 +1,6 @@ #ifdef __linux__ extern "C" { -#include +#include } #endif From 3f45b5dcfaadd842a78aceb94f0af699e10fe393 Mon Sep 17 00:00:00 2001 From: elsa Date: Wed, 30 Apr 2025 13:24:51 +0000 Subject: [PATCH 5/6] fix: bazel build with customlabels * BUILD.bazel * MODULE.bazel --- BUILD.bazel | 11 ++++++++++- MODULE.bazel | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/BUILD.bazel b/BUILD.bazel index a95c9d9d..91dde1c6 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,3 +1,8 @@ +config_setting( + name = "is_linux", + constraint_values = ["@platforms//os:linux"], +) + cc_library( name = "dd_trace_cpp", srcs = [ @@ -67,6 +72,7 @@ cc_library( "src/datadog/limiter.h", "src/datadog/msgpack.h", "src/datadog/null_logger.h", + "src/datadog/otel_identifiers.h", "src/datadog/parse_util.h", "src/datadog/platform_util.h", "src/datadog/random.h", @@ -134,5 +140,8 @@ cc_library( deps = [ "@com_google_absl//absl/strings", "@com_google_absl//absl/types:optional", - ], + ] + select({ + ":is_linux": ["@customlabels//:customlabels"], + "//conditions:default": [], + }), ) diff --git a/MODULE.bazel b/MODULE.bazel index ebdca538..b3bee6c2 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -11,6 +11,14 @@ bazel_dep( name = "rules_cc", version = "0.0.9", ) +bazel_dep( + name = "customlabels", + version = "1.0.0", +) +bazel_dep( + name = "platforms", + version = "0.0.11", +) # -- bazel_dep definitions -- # non_module_dependencies = use_extension("//:extensions.bzl", "non_module_dependencies") @@ -18,3 +26,9 @@ use_repo( non_module_dependencies, "com_google_absl", ) + +git_override( + module_name = "customlabels", + remote = "https://github.com/DataDog/custom-labels.git", + commit = "1bc74232c0e414f2c63139f7cf7ac0bdce57d258", +) From 46dd99b92384b09109580a7833a8e6e500496c36 Mon Sep 17 00:00:00 2001 From: elsa Date: Wed, 7 May 2025 16:05:34 +0000 Subject: [PATCH 6/6] fix: init storage before telemetry init The telemetry initialization now takes a few seconds. This delay is problematic for trace-to-profile correlation since it delays the initialization of the process storage, which the eBPF profiler checks once when it first encounters the process. * src/datadog/tracer.cpp --- src/datadog/tracer.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index 628ab4a0..fa62dc9f 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -61,6 +61,12 @@ Tracer::Tracer(const FinalizedTracerConfig& config, baggage_injection_enabled_(false), baggage_extraction_enabled_(false), correlate_full_host_profiles_(config.correlate_full_host_profiles) { +#ifdef __linux__ + // TODO: change the way this is done to handle programs that fork. + if (correlate_full_host_profiles_) + signature_.generate_process_correlation_storage(); +#endif + telemetry::init(config.telemetry, logger_, config.http_client, config.event_scheduler, config.agent_url); if (config.report_hostname) { @@ -94,12 +100,6 @@ Tracer::Tracer(const FinalizedTracerConfig& config, } } -#ifdef __linux__ - // TODO: change the way this is done to handle programs that fork. - if (correlate_full_host_profiles_) - signature_.generate_process_correlation_storage(); -#endif - if (config.log_on_startup) { logger_->log_startup([configuration = this->config()](std::ostream& log) { log << "DATADOG TRACER CONFIGURATION - " << configuration;