feat(config): support libdatadog's library_config for config through file

ddtrace/__init__.py

@@ -1,9 +1,19 @@
 import sys
+import os
 import warnings
 
 
 LOADED_MODULES = frozenset(sys.modules.keys())
 
+# Configuration for the whole tracer from file. Do it before anything else happens.
+from ddtrace.internal.datadog.profiling import library_config
+
+library_config.set_envp(["%s=%s" % (k, v) for k, v in os.environ.items()])
+library_config.set_args(sys.argv)
+for key, value in library_config.get_config(debug_logs=False).items():
+    os.environ[key] = str(value).lower()
+
+
 from ddtrace.internal.module import ModuleWatchdog
 
 

ddtrace/internal/datadog/profiling/build_standalone.sh

@@ -78,6 +78,7 @@ target_dirs["ddup"]="ddup"
 target_dirs["crashtracker"]="crashtracker"
 target_dirs["stack_v2"]="stack_v2"
 target_dirs["dd_wrapper"]="dd_wrapper"
+target_dirs["library_config"]="library_config"
 
 # Compiler options
 declare -A compiler_args
@@ -219,6 +220,8 @@ print_help() {
   echo "  ddup_test (also builds dd_wrapper_test)"
   echo "  crashtracker (also builds dd_wrapper)"
   echo "  crashtracker_test (also builds dd_wrapper_test)"
+  echo "  library_config (also builds dd_wrapper)"
+  echo "  library_config_test (also builds dd_wrapper_test)"
 }
 
 print_cmake_args() {
@@ -343,6 +346,7 @@ add_target() {
       targets+=("stack_v2")
       targets+=("ddup")
       targets+=("crashtracker")
+      targets+=("library_config")
       ;;
     dd_wrapper)
       # `dd_wrapper` is a dependency of other targets, but the overall structure is weird when it's given explicitly
@@ -358,6 +362,9 @@ add_target() {
     crashtracker)
       targets+=("crashtracker")
       ;;
+    library_config)
+      targets+=("library_config")
+      ;;
     *)
       echo "Unknown target: $1"
       exit 1

ddtrace/internal/datadog/profiling/cmake/FindLibdatadog.cmake

@@ -14,20 +14,26 @@ set(FETCHCONTENT_DOWNLOADS_DIR
 include_guard(GLOBAL)
 include(FetchContent)
 
+# Used for local building: set the path to the local libdatadog source
+IF(DEFINED ENV{DD_LIBDATADOG_LOCAL_PATH})
+    set(FETCHCONTENT_SOURCE_DIR_LIBDATADOG "$ENV{DD_LIBDATADOG_LOCAL_PATH}")
+    message(STATUS "Using local libdatadog source at ${FETCHCONTENT_SOURCE_DIR_LIBDATADOG}")
+endif()
+
 # Set version if not already set
 if(NOT DEFINED TAG_LIBDATADOG)
     set(TAG_LIBDATADOG
-        "v14.3.1"
+        "v15.0.0"
         CACHE STRING "libdatadog github tag")
 endif()
 
 if(NOT DEFINED DD_CHECKSUMS)
     set(DD_CHECKSUMS
-        "57f83aff275628bb1af89c22bb4bd696726daf2a9e09b6cd0d966b29e65a7ad6 libdatadog-aarch64-alpine-linux-musl.tar.gz"
-        "2be2efa98dfc32f109abdd79242a8e046a7a300c77634135eb293e000ecd4a4c libdatadog-aarch64-apple-darwin.tar.gz"
-        "36db8d50ccabb71571158ea13835c0f1d05d30b32135385f97c16343cfb6ddd4 libdatadog-aarch64-unknown-linux-gnu.tar.gz"
-        "2f61fd21cf2f8147743e414b4a8c77250a17be3aecc42a69ffe54f0a603d5c92 libdatadog-x86_64-alpine-linux-musl.tar.gz"
-        "f01f05600591063eba4faf388f54c155ab4e6302e5776c7855e3734955f7daf7 libdatadog-x86_64-unknown-linux-gnu.tar.gz")
+        "d5b969b293e5a9e5e36404a553bbafdd55ff6af0b089698bd989a878534df0c7 libdatadog-aarch64-alpine-linux-musl.tar.gz"
+        "4540ffb8ccb671550a39ba79226117086582c1eaf9714180a9e26bd6bb175860 libdatadog-aarch64-apple-darwin.tar.gz"
+        "31bceab4f56873b03b3728760d30e3abc493d32ca8fdc9e1f2ec2147ef4d5424 libdatadog-aarch64-unknown-linux-gnu.tar.gz"
+        "530348c4b02cc7096de2231476ec12db82e2cc6de12a87e5b28af47ea73d4e56 libdatadog-x86_64-alpine-linux-musl.tar.gz"
+        "5073ffc657bc4698f8bdd4935475734577bfb18c54dcbebc4f7d8c7595626e52 libdatadog-x86_64-unknown-linux-gnu.tar.gz")
 endif()
 
 # Determine platform-specific tarball name in a way that conforms to the libdatadog naming scheme in Github releases

ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt

@@ -45,6 +45,8 @@ add_library(
     src/crashtracker.cpp
     src/crashtracker_interface.cpp
     src/ddup_interface.cpp
+    src/library_config.cpp
+    src/library_config_interface.cpp
     src/profile.cpp
     src/receiver_interface.cpp
     src/sample.cpp

ddtrace/internal/datadog/profiling/dd_wrapper/include/library_config.hpp

@@ -0,0 +1,43 @@
+#pragma once
+
+#include <constants.hpp>
+#include <libdatadog_helpers.hpp>
+
+#include <atomic>
+#include <optional>
+#include <vector>
+#include <string_view>
+#include <string>
+
+#include <datadog/library-config.h>
+#include <datadog/common.h>
+
+namespace Datadog {
+
+typedef struct ConfigEntry {
+    std::string key;
+    std::string value;
+} ConfigEntry;
+
+typedef struct ConfigVec {
+    ConfigEntry* ptr;
+    size_t len;
+} ConfigVec;
+
+class LibraryConfig
+{
+  private:
+    std::vector<std::string> _envp;
+    std::vector<std::string> _args;
+    ddog_Slice_CharSlice to_char_slice(const std::vector<std::string>& vec);
+
+  public:
+    // Setters -- these methods are used to set the library context
+    void set_envp(const std::vector<std::string_view>& envp);
+    void set_args(const std::vector<std::string_view>& args);
+
+    // Main method
+    ConfigVec generate_config(bool debug_logs);
+};
+
+} // namespace Datadog

ddtrace/internal/datadog/profiling/dd_wrapper/include/library_config_interface.hpp

@@ -0,0 +1,22 @@
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string_view>
+#include <vector>
+#include <unordered_map>
+#include <library_config.hpp>
+#include <datadog/common.h>
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+    void libraryconfig_set_args(const std::vector<std::string_view>& args);
+    void libraryconfig_set_envp(const std::vector<std::string_view>& envp);
+
+    Datadog::ConfigVec libraryconfig_generate_config(bool debug_logs);
+#ifdef __cplusplus
+} // extern "C"
+
+#endif

ddtrace/internal/datadog/profiling/dd_wrapper/src/crashtracker.cpp

@@ -207,7 +207,7 @@ Datadog::Crashtracker::start()
 
     auto result = ddog_crasht_init(config, receiver_config, metadata);
     ddog_Vec_Tag_drop(tags);
-    if (result.tag != DDOG_CRASHT_RESULT_OK) { // NOLINT (cppcoreguidelines-pro-type-union-access)
+    if (result.tag != DDOG_VOID_RESULT_OK) { // NOLINT (cppcoreguidelines-pro-type-union-access)
         auto err = result.err;                 // NOLINT (cppcoreguidelines-pro-type-union-access)
         std::string errmsg = err_to_msg(&err, "Error initializing crash tracker");
         std::cerr << errmsg << std::endl;

ddtrace/internal/datadog/profiling/dd_wrapper/src/library_config.cpp

@@ -0,0 +1,96 @@
+#include <iostream>
+#include <string>
+#include <string_view>
+#include <sys/stat.h>
+#include <vector>
+#include <library_config.hpp>
+#include <datadog/library-config.h>
+#include <datadog/common.h>
+
+ddog_Slice_CharSlice 
+Datadog::LibraryConfig::to_char_slice(const std::vector<std::string>& vec) {
+    ddog_CharSlice* slices = new ddog_CharSlice[vec.size()];
+    for (size_t i = 0; i < vec.size(); i++) {
+        slices[i] = ddog_CharSlice{vec[i].data(), vec[i].size()};
+    }
+    ddog_Slice_CharSlice result = {slices, vec.size()};
+    return result;
+}
+
+void
+Datadog::LibraryConfig::set_envp(const std::vector<std::string_view>& envp)
+{
+    // Copy string_views to strings to avoid garbage collection
+    std::vector<std::string> new_envp;
+    for (size_t i = 0; i < envp.size(); i++) {
+        if(!envp[i].empty()) {
+            new_envp.push_back(std::string(envp[i]));
+        }
+    }
+    _envp = std::move(new_envp);
+}
+
+void
+Datadog::LibraryConfig::set_args(const std::vector<std::string_view>& args)
+{
+    // Copy string_views to strings to avoid garbage collection
+    std::vector<std::string> new_args;
+    for (size_t i = 0; i < args.size(); i++) {
+        if(!args[i].empty()) {
+            new_args.push_back(std::string(args[i]));
+        }
+    }
+    _args = std::move(new_args);
+}
+
+Datadog::ConfigVec
+Datadog::LibraryConfig::generate_config(bool debug_logs)
+{
+    // Build process info struct
+    ddog_ProcessInfo process_info{
+        .args = to_char_slice(_args),
+        .envp = to_char_slice(_envp),
+        .language = DDOG_CHARSLICE_C("python"),
+    };
+
+    // Build config struct
+    ddog_Configurator *configurator = ddog_library_configurator_new(debug_logs);
+
+    // Compute configs
+    ddog_Result_VecLibraryConfig config_result = ddog_library_configurator_get(configurator, process_info);
+
+    // Check for errors
+    if (config_result.tag == DDOG_RESULT_VEC_LIBRARY_CONFIG_ERR_VEC_LIBRARY_CONFIG) {
+        ddog_Error err = config_result.err;
+        if (debug_logs) {
+            auto ddog_err = ddog_Error_message(&err);
+            std::string err_msg;
+            std::cerr << err_msg.assign(ddog_err.ptr, ddog_err.ptr + ddog_err.len) << std::endl;
+        }
+        ddog_Error_drop(&err);
+        return Datadog::ConfigVec{};
+    }
+
+    // Format to new type
+    ddog_Vec_LibraryConfig configs = config_result.ok;
+    Datadog::ConfigVec result{
+        .ptr = new Datadog::ConfigEntry[configs.len],
+        .len = configs.len,
+    };
+
+    for (size_t i = 0; i < configs.len; i++) {
+        const ddog_LibraryConfig *cfg = &configs.ptr[i];
+        ddog_CStr name = ddog_library_config_name_to_env(cfg->name);
+        const ddog_CString value = cfg->value;
+
+        Datadog::ConfigEntry* new_entry = new Datadog::ConfigEntry{
+            .key = name.ptr,
+            .value = value.ptr,
+        };
+
+        // Add to result
+        result.ptr[i] = *new_entry;
+    };
+
+    return result;
+}

ddtrace/internal/datadog/profiling/dd_wrapper/src/library_config_interface.cpp

@@ -0,0 +1,26 @@
+#include <library_config_interface.hpp>
+#include <library_config.hpp>
+#include <string_view>
+#include <vector>
+#include <unordered_map>
+
+// A global instance of the library config is created here.
+Datadog::LibraryConfig libconfig;
+
+void 
+libraryconfig_set_args(const std::vector<std::string_view>& args) // cppcheck-suppress unusedFunction
+{
+    libconfig.set_args(args);
+}
+
+void 
+libraryconfig_set_envp(const std::vector<std::string_view>& envp) // cppcheck-suppress unusedFunction
+{
+    libconfig.set_envp(envp);
+}
+
+Datadog::ConfigVec
+libraryconfig_generate_config(bool debug_logs) // cppcheck-suppress unusedFunction
+{
+    return libconfig.generate_config(debug_logs);
+}

ddtrace/internal/datadog/profiling/dd_wrapper/src/receiver_interface.cpp

@@ -18,8 +18,8 @@ crashtracker_receiver_entry() // cppcheck-suppress unusedFunction
 {
     // Assumes that this will be called only in the receiver binary, which is a
     // fresh process
-    ddog_crasht_Result new_result = ddog_crasht_receiver_entry_point_stdin();
-    if (new_result.tag != DDOG_CRASHT_RESULT_OK) {
+    ddog_VoidResult new_result = ddog_crasht_receiver_entry_point_stdin();
+    if (new_result.tag != DDOG_VOID_RESULT_OK) {
         ddog_CharSlice message = ddog_Error_message(&new_result.err);
 
         //`write` may not write what we want it to write, but there's nothing we can do about it,

ddtrace/internal/datadog/profiling/library_config/CMakeLists.txt

@@ -0,0 +1,77 @@
+cmake_minimum_required(VERSION 3.19)
+
+# This builds a C-extension for Python using Cython and a native dependency.  The exact name of the extension has
+# consequences for the build path, which is in turn used by setup.py to build the wheel. Otherwise, we have to propagate
+# a lot of state all around. Thus, we use the same name as the Python package (i.e., the caller sets EXTENSION_NAME)
+# library_config is used by a default for standalone/test builds.
+set(EXTENSION_NAME
+    "_library_config.so"
+    CACHE STRING "Name of the extension")
+project(${EXTENSION_NAME})
+message(STATUS "Building extension: ${EXTENSION_NAME}")
+
+# Get the cmake modules for this project
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/../cmake")
+
+# Having a common target in a subdirectory like this is a hack and a mistake, but it's fiddly to change it so we haven't
+# been able to. Instead, make sure that the binary path set in the subdirectory is stable *as a string* in order to make
+# sure the caches work.
+get_filename_component(DD_WRAPPER_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/../dd_wrapper_build ABSOLUTE)
+add_subdirectory(../dd_wrapper ${DD_WRAPPER_BUILD_DIR})
+
+find_package(Python3 COMPONENTS Interpreter Development)
+
+# Make sure we have necessary Python variables
+if(NOT Python3_INCLUDE_DIRS)
+    message(FATAL_ERROR "Python3_INCLUDE_DIRS not found")
+endif()
+
+# If we still don't have a Python executable, we can't continue
+if(NOT Python3_EXECUTABLE)
+    message(FATAL_ERROR "Python executable not found")
+endif()
+
+# This sets some parameters for the target build
+set(ENV{PY_MAJOR_VERSION} ${Python3_VERSION_MAJOR})
+set(ENV{PY_MINOR_VERSION} ${Python3_VERSION_MINOR})
+set(ENV{PY_MICRO_VERSION} ${Python3_VERSION_PATCH})
+
+# Cythonize the .pyx file
+set(DDUP_CPP_SRC ${CMAKE_CURRENT_BINARY_DIR}/_library_config.cpp)
+add_custom_command(
+    OUTPUT ${DDUP_CPP_SRC}
+    COMMAND ${Python3_EXECUTABLE} -m cython ${CMAKE_CURRENT_LIST_DIR}/_library_config.pyx -o ${DDUP_CPP_SRC}
+    DEPENDS ${CMAKE_CURRENT_LIST_DIR}/_library_config.pyx)
+
+# Specify the target C-extension that we want to build
+add_library(${EXTENSION_NAME} SHARED ${DDUP_CPP_SRC})
+
+add_ddup_config(${EXTENSION_NAME})
+# Cython generates code that produces errors for the following, so relax compile options
+target_compile_options(${EXTENSION_NAME} PRIVATE -Wno-old-style-cast -Wno-shadow -Wno-address)
+
+# cmake may mutate the name of the library (e.g., lib- and -.so for dynamic libraries). This suppresses that behavior,
+# which is required to ensure all paths can be inferred correctly by setup.py.
+set_target_properties(${EXTENSION_NAME} PROPERTIES PREFIX "")
+set_target_properties(${EXTENSION_NAME} PROPERTIES SUFFIX "")
+
+# RPATH is needed for sofile discovery at runtime, since Python packages are not installed in the system path. This is
+# typical.
+set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/..")
+target_include_directories(${EXTENSION_NAME} PRIVATE ../dd_wrapper/include ${Datadog_INCLUDE_DIRS}
+                                                     ${Python3_INCLUDE_DIRS})
+
+if(Python3_LIBRARIES)
+    target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper ${Python3_LIBRARIES})
+else()
+    target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper)
+endif()
+
+# Set the output directory for the built library
+if(LIB_INSTALL_DIR)
+    install(
+        TARGETS ${EXTENSION_NAME}
+        LIBRARY DESTINATION ${LIB_INSTALL_DIR}
+        ARCHIVE DESTINATION ${LIB_INSTALL_DIR}
+        RUNTIME DESTINATION ${LIB_INSTALL_DIR})
+endif()