feat(profiling): add DD_PROFILING_API_TIMEOUT_MS

benchmarks/bm/flask_utils.py

@@ -101,7 +101,11 @@ def setup(self):
 
         if self.profiler_enabled:
             os.environ.update(
-                {"DD_PROFILING_ENABLED": "1", "DD_PROFILING_API_TIMEOUT": "0.1", "DD_PROFILING_UPLOAD_INTERVAL": "10"}
+                {
+                    "DD_PROFILING_ENABLED": "1",
+                    "DD_PROFILING_API_TIMEOUT_MS": "100",
+                    "DD_PROFILING_UPLOAD_INTERVAL": "10",
+                }
             )
             if not self.tracer_enabled:
                 import ddtrace.profiling.auto  # noqa:F401

benchmarks/django_simple/scenario.py

@@ -38,7 +38,11 @@ def run(self):
 
         if self.profiler_enabled:
             os.environ.update(
-                {"DD_PROFILING_ENABLED": "1", "DD_PROFILING_API_TIMEOUT": "0.1", "DD_PROFILING_UPLOAD_INTERVAL": "10"}
+                {
+                    "DD_PROFILING_ENABLED": "1",
+                    "DD_PROFILING_API_TIMEOUT_MS": "100",
+                    "DD_PROFILING_UPLOAD_INTERVAL": "10",
+                }
             )
         if self.appsec_enabled:
             os.environ.update({"DD_APPSEC_ENABLED ": "1"})

ddtrace/internal/datadog/profiling/dd_wrapper/include/constants.hpp

@@ -23,3 +23,6 @@ constexpr std::string_view g_language_name = "python";
 
 // Name of the library
 constexpr std::string_view g_library_name = "dd-trace-py";
+
+// Keep this in sync with ddtrace/settings/profiling.py:ProfilingConfig.api_timeout_ms
+constexpr uint64_t g_default_max_timeout_ms = 10'000; // 10 seconds

ddtrace/internal/datadog/profiling/dd_wrapper/include/ddup_interface.hpp

@@ -25,6 +25,7 @@ extern "C"
     void ddup_config_timeline(bool enable);
     void ddup_config_output_filename(std::string_view filename);
     void ddup_config_sample_pool_capacity(uint64_t capacity);
+    void ddup_config_set_max_timeout_ms(uint64_t max_timeout_ms);
 
     void ddup_config_user_tag(std::string_view key, std::string_view val);
     void ddup_config_sample_type(unsigned int type);

ddtrace/internal/datadog/profiling/dd_wrapper/include/uploader_builder.hpp

@@ -25,6 +25,7 @@ class UploaderBuilder
     static inline std::string url{ "http://localhost:8126" };
     static inline ExporterTagset user_tags{};
     static inline std::string output_filename{ "" };
+    static inline uint64_t max_timeout_ms{ g_default_max_timeout_ms };
 
     static constexpr std::string_view language{ g_language_name };
     static constexpr std::string_view family{ g_language_name };
@@ -40,6 +41,7 @@ class UploaderBuilder
     static void set_url(std::string_view _url);
     static void set_tag(std::string_view _key, std::string_view _val);
     static void set_output_filename(std::string_view _output_filename);
+    static void set_max_timeout_ms(uint64_t _max_timeout_ms);
 
     static std::variant<Uploader, std::string> build();
 

ddtrace/internal/datadog/profiling/dd_wrapper/src/ddup_interface.cpp

@@ -127,6 +127,12 @@ ddup_config_sample_pool_capacity(uint64_t capacity) // cppcheck-suppress unusedF
     Datadog::SampleManager::set_sample_pool_capacity(capacity);
 }
 
+void
+ddup_config_set_max_timeout_ms(uint64_t max_timeout_ms)
+{
+    Datadog::UploaderBuilder::set_max_timeout_ms(max_timeout_ms);
+}
+
 bool
 ddup_is_initialized() // cppcheck-suppress unusedFunction
 {

ddtrace/internal/datadog/profiling/dd_wrapper/src/uploader_builder.cpp

@@ -90,6 +90,12 @@ Datadog::UploaderBuilder::set_output_filename(std::string_view _output_filename)
     }
 }
 
+void
+Datadog::UploaderBuilder::set_max_timeout_ms(uint64_t _max_timeout_ms)
+{
+    max_timeout_ms = _max_timeout_ms;
+}
+
 std::string
 join(const std::vector<std::string>& vec, const std::string& delim)
 {
@@ -168,8 +174,6 @@ Datadog::UploaderBuilder::build()
 
     auto ddog_exporter = &res.ok;
 
-    // 5s is a common timeout parameter for Datadog profilers
-    const uint64_t max_timeout_ms = 5000;
     auto set_timeout_result = ddog_prof_Exporter_set_timeout(ddog_exporter, max_timeout_ms);
     if (set_timeout_result.tag == DDOG_VOID_RESULT_ERR) {
         auto& err = set_timeout_result.err;

ddtrace/internal/datadog/profiling/ddup/_ddup.pyi

@@ -14,6 +14,7 @@ def config(
     timeline_enabled: Optional[bool],
     output_filename: Optional[str],
     sample_pool_capacity: Optional[int],
+    timeout: Optional[int],
 ) -> None: ...
 def start() -> None: ...
 def upload(tracer: Optional[Tracer], enable_code_provenance: Optional[bool]) -> None: ...

ddtrace/internal/datadog/profiling/ddup/_ddup.pyx

@@ -57,6 +57,7 @@ cdef extern from "ddup_interface.hpp":
     void ddup_set_runtime_id(string_view _id)
     void ddup_profile_set_endpoints(unordered_map[int64_t, string_view] span_ids_to_endpoints)
     void ddup_profile_add_endpoint_counts(unordered_map[string_view, int64_t] trace_endpoints_to_counts)
+    void ddup_config_set_max_timeout_ms(uint64_t max_timeout_ms)
     bint ddup_upload() nogil
 
     Sample *ddup_start_sample()
@@ -330,7 +331,9 @@ def config(
         max_nframes: Optional[int] = None,
         timeline_enabled: Optional[bool] = None,
         output_filename: StringType = None,
-        sample_pool_capacity: Optional[int] = None) -> None:
+        sample_pool_capacity: Optional[int] = None,
+        timeout: Optional[int] = None,
+) -> None:
 
     # Try to provide a ddtrace-specific default service if one is not given
     service = service or DEFAULT_SERVICE_NAME
@@ -361,6 +364,9 @@ def config(
     if sample_pool_capacity:
         ddup_config_sample_pool_capacity(clamp_to_uint64_unsigned(sample_pool_capacity))
 
+    if timeout is not None:
+        ddup_config_set_max_timeout_ms(clamp_to_uint64_unsigned(timeout))
+
 
 def start() -> None:
     ddup_start()

ddtrace/profiling/profiler.py

@@ -181,6 +181,7 @@ def _build_default_exporters(self):
             timeline_enabled=profiling_config.timeline_enabled,
             output_filename=profiling_config.output_pprof,
             sample_pool_capacity=profiling_config.sample_pool_capacity,
+            timeout=profiling_config.api_timeout_ms,
         )
         ddup.start()
 

ddtrace/settings/profiling.py

@@ -207,7 +207,17 @@ class ProfilingConfig(DDConfig):
         "api_timeout",
         default=10.0,
         help_type="Float",
-        help="The timeout in seconds before dropping events if the HTTP API does not reply",
+        deprecations=[("api_timeout", None, "4.0.0")],
+        help="(Deprecated) The timeout in seconds before dropping events if the HTTP API does not reply. "
+        "Use api_timeout_ms instead.",
+    )
+
+    api_timeout_ms = DDConfig.v(
+        int,
+        "api_timeout_ms",
+        default=10000,
+        help_type="Integer",
+        help="The timeout in milliseconds before dropping events if the HTTP API does not reply",
     )
 
     timeline_enabled = DDConfig.v(

releasenotes/notes/profiling-api-timeout-09bf46873195aff3.yaml

@@ -0,0 +1,7 @@
+---
+fixes:
+  - |
+    profiling: ``DD_PROFILING_API_TIMEOUT`` doesn't have any effect, and
+    is marked to be removed in upcoming 4.0 release. New environment variable
+    ``DD_PROFILING_API_TIMEOUT_MS`` is introduced to configure timeout for
+    uploading profiles to the backend. The default value is 10000 ms (10 seconds)

tests/profiling/collector/conftest.py

@@ -13,7 +13,7 @@ def tracer():
 
 @pytest.fixture
 def profiler(monkeypatch):
-    monkeypatch.setenv("DD_PROFILING_API_TIMEOUT", "0.1")
+    monkeypatch.setenv("DD_PROFILING_API_TIMEOUT_MS", "100")
     p = Profiler()
     p.start()
     try:

tests/profiling/test_main.py

@@ -12,7 +12,7 @@
 
 def test_call_script(monkeypatch):
     # Set a very short timeout to exit fast
-    monkeypatch.setenv("DD_PROFILING_API_TIMEOUT", "0.1")
+    monkeypatch.setenv("DD_PROFILING_API_TIMEOUT_MS", "100")
     monkeypatch.setenv("DD_PROFILING_ENABLED", "1")
     stdout, stderr, exitcode, _ = call_program(
         "ddtrace-run", sys.executable, os.path.join(os.path.dirname(__file__), "simple_program.py")
@@ -28,7 +28,7 @@ def test_call_script(monkeypatch):
 
 @pytest.mark.skipif(not os.getenv("DD_PROFILE_TEST_GEVENT", False), reason="Not testing gevent")
 def test_call_script_gevent(monkeypatch):
-    monkeypatch.setenv("DD_PROFILING_API_TIMEOUT", "0.1")
+    monkeypatch.setenv("DD_PROFILING_API_TIMEOUT_MS", "100")
     stdout, stderr, exitcode, pid = call_program(
         sys.executable, os.path.join(os.path.dirname(__file__), "simple_program_gevent.py")
     )
@@ -58,7 +58,7 @@ def test_call_script_pprof_output(tmp_path, monkeypatch):
 @pytest.mark.skipif(sys.platform == "win32", reason="fork only available on Unix")
 def test_fork(tmp_path, monkeypatch):
     filename = str(tmp_path / "pprof")
-    monkeypatch.setenv("DD_PROFILING_API_TIMEOUT", "0.1")
+    monkeypatch.setenv("DD_PROFILING_API_TIMEOUT_MS", "100")
     monkeypatch.setenv("DD_PROFILING_OUTPUT_PPROF", filename)
     monkeypatch.setenv("DD_PROFILING_CAPTURE_PCT", "100")
     stdout, stderr, exitcode, pid = call_program(
@@ -73,7 +73,7 @@ def test_fork(tmp_path, monkeypatch):
 @pytest.mark.skipif(sys.platform == "win32", reason="fork only available on Unix")
 @pytest.mark.skipif(not os.getenv("DD_PROFILE_TEST_GEVENT", False), reason="Not testing gevent")
 def test_fork_gevent(monkeypatch):
-    monkeypatch.setenv("DD_PROFILING_API_TIMEOUT", "0.1")
+    monkeypatch.setenv("DD_PROFILING_API_TIMEOUT_MS", "100")
     stdout, stderr, exitcode, pid = call_program("python", os.path.join(os.path.dirname(__file__), "gevent_fork.py"))
     assert exitcode == 0
 

tests/telemetry/test_writer.py

@@ -287,6 +287,7 @@ def test_app_started_event_configuration_override(test_agent_session, run_python
         {"name": "DD_METRICS_OTEL_ENABLED", "origin": "env_var", "value": True},
         {"name": "DD_PROFILING_AGENTLESS", "origin": "default", "value": False},
         {"name": "DD_PROFILING_API_TIMEOUT", "origin": "default", "value": 10.0},
+        {"name": "DD_PROFILING_API_TIMEOUT_MS", "origin": "default", "value": 10000},
         {"name": "DD_PROFILING_CAPTURE_PCT", "origin": "env_var", "value": 5.0},
         {"name": "DD_PROFILING_ENABLED", "origin": "env_var", "value": PYTHON_VERSION_INFO < (3, 14)},
         {"name": "DD_PROFILING_ENABLE_ASSERTS", "origin": "default", "value": False},