diff --git a/benchmarks/bm/flask_utils.py b/benchmarks/bm/flask_utils.py index 79d2c3c5eb9..5b4b5952e56 100644 --- a/benchmarks/bm/flask_utils.py +++ b/benchmarks/bm/flask_utils.py @@ -101,7 +101,11 @@ def setup(self): if self.profiler_enabled: os.environ.update( - {"DD_PROFILING_ENABLED": "1", "DD_PROFILING_API_TIMEOUT": "0.1", "DD_PROFILING_UPLOAD_INTERVAL": "10"} + { + "DD_PROFILING_ENABLED": "1", + "DD_PROFILING_API_TIMEOUT_MS": "100", + "DD_PROFILING_UPLOAD_INTERVAL": "10", + } ) if not self.tracer_enabled: import ddtrace.profiling.auto # noqa:F401 diff --git a/benchmarks/django_simple/scenario.py b/benchmarks/django_simple/scenario.py index fb848b36bea..284953d467a 100644 --- a/benchmarks/django_simple/scenario.py +++ b/benchmarks/django_simple/scenario.py @@ -38,7 +38,11 @@ def run(self): if self.profiler_enabled: os.environ.update( - {"DD_PROFILING_ENABLED": "1", "DD_PROFILING_API_TIMEOUT": "0.1", "DD_PROFILING_UPLOAD_INTERVAL": "10"} + { + "DD_PROFILING_ENABLED": "1", + "DD_PROFILING_API_TIMEOUT_MS": "100", + "DD_PROFILING_UPLOAD_INTERVAL": "10", + } ) if self.appsec_enabled: os.environ.update({"DD_APPSEC_ENABLED ": "1"}) diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/include/constants.hpp b/ddtrace/internal/datadog/profiling/dd_wrapper/include/constants.hpp index 43e9e77d725..f02d6539b73 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/include/constants.hpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/include/constants.hpp @@ -23,3 +23,6 @@ constexpr std::string_view g_language_name = "python"; // Name of the library constexpr std::string_view g_library_name = "dd-trace-py"; + +// Keep this in sync with ddtrace/settings/profiling.py:ProfilingConfig.api_timeout_ms +constexpr uint64_t g_default_max_timeout_ms = 10'000; // 10 seconds diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/include/ddup_interface.hpp b/ddtrace/internal/datadog/profiling/dd_wrapper/include/ddup_interface.hpp index cf09edec496..988d0160a76 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/include/ddup_interface.hpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/include/ddup_interface.hpp @@ -25,6 +25,7 @@ extern "C" void ddup_config_timeline(bool enable); void ddup_config_output_filename(std::string_view filename); void ddup_config_sample_pool_capacity(uint64_t capacity); + void ddup_config_set_max_timeout_ms(uint64_t max_timeout_ms); void ddup_config_user_tag(std::string_view key, std::string_view val); void ddup_config_sample_type(unsigned int type); diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/include/uploader_builder.hpp b/ddtrace/internal/datadog/profiling/dd_wrapper/include/uploader_builder.hpp index 332e531e85d..67f10d40039 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/include/uploader_builder.hpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/include/uploader_builder.hpp @@ -25,6 +25,7 @@ class UploaderBuilder static inline std::string url{ "http://localhost:8126" }; static inline ExporterTagset user_tags{}; static inline std::string output_filename{ "" }; + static inline uint64_t max_timeout_ms{ g_default_max_timeout_ms }; static constexpr std::string_view language{ g_language_name }; static constexpr std::string_view family{ g_language_name }; @@ -40,6 +41,7 @@ class UploaderBuilder static void set_url(std::string_view _url); static void set_tag(std::string_view _key, std::string_view _val); static void set_output_filename(std::string_view _output_filename); + static void set_max_timeout_ms(uint64_t _max_timeout_ms); static std::variant build(); diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/src/ddup_interface.cpp b/ddtrace/internal/datadog/profiling/dd_wrapper/src/ddup_interface.cpp index 580be68c9ad..cb0fd069635 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/src/ddup_interface.cpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/src/ddup_interface.cpp @@ -127,6 +127,12 @@ ddup_config_sample_pool_capacity(uint64_t capacity) // cppcheck-suppress unusedF Datadog::SampleManager::set_sample_pool_capacity(capacity); } +void +ddup_config_set_max_timeout_ms(uint64_t max_timeout_ms) +{ + Datadog::UploaderBuilder::set_max_timeout_ms(max_timeout_ms); +} + bool ddup_is_initialized() // cppcheck-suppress unusedFunction { diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/src/uploader_builder.cpp b/ddtrace/internal/datadog/profiling/dd_wrapper/src/uploader_builder.cpp index 4a9528386dd..8bcf345bb58 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/src/uploader_builder.cpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/src/uploader_builder.cpp @@ -90,6 +90,12 @@ Datadog::UploaderBuilder::set_output_filename(std::string_view _output_filename) } } +void +Datadog::UploaderBuilder::set_max_timeout_ms(uint64_t _max_timeout_ms) +{ + max_timeout_ms = _max_timeout_ms; +} + std::string join(const std::vector& vec, const std::string& delim) { @@ -168,8 +174,6 @@ Datadog::UploaderBuilder::build() auto ddog_exporter = &res.ok; - // 5s is a common timeout parameter for Datadog profilers - const uint64_t max_timeout_ms = 5000; auto set_timeout_result = ddog_prof_Exporter_set_timeout(ddog_exporter, max_timeout_ms); if (set_timeout_result.tag == DDOG_VOID_RESULT_ERR) { auto& err = set_timeout_result.err; diff --git a/ddtrace/internal/datadog/profiling/ddup/_ddup.pyi b/ddtrace/internal/datadog/profiling/ddup/_ddup.pyi index 4b76ebdd0b7..24cf06a57e9 100644 --- a/ddtrace/internal/datadog/profiling/ddup/_ddup.pyi +++ b/ddtrace/internal/datadog/profiling/ddup/_ddup.pyi @@ -14,6 +14,7 @@ def config( timeline_enabled: Optional[bool], output_filename: Optional[str], sample_pool_capacity: Optional[int], + timeout: Optional[int], ) -> None: ... def start() -> None: ... def upload(tracer: Optional[Tracer], enable_code_provenance: Optional[bool]) -> None: ... diff --git a/ddtrace/internal/datadog/profiling/ddup/_ddup.pyx b/ddtrace/internal/datadog/profiling/ddup/_ddup.pyx index 7d564044d64..efd3a4ab8ce 100644 --- a/ddtrace/internal/datadog/profiling/ddup/_ddup.pyx +++ b/ddtrace/internal/datadog/profiling/ddup/_ddup.pyx @@ -57,6 +57,7 @@ cdef extern from "ddup_interface.hpp": void ddup_set_runtime_id(string_view _id) void ddup_profile_set_endpoints(unordered_map[int64_t, string_view] span_ids_to_endpoints) void ddup_profile_add_endpoint_counts(unordered_map[string_view, int64_t] trace_endpoints_to_counts) + void ddup_config_set_max_timeout_ms(uint64_t max_timeout_ms) bint ddup_upload() nogil Sample *ddup_start_sample() @@ -330,7 +331,9 @@ def config( max_nframes: Optional[int] = None, timeline_enabled: Optional[bool] = None, output_filename: StringType = None, - sample_pool_capacity: Optional[int] = None) -> None: + sample_pool_capacity: Optional[int] = None, + timeout: Optional[int] = None, +) -> None: # Try to provide a ddtrace-specific default service if one is not given service = service or DEFAULT_SERVICE_NAME @@ -361,6 +364,9 @@ def config( if sample_pool_capacity: ddup_config_sample_pool_capacity(clamp_to_uint64_unsigned(sample_pool_capacity)) + if timeout is not None: + ddup_config_set_max_timeout_ms(clamp_to_uint64_unsigned(timeout)) + def start() -> None: ddup_start() diff --git a/ddtrace/profiling/profiler.py b/ddtrace/profiling/profiler.py index 57c72b52697..5865e33a410 100644 --- a/ddtrace/profiling/profiler.py +++ b/ddtrace/profiling/profiler.py @@ -181,6 +181,7 @@ def _build_default_exporters(self): timeline_enabled=profiling_config.timeline_enabled, output_filename=profiling_config.output_pprof, sample_pool_capacity=profiling_config.sample_pool_capacity, + timeout=profiling_config.api_timeout_ms, ) ddup.start() diff --git a/ddtrace/settings/profiling.py b/ddtrace/settings/profiling.py index 751932edce2..91076627f42 100644 --- a/ddtrace/settings/profiling.py +++ b/ddtrace/settings/profiling.py @@ -207,7 +207,17 @@ class ProfilingConfig(DDConfig): "api_timeout", default=10.0, help_type="Float", - help="The timeout in seconds before dropping events if the HTTP API does not reply", + deprecations=[("api_timeout", None, "4.0.0")], + help="(Deprecated) The timeout in seconds before dropping events if the HTTP API does not reply. " + "Use api_timeout_ms instead.", + ) + + api_timeout_ms = DDConfig.v( + int, + "api_timeout_ms", + default=10000, + help_type="Integer", + help="The timeout in milliseconds before dropping events if the HTTP API does not reply", ) timeline_enabled = DDConfig.v( diff --git a/releasenotes/notes/profiling-api-timeout-09bf46873195aff3.yaml b/releasenotes/notes/profiling-api-timeout-09bf46873195aff3.yaml new file mode 100644 index 00000000000..9bf353d5c15 --- /dev/null +++ b/releasenotes/notes/profiling-api-timeout-09bf46873195aff3.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + profiling: ``DD_PROFILING_API_TIMEOUT`` doesn't have any effect, and + is marked to be removed in upcoming 4.0 release. New environment variable + ``DD_PROFILING_API_TIMEOUT_MS`` is introduced to configure timeout for + uploading profiles to the backend. The default value is 10000 ms (10 seconds) diff --git a/tests/profiling/collector/conftest.py b/tests/profiling/collector/conftest.py index a53ac79bcad..8475c4d0a22 100644 --- a/tests/profiling/collector/conftest.py +++ b/tests/profiling/collector/conftest.py @@ -13,7 +13,7 @@ def tracer(): @pytest.fixture def profiler(monkeypatch): - monkeypatch.setenv("DD_PROFILING_API_TIMEOUT", "0.1") + monkeypatch.setenv("DD_PROFILING_API_TIMEOUT_MS", "100") p = Profiler() p.start() try: diff --git a/tests/profiling/test_main.py b/tests/profiling/test_main.py index 23add5ef75b..92d171a6f8b 100644 --- a/tests/profiling/test_main.py +++ b/tests/profiling/test_main.py @@ -12,7 +12,7 @@ def test_call_script(monkeypatch): # Set a very short timeout to exit fast - monkeypatch.setenv("DD_PROFILING_API_TIMEOUT", "0.1") + monkeypatch.setenv("DD_PROFILING_API_TIMEOUT_MS", "100") monkeypatch.setenv("DD_PROFILING_ENABLED", "1") stdout, stderr, exitcode, _ = call_program( "ddtrace-run", sys.executable, os.path.join(os.path.dirname(__file__), "simple_program.py") @@ -28,7 +28,7 @@ def test_call_script(monkeypatch): @pytest.mark.skipif(not os.getenv("DD_PROFILE_TEST_GEVENT", False), reason="Not testing gevent") def test_call_script_gevent(monkeypatch): - monkeypatch.setenv("DD_PROFILING_API_TIMEOUT", "0.1") + monkeypatch.setenv("DD_PROFILING_API_TIMEOUT_MS", "100") stdout, stderr, exitcode, pid = call_program( sys.executable, os.path.join(os.path.dirname(__file__), "simple_program_gevent.py") ) @@ -58,7 +58,7 @@ def test_call_script_pprof_output(tmp_path, monkeypatch): @pytest.mark.skipif(sys.platform == "win32", reason="fork only available on Unix") def test_fork(tmp_path, monkeypatch): filename = str(tmp_path / "pprof") - monkeypatch.setenv("DD_PROFILING_API_TIMEOUT", "0.1") + monkeypatch.setenv("DD_PROFILING_API_TIMEOUT_MS", "100") monkeypatch.setenv("DD_PROFILING_OUTPUT_PPROF", filename) monkeypatch.setenv("DD_PROFILING_CAPTURE_PCT", "100") stdout, stderr, exitcode, pid = call_program( @@ -73,7 +73,7 @@ def test_fork(tmp_path, monkeypatch): @pytest.mark.skipif(sys.platform == "win32", reason="fork only available on Unix") @pytest.mark.skipif(not os.getenv("DD_PROFILE_TEST_GEVENT", False), reason="Not testing gevent") def test_fork_gevent(monkeypatch): - monkeypatch.setenv("DD_PROFILING_API_TIMEOUT", "0.1") + monkeypatch.setenv("DD_PROFILING_API_TIMEOUT_MS", "100") stdout, stderr, exitcode, pid = call_program("python", os.path.join(os.path.dirname(__file__), "gevent_fork.py")) assert exitcode == 0 diff --git a/tests/telemetry/test_writer.py b/tests/telemetry/test_writer.py index 04a4a7e79db..27d251ff45e 100644 --- a/tests/telemetry/test_writer.py +++ b/tests/telemetry/test_writer.py @@ -287,6 +287,7 @@ def test_app_started_event_configuration_override(test_agent_session, run_python {"name": "DD_METRICS_OTEL_ENABLED", "origin": "env_var", "value": True}, {"name": "DD_PROFILING_AGENTLESS", "origin": "default", "value": False}, {"name": "DD_PROFILING_API_TIMEOUT", "origin": "default", "value": 10.0}, + {"name": "DD_PROFILING_API_TIMEOUT_MS", "origin": "default", "value": 10000}, {"name": "DD_PROFILING_CAPTURE_PCT", "origin": "env_var", "value": 5.0}, {"name": "DD_PROFILING_ENABLED", "origin": "env_var", "value": PYTHON_VERSION_INFO < (3, 14)}, {"name": "DD_PROFILING_ENABLE_ASSERTS", "origin": "default", "value": False},