From a7569a0eb6e4bb770c494b16914ef613451af69f Mon Sep 17 00:00:00 2001 From: Kyle Verhoog Date: Wed, 17 Sep 2025 10:20:35 -0400 Subject: [PATCH 1/3] fix(llmobs): ensure DD_APM_TRACING_ENABLED disables APM The existing DD_APM_TRACING_ENABLED functionality is specially implemented for asm and did not work for llmobs. Spans are still sent since some spans are required for asm. This change introduces a trace filter to drop all apm spans when llmobs is enabled. Consequently, asm + llmobs with apm disabled will not be possible with this change. I'm choosing to avoid complexity in this implementation until we see a concrete ask for this functionality. --- ddtrace/_trace/apm_filter.py | 22 +++++++++++++++++ ddtrace/llmobs/_llmobs.py | 6 +++++ .../llmobs-apm-tracing-8cd612f8a3af4960.yaml | 4 ++++ tests/llmobs/test_llmobs.py | 24 +++++++++++++++++++ 4 files changed, 56 insertions(+) create mode 100644 ddtrace/_trace/apm_filter.py create mode 100644 releasenotes/notes/llmobs-apm-tracing-8cd612f8a3af4960.yaml diff --git a/ddtrace/_trace/apm_filter.py b/ddtrace/_trace/apm_filter.py new file mode 100644 index 00000000000..7d9095ea196 --- /dev/null +++ b/ddtrace/_trace/apm_filter.py @@ -0,0 +1,22 @@ +import os +from typing import List +from typing import Optional + +from ddtrace._trace.processor import TraceProcessor +from ddtrace._trace.span import Span +from ddtrace.internal.utils.formats import asbool + + +class APMTracingEnabledFilter(TraceProcessor): + """ + Trace processor that drops all APM traces when DD_APM_TRACING_ENABLED is set to a falsy value. + """ + + def __init__(self) -> None: + super().__init__() + self._apm_tracing_enabled = asbool(os.getenv("DD_APM_TRACING_ENABLED", "true")) + + def process_trace(self, trace: List[Span]) -> Optional[List[Span]]: + if not self._apm_tracing_enabled: + return None + return trace \ No newline at end of file diff --git a/ddtrace/llmobs/_llmobs.py b/ddtrace/llmobs/_llmobs.py index 9f520e8834f..482defc1095 100644 --- a/ddtrace/llmobs/_llmobs.py +++ b/ddtrace/llmobs/_llmobs.py @@ -20,6 +20,7 @@ import ddtrace from ddtrace import config from ddtrace import patch +from ddtrace._trace.apm_filter import APMTracingEnabledFilter from ddtrace._trace.context import Context from ddtrace._trace.span import Span from ddtrace._trace.tracer import Tracer @@ -603,6 +604,11 @@ def enable( # override the default _instance with a new tracer cls._instance = cls(tracer=_tracer, span_processor=span_processor) + + # Add APM trace filter to drop all APM traces when DD_APM_TRACING_ENABLED is falsy + apm_filter = APMTracingEnabledFilter() + cls._instance.tracer._span_aggregator.user_processors.append(apm_filter) + cls.enabled = True cls._instance.start() diff --git a/releasenotes/notes/llmobs-apm-tracing-8cd612f8a3af4960.yaml b/releasenotes/notes/llmobs-apm-tracing-8cd612f8a3af4960.yaml new file mode 100644 index 00000000000..a02da063f79 --- /dev/null +++ b/releasenotes/notes/llmobs-apm-tracing-8cd612f8a3af4960.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + LLM Observability: fix DD_APM_TRACING_ENABLED flag not disabling APM when using LLM Observability. \ No newline at end of file diff --git a/tests/llmobs/test_llmobs.py b/tests/llmobs/test_llmobs.py index ffbdc7431ed..5e488ee9dc6 100644 --- a/tests/llmobs/test_llmobs.py +++ b/tests/llmobs/test_llmobs.py @@ -628,3 +628,27 @@ def test_trace_id_propagation_with_non_llm_parent(llmobs, llmobs_events): # LLMObs trace IDs should be different from APM trace ID assert first_child_event["trace_id"] != first_child_event["_dd"]["apm_trace_id"] assert second_child_event["trace_id"] != second_child_event["_dd"]["apm_trace_id"] + + +@pytest.mark.parametrize("llmobs_env", [{"DD_APM_TRACING_ENABLED": "false"}]) +def test_apm_traces_dropped_when_disabled(llmobs, llmobs_events, tracer, llmobs_env): + from tests.utils import DummyWriter + + dummy_writer = DummyWriter() + tracer._span_aggregator.writer = dummy_writer + + with tracer.trace("apm_span") as apm_span: + apm_span.set_tag("operation", "test") + + # Create an LLMObs span (should be sent to LLMObs but not APM) + with llmobs.llm(model_name="test-model") as llm_span: + llmobs.annotate(llm_span, input_data="test input", output_data="test output") + + # Check that no APM traces were sent to the writer + assert len(dummy_writer.traces) == 0, "APM traces should be dropped when DD_APM_TRACING_ENABLED=false" + + # But LLMObs events should still be sent + assert len(llmobs_events) == 1 + llm_event = llmobs_events[0] + assert llm_event["meta"]["span.kind"] == "llm" + assert llm_event["meta"]["model_name"] == "test-model" From 8a7fdf08f78ca6a9fd109841fa9c7fe7eef6ce65 Mon Sep 17 00:00:00 2001 From: kyle Date: Wed, 17 Sep 2025 15:01:45 -0400 Subject: [PATCH 2/3] Update releasenotes/notes/llmobs-apm-tracing-8cd612f8a3af4960.yaml Co-authored-by: Yun Kim <35776586+Yun-Kim@users.noreply.github.com> --- releasenotes/notes/llmobs-apm-tracing-8cd612f8a3af4960.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/releasenotes/notes/llmobs-apm-tracing-8cd612f8a3af4960.yaml b/releasenotes/notes/llmobs-apm-tracing-8cd612f8a3af4960.yaml index a02da063f79..bc3abb2f79b 100644 --- a/releasenotes/notes/llmobs-apm-tracing-8cd612f8a3af4960.yaml +++ b/releasenotes/notes/llmobs-apm-tracing-8cd612f8a3af4960.yaml @@ -1,4 +1,4 @@ --- fixes: - | - LLM Observability: fix DD_APM_TRACING_ENABLED flag not disabling APM when using LLM Observability. \ No newline at end of file + LLM Observability: ensures APM is disabled when DD_APM_TRACING_ENABLED=0 when using LLM Observability. \ No newline at end of file From ab83016f5ab0e7c0c7a9d4d2897d9fc3d86d4621 Mon Sep 17 00:00:00 2001 From: kyle Date: Wed, 17 Sep 2025 15:06:07 -0400 Subject: [PATCH 3/3] Update ddtrace/_trace/apm_filter.py Co-authored-by: Yun Kim <35776586+Yun-Kim@users.noreply.github.com> --- ddtrace/_trace/apm_filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/_trace/apm_filter.py b/ddtrace/_trace/apm_filter.py index 7d9095ea196..89ba4e93dd0 100644 --- a/ddtrace/_trace/apm_filter.py +++ b/ddtrace/_trace/apm_filter.py @@ -19,4 +19,4 @@ def __init__(self) -> None: def process_trace(self, trace: List[Span]) -> Optional[List[Span]]: if not self._apm_tracing_enabled: return None - return trace \ No newline at end of file + return trace