Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ddtrace/tracer: Report datadog.tracer.queue.enqueued.traces as health metric #3019

Merged
merged 6 commits into from
Dec 20, 2024
Merged

ddtrace/tracer: Report datadog.tracer.queue.enqueued.traces as health metric #3019

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
145e0c0
Implement datadog.tracer.queue.enqueued.traces
mtoffl01 Dec 9, 2024
84057cc
changed metric reporting to be on flush
mtoffl01 Dec 16, 2024
437600b
revert unnecessary change to ddtrace/tracer/tracer.go
mtoffl01 Dec 16, 2024
2bdd65c
Merge branch 'main' into mtoff/health-metrics
mtoffl01 Dec 18, 2024
235dc8c
fix failing tests TestTraceWriterFlushRetries
mtoffl01 Dec 19, 2024
3036c2b
Merge branch 'main' into mtoff/health-metrics
mtoffl01 Dec 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 25 additions & 1 deletion ddtrace/tracer/metrics_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,36 @@ func TestReportHealthMetrics(t *testing.T) {

tracer.StartSpan("operation").Finish()
flush(1)
tg.Wait(assert, 3, 10*time.Second)
tg.Wait(assert, 4, 10*time.Second)

counts := tg.Counts()
assert.Equal(int64(1), counts["datadog.tracer.spans_started"])
assert.Equal(int64(1), counts["datadog.tracer.spans_finished"])
assert.Equal(int64(0), counts["datadog.tracer.traces_dropped"])
assert.Equal(int64(1), counts["datadog.tracer.queue.enqueued.traces"])
}

func TestEnqueuedTracesHealthMetric(t *testing.T) {
assert := assert.New(t)
var tg statsdtest.TestStatsdClient

defer func(old time.Duration) { statsInterval = old }(statsInterval)
statsInterval = time.Nanosecond

tracer, _, flush, stop := startTestTracer(t, withStatsdClient(&tg))
defer stop()

for i := 0; i < 3; i++ {
tracer.StartSpan("operation").Finish()
}
flush(3)
tg.Wait(assert, 1, 10*time.Second)

counts := tg.Counts()
assert.Equal(int64(3), counts["datadog.tracer.queue.enqueued.traces"])
w, ok := tracer.traceWriter.(*agentTraceWriter)
assert.True(ok)
assert.Equal(uint32(0), w.tracesQueued)
}

func TestTracerMetrics(t *testing.T) {
Expand Down
5 changes: 5 additions & 0 deletions ddtrace/tracer/writer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"os"
"strconv"
"sync"
"sync/atomic"
"time"

globalinternal "gopkg.in/DataDog/dd-trace-go.v1/internal"
Expand Down Expand Up @@ -50,6 +51,8 @@ type agentTraceWriter struct {

// statsd is used to send metrics
statsd globalinternal.StatsdClient

tracesQueued uint32
}

func newAgentTraceWriter(c *config, s *prioritySampler, statsdClient globalinternal.StatsdClient) *agentTraceWriter {
Expand All @@ -67,6 +70,7 @@ func (h *agentTraceWriter) add(trace []*span) {
h.statsd.Incr("datadog.tracer.traces_dropped", []string{"reason:encoding_error"}, 1)
log.Error("Error encoding msgpack: %v", err)
}
atomic.AddUint32(&h.tracesQueued, 1) // TODO: This does not differentiate between complete traces and partial chunks
if h.payload.size() > payloadSizeLimit {
h.statsd.Incr("datadog.tracer.flush_triggered", []string{"reason:size"}, 1)
h.flush()
Expand Down Expand Up @@ -94,6 +98,7 @@ func (h *agentTraceWriter) flush() {
// collection to avoid a memory leak when references to this object
// may still be kept by faulty transport implementations or the
// standard library. See dd-trace-go#976
h.statsd.Count("datadog.tracer.queue.enqueued.traces", int64(atomic.SwapUint32(&h.tracesQueued, 0)), nil, 1)
p.clear()

<-h.climit
Expand Down
Loading