From a8665eb0b88c50b6e30477fd464f1e7e985d6a51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Mon, 16 Dec 2024 10:00:05 +0100 Subject: [PATCH 01/13] fix(.github/workflows): add tags-ignore to avoid running CI on pushing tags for contribs and other nested modules (#3005) Co-authored-by: Hannah Kim --- .github/workflows/appsec.yml | 3 +++ .github/workflows/codeql-analysis.yml | 3 +++ .github/workflows/govulncheck.yml | 5 ++++- .github/workflows/main-branch-tests.yml | 5 +++-- .github/workflows/orchestrion.yml | 3 +++ .github/workflows/parametric-tests.yml | 5 +++-- .github/workflows/pull-request.yml | 3 +++ .github/workflows/smoke-tests.yml | 5 +++-- .github/workflows/system-tests.yml | 5 +++-- 9 files changed, 28 insertions(+), 9 deletions(-) diff --git a/.github/workflows/appsec.yml b/.github/workflows/appsec.yml index 80c1fce6cf..d238b55ba3 100644 --- a/.github/workflows/appsec.yml +++ b/.github/workflows/appsec.yml @@ -19,6 +19,9 @@ on: merge_group: push: branches: release-v* + tags-ignore: + - 'contrib/**' + - 'instrumentation/**' env: DD_APPSEC_WAF_TIMEOUT: 1m diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 8d85bac9df..53fb2d4cc2 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -9,6 +9,9 @@ on: type: string push: branches: [ main, master ] + tags-ignore: + - 'contrib/**' + - 'instrumentation/**' pull_request: # The branches below must be a subset of the branches above branches: [ main ] diff --git a/.github/workflows/govulncheck.yml b/.github/workflows/govulncheck.yml index eaf5ed78d9..e7549a71fc 100644 --- a/.github/workflows/govulncheck.yml +++ b/.github/workflows/govulncheck.yml @@ -9,7 +9,10 @@ on: push: branches: - main - - release-v* + - release-v* + tags-ignore: + - 'contrib/**' + - 'instrumentation/**' schedule: - cron: '00 00 * * *' workflow_dispatch: diff --git a/.github/workflows/main-branch-tests.yml b/.github/workflows/main-branch-tests.yml index 7d738ccd9a..136805cebd 100644 --- a/.github/workflows/main-branch-tests.yml +++ b/.github/workflows/main-branch-tests.yml @@ -11,8 +11,9 @@ on: branches: - main - release-v* - tags: - - "**" + tags-ignore: + - 'contrib/**' + - 'instrumentation/**' concurrency: group: ${{ github.ref }} diff --git a/.github/workflows/orchestrion.yml b/.github/workflows/orchestrion.yml index 0603c8b139..ddd0e7435b 100644 --- a/.github/workflows/orchestrion.yml +++ b/.github/workflows/orchestrion.yml @@ -6,6 +6,9 @@ on: push: branches: - release-v* + tags-ignore: + - 'contrib/**' + - 'instrumentation/**' permissions: read-all diff --git a/.github/workflows/parametric-tests.yml b/.github/workflows/parametric-tests.yml index a25c01a2c3..4027451800 100644 --- a/.github/workflows/parametric-tests.yml +++ b/.github/workflows/parametric-tests.yml @@ -11,8 +11,9 @@ on: branches: - main - release-v* - tags: - - "**" + tags-ignore: + - 'contrib/**' + - 'instrumentation/**' pull_request: branches: - "**" diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 9513b2e328..8318181820 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -8,6 +8,9 @@ on: push: branches: - 'mq-working-branch-**' + tags-ignore: + - 'contrib/**' + - 'instrumentation/**' concurrency: group: ${{ github.ref }} diff --git a/.github/workflows/smoke-tests.yml b/.github/workflows/smoke-tests.yml index cee00b0d34..9a05bdc8ba 100644 --- a/.github/workflows/smoke-tests.yml +++ b/.github/workflows/smoke-tests.yml @@ -15,8 +15,9 @@ on: branches: - main - release-v* - tags: - - '**' + tags-ignore: + - 'contrib/**' + - 'instrumentation/**' schedule: # nightly - cron: "0 0 * * *" workflow_dispatch: {} # manually diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 90933ec1ad..f744ce1303 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -11,8 +11,9 @@ on: branches: - main - release-v* - tags: - - "**" + tags-ignore: + - 'contrib/**' + - 'instrumentation/**' pull_request: branches: - "**" From c0d5d5a9756443c34579c20b6116fc2e625e8152 Mon Sep 17 00:00:00 2001 From: Flavien Darche <11708575+e-n-0@users.noreply.github.com> Date: Mon, 16 Dec 2024 13:57:02 +0000 Subject: [PATCH 02/13] contrib/envoyproxy: envoy external processing support (#2895) This PR adds a new gRPC Interceptor (StreamServerInterceptor) to support the interception of ext_proc v3 calls to gRPC server. When the interceptor is applied, all messages of the external processing protocol are instrumented without returning an handle to the original server code Co-authored-by: Eliott Bouhana Co-authored-by: Flavien Darche --- contrib/envoyproxy/go-control-plane/envoy.go | 350 +++++++++++ .../envoyproxy/go-control-plane/envoy_test.go | 576 ++++++++++++++++++ .../go-control-plane/example_test.go | 44 ++ .../envoyproxy/go-control-plane/fakehttp.go | 189 ++++++ contrib/internal/httptrace/response_writer.go | 7 + go.mod | 3 + go.sum | 6 + internal/appsec/testdata/user_rules.json | 27 + 8 files changed, 1202 insertions(+) create mode 100644 contrib/envoyproxy/go-control-plane/envoy.go create mode 100644 contrib/envoyproxy/go-control-plane/envoy_test.go create mode 100644 contrib/envoyproxy/go-control-plane/example_test.go create mode 100644 contrib/envoyproxy/go-control-plane/fakehttp.go diff --git a/contrib/envoyproxy/go-control-plane/envoy.go b/contrib/envoyproxy/go-control-plane/envoy.go new file mode 100644 index 0000000000..52279e0138 --- /dev/null +++ b/contrib/envoyproxy/go-control-plane/envoy.go @@ -0,0 +1,350 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024 Datadog, Inc. + +package go_control_plane + +import ( + "context" + "errors" + "io" + "math" + "net/http" + "strings" + + "gopkg.in/DataDog/dd-trace-go.v1/contrib/internal/httptrace" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo" + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/emitter/waf/actions" + "gopkg.in/DataDog/dd-trace-go.v1/internal/log" + "gopkg.in/DataDog/dd-trace-go.v1/internal/telemetry" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + envoycore "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + envoyextproc "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + envoytypes "github.com/envoyproxy/go-control-plane/envoy/type/v3" +) + +const componentName = "envoyproxy/go-control-plane" + +func init() { + telemetry.LoadIntegration(componentName) + tracer.MarkIntegrationImported("github.com/envoyproxy/go-control-plane") +} + +// appsecEnvoyExternalProcessorServer is a server that implements the Envoy ExternalProcessorServer interface. +type appsecEnvoyExternalProcessorServer struct { + envoyextproc.ExternalProcessorServer +} + +// AppsecEnvoyExternalProcessorServer creates and returns a new instance of appsecEnvoyExternalProcessorServer. +func AppsecEnvoyExternalProcessorServer(userImplementation envoyextproc.ExternalProcessorServer) envoyextproc.ExternalProcessorServer { + return &appsecEnvoyExternalProcessorServer{userImplementation} +} + +type currentRequest struct { + span tracer.Span + afterHandle func() + ctx context.Context + fakeResponseWriter *fakeResponseWriter + wrappedResponseWriter http.ResponseWriter +} + +// Process handles the bidirectional stream that Envoy uses to give the server control +// over what the filter does. It processes incoming requests and sends appropriate responses +// based on the type of request received. +// +// The method receive incoming requests, processes them, and sends responses back to the client. +// It handles different types of requests such as request headers, response headers, request body, +// response body, request trailers, and response trailers. +// +// If the request is blocked, it sends an immediate response and ends the stream. If an error occurs +// during processing, it logs the error and returns an appropriate gRPC status error. +func (s *appsecEnvoyExternalProcessorServer) Process(processServer envoyextproc.ExternalProcessor_ProcessServer) error { + var ( + ctx = processServer.Context() + blocked bool + currentRequest *currentRequest + processingRequest envoyextproc.ProcessingRequest + processingResponse *envoyextproc.ProcessingResponse + ) + + // Close the span when the request is done processing + defer func() { + if currentRequest == nil { + return + } + + log.Warn("external_processing: stream stopped during a request, making sure the current span is closed\n") + currentRequest.span.Finish() + currentRequest = nil + }() + + for { + select { + case <-ctx.Done(): + if errors.Is(ctx.Err(), context.Canceled) { + return nil + } + + return ctx.Err() + default: + // no op + } + + err := processServer.RecvMsg(&processingRequest) + if err != nil { + // Note: Envoy is inconsistent with the "end_of_stream" value of its headers responses, + // so we can't fully rely on it to determine when it will close (cancel) the stream. + if s, ok := status.FromError(err); (ok && s.Code() == codes.Canceled) || err == io.EOF { + return nil + } + + log.Warn("external_processing: error receiving request/response: %v\n", err) + return status.Errorf(codes.Unknown, "Error receiving request/response: %v", err) + } + + processingResponse, err = envoyExternalProcessingRequestTypeAssert(&processingRequest) + if err != nil { + log.Error("external_processing: error asserting request type: %v\n", err) + return status.Errorf(codes.Unknown, "Error asserting request type: %v", err) + } + + switch v := processingRequest.Request.(type) { + case *envoyextproc.ProcessingRequest_RequestHeaders: + processingResponse, currentRequest, blocked, err = processRequestHeaders(ctx, v) + case *envoyextproc.ProcessingRequest_ResponseHeaders: + processingResponse, err = processResponseHeaders(v, currentRequest) + currentRequest = nil // Request is done, reset the current request + } + + if err != nil { + log.Error("external_processing: error processing request: %v\n", err) + return err + } + + // End of stream reached, no more data to process + if processingResponse == nil { + log.Debug("external_processing: end of stream reached") + return nil + } + + if err := processServer.SendMsg(processingResponse); err != nil { + log.Warn("external_processing: error sending response (probably because of an Envoy timeout): %v", err) + return status.Errorf(codes.Unknown, "Error sending response (probably because of an Envoy timeout): %v", err) + } + + if !blocked { + continue + } + + log.Debug("external_processing: request blocked, end the stream") + currentRequest = nil + return nil + } +} + +func envoyExternalProcessingRequestTypeAssert(req *envoyextproc.ProcessingRequest) (*envoyextproc.ProcessingResponse, error) { + switch r := req.Request.(type) { + case *envoyextproc.ProcessingRequest_RequestHeaders, *envoyextproc.ProcessingRequest_ResponseHeaders: + return nil, nil + + case *envoyextproc.ProcessingRequest_RequestBody: + // TODO: Handle request raw body in the WAF + return &envoyextproc.ProcessingResponse{ + Response: &envoyextproc.ProcessingResponse_RequestBody{ + RequestBody: &envoyextproc.BodyResponse{ + Response: &envoyextproc.CommonResponse{ + Status: envoyextproc.CommonResponse_CONTINUE, + }, + }, + }, + }, nil + + case *envoyextproc.ProcessingRequest_RequestTrailers: + return &envoyextproc.ProcessingResponse{ + Response: &envoyextproc.ProcessingResponse_RequestTrailers{}, + }, nil + + case *envoyextproc.ProcessingRequest_ResponseBody: + // Note: The end of stream bool value is not reliable + // Sometimes it's not set to true even if there is no more data to process + if r.ResponseBody.GetEndOfStream() { + return nil, nil + } + + // TODO: Handle response raw body in the WAF + return &envoyextproc.ProcessingResponse{ + Response: &envoyextproc.ProcessingResponse_ResponseBody{}, + }, nil + + case *envoyextproc.ProcessingRequest_ResponseTrailers: + return &envoyextproc.ProcessingResponse{ + Response: &envoyextproc.ProcessingResponse_RequestTrailers{}, + }, nil + + default: + return nil, status.Errorf(codes.Unknown, "Unknown request type: %T", r) + } +} + +func processRequestHeaders(ctx context.Context, req *envoyextproc.ProcessingRequest_RequestHeaders) (*envoyextproc.ProcessingResponse, *currentRequest, bool, error) { + log.Debug("external_processing: received request headers: %v\n", req.RequestHeaders) + + request, err := newRequest(ctx, req) + if err != nil { + return nil, nil, false, status.Errorf(codes.InvalidArgument, "Error processing request headers from ext_proc: %v", err) + } + + var blocked bool + fakeResponseWriter := newFakeResponseWriter() + wrappedResponseWriter, request, afterHandle, blocked := httptrace.BeforeHandle(&httptrace.ServeConfig{ + SpanOpts: []ddtrace.StartSpanOption{ + tracer.Tag(ext.SpanKind, ext.SpanKindServer), + tracer.Tag(ext.Component, componentName), + }, + }, fakeResponseWriter, request) + + // Block handling: If triggered, we need to block the request, return an immediate response + if blocked { + afterHandle() + return doBlockResponse(fakeResponseWriter), nil, true, nil + } + + span, ok := tracer.SpanFromContext(request.Context()) + if !ok { + return nil, nil, false, status.Errorf(codes.Unknown, "Error getting span from context") + } + + processingResponse, err := propagationRequestHeaderMutation(span) + if err != nil { + return nil, nil, false, err + } + + return processingResponse, ¤tRequest{ + span: span, + ctx: request.Context(), + fakeResponseWriter: fakeResponseWriter, + wrappedResponseWriter: wrappedResponseWriter, + afterHandle: afterHandle, + }, false, nil +} + +func propagationRequestHeaderMutation(span ddtrace.Span) (*envoyextproc.ProcessingResponse, error) { + newHeaders := make(http.Header) + if err := tracer.Inject(span.Context(), tracer.HTTPHeadersCarrier(newHeaders)); err != nil { + return nil, status.Errorf(codes.Unknown, "Error injecting headers: %v", err) + } + + if len(newHeaders) > 0 { + log.Debug("external_processing: injecting propagation headers: %v\n", newHeaders) + } + + headerValueOptions := make([]*envoycore.HeaderValueOption, 0, len(newHeaders)) + for k, v := range newHeaders { + headerValueOptions = append(headerValueOptions, &envoycore.HeaderValueOption{ + Header: &envoycore.HeaderValue{ + Key: k, + RawValue: []byte(strings.Join(v, ",")), + }, + }) + } + + return &envoyextproc.ProcessingResponse{ + Response: &envoyextproc.ProcessingResponse_RequestHeaders{ + RequestHeaders: &envoyextproc.HeadersResponse{ + Response: &envoyextproc.CommonResponse{ + Status: envoyextproc.CommonResponse_CONTINUE, + HeaderMutation: &envoyextproc.HeaderMutation{ + SetHeaders: headerValueOptions, + }, + }, + }, + }, + }, nil +} + +func processResponseHeaders(res *envoyextproc.ProcessingRequest_ResponseHeaders, currentRequest *currentRequest) (*envoyextproc.ProcessingResponse, error) { + log.Debug("external_processing: received response headers: %v\n", res.ResponseHeaders) + + if err := createFakeResponseWriter(currentRequest.wrappedResponseWriter, res); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "Error processing response headers from ext_proc: %v", err) + } + + var blocked bool + + // Now we need to know if the request has been blocked, but we don't have any other way than to look for the operation and bind a blocking data listener to it + op, ok := dyngo.FromContext(currentRequest.ctx) + if ok { + dyngo.OnData(op, func(_ *actions.BlockHTTP) { + // We already wrote over the response writer, we need to reset it so the blocking handler can write to it + httptrace.ResetStatusCode(currentRequest.wrappedResponseWriter) + currentRequest.fakeResponseWriter.Reset() + blocked = true + }) + } + + currentRequest.afterHandle() + + if blocked { + response := doBlockResponse(currentRequest.fakeResponseWriter) + return response, nil + } + + log.Debug("external_processing: finishing request with status code: %v\n", currentRequest.fakeResponseWriter.status) + + // Note: (cf. comment in the stream error handling) + // The end of stream bool value is not reliable + if res.ResponseHeaders.GetEndOfStream() { + return nil, nil + } + + return &envoyextproc.ProcessingResponse{ + Response: &envoyextproc.ProcessingResponse_ResponseHeaders{ + ResponseHeaders: &envoyextproc.HeadersResponse{ + Response: &envoyextproc.CommonResponse{ + Status: envoyextproc.CommonResponse_CONTINUE, + }, + }, + }, + }, nil +} + +func doBlockResponse(writer *fakeResponseWriter) *envoyextproc.ProcessingResponse { + var headersMutation []*envoycore.HeaderValueOption + for k, v := range writer.headers { + headersMutation = append(headersMutation, &envoycore.HeaderValueOption{ + Header: &envoycore.HeaderValue{ + Key: k, + RawValue: []byte(strings.Join(v, ",")), + }, + }) + } + + var int32StatusCode int32 = 0 + if writer.status > 0 && writer.status <= math.MaxInt32 { + int32StatusCode = int32(writer.status) + } + + return &envoyextproc.ProcessingResponse{ + Response: &envoyextproc.ProcessingResponse_ImmediateResponse{ + ImmediateResponse: &envoyextproc.ImmediateResponse{ + Status: &envoytypes.HttpStatus{ + Code: envoytypes.StatusCode(int32StatusCode), + }, + Headers: &envoyextproc.HeaderMutation{ + SetHeaders: headersMutation, + }, + Body: string(writer.body), + GrpcStatus: &envoyextproc.GrpcStatus{ + Status: 0, + }, + }, + }, + } +} diff --git a/contrib/envoyproxy/go-control-plane/envoy_test.go b/contrib/envoyproxy/go-control-plane/envoy_test.go new file mode 100644 index 0000000000..8af05eaab3 --- /dev/null +++ b/contrib/envoyproxy/go-control-plane/envoy_test.go @@ -0,0 +1,576 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024 Datadog, Inc. + +package go_control_plane + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net" + "testing" + + envoyextproc "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + envoytypes "github.com/envoyproxy/go-control-plane/envoy/type/v3" + + ddgrpc "gopkg.in/DataDog/dd-trace-go.v1/contrib/google.golang.org/grpc" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/mocktracer" + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec" + + v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +func TestAppSec(t *testing.T) { + appsec.Start() + defer appsec.Stop() + if !appsec.Enabled() { + t.Skip("appsec disabled") + } + + setup := func() (envoyextproc.ExternalProcessorClient, mocktracer.Tracer, func()) { + rig, err := newEnvoyAppsecRig(t, false) + require.NoError(t, err) + + mt := mocktracer.Start() + + return rig.client, mt, func() { + rig.Close() + mt.Stop() + } + } + + t.Run("monitoring-event-on-request", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + end2EndStreamRequest(t, stream, "/", "GET", map[string]string{"User-Agent": "dd-test-scanner-log"}, map[string]string{}, false) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + checkForAppsecEvent(t, finished, map[string]int{"ua0-600-55x": 1}) + }) + + t.Run("blocking-event-on-request", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + err = stream.Send(&envoyextproc.ProcessingRequest{ + Request: &envoyextproc.ProcessingRequest_RequestHeaders{ + RequestHeaders: &envoyextproc.HttpHeaders{ + Headers: makeRequestHeaders(t, map[string]string{"User-Agent": "dd-test-scanner-log-block"}, "GET", "/"), + }, + }, + }) + require.NoError(t, err) + + res, err := stream.Recv() + require.Equal(t, uint32(0), res.GetImmediateResponse().GetGrpcStatus().Status) + require.Equal(t, envoytypes.StatusCode(403), res.GetImmediateResponse().GetStatus().Code) + require.Equal(t, "Content-Type", res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().Key) + require.Equal(t, "application/json", string(res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().RawValue)) + require.NoError(t, err) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + checkForAppsecEvent(t, finished, map[string]int{"ua0-600-56x": 1}) + + // Check for tags + span := finished[0] + require.Equal(t, true, span.Tag("appsec.event")) + require.Equal(t, true, span.Tag("appsec.blocked")) + }) +} + +func TestBlockingWithUserRulesFile(t *testing.T) { + t.Setenv("DD_APPSEC_RULES", "../../../internal/appsec/testdata/user_rules.json") + appsec.Start() + defer appsec.Stop() + if !appsec.Enabled() { + t.Skip("appsec disabled") + } + + setup := func() (envoyextproc.ExternalProcessorClient, mocktracer.Tracer, func()) { + rig, err := newEnvoyAppsecRig(t, false) + require.NoError(t, err) + + mt := mocktracer.Start() + + return rig.client, mt, func() { + rig.Close() + mt.Stop() + } + } + + t.Run("blocking-event-on-response", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + end2EndStreamRequest(t, stream, "/", "OPTION", map[string]string{"User-Agent": "dd-test-scanner-log-block"}, map[string]string{"User-Agent": "match-response-headers"}, true) + + // Handle the immediate response + res, err := stream.Recv() + require.Equal(t, uint32(0), res.GetImmediateResponse().GetGrpcStatus().Status) + require.Equal(t, envoytypes.StatusCode(418), res.GetImmediateResponse().GetStatus().Code) // 418 because of the rule file + require.Len(t, res.GetImmediateResponse().GetHeaders().SetHeaders, 1) + require.Equal(t, "Content-Type", res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().Key) + require.Equal(t, "application/json", string(res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().RawValue)) + require.NoError(t, err) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + checkForAppsecEvent(t, finished, map[string]int{"headers-003": 1}) + + // Check for tags + span := finished[0] + require.Equal(t, 1, span.Tag("_dd.appsec.enabled")) + require.Equal(t, true, span.Tag("appsec.event")) + require.Equal(t, true, span.Tag("appsec.blocked")) + }) + + t.Run("blocking-event-on-request-on-query", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + err = stream.Send(&envoyextproc.ProcessingRequest{ + Request: &envoyextproc.ProcessingRequest_RequestHeaders{ + RequestHeaders: &envoyextproc.HttpHeaders{ + Headers: makeRequestHeaders(t, map[string]string{"User-Agent": "Mistake Not..."}, "GET", "/hello?match=match-request-query"), + }, + }, + }) + require.NoError(t, err) + + res, err := stream.Recv() + require.Equal(t, uint32(0), res.GetImmediateResponse().GetGrpcStatus().Status) + require.Equal(t, envoytypes.StatusCode(418), res.GetImmediateResponse().GetStatus().Code) + require.Equal(t, "Content-Type", res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().Key) + require.Equal(t, "application/json", string(res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().RawValue)) + require.NoError(t, err) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + checkForAppsecEvent(t, finished, map[string]int{"query-002": 1}) + + // Check for tags + span := finished[0] + require.Equal(t, true, span.Tag("appsec.event")) + require.Equal(t, true, span.Tag("appsec.blocked")) + }) + + t.Run("blocking-event-on-request-on-cookies", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + err = stream.Send(&envoyextproc.ProcessingRequest{ + Request: &envoyextproc.ProcessingRequest_RequestHeaders{ + RequestHeaders: &envoyextproc.HttpHeaders{ + Headers: makeRequestHeaders(t, map[string]string{"Cookie": "foo=jdfoSDGFkivRG_234"}, "OPTIONS", "/"), + }, + }, + }) + require.NoError(t, err) + + res, err := stream.Recv() + require.Equal(t, uint32(0), res.GetImmediateResponse().GetGrpcStatus().Status) + require.Equal(t, envoytypes.StatusCode(418), res.GetImmediateResponse().GetStatus().Code) + require.Equal(t, "Content-Type", res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().Key) + require.Equal(t, "application/json", string(res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().RawValue)) + require.NoError(t, err) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + checkForAppsecEvent(t, finished, map[string]int{"tst-037-008": 1}) + + // Check for tags + span := finished[0] + require.Equal(t, true, span.Tag("appsec.event")) + require.Equal(t, true, span.Tag("appsec.blocked")) + }) +} + +func TestGeneratedSpan(t *testing.T) { + setup := func() (envoyextproc.ExternalProcessorClient, mocktracer.Tracer, func()) { + rig, err := newEnvoyAppsecRig(t, false) + require.NoError(t, err) + + mt := mocktracer.Start() + + return rig.client, mt, func() { + rig.Close() + mt.Stop() + } + } + + t.Run("request-span", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + end2EndStreamRequest(t, stream, "/resource-span", "GET", map[string]string{"user-agent": "Mistake Not...", "test-key": "test-value"}, map[string]string{"response-test-key": "response-test-value"}, false) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + + // Check for tags + span := finished[0] + require.Equal(t, "http.request", span.OperationName()) + require.Equal(t, "https://datadoghq.com/resource-span", span.Tag("http.url")) + require.Equal(t, "GET", span.Tag("http.method")) + require.Equal(t, "datadoghq.com", span.Tag("http.host")) + // require.Equal(t, "GET /resource-span", span.Tag("resource.name")) + require.Equal(t, "server", span.Tag("span.kind")) + require.Equal(t, "Mistake Not...", span.Tag("http.useragent")) + }) +} + +func TestXForwardedForHeaderClientIp(t *testing.T) { + t.Setenv("DD_APPSEC_RULES", "../../../internal/appsec/testdata/blocking.json") + appsec.Start() + defer appsec.Stop() + if !appsec.Enabled() { + t.Skip("appsec disabled") + } + + setup := func() (envoyextproc.ExternalProcessorClient, mocktracer.Tracer, func()) { + rig, err := newEnvoyAppsecRig(t, false) + require.NoError(t, err) + + mt := mocktracer.Start() + + return rig.client, mt, func() { + rig.Close() + mt.Stop() + } + } + + t.Run("client-ip", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + end2EndStreamRequest(t, stream, "/", "OPTION", + map[string]string{"User-Agent": "Mistake not...", "X-Forwarded-For": "18.18.18.18"}, + map[string]string{"User-Agent": "match-response-headers"}, + true) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + + // Check for tags + span := finished[0] + require.Equal(t, "18.18.18.18", span.Tag("http.client_ip")) + + // Appsec + require.Equal(t, 1, span.Tag("_dd.appsec.enabled")) + }) + + t.Run("blocking-client-ip", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + err = stream.Send(&envoyextproc.ProcessingRequest{ + Request: &envoyextproc.ProcessingRequest_RequestHeaders{ + RequestHeaders: &envoyextproc.HttpHeaders{ + Headers: makeRequestHeaders(t, map[string]string{"User-Agent": "Mistake not...", "X-Forwarded-For": "1.2.3.4"}, "GET", "/"), + }, + }, + }) + require.NoError(t, err) + + // Handle the immediate response + res, err := stream.Recv() + require.Equal(t, uint32(0), res.GetImmediateResponse().GetGrpcStatus().Status) + require.Equal(t, envoytypes.StatusCode(403), res.GetImmediateResponse().GetStatus().Code) + require.Equal(t, "Content-Type", res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().Key) + require.Equal(t, "application/json", string(res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().RawValue)) + require.NoError(t, err) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + checkForAppsecEvent(t, finished, map[string]int{"blk-001-001": 1}) + + // Check for tags + span := finished[0] + require.Equal(t, "1.2.3.4", span.Tag("http.client_ip")) + require.Equal(t, 1, span.Tag("_dd.appsec.enabled")) + require.Equal(t, true, span.Tag("appsec.event")) + require.Equal(t, true, span.Tag("appsec.blocked")) + }) +} + +func newEnvoyAppsecRig(t *testing.T, traceClient bool, interceptorOpts ...ddgrpc.Option) (*envoyAppsecRig, error) { + t.Helper() + + interceptorOpts = append([]ddgrpc.InterceptorOption{ddgrpc.WithServiceName("grpc")}, interceptorOpts...) + + server := grpc.NewServer() + + fixtureServer := new(envoyFixtureServer) + appsecSrv := AppsecEnvoyExternalProcessorServer(fixtureServer) + envoyextproc.RegisterExternalProcessorServer(server, appsecSrv) + + li, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, err + } + _, port, _ := net.SplitHostPort(li.Addr().String()) + // start our test fixtureServer. + go server.Serve(li) + + opts := []grpc.DialOption{grpc.WithInsecure()} + if traceClient { + opts = append(opts, + grpc.WithStreamInterceptor(ddgrpc.StreamClientInterceptor(interceptorOpts...)), + ) + } + conn, err := grpc.Dial(li.Addr().String(), opts...) + if err != nil { + return nil, fmt.Errorf("error dialing: %s", err) + } + return &envoyAppsecRig{ + fixtureServer: fixtureServer, + listener: li, + port: port, + server: server, + conn: conn, + client: envoyextproc.NewExternalProcessorClient(conn), + }, err +} + +// rig contains all servers and connections we'd need for a grpc integration test +type envoyAppsecRig struct { + fixtureServer *envoyFixtureServer + server *grpc.Server + port string + listener net.Listener + conn *grpc.ClientConn + client envoyextproc.ExternalProcessorClient +} + +func (r *envoyAppsecRig) Close() { + r.server.Stop() + r.conn.Close() +} + +type envoyFixtureServer struct { + envoyextproc.ExternalProcessorServer +} + +// Helper functions + +func end2EndStreamRequest(t *testing.T, stream envoyextproc.ExternalProcessor_ProcessClient, path string, method string, requestHeaders map[string]string, responseHeaders map[string]string, blockOnResponse bool) { + t.Helper() + + // First part: request + // 1- Send the headers + err := stream.Send(&envoyextproc.ProcessingRequest{ + Request: &envoyextproc.ProcessingRequest_RequestHeaders{ + RequestHeaders: &envoyextproc.HttpHeaders{ + Headers: makeRequestHeaders(t, requestHeaders, method, path), + }, + }, + }) + require.NoError(t, err) + + res, err := stream.Recv() + require.NoError(t, err) + require.Equal(t, envoyextproc.CommonResponse_CONTINUE, res.GetRequestHeaders().GetResponse().GetStatus()) + + // 2- Send the body + err = stream.Send(&envoyextproc.ProcessingRequest{ + Request: &envoyextproc.ProcessingRequest_RequestBody{ + RequestBody: &envoyextproc.HttpBody{ + Body: []byte("body"), + }, + }, + }) + require.NoError(t, err) + + res, err = stream.Recv() + require.NoError(t, err) + require.Equal(t, envoyextproc.CommonResponse_CONTINUE, res.GetRequestBody().GetResponse().GetStatus()) + + // 3- Send the trailers + err = stream.Send(&envoyextproc.ProcessingRequest{ + Request: &envoyextproc.ProcessingRequest_RequestTrailers{ + RequestTrailers: &envoyextproc.HttpTrailers{ + Trailers: &v3.HeaderMap{ + Headers: []*v3.HeaderValue{ + {Key: "key", Value: "value"}, + }, + }, + }, + }, + }) + require.NoError(t, err) + + res, err = stream.Recv() + require.NoError(t, err) + require.NotNil(t, res.GetRequestTrailers()) + + // Second part: response + // 1- Send the response headers + err = stream.Send(&envoyextproc.ProcessingRequest{ + Request: &envoyextproc.ProcessingRequest_ResponseHeaders{ + ResponseHeaders: &envoyextproc.HttpHeaders{ + Headers: makeResponseHeaders(t, responseHeaders, "200"), + }, + }, + }) + require.NoError(t, err) + + if blockOnResponse { + // Should have received an immediate response for blocking + // Let the test handle the response + return + } + + res, err = stream.Recv() + require.NoError(t, err) + require.Equal(t, envoyextproc.CommonResponse_CONTINUE, res.GetResponseHeaders().GetResponse().GetStatus()) + + // 2- Send the response body + err = stream.Send(&envoyextproc.ProcessingRequest{ + Request: &envoyextproc.ProcessingRequest_ResponseBody{ + ResponseBody: &envoyextproc.HttpBody{ + Body: []byte("body"), + EndOfStream: true, + }, + }, + }) + require.NoError(t, err) + + // The stream should now be closed + _, err = stream.Recv() + require.Equal(t, io.EOF, err) +} + +func checkForAppsecEvent(t *testing.T, finished []mocktracer.Span, expectedRuleIDs map[string]int) { + t.Helper() + + // The request should have the attack attempts + event := finished[len(finished)-1].Tag("_dd.appsec.json") + require.NotNil(t, event, "the _dd.appsec.json tag was not found") + + jsonText := event.(string) + type trigger struct { + Rule struct { + ID string `json:"id"` + } `json:"rule"` + } + var parsed struct { + Triggers []trigger `json:"triggers"` + } + err := json.Unmarshal([]byte(jsonText), &parsed) + require.NoError(t, err) + + histogram := map[string]uint8{} + for _, tr := range parsed.Triggers { + histogram[tr.Rule.ID]++ + } + + for ruleID, count := range expectedRuleIDs { + require.Equal(t, count, int(histogram[ruleID]), "rule %s has been triggered %d times but expected %d") + } + + require.Len(t, parsed.Triggers, len(expectedRuleIDs), "unexpected number of rules triggered") +} + +// Construct request headers +func makeRequestHeaders(t *testing.T, headers map[string]string, method string, path string) *v3.HeaderMap { + t.Helper() + + h := &v3.HeaderMap{} + for k, v := range headers { + h.Headers = append(h.Headers, &v3.HeaderValue{Key: k, RawValue: []byte(v)}) + } + + h.Headers = append(h.Headers, + &v3.HeaderValue{Key: ":method", RawValue: []byte(method)}, + &v3.HeaderValue{Key: ":path", RawValue: []byte(path)}, + &v3.HeaderValue{Key: ":scheme", RawValue: []byte("https")}, + &v3.HeaderValue{Key: ":authority", RawValue: []byte("datadoghq.com")}, + ) + + return h +} + +func makeResponseHeaders(t *testing.T, headers map[string]string, status string) *v3.HeaderMap { + t.Helper() + + h := &v3.HeaderMap{} + for k, v := range headers { + h.Headers = append(h.Headers, &v3.HeaderValue{Key: k, RawValue: []byte(v)}) + } + + h.Headers = append(h.Headers, &v3.HeaderValue{Key: ":status", RawValue: []byte(status)}) + + return h +} diff --git a/contrib/envoyproxy/go-control-plane/example_test.go b/contrib/envoyproxy/go-control-plane/example_test.go new file mode 100644 index 0000000000..f1e255dcaf --- /dev/null +++ b/contrib/envoyproxy/go-control-plane/example_test.go @@ -0,0 +1,44 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024 Datadog, Inc. + +package go_control_plane_test + +import ( + "log" + "net" + + "google.golang.org/grpc" + + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + gocontrolplane "gopkg.in/DataDog/dd-trace-go.v1/contrib/envoyproxy/go-control-plane" +) + +// interface fpr external processing server +type envoyExtProcServer struct { + extprocv3.ExternalProcessorServer +} + +func Example_server() { + // Create a listener for the server. + ln, err := net.Listen("tcp", ":50051") + if err != nil { + log.Fatal(err) + } + + // Initialize the grpc server as normal, using the envoy server interceptor. + s := grpc.NewServer() + srv := &envoyExtProcServer{} + + // Register the appsec envoy external processor service + appsecSrv := gocontrolplane.AppsecEnvoyExternalProcessorServer(srv) + extprocv3.RegisterExternalProcessorServer(s, appsecSrv) + + // ... register your services + + // Start serving incoming connections. + if err := s.Serve(ln); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/contrib/envoyproxy/go-control-plane/fakehttp.go b/contrib/envoyproxy/go-control-plane/fakehttp.go new file mode 100644 index 0000000000..3f20725e1b --- /dev/null +++ b/contrib/envoyproxy/go-control-plane/fakehttp.go @@ -0,0 +1,189 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024 Datadog, Inc. + +package go_control_plane + +import ( + "context" + "crypto/tls" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + extproc "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + "google.golang.org/grpc/metadata" +) + +// checkPseudoRequestHeaders Verify the required HTTP2 headers are present +// Some mandatory headers need to be set. It can happen when it wasn't a real HTTP2 request sent by Envoy, +func checkPseudoRequestHeaders(headers map[string]string) error { + for _, header := range []string{":authority", ":scheme", ":path", ":method"} { + if _, ok := headers[header]; !ok { + return fmt.Errorf("missing required headers: %q", header) + } + } + + return nil +} + +// checkPseudoResponseHeaders verifies the required HTTP2 headers are present +// Some mandatory headers need to be set. It can happen when it wasn't a real HTTP2 request sent by Envoy, +func checkPseudoResponseHeaders(headers map[string]string) error { + if _, ok := headers[":status"]; !ok { + return fmt.Errorf("missing required ':status' headers") + } + + return nil +} + +func getRemoteAddr(md metadata.MD) string { + xfwd := md.Get("x-forwarded-for") + length := len(xfwd) + if length == 0 { + return "" + } + + // Get the first right value of x-forwarded-for headers + // The rightmost IP address is the one that will be used as the remote client IP + // https://datadoghq.atlassian.net/wiki/spaces/TS/pages/2766733526/Sensitive+IP+information#Where-does-the-value-of-the-http.client_ip-tag-come-from%3F + return xfwd[length-1] +} + +// splitPseudoHeaders splits normal headers of the initial request made by the client and the pseudo headers of HTTP/2 +// - Format the headers to be used by the tracer as a map[string][]string +// - Set headers keys to be canonical +func splitPseudoHeaders(receivedHeaders []*corev3.HeaderValue) (headers map[string][]string, pseudoHeaders map[string]string) { + headers = make(map[string][]string, len(receivedHeaders)-4) + pseudoHeaders = make(map[string]string, 4) + for _, v := range receivedHeaders { + key := v.GetKey() + if key == "" { + continue + } + if key[0] == ':' { + pseudoHeaders[key] = string(v.GetRawValue()) + continue + } + + headers[http.CanonicalHeaderKey(key)] = []string{string(v.GetRawValue())} + } + return headers, pseudoHeaders +} + +func createFakeResponseWriter(w http.ResponseWriter, res *extproc.ProcessingRequest_ResponseHeaders) error { + headers, pseudoHeaders := splitPseudoHeaders(res.ResponseHeaders.GetHeaders().GetHeaders()) + + if err := checkPseudoResponseHeaders(pseudoHeaders); err != nil { + return err + } + + status, err := strconv.Atoi(pseudoHeaders[":status"]) + if err != nil { + return fmt.Errorf("error parsing status code %q: %w", pseudoHeaders[":status"], err) + } + + for k, v := range headers { + w.Header().Set(k, strings.Join(v, ",")) + } + + w.WriteHeader(status) + return nil +} + +// newRequest creates a new http.Request from an ext_proc RequestHeaders message +func newRequest(ctx context.Context, req *extproc.ProcessingRequest_RequestHeaders) (*http.Request, error) { + headers, pseudoHeaders := splitPseudoHeaders(req.RequestHeaders.GetHeaders().GetHeaders()) + if err := checkPseudoRequestHeaders(pseudoHeaders); err != nil { + return nil, err + } + + parsedURL, err := url.Parse(fmt.Sprintf("%s://%s%s", pseudoHeaders[":scheme"], pseudoHeaders[":authority"], pseudoHeaders[":path"])) + if err != nil { + return nil, fmt.Errorf( + "error building envoy URI from scheme %q, from host %q and from path %q: %w", + pseudoHeaders[":scheme"], + pseudoHeaders[":host"], + pseudoHeaders[":path"], + err) + } + + var remoteAddr string + md, ok := metadata.FromIncomingContext(ctx) + if ok { + remoteAddr = getRemoteAddr(md) + } + + var tlsState *tls.ConnectionState + if pseudoHeaders[":scheme"] == "https" { + tlsState = &tls.ConnectionState{} + } + + headers["Host"] = append(headers["Host"], pseudoHeaders[":authority"]) + + return (&http.Request{ + Method: pseudoHeaders[":method"], + Host: pseudoHeaders[":authority"], + RequestURI: pseudoHeaders[":path"], + URL: parsedURL, + Header: headers, + RemoteAddr: remoteAddr, + TLS: tlsState, + }).WithContext(ctx), nil +} + +type fakeResponseWriter struct { + mu sync.Mutex + status int + body []byte + headers http.Header +} + +// Reset resets the fakeResponseWriter to its initial state +func (w *fakeResponseWriter) Reset() { + w.mu.Lock() + defer w.mu.Unlock() + w.status = 0 + w.body = nil + w.headers = make(http.Header) +} + +// Status is not in the [http.ResponseWriter] interface, but it is cast into it by the tracing code +func (w *fakeResponseWriter) Status() int { + w.mu.Lock() + defer w.mu.Unlock() + return w.status +} + +func (w *fakeResponseWriter) WriteHeader(status int) { + w.mu.Lock() + defer w.mu.Unlock() + w.status = status +} + +func (w *fakeResponseWriter) Header() http.Header { + w.mu.Lock() + defer w.mu.Unlock() + return w.headers +} + +func (w *fakeResponseWriter) Write(b []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + w.body = append(w.body, b...) + return len(b), nil +} + +var _ http.ResponseWriter = &fakeResponseWriter{} + +// newFakeResponseWriter creates a new fakeResponseWriter that can be used to store the response a [http.Handler] made +func newFakeResponseWriter() *fakeResponseWriter { + return &fakeResponseWriter{ + headers: make(http.Header), + } +} diff --git a/contrib/internal/httptrace/response_writer.go b/contrib/internal/httptrace/response_writer.go index 2bbc31bad7..f44fff762f 100644 --- a/contrib/internal/httptrace/response_writer.go +++ b/contrib/internal/httptrace/response_writer.go @@ -16,6 +16,13 @@ type responseWriter struct { status int } +// ResetStatusCode resets the status code of the response writer. +func ResetStatusCode(w http.ResponseWriter) { + if rw, ok := w.(*responseWriter); ok { + rw.status = 0 + } +} + func newResponseWriter(w http.ResponseWriter) *responseWriter { return &responseWriter{w, 0} } diff --git a/go.mod b/go.mod index f3a841b96d..62452c745f 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( github.com/elastic/go-elasticsearch/v8 v8.4.0 github.com/emicklei/go-restful v2.16.0+incompatible github.com/emicklei/go-restful/v3 v3.11.0 + github.com/envoyproxy/go-control-plane v0.12.0 github.com/garyburd/redigo v1.6.4 github.com/gin-gonic/gin v1.9.1 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 @@ -154,6 +155,7 @@ require ( github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.0 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect + github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -162,6 +164,7 @@ require ( github.com/eapache/queue v1.1.0 // indirect github.com/ebitengine/purego v0.6.0-alpha.5 // indirect github.com/elastic/elastic-transport-go/v8 v8.1.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect github.com/fatih/color v1.16.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect diff --git a/go.sum b/go.sum index 6471b96ff5..8388b569ef 100644 --- a/go.sum +++ b/go.sum @@ -890,6 +890,8 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 h1:DBmgJDC9dTfkVyGgipamEh2BpGYxScCH1TOF1LL1cXc= +github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50/go.mod h1:5e1+Vvlzido69INQaVO6d87Qn543Xr6nooe9Kz7oBFM= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= @@ -1118,9 +1120,13 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= +github.com/envoyproxy/go-control-plane v0.12.0 h1:4X+VP1GHd1Mhj6IB5mMeGbLCleqxjletLK6K0rbxyZI= +github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= +github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= +github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= diff --git a/internal/appsec/testdata/user_rules.json b/internal/appsec/testdata/user_rules.json index 6acb14089e..a13a0d67ed 100644 --- a/internal/appsec/testdata/user_rules.json +++ b/internal/appsec/testdata/user_rules.json @@ -53,6 +53,33 @@ "block" ] }, + { + "id": "tst-037-008", + "name": "Test block on cookies", + "tags": { + "type": "lfi", + "crs_id": "000008", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + } + ], + "regex": "jdfoSDGFkivRG_234" + }, + "operator": "match_regex" + } + ], + "transformers": [], + "on_match": [ + "block" + ] + }, + { "id": "headers-003", "name": "query match", From 5a8a82ce66fd77fcefa4f983968ab8f0ea89556a Mon Sep 17 00:00:00 2001 From: Quinna Halim Date: Mon, 16 Dec 2024 10:35:33 -0500 Subject: [PATCH 03/13] add go mod to workflows/apps (#3036) --- .github/workflows/apps/go.mod | 16 ++++++++++++++++ .github/workflows/apps/go.sum | 16 ++++++++++++++++ .github/workflows/apps/latest_major_version.go | 2 +- 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/apps/go.mod create mode 100644 .github/workflows/apps/go.sum diff --git a/.github/workflows/apps/go.mod b/.github/workflows/apps/go.mod new file mode 100644 index 0000000000..19c2d9dfde --- /dev/null +++ b/.github/workflows/apps/go.mod @@ -0,0 +1,16 @@ +module github.com/DataDog/dd-trace-go/.github/workflows/apps + +go 1.23.3 + +require ( + github.com/Masterminds/semver/v3 v3.3.1 + github.com/stretchr/testify v1.10.0 + golang.org/x/mod v0.22.0 + gopkg.in/DataDog/dd-trace-go.v1 v1.70.1 +) + +require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/.github/workflows/apps/go.sum b/.github/workflows/apps/go.sum new file mode 100644 index 0000000000..4958ac6c0a --- /dev/null +++ b/.github/workflows/apps/go.sum @@ -0,0 +1,16 @@ +github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7rj+4nv4= +github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +gopkg.in/DataDog/dd-trace-go.v1 v1.70.1 h1:ZIRxAKlr3xr6xbMUDs3IDa6xq+ISv9zxyjaDCfwDjMY= +gopkg.in/DataDog/dd-trace-go.v1 v1.70.1/go.mod h1:PMOSkeY4VfXiuPvGodeNLCZCFYU2VfOvjVI6cX5bGrc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/.github/workflows/apps/latest_major_version.go b/.github/workflows/apps/latest_major_version.go index d1a7a5ccba..f471afe580 100644 --- a/.github/workflows/apps/latest_major_version.go +++ b/.github/workflows/apps/latest_major_version.go @@ -1,7 +1,7 @@ // Unless explicitly stated otherwise all files in this repository are licensed // under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016 Datadog, Inc. +// Copyright 2024 Datadog, Inc. package main From cd21985fed1106b3039d65b9c3be3281a7d2ee79 Mon Sep 17 00:00:00 2001 From: Eliott Bouhana <47679741+eliottness@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:10:02 +0100 Subject: [PATCH 04/13] go.mod: module go.opentelemetry.io/collector/pdata@latest found (v1.21.0), but does not contain package go.opentelemetry.io/collector/pdata/internal/data/protogen/profiles/v1experimental (#3042) Signed-off-by: Eliott Bouhana --- go.mod | 1 + go.sum | 2 ++ internal/agent.go | 8 ++++++++ internal/apps/go.mod | 1 + internal/apps/go.sum | 2 ++ internal/exectracetest/go.mod | 1 + internal/exectracetest/go.sum | 2 ++ 7 files changed, 17 insertions(+) diff --git a/go.mod b/go.mod index 62452c745f..e301a9da7a 100644 --- a/go.mod +++ b/go.mod @@ -94,6 +94,7 @@ require ( github.com/vektah/gqlparser/v2 v2.5.16 github.com/zenazn/goji v1.0.1 go.mongodb.org/mongo-driver v1.12.1 + go.opentelemetry.io/collector/pdata/pprofile v0.104.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go.opentelemetry.io/otel v1.27.0 go.opentelemetry.io/otel/trace v1.27.0 diff --git a/go.sum b/go.sum index 8388b569ef..ba7cea7f7c 100644 --- a/go.sum +++ b/go.sum @@ -2213,6 +2213,8 @@ go.opentelemetry.io/collector/config/configtelemetry v0.104.0 h1:eHv98XIhapZA8Mg go.opentelemetry.io/collector/config/configtelemetry v0.104.0/go.mod h1:WxWKNVAQJg/Io1nA3xLgn/DWLE/W1QOB2+/Js3ACi40= go.opentelemetry.io/collector/pdata v1.11.0 h1:rzYyV1zfTQQz1DI9hCiaKyyaczqawN75XO9mdXmR/hE= go.opentelemetry.io/collector/pdata v1.11.0/go.mod h1:IHxHsp+Jq/xfjORQMDJjSH6jvedOSTOyu3nbxqhWSYE= +go.opentelemetry.io/collector/pdata/pprofile v0.104.0 h1:MYOIHvPlKEJbWLiBKFQWGD0xd2u22xGVLt4jPbdxP4Y= +go.opentelemetry.io/collector/pdata/pprofile v0.104.0/go.mod h1:7WpyHk2wJZRx70CGkBio8klrYTTXASbyIhf+rH4FKnA= go.opentelemetry.io/collector/semconv v0.104.0 h1:dUvajnh+AYJLEW/XOPk0T0BlwltSdi3vrjO7nSOos3k= go.opentelemetry.io/collector/semconv v0.104.0/go.mod h1:yMVUCNoQPZVq/IPfrHrnntZTWsLf5YGZ7qwKulIl5hw= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= diff --git a/internal/agent.go b/internal/agent.go index f4bcdce8b0..8b0023274a 100644 --- a/internal/agent.go +++ b/internal/agent.go @@ -11,6 +11,14 @@ import ( "os" "gopkg.in/DataDog/dd-trace-go.v1/internal/log" + + // OTel did a breaking change to the module go.opentelemetry.io/collector/pdata which is imported by the agent + // and go.opentelemetry.io/collector/pdata/pprofile depends on it and is breaking because of it + // For some reason the dependency closure won't let use upgrade this module past the point where it does not break anymore + // So we are forced to add a blank import of this module to give us back the control over its version + // + // TODO: remove this once github.com/datadog-agent/pkg/trace has upgraded both modules past the breaking change + _ "go.opentelemetry.io/collector/pdata/pprofile" ) const ( diff --git a/internal/apps/go.mod b/internal/apps/go.mod index d90e69002e..76197025d9 100644 --- a/internal/apps/go.mod +++ b/internal/apps/go.mod @@ -44,6 +44,7 @@ require ( go.opentelemetry.io/collector/component v0.104.0 // indirect go.opentelemetry.io/collector/config/configtelemetry v0.104.0 // indirect go.opentelemetry.io/collector/pdata v1.11.0 // indirect + go.opentelemetry.io/collector/pdata/pprofile v0.104.0 // indirect go.opentelemetry.io/collector/semconv v0.104.0 // indirect go.opentelemetry.io/otel v1.27.0 // indirect go.opentelemetry.io/otel/metric v1.27.0 // indirect diff --git a/internal/apps/go.sum b/internal/apps/go.sum index 1ca195d002..a0e2472c51 100644 --- a/internal/apps/go.sum +++ b/internal/apps/go.sum @@ -195,6 +195,8 @@ go.opentelemetry.io/collector/config/configtelemetry v0.104.0 h1:eHv98XIhapZA8Mg go.opentelemetry.io/collector/config/configtelemetry v0.104.0/go.mod h1:WxWKNVAQJg/Io1nA3xLgn/DWLE/W1QOB2+/Js3ACi40= go.opentelemetry.io/collector/pdata v1.11.0 h1:rzYyV1zfTQQz1DI9hCiaKyyaczqawN75XO9mdXmR/hE= go.opentelemetry.io/collector/pdata v1.11.0/go.mod h1:IHxHsp+Jq/xfjORQMDJjSH6jvedOSTOyu3nbxqhWSYE= +go.opentelemetry.io/collector/pdata/pprofile v0.104.0 h1:MYOIHvPlKEJbWLiBKFQWGD0xd2u22xGVLt4jPbdxP4Y= +go.opentelemetry.io/collector/pdata/pprofile v0.104.0/go.mod h1:7WpyHk2wJZRx70CGkBio8klrYTTXASbyIhf+rH4FKnA= go.opentelemetry.io/collector/semconv v0.104.0 h1:dUvajnh+AYJLEW/XOPk0T0BlwltSdi3vrjO7nSOos3k= go.opentelemetry.io/collector/semconv v0.104.0/go.mod h1:yMVUCNoQPZVq/IPfrHrnntZTWsLf5YGZ7qwKulIl5hw= go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= diff --git a/internal/exectracetest/go.mod b/internal/exectracetest/go.mod index 33a74a94ba..9292ebda48 100644 --- a/internal/exectracetest/go.mod +++ b/internal/exectracetest/go.mod @@ -57,6 +57,7 @@ require ( go.opentelemetry.io/collector/component v0.104.0 // indirect go.opentelemetry.io/collector/config/configtelemetry v0.104.0 // indirect go.opentelemetry.io/collector/pdata v1.11.0 // indirect + go.opentelemetry.io/collector/pdata/pprofile v0.104.0 // indirect go.opentelemetry.io/collector/semconv v0.104.0 // indirect go.opentelemetry.io/otel v1.27.0 // indirect go.opentelemetry.io/otel/metric v1.27.0 // indirect diff --git a/internal/exectracetest/go.sum b/internal/exectracetest/go.sum index 7321d5519b..00a653dc7e 100644 --- a/internal/exectracetest/go.sum +++ b/internal/exectracetest/go.sum @@ -201,6 +201,8 @@ go.opentelemetry.io/collector/config/configtelemetry v0.104.0 h1:eHv98XIhapZA8Mg go.opentelemetry.io/collector/config/configtelemetry v0.104.0/go.mod h1:WxWKNVAQJg/Io1nA3xLgn/DWLE/W1QOB2+/Js3ACi40= go.opentelemetry.io/collector/pdata v1.11.0 h1:rzYyV1zfTQQz1DI9hCiaKyyaczqawN75XO9mdXmR/hE= go.opentelemetry.io/collector/pdata v1.11.0/go.mod h1:IHxHsp+Jq/xfjORQMDJjSH6jvedOSTOyu3nbxqhWSYE= +go.opentelemetry.io/collector/pdata/pprofile v0.104.0 h1:MYOIHvPlKEJbWLiBKFQWGD0xd2u22xGVLt4jPbdxP4Y= +go.opentelemetry.io/collector/pdata/pprofile v0.104.0/go.mod h1:7WpyHk2wJZRx70CGkBio8klrYTTXASbyIhf+rH4FKnA= go.opentelemetry.io/collector/semconv v0.104.0 h1:dUvajnh+AYJLEW/XOPk0T0BlwltSdi3vrjO7nSOos3k= go.opentelemetry.io/collector/semconv v0.104.0/go.mod h1:yMVUCNoQPZVq/IPfrHrnntZTWsLf5YGZ7qwKulIl5hw= go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= From e8b2b8c7316b415c2657c407baee163b835b60b5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:44:44 -0500 Subject: [PATCH 05/13] chore: update latest majors (#2993) --- latests.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 latests.txt diff --git a/latests.txt b/latests.txt new file mode 100644 index 0000000000..49ed641861 --- /dev/null +++ b/latests.txt @@ -0,0 +1,12 @@ +Latest DD major version of confluentinc/confluent-kafka-go: 2 +Latest DD major version of dimfeld/httptreemux: 5 +Latest DD major version of elastic/go-elasticsearch: 8 +Latest DD major version of emicklei/go-restful: 3 +Latest DD major version of go-chi/chi: 5 +Latest DD major version of go-pg/pg: 10 +Latest DD major version of go-redis/redis: 9 +Latest DD major version of gofiber/fiber: 2 +Latest DD major version of jackc/pgx: 5 +Latest DD major version of labstack/echo: 4 +Latest DD major version of redis/go-redis: 9 +Latest DD major version of vektah/gqlparser: 2 From d15e61ad1b9c08739c94b87ea8ec4b957a64cad4 Mon Sep 17 00:00:00 2001 From: Eliott Bouhana <47679741+eliottness@users.noreply.github.com> Date: Wed, 18 Dec 2024 10:42:18 +0100 Subject: [PATCH 06/13] appsec: stop storing span tags, directly call span.SetTag (#3044) Signed-off-by: Eliott Bouhana --- contrib/99designs/gqlgen/tracer.go | 4 +-- contrib/google.golang.org/grpc/appsec.go | 8 +++--- contrib/graph-gophers/graphql-go/graphql.go | 4 +-- contrib/graphql-go/graphql/graphql.go | 4 +-- internal/appsec/emitter/graphqlsec/request.go | 8 +++--- internal/appsec/emitter/grpcsec/grpc.go | 8 +++--- internal/appsec/emitter/httpsec/http.go | 13 +++++---- .../emitter/trace/service_entry_span.go | 28 +++++++++---------- internal/appsec/emitter/waf/context.go | 8 +++--- 9 files changed, 42 insertions(+), 43 deletions(-) diff --git a/contrib/99designs/gqlgen/tracer.go b/contrib/99designs/gqlgen/tracer.go index ffdcb0c550..d4e1e0f52a 100644 --- a/contrib/99designs/gqlgen/tracer.go +++ b/contrib/99designs/gqlgen/tracer.go @@ -103,7 +103,7 @@ func (t *gqlTracer) Validate(_ graphql.ExecutableSchema) error { func (t *gqlTracer) InterceptOperation(ctx context.Context, next graphql.OperationHandler) graphql.ResponseHandler { opCtx := graphql.GetOperationContext(ctx) span, ctx := t.createRootSpan(ctx, opCtx) - ctx, req := graphqlsec.StartRequestOperation(ctx, graphqlsec.RequestOperationArgs{ + ctx, req := graphqlsec.StartRequestOperation(ctx, span, graphqlsec.RequestOperationArgs{ RawQuery: opCtx.RawQuery, OperationName: opCtx.OperationName, Variables: opCtx.Variables, @@ -137,7 +137,7 @@ func (t *gqlTracer) InterceptOperation(ctx context.Context, next graphql.Operati } query.Finish(executionOperationRes) - req.Finish(span, requestOperationRes) + req.Finish(requestOperationRes) return response } } diff --git a/contrib/google.golang.org/grpc/appsec.go b/contrib/google.golang.org/grpc/appsec.go index f7b4aecf53..4fb2f47a28 100644 --- a/contrib/google.golang.org/grpc/appsec.go +++ b/contrib/google.golang.org/grpc/appsec.go @@ -44,7 +44,7 @@ func appsecUnaryHandlerMiddleware(method string, span ddtrace.Span, handler grpc remoteAddr = p.Addr.String() } - ctx, op, blockAtomic := grpcsec.StartHandlerOperation(ctx, grpcsec.HandlerOperationArgs{ + ctx, op, blockAtomic := grpcsec.StartHandlerOperation(ctx, span, grpcsec.HandlerOperationArgs{ Method: method, Metadata: md, RemoteAddr: remoteAddr, @@ -55,7 +55,7 @@ func appsecUnaryHandlerMiddleware(method string, span ddtrace.Span, handler grpc if statusErr, ok := rpcErr.(interface{ GRPCStatus() *status.Status }); ok && !applyAction(blockAtomic, &rpcErr) { statusCode = int(statusErr.GRPCStatus().Code()) } - op.Finish(span, grpcsec.HandlerOperationRes{StatusCode: statusCode}) + op.Finish(grpcsec.HandlerOperationRes{StatusCode: statusCode}) applyAction(blockAtomic, &rpcErr) }() @@ -90,7 +90,7 @@ func appsecStreamHandlerMiddleware(method string, span ddtrace.Span, handler grp } // Create the handler operation and listen to blocking gRPC actions to detect a blocking condition - ctx, op, blockAtomic := grpcsec.StartHandlerOperation(ctx, grpcsec.HandlerOperationArgs{ + ctx, op, blockAtomic := grpcsec.StartHandlerOperation(ctx, span, grpcsec.HandlerOperationArgs{ Method: method, Metadata: md, RemoteAddr: remoteAddr, @@ -104,7 +104,7 @@ func appsecStreamHandlerMiddleware(method string, span ddtrace.Span, handler grp statusCode = int(res.Status()) } - op.Finish(span, grpcsec.HandlerOperationRes{StatusCode: statusCode}) + op.Finish(grpcsec.HandlerOperationRes{StatusCode: statusCode}) applyAction(blockAtomic, &rpcErr) }() diff --git a/contrib/graph-gophers/graphql-go/graphql.go b/contrib/graph-gophers/graphql-go/graphql.go index 040e7dff04..40f026a842 100644 --- a/contrib/graph-gophers/graphql-go/graphql.go +++ b/contrib/graph-gophers/graphql-go/graphql.go @@ -70,7 +70,7 @@ func (t *Tracer) TraceQuery(ctx context.Context, queryString, operationName stri } span, ctx := ddtracer.StartSpanFromContext(ctx, t.cfg.querySpanName, opts...) - ctx, request := graphqlsec.StartRequestOperation(ctx, graphqlsec.RequestOperationArgs{ + ctx, request := graphqlsec.StartRequestOperation(ctx, span, graphqlsec.RequestOperationArgs{ RawQuery: queryString, OperationName: operationName, Variables: variables, @@ -92,7 +92,7 @@ func (t *Tracer) TraceQuery(ctx context.Context, queryString, operationName stri err = fmt.Errorf("%s (and %d more errors)", errs[0], n-1) } defer span.Finish(ddtracer.WithError(err)) - defer request.Finish(span, graphqlsec.RequestOperationRes{Error: err}) + defer request.Finish(graphqlsec.RequestOperationRes{Error: err}) query.Finish(graphqlsec.ExecutionOperationRes{Error: err}) } } diff --git a/contrib/graphql-go/graphql/graphql.go b/contrib/graphql-go/graphql/graphql.go index 70c131d314..5d646a7f73 100644 --- a/contrib/graphql-go/graphql/graphql.go +++ b/contrib/graphql-go/graphql/graphql.go @@ -72,7 +72,7 @@ type contextData struct { // finish closes the top-level request operation, as well as the server span. func (c *contextData) finish(data any, err error) { defer c.serverSpan.Finish(tracer.WithError(err)) - c.requestOp.Finish(c.serverSpan, graphqlsec.RequestOperationRes{Data: data, Error: err}) + c.requestOp.Finish(graphqlsec.RequestOperationRes{Data: data, Error: err}) } var extensionName = reflect.TypeOf((*datadogExtension)(nil)).Elem().Name() @@ -97,7 +97,7 @@ func (i datadogExtension) Init(ctx context.Context, params *graphql.Params) cont tracer.Tag(ext.Component, componentName), tracer.Measured(), ) - ctx, request := graphqlsec.StartRequestOperation(ctx, graphqlsec.RequestOperationArgs{ + ctx, request := graphqlsec.StartRequestOperation(ctx, span, graphqlsec.RequestOperationArgs{ RawQuery: params.RequestString, Variables: params.VariableValues, OperationName: params.OperationName, diff --git a/internal/appsec/emitter/graphqlsec/request.go b/internal/appsec/emitter/graphqlsec/request.go index 20ae575660..6153c8b4c6 100644 --- a/internal/appsec/emitter/graphqlsec/request.go +++ b/internal/appsec/emitter/graphqlsec/request.go @@ -44,10 +44,10 @@ type ( // Finish the GraphQL query operation, along with the given results, and emit a finish event up in // the operation stack. -func (op *RequestOperation) Finish(span trace.TagSetter, res RequestOperationRes) { +func (op *RequestOperation) Finish(res RequestOperationRes) { dyngo.FinishOperation(op, res) if op.wafContextOwner { - op.ContextOperation.Finish(span) + op.ContextOperation.Finish() } } @@ -58,10 +58,10 @@ func (RequestOperationRes) IsResultOf(*RequestOperation) {} // emits a start event up in the operation stack. The operation is usually linked to tge global root // operation. The operation is tracked on the returned context, and can be extracted later on using // FromContext. -func StartRequestOperation(ctx context.Context, args RequestOperationArgs) (context.Context, *RequestOperation) { +func StartRequestOperation(ctx context.Context, span trace.TagSetter, args RequestOperationArgs) (context.Context, *RequestOperation) { wafOp, found := dyngo.FindOperation[waf.ContextOperation](ctx) if !found { // Usually we can find the HTTP Handler Operation as the parent, but it's technically optional - wafOp, ctx = waf.StartContextOperation(ctx) + wafOp, ctx = waf.StartContextOperation(ctx, span) } op := &RequestOperation{ diff --git a/internal/appsec/emitter/grpcsec/grpc.go b/internal/appsec/emitter/grpcsec/grpc.go index 2495e8bbd5..1d0e305bd9 100644 --- a/internal/appsec/emitter/grpcsec/grpc.go +++ b/internal/appsec/emitter/grpcsec/grpc.go @@ -77,10 +77,10 @@ func (HandlerOperationRes) IsResultOf(*HandlerOperation) {} // given arguments and parent operation, and emits a start event up in the // operation stack. When parent is nil, the operation is linked to the global // root operation. -func StartHandlerOperation(ctx context.Context, args HandlerOperationArgs) (context.Context, *HandlerOperation, *atomic.Pointer[actions.BlockGRPC]) { +func StartHandlerOperation(ctx context.Context, span trace.TagSetter, args HandlerOperationArgs) (context.Context, *HandlerOperation, *atomic.Pointer[actions.BlockGRPC]) { wafOp, found := dyngo.FindOperation[waf.ContextOperation](ctx) if !found { - wafOp, ctx = waf.StartContextOperation(ctx) + wafOp, ctx = waf.StartContextOperation(ctx, span) } op := &HandlerOperation{ Operation: dyngo.NewOperation(wafOp), @@ -117,9 +117,9 @@ func MonitorResponseMessage(ctx context.Context, msg any) error { // Finish the gRPC handler operation, along with the given results, and emit a // finish event up in the operation stack. -func (op *HandlerOperation) Finish(span trace.TagSetter, res HandlerOperationRes) { +func (op *HandlerOperation) Finish(res HandlerOperationRes) { dyngo.FinishOperation(op, res) if op.wafContextOwner { - op.ContextOperation.Finish(span) + op.ContextOperation.Finish() } } diff --git a/internal/appsec/emitter/httpsec/http.go b/internal/appsec/emitter/httpsec/http.go index 41ebfa7e23..16cfa2ba3e 100644 --- a/internal/appsec/emitter/httpsec/http.go +++ b/internal/appsec/emitter/httpsec/http.go @@ -19,6 +19,7 @@ import ( "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo" + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/emitter/trace" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/emitter/waf" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/emitter/waf/actions" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/emitter/waf/addresses" @@ -57,10 +58,10 @@ type ( func (HandlerOperationArgs) IsArgOf(*HandlerOperation) {} func (HandlerOperationRes) IsResultOf(*HandlerOperation) {} -func StartOperation(ctx context.Context, args HandlerOperationArgs) (*HandlerOperation, *atomic.Pointer[actions.BlockHTTP], context.Context) { +func StartOperation(ctx context.Context, args HandlerOperationArgs, span trace.TagSetter) (*HandlerOperation, *atomic.Pointer[actions.BlockHTTP], context.Context) { wafOp, found := dyngo.FindOperation[waf.ContextOperation](ctx) if !found { - wafOp, ctx = waf.StartContextOperation(ctx) + wafOp, ctx = waf.StartContextOperation(ctx, span) } op := &HandlerOperation{ @@ -79,10 +80,10 @@ func StartOperation(ctx context.Context, args HandlerOperationArgs) (*HandlerOpe } // Finish the HTTP handler operation and its children operations and write everything to the service entry span. -func (op *HandlerOperation) Finish(res HandlerOperationRes, span ddtrace.Span) { +func (op *HandlerOperation) Finish(res HandlerOperationRes) { dyngo.FinishOperation(op, res) if op.wafContextOwner { - op.ContextOperation.Finish(span) + op.ContextOperation.Finish() } } @@ -142,7 +143,7 @@ func BeforeHandle( Cookies: makeCookies(r.Cookies()), QueryParams: r.URL.Query(), PathParams: pathParams, - }) + }, span) tr := r.WithContext(ctx) var blocked atomic.Bool @@ -154,7 +155,7 @@ func BeforeHandle( op.Finish(HandlerOperationRes{ Headers: opts.ResponseHeaderCopier(w), StatusCode: statusCode, - }, span) + }) if blockPtr := blockAtomic.Swap(nil); blockPtr != nil { blockPtr.Handler.ServeHTTP(w, tr) diff --git a/internal/appsec/emitter/trace/service_entry_span.go b/internal/appsec/emitter/trace/service_entry_span.go index 98e14b092f..802a82e7e8 100644 --- a/internal/appsec/emitter/trace/service_entry_span.go +++ b/internal/appsec/emitter/trace/service_entry_span.go @@ -18,9 +18,9 @@ type ( // ServiceEntrySpanOperation is a dyngo.Operation that holds a the first span of a service. Usually a http or grpc span. ServiceEntrySpanOperation struct { dyngo.Operation - tags map[string]any - jsonTags map[string]any - mu sync.Mutex + jsonTags map[string]any + tagSetter TagSetter + mu sync.Mutex } // ServiceEntrySpanArgs is the arguments for a ServiceEntrySpanOperation @@ -52,7 +52,7 @@ func (ServiceEntrySpanArgs) IsArgOf(*ServiceEntrySpanOperation) {} func (op *ServiceEntrySpanOperation) SetTag(key string, value any) { op.mu.Lock() defer op.mu.Unlock() - op.tags[key] = value + op.tagSetter.SetTag(key, value) } // SetSerializableTag adds the key/value pair to the tags to add to the service entry span. @@ -76,7 +76,7 @@ func (op *ServiceEntrySpanOperation) SetSerializableTags(tags map[string]any) { func (op *ServiceEntrySpanOperation) setSerializableTag(key string, value any) { switch value.(type) { case string, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float32, float64, bool: - op.tags[key] = value + op.tagSetter.SetTag(key, value) default: op.jsonTags[key] = value } @@ -87,7 +87,7 @@ func (op *ServiceEntrySpanOperation) SetTags(tags map[string]any) { op.mu.Lock() defer op.mu.Unlock() for k, v := range tags { - op.tags[k] = v + op.tagSetter.SetTag(k, v) } } @@ -96,7 +96,7 @@ func (op *ServiceEntrySpanOperation) SetStringTags(tags map[string]string) { op.mu.Lock() defer op.mu.Unlock() for k, v := range tags { - op.tags[k] = v + op.tagSetter.SetTag(k, v) } } @@ -126,17 +126,18 @@ func (op *ServiceEntrySpanOperation) OnSpanTagEvent(tag SpanTag) { op.SetTag(tag.Key, tag.Value) } -func StartServiceEntrySpanOperation(ctx context.Context) (*ServiceEntrySpanOperation, context.Context) { +func StartServiceEntrySpanOperation(ctx context.Context, span TagSetter) (*ServiceEntrySpanOperation, context.Context) { parent, _ := dyngo.FromContext(ctx) op := &ServiceEntrySpanOperation{ Operation: dyngo.NewOperation(parent), - tags: make(map[string]any), - jsonTags: make(map[string]any), + jsonTags: make(map[string]any, 2), + tagSetter: span, } return op, dyngo.StartAndRegisterOperation(ctx, op, ServiceEntrySpanArgs{}) } -func (op *ServiceEntrySpanOperation) Finish(span TagSetter) { +func (op *ServiceEntrySpanOperation) Finish() { + span := op.tagSetter if _, ok := span.(*NoopTagSetter); ok { // If the span is a NoopTagSetter or is nil, we don't need to set any tags return } @@ -144,14 +145,11 @@ func (op *ServiceEntrySpanOperation) Finish(span TagSetter) { op.mu.Lock() defer op.mu.Unlock() - for k, v := range op.tags { - span.SetTag(k, v) - } - for k, v := range op.jsonTags { strValue, err := json.Marshal(v) if err != nil { log.Debug("appsec: failed to marshal tag %s: %v", k, err) + continue } span.SetTag(k, string(strValue)) } diff --git a/internal/appsec/emitter/waf/context.go b/internal/appsec/emitter/waf/context.go index 698e721880..e88e03b6b8 100644 --- a/internal/appsec/emitter/waf/context.go +++ b/internal/appsec/emitter/waf/context.go @@ -61,8 +61,8 @@ type ( func (ContextArgs) IsArgOf(*ContextOperation) {} func (ContextRes) IsResultOf(*ContextOperation) {} -func StartContextOperation(ctx context.Context) (*ContextOperation, context.Context) { - entrySpanOp, ctx := trace.StartServiceEntrySpanOperation(ctx) +func StartContextOperation(ctx context.Context, span trace.TagSetter) (*ContextOperation, context.Context) { + entrySpanOp, ctx := trace.StartServiceEntrySpanOperation(ctx, span) op := &ContextOperation{ Operation: dyngo.NewOperation(entrySpanOp), ServiceEntrySpanOperation: entrySpanOp, @@ -70,9 +70,9 @@ func StartContextOperation(ctx context.Context) (*ContextOperation, context.Cont return op, dyngo.StartAndRegisterOperation(ctx, op, ContextArgs{}) } -func (op *ContextOperation) Finish(span trace.TagSetter) { +func (op *ContextOperation) Finish() { dyngo.FinishOperation(op, ContextRes{}) - op.ServiceEntrySpanOperation.Finish(span) + op.ServiceEntrySpanOperation.Finish() } func (op *ContextOperation) SwapContext(ctx *waf.Context) *waf.Context { From 2dfcb2a5eb6d024e428159095260bf6125687f31 Mon Sep 17 00:00:00 2001 From: Eliott Bouhana <47679741+eliottness@users.noreply.github.com> Date: Thu, 19 Dec 2024 13:57:36 +0100 Subject: [PATCH 07/13] ddtrace/tracer: Tracing as transport-only mode (APPSEC_STANDALONE) (#3033) Signed-off-by: Eliott Bouhana --- .github/workflows/system-tests.yml | 2 + contrib/net/http/roundtripper.go | 1 - ddtrace/tracer/log.go | 2 + ddtrace/tracer/log_test.go | 10 ++--- ddtrace/tracer/option.go | 42 +++++++++++++++++++ ddtrace/tracer/rules_sampler.go | 38 ++++------------- ddtrace/tracer/sampler_test.go | 32 +++++++------- ddtrace/tracer/span.go | 6 +++ ddtrace/tracer/tracer.go | 34 ++++++++++++--- ddtrace/tracer/transport.go | 4 +- internal/appsec/emitter/waf/context.go | 3 ++ internal/appsec/emitter/waf/run.go | 2 +- internal/appsec/listener/grpcsec/grpc_test.go | 6 ++- .../appsec/listener/httpsec/request_test.go | 9 +++- internal/appsec/listener/waf/tags.go | 32 ++------------ internal/appsec/listener/waf/waf.go | 20 +++++---- ...{meta_struct.go => meta_internal_types.go} | 6 +++ 17 files changed, 153 insertions(+), 96 deletions(-) rename internal/{meta_struct.go => meta_internal_types.go} (69%) diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index f744ce1303..c8a0b287f7 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -79,6 +79,8 @@ jobs: scenario: APPSEC_CORRUPTED_RULES - weblog-variant: net-http scenario: APPSEC_LOW_WAF_TIMEOUT + - weblog-variant: net-http + scenario: APPSEC_STANDALONE - weblog-variant: net-http scenario: APPSEC_CUSTOM_OBFUSCATION # APM scenarios requiring specific environment settings diff --git a/contrib/net/http/roundtripper.go b/contrib/net/http/roundtripper.go index 7e47c530b6..b9ee812508 100644 --- a/contrib/net/http/roundtripper.go +++ b/contrib/net/http/roundtripper.go @@ -14,7 +14,6 @@ import ( "strings" "gopkg.in/DataDog/dd-trace-go.v1/appsec/events" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" diff --git a/ddtrace/tracer/log.go b/ddtrace/tracer/log.go index 5b6e056f28..f79dfe4df9 100644 --- a/ddtrace/tracer/log.go +++ b/ddtrace/tracer/log.go @@ -59,6 +59,7 @@ type startupInfo struct { FeatureFlags []string `json:"feature_flags"` PropagationStyleInject string `json:"propagation_style_inject"` // Propagation style for inject PropagationStyleExtract string `json:"propagation_style_extract"` // Propagation style for extract + TracingAsTransport bool `json:"tracing_as_transport"` // Whether the tracer is disabled and other products are using it as a transport } // checkEndpoint tries to connect to the URL specified by endpoint. @@ -147,6 +148,7 @@ func logStartup(t *tracer) { FeatureFlags: featureFlags, PropagationStyleInject: injectorNames, PropagationStyleExtract: extractorNames, + TracingAsTransport: t.config.tracingAsTransport, } if _, _, err := samplingRulesFromEnv(); err != nil { info.SamplingRulesError = fmt.Sprintf("%s", err) diff --git a/ddtrace/tracer/log_test.go b/ddtrace/tracer/log_test.go index f4467ca009..268e0ad015 100644 --- a/ddtrace/tracer/log_test.go +++ b/ddtrace/tracer/log_test.go @@ -33,7 +33,7 @@ func TestStartupLog(t *testing.T) { tp.Ignore("appsec: ", telemetry.LogPrefix) logStartup(tracer) require.Len(t, tp.Logs(), 2) - assert.Regexp(logPrefixRegexp+` INFO: DATADOG TRACER CONFIGURATION {"date":"[^"]*","os_name":"[^"]*","os_version":"[^"]*","version":"[^"]*","lang":"Go","lang_version":"[^"]*","env":"","service":"tracer\.test(\.exe)?","agent_url":"http://localhost:9/v0.4/traces","agent_error":"Post .*","debug":false,"analytics_enabled":false,"sample_rate":"NaN","sample_rate_limit":"disabled","trace_sampling_rules":null,"span_sampling_rules":null,"sampling_rules_error":"","service_mappings":null,"tags":{"runtime-id":"[^"]*"},"runtime_metrics_enabled":false,"runtime_metrics_v2_enabled":false,"profiler_code_hotspots_enabled":((false)|(true)),"profiler_endpoints_enabled":((false)|(true)),"dd_version":"","architecture":"[^"]*","global_service":"","lambda_mode":"false","appsec":((true)|(false)),"agent_features":{"DropP0s":((true)|(false)),"Stats":((true)|(false)),"StatsdPort":(0|8125)},"integrations":{.*},"partial_flush_enabled":false,"partial_flush_min_spans":1000,"orchestrion":{"enabled":false},"feature_flags":\[\],"propagation_style_inject":"datadog,tracecontext","propagation_style_extract":"datadog,tracecontext"}`, tp.Logs()[1]) + assert.Regexp(logPrefixRegexp+` INFO: DATADOG TRACER CONFIGURATION {"date":"[^"]*","os_name":"[^"]*","os_version":"[^"]*","version":"[^"]*","lang":"Go","lang_version":"[^"]*","env":"","service":"tracer\.test(\.exe)?","agent_url":"http://localhost:9/v0.4/traces","agent_error":"Post .*","debug":false,"analytics_enabled":false,"sample_rate":"NaN","sample_rate_limit":"disabled","trace_sampling_rules":null,"span_sampling_rules":null,"sampling_rules_error":"","service_mappings":null,"tags":{"runtime-id":"[^"]*"},"runtime_metrics_enabled":false,"runtime_metrics_v2_enabled":false,"profiler_code_hotspots_enabled":((false)|(true)),"profiler_endpoints_enabled":((false)|(true)),"dd_version":"","architecture":"[^"]*","global_service":"","lambda_mode":"false","appsec":((true)|(false)),"agent_features":{"DropP0s":((true)|(false)),"Stats":((true)|(false)),"StatsdPort":(0|8125)},"integrations":{.*},"partial_flush_enabled":false,"partial_flush_min_spans":1000,"orchestrion":{"enabled":false},"feature_flags":\[\],"propagation_style_inject":"datadog,tracecontext","propagation_style_extract":"datadog,tracecontext","tracing_as_transport":false}`, tp.Logs()[1]) }) t.Run("configured", func(t *testing.T) { @@ -65,7 +65,7 @@ func TestStartupLog(t *testing.T) { tp.Ignore("appsec: ", telemetry.LogPrefix) logStartup(tracer) require.Len(t, tp.Logs(), 2) - assert.Regexp(logPrefixRegexp+` INFO: DATADOG TRACER CONFIGURATION {"date":"[^"]*","os_name":"[^"]*","os_version":"[^"]*","version":"[^"]*","lang":"Go","lang_version":"[^"]*","env":"configuredEnv","service":"configured.service","agent_url":"http://localhost:9/v0.4/traces","agent_error":"Post .*","debug":true,"analytics_enabled":true,"sample_rate":"0\.123000","sample_rate_limit":"100","trace_sampling_rules":\[{"service":"mysql","sample_rate":0\.75}\],"span_sampling_rules":null,"sampling_rules_error":"","service_mappings":{"initial_service":"new_service"},"tags":{"runtime-id":"[^"]*","tag":"value","tag2":"NaN"},"runtime_metrics_enabled":true,"runtime_metrics_v2_enabled":false,"profiler_code_hotspots_enabled":((false)|(true)),"profiler_endpoints_enabled":((false)|(true)),"dd_version":"2.3.4","architecture":"[^"]*","global_service":"configured.service","lambda_mode":"false","appsec":((true)|(false)),"agent_features":{"DropP0s":false,"Stats":false,"StatsdPort":(0|8125)},"integrations":{.*},"partial_flush_enabled":false,"partial_flush_min_spans":1000,"orchestrion":{"enabled":true,"metadata":{"version":"v1"}},"feature_flags":\["discovery"\],"propagation_style_inject":"datadog,tracecontext","propagation_style_extract":"datadog,tracecontext"}`, tp.Logs()[1]) + assert.Regexp(logPrefixRegexp+` INFO: DATADOG TRACER CONFIGURATION {"date":"[^"]*","os_name":"[^"]*","os_version":"[^"]*","version":"[^"]*","lang":"Go","lang_version":"[^"]*","env":"configuredEnv","service":"configured.service","agent_url":"http://localhost:9/v0.4/traces","agent_error":"Post .*","debug":true,"analytics_enabled":true,"sample_rate":"0\.123000","sample_rate_limit":"100","trace_sampling_rules":\[{"service":"mysql","sample_rate":0\.75}\],"span_sampling_rules":null,"sampling_rules_error":"","service_mappings":{"initial_service":"new_service"},"tags":{"runtime-id":"[^"]*","tag":"value","tag2":"NaN"},"runtime_metrics_enabled":true,"runtime_metrics_v2_enabled":false,"profiler_code_hotspots_enabled":((false)|(true)),"profiler_endpoints_enabled":((false)|(true)),"dd_version":"2.3.4","architecture":"[^"]*","global_service":"configured.service","lambda_mode":"false","appsec":((true)|(false)),"agent_features":{"DropP0s":false,"Stats":false,"StatsdPort":(0|8125)},"integrations":{.*},"partial_flush_enabled":false,"partial_flush_min_spans":1000,"orchestrion":{"enabled":true,"metadata":{"version":"v1"}},"feature_flags":\["discovery"\],"propagation_style_inject":"datadog,tracecontext","propagation_style_extract":"datadog,tracecontext","tracing_as_transport":false}`, tp.Logs()[1]) }) t.Run("limit", func(t *testing.T) { @@ -95,7 +95,7 @@ func TestStartupLog(t *testing.T) { tp.Ignore("appsec: ", telemetry.LogPrefix) logStartup(tracer) require.Len(t, tp.Logs(), 2) - assert.Regexp(logPrefixRegexp+` INFO: DATADOG TRACER CONFIGURATION {"date":"[^"]*","os_name":"[^"]*","os_version":"[^"]*","version":"[^"]*","lang":"Go","lang_version":"[^"]*","env":"configuredEnv","service":"configured.service","agent_url":"http://localhost:9/v0.4/traces","agent_error":"Post .*","debug":true,"analytics_enabled":true,"sample_rate":"0\.123000","sample_rate_limit":"1000.001","trace_sampling_rules":\[{"service":"mysql","sample_rate":0\.75}\],"span_sampling_rules":null,"sampling_rules_error":"","service_mappings":{"initial_service":"new_service"},"tags":{"runtime-id":"[^"]*","tag":"value","tag2":"NaN"},"runtime_metrics_enabled":true,"runtime_metrics_v2_enabled":false,"profiler_code_hotspots_enabled":((false)|(true)),"profiler_endpoints_enabled":((false)|(true)),"dd_version":"2.3.4","architecture":"[^"]*","global_service":"configured.service","lambda_mode":"false","appsec":((true)|(false)),"agent_features":{"DropP0s":false,"Stats":false,"StatsdPort":(0|8125)},"integrations":{.*},"partial_flush_enabled":false,"partial_flush_min_spans":1000,"orchestrion":{"enabled":false},"feature_flags":\[\],"propagation_style_inject":"datadog,tracecontext","propagation_style_extract":"datadog,tracecontext"}`, tp.Logs()[1]) + assert.Regexp(logPrefixRegexp+` INFO: DATADOG TRACER CONFIGURATION {"date":"[^"]*","os_name":"[^"]*","os_version":"[^"]*","version":"[^"]*","lang":"Go","lang_version":"[^"]*","env":"configuredEnv","service":"configured.service","agent_url":"http://localhost:9/v0.4/traces","agent_error":"Post .*","debug":true,"analytics_enabled":true,"sample_rate":"0\.123000","sample_rate_limit":"1000.001","trace_sampling_rules":\[{"service":"mysql","sample_rate":0\.75}\],"span_sampling_rules":null,"sampling_rules_error":"","service_mappings":{"initial_service":"new_service"},"tags":{"runtime-id":"[^"]*","tag":"value","tag2":"NaN"},"runtime_metrics_enabled":true,"runtime_metrics_v2_enabled":false,"profiler_code_hotspots_enabled":((false)|(true)),"profiler_endpoints_enabled":((false)|(true)),"dd_version":"2.3.4","architecture":"[^"]*","global_service":"configured.service","lambda_mode":"false","appsec":((true)|(false)),"agent_features":{"DropP0s":false,"Stats":false,"StatsdPort":(0|8125)},"integrations":{.*},"partial_flush_enabled":false,"partial_flush_min_spans":1000,"orchestrion":{"enabled":false},"feature_flags":\[\],"propagation_style_inject":"datadog,tracecontext","propagation_style_extract":"datadog,tracecontext","tracing_as_transport":false}`, tp.Logs()[1]) }) t.Run("errors", func(t *testing.T) { @@ -110,7 +110,7 @@ func TestStartupLog(t *testing.T) { logStartup(tracer) require.Len(t, tp.Logs(), 2) fmt.Println(tp.Logs()[1]) - assert.Regexp(logPrefixRegexp+` INFO: DATADOG TRACER CONFIGURATION {"date":"[^"]*","os_name":"[^"]*","os_version":"[^"]*","version":"[^"]*","lang":"Go","lang_version":"[^"]*","env":"","service":"tracer\.test(\.exe)?","agent_url":"http://localhost:9/v0.4/traces","agent_error":"Post .*","debug":false,"analytics_enabled":false,"sample_rate":"NaN","sample_rate_limit":"100","trace_sampling_rules":\[{"service":"some\.service","sample_rate":0\.234}\],"span_sampling_rules":null,"sampling_rules_error":"\\n\\tat index 1: ignoring rule {Service:other.service Rate:2}: rate is out of \[0\.0, 1\.0] range","service_mappings":null,"tags":{"runtime-id":"[^"]*"},"runtime_metrics_enabled":false,"runtime_metrics_v2_enabled":false,"profiler_code_hotspots_enabled":((false)|(true)),"profiler_endpoints_enabled":((false)|(true)),"dd_version":"","architecture":"[^"]*","global_service":"","lambda_mode":"false","appsec":((true)|(false)),"agent_features":{"DropP0s":((true)|(false)),"Stats":((true)|(false)),"StatsdPort":(0|8125)},"integrations":{.*},"partial_flush_enabled":false,"partial_flush_min_spans":1000,"orchestrion":{"enabled":false},"feature_flags":\[\],"propagation_style_inject":"datadog,tracecontext","propagation_style_extract":"datadog,tracecontext"}`, tp.Logs()[1]) + assert.Regexp(logPrefixRegexp+` INFO: DATADOG TRACER CONFIGURATION {"date":"[^"]*","os_name":"[^"]*","os_version":"[^"]*","version":"[^"]*","lang":"Go","lang_version":"[^"]*","env":"","service":"tracer\.test(\.exe)?","agent_url":"http://localhost:9/v0.4/traces","agent_error":"Post .*","debug":false,"analytics_enabled":false,"sample_rate":"NaN","sample_rate_limit":"100","trace_sampling_rules":\[{"service":"some\.service","sample_rate":0\.234}\],"span_sampling_rules":null,"sampling_rules_error":"\\n\\tat index 1: ignoring rule {Service:other.service Rate:2}: rate is out of \[0\.0, 1\.0] range","service_mappings":null,"tags":{"runtime-id":"[^"]*"},"runtime_metrics_enabled":false,"runtime_metrics_v2_enabled":false,"profiler_code_hotspots_enabled":((false)|(true)),"profiler_endpoints_enabled":((false)|(true)),"dd_version":"","architecture":"[^"]*","global_service":"","lambda_mode":"false","appsec":((true)|(false)),"agent_features":{"DropP0s":((true)|(false)),"Stats":((true)|(false)),"StatsdPort":(0|8125)},"integrations":{.*},"partial_flush_enabled":false,"partial_flush_min_spans":1000,"orchestrion":{"enabled":false},"feature_flags":\[\],"propagation_style_inject":"datadog,tracecontext","propagation_style_extract":"datadog,tracecontext","tracing_as_transport":false}`, tp.Logs()[1]) }) t.Run("lambda", func(t *testing.T) { @@ -123,7 +123,7 @@ func TestStartupLog(t *testing.T) { tp.Ignore("appsec: ", telemetry.LogPrefix) logStartup(tracer) assert.Len(tp.Logs(), 1) - assert.Regexp(logPrefixRegexp+` INFO: DATADOG TRACER CONFIGURATION {"date":"[^"]*","os_name":"[^"]*","os_version":"[^"]*","version":"[^"]*","lang":"Go","lang_version":"[^"]*","env":"","service":"tracer\.test(\.exe)?","agent_url":"http://localhost:9/v0.4/traces","agent_error":"","debug":false,"analytics_enabled":false,"sample_rate":"NaN","sample_rate_limit":"disabled","trace_sampling_rules":null,"span_sampling_rules":null,"sampling_rules_error":"","service_mappings":null,"tags":{"runtime-id":"[^"]*"},"runtime_metrics_enabled":false,"runtime_metrics_v2_enabled":false,"profiler_code_hotspots_enabled":((false)|(true)),"profiler_endpoints_enabled":((false)|(true)),"dd_version":"","architecture":"[^"]*","global_service":"","lambda_mode":"true","appsec":((true)|(false)),"agent_features":{"DropP0s":false,"Stats":false,"StatsdPort":(0|8125)},"integrations":{.*},"partial_flush_enabled":false,"partial_flush_min_spans":1000,"orchestrion":{"enabled":false},"feature_flags":\[\],"propagation_style_inject":"datadog,tracecontext","propagation_style_extract":"datadog,tracecontext"}`, tp.Logs()[0]) + assert.Regexp(logPrefixRegexp+` INFO: DATADOG TRACER CONFIGURATION {"date":"[^"]*","os_name":"[^"]*","os_version":"[^"]*","version":"[^"]*","lang":"Go","lang_version":"[^"]*","env":"","service":"tracer\.test(\.exe)?","agent_url":"http://localhost:9/v0.4/traces","agent_error":"","debug":false,"analytics_enabled":false,"sample_rate":"NaN","sample_rate_limit":"disabled","trace_sampling_rules":null,"span_sampling_rules":null,"sampling_rules_error":"","service_mappings":null,"tags":{"runtime-id":"[^"]*"},"runtime_metrics_enabled":false,"runtime_metrics_v2_enabled":false,"profiler_code_hotspots_enabled":((false)|(true)),"profiler_endpoints_enabled":((false)|(true)),"dd_version":"","architecture":"[^"]*","global_service":"","lambda_mode":"true","appsec":((true)|(false)),"agent_features":{"DropP0s":false,"Stats":false,"StatsdPort":(0|8125)},"integrations":{.*},"partial_flush_enabled":false,"partial_flush_min_spans":1000,"orchestrion":{"enabled":false},"feature_flags":\[\],"propagation_style_inject":"datadog,tracecontext","propagation_style_extract":"datadog,tracecontext","tracing_as_transport":false}`, tp.Logs()[0]) }) t.Run("integrations", func(t *testing.T) { diff --git a/ddtrace/tracer/option.go b/ddtrace/tracer/option.go index fd42e3c4e4..c1ad01edf9 100644 --- a/ddtrace/tracer/option.go +++ b/ddtrace/tracer/option.go @@ -111,6 +111,9 @@ var ( // defaultMaxTagsHeaderLen specifies the default maximum length of the X-Datadog-Tags header value. defaultMaxTagsHeaderLen = 128 + + // defaultRateLimit specifies the default trace rate limit used when DD_TRACE_RATE_LIMIT is not set. + defaultRateLimit = 100.0 ) // config holds the tracer configuration. @@ -298,6 +301,12 @@ type config struct { // logDirectory is directory for tracer logs specified by user-setting DD_TRACE_LOG_DIRECTORY. default empty/unused logDirectory string + + // tracingAsTransport specifies whether the tracer is running in transport-only mode, where traces are only sent when other products request it. + tracingAsTransport bool + + // traceRateLimitPerSecond specifies the rate limit for traces. + traceRateLimitPerSecond float64 } // orchestrionConfig contains Orchestrion configuration. @@ -344,6 +353,22 @@ func newConfig(opts ...StartOption) *config { c.globalSampleRate = sampleRate c.httpClientTimeout = time.Second * 10 // 10 seconds + c.traceRateLimitPerSecond = defaultRateLimit + origin := telemetry.OriginDefault + if v, ok := os.LookupEnv("DD_TRACE_RATE_LIMIT"); ok { + l, err := strconv.ParseFloat(v, 64) + if err != nil { + log.Warn("DD_TRACE_RATE_LIMIT invalid, using default value %f: %v", defaultRateLimit, err) + } else if l < 0.0 { + log.Warn("DD_TRACE_RATE_LIMIT negative, using default value %f", defaultRateLimit) + } else { + c.traceRateLimitPerSecond = l + origin = telemetry.OriginEnvVar + } + } + + reportTelemetryOnAppStarted(telemetry.Configuration{Name: "trace_rate_limit", Value: c.traceRateLimitPerSecond, Origin: origin}) + if v := os.Getenv("OTEL_LOGS_EXPORTER"); v != "" { log.Warn("OTEL_LOGS_EXPORTER is not supported") } @@ -562,6 +587,23 @@ func newConfig(opts ...StartOption) *config { // This allows persisting the initial value of globalTags for future resets and updates. globalTagsOrigin := c.globalTags.cfgOrigin c.initGlobalTags(c.globalTags.get(), globalTagsOrigin) + + // TODO: change the name once APM Platform RFC is approved + if internal.BoolEnv("DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED", false) { + // Enable tracing as transport layer mode + // This means to stop sending trace metrics, send one trace per minute and those force-kept by other products + // using the tracer as transport layer for their data. And finally adding the _dd.apm.enabled=0 tag to all traces + // to let the backend know that it needs to keep APM UI disabled. + c.globalSampleRate = 1.0 + c.traceRateLimitPerSecond = 1.0 / 60 + c.tracingAsTransport = true + WithGlobalTag("_dd.apm.enabled", 0)(c) + // Disable runtime metrics. In `tracingAsTransport` mode, we'll still + // tell the agent we computed them, so it doesn't do it either. + c.runtimeMetrics = false + c.runtimeMetricsV2 = false + } + return c } diff --git a/ddtrace/tracer/rules_sampler.go b/ddtrace/tracer/rules_sampler.go index 037e393642..0edfa98b99 100644 --- a/ddtrace/tracer/rules_sampler.go +++ b/ddtrace/tracer/rules_sampler.go @@ -16,12 +16,11 @@ import ( "sync" "time" + "golang.org/x/time/rate" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" "gopkg.in/DataDog/dd-trace-go.v1/internal/log" "gopkg.in/DataDog/dd-trace-go.v1/internal/samplernames" - "gopkg.in/DataDog/dd-trace-go.v1/internal/telemetry" - - "golang.org/x/time/rate" ) // rulesSampler holds instances of trace sampler and single span sampler, that are configured with the given set of rules. @@ -38,9 +37,9 @@ type rulesSampler struct { // Rules are split between trace and single span sampling rules according to their type. // Such rules are user-defined through environment variable or WithSamplingRules option. // Invalid rules or environment variable values are tolerated, by logging warnings and then ignoring them. -func newRulesSampler(traceRules, spanRules []SamplingRule, traceSampleRate float64) *rulesSampler { +func newRulesSampler(traceRules, spanRules []SamplingRule, traceSampleRate, rateLimitPerSecond float64) *rulesSampler { return &rulesSampler{ - traces: newTraceRulesSampler(traceRules, traceSampleRate), + traces: newTraceRulesSampler(traceRules, traceSampleRate, rateLimitPerSecond), spans: newSingleSpanRulesSampler(spanRules), } } @@ -373,11 +372,11 @@ type traceRulesSampler struct { // newTraceRulesSampler configures a *traceRulesSampler instance using the given set of rules. // Invalid rules or environment variable values are tolerated, by logging warnings and then ignoring them. -func newTraceRulesSampler(rules []SamplingRule, traceSampleRate float64) *traceRulesSampler { +func newTraceRulesSampler(rules []SamplingRule, traceSampleRate, rateLimitPerSecond float64) *traceRulesSampler { return &traceRulesSampler{ rules: rules, globalRate: traceSampleRate, - limiter: newRateLimiter(), + limiter: newRateLimiter(rateLimitPerSecond), } } @@ -387,7 +386,7 @@ func (rs *traceRulesSampler) enabled() bool { return len(rs.rules) > 0 || !math.IsNaN(rs.globalRate) } -// Tests whether two sets of the rules are the same. +// EqualsFalseNegative tests whether two sets of the rules are the same. // This returns result that can be false negative. If the result is true, then the two sets of rules // are guaranteed to be the same. // On the other hand, false can be returned while the two rulesets are logically the same. @@ -527,30 +526,11 @@ func (rs *traceRulesSampler) limit() (float64, bool) { return math.NaN(), false } -// defaultRateLimit specifies the default trace rate limit used when DD_TRACE_RATE_LIMIT is not set. -const defaultRateLimit = 100.0 - // newRateLimiter returns a rate limiter which restricts the number of traces sampled per second. // The limit is DD_TRACE_RATE_LIMIT if set, `defaultRateLimit` otherwise. -func newRateLimiter() *rateLimiter { - limit := defaultRateLimit - origin := telemetry.OriginDefault - v := os.Getenv("DD_TRACE_RATE_LIMIT") - if v != "" { - l, err := strconv.ParseFloat(v, 64) - if err != nil { - log.Warn("DD_TRACE_RATE_LIMIT invalid, using default value %f: %v", limit, err) - } else if l < 0.0 { - log.Warn("DD_TRACE_RATE_LIMIT negative, using default value %f", limit) - } else { - // override the default limit - origin = telemetry.OriginEnvVar - limit = l - } - } - reportTelemetryOnAppStarted(telemetry.Configuration{Name: "trace_rate_limit", Value: limit, Origin: origin}) +func newRateLimiter(ratePerSecond float64) *rateLimiter { return &rateLimiter{ - limiter: rate.NewLimiter(rate.Limit(limit), int(math.Ceil(limit))), + limiter: rate.NewLimiter(rate.Limit(ratePerSecond), int(math.Ceil(ratePerSecond))), prevTime: time.Now(), } } diff --git a/ddtrace/tracer/sampler_test.go b/ddtrace/tracer/sampler_test.go index 117518b533..22678482c9 100644 --- a/ddtrace/tracer/sampler_test.go +++ b/ddtrace/tracer/sampler_test.go @@ -246,7 +246,7 @@ func TestRuleEnvVars(t *testing.T) { {in: "1point0", out: rate.NewLimiter(100.0, 100)}, // default if invalid value } { t.Setenv("DD_TRACE_RATE_LIMIT", tt.in) - res := newRateLimiter() + res := newRateLimiter(newConfig().traceRateLimitPerSecond) assert.Equal(tt.out, res.limiter) } }) @@ -477,7 +477,7 @@ func TestRulesSampler(t *testing.T) { } t.Run("no-rules", func(t *testing.T) { assert := assert.New(t) - rs := newRulesSampler(nil, nil, newConfig().globalSampleRate) + rs := newRulesSampler(nil, nil, newConfig().globalSampleRate, newConfig().traceRateLimitPerSecond) span := makeSpan("http.request", "test-service") result := rs.SampleTrace(span) @@ -542,7 +542,7 @@ func TestRulesSampler(t *testing.T) { assert.Nil(t, err) assert := assert.New(t) - rs := newRulesSampler(rules, nil, newConfig().globalSampleRate) + rs := newRulesSampler(rules, nil, newConfig().globalSampleRate, newConfig().traceRateLimitPerSecond) span := makeFinishedSpan(tt.spanName, tt.spanSrv, tt.spanRsc, tt.spanTags) @@ -566,7 +566,7 @@ func TestRulesSampler(t *testing.T) { for _, v := range traceRules { t.Run("", func(t *testing.T) { assert := assert.New(t) - rs := newRulesSampler(v, nil, newConfig().globalSampleRate) + rs := newRulesSampler(v, nil, newConfig().globalSampleRate, newConfig().traceRateLimitPerSecond) span := makeSpan("http.request", "test-service") result := rs.SampleTrace(span) @@ -592,7 +592,7 @@ func TestRulesSampler(t *testing.T) { for _, v := range traceRules { t.Run("", func(t *testing.T) { assert := assert.New(t) - rs := newRulesSampler(v, nil, newConfig().globalSampleRate) + rs := newRulesSampler(v, nil, newConfig().globalSampleRate, newConfig().traceRateLimitPerSecond) span := makeSpan("http.request", "test-service") result := rs.SampleTrace(span) @@ -638,7 +638,7 @@ func TestRulesSampler(t *testing.T) { _, rules, err := samplingRulesFromEnv() assert.Nil(t, err) assert := assert.New(t) - rs := newRulesSampler(nil, rules, newConfig().globalSampleRate) + rs := newRulesSampler(nil, rules, newConfig().globalSampleRate, newConfig().traceRateLimitPerSecond) span := makeFinishedSpan(tt.spanName, tt.spanSrv, "res-10", map[string]interface{}{"hostname": "hn-30"}) @@ -761,7 +761,7 @@ func TestRulesSampler(t *testing.T) { t.Run(fmt.Sprintf("%v", i), func(t *testing.T) { assert := assert.New(t) c := newConfig(WithSamplingRules(tt.rules)) - rs := newRulesSampler(nil, c.spanRules, newConfig().globalSampleRate) + rs := newRulesSampler(nil, c.spanRules, newConfig().globalSampleRate, newConfig().traceRateLimitPerSecond) span := makeFinishedSpan(tt.spanName, tt.spanSrv, "res-10", map[string]interface{}{"hostname": "hn-30", "tag": 20.1, @@ -830,7 +830,7 @@ func TestRulesSampler(t *testing.T) { assert := assert.New(t) sampleRate := newConfig().globalSampleRate - rs := newRulesSampler(nil, rules, sampleRate) + rs := newRulesSampler(nil, rules, sampleRate, newConfig().globalSampleRate) span := makeFinishedSpan(tt.spanName, tt.spanSrv, tt.resName, map[string]interface{}{"hostname": "hn-30"}) result := rs.SampleSpan(span) @@ -940,7 +940,7 @@ func TestRulesSampler(t *testing.T) { t.Run("", func(t *testing.T) { assert := assert.New(t) c := newConfig(WithSamplingRules(tt.rules)) - rs := newRulesSampler(nil, c.spanRules, newConfig().globalSampleRate) + rs := newRulesSampler(nil, c.spanRules, newConfig().globalSampleRate, newConfig().traceRateLimitPerSecond) span := makeFinishedSpan(tt.spanName, tt.spanSrv, "res-10", map[string]interface{}{"hostname": "hn-30", "tag": 20.1, @@ -969,7 +969,7 @@ func TestRulesSampler(t *testing.T) { t.Run("", func(t *testing.T) { assert := assert.New(t) t.Setenv("DD_TRACE_SAMPLE_RATE", fmt.Sprint(rate)) - rs := newRulesSampler(nil, rules, newConfig().globalSampleRate) + rs := newRulesSampler(nil, rules, newConfig().globalSampleRate, newConfig().traceRateLimitPerSecond) span := makeSpan("http.request", "test-service") result := rs.SampleTrace(span) @@ -1250,7 +1250,7 @@ func TestRulesSamplerInternals(t *testing.T) { t.Run("full-rate", func(t *testing.T) { assert := assert.New(t) now := time.Now() - rs := newRulesSampler(nil, nil, newConfig().globalSampleRate) + rs := newRulesSampler(nil, nil, newConfig().globalSampleRate, newConfig().traceRateLimitPerSecond) // set samplingLimiter to specific state rs.traces.limiter.prevTime = now.Add(-1 * time.Second) rs.traces.limiter.allowed = 1 @@ -1265,7 +1265,7 @@ func TestRulesSamplerInternals(t *testing.T) { t.Run("limited-rate", func(t *testing.T) { assert := assert.New(t) now := time.Now() - rs := newRulesSampler(nil, nil, newConfig().globalSampleRate) + rs := newRulesSampler(nil, nil, newConfig().globalSampleRate, newConfig().traceRateLimitPerSecond) // force sampling limiter to 1.0 spans/sec rs.traces.limiter.limiter = rate.NewLimiter(rate.Limit(1.0), 1) rs.traces.limiter.prevTime = now.Add(-1 * time.Second) @@ -1288,7 +1288,7 @@ func TestRulesSamplerInternals(t *testing.T) { func TestSamplingLimiter(t *testing.T) { t.Run("resets-every-second", func(t *testing.T) { assert := assert.New(t) - sl := newRateLimiter() + sl := newRateLimiter(defaultRateLimit) sl.prevSeen = 100 sl.prevAllowed = 99 sl.allowed = 42 @@ -1307,7 +1307,7 @@ func TestSamplingLimiter(t *testing.T) { t.Run("averages-rates", func(t *testing.T) { assert := assert.New(t) - sl := newRateLimiter() + sl := newRateLimiter(defaultRateLimit) sl.prevSeen = 100 sl.prevAllowed = 42 sl.allowed = 41 @@ -1325,7 +1325,7 @@ func TestSamplingLimiter(t *testing.T) { t.Run("discards-rate", func(t *testing.T) { assert := assert.New(t) - sl := newRateLimiter() + sl := newRateLimiter(defaultRateLimit) sl.prevSeen = 100 sl.prevAllowed = 42 sl.allowed = 42 @@ -1614,7 +1614,7 @@ func BenchmarkGlobMatchSpan(b *testing.B) { } func TestSetGlobalSampleRate(t *testing.T) { - rs := newTraceRulesSampler(nil, math.NaN()) + rs := newTraceRulesSampler(nil, math.NaN(), defaultRateLimit) assert.True(t, math.IsNaN(rs.globalRate)) // Comparing NaN values diff --git a/ddtrace/tracer/span.go b/ddtrace/tracer/span.go index 27437ff5c4..19da3779e3 100644 --- a/ddtrace/tracer/span.go +++ b/ddtrace/tracer/span.go @@ -193,6 +193,12 @@ func (s *span) SetTag(key string, value interface{}) { s.setMetaStruct(key, v.Value) return } + + // Add this tag to propagating tags and to span tags + // reserved for internal use only + if v, ok := value.(sharedinternal.PropagatingTagValue); ok { + s.context.trace.setPropagatingTag(key, v.Value) + } } // not numeric, not a string, not a fmt.Stringer, not a bool, and not an error diff --git a/ddtrace/tracer/tracer.go b/ddtrace/tracer/tracer.go index e6bfcc7b49..a345fd6366 100644 --- a/ddtrace/tracer/tracer.go +++ b/ddtrace/tracer/tracer.go @@ -159,9 +159,6 @@ func Start(opts ...StartOption) { return } internal.SetGlobalTracer(t) - if t.config.logStartup { - logStartup(t) - } if t.dataStreams != nil { t.dataStreams.Start() } @@ -200,6 +197,11 @@ func Start(opts ...StartOption) { appsecopts = append(appsecopts, t.config.appsecStartOptions...) appsecopts = append(appsecopts, appsecConfig.WithRCConfig(cfg)) appsec.Start(appsecopts...) + + if t.config.logStartup { + logStartup(t) + } + _ = t.hostname() // Prime the hostname cache } @@ -280,7 +282,8 @@ func newUnstartedTracer(opts ...StartOption) *tracer { if spans != nil { c.spanRules = spans } - rulesSampler := newRulesSampler(c.traceRules, c.spanRules, c.globalSampleRate) + + rulesSampler := newRulesSampler(c.traceRules, c.spanRules, c.globalSampleRate, c.traceRateLimitPerSecond) c.traceSampleRate = newDynamicConfig("trace_sample_rate", c.globalSampleRate, rulesSampler.traces.setGlobalSampleRate, equal[float64]) // If globalSampleRate returns NaN, it means the environment variable was not set or valid. // We could always set the origin to "env_var" inconditionally, but then it wouldn't be possible @@ -423,7 +426,9 @@ func (t *tracer) worker(tick <-chan time.Time) { t.statsd.Incr("datadog.tracer.flush_triggered", []string{"reason:invoked"}, 1) t.traceWriter.flush() t.statsd.Flush() - t.stats.flushAndSend(time.Now(), withCurrentBucket) + if !t.config.tracingAsTransport { + t.stats.flushAndSend(time.Now(), withCurrentBucket) + } // TODO(x): In reality, the traceWriter.flush() call is not synchronous // when using the agent traceWriter. However, this functionality is used // in Lambda so for that purpose this mechanism should suffice. @@ -729,6 +734,15 @@ func (t *tracer) Inject(ctx ddtrace.SpanContext, carrier interface{}) error { if !t.config.enabled.current { return nil } + + if t.config.tracingAsTransport { + // in tracing as transport mode, only propagate when there is an upstream appsec event + // TODO: replace with _dd.p.ts in the next iteration standardizing this for other products, comparing enabled products in `t.config` with their corresponding `_dd.p.ts` bitfields + if ctx, ok := ctx.(*spanContext); ok && ctx.trace != nil && ctx.trace.propagatingTag("_dd.p.appsec") != "1" { + return nil + } + } + t.updateSampling(ctx) return t.config.propagator.Inject(ctx, carrier) } @@ -768,7 +782,15 @@ func (t *tracer) Extract(carrier interface{}) (ddtrace.SpanContext, error) { if !t.config.enabled.current { return internal.NoopSpanContext{}, nil } - return t.config.propagator.Extract(carrier) + ctx, err := t.config.propagator.Extract(carrier) + if t.config.tracingAsTransport { + // in tracing as transport mode, reset upstream sampling decision to make sure we keep 1 trace/minute + // TODO: replace with _dd.p.ts in the next iteration standardizing this for other products, comparing enabled products in `t.config` with their corresponding `_dd.p.ts` bitfields + if ctx, ok := ctx.(*spanContext); ok && ctx.trace.propagatingTag("_dd.p.appsec") != "1" { + ctx.trace.priority = nil + } + } + return ctx, err } // sampleRateMetricKey is the metric key holding the applied sample rate. Has to be the same as the Agent. diff --git a/ddtrace/tracer/transport.go b/ddtrace/tracer/transport.go index 53157c834f..a8ad9cfe8e 100644 --- a/ddtrace/tracer/transport.go +++ b/ddtrace/tracer/transport.go @@ -152,7 +152,9 @@ func (t *httpTransport) send(p *payload) (body io.ReadCloser, err error) { req.Header.Set(traceCountHeader, strconv.Itoa(p.itemCount())) req.Header.Set(headerComputedTopLevel, "yes") if t, ok := traceinternal.GetGlobalTracer().(*tracer); ok { - if t.config.canComputeStats() { + if t.config.tracingAsTransport || t.config.canComputeStats() { + // tracingAsTransport uses this header to disable the trace agent's stats computation + // while making canComputeStats() always false to also disable client stats computation. req.Header.Set("Datadog-Client-Computed-Stats", "yes") } droppedTraces := int(atomic.SwapUint32(&t.droppedP0Traces, 0)) diff --git a/internal/appsec/emitter/waf/context.go b/internal/appsec/emitter/waf/context.go index e88e03b6b8..7e53e7d3a8 100644 --- a/internal/appsec/emitter/waf/context.go +++ b/internal/appsec/emitter/waf/context.go @@ -56,6 +56,9 @@ type ( waf.RunAddressData dyngo.Operation } + + // SecurityEvent is a dyngo data event sent when a security event is detected by the WAF + SecurityEvent struct{} ) func (ContextArgs) IsArgOf(*ContextOperation) {} diff --git a/internal/appsec/emitter/waf/run.go b/internal/appsec/emitter/waf/run.go index a77abd5b20..19abbf4fba 100644 --- a/internal/appsec/emitter/waf/run.go +++ b/internal/appsec/emitter/waf/run.go @@ -53,7 +53,7 @@ func (op *ContextOperation) Run(eventReceiver dyngo.Operation, addrs waf.RunAddr actions.SendActionEvents(eventReceiver, result.Actions) if result.HasEvents() { - log.Debug("appsec: WAF detected a suspicious event") + dyngo.EmitData(op, &SecurityEvent{}) } } diff --git a/internal/appsec/listener/grpcsec/grpc_test.go b/internal/appsec/listener/grpcsec/grpc_test.go index bfb8520b5c..b8ac736fc4 100644 --- a/internal/appsec/listener/grpcsec/grpc_test.go +++ b/internal/appsec/listener/grpcsec/grpc_test.go @@ -6,6 +6,7 @@ package grpcsec import ( + "encoding/json" "fmt" "testing" @@ -94,11 +95,14 @@ func TestTags(t *testing.T) { metadataCase := metadataCase t.Run(fmt.Sprintf("%s-%s", eventCase.name, metadataCase.name), func(t *testing.T) { var span MockSpan - err := waf.SetEventSpanTags(&span, eventCase.events) + waf.SetEventSpanTags(&span) + value, err := json.Marshal(map[string][]any{"triggers": eventCase.events}) if eventCase.expectedError { require.Error(t, err) return } + + span.SetTag("_dd.appsec.json", string(value)) require.NoError(t, err) SetRequestMetadataTags(&span, metadataCase.md) diff --git a/internal/appsec/listener/httpsec/request_test.go b/internal/appsec/listener/httpsec/request_test.go index 38052cbb96..08d27529cd 100644 --- a/internal/appsec/listener/httpsec/request_test.go +++ b/internal/appsec/listener/httpsec/request_test.go @@ -6,6 +6,7 @@ package httpsec import ( + "encoding/json" "fmt" "net" "net/netip" @@ -13,6 +14,7 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/grpc/metadata" + "gopkg.in/DataDog/dd-trace-go.v1/internal" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/listener/waf" @@ -209,11 +211,15 @@ func TestTags(t *testing.T) { respHeadersCase := respHeadersCase t.Run(fmt.Sprintf("%s-%s-%s", eventCase.name, reqHeadersCase.name, respHeadersCase.name), func(t *testing.T) { var span MockSpan - err := waf.SetEventSpanTags(&span, eventCase.events) + waf.SetEventSpanTags(&span) + value, err := json.Marshal(map[string][]any{"triggers": eventCase.events}) if eventCase.expectedError { require.Error(t, err) return } + + span.SetTag("_dd.appsec.json", string(value)) + require.NoError(t, err) setRequestHeadersTags(&span, reqHeadersCase.headers) setResponseHeadersTags(&span, respHeadersCase.headers) @@ -224,6 +230,7 @@ func TestTags(t *testing.T) { "manual.keep": true, "appsec.event": true, "_dd.origin": "appsec", + "_dd.p.appsec": internal.PropagatingTagValue{Value: "1"}, }) } diff --git a/internal/appsec/listener/waf/tags.go b/internal/appsec/listener/waf/tags.go index bac41ce5ed..45d9f082d2 100644 --- a/internal/appsec/listener/waf/tags.go +++ b/internal/appsec/listener/waf/tags.go @@ -7,10 +7,10 @@ package waf import ( "encoding/json" - "fmt" waf "github.com/DataDog/go-libddwaf/v3" + "gopkg.in/DataDog/dd-trace-go.v1/internal" "gopkg.in/DataDog/dd-trace-go.v1/internal/log" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" @@ -61,18 +61,8 @@ func AddWAFMonitoringTags(th trace.TagSetter, rulesVersion string, stats map[str } } -// SetEventSpanTags sets the security event span tags into the service entry span. -func SetEventSpanTags(span trace.TagSetter, events []any) error { - if len(events) == 0 { - return nil - } - - // Set the appsec event span tag - val, err := makeEventTagValue(events) - if err != nil { - return err - } - span.SetTag("_dd.appsec.json", string(val)) +// SetEventSpanTags sets the security event span tags related to an appsec event +func SetEventSpanTags(span trace.TagSetter) { // Keep this span due to the security event // // This is a workaround to tell the tracer that the trace was kept by AppSec. @@ -83,19 +73,5 @@ func SetEventSpanTags(span trace.TagSetter, events []any) error { span.SetTag("_dd.origin", "appsec") // Set the appsec.event tag needed by the appsec backend span.SetTag("appsec.event", true) - return nil -} - -// Create the value of the security event tag. -func makeEventTagValue(events []any) (json.RawMessage, error) { - type eventTagValue struct { - Triggers []any `json:"triggers"` - } - - tag, err := json.Marshal(eventTagValue{events}) - if err != nil { - return nil, fmt.Errorf("unexpected error while serializing the appsec event span tag: %v", err) - } - - return tag, nil + span.SetTag("_dd.p.appsec", internal.PropagatingTagValue{Value: "1"}) } diff --git a/internal/appsec/listener/waf/waf.go b/internal/appsec/listener/waf/waf.go index 308eaa25d7..7607f3245b 100644 --- a/internal/appsec/listener/waf/waf.go +++ b/internal/appsec/listener/waf/waf.go @@ -87,15 +87,22 @@ func (waf *Feature) onStart(op *waf.ContextOperation, _ waf.ContextArgs) { waf.SetupActionHandlers(op) } -func (waf *Feature) SetupActionHandlers(op *waf.ContextOperation) { +func (*Feature) SetupActionHandlers(op *waf.ContextOperation) { // Set the blocking tag on the operation when a blocking event is received - dyngo.OnData(op, func(_ *events.BlockingSecurityEvent) { + dyngo.OnData(op, func(*events.BlockingSecurityEvent) { + log.Debug("appsec: blocking event detected") op.SetTag(BlockedRequestTag, true) }) // Register the stacktrace if one is requested by a WAF action - dyngo.OnData(op, func(err *actions.StackTraceAction) { - op.AddStackTraces(err.Event) + dyngo.OnData(op, func(action *actions.StackTraceAction) { + log.Debug("appsec: registering stack trace for security purposes") + op.AddStackTraces(action.Event) + }) + + dyngo.OnData(op, func(*waf.SecurityEvent) { + log.Debug("appsec: WAF detected a suspicious event") + SetEventSpanTags(op) }) } @@ -108,10 +115,9 @@ func (waf *Feature) onFinish(op *waf.ContextOperation, _ waf.ContextRes) { ctx.Close() AddWAFMonitoringTags(op, waf.handle.Diagnostics().Version, ctx.Stats().Metrics()) - if err := SetEventSpanTags(op, op.Events()); err != nil { - log.Debug("appsec: failed to set event span tags: %v", err) + if wafEvents := op.Events(); len(wafEvents) > 0 { + op.SetSerializableTag("_dd.appsec.json", map[string][]any{"triggers": op.Events()}) } - op.SetSerializableTags(op.Derivatives()) if stacks := op.StackTraces(); len(stacks) > 0 { op.SetTag(stacktrace.SpanKey, stacktrace.GetSpanValue(stacks...)) diff --git a/internal/meta_struct.go b/internal/meta_internal_types.go similarity index 69% rename from internal/meta_struct.go rename to internal/meta_internal_types.go index 6a8404d9c7..6562cc0f7f 100644 --- a/internal/meta_struct.go +++ b/internal/meta_internal_types.go @@ -10,3 +10,9 @@ package internal type MetaStructValue struct { Value any // TODO: further constraining Value's type, especially if it becomes public } + +// PropagatingTagValue is a custom type wrapper used to create tags that will be propagated +// to downstream distributed traces via the `X-Datadog-Tags` HTTP header for example. +type PropagatingTagValue struct { + Value string +} From 76d48962c09d49399814125eebed0e3b6af6ac37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Thu, 19 Dec 2024 15:47:18 +0100 Subject: [PATCH 08/13] fix: improving test logic for TestStreamSendsErrorCode to avoid flakiness (#3049) --- contrib/google.golang.org/grpc/grpc_test.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/contrib/google.golang.org/grpc/grpc_test.go b/contrib/google.golang.org/grpc/grpc_test.go index 2534630689..96a80fe1be 100644 --- a/contrib/google.golang.org/grpc/grpc_test.go +++ b/contrib/google.golang.org/grpc/grpc_test.go @@ -520,20 +520,20 @@ func TestStreamSendsErrorCode(t *testing.T) { // to flush the spans _, _ = stream.Recv() - containsErrorCode := false spans := mt.FinishedSpans() - // check if at least one span has error code + // check if at least one span with spank.kind=server has error code + var span mocktracer.Span for _, s := range spans { - if s.Tag(tagCode) == wantCode { - containsErrorCode = true + if s.Tag(tagCode) != wantCode { + continue } + if s.Tag(ext.SpanKind) != ext.SpanKindServer { + continue + } + span = s } - assert.True(t, containsErrorCode, "at least one span should contain error code, the spans were:\n%v", spans) - - // ensure that last span contains error code also - gotLastSpanCode := spans[len(spans)-1].Tag(tagCode) - assert.Equal(t, wantCode, gotLastSpanCode, "last span should contain error code") + assert.NotNilf(t, span, "at least one span should contain error code, the spans were:\n%v", spans) } // fixtureServer a dummy implementation of our grpc fixtureServer. From c536c42bb9f1b766e5504b29e83b6171e139d066 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Thu, 19 Dec 2024 16:57:42 +0100 Subject: [PATCH 09/13] vuln: upgrade golang.org/x/{crypto,net} to non-vulnerable versions (#3050) --- go.mod | 12 ++++++------ go.sum | 24 ++++++++++++------------ internal/apps/go.mod | 8 ++++---- internal/apps/go.sum | 20 ++++++++++---------- internal/exectracetest/go.mod | 8 ++++---- internal/exectracetest/go.sum | 20 ++++++++++---------- 6 files changed, 46 insertions(+), 46 deletions(-) diff --git a/go.mod b/go.mod index e301a9da7a..521ceb0374 100644 --- a/go.mod +++ b/go.mod @@ -101,7 +101,7 @@ require ( go.uber.org/goleak v1.3.0 golang.org/x/mod v0.20.0 golang.org/x/oauth2 v0.18.0 - golang.org/x/sys v0.24.0 + golang.org/x/sys v0.28.0 golang.org/x/time v0.6.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 google.golang.org/api v0.169.0 @@ -292,12 +292,12 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/arch v0.4.0 // indirect - golang.org/x/crypto v0.26.0 // indirect + golang.org/x/crypto v0.31.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/net v0.28.0 // indirect - golang.org/x/sync v0.8.0 // indirect - golang.org/x/term v0.23.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/tools v0.24.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect diff --git a/go.sum b/go.sum index ba7cea7f7c..e13756770a 100644 --- a/go.sum +++ b/go.sum @@ -2312,8 +2312,8 @@ golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2459,8 +2459,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2508,8 +2508,8 @@ golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -2658,8 +2658,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= -golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -2674,8 +2674,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2693,8 +2693,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/internal/apps/go.mod b/internal/apps/go.mod index 76197025d9..1071a2037e 100644 --- a/internal/apps/go.mod +++ b/internal/apps/go.mod @@ -3,7 +3,7 @@ module github.com/DataDog/dd-trace-go/internal/apps go 1.23.0 require ( - golang.org/x/sync v0.8.0 + golang.org/x/sync v0.10.0 gopkg.in/DataDog/dd-trace-go.v1 v1.64.0 ) @@ -53,8 +53,8 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/mod v0.20.0 // indirect - golang.org/x/net v0.28.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/tools v0.24.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect google.golang.org/grpc v1.64.1 // indirect @@ -82,7 +82,7 @@ require ( github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/stretchr/testify v1.9.0 github.com/tinylib/msgp v1.2.1 // indirect - golang.org/x/sys v0.24.0 // indirect + golang.org/x/sys v0.28.0 // indirect golang.org/x/time v0.6.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/protobuf v1.34.2 // indirect diff --git a/internal/apps/go.sum b/internal/apps/go.sum index a0e2472c51..e4beaf0c9f 100644 --- a/internal/apps/go.sum +++ b/internal/apps/go.sum @@ -223,8 +223,8 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -235,14 +235,14 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -258,13 +258,13 @@ golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= -golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/internal/exectracetest/go.mod b/internal/exectracetest/go.mod index 9292ebda48..653487ca6f 100644 --- a/internal/exectracetest/go.mod +++ b/internal/exectracetest/go.mod @@ -66,10 +66,10 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/mod v0.20.0 // indirect - golang.org/x/net v0.28.0 // indirect - golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.24.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.6.0 // indirect golang.org/x/tools v0.24.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect diff --git a/internal/exectracetest/go.sum b/internal/exectracetest/go.sum index 00a653dc7e..db01821d72 100644 --- a/internal/exectracetest/go.sum +++ b/internal/exectracetest/go.sum @@ -229,8 +229,8 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -243,14 +243,14 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -266,13 +266,13 @@ golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= -golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 144d5924232fddc9b7bfaa0e3aa1e4647da7190b Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Thu, 19 Dec 2024 11:11:16 -0500 Subject: [PATCH 10/13] contrib/miekg/dns: resolve flaky test in TestExchange* (#3045) --- contrib/miekg/dns/dns_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/miekg/dns/dns_test.go b/contrib/miekg/dns/dns_test.go index c1efa2a799..6e38fbaca5 100644 --- a/contrib/miekg/dns/dns_test.go +++ b/contrib/miekg/dns/dns_test.go @@ -33,7 +33,7 @@ func startServer(t *testing.T, traced bool) (*dns.Server, func()) { if traced { h = dnstrace.WrapHandler(h) } - addr := getFreeAddr(t).String() + addr := getAddr(t).String() server := &dns.Server{ Addr: addr, Net: "udp", @@ -190,8 +190,8 @@ func assertClientSpan(t *testing.T, s mocktracer.Span) { assert.Equal(t, ext.SpanKindClient, s.Tag(ext.SpanKind)) } -func getFreeAddr(t *testing.T) net.Addr { - li, err := net.Listen("tcp", "127.0.0.1:0") +func getAddr(t *testing.T) net.Addr { + li, err := net.Listen("tcp4", "127.0.0.1:2020") if err != nil { t.Fatal(err) } From 822d81bbc2555e86507db164dc079b2ed83248d9 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler <46911781+mtoffl01@users.noreply.github.com> Date: Thu, 19 Dec 2024 15:16:02 -0500 Subject: [PATCH 11/13] ddtrace/tracer: report datadog.tracer.api.errors health metric (#3024) --- ddtrace/tracer/transport.go | 30 ++++++++--- ddtrace/tracer/transport_test.go | 85 +++++++++++++++++++++++++++++++ internal/statsdtest/statsdtest.go | 14 +++++ 3 files changed, 123 insertions(+), 6 deletions(-) diff --git a/ddtrace/tracer/transport.go b/ddtrace/tracer/transport.go index a8ad9cfe8e..222f6c8b70 100644 --- a/ddtrace/tracer/transport.go +++ b/ddtrace/tracer/transport.go @@ -151,16 +151,18 @@ func (t *httpTransport) send(p *payload) (body io.ReadCloser, err error) { } req.Header.Set(traceCountHeader, strconv.Itoa(p.itemCount())) req.Header.Set(headerComputedTopLevel, "yes") - if t, ok := traceinternal.GetGlobalTracer().(*tracer); ok { - if t.config.tracingAsTransport || t.config.canComputeStats() { + var tr *tracer + var haveTracer bool + if tr, haveTracer = traceinternal.GetGlobalTracer().(*tracer); haveTracer { + if tr.config.tracingAsTransport || tr.config.canComputeStats() { // tracingAsTransport uses this header to disable the trace agent's stats computation // while making canComputeStats() always false to also disable client stats computation. req.Header.Set("Datadog-Client-Computed-Stats", "yes") } - droppedTraces := int(atomic.SwapUint32(&t.droppedP0Traces, 0)) - partialTraces := int(atomic.SwapUint32(&t.partialTraces, 0)) - droppedSpans := int(atomic.SwapUint32(&t.droppedP0Spans, 0)) - if stats := t.statsd; stats != nil { + droppedTraces := int(atomic.SwapUint32(&tr.droppedP0Traces, 0)) + partialTraces := int(atomic.SwapUint32(&tr.partialTraces, 0)) + droppedSpans := int(atomic.SwapUint32(&tr.droppedP0Spans, 0)) + if stats := tr.statsd; stats != nil { stats.Count("datadog.tracer.dropped_p0_traces", int64(droppedTraces), []string{fmt.Sprintf("partial:%s", strconv.FormatBool(partialTraces > 0))}, 1) stats.Count("datadog.tracer.dropped_p0_spans", int64(droppedSpans), nil, 1) @@ -170,9 +172,11 @@ func (t *httpTransport) send(p *payload) (body io.ReadCloser, err error) { } response, err := t.client.Do(req) if err != nil { + reportAPIErrorsMetric(haveTracer, response, err, tr) return nil, err } if code := response.StatusCode; code >= 400 { + reportAPIErrorsMetric(haveTracer, response, err, tr) // error, check the body for context information and // return a nice error. msg := make([]byte, 1000) @@ -187,6 +191,20 @@ func (t *httpTransport) send(p *payload) (body io.ReadCloser, err error) { return response.Body, nil } +func reportAPIErrorsMetric(haveTracer bool, response *http.Response, err error, t *tracer) { + if !haveTracer { + return + } + var reason string + if err != nil { + reason = "network_failure" + } + if response != nil { + reason = fmt.Sprintf("server_response_%d", response.StatusCode) + } + t.statsd.Incr("datadog.tracer.api.errors", []string{"reason:" + reason}, 1) +} + func (t *httpTransport) endpoint() string { return t.traceURL } diff --git a/ddtrace/tracer/transport_test.go b/ddtrace/tracer/transport_test.go index 8c1148277c..b8967713cf 100644 --- a/ddtrace/tracer/transport_test.go +++ b/ddtrace/tracer/transport_test.go @@ -20,7 +20,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + traceinternal "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/internal" "gopkg.in/DataDog/dd-trace-go.v1/internal" + "gopkg.in/DataDog/dd-trace-go.v1/internal/statsdtest" ) // getTestSpan returns a Span with different fields set @@ -241,6 +243,89 @@ func TestCustomTransport(t *testing.T) { assert.Equal(hits, 1) } +type ErrTransport struct{} + +func (t *ErrTransport) RoundTrip(req *http.Request) (*http.Response, error) { + return nil, fmt.Errorf("error in RoundTripper") +} + +type ErrResponseTransport struct{} + +func (t *ErrResponseTransport) RoundTrip(req *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: 400}, nil +} + +type OkTransport struct{} + +func (t *OkTransport) RoundTrip(req *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: 200}, nil +} + +func TestApiErrorsMetric(t *testing.T) { + t.Run("error", func(t *testing.T) { + assert := assert.New(t) + c := &http.Client{ + Transport: &ErrTransport{}, + } + var tg statsdtest.TestStatsdClient + trc := newTracer(WithHTTPClient(c), withStatsdClient(&tg)) + traceinternal.SetGlobalTracer(trc) + defer trc.Stop() + + p, err := encode(getTestTrace(1, 1)) + assert.NoError(err) + + // We're expecting an error + _, err = trc.config.transport.send(p) + assert.Error(err) + calls := statsdtest.FilterCallsByName(tg.IncrCalls(), "datadog.tracer.api.errors") + assert.Len(calls, 1) + call := calls[0] + assert.Equal([]string{"reason:network_failure"}, call.Tags()) + + }) + t.Run("response with err code", func(t *testing.T) { + assert := assert.New(t) + c := &http.Client{ + Transport: &ErrResponseTransport{}, + } + var tg statsdtest.TestStatsdClient + trc := newTracer(WithHTTPClient(c), withStatsdClient(&tg)) + traceinternal.SetGlobalTracer(trc) + defer trc.Stop() + + p, err := encode(getTestTrace(1, 1)) + assert.NoError(err) + + _, err = trc.config.transport.send(p) + assert.Error(err) + + calls := statsdtest.FilterCallsByName(tg.IncrCalls(), "datadog.tracer.api.errors") + assert.Len(calls, 1) + call := calls[0] + assert.Equal([]string{"reason:server_response_400"}, call.Tags()) + }) + t.Run("successful send - no metric", func(t *testing.T) { + assert := assert.New(t) + var tg statsdtest.TestStatsdClient + c := &http.Client{ + Transport: &OkTransport{}, + } + trc := newTracer(WithHTTPClient(c), withStatsdClient(&tg)) + traceinternal.SetGlobalTracer(trc) + defer trc.Stop() + + p, err := encode(getTestTrace(1, 1)) + assert.NoError(err) + + _, err = trc.config.transport.send(p) + assert.NoError(err) + + calls := statsdtest.FilterCallsByName(tg.IncrCalls(), "datadog.tracer.api.errors") + assert.Len(calls, 0) + }) +} + func TestWithHTTPClient(t *testing.T) { // disable instrumentation telemetry to prevent flaky number of requests t.Setenv("DD_INSTRUMENTATION_TELEMETRY_ENABLED", "false") diff --git a/internal/statsdtest/statsdtest.go b/internal/statsdtest/statsdtest.go index ffcd700cd1..bed3646fbc 100644 --- a/internal/statsdtest/statsdtest.go +++ b/internal/statsdtest/statsdtest.go @@ -49,6 +49,10 @@ type TestStatsdCall struct { rate float64 } +func (c *TestStatsdCall) Tags() []string { + return c.tags +} + func (tg *TestStatsdClient) addCount(name string, value int64) { tg.mu.Lock() defer tg.mu.Unlock() @@ -221,6 +225,16 @@ func (tg *TestStatsdClient) CallsByName() map[string]int { return counts } +func FilterCallsByName(calls []TestStatsdCall, name string) []TestStatsdCall { + var matches []TestStatsdCall + for _, c := range calls { + if c.name == name { + matches = append(matches, c) + } + } + return matches +} + func (tg *TestStatsdClient) Counts() map[string]int64 { tg.mu.RLock() defer tg.mu.RUnlock() From 637ea726559cc6a77dbd2bbff4d182691732797e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Dec 2024 12:03:58 +0000 Subject: [PATCH 12/13] build(deps): bump google.golang.org/grpc from 1.64.0 to 1.64.1 (#3001) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Rodrigo Argüello --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 521ceb0374..bbe71fe323 100644 --- a/go.mod +++ b/go.mod @@ -105,7 +105,7 @@ require ( golang.org/x/time v0.6.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 google.golang.org/api v0.169.0 - google.golang.org/grpc v1.64.0 + google.golang.org/grpc v1.64.1 google.golang.org/protobuf v1.34.2 gopkg.in/jinzhu/gorm.v1 v1.9.2 gopkg.in/olivere/elastic.v3 v3.0.75 diff --git a/go.sum b/go.sum index e13756770a..aed16d7b16 100644 --- a/go.sum +++ b/go.sum @@ -3061,8 +3061,8 @@ google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCD google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= -google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= -google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From d39332478672b57617dafc315fc258e888067143 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler <46911781+mtoffl01@users.noreply.github.com> Date: Fri, 20 Dec 2024 10:49:22 -0500 Subject: [PATCH 13/13] ddtrace/tracer: Report datadog.tracer.queue.enqueued.traces as health metric (#3019) --- ddtrace/tracer/metrics_test.go | 26 +++++++++++++++++++++++++- ddtrace/tracer/writer.go | 5 +++++ ddtrace/tracer/writer_test.go | 10 ++++++---- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/ddtrace/tracer/metrics_test.go b/ddtrace/tracer/metrics_test.go index 4aad5e373c..64548fce86 100644 --- a/ddtrace/tracer/metrics_test.go +++ b/ddtrace/tracer/metrics_test.go @@ -55,12 +55,36 @@ func TestReportHealthMetrics(t *testing.T) { tracer.StartSpan("operation").Finish() flush(1) - tg.Wait(assert, 3, 10*time.Second) + tg.Wait(assert, 4, 10*time.Second) counts := tg.Counts() assert.Equal(int64(1), counts["datadog.tracer.spans_started"]) assert.Equal(int64(1), counts["datadog.tracer.spans_finished"]) assert.Equal(int64(0), counts["datadog.tracer.traces_dropped"]) + assert.Equal(int64(1), counts["datadog.tracer.queue.enqueued.traces"]) +} + +func TestEnqueuedTracesHealthMetric(t *testing.T) { + assert := assert.New(t) + var tg statsdtest.TestStatsdClient + + defer func(old time.Duration) { statsInterval = old }(statsInterval) + statsInterval = time.Nanosecond + + tracer, _, flush, stop := startTestTracer(t, withStatsdClient(&tg)) + defer stop() + + for i := 0; i < 3; i++ { + tracer.StartSpan("operation").Finish() + } + flush(3) + tg.Wait(assert, 1, 10*time.Second) + + counts := tg.Counts() + assert.Equal(int64(3), counts["datadog.tracer.queue.enqueued.traces"]) + w, ok := tracer.traceWriter.(*agentTraceWriter) + assert.True(ok) + assert.Equal(uint32(0), w.tracesQueued) } func TestTracerMetrics(t *testing.T) { diff --git a/ddtrace/tracer/writer.go b/ddtrace/tracer/writer.go index 5ceadcb2e6..cff333c8c4 100644 --- a/ddtrace/tracer/writer.go +++ b/ddtrace/tracer/writer.go @@ -14,6 +14,7 @@ import ( "os" "strconv" "sync" + "sync/atomic" "time" globalinternal "gopkg.in/DataDog/dd-trace-go.v1/internal" @@ -50,6 +51,8 @@ type agentTraceWriter struct { // statsd is used to send metrics statsd globalinternal.StatsdClient + + tracesQueued uint32 } func newAgentTraceWriter(c *config, s *prioritySampler, statsdClient globalinternal.StatsdClient) *agentTraceWriter { @@ -67,6 +70,7 @@ func (h *agentTraceWriter) add(trace []*span) { h.statsd.Incr("datadog.tracer.traces_dropped", []string{"reason:encoding_error"}, 1) log.Error("Error encoding msgpack: %v", err) } + atomic.AddUint32(&h.tracesQueued, 1) // TODO: This does not differentiate between complete traces and partial chunks if h.payload.size() > payloadSizeLimit { h.statsd.Incr("datadog.tracer.flush_triggered", []string{"reason:size"}, 1) h.flush() @@ -94,6 +98,7 @@ func (h *agentTraceWriter) flush() { // collection to avoid a memory leak when references to this object // may still be kept by faulty transport implementations or the // standard library. See dd-trace-go#976 + h.statsd.Count("datadog.tracer.queue.enqueued.traces", int64(atomic.SwapUint32(&h.tracesQueued, 0)), nil, 1) p.clear() <-h.climit diff --git a/ddtrace/tracer/writer_test.go b/ddtrace/tracer/writer_test.go index bcdf529a58..224ae2a1ca 100644 --- a/ddtrace/tracer/writer_test.go +++ b/ddtrace/tracer/writer_test.go @@ -386,12 +386,14 @@ func TestTraceWriterFlushRetries(t *testing.T) { } sentCounts := map[string]int64{ - "datadog.tracer.decode_error": 1, - "datadog.tracer.flush_bytes": 185, - "datadog.tracer.flush_traces": 1, + "datadog.tracer.decode_error": 1, + "datadog.tracer.flush_bytes": 185, + "datadog.tracer.flush_traces": 1, + "datadog.tracer.queue.enqueued.traces": 1, } droppedCounts := map[string]int64{ - "datadog.tracer.traces_dropped": 1, + "datadog.tracer.queue.enqueued.traces": 1, + "datadog.tracer.traces_dropped": 1, } ss := []*span{makeSpan(0)}