diff --git a/.github/workflows/service-extensions-publish.yml b/.github/workflows/service-extensions-publish.yml new file mode 100644 index 0000000000..9e0c645be0 --- /dev/null +++ b/.github/workflows/service-extensions-publish.yml @@ -0,0 +1,99 @@ +name: Publish Service Extensions Callout images packages + +on: + push: + branches: + - 'flavien/service-extensions' + release: + types: + - published + workflow_dispatch: + inputs: + tag_name: + description: 'Docker image tag to use for the package' + required: true + default: 'dev' + commit_sha: + description: 'Commit SHA to checkout' + required: true + +permissions: + contents: read + packages: write + +jobs: + publish-service-extensions: + runs-on: ubuntu-latest + steps: + + - name: Get tag name + id: get_tag_name + run: | + if [ "${{ github.event_name }}" = "release" ]; then + echo "::set-output name=tag::${{ github.event.release.tag_name }}" + echo "Here1: tag=${{ github.event.release.tag_name }}" + else + if [ -z "${{ github.event.inputs.tag_name }}" ]; then + echo "::set-output name=tag::dev" + echo "Here2: tag=dev" + else + echo "::set-output name=tag::${{ github.event.inputs.tag_name }}" + echo "Here3: tag=${{ github.event.inputs.tag_name }}" + fi + fi + echo "Finally: ${{ steps.get_tag_name.outputs.tag }}" + + - name: Checkout + uses: actions/checkout@v4 + if: github.event_name == 'release' + with: + ref: ${{ steps.get_tag_name.outputs.tag }} + + - name: Checkout + uses: actions/checkout@v4 + if: github.event_name != 'release' + with: + ref: ${{ github.event.inputs.commit_sha || github.sha }} + + - name: Set up Go 1.22 + uses: actions/setup-go@v5 + with: + go-version: 1.22 + id: go + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker + shell: bash + run: docker login -u publisher -p ${{ secrets.GITHUB_TOKEN }} ghcr.io + + - name: Build and push [dev] + id: build-dev + if: github.event_name != 'release' + uses: docker/build-push-action@v6 + with: + context: . + file: ./contrib/envoyproxy/envoy/cmd/serviceextensions/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | # Use the commit SHA from the manual trigger or default to the SHA from the push event + ghcr.io/datadog/dd-trace-go/service-extensions-callout:${{ steps.get_tag_name.outputs.tag }} + ghcr.io/datadog/dd-trace-go/service-extensions-callout:${{ github.event.inputs.commit_sha || github.sha }} + + - name: Build and push [release] + id: build-release + if: github.event_name == 'release' + uses: docker/build-push-action@v6 + with: + context: . + file: ./contrib/envoyproxy/envoy/cmd/serviceextensions/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ghcr.io/datadog/dd-trace-go/service-extensions-callout:latest + ghcr.io/datadog/dd-trace-go/service-extensions-callout:${{ steps.get_tag_name.outputs.tag }} + ghcr.io/datadog/dd-trace-go/service-extensions-callout:${{ github.sha }} \ No newline at end of file diff --git a/contrib/envoyproxy/envoy/cmd/serviceextensions/.gitignore b/contrib/envoyproxy/envoy/cmd/serviceextensions/.gitignore new file mode 100644 index 0000000000..68295c4a55 --- /dev/null +++ b/contrib/envoyproxy/envoy/cmd/serviceextensions/.gitignore @@ -0,0 +1 @@ +serviceextensions \ No newline at end of file diff --git a/contrib/envoyproxy/envoy/cmd/serviceextensions/Dockerfile b/contrib/envoyproxy/envoy/cmd/serviceextensions/Dockerfile new file mode 100644 index 0000000000..87136d2cba --- /dev/null +++ b/contrib/envoyproxy/envoy/cmd/serviceextensions/Dockerfile @@ -0,0 +1,20 @@ +# Build stage +FROM golang:1.22-alpine AS builder +ENV CGO_ENABLED=1 +WORKDIR /app +COPY . . +RUN apk add --no-cache --update git build-base +RUN go build -o ./contrib/envoyproxy/envoy/cmd/serviceextensions/serviceextensions ./contrib/envoyproxy/envoy/cmd/serviceextensions + +# Runtime stage +FROM alpine:3.20.3 +RUN apk --no-cache add ca-certificates tzdata libc6-compat libgcc libstdc++ +WORKDIR /app +COPY --from=builder /app/contrib/envoyproxy/envoy/cmd/serviceextensions/serviceextensions /app/serviceextensions +COPY ./contrib/envoyproxy/envoy/cmd/serviceextensions/localhost.crt /app/localhost.crt +COPY ./contrib/envoyproxy/envoy/cmd/serviceextensions/localhost.key /app/localhost.key + +EXPOSE 80 +EXPOSE 443 + +CMD ["./serviceextensions"] diff --git a/contrib/envoyproxy/envoy/cmd/serviceextensions/localhost.crt b/contrib/envoyproxy/envoy/cmd/serviceextensions/localhost.crt new file mode 100644 index 0000000000..fc54fd492e --- /dev/null +++ b/contrib/envoyproxy/envoy/cmd/serviceextensions/localhost.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDFjCCAf4CCQCzrLIhrWa55zANBgkqhkiG9w0BAQsFADBCMQswCQYDVQQGEwJV +UzETMBEGA1UECAwKQ2FsaWZvcm5pYTEPMA0GA1UECgwGR29vZ2xlMQ0wCwYDVQQL +DARnUlBDMCAXDTE5MDYyNDIyMjIzM1oYDzIxMTkwNTMxMjIyMjMzWjBWMQswCQYD +VQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEPMA0GA1UECgwGR29vZ2xlMQ0w +CwYDVQQLDARnUlBDMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQCtCW0TjugnIUu8BEVIYvdMP+/2GENQDjZhZ8eKR5C6 +toDGbgjsDtt/GxISAg4cg70fIvy0XolnGPZodvfHDM4lJ7yHBOdZD8TXQoE6okR7 +HZuLUJ20M0pXgWqtRewKRUjuYsSDXBnzLiZw1dcv9nGpo+Bqa8NonpiGRRpEkshF +D6T9KU9Ts/x+wMQBIra2Gj0UMh79jPhUuxcYAQA0JQGivnOtdwuPiumpnUT8j8h6 +tWg5l01EsCZWJecCF85KnGpJEVYPyPqBqGsy0nGS9plGotOWF87+jyUQt+KD63xA +aBmTro86mKDDKEK4JvzjVeMGz2UbVcLPiiZnErTFaiXJAgMBAAEwDQYJKoZIhvcN +AQELBQADggEBAKsDgOPCWp5WCy17vJbRlgfgk05sVNIHZtzrmdswjBmvSg8MUpep +XqcPNUpsljAXsf9UM5IFEMRdilUsFGWvHjBEtNAW8WUK9UV18WRuU//0w1Mp5HAN +xUEKb4BoyZr65vlCnTR+AR5c9FfPvLibhr5qHs2RA8Y3GyLOcGqBWed87jhdQLCc +P1bxB+96le5JeXq0tw215lxonI2/3ZYVK4/ok9gwXrQoWm8YieJqitk/ZQ4S17/4 +pynHtDfdxLn23EXeGx+UTxJGfpRmhEZdJ+MN7QGYoomzx5qS5XoYKxRNrDlirJpr +OqXIn8E1it+6d5gOZfuHawcNGhRLplE/pfA= +-----END CERTIFICATE----- diff --git a/contrib/envoyproxy/envoy/cmd/serviceextensions/localhost.key b/contrib/envoyproxy/envoy/cmd/serviceextensions/localhost.key new file mode 100644 index 0000000000..72e2463282 --- /dev/null +++ b/contrib/envoyproxy/envoy/cmd/serviceextensions/localhost.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEArQltE47oJyFLvARFSGL3TD/v9hhDUA42YWfHikeQuraAxm4I +7A7bfxsSEgIOHIO9HyL8tF6JZxj2aHb3xwzOJSe8hwTnWQ/E10KBOqJEex2bi1Cd +tDNKV4FqrUXsCkVI7mLEg1wZ8y4mcNXXL/ZxqaPgamvDaJ6YhkUaRJLIRQ+k/SlP +U7P8fsDEASK2tho9FDIe/Yz4VLsXGAEANCUBor5zrXcLj4rpqZ1E/I/IerVoOZdN +RLAmViXnAhfOSpxqSRFWD8j6gahrMtJxkvaZRqLTlhfO/o8lELfig+t8QGgZk66P +OpigwyhCuCb841XjBs9lG1XCz4omZxK0xWolyQIDAQABAoIBADeq/Kh6JT3RfGf0 +h8WN8TlaqHxnueAbcmtL0+oss+cdp7gu1jf7X6o4r0uT1a5ew40s2Fe+wj2kzkE1 +ZOlouTlC22gkr7j7Vbxa7PBMG/Pvxoa/XL0IczZLsGImSJXVTG1E4SvRiZeulTdf +1GbdxhtpWV1jZe5Wd4Na3+SHxF5S7m3PrHiZlYdz1ND+8XZs1NlL9+ej72qSFul9 +t/QjMWJ9pky/Wad5abnRLRyOsg+BsgnXbkUy2rD89ZxFMLda9pzXo3TPyAlBHonr +mkEsE4eRMWMpjBM79JbeyDdHn/cs/LjAZrzeDf7ugXr2CHQpKaM5O0PsNHezJII9 +L5kCfzECgYEA4M/rz1UP1/BJoSqigUlSs0tPAg8a5UlkVsh6Osuq72IPNo8qg/Fw +oV/IiIS+q+obRcFj1Od3PGdTpCJwW5dzd2fXBQGmGdj0HucnCrs13RtBh91JiF5i +y/YYI9KfgOG2ZT9gG68T0gTs6jRrS3Qd83npqjrkJqMOd7s00MK9tUcCgYEAxQq7 +T541oCYHSBRIIb0IrR25krZy9caxzCqPDwOcuuhaCqCiaq+ATvOWlSfgecm4eH0K +PCH0xlWxG0auPEwm4pA8+/WR/XJwscPZMuoht1EoKy1his4eKx/s7hHNeO6KOF0V +Y/zqIiuZnEwUoKbn7EqqNFSTT65PJKyGsICJFG8CgYAfaw9yl1myfQNdQb8aQGwN +YJ33FLNWje427qeeZe5KrDKiFloDvI9YDjHRWnPnRL1w/zj7fSm9yFb5HlMDieP6 +MQnsyjEzdY2QcA+VwVoiv3dmDHgFVeOKy6bOAtaFxYWfGr9MvygO9t9BT/gawGyb +JVORlc9i0vDnrMMR1dV7awKBgBpTWLtGc/u1mPt0Wj7HtsUKV6TWY32a0l5owTxM +S0BdksogtBJ06DukJ9Y9wawD23WdnyRxlPZ6tHLkeprrwbY7dypioOKvy4a0l+xJ +g7+uRCOgqIuXBkjUtx8HmeAyXp0xMo5tWArAsIFFWOwt4IadYygitJvMuh44PraO +NcJZAoGADEiV0dheXUCVr8DrtSom8DQMj92/G/FIYjXL8OUhh0+F+YlYP0+F8PEU +yYIWEqL/S5tVKYshimUXQa537JcRKsTVJBG/ZKD2kuqgOc72zQy3oplimXeJDCXY +h2eAQ0u8GN6tN9C4t8Kp4a3y6FGsxgu+UTxdnL3YQ+yHAVhtCzo= +-----END RSA PRIVATE KEY----- diff --git a/contrib/envoyproxy/envoy/cmd/serviceextensions/main.go b/contrib/envoyproxy/envoy/cmd/serviceextensions/main.go new file mode 100644 index 0000000000..fcd86b8fb3 --- /dev/null +++ b/contrib/envoyproxy/envoy/cmd/serviceextensions/main.go @@ -0,0 +1,128 @@ +package main + +import ( + "crypto/tls" + "gopkg.in/DataDog/dd-trace-go.v1/contrib/envoyproxy/envoy" + "gopkg.in/DataDog/dd-trace-go.v1/internal/log" + "gopkg.in/DataDog/dd-trace-go.v1/internal/version" + "net" + "net/http" + "os" + + extproc "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + "github.com/gorilla/mux" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/reflection" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" +) + +// AppsecCalloutExtensionService defines the struct that follows the ExternalProcessorServer interface. +type AppsecCalloutExtensionService struct { + extproc.ExternalProcessorServer +} + +type serviceExtensionConfig struct { + extensionPort string + extensionHost string + healthcheckPort string +} + +func loadConfig() serviceExtensionConfig { + extensionPort := os.Getenv("DD_SERVICE_EXTENSION_PORT") + if extensionPort == "" { + extensionPort = "443" + } + + extensionHost := os.Getenv("DD_SERVICE_EXTENSION_HOST") + if extensionHost == "" { + extensionHost = "0.0.0.0" + } + + healthcheckPort := os.Getenv("DD_SERVICE_EXTENSION_HEALTHCHECK_PORT") + if healthcheckPort == "" { + healthcheckPort = "80" + } + + return serviceExtensionConfig{ + extensionPort: extensionPort, + extensionHost: extensionHost, + healthcheckPort: healthcheckPort, + } +} + +func main() { + var extensionService AppsecCalloutExtensionService + + // Force set ASM as enabled only if the environment variable is not set + // Note: If the environment variable is set to false, it should be disabled + if os.Getenv("DD_APPSEC_ENABLED") == "" { + if err := os.Setenv("DD_APPSEC_ENABLED", "1"); err != nil { + log.Error("service_extension: failed to set DD_APPSEC_ENABLED environment variable: %v\n", err) + } + } + + // TODO: Enable ASM standalone mode when it is developed (should be done for Q4 2024) + + // Set the DD_VERSION to the current tracer version if not set + if os.Getenv("DD_VERSION") == "" { + if err := os.Setenv("DD_VERSION", version.Tag); err != nil { + log.Error("service_extension: failed to set DD_VERSION environment variable: %v\n", err) + } + } + + config := loadConfig() + + tracer.Start() + + go StartGPRCSsl(&extensionService, config) + log.Info("service_extension: callout gRPC server started on %s:%s\n", config.extensionHost, config.extensionPort) + + go startHealthCheck(config) + log.Info("service_extension: health check server started on %s:%s\n", config.extensionHost, config.healthcheckPort) + + select {} +} + +func startHealthCheck(config serviceExtensionConfig) { + muxServer := mux.NewRouter() + muxServer.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, err := w.Write([]byte(`{"status": "ok", "library": {"language": "golang", "version": "` + version.Tag + `"}}`)) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) + }) + + server := &http.Server{ + Addr: config.extensionHost + ":" + config.healthcheckPort, + Handler: muxServer, + } + + println(server.ListenAndServe()) +} + +func StartGPRCSsl(service extproc.ExternalProcessorServer, config serviceExtensionConfig) { + cert, err := tls.LoadX509KeyPair("localhost.crt", "localhost.key") + if err != nil { + log.Error("Failed to load key pair: %v\n", err) + } + + lis, err := net.Listen("tcp", config.extensionHost+":"+config.extensionPort) + if err != nil { + log.Error("Failed to listen: %v\n", err) + } + + si := envoy.StreamServerInterceptor() + creds := credentials.NewServerTLSFromCert(&cert) + grpcServer := grpc.NewServer(grpc.StreamInterceptor(si), grpc.Creds(creds)) + + extproc.RegisterExternalProcessorServer(grpcServer, service) + reflection.Register(grpcServer) + if err := grpcServer.Serve(lis); err != nil { + log.Error("service_extension: failed to serve gRPC: %v\n", err) + } +} diff --git a/contrib/envoyproxy/envoy/envoy.go b/contrib/envoyproxy/envoy/envoy.go new file mode 100644 index 0000000000..413d475726 --- /dev/null +++ b/contrib/envoyproxy/envoy/envoy.go @@ -0,0 +1,433 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016 Datadog, Inc. + +package envoy + +import ( + "context" + "errors" + "gopkg.in/DataDog/dd-trace-go.v1/internal/telemetry" + "io" + "math" + "net/http" + "net/url" + "strconv" + "strings" + "sync/atomic" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + grpctrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/google.golang.org/grpc" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/emitter/waf/actions" + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/listener/trace" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + extproc "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + v32 "github.com/envoyproxy/go-control-plane/envoy/type/v3" + + "gopkg.in/DataDog/dd-trace-go.v1/contrib/internal/httptrace" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/emitter/httpsec" + httpsec2 "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/listener/httpsec" + "gopkg.in/DataDog/dd-trace-go.v1/internal/log" +) + +const componentName = "envoy/service/ext_proc/v3" + +func init() { + telemetry.LoadIntegration(componentName) + tracer.MarkIntegrationImported("github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3") +} + +type CurrentRequest struct { + op *httpsec.HandlerOperation + blockAction *atomic.Pointer[actions.BlockHTTP] + span tracer.Span + + remoteAddr string + parsedUrl *url.URL + requestArgs httpsec.HandlerOperationArgs + + statusCode int + blocked bool +} + +func getRemoteAddr(xfwd []string) string { + length := len(xfwd) + if length == 0 { + return "" + } + + // Get the first right value of x-forwarded-for header + // The rightmost IP address is the one that will be used as the remote client IP + // https://datadoghq.atlassian.net/wiki/spaces/TS/pages/2766733526/Sensitive+IP+information#Where-does-the-value-of-the-http.client_ip-tag-come-from%3F + return xfwd[length-1] +} + +func StreamServerInterceptor(opts ...grpctrace.Option) grpc.StreamServerInterceptor { + interceptor := grpctrace.StreamServerInterceptor(opts...) + + return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + if info.FullMethod != extproc.ExternalProcessor_Process_FullMethodName { + return interceptor(srv, ss, info, handler) + } + + ctx := ss.Context() + md, _ := metadata.FromIncomingContext(ctx) + currentRequest := &CurrentRequest{ + blocked: false, + remoteAddr: getRemoteAddr(md.Get("x-forwarded-for")), + } + + // Close the span when the request is done processing + defer func() { + closeSpan(currentRequest) + }() + + for { + select { + case <-ctx.Done(): + if errors.Is(ctx.Err(), context.Canceled) { + return nil + } + + return ctx.Err() + + default: + } + + var req extproc.ProcessingRequest + err := ss.RecvMsg(&req) + if err != nil { + // Note: Envoy is inconsistent with the "end_of_stream" value of its headers responses, + // so we can't fully rely on it to determine when it will close (cancel) the stream. + if err == io.EOF || err.(interface{ GRPCStatus() *status.Status }).GRPCStatus().Code() == codes.Canceled { + return nil + } + + log.Warn("external_processing: error receiving request/response: %v\n", err) + return status.Errorf(codes.Unknown, "Error receiving request/response: %v", err) + } + + resp, err := envoyExternalProcessingEventHandler(ctx, &req, currentRequest) + if err != nil { + log.Error("external_processing: error processing request/response: %v\n", err) + return status.Errorf(codes.Unknown, "Error processing request/response: %v", err) + } + + // End of stream reached, no more data to process + if resp == nil { + log.Debug("external_processing: end of stream reached") + return nil + } + + // Send Message could fail if envoy close the stream before the message could be sent (probably because of an Envoy timeout) + if err := ss.SendMsg(resp); err != nil { + log.Warn("external_processing: error sending response (probably because of an Envoy timeout): %v", err) + return status.Errorf(codes.Unknown, "Error sending response (probably because of an Envoy timeout): %v", err) + } + + if currentRequest.blocked { + log.Debug("external_processing: request blocked, stream ended") + return nil + } + } + } +} + +func envoyExternalProcessingEventHandler(ctx context.Context, req *extproc.ProcessingRequest, currentRequest *CurrentRequest) (*extproc.ProcessingResponse, error) { + switch v := req.Request.(type) { + case *extproc.ProcessingRequest_RequestHeaders: + return ProcessRequestHeaders(ctx, req.Request.(*extproc.ProcessingRequest_RequestHeaders), currentRequest) + + case *extproc.ProcessingRequest_RequestBody: + // TODO: Handle request raw body in the WAF + return &extproc.ProcessingResponse{ + Response: &extproc.ProcessingResponse_RequestBody{ + RequestBody: &extproc.BodyResponse{ + Response: &extproc.CommonResponse{ + Status: extproc.CommonResponse_CONTINUE, + }, + }, + }, + }, nil + + case *extproc.ProcessingRequest_RequestTrailers: + return &extproc.ProcessingResponse{ + Response: &extproc.ProcessingResponse_RequestTrailers{}, + }, nil + + case *extproc.ProcessingRequest_ResponseHeaders: + return ProcessResponseHeaders(req.Request.(*extproc.ProcessingRequest_ResponseHeaders), currentRequest) + + case *extproc.ProcessingRequest_ResponseBody: + r := req.Request.(*extproc.ProcessingRequest_ResponseBody) + + // Note: The end of stream bool value is not reliable + // Sometimes it's not set to true even if there is no more data to process + if r.ResponseBody.GetEndOfStream() { + return nil, nil + } + + // TODO: Handle response raw body in the WAF + return &extproc.ProcessingResponse{ + Response: &extproc.ProcessingResponse_ResponseBody{}, + }, nil + + case *extproc.ProcessingRequest_ResponseTrailers: + return &extproc.ProcessingResponse{ + Response: &extproc.ProcessingResponse_RequestTrailers{}, + }, nil + + default: + return nil, status.Errorf(codes.Unknown, "Unknown request type: %T", v) + } +} + +func ProcessRequestHeaders(ctx context.Context, req *extproc.ProcessingRequest_RequestHeaders, currentRequest *CurrentRequest) (*extproc.ProcessingResponse, error) { + log.Debug("external_processing: received request headers: %v\n", req.RequestHeaders) + + headers, envoyHeaders := separateEnvoyHeaders(req.RequestHeaders.GetHeaders().GetHeaders()) + + // Create args + host, scheme, path, method, err := verifyRequestHttp2RequestHeaders(envoyHeaders) + if err != nil { + return nil, err + } + + requestURI := scheme + "://" + host + path + parsedUrl, err := url.Parse(requestURI) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "Error parsing request URI: %v", err) + } + currentRequest.parsedUrl = parsedUrl + + // client ip set in the x-forwarded-for header (cf: https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#x-forwarded-for) + ipTags, _ := httpsec2.ClientIPTags(headers, true, currentRequest.remoteAddr) + + currentRequest.requestArgs = httpsec.MakeHandlerOperationArgs(headers, method, host, currentRequest.remoteAddr, parsedUrl) + headers = currentRequest.requestArgs.Headers // Replace headers with the ones from the args because it has been modified + + // Create span + currentRequest.span = createExternalProcessedSpan(ctx, headers, method, host, path, currentRequest.remoteAddr, ipTags, parsedUrl) + + // Run WAF on request data + currentRequest.op, currentRequest.blockAction, _ = httpsec.StartOperation(ctx, currentRequest.requestArgs) + + // Block handling: If triggered, we need to block the request, return an immediate response + if blockPtr := currentRequest.blockAction.Load(); blockPtr != nil { + response := doBlockRequest(currentRequest, blockPtr, headers) + return response, nil + } + + return &extproc.ProcessingResponse{ + Response: &extproc.ProcessingResponse_RequestHeaders{ + RequestHeaders: &extproc.HeadersResponse{ + Response: &extproc.CommonResponse{ + Status: extproc.CommonResponse_CONTINUE, + }, + }, + }, + }, nil +} + +// Verify the required HTTP2 headers are present +// Some mandatory headers need to be set. It can happen when it wasn't a real HTTP2 request sent by Envoy, +func verifyRequestHttp2RequestHeaders(headers map[string][]string) (string, string, string, string, error) { + // :authority, :scheme, :path, :method + + for _, header := range []string{":authority", ":scheme", ":path", ":method"} { + if _, ok := headers[header]; !ok { + return "", "", "", "", status.Errorf(codes.InvalidArgument, "Missing required header: %v", header) + } + } + + return headers[":authority"][0], headers[":scheme"][0], headers[":path"][0], headers[":method"][0], nil +} + +func verifyRequestHttp2ResponseHeaders(headers map[string][]string) (string, error) { + // :status + + if _, ok := headers[":status"]; !ok { + return "", status.Errorf(codes.InvalidArgument, "Missing required header: %v", ":status") + } + + return headers[":status"][0], nil +} + +func ProcessResponseHeaders(res *extproc.ProcessingRequest_ResponseHeaders, currentRequest *CurrentRequest) (*extproc.ProcessingResponse, error) { + log.Debug("external_processing: received response headers: %v\n", res.ResponseHeaders) + + headers, envoyHeaders := separateEnvoyHeaders(res.ResponseHeaders.GetHeaders().GetHeaders()) + + statusCodeStr, err := verifyRequestHttp2ResponseHeaders(envoyHeaders) + if err != nil { + return nil, err + } + + currentRequest.statusCode, err = strconv.Atoi(statusCodeStr) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "Error parsing response header status code: %v", err) + } + + args := httpsec.HandlerOperationRes{ + Headers: headers, + StatusCode: currentRequest.statusCode, + } + + currentRequest.op.Finish(args, currentRequest.span) + currentRequest.op = nil + + // Block handling: If triggered, we need to block the request, return an immediate response + if blockPtr := currentRequest.blockAction.Load(); blockPtr != nil { + return doBlockRequest(currentRequest, blockPtr, headers), nil + } + + httpsec2.SetResponseHeadersTags(currentRequest.span, headers) + + // Note: (cf. comment in the stream error handling) + // The end of stream bool value is not reliable + if res.ResponseHeaders.GetEndOfStream() { + return nil, nil + } + + return &extproc.ProcessingResponse{ + Response: &extproc.ProcessingResponse_ResponseHeaders{ + ResponseHeaders: &extproc.HeadersResponse{ + Response: &extproc.CommonResponse{ + Status: extproc.CommonResponse_CONTINUE, + }, + }, + }, + }, nil +} + +func createExternalProcessedSpan(ctx context.Context, headers map[string][]string, method string, host string, path string, remoteAddr string, ipTags map[string]string, parsedUrl *url.URL) tracer.Span { + userAgent := "" + if ua, ok := headers["User-Agent"]; ok { + userAgent = ua[0] + } + + span, _ := httptrace.StartHttpSpan( + ctx, + headers, + host, + method, + httptrace.UrlFromUrl(parsedUrl), + userAgent, + remoteAddr, + []ddtrace.StartSpanOption{ + func(cfg *ddtrace.StartSpanConfig) { + cfg.Tags[ext.ResourceName] = method + " " + path + cfg.Tags[ext.SpanKind] = ext.SpanKindServer + + // Add client IP tags + for k, v := range ipTags { + cfg.Tags[k] = v + } + }, + }..., + ) + + httpsec2.SetRequestHeadersTags(span, headers) + trace.SetAppsecStaticTags(span) + + return span +} + +// Separate normal headers of the initial request made by the client and the pseudo headers of HTTP/2 +// - Format the headers to be used by the tracer as a map[string][]string +// - Set header keys to be canonical +func separateEnvoyHeaders(receivedHeaders []*corev3.HeaderValue) (map[string][]string, map[string][]string) { + headers := make(map[string][]string) + pseudoHeadersHttp2 := make(map[string][]string) + for _, v := range receivedHeaders { + key := v.GetKey() + if key[0] == ':' { + pseudoHeadersHttp2[key] = []string{string(v.GetRawValue())} + } else { + headers[http.CanonicalHeaderKey(key)] = []string{string(v.GetRawValue())} + } + } + return headers, pseudoHeadersHttp2 +} + +func doBlockRequest(currentRequest *CurrentRequest, blockAction *actions.BlockHTTP, headers map[string][]string) *extproc.ProcessingResponse { + currentRequest.blocked = true + + var headerToSet map[string][]string + var body []byte + if blockAction.RedirectLocation != "" { + headerToSet, body = actions.HandleRedirectLocationString( + currentRequest.parsedUrl.Path, + blockAction.RedirectLocation, + blockAction.StatusCode, + currentRequest.requestArgs.Method, + currentRequest.requestArgs.Headers, + ) + } else { + headerToSet, body = blockAction.BlockingTemplate(headers) + } + + var headersMutation []*v3.HeaderValueOption + for k, v := range headerToSet { + headersMutation = append(headersMutation, &v3.HeaderValueOption{ + Header: &v3.HeaderValue{ + Key: k, + RawValue: []byte(strings.Join(v, ",")), + }, + }) + } + + httpsec2.SetResponseHeadersTags(currentRequest.span, headerToSet) + currentRequest.statusCode = blockAction.StatusCode + var int32StatusCode int32 = 0 + if currentRequest.statusCode > 0 && currentRequest.statusCode <= math.MaxInt32 { + int32StatusCode = int32(currentRequest.statusCode) + } + + return &extproc.ProcessingResponse{ + Response: &extproc.ProcessingResponse_ImmediateResponse{ + ImmediateResponse: &extproc.ImmediateResponse{ + Status: &v32.HttpStatus{ + Code: v32.StatusCode(int32StatusCode), + }, + Headers: &extproc.HeaderMutation{ + SetHeaders: headersMutation, + }, + Body: body, + GrpcStatus: &extproc.GrpcStatus{ + Status: 0, + }, + }, + }, + } +} + +func closeSpan(currentRequest *CurrentRequest) { + span := currentRequest.span + if span != nil { + // Finish the operation: it can be not finished when the request has been blocked or if an error occurred + // > The response hasn't been processed + if currentRequest.op != nil { + currentRequest.op.Finish(httpsec.HandlerOperationRes{}, span) + currentRequest.op = nil + } + + // Note: The status code could be 0 if an internal error occurred + statusCodeStr := strconv.Itoa(currentRequest.statusCode) + span.SetTag(ext.HTTPCode, statusCodeStr) + + span.Finish() + + log.Debug("external_processing: span closed with status code: %v\n", currentRequest.statusCode) + currentRequest.span = nil + } +} diff --git a/contrib/envoyproxy/envoy/envoy_test.go b/contrib/envoyproxy/envoy/envoy_test.go new file mode 100644 index 0000000000..e3f872f87f --- /dev/null +++ b/contrib/envoyproxy/envoy/envoy_test.go @@ -0,0 +1,569 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016 Datadog, Inc. + +// TODO: Blocking and Redirect action to test + +package envoy + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net" + "testing" + + extproc "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + typev3 "github.com/envoyproxy/go-control-plane/envoy/type/v3" + + ddgrpc "gopkg.in/DataDog/dd-trace-go.v1/contrib/google.golang.org/grpc" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/mocktracer" + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec" + + v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +func end2EndStreamRequest(t *testing.T, stream extproc.ExternalProcessor_ProcessClient, path string, method string, requestHeaders map[string]string, responseHeaders map[string]string, blockOnResponse bool) { + // First part: request + // 1- Send the headers + err := stream.Send(&extproc.ProcessingRequest{ + Request: &extproc.ProcessingRequest_RequestHeaders{ + RequestHeaders: &extproc.HttpHeaders{ + Headers: makeRequestHeaders(requestHeaders, method, path), + }, + }, + }) + require.NoError(t, err) + + res, err := stream.Recv() + require.NoError(t, err) + require.Equal(t, extproc.CommonResponse_CONTINUE, res.GetRequestHeaders().GetResponse().GetStatus()) + + // 2- Send the body + err = stream.Send(&extproc.ProcessingRequest{ + Request: &extproc.ProcessingRequest_RequestBody{ + RequestBody: &extproc.HttpBody{ + Body: []byte("body"), + }, + }, + }) + require.NoError(t, err) + + res, err = stream.Recv() + require.NoError(t, err) + require.Equal(t, extproc.CommonResponse_CONTINUE, res.GetRequestBody().GetResponse().GetStatus()) + + // 3- Send the trailers + err = stream.Send(&extproc.ProcessingRequest{ + Request: &extproc.ProcessingRequest_RequestTrailers{ + RequestTrailers: &extproc.HttpTrailers{ + Trailers: &v3.HeaderMap{ + Headers: []*v3.HeaderValue{ + {Key: "key", Value: "value"}, + }, + }, + }, + }, + }) + require.NoError(t, err) + + res, err = stream.Recv() + require.NoError(t, err) + require.NotNil(t, res.GetRequestTrailers()) + + // Second part: response + // 1- Send the response headers + err = stream.Send(&extproc.ProcessingRequest{ + Request: &extproc.ProcessingRequest_ResponseHeaders{ + ResponseHeaders: &extproc.HttpHeaders{ + Headers: makeResponseHeaders(responseHeaders, "200"), + }, + }, + }) + require.NoError(t, err) + + if blockOnResponse { + // Should have received an immediate response for blocking + // Let the test handle the response + return + } + + res, err = stream.Recv() + require.NoError(t, err) + require.Equal(t, extproc.CommonResponse_CONTINUE, res.GetResponseHeaders().GetResponse().GetStatus()) + + // 2- Send the response body + err = stream.Send(&extproc.ProcessingRequest{ + Request: &extproc.ProcessingRequest_ResponseBody{ + ResponseBody: &extproc.HttpBody{ + Body: []byte("body"), + EndOfStream: true, + }, + }, + }) + require.NoError(t, err) + + // The stream should now be closed + _, err = stream.Recv() + require.Equal(t, io.EOF, err) +} + +func checkForAppsecEvent(t *testing.T, finished []mocktracer.Span, expectedRuleIDs map[string]int) { + // The request should have the attack attempts + event := finished[len(finished)-1].Tag("_dd.appsec.json") + require.NotNil(t, event, "the _dd.appsec.json tag was not found") + + jsonText := event.(string) + type trigger struct { + Rule struct { + ID string `json:"id"` + } `json:"rule"` + } + var parsed struct { + Triggers []trigger `json:"triggers"` + } + err := json.Unmarshal([]byte(jsonText), &parsed) + require.NoError(t, err) + + histogram := map[string]uint8{} + for _, tr := range parsed.Triggers { + histogram[tr.Rule.ID]++ + } + + for ruleID, count := range expectedRuleIDs { + require.Equal(t, count, int(histogram[ruleID]), "rule %s has been triggered %d times but expected %d") + } + + require.Len(t, parsed.Triggers, len(expectedRuleIDs), "unexpected number of rules triggered") +} + +func TestAppSec(t *testing.T) { + appsec.Start() + defer appsec.Stop() + if !appsec.Enabled() { + t.Skip("appsec disabled") + } + + setup := func() (extproc.ExternalProcessorClient, mocktracer.Tracer, func()) { + rig, err := newEnvoyAppsecRig(false) + require.NoError(t, err) + + mt := mocktracer.Start() + + return rig.client, mt, func() { + rig.Close() + mt.Stop() + } + } + + t.Run("monitoring-event-on-request", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + end2EndStreamRequest(t, stream, "/", "GET", map[string]string{"User-Agent": "dd-test-scanner-log"}, map[string]string{}, false) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + checkForAppsecEvent(t, finished, map[string]int{"ua0-600-55x": 1}) + }) + + t.Run("blocking-event-on-request", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + err = stream.Send(&extproc.ProcessingRequest{ + Request: &extproc.ProcessingRequest_RequestHeaders{ + RequestHeaders: &extproc.HttpHeaders{ + Headers: makeRequestHeaders(map[string]string{"User-Agent": "dd-test-scanner-log-block"}, "GET", "/"), + }, + }, + }) + require.NoError(t, err) + + res, err := stream.Recv() + require.Equal(t, uint32(0), res.GetImmediateResponse().GetGrpcStatus().Status) + require.Equal(t, typev3.StatusCode(403), res.GetImmediateResponse().GetStatus().Code) + require.Equal(t, "Content-Type", res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().Key) + require.Equal(t, "application/json", string(res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().RawValue)) + require.NoError(t, err) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + checkForAppsecEvent(t, finished, map[string]int{"ua0-600-56x": 1}) + + // Check for tags + span := finished[0] + require.Equal(t, true, span.Tag("appsec.event")) + require.Equal(t, true, span.Tag("appsec.blocked")) + }) +} + +func TestBlockingWithUserRulesFile(t *testing.T) { + t.Setenv("DD_APPSEC_RULES", "../../../internal/appsec/testdata/user_rules.json") + appsec.Start() + defer appsec.Stop() + if !appsec.Enabled() { + t.Skip("appsec disabled") + } + + setup := func() (extproc.ExternalProcessorClient, mocktracer.Tracer, func()) { + rig, err := newEnvoyAppsecRig(false) + require.NoError(t, err) + + mt := mocktracer.Start() + + return rig.client, mt, func() { + rig.Close() + mt.Stop() + } + } + + t.Run("blocking-event-on-response", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + end2EndStreamRequest(t, stream, "/", "OPTION", map[string]string{"User-Agent": "dd-test-scanner-log-block"}, map[string]string{"User-Agent": "match-response-header"}, true) + + // Handle the immediate response + res, err := stream.Recv() + require.Equal(t, uint32(0), res.GetImmediateResponse().GetGrpcStatus().Status) + require.Equal(t, typev3.StatusCode(418), res.GetImmediateResponse().GetStatus().Code) // 418 because of the rule file + require.Equal(t, "Content-Type", res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().Key) + require.Equal(t, "application/json", string(res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().RawValue)) + require.NoError(t, err) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + checkForAppsecEvent(t, finished, map[string]int{"headers-003": 1}) + + // Check for tags + span := finished[0] + require.Equal(t, 1, span.Tag("_dd.appsec.enabled")) + require.Equal(t, true, span.Tag("appsec.event")) + require.Equal(t, true, span.Tag("appsec.blocked")) + }) + + t.Run("blocking-event-on-request-on-query", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + err = stream.Send(&extproc.ProcessingRequest{ + Request: &extproc.ProcessingRequest_RequestHeaders{ + RequestHeaders: &extproc.HttpHeaders{ + Headers: makeRequestHeaders(map[string]string{"User-Agent": "Mistake Not..."}, "GET", "/hello?match=match-request-query"), + }, + }, + }) + require.NoError(t, err) + + res, err := stream.Recv() + require.Equal(t, uint32(0), res.GetImmediateResponse().GetGrpcStatus().Status) + require.Equal(t, typev3.StatusCode(418), res.GetImmediateResponse().GetStatus().Code) + require.Equal(t, "Content-Type", res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().Key) + require.Equal(t, "application/json", string(res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().RawValue)) + require.NoError(t, err) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + checkForAppsecEvent(t, finished, map[string]int{"query-002": 1}) + + // Check for tags + span := finished[0] + require.Equal(t, true, span.Tag("appsec.event")) + require.Equal(t, true, span.Tag("appsec.blocked")) + }) + + t.Run("blocking-event-on-request-on-cookies", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + err = stream.Send(&extproc.ProcessingRequest{ + Request: &extproc.ProcessingRequest_RequestHeaders{ + RequestHeaders: &extproc.HttpHeaders{ + Headers: makeRequestHeaders(map[string]string{"Cookie": "foo=jdfoSDGFkivRG_234"}, "OPTIONS", "/"), + }, + }, + }) + require.NoError(t, err) + + res, err := stream.Recv() + require.Equal(t, uint32(0), res.GetImmediateResponse().GetGrpcStatus().Status) + require.Equal(t, typev3.StatusCode(418), res.GetImmediateResponse().GetStatus().Code) + require.Equal(t, "Content-Type", res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().Key) + require.Equal(t, "application/json", string(res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().RawValue)) + require.NoError(t, err) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + checkForAppsecEvent(t, finished, map[string]int{"tst-037-008": 1}) + + // Check for tags + span := finished[0] + require.Equal(t, true, span.Tag("appsec.event")) + require.Equal(t, true, span.Tag("appsec.blocked")) + }) +} + +func TestGeneratedSpan(t *testing.T) { + setup := func() (extproc.ExternalProcessorClient, mocktracer.Tracer, func()) { + rig, err := newEnvoyAppsecRig(false) + require.NoError(t, err) + + mt := mocktracer.Start() + + return rig.client, mt, func() { + rig.Close() + mt.Stop() + } + } + + t.Run("request-span", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + end2EndStreamRequest(t, stream, "/resource-span", "GET", map[string]string{"user-agent": "Mistake Not...", "test-key": "test-value"}, map[string]string{"response-test-key": "response-test-value"}, false) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + + // Check for tags + span := finished[0] + require.Equal(t, "http.request", span.OperationName()) + require.Equal(t, "https://datadoghq.com/resource-span", span.Tag("http.url")) + require.Equal(t, "GET", span.Tag("http.method")) + require.Equal(t, "datadoghq.com", span.Tag("http.host")) + require.Equal(t, "GET /resource-span", span.Tag("resource.name")) + require.Equal(t, "datadoghq.com", span.Tag("http.request.headers.host")) + require.Equal(t, "server", span.Tag("span.kind")) + require.Equal(t, "Mistake Not...", span.Tag("http.useragent")) + }) +} + +func TestXForwardedForHeaderClientIp(t *testing.T) { + t.Setenv("DD_APPSEC_RULES", "../../../internal/appsec/testdata/blocking.json") + appsec.Start() + defer appsec.Stop() + if !appsec.Enabled() { + t.Skip("appsec disabled") + } + + setup := func() (extproc.ExternalProcessorClient, mocktracer.Tracer, func()) { + rig, err := newEnvoyAppsecRig(false) + require.NoError(t, err) + + mt := mocktracer.Start() + + return rig.client, mt, func() { + rig.Close() + mt.Stop() + } + } + + t.Run("client-ip", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + end2EndStreamRequest(t, stream, "/", "OPTION", + map[string]string{"User-Agent": "Mistake not...", "X-Forwarded-For": "18.18.18.18"}, + map[string]string{"User-Agent": "match-response-header"}, + true) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + + // Check for tags + span := finished[0] + require.Equal(t, "18.18.18.18", span.Tag("http.client_ip")) + + // Appsec + require.Equal(t, 1, span.Tag("_dd.appsec.enabled")) + }) + + t.Run("blocking-client-ip", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + err = stream.Send(&extproc.ProcessingRequest{ + Request: &extproc.ProcessingRequest_RequestHeaders{ + RequestHeaders: &extproc.HttpHeaders{ + Headers: makeRequestHeaders(map[string]string{"User-Agent": "Mistake not...", "X-Forwarded-For": "1.2.3.4"}, "GET", "/"), + }, + }, + }) + require.NoError(t, err) + + // Handle the immediate response + res, err := stream.Recv() + require.Equal(t, uint32(0), res.GetImmediateResponse().GetGrpcStatus().Status) + require.Equal(t, typev3.StatusCode(403), res.GetImmediateResponse().GetStatus().Code) + require.Equal(t, "Content-Type", res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().Key) + require.Equal(t, "application/json", string(res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().RawValue)) + require.NoError(t, err) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + checkForAppsecEvent(t, finished, map[string]int{"blk-001-001": 1}) + + // Check for tags + span := finished[0] + require.Equal(t, "1.2.3.4", span.Tag("http.client_ip")) + require.Equal(t, 1, span.Tag("_dd.appsec.enabled")) + require.Equal(t, true, span.Tag("appsec.event")) + require.Equal(t, true, span.Tag("appsec.blocked")) + }) +} + +func newEnvoyAppsecRig(traceClient bool, interceptorOpts ...ddgrpc.Option) (*envoyAppsecRig, error) { + interceptorOpts = append([]ddgrpc.InterceptorOption{ddgrpc.WithServiceName("grpc")}, interceptorOpts...) + + server := grpc.NewServer( + grpc.StreamInterceptor(StreamServerInterceptor(interceptorOpts...)), + ) + + fixtureServer := new(envoyFixtureServer) + extproc.RegisterExternalProcessorServer(server, fixtureServer) + + li, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, err + } + _, port, _ := net.SplitHostPort(li.Addr().String()) + // start our test fixtureServer. + go server.Serve(li) + + opts := []grpc.DialOption{grpc.WithInsecure()} + if traceClient { + opts = append(opts, + grpc.WithStreamInterceptor(ddgrpc.StreamClientInterceptor(interceptorOpts...)), + ) + } + conn, err := grpc.Dial(li.Addr().String(), opts...) + if err != nil { + return nil, fmt.Errorf("error dialing: %s", err) + } + return &envoyAppsecRig{ + fixtureServer: fixtureServer, + listener: li, + port: port, + server: server, + conn: conn, + client: extproc.NewExternalProcessorClient(conn), + }, err +} + +// rig contains all servers and connections we'd need for a grpc integration test +type envoyAppsecRig struct { + fixtureServer *envoyFixtureServer + server *grpc.Server + port string + listener net.Listener + conn *grpc.ClientConn + client extproc.ExternalProcessorClient +} + +func (r *envoyAppsecRig) Close() { + r.server.Stop() + r.conn.Close() +} + +type envoyFixtureServer struct { + extproc.ExternalProcessorServer +} + +// Helper functions + +// Construct request headers +func makeRequestHeaders(headers map[string]string, method string, path string) *v3.HeaderMap { + h := &v3.HeaderMap{} + for k, v := range headers { + h.Headers = append(h.Headers, &v3.HeaderValue{Key: k, RawValue: []byte(v)}) + } + + h.Headers = append(h.Headers, + &v3.HeaderValue{Key: ":method", RawValue: []byte(method)}, + &v3.HeaderValue{Key: ":path", RawValue: []byte(path)}, + &v3.HeaderValue{Key: ":scheme", RawValue: []byte("https")}, + &v3.HeaderValue{Key: ":authority", RawValue: []byte("datadoghq.com")}, + ) + + return h +} + +func makeResponseHeaders(headers map[string]string, status string) *v3.HeaderMap { + h := &v3.HeaderMap{} + for k, v := range headers { + h.Headers = append(h.Headers, &v3.HeaderValue{Key: k, RawValue: []byte(v)}) + } + + h.Headers = append(h.Headers, &v3.HeaderValue{Key: ":status", RawValue: []byte(status)}) + + return h +} diff --git a/contrib/internal/httptrace/httptrace.go b/contrib/internal/httptrace/httptrace.go index 92069e03d8..0017159f62 100644 --- a/contrib/internal/httptrace/httptrace.go +++ b/contrib/internal/httptrace/httptrace.go @@ -11,6 +11,7 @@ import ( "context" "fmt" "net/http" + "net/url" "strconv" "strings" @@ -27,14 +28,29 @@ var ( ) // StartRequestSpan starts an HTTP request span with the standard list of HTTP request span tags (http.method, http.url, -// http.useragent). Any further span start option can be added with opts. +// http.useragent) with a http.Request object. Any further span start option can be added with opts. func StartRequestSpan(r *http.Request, opts ...ddtrace.StartSpanOption) (tracer.Span, context.Context) { + return StartHttpSpan( + r.Context(), + r.Header, + r.Host, + r.Method, + urlFromRequest(r), + r.UserAgent(), + r.RemoteAddr, + opts..., + ) +} + +// StartHttpSpan starts an HTTP request span with the standard list of HTTP request span tags (http.method, http.url, +// http.useragent). Any further span start option can be added with opts. +func StartHttpSpan(ctx context.Context, headers map[string][]string, host string, method string, url string, userAgent string, remoteAddr string, opts ...ddtrace.StartSpanOption) (tracer.Span, context.Context) { // Append our span options before the given ones so that the caller can "overwrite" them. // TODO(): rework span start option handling (https://github.com/DataDog/dd-trace-go/issues/1352) var ipTags map[string]string if cfg.traceClientIP { - ipTags, _ = httpsec.ClientIPTags(r.Header, true, r.RemoteAddr) + ipTags, _ = httpsec.ClientIPTags(headers, true, remoteAddr) } nopts := make([]ddtrace.StartSpanOption, 0, len(opts)+1+len(ipTags)) nopts = append(nopts, @@ -43,22 +59,23 @@ func StartRequestSpan(r *http.Request, opts ...ddtrace.StartSpanOption) (tracer. cfg.Tags = make(map[string]interface{}) } cfg.Tags[ext.SpanType] = ext.SpanTypeWeb - cfg.Tags[ext.HTTPMethod] = r.Method - cfg.Tags[ext.HTTPURL] = urlFromRequest(r) - cfg.Tags[ext.HTTPUserAgent] = r.UserAgent() + cfg.Tags[ext.HTTPMethod] = method + cfg.Tags[ext.HTTPURL] = url + cfg.Tags[ext.HTTPUserAgent] = userAgent cfg.Tags["_dd.measured"] = 1 - if r.Host != "" { - cfg.Tags["http.host"] = r.Host + if host != "" { + cfg.Tags["http.host"] = host } - if spanctx, err := tracer.Extract(tracer.HTTPHeadersCarrier(r.Header)); err == nil { + if spanctx, err := tracer.Extract(tracer.HTTPHeadersCarrier(headers)); err == nil { cfg.Parent = spanctx } + for k, v := range ipTags { cfg.Tags[k] = v } }) nopts = append(nopts, opts...) - return tracer.StartSpanFromContext(r.Context(), namingschema.OpName(namingschema.HTTPServer), nopts...) + return tracer.StartSpanFromContext(ctx, namingschema.OpName(namingschema.HTTPServer), nopts...) } // FinishRequestSpan finishes the given HTTP request span and sets the expected response-related tags such as the status @@ -97,27 +114,54 @@ func urlFromRequest(r *http.Request) string { // "For most requests, fields other than Path and RawQuery will be // empty. (See RFC 7230, Section 5.3)" // This is why we don't rely on url.URL.String(), url.URL.Host, url.URL.Scheme, etc... - var url string - path := r.URL.EscapedPath() scheme := "http" if r.TLS != nil { scheme = "https" } - if r.Host != "" { - url = strings.Join([]string{scheme, "://", r.Host, path}, "") + + return urlFromArgs( + r.URL.EscapedPath(), + scheme, + r.Host, + r.URL.RawQuery, + r.URL.EscapedFragment(), + ) +} + +// UrlFromUrl returns the full URL from a URL object. If query params are collected, they are obfuscated granted +// obfuscation is not disabled by the user (through DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP) +// See https://docs.datadoghq.com/tracing/configure_data_security#redacting-the-query-in-the-url for more information. +func UrlFromUrl(u *url.URL) string { + scheme := "http" + if u.Scheme != "" { + scheme = u.Scheme + } + return urlFromArgs( + u.EscapedPath(), + scheme, + u.Host, + u.RawQuery, + u.EscapedFragment(), + ) +} + +func urlFromArgs(escapedPath string, scheme string, host string, rawQuery string, escapedFragment string) string { + var url string + if host != "" { + url = strings.Join([]string{scheme, "://", host, escapedPath}, "") } else { - url = path + url = escapedPath } // Collect the query string if we are allowed to report it and obfuscate it if possible/allowed - if cfg.queryString && r.URL.RawQuery != "" { - query := r.URL.RawQuery + if cfg.queryString && rawQuery != "" { + query := rawQuery if cfg.queryStringRegexp != nil { query = cfg.queryStringRegexp.ReplaceAllLiteralString(query, "") } url = strings.Join([]string{url, query}, "?") } - if frag := r.URL.EscapedFragment(); frag != "" { - url = strings.Join([]string{url, frag}, "#") + if escapedFragment != "" { + url = strings.Join([]string{url, escapedFragment}, "#") } return url } diff --git a/go.mod b/go.mod index 41a876a7b7..02c589d98e 100644 --- a/go.mod +++ b/go.mod @@ -39,6 +39,7 @@ require ( github.com/elastic/go-elasticsearch/v8 v8.4.0 github.com/emicklei/go-restful v2.16.0+incompatible github.com/emicklei/go-restful/v3 v3.11.0 + github.com/envoyproxy/go-control-plane v0.13.0 github.com/garyburd/redigo v1.6.4 github.com/gin-gonic/gin v1.9.1 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 @@ -102,7 +103,7 @@ require ( golang.org/x/time v0.6.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 google.golang.org/api v0.192.0 - google.golang.org/grpc v1.64.1 + google.golang.org/grpc v1.65.0 google.golang.org/protobuf v1.34.2 gopkg.in/jinzhu/gorm.v1 v1.9.2 gopkg.in/olivere/elastic.v3 v3.0.75 @@ -154,6 +155,7 @@ require ( github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.0 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect + github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -162,6 +164,7 @@ require ( github.com/eapache/queue v1.1.0 // indirect github.com/ebitengine/purego v0.6.0-alpha.5 // indirect github.com/elastic/elastic-transport-go/v8 v8.1.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect github.com/fatih/color v1.16.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect @@ -240,6 +243,7 @@ require ( github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect diff --git a/go.sum b/go.sum index 19cabce2db..efb2972a1a 100644 --- a/go.sum +++ b/go.sum @@ -893,6 +893,8 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b h1:ga8SEFjZ60pxLcmhnThWgvH2wg8376yUJmPhEH4H3kw= +github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= @@ -1121,9 +1123,13 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= +github.com/envoyproxy/go-control-plane v0.13.0 h1:HzkeUz1Knt+3bK+8LG1bxOO/jzWZmdxpwC51i202les= +github.com/envoyproxy/go-control-plane v0.13.0/go.mod h1:GRaKG3dwvFoTg4nj7aXdZnvMg4d7nvT/wl9WgVXn3Q8= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= +github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= +github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -1886,6 +1892,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -3056,8 +3064,8 @@ google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCD google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= -google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= -google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= diff --git a/internal/apps/go.mod b/internal/apps/go.mod index 8b2398cb8b..be677a5009 100644 --- a/internal/apps/go.mod +++ b/internal/apps/go.mod @@ -55,7 +55,7 @@ require ( golang.org/x/text v0.17.0 // indirect golang.org/x/tools v0.24.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect - google.golang.org/grpc v1.64.1 // indirect + google.golang.org/grpc v1.65.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/internal/apps/go.sum b/internal/apps/go.sum index 133c81441d..23aee43d13 100644 --- a/internal/apps/go.sum +++ b/internal/apps/go.sum @@ -280,8 +280,8 @@ google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAs google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf h1:liao9UHurZLtiEwBgT9LMOnKYsHze6eA6w1KQCMVN2Q= google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= -google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= -google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= diff --git a/internal/appsec/emitter/httpsec/http.go b/internal/appsec/emitter/httpsec/http.go index 07f6dd9ba8..e19552853a 100644 --- a/internal/appsec/emitter/httpsec/http.go +++ b/internal/appsec/emitter/httpsec/http.go @@ -12,9 +12,12 @@ package httpsec import ( "context" + "strings" + // Blank import needed to use embed for the default blocked response payloads _ "embed" "net/http" + "net/url" "sync" "sync/atomic" @@ -189,3 +192,47 @@ func WrapHandler(handler http.Handler, span ddtrace.Span, pathParams map[string] handler.ServeHTTP(tw, tr) }) } + +// MakeHandlerOperationArgs creates the HandlerOperationArgs value. +func MakeHandlerOperationArgs(headers map[string][]string, method string, host string, remoteAddr string, url *url.URL) HandlerOperationArgs { + cookies := filterCookiesFromHeaders(headers) + + args := HandlerOperationArgs{ + Method: method, + RequestURI: url.RequestURI(), + Host: host, + RemoteAddr: remoteAddr, + Headers: headers, + Cookies: cookies, + QueryParams: url.Query(), + PathParams: map[string]string{}, + } + + args.Headers["host"] = []string{host} + return args +} + +// Separate the cookies from the headers, return the parsed cookies and remove in place the cookies from the headers. +// Headers used for `server.request.headers.no_cookies` and `server.response.headers.no_cookies` addresses for the WAF +// Cookies are used for the `server.request.cookies` address +func filterCookiesFromHeaders(headers http.Header) map[string][]string { + cookieHeader, ok := headers["Cookie"] + if !ok { + return make(http.Header) + } + + delete(headers, "Cookie") + + cookies := make(map[string][]string, len(cookieHeader)) + for _, c := range cookieHeader { + parts := strings.Split(c, ";") + for _, part := range parts { + cookie := strings.Split(part, "=") + if len(cookie) == 2 { + cookies[cookie[0]] = append(cookies[cookie[0]], cookie[1]) + } + } + } + + return cookies +} diff --git a/internal/appsec/emitter/waf/actions/block.go b/internal/appsec/emitter/waf/actions/block.go index ae802b60bd..40418f4410 100644 --- a/internal/appsec/emitter/waf/actions/block.go +++ b/internal/appsec/emitter/waf/actions/block.go @@ -69,6 +69,11 @@ type ( // BlockHTTP are actions that interact with an HTTP request flow BlockHTTP struct { http.Handler + + StatusCode int `mapstructure:"status_code"` + RedirectLocation string + // BlockingTemplate is a function that returns the headers to be added and body to be written to the response + BlockingTemplate func(headers map[string][]string) (map[string][]string, []byte) } ) @@ -125,7 +130,11 @@ func NewBlockAction(params map[string]any) []Action { } func newHTTPBlockRequestAction(status int, template string) *BlockHTTP { - return &BlockHTTP{Handler: newBlockHandler(status, template)} + return &BlockHTTP{ + Handler: newBlockHandler(status, template), + BlockingTemplate: newManualBlockHandler(template), + StatusCode: status, + } } // newBlockHandler creates, initializes and returns a new BlockRequestAction @@ -139,19 +148,50 @@ func newBlockHandler(status int, template string) http.Handler { return htmlHandler default: return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - h := jsonHandler - hdr := r.Header.Get("Accept") - htmlIdx := strings.Index(hdr, "text/html") - jsonIdx := strings.Index(hdr, "application/json") - // Switch to html handler if text/html comes before application/json in the Accept header - if htmlIdx != -1 && (jsonIdx == -1 || htmlIdx < jsonIdx) { - h = htmlHandler - } - h.ServeHTTP(w, r) + h := findCorrectTemplate(jsonHandler, htmlHandler, r.Header.Get("Accept")) + h.(http.Handler).ServeHTTP(w, r) }) } } +func newManualBlockHandler(template string) func(headers map[string][]string) (map[string][]string, []byte) { + htmlHandler := newManualBlockDataHandler("text/html", blockedTemplateHTML) + jsonHandler := newManualBlockDataHandler("application/json", blockedTemplateJSON) + switch template { + case "json": + return jsonHandler + case "html": + return htmlHandler + default: + return func(headers map[string][]string) (map[string][]string, []byte) { + acceptHeader := "" + if hdr, ok := headers["Accept"]; ok && len(hdr) > 0 { + acceptHeader = hdr[0] + } + h := findCorrectTemplate(jsonHandler, htmlHandler, acceptHeader) + return h.(func(headers map[string][]string) (map[string][]string, []byte))(headers) + } + } +} + +func findCorrectTemplate(jsonHandler interface{}, htmlHandler interface{}, acceptHeader string) interface{} { + h := jsonHandler + hdr := acceptHeader + htmlIdx := strings.Index(hdr, "text/html") + jsonIdx := strings.Index(hdr, "application/json") + // Switch to html handler if text/html comes before application/json in the Accept header + if htmlIdx != -1 && (jsonIdx == -1 || htmlIdx < jsonIdx) { + h = htmlHandler + } + return h +} + +func newManualBlockDataHandler(ct string, template []byte) func(headers map[string][]string) (map[string][]string, []byte) { + return func(headers map[string][]string) (map[string][]string, []byte) { + return map[string][]string{"Content-Type": {ct}}, template + } +} + func newBlockRequestHandler(status int, ct string, payload []byte) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", ct) diff --git a/internal/appsec/emitter/waf/actions/http_redirect.go b/internal/appsec/emitter/waf/actions/http_redirect.go index 3cdca4c818..25e209d790 100644 --- a/internal/appsec/emitter/waf/actions/http_redirect.go +++ b/internal/appsec/emitter/waf/actions/http_redirect.go @@ -7,10 +7,14 @@ package actions import ( "net/http" + "path" + "strings" "github.com/mitchellh/mapstructure" "gopkg.in/DataDog/dd-trace-go.v1/internal/log" + + urlpkg "net/url" ) // redirectActionParams are the dynamic parameters to be provided to a "redirect_request" @@ -38,9 +42,17 @@ func newRedirectRequestAction(status int, loc string) *BlockHTTP { // If location is not set we fall back on a default block action if loc == "" { - return &BlockHTTP{Handler: newBlockHandler(http.StatusForbidden, string(blockedTemplateJSON))} + return &BlockHTTP{ + Handler: newBlockHandler(http.StatusForbidden, string(blockedTemplateJSON)), + StatusCode: status, + BlockingTemplate: newManualBlockHandler("json"), + } + } + return &BlockHTTP{ + Handler: http.RedirectHandler(loc, status), + StatusCode: status, + RedirectLocation: loc, } - return &BlockHTTP{Handler: http.RedirectHandler(loc, status)} } // NewRedirectAction creates an action for the "redirect_request" action type @@ -52,3 +64,75 @@ func NewRedirectAction(params map[string]any) []Action { } return []Action{newRedirectRequestAction(p.StatusCode, p.Location)} } + +// HandleRedirectLocationString returns the headers and body to be written to the response when a redirect is needed +// Vendored from net/http/server.go +func HandleRedirectLocationString(oldpath string, url string, statusCode int, method string, h map[string][]string) (map[string][]string, []byte) { + if u, err := urlpkg.Parse(url); err == nil { + // If url was relative, make its path absolute by + // combining with request path. + // The client would probably do this for us, + // but doing it ourselves is more reliable. + // See RFC 7231, section 7.1.2 + if u.Scheme == "" && u.Host == "" { + if oldpath == "" { // should not happen, but avoid a crash if it does + oldpath = "/" + } + + // no leading http://server + if url == "" || url[0] != '/' { + // make relative path absolute + olddir, _ := path.Split(oldpath) + url = olddir + url + } + + var query string + if i := strings.Index(url, "?"); i != -1 { + url, query = url[:i], url[i:] + } + + // clean up but preserve trailing slash + trailing := strings.HasSuffix(url, "/") + url = path.Clean(url) + if trailing && !strings.HasSuffix(url, "/") { + url += "/" + } + url += query + } + } + + // RFC 7231 notes that a short HTML body is usually included in + // the response because older user agents may not understand 301/307. + // Do it only if the request didn't already have a Content-Type header. + _, hadCT := h["content-type"] + newHeaders := make(map[string][]string, 2) + + newHeaders["location"] = []string{url} + if !hadCT && (method == "GET" || method == "HEAD") { + newHeaders["content-length"] = []string{"text/html; charset=utf-8"} + } + + // Shouldn't send the body for POST or HEAD; that leaves GET. + var body []byte + if !hadCT && method == "GET" { + body = []byte("" + http.StatusText(statusCode) + ".\n") + } + + return newHeaders, body +} + +// Vendored from net/http/server.go +var htmlReplacer = strings.NewReplacer( + "&", "&", + "<", "<", + ">", ">", + // """ is shorter than """. + `"`, """, + // "'" is shorter than "'" and apos was not in HTML until HTML5. + "'", "'", +) + +// htmlEscape escapes special characters like "<" to become "<". +func htmlEscape(s string) string { + return htmlReplacer.Replace(s) +} diff --git a/internal/appsec/listener/httpsec/http.go b/internal/appsec/listener/httpsec/http.go index 08b9e853dd..804d115fcd 100644 --- a/internal/appsec/listener/httpsec/http.go +++ b/internal/appsec/listener/httpsec/http.go @@ -59,7 +59,7 @@ func (feature *Feature) OnRequest(op *httpsec.HandlerOperation, args httpsec.Han headers := headersRemoveCookies(args.Headers) headers["host"] = []string{args.Host} - setRequestHeadersTags(op, headers) + SetRequestHeadersTags(op, headers) op.Run(op, addresses.NewAddressesBuilder(). @@ -76,7 +76,7 @@ func (feature *Feature) OnRequest(op *httpsec.HandlerOperation, args httpsec.Han func (feature *Feature) OnResponse(op *httpsec.HandlerOperation, resp httpsec.HandlerOperationRes) { headers := headersRemoveCookies(resp.Headers) - setResponseHeadersTags(op, headers) + SetResponseHeadersTags(op, headers) builder := addresses.NewAddressesBuilder(). WithResponseHeadersNoCookies(headers). diff --git a/internal/appsec/listener/httpsec/request.go b/internal/appsec/listener/httpsec/request.go index abd3983183..cb181be81e 100644 --- a/internal/appsec/listener/httpsec/request.go +++ b/internal/appsec/listener/httpsec/request.go @@ -149,13 +149,13 @@ func readMonitoredClientIPHeadersConfig() { } } -// setRequestHeadersTags sets the AppSec-specific request headers span tags. -func setRequestHeadersTags(span trace.TagSetter, headers map[string][]string) { +// SetRequestHeadersTags sets the AppSec-specific request headers span tags. +func SetRequestHeadersTags(span trace.TagSetter, headers map[string][]string) { setHeadersTags(span, "http.request.headers.", headers) } -// setResponseHeadersTags sets the AppSec-specific response headers span tags. -func setResponseHeadersTags(span trace.TagSetter, headers map[string][]string) { +// SetResponseHeadersTags sets the AppSec-specific response headers span tags. +func SetResponseHeadersTags(span trace.TagSetter, headers map[string][]string) { setHeadersTags(span, "http.response.headers.", headers) } diff --git a/internal/appsec/listener/httpsec/request_test.go b/internal/appsec/listener/httpsec/request_test.go index 38052cbb96..badac1891c 100644 --- a/internal/appsec/listener/httpsec/request_test.go +++ b/internal/appsec/listener/httpsec/request_test.go @@ -215,8 +215,8 @@ func TestTags(t *testing.T) { return } require.NoError(t, err) - setRequestHeadersTags(&span, reqHeadersCase.headers) - setResponseHeadersTags(&span, respHeadersCase.headers) + SetRequestHeadersTags(&span, reqHeadersCase.headers) + SetResponseHeadersTags(&span, respHeadersCase.headers) if eventCase.events != nil { require.Subset(t, span.Tags, map[string]interface{}{ diff --git a/internal/appsec/listener/trace/trace.go b/internal/appsec/listener/trace/trace.go index 45fb28e99f..709ed31ecf 100644 --- a/internal/appsec/listener/trace/trace.go +++ b/internal/appsec/listener/trace/trace.go @@ -6,6 +6,7 @@ package trace import ( + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/config" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/emitter/trace" @@ -19,6 +20,12 @@ var staticAppsecTags = map[string]any{ "_dd.runtime_family": "go", } +func SetAppsecStaticTags(span ddtrace.Span) { + for key, value := range staticAppsecTags { + span.SetTag(key, value) + } +} + type AppsecSpanTransport struct{} func (*AppsecSpanTransport) String() string { diff --git a/internal/appsec/testdata/user_rules.json b/internal/appsec/testdata/user_rules.json index 6acb14089e..a13a0d67ed 100644 --- a/internal/appsec/testdata/user_rules.json +++ b/internal/appsec/testdata/user_rules.json @@ -53,6 +53,33 @@ "block" ] }, + { + "id": "tst-037-008", + "name": "Test block on cookies", + "tags": { + "type": "lfi", + "crs_id": "000008", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + } + ], + "regex": "jdfoSDGFkivRG_234" + }, + "operator": "match_regex" + } + ], + "transformers": [], + "on_match": [ + "block" + ] + }, + { "id": "headers-003", "name": "query match", diff --git a/internal/exectracetest/go.mod b/internal/exectracetest/go.mod index 18bc552f29..9031c4497f 100644 --- a/internal/exectracetest/go.mod +++ b/internal/exectracetest/go.mod @@ -72,7 +72,7 @@ require ( golang.org/x/tools v0.24.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect - google.golang.org/grpc v1.64.1 // indirect + google.golang.org/grpc v1.65.0 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/internal/exectracetest/go.sum b/internal/exectracetest/go.sum index 5e9a403a39..bcd7a0e887 100644 --- a/internal/exectracetest/go.sum +++ b/internal/exectracetest/go.sum @@ -288,8 +288,8 @@ google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAs google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf h1:liao9UHurZLtiEwBgT9LMOnKYsHze6eA6w1KQCMVN2Q= google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= -google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= -google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=