Merge pull request #1582 from ministryofjustice/MLPAB-2607-add-schedu…

…ler-xray

MLPAB-2607: Add xray to schedule runner lambda

Makefile

@@ -176,3 +176,6 @@ delete-lpa-index: ##@opensearch deletes the lpa index
 
 add-scheduled-events:
 	go run ./scripts/add-scheduled-items.go
+
+run-schedule-runner: ##@scheduler invokes the schedule-runner lambda
+	curl "http://localhost:9002/2015-03-31/functions/function/invocations"

cmd/schedule-runner/main.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/aws/aws-lambda-go/lambda"
+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/ministryofjustice/opg-modernising-lpa/internal/donor"
 	"github.com/ministryofjustice/opg-modernising-lpa/internal/dynamo"
@@ -19,29 +20,36 @@ import (
 	"github.com/ministryofjustice/opg-modernising-lpa/internal/scheduled"
 	"github.com/ministryofjustice/opg-modernising-lpa/internal/search"
 	"github.com/ministryofjustice/opg-modernising-lpa/internal/secrets"
+	"github.com/ministryofjustice/opg-modernising-lpa/internal/telemetry"
+	"go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda"
+	"go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda/xrayconfig"
+	"go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-sdk-go-v2/otelaws"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"google.golang.org/appengine/log"
+)
+
+var (
+	awsBaseURL            = os.Getenv("AWS_BASE_URL")
+	eventBusName          = cmp.Or(os.Getenv("EVENT_BUS_NAME"), "default")
+	notifyBaseURL         = os.Getenv("GOVUK_NOTIFY_BASE_URL")
+	notifyIsProduction    = os.Getenv("GOVUK_NOTIFY_IS_PRODUCTION") == "1"
+	searchEndpoint        = os.Getenv("SEARCH_ENDPOINT")
+	searchIndexName       = cmp.Or(os.Getenv("SEARCH_INDEX_NAME"), "lpas")
+	searchIndexingEnabled = os.Getenv("SEARCH_INDEXING_DISABLED") != "1"
+	tableName             = os.Getenv("LPAS_TABLE")
+	xrayEnabled           = os.Getenv("XRAY_ENABLED") == "1"
+
+	httpClient *http.Client
+	cfg        aws.Config
+	logHandler slog.Handler
 )
 
 func handleRunSchedule(ctx context.Context) error {
-	var (
-		eventBusName          = cmp.Or(os.Getenv("EVENT_BUS_NAME"), "default")
-		notifyBaseURL         = os.Getenv("GOVUK_NOTIFY_BASE_URL")
-		notifyIsProduction    = os.Getenv("GOVUK_NOTIFY_IS_PRODUCTION") == "1"
-		searchEndpoint        = os.Getenv("SEARCH_ENDPOINT")
-		searchIndexName       = cmp.Or(os.Getenv("SEARCH_INDEX_NAME"), "lpas")
-		searchIndexingEnabled = os.Getenv("SEARCH_INDEXING_DISABLED") != "1"
-		tableName             = os.Getenv("LPAS_TABLE")
-	)
-
-	logger := slog.New(slog.NewJSONHandler(os.Stdout, nil).
+	logger := slog.New(logHandler.
 		WithAttrs([]slog.Attr{
 			slog.String("service_name", "opg-modernising-lpa/schedule-runner"),
 		}))
 
-	cfg, err := config.LoadDefaultConfig(ctx)
-	if err != nil {
-		return fmt.Errorf("failed to load default config: %w", err)
-	}
-
 	secretsClient, err := secrets.NewClient(cfg, time.Hour)
 	if err != nil {
 		return err
@@ -57,7 +65,7 @@ func handleRunSchedule(ctx context.Context) error {
 		return err
 	}
 
-	notifyClient, err := notify.New(logger, notifyIsProduction, notifyBaseURL, notifyApiKey, http.DefaultClient, event.NewClient(cfg, eventBusName), bundle)
+	notifyClient, err := notify.New(logger, notifyIsProduction, notifyBaseURL, notifyApiKey, httpClient, event.NewClient(cfg, eventBusName), bundle)
 	if err != nil {
 		return err
 	}
@@ -88,5 +96,56 @@ func handleRunSchedule(ctx context.Context) error {
 }
 
 func main() {
-	lambda.Start(handleRunSchedule)
+	ctx := context.Background()
+
+	var err error
+	cfg, err = config.LoadDefaultConfig(ctx)
+	if err != nil {
+		log.Errorf(ctx, "failed to load default config: %v", err)
+		return
+	}
+
+	httpClient = &http.Client{Timeout: time.Second * 30}
+
+	if len(awsBaseURL) > 0 {
+		cfg.BaseEndpoint = aws.String(awsBaseURL)
+	}
+
+	if xrayEnabled {
+		tp, err := telemetry.SetupLambda(ctx)
+		if err != nil {
+			fmt.Printf("error creating tracer provider: %v", err)
+		}
+
+		otelaws.AppendMiddlewares(&cfg.APIOptions)
+		httpClient.Transport = otelhttp.NewTransport(httpClient.Transport)
+
+		defer func(ctx context.Context) {
+			if err := tp.Shutdown(ctx); err != nil {
+				fmt.Printf("error shutting down tracer provider: %v", err)
+			}
+		}(ctx)
+
+		logHandler = telemetry.NewSlogHandler(slog.
+			NewJSONHandler(os.Stdout, &slog.HandlerOptions{
+				ReplaceAttr: func(_ []string, a slog.Attr) slog.Attr {
+					switch a.Value.Kind() {
+					case slog.KindAny:
+						switch v := a.Value.Any().(type) {
+						case *http.Request:
+							return slog.Group(a.Key,
+								slog.String("method", v.Method),
+								slog.String("uri", v.URL.String()))
+						}
+					}
+
+					return a
+				},
+			}))
+
+		lambda.Start(otellambda.InstrumentHandler(handleRunSchedule, xrayconfig.WithRecommendedOptions(tp)...))
+	} else {
+		logHandler = slog.NewJSONHandler(os.Stdout, nil)
+		lambda.Start(handleRunSchedule)
+	}
 }

docker/adot-collector/config/config.yaml

@@ -0,0 +1,23 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: "localhost:4317"
+      http:
+        endpoint: "localhost:4318"
+
+exporters:
+  logging:
+  awsxray:
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [awsxray]
+    metrics:
+      receivers: [otlp]
+      exporters: [logging]
+  telemetry:
+    metrics:
+      address: localhost:8888

docker/docker-compose.dev.yml

@@ -58,7 +58,7 @@ services:
       platforms:
         - "linux/amd64"
         - "linux/arm64"
-    container_name: schedule-runner
+    container_name: schedule-runner-lambda
     environment:
       - AWS_ACCESS_KEY_ID=fakeKeyId
       - AWS_BASE_URL=http://localstack:4566
@@ -68,6 +68,7 @@ services:
       - GOVUK_NOTIFY_IS_PRODUCTION=0
       - GOVUK_NOTIFY_BASE_URL=http://mock-notify:8080
       - SEARCH_ENDPOINT=http://my-domain.eu-west-1.opensearch.localhost.localstack.cloud:4566
+      - XRAY_ENABLED=1
     ports:
       - "9002:8080"
     entrypoint: aws-lambda-rie /var/task/schedule-runner

docker/localstack/localstack-init.sh

@@ -79,7 +79,8 @@ awslocal lambda create-function \
   --runtime provided.al2023 \
   --role arn:aws:iam::000000000000:role/lambda-role \
   --zip-file fileb:///etc/schedule-runner.zip \
-  --timeout 900
+  --timeout 900 \
+  --tracing-config Mode=Active
 
 awslocal lambda wait function-active-v2 --region eu-west-1 --function-name schedule-runner
 

docker/schedule-runner/Dockerfile

@@ -15,9 +15,13 @@ FROM public.ecr.aws/lambda/provided:al2023.2024.10.14.12 AS dev
 WORKDIR /app
 
 COPY --from=build /app/schedule-runner /var/task/schedule-runner
-COPY --link lang /var/task/lang
+COPY --link lang ./lang
+COPY --link ./docker/adot-collector/ /opt
 COPY --link docker/aws-lambda-rie ./aws-lambda-rie
 
+ENV AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-handler
+ENV OPENTELEMETRY_COLLECTOR_CONFIG_FILE="/opt/config/config.yaml"
+
 ENTRYPOINT ["./schedule-runner"]
 
 FROM public.ecr.aws/lambda/provided:al2023.2024.10.14.12 AS production
@@ -30,5 +34,11 @@ RUN chmod +x "/app/install_lambda_insights.sh" \
 
 COPY --from=build /app/schedule-runner ./schedule-runner
 COPY --link lang ./lang
+COPY --link ./docker/adot-collector/ /opt
+
+RUN chmod 755 /opt/config/config.yaml
+
+ENV AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-handler
+ENV OPENTELEMETRY_COLLECTOR_CONFIG_FILE="/opt/config/config.yaml"
 
 ENTRYPOINT ["./schedule-runner"]

docs/runbooks/adding-otel-collector-to-lambdas.md

@@ -0,0 +1,47 @@
+# Setting up the AWS Distro for OpenTelemetry Collector (ADOT Collector) for Go Lambda Function Images
+
+## Introduction
+
+The AWS Distro for OpenTelemetry Collector (ADOT Collector) is a distribution of the OpenTelemetry Collector that is optimized for use with AWS services. The ADOT Collector can be used to collect traces from AWS Lambda functions and send them to AWS X-Ray, AWS CloudWatch, or other AWS services.
+
+Instructions for setting up the ADOT Collector for Go Lambda functions using lambda layers are provided in the ADOT Collector documentation: <https://aws-otel.github.io/docs/getting-started/lambda/lambda-go#lambda-layer>, including code examples for using the ADOT Collector with Go Lambda functions.
+
+Layers aren't supported for Image based lambdas, so the ADOT Collector binary needs to be included in the Lambda image. This document provides instructions for including the ADOT Collector binary in a Go Lambda image.
+
+## Downloading the ADOT Collector binary
+
+This method is adapted from <https://github.com/nvsecurity/aws-otel-docker-lambda-layers> and <https://aws.amazon.com/blogs/compute/working-with-lambda-layers-and-extensions-in-container-images/>
+
+The following instructions fetch the URL for the Lambda layer from the official AWS Lambda Layer ARN: <https://aws-otel.github.io/docs/getting-started/lambda/lambda-go>
+
+The URL is used to download the layer zip which is then unzipped to retrieve the ADOT Collector binary.
+
+```sh
+export VERSION=0-112-0
+URL=$(aws-vault exec management-operator -- aws lambda get-layer-version-by-arn --arn arn:aws:lambda:eu-west-1:901920570463:layer:aws-otel-collector-amd64-ver-${VERSION}:1 --query Content.Location --output text)
+curl $URL -o aws-otel-collector-amd64-ver-${VERSION}.zip
+unzip aws-otel-collector-amd64-ver-${VERSION}.zip
+```
+
+```sh
+.
+├── aws-otel-collector-amd64-ver-0-102-1.zip
+├── collector-config
+│   └── config.yaml
+└── extensions
+    └── collector
+```
+
+This method can be used to retrieve the ADOT Collector binary for other languages of the AWS Otel Lambda Layer. The version can be changed by updating the VERSION variable in the script.
+
+## Using the ADOT Collector in a docker container
+
+The following Dockerfile instructions can be used to set up a docker container with the ADOT Collector binary.
+
+```Dockerfile
+COPY collector-config/config.yaml /opt/config/config.yaml
+COPY extensions/collector /opt/extensions/collector
+RUN chmod 755 /opt/config.yaml
+ENV AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-handler
+ENV OPENTELEMETRY_COLLECTOR_CONFIG_FILE="/opt/config/config.yaml"
+```

go.mod

@@ -31,6 +31,8 @@ require (
 	github.com/vektra/mockery/v2 v2.46.3
 	github.com/xeipuuv/gojsonschema v1.2.0
 	go.opentelemetry.io/contrib/detectors/aws/ecs v1.31.0
+	go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda v0.56.0
+	go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda/xrayconfig v0.56.0
 	go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-sdk-go-v2/otelaws v0.56.0
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0
 	go.opentelemetry.io/contrib/propagators/aws v1.31.0
@@ -41,6 +43,7 @@ require (
 	golang.org/x/mod v0.21.0
 	golang.org/x/time v0.7.0
 	golang.org/x/tools v0.26.0
+	google.golang.org/appengine v1.6.7
 )
 
 require (
@@ -66,6 +69,7 @@ require (
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/gorilla/securecookie v1.1.2 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect
 	github.com/hashicorp/go-version v1.7.0 // indirect
@@ -96,11 +100,11 @@ require (
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	go.opentelemetry.io/contrib/detectors/aws/lambda v0.56.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0 // indirect
 	go.opentelemetry.io/otel/metric v1.31.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
-	go.uber.org/atomic v1.9.0 // indirect
-	go.uber.org/multierr v1.9.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect
 	golang.org/x/net v0.30.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect