Merge 8ba2377 into fb67c62

Makefile

@@ -176,3 +176,6 @@ delete-lpa-index: ##@opensearch deletes the lpa index
 
 add-scheduled-events:
 	go run ./scripts/add-scheduled-items.go
+
+run-schedule-runner: ##@scheduler invokes the schedule-runner lambda
+	curl "http://localhost:9002/2015-03-31/functions/function/invocations"

cmd/mlpa/main.go

@@ -130,7 +130,7 @@ func run(ctx context.Context, logger *slog.Logger) error {
 			return err
 		}
 
-		shutdown, err := telemetry.Setup(ctx, resource)
+		shutdown, err := telemetry.Setup(ctx, resource, false)
 		if err != nil {
 			return err
 		}

cmd/schedule-runner/main.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/aws/aws-lambda-go/lambda"
+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/ministryofjustice/opg-modernising-lpa/internal/donor"
 	"github.com/ministryofjustice/opg-modernising-lpa/internal/dynamo"
@@ -19,29 +20,35 @@ import (
 	"github.com/ministryofjustice/opg-modernising-lpa/internal/scheduled"
 	"github.com/ministryofjustice/opg-modernising-lpa/internal/search"
 	"github.com/ministryofjustice/opg-modernising-lpa/internal/secrets"
+	"github.com/ministryofjustice/opg-modernising-lpa/internal/telemetry"
+	"go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda"
+	"go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda/xrayconfig"
+	"go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-sdk-go-v2/otelaws"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"google.golang.org/appengine/log"
 )
 
-func handleRunSchedule(ctx context.Context) error {
-	var (
-		eventBusName          = cmp.Or(os.Getenv("EVENT_BUS_NAME"), "default")
-		notifyBaseURL         = os.Getenv("GOVUK_NOTIFY_BASE_URL")
-		notifyIsProduction    = os.Getenv("GOVUK_NOTIFY_IS_PRODUCTION") == "1"
-		searchEndpoint        = os.Getenv("SEARCH_ENDPOINT")
-		searchIndexName       = cmp.Or(os.Getenv("SEARCH_INDEX_NAME"), "lpas")
-		searchIndexingEnabled = os.Getenv("SEARCH_INDEXING_DISABLED") != "1"
-		tableName             = os.Getenv("LPAS_TABLE")
-	)
+var (
+	awsBaseURL            = os.Getenv("AWS_BASE_URL")
+	eventBusName          = cmp.Or(os.Getenv("EVENT_BUS_NAME"), "default")
+	notifyBaseURL         = os.Getenv("GOVUK_NOTIFY_BASE_URL")
+	notifyIsProduction    = os.Getenv("GOVUK_NOTIFY_IS_PRODUCTION") == "1"
+	searchEndpoint        = os.Getenv("SEARCH_ENDPOINT")
+	searchIndexName       = cmp.Or(os.Getenv("SEARCH_INDEX_NAME"), "lpas")
+	searchIndexingEnabled = os.Getenv("SEARCH_INDEXING_DISABLED") != "1"
+	tableName             = os.Getenv("LPAS_TABLE")
+	xrayEnabled           = os.Getenv("XRAY_ENABLED") == "1"
+
+	httpClient *http.Client
+	cfg        aws.Config
+)
 
+func handleRunSchedule(ctx context.Context) error {
 	logger := slog.New(slog.NewJSONHandler(os.Stdout, nil).
 		WithAttrs([]slog.Attr{
 			slog.String("service_name", "opg-modernising-lpa/schedule-runner"),
 		}))
 
-	cfg, err := config.LoadDefaultConfig(ctx)
-	if err != nil {
-		return fmt.Errorf("failed to load default config: %w", err)
-	}
-
 	secretsClient, err := secrets.NewClient(cfg, time.Hour)
 	if err != nil {
 		return err
@@ -57,7 +64,7 @@ func handleRunSchedule(ctx context.Context) error {
 		return err
 	}
 
-	notifyClient, err := notify.New(logger, notifyIsProduction, notifyBaseURL, notifyApiKey, http.DefaultClient, event.NewClient(cfg, eventBusName), bundle)
+	notifyClient, err := notify.New(logger, notifyIsProduction, notifyBaseURL, notifyApiKey, httpClient, event.NewClient(cfg, eventBusName), bundle)
 	if err != nil {
 		return err
 	}
@@ -88,5 +95,38 @@ func handleRunSchedule(ctx context.Context) error {
 }
 
 func main() {
-	lambda.Start(handleRunSchedule)
+	ctx := context.Background()
+
+	var err error
+	cfg, err = config.LoadDefaultConfig(ctx)
+	if err != nil {
+		log.Errorf(ctx, "failed to load default config: %v", err)
+		return
+	}
+
+	httpClient = &http.Client{Timeout: time.Second * 30}
+
+	if len(awsBaseURL) > 0 {
+		cfg.BaseEndpoint = aws.String(awsBaseURL)
+	}
+
+	if xrayEnabled {
+		tp, err := telemetry.SetupLambda(ctx)
+		if err != nil {
+			fmt.Printf("error creating tracer provider: %v", err)
+		}
+
+		otelaws.AppendMiddlewares(&cfg.APIOptions)
+		httpClient.Transport = otelhttp.NewTransport(httpClient.Transport)
+
+		defer func(ctx context.Context) {
+			if err := tp.Shutdown(ctx); err != nil {
+				fmt.Printf("error shutting down tracer provider: %v", err)
+			}
+		}(ctx)
+
+		lambda.Start(otellambda.InstrumentHandler(handleRunSchedule, xrayconfig.WithRecommendedOptions(tp)...))
+	} else {
+		lambda.Start(handleRunSchedule)
+	}
 }

docker/adot-collector/config/config.yaml

@@ -0,0 +1,23 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: "localhost:4317"
+      http:
+        endpoint: "localhost:4318"
+
+exporters:
+  logging:
+  awsxray:
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [awsxray]
+    metrics:
+      receivers: [otlp]
+      exporters: [logging]
+  telemetry:
+    metrics:
+      address: localhost:8888

docker/docker-compose.dev.yml

@@ -58,7 +58,7 @@ services:
       platforms:
         - "linux/amd64"
         - "linux/arm64"
-    container_name: schedule-runner
+    container_name: schedule-runner-lambda
     environment:
       - AWS_ACCESS_KEY_ID=fakeKeyId
       - AWS_BASE_URL=http://localstack:4566
@@ -68,6 +68,7 @@ services:
       - GOVUK_NOTIFY_IS_PRODUCTION=0
       - GOVUK_NOTIFY_BASE_URL=http://mock-notify:8080
       - SEARCH_ENDPOINT=http://my-domain.eu-west-1.opensearch.localhost.localstack.cloud:4566
+      - XRAY_ENABLED=1
     ports:
       - "9002:8080"
     entrypoint: aws-lambda-rie /var/task/schedule-runner

docker/localstack/localstack-init.sh

@@ -79,7 +79,8 @@ awslocal lambda create-function \
   --runtime provided.al2023 \
   --role arn:aws:iam::000000000000:role/lambda-role \
   --zip-file fileb:///etc/schedule-runner.zip \
-  --timeout 900
+  --timeout 900 \
+  --tracing-config Mode=Active
 
 awslocal lambda wait function-active-v2 --region eu-west-1 --function-name schedule-runner
 

docker/schedule-runner/Dockerfile

@@ -10,16 +10,6 @@ COPY --link internal ./internal
 
 RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 go build -o schedule-runner ./cmd/schedule-runner
 
-FROM public.ecr.aws/lambda/provided:al2023.2024.10.14.12 AS dev
-
-WORKDIR /app
-
-COPY --from=build /app/schedule-runner /var/task/schedule-runner
-COPY --link lang /var/task/lang
-COPY --link docker/aws-lambda-rie ./aws-lambda-rie
-
-ENTRYPOINT ["./schedule-runner"]
-
 FROM public.ecr.aws/lambda/provided:al2023.2024.10.14.12 AS production
 
 WORKDIR /app
@@ -30,5 +20,23 @@ RUN chmod +x "/app/install_lambda_insights.sh" \
 
 COPY --from=build /app/schedule-runner ./schedule-runner
 COPY --link lang ./lang
+COPY ./docker/adot-collector/ /opt
+
+RUN chmod 755 /opt/config/config.yaml
+
+ENV AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-handler
+ENV OPENTELEMETRY_COLLECTOR_CONFIG_FILE="/opt/config/config.yaml"
+
+ENTRYPOINT ["./schedule-runner"]
+
+FROM production AS dev
+
+WORKDIR /app
+
+COPY --from=build /app/schedule-runner /var/task/schedule-runner
+COPY --link docker/aws-lambda-rie ./aws-lambda-rie
+
+ENV AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-handler
+ENV OPENTELEMETRY_COLLECTOR_CONFIG_FILE="/opt/config/config.yaml"
 
 ENTRYPOINT ["./schedule-runner"]

go.mod

@@ -31,16 +31,20 @@ require (
 	github.com/vektra/mockery/v2 v2.46.3
 	github.com/xeipuuv/gojsonschema v1.2.0
 	go.opentelemetry.io/contrib/detectors/aws/ecs v1.31.0
+	go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda v0.56.0
+	go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda/xrayconfig v0.56.0
 	go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-sdk-go-v2/otelaws v0.56.0
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0
 	go.opentelemetry.io/contrib/propagators/aws v1.31.0
 	go.opentelemetry.io/otel v1.31.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.31.0
+	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0
 	go.opentelemetry.io/otel/sdk v1.31.0
 	go.opentelemetry.io/otel/trace v1.31.0
 	golang.org/x/mod v0.21.0
 	golang.org/x/time v0.7.0
 	golang.org/x/tools v0.26.0
+	google.golang.org/appengine v1.6.7
 )
 
 require (
@@ -66,6 +70,7 @@ require (
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/gorilla/securecookie v1.1.2 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect
 	github.com/hashicorp/go-version v1.7.0 // indirect
@@ -96,11 +101,11 @@ require (
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	go.opentelemetry.io/contrib/detectors/aws/lambda v0.56.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0 // indirect
 	go.opentelemetry.io/otel/metric v1.31.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
-	go.uber.org/atomic v1.9.0 // indirect
-	go.uber.org/multierr v1.9.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect
 	golang.org/x/net v0.30.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect